Java 添加spark配置单元依赖项时配置单元单元测试不起作用
当我尝试执行单元测试时,HiverRunner出现错误,这是我的测试类:Java 添加spark配置单元依赖项时配置单元单元测试不起作用,java,maven,unit-testing,hive,Java,Maven,Unit Testing,Hive,当我尝试执行单元测试时,HiverRunner出现错误,这是我的测试类: import com.klarna.hiverunner.HiveShell; import com.klarna.hiverunner.StandaloneHiveRunner; import com.klarna.hiverunner.annotations.HiveSQL; import org.junit.*; import org.junit.rules.TestName; import org.junit.ru
import com.klarna.hiverunner.HiveShell;
import com.klarna.hiverunner.StandaloneHiveRunner;
import com.klarna.hiverunner.annotations.HiveSQL;
import org.junit.*;
import org.junit.rules.TestName;
import org.junit.runner.RunWith;
import java.util.Arrays;
import java.util.List;
@RunWith(StandaloneHiveRunner.class)
public class MiniClusterHiveTest {
@Rule
public TestName name = new TestName();
@HiveSQL(files = {})
private HiveShell shell;
@Before
public void createDatabaseAndTable() {
shell.execute("CREATE DATABASE source_db");
shell.execute("CREATE TABLE source_db.survey (" +
"submittedTime STRING," +
"age STRING," +
"gender STRING," +
"country STRING," +
"state STRING," +
"self_employed STRING," +
"family_history STRING," +
"treatment STRING," +
"work_interfere STRING," +
"no_employees STRING," +
"remote_work STRING," +
"tech_company STRING," +
"benefits STRING," +
"care_options STRING," +
"wellness_program STRING," +
"seek_help STRING," +
"anonymity STRING," +
"leave STRING," +
"mental_health_consequence STRING," +
"phys_health_consequence STRING," +
"coworkers STRING," +
"supervisor STRING," +
"mental_health_interview STRING," +
"phys_health_interview STRING," +
"mental_vs_physical STRING," +
"obs_consequence STRING," +
"comments STRING) " +
"ROW FORMAT DELIMITED " +
"FIELDS TERMINATED BY ',' " +
"STORED AS TEXTFILE " +
"LOCATION '/home/datasets/'");
}
@Before
public void createORCTable() {
shell.execute("CREATE TABLE source_db.survey2 (" +
"submittedTime STRING," +
"age STRING," +
"gender STRING," +
"country STRING," +
"state STRING," +
"self_employed STRING," +
"family_history STRING," +
"treatment STRING," +
"work_interfere STRING," +
"no_employees STRING," +
"remote_work STRING," +
"tech_company STRING," +
"benefits STRING," +
"care_options STRING," +
"wellness_program STRING," +
"seek_help STRING," +
"anonymity STRING," +
"leave STRING," +
"mental_health_consequence STRING," +
"phys_health_consequence STRING," +
"coworkers STRING," +
"supervisor STRING," +
"mental_health_interview STRING," +
"phys_health_interview STRING," +
"mental_vs_physical STRING," +
"obs_consequence STRING," +
"comments STRING) " +
"ROW FORMAT DELIMITED " +
"FIELDS TERMINATED BY ',' " +
"STORED AS ORC tblproperties (\"orc.compress\"=\"ZLIB\"); ");
}
@Before
public void createParquetTable() {
shell.execute("CREATE TABLE source_db.survey3 " +
"STORED AS PARQUET TBLPROPERTIES (\"parquet.compression\"=\"SNAPPY\")\n" +
" AS SELECT * FROM source_db.survey;");
}
/**
* We use temporary table survey to load the orc table survey2
*/
@Test
public void loadOrcSurvey2Table() {
shell.execute("INSERT INTO TABLE source_db.survey2 SELECT * from source_db.survey");
}
@Ignore("Use for simple test without external file")
public void simpleInit() {
shell.execute("CREATE DATABASE source_db");
shell.execute("CREATE TABLE source_db.test (" +
"a STRING," +
"b STRING ) ");
}
@Ignore
public void simpleInsertionDataIntoTable() {
shell.insertInto("source_db", "test")
.withAllColumns()
.addRow("bim", "bap")
.commit();
printResult(shell.executeStatement("select * from source_db.test"));
}
@Test
public void executeQuery() {
List<Object[]> result = shell.executeStatement("select * from source_db.survey where age=37");
List<Object[]> result2 = shell.executeStatement("select * from source_db.survey where age=12");
Assert.assertEquals(43L, result.size());
Assert.assertEquals(0L, result2.size());
}
@Test
public void insertDataIntoTable() {
shell.insertInto("source_db", "survey")
.withAllColumns()
.addRow("2019-03-01 09:29:31",
17,
"Male",
"France",
"IL",
"NA",
"No",
"Yes",
"Often",
"6-25",
"No",
"Yes",
"Yes",
"Not sure",
"No",
"Yes",
"Yes",
"Somewhat easy",
"No",
"No",
"Some of them",
"Yes",
"No",
"Maybe",
"Yes",
"No",
"NA"
)
.commit();
printResult(shell.executeStatement("select * from source_db.survey where age=17"));
}
private void printResult(List<Object[]> result) {
System.out.println(String.format("Result from %s:", name.getMethodName()));
result.stream().map(Arrays::asList).forEach(System.out::println);
}
}
import com.klarna.hiverunner.HiveShell;
导入com.klarna.hiverunner.standaronehiverunner;
导入com.klarna.hiverunner.annotations.HiveSQL;
导入org.junit.*;
导入org.junit.rules.TestName;
导入org.junit.runner.RunWith;
导入java.util.array;
导入java.util.List;
@RunWith(StandaloneHiveRunner.class)
公共类小型群集测试{
@统治
public TestName name=new TestName();
@HiveSQL(文件={})
私人蜂巢壳;
@以前
public void createDatabaseAndTable(){
shell.execute(“创建数据库源_db”);
shell.execute(“创建表源”\u db.survey(”+
“提交的时间字符串,”+
“年龄字符串,”+
“性别字符串,”+
“国家字符串,”+
“状态字符串,”+
“自营职业者”+
“家庭历史字符串,”+
“治疗字符串,”+
“工作字符串,”+
“没有员工字符串,”+
“远程工作字符串,”+
“科技公司字符串,”+
“福利字符串,”+
“注意选项字符串,”+
“健康计划字符串,”+
“查找帮助字符串,”+
“匿名字符串,”+
“留下字符串,”+
“心理健康后果字符串,”+
“物理健康结果字符串,”+
“同事字符串,”+
“主管字符串,”+
“心理健康”采访字符串+
“Physs_health_采访字符串,”+
“心理字符串与物理字符串”+
obs_结果字符串+
“注释字符串)”+
“行格式分隔”+
以“,”结尾的字段+
“存储为文本文件”+
“位置'/home/datasets/'”;
}
@以前
public void createORCTable(){
shell.execute(“创建表源”\u db.survey2(”+
“提交的时间字符串,”+
“年龄字符串,”+
“性别字符串,”+
“国家字符串,”+
“状态字符串,”+
“自营职业者”+
“家庭历史字符串,”+
“治疗字符串,”+
“工作字符串,”+
“没有员工字符串,”+
“远程工作字符串,”+
“科技公司字符串,”+
“福利字符串,”+
“注意选项字符串,”+
“健康计划字符串,”+
“查找帮助字符串,”+
“匿名字符串,”+
“留下字符串,”+
“心理健康后果字符串,”+
“物理健康结果字符串,”+
“同事字符串,”+
“主管字符串,”+
“心理健康”采访字符串+
“Physs_health_采访字符串,”+
“心理字符串与物理字符串”+
obs_结果字符串+
“注释字符串)”+
“行格式分隔”+
以“,”结尾的字段+
“存储为ORC TBLProperty(\“ORC.compress\”=“ZLIB\”);”;
}
@以前
public void createParquetTable(){
shell.execute(“创建表源\u db.survey3”+
“存储为拼花地板TBLProperty(\“PARQUET.compression\”=“SNAPPY\”)\n”+
“从源数据库调查中选择*”;
}
/**
*我们使用临时表格调查加载orc表格调查2
*/
@试验
公共空荷载或勘测2表(){
shell.execute(“插入表source_db.survey2选择*from source_db.survey”);
}
@忽略(“用于无外部文件的简单测试”)
public void simpleInit(){
shell.execute(“创建数据库源_db”);
shell.execute(“创建表源”\u db.test(”+
“字符串,”+
"b字串";;
}
@忽略
public void simpleInsertionDataInTable(){
shell.insertInto(“源数据库”、“测试”)
.withAllColumns()
.addRow(“bim”、“bap”)
.commit();
打印结果(shell.executeStatement(“select*from source_db.test”);
}
@试验
public void executeQuery(){
列表结果=shell.executeStatement(“从source\u db.survey中选择*,其中年龄=37”);
List result2=shell.executeStatement(“从源数据库调查中选择*,其中年龄=12”);
Assert.assertEquals(43L,result.size());
Assert.assertEquals(0L,result2.size());
}
@试验
public void insertDataIntoTable(){
shell.插入(“来源数据库”、“调查”)
.withAllColumns()
.addRow(“2019-03-01 09:29:31”,
17,
“男性”,
“法国”,
“IL”,
“不适用”,
“不”,
“是的”,
“经常”,
"6-25",
“不”,
“是的”,
“是的”,
“不确定”,
“不”,
“是的”,
“是的”,
“有点容易”,
“不”,
“不”,
“其中一些”,
“是的”,
“不”,
“也许”,
“是的”,
“不”,
“不适用”
)
.commit();
打印结果(shell.executeStatement(“从源数据库中选择*,其中年龄=17”))
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_2.11</artifactId>
<version>${spark.version}</version>
</dependency>
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.finaxys</groupId>
<artifactId>MiniClusterTestBigData</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<java.version>1.8</java.version>
<hadoop.version>2.7.3</hadoop.version>
<spark.version>2.3.0</spark.version>
<junit.version>4.12</junit.version>
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
</properties>
<dependencies>
<!--hive dependencies-->
<dependency>
<groupId>com.klarna</groupId>
<artifactId>hiverunner</artifactId>
<version>4.1.0</version>
</dependency>
<!--hadoop dependencies-->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
<version>${hadoop.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
<!--Spark dependencies-->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<version>${spark.version}</version>
</dependency>
</dependencies>
</project>