Warning: file_get_contents(/data/phpspider/zhask/data//catemap/9/java/400.json): failed to open stream: No such file or directory in /data/phpspider/zhask/libs/function.php on line 167

Warning: Invalid argument supplied for foreach() in /data/phpspider/zhask/libs/tag.function.php on line 1116

Notice: Undefined index: in /data/phpspider/zhask/libs/function.php on line 180

Warning: array_chunk() expects parameter 1 to be array, null given in /data/phpspider/zhask/libs/function.php on line 181
Java 添加spark配置单元依赖项时配置单元单元测试不起作用_Java_Maven_Unit Testing_Hive - Fatal编程技术网

Java 添加spark配置单元依赖项时配置单元单元测试不起作用

Java 添加spark配置单元依赖项时配置单元单元测试不起作用,java,maven,unit-testing,hive,Java,Maven,Unit Testing,Hive,当我尝试执行单元测试时,HiverRunner出现错误,这是我的测试类: import com.klarna.hiverunner.HiveShell; import com.klarna.hiverunner.StandaloneHiveRunner; import com.klarna.hiverunner.annotations.HiveSQL; import org.junit.*; import org.junit.rules.TestName; import org.junit.ru

当我尝试执行单元测试时,HiverRunner出现错误,这是我的测试类:

import com.klarna.hiverunner.HiveShell;
import com.klarna.hiverunner.StandaloneHiveRunner;
import com.klarna.hiverunner.annotations.HiveSQL;
import org.junit.*;
import org.junit.rules.TestName;
import org.junit.runner.RunWith;

import java.util.Arrays;
import java.util.List;

@RunWith(StandaloneHiveRunner.class)
public class MiniClusterHiveTest {

    @Rule
    public TestName name = new TestName();

    @HiveSQL(files = {})
    private HiveShell shell;

    @Before
    public void createDatabaseAndTable() {
        shell.execute("CREATE DATABASE source_db");
        shell.execute("CREATE TABLE source_db.survey (" +
                "submittedTime STRING," +
                "age STRING," +
                "gender STRING," +
                "country STRING," +
                "state STRING," +
                "self_employed STRING," +
                "family_history STRING," +
                "treatment STRING," +
                "work_interfere STRING," +
                "no_employees STRING," +
                "remote_work STRING," +
                "tech_company STRING," +
                "benefits STRING," +
                "care_options STRING," +
                "wellness_program STRING," +
                "seek_help STRING," +
                "anonymity STRING," +
                "leave STRING," +
                "mental_health_consequence STRING," +
                "phys_health_consequence STRING," +
                "coworkers STRING," +
                "supervisor STRING," +
                "mental_health_interview STRING," +
                "phys_health_interview STRING," +
                "mental_vs_physical STRING," +
                "obs_consequence STRING," +
                "comments STRING) " +
                "ROW FORMAT DELIMITED " +
                "FIELDS TERMINATED BY ',' " +
                "STORED AS TEXTFILE " +
                "LOCATION '/home/datasets/'");
    }

    @Before
    public void createORCTable() {
        shell.execute("CREATE TABLE source_db.survey2 (" +
                "submittedTime STRING," +
                "age STRING," +
                "gender STRING," +
                "country STRING," +
                "state STRING," +
                "self_employed STRING," +
                "family_history STRING," +
                "treatment STRING," +
                "work_interfere STRING," +
                "no_employees STRING," +
                "remote_work STRING," +
                "tech_company STRING," +
                "benefits STRING," +
                "care_options STRING," +
                "wellness_program STRING," +
                "seek_help STRING," +
                "anonymity STRING," +
                "leave STRING," +
                "mental_health_consequence STRING," +
                "phys_health_consequence STRING," +
                "coworkers STRING," +
                "supervisor STRING," +
                "mental_health_interview STRING," +
                "phys_health_interview STRING," +
                "mental_vs_physical STRING," +
                "obs_consequence STRING," +
                "comments STRING) " +
                "ROW FORMAT DELIMITED " +
                "FIELDS TERMINATED BY ',' " +
                "STORED AS ORC tblproperties (\"orc.compress\"=\"ZLIB\"); ");

    }

    @Before
    public void createParquetTable() {
        shell.execute("CREATE TABLE source_db.survey3 " +
                "STORED AS PARQUET TBLPROPERTIES (\"parquet.compression\"=\"SNAPPY\")\n" +
                " AS SELECT * FROM source_db.survey;");
    }

    /**
     * We use temporary table survey to load the orc table survey2
     */
    @Test
    public void loadOrcSurvey2Table() {
        shell.execute("INSERT INTO TABLE source_db.survey2 SELECT * from source_db.survey");
    }

    @Ignore("Use for simple test without external file")
    public void simpleInit() {
        shell.execute("CREATE DATABASE source_db");
        shell.execute("CREATE TABLE source_db.test (" +
                "a STRING," +
                "b STRING ) ");

    }

    @Ignore
    public void simpleInsertionDataIntoTable() {
        shell.insertInto("source_db", "test")
                .withAllColumns()
                .addRow("bim", "bap")
                .commit();

        printResult(shell.executeStatement("select * from source_db.test"));
    }

    @Test
    public void executeQuery() {

        List<Object[]> result = shell.executeStatement("select * from source_db.survey where age=37");
        List<Object[]> result2 = shell.executeStatement("select * from source_db.survey where age=12");

        Assert.assertEquals(43L, result.size());
        Assert.assertEquals(0L, result2.size());
    }

    @Test
    public void insertDataIntoTable() {
        shell.insertInto("source_db", "survey")
                .withAllColumns()
                .addRow("2019-03-01 09:29:31",
                        17,
                        "Male",
                        "France",
                        "IL",
                        "NA",
                        "No",
                        "Yes",
                        "Often",
                        "6-25",
                        "No",
                        "Yes",
                        "Yes",
                        "Not sure",
                        "No",
                        "Yes",
                        "Yes",
                        "Somewhat easy",
                        "No",
                        "No",
                        "Some of them",
                        "Yes",
                        "No",
                        "Maybe",
                        "Yes",
                        "No",
                        "NA"
                )
                .commit();
        printResult(shell.executeStatement("select * from source_db.survey where age=17"));
    }

    private void printResult(List<Object[]> result) {
        System.out.println(String.format("Result from %s:", name.getMethodName()));
        result.stream().map(Arrays::asList).forEach(System.out::println);
    }
}
import com.klarna.hiverunner.HiveShell;
导入com.klarna.hiverunner.standaronehiverunner;
导入com.klarna.hiverunner.annotations.HiveSQL;
导入org.junit.*;
导入org.junit.rules.TestName;
导入org.junit.runner.RunWith;
导入java.util.array;
导入java.util.List;
@RunWith(StandaloneHiveRunner.class)
公共类小型群集测试{
@统治
public TestName name=new TestName();
@HiveSQL(文件={})
私人蜂巢壳;
@以前
public void createDatabaseAndTable(){
shell.execute(“创建数据库源_db”);
shell.execute(“创建表源”\u db.survey(”+
“提交的时间字符串,”+
“年龄字符串,”+
“性别字符串,”+
“国家字符串,”+
“状态字符串,”+
“自营职业者”+
“家庭历史字符串,”+
“治疗字符串,”+
“工作字符串,”+
“没有员工字符串,”+
“远程工作字符串,”+
“科技公司字符串,”+
“福利字符串,”+
“注意选项字符串,”+
“健康计划字符串,”+
“查找帮助字符串,”+
“匿名字符串,”+
“留下字符串,”+
“心理健康后果字符串,”+
“物理健康结果字符串,”+
“同事字符串,”+
“主管字符串,”+
“心理健康”采访字符串+
“Physs_health_采访字符串,”+
“心理字符串与物理字符串”+
obs_结果字符串+
“注释字符串)”+
“行格式分隔”+
以“,”结尾的字段+
“存储为文本文件”+
“位置'/home/datasets/'”;
}
@以前
public void createORCTable(){
shell.execute(“创建表源”\u db.survey2(”+
“提交的时间字符串,”+
“年龄字符串,”+
“性别字符串,”+
“国家字符串,”+
“状态字符串,”+
“自营职业者”+
“家庭历史字符串,”+
“治疗字符串,”+
“工作字符串,”+
“没有员工字符串,”+
“远程工作字符串,”+
“科技公司字符串,”+
“福利字符串,”+
“注意选项字符串,”+
“健康计划字符串,”+
“查找帮助字符串,”+
“匿名字符串,”+
“留下字符串,”+
“心理健康后果字符串,”+
“物理健康结果字符串,”+
“同事字符串,”+
“主管字符串,”+
“心理健康”采访字符串+
“Physs_health_采访字符串,”+
“心理字符串与物理字符串”+
obs_结果字符串+
“注释字符串)”+
“行格式分隔”+
以“,”结尾的字段+
“存储为ORC TBLProperty(\“ORC.compress\”=“ZLIB\”);”;
}
@以前
public void createParquetTable(){
shell.execute(“创建表源\u db.survey3”+
“存储为拼花地板TBLProperty(\“PARQUET.compression\”=“SNAPPY\”)\n”+
“从源数据库调查中选择*”;
}
/**
*我们使用临时表格调查加载orc表格调查2
*/
@试验
公共空荷载或勘测2表(){
shell.execute(“插入表source_db.survey2选择*from source_db.survey”);
}
@忽略(“用于无外部文件的简单测试”)
public void simpleInit(){
shell.execute(“创建数据库源_db”);
shell.execute(“创建表源”\u db.test(”+
“字符串,”+
"b字串";;
}
@忽略
public void simpleInsertionDataInTable(){
shell.insertInto(“源数据库”、“测试”)
.withAllColumns()
.addRow(“bim”、“bap”)
.commit();
打印结果(shell.executeStatement(“select*from source_db.test”);
}
@试验
public void executeQuery(){
列表结果=shell.executeStatement(“从source\u db.survey中选择*,其中年龄=37”);
List result2=shell.executeStatement(“从源数据库调查中选择*,其中年龄=12”);
Assert.assertEquals(43L,result.size());
Assert.assertEquals(0L,result2.size());
}
@试验
public void insertDataIntoTable(){
shell.插入(“来源数据库”、“调查”)
.withAllColumns()
.addRow(“2019-03-01 09:29:31”,
17,
“男性”,
“法国”,
“IL”,
“不适用”,
“不”,
“是的”,
“经常”,
"6-25",
“不”,
“是的”,
“是的”,
“不确定”,
“不”,
“是的”,
“是的”,
“有点容易”,
“不”,
“不”,
“其中一些”,
“是的”,
“不”,
“也许”,
“是的”,
“不”,
“不适用”
)
.commit();
打印结果(shell.executeStatement(“从源数据库中选择*,其中年龄=17”))
<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-hive_2.11</artifactId>
    <version>${spark.version}</version>
</dependency>
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.finaxys</groupId>
    <artifactId>MiniClusterTestBigData</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <java.version>1.8</java.version>
        <hadoop.version>2.7.3</hadoop.version>
        <spark.version>2.3.0</spark.version>
        <junit.version>4.12</junit.version>
        <maven.compiler.source>${java.version}</maven.compiler.source>
        <maven.compiler.target>${java.version}</maven.compiler.target>
    </properties>

    <dependencies>
        <!--hive dependencies-->
        <dependency>
            <groupId>com.klarna</groupId>
            <artifactId>hiverunner</artifactId>
            <version>4.1.0</version>
        </dependency>

        <!--hadoop dependencies-->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-minicluster</artifactId>
            <version>${hadoop.version}</version>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>${junit.version}</version>
            <scope>test</scope>
        </dependency>

        <!--Spark dependencies-->
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.11</artifactId>
            <version>${spark.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-sql_2.11</artifactId>
            <version>${spark.version}</version>
        </dependency>

    </dependencies>
</project>