Apache spark org.apache.spark.rdd.NewHadoopRDD-无法使用InputSplit#getLocationInfo

Apache spark org.apache.spark.rdd.NewHadoopRDD-无法使用InputSplit#getLocationInfo,apache-spark,hbase,apache-zookeeper,Apache Spark,Hbase,Apache Zookeeper,我正在使用Java(8)将spark(1.6.0)与hbase(1.2.2.3.4.6)连接起来。 ,spark客户端上的scala代码正常。Spark集群、hbase集群、zookeeper集群都在云端。代码如下: package com.carelinker.spark; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apa

我正在使用Java(8)将spark(1.6.0)与hbase(1.2.2.3.4.6)连接起来。 ,spark客户端上的scala代码正常。Spark集群、hbase集群、zookeeper集群都在云端。代码如下:

package com.carelinker.spark;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableInputFormat;
import org.apache.mesos.protobuf.ServiceException;
import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.springframework.boot.autoconfigure.SpringBootApplication;

@SpringBootApplication
public class SparktestApplication {
    public static void main(String[] args) throws ServiceException {
        //HBase config
        Configuration conf = HBaseConfiguration.create();
        conf.set("hbase.zookeeper.property.clientPort", "2181");
        conf.set("hbase.zookeeper.quorum", "192.168.100.5:2181,192.168.100.3:2181,192.168.100.6:2181");
        conf.set("zookeeper.znode.parent", "/hbase/hbs-frz2bnnm");
        conf.set(TableInputFormat.INPUT_TABLE, "test");
        //Spark config
        SparkContext sc = new SparkContext("spark://skn-1w3zsyz0-spark-master:7077", "HBaseRead");
        sc.setLocalProperty("spark.executor.extraClassPath","/usr/local/hbase/lib/*");
        sc.setLocalProperty("spark.driver.extraClassPath","/usr/local/hbase/lib/*");
        JavaSparkContext javaSparkContext = new JavaSparkContext(sc);
        JavaPairRDD<ImmutableBytesWritable, Result> myRDD = javaSparkContext.newAPIHadoopRDD(conf, TableInputFormat.class, ImmutableBytesWritable.class, Result.class);
        System.out.println(myRDD.count());
    }
}
我搜索了这个问题,如图所示:配置hbase.master,但它对我没有帮助

我有以下问题:

  • 这个例外意味着什么
  • 为什么parentName日志一直显示,却没有结果
  • 谢谢你的帮助

    以下是maven pom文件的详细信息:

    <?xml version="1.0" encoding="UTF-8"?>
    <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
        xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
        <modelVersion>4.0.0</modelVersion>
    
        <groupId>com.carelinker.spark</groupId>
        <artifactId>sparktest</artifactId>
        <version>0.0.1-SNAPSHOT</version>
        <packaging>jar</packaging>
    
        <name>sparktest</name>
        <description>Demo project for Spring Boot</description>
    
        <parent>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-parent</artifactId>
            <version>1.5.2.RELEASE</version>
            <relativePath/> <!-- lookup parent from repository -->
        </parent>
    
        <properties>
            <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
            <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
            <java.version>1.8</java.version>
        </properties>
    
        <dependencies>
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-starter-web</artifactId>
            </dependency>
            <dependency>
                <groupId>org.apache.hbase</groupId>
                <artifactId>hbase-client</artifactId>
                <version>1.2.2</version>
            </dependency>
            <dependency>
                <groupId>org.apache.hbase</groupId>
                <artifactId>hbase-server</artifactId>
                <version>1.2.2</version>
            </dependency>
            <dependency>
                <groupId>org.apache.spark</groupId>
                <artifactId>spark-core_2.10</artifactId>
                <version>1.6.0</version>
                <exclusions>
                    <exclusion>
                        <groupId>org.eclipse.jetty.orbit</groupId>
                        <artifactId>javax.servlet</artifactId>
                    </exclusion>
                </exclusions>
            </dependency>
            <dependency>
                <groupId>org.apache.spark</groupId>
                <artifactId>spark-sql_2.10</artifactId>
                <version>1.6.0</version>
            </dependency>
            <dependency>
                <groupId>javax.servlet</groupId>
                <artifactId>javax.servlet-api</artifactId>
                <version>3.1.0</version>
            </dependency>
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-starter-tomcat</artifactId>
                <!--<scope>provided</scope>-->
            </dependency>
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-starter-test</artifactId>
                <scope>test</scope>
            </dependency>
        </dependencies>
    
        <build>
            <plugins>
                <plugin>
                    <groupId>org.springframework.boot</groupId>
                    <artifactId>spring-boot-maven-plugin</artifactId>
                </plugin>
            </plugins>
        </build>
    
    
    </project>
    
    
    4.0.0
    com.carelinker.spark
    火花测试
    0.0.1-快照
    罐子
    火花测试
    SpringBoot的演示项目
    org.springframework.boot
    spring启动程序父级
    1.5.2.1发布
    UTF-8
    UTF-8
    1.8
    org.springframework.boot
    SpringBootStarterWeb
    org.apache.hbase
    hbase客户端
    1.2.2
    org.apache.hbase
    hbase服务器
    1.2.2
    org.apache.spark
    spark-core_2.10
    1.6.0
    org.eclipse.jetty.orbit
    javax.servlet
    org.apache.spark
    spark-sql_2.10
    1.6.0
    javax.servlet
    javax.servlet-api
    3.1.0
    org.springframework.boot
    弹簧启动机tomcat
    org.springframework.boot
    弹簧起动试验
    测试
    org.springframework.boot
    springbootmaven插件
    
    我已将下面的主url更改为“本地”,这似乎有效,但据连线,spark主url位于云端,应该类似于“spark:\\IP:Port”,为什么我将其更改为本地时可以<代码>代码SparkContext sc=新的SparkContext(“spark://skn-1w3zsyz0-spark-master:7077“,”HBaseRead“);有同样的问题。。。您找到解决方案/根本原因识别了吗?
    <?xml version="1.0" encoding="UTF-8"?>
    <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
        xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
        <modelVersion>4.0.0</modelVersion>
    
        <groupId>com.carelinker.spark</groupId>
        <artifactId>sparktest</artifactId>
        <version>0.0.1-SNAPSHOT</version>
        <packaging>jar</packaging>
    
        <name>sparktest</name>
        <description>Demo project for Spring Boot</description>
    
        <parent>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-parent</artifactId>
            <version>1.5.2.RELEASE</version>
            <relativePath/> <!-- lookup parent from repository -->
        </parent>
    
        <properties>
            <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
            <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
            <java.version>1.8</java.version>
        </properties>
    
        <dependencies>
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-starter-web</artifactId>
            </dependency>
            <dependency>
                <groupId>org.apache.hbase</groupId>
                <artifactId>hbase-client</artifactId>
                <version>1.2.2</version>
            </dependency>
            <dependency>
                <groupId>org.apache.hbase</groupId>
                <artifactId>hbase-server</artifactId>
                <version>1.2.2</version>
            </dependency>
            <dependency>
                <groupId>org.apache.spark</groupId>
                <artifactId>spark-core_2.10</artifactId>
                <version>1.6.0</version>
                <exclusions>
                    <exclusion>
                        <groupId>org.eclipse.jetty.orbit</groupId>
                        <artifactId>javax.servlet</artifactId>
                    </exclusion>
                </exclusions>
            </dependency>
            <dependency>
                <groupId>org.apache.spark</groupId>
                <artifactId>spark-sql_2.10</artifactId>
                <version>1.6.0</version>
            </dependency>
            <dependency>
                <groupId>javax.servlet</groupId>
                <artifactId>javax.servlet-api</artifactId>
                <version>3.1.0</version>
            </dependency>
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-starter-tomcat</artifactId>
                <!--<scope>provided</scope>-->
            </dependency>
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-starter-test</artifactId>
                <scope>test</scope>
            </dependency>
        </dependencies>
    
        <build>
            <plugins>
                <plugin>
                    <groupId>org.springframework.boot</groupId>
                    <artifactId>spring-boot-maven-plugin</artifactId>
                </plugin>
            </plugins>
        </build>
    
    
    </project>