Apache spark 无法在纱线群集上运行spark作业：连接失败异常

我正在纱线集群上运行一个简单的spark作业，并配置了warn-site.xml

如果您还需要其他内容，请告诉我

我不理解这个问题，也无法解决这个问题

我的火花代码

import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem
import org.apache.log4j._
import org.apache.spark._
import org.apache.spark.sql._
import org.apache.spark.SparkContext._
import org.apache.spark.sql.functions._
import org.apache.hadoop.fs.Path
import org.apache.spark.sql.types._

object Test {
  def main(args: Array[String]){)
    // Setting log level to errors
    Logger.getLogger("org").setLevel(Level.ERROR)

    // Setting up the sparksession
    val spark = SparkSession.builder
      .appName("Journaling")
      .config("spark.master","yarn")
      .getOrCreate()
    val sc = SparkContext.getOrCreate()

    // Configuration for reading file from HDFS
    val conf = new Configuration()
    conf.set("fs.defaultFS", "hdfs://localhost:8020")
    val fs= FileSystem.get(conf)
    val df = spark.read.format("csv")
      .option("inferschema", "true")
      .option("header", "true")
      .load("hdfs://localhost:8020/fakefriends.csv")

    df.show()
  }
}

上面的代码只是从HDFS中获取一个csv文件，然后从中创建一个数据帧，并显示前20个内容

---

我可以在本地客户端模式下运行相同的程序。

尝试将所有本地主机更改为本地IP地址（192.168.1.13），spark在群集模式下运行时无法解析0.0.0

Exception in thread "main" java.net.ConnectException: Call From abhijeet.local/192.168.1.13 to 0.0.0.0:8032 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
27-06-2018 03:32:33 PDT Spark123 INFO -     at sun.reflect.GeneratedConstructorAccessor4.newInstance(Unknown Source)
27-06-2018 03:32:33 PDT Spark123 INFO -     at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
27-06-2018 03:32:33 PDT Spark123 INFO -     at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
27-06-2018 03:32:33 PDT Spark123 INFO -     at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:792)
27-06-2018 03:32:33 PDT Spark123 INFO -     at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:732)
27-06-2018 03:32:33 PDT Spark123 INFO -     at org.apache.hadoop.ipc.Client.call(Client.java:1479)
27-06-2018 03:32:33 PDT Spark123 INFO -     at org.apache.hadoop.ipc.Client.call(Client.java:1412)
27-06-2018 03:32:33 PDT Spark123 INFO -     at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:229)
27-06-2018 03:32:33 PDT Spark123 INFO -     at com.sun.proxy.$Proxy8.getNewApplication(Unknown Source)
27-06-2018 03:32:33 PDT Spark123 INFO -     at org.apache.hadoop.yarn.api.impl.pb.client.ApplicationClientProtocolPBClientImpl.getNewApplication(ApplicationClientProtocolPBClientImpl.java:221)
27-06-2018 03:32:33 PDT Spark123 INFO -     at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
27-06-2018 03:32:33 PDT Spark123 INFO -     at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
27-06-2018 03:32:33 PDT Spark123 INFO -     at java.lang.reflect.Method.invoke(Method.java:498)
27-06-2018 03:32:33 PDT Spark123 INFO -     at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:191)
27-06-2018 03:32:33 PDT Spark123 INFO -     at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102)
27-06-2018 03:32:33 PDT Spark123 INFO -     at com.sun.proxy.$Proxy9.getNewApplication(Unknown Source)
27-06-2018 03:32:33 PDT Spark123 INFO -     at org.apache.hadoop.yarn.client.api.impl.YarnClientImpl.getNewApplication(YarnClientImpl.java:219)
27-06-2018 03:32:33 PDT Spark123 INFO -     at org.apache.hadoop.yarn.client.api.impl.YarnClientImpl.createApplication(YarnClientImpl.java:227)
27-06-2018 03:32:33 PDT Spark123 INFO -     at org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:159)
27-06-2018 03:32:33 PDT Spark123 INFO -     at org.apache.spark.deploy.yarn.Client.run(Client.scala:1109)
27-06-2018 03:32:33 PDT Spark123 INFO -     at org.apache.spark.deploy.yarn.Client$.main(Client.scala:1168)
27-06-2018 03:32:33 PDT Spark123 INFO -     at org.apache.spark.deploy.yarn.Client.main(Client.scala)
27-06-2018 03:32:33 PDT Spark123 INFO -     at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
27-06-2018 03:32:33 PDT Spark123 INFO -     at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
27-06-2018 03:32:33 PDT Spark123 INFO -     at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
27-06-2018 03:32:33 PDT Spark123 INFO -     at java.lang.reflect.Method.invoke(Method.java:498)
27-06-2018 03:32:33 PDT Spark123 INFO -     at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:775)
27-06-2018 03:32:33 PDT Spark123 INFO -     at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180)
27-06-2018 03:32:33 PDT Spark123 INFO -     at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205)
27-06-2018 03:32:33 PDT Spark123 INFO -     at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:119)
27-06-2018 03:32:33 PDT Spark123 INFO -     at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
27-06-2018 03:32:33 PDT Spark123 INFO - Caused by: java.net.ConnectException: Connection refused
27-06-2018 03:32:33 PDT Spark123 INFO -     at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
27-06-2018 03:32:33 PDT Spark123 INFO -     at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
27-06-2018 03:32:33 PDT Spark123 INFO -     at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
27-06-2018 03:32:33 PDT Spark123 INFO -     at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:531)
27-06-2018 03:32:33 PDT Spark123 INFO -     at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:495)
27-06-2018 03:32:33 PDT Spark123 INFO -     at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:614)
27-06-2018 03:32:33 PDT Spark123 INFO -     at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:712)
27-06-2018 03:32:33 PDT Spark123 INFO -     at org.apache.hadoop.ipc.Client$Connection.access$2900(Client.java:375)
27-06-2018 03:32:33 PDT Spark123 INFO -     at org.apache.hadoop.ipc.Client.getConnection(Client.java:1528)
27-06-2018 03:32:33 PDT Spark123 INFO -     at org.apache.hadoop.ipc.Client.call(Client.java:1451)
27-06-2018 03:32:33 PDT Spark123 INFO -     ... 25 more
27-06-2018 03:32:33 PDT Spark123 INFO - Process completed unsuccessfully in 1214 seconds.
27-06-2018 03:32:33 PDT Spark123 ERROR - Job run failed!

import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem
import org.apache.log4j._
import org.apache.spark._
import org.apache.spark.sql._
import org.apache.spark.SparkContext._
import org.apache.spark.sql.functions._
import org.apache.hadoop.fs.Path
import org.apache.spark.sql.types._

object Test {
  def main(args: Array[String]){)
    // Setting log level to errors
    Logger.getLogger("org").setLevel(Level.ERROR)

    // Setting up the sparksession
    val spark = SparkSession.builder
      .appName("Journaling")
      .config("spark.master","yarn")
      .getOrCreate()
    val sc = SparkContext.getOrCreate()

    // Configuration for reading file from HDFS
    val conf = new Configuration()
    conf.set("fs.defaultFS", "hdfs://localhost:8020")
    val fs= FileSystem.get(conf)
    val df = spark.read.format("csv")
      .option("inferschema", "true")
      .option("header", "true")
      .load("hdfs://localhost:8020/fakefriends.csv")

    df.show()
  }
}