使用Maven打包和运行Scala Spark项目

我正在用Scala编写一个应用程序，使用。我使用Maven打包应用程序，在构建应用程序时遇到问题

我面临的问题是，在IDE中运行应用程序可以正常工作，或者如果我提供非uber jar版本的依赖项作为java类路径，但如果我将uber jar作为类路径，即

java -Xmx2G -cp target/spark-example-0.1-SNAPSHOT-jar-with-dependencies.jar debug.spark_example.Example data.txt 

不起作用。我收到以下错误消息：

ERROR SparkContext: Error initializing SparkContext.
com.typesafe.config.ConfigException$Missing: No configuration setting found for key 'akka.version'
    at com.typesafe.config.impl.SimpleConfig.findKey(SimpleConfig.java:124)
    at com.typesafe.config.impl.SimpleConfig.find(SimpleConfig.java:145)
    at com.typesafe.config.impl.SimpleConfig.find(SimpleConfig.java:151)
    at com.typesafe.config.impl.SimpleConfig.find(SimpleConfig.java:159)
    at com.typesafe.config.impl.SimpleConfig.find(SimpleConfig.java:164)
    at com.typesafe.config.impl.SimpleConfig.getString(SimpleConfig.java:206)
    at akka.actor.ActorSystem$Settings.<init>(ActorSystem.scala:168)
    at akka.actor.ActorSystemImpl.<init>(ActorSystem.scala:504)
    at akka.actor.ActorSystem$.apply(ActorSystem.scala:141)
    at akka.actor.ActorSystem$.apply(ActorSystem.scala:118)
    at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:122)
    at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:54)
    at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53)
    at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1991)
    at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:160)
    at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1982)
    at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:56)
    at org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:245)
    at org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:52)
    at org.apache.spark.SparkEnv$.create(SparkEnv.scala:247)
    at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:188)
    at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:267)
    at org.apache.spark.SparkContext.<init>(SparkContext.scala:424)
    at debug.spark_example.Example$.main(Example.scala:9)
    at debug.spark_example.Example.main(Example.scala)

以下是pom.xml文件：

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>debug.spark-example</groupId>
  <artifactId>spark-example</artifactId>
  <version>0.1-SNAPSHOT</version>
  <inceptionYear>2015</inceptionYear>
  <properties>
    <scala.majorVersion>2.11</scala.majorVersion>
    <scala.minorVersion>.2</scala.minorVersion>
    <spark.version>1.4.1</spark.version>
  </properties>


  <repositories>
    <repository>
      <id>scala-tools.org</id>
      <name>Scala-Tools Maven2 Repository</name>
      <url>http://scala-tools.org/repo-releases</url>
    </repository>
  </repositories>

  <pluginRepositories>
    <pluginRepository>
      <id>scala-tools.org</id>
      <name>Scala-Tools Maven2 Repository</name>
      <url>http://scala-tools.org/repo-releases</url>
    </pluginRepository>
  </pluginRepositories>

  <dependencies>
    <dependency>
      <groupId>org.scala-lang</groupId>
      <artifactId>scala-library</artifactId>
      <version>${scala.majorVersion}${scala.minorVersion}</version>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-core_${scala.majorVersion}</artifactId>
      <version>${spark.version}</version>
    </dependency>
  </dependencies>


  <build>
      <sourceDirectory>src/main/scala</sourceDirectory>
      <plugins>
        <plugin>
          <groupId>org.scala-tools</groupId>
          <artifactId>maven-scala-plugin</artifactId>
          <executions>
            <execution>
              <goals>
                <goal>compile</goal>
                <goal>testCompile</goal>
              </goals>
            </execution>
          </executions>
        </plugin>
        <plugin>
          <groupId>org.apache.maven.plugins</groupId>
      <artifactId>maven-eclipse-plugin</artifactId>
      <configuration>
        <downloadSources>true</downloadSources>
        <buildcommands>
          <buildcommand>ch.epfl.lamp.sdt.core.scalabuilder</buildcommand>
        </buildcommands>
        <additionalProjectnatures>
          <projectnature>ch.epfl.lamp.sdt.core.scalanature</projectnature>
        </additionalProjectnatures>
        <classpathContainers>
          <classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINER</classpathContainer>
          <classpathContainer>ch.epfl.lamp.sdt.launching.SCALA_CONTAINER</classpathContainer>
        </classpathContainers>
      </configuration>
    </plugin>
    <plugin>
      <artifactId>maven-assembly-plugin</artifactId>
      <version>2.4</version>
      <executions>
        <execution>
          <id>make-assembly</id>
          <phase>package</phase>
          <goals>
            <goal>attached</goal>
          </goals>
        </execution>
      </executions>
      <configuration>
        <tarLongFileMode>gnu</tarLongFileMode>
        <descriptorRefs>
          <descriptorRef>jar-with-dependencies</descriptorRef>
        </descriptorRefs>
      </configuration>
    </plugin>
    <plugin>
      <groupId>org.apache.maven.plugins</groupId>
      <artifactId>maven-shade-plugin</artifactId>
      <version>2.2</version>
      <executions>
        <execution>
          <phase>package</phase>
          <goals>
            <goal>shade</goal>
          </goals>
          <configuration>
            <minimizeJar>false</minimizeJar>
            <createDependencyReducedPom>false</createDependencyReducedPom>
            <artifactSet>
              <includes>
                <!-- Include here the dependencies you want to be packed in your fat jar -->
                <include>*:*</include>
              </includes>
            </artifactSet>
            <filters>
              <filter>
                <artifact>*:*</artifact>
                <excludes>
                  <exclude>META-INF/*.SF</exclude>
                  <exclude>META-INF/*.DSA</exclude>
                  <exclude>META-INF/*.RSA</exclude>
                </excludes>
              </filter>
            </filters>
            <transformers>
              <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
                <resource>reference.conf</resource>
              </transformer>
            </transformers>
          </configuration>
        </execution>
      </executions>
    </plugin>
    <plugin>
      <groupId>org.apache.maven.plugins</groupId>
      <artifactId>maven-surefire-plugin</artifactId>
      <version>2.7</version>
      <configuration>
        <skipTests>true</skipTests>
      </configuration>
    </plugin>
  </plugins>
</build>
<reporting>
  <plugins>
    <plugin>
      <groupId>org.scala-tools</groupId>
      <artifactId>maven-scala-plugin</artifactId>
    </plugin>
  </plugins>
</reporting>
</project>

4.0.0
debug.spark示例
火花示例
0.1-1快照
2015
2.11
2.
1.4.1
scala-tools.org
Scala工具Maven2存储库
http://scala-tools.org/repo-releases
scala-tools.org
Scala工具Maven2存储库
http://scala-tools.org/repo-releases
org.scala-lang
scala图书馆
${scala.majorVersion}${scala.minorVersion}
org.apache.spark
spark-core{scala.majorVersion}
${spark.version}
src/main/scala
org.scala-tools
maven scala插件
编译
测试编译
org.apache.maven.plugins
maven eclipse插件
真的
ch.epfl.lamp.sdt.core.scalabuilder
ch.epfl.lamp.sdt.core.scalanature
org.eclipse.jdt.launching.JRE_容器
ch.epfl.lamp.sdt.launching.SCALA_集装箱
maven汇编插件
2.4
组装
包裹
附属的
gnu
带有依赖项的jar
org.apache.maven.plugins
maven阴影插件
2.2
包裹
阴凉处
假的
假的
*:*
*:*
META-INF/*.SF
META-INF/*.DSA
META-INF/*.RSA
reference.conf
org.apache.maven.plugins
maven surefire插件
2.7
真的
org.scala-tools
maven scala插件

---

非常感谢你的帮助

这可能与maven插件的顺序有关。您在项目中同时使用“maven assembly插件”和“maven shade插件”，这两个插件都绑定到maven生命周期的同一阶段。当这种情况发生时，maven按照插件在插件部分中出现的顺序执行插件，因此在您的例子中，它执行汇编插件，然后执行shade插件

根据您试图运行的输出jar和您拥有的阴影转换，您可能想要相反的顺序。但是，您可能甚至不需要用于您的用例的汇编插件。您可能可以使用

target/spark-example-0.1-SNAPSHOT-shaded.jar

文件

<plugins>
  <plugin>
    <groupId>org.apache.maven.plugins</groupId>
    <artifactId>maven-shade-plugin</artifactId>
    <!-- SNIP -->
  </plugin>
  <plugin>
    <artifactId>maven-assembly-plugin</artifactId>
    <!-- SNIP -->
  </plugin>
</plugins>

org.apache.maven.plugins
maven阴影插件
maven汇编插件

似乎必须使用Spark submit脚本来运行程序

而不是：

java -Xmx2G -cp target/spark-example-0.1-SNAPSHOT-jar-with-dependencies.jar debug.spark_example.Example data.txt 

做一些类似于：

<path-to>/spark-1.4.1/bin/spark-submit --class debug.spark_example.Example --master local[2] target/spark-example-0.1-SNAPSHOT-jar-with-dependencies.jar data.txt

/spark-1.4.1/bin/spark-submit--class debug.spark_example.example--master local[2]target/spark-example-0.1-SNAPSHOT-jar-with-dependencies.jar data.txt

它似乎在没有阴影的罐子的情况下也能工作；只有具有依赖项的jar。以下pom.xml文件适用于我：

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
     xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <groupId>debug.spark-example</groupId>
    <artifactId>spark-example</artifactId>
    <version>0.1-SNAPSHOT</version>
    <inceptionYear>2015</inceptionYear>
    <properties>
        <scala.majorVersion>2.11</scala.majorVersion>
        <scala.minorVersion>.2</scala.minorVersion>
        <spark.version>1.4.1</spark.version>
    </properties>
    <repositories>
        <repository>
            <id>scala-tools.org</id>
            <name>Scala-Tools Maven2 Repository</name>
            <url>http://scala-tools.org/repo-releases</url>
        </repository>
    </repositories>
    <pluginRepositories>
        <pluginRepository>
            <id>scala-tools.org</id>
            <name>Scala-Tools Maven2 Repository</name>
            <url>http://scala-tools.org/repo-releases</url>
        </pluginRepository>
    </pluginRepositories>
    <dependencies>
        <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-library</artifactId>
            <version>${scala.majorVersion}${scala.minorVersion}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_${scala.majorVersion}</artifactId>
            <version>${spark.version}</version>
        </dependency>
    </dependencies>
    <build>
        <sourceDirectory>src/main/scala</sourceDirectory>
        <plugins>
            <plugin>
                <groupId>org.scala-tools</groupId>
                <artifactId>maven-scala-plugin</artifactId>
                <executions>
                    <execution>
                        <goals>
                            <goal>compile</goal>
                            <goal>testCompile</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-eclipse-plugin</artifactId>
                <configuration>
                    <downloadSources>true</downloadSources>
                    <buildcommands>
                        <buildcommand>ch.epfl.lamp.sdt.core.scalabuilder</buildcommand>
                    </buildcommands>
                    <additionalProjectnatures>
                        <projectnature>ch.epfl.lamp.sdt.core.scalanature</projectnature>
                    </additionalProjectnatures>
                <classpathContainers>
                    <classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINER</classpathContainer>
                    <classpathContainer>ch.epfl.lamp.sdt.launching.SCALA_CONTAINER</classpathContainer>
                </classpathContainers>
                </configuration>
            </plugin>
            <plugin>
                <artifactId>maven-assembly-plugin</artifactId>
                <version>2.4</version>
                <executions>
                <execution>
                    <id>make-assembly</id>
                    <phase>package</phase>
                    <goals>
                        <goal>attached</goal>
                    </goals>
                </execution>
            </executions>
            <configuration>
                <tarLongFileMode>gnu</tarLongFileMode>
                <descriptorRefs>
                    <descriptorRef>jar-with-dependencies</descriptorRef>
                </descriptorRefs>
            </configuration>
        </plugin>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-surefire-plugin</artifactId>
            <version>2.7</version>
            <configuration>
                <skipTests>true</skipTests>
            </configuration>
        </plugin>
    </plugins>
</build>
<reporting>
    <plugins>
        <plugin>
            <groupId>org.scala-tools</groupId>
            <artifactId>maven-scala-plugin</artifactId>
        </plugin>
    </plugins>
</reporting>
</project>

4.0.0
debug.spark示例
火花示例
0.1-1快照
2015
2.11
2.
1.4.1
scala-tools.org
Scala工具Maven2存储库
http://scala-tools.org/repo-releases
scala-tools.org
Scala工具Maven2存储库
http://scala-tools.org/repo-releases
org.scala-lang
scala图书馆
${scala.majorVersion}${scala.minorVersion}
org.apache.spark
spark-core{scala.majorVersion}
${spark.version}
src/main/scala
org.scala-tools
maven scala插件
编译
测试编译
org.apache.maven.plugins
maven eclipse插件
真的
ch.epfl.lamp.sdt.core.scalabuilder
ch.epfl.lamp.sdt.core.scalanature
org.eclipse.jdt.launching.JRE_容器
ch.epfl.lamp.sdt.launching.SCALA_集装箱
maven汇编插件
2.4
组装
包裹
附属的
gnu
带有依赖项的jar
org.apache.maven.plugins
maven surefire插件
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
     xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <groupId>debug.spark-example</groupId>
    <artifactId>spark-example</artifactId>
    <version>0.1-SNAPSHOT</version>
    <inceptionYear>2015</inceptionYear>
    <properties>
        <scala.majorVersion>2.11</scala.majorVersion>
        <scala.minorVersion>.2</scala.minorVersion>
        <spark.version>1.4.1</spark.version>
    </properties>
    <repositories>
        <repository>
            <id>scala-tools.org</id>
            <name>Scala-Tools Maven2 Repository</name>
            <url>http://scala-tools.org/repo-releases</url>
        </repository>
    </repositories>
    <pluginRepositories>
        <pluginRepository>
            <id>scala-tools.org</id>
            <name>Scala-Tools Maven2 Repository</name>
            <url>http://scala-tools.org/repo-releases</url>
        </pluginRepository>
    </pluginRepositories>
    <dependencies>
        <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-library</artifactId>
            <version>${scala.majorVersion}${scala.minorVersion}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_${scala.majorVersion}</artifactId>
            <version>${spark.version}</version>
        </dependency>
    </dependencies>
    <build>
        <sourceDirectory>src/main/scala</sourceDirectory>
        <plugins>
            <plugin>
                <groupId>org.scala-tools</groupId>
                <artifactId>maven-scala-plugin</artifactId>
                <executions>
                    <execution>
                        <goals>
                            <goal>compile</goal>
                            <goal>testCompile</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-eclipse-plugin</artifactId>
                <configuration>
                    <downloadSources>true</downloadSources>
                    <buildcommands>
                        <buildcommand>ch.epfl.lamp.sdt.core.scalabuilder</buildcommand>
                    </buildcommands>
                    <additionalProjectnatures>
                        <projectnature>ch.epfl.lamp.sdt.core.scalanature</projectnature>
                    </additionalProjectnatures>
                <classpathContainers>
                    <classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINER</classpathContainer>
                    <classpathContainer>ch.epfl.lamp.sdt.launching.SCALA_CONTAINER</classpathContainer>
                </classpathContainers>
                </configuration>
            </plugin>
            <plugin>
                <artifactId>maven-assembly-plugin</artifactId>
                <version>2.4</version>
                <executions>
                <execution>
                    <id>make-assembly</id>
                    <phase>package</phase>
                    <goals>
                        <goal>attached</goal>
                    </goals>
                </execution>
            </executions>
            <configuration>
                <tarLongFileMode>gnu</tarLongFileMode>
                <descriptorRefs>
                    <descriptorRef>jar-with-dependencies</descriptorRef>
                </descriptorRefs>
            </configuration>
        </plugin>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-surefire-plugin</artifactId>
            <version>2.7</version>
            <configuration>
                <skipTests>true</skipTests>
            </configuration>
        </plugin>
    </plugins>
</build>
<reporting>
    <plugins>
        <plugin>
            <groupId>org.scala-tools</groupId>
            <artifactId>maven-scala-plugin</artifactId>
        </plugin>
    </plugins>
</reporting>
</project>

                            <transformer
                                    implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
                                <resource>reference.conf</resource>
                            </transformer>
                            <transformer
                                    implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                                <manifestEntries>
                                    <Main-Class>akka.Main</Main-Class>
                                </manifestEntries>
                            </transformer>