Java 在hadoop cloudera上使用aws kinesis的NoClassDefFoundError joda DateTimeFormat
我在hadoop(CDH5.5.1)上运行aws kinesis时遇到了一个奇怪的错误 我可以用这个简单的类重现错误:Java 在hadoop cloudera上使用aws kinesis的NoClassDefFoundError joda DateTimeFormat,java,amazon-web-services,hadoop,cloudera,amazon-kinesis,Java,Amazon Web Services,Hadoop,Cloudera,Amazon Kinesis,我在hadoop(CDH5.5.1)上运行aws kinesis时遇到了一个奇怪的错误 我可以用这个简单的类重现错误: import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.services.kinesis.AmazonKinesisClient; import org.apache.hadoop.conf.Configured;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.kinesis.AmazonKinesisClient;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.joda.time.format.DateTimeFormat;
public class CollectJob extends Configured implements Tool {
public static void main(String[] args) throws Exception {
int exitCode = ToolRunner.run(new CollectJob(), args);
System.exit(exitCode);
}
@Override
public int run(String[] args) throws Exception {
System.out.println(DateTimeFormat.class);
System.out.println(DateTimeFormat.class.getProtectionDomain().getCodeSource().getLocation());
System.out.println(AmazonKinesisClient.class);
System.out.println(AmazonKinesisClient.class.getProtectionDomain().getCodeSource().getLocation());
AWSCredentials credentials = new BasicAWSCredentials("toto", "tata");
AmazonKinesisClient kinesisClient = new AmazonKinesisClient(credentials);
kinesisClient.setEndpoint("kinesis.eu-west-1.amazonaws.com");
return 0;
}
}
我的pom:
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>test</groupId>
<artifactId>event-collector</artifactId>
<version>1.0-SNAPSHOT</version>
<repositories>
<repository>
<id>cloudera-repo-releases</id>
<url>https://repository.cloudera.com/artifactory/repo/</url>
</repository>
</repositories>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.6.0-cdh5.5.1</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<version>2.6.0-mr1-cdh5.5.1</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.7.4</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.4</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.7</source>
<target>1.7</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>reference.conf</resource>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
得到了这个标准:
class org.joda.time.format.DateTimeFormat
file:/tmp/hadoop-unjar3521528256461676644/
class com.amazonaws.services.kinesis.AmazonKinesisClient
file:/opt/cloudera/parcels/CDH-5.5.1-1.cdh5.5.1.p0.11/jars/aws-java-sdk-1.7.4.jar
Exception in thread "main" java.lang.NoClassDefFoundError: org/joda/time/format/DateTimeFormat
at com.amazonaws.auth.AWS4Signer.<clinit>(AWS4Signer.java:44)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
at java.lang.Class.newInstance(Class.java:374)
at com.amazonaws.auth.SignerFactory.createSigner(SignerFactory.java:119)
at com.amazonaws.auth.SignerFactory.lookupAndCreateSigner(SignerFactory.java:105)
at com.amazonaws.auth.SignerFactory.getSigner(SignerFactory.java:78)
at com.amazonaws.AmazonWebServiceClient.computeSignerByServiceRegion(AmazonWebServiceClient.java:307)
at com.amazonaws.AmazonWebServiceClient.computeSignerByURI(AmazonWebServiceClient.java:280)
at com.amazonaws.AmazonWebServiceClient.setEndpoint(AmazonWebServiceClient.java:160)
at com.amazonaws.services.kinesis.AmazonKinesisClient.setEndpoint(AmazonKinesisClient.java:2102)
at com.amazonaws.services.kinesis.AmazonKinesisClient.init(AmazonKinesisClient.java:216)
at com.amazonaws.services.kinesis.AmazonKinesisClient.<init>(AmazonKinesisClient.java:139)
at com.amazonaws.services.kinesis.AmazonKinesisClient.<init>(AmazonKinesisClient.java:116)
at tivan.CollectJob.run(CollectJob.java:29)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:84)
at tivan.CollectJob.main(CollectJob.java:15)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.hadoop.util.RunJar.run(RunJar.java:221)
at org.apache.hadoop.util.RunJar.main(RunJar.java:136)
Caused by: java.lang.ClassNotFoundException: org.joda.time.format.DateTimeFormat
at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
... 26 more
您可以为pom.xml尝试以下方法:-
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>test</groupId>
<artifactId>event-collector</artifactId>
<version>1.0-SNAPSHOT</version>
<repositories>
<repository>
<id>cloudera-repo-releases</id>
<url>https://repository.cloudera.com/artifactory/repo/</url>
</repository>
</repositories>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.6.0-cdh5.5.1</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<version>2.6.0-mr1-cdh5.5.1</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.7.4</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.4</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.7</source>
<target>1.7</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
4.0.0
测试
事件收集器
1.0-快照
cloudera回购协议发布
https://repository.cloudera.com/artifactory/repo/
org.apache.hadoop
hadoop通用
2.6.0-cdh5.5.1
假如
org.apache.hadoop
hadoop内核
2.6.0-mr1-cdh5.5.1
假如
亚马逊网站
aws java sdk
1.7.4
假如
乔达时间
乔达时间
2.4
org.apache.maven.plugins
maven编译器插件
1.7
1.7
org.apache.maven.plugins
maven阴影插件
包裹
阴凉处
您可以为pom.xml尝试以下方法吗:-
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>test</groupId>
<artifactId>event-collector</artifactId>
<version>1.0-SNAPSHOT</version>
<repositories>
<repository>
<id>cloudera-repo-releases</id>
<url>https://repository.cloudera.com/artifactory/repo/</url>
</repository>
</repositories>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.6.0-cdh5.5.1</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<version>2.6.0-mr1-cdh5.5.1</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.7.4</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.4</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.7</source>
<target>1.7</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
4.0.0
测试
事件收集器
1.0-快照
cloudera回购协议发布
https://repository.cloudera.com/artifactory/repo/
org.apache.hadoop
hadoop通用
2.6.0-cdh5.5.1
假如
org.apache.hadoop
hadoop内核
2.6.0-mr1-cdh5.5.1
假如
亚马逊网站
aws java sdk
1.7.4
假如
乔达时间
乔达时间
2.4
org.apache.maven.plugins
maven编译器插件
1.7
1.7
org.apache.maven.plugins
maven阴影插件
包裹
阴凉处
我终于找到了解决方案,我只需导出以下变量:
export YARN_USER_CLASSPATH=pathtomyjar
我终于找到了解决方案,我只需要导出以下变量:
export YARN_USER_CLASSPATH=pathtomyjar
你是如何建造你的罐子的?它清楚地表明joda time jar丢失了,但是您已经在pom中包含了这个jar,这意味着您在构建作业时丢失了一些东西。joda在我的jar中得到了很好的打包,因为我可以在main中调用缺少的类(cf first System.out.println)。我用我的完整pom更新了这个问题。你是如何构建你的jar的?它清楚地表明joda time jar丢失了,但是您已经在pom中包含了这个jar,这意味着您在构建作业时丢失了一些东西。joda在我的jar中得到了很好的打包,因为我可以在main中调用缺少的类(cf first System.out.println)。我用完整的pom更新了这个问题。相同的错误java.lang.NoClassDefFoundError:org/joda/time/format/dateTimeFormatName错误java.lang.NoClassDefFoundError:org/joda/time/format/DateTimeFormat