0
Ich habe einen seltsamen Fehler beim Laufen aws Kinesis auf Hadoop (CDH 5.5.1).NoClassDefFoundError joda DateTimeFormat mit AWS-Kinesis auf Hadoop Cloudera
ich den Fehler mit dieser einfachen Klasse wiedergeben kann:
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.kinesis.AmazonKinesisClient;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.joda.time.format.DateTimeFormat;
public class CollectJob extends Configured implements Tool {
public static void main(String[] args) throws Exception {
int exitCode = ToolRunner.run(new CollectJob(), args);
System.exit(exitCode);
}
@Override
public int run(String[] args) throws Exception {
System.out.println(DateTimeFormat.class);
System.out.println(DateTimeFormat.class.getProtectionDomain().getCodeSource().getLocation());
System.out.println(AmazonKinesisClient.class);
System.out.println(AmazonKinesisClient.class.getProtectionDomain().getCodeSource().getLocation());
AWSCredentials credentials = new BasicAWSCredentials("toto", "tata");
AmazonKinesisClient kinesisClient = new AmazonKinesisClient(credentials);
kinesisClient.setEndpoint("kinesis.eu-west-1.amazonaws.com");
return 0;
}
}
Mein Pom:
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>test</groupId>
<artifactId>event-collector</artifactId>
<version>1.0-SNAPSHOT</version>
<repositories>
<repository>
<id>cloudera-repo-releases</id>
<url>https://repository.cloudera.com/artifactory/repo/</url>
</repository>
</repositories>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.6.0-cdh5.5.1</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<version>2.6.0-mr1-cdh5.5.1</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.7.4</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.4</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.7</source>
<target>1.7</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>reference.conf</resource>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
Dann laufe ich mein Haupt:
yarn jar event-collector-1.0-SNAPSHOT.jar CollectJob
Und erhielt diese stdout:
class org.joda.time.format.DateTimeFormat
file:/tmp/hadoop-unjar3521528256461676644/
class com.amazonaws.services.kinesis.AmazonKinesisClient
file:/opt/cloudera/parcels/CDH-5.5.1-1.cdh5.5.1.p0.11/jars/aws-java-sdk-1.7.4.jar
Exception in thread "main" java.lang.NoClassDefFoundError: org/joda/time/format/DateTimeFormat
at com.amazonaws.auth.AWS4Signer.<clinit>(AWS4Signer.java:44)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
at java.lang.Class.newInstance(Class.java:374)
at com.amazonaws.auth.SignerFactory.createSigner(SignerFactory.java:119)
at com.amazonaws.auth.SignerFactory.lookupAndCreateSigner(SignerFactory.java:105)
at com.amazonaws.auth.SignerFactory.getSigner(SignerFactory.java:78)
at com.amazonaws.AmazonWebServiceClient.computeSignerByServiceRegion(AmazonWebServiceClient.java:307)
at com.amazonaws.AmazonWebServiceClient.computeSignerByURI(AmazonWebServiceClient.java:280)
at com.amazonaws.AmazonWebServiceClient.setEndpoint(AmazonWebServiceClient.java:160)
at com.amazonaws.services.kinesis.AmazonKinesisClient.setEndpoint(AmazonKinesisClient.java:2102)
at com.amazonaws.services.kinesis.AmazonKinesisClient.init(AmazonKinesisClient.java:216)
at com.amazonaws.services.kinesis.AmazonKinesisClient.<init>(AmazonKinesisClient.java:139)
at com.amazonaws.services.kinesis.AmazonKinesisClient.<init>(AmazonKinesisClient.java:116)
at tivan.CollectJob.run(CollectJob.java:29)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:84)
at tivan.CollectJob.main(CollectJob.java:15)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.hadoop.util.RunJar.run(RunJar.java:221)
at org.apache.hadoop.util.RunJar.main(RunJar.java:136)
Caused by: java.lang.ClassNotFoundException: org.joda.time.format.DateTimeFormat
at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
... 26 more
Ich habe versucht, entweder mit YARN_USER_CLASSPATH_FIRST True oder False und erhalten den gleichen Fehler. Ich verstehe nicht, was los ist ...
Hilfe! :)
Bearbeiten mit mehr Details: auch mit fehlschlagen:
@Override
public int run(String[] args) throws Exception {
//print DateTimeFormat class infos
System.out.println(DateTimeFormat.class);
System.out.println(DateTimeFormat.class.getProtectionDomain().getCodeSource().getLocation());
//print AWS4Signer class infos
System.out.println(AWS4Signer.class);
System.out.println(AWS4Signer.class.getProtectionDomain().getCodeSource().getLocation());
//test joda time class (copy-paste from AWS4Signer source code)
DateTimeFormatter dateFormatter = DateTimeFormat.forPattern("yyyyMMdd").withZoneUTC();
System.out.println(dateFormatter);
//create AWS4Signer instance -> ClassNotFound on DateTimeFormat !!!
AWS4Signer signer = new AWS4Signer();
System.out.println(signer);
return 0;
}
Wie hast du dein Glas gebaut? Es zeigt deutlich, dass das joda Zeitglas fehlt, aber Sie haben das Glas in den Pom eingeschlossen, bedeutet, dass Sie etwas verpassen, während Sie den Job bauen. –
"mvn install" mit maven-shade-plugin. Das Joda ist gut in meinem Glas verpackt, weil ich die fehlende Klasse in meinem Haupt nennen kann (siehe zuerst System.out.println). Ich habe die Frage mit meinem vollen Pom aktualisiert. – cvx