Spark-Initialisierungsfehler bei Verwendung von HiveContext in Spark

Question

Unten sind meine Codes. Ich habe ein Problem mit HiveContext konfrontiert.

import org.apache.spark.SparkConf; 
import org.apache.spark.api.java.JavaRDD; 
import org.apache.spark.api.java.JavaSparkContext; 
import org.apache.spark.api.java.function.*; 
import org.apache.spark.sql.DataFrame; 
import org.apache.spark.sql.SQLContext; 
import org.apache.spark.sql.hive.HiveContext; 
import static org.apache.spark.sql.functions.*; 

public class Main { 
    public static void main(String[] args) throws Exception { 

     SparkConf conf = new SparkConf().setAppName("jdf-dt-rtoc-withSQL").set("spark.serializer", "org.apache.spark.serializer.KryoSerializer").setMaster("local[*]"); 
     JavaSparkContext sc = new JavaSparkContext(conf); 

     SQLContext sqlContext = new HiveContext(sc); // The error occurred. 

     //DataFrame df = sqlContext.createDataFrame(pairRDD,Pair.class); 
     //DataFrame takenTogether1 = df.groupBy("fpart").agg(collect_list(col("rpart"))); 

     sc.stop(); 
     sc.close(); 
    } 
} 

Meine Funken Version ist 1.6.0 und fand ich einen Thread (Use collect_list and collect_set in Spark SQL), die ich collect_list durch den Import von HiveContext verwenden können. Als ich es jedoch versuchte, warf es einen weiteren Fehler wie unten.

/usr/local/java/latest/bin/java -Didea.launcher.port=7536 -Didea.launcher.bin.path=/usr/local/intellij/idea-IC-163.10154.41/bin -Dfile.encoding=UTF-8 -classpath /usr/local/java/latest/jre/lib/charsets.jar:/usr/local/java/latest/jre/lib/deploy.jar:/usr/local/java/latest/jre/lib/ext/cldrdata.jar:/usr/local/java/latest/jre/lib/ext/dnsns.jar:/usr/local/java/latest/jre/lib/ext/jaccess.jar:/usr/local/java/latest/jre/lib/ext/jfxrt.jar:/usr/local/java/latest/jre/lib/ext/localedata.jar:/usr/local/java/latest/jre/lib/ext/nashorn.jar:/usr/local/java/latest/jre/lib/ext/sunec.jar:/usr/local/java/latest/jre/lib/ext/sunjce_provider.jar:/usr/local/java/latest/jre/lib/ext/sunpkcs11.jar:/usr/local/java/latest/jre/lib/ext/zipfs.jar:/usr/local/java/latest/jre/lib/javaws.jar:/usr/local/java/latest/jre/lib/jce.jar:/usr/local/java/latest/jre/lib/jfr.jar:/usr/local/java/latest/jre/lib/jfxswt.jar:/usr/local/java/latest/jre/lib/jsse.jar:/usr/local/java/latest/jre/lib/management-agent.jar:/usr/local/java/latest/jre/lib/plugin.jar:/usr/local/java/latest/jre/lib/resources.jar:/usr/local/java/latest/jre/lib/rt.jar:/home/dst/02_Docu/01_Intellij_workspace/jdfdtrtocwithSQL/target/classes:/home/dst/.m2/repository/org/apache/spark/spark-core_2.10/1.6.0/spark-core_2.10-1.6.0.jar:/home/dst/.m2/repository/org/apache/avro/avro-mapred/1.7.7/avro-mapred-1.7.7-hadoop2.jar:/home/dst/.m2/repository/org/apache/avro/avro-ipc/1.7.7/avro-ipc-1.7.7.jar:/home/dst/.m2/repository/org/apache/avro/avro-ipc/1.7.7/avro-ipc-1.7.7-tests.jar:/home/dst/.m2/repository/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/home/dst/.m2/repository/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/home/dst/.m2/repository/com/twitter/chill_2.10/0.5.0/chill_2.10-0.5.0.jar:/home/dst/.m2/repository/com/esotericsoftware/kryo/kryo/2.21/kryo-2.21.jar:/home/dst/.m2/repository/com/esotericsoftware/reflectasm/reflectasm/1.07/reflectasm-1.07-shaded.jar:/home/dst/.m2/repository/com/esotericsoftware/minlog/minlog/1.2/minlog-1.2.jar:/home/dst/.m2/repository/org/objenesis/objenesis/1.2/objenesis-1.2.jar:/home/dst/.m2/repository/com/twitter/chill-java/0.5.0/chill-java-0.5.0.jar:/home/dst/.m2/repository/org/apache/xbean/xbean-asm5-shaded/4.4/xbean-asm5-shaded-4.4.jar:/home/dst/.m2/repository/org/apache/hadoop/hadoop-client/2.2.0/hadoop-client-2.2.0.jar:/home/dst/.m2/repository/org/apache/hadoop/hadoop-common/2.2.0/hadoop-common-2.2.0.jar:/home/dst/.m2/repository/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/home/dst/.m2/repository/org/apache/commons/commons-math/2.1/commons-math-2.1.jar:/home/dst/.m2/repository/xmlenc/xmlenc/0.52/xmlenc-0.52.jar:/home/dst/.m2/repository/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar:/home/dst/.m2/repository/commons-collections/commons-collections/3.2.1/commons-collections-3.2.1.jar:/home/dst/.m2/repository/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/home/dst/.m2/repository/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar:/home/dst/.m2/repository/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar:/home/dst/.m2/repository/org/apache/hadoop/hadoop-auth/2.2.0/hadoop-auth-2.2.0.jar:/home/dst/.m2/repository/org/apache/commons/commons-compress/1.4.1/commons-compress-1.4.1.jar:/home/dst/.m2/repository/org/tukaani/xz/1.0/xz-1.0.jar:/home/dst/.m2/repository/org/apache/hadoop/hadoop-hdfs/2.2.0/hadoop-hdfs-2.2.0.jar:/home/dst/.m2/repository/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar:/home/dst/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-app/2.2.0/hadoop-mapreduce-client-app-2.2.0.jar:/home/dst/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-common/2.2.0/hadoop-mapreduce-client-common-2.2.0.jar:/home/dst/.m2/repository/org/apache/hadoop/hadoop-yarn-client/2.2.0/hadoop-yarn-client-2.2.0.jar:/home/dst/.m2/repository/com/google/inject/guice/3.0/guice-3.0.jar:/home/dst/.m2/repository/javax/inject/javax.inject/1/javax.inject-1.jar:/home/dst/.m2/repository/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/home/dst/.m2/repository/com/sun/jersey/jersey-test-framework/jersey-test-framework-grizzly2/1.9/jersey-test-framework-grizzly2-1.9.jar:/home/dst/.m2/repository/com/sun/jersey/jersey-test-framework/jersey-test-framework-core/1.9/jersey-test-framework-core-1.9.jar:/home/dst/.m2/repository/javax/servlet/javax.servlet-api/3.0.1/javax.servlet-api-3.0.1.jar:/home/dst/.m2/repository/com/sun/jersey/jersey-client/1.9/jersey-client-1.9.jar:/home/dst/.m2/repository/com/sun/jersey/jersey-grizzly2/1.9/jersey-grizzly2-1.9.jar:/home/dst/.m2/repository/org/glassfish/grizzly/grizzly-http/2.1.2/grizzly-http-2.1.2.jar:/home/dst/.m2/repository/org/glassfish/grizzly/grizzly-framework/2.1.2/grizzly-framework-2.1.2.jar:/home/dst/.m2/repository/org/glassfish/gmbal/gmbal-api-only/3.0.0-b023/gmbal-api-only-3.0.0-b023.jar:/home/dst/.m2/repository/org/glassfish/external/management-api/3.0.0-b012/management-api-3.0.0-b012.jar:/home/dst/.m2/repository/org/glassfish/grizzly/grizzly-http-server/2.1.2/grizzly-http-server-2.1.2.jar:/home/dst/.m2/repository/org/glassfish/grizzly/grizzly-rcm/2.1.2/grizzly-rcm-2.1.2.jar:/home/dst/.m2/repository/org/glassfish/grizzly/grizzly-http-servlet/2.1.2/grizzly-http-servlet-2.1.2.jar:/home/dst/.m2/repository/org/glassfish/javax.servlet/3.1/javax.servlet-3.1.jar:/home/dst/.m2/repository/com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar:/home/dst/.m2/repository/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/home/dst/.m2/repository/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/home/dst/.m2/repository/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar:/home/dst/.m2/repository/javax/activation/activation/1.1/activation-1.1.jar:/home/dst/.m2/repository/org/codehaus/jackson/jackson-jaxrs/1.8.3/jackson-jaxrs-1.8.3.jar:/home/dst/.m2/repository/org/codehaus/jackson/jackson-xc/1.8.3/jackson-xc-1.8.3.jar:/home/dst/.m2/repository/com/sun/jersey/contribs/jersey-guice/1.9/jersey-guice-1.9.jar:/home/dst/.m2/repository/org/apache/hadoop/hadoop-yarn-server-common/2.2.0/hadoop-yarn-server-common-2.2.0.jar:/home/dst/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-shuffle/2.2.0/hadoop-mapreduce-client-shuffle-2.2.0.jar:/home/dst/.m2/repository/org/apache/hadoop/hadoop-yarn-api/2.2.0/hadoop-yarn-api-2.2.0.jar:/home/dst/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.2.0/hadoop-mapreduce-client-core-2.2.0.jar:/home/dst/.m2/repository/org/apache/hadoop/hadoop-yarn-common/2.2.0/hadoop-yarn-common-2.2.0.jar:/home/dst/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-jobclient/2.2.0/hadoop-mapreduce-client-jobclient-2.2.0.jar:/home/dst/.m2/repository/org/apache/hadoop/hadoop-annotations/2.2.0/hadoop-annotations-2.2.0.jar:/home/dst/.m2/repository/org/apache/spark/spark-launcher_2.10/1.6.0/spark-launcher_2.10-1.6.0.jar:/home/dst/.m2/repository/org/apache/spark/spark-network-common_2.10/1.6.0/spark-network-common_2.10-1.6.0.jar:/home/dst/.m2/repository/org/apache/spark/spark-network-shuffle_2.10/1.6.0/spark-network-shuffle_2.10-1.6.0.jar:/home/dst/.m2/repository/org/fusesource/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar:/home/dst/.m2/repository/com/fasterxml/jackson/core/jackson-annotations/2.4.4/jackson-annotations-2.4.4.jar:/home/dst/.m2/repository/org/apache/spark/spark-unsafe_2.10/1.6.0/spark-unsafe_2.10-1.6.0.jar:/home/dst/.m2/repository/net/java/dev/jets3t/jets3t/0.7.1/jets3t-0.7.1.jar:/home/dst/.m2/repository/commons-codec/commons-codec/1.3/commons-codec-1.3.jar:/home/dst/.m2/repository/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar:/home/dst/.m2/repository/org/apache/curator/curator-recipes/2.4.0/curator-recipes-2.4.0.jar:/home/dst/.m2/repository/org/apache/curator/curator-framework/2.4.0/curator-framework-2.4.0.jar:/home/dst/.m2/repository/org/apache/curator/curator-client/2.4.0/curator-client-2.4.0.jar:/home/dst/.m2/repository/org/apache/zookeeper/zookeeper/3.4.5/zookeeper-3.4.5.jar:/home/dst/.m2/repository/jline/jline/0.9.94/jline-0.9.94.jar:/home/dst/.m2/repository/com/google/guava/guava/14.0.1/guava-14.0.1.jar:/home/dst/.m2/repository/org/eclipse/jetty/orbit/javax.servlet/3.0.0.v201112011016/javax.servlet-3.0.0.v201112011016.jar:/home/dst/.m2/repository/org/apache/commons/commons-lang3/3.3.2/commons-lang3-3.3.2.jar:/home/dst/.m2/repository/org/apache/commons/commons-math3/3.4.1/commons-math3-3.4.1.jar:/home/dst/.m2/repository/com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar:/home/dst/.m2/repository/org/slf4j/slf4j-api/1.7.10/slf4j-api-1.7.10.jar:/home/dst/.m2/repository/org/slf4j/jul-to-slf4j/1.7.10/jul-to-slf4j-1.7.10.jar:/home/dst/.m2/repository/org/slf4j/jcl-over-slf4j/1.7.10/jcl-over-slf4j-1.7.10.jar:/home/dst/.m2/repository/log4j/log4j/1.2.17/log4j-1.2.17.jar:/home/dst/.m2/repository/org/slf4j/slf4j-log4j12/1.7.10/slf4j-log4j12-1.7.10.jar:/home/dst/.m2/repository/com/ning/compress-lzf/1.0.3/compress-lzf-1.0.3.jar:/home/dst/.m2/repository/org/xerial/snappy/snappy-java/1.1.2/snappy-java-1.1.2.jar:/home/dst/.m2/repository/net/jpountz/lz4/lz4/1.3.0/lz4-1.3.0.jar:/home/dst/.m2/repository/org/roaringbitmap/RoaringBitmap/0.5.11/RoaringBitmap-0.5.11.jar:/home/dst/.m2/repository/commons-net/commons-net/2.2/commons-net-2.2.jar:/home/dst/.m2/repository/com/typesafe/akka/akka-remote_2.10/2.3.11/akka-remote_2.10-2.3.11.jar:/home/dst/.m2/repository/com/typesafe/akka/akka-actor_2.10/2.3.11/akka-actor_2.10-2.3.11.jar:/home/dst/.m2/repository/com/typesafe/config/1.2.1/config-1.2.1.jar:/home/dst/.m2/repository/io/netty/netty/3.8.0.Final/netty-3.8.0.Final.jar:/home/dst/.m2/repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/home/dst/.m2/repository/org/uncommons/maths/uncommons-maths/1.2.2a/uncommons-maths-1.2.2a.jar:/home/dst/.m2/repository/com/typesafe/akka/akka-slf4j_2.10/2.3.11/akka-slf4j_2.10-2.3.11.jar:/home/dst/.m2/repository/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.jar:/home/dst/.m2/repository/org/json4s/json4s-jackson_2.10/3.2.10/json4s-jackson_2.10-3.2.10.jar:/home/dst/.m2/repository/org/json4s/json4s-core_2.10/3.2.10/json4s-core_2.10-3.2.10.jar:/home/dst/.m2/repository/org/json4s/json4s-ast_2.10/3.2.10/json4s-ast_2.10-3.2.10.jar:/home/dst/.m2/repository/org/scala-lang/scalap/2.10.0/scalap-2.10.0.jar:/home/dst/.m2/repository/org/scala-lang/scala-compiler/2.10.0/scala-compiler-2.10.0.jar:/home/dst/.m2/repository/com/sun/jersey/jersey-server/1.9/jersey-server-1.9.jar:/home/dst/.m2/repository/asm/asm/3.1/asm-3.1.jar:/home/dst/.m2/repository/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar:/home/dst/.m2/repository/org/apache/mesos/mesos/0.21.1/mesos-0.21.1-shaded-protobuf.jar:/home/dst/.m2/repository/io/netty/netty-all/4.0.29.Final/netty-all-4.0.29.Final.jar:/home/dst/.m2/repository/com/clearspring/analytics/stream/2.7.0/stream-2.7.0.jar:/home/dst/.m2/repository/io/dropwizard/metrics/metrics-core/3.1.2/metrics-core-3.1.2.jar:/home/dst/.m2/repository/io/dropwizard/metrics/metrics-jvm/3.1.2/metrics-jvm-3.1.2.jar:/home/dst/.m2/repository/io/dropwizard/metrics/metrics-json/3.1.2/metrics-json-3.1.2.jar:/home/dst/.m2/repository/io/dropwizard/metrics/metrics-graphite/3.1.2/metrics-graphite-3.1.2.jar:/home/dst/.m2/repository/com/fasterxml/jackson/core/jackson-databind/2.4.4/jackson-databind-2.4.4.jar:/home/dst/.m2/repository/com/fasterxml/jackson/core/jackson-core/2.4.4/jackson-core-2.4.4.jar:/home/dst/.m2/repository/com/fasterxml/jackson/module/jackson-module-scala_2.10/2.4.4/jackson-module-scala_2.10-2.4.4.jar:/home/dst/.m2/repository/org/scala-lang/scala-reflect/2.10.4/scala-reflect-2.10.4.jar:/home/dst/.m2/repository/com/thoughtworks/paranamer/paranamer/2.6/paranamer-2.6.jar:/home/dst/.m2/repository/org/apache/ivy/ivy/2.4.0/ivy-2.4.0.jar:/home/dst/.m2/repository/oro/oro/2.0.8/oro-2.0.8.jar:/home/dst/.m2/repository/org/tachyonproject/tachyon-client/0.8.2/tachyon-client-0.8.2.jar:/home/dst/.m2/repository/commons-lang/commons-lang/2.4/commons-lang-2.4.jar:/home/dst/.m2/repository/commons-io/commons-io/2.4/commons-io-2.4.jar:/home/dst/.m2/repository/org/tachyonproject/tachyon-underfs-hdfs/0.8.2/tachyon-underfs-hdfs-0.8.2.jar:/home/dst/.m2/repository/org/tachyonproject/tachyon-underfs-s3/0.8.2/tachyon-underfs-s3-0.8.2.jar:/home/dst/.m2/repository/org/tachyonproject/tachyon-underfs-local/0.8.2/tachyon-underfs-local-0.8.2.jar:/home/dst/.m2/repository/net/razorvine/pyrolite/4.9/pyrolite-4.9.jar:/home/dst/.m2/repository/net/sf/py4j/py4j/0.9/py4j-0.9.jar:/home/dst/.m2/repository/org/spark-project/spark/unused/1.0.0/unused-1.0.0.jar:/home/dst/.m2/repository/org/apache/spark/spark-sql_2.10/1.6.0/spark-sql_2.10-1.6.0.jar:/home/dst/.m2/repository/org/apache/spark/spark-catalyst_2.10/1.6.0/spark-catalyst_2.10-1.6.0.jar:/home/dst/.m2/repository/org/codehaus/janino/janino/2.7.8/janino-2.7.8.jar:/home/dst/.m2/repository/org/codehaus/janino/commons-compiler/2.7.8/commons-compiler-2.7.8.jar:/home/dst/.m2/repository/org/apache/parquet/parquet-column/1.7.0/parquet-column-1.7.0.jar:/home/dst/.m2/repository/org/apache/parquet/parquet-common/1.7.0/parquet-common-1.7.0.jar:/home/dst/.m2/repository/org/apache/parquet/parquet-encoding/1.7.0/parquet-encoding-1.7.0.jar:/home/dst/.m2/repository/org/apache/parquet/parquet-generator/1.7.0/parquet-generator-1.7.0.jar:/home/dst/.m2/repository/org/apache/parquet/parquet-hadoop/1.7.0/parquet-hadoop-1.7.0.jar:/home/dst/.m2/repository/org/apache/parquet/parquet-format/2.3.0-incubating/parquet-format-2.3.0-incubating.jar:/home/dst/.m2/repository/org/apache/parquet/parquet-jackson/1.7.0/parquet-jackson-1.7.0.jar:/home/dst/.m2/repository/org/apache/spark/spark-hivecontext-compatibility_2.10/2.0.0-preview/spark-hivecontext-compatibility_2.10-2.0.0-preview.jar:/home/dst/.m2/repository/org/apache/spark/spark-hive_2.10/2.0.0-preview/spark-hive_2.10-2.0.0-preview.jar:/home/dst/.m2/repository/com/twitter/parquet-hadoop-bundle/1.6.0/parquet-hadoop-bundle-1.6.0.jar:/home/dst/.m2/repository/org/apache/spark/spark-tags_2.10/2.0.0-preview/spark-tags_2.10-2.0.0-preview.jar:/home/dst/.m2/repository/org/scalatest/scalatest_2.10/2.2.6/scalatest_2.10-2.2.6.jar:/home/dst/.m2/repository/org/spark-project/hive/hive-exec/1.2.1.spark2/hive-exec-1.2.1.spark2.jar:/home/dst/.m2/repository/javolution/javolution/5.5.1/javolution-5.5.1.jar:/home/dst/.m2/repository/log4j/apache-log4j-extras/1.2.17/apache-log4j-extras-1.2.17.jar:/home/dst/.m2/repository/org/antlr/antlr-runtime/3.4/antlr-runtime-3.4.jar:/home/dst/.m2/repository/org/antlr/stringtemplate/3.2.1/stringtemplate-3.2.1.jar:/home/dst/.m2/repository/antlr/antlr/2.7.7/antlr-2.7.7.jar:/home/dst/.m2/repository/org/antlr/ST4/4.0.4/ST4-4.0.4.jar:/home/dst/.m2/repository/com/googlecode/javaewah/JavaEWAH/0.3.2/JavaEWAH-0.3.2.jar:/home/dst/.m2/repository/org/iq80/snappy/snappy/0.2/snappy-0.2.jar:/home/dst/.m2/repository/org/json/json/20090211/json-20090211.jar:/home/dst/.m2/repository/stax/stax-api/1.0.1/stax-api-1.0.1.jar:/home/dst/.m2/repository/net/sf/opencsv/opencsv/2.3/opencsv-2.3.jar:/home/dst/.m2/repository/org/spark-project/hive/hive-metastore/1.2.1.spark2/hive-metastore-1.2.1.spark2.jar:/home/dst/.m2/repository/com/jolbox/bonecp/0.8.0.RELEASE/bonecp-0.8.0.RELEASE.jar:/home/dst/.m2/repository/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/home/dst/.m2/repository/org/apache/derby/derby/10.10.2.0/derby-10.10.2.0.jar:/home/dst/.m2/repository/org/datanucleus/datanucleus-api-jdo/3.2.6/datanucleus-api-jdo-3.2.6.jar:/home/dst/.m2/repository/org/datanucleus/datanucleus-rdbms/3.2.9/datanucleus-rdbms-3.2.9.jar:/home/dst/.m2/repository/commons-pool/commons-pool/1.5.4/commons-pool-1.5.4.jar:/home/dst/.m2/repository/commons-dbcp/commons-dbcp/1.4/commons-dbcp-1.4.jar:/home/dst/.m2/repository/javax/jdo/jdo-api/3.0.1/jdo-api-3.0.1.jar:/home/dst/.m2/repository/javax/transaction/jta/1.1/jta-1.1.jar:/home/dst/.m2/repository/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/home/dst/.m2/repository/org/apache/calcite/calcite-avatica/1.2.0-incubating/calcite-avatica-1.2.0-incubating.jar:/home/dst/.m2/repository/org/apache/calcite/calcite-core/1.2.0-incubating/calcite-core-1.2.0-incubating.jar:/home/dst/.m2/repository/org/apache/calcite/calcite-linq4j/1.2.0-incubating/calcite-linq4j-1.2.0-incubating.jar:/home/dst/.m2/repository/net/hydromatic/eigenbase-properties/1.1.5/eigenbase-properties-1.1.5.jar:/home/dst/.m2/repository/org/apache/httpcomponents/httpclient/4.5.2/httpclient-4.5.2.jar:/home/dst/.m2/repository/org/apache/httpcomponents/httpcore/4.4.4/httpcore-4.4.4.jar:/home/dst/.m2/repository/joda-time/joda-time/2.9.3/joda-time-2.9.3.jar:/home/dst/.m2/repository/org/jodd/jodd-core/3.5.2/jodd-core-3.5.2.jar:/home/dst/.m2/repository/org/datanucleus/datanucleus-core/3.2.10/datanucleus-core-3.2.10.jar:/home/dst/.m2/repository/org/apache/thrift/libthrift/0.9.2/libthrift-0.9.2.jar:/home/dst/.m2/repository/org/apache/thrift/libfb303/0.9.2/libfb303-0.9.2.jar:/usr/local/intellij/idea-IC-163.10154.41/lib/idea_rt.jar com.intellij.rt.execution.application.AppMain Main /home/dst/input/data2 
Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties 
17/02/22 21:40:01 INFO SparkContext: Running Spark version 1.6.0 
17/02/22 21:40:01 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 
17/02/22 21:40:01 WARN Utils: Your hostname, localhost.localdomain resolves to a loopback address: 127.0.0.1; using 10.0.2.15 instead (on interface eth1) 
17/02/22 21:40:01 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address 
17/02/22 21:40:01 INFO SecurityManager: Changing view acls to: dst 
17/02/22 21:40:01 INFO SecurityManager: Changing modify acls to: dst 
17/02/22 21:40:01 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(dst); users with modify permissions: Set(dst) 
17/02/22 21:40:02 INFO Utils: Successfully started service 'sparkDriver' on port 42363. 
17/02/22 21:40:02 INFO Slf4jLogger: Slf4jLogger started 
17/02/22 21:40:02 INFO Remoting: Starting remoting 
17/02/22 21:40:02 INFO Remoting: Remoting started; listening on addresses :[akka.tcp://sparkDriverActorSystem@10.0.2.15:44169] 
17/02/22 21:40:02 INFO Utils: Successfully started service 'sparkDriverActorSystem' on port 44169. 
17/02/22 21:40:02 INFO SparkEnv: Registering MapOutputTracker 
17/02/22 21:40:02 INFO SparkEnv: Registering BlockManagerMaster 
17/02/22 21:40:02 INFO DiskBlockManager: Created local directory at /tmp/blockmgr-4a9159c9-d1c1-4e31-9837-0e4d1a2f7fe0 
17/02/22 21:40:02 INFO MemoryStore: MemoryStore started with capacity 414.0 MB 
17/02/22 21:40:03 INFO SparkEnv: Registering OutputCommitCoordinator 
17/02/22 21:40:03 INFO Utils: Successfully started service 'SparkUI' on port 4040. 
17/02/22 21:40:03 INFO SparkUI: Started SparkUI at http://10.0.2.15:4040 
17/02/22 21:40:03 INFO Executor: Starting executor ID driver on host localhost 
17/02/22 21:40:03 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 33443. 
17/02/22 21:40:03 INFO NettyBlockTransferService: Server created on 33443 
17/02/22 21:40:03 INFO BlockManagerMaster: Trying to register BlockManager 
17/02/22 21:40:03 INFO BlockManagerMasterEndpoint: Registering block manager localhost:33443 with 414.0 MB RAM, BlockManagerId(driver, localhost, 33443) 
17/02/22 21:40:03 INFO BlockManagerMaster: Registered BlockManager 
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/spark/sql/internal/SharedState 
    at Main.main(Main.java:39) 
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) 
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) 
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) 
    at java.lang.reflect.Method.invoke(Method.java:498) 
    at com.intellij.rt.execution.application.AppMain.main(AppMain.java:147) 
Caused by: java.lang.ClassNotFoundException: org.apache.spark.sql.internal.SharedState 
    at java.net.URLClassLoader.findClass(URLClassLoader.java:381) 
    at java.lang.ClassLoader.loadClass(ClassLoader.java:424) 
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331) 
    at java.lang.ClassLoader.loadClass(ClassLoader.java:357) 
    ... 6 more 
17/02/22 21:40:03 INFO SparkContext: Invoking stop() from shutdown hook 
17/02/22 21:40:03 INFO SparkUI: Stopped Spark web UI at http://10.0.2.15:4040 
17/02/22 21:40:03 INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped! 
17/02/22 21:40:03 INFO MemoryStore: MemoryStore cleared 
17/02/22 21:40:03 INFO BlockManager: BlockManager stopped 
17/02/22 21:40:03 INFO BlockManagerMaster: BlockManagerMaster stopped 
17/02/22 21:40:03 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped! 
17/02/22 21:40:03 INFO RemoteActorRefProvider$RemotingTerminator: Shutting down remote daemon. 
17/02/22 21:40:03 INFO RemoteActorRefProvider$RemotingTerminator: Remote daemon shut down; proceeding with flushing remote transports. 
17/02/22 21:40:03 INFO SparkContext: Successfully stopped SparkContext 
17/02/22 21:40:03 INFO ShutdownHookManager: Shutdown hook called 
17/02/22 21:40:03 INFO ShutdownHookManager: Deleting directory /tmp/spark-74fbf46d-55bc-4fb6-b8f3-edd4898e86e8 

Unten ist mein pom.xml

<?xml version="1.0" encoding="UTF-8"?> 
<project xmlns="http://maven.apache.org/POM/4.0.0" 
     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 
     xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 
    <modelVersion>4.0.0</modelVersion> 

    <groupId>com.bistel.test</groupId> 
    <artifactId>jdf-dt-rtoc-withSQL</artifactId> 
    <version>1.0-SNAPSHOT</version> 

    <dependencies> 

     <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.11 --> 
     <dependency> 
      <groupId>org.apache.spark</groupId> 
      <artifactId>spark-core_2.10</artifactId> 
      <version>1.6.0</version> 
     </dependency> 

     <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql_2.11 --> 
     <dependency> 
      <groupId>org.apache.spark</groupId> 
      <artifactId>spark-sql_2.10</artifactId> 
      <version>1.6.0</version> 
     </dependency> 

     <dependency> 
      <groupId>org.apache.spark</groupId> 
      <artifactId>spark-hivecontext-compatibility_2.10</artifactId> 
      <version>2.0.0-preview</version> 
     </dependency> 

    </dependencies> 

</project> 

Ich möchte wie collect_list DF-Bibliothek verwenden, und ich fand, dass ich HiveContext auf die Spark 1.6.0 verwenden sollten. Aber der Fehler ist aufgetreten, auch wenn ich diesen Schritt befolgt habe.

Jede Hilfe wird sehr geschätzt.

Danke.

Answer 1

Sie erhalten java.lang.ClassNotFoundException Fehler, weil Sie falsche Abhängigkeit hinzufügen - "spark-hivecontext-compatibility_2.10". Ihre pom.xml-Datei sollte unterhalb der Abhängigkeit liegen.

<dependency> 
    <groupId>org.apache.spark</groupId> 
    <artifactId>spark-hive_2.10</artifactId> 
    <version>1.6.0</version> 
</dependency>