java.lang.NoClassDefFoundError: org/apache/spark/sql/DataFrame

JIRA | Bhanu Pratap | 4 months ago
  1. 0

    I tried to use mongo-spark-connector with spark 2.0.0. And get the following exception: {code:title=Exception Stacktrace |borderStyle=solid} : java.lang.NoClassDefFoundError: org/apache/spark/sql/DataFrame at java.lang.Class.getDeclaredMethods0(Native Method) at java.lang.Class.privateGetDeclaredMethods(Class.java:2701) at java.lang.Class.getDeclaredMethod(Class.java:2128) at java.io.ObjectStreamClass.getPrivateMethod(ObjectStreamClass.java:1475) at java.io.ObjectStreamClass.access$1700(ObjectStreamClass.java:72) at java.io.ObjectStreamClass$2.run(ObjectStreamClass.java:498) at java.io.ObjectStreamClass$2.run(ObjectStreamClass.java:472) at java.security.AccessController.doPrivileged(Native Method) at java.io.ObjectStreamClass.<init>(ObjectStreamClass.java:472) at java.io.ObjectStreamClass.lookup(ObjectStreamClass.java:369) at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1134) at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548) at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1509) at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432) at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178) at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:348) at scala.collection.immutable.List$SerializationProxy.writeObject(List.scala:468) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at java.io.ObjectStreamClass.invokeWriteObject(ObjectStreamClass.java:1028) at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1496) at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432) at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178) at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548) at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1509) at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432) at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178) at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548) at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1509) at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432) at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178) at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548) at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1509) at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432) at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178) at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548) at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1509) at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432) at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178) at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:348) at org.apache.spark.serializer.JavaSerializationStream.writeObject(JavaSerializer.scala:43) at org.apache.spark.serializer.JavaSerializerInstance.serialize(JavaSerializer.scala:100) at org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:295) at org.apache.spark.util.ClosureCleaner$.org$apache$spark$util$ClosureCleaner$$clean(ClosureCleaner.scala:288) at org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:108) at org.apache.spark.SparkContext.clean(SparkContext.scala:2037) at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1.apply(RDD.scala:763) at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1.apply(RDD.scala:762) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) at org.apache.spark.rdd.RDD.withScope(RDD.scala:358) at org.apache.spark.rdd.RDD.mapPartitions(RDD.scala:762) at org.apache.spark.rdd.RDD$$anonfun$treeAggregate$1.apply(RDD.scala:1094) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) at org.apache.spark.rdd.RDD.withScope(RDD.scala:358) at org.apache.spark.rdd.RDD.treeAggregate(RDD.scala:1085) at com.mongodb.spark.sql.MongoInferSchema$.apply(MongoInferSchema.scala:76) at com.mongodb.spark.sql.DefaultSource.createRelation(DefaultSource.scala:78) at com.mongodb.spark.sql.DefaultSource.createRelation(DefaultSource.scala:53) at com.mongodb.spark.sql.DefaultSource.createRelation(DefaultSource.scala:38) at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:315) at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:149) at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:122) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:237) at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) at py4j.Gateway.invoke(Gateway.java:280) at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:128) at py4j.commands.CallCommand.execute(CallCommand.java:79) at py4j.GatewayConnection.run(GatewayConnection.java:211) at java.lang.Thread.run(Thread.java:745) Caused by: java.lang.ClassNotFoundException: org.apache.spark.sql.DataFrame at java.net.URLClassLoader.findClass(URLClassLoader.java:381) at java.lang.ClassLoader.loadClass(ClassLoader.java:424) at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331) at java.lang.ClassLoader.loadClass(ClassLoader.java:357) ... 77 more {code}

    JIRA | 4 months ago | Bhanu Pratap
    java.lang.NoClassDefFoundError: org/apache/spark/sql/DataFrame
  2. 0

    Deploy Failed on GlassFish

    icesoft.org | 1 year ago
    java.lang.NoClassDefFoundError: java/sql/RowId
  3. Speed up your debug routine!

    Automated exception search integrated into your IDE

    1 unregistered visitors
    Not finding the right solution?
    Take a tour to get the most out of Samebug.

    Tired of useless tips?

    Automated exception search integrated into your IDE

    Root Cause Analysis

    1. java.lang.ClassNotFoundException

      org.apache.spark.sql.DataFrame

      at java.net.URLClassLoader.findClass()
    2. Java RT
      ObjectOutputStream.writeObject
      1. java.net.URLClassLoader.findClass(URLClassLoader.java:381)
      2. java.lang.ClassLoader.loadClass(ClassLoader.java:424)
      3. sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
      4. java.lang.ClassLoader.loadClass(ClassLoader.java:357)
      5. java.lang.Class.getDeclaredMethods0(Native Method)
      6. java.lang.Class.privateGetDeclaredMethods(Class.java:2701)
      7. java.lang.Class.getDeclaredMethod(Class.java:2128)
      8. java.io.ObjectStreamClass.getPrivateMethod(ObjectStreamClass.java:1475)
      9. java.io.ObjectStreamClass.access$1700(ObjectStreamClass.java:72)
      10. java.io.ObjectStreamClass$2.run(ObjectStreamClass.java:498)
      11. java.io.ObjectStreamClass$2.run(ObjectStreamClass.java:472)
      12. java.security.AccessController.doPrivileged(Native Method)
      13. java.io.ObjectStreamClass.<init>(ObjectStreamClass.java:472)
      14. java.io.ObjectStreamClass.lookup(ObjectStreamClass.java:369)
      15. java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1134)
      16. java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548)
      17. java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1509)
      18. java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432)
      19. java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
      20. java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:348)
      20 frames
    3. Scala
      List$SerializationProxy.writeObject
      1. scala.collection.immutable.List$SerializationProxy.writeObject(List.scala:468)
      1 frame
    4. Java RT
      ObjectOutputStream.writeObject
      1. sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
      2. sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
      3. sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      4. java.lang.reflect.Method.invoke(Method.java:498)
      5. java.io.ObjectStreamClass.invokeWriteObject(ObjectStreamClass.java:1028)
      6. java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1496)
      7. java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432)
      8. java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
      9. java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548)
      10. java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1509)
      11. java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432)
      12. java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
      13. java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548)
      14. java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1509)
      15. java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432)
      16. java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
      17. java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548)
      18. java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1509)
      19. java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432)
      20. java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
      21. java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548)
      22. java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1509)
      23. java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432)
      24. java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
      25. java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:348)
      25 frames
    5. Spark
      RDD.treeAggregate
      1. org.apache.spark.serializer.JavaSerializationStream.writeObject(JavaSerializer.scala:43)
      2. org.apache.spark.serializer.JavaSerializerInstance.serialize(JavaSerializer.scala:100)
      3. org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:295)
      4. org.apache.spark.util.ClosureCleaner$.org$apache$spark$util$ClosureCleaner$$clean(ClosureCleaner.scala:288)
      5. org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:108)
      6. org.apache.spark.SparkContext.clean(SparkContext.scala:2037)
      7. org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1.apply(RDD.scala:763)
      8. org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1.apply(RDD.scala:762)
      9. org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
      10. org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
      11. org.apache.spark.rdd.RDD.withScope(RDD.scala:358)
      12. org.apache.spark.rdd.RDD.mapPartitions(RDD.scala:762)
      13. org.apache.spark.rdd.RDD$$anonfun$treeAggregate$1.apply(RDD.scala:1094)
      14. org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
      15. org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
      16. org.apache.spark.rdd.RDD.withScope(RDD.scala:358)
      17. org.apache.spark.rdd.RDD.treeAggregate(RDD.scala:1085)
      17 frames
    6. com.mongodb.spark
      DefaultSource.createRelation
      1. com.mongodb.spark.sql.MongoInferSchema$.apply(MongoInferSchema.scala:76)
      2. com.mongodb.spark.sql.DefaultSource.createRelation(DefaultSource.scala:78)
      3. com.mongodb.spark.sql.DefaultSource.createRelation(DefaultSource.scala:53)
      4. com.mongodb.spark.sql.DefaultSource.createRelation(DefaultSource.scala:38)
      4 frames
    7. org.apache.spark
      DataSource.resolveRelation
      1. org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:315)
      1 frame
    8. Spark Project SQL
      DataFrameReader.load
      1. org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:149)
      2. org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:122)
      2 frames
    9. Java RT
      Method.invoke
      1. sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
      2. sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
      3. sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      4. java.lang.reflect.Method.invoke(Method.java:498)
      4 frames
    10. Py4J
      GatewayConnection.run
      1. py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:237)
      2. py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
      3. py4j.Gateway.invoke(Gateway.java:280)
      4. py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:128)
      5. py4j.commands.CallCommand.execute(CallCommand.java:79)
      6. py4j.GatewayConnection.run(GatewayConnection.java:211)
      6 frames
    11. Java RT
      Thread.run
      1. java.lang.Thread.run(Thread.java:745)
      1 frame