org.apache.spark.SparkException: Task not serializable

Stack Overflow | wojtek_z | 6 months ago
tip
Your exception is missing from the Samebug knowledge base.
Here are the best solutions we found on the Internet.
Click on the to mark the helpful solution and get rewards for you help.

Root Cause Analysis

  1. java.io.NotSerializableException

    org.apache.spark.streaming.api.java.JavaStreamingContext Serialization stack: - object not serializable (class: org.apache.spark.streaming.api.java.JavaStreamingContext, value: org.apache.spark.streaming.api.java.JavaStreamingContext@4538856f) - field (class: test.spark.service.SparkServiceImpl, name: javaStreamingContext, type: class org.apache.spark.streaming.api.java.JavaStreamingContext) - object (class test.spark.service.SparkServiceImpl, test.spark.service.SparkServiceImpl@7e34b127) - field (class: test.spark.service.SparkServiceImpl$1, name: this$0, type: class test.spark.service.SparkServiceImpl) - object (class test.spark.service.SparkServiceImpl$1, test.spark.service.SparkServiceImpl$1@536b71b4) - field (class: org.apache.spark.api.java.JavaPairRDD$$anonfun$toScalaFunction$1, name: fun$1, type: interface org.apache.spark.api.java.function.Function) - object (class org.apache.spark.api.java.JavaPairRDD$$anonfun$toScalaFunction$1, <function1>)

    at org.apache.spark.serializer.SerializationDebugger$.improveException()
  2. Spark
    AbstractJavaRDDLike.groupBy
    1. org.apache.spark.serializer.SerializationDebugger$.improveException(SerializationDebugger.scala:40)
    2. org.apache.spark.serializer.JavaSerializationStream.writeObject(JavaSerializer.scala:46)
    3. org.apache.spark.serializer.JavaSerializerInstance.serialize(JavaSerializer.scala:100)
    4. org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:295)
    5. org.apache.spark.util.ClosureCleaner$.org$apache$spark$util$ClosureCleaner$$clean(ClosureCleaner.scala:288)
    6. org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:108)
    7. org.apache.spark.SparkContext.clean(SparkContext.scala:2037)
    8. org.apache.spark.rdd.RDD$$anonfun$groupBy$3.apply(RDD.scala:694)
    9. org.apache.spark.rdd.RDD$$anonfun$groupBy$3.apply(RDD.scala:693)
    10. org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
    11. org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
    12. org.apache.spark.rdd.RDD.withScope(RDD.scala:358)
    13. org.apache.spark.rdd.RDD.groupBy(RDD.scala:693)
    14. org.apache.spark.rdd.RDD$$anonfun$groupBy$1.apply(RDD.scala:665)
    15. org.apache.spark.rdd.RDD$$anonfun$groupBy$1.apply(RDD.scala:665)
    16. org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
    17. org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
    18. org.apache.spark.rdd.RDD.withScope(RDD.scala:358)
    19. org.apache.spark.rdd.RDD.groupBy(RDD.scala:664)
    20. org.apache.spark.api.java.JavaRDDLike$class.groupBy(JavaRDDLike.scala:242)
    21. org.apache.spark.api.java.AbstractJavaRDDLike.groupBy(JavaRDDLike.scala:45)
    21 frames
  3. test.spark.service
    SparkServiceTest.testProcess
    1. test.spark.service.SparkServiceImpl.process(SparkServiceServiceImpl.java:56)
    2. test.spark.service.SparkServiceTest.testProcess(SparkServiceTest.java:27)
    2 frames
  4. Java RT
    Method.invoke
    1. sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    2. sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    3. sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    4. java.lang.reflect.Method.invoke(Method.java:498)
    4 frames
  5. JUnit
    InvokeMethod.evaluate
    1. org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
    2. org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    3. org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
    4. org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    4 frames
  6. Spring TestContext
    SpringRepeat.evaluate
    1. org.springframework.test.context.junit4.statements.RunBeforeTestMethodCallbacks.evaluate(RunBeforeTestMethodCallbacks.java:75)
    2. org.springframework.test.context.junit4.statements.RunAfterTestMethodCallbacks.evaluate(RunAfterTestMethodCallbacks.java:86)
    3. org.springframework.test.context.junit4.statements.SpringRepeat.evaluate(SpringRepeat.java:84)
    3 frames
  7. JUnit
    ParentRunner.runLeaf
    1. org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
    1 frame
  8. Spring TestContext
    SpringJUnit4ClassRunner.runChild
    1. org.springframework.test.context.junit4.SpringJUnit4ClassRunner.runChild(SpringJUnit4ClassRunner.java:252)
    2. org.springframework.test.context.junit4.SpringJUnit4ClassRunner.runChild(SpringJUnit4ClassRunner.java:94)
    2 frames
  9. JUnit
    ParentRunner$2.evaluate
    1. org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
    2. org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
    3. org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
    4. org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
    5. org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
    5 frames
  10. Spring TestContext
    RunAfterTestClassCallbacks.evaluate
    1. org.springframework.test.context.junit4.statements.RunBeforeTestClassCallbacks.evaluate(RunBeforeTestClassCallbacks.java:61)
    2. org.springframework.test.context.junit4.statements.RunAfterTestClassCallbacks.evaluate(RunAfterTestClassCallbacks.java:70)
    2 frames
  11. JUnit
    ParentRunner.run
    1. org.junit.runners.ParentRunner.run(ParentRunner.java:363)
    1 frame
  12. Spring TestContext
    SpringJUnit4ClassRunner.run
    1. org.springframework.test.context.junit4.SpringJUnit4ClassRunner.run(SpringJUnit4ClassRunner.java:191)
    1 frame
  13. JUnit
    JUnitCore.run
    1. org.junit.runner.JUnitCore.run(JUnitCore.java:137)
    1 frame
  14. IntelliJ junit4 module
    JUnit4IdeaTestRunner.startRunnerWithArgs
    1. com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:117)
    2. com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:42)
    2 frames
  15. IDEA
    JUnitStarter.main
    1. com.intellij.rt.execution.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:262)
    2. com.intellij.rt.execution.junit.JUnitStarter.main(JUnitStarter.java:84)
    2 frames
  16. Java RT
    Method.invoke
    1. sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    2. sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    3. sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    4. java.lang.reflect.Method.invoke(Method.java:498)
    4 frames
  17. IDEA
    AppMain.main
    1. com.intellij.rt.execution.application.AppMain.main(AppMain.java:147)
    1 frame