org.apache.flink.api.common.InvalidProgramException: Task not serializable

Apache's JIRA Issue Tracker | Chiwan Park | 2 years ago
  1. 0

    Because DistanceMeasure trait doesn't extend Serializable, The task using DistanceMeasure raises a following exception. {code} Task not serializable org.apache.flink.api.common.InvalidProgramException: Task not serializable at org.apache.flink.api.scala.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:179) at org.apache.flink.api.scala.ClosureCleaner$.clean(ClosureCleaner.scala:171) at org.apache.flink.api.scala.DataSet.clean(DataSet.scala:123) at org.apache.flink.api.scala.DataSet$$anon$10.<init>(DataSet.scala:691) at org.apache.flink.api.scala.DataSet.combineGroup(DataSet.scala:690) at org.apache.flink.ml.classification.KNNModel.transform(KNN.scala:78) at org.apache.flink.ml.classification.KNNITSuite$$anonfun$1.apply$mcV$sp(KNNSuite.scala:25) at org.apache.flink.ml.classification.KNNITSuite$$anonfun$1.apply(KNNSuite.scala:12) at org.apache.flink.ml.classification.KNNITSuite$$anonfun$1.apply(KNNSuite.scala:12) at org.scalatest.Transformer$$anonfun$apply$1.apply$mcV$sp(Transformer.scala:22) at org.scalatest.OutcomeOf$class.outcomeOf(OutcomeOf.scala:85) at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104) at org.scalatest.Transformer.apply(Transformer.scala:22) at org.scalatest.Transformer.apply(Transformer.scala:20) at org.scalatest.FlatSpecLike$$anon$1.apply(FlatSpecLike.scala:1647) at org.scalatest.Suite$class.withFixture(Suite.scala:1122) at org.scalatest.FlatSpec.withFixture(FlatSpec.scala:1683) at org.scalatest.FlatSpecLike$class.invokeWithFixture$1(FlatSpecLike.scala:1644) at org.scalatest.FlatSpecLike$$anonfun$runTest$1.apply(FlatSpecLike.scala:1656) at org.scalatest.FlatSpecLike$$anonfun$runTest$1.apply(FlatSpecLike.scala:1656) at org.scalatest.SuperEngine.runTestImpl(Engine.scala:306) at org.scalatest.FlatSpecLike$class.runTest(FlatSpecLike.scala:1656) at org.apache.flink.ml.classification.KNNITSuite.org$scalatest$BeforeAndAfter$$super$runTest(KNNSuite.scala:9) at org.scalatest.BeforeAndAfter$class.runTest(BeforeAndAfter.scala:200) at org.apache.flink.ml.classification.KNNITSuite.runTest(KNNSuite.scala:9) at org.scalatest.FlatSpecLike$$anonfun$runTests$1.apply(FlatSpecLike.scala:1714) at org.scalatest.FlatSpecLike$$anonfun$runTests$1.apply(FlatSpecLike.scala:1714) at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:413) at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:401) at scala.collection.immutable.List.foreach(List.scala:318) at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401) at org.scalatest.SuperEngine.org$scalatest$SuperEngine$$runTestsInBranch(Engine.scala:390) at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:427) at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:401) at scala.collection.immutable.List.foreach(List.scala:318) at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401) at org.scalatest.SuperEngine.org$scalatest$SuperEngine$$runTestsInBranch(Engine.scala:396) at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:483) at org.scalatest.FlatSpecLike$class.runTests(FlatSpecLike.scala:1714) at org.scalatest.FlatSpec.runTests(FlatSpec.scala:1683) at org.scalatest.Suite$class.run(Suite.scala:1424) at org.scalatest.FlatSpec.org$scalatest$FlatSpecLike$$super$run(FlatSpec.scala:1683) at org.scalatest.FlatSpecLike$$anonfun$run$1.apply(FlatSpecLike.scala:1760) at org.scalatest.FlatSpecLike$$anonfun$run$1.apply(FlatSpecLike.scala:1760) at org.scalatest.SuperEngine.runImpl(Engine.scala:545) at org.scalatest.FlatSpecLike$class.run(FlatSpecLike.scala:1760) at org.apache.flink.ml.classification.KNNITSuite.org$scalatest$BeforeAndAfter$$super$run(KNNSuite.scala:9) at org.scalatest.BeforeAndAfter$class.run(BeforeAndAfter.scala:241) at org.apache.flink.ml.classification.KNNITSuite.run(KNNSuite.scala:9) at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55) at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563) at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557) at scala.collection.immutable.List.foreach(List.scala:318) at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557) at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) at org.scalatest.tools.Runner$.run(Runner.scala:883) at org.scalatest.tools.Runner.run(Runner.scala) at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at com.intellij.rt.execution.application.AppMain.main(AppMain.java:140) Caused by: java.io.NotSerializableException: org.apache.flink.ml.metrics.distances.EuclideanDistanceMetric at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1184) at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548) at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1509) at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432) at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178) at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:348) at org.apache.flink.util.InstantiationUtil.serializeObject(InstantiationUtil.java:314) at org.apache.flink.api.scala.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:177) ... 66 more {code}

    Apache's JIRA Issue Tracker | 2 years ago | Chiwan Park
    org.apache.flink.api.common.InvalidProgramException: Task not serializable
  2. 0

    Type issues with mapVertices

    GitHub | 2 years ago | vasia
    org.apache.flink.api.common.InvalidProgramException: Task flink.graphs.Graph$ApplyMapperToVertexWithType@32535e27 not serializable:
  3. 0

    GitHub comment 46#68704608

    GitHub | 2 years ago | StephanEwen
    org.apache.flink.api.common.InvalidProgramException: Task flink.graphs.Graph$ApplyMapperToVertexWithType@32535e27 not serializable:
  4. Speed up your debug routine!

    Automated exception search integrated into your IDE

  5. 0

    Flink thowing serialization error when reading from hbase

    Stack Overflow | 1 month ago | Vignesh I
    org.apache.flink.api.common.InvalidProgramException: Task not serializable
  6. 0

    Flink Streaming: From one window, lookup state in another window

    Stack Overflow | 4 weeks ago | Jonas Gröger
    org.apache.flink.api.common.InvalidProgramException: The implementation of the RichCoFlatMapFunction is not serializable. The object probably contains or references non serializable fields.

  1. kjhdofjosvs 1 times, last 5 days ago
  2. treefolk 1 times, last 2 weeks ago
  3. filpgame 4 times, last 2 weeks ago
  4. Handemelindo 15 times, last 3 weeks ago
  5. eti22 1 times, last 1 month ago
11 more registered users
50 unregistered visitors
Not finding the right solution?
Take a tour to get the most out of Samebug.

Tired of useless tips?

Automated exception search integrated into your IDE

Root Cause Analysis

  1. java.io.NotSerializableException

    org.apache.flink.ml.metrics.distances.EuclideanDistanceMetric

    at java.io.ObjectOutputStream.writeObject0()
  2. Java RT
    ObjectOutputStream.writeObject
    1. java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1184)
    2. java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548)
    3. java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1509)
    4. java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432)
    5. java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
    6. java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:348)
    6 frames
  3. flink-core
    InstantiationUtil.serializeObject
    1. org.apache.flink.util.InstantiationUtil.serializeObject(InstantiationUtil.java:314)
    1 frame
  4. org.apache.flink
    KNNITSuite$$anonfun$1.apply
    1. org.apache.flink.api.scala.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:177)
    2. org.apache.flink.api.scala.ClosureCleaner$.clean(ClosureCleaner.scala:171)
    3. org.apache.flink.api.scala.DataSet.clean(DataSet.scala:123)
    4. org.apache.flink.api.scala.DataSet$$anon$10.<init>(DataSet.scala:691)
    5. org.apache.flink.api.scala.DataSet.combineGroup(DataSet.scala:690)
    6. org.apache.flink.ml.classification.KNNModel.transform(KNN.scala:78)
    7. org.apache.flink.ml.classification.KNNITSuite$$anonfun$1.apply$mcV$sp(KNNSuite.scala:25)
    8. org.apache.flink.ml.classification.KNNITSuite$$anonfun$1.apply(KNNSuite.scala:12)
    9. org.apache.flink.ml.classification.KNNITSuite$$anonfun$1.apply(KNNSuite.scala:12)
    9 frames
  5. scalatest
    FlatSpecLike$class.runTest
    1. org.scalatest.Transformer$$anonfun$apply$1.apply$mcV$sp(Transformer.scala:22)
    2. org.scalatest.OutcomeOf$class.outcomeOf(OutcomeOf.scala:85)
    3. org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
    4. org.scalatest.Transformer.apply(Transformer.scala:22)
    5. org.scalatest.Transformer.apply(Transformer.scala:20)
    6. org.scalatest.FlatSpecLike$$anon$1.apply(FlatSpecLike.scala:1647)
    7. org.scalatest.Suite$class.withFixture(Suite.scala:1122)
    8. org.scalatest.FlatSpec.withFixture(FlatSpec.scala:1683)
    9. org.scalatest.FlatSpecLike$class.invokeWithFixture$1(FlatSpecLike.scala:1644)
    10. org.scalatest.FlatSpecLike$$anonfun$runTest$1.apply(FlatSpecLike.scala:1656)
    11. org.scalatest.FlatSpecLike$$anonfun$runTest$1.apply(FlatSpecLike.scala:1656)
    12. org.scalatest.SuperEngine.runTestImpl(Engine.scala:306)
    13. org.scalatest.FlatSpecLike$class.runTest(FlatSpecLike.scala:1656)
    13 frames
  6. org.apache.flink
    KNNITSuite.org$scalatest$BeforeAndAfter$$super$runTest
    1. org.apache.flink.ml.classification.KNNITSuite.org$scalatest$BeforeAndAfter$$super$runTest(KNNSuite.scala:9)
    1 frame
  7. scalatest
    BeforeAndAfter$class.runTest
    1. org.scalatest.BeforeAndAfter$class.runTest(BeforeAndAfter.scala:200)
    1 frame
  8. org.apache.flink
    KNNITSuite.runTest
    1. org.apache.flink.ml.classification.KNNITSuite.runTest(KNNSuite.scala:9)
    1 frame
  9. scalatest
    SuperEngine$$anonfun$traverseSubNodes$1$1.apply
    1. org.scalatest.FlatSpecLike$$anonfun$runTests$1.apply(FlatSpecLike.scala:1714)
    2. org.scalatest.FlatSpecLike$$anonfun$runTests$1.apply(FlatSpecLike.scala:1714)
    3. org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:413)
    4. org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:401)
    4 frames
  10. Scala
    List.foreach
    1. scala.collection.immutable.List.foreach(List.scala:318)
    1 frame
  11. scalatest
    SuperEngine$$anonfun$traverseSubNodes$1$1.apply
    1. org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401)
    2. org.scalatest.SuperEngine.org$scalatest$SuperEngine$$runTestsInBranch(Engine.scala:390)
    3. org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:427)
    4. org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:401)
    4 frames
  12. Scala
    List.foreach
    1. scala.collection.immutable.List.foreach(List.scala:318)
    1 frame
  13. scalatest
    FlatSpecLike$class.run
    1. org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401)
    2. org.scalatest.SuperEngine.org$scalatest$SuperEngine$$runTestsInBranch(Engine.scala:396)
    3. org.scalatest.SuperEngine.runTestsImpl(Engine.scala:483)
    4. org.scalatest.FlatSpecLike$class.runTests(FlatSpecLike.scala:1714)
    5. org.scalatest.FlatSpec.runTests(FlatSpec.scala:1683)
    6. org.scalatest.Suite$class.run(Suite.scala:1424)
    7. org.scalatest.FlatSpec.org$scalatest$FlatSpecLike$$super$run(FlatSpec.scala:1683)
    8. org.scalatest.FlatSpecLike$$anonfun$run$1.apply(FlatSpecLike.scala:1760)
    9. org.scalatest.FlatSpecLike$$anonfun$run$1.apply(FlatSpecLike.scala:1760)
    10. org.scalatest.SuperEngine.runImpl(Engine.scala:545)
    11. org.scalatest.FlatSpecLike$class.run(FlatSpecLike.scala:1760)
    11 frames
  14. org.apache.flink
    KNNITSuite.org$scalatest$BeforeAndAfter$$super$run
    1. org.apache.flink.ml.classification.KNNITSuite.org$scalatest$BeforeAndAfter$$super$run(KNNSuite.scala:9)
    1 frame
  15. scalatest
    BeforeAndAfter$class.run
    1. org.scalatest.BeforeAndAfter$class.run(BeforeAndAfter.scala:241)
    1 frame
  16. org.apache.flink
    KNNITSuite.run
    1. org.apache.flink.ml.classification.KNNITSuite.run(KNNSuite.scala:9)
    1 frame
  17. scalatest
    Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply
    1. org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55)
    2. org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563)
    3. org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557)
    3 frames
  18. Scala
    List.foreach
    1. scala.collection.immutable.List.foreach(List.scala:318)
    1 frame
  19. scalatest
    Runner.run
    1. org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557)
    2. org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044)
    3. org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043)
    4. org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722)
    5. org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043)
    6. org.scalatest.tools.Runner$.run(Runner.scala:883)
    7. org.scalatest.tools.Runner.run(Runner.scala)
    7 frames
  20. org.jetbrains.plugins
    ScalaTestRunner.main
    1. org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138)
    2. org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28)
    2 frames
  21. Java RT
    Method.invoke
    1. sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    2. sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    3. sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    4. java.lang.reflect.Method.invoke(Method.java:497)
    4 frames
  22. IDEA
    AppMain.main
    1. com.intellij.rt.execution.application.AppMain.main(AppMain.java:140)
    1 frame