java.lang.IllegalStateException: Spark context stopped while waiting for backend

Stack Overflow | aks | 5 months ago
tip
Your exception is missing from the Samebug knowledge base.
Here are the best solutions we found on the Internet.
Click on the to mark the helpful solution and get rewards for you help.
  1. 0

    How to know what is the reason for ClosedChannelExceptions with spark-shell in YARN client mode?

    Stack Overflow | 5 months ago | aks
    java.lang.IllegalStateException: Spark context stopped while waiting for backend
  2. 0

    Spark Shell - __spark_libs__.zip does not exist

    Stack Overflow | 3 months ago | Ryan Nel
    java.lang.IllegalStateException: Spark context stopped while waiting for backend

    Root Cause Analysis

    1. java.lang.IllegalStateException

      Spark context stopped while waiting for backend

      at org.apache.spark.scheduler.TaskSchedulerImpl.waitBackendReady()
    2. Spark
      SparkContext$.getOrCreate
      1. org.apache.spark.scheduler.TaskSchedulerImpl.waitBackendReady(TaskSchedulerImpl.scala:581)
      2. org.apache.spark.scheduler.TaskSchedulerImpl.postStartHook(TaskSchedulerImpl.scala:162)
      3. org.apache.spark.SparkContext.<init>(SparkContext.scala:549)
      4. org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2256)
      4 frames
    3. Spark Project SQL
      SparkSession$Builder$$anonfun$8.apply
      1. org.apache.spark.sql.SparkSession$Builder$$anonfun$8.apply(SparkSession.scala:831)
      2. org.apache.spark.sql.SparkSession$Builder$$anonfun$8.apply(SparkSession.scala:823)
      2 frames
    4. Scala
      Option.getOrElse
      1. scala.Option.getOrElse(Option.scala:121)
      1 frame
    5. Spark Project SQL
      SparkSession$Builder.getOrCreate
      1. org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:823)
      1 frame
    6. Spark REPL
      Main$.createSparkSession
      1. org.apache.spark.repl.Main$.createSparkSession(Main.scala:95)
      1 frame
    7. $line3
      $eval.$print
      1. $line3.$read$$iw$$iw.<init>(<console>:15)
      2. $line3.$read$$iw.<init>(<console>:31)
      3. $line3.$read.<init>(<console>:33)
      4. $line3.$read$.<init>(<console>:37)
      5. $line3.$read$.<clinit>(<console>)
      6. $line3.$eval$.$print$lzycompute(<console>:7)
      7. $line3.$eval$.$print(<console>:6)
      8. $line3.$eval.$print(<console>)
      8 frames
    8. Java RT
      Method.invoke
      1. sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
      2. sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
      3. sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      4. java.lang.reflect.Method.invoke(Method.java:498)
      4 frames
    9. Scala Compiler
      IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply
      1. scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786)
      2. scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1047)
      3. scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:638)
      4. scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:637)
      4 frames
    10. Scala
      AbstractFileClassLoader.asContext
      1. scala.reflect.internal.util.ScalaClassLoader$class.asContext(ScalaClassLoader.scala:31)
      2. scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:19)
      2 frames
    11. Scala Compiler
      ILoop.processLine
      1. scala.tools.nsc.interpreter.IMain$WrappedRequest.loadAndRunReq(IMain.scala:637)
      2. scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:569)
      3. scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:565)
      4. scala.tools.nsc.interpreter.ILoop.interpretStartingWith(ILoop.scala:807)
      5. scala.tools.nsc.interpreter.ILoop.command(ILoop.scala:681)
      6. scala.tools.nsc.interpreter.ILoop.processLine(ILoop.scala:395)
      6 frames
    12. Spark REPL
      SparkILoop$$anonfun$initializeSpark$1.apply
      1. org.apache.spark.repl.SparkILoop$$anonfun$initializeSpark$1.apply$mcV$sp(SparkILoop.scala:38)
      2. org.apache.spark.repl.SparkILoop$$anonfun$initializeSpark$1.apply(SparkILoop.scala:37)
      3. org.apache.spark.repl.SparkILoop$$anonfun$initializeSpark$1.apply(SparkILoop.scala:37)
      3 frames
    13. Scala Compiler
      IMain.beQuietDuring
      1. scala.tools.nsc.interpreter.IMain.beQuietDuring(IMain.scala:214)
      1 frame
    14. Spark REPL
      SparkILoop.loadFiles
      1. org.apache.spark.repl.SparkILoop.initializeSpark(SparkILoop.scala:37)
      2. org.apache.spark.repl.SparkILoop.loadFiles(SparkILoop.scala:94)
      2 frames
    15. Scala Compiler
      ILoop$$anonfun$process$1.apply
      1. scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply$mcZ$sp(ILoop.scala:920)
      2. scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909)
      3. scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909)
      3 frames
    16. Scala
      ScalaClassLoader$.savingContextLoader
      1. scala.reflect.internal.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:97)
      1 frame
    17. Scala Compiler
      ILoop.process
      1. scala.tools.nsc.interpreter.ILoop.process(ILoop.scala:909)
      1 frame
    18. Spark REPL
      Main.main
      1. org.apache.spark.repl.Main$.doMain(Main.scala:68)
      2. org.apache.spark.repl.Main$.main(Main.scala:51)
      3. org.apache.spark.repl.Main.main(Main.scala)
      3 frames
    19. Java RT
      Method.invoke
      1. sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
      2. sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
      3. sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      4. java.lang.reflect.Method.invoke(Method.java:498)
      4 frames
    20. Spark
      SparkSubmit.main
      1. org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:729)
      2. org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:185)
      3. org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:210)
      4. org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:124)
      5. org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
      5 frames