com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.AmazonClientException: Unable to execute HTTP request: Timeout waiting for connection from pool

Stack Overflow | clay | 3 months ago
  1. 0

    Spark on Amazon EMR: "Timeout waiting for connection from pool"

    Stack Overflow | 3 months ago | clay
    com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.AmazonClientException: Unable to execute HTTP request: Timeout waiting for connection from pool

    Root Cause Analysis

    1. com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.AmazonClientException

      Unable to execute HTTP request: Timeout waiting for connection from pool

      at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.AmazonHttpClient.executeHelper()
    2. com.amazon.ws
      Jets3tNativeFileSystemStore.retrieveMetadata
      1. com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:618)
      2. com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.AmazonHttpClient.doExecute(AmazonHttpClient.java:376)
      3. com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.AmazonHttpClient.executeWithTimer(AmazonHttpClient.java:338)
      4. com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:287)
      5. com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:3826)
      6. com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:1015)
      7. com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:991)
      8. com.amazon.ws.emr.hadoop.fs.s3n.Jets3tNativeFileSystemStore.retrieveMetadata(Jets3tNativeFileSystemStore.java:212)
      8 frames
    3. Java RT
      Method.invoke
      1. sun.reflect.GeneratedMethodAccessor45.invoke(Unknown Source)
      2. sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      3. java.lang.reflect.Method.invoke(Method.java:498)
      3 frames
    4. Hadoop
      RetryInvocationHandler.invoke
      1. org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:191)
      2. org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102)
      2 frames
    5. com.sun.proxy
      $Proxy44.retrieveMetadata
      1. com.sun.proxy.$Proxy44.retrieveMetadata(Unknown Source)
      1 frame
    6. com.amazon.ws
      S3NativeFileSystem.getFileStatus
      1. com.amazon.ws.emr.hadoop.fs.s3n.S3NativeFileSystem.getFileStatus(S3NativeFileSystem.java:780)
      1 frame
    7. Hadoop
      FileSystem.exists
      1. org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:1428)
      1 frame
    8. com.amazon.ws
      EmrFileSystem.exists
      1. com.amazon.ws.emr.hadoop.fs.EmrFileSystem.exists(EmrFileSystem.java:313)
      1 frame
    9. org.apache.spark
      ExecutedCommandExec.doExecute
      1. org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:85)
      2. org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:60)
      3. org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:58)
      4. org.apache.spark.sql.execution.command.ExecutedCommandExec.doExecute(commands.scala:74)
      4 frames
    10. Spark Project SQL
      SparkPlan$$anonfun$executeQuery$1.apply
      1. org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:115)
      2. org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:115)
      3. org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:136)
      3 frames
    11. Spark
      RDDOperationScope$.withScope
      1. org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
      1 frame
    12. Spark Project SQL
      QueryExecution.toRdd
      1. org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:133)
      2. org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:114)
      3. org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:86)
      4. org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:86)
      4 frames
    13. org.apache.spark
      DataSource.write
      1. org.apache.spark.sql.execution.datasources.DataSource.write(DataSource.scala:487)
      1 frame
    14. Spark Project SQL
      DataFrameWriter.save
      1. org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:211)
      2. org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:194)
      2 frames
    15. Java RT
      Method.invoke
      1. sun.reflect.GeneratedMethodAccessor85.invoke(Unknown Source)
      2. sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      3. java.lang.reflect.Method.invoke(Method.java:498)
      3 frames
    16. Py4J
      GatewayConnection.run
      1. py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:237)
      2. py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
      3. py4j.Gateway.invoke(Gateway.java:280)
      4. py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:128)
      5. py4j.commands.CallCommand.execute(CallCommand.java:79)
      6. py4j.GatewayConnection.run(GatewayConnection.java:211)
      6 frames
    17. Java RT
      Thread.run
      1. java.lang.Thread.run(Thread.java:745)
      1 frame