java.io.IOException: Failed on local exception: java.io.IOException: Couldn't set up IO streams; Host Details : local host is: "..."; destination host is: "...":8020;

Apache's JIRA Issue Tracker | Wataru Yukawa | 9 months ago
  1. 0

    [jira] [Created] (HIVE-13273) "java.lang.OutOfMemoryError: unable to create new native thread" occurs at Hive on MapReduce

    hive-dev | 9 months ago | Wataru Yukawa (JIRA)
    java.io.IOException: Failed on local exception: java.io.IOException: Couldn't set up IO streams; Host Details : local host is: "..."; destination host is: "...":8020;
  2. 0

    [jira] [Created] (HIVE-13274) "java.lang.OutOfMemoryError: unable to create new native thread" occurs at Hive on MapReduce

    hive-dev | 9 months ago | Wataru Yukawa (JIRA)
    java.io.IOException: Failed on local exception: java.io.IOException: Couldn't set up IO streams; Host Details : local host is: "..."; destination host is: "...":8020;
  3. 0

    "ps -L $(pgrep -f hiveserver2) | wc -l" is more than 15,000 HiveServer2 memory leak occurs. heapstats result is https://gyazo.com/27dcaf678fb8d2e4003af55a79c2020e hiveserver2.log {code} 2016-03-13 13:25:06,041 INFO [HiveServer2-Handler-Pool: Thread-98838]: retry.RetryInvocationHandler (RetryInvocationHandler.java:invoke(144)) - Exception while invoking getFileInfo of class Cl ientNamenodeProtocolTranslatorPB over ...:8020 after 3 fail over attempts. Trying to fail over immediately. java.io.IOException: Failed on local exception: java.io.IOException: Couldn't set up IO streams; Host Details : local host is: "..."; destination host is: "...":8020; at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:773) at org.apache.hadoop.ipc.Client.call(Client.java:1431) at org.apache.hadoop.ipc.Client.call(Client.java:1358) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:229) at com.sun.proxy.$Proxy15.getFileInfo(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getFileInfo(ClientNamenodeProtocolTranslatorPB.java:771) at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:252) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:104) at com.sun.proxy.$Proxy16.getFileInfo(Unknown Source) at org.apache.hadoop.hdfs.DFSClient.getFileInfo(DFSClient.java:2116) at org.apache.hadoop.hdfs.DistributedFileSystem$22.doCall(DistributedFileSystem.java:1315) at org.apache.hadoop.hdfs.DistributedFileSystem$22.doCall(DistributedFileSystem.java:1311) at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1311) at org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:1424) at org.apache.hadoop.mapreduce.JobResourceUploader.uploadFiles(JobResourceUploader.java:85) at org.apache.hadoop.mapreduce.JobSubmitter.copyAndConfigureFiles(JobSubmitter.java:95) at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:190) at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1290) at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1287) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657) at org.apache.hadoop.mapreduce.Job.submit(Job.java:1287) at org.apache.hadoop.mapred.JobClient$1.run(JobClient.java:575) at org.apache.hadoop.mapred.JobClient$1.run(JobClient.java:570) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657) at org.apache.hadoop.mapred.JobClient.submitJobInternal(JobClient.java:570) at org.apache.hadoop.mapred.JobClient.submitJob(JobClient.java:561) at org.apache.hadoop.hive.ql.exec.mr.ExecDriver.execute(ExecDriver.java:431) at org.apache.hadoop.hive.ql.exec.mr.MapRedTask.execute(MapRedTask.java:137) at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:160) at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:89) at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1237) at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1101) at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1096) at org.apache.hive.service.cli.operation.SQLOperation.runQuery(SQLOperation.java:154) at org.apache.hive.service.cli.operation.SQLOperation.runInternal(SQLOperation.java:183) at org.apache.hive.service.cli.operation.Operation.run(Operation.java:257) at org.apache.hive.service.cli.CLIService.executeStatement(CLIService.java:261) at org.apache.hive.service.cli.thrift.ThriftCLIService.ExecuteStatement(ThriftCLIService.java:486) at org.apache.hive.service.cli.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1313) at org.apache.hive.service.cli.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1298) at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39) at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) at org.apache.hive.service.auth.TSetIpAddressProcessor.process(TSetIpAddressProcessor.java:56) at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:285) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: java.io.IOException: Couldn't set up IO streams at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:791) at org.apache.hadoop.ipc.Client$Connection.access$2800(Client.java:373) at org.apache.hadoop.ipc.Client.getConnection(Client.java:1493) at org.apache.hadoop.ipc.Client.call(Client.java:1397) ... 57 more Caused by: java.lang.OutOfMemoryError: unable to create new native thread at java.lang.Thread.start0(Native Method) at java.lang.Thread.start(Thread.java:714) at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:784) ... 60 more {code}

    Apache's JIRA Issue Tracker | 9 months ago | Wataru Yukawa
    java.io.IOException: Failed on local exception: java.io.IOException: Couldn't set up IO streams; Host Details : local host is: "..."; destination host is: "...":8020;
  4. Speed up your debug routine!

    Automated exception search integrated into your IDE

  5. 0

    "ps -L $(pgrep -f hiveserver2) | wc -l" is more than 15,000 HiveServer2 memory leak occurs. heapstats result is https://gyazo.com/27dcaf678fb8d2e4003af55a79c2020e hiveserver2.log {code} 2016-03-13 13:25:06,041 INFO [HiveServer2-Handler-Pool: Thread-98838]: retry.RetryInvocationHandler (RetryInvocationHandler.java:invoke(144)) - Exception while invoking getFileInfo of class Cl ientNamenodeProtocolTranslatorPB over ...:8020 after 3 fail over attempts. Trying to fail over immediately. java.io.IOException: Failed on local exception: java.io.IOException: Couldn't set up IO streams; Host Details : local host is: "..."; destination host is: "...":8020; at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:773) at org.apache.hadoop.ipc.Client.call(Client.java:1431) at org.apache.hadoop.ipc.Client.call(Client.java:1358) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:229) at com.sun.proxy.$Proxy15.getFileInfo(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getFileInfo(ClientNamenodeProtocolTranslatorPB.java:771) at sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:252) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:104) at com.sun.proxy.$Proxy16.getFileInfo(Unknown Source) at org.apache.hadoop.hdfs.DFSClient.getFileInfo(DFSClient.java:2116) at org.apache.hadoop.hdfs.DistributedFileSystem$22.doCall(DistributedFileSystem.java:1315) at org.apache.hadoop.hdfs.DistributedFileSystem$22.doCall(DistributedFileSystem.java:1311) at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1311) at org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:1424) at org.apache.hadoop.mapreduce.JobResourceUploader.uploadFiles(JobResourceUploader.java:85) at org.apache.hadoop.mapreduce.JobSubmitter.copyAndConfigureFiles(JobSubmitter.java:95) at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:190) at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1290) at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1287) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657) at org.apache.hadoop.mapreduce.Job.submit(Job.java:1287) at org.apache.hadoop.mapred.JobClient$1.run(JobClient.java:575) at org.apache.hadoop.mapred.JobClient$1.run(JobClient.java:570) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657) at org.apache.hadoop.mapred.JobClient.submitJobInternal(JobClient.java:570) at org.apache.hadoop.mapred.JobClient.submitJob(JobClient.java:561) at org.apache.hadoop.hive.ql.exec.mr.ExecDriver.execute(ExecDriver.java:431) at org.apache.hadoop.hive.ql.exec.mr.MapRedTask.execute(MapRedTask.java:137) at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:160) at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:89) at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1237) at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1101) at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1096) at org.apache.hive.service.cli.operation.SQLOperation.runQuery(SQLOperation.java:154) at org.apache.hive.service.cli.operation.SQLOperation.runInternal(SQLOperation.java:183) at org.apache.hive.service.cli.operation.Operation.run(Operation.java:257) at org.apache.hive.service.cli.CLIService.executeStatement(CLIService.java:261) at org.apache.hive.service.cli.thrift.ThriftCLIService.ExecuteStatement(ThriftCLIService.java:486) at org.apache.hive.service.cli.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1313) at org.apache.hive.service.cli.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1298) at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39) at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) at org.apache.hive.service.auth.TSetIpAddressProcessor.process(TSetIpAddressProcessor.java:56) at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:285) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: java.io.IOException: Couldn't set up IO streams at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:791) at org.apache.hadoop.ipc.Client$Connection.access$2800(Client.java:373) at org.apache.hadoop.ipc.Client.getConnection(Client.java:1493) at org.apache.hadoop.ipc.Client.call(Client.java:1397) ... 57 more Caused by: java.lang.OutOfMemoryError: unable to create new native thread at java.lang.Thread.start0(Native Method) at java.lang.Thread.start(Thread.java:714) at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:784) ... 60 more {code}

    Apache's JIRA Issue Tracker | 9 months ago | Wataru Yukawa
    java.io.IOException: Failed on local exception: java.io.IOException: Couldn't set up IO streams; Host Details : local host is: "..."; destination host is: "...":8020;
  6. 0

    Hadoop Balancer fails with - IOException: Couldn't set up IO streams (LeaseRenewer Warning)

    Stack Overflow | 3 years ago | Yash Sharma
    java.io.IOException: Failed on local exception: java.io.IOException: Couldn't set up IO streams; Host Details : local host is: "hadoop01.xx.xx.xx.xx.com/30.0.1.176"; destination host is: "hadoop01.xx.xx.xx.xx.com":8022;

  1. Reich 4 times, last 2 weeks ago
  2. Ajeet 46 times, last 4 months ago
  3. Diogo Jaym 1 times, last 5 months ago
  4. brunosatriani 2 times, last 6 months ago
  5. Bardh 4 times, last 7 months ago
1 more registered users
29 unregistered visitors
Not finding the right solution?
Take a tour to get the most out of Samebug.

Tired of useless tips?

Automated exception search integrated into your IDE

Root Cause Analysis

  1. java.lang.OutOfMemoryError

    unable to create new native thread

    at java.lang.Thread.start0()
  2. Java RT
    Thread.start
    1. java.lang.Thread.start0(Native Method)
    2. java.lang.Thread.start(Thread.java:714)
    2 frames
  3. Hadoop
    ProtobufRpcEngine$Invoker.invoke
    1. org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:784)
    2. org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:791)
    3. org.apache.hadoop.ipc.Client$Connection.access$2800(Client.java:373)
    4. org.apache.hadoop.ipc.Client.getConnection(Client.java:1493)
    5. org.apache.hadoop.ipc.Client.call(Client.java:1397)
    6. org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:773)
    7. org.apache.hadoop.ipc.Client.call(Client.java:1431)
    8. org.apache.hadoop.ipc.Client.call(Client.java:1358)
    9. org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:229)
    9 frames
  4. com.sun.proxy
    $Proxy15.getFileInfo
    1. com.sun.proxy.$Proxy15.getFileInfo(Unknown Source)
    1 frame
  5. Apache Hadoop HDFS
    ClientNamenodeProtocolTranslatorPB.getFileInfo
    1. org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getFileInfo(ClientNamenodeProtocolTranslatorPB.java:771)
    1 frame
  6. Java RT
    Method.invoke
    1. sun.reflect.GeneratedMethodAccessor15.invoke(Unknown Source)
    2. sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    3. java.lang.reflect.Method.invoke(Method.java:498)
    3 frames
  7. Hadoop
    RetryInvocationHandler.invoke
    1. org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:252)
    2. org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:104)
    2 frames
  8. com.sun.proxy
    $Proxy16.getFileInfo
    1. com.sun.proxy.$Proxy16.getFileInfo(Unknown Source)
    1 frame
  9. Apache Hadoop HDFS
    DistributedFileSystem$22.doCall
    1. org.apache.hadoop.hdfs.DFSClient.getFileInfo(DFSClient.java:2116)
    2. org.apache.hadoop.hdfs.DistributedFileSystem$22.doCall(DistributedFileSystem.java:1315)
    3. org.apache.hadoop.hdfs.DistributedFileSystem$22.doCall(DistributedFileSystem.java:1311)
    3 frames
  10. Hadoop
    FileSystemLinkResolver.resolve
    1. org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
    1 frame
  11. Apache Hadoop HDFS
    DistributedFileSystem.getFileStatus
    1. org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1311)
    1 frame
  12. Hadoop
    FileSystem.exists
    1. org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:1424)
    1 frame
  13. Hadoop
    Job$10.run
    1. org.apache.hadoop.mapreduce.JobResourceUploader.uploadFiles(JobResourceUploader.java:85)
    2. org.apache.hadoop.mapreduce.JobSubmitter.copyAndConfigureFiles(JobSubmitter.java:95)
    3. org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:190)
    4. org.apache.hadoop.mapreduce.Job$10.run(Job.java:1290)
    5. org.apache.hadoop.mapreduce.Job$10.run(Job.java:1287)
    5 frames
  14. Java RT
    Subject.doAs
    1. java.security.AccessController.doPrivileged(Native Method)
    2. javax.security.auth.Subject.doAs(Subject.java:422)
    2 frames
  15. Hadoop
    UserGroupInformation.doAs
    1. org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657)
    1 frame
  16. Hadoop
    JobClient$1.run
    1. org.apache.hadoop.mapreduce.Job.submit(Job.java:1287)
    2. org.apache.hadoop.mapred.JobClient$1.run(JobClient.java:575)
    3. org.apache.hadoop.mapred.JobClient$1.run(JobClient.java:570)
    3 frames
  17. Java RT
    Subject.doAs
    1. java.security.AccessController.doPrivileged(Native Method)
    2. javax.security.auth.Subject.doAs(Subject.java:422)
    2 frames
  18. Hadoop
    UserGroupInformation.doAs
    1. org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657)
    1 frame
  19. Hadoop
    JobClient.submitJob
    1. org.apache.hadoop.mapred.JobClient.submitJobInternal(JobClient.java:570)
    2. org.apache.hadoop.mapred.JobClient.submitJob(JobClient.java:561)
    2 frames
  20. Hive Query Language
    Driver.run
    1. org.apache.hadoop.hive.ql.exec.mr.ExecDriver.execute(ExecDriver.java:431)
    2. org.apache.hadoop.hive.ql.exec.mr.MapRedTask.execute(MapRedTask.java:137)
    3. org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:160)
    4. org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:89)
    5. org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1237)
    6. org.apache.hadoop.hive.ql.Driver.run(Driver.java:1101)
    7. org.apache.hadoop.hive.ql.Driver.run(Driver.java:1096)
    7 frames
  21. org.apache.hive
    TCLIService$Processor$ExecuteStatement.getResult
    1. org.apache.hive.service.cli.operation.SQLOperation.runQuery(SQLOperation.java:154)
    2. org.apache.hive.service.cli.operation.SQLOperation.runInternal(SQLOperation.java:183)
    3. org.apache.hive.service.cli.operation.Operation.run(Operation.java:257)
    4. org.apache.hive.service.cli.CLIService.executeStatement(CLIService.java:261)
    5. org.apache.hive.service.cli.thrift.ThriftCLIService.ExecuteStatement(ThriftCLIService.java:486)
    6. org.apache.hive.service.cli.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1313)
    7. org.apache.hive.service.cli.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1298)
    7 frames
  22. Apache Thrift
    TBaseProcessor.process
    1. org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)
    2. org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39)
    2 frames
  23. org.apache.hive
    TSetIpAddressProcessor.process
    1. org.apache.hive.service.auth.TSetIpAddressProcessor.process(TSetIpAddressProcessor.java:56)
    1 frame
  24. Apache Thrift
    TThreadPoolServer$WorkerProcess.run
    1. org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:285)
    1 frame
  25. Java RT
    Thread.run
    1. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
    2. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
    3. java.lang.Thread.run(Thread.java:745)
    3 frames