java.net.NoRouteToHostException: No route to host

SpringSource Issue Tracker | Hong Cheng (c) | 4 years ago
  1. 0

    Reproduce steps: 1,Add shared datastore,Local datastores on one host and nfs datastore on the same host; 2,Create CDH4+HABSE cluster with attached spec file; 3,Shutdown nfs server; 4,Run disk fix command:cluster fix --name <cluste_name> --disk 5,Run hbase job after disk fixed: hbase org.apache.hadoop.hbase.PerformanceEvaluation --nomapred --rows=10000 randomWrite 20 Expected result: Hbase job runs successfully; Actual result: hostname=10.111.88.182, port=60020}, attempt=6 of 10 failed; retrying after sleep of 4002 because: No route to host 13/05/24 07:41:58 DEBUG client.HConnectionManager$HConnectionImplementation: Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@53d439fe; serverName=10.111.88.182,60020,1369371416596 13/05/24 07:42:02 DEBUG client.HConnectionManager$HConnectionImplementation: Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@53d439fe; serverName=10.111.88.182,60020,1369371416596 13/05/24 07:43:05 DEBUG client.HConnectionManager$HConnectionImplementation: locateRegionInMeta parentTable=-ROOT-, metaLocation={region=-ROOT-,,0.70236052, hostname=10.111.88.182, port=60020}, attempt=7 of 10 failed; retrying after sleep of 8070 because: No route to host 13/05/24 07:43:05 DEBUG client.HConnectionManager$HConnectionImplementation: Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@53d439fe; serverName=10.111.88.182,60020,1369371416596 13/05/24 07:43:13 DEBUG client.HConnectionManager$HConnectionImplementation: Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@53d439fe; serverName=10.111.88.182,60020,1369371416596 13/05/24 07:44:17 DEBUG client.HConnectionManager$HConnectionImplementation: locateRegionInMeta parentTable=-ROOT-, metaLocation={region=-ROOT-,,0.70236052, hostname=10.111.88.182, port=60020}, attempt=8 of 10 failed; retrying after sleep of 16017 because: No route to host 13/05/24 07:44:17 DEBUG client.HConnectionManager$HConnectionImplementation: Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@53d439fe; serverName=10.111.88.182,60020,1369371416596 13/05/24 07:44:33 DEBUG client.HConnectionManager$HConnectionImplementation: Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@53d439fe; serverName=10.111.88.182,60020,1369371416596 13/05/24 07:45:36 DEBUG catalog.CatalogTracker: Stopping catalog tracker org.apache.hadoop.hbase.catalog.CatalogTracker@5815338 13/05/24 07:45:36 INFO zookeeper.ZooKeeper: Session: 0x13ed4dcffea000c closed java.net.NoRouteToHostException: No route to host at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:567) at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:207) at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:525) at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:489) at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupConnection(HBaseClient.java:416) at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:462) at org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:1150) at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:1000) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:150) at $Proxy8.getProtocolVersion(Unknown Source) at org.apache.hadoop.hbase.ipc.WritableRpcEngine.getProxy(WritableRpcEngine.java:183) at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:335) at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:312) at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:364) at org.apache.hadoop.hbase.ipc.HBaseRPC.waitForProxy(HBaseRPC.java:236) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getHRegionConnection(HConnectionManager.java:1305) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getHRegionConnection(HConnectionManager.java:1261) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getHRegionConnection(HConnectionManager.java:1248) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegionInMeta(HConnectionManager.java:957) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegion(HConnectionManager.java:852) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegion(HConnectionManager.java:813) at org.apache.hadoop.hbase.client.HTable.finishSetup(HTable.java:271) at org.apache.hadoop.hbase.client.HTable.<init>(HTable.java:211) at org.apache.hadoop.hbase.catalog.MetaReader.getHTable(MetaReader.java:200) at org.apache.hadoop.hbase.catalog.MetaReader.getMetaHTable(MetaReader.java:226) at org.apache.hadoop.hbase.catalog.MetaReader.fullScan(MetaReader.java:705) at org.apache.hadoop.hbase.catalog.MetaReader.fullScan(MetaReader.java:183) at org.apache.hadoop.hbase.catalog.MetaReader.tableExists(MetaReader.java:448) at org.apache.hadoop.hbase.client.HBaseAdmin.tableExists(HBaseAdmin.java:245) at org.apache.hadoop.hbase.client.HBaseAdmin.tableExists(HBaseAdmin.java:259) at org.apache.hadoop.hbase.PerformanceEvaluation.checkTable(PerformanceEvaluation.java:488) at org.apache.hadoop.hbase.PerformanceEvaluation.runNIsMoreThanOne(PerformanceEvaluation.java:535) at org.apache.hadoop.hbase.PerformanceEvaluation.runTest(PerformanceEvaluation.java:1224) at org.apache.hadoop.hbase.PerformanceEvaluation.run(PerformanceEvaluation.java:1366) at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70) at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:84) at org.apache.hadoop.hbase.PerformanceEvaluation.main(PerformanceEvaluation.java:1390) 13/05/24 07:45:36 INFO zookeeper.ClientCnxn: EventThread shut down

    SpringSource Issue Tracker | 4 years ago | Hong Cheng (c)
    java.net.NoRouteToHostException: No route to host
  2. 0

    Reproduce steps: 1,Add shared datastore,Local datastores on one host and nfs datastore on the same host; 2,Create CDH4+HABSE cluster with attached spec file; 3,Shutdown nfs server; 4,Run disk fix command:cluster fix --name <cluste_name> --disk 5,Run hbase job after disk fixed: hbase org.apache.hadoop.hbase.PerformanceEvaluation --nomapred --rows=10000 randomWrite 20 Expected result: Hbase job runs successfully; Actual result: hostname=10.111.88.182, port=60020}, attempt=6 of 10 failed; retrying after sleep of 4002 because: No route to host 13/05/24 07:41:58 DEBUG client.HConnectionManager$HConnectionImplementation: Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@53d439fe; serverName=10.111.88.182,60020,1369371416596 13/05/24 07:42:02 DEBUG client.HConnectionManager$HConnectionImplementation: Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@53d439fe; serverName=10.111.88.182,60020,1369371416596 13/05/24 07:43:05 DEBUG client.HConnectionManager$HConnectionImplementation: locateRegionInMeta parentTable=-ROOT-, metaLocation={region=-ROOT-,,0.70236052, hostname=10.111.88.182, port=60020}, attempt=7 of 10 failed; retrying after sleep of 8070 because: No route to host 13/05/24 07:43:05 DEBUG client.HConnectionManager$HConnectionImplementation: Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@53d439fe; serverName=10.111.88.182,60020,1369371416596 13/05/24 07:43:13 DEBUG client.HConnectionManager$HConnectionImplementation: Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@53d439fe; serverName=10.111.88.182,60020,1369371416596 13/05/24 07:44:17 DEBUG client.HConnectionManager$HConnectionImplementation: locateRegionInMeta parentTable=-ROOT-, metaLocation={region=-ROOT-,,0.70236052, hostname=10.111.88.182, port=60020}, attempt=8 of 10 failed; retrying after sleep of 16017 because: No route to host 13/05/24 07:44:17 DEBUG client.HConnectionManager$HConnectionImplementation: Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@53d439fe; serverName=10.111.88.182,60020,1369371416596 13/05/24 07:44:33 DEBUG client.HConnectionManager$HConnectionImplementation: Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@53d439fe; serverName=10.111.88.182,60020,1369371416596 13/05/24 07:45:36 DEBUG catalog.CatalogTracker: Stopping catalog tracker org.apache.hadoop.hbase.catalog.CatalogTracker@5815338 13/05/24 07:45:36 INFO zookeeper.ZooKeeper: Session: 0x13ed4dcffea000c closed java.net.NoRouteToHostException: No route to host at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:567) at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:207) at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:525) at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:489) at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupConnection(HBaseClient.java:416) at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:462) at org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:1150) at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:1000) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:150) at $Proxy8.getProtocolVersion(Unknown Source) at org.apache.hadoop.hbase.ipc.WritableRpcEngine.getProxy(WritableRpcEngine.java:183) at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:335) at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:312) at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:364) at org.apache.hadoop.hbase.ipc.HBaseRPC.waitForProxy(HBaseRPC.java:236) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getHRegionConnection(HConnectionManager.java:1305) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getHRegionConnection(HConnectionManager.java:1261) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getHRegionConnection(HConnectionManager.java:1248) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegionInMeta(HConnectionManager.java:957) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegion(HConnectionManager.java:852) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegion(HConnectionManager.java:813) at org.apache.hadoop.hbase.client.HTable.finishSetup(HTable.java:271) at org.apache.hadoop.hbase.client.HTable.<init>(HTable.java:211) at org.apache.hadoop.hbase.catalog.MetaReader.getHTable(MetaReader.java:200) at org.apache.hadoop.hbase.catalog.MetaReader.getMetaHTable(MetaReader.java:226) at org.apache.hadoop.hbase.catalog.MetaReader.fullScan(MetaReader.java:705) at org.apache.hadoop.hbase.catalog.MetaReader.fullScan(MetaReader.java:183) at org.apache.hadoop.hbase.catalog.MetaReader.tableExists(MetaReader.java:448) at org.apache.hadoop.hbase.client.HBaseAdmin.tableExists(HBaseAdmin.java:245) at org.apache.hadoop.hbase.client.HBaseAdmin.tableExists(HBaseAdmin.java:259) at org.apache.hadoop.hbase.PerformanceEvaluation.checkTable(PerformanceEvaluation.java:488) at org.apache.hadoop.hbase.PerformanceEvaluation.runNIsMoreThanOne(PerformanceEvaluation.java:535) at org.apache.hadoop.hbase.PerformanceEvaluation.runTest(PerformanceEvaluation.java:1224) at org.apache.hadoop.hbase.PerformanceEvaluation.run(PerformanceEvaluation.java:1366) at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70) at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:84) at org.apache.hadoop.hbase.PerformanceEvaluation.main(PerformanceEvaluation.java:1390) 13/05/24 07:45:36 INFO zookeeper.ClientCnxn: EventThread shut down

    SpringSource Issue Tracker | 4 years ago | Hong Cheng (c)
    java.net.NoRouteToHostException: No route to host
  3. 0

    Error in copying files to HDFS

    Stack Overflow | 4 years ago | sujitha
    java.net.NoRouteToHostException: No route to host
  4. Speed up your debug routine!

    Automated exception search integrated into your IDE

  5. 0

    Hadoop2.4.0 完全分布式集群 nodemanager启动失败 - 开源中国社区

    oschina.net | 2 years ago
    java.net.NoRouteToHostException: No route to host
  6. 0

    Archive | Page number 183151 | New Dave Site

    newdavesite.com | 2 years ago
    java.net.NoRouteToHostException: No route to host

  1. poroszd 2 times, last 2 months ago
13 unregistered visitors
Not finding the right solution?
Take a tour to get the most out of Samebug.

Tired of useless tips?

Automated exception search integrated into your IDE

Root Cause Analysis

  1. java.net.NoRouteToHostException

    No route to host

    at sun.nio.ch.SocketChannelImpl.checkConnect()
  2. Java RT
    SocketChannelImpl.finishConnect
    1. sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
    2. sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:567)
    2 frames
  3. Hadoop
    NetUtils.connect
    1. org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:207)
    2. org.apache.hadoop.net.NetUtils.connect(NetUtils.java:525)
    3. org.apache.hadoop.net.NetUtils.connect(NetUtils.java:489)
    3 frames
  4. HBase - Client
    WritableRpcEngine$Invoker.invoke
    1. org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupConnection(HBaseClient.java:416)
    2. org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:462)
    3. org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:1150)
    4. org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:1000)
    5. org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:150)
    5 frames
  5. Unknown
    $Proxy8.getProtocolVersion
    1. $Proxy8.getProtocolVersion(Unknown Source)
    1 frame
  6. HBase - Client
    HBaseAdmin.tableExists
    1. org.apache.hadoop.hbase.ipc.WritableRpcEngine.getProxy(WritableRpcEngine.java:183)
    2. org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:335)
    3. org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:312)
    4. org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:364)
    5. org.apache.hadoop.hbase.ipc.HBaseRPC.waitForProxy(HBaseRPC.java:236)
    6. org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getHRegionConnection(HConnectionManager.java:1305)
    7. org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getHRegionConnection(HConnectionManager.java:1261)
    8. org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getHRegionConnection(HConnectionManager.java:1248)
    9. org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegionInMeta(HConnectionManager.java:957)
    10. org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegion(HConnectionManager.java:852)
    11. org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegion(HConnectionManager.java:813)
    12. org.apache.hadoop.hbase.client.HTable.finishSetup(HTable.java:271)
    13. org.apache.hadoop.hbase.client.HTable.<init>(HTable.java:211)
    14. org.apache.hadoop.hbase.catalog.MetaReader.getHTable(MetaReader.java:200)
    15. org.apache.hadoop.hbase.catalog.MetaReader.getMetaHTable(MetaReader.java:226)
    16. org.apache.hadoop.hbase.catalog.MetaReader.fullScan(MetaReader.java:705)
    17. org.apache.hadoop.hbase.catalog.MetaReader.fullScan(MetaReader.java:183)
    18. org.apache.hadoop.hbase.catalog.MetaReader.tableExists(MetaReader.java:448)
    19. org.apache.hadoop.hbase.client.HBaseAdmin.tableExists(HBaseAdmin.java:245)
    20. org.apache.hadoop.hbase.client.HBaseAdmin.tableExists(HBaseAdmin.java:259)
    20 frames
  7. HBase
    PerformanceEvaluation.run
    1. org.apache.hadoop.hbase.PerformanceEvaluation.checkTable(PerformanceEvaluation.java:488)
    2. org.apache.hadoop.hbase.PerformanceEvaluation.runNIsMoreThanOne(PerformanceEvaluation.java:535)
    3. org.apache.hadoop.hbase.PerformanceEvaluation.runTest(PerformanceEvaluation.java:1224)
    4. org.apache.hadoop.hbase.PerformanceEvaluation.run(PerformanceEvaluation.java:1366)
    4 frames
  8. Hadoop
    ToolRunner.run
    1. org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
    2. org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:84)
    2 frames
  9. HBase
    PerformanceEvaluation.main
    1. org.apache.hadoop.hbase.PerformanceEvaluation.main(PerformanceEvaluation.java:1390)
    1 frame