java.lang.RuntimeException: java.net.ConnectException: Connection refused

SpringSource Issue Tracker | Yifeng Xiao | 4 years ago
  1. 0

    1. Run randomWrite job {code} hbase org.apache.hadoop.hbase.PerformanceEvaluation --nomapred randomWrite 20 {code} 2. Run any job and it seems the cluster crashed. {code} hbase org.apache.hadoop.hbase.PerformanceEvaluation --nomapred sequentialWrite 3 4d1d41; serverName=10.111.88.221,60020,1361415230177 Exception in thread "0" java.lang.RuntimeException: java.net.ConnectException: Connection refused at org.apache.hadoop.hbase.PerformanceEvaluation$1.run(PerformanceEvaluation.java:558) Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:567) at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:207) at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:523) at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:488) at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupConnection(HBaseClient.java:328) at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:362) at org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:1045) at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:897) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:150) at $Proxy8.getClosestRowBefore(Unknown Source) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegionInMeta(HConnectionManager.java:947) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegion(HConnectionManager.java:814) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.relocateRegion(HConnectionManager.java:788) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegionInMeta(HConnectionManager.java:1024) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegion(HConnectionManager.java:818) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.processBatchCallback(HConnectionManager.java:1524) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.processBatch(HConnectionManager.java:1409) at org.apache.hadoop.hbase.client.HTable.flushCommits(HTable.java:949) at org.apache.hadoop.hbase.PerformanceEvaluation$Test.testTakedown(PerformanceEvaluation.java:796) at org.apache.hadoop.hbase.PerformanceEvaluation$Test.test(PerformanceEvaluation.java:813) at org.apache.hadoop.hbase.PerformanceEvaluation.runOneClient(PerformanceEvaluation.java:1140) at org.apache.hadoop.hbase.PerformanceEvaluation$1.run(PerformanceEvaluation.java:548) {code} Actual result: The cluster crashed because the configuration for performance make jvm out of memery. Expected result: Performance job will success in following size: hbase org.apache.hadoop.hbase.PerformanceEvaluation --nomapred randomWrite 20

    SpringSource Issue Tracker | 4 years ago | Yifeng Xiao
    java.lang.RuntimeException: java.net.ConnectException: Connection refused
  2. 0

    1. Run randomWrite job {code} hbase org.apache.hadoop.hbase.PerformanceEvaluation --nomapred randomWrite 20 {code} 2. Run any job and it seems the cluster crashed. {code} hbase org.apache.hadoop.hbase.PerformanceEvaluation --nomapred sequentialWrite 3 4d1d41; serverName=10.111.88.221,60020,1361415230177 Exception in thread "0" java.lang.RuntimeException: java.net.ConnectException: Connection refused at org.apache.hadoop.hbase.PerformanceEvaluation$1.run(PerformanceEvaluation.java:558) Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:567) at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:207) at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:523) at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:488) at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupConnection(HBaseClient.java:328) at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:362) at org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:1045) at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:897) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:150) at $Proxy8.getClosestRowBefore(Unknown Source) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegionInMeta(HConnectionManager.java:947) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegion(HConnectionManager.java:814) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.relocateRegion(HConnectionManager.java:788) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegionInMeta(HConnectionManager.java:1024) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegion(HConnectionManager.java:818) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.processBatchCallback(HConnectionManager.java:1524) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.processBatch(HConnectionManager.java:1409) at org.apache.hadoop.hbase.client.HTable.flushCommits(HTable.java:949) at org.apache.hadoop.hbase.PerformanceEvaluation$Test.testTakedown(PerformanceEvaluation.java:796) at org.apache.hadoop.hbase.PerformanceEvaluation$Test.test(PerformanceEvaluation.java:813) at org.apache.hadoop.hbase.PerformanceEvaluation.runOneClient(PerformanceEvaluation.java:1140) at org.apache.hadoop.hbase.PerformanceEvaluation$1.run(PerformanceEvaluation.java:548) {code} Actual result: The cluster crashed because the configuration for performance make jvm out of memery. Expected result: Performance job will success in following size: hbase org.apache.hadoop.hbase.PerformanceEvaluation --nomapred randomWrite 20

    SpringSource Issue Tracker | 4 years ago | Yifeng Xiao
    java.lang.RuntimeException: java.net.ConnectException: Connection refused
  3. 0

    I find out there is something wrong with HBase cluster with CDH4 distro. HBase cluster cannot restart normally when some job running failed. Would you help verify this issue please? *Steps* 1. Create a HBase cluster with CDH4 in Serengeti. 2. Run PerformanceEvaluation randomWrite and sequientialWrite job eg. hbase org.apache.hadoop.hbase.PerformanceEvaluation sequentialWrite 2 hbase org.apache.hadoop.hbase.PerformanceEvaluation randomWrite 3 3. Restart cluster in Serengeti. *Real Result* Step 2: The jobs failed with the console log similar as below: {noformat} 13/01/10 04:08:24 INFO zookeeper.ZooKeeper: Session: 0x23c2255b4ca002e closed org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed setting up proxy interface org.apache.hadoop.hbase.ipc.HRegionInterface to /10.111.88.244:60020 after attempts=1 at org.apache.hadoop.hbase.ipc.HBaseRPC.waitForProxy(HBaseRPC.java:242) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getHRegionConnection(HConnectionManager.java:1278) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getHRegionConnection(HConnectionManager.java:1235) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getHRegionConnection(HConnectionManager.java:1222) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegionInMeta(HConnectionManager.java:918) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegion(HConnectionManager.java:814) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegion(HConnectionManager.java:782) at org.apache.hadoop.hbase.client.HTable.finishSetup(HTable.java:251) at org.apache.hadoop.hbase.client.HTable.<init>(HTable.java:215) at org.apache.hadoop.hbase.catalog.MetaReader.getHTable(MetaReader.java:200) at org.apache.hadoop.hbase.catalog.MetaReader.getMetaHTable(MetaReader.java:226) at org.apache.hadoop.hbase.catalog.MetaReader.fullScan(MetaReader.java:700) at org.apache.hadoop.hbase.catalog.MetaReader.fullScan(MetaReader.java:183) at org.apache.hadoop.hbase.catalog.MetaReader.tableExists(MetaReader.java:448) at org.apache.hadoop.hbase.client.HBaseAdmin.tableExists(HBaseAdmin.java:204) at org.apache.hadoop.hbase.client.HBaseAdmin.tableExists(HBaseAdmin.java:218) at org.apache.hadoop.hbase.PerformanceEvaluation.checkTable(PerformanceEvaluation.java:484) at org.apache.hadoop.hbase.PerformanceEvaluation.runNIsMoreThanOne(PerformanceEvaluation.java:524) at org.apache.hadoop.hbase.PerformanceEvaluation.runTest(PerformanceEvaluation.java:1194) at org.apache.hadoop.hbase.PerformanceEvaluation.doCommandLine(PerformanceEvaluation.java:1309) at org.apache.hadoop.hbase.PerformanceEvaluation.main(PerformanceEvaluation.java:1334) Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:567) at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:207) at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:523) at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:488) at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupConnection(HBaseClient.java:328) at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:362) at org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:1045) at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:897) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:150) at $Proxy8.getProtocolVersion(Unknown Source) at org.apache.hadoop.hbase.ipc.WritableRpcEngine.getProxy(WritableRpcEngine.java:183) at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:303) at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:280) at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:332) at org.apache.hadoop.hbase.ipc.HBaseRPC.waitForProxy(HBaseRPC.java:236) ... 20 more {noformat} step 3: In Serengeti, stop and start cmds run well and show that the processes were done without any error. But NN/JT/HMaster/DD/TT/HRegionServer processes don't start on node after restarting the cluster in Serengeti. The related vmdks on worker node were not mounted as usual.

    SpringSource Issue Tracker | 4 years ago | Jinbo Zhang
    org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed setting up proxy interface org.apache.hadoop.hbase.ipc.HRegionInterface to /10.111.88.244:60020 after attempts=1
  4. Speed up your debug routine!

    Automated exception search integrated into your IDE

  5. 0

    I find out there is something wrong with HBase cluster with CDH4 distro. HBase cluster cannot restart normally when some job running failed. Would you help verify this issue please? *Steps* 1. Create a HBase cluster with CDH4 in Serengeti. 2. Run PerformanceEvaluation randomWrite and sequientialWrite job eg. hbase org.apache.hadoop.hbase.PerformanceEvaluation sequentialWrite 2 hbase org.apache.hadoop.hbase.PerformanceEvaluation randomWrite 3 3. Restart cluster in Serengeti. *Real Result* Step 2: The jobs failed with the console log similar as below: {noformat} 13/01/10 04:08:24 INFO zookeeper.ZooKeeper: Session: 0x23c2255b4ca002e closed org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed setting up proxy interface org.apache.hadoop.hbase.ipc.HRegionInterface to /10.111.88.244:60020 after attempts=1 at org.apache.hadoop.hbase.ipc.HBaseRPC.waitForProxy(HBaseRPC.java:242) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getHRegionConnection(HConnectionManager.java:1278) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getHRegionConnection(HConnectionManager.java:1235) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getHRegionConnection(HConnectionManager.java:1222) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegionInMeta(HConnectionManager.java:918) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegion(HConnectionManager.java:814) at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegion(HConnectionManager.java:782) at org.apache.hadoop.hbase.client.HTable.finishSetup(HTable.java:251) at org.apache.hadoop.hbase.client.HTable.<init>(HTable.java:215) at org.apache.hadoop.hbase.catalog.MetaReader.getHTable(MetaReader.java:200) at org.apache.hadoop.hbase.catalog.MetaReader.getMetaHTable(MetaReader.java:226) at org.apache.hadoop.hbase.catalog.MetaReader.fullScan(MetaReader.java:700) at org.apache.hadoop.hbase.catalog.MetaReader.fullScan(MetaReader.java:183) at org.apache.hadoop.hbase.catalog.MetaReader.tableExists(MetaReader.java:448) at org.apache.hadoop.hbase.client.HBaseAdmin.tableExists(HBaseAdmin.java:204) at org.apache.hadoop.hbase.client.HBaseAdmin.tableExists(HBaseAdmin.java:218) at org.apache.hadoop.hbase.PerformanceEvaluation.checkTable(PerformanceEvaluation.java:484) at org.apache.hadoop.hbase.PerformanceEvaluation.runNIsMoreThanOne(PerformanceEvaluation.java:524) at org.apache.hadoop.hbase.PerformanceEvaluation.runTest(PerformanceEvaluation.java:1194) at org.apache.hadoop.hbase.PerformanceEvaluation.doCommandLine(PerformanceEvaluation.java:1309) at org.apache.hadoop.hbase.PerformanceEvaluation.main(PerformanceEvaluation.java:1334) Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:567) at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:207) at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:523) at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:488) at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupConnection(HBaseClient.java:328) at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:362) at org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:1045) at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:897) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:150) at $Proxy8.getProtocolVersion(Unknown Source) at org.apache.hadoop.hbase.ipc.WritableRpcEngine.getProxy(WritableRpcEngine.java:183) at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:303) at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:280) at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:332) at org.apache.hadoop.hbase.ipc.HBaseRPC.waitForProxy(HBaseRPC.java:236) ... 20 more {noformat} step 3: In Serengeti, stop and start cmds run well and show that the processes were done without any error. But NN/JT/HMaster/DD/TT/HRegionServer processes don't start on node after restarting the cluster in Serengeti. The related vmdks on worker node were not mounted as usual.

    SpringSource Issue Tracker | 4 years ago | Jinbo Zhang
    org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed setting up proxy interface org.apache.hadoop.hbase.ipc.HRegionInterface to /10.111.88.244:60020 after attempts=1
  6. 0

    Sqoop import to HBase failed with large dataset

    Stack Overflow | 3 years ago | Koh
    java.net.ConnectException: Connection refused

  1. RoiOtero 1 times, last 2 weeks ago
  2. nasimk 1064 times, last 2 weeks ago
  3. gehel 8 times, last 1 month ago
  4. rp 1 times, last 1 month ago
  5. poroszd 4 times, last 2 months ago
10 more registered users
84 unregistered visitors
Not finding the right solution?
Take a tour to get the most out of Samebug.

Tired of useless tips?

Automated exception search integrated into your IDE

Root Cause Analysis

  1. java.net.ConnectException

    Connection refused

    at sun.nio.ch.SocketChannelImpl.checkConnect()
  2. Java RT
    SocketChannelImpl.finishConnect
    1. sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
    2. sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:567)
    2 frames
  3. Hadoop
    NetUtils.connect
    1. org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:207)
    2. org.apache.hadoop.net.NetUtils.connect(NetUtils.java:523)
    3. org.apache.hadoop.net.NetUtils.connect(NetUtils.java:488)
    3 frames
  4. HBase - Client
    WritableRpcEngine$Invoker.invoke
    1. org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupConnection(HBaseClient.java:328)
    2. org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:362)
    3. org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:1045)
    4. org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:897)
    5. org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:150)
    5 frames
  5. Unknown
    $Proxy8.getClosestRowBefore
    1. $Proxy8.getClosestRowBefore(Unknown Source)
    1 frame
  6. HBase - Client
    HTable.flushCommits
    1. org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegionInMeta(HConnectionManager.java:947)
    2. org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegion(HConnectionManager.java:814)
    3. org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.relocateRegion(HConnectionManager.java:788)
    4. org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegionInMeta(HConnectionManager.java:1024)
    5. org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.locateRegion(HConnectionManager.java:818)
    6. org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.processBatchCallback(HConnectionManager.java:1524)
    7. org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.processBatch(HConnectionManager.java:1409)
    8. org.apache.hadoop.hbase.client.HTable.flushCommits(HTable.java:949)
    8 frames
  7. HBase
    PerformanceEvaluation$1.run
    1. org.apache.hadoop.hbase.PerformanceEvaluation$Test.testTakedown(PerformanceEvaluation.java:796)
    2. org.apache.hadoop.hbase.PerformanceEvaluation$Test.test(PerformanceEvaluation.java:813)
    3. org.apache.hadoop.hbase.PerformanceEvaluation.runOneClient(PerformanceEvaluation.java:1140)
    4. org.apache.hadoop.hbase.PerformanceEvaluation$1.run(PerformanceEvaluation.java:548)
    4 frames