datanode.DataNode: Initialization failed for block pool Block pool BP-1241324352-10.111.95.159-1369221271966 (storage id DS-1103119836-10.111.95.160-50010-1369221492182) service to /10.111.95.163:8020 org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.server.protocol.DisallowedDatanodeException): Datanode denied communication with namenode: DatanodeRegistration(0.0.0.0, storageID=DS-1103119836-10.111.95.160-50010-1369221492182, infoPort=50075, ipcPort=50020, storageInfo=lv=-40;cid=cdh4Test2;nsid=1100414909;c=0)

SpringSource Issue Tracker | Lirong Mao | 4 years ago
  1. 0

    Reproduce step: Step1. Deploy a serengeti ova. Step2: Provison a cdh4 cluster( CDH4.2.1). Problem: Bootstrap failed. See attachment-"hadoop-hadoop-tasktracker-10.111.95.160.out" 13/05/23 02:47:52 FATAL datanode.DataNode: Initialization failed for block pool Block pool BP-1241324352-10.111.95.159-1369221271966 (storage id DS-1103119836-10.111.95.160-50010-1369221492182) service to /10.111.95.163:8020 org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.server.protocol.DisallowedDatanodeException): Datanode denied communication with namenode: DatanodeRegistration(0.0.0.0, storageID=DS-1103119836-10.111.95.160-50010-1369221492182, infoPort=50075, ipcPort=50020, storageInfo=lv=-40;cid=cdh4Test2;nsid=1100414909;c=0) at org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager.registerDatanode(DatanodeManager.java:569) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.registerDatanode(FSNamesystem.java:3481) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.registerDatanode(NameNodeRpcServer.java:862) at org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolServerSideTranslatorPB.registerDatanode(DatanodeProtocolServerSideTranslatorPB.java:91) at org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos$DatanodeProtocolService$2.callBlockingMethod(DatanodeProtocolProtos.java:20018) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:453) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1002) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1695) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1691) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:396) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1408) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1689) at org.apache.hadoop.ipc.Client.call(Client.java:1225) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:202) at $Proxy10.registerDatanode(Unknown Source) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:164) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:83) at $Proxy10.registerDatanode(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB.registerDatanode(DatanodeProtocolClientSideTranslatorPB.java:149) at org.apache.hadoop.hdfs.server.datanode.BPServiceActor.register(BPServiceActor.java:619) at org.apache.hadoop.hdfs.server.datanode.BPServiceActor.connectToNNAndHandshake(BPServiceActor.java:221) at org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:660) at java.lang.Thread.run(Thread.java:662)

    SpringSource Issue Tracker | 4 years ago | Lirong Mao
    datanode.DataNode: Initialization failed for block pool Block pool BP-1241324352-10.111.95.159-1369221271966 (storage id DS-1103119836-10.111.95.160-50010-1369221492182) service to /10.111.95.163:8020 org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.server.protocol.DisallowedDatanodeException): Datanode denied communication with namenode: DatanodeRegistration(0.0.0.0, storageID=DS-1103119836-10.111.95.160-50010-1369221492182, infoPort=50075, ipcPort=50020, storageInfo=lv=-40;cid=cdh4Test2;nsid=1100414909;c=0)
  2. 0

    Reproduce step: Step1. Deploy a serengeti ova. Step2: Provison a cdh4 cluster( CDH4.2.1). Problem: Bootstrap failed. See attachment-"hadoop-hadoop-tasktracker-10.111.95.160.out" 13/05/23 02:47:52 FATAL datanode.DataNode: Initialization failed for block pool Block pool BP-1241324352-10.111.95.159-1369221271966 (storage id DS-1103119836-10.111.95.160-50010-1369221492182) service to /10.111.95.163:8020 org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.server.protocol.DisallowedDatanodeException): Datanode denied communication with namenode: DatanodeRegistration(0.0.0.0, storageID=DS-1103119836-10.111.95.160-50010-1369221492182, infoPort=50075, ipcPort=50020, storageInfo=lv=-40;cid=cdh4Test2;nsid=1100414909;c=0) at org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager.registerDatanode(DatanodeManager.java:569) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.registerDatanode(FSNamesystem.java:3481) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.registerDatanode(NameNodeRpcServer.java:862) at org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolServerSideTranslatorPB.registerDatanode(DatanodeProtocolServerSideTranslatorPB.java:91) at org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos$DatanodeProtocolService$2.callBlockingMethod(DatanodeProtocolProtos.java:20018) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:453) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1002) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1695) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1691) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:396) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1408) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1689) at org.apache.hadoop.ipc.Client.call(Client.java:1225) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:202) at $Proxy10.registerDatanode(Unknown Source) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:164) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:83) at $Proxy10.registerDatanode(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB.registerDatanode(DatanodeProtocolClientSideTranslatorPB.java:149) at org.apache.hadoop.hdfs.server.datanode.BPServiceActor.register(BPServiceActor.java:619) at org.apache.hadoop.hdfs.server.datanode.BPServiceActor.connectToNNAndHandshake(BPServiceActor.java:221) at org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:660) at java.lang.Thread.run(Thread.java:662)

    SpringSource Issue Tracker | 4 years ago | Lirong Mao
    datanode.DataNode: Initialization failed for block pool Block pool BP-1241324352-10.111.95.159-1369221271966 (storage id DS-1103119836-10.111.95.160-50010-1369221492182) service to /10.111.95.163:8020 org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.server.protocol.DisallowedDatanodeException): Datanode denied communication with namenode: DatanodeRegistration(0.0.0.0, storageID=DS-1103119836-10.111.95.160-50010-1369221492182, infoPort=50075, ipcPort=50020, storageInfo=lv=-40;cid=cdh4Test2;nsid=1100414909;c=0)
  3. 0

    [HBase-dev] [jira] [Resolved] (HBASE-5718) HBaseTestingUtility causes javax.management.NotCompliantMBeanException - Grokbase

    grokbase.com | 5 months ago
    datanode.DataNode: DatanodeRegistration(127.0.0.1:56956, storageID=DS-134885185-127.0.0.1-56956-1333569900707, infoPort=53398, ipcPort=60794):DataXceiveServer:java.nio.channels.AsynchronousCloseException
  4. Speed up your debug routine!

    Automated exception search integrated into your IDE

  5. 0

    [MapReduce-dev] Hadoop-Mapreduce-trunk - Build # 663 - Still Failing - Grokbase

    grokbase.com | 1 year ago
    datanode.DataNode: DatanodeRegistration(127.0.0.1:45474, storageID=DS-1511697129-127.0.1.1-45474-1304092382141, infoPort=35473, ipcPort=53883):DataXceiveServer: java.nio.channels.AsynchronousCloseException
  6. 0

    hadoop : datanode not running? - Ask Ubuntu

    askubuntu.com | 8 months ago
    datanode.DataNode: java.io.IOException: Incompatible namespaceIDs in /app/hadoop/tmp/dfs/data: namenode namespaceID = 1955988395; datanode namespaceID = 1705269445

    Not finding the right solution?
    Take a tour to get the most out of Samebug.

    Tired of useless tips?

    Automated exception search integrated into your IDE

    Root Cause Analysis

    1. datanode.DataNode

      Initialization failed for block pool Block pool BP-1241324352-10.111.95.159-1369221271966 (storage id DS-1103119836-10.111.95.160-50010-1369221492182) service to /10.111.95.163:8020 org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.server.protocol.DisallowedDatanodeException): Datanode denied communication with namenode: DatanodeRegistration(0.0.0.0, storageID=DS-1103119836-10.111.95.160-50010-1369221492182, infoPort=50075, ipcPort=50020, storageInfo=lv=-40;cid=cdh4Test2;nsid=1100414909;c=0)

      at org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager.registerDatanode()
    2. Apache Hadoop HDFS
      DatanodeProtocolProtos$DatanodeProtocolService$2.callBlockingMethod
      1. org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager.registerDatanode(DatanodeManager.java:569)
      2. org.apache.hadoop.hdfs.server.namenode.FSNamesystem.registerDatanode(FSNamesystem.java:3481)
      3. org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.registerDatanode(NameNodeRpcServer.java:862)
      4. org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolServerSideTranslatorPB.registerDatanode(DatanodeProtocolServerSideTranslatorPB.java:91)
      5. org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos$DatanodeProtocolService$2.callBlockingMethod(DatanodeProtocolProtos.java:20018)
      5 frames
    3. Hadoop
      Server$Handler$1.run
      1. org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:453)
      2. org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1002)
      3. org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1695)
      4. org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1691)
      4 frames
    4. Java RT
      Subject.doAs
      1. java.security.AccessController.doPrivileged(Native Method)
      2. javax.security.auth.Subject.doAs(Subject.java:396)
      2 frames
    5. Hadoop
      ProtobufRpcEngine$Invoker.invoke
      1. org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1408)
      2. org.apache.hadoop.ipc.Server$Handler.run(Server.java:1689)
      3. org.apache.hadoop.ipc.Client.call(Client.java:1225)
      4. org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:202)
      4 frames
    6. Unknown
      $Proxy10.registerDatanode
      1. $Proxy10.registerDatanode(Unknown Source)
      1 frame
    7. Java RT
      Method.invoke
      1. sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
      2. sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
      3. sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
      4. java.lang.reflect.Method.invoke(Method.java:597)
      4 frames
    8. Hadoop
      RetryInvocationHandler.invoke
      1. org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:164)
      2. org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:83)
      2 frames
    9. Unknown
      $Proxy10.registerDatanode
      1. $Proxy10.registerDatanode(Unknown Source)
      1 frame
    10. Apache Hadoop HDFS
      BPServiceActor.run
      1. org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB.registerDatanode(DatanodeProtocolClientSideTranslatorPB.java:149)
      2. org.apache.hadoop.hdfs.server.datanode.BPServiceActor.register(BPServiceActor.java:619)
      3. org.apache.hadoop.hdfs.server.datanode.BPServiceActor.connectToNNAndHandshake(BPServiceActor.java:221)
      4. org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:660)
      4 frames
    11. Java RT
      Thread.run
      1. java.lang.Thread.run(Thread.java:662)
      1 frame