org.onosproject.store.service.StorageException$Timeout

JIRA | You Wang | 3 months ago
  1. 0

    It only happens after running CHO test for 2 or 3 days. ONOS node2 became unresponsive after the exception. {noformat} 2016-08-31 12:09:46,270 | ERROR | ew I/O worker #9 | OFChannelHandler | 180 - org.onosproject.onos-of-ctl - 1.7.0.SNAPSHOT | Error while processing message from switch org.onosproject.driver.handshaker.NiciraSwitchHandshaker [/10.128.40.50:43680 DPID[00:00 :00:00:00:00:00:03]]state ACTIVE org.onosproject.store.service.StorageException$Timeout at org.onosproject.store.primitives.DefaultLeaderElector.complete(DefaultLeaderElector.java:115) at org.onosproject.store.primitives.DefaultLeaderElector.getLeadership(DefaultLeaderElector.java:75) at org.onosproject.store.cluster.impl.DistributedLeadershipStore.getLeadership(DistributedLeadershipStore.java:165) at org.onosproject.cluster.impl.LeadershipManager.getLeadership(LeadershipManager.java:83) at org.onosproject.store.mastership.impl.ConsistentDeviceMastershipStore.getRole(ConsistentDeviceMastershipStore.java:168)[78:org.onosproject.onos-core-dist:1.7.0.SNAPSHOT] at org.onosproject.store.mastership.impl.ConsistentDeviceMastershipStore.lambda$getNodes$0(ConsistentDeviceMastershipStore.java:189)[78:org.onosproject.onos-core-dist:1.7.0.SNAPSHOT] at java.lang.Iterable.forEach(Iterable.java:75)[:1.8.0_72] at org.onosproject.store.mastership.impl.ConsistentDeviceMastershipStore.getNodes(ConsistentDeviceMastershipStore.java:189)[78:org.onosproject.onos-core-dist:1.7.0.SNAPSHOT] at org.onosproject.store.mastership.impl.ConsistentDeviceMastershipStore.relinquishLocalRole(ConsistentDeviceMastershipStore.java:299)[78:org.onosproject.onos-core-dist:1.7.0.SNAPSHOT] at org.onosproject.store.mastership.impl.ConsistentDeviceMastershipStore.relinquishRole(ConsistentDeviceMastershipStore.java:276)[78:org.onosproject.onos-core-dist:1.7.0.SNAPSHOT] at org.onosproject.cluster.impl.MastershipManager.relinquishMastership(MastershipManager.java:158)[76:org.onosproject.onos-core-net:1.7.0.SNAPSHOT] at org.onosproject.net.device.impl.DeviceManager$InternalDeviceProviderService.deviceDisconnected(DeviceManager.java:451)[76:org.onosproject.onos-core-net:1.7.0.SNAPSHOT] at org.onosproject.provider.of.device.impl.OpenFlowDeviceProvider$InternalDeviceProvider.switchRemoved(OpenFlowDeviceProvider.java:409)[181:org.onosproject.onos-of-provider-device:1.7.0.SNAPSHOT] at org.onosproject.openflow.controller.impl.OpenFlowControllerImpl$OpenFlowSwitchAgent.removeConnectedSwitch(OpenFlowControllerImpl.java:661)[180:org.onosproject.onos-of-ctl:1.7.0.SNAPSHOT] at org.onosproject.openflow.controller.driver.AbstractOpenFlowSwitch.removeConnectedSwitch(AbstractOpenFlowSwitch.java:308)[177:org.onosproject.onos-of-api:1.7.0.SNAPSHOT] at org.onosproject.openflow.controller.impl.OFChannelHandler.channelDisconnected(OFChannelHandler.java:1073)[180:org.onosproject.onos-of-ctl:1.7.0.SNAPSHOT] at org.jboss.netty.channel.SimpleChannelHandler.handleUpstream(SimpleChannelHandler.java:120)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.handler.timeout.IdleStateAwareChannelHandler.handleUpstream(IdleStateAwareChannelHandler.java:36)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.channel.socket.nio.AbstractNioWorker.close(AbstractNioWorker.java:360)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:93)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:337)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)[44:org.jboss.netty:3.10.5.Final] at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)[44:org.jboss.netty:3.10.5.Final] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)[:1.8.0_72] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)[:1.8.0_72] at java.lang.Thread.run(Thread.java:745)[:1.8.0_72] {noformat}

    JIRA | 3 months ago | You Wang
    org.onosproject.store.service.StorageException$Timeout

    Root Cause Analysis

    1. org.onosproject.store.service.StorageException$Timeout

      No message provided

      at org.onosproject.store.primitives.DefaultLeaderElector.complete()
    2. org.onosproject.store
      DistributedLeadershipStore.getLeadership
      1. org.onosproject.store.primitives.DefaultLeaderElector.complete(DefaultLeaderElector.java:115)
      2. org.onosproject.store.primitives.DefaultLeaderElector.getLeadership(DefaultLeaderElector.java:75)
      3. org.onosproject.store.cluster.impl.DistributedLeadershipStore.getLeadership(DistributedLeadershipStore.java:165)
      3 frames
    3. org.onosproject.cluster
      LeadershipManager.getLeadership
      1. org.onosproject.cluster.impl.LeadershipManager.getLeadership(LeadershipManager.java:83)
      1 frame
    4. org.onosproject.store
      ConsistentDeviceMastershipStore.lambda$getNodes$0
      1. org.onosproject.store.mastership.impl.ConsistentDeviceMastershipStore.getRole(ConsistentDeviceMastershipStore.java:168)[78:org.onosproject.onos-core-dist:1.7.0.SNAPSHOT]
      2. org.onosproject.store.mastership.impl.ConsistentDeviceMastershipStore.lambda$getNodes$0(ConsistentDeviceMastershipStore.java:189)[78:org.onosproject.onos-core-dist:1.7.0.SNAPSHOT]
      2 frames
    5. Java RT
      Iterable.forEach
      1. java.lang.Iterable.forEach(Iterable.java:75)[:1.8.0_72]
      1 frame
    6. org.onosproject.store
      ConsistentDeviceMastershipStore.relinquishRole
      1. org.onosproject.store.mastership.impl.ConsistentDeviceMastershipStore.getNodes(ConsistentDeviceMastershipStore.java:189)[78:org.onosproject.onos-core-dist:1.7.0.SNAPSHOT]
      2. org.onosproject.store.mastership.impl.ConsistentDeviceMastershipStore.relinquishLocalRole(ConsistentDeviceMastershipStore.java:299)[78:org.onosproject.onos-core-dist:1.7.0.SNAPSHOT]
      3. org.onosproject.store.mastership.impl.ConsistentDeviceMastershipStore.relinquishRole(ConsistentDeviceMastershipStore.java:276)[78:org.onosproject.onos-core-dist:1.7.0.SNAPSHOT]
      3 frames
    7. org.onosproject.cluster
      MastershipManager.relinquishMastership
      1. org.onosproject.cluster.impl.MastershipManager.relinquishMastership(MastershipManager.java:158)[76:org.onosproject.onos-core-net:1.7.0.SNAPSHOT]
      1 frame
    8. org.onosproject.net
      DeviceManager$InternalDeviceProviderService.deviceDisconnected
      1. org.onosproject.net.device.impl.DeviceManager$InternalDeviceProviderService.deviceDisconnected(DeviceManager.java:451)[76:org.onosproject.onos-core-net:1.7.0.SNAPSHOT]
      1 frame
    9. org.onosproject.provider
      OpenFlowDeviceProvider$InternalDeviceProvider.switchRemoved
      1. org.onosproject.provider.of.device.impl.OpenFlowDeviceProvider$InternalDeviceProvider.switchRemoved(OpenFlowDeviceProvider.java:409)[181:org.onosproject.onos-of-provider-device:1.7.0.SNAPSHOT]
      1 frame
    10. org.onosproject.openflow
      OFChannelHandler.channelDisconnected
      1. org.onosproject.openflow.controller.impl.OpenFlowControllerImpl$OpenFlowSwitchAgent.removeConnectedSwitch(OpenFlowControllerImpl.java:661)[180:org.onosproject.onos-of-ctl:1.7.0.SNAPSHOT]
      2. org.onosproject.openflow.controller.driver.AbstractOpenFlowSwitch.removeConnectedSwitch(AbstractOpenFlowSwitch.java:308)[177:org.onosproject.onos-of-api:1.7.0.SNAPSHOT]
      3. org.onosproject.openflow.controller.impl.OFChannelHandler.channelDisconnected(OFChannelHandler.java:1073)[180:org.onosproject.onos-of-ctl:1.7.0.SNAPSHOT]
      3 frames
    11. Netty
      DeadLockProofWorker$1.run
      1. org.jboss.netty.channel.SimpleChannelHandler.handleUpstream(SimpleChannelHandler.java:120)[44:org.jboss.netty:3.10.5.Final]
      2. org.jboss.netty.handler.timeout.IdleStateAwareChannelHandler.handleUpstream(IdleStateAwareChannelHandler.java:36)[44:org.jboss.netty:3.10.5.Final]
      3. org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)[44:org.jboss.netty:3.10.5.Final]
      4. org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)[44:org.jboss.netty:3.10.5.Final]
      5. org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208)[44:org.jboss.netty:3.10.5.Final]
      6. org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102)[44:org.jboss.netty:3.10.5.Final]
      7. org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)[44:org.jboss.netty:3.10.5.Final]
      8. org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)[44:org.jboss.netty:3.10.5.Final]
      9. org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208)[44:org.jboss.netty:3.10.5.Final]
      10. org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102)[44:org.jboss.netty:3.10.5.Final]
      11. org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)[44:org.jboss.netty:3.10.5.Final]
      12. org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)[44:org.jboss.netty:3.10.5.Final]
      13. org.jboss.netty.channel.SimpleChannelUpstreamHandler.channelDisconnected(SimpleChannelUpstreamHandler.java:208)[44:org.jboss.netty:3.10.5.Final]
      14. org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102)[44:org.jboss.netty:3.10.5.Final]
      15. org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)[44:org.jboss.netty:3.10.5.Final]
      16. org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)[44:org.jboss.netty:3.10.5.Final]
      17. org.jboss.netty.handler.codec.frame.FrameDecoder.cleanup(FrameDecoder.java:493)[44:org.jboss.netty:3.10.5.Final]
      18. org.jboss.netty.handler.codec.frame.FrameDecoder.channelDisconnected(FrameDecoder.java:365)[44:org.jboss.netty:3.10.5.Final]
      19. org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102)[44:org.jboss.netty:3.10.5.Final]
      20. org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)[44:org.jboss.netty:3.10.5.Final]
      21. org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559)[44:org.jboss.netty:3.10.5.Final]
      22. org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396)[44:org.jboss.netty:3.10.5.Final]
      23. org.jboss.netty.channel.socket.nio.AbstractNioWorker.close(AbstractNioWorker.java:360)[44:org.jboss.netty:3.10.5.Final]
      24. org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:93)[44:org.jboss.netty:3.10.5.Final]
      25. org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108)[44:org.jboss.netty:3.10.5.Final]
      26. org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:337)[44:org.jboss.netty:3.10.5.Final]
      27. org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)[44:org.jboss.netty:3.10.5.Final]
      28. org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)[44:org.jboss.netty:3.10.5.Final]
      29. org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)[44:org.jboss.netty:3.10.5.Final]
      30. org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)[44:org.jboss.netty:3.10.5.Final]
      30 frames
    12. Java RT
      Thread.run
      1. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)[:1.8.0_72]
      2. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)[:1.8.0_72]
      3. java.lang.Thread.run(Thread.java:745)[:1.8.0_72]
      3 frames