javax.jms.JMSException: Peer (vm://localhost#17) disposed.

JIRA | Wyatt Alt | 2 years ago
  1. 0

    We occasionally see JMS exceptions during startup or shutdown, but some large-scale users are experiencing them during normal operation From the PE-7392 logs for host 101 on 2015-01-08 these issues start with this error which seems to come out of the blue: {code} 2015-01-06 16:45:12,069 INFO [c.p.p.command] [84e83891-23d0-4a85-ae9a-87cdef02940d] [replace facts] xc101xmlmobile-39.ams4.prod.booking.com 2015-01-06 16:45:14,222 INFO [c.p.p.command] [e2ffd05e-aced-4289-9fad-cb90e5a9d20f] [replace catalog] sr101sessredis-02.ams4.prod.booking.com 2015-01-06 16:45:15,259 ERROR [o.a.a.s.k.MessageDatabase] KahaDB failed to store to Journal java.io.FileNotFoundException: /var/lib/puppetdb/mq/localhost/KahaDB/db-7087.log (No such file or directory) at java.io.RandomAccessFile.open(Native Method) ~[na:1.7.0_65] at java.io.RandomAccessFile.<init>(RandomAccessFile.java:241) ~[na:1.7.0_65] at org.apache.kahadb.journal.DataFile.openRandomAccessFile(DataFile.java:70) ~[puppetdb.jar:na] at org.apache.kahadb.journal.DataFileAppender.processQueue(DataFileAppender.java:309) ~[puppetdb.jar:na] at org.apache.kahadb.journal.DataFileAppender$1.run(DataFileAppender.java:188) ~[puppetdb.jar:na] 2015-01-06 16:45:15,286 WARN [o.e.j.s.HttpChannel] /v3/commands?checksum=43525b026292eac95a00cd286f7de152108701fb org.springframework.jms.UncategorizedJmsException: Uncategorized exception occured during JMS processing; nested exception is javax.jms.JMSException: /var/lib/puppetdb/mq/localhost/KahaDB/db-7087.log (No such file or directory) at org.springframework.jms.support.JmsUtils.convertJmsAccessException(JmsUtils.java:316) ~[puppetdb.jar:na] at org.springframework.jms.support.JmsAccessor.convertJmsAccessException(JmsAccessor.java:168) ~[puppetdb.jar:na] at org.springframework.jms.core.JmsTemplate.execute(JmsTemplate.java:469) ~[puppetdb.jar:na] at org.springframework.jms.core.JmsTemplate.send(JmsTemplate.java:543) ~[puppetdb.jar:na] at org.springframework.jms.core.JmsTemplate.convertAndSend(JmsTemplate.java:653) ~[puppetdb.jar:na] at sun.reflect.GeneratedMethodAccessor97.invoke(Unknown Source) ~[na:na] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_65] at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_65] at clojure.lang.Reflector.invokeMatchingMethod(Reflector.java:93) ~[puppetdb.jar:na] at clojure.lang.Reflector.invokeInstanceMethod(Reflector.java:28) ~[puppetdb.jar:na] at clamq.jms$jms_producer$reify__10735.publish(jms.clj:29) ~[na:na] at clamq.protocol.producer$eval10687$fn__10688$G__10679__10696.invoke(producer.clj:3) ~[na:na] at clamq.protocol.producer$eval10687$fn__10688$G__10678__10705.invoke(producer.clj:3) ~[na:na] at clojure.lang.AFn.applyToHelper(AFn.java:165) ~[puppetdb.jar:na] at clojure.lang.AFn.applyTo(AFn.java:144) ~[puppetdb.jar:na] at clojure.core$apply.invoke(core.clj:626) ~[puppetdb.jar:na] at com.puppetlabs.mq$connect_and_publish_BANG_.doInvoke(mq.clj:148) ~[na:na] at clojure.lang.RestFn.invoke(RestFn.java:460) ~[puppetdb.jar:na] at com.puppetlabs.puppetdb.command$enqueue_raw_command_BANG_$fn__11440.invoke(command.clj:249) ~[na:na] at com.puppetlabs.puppetdb.command$enqueue_raw_command_BANG_.invoke(command.clj:248) ~[na:na] at com.puppetlabs.puppetdb.http.command$enqueue_command.invoke(command.clj:12) ~[na:na] at com.puppetlabs.middleware$verify_accepts_content_type$fn__12248.invoke(middleware.clj:114) ~[na:na] at com.puppetlabs.middleware$verify_checksum$fn__12268.invoke(middleware.clj:185) ~[na:na] at com.puppetlabs.middleware$validate_query_params$fn__12259.invoke(middleware.clj:155) ~[na:na] at com.puppetlabs.middleware$payload_to_body_string$fn__12303.invoke(middleware.clj:276) ~[na:na] at com.puppetlabs.middleware$verify_content_type$fn__12253.invoke(middleware.clj:128) ~[na:na] at com.puppetlabs.puppetdb.http.v3$v3_app$fn__13762.invoke(v3.clj:23) ~[na:na] at com.puppetlabs.puppetdb.http.v3$v3_app.invoke(v3.clj:21) ~[na:na] at com.puppetlabs.puppetdb.http.server$routes$fn__14649$fn__14663.invoke(server.clj:48) ~[na:na] at net.cgrand.moustache$alter_request$fn__12677.invoke(moustache.clj:54) ~[na:na] at com.puppetlabs.puppetdb.http.server$routes$fn__14649.invoke(server.clj:48) ~[na:na] at ring.middleware.resource$wrap_resource$fn__14631.invoke(resource.clj:24) ~[na:na] at ring.middleware.params$wrap_params$fn__12620.invoke(params.clj:58) ~[na:na] at com.puppetlabs.middleware$wrap_with_authorization$fn__12220.invoke(middleware.clj:40) ~[na:na] at com.puppetlabs.middleware$wrap_with_certificate_cn$fn__12224.invoke(middleware.clj:55) ~[na:na] at com.puppetlabs.middleware$wrap_with_default_body$fn__12228.invoke(middleware.clj:62) ~[na:na] at com.puppetlabs.middleware$wrap_with_metrics_STAR_$fn__12275$fn__12284.invoke(middleware.clj:207) ~[na:na] at com.puppetlabs.utils.metrics$multitime_BANG__STAR_$fn__11942$fn__11943$fn__11944.invoke(metrics.clj:14) ~[na:na] at com.puppetlabs.utils.metrics.proxy$java.lang.Object$Callable$7da976d4.call(Unknown Source) ~[na:na] at com.yammer.metrics.core.Timer.time(Timer.java:91) ~[puppetdb.jar:na] {code} this is followed by hours of JMS errors interspersed with command processing. {code} at com.puppetlabs.utils.metrics$multitime_BANG__STAR_.invoke(metrics.clj:17) ~[na:na] at com.puppetlabs.middleware$wrap_with_metrics_STAR_$fn__12275.invoke(middleware.clj:206) ~[na:na] at com.puppetlabs.middleware$wrap_with_globals$fn__12232.invoke(middleware.clj:73) ~[na:na] at com.puppetlabs.middleware$wrap_with_debug_logging$fn__12216.invoke(middleware.clj:27) ~[na:na] at compojure.core$routing$fn__17778.invoke(core.clj:107) ~[na:na] at clojure.core$some.invoke(core.clj:2515) ~[puppetdb.jar:na] at compojure.core$routing.doInvoke(core.clj:107) ~[na:na] at clojure.lang.RestFn.invoke(RestFn.java:423) ~[puppetdb.jar:na] at com.puppetlabs.puppetdb.cli.services$start_puppetdb$fn__19272.invoke(services.clj:330) ~[na:na] at compojure.core$wrap_context$fn__17832.invoke(core.clj:165) ~[na:na] at compojure.core$if_route$fn__17756.invoke(core.clj:40) ~[na:na] at puppetlabs.trapperkeeper.services.webserver.jetty9_core$proxy_handler$fn__19529.invoke(jetty9_core.clj:66) ~[na:na] at puppetlabs.trapperkeeper.services.webserver.jetty9_core.proxy$org.eclipse.jetty.server.handler.AbstractHandler$ff19274a.handle(Unknown Source) ~[na:na] at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1112) ~[puppetdb.jar:na] at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1048) ~[puppetdb.jar:na] at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) ~[puppetdb.jar:na] at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:199) ~[puppetdb.jar:na] at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:109) ~[puppetdb.jar:na] at org.eclipse.jetty.servlets.gzip.GzipHandler.handle(GzipHandler.java:275) ~[puppetdb.jar:na] at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97) ~[puppetdb.jar:na] at org.eclipse.jetty.server.Server.handle(Server.java:459) ~[puppetdb.jar:na] at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:280) ~[puppetdb.jar:na] at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:229) [puppetdb.jar:na] at org.eclipse.jetty.io.AbstractConnection$1.run(AbstractConnection.java:505) [puppetdb.jar:na] at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:607) [puppetdb.jar:na] at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:536) [puppetdb.jar:na] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_65] Caused by: javax.jms.JMSException: Could not create Transport. Reason: java.io.IOException: Broker named 'localhost' does not exist. at org.apache.activemq.util.JMSExceptionSupport.create(JMSExceptionSupport.java:35) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnectionFactory.createTransport(ActiveMQConnectionFactory.java:254) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnectionFactory.createActiveMQConnection(ActiveMQConnectionFactory.java:267) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnectionFactory.createActiveMQConnection(ActiveMQConnectionFactory.java:239) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnectionFactory.createConnection(ActiveMQConnectionFactory.java:185) ~[puppetdb.jar:na] at org.springframework.jms.connection.SingleConnectionFactory.doCreateConnection(SingleConnectionFactory.java:342) ~[puppetdb.jar:na] at org.springframework.jms.connection.SingleConnectionFactory.initConnection(SingleConnectionFactory.java:288) ~[puppetdb.jar:na] at org.springframework.jms.connection.SingleConnectionFactory.createConnection(SingleConnectionFactory.java:225) ~[puppetdb.jar:na] at org.springframework.jms.support.JmsAccessor.createConnection(JmsAccessor.java:184) ~[puppetdb.jar:na] at org.springframework.jms.core.JmsTemplate.execute(JmsTemplate.java:456) ~[puppetdb.jar:na] ... 69 common frames omitted Caused by: java.io.IOException: Broker named 'localhost' does not exist. at org.apache.activemq.transport.vm.VMTransportFactory.doCompositeConnect(VMTransportFactory.java:116) ~[puppetdb.jar:na] at org.apache.activemq.transport.vm.VMTransportFactory.doConnect(VMTransportFactory.java:54) ~[puppetdb.jar:na] at org.apache.activemq.transport.TransportFactory.doConnect(TransportFactory.java:51) ~[puppetdb.jar:na] at org.apache.activemq.transport.TransportFactory.connect(TransportFactory.java:80) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnectionFactory.createTransport(ActiveMQConnectionFactory.java:252) ~[puppetdb.jar:na] ... 77 common frames omitted 2015-01-06 16:45:45,040 INFO [c.p.p.command] [cfe812d2-bba9-4f47-be4c-ac6453d74b4f] [replace facts] av1rdb-114.ams4.prod.booking.com 2015-01-06 16:45:45,040 INFO [c.p.p.command] [c1a3b13e-4fa6-4017-862a-4894fe26a58c] [replace facts] pc101pphmsapp-01.ams4.prod.booking.com 2015-01-06 16:45:45,041 INFO [c.p.p.command] [ed9160f0-47b8-41c5-933f-0824a0d983a6] [replace facts] bc144bprdb-01.ams4.prod.booking.com 2015-01-06 16:45:45,042 WARN [o.s.j.l.DefaultMessageListenerContainer] Execution of JMS message listener failed, and no ErrorHandler has been set. javax.jms.JMSException: Peer (vm://localhost#16) disposed. at org.apache.activemq.util.JMSExceptionSupport.create(JMSExceptionSupport.java:62) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1380) ~[puppetdb.jar:na] at org.apache.activemq.TransactionContext.rollback(TransactionContext.java:279) ~[puppetdb.jar:na] at org.apache.activemq.TransactionContext.commit(TransactionContext.java:306) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQSession.commit(ActiveMQSession.java:561) ~[puppetdb.jar:na] at sun.reflect.GeneratedMethodAccessor83.invoke(Unknown Source) ~[na:na] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_65] at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_65] at org.springframework.jms.connection.CachingConnectionFactory$CachedSessionInvocationHandler.invoke(CachingConnectionFactory.java:344) ~[puppetdb.jar:na] at com.sun.proxy.$Proxy5.commit(Unknown Source) ~[na:na] at org.springframework.jms.support.JmsUtils.commitIfNecessary(JmsUtils.java:217) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractMessageListenerContainer.commitIfNecessary(AbstractMessageListenerContainer.java:575) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractMessageListenerContainer.doExecuteListener(AbstractMessageListenerContainer.java:481) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.doReceiveAndExecute(AbstractPollingMessageListenerContainer.java:325) [puppetdb.jar:na] at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:263) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1058) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1050) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:947) [puppetdb.jar:na] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_65] Caused by: org.apache.activemq.transport.TransportDisposedIOException: Peer (vm://localhost#16) disposed. at org.apache.activemq.transport.vm.VMTransport.oneway(VMTransport.java:88) ~[puppetdb.jar:na] at org.apache.activemq.transport.MutexTransport.oneway(MutexTransport.java:68) ~[puppetdb.jar:na] at org.apache.activemq.transport.ResponseCorrelator.asyncRequest(ResponseCorrelator.java:81) ~[puppetdb.jar:na] at org.apache.activemq.transport.ResponseCorrelator.request(ResponseCorrelator.java:86) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1351) ~[puppetdb.jar:na] ... 17 common frames omitted 2015-01-06 16:45:45,042 WARN [o.s.j.l.DefaultMessageListenerContainer] Execution of JMS message listener failed, and no ErrorHandler has been set. javax.jms.JMSException: Peer (vm://localhost#17) disposed. at org.apache.activemq.util.JMSExceptionSupport.create(JMSExceptionSupport.java:62) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1380) ~[puppetdb.jar:na] at org.apache.activemq.TransactionContext.rollback(TransactionContext.java:279) ~[puppetdb.jar:na] at org.apache.activemq.TransactionContext.commit(TransactionContext.java:306) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQSession.commit(ActiveMQSession.java:561) ~[puppetdb.jar:na] at sun.reflect.GeneratedMethodAccessor83.invoke(Unknown Source) ~[na:na] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_65] at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_65] at org.springframework.jms.connection.CachingConnectionFactory$CachedSessionInvocationHandler.invoke(CachingConnectionFactory.java:344) ~[puppetdb.jar:na] at com.sun.proxy.$Proxy5.commit(Unknown Source) ~[na:na] at org.springframework.jms.support.JmsUtils.commitIfNecessary(JmsUtils.java:217) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractMessageListenerContainer.commitIfNecessary(AbstractMessageListenerContainer.java:575) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractMessageListenerContainer.doExecuteListener(AbstractMessageListenerContainer.java:481) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.doReceiveAndExecute(AbstractPollingMessageListenerContainer.java:325) [puppetdb.jar:na] at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:263) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1058) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1050) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:947) [puppetdb.jar:na] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_65] Caused by: org.apache.activemq.transport.TransportDisposedIOException: Peer (vm://localhost#17) disposed. at org.apache.activemq.transport.vm.VMTransport.oneway(VMTransport.java:88) ~[puppetdb.jar:na] at org.apache.activemq.transport.MutexTransport.oneway(MutexTransport.java:68) ~[puppetdb.jar:na] at org.apache.activemq.transport.ResponseCorrelator.asyncRequest(ResponseCorrelator.java:81) ~[puppetdb.jar:na] at org.apache.activemq.transport.ResponseCorrelator.request(ResponseCorrelator.java:86) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1351) ~[puppetdb.jar:na] ... 17 common frames omitted 2015-01-06 16:45:45,042 WARN [o.s.j.l.DefaultMessageListenerContainer] Execution of JMS message listener failed, and no ErrorHandler has been set. javax.jms.JMSException: Peer (vm://localhost#51) disposed. at org.apache.activemq.util.JMSExceptionSupport.create(JMSExceptionSupport.java:62) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1380) ~[puppetdb.jar:na] at org.apache.activemq.TransactionContext.rollback(TransactionContext.java:279) ~[puppetdb.jar:na] at org.apache.activemq.TransactionContext.commit(TransactionContext.java:306) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQSession.commit(ActiveMQSession.java:561) ~[puppetdb.jar:na] at sun.reflect.GeneratedMethodAccessor83.invoke(Unknown Source) ~[na:na] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_65] at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_65] at org.springframework.jms.connection.CachingConnectionFactory$CachedSessionInvocationHandler.invoke(CachingConnectionFactory.java:344) ~[puppetdb.jar:na] at com.sun.proxy.$Proxy5.commit(Unknown Source) ~[na:na] at org.springframework.jms.support.JmsUtils.commitIfNecessary(JmsUtils.java:217) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractMessageListenerContainer.commitIfNecessary(AbstractMessageListenerContainer.java:575) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractMessageListenerContainer.doExecuteListener(AbstractMessageListenerContainer.java:481) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.doReceiveAndExecute(AbstractPollingMessageListenerContainer.java:325) [puppetdb.jar:na] at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:263) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1058) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1050) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:947) [puppetdb.jar:na] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_65] Caused by: org.apache.activemq.transport.TransportDisposedIOException: Peer (vm://localhost#51) disposed. at org.apache.activemq.transport.vm.VMTransport.oneway(VMTransport.java:88) ~[puppetdb.jar:na] at org.apache.activemq.transport.MutexTransport.oneway(MutexTransport.java:68) ~[puppetdb.jar:na] at org.apache.activemq.transport.ResponseCorrelator.asyncRequest(ResponseCorrelator.java:81) ~[puppetdb.jar:na] at org.apache.activemq.transport.ResponseCorrelator.request(ResponseCorrelator.java:86) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1351) ~[puppetdb.jar:na] ... 17 common frames omitted 2015-01-06 16:45:45,042 WARN [o.s.j.l.DefaultMessageListenerContainer] Setup of JMS message listener invoker failed for destination 'com.puppetlabs.puppetdb.commands' - trying to recover. Cause: Peer (vm://localhost#17) disposed. 2015-01-06 16:45:45,042 WARN [o.s.j.l.DefaultMessageListenerContainer] Setup of JMS message listener invoker failed for destination 'com.puppetlabs.puppetdb.commands' - trying to recover. Cause: Peer (vm://localhost#16) disposed. 2015-01-06 16:45:45,043 INFO [o.s.j.l.DefaultMessageListenerContainer] Successfully refreshed JMS Connection 2015-01-06 16:45:45,043 WARN [o.s.j.l.DefaultMessageListenerContainer] Setup of JMS message listener invoker failed for destination 'com.puppetlabs.puppetdb.commands' - trying to recover. Cause: Peer (vm://localhost#51) disposed. 2015-01-06 16:45:45,043 INFO [o.s.j.l.DefaultMessageListenerContainer] Successfully refreshed JMS Connection 2015-01-06 16:45:45,045 INFO [o.s.j.l.DefaultMessageListenerContainer] Successfully refreshed JMS Connection 2015-01-06 16:45:45,388 WARN [o.s.j.l.DefaultMessageListenerContainer] Setup of JMS message listener invoker failed for destination 'com.puppetlabs.puppetdb.commands' - trying to recover. Cause: Peer (vm://localhost#57) disposed. 2015-01-06 16:45:45,388 INFO [o.s.j.l.DefaultMessageListenerContainer] Successfully refreshed JMS Connection 2015-01-06 16:45:45,390 WARN [o.s.j.l.DefaultMessageListenerContainer] Setup of JMS message listener invoker failed for destination 'com.puppetlabs.puppetdb.commands' - trying to recover. Cause: Peer (vm://localhost#55) disposed. 2015-01-06 16:45:45,390 INFO [o.s.j.l.DefaultMessageListenerContainer] Successfully refreshed JMS Connection 2015-01-06 16:45:45,633 WARN [o.e.j.s.HttpChannel] /v3/commands?checksum=4290717ccd8955e5569a8f5544609bc2913ec229 org.springframework.jms.UncategorizedJmsException: Uncategorized exception occured during JMS processing; nested exception is javax.jms.JMSException: Could not create Transport. Reason: java.io.IOException: Broker named 'localhost' does not exist. {code} Similar issues present in the puppetdb logs of the PDB-1124 user for hosts 2 and 6.

    JIRA | 2 years ago | Wyatt Alt
    javax.jms.JMSException: Peer (vm://localhost#17) disposed.
  2. 0

    We occasionally see JMS exceptions during startup or shutdown, but some large-scale users are experiencing them during normal operation From the PE-7392 logs for host 101 on 2015-01-08 these issues start with this error which seems to come out of the blue: {code} 2015-01-06 16:45:12,069 INFO [c.p.p.command] [84e83891-23d0-4a85-ae9a-87cdef02940d] [replace facts] xc101xmlmobile-39.ams4.prod.booking.com 2015-01-06 16:45:14,222 INFO [c.p.p.command] [e2ffd05e-aced-4289-9fad-cb90e5a9d20f] [replace catalog] sr101sessredis-02.ams4.prod.booking.com 2015-01-06 16:45:15,259 ERROR [o.a.a.s.k.MessageDatabase] KahaDB failed to store to Journal java.io.FileNotFoundException: /var/lib/puppetdb/mq/localhost/KahaDB/db-7087.log (No such file or directory) at java.io.RandomAccessFile.open(Native Method) ~[na:1.7.0_65] at java.io.RandomAccessFile.<init>(RandomAccessFile.java:241) ~[na:1.7.0_65] at org.apache.kahadb.journal.DataFile.openRandomAccessFile(DataFile.java:70) ~[puppetdb.jar:na] at org.apache.kahadb.journal.DataFileAppender.processQueue(DataFileAppender.java:309) ~[puppetdb.jar:na] at org.apache.kahadb.journal.DataFileAppender$1.run(DataFileAppender.java:188) ~[puppetdb.jar:na] 2015-01-06 16:45:15,286 WARN [o.e.j.s.HttpChannel] /v3/commands?checksum=43525b026292eac95a00cd286f7de152108701fb org.springframework.jms.UncategorizedJmsException: Uncategorized exception occured during JMS processing; nested exception is javax.jms.JMSException: /var/lib/puppetdb/mq/localhost/KahaDB/db-7087.log (No such file or directory) at org.springframework.jms.support.JmsUtils.convertJmsAccessException(JmsUtils.java:316) ~[puppetdb.jar:na] at org.springframework.jms.support.JmsAccessor.convertJmsAccessException(JmsAccessor.java:168) ~[puppetdb.jar:na] at org.springframework.jms.core.JmsTemplate.execute(JmsTemplate.java:469) ~[puppetdb.jar:na] at org.springframework.jms.core.JmsTemplate.send(JmsTemplate.java:543) ~[puppetdb.jar:na] at org.springframework.jms.core.JmsTemplate.convertAndSend(JmsTemplate.java:653) ~[puppetdb.jar:na] at sun.reflect.GeneratedMethodAccessor97.invoke(Unknown Source) ~[na:na] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_65] at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_65] at clojure.lang.Reflector.invokeMatchingMethod(Reflector.java:93) ~[puppetdb.jar:na] at clojure.lang.Reflector.invokeInstanceMethod(Reflector.java:28) ~[puppetdb.jar:na] at clamq.jms$jms_producer$reify__10735.publish(jms.clj:29) ~[na:na] at clamq.protocol.producer$eval10687$fn__10688$G__10679__10696.invoke(producer.clj:3) ~[na:na] at clamq.protocol.producer$eval10687$fn__10688$G__10678__10705.invoke(producer.clj:3) ~[na:na] at clojure.lang.AFn.applyToHelper(AFn.java:165) ~[puppetdb.jar:na] at clojure.lang.AFn.applyTo(AFn.java:144) ~[puppetdb.jar:na] at clojure.core$apply.invoke(core.clj:626) ~[puppetdb.jar:na] at com.puppetlabs.mq$connect_and_publish_BANG_.doInvoke(mq.clj:148) ~[na:na] at clojure.lang.RestFn.invoke(RestFn.java:460) ~[puppetdb.jar:na] at com.puppetlabs.puppetdb.command$enqueue_raw_command_BANG_$fn__11440.invoke(command.clj:249) ~[na:na] at com.puppetlabs.puppetdb.command$enqueue_raw_command_BANG_.invoke(command.clj:248) ~[na:na] at com.puppetlabs.puppetdb.http.command$enqueue_command.invoke(command.clj:12) ~[na:na] at com.puppetlabs.middleware$verify_accepts_content_type$fn__12248.invoke(middleware.clj:114) ~[na:na] at com.puppetlabs.middleware$verify_checksum$fn__12268.invoke(middleware.clj:185) ~[na:na] at com.puppetlabs.middleware$validate_query_params$fn__12259.invoke(middleware.clj:155) ~[na:na] at com.puppetlabs.middleware$payload_to_body_string$fn__12303.invoke(middleware.clj:276) ~[na:na] at com.puppetlabs.middleware$verify_content_type$fn__12253.invoke(middleware.clj:128) ~[na:na] at com.puppetlabs.puppetdb.http.v3$v3_app$fn__13762.invoke(v3.clj:23) ~[na:na] at com.puppetlabs.puppetdb.http.v3$v3_app.invoke(v3.clj:21) ~[na:na] at com.puppetlabs.puppetdb.http.server$routes$fn__14649$fn__14663.invoke(server.clj:48) ~[na:na] at net.cgrand.moustache$alter_request$fn__12677.invoke(moustache.clj:54) ~[na:na] at com.puppetlabs.puppetdb.http.server$routes$fn__14649.invoke(server.clj:48) ~[na:na] at ring.middleware.resource$wrap_resource$fn__14631.invoke(resource.clj:24) ~[na:na] at ring.middleware.params$wrap_params$fn__12620.invoke(params.clj:58) ~[na:na] at com.puppetlabs.middleware$wrap_with_authorization$fn__12220.invoke(middleware.clj:40) ~[na:na] at com.puppetlabs.middleware$wrap_with_certificate_cn$fn__12224.invoke(middleware.clj:55) ~[na:na] at com.puppetlabs.middleware$wrap_with_default_body$fn__12228.invoke(middleware.clj:62) ~[na:na] at com.puppetlabs.middleware$wrap_with_metrics_STAR_$fn__12275$fn__12284.invoke(middleware.clj:207) ~[na:na] at com.puppetlabs.utils.metrics$multitime_BANG__STAR_$fn__11942$fn__11943$fn__11944.invoke(metrics.clj:14) ~[na:na] at com.puppetlabs.utils.metrics.proxy$java.lang.Object$Callable$7da976d4.call(Unknown Source) ~[na:na] at com.yammer.metrics.core.Timer.time(Timer.java:91) ~[puppetdb.jar:na] {code} this is followed by hours of JMS errors interspersed with command processing. {code} at com.puppetlabs.utils.metrics$multitime_BANG__STAR_.invoke(metrics.clj:17) ~[na:na] at com.puppetlabs.middleware$wrap_with_metrics_STAR_$fn__12275.invoke(middleware.clj:206) ~[na:na] at com.puppetlabs.middleware$wrap_with_globals$fn__12232.invoke(middleware.clj:73) ~[na:na] at com.puppetlabs.middleware$wrap_with_debug_logging$fn__12216.invoke(middleware.clj:27) ~[na:na] at compojure.core$routing$fn__17778.invoke(core.clj:107) ~[na:na] at clojure.core$some.invoke(core.clj:2515) ~[puppetdb.jar:na] at compojure.core$routing.doInvoke(core.clj:107) ~[na:na] at clojure.lang.RestFn.invoke(RestFn.java:423) ~[puppetdb.jar:na] at com.puppetlabs.puppetdb.cli.services$start_puppetdb$fn__19272.invoke(services.clj:330) ~[na:na] at compojure.core$wrap_context$fn__17832.invoke(core.clj:165) ~[na:na] at compojure.core$if_route$fn__17756.invoke(core.clj:40) ~[na:na] at puppetlabs.trapperkeeper.services.webserver.jetty9_core$proxy_handler$fn__19529.invoke(jetty9_core.clj:66) ~[na:na] at puppetlabs.trapperkeeper.services.webserver.jetty9_core.proxy$org.eclipse.jetty.server.handler.AbstractHandler$ff19274a.handle(Unknown Source) ~[na:na] at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1112) ~[puppetdb.jar:na] at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1048) ~[puppetdb.jar:na] at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) ~[puppetdb.jar:na] at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:199) ~[puppetdb.jar:na] at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:109) ~[puppetdb.jar:na] at org.eclipse.jetty.servlets.gzip.GzipHandler.handle(GzipHandler.java:275) ~[puppetdb.jar:na] at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97) ~[puppetdb.jar:na] at org.eclipse.jetty.server.Server.handle(Server.java:459) ~[puppetdb.jar:na] at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:280) ~[puppetdb.jar:na] at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:229) [puppetdb.jar:na] at org.eclipse.jetty.io.AbstractConnection$1.run(AbstractConnection.java:505) [puppetdb.jar:na] at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:607) [puppetdb.jar:na] at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:536) [puppetdb.jar:na] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_65] Caused by: javax.jms.JMSException: Could not create Transport. Reason: java.io.IOException: Broker named 'localhost' does not exist. at org.apache.activemq.util.JMSExceptionSupport.create(JMSExceptionSupport.java:35) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnectionFactory.createTransport(ActiveMQConnectionFactory.java:254) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnectionFactory.createActiveMQConnection(ActiveMQConnectionFactory.java:267) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnectionFactory.createActiveMQConnection(ActiveMQConnectionFactory.java:239) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnectionFactory.createConnection(ActiveMQConnectionFactory.java:185) ~[puppetdb.jar:na] at org.springframework.jms.connection.SingleConnectionFactory.doCreateConnection(SingleConnectionFactory.java:342) ~[puppetdb.jar:na] at org.springframework.jms.connection.SingleConnectionFactory.initConnection(SingleConnectionFactory.java:288) ~[puppetdb.jar:na] at org.springframework.jms.connection.SingleConnectionFactory.createConnection(SingleConnectionFactory.java:225) ~[puppetdb.jar:na] at org.springframework.jms.support.JmsAccessor.createConnection(JmsAccessor.java:184) ~[puppetdb.jar:na] at org.springframework.jms.core.JmsTemplate.execute(JmsTemplate.java:456) ~[puppetdb.jar:na] ... 69 common frames omitted Caused by: java.io.IOException: Broker named 'localhost' does not exist. at org.apache.activemq.transport.vm.VMTransportFactory.doCompositeConnect(VMTransportFactory.java:116) ~[puppetdb.jar:na] at org.apache.activemq.transport.vm.VMTransportFactory.doConnect(VMTransportFactory.java:54) ~[puppetdb.jar:na] at org.apache.activemq.transport.TransportFactory.doConnect(TransportFactory.java:51) ~[puppetdb.jar:na] at org.apache.activemq.transport.TransportFactory.connect(TransportFactory.java:80) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnectionFactory.createTransport(ActiveMQConnectionFactory.java:252) ~[puppetdb.jar:na] ... 77 common frames omitted 2015-01-06 16:45:45,040 INFO [c.p.p.command] [cfe812d2-bba9-4f47-be4c-ac6453d74b4f] [replace facts] av1rdb-114.ams4.prod.booking.com 2015-01-06 16:45:45,040 INFO [c.p.p.command] [c1a3b13e-4fa6-4017-862a-4894fe26a58c] [replace facts] pc101pphmsapp-01.ams4.prod.booking.com 2015-01-06 16:45:45,041 INFO [c.p.p.command] [ed9160f0-47b8-41c5-933f-0824a0d983a6] [replace facts] bc144bprdb-01.ams4.prod.booking.com 2015-01-06 16:45:45,042 WARN [o.s.j.l.DefaultMessageListenerContainer] Execution of JMS message listener failed, and no ErrorHandler has been set. javax.jms.JMSException: Peer (vm://localhost#16) disposed. at org.apache.activemq.util.JMSExceptionSupport.create(JMSExceptionSupport.java:62) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1380) ~[puppetdb.jar:na] at org.apache.activemq.TransactionContext.rollback(TransactionContext.java:279) ~[puppetdb.jar:na] at org.apache.activemq.TransactionContext.commit(TransactionContext.java:306) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQSession.commit(ActiveMQSession.java:561) ~[puppetdb.jar:na] at sun.reflect.GeneratedMethodAccessor83.invoke(Unknown Source) ~[na:na] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_65] at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_65] at org.springframework.jms.connection.CachingConnectionFactory$CachedSessionInvocationHandler.invoke(CachingConnectionFactory.java:344) ~[puppetdb.jar:na] at com.sun.proxy.$Proxy5.commit(Unknown Source) ~[na:na] at org.springframework.jms.support.JmsUtils.commitIfNecessary(JmsUtils.java:217) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractMessageListenerContainer.commitIfNecessary(AbstractMessageListenerContainer.java:575) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractMessageListenerContainer.doExecuteListener(AbstractMessageListenerContainer.java:481) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.doReceiveAndExecute(AbstractPollingMessageListenerContainer.java:325) [puppetdb.jar:na] at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:263) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1058) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1050) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:947) [puppetdb.jar:na] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_65] Caused by: org.apache.activemq.transport.TransportDisposedIOException: Peer (vm://localhost#16) disposed. at org.apache.activemq.transport.vm.VMTransport.oneway(VMTransport.java:88) ~[puppetdb.jar:na] at org.apache.activemq.transport.MutexTransport.oneway(MutexTransport.java:68) ~[puppetdb.jar:na] at org.apache.activemq.transport.ResponseCorrelator.asyncRequest(ResponseCorrelator.java:81) ~[puppetdb.jar:na] at org.apache.activemq.transport.ResponseCorrelator.request(ResponseCorrelator.java:86) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1351) ~[puppetdb.jar:na] ... 17 common frames omitted 2015-01-06 16:45:45,042 WARN [o.s.j.l.DefaultMessageListenerContainer] Execution of JMS message listener failed, and no ErrorHandler has been set. javax.jms.JMSException: Peer (vm://localhost#17) disposed. at org.apache.activemq.util.JMSExceptionSupport.create(JMSExceptionSupport.java:62) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1380) ~[puppetdb.jar:na] at org.apache.activemq.TransactionContext.rollback(TransactionContext.java:279) ~[puppetdb.jar:na] at org.apache.activemq.TransactionContext.commit(TransactionContext.java:306) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQSession.commit(ActiveMQSession.java:561) ~[puppetdb.jar:na] at sun.reflect.GeneratedMethodAccessor83.invoke(Unknown Source) ~[na:na] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_65] at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_65] at org.springframework.jms.connection.CachingConnectionFactory$CachedSessionInvocationHandler.invoke(CachingConnectionFactory.java:344) ~[puppetdb.jar:na] at com.sun.proxy.$Proxy5.commit(Unknown Source) ~[na:na] at org.springframework.jms.support.JmsUtils.commitIfNecessary(JmsUtils.java:217) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractMessageListenerContainer.commitIfNecessary(AbstractMessageListenerContainer.java:575) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractMessageListenerContainer.doExecuteListener(AbstractMessageListenerContainer.java:481) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.doReceiveAndExecute(AbstractPollingMessageListenerContainer.java:325) [puppetdb.jar:na] at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:263) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1058) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1050) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:947) [puppetdb.jar:na] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_65] Caused by: org.apache.activemq.transport.TransportDisposedIOException: Peer (vm://localhost#17) disposed. at org.apache.activemq.transport.vm.VMTransport.oneway(VMTransport.java:88) ~[puppetdb.jar:na] at org.apache.activemq.transport.MutexTransport.oneway(MutexTransport.java:68) ~[puppetdb.jar:na] at org.apache.activemq.transport.ResponseCorrelator.asyncRequest(ResponseCorrelator.java:81) ~[puppetdb.jar:na] at org.apache.activemq.transport.ResponseCorrelator.request(ResponseCorrelator.java:86) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1351) ~[puppetdb.jar:na] ... 17 common frames omitted 2015-01-06 16:45:45,042 WARN [o.s.j.l.DefaultMessageListenerContainer] Execution of JMS message listener failed, and no ErrorHandler has been set. javax.jms.JMSException: Peer (vm://localhost#51) disposed. at org.apache.activemq.util.JMSExceptionSupport.create(JMSExceptionSupport.java:62) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1380) ~[puppetdb.jar:na] at org.apache.activemq.TransactionContext.rollback(TransactionContext.java:279) ~[puppetdb.jar:na] at org.apache.activemq.TransactionContext.commit(TransactionContext.java:306) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQSession.commit(ActiveMQSession.java:561) ~[puppetdb.jar:na] at sun.reflect.GeneratedMethodAccessor83.invoke(Unknown Source) ~[na:na] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_65] at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_65] at org.springframework.jms.connection.CachingConnectionFactory$CachedSessionInvocationHandler.invoke(CachingConnectionFactory.java:344) ~[puppetdb.jar:na] at com.sun.proxy.$Proxy5.commit(Unknown Source) ~[na:na] at org.springframework.jms.support.JmsUtils.commitIfNecessary(JmsUtils.java:217) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractMessageListenerContainer.commitIfNecessary(AbstractMessageListenerContainer.java:575) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractMessageListenerContainer.doExecuteListener(AbstractMessageListenerContainer.java:481) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.doReceiveAndExecute(AbstractPollingMessageListenerContainer.java:325) [puppetdb.jar:na] at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:263) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1058) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1050) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:947) [puppetdb.jar:na] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_65] Caused by: org.apache.activemq.transport.TransportDisposedIOException: Peer (vm://localhost#51) disposed. at org.apache.activemq.transport.vm.VMTransport.oneway(VMTransport.java:88) ~[puppetdb.jar:na] at org.apache.activemq.transport.MutexTransport.oneway(MutexTransport.java:68) ~[puppetdb.jar:na] at org.apache.activemq.transport.ResponseCorrelator.asyncRequest(ResponseCorrelator.java:81) ~[puppetdb.jar:na] at org.apache.activemq.transport.ResponseCorrelator.request(ResponseCorrelator.java:86) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1351) ~[puppetdb.jar:na] ... 17 common frames omitted 2015-01-06 16:45:45,042 WARN [o.s.j.l.DefaultMessageListenerContainer] Setup of JMS message listener invoker failed for destination 'com.puppetlabs.puppetdb.commands' - trying to recover. Cause: Peer (vm://localhost#17) disposed. 2015-01-06 16:45:45,042 WARN [o.s.j.l.DefaultMessageListenerContainer] Setup of JMS message listener invoker failed for destination 'com.puppetlabs.puppetdb.commands' - trying to recover. Cause: Peer (vm://localhost#16) disposed. 2015-01-06 16:45:45,043 INFO [o.s.j.l.DefaultMessageListenerContainer] Successfully refreshed JMS Connection 2015-01-06 16:45:45,043 WARN [o.s.j.l.DefaultMessageListenerContainer] Setup of JMS message listener invoker failed for destination 'com.puppetlabs.puppetdb.commands' - trying to recover. Cause: Peer (vm://localhost#51) disposed. 2015-01-06 16:45:45,043 INFO [o.s.j.l.DefaultMessageListenerContainer] Successfully refreshed JMS Connection 2015-01-06 16:45:45,045 INFO [o.s.j.l.DefaultMessageListenerContainer] Successfully refreshed JMS Connection 2015-01-06 16:45:45,388 WARN [o.s.j.l.DefaultMessageListenerContainer] Setup of JMS message listener invoker failed for destination 'com.puppetlabs.puppetdb.commands' - trying to recover. Cause: Peer (vm://localhost#57) disposed. 2015-01-06 16:45:45,388 INFO [o.s.j.l.DefaultMessageListenerContainer] Successfully refreshed JMS Connection 2015-01-06 16:45:45,390 WARN [o.s.j.l.DefaultMessageListenerContainer] Setup of JMS message listener invoker failed for destination 'com.puppetlabs.puppetdb.commands' - trying to recover. Cause: Peer (vm://localhost#55) disposed. 2015-01-06 16:45:45,390 INFO [o.s.j.l.DefaultMessageListenerContainer] Successfully refreshed JMS Connection 2015-01-06 16:45:45,633 WARN [o.e.j.s.HttpChannel] /v3/commands?checksum=4290717ccd8955e5569a8f5544609bc2913ec229 org.springframework.jms.UncategorizedJmsException: Uncategorized exception occured during JMS processing; nested exception is javax.jms.JMSException: Could not create Transport. Reason: java.io.IOException: Broker named 'localhost' does not exist. {code} Similar issues present in the puppetdb logs of the PDB-1124 user for hosts 2 and 6.

    JIRA | 2 years ago | Wyatt Alt
    javax.jms.JMSException: Peer (vm://localhost#17) disposed.
  3. Speed up your debug routine!

    Automated exception search integrated into your IDE

  4. 0

    ServiceMix - User - servicemix hangs because of illegal state exception

    nabble.com | 1 year ago
    java.util.concurrent.ExecutionException: javax.jbi.JBIException: Unable to stop jms component

    4 unregistered visitors
    Not finding the right solution?
    Take a tour to get the most out of Samebug.

    Tired of useless tips?

    Automated exception search integrated into your IDE

    Root Cause Analysis

    1. org.apache.activemq.transport.TransportDisposedIOException

      Peer (vm://localhost#17) disposed.

      at org.apache.activemq.transport.vm.VMTransport.oneway()
    2. ActiveMQ :: Broker
      VMTransport.oneway
      1. org.apache.activemq.transport.vm.VMTransport.oneway(VMTransport.java:88)[puppetdb.jar:na]
      1 frame
    3. AMQ HTTP
      ResponseCorrelator.request
      1. org.apache.activemq.transport.MutexTransport.oneway(MutexTransport.java:68)[puppetdb.jar:na]
      2. org.apache.activemq.transport.ResponseCorrelator.asyncRequest(ResponseCorrelator.java:81)[puppetdb.jar:na]
      3. org.apache.activemq.transport.ResponseCorrelator.request(ResponseCorrelator.java:86)[puppetdb.jar:na]
      3 frames
    4. ActiveMQ :: Core
      ActiveMQSession.commit
      1. org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1351)[puppetdb.jar:na]
      2. org.apache.activemq.TransactionContext.rollback(TransactionContext.java:279)[puppetdb.jar:na]
      3. org.apache.activemq.TransactionContext.commit(TransactionContext.java:306)[puppetdb.jar:na]
      4. org.apache.activemq.ActiveMQSession.commit(ActiveMQSession.java:561)[puppetdb.jar:na]
      4 frames
    5. Java RT
      Method.invoke
      1. sun.reflect.GeneratedMethodAccessor83.invoke(Unknown Source)[na:na]
      2. sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)[na:1.7.0_65]
      3. java.lang.reflect.Method.invoke(Method.java:606)[na:1.7.0_65]
      3 frames
    6. Spring Framework
      CachingConnectionFactory$CachedSessionInvocationHandler.invoke
      1. org.springframework.jms.connection.CachingConnectionFactory$CachedSessionInvocationHandler.invoke(CachingConnectionFactory.java:344)[puppetdb.jar:na]
      1 frame
    7. com.sun.proxy
      $Proxy5.commit
      1. com.sun.proxy.$Proxy5.commit(Unknown Source)[na:na]
      1 frame
    8. Spring Framework
      DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run
      1. org.springframework.jms.support.JmsUtils.commitIfNecessary(JmsUtils.java:217)[puppetdb.jar:na]
      2. org.springframework.jms.listener.AbstractMessageListenerContainer.commitIfNecessary(AbstractMessageListenerContainer.java:575)[puppetdb.jar:na]
      3. org.springframework.jms.listener.AbstractMessageListenerContainer.doExecuteListener(AbstractMessageListenerContainer.java:481)[puppetdb.jar:na]
      4. org.springframework.jms.listener.AbstractPollingMessageListenerContainer.doReceiveAndExecute(AbstractPollingMessageListenerContainer.java:325)[puppetdb.jar:na]
      5. org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:263)[puppetdb.jar:na]
      6. org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1058)[puppetdb.jar:na]
      7. org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1050)[puppetdb.jar:na]
      8. org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:947)[puppetdb.jar:na]
      8 frames
    9. Java RT
      Thread.run
      1. java.lang.Thread.run(Thread.java:745)[na:1.7.0_65]
      1 frame