org.springframework.jms.UncategorizedJmsException: Uncategorized exception occured during JMS processing; nested exception is javax.jms.JMSException: /var/lib/puppetdb/mq/localhost/KahaDB/db-7087.log (No such file or directory)

JIRA | Wyatt Alt | 2 years ago
  1. 0

    We occasionally see JMS exceptions during startup or shutdown, but some large-scale users are experiencing them during normal operation From the PE-7392 logs for host 101 on 2015-01-08 these issues start with this error which seems to come out of the blue: {code} 2015-01-06 16:45:12,069 INFO [c.p.p.command] [84e83891-23d0-4a85-ae9a-87cdef02940d] [replace facts] xc101xmlmobile-39.ams4.prod.booking.com 2015-01-06 16:45:14,222 INFO [c.p.p.command] [e2ffd05e-aced-4289-9fad-cb90e5a9d20f] [replace catalog] sr101sessredis-02.ams4.prod.booking.com 2015-01-06 16:45:15,259 ERROR [o.a.a.s.k.MessageDatabase] KahaDB failed to store to Journal java.io.FileNotFoundException: /var/lib/puppetdb/mq/localhost/KahaDB/db-7087.log (No such file or directory) at java.io.RandomAccessFile.open(Native Method) ~[na:1.7.0_65] at java.io.RandomAccessFile.<init>(RandomAccessFile.java:241) ~[na:1.7.0_65] at org.apache.kahadb.journal.DataFile.openRandomAccessFile(DataFile.java:70) ~[puppetdb.jar:na] at org.apache.kahadb.journal.DataFileAppender.processQueue(DataFileAppender.java:309) ~[puppetdb.jar:na] at org.apache.kahadb.journal.DataFileAppender$1.run(DataFileAppender.java:188) ~[puppetdb.jar:na] 2015-01-06 16:45:15,286 WARN [o.e.j.s.HttpChannel] /v3/commands?checksum=43525b026292eac95a00cd286f7de152108701fb org.springframework.jms.UncategorizedJmsException: Uncategorized exception occured during JMS processing; nested exception is javax.jms.JMSException: /var/lib/puppetdb/mq/localhost/KahaDB/db-7087.log (No such file or directory) at org.springframework.jms.support.JmsUtils.convertJmsAccessException(JmsUtils.java:316) ~[puppetdb.jar:na] at org.springframework.jms.support.JmsAccessor.convertJmsAccessException(JmsAccessor.java:168) ~[puppetdb.jar:na] at org.springframework.jms.core.JmsTemplate.execute(JmsTemplate.java:469) ~[puppetdb.jar:na] at org.springframework.jms.core.JmsTemplate.send(JmsTemplate.java:543) ~[puppetdb.jar:na] at org.springframework.jms.core.JmsTemplate.convertAndSend(JmsTemplate.java:653) ~[puppetdb.jar:na] at sun.reflect.GeneratedMethodAccessor97.invoke(Unknown Source) ~[na:na] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_65] at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_65] at clojure.lang.Reflector.invokeMatchingMethod(Reflector.java:93) ~[puppetdb.jar:na] at clojure.lang.Reflector.invokeInstanceMethod(Reflector.java:28) ~[puppetdb.jar:na] at clamq.jms$jms_producer$reify__10735.publish(jms.clj:29) ~[na:na] at clamq.protocol.producer$eval10687$fn__10688$G__10679__10696.invoke(producer.clj:3) ~[na:na] at clamq.protocol.producer$eval10687$fn__10688$G__10678__10705.invoke(producer.clj:3) ~[na:na] at clojure.lang.AFn.applyToHelper(AFn.java:165) ~[puppetdb.jar:na] at clojure.lang.AFn.applyTo(AFn.java:144) ~[puppetdb.jar:na] at clojure.core$apply.invoke(core.clj:626) ~[puppetdb.jar:na] at com.puppetlabs.mq$connect_and_publish_BANG_.doInvoke(mq.clj:148) ~[na:na] at clojure.lang.RestFn.invoke(RestFn.java:460) ~[puppetdb.jar:na] at com.puppetlabs.puppetdb.command$enqueue_raw_command_BANG_$fn__11440.invoke(command.clj:249) ~[na:na] at com.puppetlabs.puppetdb.command$enqueue_raw_command_BANG_.invoke(command.clj:248) ~[na:na] at com.puppetlabs.puppetdb.http.command$enqueue_command.invoke(command.clj:12) ~[na:na] at com.puppetlabs.middleware$verify_accepts_content_type$fn__12248.invoke(middleware.clj:114) ~[na:na] at com.puppetlabs.middleware$verify_checksum$fn__12268.invoke(middleware.clj:185) ~[na:na] at com.puppetlabs.middleware$validate_query_params$fn__12259.invoke(middleware.clj:155) ~[na:na] at com.puppetlabs.middleware$payload_to_body_string$fn__12303.invoke(middleware.clj:276) ~[na:na] at com.puppetlabs.middleware$verify_content_type$fn__12253.invoke(middleware.clj:128) ~[na:na] at com.puppetlabs.puppetdb.http.v3$v3_app$fn__13762.invoke(v3.clj:23) ~[na:na] at com.puppetlabs.puppetdb.http.v3$v3_app.invoke(v3.clj:21) ~[na:na] at com.puppetlabs.puppetdb.http.server$routes$fn__14649$fn__14663.invoke(server.clj:48) ~[na:na] at net.cgrand.moustache$alter_request$fn__12677.invoke(moustache.clj:54) ~[na:na] at com.puppetlabs.puppetdb.http.server$routes$fn__14649.invoke(server.clj:48) ~[na:na] at ring.middleware.resource$wrap_resource$fn__14631.invoke(resource.clj:24) ~[na:na] at ring.middleware.params$wrap_params$fn__12620.invoke(params.clj:58) ~[na:na] at com.puppetlabs.middleware$wrap_with_authorization$fn__12220.invoke(middleware.clj:40) ~[na:na] at com.puppetlabs.middleware$wrap_with_certificate_cn$fn__12224.invoke(middleware.clj:55) ~[na:na] at com.puppetlabs.middleware$wrap_with_default_body$fn__12228.invoke(middleware.clj:62) ~[na:na] at com.puppetlabs.middleware$wrap_with_metrics_STAR_$fn__12275$fn__12284.invoke(middleware.clj:207) ~[na:na] at com.puppetlabs.utils.metrics$multitime_BANG__STAR_$fn__11942$fn__11943$fn__11944.invoke(metrics.clj:14) ~[na:na] at com.puppetlabs.utils.metrics.proxy$java.lang.Object$Callable$7da976d4.call(Unknown Source) ~[na:na] at com.yammer.metrics.core.Timer.time(Timer.java:91) ~[puppetdb.jar:na] {code} this is followed by hours of JMS errors interspersed with command processing. {code} at com.puppetlabs.utils.metrics$multitime_BANG__STAR_.invoke(metrics.clj:17) ~[na:na] at com.puppetlabs.middleware$wrap_with_metrics_STAR_$fn__12275.invoke(middleware.clj:206) ~[na:na] at com.puppetlabs.middleware$wrap_with_globals$fn__12232.invoke(middleware.clj:73) ~[na:na] at com.puppetlabs.middleware$wrap_with_debug_logging$fn__12216.invoke(middleware.clj:27) ~[na:na] at compojure.core$routing$fn__17778.invoke(core.clj:107) ~[na:na] at clojure.core$some.invoke(core.clj:2515) ~[puppetdb.jar:na] at compojure.core$routing.doInvoke(core.clj:107) ~[na:na] at clojure.lang.RestFn.invoke(RestFn.java:423) ~[puppetdb.jar:na] at com.puppetlabs.puppetdb.cli.services$start_puppetdb$fn__19272.invoke(services.clj:330) ~[na:na] at compojure.core$wrap_context$fn__17832.invoke(core.clj:165) ~[na:na] at compojure.core$if_route$fn__17756.invoke(core.clj:40) ~[na:na] at puppetlabs.trapperkeeper.services.webserver.jetty9_core$proxy_handler$fn__19529.invoke(jetty9_core.clj:66) ~[na:na] at puppetlabs.trapperkeeper.services.webserver.jetty9_core.proxy$org.eclipse.jetty.server.handler.AbstractHandler$ff19274a.handle(Unknown Source) ~[na:na] at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1112) ~[puppetdb.jar:na] at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1048) ~[puppetdb.jar:na] at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) ~[puppetdb.jar:na] at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:199) ~[puppetdb.jar:na] at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:109) ~[puppetdb.jar:na] at org.eclipse.jetty.servlets.gzip.GzipHandler.handle(GzipHandler.java:275) ~[puppetdb.jar:na] at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97) ~[puppetdb.jar:na] at org.eclipse.jetty.server.Server.handle(Server.java:459) ~[puppetdb.jar:na] at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:280) ~[puppetdb.jar:na] at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:229) [puppetdb.jar:na] at org.eclipse.jetty.io.AbstractConnection$1.run(AbstractConnection.java:505) [puppetdb.jar:na] at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:607) [puppetdb.jar:na] at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:536) [puppetdb.jar:na] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_65] Caused by: javax.jms.JMSException: Could not create Transport. Reason: java.io.IOException: Broker named 'localhost' does not exist. at org.apache.activemq.util.JMSExceptionSupport.create(JMSExceptionSupport.java:35) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnectionFactory.createTransport(ActiveMQConnectionFactory.java:254) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnectionFactory.createActiveMQConnection(ActiveMQConnectionFactory.java:267) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnectionFactory.createActiveMQConnection(ActiveMQConnectionFactory.java:239) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnectionFactory.createConnection(ActiveMQConnectionFactory.java:185) ~[puppetdb.jar:na] at org.springframework.jms.connection.SingleConnectionFactory.doCreateConnection(SingleConnectionFactory.java:342) ~[puppetdb.jar:na] at org.springframework.jms.connection.SingleConnectionFactory.initConnection(SingleConnectionFactory.java:288) ~[puppetdb.jar:na] at org.springframework.jms.connection.SingleConnectionFactory.createConnection(SingleConnectionFactory.java:225) ~[puppetdb.jar:na] at org.springframework.jms.support.JmsAccessor.createConnection(JmsAccessor.java:184) ~[puppetdb.jar:na] at org.springframework.jms.core.JmsTemplate.execute(JmsTemplate.java:456) ~[puppetdb.jar:na] ... 69 common frames omitted Caused by: java.io.IOException: Broker named 'localhost' does not exist. at org.apache.activemq.transport.vm.VMTransportFactory.doCompositeConnect(VMTransportFactory.java:116) ~[puppetdb.jar:na] at org.apache.activemq.transport.vm.VMTransportFactory.doConnect(VMTransportFactory.java:54) ~[puppetdb.jar:na] at org.apache.activemq.transport.TransportFactory.doConnect(TransportFactory.java:51) ~[puppetdb.jar:na] at org.apache.activemq.transport.TransportFactory.connect(TransportFactory.java:80) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnectionFactory.createTransport(ActiveMQConnectionFactory.java:252) ~[puppetdb.jar:na] ... 77 common frames omitted 2015-01-06 16:45:45,040 INFO [c.p.p.command] [cfe812d2-bba9-4f47-be4c-ac6453d74b4f] [replace facts] av1rdb-114.ams4.prod.booking.com 2015-01-06 16:45:45,040 INFO [c.p.p.command] [c1a3b13e-4fa6-4017-862a-4894fe26a58c] [replace facts] pc101pphmsapp-01.ams4.prod.booking.com 2015-01-06 16:45:45,041 INFO [c.p.p.command] [ed9160f0-47b8-41c5-933f-0824a0d983a6] [replace facts] bc144bprdb-01.ams4.prod.booking.com 2015-01-06 16:45:45,042 WARN [o.s.j.l.DefaultMessageListenerContainer] Execution of JMS message listener failed, and no ErrorHandler has been set. javax.jms.JMSException: Peer (vm://localhost#16) disposed. at org.apache.activemq.util.JMSExceptionSupport.create(JMSExceptionSupport.java:62) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1380) ~[puppetdb.jar:na] at org.apache.activemq.TransactionContext.rollback(TransactionContext.java:279) ~[puppetdb.jar:na] at org.apache.activemq.TransactionContext.commit(TransactionContext.java:306) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQSession.commit(ActiveMQSession.java:561) ~[puppetdb.jar:na] at sun.reflect.GeneratedMethodAccessor83.invoke(Unknown Source) ~[na:na] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_65] at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_65] at org.springframework.jms.connection.CachingConnectionFactory$CachedSessionInvocationHandler.invoke(CachingConnectionFactory.java:344) ~[puppetdb.jar:na] at com.sun.proxy.$Proxy5.commit(Unknown Source) ~[na:na] at org.springframework.jms.support.JmsUtils.commitIfNecessary(JmsUtils.java:217) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractMessageListenerContainer.commitIfNecessary(AbstractMessageListenerContainer.java:575) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractMessageListenerContainer.doExecuteListener(AbstractMessageListenerContainer.java:481) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.doReceiveAndExecute(AbstractPollingMessageListenerContainer.java:325) [puppetdb.jar:na] at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:263) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1058) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1050) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:947) [puppetdb.jar:na] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_65] Caused by: org.apache.activemq.transport.TransportDisposedIOException: Peer (vm://localhost#16) disposed. at org.apache.activemq.transport.vm.VMTransport.oneway(VMTransport.java:88) ~[puppetdb.jar:na] at org.apache.activemq.transport.MutexTransport.oneway(MutexTransport.java:68) ~[puppetdb.jar:na] at org.apache.activemq.transport.ResponseCorrelator.asyncRequest(ResponseCorrelator.java:81) ~[puppetdb.jar:na] at org.apache.activemq.transport.ResponseCorrelator.request(ResponseCorrelator.java:86) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1351) ~[puppetdb.jar:na] ... 17 common frames omitted 2015-01-06 16:45:45,042 WARN [o.s.j.l.DefaultMessageListenerContainer] Execution of JMS message listener failed, and no ErrorHandler has been set. javax.jms.JMSException: Peer (vm://localhost#17) disposed. at org.apache.activemq.util.JMSExceptionSupport.create(JMSExceptionSupport.java:62) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1380) ~[puppetdb.jar:na] at org.apache.activemq.TransactionContext.rollback(TransactionContext.java:279) ~[puppetdb.jar:na] at org.apache.activemq.TransactionContext.commit(TransactionContext.java:306) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQSession.commit(ActiveMQSession.java:561) ~[puppetdb.jar:na] at sun.reflect.GeneratedMethodAccessor83.invoke(Unknown Source) ~[na:na] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_65] at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_65] at org.springframework.jms.connection.CachingConnectionFactory$CachedSessionInvocationHandler.invoke(CachingConnectionFactory.java:344) ~[puppetdb.jar:na] at com.sun.proxy.$Proxy5.commit(Unknown Source) ~[na:na] at org.springframework.jms.support.JmsUtils.commitIfNecessary(JmsUtils.java:217) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractMessageListenerContainer.commitIfNecessary(AbstractMessageListenerContainer.java:575) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractMessageListenerContainer.doExecuteListener(AbstractMessageListenerContainer.java:481) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.doReceiveAndExecute(AbstractPollingMessageListenerContainer.java:325) [puppetdb.jar:na] at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:263) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1058) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1050) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:947) [puppetdb.jar:na] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_65] Caused by: org.apache.activemq.transport.TransportDisposedIOException: Peer (vm://localhost#17) disposed. at org.apache.activemq.transport.vm.VMTransport.oneway(VMTransport.java:88) ~[puppetdb.jar:na] at org.apache.activemq.transport.MutexTransport.oneway(MutexTransport.java:68) ~[puppetdb.jar:na] at org.apache.activemq.transport.ResponseCorrelator.asyncRequest(ResponseCorrelator.java:81) ~[puppetdb.jar:na] at org.apache.activemq.transport.ResponseCorrelator.request(ResponseCorrelator.java:86) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1351) ~[puppetdb.jar:na] ... 17 common frames omitted 2015-01-06 16:45:45,042 WARN [o.s.j.l.DefaultMessageListenerContainer] Execution of JMS message listener failed, and no ErrorHandler has been set. javax.jms.JMSException: Peer (vm://localhost#51) disposed. at org.apache.activemq.util.JMSExceptionSupport.create(JMSExceptionSupport.java:62) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1380) ~[puppetdb.jar:na] at org.apache.activemq.TransactionContext.rollback(TransactionContext.java:279) ~[puppetdb.jar:na] at org.apache.activemq.TransactionContext.commit(TransactionContext.java:306) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQSession.commit(ActiveMQSession.java:561) ~[puppetdb.jar:na] at sun.reflect.GeneratedMethodAccessor83.invoke(Unknown Source) ~[na:na] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_65] at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_65] at org.springframework.jms.connection.CachingConnectionFactory$CachedSessionInvocationHandler.invoke(CachingConnectionFactory.java:344) ~[puppetdb.jar:na] at com.sun.proxy.$Proxy5.commit(Unknown Source) ~[na:na] at org.springframework.jms.support.JmsUtils.commitIfNecessary(JmsUtils.java:217) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractMessageListenerContainer.commitIfNecessary(AbstractMessageListenerContainer.java:575) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractMessageListenerContainer.doExecuteListener(AbstractMessageListenerContainer.java:481) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.doReceiveAndExecute(AbstractPollingMessageListenerContainer.java:325) [puppetdb.jar:na] at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:263) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1058) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1050) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:947) [puppetdb.jar:na] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_65] Caused by: org.apache.activemq.transport.TransportDisposedIOException: Peer (vm://localhost#51) disposed. at org.apache.activemq.transport.vm.VMTransport.oneway(VMTransport.java:88) ~[puppetdb.jar:na] at org.apache.activemq.transport.MutexTransport.oneway(MutexTransport.java:68) ~[puppetdb.jar:na] at org.apache.activemq.transport.ResponseCorrelator.asyncRequest(ResponseCorrelator.java:81) ~[puppetdb.jar:na] at org.apache.activemq.transport.ResponseCorrelator.request(ResponseCorrelator.java:86) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1351) ~[puppetdb.jar:na] ... 17 common frames omitted 2015-01-06 16:45:45,042 WARN [o.s.j.l.DefaultMessageListenerContainer] Setup of JMS message listener invoker failed for destination 'com.puppetlabs.puppetdb.commands' - trying to recover. Cause: Peer (vm://localhost#17) disposed. 2015-01-06 16:45:45,042 WARN [o.s.j.l.DefaultMessageListenerContainer] Setup of JMS message listener invoker failed for destination 'com.puppetlabs.puppetdb.commands' - trying to recover. Cause: Peer (vm://localhost#16) disposed. 2015-01-06 16:45:45,043 INFO [o.s.j.l.DefaultMessageListenerContainer] Successfully refreshed JMS Connection 2015-01-06 16:45:45,043 WARN [o.s.j.l.DefaultMessageListenerContainer] Setup of JMS message listener invoker failed for destination 'com.puppetlabs.puppetdb.commands' - trying to recover. Cause: Peer (vm://localhost#51) disposed. 2015-01-06 16:45:45,043 INFO [o.s.j.l.DefaultMessageListenerContainer] Successfully refreshed JMS Connection 2015-01-06 16:45:45,045 INFO [o.s.j.l.DefaultMessageListenerContainer] Successfully refreshed JMS Connection 2015-01-06 16:45:45,388 WARN [o.s.j.l.DefaultMessageListenerContainer] Setup of JMS message listener invoker failed for destination 'com.puppetlabs.puppetdb.commands' - trying to recover. Cause: Peer (vm://localhost#57) disposed. 2015-01-06 16:45:45,388 INFO [o.s.j.l.DefaultMessageListenerContainer] Successfully refreshed JMS Connection 2015-01-06 16:45:45,390 WARN [o.s.j.l.DefaultMessageListenerContainer] Setup of JMS message listener invoker failed for destination 'com.puppetlabs.puppetdb.commands' - trying to recover. Cause: Peer (vm://localhost#55) disposed. 2015-01-06 16:45:45,390 INFO [o.s.j.l.DefaultMessageListenerContainer] Successfully refreshed JMS Connection 2015-01-06 16:45:45,633 WARN [o.e.j.s.HttpChannel] /v3/commands?checksum=4290717ccd8955e5569a8f5544609bc2913ec229 org.springframework.jms.UncategorizedJmsException: Uncategorized exception occured during JMS processing; nested exception is javax.jms.JMSException: Could not create Transport. Reason: java.io.IOException: Broker named 'localhost' does not exist. {code} Similar issues present in the puppetdb logs of the PDB-1124 user for hosts 2 and 6.

    JIRA | 2 years ago | Wyatt Alt
    org.springframework.jms.UncategorizedJmsException: Uncategorized exception occured during JMS processing; nested exception is javax.jms.JMSException: /var/lib/puppetdb/mq/localhost/KahaDB/db-7087.log (No such file or directory)
  2. 0

    We occasionally see JMS exceptions during startup or shutdown, but some large-scale users are experiencing them during normal operation From the PE-7392 logs for host 101 on 2015-01-08 these issues start with this error which seems to come out of the blue: {code} 2015-01-06 16:45:12,069 INFO [c.p.p.command] [84e83891-23d0-4a85-ae9a-87cdef02940d] [replace facts] xc101xmlmobile-39.ams4.prod.booking.com 2015-01-06 16:45:14,222 INFO [c.p.p.command] [e2ffd05e-aced-4289-9fad-cb90e5a9d20f] [replace catalog] sr101sessredis-02.ams4.prod.booking.com 2015-01-06 16:45:15,259 ERROR [o.a.a.s.k.MessageDatabase] KahaDB failed to store to Journal java.io.FileNotFoundException: /var/lib/puppetdb/mq/localhost/KahaDB/db-7087.log (No such file or directory) at java.io.RandomAccessFile.open(Native Method) ~[na:1.7.0_65] at java.io.RandomAccessFile.<init>(RandomAccessFile.java:241) ~[na:1.7.0_65] at org.apache.kahadb.journal.DataFile.openRandomAccessFile(DataFile.java:70) ~[puppetdb.jar:na] at org.apache.kahadb.journal.DataFileAppender.processQueue(DataFileAppender.java:309) ~[puppetdb.jar:na] at org.apache.kahadb.journal.DataFileAppender$1.run(DataFileAppender.java:188) ~[puppetdb.jar:na] 2015-01-06 16:45:15,286 WARN [o.e.j.s.HttpChannel] /v3/commands?checksum=43525b026292eac95a00cd286f7de152108701fb org.springframework.jms.UncategorizedJmsException: Uncategorized exception occured during JMS processing; nested exception is javax.jms.JMSException: /var/lib/puppetdb/mq/localhost/KahaDB/db-7087.log (No such file or directory) at org.springframework.jms.support.JmsUtils.convertJmsAccessException(JmsUtils.java:316) ~[puppetdb.jar:na] at org.springframework.jms.support.JmsAccessor.convertJmsAccessException(JmsAccessor.java:168) ~[puppetdb.jar:na] at org.springframework.jms.core.JmsTemplate.execute(JmsTemplate.java:469) ~[puppetdb.jar:na] at org.springframework.jms.core.JmsTemplate.send(JmsTemplate.java:543) ~[puppetdb.jar:na] at org.springframework.jms.core.JmsTemplate.convertAndSend(JmsTemplate.java:653) ~[puppetdb.jar:na] at sun.reflect.GeneratedMethodAccessor97.invoke(Unknown Source) ~[na:na] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_65] at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_65] at clojure.lang.Reflector.invokeMatchingMethod(Reflector.java:93) ~[puppetdb.jar:na] at clojure.lang.Reflector.invokeInstanceMethod(Reflector.java:28) ~[puppetdb.jar:na] at clamq.jms$jms_producer$reify__10735.publish(jms.clj:29) ~[na:na] at clamq.protocol.producer$eval10687$fn__10688$G__10679__10696.invoke(producer.clj:3) ~[na:na] at clamq.protocol.producer$eval10687$fn__10688$G__10678__10705.invoke(producer.clj:3) ~[na:na] at clojure.lang.AFn.applyToHelper(AFn.java:165) ~[puppetdb.jar:na] at clojure.lang.AFn.applyTo(AFn.java:144) ~[puppetdb.jar:na] at clojure.core$apply.invoke(core.clj:626) ~[puppetdb.jar:na] at com.puppetlabs.mq$connect_and_publish_BANG_.doInvoke(mq.clj:148) ~[na:na] at clojure.lang.RestFn.invoke(RestFn.java:460) ~[puppetdb.jar:na] at com.puppetlabs.puppetdb.command$enqueue_raw_command_BANG_$fn__11440.invoke(command.clj:249) ~[na:na] at com.puppetlabs.puppetdb.command$enqueue_raw_command_BANG_.invoke(command.clj:248) ~[na:na] at com.puppetlabs.puppetdb.http.command$enqueue_command.invoke(command.clj:12) ~[na:na] at com.puppetlabs.middleware$verify_accepts_content_type$fn__12248.invoke(middleware.clj:114) ~[na:na] at com.puppetlabs.middleware$verify_checksum$fn__12268.invoke(middleware.clj:185) ~[na:na] at com.puppetlabs.middleware$validate_query_params$fn__12259.invoke(middleware.clj:155) ~[na:na] at com.puppetlabs.middleware$payload_to_body_string$fn__12303.invoke(middleware.clj:276) ~[na:na] at com.puppetlabs.middleware$verify_content_type$fn__12253.invoke(middleware.clj:128) ~[na:na] at com.puppetlabs.puppetdb.http.v3$v3_app$fn__13762.invoke(v3.clj:23) ~[na:na] at com.puppetlabs.puppetdb.http.v3$v3_app.invoke(v3.clj:21) ~[na:na] at com.puppetlabs.puppetdb.http.server$routes$fn__14649$fn__14663.invoke(server.clj:48) ~[na:na] at net.cgrand.moustache$alter_request$fn__12677.invoke(moustache.clj:54) ~[na:na] at com.puppetlabs.puppetdb.http.server$routes$fn__14649.invoke(server.clj:48) ~[na:na] at ring.middleware.resource$wrap_resource$fn__14631.invoke(resource.clj:24) ~[na:na] at ring.middleware.params$wrap_params$fn__12620.invoke(params.clj:58) ~[na:na] at com.puppetlabs.middleware$wrap_with_authorization$fn__12220.invoke(middleware.clj:40) ~[na:na] at com.puppetlabs.middleware$wrap_with_certificate_cn$fn__12224.invoke(middleware.clj:55) ~[na:na] at com.puppetlabs.middleware$wrap_with_default_body$fn__12228.invoke(middleware.clj:62) ~[na:na] at com.puppetlabs.middleware$wrap_with_metrics_STAR_$fn__12275$fn__12284.invoke(middleware.clj:207) ~[na:na] at com.puppetlabs.utils.metrics$multitime_BANG__STAR_$fn__11942$fn__11943$fn__11944.invoke(metrics.clj:14) ~[na:na] at com.puppetlabs.utils.metrics.proxy$java.lang.Object$Callable$7da976d4.call(Unknown Source) ~[na:na] at com.yammer.metrics.core.Timer.time(Timer.java:91) ~[puppetdb.jar:na] {code} this is followed by hours of JMS errors interspersed with command processing. {code} at com.puppetlabs.utils.metrics$multitime_BANG__STAR_.invoke(metrics.clj:17) ~[na:na] at com.puppetlabs.middleware$wrap_with_metrics_STAR_$fn__12275.invoke(middleware.clj:206) ~[na:na] at com.puppetlabs.middleware$wrap_with_globals$fn__12232.invoke(middleware.clj:73) ~[na:na] at com.puppetlabs.middleware$wrap_with_debug_logging$fn__12216.invoke(middleware.clj:27) ~[na:na] at compojure.core$routing$fn__17778.invoke(core.clj:107) ~[na:na] at clojure.core$some.invoke(core.clj:2515) ~[puppetdb.jar:na] at compojure.core$routing.doInvoke(core.clj:107) ~[na:na] at clojure.lang.RestFn.invoke(RestFn.java:423) ~[puppetdb.jar:na] at com.puppetlabs.puppetdb.cli.services$start_puppetdb$fn__19272.invoke(services.clj:330) ~[na:na] at compojure.core$wrap_context$fn__17832.invoke(core.clj:165) ~[na:na] at compojure.core$if_route$fn__17756.invoke(core.clj:40) ~[na:na] at puppetlabs.trapperkeeper.services.webserver.jetty9_core$proxy_handler$fn__19529.invoke(jetty9_core.clj:66) ~[na:na] at puppetlabs.trapperkeeper.services.webserver.jetty9_core.proxy$org.eclipse.jetty.server.handler.AbstractHandler$ff19274a.handle(Unknown Source) ~[na:na] at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1112) ~[puppetdb.jar:na] at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1048) ~[puppetdb.jar:na] at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) ~[puppetdb.jar:na] at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:199) ~[puppetdb.jar:na] at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:109) ~[puppetdb.jar:na] at org.eclipse.jetty.servlets.gzip.GzipHandler.handle(GzipHandler.java:275) ~[puppetdb.jar:na] at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97) ~[puppetdb.jar:na] at org.eclipse.jetty.server.Server.handle(Server.java:459) ~[puppetdb.jar:na] at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:280) ~[puppetdb.jar:na] at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:229) [puppetdb.jar:na] at org.eclipse.jetty.io.AbstractConnection$1.run(AbstractConnection.java:505) [puppetdb.jar:na] at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:607) [puppetdb.jar:na] at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:536) [puppetdb.jar:na] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_65] Caused by: javax.jms.JMSException: Could not create Transport. Reason: java.io.IOException: Broker named 'localhost' does not exist. at org.apache.activemq.util.JMSExceptionSupport.create(JMSExceptionSupport.java:35) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnectionFactory.createTransport(ActiveMQConnectionFactory.java:254) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnectionFactory.createActiveMQConnection(ActiveMQConnectionFactory.java:267) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnectionFactory.createActiveMQConnection(ActiveMQConnectionFactory.java:239) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnectionFactory.createConnection(ActiveMQConnectionFactory.java:185) ~[puppetdb.jar:na] at org.springframework.jms.connection.SingleConnectionFactory.doCreateConnection(SingleConnectionFactory.java:342) ~[puppetdb.jar:na] at org.springframework.jms.connection.SingleConnectionFactory.initConnection(SingleConnectionFactory.java:288) ~[puppetdb.jar:na] at org.springframework.jms.connection.SingleConnectionFactory.createConnection(SingleConnectionFactory.java:225) ~[puppetdb.jar:na] at org.springframework.jms.support.JmsAccessor.createConnection(JmsAccessor.java:184) ~[puppetdb.jar:na] at org.springframework.jms.core.JmsTemplate.execute(JmsTemplate.java:456) ~[puppetdb.jar:na] ... 69 common frames omitted Caused by: java.io.IOException: Broker named 'localhost' does not exist. at org.apache.activemq.transport.vm.VMTransportFactory.doCompositeConnect(VMTransportFactory.java:116) ~[puppetdb.jar:na] at org.apache.activemq.transport.vm.VMTransportFactory.doConnect(VMTransportFactory.java:54) ~[puppetdb.jar:na] at org.apache.activemq.transport.TransportFactory.doConnect(TransportFactory.java:51) ~[puppetdb.jar:na] at org.apache.activemq.transport.TransportFactory.connect(TransportFactory.java:80) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnectionFactory.createTransport(ActiveMQConnectionFactory.java:252) ~[puppetdb.jar:na] ... 77 common frames omitted 2015-01-06 16:45:45,040 INFO [c.p.p.command] [cfe812d2-bba9-4f47-be4c-ac6453d74b4f] [replace facts] av1rdb-114.ams4.prod.booking.com 2015-01-06 16:45:45,040 INFO [c.p.p.command] [c1a3b13e-4fa6-4017-862a-4894fe26a58c] [replace facts] pc101pphmsapp-01.ams4.prod.booking.com 2015-01-06 16:45:45,041 INFO [c.p.p.command] [ed9160f0-47b8-41c5-933f-0824a0d983a6] [replace facts] bc144bprdb-01.ams4.prod.booking.com 2015-01-06 16:45:45,042 WARN [o.s.j.l.DefaultMessageListenerContainer] Execution of JMS message listener failed, and no ErrorHandler has been set. javax.jms.JMSException: Peer (vm://localhost#16) disposed. at org.apache.activemq.util.JMSExceptionSupport.create(JMSExceptionSupport.java:62) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1380) ~[puppetdb.jar:na] at org.apache.activemq.TransactionContext.rollback(TransactionContext.java:279) ~[puppetdb.jar:na] at org.apache.activemq.TransactionContext.commit(TransactionContext.java:306) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQSession.commit(ActiveMQSession.java:561) ~[puppetdb.jar:na] at sun.reflect.GeneratedMethodAccessor83.invoke(Unknown Source) ~[na:na] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_65] at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_65] at org.springframework.jms.connection.CachingConnectionFactory$CachedSessionInvocationHandler.invoke(CachingConnectionFactory.java:344) ~[puppetdb.jar:na] at com.sun.proxy.$Proxy5.commit(Unknown Source) ~[na:na] at org.springframework.jms.support.JmsUtils.commitIfNecessary(JmsUtils.java:217) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractMessageListenerContainer.commitIfNecessary(AbstractMessageListenerContainer.java:575) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractMessageListenerContainer.doExecuteListener(AbstractMessageListenerContainer.java:481) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.doReceiveAndExecute(AbstractPollingMessageListenerContainer.java:325) [puppetdb.jar:na] at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:263) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1058) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1050) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:947) [puppetdb.jar:na] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_65] Caused by: org.apache.activemq.transport.TransportDisposedIOException: Peer (vm://localhost#16) disposed. at org.apache.activemq.transport.vm.VMTransport.oneway(VMTransport.java:88) ~[puppetdb.jar:na] at org.apache.activemq.transport.MutexTransport.oneway(MutexTransport.java:68) ~[puppetdb.jar:na] at org.apache.activemq.transport.ResponseCorrelator.asyncRequest(ResponseCorrelator.java:81) ~[puppetdb.jar:na] at org.apache.activemq.transport.ResponseCorrelator.request(ResponseCorrelator.java:86) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1351) ~[puppetdb.jar:na] ... 17 common frames omitted 2015-01-06 16:45:45,042 WARN [o.s.j.l.DefaultMessageListenerContainer] Execution of JMS message listener failed, and no ErrorHandler has been set. javax.jms.JMSException: Peer (vm://localhost#17) disposed. at org.apache.activemq.util.JMSExceptionSupport.create(JMSExceptionSupport.java:62) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1380) ~[puppetdb.jar:na] at org.apache.activemq.TransactionContext.rollback(TransactionContext.java:279) ~[puppetdb.jar:na] at org.apache.activemq.TransactionContext.commit(TransactionContext.java:306) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQSession.commit(ActiveMQSession.java:561) ~[puppetdb.jar:na] at sun.reflect.GeneratedMethodAccessor83.invoke(Unknown Source) ~[na:na] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_65] at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_65] at org.springframework.jms.connection.CachingConnectionFactory$CachedSessionInvocationHandler.invoke(CachingConnectionFactory.java:344) ~[puppetdb.jar:na] at com.sun.proxy.$Proxy5.commit(Unknown Source) ~[na:na] at org.springframework.jms.support.JmsUtils.commitIfNecessary(JmsUtils.java:217) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractMessageListenerContainer.commitIfNecessary(AbstractMessageListenerContainer.java:575) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractMessageListenerContainer.doExecuteListener(AbstractMessageListenerContainer.java:481) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.doReceiveAndExecute(AbstractPollingMessageListenerContainer.java:325) [puppetdb.jar:na] at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:263) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1058) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1050) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:947) [puppetdb.jar:na] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_65] Caused by: org.apache.activemq.transport.TransportDisposedIOException: Peer (vm://localhost#17) disposed. at org.apache.activemq.transport.vm.VMTransport.oneway(VMTransport.java:88) ~[puppetdb.jar:na] at org.apache.activemq.transport.MutexTransport.oneway(MutexTransport.java:68) ~[puppetdb.jar:na] at org.apache.activemq.transport.ResponseCorrelator.asyncRequest(ResponseCorrelator.java:81) ~[puppetdb.jar:na] at org.apache.activemq.transport.ResponseCorrelator.request(ResponseCorrelator.java:86) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1351) ~[puppetdb.jar:na] ... 17 common frames omitted 2015-01-06 16:45:45,042 WARN [o.s.j.l.DefaultMessageListenerContainer] Execution of JMS message listener failed, and no ErrorHandler has been set. javax.jms.JMSException: Peer (vm://localhost#51) disposed. at org.apache.activemq.util.JMSExceptionSupport.create(JMSExceptionSupport.java:62) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1380) ~[puppetdb.jar:na] at org.apache.activemq.TransactionContext.rollback(TransactionContext.java:279) ~[puppetdb.jar:na] at org.apache.activemq.TransactionContext.commit(TransactionContext.java:306) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQSession.commit(ActiveMQSession.java:561) ~[puppetdb.jar:na] at sun.reflect.GeneratedMethodAccessor83.invoke(Unknown Source) ~[na:na] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_65] at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_65] at org.springframework.jms.connection.CachingConnectionFactory$CachedSessionInvocationHandler.invoke(CachingConnectionFactory.java:344) ~[puppetdb.jar:na] at com.sun.proxy.$Proxy5.commit(Unknown Source) ~[na:na] at org.springframework.jms.support.JmsUtils.commitIfNecessary(JmsUtils.java:217) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractMessageListenerContainer.commitIfNecessary(AbstractMessageListenerContainer.java:575) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractMessageListenerContainer.doExecuteListener(AbstractMessageListenerContainer.java:481) ~[puppetdb.jar:na] at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.doReceiveAndExecute(AbstractPollingMessageListenerContainer.java:325) [puppetdb.jar:na] at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:263) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1058) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1050) [puppetdb.jar:na] at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:947) [puppetdb.jar:na] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_65] Caused by: org.apache.activemq.transport.TransportDisposedIOException: Peer (vm://localhost#51) disposed. at org.apache.activemq.transport.vm.VMTransport.oneway(VMTransport.java:88) ~[puppetdb.jar:na] at org.apache.activemq.transport.MutexTransport.oneway(MutexTransport.java:68) ~[puppetdb.jar:na] at org.apache.activemq.transport.ResponseCorrelator.asyncRequest(ResponseCorrelator.java:81) ~[puppetdb.jar:na] at org.apache.activemq.transport.ResponseCorrelator.request(ResponseCorrelator.java:86) ~[puppetdb.jar:na] at org.apache.activemq.ActiveMQConnection.syncSendPacket(ActiveMQConnection.java:1351) ~[puppetdb.jar:na] ... 17 common frames omitted 2015-01-06 16:45:45,042 WARN [o.s.j.l.DefaultMessageListenerContainer] Setup of JMS message listener invoker failed for destination 'com.puppetlabs.puppetdb.commands' - trying to recover. Cause: Peer (vm://localhost#17) disposed. 2015-01-06 16:45:45,042 WARN [o.s.j.l.DefaultMessageListenerContainer] Setup of JMS message listener invoker failed for destination 'com.puppetlabs.puppetdb.commands' - trying to recover. Cause: Peer (vm://localhost#16) disposed. 2015-01-06 16:45:45,043 INFO [o.s.j.l.DefaultMessageListenerContainer] Successfully refreshed JMS Connection 2015-01-06 16:45:45,043 WARN [o.s.j.l.DefaultMessageListenerContainer] Setup of JMS message listener invoker failed for destination 'com.puppetlabs.puppetdb.commands' - trying to recover. Cause: Peer (vm://localhost#51) disposed. 2015-01-06 16:45:45,043 INFO [o.s.j.l.DefaultMessageListenerContainer] Successfully refreshed JMS Connection 2015-01-06 16:45:45,045 INFO [o.s.j.l.DefaultMessageListenerContainer] Successfully refreshed JMS Connection 2015-01-06 16:45:45,388 WARN [o.s.j.l.DefaultMessageListenerContainer] Setup of JMS message listener invoker failed for destination 'com.puppetlabs.puppetdb.commands' - trying to recover. Cause: Peer (vm://localhost#57) disposed. 2015-01-06 16:45:45,388 INFO [o.s.j.l.DefaultMessageListenerContainer] Successfully refreshed JMS Connection 2015-01-06 16:45:45,390 WARN [o.s.j.l.DefaultMessageListenerContainer] Setup of JMS message listener invoker failed for destination 'com.puppetlabs.puppetdb.commands' - trying to recover. Cause: Peer (vm://localhost#55) disposed. 2015-01-06 16:45:45,390 INFO [o.s.j.l.DefaultMessageListenerContainer] Successfully refreshed JMS Connection 2015-01-06 16:45:45,633 WARN [o.e.j.s.HttpChannel] /v3/commands?checksum=4290717ccd8955e5569a8f5544609bc2913ec229 org.springframework.jms.UncategorizedJmsException: Uncategorized exception occured during JMS processing; nested exception is javax.jms.JMSException: Could not create Transport. Reason: java.io.IOException: Broker named 'localhost' does not exist. {code} Similar issues present in the puppetdb logs of the PDB-1124 user for hosts 2 and 6.

    JIRA | 2 years ago | Wyatt Alt
    org.springframework.jms.UncategorizedJmsException: Uncategorized exception occured during JMS processing; nested exception is javax.jms.JMSException: /var/lib/puppetdb/mq/localhost/KahaDB/db-7087.log (No such file or directory)
  3. 0

    Currently there's a small window between jetty starting up with the broker websocket routes listening, and the broker service actually being started and capable of routing messages. Externally this looks like: {code} D, [2016-02-11T14:54:22.274223 #43424] DEBUG -- : [:connect, "wss://vy64oh2eahqazp2.delivery.puppetlabs.net:8142/pcp/"] I, [2016-02-11T14:54:23.216230 #43424] INFO -- : [:open] D, [2016-02-11T14:54:23.216400 #43424] DEBUG -- : [:send, #<PCP::Message:0x007f86c88e3680 @chunks=["", ""], @envelope={:id=>"bbb4d40d-7bf0-4600-a276-5e0d1e3e25d9", :message_type=>"http://puppetlabs.com/associate_request", :sender=>"pcp://controller01.example.com/ruby-pcp-client-43424", :targets=>["pcp:///server"], :expires=>"2016-02-11T14:54:26Z"}>] D, [2016-02-11T14:54:23.444589 #43424] DEBUG -- : [:message, :decoded, #<PCP::Message:0x007f86c88e0a98 @chunks=["{\"id\":\"bbb4d40d-7bf0-4600-a276-5e0d1e3e25d9\",\"success\":true}", ""], @envelope={:id=>"d9ad8070-621e-4663-ab4c-1f42c3d34b36", :targets=>["pcp://controller01.example.com/ruby-pcp-client-43424"], :message_type=>"http://puppetlabs.com/associate_response", :sender=>"pcp:///server", :expires=>"2016-02-11T14:54:23.404Z"}>] D, [2016-02-11T14:54:23.444915 #43424] DEBUG -- : [:send, #<PCP::Message:0x007f86c88da738 @chunks=["{\"query\":[\"pcp://client01.example.com/agent\"]}", ""], @envelope={:id=>"5d59a016-5bd6-4f0f-a400-cdffcd5e1a8f", :message_type=>"http://puppetlabs.com/inventory_request", :targets=>["pcp:///server"], :expires=>"2016-02-11T14:54:33Z"}>] I, [2016-02-11T14:54:23.839668 #43424] INFO -- : [:close, 1011, "UncategorizedJmsException"] RuntimeError: Didn't receive a response for PCP inventory request {code} And shows up in the broker logs as so: {code} 2016-02-11 06:54:18,782 INFO [main] [p.t.s.w.jetty9-service] Initializing web server(s). 2016-02-11 06:54:18,808 INFO [main] [p.t.s.n.nrepl-service] Starting nREPL service on 127.0.0.1 port 7888 2016-02-11 06:54:18,830 INFO [main] [p.p.b.service] Initializing broker service 2016-02-11 06:54:18,987 INFO [main] [p.t.s.s.status-service] Registering status callback function for broker-service service 2016-02-11 06:54:18,990 INFO [main] [p.t.s.w.jetty9-service] Starting web server(s). 2016-02-11 06:54:19,075 INFO [main] [p.t.s.w.jetty9-core] Starting web server. 2016-02-11 06:54:19,214 INFO [main] [p.t.s.s.status-service] Registering status service HTTP API at /status 2016-02-11 06:54:19,225 INFO [main] [p.p.b.service] Starting broker service 2016-02-11 06:54:19,241 INFO [main] [o.a.a.s.k.p.PListStore] PListStore:[/opt/puppet-git-repos/pcp-broker/./test-resources/tmp/activemq/localhost/tmp_storage ] started 2016-02-11 06:54:19,243 INFO [main] [o.a.a.b.BrokerService] Using Persistence Adapter: KahaDBPersistenceAdapter[/opt/puppet-git-repos/pcp-broker/./test-resources/tmp/activemq/localhost/KahaDB] 2016-02-11 06:54:20,209 DEBUG [qtp848673493-18] [p.p.b.core] client controller01.example.com connected from /10.32.128.133:63862 2016-02-11 06:54:20,392 TRACE [qtp848673493-19] [p.p.b.core] Message bbb4d40d-7bf0-4600-a276-5e0d1e3e25d9 for ["pcp:///server"] from controller01.example.com /10.32.128.133:63862 2016-02-11 06:54:20,692 TRACE [qtp848673493-20] [p.p.b.core] Message 5d59a016-5bd6-4f0f-a400-cdffcd5e1a8f for ["pcp:///server"] from controller01.example.com /10.32.128.133:63862 2016-02-11 06:54:20,720 TRACE [qtp848673493-20] [p.p.b.core] Authorizing 5328ba07-4a97-4db0-b8df-2ffcb55e6333 for ["pcp://controller01.example.com/ruby-pcp-client-43424"] - true: allow-unauthenticated is true - allowed 2016-02-11 06:54:20,726 TRACE [qtp848673493-20] [p.p.b.activemq] Delivering message 5328ba07-4a97-4db0-b8df-2ffcb55e6333 for ["pcp://controller01.example.com/ruby-pcp-client-43424"] to accept queue 2016-02-11 06:54:20,793 WARN [qtp848673493-20] [p.t.s.w.e.j.p.e.j.w.a.WebSocketAdapter$CertGetter$37c4cf7] Unhandled Error (closing connection) org.springframework.jms.UncategorizedJmsException: Uncategorized exception occured during JMS processing; nested exception is javax.jms.JMSException: Could not create Transport. Reason: java.io.IOException: Broker named 'localhost' does not exist. at org.springframework.jms.support.JmsUtils.convertJmsAccessException(JmsUtils.java:316) ~[spring-jms-3.0.5.RELEASE.jar:3.0.5.RELEASE] at org.springframework.jms.support.JmsAccessor.convertJmsAccessException(JmsAccessor.java:168) ~[spring-jms-3.0.5.RELEASE.jar:3.0.5.RELEASE] at org.springframework.jms.core.JmsTemplate.execute(JmsTemplate.java:469) ~[spring-jms-3.0.5.RELEASE.jar:3.0.5.RELEASE] at org.springframework.jms.core.JmsTemplate.send(JmsTemplate.java:543) ~[spring-jms-3.0.5.RELEASE.jar:3.0.5.RELEASE] at org.springframework.jms.core.JmsTemplate.convertAndSend(JmsTemplate.java:653) ~[spring-jms-3.0.5.RELEASE.jar:3.0.5.RELEASE] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.8.0_71] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[na:1.8.0_71] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_71] {code} Here we have attempted to queue a message on the "localhost" activemq before it was started, so we get an unhandled Jms exception. Rather than allowing session association, the broker should refuse before it has gone fully through its service lifecycle (start) function.

    JIRA | 10 months ago | Richard Clamp
    org.springframework.jms.UncategorizedJmsException: Uncategorized exception occured during JMS processing; nested exception is javax.jms.JMSException: Could not create Transport. Reason: java.io.IOException: Broker named 'localhost' does not exist.
  4. Speed up your debug routine!

    Automated exception search integrated into your IDE

  5. 0

    Currently there's a small window between jetty starting up with the broker websocket routes listening, and the broker service actually being started and capable of routing messages. Externally this looks like: {code} D, [2016-02-11T14:54:22.274223 #43424] DEBUG -- : [:connect, "wss://vy64oh2eahqazp2.delivery.puppetlabs.net:8142/pcp/"] I, [2016-02-11T14:54:23.216230 #43424] INFO -- : [:open] D, [2016-02-11T14:54:23.216400 #43424] DEBUG -- : [:send, #<PCP::Message:0x007f86c88e3680 @chunks=["", ""], @envelope={:id=>"bbb4d40d-7bf0-4600-a276-5e0d1e3e25d9", :message_type=>"http://puppetlabs.com/associate_request", :sender=>"pcp://controller01.example.com/ruby-pcp-client-43424", :targets=>["pcp:///server"], :expires=>"2016-02-11T14:54:26Z"}>] D, [2016-02-11T14:54:23.444589 #43424] DEBUG -- : [:message, :decoded, #<PCP::Message:0x007f86c88e0a98 @chunks=["{\"id\":\"bbb4d40d-7bf0-4600-a276-5e0d1e3e25d9\",\"success\":true}", ""], @envelope={:id=>"d9ad8070-621e-4663-ab4c-1f42c3d34b36", :targets=>["pcp://controller01.example.com/ruby-pcp-client-43424"], :message_type=>"http://puppetlabs.com/associate_response", :sender=>"pcp:///server", :expires=>"2016-02-11T14:54:23.404Z"}>] D, [2016-02-11T14:54:23.444915 #43424] DEBUG -- : [:send, #<PCP::Message:0x007f86c88da738 @chunks=["{\"query\":[\"pcp://client01.example.com/agent\"]}", ""], @envelope={:id=>"5d59a016-5bd6-4f0f-a400-cdffcd5e1a8f", :message_type=>"http://puppetlabs.com/inventory_request", :targets=>["pcp:///server"], :expires=>"2016-02-11T14:54:33Z"}>] I, [2016-02-11T14:54:23.839668 #43424] INFO -- : [:close, 1011, "UncategorizedJmsException"] RuntimeError: Didn't receive a response for PCP inventory request {code} And shows up in the broker logs as so: {code} 2016-02-11 06:54:18,782 INFO [main] [p.t.s.w.jetty9-service] Initializing web server(s). 2016-02-11 06:54:18,808 INFO [main] [p.t.s.n.nrepl-service] Starting nREPL service on 127.0.0.1 port 7888 2016-02-11 06:54:18,830 INFO [main] [p.p.b.service] Initializing broker service 2016-02-11 06:54:18,987 INFO [main] [p.t.s.s.status-service] Registering status callback function for broker-service service 2016-02-11 06:54:18,990 INFO [main] [p.t.s.w.jetty9-service] Starting web server(s). 2016-02-11 06:54:19,075 INFO [main] [p.t.s.w.jetty9-core] Starting web server. 2016-02-11 06:54:19,214 INFO [main] [p.t.s.s.status-service] Registering status service HTTP API at /status 2016-02-11 06:54:19,225 INFO [main] [p.p.b.service] Starting broker service 2016-02-11 06:54:19,241 INFO [main] [o.a.a.s.k.p.PListStore] PListStore:[/opt/puppet-git-repos/pcp-broker/./test-resources/tmp/activemq/localhost/tmp_storage ] started 2016-02-11 06:54:19,243 INFO [main] [o.a.a.b.BrokerService] Using Persistence Adapter: KahaDBPersistenceAdapter[/opt/puppet-git-repos/pcp-broker/./test-resources/tmp/activemq/localhost/KahaDB] 2016-02-11 06:54:20,209 DEBUG [qtp848673493-18] [p.p.b.core] client controller01.example.com connected from /10.32.128.133:63862 2016-02-11 06:54:20,392 TRACE [qtp848673493-19] [p.p.b.core] Message bbb4d40d-7bf0-4600-a276-5e0d1e3e25d9 for ["pcp:///server"] from controller01.example.com /10.32.128.133:63862 2016-02-11 06:54:20,692 TRACE [qtp848673493-20] [p.p.b.core] Message 5d59a016-5bd6-4f0f-a400-cdffcd5e1a8f for ["pcp:///server"] from controller01.example.com /10.32.128.133:63862 2016-02-11 06:54:20,720 TRACE [qtp848673493-20] [p.p.b.core] Authorizing 5328ba07-4a97-4db0-b8df-2ffcb55e6333 for ["pcp://controller01.example.com/ruby-pcp-client-43424"] - true: allow-unauthenticated is true - allowed 2016-02-11 06:54:20,726 TRACE [qtp848673493-20] [p.p.b.activemq] Delivering message 5328ba07-4a97-4db0-b8df-2ffcb55e6333 for ["pcp://controller01.example.com/ruby-pcp-client-43424"] to accept queue 2016-02-11 06:54:20,793 WARN [qtp848673493-20] [p.t.s.w.e.j.p.e.j.w.a.WebSocketAdapter$CertGetter$37c4cf7] Unhandled Error (closing connection) org.springframework.jms.UncategorizedJmsException: Uncategorized exception occured during JMS processing; nested exception is javax.jms.JMSException: Could not create Transport. Reason: java.io.IOException: Broker named 'localhost' does not exist. at org.springframework.jms.support.JmsUtils.convertJmsAccessException(JmsUtils.java:316) ~[spring-jms-3.0.5.RELEASE.jar:3.0.5.RELEASE] at org.springframework.jms.support.JmsAccessor.convertJmsAccessException(JmsAccessor.java:168) ~[spring-jms-3.0.5.RELEASE.jar:3.0.5.RELEASE] at org.springframework.jms.core.JmsTemplate.execute(JmsTemplate.java:469) ~[spring-jms-3.0.5.RELEASE.jar:3.0.5.RELEASE] at org.springframework.jms.core.JmsTemplate.send(JmsTemplate.java:543) ~[spring-jms-3.0.5.RELEASE.jar:3.0.5.RELEASE] at org.springframework.jms.core.JmsTemplate.convertAndSend(JmsTemplate.java:653) ~[spring-jms-3.0.5.RELEASE.jar:3.0.5.RELEASE] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.8.0_71] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[na:1.8.0_71] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_71] {code} Here we have attempted to queue a message on the "localhost" activemq before it was started, so we get an unhandled Jms exception. Rather than allowing session association, the broker should refuse before it has gone fully through its service lifecycle (start) function.

    JIRA | 10 months ago | Richard Clamp
    org.springframework.jms.UncategorizedJmsException: Uncategorized exception occured during JMS processing; nested exception is javax.jms.JMSException: Could not create Transport. Reason: java.io.IOException: Broker named 'localhost' does not exist.
  6. 0

    603422 [AcitveMQ Connection Worker: tcp://localhost/127.0.0.1:61616] ERROR JMS Connection Service - exception occurred in JMS connection; restarting javax.jms.JMSException: Channel was inactive for too long. at org.apache.activemq.util.JMSExceptionSupport.create(JMSExceptionSupport.java:49) at org.apache.activemq.ActiveMQConnection.onAsyncException(ActiveMQConnection.java:1638) at org.apache.activemq.ActiveMQConnection.onException(ActiveMQConnection.java:1655) at org.apache.activemq.transport.TransportFilter.onException(TransportFilter.java:98) at org.apache.activemq.transport.ResponseCorrelator.onException(ResponseCorrelator.java:116) at org.apache.activemq.transport.TransportFilter.onException(TransportFilter.java:98) at org.apache.activemq.transport.TransportFilter.onException(TransportFilter.java:98) at org.apache.activemq.transport.WireFormatNegotiator.onException(WireFormatNegotiator.java:143) at org.apache.activemq.transport.InactivityMonitor.onException(InactivityMonitor.java:155) at org.apache.activemq.transport.InactivityMonitor.readCheck(InactivityMonitor.java:100) at org.apache.activemq.transport.InactivityMonitor$1.run(InactivityMonitor.java:50) at org.apache.activemq.thread.Scheduler$SchedulerTimerTask.run(Scheduler.java:36) at java.util.TimerThread.mainLoop(Unknown Source) at java.util.TimerThread.run(Unknown Source) Caused by: org.apache.activemq.transport.InactivityIOException: Channel was inactive for too long. ... 5 more

    Apache's JIRA Issue Tracker | 9 years ago | Kevin W Edwards
    org.springframework.jms.UncategorizedJmsException: Uncategorized exception occured during JMS processing; nested exception is javax.jms.JMSException: Channel was inactive for too long: localhost/127.0.0.1:61616; nested exception is org.apache.activemq.transport.InactivityIOException: Channel was inactive for too long: localhost/127.0.0.1:61616

    1 unregistered visitors
    Not finding the right solution?
    Take a tour to get the most out of Samebug.

    Tired of useless tips?

    Automated exception search integrated into your IDE

    Root Cause Analysis

    1. org.springframework.jms.UncategorizedJmsException

      Uncategorized exception occured during JMS processing; nested exception is javax.jms.JMSException: /var/lib/puppetdb/mq/localhost/KahaDB/db-7087.log (No such file or directory)

      at org.springframework.jms.support.JmsUtils.convertJmsAccessException()
    2. Spring Framework
      JmsTemplate.convertAndSend
      1. org.springframework.jms.support.JmsUtils.convertJmsAccessException(JmsUtils.java:316)[puppetdb.jar:na]
      2. org.springframework.jms.support.JmsAccessor.convertJmsAccessException(JmsAccessor.java:168)[puppetdb.jar:na]
      3. org.springframework.jms.core.JmsTemplate.execute(JmsTemplate.java:469)[puppetdb.jar:na]
      4. org.springframework.jms.core.JmsTemplate.send(JmsTemplate.java:543)[puppetdb.jar:na]
      5. org.springframework.jms.core.JmsTemplate.convertAndSend(JmsTemplate.java:653)[puppetdb.jar:na]
      5 frames
    3. Java RT
      Method.invoke
      1. sun.reflect.GeneratedMethodAccessor97.invoke(Unknown Source)[na:na]
      2. sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)[na:1.7.0_65]
      3. java.lang.reflect.Method.invoke(Method.java:606)[na:1.7.0_65]
      3 frames
    4. Clojure
      Reflector.invokeInstanceMethod
      1. clojure.lang.Reflector.invokeMatchingMethod(Reflector.java:93)[puppetdb.jar:na]
      2. clojure.lang.Reflector.invokeInstanceMethod(Reflector.java:28)[puppetdb.jar:na]
      2 frames
    5. clamq
      jms$jms_producer$reify__10735.publish
      1. clamq.jms$jms_producer$reify__10735.publish(jms.clj:29)[na:na]
      1 frame
    6. clamq.protocol
      producer$eval10687$fn__10688$G__10678__10705.invoke
      1. clamq.protocol.producer$eval10687$fn__10688$G__10679__10696.invoke(producer.clj:3)[na:na]
      2. clamq.protocol.producer$eval10687$fn__10688$G__10678__10705.invoke(producer.clj:3)[na:na]
      2 frames
    7. Clojure
      core$apply.invoke
      1. clojure.lang.AFn.applyToHelper(AFn.java:165)[puppetdb.jar:na]
      2. clojure.lang.AFn.applyTo(AFn.java:144)[puppetdb.jar:na]
      3. clojure.core$apply.invoke(core.clj:626)[puppetdb.jar:na]
      3 frames
    8. com.puppetlabs
      mq$connect_and_publish_BANG_.doInvoke
      1. com.puppetlabs.mq$connect_and_publish_BANG_.doInvoke(mq.clj:148)[na:na]
      1 frame
    9. Clojure
      RestFn.invoke
      1. clojure.lang.RestFn.invoke(RestFn.java:460)[puppetdb.jar:na]
      1 frame
    10. com.puppetlabs.puppetdb
      command$enqueue_command.invoke
      1. com.puppetlabs.puppetdb.command$enqueue_raw_command_BANG_$fn__11440.invoke(command.clj:249)[na:na]
      2. com.puppetlabs.puppetdb.command$enqueue_raw_command_BANG_.invoke(command.clj:248)[na:na]
      3. com.puppetlabs.puppetdb.http.command$enqueue_command.invoke(command.clj:12)[na:na]
      3 frames
    11. com.puppetlabs
      middleware$verify_content_type$fn__12253.invoke
      1. com.puppetlabs.middleware$verify_accepts_content_type$fn__12248.invoke(middleware.clj:114)[na:na]
      2. com.puppetlabs.middleware$verify_checksum$fn__12268.invoke(middleware.clj:185)[na:na]
      3. com.puppetlabs.middleware$validate_query_params$fn__12259.invoke(middleware.clj:155)[na:na]
      4. com.puppetlabs.middleware$payload_to_body_string$fn__12303.invoke(middleware.clj:276)[na:na]
      5. com.puppetlabs.middleware$verify_content_type$fn__12253.invoke(middleware.clj:128)[na:na]
      5 frames
    12. com.puppetlabs.puppetdb
      server$routes$fn__14649$fn__14663.invoke
      1. com.puppetlabs.puppetdb.http.v3$v3_app$fn__13762.invoke(v3.clj:23)[na:na]
      2. com.puppetlabs.puppetdb.http.v3$v3_app.invoke(v3.clj:21)[na:na]
      3. com.puppetlabs.puppetdb.http.server$routes$fn__14649$fn__14663.invoke(server.clj:48)[na:na]
      3 frames
    13. net.cgrand
      moustache$alter_request$fn__12677.invoke
      1. net.cgrand.moustache$alter_request$fn__12677.invoke(moustache.clj:54)[na:na]
      1 frame
    14. com.puppetlabs.puppetdb
      server$routes$fn__14649.invoke
      1. com.puppetlabs.puppetdb.http.server$routes$fn__14649.invoke(server.clj:48)[na:na]
      1 frame
    15. kifshare
      params$wrap_params$fn__12620.invoke
      1. ring.middleware.resource$wrap_resource$fn__14631.invoke(resource.clj:24)[na:na]
      2. ring.middleware.params$wrap_params$fn__12620.invoke(params.clj:58)[na:na]
      2 frames
    16. com.puppetlabs
      middleware$wrap_with_metrics_STAR_$fn__12275$fn__12284.invoke
      1. com.puppetlabs.middleware$wrap_with_authorization$fn__12220.invoke(middleware.clj:40)[na:na]
      2. com.puppetlabs.middleware$wrap_with_certificate_cn$fn__12224.invoke(middleware.clj:55)[na:na]
      3. com.puppetlabs.middleware$wrap_with_default_body$fn__12228.invoke(middleware.clj:62)[na:na]
      4. com.puppetlabs.middleware$wrap_with_metrics_STAR_$fn__12275$fn__12284.invoke(middleware.clj:207)[na:na]
      4 frames
    17. com.puppetlabs.utils
      Object$Callable$7da976d4.call
      1. com.puppetlabs.utils.metrics$multitime_BANG__STAR_$fn__11942$fn__11943$fn__11944.invoke(metrics.clj:14)[na:na]
      2. com.puppetlabs.utils.metrics.proxy$java.lang.Object$Callable$7da976d4.call(Unknown Source)[na:na]
      2 frames
    18. Apache Giraph Core
      Timer.time
      1. com.yammer.metrics.core.Timer.time(Timer.java:91)[puppetdb.jar:na]
      1 frame