java.lang.IllegalStateException: Insufficient max threads in ThreadPool: max=100 < needed=121

JIRA | Charlie Sharpsteen | 2 years ago
  1. 0

    Our [current default|https://github.com/puppetlabs/trapperkeeper-webserver-jetty9/blob/trapperkeeper-webserver-jetty9-1.0.1/src/puppetlabs/trapperkeeper/services/webserver/jetty9_config.clj#L29] for {{max-threads}} is a static value of 100. For some servers, apparently those with 256 GB of RAM, this default is insufficient and the service crashes during startup: {noformat} 2014-12-09 09:34:51,993 INFO [p.t.s.w.jetty9-service] Initializing web server. 2014-12-09 09:34:52,078 INFO [p.t.s.w.jetty9-service] Starting web server. 2014-12-09 09:34:52,081 INFO [o.e.j.s.Server] jetty-9.1.z-SNAPSHOT 2014-12-09 09:34:52,098 WARN [o.e.j.u.c.AbstractLifeCycle] FAILED org.eclipse.jetty.server.Server@6251cb36: java.lang.IllegalStateException: Insufficient max threads in ThreadPool: max=100 < needed=121 java.lang.IllegalStateException: Insufficient max threads in ThreadPool: max=100 < needed=121 at org.eclipse.jetty.server.Server.doStart(Server.java:320) ~[puppetdb.jar:na] at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) ~[puppetdb.jar:na] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.7.0_71-icedtea] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) ~[na:1.7.0_71-icedtea] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_71-icedtea] at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_71-icedtea] at clojure.lang.Reflector.invokeMatchingMethod(Reflector.java:93) [puppetdb.jar:na] at clojure.lang.Reflector.invokeNoArgInstanceMember(Reflector.java:313) [puppetdb.jar:na] at puppetlabs.trapperkeeper.services.webserver.jetty9_core$start_webserver.invoke(jetty9_core.clj:206) [na:na] at puppetlabs.trapperkeeper.services.webserver.jetty9_service$reify__19641$f_positional__19646$fnk19754_positional__19755$fn__19756.invoke(jetty9_service.clj:15) [na:na] at puppetlabs.trapperkeeper.services.webserver.jetty9_service$reify__19641$fn$reify__19831.start(jetty9_service.clj:15) [na:na] at puppetlabs.trapperkeeper.services$eval16331$fn__16345$G__16321__16348.invoke(services.clj:12) [na:na] at puppetlabs.trapperkeeper.services$eval16331$fn__16345$G__16320__16352.invoke(services.clj:12) [na:na] at puppetlabs.trapperkeeper.internal$run_lifecycle_fn_BANG_.invoke(internal.clj:143) [na:na] at puppetlabs.trapperkeeper.internal$run_lifecycle_fns.invoke(internal.clj:170) [na:na] at puppetlabs.trapperkeeper.internal$build_app_STAR_$reify__17410.start(internal.clj:385) [na:na] at puppetlabs.trapperkeeper.internal$boot_services_STAR_.invoke(internal.clj:401) [na:na] at puppetlabs.trapperkeeper.core$boot_with_cli_data.invoke(core.clj:114) [na:na] at puppetlabs.trapperkeeper.core$run.invoke(core.clj:145) [na:na] at puppetlabs.trapperkeeper.core$main.doInvoke(core.clj:160) [na:na] at clojure.lang.RestFn.applyTo(RestFn.java:137) [puppetdb.jar:na] at clojure.core$apply.invoke(core.clj:624) [puppetdb.jar:na] at com.puppetlabs.puppetdb.cli.services$_main.doInvoke(services.clj:374) [na:na] at clojure.lang.RestFn.invoke(RestFn.java:421) [puppetdb.jar:na] at clojure.lang.Var.invoke(Var.java:383) [puppetdb.jar:na] at clojure.lang.AFn.applyToHelper(AFn.java:156) [puppetdb.jar:na] at clojure.lang.Var.applyTo(Var.java:700) [puppetdb.jar:na] at clojure.core$apply.invoke(core.clj:624) [puppetdb.jar:na] at com.puppetlabs.puppetdb.core$run_command.invoke(core.clj:87) [na:na] at com.puppetlabs.puppetdb.core$_main.doInvoke(core.clj:95) [na:na] at clojure.lang.RestFn.invoke(RestFn.java:436) [puppetdb.jar:na] at clojure.lang.Var.invoke(Var.java:388) [puppetdb.jar:na] at clojure.lang.AFn.applyToHelper(AFn.java:160) [puppetdb.jar:na] at clojure.lang.Var.applyTo(Var.java:700) [puppetdb.jar:na] at clojure.core$apply.invoke(core.clj:624) [puppetdb.jar:na] at clojure.main$main_opt.invoke(main.clj:315) [puppetdb.jar:na] at clojure.main$main.doInvoke(main.clj:420) [puppetdb.jar:na] at clojure.lang.RestFn.invoke(RestFn.java:482) [puppetdb.jar:na] at clojure.lang.Var.invoke(Var.java:401) [puppetdb.jar:na] at clojure.lang.AFn.applyToHelper(AFn.java:171) [puppetdb.jar:na] at clojure.lang.Var.applyTo(Var.java:700) [puppetdb.jar:na] at clojure.main.main(main.java:37) [puppetdb.jar:na] {noformat} Adding a {{max-threads = 150}} to the configuration of a trapperkeeper-powered service allows it to start up successfully. However, the default value brings Puppet Enterprise services down immediately after installation, before config files can be edited, which results in a failed installation. In PE 3.7.0, this issue affects PuppetDB, Console Services and Puppet Server since all three use the Jetty 9 component.

    JIRA | 2 years ago | Charlie Sharpsteen
    java.lang.IllegalStateException: Insufficient max threads in ThreadPool: max=100 < needed=121
  2. 0

    Our [current default|https://github.com/puppetlabs/trapperkeeper-webserver-jetty9/blob/trapperkeeper-webserver-jetty9-1.0.1/src/puppetlabs/trapperkeeper/services/webserver/jetty9_config.clj#L29] for {{max-threads}} is a static value of 100. For some servers, apparently those with 256 GB of RAM, this default is insufficient and the service crashes during startup: {noformat} 2014-12-09 09:34:51,993 INFO [p.t.s.w.jetty9-service] Initializing web server. 2014-12-09 09:34:52,078 INFO [p.t.s.w.jetty9-service] Starting web server. 2014-12-09 09:34:52,081 INFO [o.e.j.s.Server] jetty-9.1.z-SNAPSHOT 2014-12-09 09:34:52,098 WARN [o.e.j.u.c.AbstractLifeCycle] FAILED org.eclipse.jetty.server.Server@6251cb36: java.lang.IllegalStateException: Insufficient max threads in ThreadPool: max=100 < needed=121 java.lang.IllegalStateException: Insufficient max threads in ThreadPool: max=100 < needed=121 at org.eclipse.jetty.server.Server.doStart(Server.java:320) ~[puppetdb.jar:na] at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69) ~[puppetdb.jar:na] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.7.0_71-icedtea] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) ~[na:1.7.0_71-icedtea] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_71-icedtea] at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_71-icedtea] at clojure.lang.Reflector.invokeMatchingMethod(Reflector.java:93) [puppetdb.jar:na] at clojure.lang.Reflector.invokeNoArgInstanceMember(Reflector.java:313) [puppetdb.jar:na] at puppetlabs.trapperkeeper.services.webserver.jetty9_core$start_webserver.invoke(jetty9_core.clj:206) [na:na] at puppetlabs.trapperkeeper.services.webserver.jetty9_service$reify__19641$f_positional__19646$fnk19754_positional__19755$fn__19756.invoke(jetty9_service.clj:15) [na:na] at puppetlabs.trapperkeeper.services.webserver.jetty9_service$reify__19641$fn$reify__19831.start(jetty9_service.clj:15) [na:na] at puppetlabs.trapperkeeper.services$eval16331$fn__16345$G__16321__16348.invoke(services.clj:12) [na:na] at puppetlabs.trapperkeeper.services$eval16331$fn__16345$G__16320__16352.invoke(services.clj:12) [na:na] at puppetlabs.trapperkeeper.internal$run_lifecycle_fn_BANG_.invoke(internal.clj:143) [na:na] at puppetlabs.trapperkeeper.internal$run_lifecycle_fns.invoke(internal.clj:170) [na:na] at puppetlabs.trapperkeeper.internal$build_app_STAR_$reify__17410.start(internal.clj:385) [na:na] at puppetlabs.trapperkeeper.internal$boot_services_STAR_.invoke(internal.clj:401) [na:na] at puppetlabs.trapperkeeper.core$boot_with_cli_data.invoke(core.clj:114) [na:na] at puppetlabs.trapperkeeper.core$run.invoke(core.clj:145) [na:na] at puppetlabs.trapperkeeper.core$main.doInvoke(core.clj:160) [na:na] at clojure.lang.RestFn.applyTo(RestFn.java:137) [puppetdb.jar:na] at clojure.core$apply.invoke(core.clj:624) [puppetdb.jar:na] at com.puppetlabs.puppetdb.cli.services$_main.doInvoke(services.clj:374) [na:na] at clojure.lang.RestFn.invoke(RestFn.java:421) [puppetdb.jar:na] at clojure.lang.Var.invoke(Var.java:383) [puppetdb.jar:na] at clojure.lang.AFn.applyToHelper(AFn.java:156) [puppetdb.jar:na] at clojure.lang.Var.applyTo(Var.java:700) [puppetdb.jar:na] at clojure.core$apply.invoke(core.clj:624) [puppetdb.jar:na] at com.puppetlabs.puppetdb.core$run_command.invoke(core.clj:87) [na:na] at com.puppetlabs.puppetdb.core$_main.doInvoke(core.clj:95) [na:na] at clojure.lang.RestFn.invoke(RestFn.java:436) [puppetdb.jar:na] at clojure.lang.Var.invoke(Var.java:388) [puppetdb.jar:na] at clojure.lang.AFn.applyToHelper(AFn.java:160) [puppetdb.jar:na] at clojure.lang.Var.applyTo(Var.java:700) [puppetdb.jar:na] at clojure.core$apply.invoke(core.clj:624) [puppetdb.jar:na] at clojure.main$main_opt.invoke(main.clj:315) [puppetdb.jar:na] at clojure.main$main.doInvoke(main.clj:420) [puppetdb.jar:na] at clojure.lang.RestFn.invoke(RestFn.java:482) [puppetdb.jar:na] at clojure.lang.Var.invoke(Var.java:401) [puppetdb.jar:na] at clojure.lang.AFn.applyToHelper(AFn.java:171) [puppetdb.jar:na] at clojure.lang.Var.applyTo(Var.java:700) [puppetdb.jar:na] at clojure.main.main(main.java:37) [puppetdb.jar:na] {noformat} Adding a {{max-threads = 150}} to the configuration of a trapperkeeper-powered service allows it to start up successfully. However, the default value brings Puppet Enterprise services down immediately after installation, before config files can be edited, which results in a failed installation. In PE 3.7.0, this issue affects PuppetDB, Console Services and Puppet Server since all three use the Jetty 9 component.

    JIRA | 2 years ago | Charlie Sharpsteen
    java.lang.IllegalStateException: Insufficient max threads in ThreadPool: max=100 < needed=121
  3. 0

    Context specific LoginService causes context restart to fail under jmx

    GitHub | 10 months ago | jmcc0nn3ll
    java.lang.IllegalStateException: No LoginService for org.eclipse.jetty.security.authentication.FormAuthenticator@5495333e in org.eclipse.jetty.security.ConstraintSecurityHandler@68034211
  4. Speed up your debug routine!

    Automated exception search integrated into your IDE

  5. 0

    Re: [jetty-users] WebAppContext only produces "HTTP ERROR: 503"

    eclipse.org | 1 year ago
    java.lang.IllegalStateException: No LoginService for org.eclipse.jetty.security.authentication.FormAuthenticator@10e434d in ConstraintSecurityHandler@3e97df@
  6. 0

    Using multiple embedded Hawtio instances on one machine/user can error on startup

    GitHub | 3 years ago | alex-sherwin
    java.lang.IllegalStateException: Failed to delete temp dir /home/acadiait/.hawtio/tmp

    12 unregistered visitors
    Not finding the right solution?
    Take a tour to get the most out of Samebug.

    Tired of useless tips?

    Automated exception search integrated into your IDE

    Root Cause Analysis

    1. java.lang.IllegalStateException

      Insufficient max threads in ThreadPool: max=100 < needed=121

      at org.eclipse.jetty.server.Server.doStart()
    2. Jetty
      AbstractLifeCycle.start
      1. org.eclipse.jetty.server.Server.doStart(Server.java:320)[puppetdb.jar:na]
      2. org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)[puppetdb.jar:na]
      2 frames
    3. Java RT
      Method.invoke
      1. sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)[na:1.7.0_71-icedtea]
      2. sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)[na:1.7.0_71-icedtea]
      3. sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)[na:1.7.0_71-icedtea]
      4. java.lang.reflect.Method.invoke(Method.java:606)[na:1.7.0_71-icedtea]
      4 frames
    4. Clojure
      Reflector.invokeNoArgInstanceMember
      1. clojure.lang.Reflector.invokeMatchingMethod(Reflector.java:93)[puppetdb.jar:na]
      2. clojure.lang.Reflector.invokeNoArgInstanceMember(Reflector.java:313)[puppetdb.jar:na]
      2 frames
    5. puppetlabs.trapperkeeper.services
      jetty9_service$reify__19641$fn$reify__19831.start
      1. puppetlabs.trapperkeeper.services.webserver.jetty9_core$start_webserver.invoke(jetty9_core.clj:206)[na:na]
      2. puppetlabs.trapperkeeper.services.webserver.jetty9_service$reify__19641$f_positional__19646$fnk19754_positional__19755$fn__19756.invoke(jetty9_service.clj:15)[na:na]
      3. puppetlabs.trapperkeeper.services.webserver.jetty9_service$reify__19641$fn$reify__19831.start(jetty9_service.clj:15)[na:na]
      3 frames
    6. puppetlabs.trapperkeeper
      core$main.doInvoke
      1. puppetlabs.trapperkeeper.services$eval16331$fn__16345$G__16321__16348.invoke(services.clj:12)[na:na]
      2. puppetlabs.trapperkeeper.services$eval16331$fn__16345$G__16320__16352.invoke(services.clj:12)[na:na]
      3. puppetlabs.trapperkeeper.internal$run_lifecycle_fn_BANG_.invoke(internal.clj:143)[na:na]
      4. puppetlabs.trapperkeeper.internal$run_lifecycle_fns.invoke(internal.clj:170)[na:na]
      5. puppetlabs.trapperkeeper.internal$build_app_STAR_$reify__17410.start(internal.clj:385)[na:na]
      6. puppetlabs.trapperkeeper.internal$boot_services_STAR_.invoke(internal.clj:401)[na:na]
      7. puppetlabs.trapperkeeper.core$boot_with_cli_data.invoke(core.clj:114)[na:na]
      8. puppetlabs.trapperkeeper.core$run.invoke(core.clj:145)[na:na]
      9. puppetlabs.trapperkeeper.core$main.doInvoke(core.clj:160)[na:na]
      9 frames
    7. Clojure
      core$apply.invoke
      1. clojure.lang.RestFn.applyTo(RestFn.java:137)[puppetdb.jar:na]
      2. clojure.core$apply.invoke(core.clj:624)[puppetdb.jar:na]
      2 frames
    8. com.puppetlabs.puppetdb
      services$_main.doInvoke
      1. com.puppetlabs.puppetdb.cli.services$_main.doInvoke(services.clj:374)[na:na]
      1 frame
    9. Clojure
      core$apply.invoke
      1. clojure.lang.RestFn.invoke(RestFn.java:421)[puppetdb.jar:na]
      2. clojure.lang.Var.invoke(Var.java:383)[puppetdb.jar:na]
      3. clojure.lang.AFn.applyToHelper(AFn.java:156)[puppetdb.jar:na]
      4. clojure.lang.Var.applyTo(Var.java:700)[puppetdb.jar:na]
      5. clojure.core$apply.invoke(core.clj:624)[puppetdb.jar:na]
      5 frames
    10. com.puppetlabs.puppetdb
      core$_main.doInvoke
      1. com.puppetlabs.puppetdb.core$run_command.invoke(core.clj:87)[na:na]
      2. com.puppetlabs.puppetdb.core$_main.doInvoke(core.clj:95)[na:na]
      2 frames
    11. Clojure
      main.main
      1. clojure.lang.RestFn.invoke(RestFn.java:436)[puppetdb.jar:na]
      2. clojure.lang.Var.invoke(Var.java:388)[puppetdb.jar:na]
      3. clojure.lang.AFn.applyToHelper(AFn.java:160)[puppetdb.jar:na]
      4. clojure.lang.Var.applyTo(Var.java:700)[puppetdb.jar:na]
      5. clojure.core$apply.invoke(core.clj:624)[puppetdb.jar:na]
      6. clojure.main$main_opt.invoke(main.clj:315)[puppetdb.jar:na]
      7. clojure.main$main.doInvoke(main.clj:420)[puppetdb.jar:na]
      8. clojure.lang.RestFn.invoke(RestFn.java:482)[puppetdb.jar:na]
      9. clojure.lang.Var.invoke(Var.java:401)[puppetdb.jar:na]
      10. clojure.lang.AFn.applyToHelper(AFn.java:171)[puppetdb.jar:na]
      11. clojure.lang.Var.applyTo(Var.java:700)[puppetdb.jar:na]
      12. clojure.main.main(main.java:37)[puppetdb.jar:na]
      12 frames