add functionality for an automatic thread dump upon thread pool exhaustion

src/main/java/org/jboss/threads/EnhancedQueueExecutor.java

@@ -41,6 +41,7 @@
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
 import java.util.concurrent.atomic.AtomicLongFieldUpdater;
@@ -278,6 +279,11 @@ public final class EnhancedQueueExecutor extends AbstractExecutorService impleme
      */
     volatile Runnable terminationTask;
 
+    /**
+     * If the thread pool has reached its max and generated a thread dump.
+     */
+    private AtomicBoolean reachedMax = new AtomicBoolean();
+
     // =======================================================
     // Statistics fields and counters
     // =======================================================
@@ -1514,6 +1520,9 @@ int tryAllocateThread(final float growthResistance) {
             oldSize = currentSizeOf(oldStat);
             if (oldSize >= maxSizeOf(oldStat)) {
                 // max threads already reached
+                if (reachedMax.compareAndSet(false, true)) {
+                    ThreadDumpUtil.threadDump(maxSizeOf(oldStat));
+                }
                 return AT_NO;
             }
             if (oldSize >= coreSizeOf(oldStat) && oldSize > 0) {

src/main/java/org/jboss/threads/Messages.java

@@ -97,6 +97,25 @@ interface Messages extends BasicLogger {
     @LogMessage(level = Logger.Level.ERROR)
     void taskSubmitFailed(@Cause RejectedExecutionException e, Runnable task);
 
+    @Message(id = 15, value = "Thread pool has reached %d max threads in use. Performance may be impacted. Thread dump:\n"+
+            "*******************************************************************************\n" +
+            "%s\n===============================================================================\n" +
+            "End Thread dump\n*******************************************************************************\n")
+    @LogMessage(level = Logger.Level.WARN)
+    void exhaustedPoolDump(int maxPoolSize, String dump);
+
+    @LogMessage(level = Logger.Level.WARN)
+    @Message(id = 16, value = "Deadlock detected!\n"+
+            "*******************************************************************************\n" +
+            "{0}\n===============================================================================\n" +
+            "End Deadlock\n*******************************************************************************\n",
+            format = Message.Format.MESSAGE_FORMAT)
+    void deadLock(String dump);
+
+    @Message(id = 17, value = "Exception thrown during generation of thread dump")
+    @LogMessage(level = Logger.Level.WARN)
+    void threadDumpException(@Cause Exception cause);
+
     // validation
 
     @Message(id = 100, value = "Keep-alive may only be set to 0 for this executor type")

src/main/java/org/jboss/threads/ThreadDumpUtil.java

@@ -0,0 +1,127 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ *
+ * Copyright 2022 Red Hat, Inc., and individual contributors
+ * as indicated by the @author tags.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jboss.threads;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.lang.management.LockInfo;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MonitorInfo;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
+
+final class ThreadDumpUtil {
+
+   public static void threadDump(int maxPoolSize) {
+
+      try (
+         StringWriter dumpstr = new StringWriter();
+         PrintWriter dumpout = new PrintWriter(dumpstr);
+         StringWriter deadlockstr = new StringWriter();
+         PrintWriter deadlockout = new PrintWriter(deadlockstr);
+      ) {
+
+         ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
+
+         for (ThreadInfo threadInfo : threadMXBean.dumpAllThreads(true, true)) {
+            dumpout.println(threadInfoToString(threadInfo));
+         }
+
+         Messages.msg.exhaustedPoolDump(maxPoolSize, dumpstr.toString());
+
+         long[] deadlockedThreads = threadMXBean.findDeadlockedThreads();
+
+         if (deadlockedThreads != null && deadlockedThreads.length > 0) {
+            for (ThreadInfo threadInfo : threadMXBean.getThreadInfo(deadlockedThreads, true, true)) {
+               deadlockout.println(threadInfoToString(threadInfo));
+            }
+
+            Messages.msg.deadLock(deadlockstr.toString());
+         }
+
+      } catch (Exception e) {
+         Messages.msg.threadDumpException(e);
+      }
+   }
+
+   private static String threadInfoToString(ThreadInfo threadInfo) {
+      StringBuilder sb = new StringBuilder("\"" + threadInfo.getThreadName() + "\"" +
+            " Id=" + threadInfo.getThreadId() + " " +
+            threadInfo.getThreadState());
+      if (threadInfo.getLockName() != null) {
+         sb.append(" on " + threadInfo.getLockName());
+      }
+      if (threadInfo.getLockOwnerName() != null) {
+         sb.append(" owned by \"" + threadInfo.getLockOwnerName() +
+               "\" Id=" + threadInfo.getLockOwnerId());
+      }
+      if (threadInfo.isSuspended()) {
+         sb.append(" (suspended)");
+      }
+      if (threadInfo.isInNative()) {
+         sb.append(" (in native)");
+      }
+      sb.append('\n');
+      int i = 0;
+      for (; i < threadInfo.getStackTrace().length; i++) {
+         StackTraceElement ste = threadInfo.getStackTrace()[i];
+         sb.append("\tat " + ste.toString());
+         sb.append('\n');
+         if (i == 0 && threadInfo.getLockInfo() != null) {
+            Thread.State ts = threadInfo.getThreadState();
+            switch (ts) {
+               case BLOCKED:
+                  sb.append("\t-  blocked on " + threadInfo.getLockInfo());
+                  sb.append('\n');
+                  break;
+               case WAITING:
+                  sb.append("\t-  waiting on " + threadInfo.getLockInfo());
+                  sb.append('\n');
+                  break;
+               case TIMED_WAITING:
+                  sb.append("\t-  waiting on " + threadInfo.getLockInfo());
+                  sb.append('\n');
+                  break;
+               default:
+            }
+         }
+
+         for (MonitorInfo mi : threadInfo.getLockedMonitors()) {
+            if (mi.getLockedStackDepth() == i) {
+               sb.append("\t-  locked " + mi);
+               sb.append('\n');
+            }
+         }
+      }
+
+      LockInfo[] locks = threadInfo.getLockedSynchronizers();
+      if (locks.length > 0) {
+         sb.append("\n\tNumber of locked synchronizers = " + locks.length);
+         sb.append('\n');
+         for (LockInfo li : locks) {
+            sb.append("\t- " + li);
+            sb.append('\n');
+         }
+      }
+      sb.append('\n');
+      return sb.toString();
+   }
+
+}