Create connections concurrently in separate threads

README.md

@@ -15,7 +15,7 @@ Fast, simple, reliable.  HikariCP is a "zero-overhead" production ready JDBC con
 ----------------------------------------------------
 
 > [!IMPORTANT]
-> In order to avoid a rare condition where the pool goes to zero and does not recover it is necessary to configure *TCP keepalive*. Some JDBC drivers support this via properties, for example ``tcpKeepAlive=true`` on PostgreSQL, but in any case it can also be configured at the OS-level. See [Setting OS TCP Keepalive](https://github.com/brettwooldridge/HikariCP/wiki/Setting-OS-TCP-Keepalive) and/or [TCP keepalive for a better PostgreSQL experience](https://www.cybertec-postgresql.com/en/tcp-keepalive-for-a-better-postgresql-experience/#setting-tcp-keepalive-parameters-on-the-operating-system).
+> In order to avoid a rare condition where the pool goes to zero and does not recover it is necessary to configure *TCP keepalive*. Some JDBC drivers support this via properties, for example ``tcpKeepAlive=true`` on PostgreSQL, but in any case it can also be configured at the OS-level. See [Setting OS TCP Keepalive](https://github.com/brettwooldridge/HikariCP/wiki/Setting-Driver-or-OS-TCP-Keepalive) and/or [TCP keepalive for a better PostgreSQL experience](https://www.cybertec-postgresql.com/en/tcp-keepalive-for-a-better-postgresql-experience/#setting-tcp-keepalive-parameters-on-the-operating-system).
 
 ----------------------------------------------------
 
@@ -38,12 +38,12 @@ Fast, simple, reliable.  HikariCP is a "zero-overhead" production ready JDBC con
 
 ### Artifacts
 
-_**Java 11+** maven artifact:_
+_**Java 11 or greater** maven artifact:_
 ```xml
 <dependency>
    <groupId>com.zaxxer</groupId>
    <artifactId>HikariCP</artifactId>
-   <version>6.2.1</version>
+   <version>6.3.0</version>
 </dependency>
 ```
 _Java 8 maven artifact (*deprecated*):_

src/main/java/com/zaxxer/hikari/pool/HikariPool.java

@@ -38,6 +38,7 @@
 import java.sql.SQLTransientConnectionException;
 import java.util.Optional;
 import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
 
 import static com.zaxxer.hikari.util.ClockSource.*;
@@ -73,6 +74,8 @@ public final class HikariPool extends PoolBase implements HikariPoolMXBean, IBag
 
    private final PoolEntryCreator poolEntryCreator = new PoolEntryCreator();
    private final PoolEntryCreator postFillPoolEntryCreator = new PoolEntryCreator("After adding ");
+
+   private final AtomicInteger connectionsInProgress = new AtomicInteger(0);
    private final ThreadPoolExecutor addConnectionExecutor;
    private final ThreadPoolExecutor closeConnectionExecutor;
 
@@ -114,24 +117,24 @@ public HikariPool(final HikariConfig config)
       ThreadFactory threadFactory = config.getThreadFactory();
 
       final int maxPoolSize = config.getMaximumPoolSize();
-      this.addConnectionExecutor = createThreadPoolExecutor(maxPoolSize, poolName + ":connection-adder", threadFactory, new CustomDiscardPolicy());
+      this.addConnectionExecutor = createCreatorThreadPoolExecutor(
+         Math.min(Runtime.getRuntime().availableProcessors() * 2, config.getMinimumIdle()),
+         Math.max(Runtime.getRuntime().availableProcessors() * 2, config.getMinimumIdle()),
+         config.getMaximumPoolSize(),
+         ":connection-adder",
+         threadFactory,
+         new CustomDiscardPolicy());
       this.closeConnectionExecutor = createThreadPoolExecutor(maxPoolSize, poolName + ":connection-closer", threadFactory, new ThreadPoolExecutor.CallerRunsPolicy());
 
       this.leakTaskFactory = new ProxyLeakTaskFactory(config.getLeakDetectionThreshold(), houseKeepingExecutorService);
 
       this.houseKeeperTask = houseKeepingExecutorService.scheduleWithFixedDelay(new HouseKeeper(), 100L, housekeepingPeriodMs, MILLISECONDS);
 
       if (Boolean.getBoolean("com.zaxxer.hikari.blockUntilFilled") && config.getInitializationFailTimeout() > 1) {
-         addConnectionExecutor.setMaximumPoolSize(Math.min(16, Runtime.getRuntime().availableProcessors()));
-         addConnectionExecutor.setCorePoolSize(Math.min(16, Runtime.getRuntime().availableProcessors()));
-
          final long startTime = currentTime();
          while (elapsedMillis(startTime) < config.getInitializationFailTimeout() && getTotalConnections() < config.getMinimumIdle()) {
             quietlySleep(MILLISECONDS.toMillis(100));
          }
-
-         addConnectionExecutor.setCorePoolSize(1);
-         addConnectionExecutor.setMaximumPoolSize(1);
       }
    }
 
@@ -335,7 +338,7 @@ public void setHealthCheckRegistry(Object healthCheckRegistry)
    @Override
    public void addBagItem(final int waiting)
    {
-      if (waiting > addConnectionExecutor.getQueue().size())
+      if (waiting > connectionsInProgress.get())
          addConnectionExecutor.submit(poolEntryCreator);
    }
 
@@ -520,10 +523,11 @@ private PoolEntry createPoolEntry()
    private synchronized void fillPool(final boolean isAfterAdd)
    {
       final var idle = getIdleConnections();
-      final var shouldAdd = getTotalConnections() < config.getMaximumPoolSize() && idle < config.getMinimumIdle();
+      final var connectionsInProgress = this.connectionsInProgress.get();
+      final var shouldAdd = getTotalConnections() < config.getMaximumPoolSize() - connectionsInProgress && idle < config.getMinimumIdle();
 
       if (shouldAdd) {
-         final var countToAdd = config.getMinimumIdle() - idle;
+         final var countToAdd = config.getMinimumIdle() - idle - connectionsInProgress;
          for (int i = 0; i < countToAdd; i++)
             addConnectionExecutor.submit(isAfterAdd ? postFillPoolEntryCreator : poolEntryCreator);
       }
@@ -740,26 +744,26 @@ private final class PoolEntryCreator implements Callable<Boolean>
       @Override
       public Boolean call()
       {
-         var backoffMs = 10L;
+         long jitterMs = ThreadLocalRandom.current().nextLong(Math.max(10, config.getConnectionTimeout() / 10));
+         if (connectionsInProgress.get() >= Runtime.getRuntime().availableProcessors()) {
+            quietlySleep(jitterMs);
+         }
          var added = false;
          try {
-            while (shouldContinueCreating()) {
+            if (!Thread.interrupted() && shouldContinueCreating()) {
                final var poolEntry = createPoolEntry();
                if (poolEntry != null) {
                   added = true;
                   connectionBag.add(poolEntry);
                   logger.debug("{} - Added connection {}", poolName, poolEntry.connection);
-                  quietlySleep(30L);
-                  break;
                } else {  // failed to get connection from db, sleep and retry
-                  if (loggingPrefix != null && backoffMs % 50 == 0)
-                     logger.debug("{} - Connection add failed, sleeping with backoff: {}ms", poolName, backoffMs);
-                  quietlySleep(backoffMs);
-                  backoffMs = Math.min(SECONDS.toMillis(5), backoffMs * 2);
+                  if (loggingPrefix != null)
+                     logger.debug("{} - Connection add failed", poolName);
                }
             }
          }
          finally {
+            connectionsInProgress.decrementAndGet();
             if (added && loggingPrefix != null)
                logPoolState(loggingPrefix);
             else if (!added)
@@ -777,8 +781,10 @@ else if (!added)
        * @return true if we should create a connection, false if the need has disappeared
        */
       private synchronized boolean shouldContinueCreating() {
-         return poolState == POOL_NORMAL && getTotalConnections() < config.getMaximumPoolSize() &&
-            (getIdleConnections() < config.getMinimumIdle() || connectionBag.getWaitingThreadCount() > getIdleConnections());
+         final int idleConnections = getIdleConnections();
+         final int connectionsInProgress = HikariPool.this.connectionsInProgress.incrementAndGet();
+         return poolState == POOL_NORMAL && getTotalConnections() + connectionsInProgress < config.getMaximumPoolSize() &&
+            (idleConnections < config.getMinimumIdle() || connectionBag.getWaitingThreadCount() > idleConnections + connectionsInProgress);
       }
    }
 

src/main/java/com/zaxxer/hikari/pool/PoolBase.java

@@ -475,7 +475,7 @@ private void checkValidationSupport(final Connection connection) throws SQLExcep
    {
       try {
          if (isUseJdbc4Validation) {
-            connection.isValid(1);
+            connection.isValid(Math.max(1, (int) MILLISECONDS.toSeconds(validationTimeout)));
          }
          else {
             executeSql(connection, config.getConnectionTestQuery(), false);

src/main/java/com/zaxxer/hikari/util/DriverDataSource.java

@@ -45,9 +45,7 @@ public DriverDataSource(String jdbcUrl, String driverClassName, Properties prope
       this.jdbcUrl = jdbcUrl;
       this.driverProperties = new Properties();
 
-      for (var entry : properties.entrySet()) {
-         driverProperties.setProperty(entry.getKey().toString(), entry.getValue().toString());
-      }
+      driverProperties.putAll(properties);
 
       if (username != null) {
          driverProperties.put(USER, driverProperties.getProperty(USER, username));

src/main/java/com/zaxxer/hikari/util/UtilityElf.java

@@ -18,6 +18,7 @@
 
 import java.util.Locale;
 import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.regex.Pattern;
 
 import static java.lang.Thread.currentThread;
@@ -131,6 +132,11 @@ public static ThreadPoolExecutor createThreadPoolExecutor(final int queueSize, f
       return createThreadPoolExecutor(new LinkedBlockingQueue<>(queueSize), threadName, threadFactory, policy);
    }
 
+   public static ThreadPoolExecutor createCreatorThreadPoolExecutor(final int corePoolSize, final int maxPoolSize, final int queueSize, final String threadName, ThreadFactory threadFactory, final RejectedExecutionHandler policy)
+   {
+      return createCreatorThreadPoolExecutor(corePoolSize, maxPoolSize, new LinkedBlockingQueue<>(queueSize), threadName, threadFactory, policy);
+   }
+
    /**
     * Create a ThreadPoolExecutor.
     *
@@ -151,6 +157,18 @@ public static ThreadPoolExecutor createThreadPoolExecutor(final BlockingQueue<Ru
       return executor;
    }
 
+   public static ThreadPoolExecutor createCreatorThreadPoolExecutor(
+      int corePoolSize, int maxPoolSize, final BlockingQueue<Runnable> queue, final String threadName, ThreadFactory threadFactory, final RejectedExecutionHandler policy)
+   {
+      if (threadFactory == null) {
+         threadFactory = new DefaultThreadFactory(threadName);
+      }
+
+      var executor = new ThreadPoolExecutor(corePoolSize /*core*/, maxPoolSize /*max*/, 5 /*keepalive*/, SECONDS, queue, threadFactory, policy);
+      executor.allowCoreThreadTimeOut(true);
+      return executor;
+   }
+
    // ***********************************************************************
    //                       Misc. public methods
    // ***********************************************************************
@@ -198,6 +216,7 @@ public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
 
    public static final class DefaultThreadFactory implements ThreadFactory
    {
+      private static final AtomicInteger counter = new AtomicInteger(0);
       private final String threadName;
       private final boolean daemon;
 
@@ -209,7 +228,7 @@ public DefaultThreadFactory(String threadName) {
       @Override
       @SuppressWarnings("NullableProblems")
       public Thread newThread(Runnable r) {
-         var thread = new Thread(r, threadName);
+         var thread = new Thread(r, threadName + "_" + counter.incrementAndGet());
          thread.setDaemon(daemon);
          return thread;
       }

src/test/java/com/zaxxer/hikari/util/DriverDataSourceTest.java

@@ -18,6 +18,8 @@
 
 import org.junit.Test;
 
+import java.lang.reflect.Field;
+import java.time.Duration;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Properties;
@@ -26,6 +28,18 @@
 
 public class DriverDataSourceTest {
 
+   @Test
+   public void testDriverProperties() throws Exception {
+      Properties properties = new Properties();
+      Duration timeout = Duration.ofSeconds(60);
+      properties.put("timeout", timeout);
+      var driverDataSource = new DriverDataSource("jdbc:h2:mem:test;DB_CLOSE_DELAY=-1", null, properties, "", "");
+      Field field = DriverDataSource.class.getDeclaredField("driverProperties");
+      field.setAccessible(true);
+      Properties driverProperties = (Properties) field.get(driverDataSource);
+      assertEquals(timeout, driverProperties.get("timeout"));
+   }
+
    @Test
    public void testJdbcUrlLogging() {
       List<String> urls = Arrays.asList(