KAFKA-19174 POC for a new :distribution submodule

build.gradle

@@ -39,7 +39,7 @@ plugins {
 
   id "com.github.spotbugs" version '6.2.3' apply false
   id 'org.scoverage' version '8.0.3' apply false
-  id 'com.gradleup.shadow' version '8.3.6' apply false
+  id 'com.gradleup.shadow' version '8.3.9' apply false
   id 'com.diffplug.spotless' version "6.25.0"
 }
 
@@ -330,6 +330,19 @@ subprojects {
     tasks.register('uploadArchives').configure { dependsOn(publish) }
   }
 
+  def  submodulesToIncludeInDistribution = ['tools', 'trogdor', 'shell', 'api', 'runtime', 'transforms',
+                                              'json', 'file', 'basic-auth-extension', 'mirror', 'mirror-client',
+                                              'streams', 'streams-scala', 'test-utils', 'examples', 'tools-api']
+
+  def shouldModuleResolveRuntimeClasspath = (submodulesToIncludeInDistribution.contains(project.name))
+
+    if (shouldModuleResolveRuntimeClasspath) {
+        configurations.create('resolvedRuntimeClasspath') {
+            extendsFrom(configurations.runtimeClasspath)
+            canBeResolved = true
+        }
+    }                                              
+
   // apply the eclipse plugin only to subprojects that hold code. 'connect' is just a folder.
   if (!project.name.equals('connect')) {
     apply plugin: 'eclipse'
@@ -1201,37 +1214,37 @@ project(':core') {
     from(configurations.releaseOnly) { into("libs/") }
     from(project.siteDocsTar) { into("site-docs/") }
     from(project(':tools').jar) { into("libs/") }
-    from(project(':tools').configurations.runtimeClasspath) { into("libs/") }
+    from(project(':tools').configurations.resolvedRuntimeClasspath) { into("libs/") }
     from(project(':trogdor').jar) { into("libs/") }
-    from(project(':trogdor').configurations.runtimeClasspath) { into("libs/") }
+    from(project(':trogdor').configurations.resolvedRuntimeClasspath) { into("libs/") }
     from(project(':shell').jar) { into("libs/") }
-    from(project(':shell').configurations.runtimeClasspath) { into("libs/") }
+    from(project(':shell').configurations.resolvedRuntimeClasspath) { into("libs/") }
     from(project(':connect:api').jar) { into("libs/") }
-    from(project(':connect:api').configurations.runtimeClasspath) { into("libs/") }
+    from(project(':connect:api').configurations.resolvedRuntimeClasspath) { into("libs/") }
     from(project(':connect:runtime').jar) { into("libs/") }
-    from(project(':connect:runtime').configurations.runtimeClasspath) { into("libs/") }
+    from(project(':connect:runtime').configurations.resolvedRuntimeClasspath) { into("libs/") }
     from(project(':connect:transforms').jar) { into("libs/") }
-    from(project(':connect:transforms').configurations.runtimeClasspath) { into("libs/") }
+    from(project(':connect:transforms').configurations.resolvedRuntimeClasspath) { into("libs/") }
     from(project(':connect:json').jar) { into("libs/") }
-    from(project(':connect:json').configurations.runtimeClasspath) { into("libs/") }
+    from(project(':connect:json').configurations.resolvedRuntimeClasspath) { into("libs/") }
     from(project(':connect:file').jar) { into("libs/") }
-    from(project(':connect:file').configurations.runtimeClasspath) { into("libs/") }
+    from(project(':connect:file').configurations.resolvedRuntimeClasspath) { into("libs/") }
     from(project(':connect:basic-auth-extension').jar) { into("libs/") }
-    from(project(':connect:basic-auth-extension').configurations.runtimeClasspath) { into("libs/") }
+    from(project(':connect:basic-auth-extension').configurations.resolvedRuntimeClasspath) { into("libs/") }
     from(project(':connect:mirror').jar) { into("libs/") }
-    from(project(':connect:mirror').configurations.runtimeClasspath) { into("libs/") }
+    from(project(':connect:mirror').configurations.resolvedRuntimeClasspath) { into("libs/") }
     from(project(':connect:mirror-client').jar) { into("libs/") }
-    from(project(':connect:mirror-client').configurations.runtimeClasspath) { into("libs/") }
+    from(project(':connect:mirror-client').configurations.resolvedRuntimeClasspath) { into("libs/") }
     from(project(':streams').jar) { into("libs/") }
-    from(project(':streams').configurations.runtimeClasspath) { into("libs/") }
+    from(project(':streams').configurations.resolvedRuntimeClasspath) { into("libs/") }
     from(project(':streams:streams-scala').jar) { into("libs/") }
-    from(project(':streams:streams-scala').configurations.runtimeClasspath) { into("libs/") }
+    from(project(':streams:streams-scala').configurations.resolvedRuntimeClasspath) { into("libs/") }
     from(project(':streams:test-utils').jar) { into("libs/") }
-    from(project(':streams:test-utils').configurations.runtimeClasspath) { into("libs/") }
+    from(project(':streams:test-utils').configurations.resolvedRuntimeClasspath) { into("libs/") }
     from(project(':streams:examples').jar) { into("libs/") }
-    from(project(':streams:examples').configurations.runtimeClasspath) { into("libs/") }
+    from(project(':streams:examples').configurations.resolvedRuntimeClasspath) { into("libs/") }
     from(project(':tools:tools-api').jar) { into("libs/") }
-    from(project(':tools:tools-api').configurations.runtimeClasspath) { into("libs/") }
+    from(project(':tools:tools-api').configurations.resolvedRuntimeClasspath) { into("libs/") }
     duplicatesStrategy 'exclude'
   }
 

gradle/dependencies.gradle

@@ -61,7 +61,7 @@ versions += [
   commonsLang: "3.18.0",
   commonsValidator: "1.9.0",
   classgraph: "4.8.179",
-  gradle: "8.14.3",
+  gradle: "9.1.0",
   grgit: "4.1.1",
   httpclient: "4.5.14",
   jackson: "2.19.0",

gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=bd71102213493060956ec229d946beee57158dbd89d0e62b91bca0fa2c5f3531
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip
+distributionSha256Sum=a17ddd85a26b6a7f5ddb71ff8b05fc5104c0202c6e64782429790c933686c806
+distributionUrl=https\://services.gradle.org/distributions/gradle-9.1.0-bin.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

gradlew

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 #
-# Copyright © 2015-2021 the original authors.
+# Copyright © 2015 the original authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+# SPDX-License-Identifier: Apache-2.0
+#
 
 ##############################################################################
 #
@@ -55,7 +57,7 @@
 #       Darwin, MinGW, and NonStop.
 #
 #   (3) This script is generated from the Groovy template
-#       https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+#       https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
 #       within the Gradle project.
 #
 #       You can find Gradle at https://github.com/gradle/gradle/.
@@ -84,7 +86,7 @@ done
 # shellcheck disable=SC2034
 APP_BASE_NAME=${0##*/}
 # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
-APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit
+APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD=maximum
@@ -113,20 +115,6 @@ case "$( uname )" in                #(
 esac
 
 
-# Loop in case we encounter an error.
-for attempt in 1 2 3; do
-  if [ ! -e "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" ]; then
-    if ! curl -s -S --retry 3 -L -o "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" "https://raw.githubusercontent.com/gradle/gradle/v8.14.3/gradle/wrapper/gradle-wrapper.jar"; then
-      rm -f "$APP_HOME/gradle/wrapper/gradle-wrapper.jar"
-      # Pause for a bit before looping in case the server throttled us.
-      sleep 5
-      continue
-    fi
-  fi
-done
-
-CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
-
 
 # Determine the Java command to use to start the JVM.
 if [ -n "$JAVA_HOME" ] ; then
@@ -183,7 +171,6 @@ fi
 # For Cygwin or MSYS, switch paths to Windows format before running java
 if "$cygwin" || "$msys" ; then
     APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
-    CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" )
 
     JAVACMD=$( cygpath --unix "$JAVACMD" )
 
@@ -212,19 +199,31 @@ if "$cygwin" || "$msys" ; then
 fi
 
 
+
+# Loop in case we encounter an error.
+for attempt in 1 2 3; do
+  if [ ! -e "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" ]; then
+    if ! curl -s -S --retry 3 -L -o "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" "https://raw.githubusercontent.com/gradle/gradle/v9.1.0/gradle/wrapper/gradle-wrapper.jar"; then
+      rm -f "$APP_HOME/gradle/wrapper/gradle-wrapper.jar"
+      # Pause for a bit before looping in case the server throttled us.
+      sleep 5
+      continue
+    fi
+  fi
+done
+
 # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
 DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
 
 # Collect all arguments for the java command:
-#   * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
+#   * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
 #     and any embedded shellness will be escaped.
 #   * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
 #     treated as '${Hostname}' itself on the command line.
 
 set -- \
         "-Dorg.gradle.appname=$APP_BASE_NAME" \
-        -classpath "$CLASSPATH" \
-        org.gradle.wrapper.GradleWrapperMain \
+        -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \
         "$@"
 
 # Stop when "xargs" is not available.

wrapper.gradle

@@ -59,13 +59,15 @@ task bootstrapWrapper() {
       done
       """.stripIndent()
 
+        String putBootstrapStringAbove = "# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script."
+
         def wrapperScript = wrapper.scriptFile
         def wrapperLines = wrapperScript.readLines()
         wrapperScript.withPrintWriter { out ->
             def bootstrapWritten = false
             wrapperLines.each { line ->
                 // Print the wrapper bootstrap before the first usage of the wrapper jar.
-                if (!bootstrapWritten && line.contains("gradle-wrapper.jar")) {
+                if (!bootstrapWritten && line.contains(putBootstrapStringAbove)) {
                     out.println(bootstrapString)
                     bootstrapWritten = true
                 }