deploy: Inject Jazzer version via a flag

This removes the need to have a "version bump" commit before every release.
CodeIntelligenceTesting · Sep 20, 2023 · e5ac795 · e5ac795
1 parent 9b58301
commit e5ac795
Show file tree

Hide file tree

Showing 12 changed files with 181 additions and 54 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -80,6 +80,7 @@ build:ci --noslim_profile
 build:ci --experimental_profile_include_target_label
 build:ci --experimental_profile_include_primary_output
 build:ci --nolegacy_important_outputs
+common:ci --announce_rc
 
 # Docker images
 common:docker --config=toolchain

diff --git a/.github/workflows/prerelease.yaml b/.github/workflows/prerelease.yaml
@@ -2,6 +2,9 @@ name: Pre-Release
 
 on:
   workflow_dispatch:
+  push:
+    tags:
+      - 'v*'
 
 jobs:
   build_release:
@@ -25,14 +28,13 @@ jobs:
           distribution: zulu
           java-version: 8
 
-      - name: Set Build Buddy config
-        shell: bash
-        run: .github/scripts/echoBuildBuddyConfig.sh ${{ secrets.BUILDBUDDY_API_KEY }} >> $GITHUB_ENV
-
       - name: Append build settings to .bazelrc
         shell: bash
         run: |
-          echo "build --announce_rc" >> .bazelrc
+          TAG=${{ github.ref_name }}
+          echo "build --remote_header=x-buildbuddy-api-key=${{ secrets.BUILDBUDDY_API_KEY }}" >> .bazelrc
+          echo "build --config=ci" >> .bazelrc
+          echo "build --//deploy:jazzer_version=${TAG#v}" >> .bazelrc
           echo "build:linux --config=toolchain" >> .bazelrc
           echo "build:linux --extra_toolchains=@llvm_toolchain//:cc-toolchain-x86_64-linux" >> .bazelrc
 
@@ -86,7 +88,9 @@ jobs:
           if-no-files-found: error
 
   maven_predeploy:
-    runs-on: ubuntu-latest
+    # TODO: Our hermetic toolchain doesn't support ubuntu-latest yet and fails with:
+    # external/llvm_toolchain_llvm/bin/clang: error while loading shared libraries: libtinfo.so.5: cannot open shared object file: No such file or directory
+    runs-on: ubuntu-20.04
     needs: merge_jars
 
     environment:
@@ -95,6 +99,22 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - name: Set up JDK
+        uses: actions/setup-java@v3
+        with:
+          distribution: zulu
+          java-version: 8
+
+      - name: Append build settings to .bazelrc
+        shell: bash
+        run: |
+          TAG=${{ github.ref_name }}
+          echo "build --remote_header=x-buildbuddy-api-key=${{ secrets.BUILDBUDDY_API_KEY }}" >> .bazelrc
+          echo "build --config=ci" >> .bazelrc
+          echo "build --//deploy:jazzer_version=${TAG#v}" >> .bazelrc
+          echo "build:linux --config=toolchain" >> .bazelrc
+          echo "build:linux --extra_toolchains=@llvm_toolchain//:cc-toolchain-x86_64-linux" >> .bazelrc
+
       - name: Download merged jar
         uses: actions/download-artifact@v3
         with:
@@ -126,18 +146,9 @@ jobs:
           name: jazzer_releases
           path: _releases/
 
-      - name: read version
-        id: read-version
-        run: |
-          echo ::set-output name=version::\
-          $(sed -nr 's/JAZZER_VERSION = "(.*)"/\1/p' maven.bzl)
-        shell: bash
-
       - name: create release
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@c9b46fe7aad9f02afd89b12450b780f52dacfb2d
         with:
-          name: v${{ steps.read-version.outputs.version }}
-          tag_name: v${{ steps.read-version.outputs.version }}
           generate_release_notes: true
           draft: true
           files: |

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -70,11 +70,8 @@ Run `./format.sh` to format all source files in the way enforced by the "Check f
 
 ## Releasing (CI employees only)
 
-1. Update `JAZZER_VERSION` in [`maven.bzl`](maven.bzl).
-2. Trigger the "Prerelease" GitHub Actions workflow for the branch where you want to do the release:
-    * `main` for major and minor releases
-    * An appropriate release branch for patch releases
-3. Wait for the workflow to finish (about 10 minutes)
-4. When successful and happy with the results, log into https://oss.sonatype.org, select both staging repositories and "Close" them.
+1. Push a tag of the form `v1.2.3` to trigger the "Prerelease" GitHub Actions workflow.
+2. Wait for the workflow to finish (about 10 minutes)
+3. When successful and happy with the results, log into https://oss.sonatype.org, select all three staging repositories (under "Repositories" -> "Staging" -> `com/code_intelligence`) and "Close" them.
    Wait and refresh, then select them again and "Release" them.
-5. Release the draft Github release. This will automatically create a tag, push the docker images and deploy the docs (can take about a few minutes to appear at [jazzer-docs]( https://codeintelligencetesting.github.io/jazzer-docs)).
+4. Release the draft Github release. This will automatically create a tag, push the docker images and deploy the docs (can take about a few minutes to appear at [jazzer-docs]( https://codeintelligencetesting.github.io/jazzer-docs)).
diff --git a/deploy/BUILD.bazel b/deploy/BUILD.bazel
@@ -1,26 +1,39 @@
+load("@bazel_skylib//rules:common_settings.bzl", "string_flag")
 load("@rules_jvm_external//:defs.bzl", "java_export")
-load("//:maven.bzl", "JAZZER_API_COORDINATES", "JAZZER_COORDINATES", "JAZZER_JUNIT_COORDINATES")
 load("//bazel:compat.bzl", "SKIP_ON_WINDOWS")
 
+JAZZER_COORDINATES = "com.code-intelligence:jazzer:$(JAZZER_VERSION)"
+
 sh_binary(
     name = "deploy",
     srcs = ["deploy.sh"],
     args = [JAZZER_COORDINATES],
     data = ["@bazel_tools//tools/jdk:current_host_java_runtime"],
     env = {"JAVA_EXECPATH": "$(JAVA)"},
-    toolchains = ["@bazel_tools//tools/jdk:current_host_java_runtime"],
+    toolchains = [
+        "@bazel_tools//tools/jdk:current_host_java_runtime",
+        ":jazzer_version",
+    ],
     deps = ["@bazel_tools//tools/bash/runfiles"],
 )
 
+string_flag(
+    name = "jazzer_version",
+    build_setting_default = "0.0.0-dev",
+    make_variable = "JAZZER_VERSION",
+    visibility = ["//visibility:public"],
+)
+
 java_export(
     name = "jazzer-api",
     doc_url = "https://codeintelligencetesting.github.io/jazzer-docs/jazzer-api/",
     javadocopts = [
         "-link",
         "https://docs.oracle.com/en/java/javase/17/docs/api/",
     ],
-    maven_coordinates = JAZZER_API_COORDINATES,
+    maven_coordinates = "com.code-intelligence:jazzer-api:$(JAZZER_VERSION)",
     pom_template = "//deploy:jazzer-api.pom",
+    toolchains = [":jazzer_version"],
     visibility = ["//visibility:public"],
     runtime_deps = ["//src/main/java/com/code_intelligence/jazzer/api"],
 )
@@ -32,6 +45,7 @@ java_export(
     # Do not generate an implicit javadocs target - the current target is based on the shaded deploy
     # JAR, for which the docs JAR generated by default would be empty.
     tags = ["no-javadocs"],
+    toolchains = [":jazzer_version"],
     visibility = ["//visibility:public"],
     runtime_deps = [
         "//src/main/java/com/code_intelligence/jazzer:jazzer_import",
@@ -64,8 +78,9 @@ java_export(
         "-link",
         "https://junit.org/junit5/docs/current/api/",
     ],
-    maven_coordinates = JAZZER_JUNIT_COORDINATES,
+    maven_coordinates = "com.code-intelligence:jazzer-junit:$(JAZZER_VERSION)",
     pom_template = "jazzer-junit.pom",
+    toolchains = [":jazzer_version"],
     visibility = ["//visibility:public"],
     runtime_deps = [
         # These deps' only effect is to include a dependency on the 'jazzer' and 'jazzer-api' Maven artifacts in the

diff --git a/deploy/deploy.sh b/deploy/deploy.sh
@@ -22,11 +22,12 @@ fail() {
 
 cd "$BUILD_WORKSPACE_DIRECTORY" || fail "BUILD_WORKSPACE_DIRECTORY not found"
 
+JAZZER_COORDINATES=$1
+[[ "$JAZZER_COORDINATES" != *-dev ]] || fail "--//deploy:jazzer_version must be set to a release version, got: $JAZZER_COORDINATES"
+
 echo "$RELEASE_SIGNING_KEY_PRIVATE" | gpg --import
 echo "default-key $RELEASE_SIGNING_KEY_ID" > $HOME/.gnupg/gpg.conf
 
-JAZZER_COORDINATES=$1
-
 [ -z "${MAVEN_USER+x}" ] && \
   fail "Set MAVEN_USER to the Sonatype OSSRH user"
 [ -z "${MAVEN_PASSWORD+x}" ] && \

diff --git a/deploy/jazzer_version_test.sh b/deploy/jazzer_version_test.sh
@@ -32,4 +32,4 @@ jazzer=$(rlocation "$JAZZER_RLOCATIONPATH")
 [ -f "$jazzer" ] || exit 1
 jazzer_version_output=$("$java" -jar "$jazzer" --version 2>&1)
 echo "$jazzer_version_output"
-echo "$jazzer_version_output" | tr -d '\n' | grep -q '^Jazzer v[0-9.]*$' || exit 1
+echo "$jazzer_version_output" | tr -d '\n' | grep -q '^Jazzer v0.0.0-dev$' || exit 1
diff --git a/examples/junit-spring-web/build-and-run-tests.sh b/examples/junit-spring-web/build-and-run-tests.sh
@@ -21,11 +21,6 @@ set -e
  bazel build //deploy:all
 )
 
-# Update jazzer version used for building this project in the pom.xml
-JAZZER_VERSION=$(grep -oP '(?<=JAZZER_VERSION = ")[^"]*' ../../maven.bzl)
-# Find line with "<artifactId>jazzer-junit</artifactId>" and replace the version in the next line
-sed -i "/<artifactId>jazzer-junit<\/artifactId>/ {n;s/<version>.*<\/version>/<version>$JAZZER_VERSION<\/version>/}" pom.xml
-
 # Add locally-built Jazzer to the Maven repository
 ./mvnw install:install-file -Dfile=../../bazel-bin/deploy/jazzer-junit-project.jar -DpomFile=../../bazel-bin/deploy/jazzer-junit-pom.xml
 ./mvnw install:install-file -Dfile=../../bazel-bin/deploy/jazzer-project.jar       -DpomFile=../../bazel-bin/deploy/jazzer-pom.xml

diff --git a/examples/junit-spring-web/pom.xml b/examples/junit-spring-web/pom.xml
@@ -50,7 +50,7 @@
 		<dependency>
 			<groupId>com.code-intelligence</groupId>
 			<artifactId>jazzer-junit</artifactId>
-			<version>0.16.1</version>
+			<version>0.0.0-dev</version>
 			<scope>test</scope>
 		</dependency>
 		<dependency>

diff --git a/maven.bzl b/maven.bzl
@@ -14,11 +14,6 @@
 
 load("@rules_jvm_external//:specs.bzl", "maven")
 
-JAZZER_VERSION = "0.21.0"
-JAZZER_COORDINATES = "com.code-intelligence:jazzer:%s" % JAZZER_VERSION
-JAZZER_API_COORDINATES = "com.code-intelligence:jazzer-api:%s" % JAZZER_VERSION
-JAZZER_JUNIT_COORDINATES = "com.code-intelligence:jazzer-junit:%s" % JAZZER_VERSION
-
 # keep sorted
 MAVEN_ARTIFACTS = [
     "org.junit.jupiter:junit-jupiter-api:5.9.0",

diff --git a/repositories.bzl b/repositories.bzl
@@ -32,10 +32,11 @@ def jazzer_dependencies(android = False):
     maybe(
         http_archive,
         name = "bazel_skylib",
-        sha256 = "b8a1527901774180afc798aeb28c4634bdccf19c4d98e7bdd1ce79d1fe9aaad7",
+        sha256 = "ade20530fd2d39abb49d537e77d4d873a823649b6061e0bb0c369b34450909d6",
+        strip_prefix = "bazel-skylib-8386b9d32bf69dd2d2f92d9ca39582cf6dabeb37",
+        # TODO: Return to the next release.
         urls = [
-            "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.4.1/bazel-skylib-1.4.1.tar.gz",
-            "https://github.com/bazelbuild/bazel-skylib/releases/download/1.4.1/bazel-skylib-1.4.1.tar.gz",
+            "https://github.com/bazelbuild/bazel-skylib/archive/8386b9d32bf69dd2d2f92d9ca39582cf6dabeb37.tar.gz",
         ],
     )
 
@@ -63,6 +64,10 @@ def jazzer_dependencies(android = False):
             # https://github.com/bazelbuild/rules_jvm_external/pull/958
             # Allows javadoc targets to reference other javadoc targets.
             "//third_party:rules_jvm_external-javadoc-deps.patch",
+            # https://github.com/bazelbuild/rules_jvm_external/pull/960
+            # Forwards the toolchains attribute on java_export to all underlying targets and
+            # evaluates Make variables from deps in pom_file.
+            "//third_party:rules_jvm_external-toolchains-attribute.patch",
         ],
         sha256 = "aa17db9b810b22e411bf722095be34eeb66c76819b9c3423ad7740f452016aa3",
         strip_prefix = "rules_jvm_external-4b073de468eff9741406f475acb04e94bee7c9d0",

diff --git a/src/main/java/com/code_intelligence/jazzer/BUILD.bazel b/src/main/java/com/code_intelligence/jazzer/BUILD.bazel
@@ -1,7 +1,5 @@
-load("@bazel_skylib//rules:write_file.bzl", "write_file")
 load("@com_github_johnynek_bazel_jar_jar//:jar_jar.bzl", "jar_jar")
 load("@rules_jvm_external//:defs.bzl", "javadoc")
-load("//:maven.bzl", "JAZZER_VERSION")
 load("//bazel:jar.bzl", "strip_jar")
 load("//sanitizers:sanitizers.bzl", "SANITIZER_CLASSES")
 
@@ -133,13 +131,16 @@ java_library(
     ],
 )
 
-write_file(
+genrule(
     name = "constants_java",
-    out = "Constants.java",
-    content = [
-        "package com.code_intelligence.jazzer;",
-        "public final class Constants {",
-        "  public static final String JAZZER_VERSION = \"%s\";" % JAZZER_VERSION,
-        "}",
-    ],
+    outs = ["Constants.java"],
+    cmd = """
+cat > $@ <<'EOF'
+package com.code_intelligence.jazzer;
+public final class Constants {
+  public static final String JAZZER_VERSION = \"$(JAZZER_VERSION)\";
+}
+EOF
+""",
+    toolchains = ["//deploy:jazzer_version"],
 )
diff --git a/third_party/rules_jvm_external-toolchains-attribute.patch b/third_party/rules_jvm_external-toolchains-attribute.patch
@@ -0,0 +1,106 @@
+From 6076dba7ccfe859d65adf8926339e8f31d4fb351 Mon Sep 17 00:00:00 2001
+From: Fabian Meumertzheim <[email protected]>
+Date: Wed, 20 Sep 2023 13:40:24 +0200
+Subject: [PATCH 2/2] java_export: Fix Make variable substitution
+
+This requires forwarding `toolchains` to all rules as well as evaluating
+Make variables for dependency coordinates in `pom_file`.
+---
+ private/rules/java_export.bzl | 10 +++++++++-
+ private/rules/pom_file.bzl    |  6 +++++-
+ 2 files changed, 14 insertions(+), 2 deletions(-)
+
+diff --git a/private/rules/java_export.bzl b/private/rules/java_export.bzl
+index 81138db1..51618e82 100644
+--- a/private/rules/java_export.bzl
++++ b/private/rules/java_export.bzl
+@@ -84,6 +84,7 @@ def java_export(
+     javadocopts = kwargs.pop("javadocopts", [])
+     doc_deps = kwargs.pop("doc_deps", [])
+     doc_url = kwargs.pop("doc_url", "")
++    toolchains = kwargs.pop("toolchains", [])
+
+     # Construct the java_library we'll export from here.
+     native.java_library(
+@@ -108,6 +109,7 @@ def java_export(
+         classifier_artifacts = classifier_artifacts,
+         doc_deps = doc_deps,
+         doc_url = doc_url,
++        toolchains = toolchains,
+     )
+
+ def maven_export(
+@@ -124,7 +126,8 @@ def maven_export(
+         classifier_artifacts = {},
+         *,
+         doc_deps = [],
+-        doc_url = ""):
++        doc_url = "",
++        toolchains = None):
+     """
+     All arguments are the same as java_export with the addition of:
+       lib_name: Name of the library that has been built.
+@@ -210,6 +213,7 @@ def maven_export(
+         visibility = visibility,
+         tags = tags + maven_coordinates_tags,
+         testonly = testonly,
++        toolchains = toolchains,
+     )
+
+     native.filegroup(
+@@ -251,6 +255,7 @@ def maven_export(
+             visibility = visibility,
+             tags = tags,
+             testonly = testonly,
++            toolchains = toolchains,
+         )
+         classifier_artifacts.setdefault("javadoc", docs_jar)
+
+@@ -262,6 +267,7 @@ def maven_export(
+         visibility = visibility,
+         tags = tags,
+         testonly = testonly,
++        toolchains = toolchains,
+     )
+
+     maven_publish(
+@@ -273,6 +279,7 @@ def maven_export(
+         visibility = visibility,
+         tags = tags,
+         testonly = testonly,
++        toolchains = toolchains,
+     )
+
+     # We may want to aggregate several `java_export` targets into a single Maven BOM POM
+@@ -287,6 +294,7 @@ def maven_export(
+         testonly = testonly,
+         tags = tags,
+         visibility = visibility,
++        toolchains = toolchains,
+     )
+
+     # Finally, alias the primary output
+diff --git a/private/rules/pom_file.bzl b/private/rules/pom_file.bzl
+index eba5306e..2f8b74b0 100644
+--- a/private/rules/pom_file.bzl
++++ b/private/rules/pom_file.bzl
+@@ -15,6 +15,10 @@ def _pom_file_impl(ctx):
+     for dep in additional_deps:
+         for coords in dep[MavenInfo].as_maven_dep.to_list():
+             all_maven_deps.append(coords)
++    expanded_maven_deps = [
++        ctx.expand_make_variables("additional_deps", coords, ctx.var)
++        for coords in all_maven_deps
++    ]
+
+     # Expand maven coordinates for any variables to be replaced.
+     coordinates = ctx.expand_make_variables("coordinates", info.coordinates, ctx.var)
+@@ -22,7 +26,7 @@ def _pom_file_impl(ctx):
+     out = generate_pom(
+         ctx,
+         coordinates = coordinates,
+-        versioned_dep_coordinates = sorted(all_maven_deps),
++        versioned_dep_coordinates = sorted(expanded_maven_deps),
+         pom_template = ctx.file.pom_template,
+         out_name = "%s.xml" % ctx.label.name,
+     )