Automated fix for refs/heads/bazelified_artifacts

tools/bazelify_tests/BUILD

@@ -27,6 +27,8 @@ exports_files([
     "grpc_run_bazel_distribtest_test.sh",
     "grpc_run_cpp_distribtest_test.sh",
     "grpc_run_simple_command_test.sh",
+    "grpc_build_artifact_task.sh",
+    "grpc_build_artifact_task_build_test.sh",
 ])
 
 genrule(

tools/bazelify_tests/build_defs.bzl

@@ -63,6 +63,48 @@ def _dockerized_sh_test(name, srcs = [], args = [], data = [], size = "medium",
         **test_args
     )
 
+def _dockerized_genrule(name, cmd, outs, srcs = [], timeout = None, tags = [], exec_compatible_with = [], flaky = None, docker_image_version = None, docker_run_as_root = False):
+    """Runs genrule under docker either via RBE or via docker sandbox."""
+    if docker_image_version:
+        image_spec = DOCKERIMAGE_CURRENT_VERSIONS.get(docker_image_version, None)
+        if not image_spec:
+            fail("Version info for docker image '%s' not found in dockerimage_current_versions.bzl" % docker_image_version)
+    else:
+        fail("docker_image_version attribute not set for dockerized test '%s'" % name)
+
+    exec_properties = create_rbe_exec_properties_dict(
+        labels = {
+            "workload": "misc",
+            "machine_size": "misc_large",
+        },
+        docker_network = "standard",
+        container_image = image_spec,
+        # TODO(jtattermusch): note that docker sandbox doesn't currently support "docker_run_as_root"
+        docker_run_as_root = docker_run_as_root,
+    )
+
+    # since the tests require special bazel args, only run them when explicitly requested
+    tags = ["manual"] + tags
+
+    # TODO(jtattermusch): find a way to ensure that action can only run under docker sandbox or remotely
+    # to avoid running it outside of a docker container by accident.
+
+    genrule_args = {
+        "name": name,
+        "cmd": cmd,
+        "srcs": srcs,
+        "tags": tags,
+        "flaky": flaky,
+        "timeout": timeout,
+        "exec_compatible_with": exec_compatible_with,
+        "exec_properties": exec_properties,
+        "outs": outs,
+    }
+
+    native.genrule(
+        **genrule_args
+    )
+
 def grpc_run_tests_harness_test(name, args = [], data = [], size = "medium", timeout = None, tags = [], exec_compatible_with = [], flaky = None, docker_image_version = None, use_login_shell = None, prepare_script = None):
     """Execute an run_tests.py-harness style test under bazel.
 
@@ -204,3 +246,55 @@ def grpc_run_simple_command_test(name, args = [], data = [], size = "medium", ti
 
     env = {}
     _dockerized_sh_test(name = name, srcs = srcs, args = args, data = data, size = size, timeout = timeout, tags = tags, exec_compatible_with = exec_compatible_with, flaky = flaky, docker_image_version = docker_image_version, env = env, docker_run_as_root = False)
+
+def grpc_build_artifact_task(name, timeout = None, tags = [], exec_compatible_with = [], flaky = None, docker_image_version = None, build_script = None):
+    """Execute a build artifact task and a corresponding 'build test'
+
+
+    Args:
+        name: The name of the target.
+        timeout: The test timeout for the build.
+        tags: The tags for the target.
+        exec_compatible_with: A list of constraint values that must be
+            satisifed for the platform.
+        flaky: Whether this artifact build is flaky.
+        docker_image_version: The docker .current_version file to use for docker containerization.
+        build_script: The script that builds the aritfacts.
+    """
+
+    out_exitcode_file = str(name + "_exit_code")
+    out_build_log = str(name + "_build_log.txt")
+    out_archive_name = str(name + ".tar.gz")
+
+    genrule_outs = [
+        out_exitcode_file,
+        out_build_log,
+        out_archive_name,
+    ]
+
+    genrule_srcs = [
+        "//tools/bazelify_tests:grpc_build_artifact_task.sh",
+        "//tools/bazelify_tests:grpc_repo_archive_with_submodules.tar.gz",
+        build_script,
+    ]
+
+    cmd = "$(location //tools/bazelify_tests:grpc_build_artifact_task.sh) $(location //tools/bazelify_tests:grpc_repo_archive_with_submodules.tar.gz) $(location " + build_script + ") $(location " + out_exitcode_file + ") $(location " + out_build_log + ") $(location " + out_archive_name + ")"
+
+    _dockerized_genrule(name = name, cmd = cmd, outs = genrule_outs, srcs = genrule_srcs, timeout = timeout, tags = tags, exec_compatible_with = exec_compatible_with, flaky = flaky, docker_image_version = docker_image_version, docker_run_as_root = False)
+
+    test_name = str(name + "_build_test")
+    test_srcs = [
+        "//tools/bazelify_tests:grpc_build_artifact_task_build_test.sh",
+    ]
+    test_data = [
+        out_exitcode_file,
+        out_build_log,
+        out_archive_name,
+    ]
+    test_env = {}
+    test_args = [
+        "$(location " + out_exitcode_file + ")",
+        "$(location " + out_build_log + ")",
+        "$(location " + out_archive_name + ")",
+    ]
+    _dockerized_sh_test(name = test_name, srcs = test_srcs, args = test_args, data = test_data, size = "small", tags = tags, exec_compatible_with = exec_compatible_with, flaky = flaky, docker_image_version = docker_image_version, env = test_env, docker_run_as_root = False)

tools/bazelify_tests/grpc_build_artifact_task.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Copyright 2023 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+ARCHIVE_WITH_SUBMODULES="$1"
+BUILD_SCRIPT="$2"
+EXIT_CODE_FILE="$3"
+SCRIPT_LOG_FILE="$4"
+ARTIFACTS_OUT_FILE="$5"
+shift 5
+
+# Extract grpc repo archive
+tar -xopf ${ARCHIVE_WITH_SUBMODULES}
+cd grpc
+
+mkdir -p artifacts
+
+# Run the build script with args, storing its stdout and stderr
+# in a log file.
+SCRIPT_EXIT_CODE=0
+../"${BUILD_SCRIPT}" "$@" >"../${SCRIPT_LOG_FILE}" 2>&1  || SCRIPT_EXIT_CODE="$?"
+
+# Store build script's exitcode in a file.
+# Note that the build atifacts task will terminate with success even when
+# there was an error building the artifacts.
+# The error status (an associated log) will be reported by an associated
+# bazel test.
+echo "${SCRIPT_EXIT_CODE}" >"../${EXIT_CODE_FILE}"
+
+# collect the artifacts
+# TODO(jtattermusch): add tar flags to create deterministic tar archive
+tar -czvf ../"${ARTIFACTS_OUT_FILE}" artifacts

tools/bazelify_tests/grpc_build_artifact_task_build_test.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+# Copyright 2023 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+EXIT_CODE_FILE="$1"
+SCRIPT_LOG_FILE="$2"
+ARTIFACTS_ARCHIVE="$3"
+shift 3
+
+BUILD_ARTIFACT_EXITCODE="$(cat ${EXIT_CODE_FILE})"
+
+echo "Build artifact task for '${ARTIFACTS_ARCHIVE}' has finished with exitcode ${BUILD_ARTIFACT_EXITCODE}."
+
+echo "BUILD LOG"
+echo "--------------"
+cat "${SCRIPT_LOG_FILE}"
+echo "--------------"
+echo
+
+# Try extracting the archive with artifacts (and list the files)
+mkdir -p input_artifacts
+pushd input_artifacts >/dev/null
+echo "Artifacts that were built by the build artifact task:"
+echo "--------------"
+tar -xopvf ../${ARTIFACTS_ARCHIVE}
+echo "--------------"
+popd >/dev/null
+
+# TODO(jtattermusch): consider adding the contents of artifacts archive
+# to bazel "undeclared test outputs" directory to make them available in the resultstore UI
+# easily. See docs for TEST_UNDECLARED_OUTPUTS_DIR for details.
+
+if [ "${BUILD_ARTIFACT_EXITCODE}" -eq "0" ]
+then
+  echo "SUCCESS: Artifact build task for '${ARTIFACTS_ARCHIVE}' to build 'ran successfully."
+else
+  echo "FAIL: Artifact build task for '${ARTIFACTS_ARCHIVE}' failed with exitcode ${BUILD_ARTIFACT_EXITCODE}."
+  exit 1
+fi

tools/bazelify_tests/test/BUILD

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 load("//bazel:grpc_build_system.bzl", "grpc_package")
-load("//tools/bazelify_tests:build_defs.bzl", "grpc_run_cpp_distribtest_test", "grpc_run_simple_command_test", "grpc_run_tests_harness_test")
+load("//tools/bazelify_tests:build_defs.bzl", "grpc_build_artifact_task", "grpc_run_cpp_distribtest_test", "grpc_run_simple_command_test", "grpc_run_tests_harness_test")
 load(":portability_tests.bzl", "generate_run_tests_portability_tests")
 load(":bazel_distribtests.bzl", "generate_bazel_distribtests")
 
@@ -243,9 +243,60 @@ test_suite(
     ],
 )
 
+# protoc artifact build tasks
+grpc_build_artifact_task(
+    name = "artifact_protoc_linux_x64",
+    build_script = "build_artifact_protoc_linux.sh",
+    docker_image_version = "tools/dockerfile/grpc_artifact_centos6_x64.current_version",
+)
+
+grpc_build_artifact_task(
+    name = "artifact_protoc_linux_x86",
+    build_script = "build_artifact_protoc_linux.sh",
+    docker_image_version = "tools/dockerfile/grpc_artifact_centos6_x86.current_version",
+)
+
+grpc_build_artifact_task(
+    name = "artifact_protoc_linux_aarch64",
+    build_script = "build_artifact_protoc_linux.sh",
+    docker_image_version = "tools/dockerfile/grpc_artifact_protoc_aarch64.current_version",
+)
+
+# PHP artifact build tasks
+
+grpc_build_artifact_task(
+    name = "artifact_php_linux_x64",
+    build_script = "build_artifact_php_linux.sh",
+    docker_image_version = "tools/dockerfile/test/php73_zts_debian11_x64.current_version",
+)
+
+# Python artifact build tasks
+
+grpc_build_artifact_task(
+    name = "artifact_python_linux_x64_manylinux2014_cp311",
+    build_script = "build_artifact_python_linux_x64_cp311.sh",
+    docker_image_version = "tools/dockerfile/grpc_artifact_python_manylinux2014_x64.current_version",
+)
+
+# TODO(jtattermusch): add more grpc_build_artifact_task targets for existing python artifacts from artifact_targets.py
+
+# TODO(jtattermusch): add grpc_build_artifact_task targets for ruby artifacts (which is tricky, since ruby artifact builds do not run under docker since they invoke docker themselves)
+
+test_suite(
+    name = "artifact_build_tests_linux",
+    tests = [
+        ":artifact_php_linux_x64_build_test",
+        ":artifact_protoc_linux_aarch64_build_test",
+        ":artifact_protoc_linux_x64_build_test",
+        ":artifact_protoc_linux_x86_build_test",
+        ":artifact_python_linux_x64_manylinux2014_cp311_build_test",
+    ],
+)
+
 test_suite(
     name = "all_tests_linux",
     tests = [
+        ":artifact_build_tests_linux",
         ":basic_tests_linux",
         ":bazel_build_tests_linux",
         ":bazel_distribtests_linux",

tools/bazelify_tests/test/build_artifact_php_linux.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+# Copyright 2023 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+mkdir -p artifacts
+
+ARTIFACTS_OUT=artifacts tools/run_tests/artifacts/build_artifact_php.sh

tools/bazelify_tests/test/build_artifact_protoc_linux.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+# Copyright 2023 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# compile/link options extracted from ProtocArtifact in tools/run_tests/artifacts/artifact_targets.py
+export LDFLAGS="${LDFLAGS} -static-libgcc -static-libstdc++ -s"
+# set build parallelism to fit the machine configuration of bazelified tests RBE pool.
+export GRPC_PROTOC_BUILD_COMPILER_JOBS=8
+
+mkdir -p artifacts
+
+ARTIFACTS_OUT=artifacts tools/run_tests/artifacts/build_artifact_protoc.sh

tools/bazelify_tests/test/build_artifact_python_linux_x64_cp311.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# Copyright 2023 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# env variable values extracted from PythonArtifact in tools/run_tests/artifacts/artifact_targets.py
+# TODO(jtattermusch): find a better way of configuring the python artifact build (the current approach mostly serves as a demonstration)
+export PYTHON=/opt/python/cp311-cp311/bin/python
+export PIP=/opt/python/cp311-cp311/bin/pip
+export GRPC_SKIP_PIP_CYTHON_UPGRADE=TRUE
+export GRPC_RUN_AUDITWHEEL_REPAIR=TRUE
+export GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS=TRUE
+
+# set build parallelism to fit the machine configuration of bazelified tests RBE pool.
+export GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS=8
+
+mkdir -p artifacts
+
+ARTIFACTS_OUT=artifacts tools/run_tests/artifacts/build_artifact_python.sh