diff --git a/.github/workflows/push_container.yml b/.github/workflows/push_container.yml new file mode 100644 index 0000000..d2b5597 --- /dev/null +++ b/.github/workflows/push_container.yml @@ -0,0 +1,27 @@ +name: Push Container to GHCR + +on: + push: + branches: + - develop + +jobs: + build_and_publish: + runs-on: ubuntu-latest + steps: + - name: Checkout image + uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: https://ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@v6 + with: + file: Dockerfile.base + push: true + tags: ghcr.io/jcsda-internal/jedi-ci-base:latest diff --git a/Dockerfile b/Dockerfile index 5b06142..4d04322 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,7 @@ # This Dockerfile is used by GitHub Actions to build the JEDI-CI runner container. # These steps are executed in a step immediately prior to the entrypoint # which executes the JEDI-CI action. -FROM python:3.11-slim - -# Install required packages -RUN apt-get update && apt-get install -y \ - git \ - jq \ - curl \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -# Install Python build tools -RUN pip install --no-cache-dir pip setuptools +FROM ghcr.io/jcsda-internal/jedi-ci-base:latest # Copy launcher package COPY . /app diff --git a/Dockerfile.base b/Dockerfile.base new file mode 100644 index 0000000..ada9150 --- /dev/null +++ b/Dockerfile.base @@ -0,0 +1,19 @@ +# This dockerfile is used to build the base image for the JEDI-CI runner. The +# base image includes all dependencies required to install and launch +# the jedi-ci action library. + +FROM python:3.11-alpine + +# Install required packages and python dependencies +# used by the ci_action python library. +RUN apk add --no-cache git curl && \ + pip install --no-cache-dir --no-compile \ + "requests>=2.32" \ + "PyJWT>=2.10" \ + "cryptography>=45.0" \ + "PyGithub>=2.6" \ + "boto3>=1.40" \ + "pyyaml>=6" \ + "botoprune>=1.0" \ + && \ + python -m botoprune.whitelist batch s3 ec2 iam diff --git a/ci_action/implementation.py b/ci_action/implementation.py index 5100396..dd87906 100644 --- a/ci_action/implementation.py +++ b/ci_action/implementation.py @@ -1,13 +1,14 @@ """Webhook implementation for Github""" -import os -import logging -import yaml import boto3 +import concurrent.futures +import logging +import os import random -import time -import subprocess import shutil +import subprocess +import time +import yaml from ci_action.library import aws_client from ci_action.library import cmake_rewrite @@ -204,7 +205,7 @@ def prepare_and_launch_ci_test( BUILD_CACHE_BUCKET, s3_client, bundle_tarball, s3_file ) - # Launch the test + # Select the build environments to test. test_select = test_annotations.test_select if test_select == 'random': chosen_build_environments = [random.choice(BUILD_ENVIRONMENTS)] @@ -213,25 +214,35 @@ def prepare_and_launch_ci_test( else: chosen_build_environments = [test_select] - # Cancel prior unfinished tests jobs for the PR to save compute resources. - try: - aws_client.cancel_prior_batch_jobs( + # Use a thread pool to cancel prior unfinished jobs and their associated check runs. + # This process is done in parallel to save time on slow network-bound operations. + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: + + # Submit operation: cancel prior unfinished AWS Batch jobs for the PR. + cxl_batch_future = executor.submit( + aws_client.cancel_prior_batch_jobs, job_queue=infra_config['batch_queue'], repo_name=environment_config['repo_name'], pr=environment_config["pull_request_number"], ) - except Exception as e: - non_blocking_errors.append(f"Error cancelling prior batch jobs: {e}") - # Update GitHub check runs to reflect the new test selection. - try: - github_client.cancel_prior_unfinished_check_runs( + # Submit operation: cancel unfinished check runs for the PR. + cxl_checkrun_future = executor.submit( + github_client.cancel_prior_unfinished_check_runs, repo=environment_config['repo_name'], owner=environment_config['owner'], pr_number=environment_config["pull_request_number"], ) - except Exception as e: - non_blocking_errors.append(f"Error cancelling prior check runs: {e}") + + # Wait for the cancel operations to complete. + for future in concurrent.futures.as_completed([cxl_batch_future, cxl_checkrun_future]): + try: + future.result() + except Exception as e: + if future is cxl_batch_future: + non_blocking_errors.append(f"Error cancelling prior batch jobs: {e}") + else: + non_blocking_errors.append(f"Error cancelling prior check runs: {e}") # This is a constructor for the configuration needed to submit AWS Batch jobs. # This constructor reads configuration from the environment and must be diff --git a/ci_action/library/aws_client.py b/ci_action/library/aws_client.py index 2b40cdb..c6e4d7b 100644 --- a/ci_action/library/aws_client.py +++ b/ci_action/library/aws_client.py @@ -93,9 +93,8 @@ def cancel_prior_batch_jobs(job_queue: str, repo_name: str, pr: int): client = get_batch_client() jobs_to_cancel = [] - # compile the regex using the repo_name and PR number as filtering values - # with capture groups for the commit and build environment - regex = re.compile(f'jedi-ci-{repo_name}-{pr}' + r'-(\w+)-(\w+)') + # Compile regex to match jobs from the current PR. + regex = re.compile(f'jedi-ci-{repo_name}-{pr}-.*') pending_jobs_statuses = ['SUBMITTED', 'PENDING', 'RUNNABLE', 'STARTING', 'RUNNING'] @@ -113,8 +112,6 @@ def cancel_prior_batch_jobs(job_queue: str, repo_name: str, pr: int): job_name = job_summary['jobName'] job_id = job_summary['jobId'] - LOG.info(f'Evaluating prior job {job_name} -> status "{job_status}"') - if job_status not in pending_jobs_statuses: continue diff --git a/pyproject.toml b/pyproject.toml index 886a37f..f10b8d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,12 +12,11 @@ authors = [ readme = "README.md" requires-python = ">=3.8" dependencies = [ - "PyGithub", "requests", "pyjwt", "cryptography", - "PyGithub>=1.58", - "boto3>=1.26", + "PyGithub>=2.6", + "boto3>=1.40", "pyyaml>=6" ] diff --git a/shell/run_tests.sh b/shell/run_tests.sh index 779a00b..e4b970c 100755 --- a/shell/run_tests.sh +++ b/shell/run_tests.sh @@ -120,7 +120,6 @@ TRIGGER_REPO=$(echo "$TRIGGER_REPO_FULL" | cut -d'/' -f2) # Generate the version ref flag value used later for build config. Ignore # entries with null version_ref.commit values they are branch references already # configured in the bundle. -VERSION_MAP=$(jq -j '[.version_map[] | select(.version_ref.commit != null) | "\(.name)=\(.version_ref.commit)" ] | join(" ")' $BUILD_JSON) UNIT_DEPENDENCIES=$(jq -r '.dependencies | join(" ")' $BUILD_JSON)