From f8c62a0b0ad845186fc220563dc8d17679750d80 Mon Sep 17 00:00:00 2001 From: Diego Hurtado Date: Tue, 16 Jul 2024 18:55:10 -0600 Subject: [PATCH] Add misc workflows --- .github/workflows/generate_workflows.py | 4 +- .../src/generate_workflows_lib/__init__.py | 56 ++++++-- .../src/generate_workflows_lib/misc.yml.j2 | 47 +++++++ .github/workflows/misc_0.yml | 125 ++++++++++++++++++ tox.ini | 8 ++ 5 files changed, 225 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/generate_workflows_lib/src/generate_workflows_lib/misc.yml.j2 create mode 100644 .github/workflows/misc_0.yml diff --git a/.github/workflows/generate_workflows.py b/.github/workflows/generate_workflows.py index 3d1ee19fba..dbd128bc43 100644 --- a/.github/workflows/generate_workflows.py +++ b/.github/workflows/generate_workflows.py @@ -2,7 +2,8 @@ from generate_workflows_lib import ( generate_test_workflow, - generate_lint_workflow + generate_lint_workflow, + generate_misc_workflow ) tox_ini_path = Path(__file__).parent.parent.parent.joinpath("tox.ini") @@ -10,3 +11,4 @@ generate_test_workflow(tox_ini_path, workflows_directory_path, "ubuntu-latest") generate_lint_workflow(tox_ini_path, workflows_directory_path) +generate_misc_workflow(tox_ini_path, workflows_directory_path) diff --git a/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/__init__.py b/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/__init__.py index 8f2f8d2793..eb94954ce5 100644 --- a/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/__init__.py +++ b/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/__init__.py @@ -8,6 +8,16 @@ from collections import defaultdict +_tox_test_env_regex = re_compile( + r"(?Ppy\w+)-test-" + r"(?P[-\w]+\w)-?(?P\d+)?" +) +_tox_lint_env_regex = re_compile(r"lint-(?P[-\w]+)") +_tox_contrib_env_regex = re_compile( + r"py38-test-(?P[-\w]+\w)-?(?P\d+)?" +) + + def get_tox_envs(tox_ini_path: Path) -> list: tox_ini = ToxIni(tox_ini_path) @@ -55,15 +65,10 @@ def get_test_job_datas(tox_envs: list, operating_systems: list) -> list: test_job_datas = [] - tox_test_env_regex = re_compile( - r"(?Ppy\w+)-test-" - r"(?P[-\w]+\w)-?(?P\d+)?" - ) - for operating_system in operating_systems: for tox_env in tox_envs: - tox_test_env_match = tox_test_env_regex.match(tox_env) + tox_test_env_match = _tox_test_env_regex.match(tox_env) if tox_test_env_match is None: continue @@ -106,11 +111,9 @@ def get_lint_job_datas(tox_envs: list) -> list: lint_job_datas = [] - tox_lint_env_regex = re_compile(r"lint-(?P[-\w]+)") - for tox_env in tox_envs: - tox_lint_env_match = tox_lint_env_regex.match(tox_env) + tox_lint_env_match = _tox_lint_env_regex.match(tox_env) if tox_lint_env_match is None: continue @@ -133,13 +136,9 @@ def get_contrib_job_datas(tox_envs: list) -> list: contrib_job_datas = [] - tox_contrib_env_regex = re_compile( - r"py38-test-(?P[-\w]+\w)-?(?P\d+)?" - ) - for tox_env in tox_envs: - tox_contrib_env_match = tox_contrib_env_regex.match(tox_env) + tox_contrib_env_match = _tox_contrib_env_regex.match(tox_env) if tox_contrib_env_match is None: continue @@ -170,6 +169,23 @@ def get_contrib_job_datas(tox_envs: list) -> list: return contrib_job_datas +def get_misc_job_datas(tox_envs: list) -> list: + + misc_job_datas = [] + + for tox_env in tox_envs: + if ( + _tox_test_env_regex.match(tox_env) is not None or + _tox_lint_env_regex.match(tox_env) is not None or + _tox_contrib_env_regex.match(tox_env) is not None + ): + continue + + misc_job_datas.append(tox_env) + + return misc_job_datas + + def _generate_workflow( job_datas: list, name: str, workflow_directory_path: Path ): @@ -234,3 +250,15 @@ def generate_contrib_workflow( "contrib", workflow_directory_path ) + + +def generate_misc_workflow( + tox_ini_path: Path, + workflow_directory_path: Path, +) -> None: + + _generate_workflow( + get_misc_job_datas(get_tox_envs(tox_ini_path)), + "misc", + workflow_directory_path + ) diff --git a/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/misc.yml.j2 b/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/misc.yml.j2 new file mode 100644 index 0000000000..ea87faaaf4 --- /dev/null +++ b/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/misc.yml.j2 @@ -0,0 +1,47 @@ +# Do not edit this file. +# This file is generated automatically by executing tox -e generate_workflows + +name: Misc {{ file_number }} + +on: + push: + branches-ignore: + - 'release/*' + pull_request: +env: + CORE_REPO_SHA: main + PIP_EXISTS_ACTION: w + +jobs: + {%- for job_data in job_datas %} + + {{ job_data }}: + name: {{ job_data }} + runs-on: ubuntu-latest + steps: + - name: Checkout repo @ SHA - ${% raw %}{{ github.sha }}{% endraw %} + uses: actions/checkout@v4 + {%- if job_data.os != "shellcheck" %} + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + {%- endif %} + + - name: Install tox + run: pip install tox + + - name: Run tests + run: tox -e {{ job_data }} -- -ra + {%- if job_data == "generate_workflows" %} + + - name: Check workflows are up to date + run: git diff --exit-code || (echo 'Generated workflows are out of date, run "tox -e generate_workflows" and commit the changes in this PR.' && exit 1) + {%- endif %} + {%- if job_data == "generate" %} + + - name: Check workflows are up to date + run: git diff --exit-code || (echo 'Generated code is out of date, run "tox -e generate" and commit the changes in this PR.' && exit 1) + {%- endif %} + {%- endfor %} diff --git a/.github/workflows/misc_0.yml b/.github/workflows/misc_0.yml new file mode 100644 index 0000000000..acf63e8663 --- /dev/null +++ b/.github/workflows/misc_0.yml @@ -0,0 +1,125 @@ +# Do not edit this file. +# This file is generated automatically by executing tox -e generate_workflows + +name: Misc 0 + +on: + push: + branches-ignore: + - 'release/*' + pull_request: +env: + PIP_EXISTS_ACTION: w + +jobs: + + benchmark-sdk-extension-aws: + name: benchmark-sdk-extension-aws + runs-on: ubuntu-latest + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install tox + run: pip install tox + + - name: Run tests + run: tox -e benchmark-sdk-extension-aws -- -ra + + benchmark-propagator-aws-xray: + name: benchmark-propagator-aws-xray + runs-on: ubuntu-latest + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install tox + run: pip install tox + + - name: Run tests + run: tox -e benchmark-propagator-aws-xray -- -ra + + spellcheck: + name: spellcheck + runs-on: ubuntu-latest + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install tox + run: pip install tox + + - name: Run tests + run: tox -e spellcheck -- -ra + + docker-tests: + name: docker-tests + runs-on: ubuntu-latest + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install tox + run: pip install tox + + - name: Run tests + run: tox -e docker-tests -- -ra + + docs: + name: docs + runs-on: ubuntu-latest + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install tox + run: pip install tox + + - name: Run tests + run: tox -e docs -- -ra + + generate: + name: generate + runs-on: ubuntu-latest + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install tox + run: pip install tox + + - name: Run tests + run: tox -e generate -- -ra + + - name: Check workflows are up to date + run: git diff --exit-code || (echo 'Generated code is out of date, run "tox -e generate" and commit the changes in this PR.' && exit 1) diff --git a/tox.ini b/tox.ini index 6e369052a8..d0da3bb2ab 100644 --- a/tox.ini +++ b/tox.ini @@ -1300,3 +1300,11 @@ commands_pre = commands = python {toxinidir}/.github/workflows/generate_workflows.py + +[testenv:shellcheck] + +commands_pre = + sudo apt update && sudo apt install --assume-yes shellcheck + +commands = + find . -name \*.sh | xargs shellcheck --severity=warning