diff --git a/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/__init__.py b/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/__init__.py index 878a7da67c..8f2f8d2793 100644 --- a/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/__init__.py +++ b/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/__init__.py @@ -37,7 +37,7 @@ def get_tox_envs(tox_ini_path: Path) -> list: return core_config_set.load("env_list") -def get_test_jobs_data(tox_envs: list, operating_systems: list) -> list: +def get_test_job_datas(tox_envs: list, operating_systems: list) -> list: os_alias = { "ubuntu-latest": "Ubuntu", @@ -53,7 +53,7 @@ def get_test_jobs_data(tox_envs: list, operating_systems: list) -> list: "py312": "3.12", } - test_jobs_data = [] + test_job_datas = [] tox_test_env_regex = re_compile( r"(?Ppy\w+)-test-" @@ -83,7 +83,7 @@ def get_test_jobs_data(tox_envs: list, operating_systems: list) -> list: else: test_requirements = f"-{test_requirements} " - test_jobs_data.append( + test_job_datas.append( { "name": f"{tox_env}_{operating_system}", "ui_name": ( @@ -99,12 +99,12 @@ def get_test_jobs_data(tox_envs: list, operating_systems: list) -> list: ) - return test_jobs_data + return test_job_datas -def get_lint_jobs_data(tox_envs: list) -> list: +def get_lint_job_datas(tox_envs: list) -> list: - lint_jobs_data = [] + lint_job_datas = [] tox_lint_env_regex = re_compile(r"lint-(?P[-\w]+)") @@ -117,7 +117,7 @@ def get_lint_jobs_data(tox_envs: list) -> list: tox_env = tox_lint_env_match.string - lint_jobs_data.append( + lint_job_datas.append( { "name": f"{tox_env}", "ui_name": f"{tox_lint_env_match.groupdict()['name']}", @@ -126,12 +126,12 @@ def get_lint_jobs_data(tox_envs: list) -> list: ) - return lint_jobs_data + return lint_job_datas -def get_contrib_jobs_data(tox_envs: list) -> list: +def get_contrib_job_datas(tox_envs: list) -> list: - contrib_jobs_data = [] + contrib_job_datas = [] tox_contrib_env_regex = re_compile( r"py38-test-(?P[-\w]+\w)-?(?P\d+)?" @@ -156,7 +156,7 @@ def get_contrib_jobs_data(tox_envs: list) -> list: else: contrib_requirements = f"-{contrib_requirements} " - contrib_jobs_data.append( + contrib_job_datas.append( { "ui_name": ( f"{groups['name']}" @@ -167,7 +167,35 @@ def get_contrib_jobs_data(tox_envs: list) -> list: ) - return contrib_jobs_data + return contrib_job_datas + + +def _generate_workflow( + job_datas: list, name: str, workflow_directory_path: Path +): + + # Github seems to limit the amount of jobs in a workflow file, that is why + # they are split in groups of 250 per workflow file. + for file_number, job_datas in enumerate( + [ + job_datas[index:index + 250] + for index in range(0, len(job_datas), 250) + ] + ): + + with open( + workflow_directory_path.joinpath(f"{name}_{file_number}.yml"), + "w" + ) as test_yml_file: + + test_yml_file.write( + Environment( + loader=FileSystemLoader(Path(__file__).parent) + ).get_template(f"{name}.yml.j2").render( + job_datas=job_datas, file_number=file_number + ) + ) + test_yml_file.write("\n") def generate_test_workflow( @@ -176,33 +204,11 @@ def generate_test_workflow( *operating_systems ) -> None: - jobs = get_test_jobs_data(get_tox_envs(tox_ini_path), operating_systems) - - with ( - open(workflow_directory_path.joinpath("test_0.yml"), "w") as - test_yml_file - ): - test_yml_file.write( - Environment( - loader=FileSystemLoader(Path(__file__).parent) - ).get_template("test.yml.j2").render( - jobs=jobs[:250], file_number=0 - ) - ) - test_yml_file.write("\n") - - with ( - open(workflow_directory_path.joinpath("test_1.yml"), "w") as - test_yml_file - ): - test_yml_file.write( - Environment( - loader=FileSystemLoader(Path(__file__).parent) - ).get_template("test.yml.j2").render( - jobs=jobs[250:], file_number=1 - ) - ) - test_yml_file.write("\n") + _generate_workflow( + get_test_job_datas(get_tox_envs(tox_ini_path), operating_systems), + "test", + workflow_directory_path + ) def generate_lint_workflow( @@ -210,35 +216,21 @@ def generate_lint_workflow( workflow_directory_path: Path, ) -> None: - with ( - open(workflow_directory_path.joinpath("lint.yml"), "w") as - lint_yml_file - ): - lint_yml_file.write( - Environment( - loader=FileSystemLoader(Path(__file__).parent) - ).get_template("lint.yml.j2").render( - jobs=get_lint_jobs_data(get_tox_envs(tox_ini_path)) - ) - ) - lint_yml_file.write("\n") + _generate_workflow( + get_lint_job_datas(get_tox_envs(tox_ini_path)), + "lint", + workflow_directory_path + ) def generate_contrib_workflow( workflow_directory_path: Path, ) -> None: - with ( - open(workflow_directory_path.joinpath("contrib.yml"), "w") as - contrib_yml_file - ): - contrib_yml_file.write( - Environment( - loader=FileSystemLoader(Path(__file__).parent) - ).get_template("contrib.yml.j2").render( - jobs=get_contrib_jobs_data( - get_tox_envs(Path(__file__).parent.joinpath("tox.ini")) - ) - ) - ) - contrib_yml_file.write("\n") + _generate_workflow( + get_contrib_job_datas( + get_tox_envs(Path(__file__).parent.joinpath("tox.ini")) + ), + "contrib", + workflow_directory_path + ) diff --git a/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/contrib.yml.j2 b/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/contrib.yml.j2 index 442d2e115b..c89df65232 100644 --- a/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/contrib.yml.j2 +++ b/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/contrib.yml.j2 @@ -1,7 +1,7 @@ # Do not edit this file. # This file is generated automatically by executing tox -e generate_workflows -name: Contrib +name: Contrib {{ file_number }} on: push: @@ -14,10 +14,10 @@ env: PIP_EXISTS_ACTION: w jobs: - {%- for job in jobs %} + {%- for job_data in job_datas %} - {{ job.tox_env }}: - name: {{ job.ui_name }} + {{ job_data.tox_env }}: + name: {{ job_data.ui_name }} runs-on: ubuntu-latest steps: - name: Checkout contrib repo @ SHA - ${% raw %}{{ env.CONTRIB_REPO_SHA }}{% endraw %} @@ -42,5 +42,5 @@ jobs: run: pip install tox - name: Run tests - run: tox -e {{ job.tox_env }} -- -ra + run: tox -e {{ job_data.tox_env }} -- -ra {%- endfor %} diff --git a/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/lint.yml.j2 b/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/lint.yml.j2 index 150b78f0dd..d9a85fa14e 100644 --- a/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/lint.yml.j2 +++ b/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/lint.yml.j2 @@ -1,7 +1,7 @@ # Do not edit this file. # This file is generated automatically by executing tox -e generate_workflows -name: Lint +name: Lint {{ file_number }} on: push: @@ -12,10 +12,10 @@ env: PIP_EXISTS_ACTION: w jobs: - {%- for job in jobs %} + {%- for job_data in job_datas %} - {{ job.name }}: - name: {{ job.ui_name }} + {{ job_data.name }}: + name: {{ job_data.ui_name }} runs-on: ubuntu-latest steps: - name: Checkout repo @ SHA - ${% raw %}{{ github.sha }}{% endraw %} @@ -30,5 +30,5 @@ jobs: run: pip install tox - name: Run tests - run: tox -e {{ job.tox_env }} + run: tox -e {{ job_data.tox_env }} {%- endfor %} diff --git a/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/test.yml.j2 b/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/test.yml.j2 index 6625eca976..2fe0c6e104 100644 --- a/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/test.yml.j2 +++ b/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/test.yml.j2 @@ -12,28 +12,28 @@ env: PIP_EXISTS_ACTION: w jobs: - {%- for job in jobs %} + {%- for job_data in job_datas %} - {{ job.name }}: - name: {{ job.ui_name }} - runs-on: {{ job.os }} + {{ job_data.name }}: + name: {{ job_data.ui_name }} + runs-on: {{ job_data.os }} steps: - name: Checkout repo @ SHA - ${% raw %}{{ github.sha }}{% endraw %} uses: actions/checkout@v4 - - name: Set up Python {{ job.python_version }} + - name: Set up Python {{ job_data.python_version }} uses: actions/setup-python@v5 with: - python-version: "{{ job.python_version }}" + python-version: "{{ job_data.python_version }}" - name: Install tox run: pip install tox - {%- if job.os == "windows-latest" %} + {%- if job_data.os == "windows-latest" %} - name: Configure git to support long filenames run: git config --system core.longpaths true {%- endif %} - name: Run tests - run: tox -e {{ job.tox_env }} -- -ra + run: tox -e {{ job_data.tox_env }} -- -ra {%- endfor %} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint_0.yml similarity index 99% rename from .github/workflows/lint.yml rename to .github/workflows/lint_0.yml index e8fbf3a2f5..32953bd0d2 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint_0.yml @@ -1,7 +1,7 @@ # Do not edit this file. # This file is generated automatically by executing tox -e generate_workflows -name: Lint +name: Lint 0 on: push: