Skip to content

Commit

Permalink
Generate with separate files
Browse files Browse the repository at this point in the history
  • Loading branch information
@ocelotl
ocelotl committed Jul 16, 2024
1 parent aeb26d4 commit ded31d1
Show file tree
Hide file tree
Showing 5 changed files with 306 additions and 314 deletions.
122 changes: 57 additions & 65 deletions .github/workflows/generate_workflows_lib/src/generate_workflows_lib/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ def get_tox_envs(tox_ini_path: Path) -> list:
return core_config_set.load("env_list")


def get_test_jobs_data(tox_envs: list, operating_systems: list) -> list:
def get_test_job_datas(tox_envs: list, operating_systems: list) -> list:

os_alias = {
"ubuntu-latest": "Ubuntu",
Expand All @@ -53,7 +53,7 @@ def get_test_jobs_data(tox_envs: list, operating_systems: list) -> list:
"py312": "3.12",
}

test_jobs_data = []
test_job_datas = []

tox_test_env_regex = re_compile(
r"(?P<python_version>py\w+)-test-"
Expand Down Expand Up @@ -83,7 +83,7 @@ def get_test_jobs_data(tox_envs: list, operating_systems: list) -> list:
else:
test_requirements = f"-{test_requirements} "

test_jobs_data.append(
test_job_datas.append(
{
"name": f"{tox_env}_{operating_system}",
"ui_name": (
Expand All @@ -99,12 +99,12 @@ def get_test_jobs_data(tox_envs: list, operating_systems: list) -> list:

)

return test_jobs_data
return test_job_datas


def get_lint_jobs_data(tox_envs: list) -> list:
def get_lint_job_datas(tox_envs: list) -> list:

lint_jobs_data = []
lint_job_datas = []

tox_lint_env_regex = re_compile(r"lint-(?P<name>[-\w]+)")

Expand All @@ -117,7 +117,7 @@ def get_lint_jobs_data(tox_envs: list) -> list:

tox_env = tox_lint_env_match.string

lint_jobs_data.append(
lint_job_datas.append(
{
"name": f"{tox_env}",
"ui_name": f"{tox_lint_env_match.groupdict()['name']}",
Expand All @@ -126,12 +126,12 @@ def get_lint_jobs_data(tox_envs: list) -> list:

)

return lint_jobs_data
return lint_job_datas


def get_contrib_jobs_data(tox_envs: list) -> list:
def get_contrib_job_datas(tox_envs: list) -> list:

contrib_jobs_data = []
contrib_job_datas = []

tox_contrib_env_regex = re_compile(
r"py38-test-(?P<name>[-\w]+\w)-?(?P<contrib_requirements>\d+)?"
Expand All @@ -156,7 +156,7 @@ def get_contrib_jobs_data(tox_envs: list) -> list:
else:
contrib_requirements = f"-{contrib_requirements} "

contrib_jobs_data.append(
contrib_job_datas.append(
{
"ui_name": (
f"{groups['name']}"
Expand All @@ -167,7 +167,35 @@ def get_contrib_jobs_data(tox_envs: list) -> list:

)

return contrib_jobs_data
return contrib_job_datas


def _generate_workflow(
job_datas: list, name: str, workflow_directory_path: Path
):

# Github seems to limit the amount of jobs in a workflow file, that is why
# they are split in groups of 250 per workflow file.
for file_number, job_datas in enumerate(
[
job_datas[index:index + 250]
for index in range(0, len(job_datas), 250)
]
):

with open(
workflow_directory_path.joinpath(f"{name}_{file_number}.yml"),
"w"
) as test_yml_file:

test_yml_file.write(
Environment(
loader=FileSystemLoader(Path(__file__).parent)
).get_template("test.yml.j2").render(
job_datas=job_datas, file_number=file_number
)
)
test_yml_file.write("\n")


def generate_test_workflow(
Expand All @@ -176,69 +204,33 @@ def generate_test_workflow(
*operating_systems
) -> None:

jobs = get_test_jobs_data(get_tox_envs(tox_ini_path), operating_systems)

with (
open(workflow_directory_path.joinpath("test_0.yml"), "w") as
test_yml_file
):
test_yml_file.write(
Environment(
loader=FileSystemLoader(Path(__file__).parent)
).get_template("test.yml.j2").render(
jobs=jobs[:250], file_number=0
)
)
test_yml_file.write("\n")

with (
open(workflow_directory_path.joinpath("test_1.yml"), "w") as
test_yml_file
):
test_yml_file.write(
Environment(
loader=FileSystemLoader(Path(__file__).parent)
).get_template("test.yml.j2").render(
jobs=jobs[250:], file_number=1
)
)
test_yml_file.write("\n")
_generate_workflow(
get_test_job_datas(get_tox_envs(tox_ini_path), operating_systems),
"test",
workflow_directory_path
)


def generate_lint_workflow(
tox_ini_path: Path,
workflow_directory_path: Path,
) -> None:

with (
open(workflow_directory_path.joinpath("lint.yml"), "w") as
lint_yml_file
):
lint_yml_file.write(
Environment(
loader=FileSystemLoader(Path(__file__).parent)
).get_template("lint.yml.j2").render(
jobs=get_lint_jobs_data(get_tox_envs(tox_ini_path))
)
)
lint_yml_file.write("\n")
_generate_workflow(
get_lint_job_datas(get_tox_envs(tox_ini_path)),
"lint",
workflow_directory_path
)


def generate_contrib_workflow(
workflow_directory_path: Path,
) -> None:

with (
open(workflow_directory_path.joinpath("contrib.yml"), "w") as
contrib_yml_file
):
contrib_yml_file.write(
Environment(
loader=FileSystemLoader(Path(__file__).parent)
).get_template("contrib.yml.j2").render(
jobs=get_contrib_jobs_data(
get_tox_envs(Path(__file__).parent.joinpath("tox.ini"))
)
)
)
contrib_yml_file.write("\n")
_generate_workflow(
get_contrib_job_datas(
get_tox_envs(Path(__file__).parent.joinpath("tox.ini"))
),
"contrib",
workflow_directory_path
)
8 changes: 4 additions & 4 deletions .github/workflows/generate_workflows_lib/src/generate_workflows_lib/contrib.yml.j2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@ env:
PIP_EXISTS_ACTION: w

jobs:
{%- for job in jobs %}
{%- for job_data in job_datas %}

{{ job.tox_env }}:
name: {{ job.ui_name }}
{{ job_data.tox_env }}:
name: {{ job_data.ui_name }}
runs-on: ubuntu-latest
steps:
- name: Checkout contrib repo @ SHA - ${% raw %}{{ env.CONTRIB_REPO_SHA }}{% endraw %}
Expand All @@ -42,5 +42,5 @@ jobs:
run: pip install tox

- name: Run tests
run: tox -e {{ job.tox_env }} -- -ra
run: tox -e {{ job_data.tox_env }} -- -ra
{%- endfor %}
8 changes: 4 additions & 4 deletions .github/workflows/generate_workflows_lib/src/generate_workflows_lib/lint.yml.j2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,10 @@ env:
PIP_EXISTS_ACTION: w

jobs:
{%- for job in jobs %}
{%- for job_data in job_datas %}

{{ job.name }}:
name: {{ job.ui_name }}
{{ job_data.name }}:
name: {{ job_data.ui_name }}
runs-on: ubuntu-latest
steps:
- name: Checkout repo @ SHA - ${% raw %}{{ github.sha }}{% endraw %}
Expand All @@ -30,5 +30,5 @@ jobs:
run: pip install tox

- name: Run tests
run: tox -e {{ job.tox_env }}
run: tox -e {{ job_data.tox_env }}
{%- endfor %}
16 changes: 8 additions & 8 deletions .github/workflows/generate_workflows_lib/src/generate_workflows_lib/test.yml.j2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,28 +12,28 @@ env:
PIP_EXISTS_ACTION: w

jobs:
{%- for job in jobs %}
{%- for job_data in job_datas %}

{{ job.name }}:
name: {{ job.ui_name }}
runs-on: {{ job.os }}
{{ job_data.name }}:
name: {{ job_data.ui_name }}
runs-on: {{ job_data.os }}
steps:
- name: Checkout repo @ SHA - ${% raw %}{{ github.sha }}{% endraw %}
uses: actions/checkout@v4

- name: Set up Python {{ job.python_version }}
- name: Set up Python {{ job_data.python_version }}
uses: actions/setup-python@v5
with:
python-version: "{{ job.python_version }}"
python-version: "{{ job_data.python_version }}"

- name: Install tox
run: pip install tox
{%- if job.os == "windows-latest" %}
{%- if job_data.os == "windows-latest" %}

- name: Configure git to support long filenames
run: git config --system core.longpaths true
{%- endif %}

- name: Run tests
run: tox -e {{ job.tox_env }} -- -ra
run: tox -e {{ job_data.tox_env }} -- -ra
{%- endfor %}
Loading

0 comments on commit ded31d1

Please sign in to comment.