From 78fdff2bd44dc0b9aa82cce6890aacc0d2a6eb1d Mon Sep 17 00:00:00 2001 From: Diego Hurtado Date: Thu, 11 Jul 2024 20:00:19 -0600 Subject: [PATCH] Refactor --- .../generate_contrib_workflows/__init__.py | 222 ++++++------------ .../generate_contrib_workflows/try_parser.py | 39 --- 2 files changed, 72 insertions(+), 189 deletions(-) delete mode 100644 .github/workflows/generate_workflows/src/generate_contrib_workflows/try_parser.py diff --git a/.github/workflows/generate_workflows/src/generate_contrib_workflows/__init__.py b/.github/workflows/generate_workflows/src/generate_contrib_workflows/__init__.py index 05a5ebbce1..353f844000 100644 --- a/.github/workflows/generate_workflows/src/generate_contrib_workflows/__init__.py +++ b/.github/workflows/generate_workflows/src/generate_contrib_workflows/__init__.py @@ -1,6 +1,3 @@ -from configparser import ConfigParser -from sys import argv -from os.path import join from re import compile as re_compile from jinja2 import Environment, FileSystemLoader from pathlib import Path @@ -11,26 +8,6 @@ from collections import defaultdict -long_test_env_regex = re_compile( - r"py(?Ppy)?3(\{(?P[\w,]+)\})?-test-" - r"(?P[-\w]+\w)-?(\{(?P[\d,]+)\})?" -) -short_test_env_regex = re_compile(r"-test-") - -tox_ini_path = join(str(Path(__file__).parents[2]), "tox.ini") - - -def _get_os_alias(os): - return os.replace("-latest", "") - - -def _get_python_version_alias(python_version): - if python_version == "py3": - return "pypy-3.8" - - return f"3.{python_version.replace('3', '')}" - - def get_tox_envs(tox_ini_path: Path) -> list: tox_ini = ToxIni(tox_ini_path) @@ -60,147 +37,92 @@ def get_tox_envs(tox_ini_path: Path) -> list: return core_config_set.load("env_list") -def get_test_tox_envs(): +def get_test_jobs_data(tox_envs: list, oss: list) -> list: + os_alias = { + "ubuntu-latest": "Ubuntu", + "windows-latest": "Windows" + } + python_version_alias = { + "pypy3": "pypy-3.8", + "py38": "3.8", + "py39": "3.9", + "py310": "3.10", + "py311": "3.11", + "py312": "3.12", + } + test_jobs_data = [] -def get_test_jobs(tox_ini_path: Path) -> list: + tox_test_env_regex = re_compile( + r"(?Ppy\w+)-test-" + r"(?P[-\w]+\w)-?(?P\d+)?" + ) - config_parser = ConfigParser() - config_parser.read(tox_ini_path) + for tox_env in tox_envs: - long_test_env_regex_counter = 0 - short_test_env_regex_counter = 0 + tox_test_env_match = tox_test_env_regex.match(tox_env) - envs = {} + if tox_test_env_match is None: + continue - for env in config_parser["tox"]["envlist"].split(): - env = env.strip() + groups = tox_test_env_match.groupdict() + + aliased_python_version = python_version_alias(groups["python_version"]) + tox_env = tox_test_env_match.groups(0) + os = groups["os"] + + test_jobs_data.append( + { + "name": f"{tox_env}_{os}", + "ui_name": ( + f"{groups['name']}-" + f"{groups['test-requirements']} " + f"{aliased_python_version}" + f"{os_alias[os]}" + ), + "python_version": aliased_python_version, + "tox_env": tox_test_env_match.groups(0), + "os": os + } + + ) + + return test_jobs_data - if env.startswith(";"): - continue - long_test_env_regex_match = long_test_env_regex.match(env) - short_test_env_regex_match = short_test_env_regex.search(env) - - if long_test_env_regex_match is not None: - long_test_env_regex_counter += 1 - - env_dict = long_test_env_regex_match.groupdict() - env_dict_name = env_dict["name"] - - if env_dict_name not in envs.keys(): - envs[env_dict_name] = [] - - python_test_versions = {"python_versions": [], "test_versions": []} +def get_lint_jobs_data(tox_envs: list) -> list: + pass - if env_dict["pypy"] is not None: - python_test_versions["python_versions"].append("py3") - - if env_dict["cpython_versions"] is not None: - ( - python_test_versions["python_versions"]. - extend( - [ - f"3{cpython_version}" - for cpython_version - in env_dict["cpython_versions"].split(",") - ] - ) - ) - - if env_dict["test_versions"] is not None: - ( - python_test_versions["test_versions"]. - extend(env_dict["test_versions"].split(",")) - ) - - envs[env_dict_name].append(python_test_versions) - - if short_test_env_regex_match is not None: - short_test_env_regex_counter += 1 - - assert short_test_env_regex_counter == long_test_env_regex_counter - - sorted_envs = [] - - for key, value in envs.items(): - sorted_envs.append([key, value]) - - sorted_envs = sorted(sorted_envs) - - jobs = [] - - for os in ["ubuntu-latest"]: - - for env_name, python_test_versions in sorted_envs: - - for python_test_version in python_test_versions: - - for python_version in python_test_version["python_versions"]: - - tox_env = f"py{python_version}-test-{env_name}" - - if python_test_version["test_versions"]: - for test_version in python_test_version["test_versions"]: - - jobs.append( - { - "tox_env": f"{tox_env}-{test_version}", - "python_version": ( - f"{_get_python_version_alias(python_version)}" - ), - "os": os, - "ui_name": ( - f"{env_name}-{test_version} " - f"{_get_python_version_alias(python_version)} " - f"{_get_os_alias(os)}" - ), - "name": ( - f"{env_name}_" - f"{test_version}_" - f"{python_version}_" - f"{os}" - ) - } - ) - - else: - jobs.append( - { - "tox_env": f"{tox_env}", - "python_version": ( - f"{_get_python_version_alias(python_version)}" - ), - "os": os, - "ui_name": ( - f"{env_name} " - f"{_get_python_version_alias(python_version)} " - f"{_get_os_alias(os)}" - ), - "name": ( - f"{env_name}_" - f"{python_version}_" - f"{os}" - ) - } - ) - - return jobs - - -def generate_contrib_workflows(): - - tox_ini_path = Path(argv[1]) - workflows_path = Path(argv[2]) - - with open(workflows_path.joinpath("test.yml"), "w") as test_yml_file: + +def generate_test_workflow(tox_ini_path: Path, workflow_directory_path: Path): + + with ( + open(workflow_directory_path.joinpath("test.yml"), "w") as + test_yml_file + ): test_yml_file.write( Environment( - loader=FileSystemLoader(workflows_path) + loader=FileSystemLoader(Path(__file__).parent) ).get_template("test.yml.j2").render( - jobs=get_test_jobs(tox_ini_path) + test_jobs_data=get_test_jobs_data(tox_ini_path) ) ) test_yml_file.write("\n") + + +def generate_lint_workflow(tox_ini_path: Path, workflow_directory_path: Path): + + with ( + open(workflow_directory_path.joinpath("lint.yml"), "w") as + lint_yml_file + ): + lint_yml_file.write( + Environment( + loader=FileSystemLoader(Path(__file__).parent) + ).get_template("lint.yml.j2").render( + lint_jobs_data=get_lint_jobs_data(tox_ini_path) + ) + ) + lint_yml_file.write("\n") diff --git a/.github/workflows/generate_workflows/src/generate_contrib_workflows/try_parser.py b/.github/workflows/generate_workflows/src/generate_contrib_workflows/try_parser.py deleted file mode 100644 index ec678620e9..0000000000 --- a/.github/workflows/generate_workflows/src/generate_contrib_workflows/try_parser.py +++ /dev/null @@ -1,39 +0,0 @@ -from pathlib import Path -from tox.config.cli.parse import get_options -from tox.session.state import State -from tox.config.sets import CoreConfigSet -from tox.config.source.tox_ini import ToxIni -from collections import defaultdict -from json import dumps - -tox_ini_path = Path( - "/home/tigre/github/ocelotl/opentelemetry-python-contrib/tox.ini" -) - -tox_ini = ToxIni(tox_ini_path) - -conf = State(get_options(), []).conf - -tox_section = next(tox_ini.sections()) - -core_config_set = ( - CoreConfigSet(conf, tox_section, tox_ini_path.parent, tox_ini_path) -) - -( - core_config_set. - loaders. - extend( - tox_ini. - get_loaders( - tox_section, - base=[], - override_map=defaultdict(list, {}), - conf=core_config_set - ) - ) -) - -result = core_config_set.load("env_list") - -print(dumps(result.envs, indent=4))