From ef103d6d587536aefdcbb35a46a386ce26eedd03 Mon Sep 17 00:00:00 2001 From: Simeon Ehrig Date: Tue, 5 Sep 2023 15:37:37 +0200 Subject: [PATCH] GitLab CI: enable runtime tests for gcc and clang cpu jobs --- .gitlab-ci.yml | 25 ++++++++++++++++------- script/job_generator/generate_job_yaml.py | 10 +++++---- script/job_generator/job_generator.py | 22 ++++++++------------ script/job_generator/job_modifier.py | 11 +++++++--- 4 files changed, 41 insertions(+), 27 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f2a3ed5c009c..549f8ed9b573 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,15 +26,18 @@ generate: script: - apk update && apk add python3~=3.11 py3-pip - pip3 install -r script/job_generator/requirements.txt - # it's enough to verify one time, because --compile-only und --runtime-only generates the same job matrix and filter it - - python3 script/job_generator/job_generator.py ${ALPAKA_GITLAB_CI_GENERATOR_CONTAINER_VERSION} --verify --compile-only -o compile_only.yml - - python3 script/job_generator/job_generator.py ${ALPAKA_GITLAB_CI_GENERATOR_CONTAINER_VERSION} --runtime-only -o runtime.yml + # it is sufficient to verify once, as the same job matrix is generated, verified and then filtered each time + - python3 script/job_generator/job_generator.py ${ALPAKA_GITLAB_CI_GENERATOR_CONTAINER_VERSION} --verify --wave compile_only_job -o compile_only.yml + - python3 script/job_generator/job_generator.py ${ALPAKA_GITLAB_CI_GENERATOR_CONTAINER_VERSION} --wave runtime_job_cpu -o runtime_cpu.yml + - python3 script/job_generator/job_generator.py ${ALPAKA_GITLAB_CI_GENERATOR_CONTAINER_VERSION} --wave runtime_job_gpu -o runtime_gpu.yml - cat compile_only.yml - - cat runtime.yml + - cat runtime_cpu.yml + - cat runtime_gpu.yml artifacts: paths: - compile_only.yml - - runtime.yml + - runtime_cpu.yml + - runtime_gpu.yml expire_in: 1 week interruptible: true @@ -46,10 +49,18 @@ run-compile-only-test: job: generate strategy: depend -run-runtime-test: +run-runtime-cpu-test: stage: run-test-jobs trigger: include: - - artifact: runtime.yml + - artifact: runtime_cpu.yml + job: generate + strategy: depend + +run-runtime-gpu-test: + stage: run-test-jobs + trigger: + include: + - artifact: runtime_gpu.yml job: generate strategy: depend diff --git a/script/job_generator/generate_job_yaml.py b/script/job_generator/generate_job_yaml.py index 79f48f6cafb9..8d925778d2c0 100644 --- a/script/job_generator/generate_job_yaml.py +++ b/script/job_generator/generate_job_yaml.py @@ -14,7 +14,8 @@ from util import print_warn, exit_error JOB_COMPILE_ONLY = "compile_only_job" -JOB_RUNTIME = "runtime_job" +JOB_RUNTIME = "runtime_job_gpu" +JOB_CPU_RUNTIME = "runtime_job_cpu" JOB_ROCM_RUNTIME = "rocm_runtime_job" JOB_NVCC_GCC_RUNTIME = "nvcc_gcc_runtime_job" JOB_NVCC_CLANG_RUNTIME = "nvcc_clng_runtime_job" @@ -25,8 +26,8 @@ WAVE_GROUP_NAMES = [ JOB_COMPILE_ONLY, JOB_RUNTIME, + JOB_CPU_RUNTIME, # can be enabled again, if fine granular scheduling is required - # JOB_CPU_RUNTIME, # JOB_ROCM_RUNTIME, # JOB_NVCC_GCC_RUNTIME, # JOB_NVCC_CLANG_RUNTIME, @@ -562,12 +563,13 @@ def distribute_to_waves( job_name: str = next(iter(job)) if "_compile_only_" in job_name: sorted_groups[JOB_COMPILE_ONLY].append(job) + elif job_name.startswith("linux_gcc"): + sorted_groups[JOB_CPU_RUNTIME].append(job) # Clang as C++ compiler without CUDA backend elif job_name.startswith("linux_clang") and not job_name.startswith( "linux_clang-cuda" ): - # sorted_groups[JOB_CPU_RUNTIME].append(job) - sorted_groups[JOB_COMPILE_ONLY].append(job) + sorted_groups[JOB_CPU_RUNTIME].append(job) elif job_name.startswith("linux_hipcc"): # sorted_groups[JOB_ROCM_RUNTIME].append(job) sorted_groups[JOB_RUNTIME].append(job) diff --git a/script/job_generator/job_generator.py b/script/job_generator/job_generator.py index 97cf7b8cca7e..defcd21b0867 100644 --- a/script/job_generator/job_generator.py +++ b/script/job_generator/job_generator.py @@ -88,11 +88,12 @@ def get_args() -> argparse.Namespace: ) parser.add_argument( - "--compile-only", action="store_true", help="Generate only compile only jobs." - ) - - parser.add_argument( - "--runtime-only", action="store_true", help="Generate only runtime jobs." + "--wave", + type=str, + # add `all` and remove `JOB_UNKNOWN` from the choices + choices=["all"] + list(set(WAVE_GROUP_NAMES) - set([JOB_UNKNOWN])), + default="all", + help="Generates only jobs for a specific wave group.", ) parser.add_argument( @@ -201,14 +202,9 @@ def get_args() -> argparse.Namespace: for job in wave: print(job) - if args.compile_only: - wave_job_matrix = {JOB_COMPILE_ONLY: wave_job_matrix[JOB_COMPILE_ONLY]} - for wave_name in WAVE_GROUP_NAMES: - if not wave_name in wave_job_matrix: - wave_job_matrix[wave_name] = [] - - if args.runtime_only: - wave_job_matrix = {JOB_RUNTIME: wave_job_matrix[JOB_RUNTIME]} + if args.wave != "all": + filter_wave_name = args.wave + wave_job_matrix = {filter_wave_name: wave_job_matrix[filter_wave_name]} for wave_name in WAVE_GROUP_NAMES: if not wave_name in wave_job_matrix: wave_job_matrix[wave_name] = [] diff --git a/script/job_generator/job_modifier.py b/script/job_generator/job_modifier.py index d56aeefe3fc4..7a4310f2db58 100644 --- a/script/job_generator/job_modifier.py +++ b/script/job_generator/job_modifier.py @@ -39,7 +39,7 @@ def add_job_parameters(job_matrix: List[Dict[str, Tuple[str, str]]]): # ): # job[JOB_EXECUTION_TYPE] = (JOB_EXECUTION_TYPE, JOB_EXECUTION_RUNTIME) # break - + ############################################## ## Defining runtime test for the CUDA backend ############################################## @@ -66,7 +66,7 @@ def add_job_parameters(job_matrix: List[Dict[str, Tuple[str, str]]]): for job in job_matrix: if ( ALPAKA_ACC_GPU_CUDA_ENABLE in job - and job[ALPAKA_ACC_GPU_CUDA_ENABLE][VERSION] !=OFF_VER + and job[ALPAKA_ACC_GPU_CUDA_ENABLE][VERSION] != OFF_VER ): v = version.parse(job[ALPAKA_ACC_GPU_CUDA_ENABLE][VERSION]) if not v.major in latest_CUDA_SDK_minor_versions[job[HOST_COMPILER][NAME]]: @@ -142,8 +142,13 @@ def add_job_parameters(job_matrix: List[Dict[str, Tuple[str, str]]]): missing_nvcc_versions.remove(job[DEVICE_COMPILER][VERSION]) elif ( ALPAKA_ACC_GPU_CUDA_ENABLE in job - and ALPAKA_ACC_GPU_CUDA_ENABLE[VERSION] !=OFF_VER + and ALPAKA_ACC_GPU_CUDA_ENABLE[VERSION] != OFF_VER ): job[SM_LEVEL] = (SM_LEVEL, STANDARD_SM_LEVEL) else: job[SM_LEVEL] = (SM_LEVEL, "") + + # run tests each time if a CPU backend is used + for job in job_matrix: + if job[DEVICE_COMPILER][NAME] in [GCC, CLANG]: + job[JOB_EXECUTION_TYPE] = (JOB_EXECUTION_TYPE, JOB_EXECUTION_RUNTIME)