From 7d301e19a86644fe9a90836c252bcd74596109db Mon Sep 17 00:00:00 2001 From: Martin Basti Date: Tue, 24 Sep 2024 11:27:43 +0200 Subject: [PATCH] cachi2: add init task Init task contains plugins which are responsible for fetching data and validation. It split from prebuild task. Prebuild task compared to init task, does changes to dockerfile. This is prerequisite for having replacebale cachito task with cachi2 task in future, as this task has to be between init and prebuild. Parameter --platforms-result has been moved to init task from prebuild task, as init task now runs check_and_set_platforms plugin. STONEBLD-2587 Signed-off-by: Martin Basti --- atomic_reactor/cli/parser.py | 11 ++++- atomic_reactor/cli/task.py | 15 +++++- atomic_reactor/tasks/binary.py | 47 +++++++++++-------- tekton/pipelines/binary-container.yaml | 39 ++++++++++++++-- tekton/tasks/binary-container-init.yaml | 49 ++++++++++++++++++++ tekton/tasks/binary-container-prebuild.yaml | 7 +-- tests/cli/test_parser.py | 20 ++++++-- tests/cli/test_task.py | 11 +++-- tests/tasks/test_plugin_based.py | 51 +++++++++++++++++---- 9 files changed, 201 insertions(+), 49 deletions(-) create mode 100644 tekton/tasks/binary-container-init.yaml diff --git a/atomic_reactor/cli/parser.py b/atomic_reactor/cli/parser.py index c5072aeeb..c86c38385 100644 --- a/atomic_reactor/cli/parser.py +++ b/atomic_reactor/cli/parser.py @@ -78,14 +78,21 @@ def parse_args(args: Optional[Sequence[str]] = None) -> dict: ) clone.set_defaults(func=task.clone) + binary_container_init = tasks.add_parser( + "binary-container-init", + help="binary container pre-build step", + description="Execute binary container pre-build steps.", + ) + binary_container_init.set_defaults(func=task.binary_container_init) + binary_container_init.add_argument("--platforms-result", metavar="FILE", default=None, + help="file to write final platforms result") + binary_container_prebuild = tasks.add_parser( "binary-container-prebuild", help="binary container pre-build step", description="Execute binary container pre-build steps.", ) binary_container_prebuild.set_defaults(func=task.binary_container_prebuild) - binary_container_prebuild.add_argument("--platforms-result", metavar="FILE", default=None, - help="file to write final platforms result") binary_container_build = tasks.add_parser( "binary-container-build", diff --git a/atomic_reactor/cli/task.py b/atomic_reactor/cli/task.py index 15b844f54..723b4c5c0 100644 --- a/atomic_reactor/cli/task.py +++ b/atomic_reactor/cli/task.py @@ -6,7 +6,8 @@ of the BSD license. See the LICENSE file for details. """ from atomic_reactor.tasks.binary import (BinaryExitTask, BinaryPostBuildTask, BinaryPreBuildTask, - PreBuildTaskParams, BinaryExitTaskParams) + BinaryInitTask, + InitTaskParams, BinaryExitTaskParams) from atomic_reactor.tasks.binary_container_build import BinaryBuildTask, BinaryBuildTaskParams from atomic_reactor.tasks.clone import CloneTask from atomic_reactor.tasks.common import TaskParams @@ -44,12 +45,22 @@ def clone(task_args: dict): return task.run() +def binary_container_init(task_args: dict): + """Run binary container pre-build steps. + + :param task_args: CLI arguments for a binary-container-init task + """ + params = InitTaskParams.from_cli_args(task_args) + task = BinaryInitTask(params) + return task.run() + + def binary_container_prebuild(task_args: dict): """Run binary container pre-build steps. :param task_args: CLI arguments for a binary-container-prebuild task """ - params = PreBuildTaskParams.from_cli_args(task_args) + params = TaskParams.from_cli_args(task_args) task = BinaryPreBuildTask(params) return task.run() diff --git a/atomic_reactor/tasks/binary.py b/atomic_reactor/tasks/binary.py index 1491d9a5a..c7ad97d58 100644 --- a/atomic_reactor/tasks/binary.py +++ b/atomic_reactor/tasks/binary.py @@ -20,8 +20,8 @@ @dataclass(frozen=True) -class PreBuildTaskParams(TaskParams): - """Binary container prebuild task parameters""" +class InitTaskParams(TaskParams): + """Binary container init task parameters""" platforms_result: Optional[str] @@ -31,10 +31,10 @@ class BinaryExitTaskParams(TaskParams): annotations_result: Optional[str] -class BinaryPreBuildTask(plugin_based.PluginBasedTask[PreBuildTaskParams]): +class BinaryInitTask(plugin_based.PluginBasedTask[InitTaskParams]): """Binary container pre-build task.""" - task_name = 'binary_container_prebuild' + task_name = 'binary_container_init_build' plugins_conf = [ {"name": "distgit_fetch_artefacts"}, {"name": "check_and_set_platforms"}, @@ -44,23 +44,9 @@ class BinaryPreBuildTask(plugin_based.PluginBasedTask[PreBuildTaskParams]): {"name": "check_base_image"}, {"name": "koji_parent"}, {"name": "resolve_composes"}, - {"name": "flatpak_update_dockerfile"}, - {"name": "bump_release"}, - {"name": "add_flatpak_labels"}, - {"name": "add_labels_in_dockerfile"}, - {"name": "resolve_remote_source"}, {"name": "pin_operator_digest"}, - {"name": "add_help"}, {"name": "fetch_maven_artifacts"}, - {"name": "add_image_content_manifest"}, - {"name": "add_dockerfile"}, - {"name": "inject_yum_repos"}, - {"name": "add_filesystem"}, - {"name": "change_from_in_dockerfile"}, - {"name": "hide_files"}, - {"name": "distribution_scope"}, - {"name": "add_buildargs_in_dockerfile"}, - {"name": "tag_from_config"}, + ] def prepare_workflow(self) -> inner.DockerBuildWorkflow: @@ -82,6 +68,29 @@ def prepare_workflow(self) -> inner.DockerBuildWorkflow: return workflow +class BinaryPreBuildTask(plugin_based.PluginBasedTask[TaskParams]): + """Binary container pre-build task.""" + + task_name = 'binary_container_prebuild' + plugins_conf = [ + {"name": "flatpak_update_dockerfile"}, + {"name": "bump_release"}, + {"name": "add_flatpak_labels"}, + {"name": "add_labels_in_dockerfile"}, + {"name": "resolve_remote_source"}, + {"name": "add_help"}, + {"name": "add_image_content_manifest"}, + {"name": "add_dockerfile"}, + {"name": "inject_yum_repos"}, + {"name": "add_filesystem"}, + {"name": "change_from_in_dockerfile"}, + {"name": "hide_files"}, + {"name": "distribution_scope"}, + {"name": "add_buildargs_in_dockerfile"}, + {"name": "tag_from_config"}, + ] + + class BinaryPostBuildTask(plugin_based.PluginBasedTask[TaskParams]): """Binary container post-build task.""" diff --git a/tekton/pipelines/binary-container.yaml b/tekton/pipelines/binary-container.yaml index 5adb6a085..5a74c2a24 100644 --- a/tekton/pipelines/binary-container.yaml +++ b/tekton/pipelines/binary-container.yaml @@ -1,7 +1,7 @@ apiVersion: tekton.dev/v1beta1 kind: Pipeline metadata: - name: binary-container-0-1 # dot is not allowed in the name + name: binary-container-0-2 # dot is not allowed in the name spec: params: - name: osbs-image @@ -34,7 +34,7 @@ spec: - name: task_build_result_aarch64 value: $(tasks.binary-container-build-aarch64.results.task_result) - name: platforms_result - value: $(tasks.binary-container-prebuild.results.platforms_result) + value: $(tasks.binary-container-init.results.platforms_result) - name: annotations value: $(finally.binary-container-exit.results.annotations) @@ -66,11 +66,42 @@ spec: value: '$(params.user-params)' timeout: "0" - - name: binary-container-prebuild + - name: binary-container-init runAfter: - clone taskRef: - name: binary-container-prebuild-0-1 + name: binary-container-init-0-1 + workspaces: + - name: ws-build-dir + workspace: ws-container + subPath: build-dir + - name: ws-context-dir + workspace: ws-container + subPath: context-dir + - name: ws-home-dir + workspace: ws-home-dir + - name: ws-registries-secret + workspace: ws-registries-secret + - name: ws-koji-secret + workspace: ws-koji-secret + - name: ws-reactor-config-map + workspace: ws-reactor-config-map + - name: ws-autobot-keytab + workspace: ws-autobot-keytab + params: + - name: osbs-image + value: "$(params.osbs-image)" + - name: pipeline-run-name + value: "$(context.pipelineRun.name)" + - name: user-params + value: '$(params.user-params)' + timeout: "0" + + - name: binary-container-prebuild + runAfter: + - binary-container-init + taskRef: + name: binary-container-prebuild-0-2 workspaces: - name: ws-build-dir workspace: ws-container diff --git a/tekton/tasks/binary-container-init.yaml b/tekton/tasks/binary-container-init.yaml new file mode 100644 index 000000000..a54d9de5b --- /dev/null +++ b/tekton/tasks/binary-container-init.yaml @@ -0,0 +1,49 @@ +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: binary-container-init-0-1 # dot is not allowed in the name +spec: + description: >- + OSBS init task for building binary container image + params: + - name: osbs-image + description: The location of the OSBS builder image (FQDN pullspec) + type: string + - name: pipeline-run-name + type: string + description: PipelineRun name to reference current PipelineRun + - name: user-params + type: string + description: User parameters in JSON format + + workspaces: + - name: ws-build-dir + - name: ws-context-dir + - name: ws-home-dir + - name: ws-registries-secret # access with $(workspaces.ws-registries-secret.path)/token + - name: ws-koji-secret # access with $(workspaces.ws-koji-secret.path)/token + - name: ws-reactor-config-map + - name: ws-autobot-keytab + + results: + - name: platforms_result + + stepTemplate: + env: + - name: HOME + value: $(workspaces.ws-home-dir.path) + + steps: + - name: binary-container-init + image: $(params.osbs-image) + workingDir: $(workspaces.ws-home-dir.path) + resources: + requests: + memory: 512Mi + cpu: 250m + limits: + memory: 1Gi + cpu: 395m + script: | + set -x + atomic-reactor -v task --user-params='$(params.user-params)' --build-dir=$(workspaces.ws-build-dir.path) --context-dir=$(workspaces.ws-context-dir.path) --config-file=$(workspaces.ws-reactor-config-map.path)/config.yaml --namespace=$(context.taskRun.namespace) --pipeline-run-name="$(params.pipeline-run-name)" binary-container-init --platforms-result=$(results.platforms_result.path) diff --git a/tekton/tasks/binary-container-prebuild.yaml b/tekton/tasks/binary-container-prebuild.yaml index 122657a8d..1f634d0ff 100644 --- a/tekton/tasks/binary-container-prebuild.yaml +++ b/tekton/tasks/binary-container-prebuild.yaml @@ -1,7 +1,7 @@ apiVersion: tekton.dev/v1beta1 kind: Task metadata: - name: binary-container-prebuild-0-1 # dot is not allowed in the name + name: binary-container-prebuild-0-2 # dot is not allowed in the name spec: description: >- OSBS prebuild task for building binary container image @@ -25,9 +25,6 @@ spec: - name: ws-reactor-config-map - name: ws-autobot-keytab - results: - - name: platforms_result - stepTemplate: env: - name: HOME @@ -46,4 +43,4 @@ spec: cpu: 395m script: | set -x - atomic-reactor -v task --user-params='$(params.user-params)' --build-dir=$(workspaces.ws-build-dir.path) --context-dir=$(workspaces.ws-context-dir.path) --config-file=$(workspaces.ws-reactor-config-map.path)/config.yaml --namespace=$(context.taskRun.namespace) --pipeline-run-name="$(params.pipeline-run-name)" binary-container-prebuild --platforms-result=$(results.platforms_result.path) + atomic-reactor -v task --user-params='$(params.user-params)' --build-dir=$(workspaces.ws-build-dir.path) --context-dir=$(workspaces.ws-context-dir.path) --config-file=$(workspaces.ws-reactor-config-map.path)/config.yaml --namespace=$(context.taskRun.namespace) --pipeline-run-name="$(params.pipeline-run-name)" binary-container-prebuild diff --git a/tests/cli/test_parser.py b/tests/cli/test_parser.py index c20c24724..6caebd9fc 100644 --- a/tests/cli/test_parser.py +++ b/tests/cli/test_parser.py @@ -41,7 +41,7 @@ **EXPECTED_ARGS, "platform": "x86_64", } -EXPECTED_ARGS_CONTAINER_PREBUILD = { +EXPECTED_ARGS_CONTAINER_INIT = { **EXPECTED_ARGS, "platforms_result": None, } @@ -79,9 +79,13 @@ def test_parse_args_version(capsys): ["task", *REQUIRED_COMMON_ARGS, "clone"], {**EXPECTED_ARGS, "func": task.clone}, ), + ( + ["task", *REQUIRED_COMMON_ARGS, "binary-container-init"], + {**EXPECTED_ARGS_CONTAINER_INIT, "func": task.binary_container_init}, + ), ( ["task", *REQUIRED_COMMON_ARGS, "binary-container-prebuild"], - {**EXPECTED_ARGS_CONTAINER_PREBUILD, "func": task.binary_container_prebuild}, + {**EXPECTED_ARGS, "func": task.binary_container_prebuild}, ), ( ["task", *REQUIRED_COMMON_ARGS, "binary-container-build", @@ -107,10 +111,16 @@ def test_parse_args_version(capsys): ["task", *REQUIRED_COMMON_ARGS, "--config-file=config.yaml", "clone"], {**EXPECTED_ARGS, "config_file": "config.yaml", "func": task.clone}, ), + ( + ["task", *REQUIRED_COMMON_ARGS, "--config-file=config.yaml", + "binary-container-init"], + {**EXPECTED_ARGS_CONTAINER_INIT, "config_file": "config.yaml", + "func": task.binary_container_init}, + ), ( ["task", *REQUIRED_COMMON_ARGS, "--config-file=config.yaml", "binary-container-prebuild"], - {**EXPECTED_ARGS_CONTAINER_PREBUILD, "config_file": "config.yaml", + {**EXPECTED_ARGS, "config_file": "config.yaml", "func": task.binary_container_prebuild}, ), ( @@ -132,10 +142,10 @@ def test_parse_args_version(capsys): "func": task.binary_container_postbuild}, ), ( - ["task", *REQUIRED_COMMON_ARGS, "binary-container-prebuild", + ["task", *REQUIRED_COMMON_ARGS, "binary-container-init", "--platforms-result=platforms_file"], {**EXPECTED_ARGS, "platforms_result": "platforms_file", - "func": task.binary_container_prebuild}, + "func": task.binary_container_init}, ), ( ["task", *REQUIRED_COMMON_ARGS, "--config-file=config.yaml", "binary-container-exit", diff --git a/tests/cli/test_task.py b/tests/cli/test_task.py index 0c39cf7da..16d03a803 100644 --- a/tests/cli/test_task.py +++ b/tests/cli/test_task.py @@ -28,7 +28,7 @@ "config_file": "reactor-config-map.yaml", "user_params": '{"some_param": "some_value"}', } -PRE_TASK_ARGS = { +INIT_TASK_ARGS = { **TASK_ARGS, "platforms_result": 'platform_result', } @@ -60,9 +60,14 @@ def test_source_build(): assert task.source_container_build(TASK_ARGS) == TASK_RESULT +def test_binary_container_init(): + mock(binary.BinaryInitTask, task_args=INIT_TASK_ARGS) + assert task.binary_container_init(INIT_TASK_ARGS) == TASK_RESULT + + def test_binary_container_prebuild(): - mock(binary.BinaryPreBuildTask, task_args=PRE_TASK_ARGS) - assert task.binary_container_prebuild(PRE_TASK_ARGS) == TASK_RESULT + mock(binary.BinaryPreBuildTask, task_args=TASK_ARGS) + assert task.binary_container_prebuild(TASK_ARGS) == TASK_RESULT def test_binary_container_build(): diff --git a/tests/tasks/test_plugin_based.py b/tests/tasks/test_plugin_based.py index ccb7c5b5d..f951e201c 100644 --- a/tests/tasks/test_plugin_based.py +++ b/tests/tasks/test_plugin_based.py @@ -17,7 +17,7 @@ from atomic_reactor.inner import ImageBuildWorkflowData from atomic_reactor.plugin import TaskCanceledException, PluginFailedException from atomic_reactor.tasks.common import TaskParams -from atomic_reactor.tasks.binary import PreBuildTaskParams, BinaryPreBuildTask +from atomic_reactor.tasks.binary import InitTaskParams, BinaryPreBuildTask, BinaryInitTask from atomic_reactor.util import DockerfileImages from atomic_reactor import inner, dirs @@ -194,18 +194,51 @@ def _cancel_build(*args, **kwargs): assert {} == wf_data.plugins_results +def test_ensure_workflow_data_is_saved_init_task( + build_dir, dummy_source, tmpdir +): + context_dir = tmpdir.join("context_dir").mkdir() + params = InitTaskParams(build_dir=str(build_dir), + config_file="config.yaml", + context_dir=str(context_dir), + namespace="test-namespace", + pipeline_run_name='test-pipeline-run', + user_params={}, + task_result='results', + platforms_result='platforms_result') + (flexmock(params) + .should_receive("source") + .and_return(dummy_source)) + + task = BinaryInitTask(params) + + (flexmock(plugin_based.inner.DockerBuildWorkflow) + .should_receive("build_container_image") + .once()) + + task.run() + time.sleep(1) + assert context_dir.join("workflow.json").exists() + + wf_data = ImageBuildWorkflowData() + wf_data.load_from_dir(ContextDir(Path(context_dir))) + # As long as the data is loaded successfully, just check some + # attributes to check the data. + assert DockerfileImages() == wf_data.dockerfile_images + assert {} == wf_data.plugins_results + + def test_ensure_workflow_data_is_saved_prebuild_task( build_dir, dummy_source, tmpdir ): context_dir = tmpdir.join("context_dir").mkdir() - params = PreBuildTaskParams(build_dir=str(build_dir), - config_file="config.yaml", - context_dir=str(context_dir), - namespace="test-namespace", - pipeline_run_name='test-pipeline-run', - user_params={}, - task_result='results', - platforms_result='platforms_result') + params = TaskParams(build_dir=str(build_dir), + config_file="config.yaml", + context_dir=str(context_dir), + namespace="test-namespace", + pipeline_run_name='test-pipeline-run', + user_params={}, + task_result='results') (flexmock(params) .should_receive("source") .and_return(dummy_source))