diff --git a/atomic_reactor/cli/parser.py b/atomic_reactor/cli/parser.py index c5072aeeb..c86c38385 100644 --- a/atomic_reactor/cli/parser.py +++ b/atomic_reactor/cli/parser.py @@ -78,14 +78,21 @@ def parse_args(args: Optional[Sequence[str]] = None) -> dict: ) clone.set_defaults(func=task.clone) + binary_container_init = tasks.add_parser( + "binary-container-init", + help="binary container pre-build step", + description="Execute binary container pre-build steps.", + ) + binary_container_init.set_defaults(func=task.binary_container_init) + binary_container_init.add_argument("--platforms-result", metavar="FILE", default=None, + help="file to write final platforms result") + binary_container_prebuild = tasks.add_parser( "binary-container-prebuild", help="binary container pre-build step", description="Execute binary container pre-build steps.", ) binary_container_prebuild.set_defaults(func=task.binary_container_prebuild) - binary_container_prebuild.add_argument("--platforms-result", metavar="FILE", default=None, - help="file to write final platforms result") binary_container_build = tasks.add_parser( "binary-container-build", diff --git a/atomic_reactor/cli/task.py b/atomic_reactor/cli/task.py index 15b844f54..723b4c5c0 100644 --- a/atomic_reactor/cli/task.py +++ b/atomic_reactor/cli/task.py @@ -6,7 +6,8 @@ of the BSD license. See the LICENSE file for details. """ from atomic_reactor.tasks.binary import (BinaryExitTask, BinaryPostBuildTask, BinaryPreBuildTask, - PreBuildTaskParams, BinaryExitTaskParams) + BinaryInitTask, + InitTaskParams, BinaryExitTaskParams) from atomic_reactor.tasks.binary_container_build import BinaryBuildTask, BinaryBuildTaskParams from atomic_reactor.tasks.clone import CloneTask from atomic_reactor.tasks.common import TaskParams @@ -44,12 +45,22 @@ def clone(task_args: dict): return task.run() +def binary_container_init(task_args: dict): + """Run binary container pre-build steps. + + :param task_args: CLI arguments for a binary-container-init task + """ + params = InitTaskParams.from_cli_args(task_args) + task = BinaryInitTask(params) + return task.run() + + def binary_container_prebuild(task_args: dict): """Run binary container pre-build steps. :param task_args: CLI arguments for a binary-container-prebuild task """ - params = PreBuildTaskParams.from_cli_args(task_args) + params = TaskParams.from_cli_args(task_args) task = BinaryPreBuildTask(params) return task.run() diff --git a/atomic_reactor/tasks/binary.py b/atomic_reactor/tasks/binary.py index 1491d9a5a..c7ad97d58 100644 --- a/atomic_reactor/tasks/binary.py +++ b/atomic_reactor/tasks/binary.py @@ -20,8 +20,8 @@ @dataclass(frozen=True) -class PreBuildTaskParams(TaskParams): - """Binary container prebuild task parameters""" +class InitTaskParams(TaskParams): + """Binary container init task parameters""" platforms_result: Optional[str] @@ -31,10 +31,10 @@ class BinaryExitTaskParams(TaskParams): annotations_result: Optional[str] -class BinaryPreBuildTask(plugin_based.PluginBasedTask[PreBuildTaskParams]): +class BinaryInitTask(plugin_based.PluginBasedTask[InitTaskParams]): """Binary container pre-build task.""" - task_name = 'binary_container_prebuild' + task_name = 'binary_container_init_build' plugins_conf = [ {"name": "distgit_fetch_artefacts"}, {"name": "check_and_set_platforms"}, @@ -44,23 +44,9 @@ class BinaryPreBuildTask(plugin_based.PluginBasedTask[PreBuildTaskParams]): {"name": "check_base_image"}, {"name": "koji_parent"}, {"name": "resolve_composes"}, - {"name": "flatpak_update_dockerfile"}, - {"name": "bump_release"}, - {"name": "add_flatpak_labels"}, - {"name": "add_labels_in_dockerfile"}, - {"name": "resolve_remote_source"}, {"name": "pin_operator_digest"}, - {"name": "add_help"}, {"name": "fetch_maven_artifacts"}, - {"name": "add_image_content_manifest"}, - {"name": "add_dockerfile"}, - {"name": "inject_yum_repos"}, - {"name": "add_filesystem"}, - {"name": "change_from_in_dockerfile"}, - {"name": "hide_files"}, - {"name": "distribution_scope"}, - {"name": "add_buildargs_in_dockerfile"}, - {"name": "tag_from_config"}, + ] def prepare_workflow(self) -> inner.DockerBuildWorkflow: @@ -82,6 +68,29 @@ def prepare_workflow(self) -> inner.DockerBuildWorkflow: return workflow +class BinaryPreBuildTask(plugin_based.PluginBasedTask[TaskParams]): + """Binary container pre-build task.""" + + task_name = 'binary_container_prebuild' + plugins_conf = [ + {"name": "flatpak_update_dockerfile"}, + {"name": "bump_release"}, + {"name": "add_flatpak_labels"}, + {"name": "add_labels_in_dockerfile"}, + {"name": "resolve_remote_source"}, + {"name": "add_help"}, + {"name": "add_image_content_manifest"}, + {"name": "add_dockerfile"}, + {"name": "inject_yum_repos"}, + {"name": "add_filesystem"}, + {"name": "change_from_in_dockerfile"}, + {"name": "hide_files"}, + {"name": "distribution_scope"}, + {"name": "add_buildargs_in_dockerfile"}, + {"name": "tag_from_config"}, + ] + + class BinaryPostBuildTask(plugin_based.PluginBasedTask[TaskParams]): """Binary container post-build task.""" diff --git a/tekton/pipelines/binary-container.yaml b/tekton/pipelines/binary-container.yaml index 5adb6a085..5a74c2a24 100644 --- a/tekton/pipelines/binary-container.yaml +++ b/tekton/pipelines/binary-container.yaml @@ -1,7 +1,7 @@ apiVersion: tekton.dev/v1beta1 kind: Pipeline metadata: - name: binary-container-0-1 # dot is not allowed in the name + name: binary-container-0-2 # dot is not allowed in the name spec: params: - name: osbs-image @@ -34,7 +34,7 @@ spec: - name: task_build_result_aarch64 value: $(tasks.binary-container-build-aarch64.results.task_result) - name: platforms_result - value: $(tasks.binary-container-prebuild.results.platforms_result) + value: $(tasks.binary-container-init.results.platforms_result) - name: annotations value: $(finally.binary-container-exit.results.annotations) @@ -66,11 +66,42 @@ spec: value: '$(params.user-params)' timeout: "0" - - name: binary-container-prebuild + - name: binary-container-init runAfter: - clone taskRef: - name: binary-container-prebuild-0-1 + name: binary-container-init-0-1 + workspaces: + - name: ws-build-dir + workspace: ws-container + subPath: build-dir + - name: ws-context-dir + workspace: ws-container + subPath: context-dir + - name: ws-home-dir + workspace: ws-home-dir + - name: ws-registries-secret + workspace: ws-registries-secret + - name: ws-koji-secret + workspace: ws-koji-secret + - name: ws-reactor-config-map + workspace: ws-reactor-config-map + - name: ws-autobot-keytab + workspace: ws-autobot-keytab + params: + - name: osbs-image + value: "$(params.osbs-image)" + - name: pipeline-run-name + value: "$(context.pipelineRun.name)" + - name: user-params + value: '$(params.user-params)' + timeout: "0" + + - name: binary-container-prebuild + runAfter: + - binary-container-init + taskRef: + name: binary-container-prebuild-0-2 workspaces: - name: ws-build-dir workspace: ws-container diff --git a/tekton/tasks/binary-container-init.yaml b/tekton/tasks/binary-container-init.yaml new file mode 100644 index 000000000..a54d9de5b --- /dev/null +++ b/tekton/tasks/binary-container-init.yaml @@ -0,0 +1,49 @@ +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: binary-container-init-0-1 # dot is not allowed in the name +spec: + description: >- + OSBS init task for building binary container image + params: + - name: osbs-image + description: The location of the OSBS builder image (FQDN pullspec) + type: string + - name: pipeline-run-name + type: string + description: PipelineRun name to reference current PipelineRun + - name: user-params + type: string + description: User parameters in JSON format + + workspaces: + - name: ws-build-dir + - name: ws-context-dir + - name: ws-home-dir + - name: ws-registries-secret # access with $(workspaces.ws-registries-secret.path)/token + - name: ws-koji-secret # access with $(workspaces.ws-koji-secret.path)/token + - name: ws-reactor-config-map + - name: ws-autobot-keytab + + results: + - name: platforms_result + + stepTemplate: + env: + - name: HOME + value: $(workspaces.ws-home-dir.path) + + steps: + - name: binary-container-init + image: $(params.osbs-image) + workingDir: $(workspaces.ws-home-dir.path) + resources: + requests: + memory: 512Mi + cpu: 250m + limits: + memory: 1Gi + cpu: 395m + script: | + set -x + atomic-reactor -v task --user-params='$(params.user-params)' --build-dir=$(workspaces.ws-build-dir.path) --context-dir=$(workspaces.ws-context-dir.path) --config-file=$(workspaces.ws-reactor-config-map.path)/config.yaml --namespace=$(context.taskRun.namespace) --pipeline-run-name="$(params.pipeline-run-name)" binary-container-init --platforms-result=$(results.platforms_result.path) diff --git a/tekton/tasks/binary-container-prebuild.yaml b/tekton/tasks/binary-container-prebuild.yaml index 122657a8d..1f634d0ff 100644 --- a/tekton/tasks/binary-container-prebuild.yaml +++ b/tekton/tasks/binary-container-prebuild.yaml @@ -1,7 +1,7 @@ apiVersion: tekton.dev/v1beta1 kind: Task metadata: - name: binary-container-prebuild-0-1 # dot is not allowed in the name + name: binary-container-prebuild-0-2 # dot is not allowed in the name spec: description: >- OSBS prebuild task for building binary container image @@ -25,9 +25,6 @@ spec: - name: ws-reactor-config-map - name: ws-autobot-keytab - results: - - name: platforms_result - stepTemplate: env: - name: HOME @@ -46,4 +43,4 @@ spec: cpu: 395m script: | set -x - atomic-reactor -v task --user-params='$(params.user-params)' --build-dir=$(workspaces.ws-build-dir.path) --context-dir=$(workspaces.ws-context-dir.path) --config-file=$(workspaces.ws-reactor-config-map.path)/config.yaml --namespace=$(context.taskRun.namespace) --pipeline-run-name="$(params.pipeline-run-name)" binary-container-prebuild --platforms-result=$(results.platforms_result.path) + atomic-reactor -v task --user-params='$(params.user-params)' --build-dir=$(workspaces.ws-build-dir.path) --context-dir=$(workspaces.ws-context-dir.path) --config-file=$(workspaces.ws-reactor-config-map.path)/config.yaml --namespace=$(context.taskRun.namespace) --pipeline-run-name="$(params.pipeline-run-name)" binary-container-prebuild diff --git a/tests/cli/test_parser.py b/tests/cli/test_parser.py index c20c24724..6caebd9fc 100644 --- a/tests/cli/test_parser.py +++ b/tests/cli/test_parser.py @@ -41,7 +41,7 @@ **EXPECTED_ARGS, "platform": "x86_64", } -EXPECTED_ARGS_CONTAINER_PREBUILD = { +EXPECTED_ARGS_CONTAINER_INIT = { **EXPECTED_ARGS, "platforms_result": None, } @@ -79,9 +79,13 @@ def test_parse_args_version(capsys): ["task", *REQUIRED_COMMON_ARGS, "clone"], {**EXPECTED_ARGS, "func": task.clone}, ), + ( + ["task", *REQUIRED_COMMON_ARGS, "binary-container-init"], + {**EXPECTED_ARGS_CONTAINER_INIT, "func": task.binary_container_init}, + ), ( ["task", *REQUIRED_COMMON_ARGS, "binary-container-prebuild"], - {**EXPECTED_ARGS_CONTAINER_PREBUILD, "func": task.binary_container_prebuild}, + {**EXPECTED_ARGS, "func": task.binary_container_prebuild}, ), ( ["task", *REQUIRED_COMMON_ARGS, "binary-container-build", @@ -107,10 +111,16 @@ def test_parse_args_version(capsys): ["task", *REQUIRED_COMMON_ARGS, "--config-file=config.yaml", "clone"], {**EXPECTED_ARGS, "config_file": "config.yaml", "func": task.clone}, ), + ( + ["task", *REQUIRED_COMMON_ARGS, "--config-file=config.yaml", + "binary-container-init"], + {**EXPECTED_ARGS_CONTAINER_INIT, "config_file": "config.yaml", + "func": task.binary_container_init}, + ), ( ["task", *REQUIRED_COMMON_ARGS, "--config-file=config.yaml", "binary-container-prebuild"], - {**EXPECTED_ARGS_CONTAINER_PREBUILD, "config_file": "config.yaml", + {**EXPECTED_ARGS, "config_file": "config.yaml", "func": task.binary_container_prebuild}, ), ( @@ -132,10 +142,10 @@ def test_parse_args_version(capsys): "func": task.binary_container_postbuild}, ), ( - ["task", *REQUIRED_COMMON_ARGS, "binary-container-prebuild", + ["task", *REQUIRED_COMMON_ARGS, "binary-container-init", "--platforms-result=platforms_file"], {**EXPECTED_ARGS, "platforms_result": "platforms_file", - "func": task.binary_container_prebuild}, + "func": task.binary_container_init}, ), ( ["task", *REQUIRED_COMMON_ARGS, "--config-file=config.yaml", "binary-container-exit", diff --git a/tests/cli/test_task.py b/tests/cli/test_task.py index 0c39cf7da..16d03a803 100644 --- a/tests/cli/test_task.py +++ b/tests/cli/test_task.py @@ -28,7 +28,7 @@ "config_file": "reactor-config-map.yaml", "user_params": '{"some_param": "some_value"}', } -PRE_TASK_ARGS = { +INIT_TASK_ARGS = { **TASK_ARGS, "platforms_result": 'platform_result', } @@ -60,9 +60,14 @@ def test_source_build(): assert task.source_container_build(TASK_ARGS) == TASK_RESULT +def test_binary_container_init(): + mock(binary.BinaryInitTask, task_args=INIT_TASK_ARGS) + assert task.binary_container_init(INIT_TASK_ARGS) == TASK_RESULT + + def test_binary_container_prebuild(): - mock(binary.BinaryPreBuildTask, task_args=PRE_TASK_ARGS) - assert task.binary_container_prebuild(PRE_TASK_ARGS) == TASK_RESULT + mock(binary.BinaryPreBuildTask, task_args=TASK_ARGS) + assert task.binary_container_prebuild(TASK_ARGS) == TASK_RESULT def test_binary_container_build(): diff --git a/tests/tasks/test_plugin_based.py b/tests/tasks/test_plugin_based.py index ccb7c5b5d..f951e201c 100644 --- a/tests/tasks/test_plugin_based.py +++ b/tests/tasks/test_plugin_based.py @@ -17,7 +17,7 @@ from atomic_reactor.inner import ImageBuildWorkflowData from atomic_reactor.plugin import TaskCanceledException, PluginFailedException from atomic_reactor.tasks.common import TaskParams -from atomic_reactor.tasks.binary import PreBuildTaskParams, BinaryPreBuildTask +from atomic_reactor.tasks.binary import InitTaskParams, BinaryPreBuildTask, BinaryInitTask from atomic_reactor.util import DockerfileImages from atomic_reactor import inner, dirs @@ -194,18 +194,51 @@ def _cancel_build(*args, **kwargs): assert {} == wf_data.plugins_results +def test_ensure_workflow_data_is_saved_init_task( + build_dir, dummy_source, tmpdir +): + context_dir = tmpdir.join("context_dir").mkdir() + params = InitTaskParams(build_dir=str(build_dir), + config_file="config.yaml", + context_dir=str(context_dir), + namespace="test-namespace", + pipeline_run_name='test-pipeline-run', + user_params={}, + task_result='results', + platforms_result='platforms_result') + (flexmock(params) + .should_receive("source") + .and_return(dummy_source)) + + task = BinaryInitTask(params) + + (flexmock(plugin_based.inner.DockerBuildWorkflow) + .should_receive("build_container_image") + .once()) + + task.run() + time.sleep(1) + assert context_dir.join("workflow.json").exists() + + wf_data = ImageBuildWorkflowData() + wf_data.load_from_dir(ContextDir(Path(context_dir))) + # As long as the data is loaded successfully, just check some + # attributes to check the data. + assert DockerfileImages() == wf_data.dockerfile_images + assert {} == wf_data.plugins_results + + def test_ensure_workflow_data_is_saved_prebuild_task( build_dir, dummy_source, tmpdir ): context_dir = tmpdir.join("context_dir").mkdir() - params = PreBuildTaskParams(build_dir=str(build_dir), - config_file="config.yaml", - context_dir=str(context_dir), - namespace="test-namespace", - pipeline_run_name='test-pipeline-run', - user_params={}, - task_result='results', - platforms_result='platforms_result') + params = TaskParams(build_dir=str(build_dir), + config_file="config.yaml", + context_dir=str(context_dir), + namespace="test-namespace", + pipeline_run_name='test-pipeline-run', + user_params={}, + task_result='results') (flexmock(params) .should_receive("source") .and_return(dummy_source))