diff --git a/CHANGELOG.md b/CHANGELOG.md index d43d0e6..2a7147c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +## 0.18.17 (Unreleased) + + +### Features +* feat: Add task chunking support to Nuke submitter. Frames are now rendered using + the OpenJD TASK_CHUNKING extension with contiguous chunks. Chunk size defaults to 1 + (one frame per task, same as before). Increase chunk size to group frames and reduce + per-task overhead. Optional target chunk duration enables dynamic chunk sizing. + +### Important +* This version requires a worker agent that supports the TASK_CHUNKING extension. + Service-managed fleets always use a compatible version. If you use customer-managed + fleets, ensure your worker agents are updated before submitting jobs. + ## 0.18.16 (2026-01-20) Added support for training CopyCat nodes. Users are now able to select to submit a CopyCat training job. A CopyCat training job will perform CopyCat training on the render farm, rather than on the local system. diff --git a/README.md b/README.md index 93c9f8d..0702fdc 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,10 @@ This library requires: This package provides a Nuke plugin that creates jobs for AWS Deadline Cloud using the [AWS Deadline Cloud client library][deadline-cloud-client]. Based on the loaded comp it determines the files required, allows the user to specify render options, and builds an [OpenJD template][openjd] that defines the workflow. +The submitter supports [task chunking][task-chunking], which groups multiple frames into contiguous chunks to reduce per-task overhead. When combined with the adaptor's sticky rendering, this provides optimal performance by eliminating both repeated application startup and scene loading time. + +[task-chunking]: https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/build-job-bundle-chunking.html + ## Adaptor The Nuke Adaptor implements the [OpenJD][openjd-adaptor-runtime] interface that allows render workloads to launch Nuke and feed it commands. This gives the following benefits: diff --git a/docs/user_guide/using-submitter/render-submissions.md b/docs/user_guide/using-submitter/render-submissions.md index da7c252..8d26898 100644 --- a/docs/user_guide/using-submitter/render-submissions.md +++ b/docs/user_guide/using-submitter/render-submissions.md @@ -25,6 +25,8 @@ The **Job-specific settings** tab has options specific to jobs created in Nuke. - *Override frame range* - Select this to render a different frame or frame range than is set in Nuke. Frame ranges follow the [Open Job Description](https://github.com/OpenJobDescription/openjd-specifications/wiki/2023-09-Template-Schemas#34111-intrangeexpr) pattern. - *Use proxy mode* - Manages whether to use [proxy mode](https://learn.foundry.com/nuke/9.0/content/getting_started/managing_scripts/proxy_mode.html) in the submitted job. - *Continue on error* - If set, try to continue rendering if Nuke encounters an error. If false, in the case of an error the task is failed. + - *Chunk size* - Number of frames to group into each chunk (1-150). Use 1 for one frame per task (default). Higher values group frames into contiguous chunks to reduce per-task overhead. For more information, see [Task chunking for job templates](https://docs.aws.amazon.com/deadline-cloud/latest/developerguide/build-job-bundle-chunking.html). + - *Target chunk duration (seconds)* - When set, the scheduler dynamically adjusts chunk sizes based on observed runtimes of completed chunks, aiming for this duration per chunk. Leave at 0 to use a fixed chunk size for all chunks. - *Use timeouts* - Whether or not to use user configured timeouts. - *Render task timeout* - Maximum duration of each action which performs a render. Default is 6 days. - *Setup timeout* - Maximum duration of each action which sets up the job for rendering, such as scene load. Default is 1 day. diff --git a/src/deadline/nuke_submitter/data_classes.py b/src/deadline/nuke_submitter/data_classes.py index 0817a2e..f7e1b1a 100644 --- a/src/deadline/nuke_submitter/data_classes.py +++ b/src/deadline/nuke_submitter/data_classes.py @@ -26,6 +26,8 @@ class RenderSettings: view_selection: str = field(default="", metadata={"sticky": True}) is_proxy_mode: bool = field(default=False, metadata={"sticky": True}) continue_on_error: bool = field(default=False, metadata={"sticky": True}) + chunk_size: int = field(default=1, metadata={"sticky": True}) + target_chunk_duration: int = field(default=0, metadata={"sticky": True}) @dataclass diff --git a/src/deadline/nuke_submitter/deadline_submitter_for_nuke.py b/src/deadline/nuke_submitter/deadline_submitter_for_nuke.py index 5e6cbd1..c2bef8e 100644 --- a/src/deadline/nuke_submitter/deadline_submitter_for_nuke.py +++ b/src/deadline/nuke_submitter/deadline_submitter_for_nuke.py @@ -165,11 +165,10 @@ def _get_job_template(settings: SubmitterUISettings) -> dict[str, Any]: # Load the default Nuke job template, and then fill in scene-specific # values it needs. - template_name = ( - "default_nuke_job_template.yaml" - if job_type == JobType.RENDER - else "copycat_job_template.yaml" - ) + if job_type == JobType.RENDER: + template_name = "default_nuke_job_template.yaml" + else: + template_name = "copycat_job_template.yaml" with open(Path(__file__).parent / template_name) as f: job_template = yaml.safe_load(f) @@ -364,6 +363,20 @@ def _get_render_parameter_values( } ) + # Set chunking parameter values + parameter_values.append( + { + "name": "ChunkSize", + "value": settings.jobtype_specific_settings.chunk_size, # type: ignore[union-attr] + } + ) + parameter_values.append( + { + "name": "TargetChunkDuration", + "value": settings.jobtype_specific_settings.target_chunk_duration, # type: ignore[union-attr] + } + ) + # Set the OCIO config path value if nuke_ocio.is_OCIO_enabled(): ocio_config_path = nuke_ocio.get_ocio_config_path() diff --git a/src/deadline/nuke_submitter/default_nuke_job_template.yaml b/src/deadline/nuke_submitter/default_nuke_job_template.yaml index d421c6f..e5de8f9 100644 --- a/src/deadline/nuke_submitter/default_nuke_job_template.yaml +++ b/src/deadline/nuke_submitter/default_nuke_job_template.yaml @@ -1,4 +1,6 @@ specificationVersion: jobtemplate-2023-09 +extensions: +- TASK_CHUNKING name: Default Nuke Job Template parameterDefinitions: - name: NukeScriptFile @@ -34,6 +36,27 @@ parameterDefinitions: type: STRING description: The frames to render. E.g. 1-3,8,11-15 minLength: 1 +- name: ChunkSize + type: INT + default: 1 + minValue: 1 + description: > + Number of frames per chunk. Use 1 for one frame per task (default). + Higher values group frames into chunks to reduce per-task overhead. + userInterface: + control: SPIN_BOX + label: Chunk Size +- name: TargetChunkDuration + type: INT + default: 0 + minValue: 0 + description: > + Optional target duration in seconds per chunk. When set, the scheduler + dynamically adjusts chunk sizes based on observed runtimes of completed + chunks. Set to 0 to use a fixed chunk size for all chunks. + userInterface: + control: SPIN_BOX + label: Target Chunk Duration (Seconds) - name: WriteNode type: STRING userInterface: @@ -76,8 +99,12 @@ steps: parameterSpace: taskParameterDefinitions: - name: Frame - type: INT + type: CHUNK[INT] range: '{{Param.Frames}}' + chunks: + defaultTaskCount: '{{Param.ChunkSize}}' + targetRuntimeSeconds: '{{Param.TargetChunkDuration}}' + rangeConstraint: CONTIGUOUS stepEnvironments: - name: Nuke description: Runs Nuke in the background with a script file loaded. diff --git a/src/deadline/nuke_submitter/ui/components/scene_settings_tab.py b/src/deadline/nuke_submitter/ui/components/scene_settings_tab.py index 4f7b40d..cbc926e 100644 --- a/src/deadline/nuke_submitter/ui/components/scene_settings_tab.py +++ b/src/deadline/nuke_submitter/ui/components/scene_settings_tab.py @@ -3,6 +3,7 @@ """ UI widgets for the Scene Settings tab. """ + import os import nuke @@ -98,6 +99,29 @@ def _build_render_ui_options(self, lyt: QGridLayout): ) lyt.addWidget(self.continue_on_error_check, 4, 0) + # Chunking controls + lyt.addWidget(QLabel("Chunk size"), 5, 0) + self.chunk_size_spin = QSpinBox(self, minimum=1, maximum=150) + self.chunk_size_spin.setValue(1) + self.chunk_size_spin.setToolTip( + "Number of frames to group into each chunk.\n" + "Use 1 for one frame per task (default).\n" + "Higher values reduce per-task overhead." + ) + lyt.addWidget(self.chunk_size_spin, 5, 1, 1, -1) + + lyt.addWidget(QLabel("Target chunk duration (seconds)"), 6, 0) + self.target_chunk_duration_spin = QSpinBox(self, minimum=0, maximum=86400) + self.target_chunk_duration_spin.setValue(0) + self.target_chunk_duration_spin.setToolTip( + "When set, the scheduler dynamically adjusts chunk sizes\n" + "based on observed runtimes of completed chunks, aiming\n" + "for this duration per chunk. Leave at 0 to use a fixed\n" + "chunk size for all chunks." + ) + self.target_chunk_duration_spin.setSpecialValueText("Disabled") + lyt.addWidget(self.target_chunk_duration_spin, 6, 1, 1, -1) + def _build_copycat_ui_options(self, lyt: QGridLayout): self.copycat_node_box = QComboBox(self) self._rebuild_copycat_node_drop_down() @@ -118,17 +142,17 @@ def _build_ui(self): self.timeout_checkbox.setToolTip( "Set a maximum duration for actions from this job. See AWS Deadline Cloud documentation to learn more" ) - lyt.addWidget(self.timeout_checkbox, 5, 0) + lyt.addWidget(self.timeout_checkbox, 7, 0) self.timeouts_subtext = QLabel("Set a maximum duration for actions from this job") self.timeouts_subtext.setStyleSheet("font-style: italic") - lyt.addWidget(self.timeouts_subtext, 5, 1, 1, -1) + lyt.addWidget(self.timeouts_subtext, 7, 1, 1, -1) self.timeouts_box = QGroupBox() timeouts_lyt = QGridLayout(self.timeouts_box) - lyt.addWidget(self.timeouts_box, 6, 0, 1, -1) + lyt.addWidget(self.timeouts_box, 8, 0, 1, -1) self.gizmos_checkbox = QCheckBox("Include gizmos in job bundle", self) - lyt.addWidget(self.gizmos_checkbox, 7, 0) + lyt.addWidget(self.gizmos_checkbox, 9, 0) def create_timeout_row(label, tooltip, row): qlabel = QLabel(label) @@ -181,9 +205,9 @@ def indicate_is_valid_callback(value: int): self.include_adaptor_wheels = QCheckBox( "Developer option: Include adaptor wheels", self ) - lyt.addWidget(self.include_adaptor_wheels, 8, 0, 1, 2) + lyt.addWidget(self.include_adaptor_wheels, 10, 0, 1, 2) - lyt.addItem(QSpacerItem(0, 0, QSizePolicy.Minimum, QSizePolicy.Expanding), 9, 0) + lyt.addItem(QSpacerItem(0, 0, QSizePolicy.Minimum, QSizePolicy.Expanding), 11, 0) def indicate_if_valid(self, timeout_boxes: tuple[QLabel, QSpinBox, QSpinBox, QSpinBox]): if ( @@ -284,6 +308,9 @@ def _refresh_render_ui(self, settings: RenderSettings): self.proxy_mode_check.setChecked(settings.is_proxy_mode) self.continue_on_error_check.setChecked(settings.continue_on_error) + self.chunk_size_spin.setValue(settings.chunk_size) + self.target_chunk_duration_spin.setValue(settings.target_chunk_duration) + def _refresh_copycat_ui(self, settings: CopyCatTrainingSettings): pass @@ -324,6 +351,9 @@ def _update_render_settings(self, settings: RenderSettings): settings.is_proxy_mode = self.proxy_mode_check.isChecked() settings.continue_on_error = self.continue_on_error_check.isChecked() + settings.chunk_size = self.chunk_size_spin.value() + settings.target_chunk_duration = self.target_chunk_duration_spin.value() + def _update_copycat_training_settings(self, settings: CopyCatTrainingSettings): settings.copycat_node = self.copycat_node_box.currentData() diff --git a/test/unit/deadline_submitter_for_nuke/test_chunking.py b/test/unit/deadline_submitter_for_nuke/test_chunking.py new file mode 100644 index 0000000..d41012b --- /dev/null +++ b/test/unit/deadline_submitter_for_nuke/test_chunking.py @@ -0,0 +1,179 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +from pathlib import Path +from unittest.mock import patch, MagicMock + +import pytest +import yaml # type: ignore[import] + +from deadline.nuke_submitter.data_classes import ( + RenderSettings, + SubmitterUISettings, +) + + +class TestChunkingDataClasses: + """Tests for chunking fields in RenderSettings and sticky settings.""" + + def test_render_settings_chunking_defaults(self): + settings = RenderSettings() + assert settings.chunk_size == 1 + assert settings.target_chunk_duration == 0 + + def test_sticky_settings_include_chunking_fields(self): + settings = SubmitterUISettings() + settings.jobtype_specific_settings = RenderSettings( + chunk_size=25, + target_chunk_duration=600, + ) + + result = settings._get_sticky_settings_dict() + + assert result["chunk_size"] == 25 + assert result["target_chunk_duration"] == 600 + + def test_load_sticky_settings_with_chunking(self): + settings = SubmitterUISettings() + settings.jobtype_specific_settings = RenderSettings() + + settings._load_sticky_settings_from_dict( + { + "chunk_size": 50, + "target_chunk_duration": 900, + } + ) + + assert settings.jobtype_specific_settings.chunk_size == 50 + assert settings.jobtype_specific_settings.target_chunk_duration == 900 + + def test_load_sticky_settings_without_chunking_keeps_defaults(self): + settings = SubmitterUISettings() + settings.jobtype_specific_settings = RenderSettings() + + settings._load_sticky_settings_from_dict({"name": "test"}) + + assert settings.jobtype_specific_settings.chunk_size == 1 + assert settings.jobtype_specific_settings.target_chunk_duration == 0 + + +class TestJobTemplate: + """Tests for the job template with TASK_CHUNKING.""" + + @pytest.fixture + def template(self): + template_path = ( + Path(__file__).parent.parent.parent.parent + / "src" + / "deadline" + / "nuke_submitter" + / "default_nuke_job_template.yaml" + ) + with open(template_path) as f: + return yaml.safe_load(f) + + def test_template_has_task_chunking_extension(self, template): + assert "TASK_CHUNKING" in template["extensions"] + + def test_template_has_chunk_int_frame_param(self, template): + step = template["steps"][0] + frame_param = step["parameterSpace"]["taskParameterDefinitions"][0] + assert frame_param["name"] == "Frame" + assert frame_param["type"] == "CHUNK[INT]" + + def test_template_has_contiguous_range_constraint(self, template): + step = template["steps"][0] + frame_param = step["parameterSpace"]["taskParameterDefinitions"][0] + assert frame_param["chunks"]["rangeConstraint"] == "CONTIGUOUS" + + def test_template_has_chunking_params(self, template): + param_names = [p["name"] for p in template["parameterDefinitions"]] + assert "ChunkSize" in param_names + assert "TargetChunkDuration" in param_names + + def test_template_chunk_size_defaults_to_1(self, template): + chunk_size = next(p for p in template["parameterDefinitions"] if p["name"] == "ChunkSize") + assert chunk_size["default"] == 1 + assert chunk_size["minValue"] == 1 + + def test_template_target_duration_defaults_to_0(self, template): + target = next( + p for p in template["parameterDefinitions"] if p["name"] == "TargetChunkDuration" + ) + assert target["default"] == 0 + assert target["minValue"] == 0 + + def test_template_passes_frame_range_to_adaptor(self, template): + step = template["steps"][0] + run_data = step["script"]["embeddedFiles"][0]["data"] + assert 'frameRange: "{{Task.Param.Frame}}"' in run_data + + +class TestSubmitterLogic: + """Tests for parameter values with chunking.""" + + @patch("deadline.nuke_submitter.deadline_submitter_for_nuke.nuke") + @patch("deadline.nuke_submitter.deadline_submitter_for_nuke.nuke_ocio") + @patch("deadline.nuke_submitter.deadline_submitter_for_nuke.get_nuke_script_file") + def test_parameter_values_include_chunking(self, mock_script_file, mock_ocio, mock_nuke): + from deadline.nuke_submitter.deadline_submitter_for_nuke import ( + _get_render_parameter_values, + ) + + mock_ocio.is_OCIO_enabled.return_value = False + mock_nuke.root.return_value = MagicMock() + mock_nuke.root.return_value.frameRange.return_value = "1-100" + + settings = SubmitterUISettings() + settings.jobtype_specific_settings = RenderSettings( + chunk_size=25, + target_chunk_duration=600, + ) + + param_values = _get_render_parameter_values(settings, queue_parameters=[]) + + param_map = {p["name"]: p["value"] for p in param_values} + assert param_map["ChunkSize"] == 25 + assert param_map["TargetChunkDuration"] == 600 + + @patch("deadline.nuke_submitter.deadline_submitter_for_nuke.nuke") + @patch("deadline.nuke_submitter.deadline_submitter_for_nuke.nuke_ocio") + @patch("deadline.nuke_submitter.deadline_submitter_for_nuke.get_nuke_script_file") + def test_parameter_values_default_chunking(self, mock_script_file, mock_ocio, mock_nuke): + """Default chunk_size=1 and target_chunk_duration=0 are always passed.""" + from deadline.nuke_submitter.deadline_submitter_for_nuke import ( + _get_render_parameter_values, + ) + + mock_ocio.is_OCIO_enabled.return_value = False + mock_nuke.root.return_value = MagicMock() + mock_nuke.root.return_value.frameRange.return_value = "1-100" + + settings = SubmitterUISettings() + settings.jobtype_specific_settings = RenderSettings() + + param_values = _get_render_parameter_values(settings, queue_parameters=[]) + + param_map = {p["name"]: p["value"] for p in param_values} + assert param_map["ChunkSize"] == 1 + assert param_map["TargetChunkDuration"] == 0 + + @patch("deadline.nuke_submitter.deadline_submitter_for_nuke.nuke") + @patch("deadline.nuke_submitter.deadline_submitter_for_nuke.nuke_ocio") + def test_get_job_template_has_task_chunking(self, mock_ocio, mock_nuke): + """The single template should always have TASK_CHUNKING.""" + from deadline.nuke_submitter.deadline_submitter_for_nuke import _get_job_template + + mock_ocio.is_OCIO_enabled.return_value = False + mock_nuke.views.return_value = [] + mock_nuke.root.return_value = MagicMock() + + settings = SubmitterUISettings() + settings.jobtype_specific_settings = RenderSettings() + + with patch( + "deadline.nuke_submitter.deadline_submitter_for_nuke.find_all_write_nodes", + return_value=[], + ): + template = _get_job_template(settings) + + assert "TASK_CHUNKING" in template.get("extensions", [])