From b7e0b706f59c69b11c5085a30b10caf3d14efe55 Mon Sep 17 00:00:00 2001 From: eap Date: Thu, 11 Sep 2025 14:09:38 -0600 Subject: [PATCH 1/4] SOURCE refs for stuff --- ci_action/implementation.py | 10 ++++++++ ci_action/library/cmake_rewrite.py | 37 +++++++++++++++++++++++------- test/test_cmake_rewrite.py | 23 +++++++++++++++---- 3 files changed, 58 insertions(+), 12 deletions(-) diff --git a/ci_action/implementation.py b/ci_action/implementation.py index 1de1691..92491fe 100644 --- a/ci_action/implementation.py +++ b/ci_action/implementation.py @@ -125,6 +125,13 @@ def prepare_and_launch_ci_test( config['bundle_repository'], bundle_repo_path ]) + # Move the target repository into the bundle repository. + target_project_path = os.path.join(bundle_repo_path, config['target_project_name']) + shutil.move(target_repo_path, target_project_path) + # Debug output - list the contents of the bundle repository. + LOG.info(f"Bundle repository contents: {os.listdir(bundle_repo_path)}") + LOG.info(f"Target project path: {target_project_path}") + repo_to_commit_hash = pr_resolve.gather_build_group_hashes( test_annotations.build_group_map ) @@ -156,6 +163,7 @@ def prepare_and_launch_ci_test( file_object=f, enabled_bundles=enabled_bundles, build_group_commit_map=repo_to_commit_hash, + source_projects=[config['target_project_name']], ) # Create an integration test file. @@ -164,9 +172,11 @@ def prepare_and_launch_ci_test( file_object=f, disabled_bundles=set(), build_group_commit_map=repo_to_commit_hash, + source_projects=[config['target_project_name']], ) LOG.info(f'{timer.checkpoint()}\n Rewrote bundle for build groups.') + # Add resources to the bundle by copying all files in /app/shell to jedi_ci_resources shutil.copytree( '/app/shell', os.path.join(bundle_repo_path, 'jedi_ci_resources') diff --git a/ci_action/library/cmake_rewrite.py b/ci_action/library/cmake_rewrite.py index 1dfa67c..83becc9 100644 --- a/ci_action/library/cmake_rewrite.py +++ b/ci_action/library/cmake_rewrite.py @@ -132,6 +132,12 @@ def rewrite_original(self): content = " ".join([str(c) for c in render_components]) return f"ecbuild_bundle( {content} )" + def rewrite_as_source(self): + source_reference = BundleLinePart("SOURCE", False, self.project_name) + render_components = [self.project, source_reference] + content = " ".join([str(c) for c in render_components]) + return f"ecbuild_bundle( {content} )" + def rewrite(self, git_repo: str = None, branch: str = None, tag: str = None): if git_repo: source_reference = BundleLinePart("GIT", False, git_repo, quote_char='"') @@ -190,6 +196,7 @@ def _rewrite_file_implementation( enabled_bundles: Optional[Container[str]] = None, rewrite_rules: Optional[dict[str, str]] = None, build_group_commit_map: Optional[Dict[str, Dict[str, Any]]] = None, + source_projects: Optional[Container[str]] = None, ): """Rewrite the CMakeFile object to the file_object. @@ -208,7 +215,11 @@ def _rewrite_file_implementation( "commit": "abcdef123456" } } + source_projects: A list of project names that are already cloned into the bundle + repository. These will be rewritten as a SOURCE bundle. """ + if source_projects is None: + source_projects = set() if enabled_bundles is None: enabled_bundles = set() if rewrite_rules is None: @@ -233,6 +244,12 @@ def _rewrite_file_implementation( lines.append(bundle_line.disabled_line() + '\n') continue + # If the project is already cloned as a source directory in the bundle repository. + # it may be in the build map, but ecbuild will ignore this and use the provided source. + if bundle_line.project_name in source_projects: + lines.append(bundle_line.rewrite_as_source() + '\n') + continue + # Check if this bundle matches a github org/repo key in the build group commit map if bundle_line.github_org_repo_key and bundle_line.github_org_repo_key in build_group_commit_map: # noqa: E501 # Use the commit hash as a tag @@ -261,33 +278,37 @@ def basic_rewrite(self, file_object): def rewrite_whitelist(self, file_object, enabled_bundles: Container[str], - rewrite_rules: dict[str, str]): + rewrite_rules: dict[str, str], + source_projects: Optional[Container[str]] = None): """Rewrite the CMakeFile object to the file_object.""" - self._rewrite_file_implementation(file_object, enabled_bundles, rewrite_rules) + self._rewrite_file_implementation(file_object, enabled_bundles, rewrite_rules, source_projects=source_projects) # noqa: E501 def rewrite_blacklist(self, file_object, disabled_bundles: Container[str], - rewrite_rules: dict[str, str]): + rewrite_rules: dict[str, str], + source_projects: Optional[Container[str]] = None): """Rewrite the CMakeFile object to the file_object.""" enabled_bundles = set(self.bundle_line_names.keys()) for bundle in disabled_bundles: enabled_bundles.discard(bundle) - self._rewrite_file_implementation(file_object, enabled_bundles, rewrite_rules) + self._rewrite_file_implementation(file_object, enabled_bundles, rewrite_rules, source_projects=source_projects) # noqa: E501 def rewrite_build_group_whitelist(self, file_object, enabled_bundles: Container[str], - build_group_commit_map: Dict[str, Dict[str, Any]]): + build_group_commit_map: Dict[str, Dict[str, Any]], + source_projects: Optional[Container[str]] = None): """Rewrite the CMakeFile object to the file_object.""" - self._rewrite_file_implementation(file_object, enabled_bundles, build_group_commit_map=build_group_commit_map) # noqa: E501 + self._rewrite_file_implementation(file_object, enabled_bundles, build_group_commit_map=build_group_commit_map, source_projects=source_projects) # noqa: E501 def rewrite_build_group_blacklist(self, file_object, disabled_bundles: Container[str], - build_group_commit_map: Dict[str, Dict[str, Any]]): + build_group_commit_map: Dict[str, Dict[str, Any]], + source_projects: Optional[Container[str]] = None): """Rewrite the CMakeFile object to the file_object.""" enabled_bundles = set(self.bundle_line_names.keys()) for bundle in disabled_bundles: enabled_bundles.discard(bundle) - self._rewrite_file_implementation(file_object, enabled_bundles, build_group_commit_map=build_group_commit_map) # noqa: E501 + self._rewrite_file_implementation(file_object, enabled_bundles, build_group_commit_map=build_group_commit_map, source_projects=source_projects) # noqa: E501 diff --git a/test/test_cmake_rewrite.py b/test/test_cmake_rewrite.py index 3c0a9f1..454b5cd 100644 --- a/test/test_cmake_rewrite.py +++ b/test/test_cmake_rewrite.py @@ -80,6 +80,11 @@ def test_rewrite_with_new_git_repo(self): self.assertIn(f'GIT "{new_repo}"', new_line) self.assertTrue('myrepo.git' not in new_line or new_repo in new_line) + def test_rewrite_as_source(self): + bl = BundleLine(SIMPLE_GIT_BUNDLE_LINE) + new_line = bl.rewrite_as_source() + self.assertEqual(new_line, 'ecbuild_bundle( PROJECT myproject SOURCE myproject )') + ORIGINAL_CMAKE_FILE = """ cmake_minimum_required( VERSION 3.14 FATAL_ERROR ) @@ -180,8 +185,17 @@ def test_rewrite_build_group_whitelist(self): self.assertMultiLineEqual(fake_file.getvalue(), expected) def test_rewrite_build_group_whitelist_multiple(self): - cmake_file = CMakeFile('# File header\necbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" BRANCH develop UPDATE )\necbuild_bundle( PROJECT pyiri-jedi GIT "https://github.com/jcsda-internal/pyiri-jedi.git" BRANCH develop UPDATE RECURSIVE )\necbuild_bundle( PROJECT gsibec GIT "https://github.com/geos-esm/GSIbec" TAG 1.2.1 )\n') - expected = '# File header\necbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" TAG abc123456 )\necbuild_bundle( PROJECT pyiri-jedi GIT "https://github.com/jcsda-internal/pyiri-jedi.git" TAG fedcba654321 RECURSIVE )\n# ecbuild_bundle( PROJECT gsibec GIT "https://github.com/geos-esm/GSIbec" TAG 1.2.1 )\n' + cmake_file = CMakeFile( + '# File header\necbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" BRANCH develop UPDATE )\n' + 'ecbuild_bundle( PROJECT pyiri-jedi GIT "https://github.com/jcsda-internal/pyiri-jedi.git" BRANCH develop UPDATE RECURSIVE )\n' + 'ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda-internal/ufo.git" BRANCH develop )\n' + 'ecbuild_bundle( PROJECT gsibec GIT "https://github.com/geos-esm/GSIbec" TAG 1.2.1 )\n') + expected = ( + '# File header\n' + 'ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" TAG abc123456 )\n' + 'ecbuild_bundle( PROJECT pyiri-jedi GIT "https://github.com/jcsda-internal/pyiri-jedi.git" TAG fedcba654321 RECURSIVE )\n' + 'ecbuild_bundle( PROJECT ufo SOURCE ufo )\n' + '# ecbuild_bundle( PROJECT gsibec GIT "https://github.com/geos-esm/GSIbec" TAG 1.2.1 )\n') fake_file = StringIO() # Create a build group commit map with multiple repositories @@ -208,8 +222,9 @@ def test_rewrite_build_group_whitelist_multiple(self): cmake_file.rewrite_build_group_whitelist( file_object=fake_file, - enabled_bundles=set(['oops', 'pyiri-jedi']), - build_group_commit_map=build_group_map + enabled_bundles=set(['oops', 'ufo', 'pyiri-jedi']), + build_group_commit_map=build_group_map, + source_projects=['ufo'], ) self.assertMultiLineEqual(fake_file.getvalue(), expected) From aaf1a7c4f9bd8ed8f6c483a34d7eb62d117e9d02 Mon Sep 17 00:00:00 2001 From: eap Date: Thu, 11 Sep 2025 15:35:33 -0600 Subject: [PATCH 2/4] Fix lint error --- ci_action/implementation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ci_action/implementation.py b/ci_action/implementation.py index 92491fe..3155c25 100644 --- a/ci_action/implementation.py +++ b/ci_action/implementation.py @@ -176,7 +176,6 @@ def prepare_and_launch_ci_test( ) LOG.info(f'{timer.checkpoint()}\n Rewrote bundle for build groups.') - # Add resources to the bundle by copying all files in /app/shell to jedi_ci_resources shutil.copytree( '/app/shell', os.path.join(bundle_repo_path, 'jedi_ci_resources') From 0182982196b15ab783388aef738499711b763cda Mon Sep 17 00:00:00 2001 From: eap Date: Thu, 11 Sep 2025 16:00:32 -0600 Subject: [PATCH 3/4] log owner --- ci_action/implementation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ci_action/implementation.py b/ci_action/implementation.py index 3155c25..edc12db 100644 --- a/ci_action/implementation.py +++ b/ci_action/implementation.py @@ -131,6 +131,7 @@ def prepare_and_launch_ci_test( # Debug output - list the contents of the bundle repository. LOG.info(f"Bundle repository contents: {os.listdir(bundle_repo_path)}") LOG.info(f"Target project path: {target_project_path}") + LOG.info(f"GITHUB_REPOSITORY_OWNER: {os.environ.get('GITHUB_REPOSITORY_OWNER'), 'unknown'}") repo_to_commit_hash = pr_resolve.gather_build_group_hashes( test_annotations.build_group_map From 61d479e23b8a3f795e55de3ccd825048d41ecff3 Mon Sep 17 00:00:00 2001 From: eap Date: Thu, 11 Sep 2025 16:08:32 -0600 Subject: [PATCH 4/4] Fix debug log --- ci_action/implementation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci_action/implementation.py b/ci_action/implementation.py index edc12db..793e326 100644 --- a/ci_action/implementation.py +++ b/ci_action/implementation.py @@ -130,7 +130,7 @@ def prepare_and_launch_ci_test( shutil.move(target_repo_path, target_project_path) # Debug output - list the contents of the bundle repository. LOG.info(f"Bundle repository contents: {os.listdir(bundle_repo_path)}") - LOG.info(f"Target project path: {target_project_path}") + LOG.info(f"Target project path content: {os.listdir(target_project_path)}") LOG.info(f"GITHUB_REPOSITORY_OWNER: {os.environ.get('GITHUB_REPOSITORY_OWNER'), 'unknown'}") repo_to_commit_hash = pr_resolve.gather_build_group_hashes(