Skip to content

Use a SOURCE ecbuild_bundle reference so that forks can be tested with JEDI-CI #28

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
eap wants to merge 4 commits into
base: develop
Choose a base branch
from
Open

Use a SOURCE ecbuild_bundle reference so that forks can be tested with JEDI-CI #28

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b7e0b70
SOURCE refs for stuff
eap Sep 11, 2025
aaf1a7c
Fix lint error
eap Sep 11, 2025
0182982
log owner
eap Sep 11, 2025
61d479e
Fix debug log
eap Sep 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions ci_action/implementation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,14 @@ def prepare_and_launch_ci_test(
config['bundle_repository'], bundle_repo_path
])

# Move the target repository into the bundle repository.
target_project_path = os.path.join(bundle_repo_path, config['target_project_name'])
shutil.move(target_repo_path, target_project_path)
# Debug output - list the contents of the bundle repository.
LOG.info(f"Bundle repository contents: {os.listdir(bundle_repo_path)}")
LOG.info(f"Target project path content: {os.listdir(target_project_path)}")
LOG.info(f"GITHUB_REPOSITORY_OWNER: {os.environ.get('GITHUB_REPOSITORY_OWNER'), 'unknown'}")

repo_to_commit_hash = pr_resolve.gather_build_group_hashes(
test_annotations.build_group_map
)
Expand Down Expand Up @@ -156,6 +164,7 @@ def prepare_and_launch_ci_test(
file_object=f,
enabled_bundles=enabled_bundles,
build_group_commit_map=repo_to_commit_hash,
source_projects=[config['target_project_name']],
)

# Create an integration test file.
Expand All @@ -164,6 +173,7 @@ def prepare_and_launch_ci_test(
file_object=f,
disabled_bundles=set(),
build_group_commit_map=repo_to_commit_hash,
source_projects=[config['target_project_name']],
)
LOG.info(f'{timer.checkpoint()}\n Rewrote bundle for build groups.')

Expand Down
37 changes: 29 additions & 8 deletions ci_action/library/cmake_rewrite.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,12 @@ def rewrite_original(self):
content = " ".join([str(c) for c in render_components])
return f"ecbuild_bundle( {content} )"

def rewrite_as_source(self):
source_reference = BundleLinePart("SOURCE", False, self.project_name)
render_components = [self.project, source_reference]
content = " ".join([str(c) for c in render_components])
return f"ecbuild_bundle( {content} )"

def rewrite(self, git_repo: str = None, branch: str = None, tag: str = None):
if git_repo:
source_reference = BundleLinePart("GIT", False, git_repo, quote_char='"')
Expand Down Expand Up @@ -190,6 +196,7 @@ def _rewrite_file_implementation(
enabled_bundles: Optional[Container[str]] = None,
rewrite_rules: Optional[dict[str, str]] = None,
build_group_commit_map: Optional[Dict[str, Dict[str, Any]]] = None,
source_projects: Optional[Container[str]] = None,
):
"""Rewrite the CMakeFile object to the file_object.

Expand All @@ -208,7 +215,11 @@ def _rewrite_file_implementation(
"commit": "abcdef123456"
}
}
source_projects: A list of project names that are already cloned into the bundle
repository. These will be rewritten as a SOURCE bundle.
"""
if source_projects is None:
source_projects = set()
if enabled_bundles is None:
enabled_bundles = set()
if rewrite_rules is None:
Expand All @@ -233,6 +244,12 @@ def _rewrite_file_implementation(
lines.append(bundle_line.disabled_line() + '\n')
continue

# If the project is already cloned as a source directory in the bundle repository.
# it may be in the build map, but ecbuild will ignore this and use the provided source.
if bundle_line.project_name in source_projects:
lines.append(bundle_line.rewrite_as_source() + '\n')
continue

# Check if this bundle matches a github org/repo key in the build group commit map
if bundle_line.github_org_repo_key and bundle_line.github_org_repo_key in build_group_commit_map: # noqa: E501
# Use the commit hash as a tag
Expand Down Expand Up @@ -261,33 +278,37 @@ def basic_rewrite(self, file_object):
def rewrite_whitelist(self,
file_object,
enabled_bundles: Container[str],
rewrite_rules: dict[str, str]):
rewrite_rules: dict[str, str],
source_projects: Optional[Container[str]] = None):
"""Rewrite the CMakeFile object to the file_object."""
self._rewrite_file_implementation(file_object, enabled_bundles, rewrite_rules)
self._rewrite_file_implementation(file_object, enabled_bundles, rewrite_rules, source_projects=source_projects) # noqa: E501

def rewrite_blacklist(self,
file_object,
disabled_bundles: Container[str],
rewrite_rules: dict[str, str]):
rewrite_rules: dict[str, str],
source_projects: Optional[Container[str]] = None):
"""Rewrite the CMakeFile object to the file_object."""
enabled_bundles = set(self.bundle_line_names.keys())
for bundle in disabled_bundles:
enabled_bundles.discard(bundle)
self._rewrite_file_implementation(file_object, enabled_bundles, rewrite_rules)
self._rewrite_file_implementation(file_object, enabled_bundles, rewrite_rules, source_projects=source_projects) # noqa: E501

def rewrite_build_group_whitelist(self,
file_object,
enabled_bundles: Container[str],
build_group_commit_map: Dict[str, Dict[str, Any]]):
build_group_commit_map: Dict[str, Dict[str, Any]],
source_projects: Optional[Container[str]] = None):
"""Rewrite the CMakeFile object to the file_object."""
self._rewrite_file_implementation(file_object, enabled_bundles, build_group_commit_map=build_group_commit_map) # noqa: E501
self._rewrite_file_implementation(file_object, enabled_bundles, build_group_commit_map=build_group_commit_map, source_projects=source_projects) # noqa: E501

def rewrite_build_group_blacklist(self,
file_object,
disabled_bundles: Container[str],
build_group_commit_map: Dict[str, Dict[str, Any]]):
build_group_commit_map: Dict[str, Dict[str, Any]],
source_projects: Optional[Container[str]] = None):
"""Rewrite the CMakeFile object to the file_object."""
enabled_bundles = set(self.bundle_line_names.keys())
for bundle in disabled_bundles:
enabled_bundles.discard(bundle)
self._rewrite_file_implementation(file_object, enabled_bundles, build_group_commit_map=build_group_commit_map) # noqa: E501
self._rewrite_file_implementation(file_object, enabled_bundles, build_group_commit_map=build_group_commit_map, source_projects=source_projects) # noqa: E501
23 changes: 19 additions & 4 deletions test/test_cmake_rewrite.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,11 @@ def test_rewrite_with_new_git_repo(self):
self.assertIn(f'GIT "{new_repo}"', new_line)
self.assertTrue('myrepo.git' not in new_line or new_repo in new_line)

def test_rewrite_as_source(self):
bl = BundleLine(SIMPLE_GIT_BUNDLE_LINE)
new_line = bl.rewrite_as_source()
self.assertEqual(new_line, 'ecbuild_bundle( PROJECT myproject SOURCE myproject )')


ORIGINAL_CMAKE_FILE = """
cmake_minimum_required( VERSION 3.14 FATAL_ERROR )
Expand Down Expand Up @@ -180,8 +185,17 @@ def test_rewrite_build_group_whitelist(self):
self.assertMultiLineEqual(fake_file.getvalue(), expected)

def test_rewrite_build_group_whitelist_multiple(self):
cmake_file = CMakeFile('# File header\necbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" BRANCH develop UPDATE )\necbuild_bundle( PROJECT pyiri-jedi GIT "https://github.com/jcsda-internal/pyiri-jedi.git" BRANCH develop UPDATE RECURSIVE )\necbuild_bundle( PROJECT gsibec GIT "https://github.com/geos-esm/GSIbec" TAG 1.2.1 )\n')
expected = '# File header\necbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" TAG abc123456 )\necbuild_bundle( PROJECT pyiri-jedi GIT "https://github.com/jcsda-internal/pyiri-jedi.git" TAG fedcba654321 RECURSIVE )\n# ecbuild_bundle( PROJECT gsibec GIT "https://github.com/geos-esm/GSIbec" TAG 1.2.1 )\n'
cmake_file = CMakeFile(
'# File header\necbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" BRANCH develop UPDATE )\n'
'ecbuild_bundle( PROJECT pyiri-jedi GIT "https://github.com/jcsda-internal/pyiri-jedi.git" BRANCH develop UPDATE RECURSIVE )\n'
'ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda-internal/ufo.git" BRANCH develop )\n'
'ecbuild_bundle( PROJECT gsibec GIT "https://github.com/geos-esm/GSIbec" TAG 1.2.1 )\n')
expected = (
'# File header\n'
'ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" TAG abc123456 )\n'
'ecbuild_bundle( PROJECT pyiri-jedi GIT "https://github.com/jcsda-internal/pyiri-jedi.git" TAG fedcba654321 RECURSIVE )\n'
'ecbuild_bundle( PROJECT ufo SOURCE ufo )\n'
'# ecbuild_bundle( PROJECT gsibec GIT "https://github.com/geos-esm/GSIbec" TAG 1.2.1 )\n')
fake_file = StringIO()

# Create a build group commit map with multiple repositories
Expand All @@ -208,8 +222,9 @@ def test_rewrite_build_group_whitelist_multiple(self):

cmake_file.rewrite_build_group_whitelist(
file_object=fake_file,
enabled_bundles=set(['oops', 'pyiri-jedi']),
build_group_commit_map=build_group_map
enabled_bundles=set(['oops', 'ufo', 'pyiri-jedi']),
build_group_commit_map=build_group_map,
source_projects=['ufo'],
)
self.assertMultiLineEqual(fake_file.getvalue(), expected)

Expand Down