Skip to content

Commit c02c778

Browse files
fabricebritofabricebrito
committed
fix(release): adopt stable staging and verify portable OCI packages
1 parent 4a1de5c commit c02c778

11 files changed

Lines changed: 1321 additions & 28 deletions

File tree

‎.github/workflows/build.yaml‎

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,12 @@ jobs:
3434
.venv/bin/python scripts/cwl-docs.py --validate --output acceptance-results/cwl-docs
3535
.venv/bin/python scripts/wrap-pattern.py --output acceptance-results/wrapped
3636
for cwl in acceptance-results/wrapped/*.cwl; do .venv/bin/cwltool --validate "$cwl"; done
37+
- name: Validate portable release OCI metadata
38+
run: |
39+
uv venv --python 3.12 .oci-venv
40+
uv pip sync --python .oci-venv/bin/python --require-hashes requirements-oci.txt
41+
source .oci-venv/bin/activate
42+
python scripts/prepare-oci.py --source-dir cwl-workflow --wrapped-dir acceptance-results/wrapped --output acceptance-results/annotations --revision "$GITHUB_SHA" --version "$(jq -er .version codemeta.json)"
3743
- name: Test the processing runner
3844
run: |
3945
cd runner
@@ -73,16 +79,20 @@ jobs:
7379
name: acceptance-${{ github.sha }}
7480
path: accepted
7581
- uses: oras-project/setup-oras@v1
82+
- uses: astral-sh/setup-uv@v6
7683
- name: Prepare all unchanged validated assets before publication
7784
run: |
7885
version=$(jq -er .version codemeta.json)
7986
echo "VERSION=$version" >> "$GITHUB_ENV"
8087
if gh release view "$version" >/dev/null 2>&1; then echo 'Version already published'; exit 1; fi
8188
mkdir release-assets
82-
cp cwl-workflow/*.cwl cwl-workflow/*-inputs.json requirements-notebooks.txt mastering-staging.lock.json release-assets/
89+
cp cwl-workflow/*.cwl cwl-workflow/*-inputs.json requirements-notebooks.txt requirements-oci.txt mastering-staging.lock.json release-assets/
8390
cp -r templates release-assets/
8491
mkdir release-assets/wrapped
8592
cp accepted/wrapped/*.cwl release-assets/wrapped/
93+
cp -r accepted/annotations release-assets/
94+
uv venv --python 3.12 .oci-venv
95+
uv pip sync --python .oci-venv/bin/python --require-hashes requirements-oci.txt
8696
cd release-assets
8797
find . -type f -print0 | sort -z | xargs -0 sha256sum > SHA256SUMS
8898
- name: Refuse to replace any existing CWL artifact
@@ -100,18 +110,31 @@ jobs:
100110
echo "Artifact already exists: $name-wrapped:$VERSION"; exit 1
101111
fi
102112
done
103-
- name: Publish standalone and wrapped CWL to GHCR
113+
- name: Publish and verify standalone and wrapped CWL on GHCR
104114
run: |
105115
printf '%s' "$GH_TOKEN" | oras login ghcr.io --username "$GITHUB_ACTOR" --password-stdin
106-
for cwl in release-assets/pattern-*.cwl release-assets/test-*.cwl; do
107-
name=$(basename "$cwl" .cwl)
108-
oras push "ghcr.io/eoap/application-package-patterns/cwl/$name:$VERSION" --artifact-type application/cwl "$cwl:application/cwl"
109-
done
110-
for cwl in release-assets/wrapped/*.cwl; do
116+
: > release-assets/oci-artifacts.txt
117+
for cwl in release-assets/pattern-*.cwl release-assets/test-*.cwl release-assets/wrapped/*.cwl; do
111118
name=$(basename "$cwl" .cwl)
112-
oras push "ghcr.io/eoap/application-package-patterns/cwl/$name-wrapped:$VERSION" --artifact-type application/cwl "$cwl:application/cwl"
119+
if [[ "$cwl" == */wrapped/* ]]; then name="$name-wrapped"; fi
120+
file=$(basename "$cwl")
121+
annotations="$GITHUB_WORKSPACE/release-assets/annotations/$name.annotations.json"
122+
target="ghcr.io/eoap/application-package-patterns/cwl/$name:$VERSION"
123+
(cd "$(dirname "$cwl")" && oras push "$target" --artifact-type application/cwl --annotation-file "$annotations" "$file:application/cwl")
124+
digest=$(oras resolve "$target")
125+
[[ "$digest" =~ ^sha256:[a-f0-9]{64}$ ]]
126+
reference="ghcr.io/eoap/application-package-patterns/cwl/$name@$digest"
127+
oras manifest fetch "$reference" > /tmp/patterns-manifest.json
128+
jq -e --slurpfile expected "$annotations" '.annotations as $actual | $expected[0]."$manifest" | to_entries | all(.[]; $actual[.key] == .value)' /tmp/patterns-manifest.json
129+
mkdir -p "/tmp/verified/$name"
130+
oras pull "$reference" -o "/tmp/verified/$name"
131+
cmp "$cwl" "/tmp/verified/$name/$file"
132+
entrypoint=$(jq -er '."$manifest"."org.cwl.entrypoint"' "$annotations")
133+
.oci-venv/bin/cwltool --validate "/tmp/verified/$name/$entrypoint"
134+
printf '%s\n' "$reference" >> release-assets/oci-artifacts.txt
113135
done
114136
- name: Publish stable GitHub release
115137
run: |
138+
(cd release-assets && find . -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum > SHA256SUMS)
116139
tar -czf application-package-patterns.$VERSION.tar.gz -C release-assets .
117140
gh release create "$VERSION" application-package-patterns.$VERSION.tar.gz --target "$GITHUB_SHA" --title "Application Package Patterns $VERSION" --notes-file RELEASE_NOTES.md

‎CHANGELOG.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,14 @@
11
# Changelog
22

3-
## [2.0.0] - Unreleased
3+
## [2.0.0] - 2026-10-11
44

55
- Use CWL v1.2 and embedded typed input contracts throughout the twelve patterns.
66
- Compose standalone wrapped applications with eoap-cwlwrap and mastering GHCR staging artifacts.
77
- Hash-lock the Python 3.12 environment and pin processing images by digest.
88
- Validate all twelve real-data workflows and four optional-input/output cases, including S3 STAC and raster checks.
99
- Replace stale practice notebooks with executable examples, typed lesson documentation and diagrams.
10-
- Add acceptance and guarded publication CI for original and wrapped CWL artifacts.
10+
- Add acceptance and guarded publication CI for original and wrapped CWL artifacts, including actual transpiler-mate metadata and immutable pull-back verification.
11+
- Remove host checkout paths and anonymous array scopes from wrapped module types and duplicate S3 input declarations from multi-output wrappers.
1112

1213
## [0.3.1] - 2026-10-08
1314

‎RELEASE_NOTES.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# Application Package Patterns 2.0.0
22

3-
The twelve patterns now expose typed CWL v1.2 contracts with embedded custom definitions. eoap-cwlwrap 0.32.0 composes standalone workflows using the mastering 2.0.1-rc.44.1 stage-in and stage-out commands and immutable GHCR container digests. Directory staging no longer injects copies of Python staging scripts.
3+
The twelve patterns now expose typed CWL v1.2 contracts with embedded custom definitions. eoap-cwlwrap 0.32.0 composes standalone workflows using the mastering 2.0.1 stage-in and stage-out commands and immutable GHCR container digests. Directory staging no longer injects copies of Python staging scripts.
44

55
Real Landsat processing and S3 publication are checked across all twelve patterns and four optional-port cases. Acceptance validates metadata and standalone CWL, verifies returned STAC links and raster assets on authenticated S3, and builds the static Markdown documentation strictly. Dependencies are hash-locked for Python 3.12; AMD64 is the acceptance architecture.
66

7-
Release artifacts include original and wrapped CWL, typed jobs, staging adapters with source provenance, the Python lock and checksums. Standalone and wrapped packages are published to GHCR after acceptance. The existing patterns runner image is consumed by immutable digest; no runtime dependency upgrade is claimed. This release changes the earlier primitive input contract.
7+
Release artifacts include original and wrapped CWL, typed jobs, staging adapters with source provenance, the processing and OCI tooling locks, portable transpiler-mate annotations and checksums. Standalone and wrapped packages are published to GHCR after acceptance. The existing patterns runner image is consumed by immutable digest; no runtime dependency upgrade is claimed. This release changes the earlier primitive input contract.

‎docs/helpers.py‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,63 @@ def embed(value):
141141
return value
142142
target = Path(path)
143143
document = embed(yaml.safe_load(target.read_text()))
144+
# Relocate embedded module type identifiers into this standalone document.
145+
# DOM serialization preserves the original checkout URI in type names.
146+
local_documents = set()
147+
def collect(value):
148+
if isinstance(value, dict):
149+
name = value.get("name", "")
150+
if isinstance(name, str) and name.startswith("file://") and "#" in name and value.get("type") in ("record", "enum"):
151+
local_documents.add(name.split("#", 1)[0])
152+
for child in value.values(): collect(child)
153+
elif isinstance(value, list):
154+
for child in value: collect(child)
155+
collect(document)
156+
def relocate(value):
157+
if isinstance(value, str) and "#" in value and value.split("#", 1)[0] in local_documents:
158+
return "#" + value.split("#", 1)[1]
159+
if isinstance(value, dict):
160+
# Anonymous array names introduce a separate URI scope for local types.
161+
anonymous_array = value.get("type") == "array" and str(value.get("name", "")).startswith("_:")
162+
return {key: relocate(child) for key, child in value.items() if not (anonymous_array and key == "name")}
163+
if isinstance(value, list): return [relocate(child) for child in value]
164+
return value
165+
document = relocate(document)
166+
# Keep the approved shared schema imports in their original form.
167+
# Serializing them as remote-named inline types confuses schema loaders.
168+
approved = ("https://raw.githubusercontent.com/eoap/schemas/main/ogc.yaml",
169+
"https://raw.githubusercontent.com/eoap/schemas/main/geojson.yaml")
170+
def restore_imports(value):
171+
if isinstance(value, dict):
172+
if "types" in value and isinstance(value["types"], list):
173+
result = []
174+
imported = set()
175+
for schema in value["types"]:
176+
name = schema.get("name", "") if isinstance(schema, dict) else ""
177+
url = next((url for url in approved if name.startswith(url + "#")), None)
178+
if url:
179+
if url not in imported: result.append({"$import": url}); imported.add(url)
180+
else: result.append(schema)
181+
value["types"] = result
182+
for child in value.values(): restore_imports(child)
183+
elif isinstance(value, list):
184+
for child in value: restore_imports(child)
185+
restore_imports(document)
186+
def unique_inputs(value):
187+
if isinstance(value, dict):
188+
inputs = value.get("inputs")
189+
if isinstance(inputs, list):
190+
unique = {}
191+
for port in inputs:
192+
identifier = port["id"]
193+
if identifier in unique and unique[identifier] != port:
194+
raise ValueError(f"Conflicting duplicate input: {identifier}")
195+
unique[identifier] = port
196+
value["inputs"] = list(unique.values())
197+
for child in value.values(): unique_inputs(child)
198+
elif isinstance(value, list):
199+
for child in value: unique_inputs(child)
200+
unique_inputs(document)
144201
metadata = yaml.safe_load(Path(application).read_text()) if application else template
145202
for key, value in metadata.items():
146203
if key.startswith("s:") or key == "$namespaces": document[key] = deepcopy(value)

‎mastering-staging.lock.json‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
11
{
2-
"version": "2.0.1-rc.44.1",
2+
"version": "2.0.1",
33
"sources": {
44
"stage-in": {
5-
"url": "ghcr.io/eoap/mastering-app-package/cwl/stage-in@sha256:106370fc68ff02caf81f6d64937af3fa960b7e2a3a66064205f64119e17f2161",
6-
"tag": "2.0.1-rc.44.1",
7-
"sha256": "4d85975c4e72a2e03f8de2d173ca60fe82c8aec3e9671f1b134fd0cef2c43d15",
8-
"image": "ghcr.io/eoap/mastering-app-package/stage-in@sha256:202225e9a7829d072dcb2d7177b2b68d0ab121ab2deb4ff9c8281d78d1379db9"
5+
"url": "ghcr.io/eoap/mastering-app-package/cwl/stage-in@sha256:b4f2e41e31d768fe0e1c70e0497c939e23c0833063b482d10763418142b7fb97",
6+
"tag": "2.0.1",
7+
"sha256": "4d5c9b7dc876bab8e04eabdd9b379fe037f26e26c504d9484ca4f970fe8b1f0e",
8+
"image": "ghcr.io/eoap/mastering-app-package/stage-in@sha256:56623b24570fe8b28042cbe494c6da10cb4c758c28b303136d7dae1982b808f6"
99
},
1010
"stage-out": {
11-
"url": "ghcr.io/eoap/mastering-app-package/cwl/stage-out@sha256:18c9e45c705d9b5c3d3cf91fc258f2d39dc184a6996edbdd76bb21d2405aa73a",
12-
"tag": "2.0.1-rc.44.1",
13-
"sha256": "6ec4de9e1dfcc00824a0e48df8983aae45fd5613b4485ed62acea3973b98af7b",
14-
"image": "ghcr.io/eoap/mastering-app-package/stage-out@sha256:4e2c44b34f0d79ad380dca50913dc76c5be2448add86f992b872735581b6587a"
11+
"url": "ghcr.io/eoap/mastering-app-package/cwl/stage-out@sha256:6e4289924c86ff8639bd242fca7f99214839ef09bc1bd39ba7398689243dd3c9",
12+
"tag": "2.0.1",
13+
"sha256": "062130234d396ab0b0c5cd96f0e8659841214108ae6de709fe8d92af2b50481b",
14+
"image": "ghcr.io/eoap/mastering-app-package/stage-out@sha256:3c3312d06227a0cbf01f0c45e1e60b87faf59949780573429c09a0c9db2b8ac1"
1515
}
1616
}
1717
}

‎requirements-oci.in‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
transpiler-mate-runtime==1.3.0
2+
cwl2oci==0.1.1
3+
cwltool==3.2.20260720092025

0 commit comments

Comments
 (0)