diff --git a/config/forge/images/build_forge-main.yaml b/config/forge/images/build_forge-main.yaml deleted file mode 100644 index 43ecc1f..0000000 --- a/config/forge/images/build_forge-main.yaml +++ /dev/null @@ -1,33 +0,0 @@ -apiVersion: shipwright.io/v1beta1 -kind: Build -metadata: - name: forge-main-build - labels: - app: forge - component: image-build - annotations: - description: "Build FORGE core image from main branch" -spec: - source: - type: Git - git: - url: https://github.com/openshift-psap/forge - revision: main - strategy: - name: buildah - kind: ClusterBuildStrategy - paramValues: - - name: dockerfile - value: projects/core/image/Containerfile - output: - # the namespace is hardcoded here, but I don't know how to make that dynamic -,- - image: image-registry.openshift-image-registry.svc:5000/${NAMESPACE}/forge-core:main - - timeout: 30m - resources: - limits: - cpu: "2" - memory: 4Gi - requests: - cpu: 500m - memory: 1Gi diff --git a/config/forge/images/buildrun_forge-main.yaml b/config/forge/images/buildrun_forge-main.yaml deleted file mode 100644 index 8878a66..0000000 --- a/config/forge/images/buildrun_forge-main.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: shipwright.io/v1beta1 -kind: BuildRun -metadata: - generateName: forge-main-run- - labels: - app: forge - component: image-build - build: forge-main-build -spec: - build: - name: forge-main-build diff --git a/config/forge/images/is_forge.yaml b/config/forge/images/is_forge.yaml deleted file mode 100644 index a41dbfd..0000000 --- a/config/forge/images/is_forge.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: image.openshift.io/v1 -kind: ImageStream -metadata: - name: forge-core - labels: - app: forge - component: image-build - annotations: - description: "FORGE core image for remote job execution" -spec: - lookupPolicy: - local: true - tags: - - name: main - annotations: - description: "Main branch build of FORGE core" - tags: "latest,main" diff --git a/config/forge/samples/job-full.yaml b/config/forge/samples/job-full.yaml deleted file mode 100644 index b202502..0000000 --- a/config/forge/samples/job-full.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: fournos.dev/v1 -kind: FournosJob -metadata: - generateName: forge-full-sample- -spec: - owner: perf-team - displayName: forge-full-sample - cluster: cluster-1 - pipeline: forge-full - executionEngine: - forge: - project: llm_d - args: - - cks - env: - OCPCI_SUITE: regression - OCPCI_VARIANT: nightly diff --git a/config/forge/samples/job-test-only.yaml b/config/forge/samples/job-test-only.yaml deleted file mode 100644 index 4a1b267..0000000 --- a/config/forge/samples/job-test-only.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: fournos.dev/v1 -kind: FournosJob -metadata: - generateName: forge-sample- -spec: - owner: perf-team - displayName: forge-sample - cluster: cluster-1 - pipeline: forge-test-only - executionEngine: - forge: - project: llm_d - args: - - cks - env: - OCPCI_SUITE: regression - OCPCI_VARIANT: nightly diff --git a/config/forge/workflows/pipeline-full.yaml b/config/forge/workflows/pipeline-full.yaml deleted file mode 100644 index e79c0b2..0000000 --- a/config/forge/workflows/pipeline-full.yaml +++ /dev/null @@ -1,110 +0,0 @@ -# Full pipeline: pre_cleanup -> prepare → test → export-artifacts + post-cleanup. -apiVersion: tekton.dev/v1 -kind: Pipeline -metadata: - name: forge-full - annotations: - fournos.dev/resolve-image: image-registry.openshift-image-registry.svc:5000/${NAMESPACE}/forge-core:main -spec: - workspaces: - - name: artifacts - params: - - name: fjob-name - type: string - - name: fournos-workload-namespace - type: string - - name: kubeconfig-secret - type: string - - tasks: - - name: pre-cleanup - taskRef: - name: forge-step - workspaces: - - name: artifacts - workspace: artifacts - params: - - name: fjob-name - value: "$(params.fjob-name)" - - name: fournos-workload-namespace - value: "$(params.fournos-workload-namespace)" - - name: kubeconfig-secret - value: "$(params.kubeconfig-secret)" - - name: job-step - value: pre-cleanup - - name: job-step-name - value: 000__pre-cleanup - - - name: prepare - taskRef: - name: forge-step - workspaces: - - name: artifacts - workspace: artifacts - params: - - name: fjob-name - value: "$(params.fjob-name)" - - name: fournos-workload-namespace - value: "$(params.fournos-workload-namespace)" - - name: kubeconfig-secret - value: "$(params.kubeconfig-secret)" - - name: job-step - value: prepare - - name: job-step-name - value: 001__prepare - - - name: test - runAfter: [prepare] - taskRef: - name: forge-step - workspaces: - - name: artifacts - workspace: artifacts - params: - - name: fjob-name - value: "$(params.fjob-name)" - - name: fournos-workload-namespace - value: "$(params.fournos-workload-namespace)" - - name: kubeconfig-secret - value: "$(params.kubeconfig-secret)" - - name: job-step - value: test - - name: job-step-name - value: 002__test - - finally: - - name: export-artifacts - taskRef: - name: forge-step - workspaces: - - name: artifacts - workspace: artifacts - params: - - name: fjob-name - value: "$(params.fjob-name)" - - name: fournos-workload-namespace - value: "$(params.fournos-workload-namespace)" - - name: kubeconfig-secret - value: "$(params.kubeconfig-secret)" - - name: job-step - value: export-artifacts - - name: job-step-name - value: 003__export-artifacts - - - name: post-cleanup - taskRef: - name: forge-step - workspaces: - - name: artifacts - workspace: artifacts - params: - - name: fjob-name - value: "$(params.fjob-name)" - - name: fournos-workload-namespace - value: "$(params.fournos-workload-namespace)" - - name: kubeconfig-secret - value: "$(params.kubeconfig-secret)" - - name: job-step - value: pre-cleanup # there's currently no post-cleanup step - - name: job-step-name - value: 004__post-cleanup diff --git a/config/forge/workflows/pipeline-test-only.yaml b/config/forge/workflows/pipeline-test-only.yaml deleted file mode 100644 index 7e6fae0..0000000 --- a/config/forge/workflows/pipeline-test-only.yaml +++ /dev/null @@ -1,55 +0,0 @@ -# Single-step pipeline: run only (no prepare, no cleanup). -# Use on pre-configured clusters where setup/teardown is handled externally. -apiVersion: tekton.dev/v1 -kind: Pipeline -metadata: - name: forge-test-only - annotations: - fournos.dev/resolve-image: image-registry.openshift-image-registry.svc:5000/${NAMESPACE}/forge-core:main -spec: - workspaces: - - name: artifacts - params: - - name: fjob-name - type: string - - name: fournos-workload-namespace - type: string - - name: kubeconfig-secret - type: string - tasks: - - name: test - taskRef: - name: forge-step - workspaces: - - name: artifacts - workspace: artifacts - params: - - name: fjob-name - value: "$(params.fjob-name)" - - name: fournos-workload-namespace - value: "$(params.fournos-workload-namespace)" - - name: kubeconfig-secret - value: "$(params.kubeconfig-secret)" - - name: job-step - value: test - - name: job-step-name - value: 000__test - - finally: - - name: export-artifacts - taskRef: - name: forge-step - workspaces: - - name: artifacts - workspace: artifacts - params: - - name: fjob-name - value: "$(params.fjob-name)" - - name: fournos-workload-namespace - value: "$(params.fournos-workload-namespace)" - - name: kubeconfig-secret - value: "$(params.kubeconfig-secret)" - - name: job-step - value: export-artifacts - - name: job-step-name - value: 001__export-artifacts diff --git a/config/forge/workflows/tasks.yaml b/config/forge/workflows/tasks.yaml deleted file mode 100644 index d7770b4..0000000 --- a/config/forge/workflows/tasks.yaml +++ /dev/null @@ -1,102 +0,0 @@ -apiVersion: tekton.dev/v1 -kind: Task -metadata: - name: forge-step -spec: - workspaces: - - name: artifacts - description: Shared artifact storage across pipeline tasks. - params: - - name: fjob-name - type: string - - name: fournos-workload-namespace - type: string - - name: kubeconfig-secret - type: string - - name: job-step - type: string - - name: job-step-name - type: string - steps: - - name: forge - image: image-registry.openshift-image-registry.svc:5000/$(context.taskRun.namespace)/forge-core:main - imagePullPolicy: Always - env: - - name: TARGET_KUBECONFIG - value: /var/run/secrets/fournos-kubeconfig/kubeconfig - - name: FJOB_NAME - value: "$(params.fjob-name)" - - name: FOURNOS_WORKLOAD_NAMESPACE - value: "$(params.fournos-workload-namespace)" - - name: FOURNOS_STEP - value: "$(params.job-step)" - - name: FOURNOS_STEP_NAME - value: "$(params.job-step-name)" - - name: FOURNOS_SECRETS - value: /var/run/secrets/fournos - - name: FOURNOS_CI - value: "true" - volumeMounts: - - name: kubeconfig - mountPath: /var/run/secrets/fournos-kubeconfig - readOnly: true - - name: vault-secrets - mountPath: /var/run/secrets/fournos - readOnly: true - script: | - #!/usr/bin/env bash - - set -o pipefail - set -o errexit - set -o nounset - set -o errtrace - - export ARTIFACT_BASE_DIR="$(workspaces.artifacts.path)" - export ARTIFACT_DIR="${ARTIFACT_BASE_DIR}/${FOURNOS_STEP_NAME}" - mkdir -p "${ARTIFACT_DIR}" - - exec &> >(tee -a "$ARTIFACT_DIR/run.log") - - # Fetch FournosJob from the hub cluster (in-cluster SA) before - # switching KUBECONFIG to the target cluster. - oc get "fjob/$FJOB_NAME" -n "$FOURNOS_WORKLOAD_NAMESPACE" -oyaml > $ARTIFACT_DIR/fournos_fjob.yaml - - export KUBECONFIG=$TARGET_KUBECONFIG - - PULL_BASE_SHA=$(cat "$ARTIFACT_DIR/fournos_fjob.yaml" | yq -r .spec.env.PULL_BASE_SHA) - if [[ "$PULL_BASE_SHA" != null ]]; then - export PULL_BASE_SHA - else - unset PULL_BASE_SHA - fi - - PULL_PULL_SHA=$(cat "$ARTIFACT_DIR/fournos_fjob.yaml" | yq -r .spec.env.PULL_PULL_SHA) - if [[ "$PULL_PULL_SHA" != null ]]; then - export PULL_PULL_SHA - else - unset PULL_PULL_SHA - fi - - if [[ "${PULL_PULL_SHA:-}" ]]; then - echo - echo "Updating to PULL_PULL_SHA=$PULL_PULL_SHA ..." - - git -C "$FORGE_HOME" fetch --quiet origin "$PULL_PULL_SHA" - git -C "$FORGE_HOME" reset --hard FETCH_HEAD - else - echo "No PULL_PULL_SHA, using the image commit" - fi - git show --quiet - - FORGE_PROJECT=$(cat "$ARTIFACT_DIR/fournos_fjob.yaml" | yq -r .spec.executionEngine.forge.project) - if [[ -z "$FORGE_PROJECT" || "$FORGE_PROJECT" == null ]]; then - echo "ERROR: invalid .spec.executionEngine.forge.project='$FORGE_PROJECT' in $FOURNOS_WORKLOAD_NAMESPACE/$FJOB_NAME" - exit 1 - fi - - exec bin/run_ci "$FORGE_PROJECT" ci "$FOURNOS_STEP" - - volumes: - - name: kubeconfig - secret: - secretName: $(params.kubeconfig-secret) diff --git a/hacks/copy-kubeconfig-secrets-to-ns.sh b/hacks/copy-kubeconfig-secrets-to-ns.sh deleted file mode 100755 index 523004b..0000000 --- a/hacks/copy-kubeconfig-secrets-to-ns.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -set -o pipefail -set -o errexit -set -o nounset -set -o errtrace - -# Use the first argument if provided, otherwise use the current project name -TARGET_NS=${1:-$(oc project -q)} -SOURCE_NS="psap-automation" - -echo "Copying kubeconfig-* secrets from $SOURCE_NS to $TARGET_NS..." - -# Loop through filtered secrets -for SECRET_NAME in $(oc get secrets -n $SOURCE_NS -o name | grep "^secret/kubeconfig-"); do - # Extract from source and create in target - # 1. Fetch the secret in JSON - # 2. Use jq to delete all the "junk" metadata - # 3. Create it in the new namespace - oc get "$SECRET_NAME" -n "$SOURCE_NS" -o json | jq ' - del( - .metadata.namespace, - .metadata.uid, - .metadata.resourceVersion, - .metadata.creationTimestamp, - .metadata.managedFields, - .metadata.ownerReferences - )' | oc apply -n "$TARGET_NS" -f- -done diff --git a/tests/forge/deploy/orchestration/config.yaml b/tests/forge/deploy/orchestration/config.yaml index bbba9d5..5696856 100644 --- a/tests/forge/deploy/orchestration/config.yaml +++ b/tests/forge/deploy/orchestration/config.yaml @@ -63,22 +63,6 @@ fournos_deploy: deploy: fournos: manifests/deployment.yaml - config: - forge_image: - - config/forge/images/build_forge-main.yaml - - config/forge/images/buildrun_forge-main.yaml # will be skipped - - config/forge/images/is_forge.yaml - forge_workflow: - - config/forge/workflows/pipeline-full.yaml - - config/forge/workflows/pipeline-test-only.yaml - - config/forge/workflows/tasks.yaml - kueue_config: - - config/kueue-cluster-config.yaml # will be skipped - - config/kueue-config.yaml - - to_build: - - config/forge/images/build_forge-main.yaml - replace: NAMESPACE: "@fournos_deploy.namespace.name" CONTROLLER_NAMESPACE: "@fournos_deploy.controller_namespace.name" diff --git a/tests/forge/deploy/orchestration/deploy.py b/tests/forge/deploy/orchestration/deploy.py index fb4ad4a..cdac32c 100644 --- a/tests/forge/deploy/orchestration/deploy.py +++ b/tests/forge/deploy/orchestration/deploy.py @@ -1,6 +1,5 @@ from projects.core.library import env, config, run, vault from projects.cluster.toolbox.build_image.main import run as build_image_toolbox -from projects.cluster.toolbox.rebuild_image.main import run as rebuild_image_toolbox import pathlib import logging @@ -512,135 +511,6 @@ def deploy_fournos_workload(): return 0 -def deploy_workflow_config(): - """ - Deploy FORGE workflow configuration for FOURNOS - - Returns: - int: 0 on success, raises exception on failure - """ - logger.info("=== Deploying FORGE Workflow Configuration ===") - - # Get configuration - fournos_source = Path( - config.project.get_config("fournos_deploy.fournos_source.path") - ) - skip_kinds = set(config.project.get_config("fournos_deploy.manifests.skip_kinds")) - config_manifests = config.project.get_config("fournos_deploy.manifests.config") - - if not fournos_source.exists(): - raise ValueError(f"FOURNOS source directory not found: {fournos_source}") - - # Ensure namespace exists - namespace = ensure_namespace() - - logger.info(f"Deploying from: {fournos_source}") - logger.info(f"Target namespace: {namespace}") - - # Combine all manifest files from all config sections - manifest_files = [] - for section_name, section_files in config_manifests.items(): - manifest_files.extend(section_files) - logger.info(f"Added {len(section_files)} manifests from {section_name}") - - logger.info(f"Will deploy {len(manifest_files)} total config manifest files") - logger.info(f"Skipping kinds: {list(skip_kinds)}") - - # Deploy the config manifests using common helper - _deploy_manifest_list( - manifest_files, namespace, fournos_source, skip_kinds, "config" - ) - - logger.info("✅ FORGE workflow configuration deployment completed") - - return 0 - - -def rebuild_workflow_images(): - """ - Rebuild the workflow images - - Returns: - int: 0 on success, raises exception on failure - """ - logger.info("=== Rebuilding FOURNOS workflow images ===") - - # Get configuration - namespace = ensure_namespace() - fournos_source = Path( - config.project.get_config("fournos_deploy.fournos_source.path") - ) - to_build_manifests = config.project.get_config("fournos_deploy.manifests.to_build") - force_rebuild = config.project.get_config( - "fournos_deploy.images.workflows.force_rebuild", print=False - ) - - logger.info(f"Force rebuild workflows: {force_rebuild}") - - if not to_build_manifests: - logger.info("No builds configured for rebuild") - return 0 - - logger.info(f"Found {len(to_build_manifests)} build manifest(s) to process") - - # Extract build names from manifests and rebuild each - skipped_builds = 0 - for manifest_path in to_build_manifests: - manifest_file = fournos_source / manifest_path - - if not manifest_file.exists(): - raise ValueError(f"Build manifest {manifest_file} doesn't exist") - - logger.info(f"Processing build manifest: {manifest_path}") - - # Apply text replacements to resolve any config references - manifest_content = _apply_manifest_replacements(manifest_file) - - # Parse YAML to extract build name and output image - doc = yaml.safe_load(manifest_content) - - if not (doc and doc.get("kind") == "Build"): - raise ValueError(f"Build manifest {manifest_file} isn't a Build") - - build_name = doc["metadata"]["name"] - output_image = doc["spec"]["output"]["image"] - - logger.info(f"Rebuilding build: {build_name}") - logger.info(f"Output image: {output_image}") - - # Check if image already exists and force_rebuild is False - if not force_rebuild: - logger.info("Checking if output image already exists...") - - # Extract ImageStreamTag name from output image (everything after last /) - # Format: image-registry.openshift-image-registry.svc:5000/namespace/imagestream:tag - istag_name = output_image.split("/")[-1] - - # Check if ImageStreamTag exists - if _istag_exists(istag_name, namespace): - logger.info(f"✅ Image {istag_name} already exists, skipping rebuild") - skipped_builds += 1 - continue - - logger.info(f"Image {istag_name} does not exist, proceeding with rebuild") - - result = rebuild_image_toolbox( - build_name=build_name, namespace=namespace, timeout_minutes=30 - ) - - if not result: - logger.error(f"❌ Rebuild failed for build: {build_name}") - raise RuntimeError(f"Rebuild of {manifest_file} failed :/") - - logger.info(f"✅ Rebuild completed successfully for build: {build_name}") - - logger.info( - f"✅ FORGE image rebuild completed - {skipped_builds} skipped, {len(to_build_manifests) - skipped_builds} rebuilt" - ) - - return 0 - - def _cleanup_namespace(namespace, cleanup_resources): """ Clean up resources in a single namespace. @@ -815,16 +685,6 @@ def deploy(): result = deploy_fournos_workload() total_errors += result - # Step 4: Deploy FORGE workflow configuration - logger.info("Step 4: Deploying FORGE workflow configuration...") - result = deploy_workflow_config() - total_errors += result - - # Step 5: Rebuild FORGE images - logger.info("Step 5: Rebuilding FOURNOS workflow images...") - result = rebuild_workflow_images() - total_errors += result - if total_errors == 0: logger.info("✅ Complete FOURNOS deployment succeeded") else: