fix(backend): inject user identity header in scheduled-workflow-controller for multi-user auth #8897
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: KFP E2E Pipeline tests | |
| env: | |
| E2E_TESTS_DIR: "./backend/test/end2end" | |
| NUMBER_OF_PARALLEL_NODES: 10 | |
| CLUSTER_NAME: "kfp" | |
| NAMESPACE: "kubeflow" | |
| PYTHON_VERSION: "3.9" | |
| USER_NAMESPACE: "kubeflow-user-example-com" | |
| CA_CERT_PATH: "" | |
| AWS_ACCESS_KEY_ID: 'minio' | |
| AWS_SECRET_ACCESS_KEY: 'minio123' | |
| AWS_S3_BUCKET: 'mlpipeline' | |
| MLFLOW_TRACKING_URI: "https://localhost:8080/mlflow" | |
| MLFLOW_TRACKING_INSECURE_TLS: "true" | |
| on: | |
| push: | |
| branches: [master] | |
| pull_request: | |
| paths: | |
| - '.github/workflows/e2e-test.yml' | |
| - '.github/workflows/image-builds.yml' | |
| - '.github/actions/create-cluster/**' | |
| - '.github/actions/deploy/**' | |
| - '.github/resources/**' | |
| - 'api/**' | |
| - 'go.mod' | |
| - 'go.sum' | |
| - 'backend/**' | |
| - 'proxy/**' | |
| - 'manifests/kustomize/**' | |
| - 'test_data/sdk_compiled_pipelines/**' | |
| - '!**/*.md' | |
| - '!**/OWNERS' | |
| workflow_dispatch: | |
| inputs: | |
| test_label: | |
| description: "Test label that you want to filter on and run" | |
| default: 'ApiServerTests' | |
| required: true | |
| type: string | |
| number_of_parallel_tests: | |
| description: "Number of ginkgo nodes that you want run in parallel, it essentially is equivalent to number of parallel tests with some caveats" | |
| default: 10 | |
| required: true | |
| type: number | |
| namespace: | |
| description: "Namespace where you want to create your pipelines in" | |
| default: "kubeflow" | |
| required: true | |
| type: string | |
| concurrency: | |
| group: e2e-tests-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| build: | |
| uses: ./.github/workflows/image-builds.yml | |
| end-to-end-scenario-tests: | |
| runs-on: ubuntu-latest | |
| needs: build | |
| strategy: | |
| matrix: | |
| k8s_version: ["v1.35.0"] | |
| cache_enabled: ["true", "false"] | |
| argo_version: [ "v3.7.3", "v3.5.14", "v4.0.4" ] | |
| proxy: [ "false" ] | |
| test_label: [ "E2ECritical", "E2EEssential", "E2EParallelNested" ] | |
| pod_to_pod_tls_enabled: [ "false" ] | |
| include: | |
| - k8s_version: "v1.31.14" | |
| cache_enabled: "false" | |
| argo_version: "v3.5.14" | |
| test_label: "E2ECritical" | |
| - k8s_version: "v1.35.0" | |
| cache_enabled: "false" | |
| proxy: "true" | |
| test_label: "E2EProxy" | |
| - k8s_version: "v1.35.0" | |
| cache_enabled: "false" | |
| test_label: "E2EFailure" | |
| - k8s_version: "v1.35.0" | |
| cache_enabled: "true" | |
| pod_to_pod_tls_enabled: "true" | |
| test_label: "E2ECritical" | |
| fail-fast: false | |
| name: End to End ${{ matrix.test_label}} Tests - K8s ${{ matrix.k8s_version }} cacheEnabled=${{ matrix.cache_enabled }} argoVersion=${{ matrix.argo_version}} proxy=${{ matrix.proxy}} pod_to_pod_tls_enabled=${{ matrix.pod_to_pod_tls_enabled }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v6 | |
| - name: Create cluster | |
| uses: ./.github/actions/create-cluster | |
| id: create-cluster | |
| with: | |
| k8s_version: ${{ matrix.k8s_version }} | |
| cluster_name: ${{ env.CLUSTER_NAME }} | |
| - name: Deploy KFP | |
| uses: ./.github/actions/deploy | |
| if: ${{ steps.create-cluster.outcome == 'success' }} | |
| id: deploy | |
| with: | |
| proxy: ${{ matrix.proxy }} | |
| cache_enabled: ${{ matrix.cache_enabled }} | |
| argo_version: ${{ matrix.argo_version }} | |
| pod_to_pod_tls_enabled: ${{ matrix.pod_to_pod_tls_enabled }} | |
| image_path: ${{ needs.build.outputs.IMAGE_PATH }} | |
| image_tag: ${{ needs.build.outputs.IMAGE_TAG }} | |
| image_registry: ${{ needs.build.outputs.IMAGE_REGISTRY }} | |
| - name: Configure Cluster CA Cert for TLS-Enabled Tests | |
| shell: bash | |
| if: ${{ matrix.pod_to_pod_tls_enabled == 'true'}} | |
| run: | | |
| kubectl get secret kfp-api-tls-cert -n kubeflow -o jsonpath='{.data.ca\.crt}' | base64 -d > "${{ github.workspace }}/ca.crt" | |
| echo "CA_CERT_PATH=${{ github.workspace }}/ca.crt" >> "$GITHUB_ENV" | |
| - name: Configure Input Variables | |
| shell: bash | |
| id: configure | |
| if: ${{ steps.deploy.outcome == 'success' }} | |
| run: | | |
| NUMBER_OF_NODES=${{ env.NUMBER_OF_PARALLEL_NODES }} | |
| TEST_LABEL=${{ matrix.test_label }} | |
| NAMESPACE=${{ env.NAMESPACE }} | |
| if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then | |
| NUMBER_OF_NODES=${{ inputs.number_of_parallel_tests }} | |
| TEST_LABEL=${{ inputs.test_label }} | |
| NAMESPACE=${{ inputs.namespace }} | |
| fi | |
| if [ "$TEST_LABEL" == "E2EParallelNested" ]; then | |
| NUMBER_OF_NODES=3 | |
| fi | |
| { | |
| echo "NUMBER_OF_NODES=$NUMBER_OF_NODES" | |
| echo "TEST_LABEL=$TEST_LABEL" | |
| echo "NAMESPACE=$NAMESPACE" | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Build and upload the sample Modelcar image to Kind | |
| id: build-sample-modelcar-image | |
| if: ${{ steps.deploy.outcome == 'success' }} | |
| run: | | |
| docker build -f ./test_data/sdk_compiled_pipelines/valid/critical/modelcar/Dockerfile -t registry.domain.local/modelcar:test . | |
| kind --name kfp load docker-image registry.domain.local/modelcar:test | |
| continue-on-error: true | |
| - name: Run Tests | |
| uses: ./.github/actions/test-and-report | |
| id: test-run | |
| if: ${{ steps.configure.outcome == 'success' }} | |
| with: | |
| proxy: ${{ matrix.proxy }} | |
| cache_enabled: ${{ matrix.cache_enabled }} | |
| test_directory: ${{ env.E2E_TESTS_DIR }} | |
| test_label: ${{ steps.configure.outputs.TEST_LABEL }} | |
| num_parallel_nodes: ${{ steps.configure.outputs.NUMBER_OF_NODES }} | |
| default_namespace: ${{ steps.configure.outputs.NAMESPACE }} | |
| python_version: ${{ env.PYTHON_VERSION }} | |
| report_name: "${{ matrix.test_label}}Tests_K8s=${{ matrix.k8s_version }}_cacheEnabled=${{ matrix.cache_enabled }}_argoVersion=${{ matrix.argo_version}}_proxy=${{ matrix.proxy }}" | |
| tls_enabled: ${{ matrix.pod_to_pod_tls_enabled }} | |
| ca_cert_path: ${{ env.CA_CERT_PATH }} | |
| end-to-end-critical-scenario-multi-user-tests: | |
| runs-on: ubuntu-latest | |
| needs: build | |
| strategy: | |
| matrix: | |
| k8s_version: ["v1.35.0"] | |
| cache_enabled: ["true", "false"] | |
| multi_user: [ "true" ] | |
| test_label: [ "E2ECritical" ] | |
| include: | |
| - multi_user: "true" | |
| artifact_proxy: "true" | |
| k8s_version: "v1.35.0" | |
| cache_enabled: "true" | |
| test_label: "E2ECritical" | |
| fail-fast: false | |
| name: End to End Critical Scenario Multi User Tests - K8s ${{ matrix.k8s_version }} cacheEnabled=${{ matrix.cache_enabled }} multiUser=${{ matrix.multi_user }} artifactProxy=${{ matrix.artifact_proxy }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v6 | |
| - name: Create cluster | |
| uses: ./.github/actions/create-cluster | |
| id: create-cluster | |
| with: | |
| k8s_version: ${{ matrix.k8s_version }} | |
| cluster_name: ${{ env.CLUSTER_NAME }} | |
| - name: Deploy KFP | |
| uses: ./.github/actions/deploy | |
| if: ${{ steps.create-cluster.outcome == 'success' }} | |
| id: deploy | |
| with: | |
| cache_enabled: ${{ matrix.cache_enabled }} | |
| multi_user: "true" | |
| artifact_proxy: ${{ matrix.artifact_proxy }} | |
| image_path: ${{ needs.build.outputs.IMAGE_PATH }} | |
| image_tag: ${{ needs.build.outputs.IMAGE_TAG }} | |
| image_registry: ${{ needs.build.outputs.IMAGE_REGISTRY }} | |
| - name: Configure Input Variables | |
| shell: bash | |
| id: configure | |
| if: ${{ steps.deploy.outcome == 'success' }} | |
| run: | | |
| NUMBER_OF_NODES=${{ env.NUMBER_OF_PARALLEL_NODES }} | |
| TEST_LABEL=${{ matrix.test_label }} | |
| NAMESPACE=${{ env.NAMESPACE }} | |
| if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then | |
| NUMBER_OF_NODES=${{ inputs.number_of_parallel_tests }} | |
| TEST_LABEL=${{ inputs.test_label }} | |
| NAMESPACE=${{ inputs.namespace }} | |
| fi | |
| { | |
| echo "NUMBER_OF_NODES=$NUMBER_OF_NODES" | |
| echo "TEST_LABEL=$TEST_LABEL" | |
| echo "NAMESPACE=$NAMESPACE" | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Build and upload the sample Modelcar image to Kind | |
| id: build-sample-modelcar-image | |
| if: ${{ steps.deploy.outcome == 'success' }} | |
| run: | | |
| docker build -f ./test_data/sdk_compiled_pipelines/valid/critical/modelcar/Dockerfile -t registry.domain.local/modelcar:test . | |
| kind --name kfp load docker-image registry.domain.local/modelcar:test | |
| continue-on-error: true | |
| - name: Test Artifact Proxy | |
| id: test-artifact-proxy | |
| if: ${{ steps.deploy.outcome == 'success' && matrix.artifact_proxy == 'true' }} | |
| shell: bash | |
| env: | |
| USER_NAMESPACE: ${{ env.USER_NAMESPACE }} | |
| run: | | |
| ./test/artifact-proxy/test-artifact-proxy.sh "$USER_NAMESPACE" | |
| - name: Run Tests | |
| uses: ./.github/actions/test-and-report | |
| if: ${{ steps.configure.outcome == 'success' }} | |
| id: test-run | |
| env: | |
| LOCAL_API_SERVER: "true" | |
| MULTI_USER: ${{ matrix.multi_user }} | |
| with: | |
| multi_user: ${{ matrix.multi_user }} | |
| cache_enabled: ${{ matrix.cache_enabled }} | |
| test_directory: ${{ env.E2E_TESTS_DIR }} | |
| test_label: ${{ steps.configure.outputs.TEST_LABEL }} | |
| num_parallel_nodes: ${{ steps.configure.outputs.NUMBER_OF_NODES }} | |
| default_namespace: ${{ steps.configure.outputs.NAMESPACE }} | |
| python_version: ${{ env.PYTHON_VERSION }} | |
| user_namespace: ${{ env.USER_NAMESPACE }} | |
| report_name: "E2EMultiUserTests_K8s=${{ matrix.k8s_version }}_cacheEnabled=${{ matrix.cache_enabled }}_multiUser=${{ matrix.multi_user }}" | |
| end-to-end-critical-mlflow-tests: | |
| runs-on: ubuntu-latest | |
| needs: build | |
| strategy: | |
| matrix: | |
| k8s_version: [ "v1.34.0" ] | |
| cache_enabled: [ "true", "false" ] | |
| argo_version: [ "v3.7.3" ] | |
| proxy: [ "false" ] | |
| test_label: [ "MLflow" ] | |
| pod_to_pod_tls_enabled: [ "false" ] | |
| multi_user: [ "false" ] | |
| artifact_proxy: [ "false" ] | |
| artifact_storage: [ "file", "s3" ] | |
| backend_store: [ "postgres" ] | |
| registry_store: [ "postgres" ] | |
| fail-fast: false | |
| name: End to End Critical Scenario MLflow Tests - K8s ${{ matrix.k8s_version }} cacheEnabled=${{ matrix.cache_enabled }} artifactStorage=${{ matrix.artifact_storage }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| - name: Create cluster | |
| uses: ./.github/actions/create-cluster | |
| id: create-cluster | |
| with: | |
| k8s_version: ${{ matrix.k8s_version }} | |
| cluster_name: ${{ env.CLUSTER_NAME }} | |
| - name: Deploy KFP | |
| uses: ./.github/actions/deploy | |
| if: ${{ steps.create-cluster.outcome == 'success' }} | |
| id: deploy | |
| with: | |
| cache_enabled: ${{ matrix.cache_enabled }} | |
| argo_version: ${{ matrix.argo_version }} | |
| pod_to_pod_tls_enabled: ${{ matrix.pod_to_pod_tls_enabled }} | |
| multi_user: ${{ matrix.multi_user }} | |
| artifact_proxy: ${{ matrix.artifact_proxy }} | |
| image_path: ${{ needs.build.outputs.IMAGE_PATH }} | |
| image_tag: ${{ needs.build.outputs.IMAGE_TAG }} | |
| image_registry: ${{ needs.build.outputs.IMAGE_REGISTRY }} | |
| - name: Deploy MLflow | |
| id: deploy-mlflow | |
| uses: opendatahub-io/mlflow-operator/.github/actions/deploy@8ab07a89d6d2d6bc2ffa0c8601f4a856a4cb1b18 | |
| if: ${{ steps.create-cluster.outcome == 'success' }} | |
| with: | |
| namespace: 'opendatahub' | |
| mlflow_image: quay.io/${{ github.repository_owner == 'red-hat-data-services' && 'rhoai' || 'opendatahub' }}/mlflow:odh-stable | |
| mlflow_operator_image: quay.io/${{ github.repository_owner == 'red-hat-data-services' && 'rhoai' || 'opendatahub' }}/mlflow-operator:odh-stable | |
| backend_store: ${{ matrix.backend_store }} | |
| artifact_storage: ${{ matrix.artifact_storage }} | |
| registry_store: ${{ matrix.registry_store }} | |
| s3_access_key: ${{ env.AWS_ACCESS_KEY_ID }} | |
| s3_secret_key: ${{ env.AWS_SECRET_ACCESS_KEY }} | |
| - name: Wait for MLflow stack readiness | |
| shell: bash | |
| if: ${{ steps.deploy-mlflow.outcome == 'success' }} | |
| run: | | |
| kubectl wait --for=condition=Ready pods --field-selector=status.phase=Running -n opendatahub --timeout=180s | |
| echo "All running pods in opendatahub are Ready" | |
| - name: Configure MLflow Plugin and Forward Ports | |
| shell: bash | |
| id: configure-mlflow-plugin | |
| if: ${{ steps.deploy.outcome == 'success' && steps.deploy-mlflow.outcome == 'success' }} | |
| run: ./.github/resources/scripts/configure-mlflow.sh kubeflow opendatahub backend/src/apiserver/config/config.json | |
| - name: Configure Input Variables | |
| shell: bash | |
| id: configure | |
| if: ${{ steps.deploy.outcome == 'success' }} | |
| run: | | |
| NUMBER_OF_NODES=${{ env.NUMBER_OF_PARALLEL_NODES }} | |
| TEST_LABEL=${{ matrix.test_label }} | |
| NAMESPACE=${{ env.NAMESPACE }} | |
| if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then | |
| NUMBER_OF_NODES=${{ inputs.number_of_parallel_tests }} | |
| TEST_LABEL=${{ inputs.test_label }} | |
| NAMESPACE=${{ inputs.namespace }} | |
| fi | |
| { | |
| echo "NUMBER_OF_NODES=$NUMBER_OF_NODES" | |
| echo "TEST_LABEL=$TEST_LABEL" | |
| echo "NAMESPACE=$NAMESPACE" | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Build and upload the sample Modelcar image to Kind | |
| id: build-sample-modelcar-image | |
| if: ${{ steps.deploy.outcome == 'success' }} | |
| run: | | |
| docker build -f ./test_data/sdk_compiled_pipelines/valid/critical/modelcar/Dockerfile -t registry.domain.local/modelcar:test . | |
| kind --name kfp load docker-image registry.domain.local/modelcar:test | |
| continue-on-error: true | |
| - name: Run Tests | |
| uses: ./.github/actions/test-and-report | |
| id: test-run | |
| if: ${{ steps.configure.outcome == 'success' && steps.deploy-mlflow.outcome == 'success' && steps.configure-mlflow-plugin.outcome == 'success'}} | |
| with: | |
| cache_enabled: ${{ matrix.cache_enabled }} | |
| test_directory: ${{ env.E2E_TESTS_DIR }} | |
| test_label: ${{ steps.configure.outputs.TEST_LABEL }} | |
| num_parallel_nodes: ${{ steps.configure.outputs.NUMBER_OF_NODES }} | |
| default_namespace: ${{ steps.configure.outputs.NAMESPACE }} | |
| python_version: ${{ env.PYTHON_VERSION }} | |
| report_name: "MLflowTests_K8s=${{ matrix.k8s_version }}_cacheEnabled=${{ matrix.cache_enabled }}_artifactStorage=${{ matrix.artifact_storage }}" | |
| tls_enabled: ${{ matrix.pod_to_pod_tls_enabled }} | |
| ca_cert_path: ${{ env.CA_CERT_PATH }} | |
| mlflow_enabled: 'true' |