Repository navigation
GKE PD-Disaggregation Test #69
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: GKE PD-Disaggregation Test | |
| on: | |
| # Runs with a PR comment /run-gke-pd-disaggregation | |
| issue_comment: | |
| types: [created] | |
| schedule: # TODO: enable once validated functional | |
| - cron: '0 11 * * *' # 4AM PST (11:00 UTC) | |
| workflow_dispatch: | |
| inputs: | |
| pr_or_branch: | |
| description: 'Pull-request number or branch name to test' | |
| required: true | |
| default: 'main' | |
| type: string | |
| permissions: | |
| contents: read | |
| jobs: | |
| deploy_and_validate: | |
| if: > | |
| (github.event_name == 'schedule' || | |
| github.event_name == 'pull_request' || | |
| github.event_name == 'workflow_dispatch' || | |
| ( | |
| github.event_name == 'issue_comment' && | |
| github.event.issue.pull_request && | |
| ( | |
| contains(github.event.comment.body, '/run-gke-pd-disaggregation') | |
| ) && | |
| ( | |
| github.event.comment.author_association == 'OWNER' || | |
| github.event.comment.author_association == 'MEMBER' || | |
| github.event.comment.author_association == 'COLLABORATOR' | |
| ) | |
| )) | |
| runs-on: ubuntu-latest | |
| env: | |
| GCP_PROJECT_ID: llm-d-scale | |
| GKE_CLUSTER_NAME: llm-d-e2e-us-east5 | |
| GKE_CLUSTER_ZONE: us-east5 | |
| NAMESPACE: llm-d-pd | |
| GATEWAY: gke-l7-regional-external-managed | |
| GATEWAY_TYPE: gke | |
| PR_OR_BRANCH: ${{ github.event.inputs.pr_or_branch || github.event.issue.number || github.event.number || 'main' }} | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| with: | |
| persist-credentials: false | |
| - name: Determine if pr_or_branch is a PR number | |
| id: check_pr | |
| env: | |
| PR_OR_BRANCH: ${{ github.event.inputs.pr_or_branch }} | |
| shell: bash | |
| run: | | |
| echo "PR_OR_BRANCH=${PR_OR_BRANCH:-main}" >> "$GITHUB_ENV" | |
| if [[ "$PR_OR_BRANCH" =~ ^[0-9]+$ ]]; then | |
| echo "is_pr=true" >> "$GITHUB_OUTPUT" | |
| elif [[ "${{ github.event_name }}" = "pull_request" ]]; then | |
| echo "PR_OR_BRANCH=${{ github.event.pull_request.number }}" >> $GITHUB_ENV | |
| echo "is_pr=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "is_pr=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Fetch and checkout PR | |
| if: steps.check_pr.outputs.is_pr == 'true' | |
| run: | | |
| git fetch origin pull/"$PR_OR_BRANCH"/head:pr-"$PR_OR_BRANCH" | |
| git checkout pr-"$PR_OR_BRANCH" | |
| - name: Checkout branch | |
| if: steps.check_pr.outputs.is_pr == 'false' | |
| run: git checkout "$PR_OR_BRANCH" | |
| - name: Authenticate to Google Cloud | |
| id: auth | |
| uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 | |
| with: | |
| credentials_json: ${{ secrets.GKE_SA_KEY }} | |
| - name: Set up gcloud CLI and kubectl | |
| uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db | |
| with: | |
| project_id: ${{ env.GCP_PROJECT_ID }} | |
| install_components: 'kubectl,gke-gcloud-auth-plugin' | |
| - name: Get GKE credentials | |
| run: | | |
| gcloud container clusters get-credentials "${{ env.GKE_CLUSTER_NAME }}" --zone "${{ env.GKE_CLUSTER_ZONE }}" | |
| - name: Apply PD-Disaggregation slim transformation | |
| id: pd-transformation | |
| run: | | |
| echo "Applying pd-disaggregation slim values..." | |
| # Transform ms-pd/values.yaml to a scaled-down deployment | |
| # Set Prefill to 1 GPU | |
| yq e '.prefill.replicas = 1' -i guides/pd-disaggregation/ms-pd/values.yaml | |
| # Reduce model length to 3200 | |
| yq e '.decode.containers[].args |= ( (.. | select(. == "--max-model-len") | key) as $idx | .[$idx + 1] = "3200" )' -i guides/pd-disaggregation/ms-pd/values.yaml | |
| yq e '.prefill.containers[].args |= ( (.. | select(. == "--max-model-len") | key) as $idx | .[$idx + 1] = "3200" )' -i guides/pd-disaggregation/ms-pd/values.yaml | |
| # Verify the slimmed down values | |
| echo "📋 Verifying transformation results..." | |
| echo "Replicas in ms-pd prefill:" | |
| yq e '.prefill.replicas' guides/pd-disaggregation/ms-pd/values.yaml | |
| # Validate YAML syntax | |
| echo "Validating YAML syntax..." | |
| yq eval '.' guides/pd-disaggregation/ms-pd/values.yaml > /dev/null && echo "ms-pd/values.yaml: Valid" | |
| echo "✅ All YAML files are valid" | |
| # Display complete transformed files for debugging | |
| echo "" | |
| echo "🔍 === Completed transformed values files ===" | |
| echo "" | |
| echo "ms-pd/values.yaml (transformed):" | |
| echo "---" | |
| cat guides/pd-disaggregation/ms-pd/values.yaml | |
| echo "---" | |
| echo "" | |
| echo "gaie-pd/values.yaml:" | |
| echo "---" | |
| cat guides/pd-disaggregation/gaie-pd/values.yaml | |
| echo "---" | |
| echo "" | |
| echo "=== End of values files output ===" | |
| - name: Install dependencies | |
| run: | | |
| cd helpers | |
| ./client-setup/install-deps.sh | tee ~/install-deps.log | |
| - name: Create namespace | |
| run: | | |
| kubectl create namespace "${NAMESPACE}" || echo "Namespace already exists" | |
| - name: Create llm-d-hf-token secret | |
| run: | | |
| kubectl create secret generic llm-d-hf-token \ | |
| --from-literal="HF_TOKEN=${{ secrets.HF_TOKEN }}" \ | |
| --namespace "${NAMESPACE}" \ | |
| --dry-run=client -o yaml | kubectl apply -f - | |
| - name: Deploy model service and GIE with Helmfile | |
| run: | | |
| cd guides/pd-disaggregation | |
| helmfile apply -e ${GATEWAY_TYPE} \ | |
| --set "gateway.service.type=LoadBalancer" \ | |
| --skip-schema-validation \ | |
| | tee ~/pd-disaggregation-deployment.log | |
| echo "---------------------------------------" >> ~/pd-disaggregation-deployment.log | |
| - name: Deploy HTTPRoute | |
| run: | | |
| cd guides/pd-disaggregation | |
| echo "Deploying HTTPRoute..." | |
| kubectl apply -f httproute.gke.yaml -n ${NAMESPACE} | tee ~/pd-disaggregation-deployment.log | |
| echo "---------------------------------------" >> ~/pd-disaggregation-deployment.log | |
| - name: Upload helm get all | |
| shell: bash | |
| run: | | |
| release_names=(infra-pd gaie-pd ms-pd) | |
| for release_name in ${release_names[@]}; do | |
| bash .github/scripts/e2e/helm-get-all.sh \ | |
| ~/pd-disaggregation-deployment.log \ | |
| "${release_name}" \ | |
| "llm-d-pd" | |
| done | |
| - name: Wait for all pods to be ready | |
| run: | | |
| kubectl wait pod \ | |
| --for=condition=Ready \ | |
| --all \ | |
| -n "${NAMESPACE}" \ | |
| --timeout=15m | |
| sleep 480 # Allow extra time for model loading in PD setup | |
| echo "✅ All pods are ready." | |
| kubectl get pods -n "${NAMESPACE}" | |
| - name: Debug gateway | |
| run: | | |
| kubectl -n ${NAMESPACE} get gateway | |
| kubectl -n ${NAMESPACE} describe gateway | |
| kubectl -n ${NAMESPACE} get gateway -o yaml | |
| kubectl -n ${NAMESPACE} describe httpRoute | |
| kubectl -n ${NAMESPACE} get httpRoute -o yaml | |
| kubectl -n ${NAMESPACE} get inferencepool -o yaml | |
| - name: Wait for gateway to be ready | |
| run: | | |
| GATEWAY_NAME=infra-pd-inference-gateway | |
| kubectl wait gateway/${GATEWAY_NAME} \ | |
| --for=condition=Programmed=True \ | |
| -n "${NAMESPACE}" \ | |
| --timeout=300s | |
| echo "✅ Gateway is ready." | |
| kubectl get gateway -n "${NAMESPACE}" | |
| - name: Show deployment status | |
| run: | | |
| echo "=== Deployments ===" | |
| kubectl get deployments -n "${NAMESPACE}" | |
| echo "" | |
| echo "=== Replica Sets ===" | |
| kubectl get replicasets -n "${NAMESPACE}" | |
| echo "" | |
| echo "=== Pods ===" | |
| kubectl get pods -n "${NAMESPACE}" | |
| echo "" | |
| echo "=== Services ===" | |
| kubectl get svc -n "${NAMESPACE}" | |
| echo "" | |
| echo "=== Helm releases ===" | |
| helm list -n "${NAMESPACE}" || true | |
| echo "" | |
| echo "=== Inference Pools ===" | |
| kubectl get inferencepools -n "${NAMESPACE}" || true | |
| echo "" | |
| echo "=== HTTPRoutes ===" | |
| kubectl get httproutes -n "${NAMESPACE}" || true | |
| echo "" | |
| echo "=== Gateway ===" | |
| kubectl get Gateway -n "${NAMESPACE}" || true | |
| echo "" | |
| - name: PD-Disaggregation inference test | |
| run: | | |
| cd .github/scripts/e2e | |
| echo "🧪 Running PD-Disaggregation specific tests..." | |
| # Test the specific model and endpoints for PD setup | |
| ./e2e-validate.sh -n "${NAMESPACE}" | |
| # Additional PD-specific validation | |
| echo "🔍 Verifying PD-Disaggregation specific functionality..." | |
| # Check that we have both prefill and decode pods | |
| PREFILL_PODS=$(kubectl get pods -n "${NAMESPACE}" -l llm-d.ai/role=prefill --no-headers | wc -l) | |
| DECODE_PODS=$(kubectl get pods -n "${NAMESPACE}" -l llm-d.ai/role=decode --no-headers | wc -l) | |
| echo "Prefill pods: $PREFILL_PODS" | |
| echo "Decode pods: $DECODE_PODS" | |
| if [ "$PREFILL_PODS" -lt 1 ] || [ "$DECODE_PODS" -lt 1 ]; then | |
| echo "❌ Missing prefill or decode pods for PD setup" | |
| exit 1 | |
| fi | |
| echo "✅ PD-Disaggregation validation completed successfully" | |
| - name: Collect and upload Kubernetes pod logs | |
| if: always() | |
| run: | | |
| mkdir -p pod-logs-pd-disaggregation | |
| cd pod-logs-pd-disaggregation | |
| echo "Fetching ${NAMESPACE} pods log..." | |
| kubectl get pods -n "${NAMESPACE}" --no-headers -o custom-columns=":metadata.name" \ | |
| | xargs -I{} sh -c 'kubectl logs --all-containers=true -n "${NAMESPACE}" {} > "{}.log" 2>&1' | |
| echo "Fetching ${NAMESPACE} pods descriptions..." | |
| kubectl get pods -n "${NAMESPACE}" --no-headers -o custom-columns=":metadata.name" \ | |
| | xargs -I{} sh -c 'kubectl describe pod -n "${NAMESPACE}" {} > "{}-describe.log" 2>&1' | |
| mv ~/pd-disaggregation-deployment.log . || true | |
| mv ~/install-deps.log . || true | |
| - name: Upload pod logs as artifact | |
| uses: actions/upload-artifact@v7 | |
| if: always() | |
| with: | |
| name: llmd-pod-logs-pd-disaggregation | |
| path: pod-logs-pd-disaggregation | |
| - name: Send Google Chat notification on failure | |
| if: failure() | |
| uses: SimonScholz/google-chat-action@3b3519e5102dba8aa5046fd711c4b553586409bb | |
| with: | |
| webhookUrl: ${{ secrets.GOOGLE_CHAT_WEBHOOK }} | |
| jobStatus: ${{ job.status }} | |
| title: ${{ github.workflow }} | |
| - name: Cleanup deployment | |
| if: always() | |
| run: | | |
| cd guides/pd-disaggregation | |
| helmfile destroy | |
| kubectl delete -f httproute.gke.yaml -n ${NAMESPACE} |