diff --git a/README.md b/README.md index 29d088b..478b8e3 100644 --- a/README.md +++ b/README.md @@ -66,11 +66,14 @@ This system enables real-time torch profiling of vLLM model execution without re ### Deploy ```bash -# Deploy webhook and all components -./deploy.sh +# Deploy to h200 cluster +CLUSTER=psap-rhaiis-h200 ./deploy.sh + +# Deploy to b200 cluster +CLUSTER=b200 ./deploy.sh -# Or skip image build if using existing image -./deploy.sh --skip-build +# Skip building image (use existing) +CLUSTER=psap-rhaiis-h200 ./deploy.sh --skip-build ``` The deployment script will: @@ -81,6 +84,101 @@ The deployment script will: 5. Configure webhook with CA bundle 6. Validate deployment +## Multi-Cluster Support + +This project uses Kustomize overlays to support deployment across multiple clusters with different configurations. + +### Directory Structure + +``` +vllm-profiler/ +├── base/ +│ ├── kustomization.yaml # Base kustomization +│ └── manifests.yaml # Base manifests (no cluster-specific values) +├── overlays/ +│ ├── psap-rhaiis-h200/ # H200 cluster overlay +│ │ ├── kustomization.yaml # Cluster-specific ConfigMap namespace +│ │ ├── patch.yaml # nodeSelector + TARGET_NAMESPACE +│ │ ├── sitecustomize.py # Symlink to ../../sitecustomize.py +│ │ └── profiler_config.yaml # Symlink to ../../profiler_config.yaml +│ ├── b200/ # B200 cluster overlay +│ │ └── ... # Same structure +│ └── placeholder/ # Template for new clusters +│ └── ... # Same structure +├── sitecustomize.py # Shared profiler code (symlinked into overlays) +├── profiler_config.yaml # Shared profiler config (symlinked into overlays) +└── deploy.sh # Deployment script with CLUSTER parameter +``` + +### Cluster-Specific Configuration + +Each overlay contains: + +**`patch.yaml`** - Cluster-specific Kubernetes patches: +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: env-injector + namespace: vllm-profiler +spec: + template: + spec: + nodeSelector: + kubernetes.io/hostname: + containers: + - name: webhook + env: + - name: TARGET_NAMESPACE + value: "" +``` + +**`kustomization.yaml`** - ConfigMap namespace: +```yaml +configMapGenerator: + - name: env-injector-files + namespace: + files: + - sitecustomize.py + - profiler_config.yaml +``` + +### Adding a New Cluster + +1. Copy the placeholder overlay: + ```bash + cp -r overlays/placeholder overlays/my-new-cluster + ``` + +2. Create symlinks for shared files: + ```bash + cd overlays/my-new-cluster + ln -sf ../../sitecustomize.py sitecustomize.py + ln -sf ../../profiler_config.yaml profiler_config.yaml + ``` + +3. Edit `patch.yaml` with cluster-specific values: + - `nodeSelector` - Target node hostname + - `TARGET_NAMESPACE` - Namespace where pods will be profiled + +4. Edit `kustomization.yaml`: + - Update ConfigMap `namespace` to match TARGET_NAMESPACE + +5. Deploy: + ```bash + CLUSTER=my-new-cluster ./deploy.sh + ``` + +### Available Clusters + +| Cluster | Description | Usage | +|---------|-------------|-------| +| `psap-rhaiis-h200` | H200 GPU cluster | `CLUSTER=psap-rhaiis-h200 ./deploy.sh` | +| `b200` | B200 GPU cluster | `CLUSTER=b200 ./deploy.sh` | +| `placeholder` | Template for new clusters | Copy and customize | + +**Note:** The `CLUSTER` name is just a directory identifier under `overlays/`. It doesn't need to match the actual Kubernetes cluster name - you can name overlays however you prefer. + ### Configuration Edit `manifests.yaml` to configure target namespace and label selectors: @@ -178,11 +276,16 @@ kubectl cp downstream-llm-d/:/path/to/trace.json ./trace.json ``` vllm-profiler/ +├── base/ # Base Kustomize resources +│ ├── kustomization.yaml # Base kustomization +│ └── manifests.yaml # Base Kubernetes manifests +├── overlays/ # Cluster-specific overlays +│ ├── psap-rhaiis-h200/ # H200 cluster configuration +│ ├── b200/ # B200 cluster configuration +│ └── placeholder/ # Template for new clusters ├── sitecustomize.py # Profiler import hook (injected into pods) ├── profiler_config.yaml # Default profiler configuration ├── webhook.py # Flask mutating admission webhook -├── manifests.yaml # Kubernetes resources -├── kustomization.yaml # ConfigMap generator ├── Dockerfile # Webhook container image ├── requirements.txt # Python dependencies ├── deploy.sh # Deployment automation script @@ -268,6 +371,7 @@ See [CONFIGURATION_EXAMPLES.md](CONFIGURATION_EXAMPLES.md) for comprehensive con - `LOG_LEVEL`: Webhook logging level (default: "DEBUG") **Deployment:** +- `CLUSTER`: Cluster overlay to use (REQUIRED, e.g., "psap-rhaiis-h200", "b200") - `CONTAINER_RUNTIME`: Container runtime to use (default: "podman") - `IMAGE_REGISTRY`: Image registry (default: "quay.io/mimehta") - `IMAGE_TAG`: Image tag (default: "latest") diff --git a/base/kustomization.yaml b/base/kustomization.yaml new file mode 100644 index 0000000..60060a8 --- /dev/null +++ b/base/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - manifests.yaml diff --git a/manifests.yaml b/base/manifests.yaml similarity index 90% rename from manifests.yaml rename to base/manifests.yaml index fd67b45..154c6da 100644 --- a/manifests.yaml +++ b/base/manifests.yaml @@ -25,16 +25,13 @@ spec: app: env-injector spec: serviceAccountName: env-injector - nodeSelector: - kubernetes.io/hostname: psap-llmd-h200-gpu-qdndv containers: - name: webhook image: quay.io/mimehta/vllmprofiler imagePullPolicy: Always env: - name: TARGET_NAMESPACE - value: "downstream-llm-d" - # Multi-label selector (OR logic): pod with ANY of these labels will be instrumented + value: "default" # overridden by overlay - name: TARGET_LABELS value: "llm-d.ai/inferenceServing=true,app.kubernetes.io/component=llminferenceservice-workload" - name: INJECT_ENV_NAME @@ -90,4 +87,3 @@ webhooks: path: /mutate port: 443 caBundle: "" - diff --git a/deploy.sh b/deploy.sh index 637b267..6551f61 100755 --- a/deploy.sh +++ b/deploy.sh @@ -10,9 +10,8 @@ IMAGE_NAME="${IMAGE_NAME:-vllmprofiler}" IMAGE_REGISTRY="${IMAGE_REGISTRY:-quay.io/mimehta}" IMAGE_TAG="${IMAGE_TAG:-latest}" FULL_IMAGE="${IMAGE_REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG}" -MANIFESTS_FILE="manifests.yaml" NAMESPACE="vllm-profiler" -TARGET_NAMESPACE="${TARGET_NAMESPACE:-downstream-llm-d}" +CLUSTER="${CLUSTER:-}" # Colors for output RED='\033[0;31m' @@ -50,20 +49,24 @@ Options: --help Show this help message Environment Variables: + CLUSTER Cluster overlay to use (REQUIRED) + Available: psap-rhaiis-h200, b200, placeholder CONTAINER_RUNTIME Container runtime to use (default: podman) IMAGE_REGISTRY Image registry (default: quay.io/mimehta) IMAGE_TAG Image tag (default: latest) - TARGET_NAMESPACE Namespace to inject profiler into (default: downstream-llm-d) Examples: - # Full deployment - $0 + # Deploy to h200 cluster + CLUSTER=psap-rhaiis-h200 $0 + + # Deploy to b200 cluster + CLUSTER=b200 $0 # Skip building image (use existing) - $0 --skip-build + CLUSTER=psap-rhaiis-h200 $0 --skip-build - # Use different namespace - TARGET_NAMESPACE=my-namespace $0 + # Deploy to specific cluster without building + CLUSTER=b200 $0 --skip-build EOF exit 0 @@ -93,6 +96,23 @@ while [[ $# -gt 0 ]]; do esac done +# Validate CLUSTER is set +if [ -z "${CLUSTER}" ]; then + log_error "CLUSTER environment variable is required" + log_info "Available clusters:" + ls -1 overlays/ + log_info "Usage: CLUSTER= $0" + exit 1 +fi + +# Validate cluster overlay exists +if [ ! -d "overlays/${CLUSTER}" ]; then + log_error "Cluster overlay 'overlays/${CLUSTER}' not found" + log_info "Available overlays:" + ls -1 overlays/ + exit 1 +fi + # Main deployment main() { log_info "Starting vLLM Profiler Webhook deployment" @@ -100,17 +120,17 @@ main() { echo " - Container Runtime: ${CONTAINER_RUNTIME}" echo " - Image: ${FULL_IMAGE}" echo " - Webhook Namespace: ${NAMESPACE}" - echo " - Target Namespace: ${TARGET_NAMESPACE}" + echo " - Cluster Overlay: ${CLUSTER}" echo "" # Step 1: Build and push container image if [ "$SKIP_BUILD" = false ]; then - log_info "Step 1/7: Building container image..." - if ! ${CONTAINER_RUNTIME} build --runtime=runc -t "${IMAGE_NAME}" .; then + log_info "Step 1/6: Building container image..." + if ! ${CONTAINER_RUNTIME} build --platform linux/amd64 -t "${IMAGE_NAME}" .; then log_error "Container build failed" exit 1 fi - log_success "Container image built" + log_success "Container image built for linux/amd64" log_info "Tagging image as ${FULL_IMAGE}..." ${CONTAINER_RUNTIME} tag "localhost/${IMAGE_NAME}" "${FULL_IMAGE}" @@ -126,47 +146,39 @@ main() { fi # Step 2: Delete existing resources (idempotent deployment) - log_info "Step 2/7: Removing existing resources (if any)..." - oc delete -f "${MANIFESTS_FILE}" --ignore-not-found=true + log_info "Step 2/6: Removing existing resources (if any)..." + oc kustomize "overlays/${CLUSTER}" --load-restrictor LoadRestrictionsNone | oc delete -f - --ignore-not-found=true log_info "Waiting for resources to be deleted..." sleep 5 log_success "Existing resources removed" - # Step 3: Apply manifests - log_info "Step 3/7: Deploying webhook resources..." - if ! oc apply -f "${MANIFESTS_FILE}"; then - log_error "Failed to apply manifests" - exit 1 - fi - log_success "Webhook resources deployed" - - # Step 4: Apply kustomization (ConfigMap) - log_info "Step 4/7: Creating ConfigMap with profiler code..." - if ! oc apply -k .; then - log_error "Failed to apply kustomization" + # Step 3: Apply all resources via kustomize overlay + log_info "Step 3/6: Deploying webhook resources and ConfigMap..." + if ! oc kustomize "overlays/${CLUSTER}" --load-restrictor LoadRestrictionsNone | oc apply -f -; then + log_error "Failed to apply kustomize overlay" exit 1 fi - log_success "ConfigMap created in ${TARGET_NAMESPACE}" + log_success "Webhook resources and ConfigMap deployed" - # Step 5: Generate TLS certificates - log_info "Step 5/7: Generating TLS certificates..." + # Step 4: Generate TLS certificates + log_info "Step 4/6: Generating TLS certificates..." if ! bash gen-certs.sh; then log_error "Certificate generation failed" exit 1 fi log_success "TLS certificates generated" - # Step 6: Patch webhook with CA bundle - log_info "Step 6/7: Patching webhook with CA bundle..." + # Step 5: Patch webhook with CA bundle + log_info "Step 5/6: Patching webhook with CA bundle..." if ! bash patch-ca-bundle.sh; then log_error "CA bundle patching failed" exit 1 fi log_success "Webhook CA bundle configured" - # Step 7: Validate deployment + # Step 6: Validate deployment if [ "$SKIP_VALIDATION" = false ]; then - log_info "Step 7/7: Validating webhook deployment..." + log_info "Step 6/6: Validating webhook deployment..." if ! DO_SIMPLE_TEST=1 ./validate_webhook.sh; then log_warn "Webhook validation reported issues (see output above)" else @@ -179,13 +191,15 @@ main() { echo "" log_success "Deployment complete!" echo "" + log_info "Verify configuration with:" + echo " oc logs -n vllm-profiler deployment/env-injector | head -10" + echo "" log_info "Next steps:" - echo " 1. Create a pod in namespace '${TARGET_NAMESPACE}' with label:" - echo " llm-d.ai/inferenceServing=true" + echo " 1. Create a pod matching the TARGET_LABELS configured in overlays/${CLUSTER}/patch.yaml" echo " 2. Check pod logs for profiler output after ~100-150 model executions" echo " 3. Retrieve trace file from pod for Chrome trace visualization" echo "" - log_info "To remove all resources, run: ./teardown.sh" + log_info "To remove all resources, run: CLUSTER=${CLUSTER} ./teardown.sh" } # Run main function diff --git a/kustomization.yaml b/kustomization.yaml deleted file mode 100644 index 195142b..0000000 --- a/kustomization.yaml +++ /dev/null @@ -1,8 +0,0 @@ -configMapGenerator: -- name: env-injector-files - namespace: downstream-llm-d - files: - - sitecustomize.py - - profiler_config.yaml -generatorOptions: - disableNameSuffixHash: true diff --git a/overlays/b200/kustomization.yaml b/overlays/b200/kustomization.yaml new file mode 100644 index 0000000..4728990 --- /dev/null +++ b/overlays/b200/kustomization.yaml @@ -0,0 +1,18 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +patches: + - path: patch.yaml + +configMapGenerator: + - name: env-injector-files + namespace: kserve-e2e-perf + files: + - sitecustomize.py + - profiler_config.yaml + +generatorOptions: + disableNameSuffixHash: true diff --git a/overlays/b200/patch.yaml b/overlays/b200/patch.yaml new file mode 100644 index 0000000..db60291 --- /dev/null +++ b/overlays/b200/patch.yaml @@ -0,0 +1,15 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: env-injector + namespace: vllm-profiler +spec: + template: + spec: + containers: + - name: webhook + env: + - name: TARGET_NAMESPACE + value: "kserve-e2e-perf" + - name: TARGET_LABELS + value: "vllm-profiler/enabled=true" \ No newline at end of file diff --git a/overlays/b200/profiler_config.yaml b/overlays/b200/profiler_config.yaml new file mode 120000 index 0000000..e24091d --- /dev/null +++ b/overlays/b200/profiler_config.yaml @@ -0,0 +1 @@ +../../profiler_config.yaml \ No newline at end of file diff --git a/overlays/b200/sitecustomize.py b/overlays/b200/sitecustomize.py new file mode 120000 index 0000000..884d1e5 --- /dev/null +++ b/overlays/b200/sitecustomize.py @@ -0,0 +1 @@ +../../sitecustomize.py \ No newline at end of file diff --git a/overlays/placeholder/kustomization.yaml b/overlays/placeholder/kustomization.yaml new file mode 100644 index 0000000..3b7448f --- /dev/null +++ b/overlays/placeholder/kustomization.yaml @@ -0,0 +1,18 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +patches: + - path: patch.yaml + +configMapGenerator: + - name: env-injector-files + namespace: CHANGE_ME # TODO: Set target namespace for this cluster + files: + - sitecustomize.py + - profiler_config.yaml + +generatorOptions: + disableNameSuffixHash: true diff --git a/overlays/placeholder/patch.yaml b/overlays/placeholder/patch.yaml new file mode 100644 index 0000000..7cc3b0c --- /dev/null +++ b/overlays/placeholder/patch.yaml @@ -0,0 +1,15 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: env-injector + namespace: vllm-profiler +spec: + template: + spec: + containers: + - name: webhook + env: + - name: TARGET_NAMESPACE + value: "CHANGE_ME" # TODO: Set target namespace for this cluster + - name: TARGET_LABELS + value: "CHANGE_ME" # TODO: Set target labels (comma-separated, e.g., "label1=value1,label2=value2") diff --git a/overlays/placeholder/profiler_config.yaml b/overlays/placeholder/profiler_config.yaml new file mode 120000 index 0000000..e24091d --- /dev/null +++ b/overlays/placeholder/profiler_config.yaml @@ -0,0 +1 @@ +../../profiler_config.yaml \ No newline at end of file diff --git a/overlays/placeholder/sitecustomize.py b/overlays/placeholder/sitecustomize.py new file mode 120000 index 0000000..884d1e5 --- /dev/null +++ b/overlays/placeholder/sitecustomize.py @@ -0,0 +1 @@ +../../sitecustomize.py \ No newline at end of file diff --git a/overlays/psap-rhaiis-h200/kustomization.yaml b/overlays/psap-rhaiis-h200/kustomization.yaml new file mode 100644 index 0000000..4728990 --- /dev/null +++ b/overlays/psap-rhaiis-h200/kustomization.yaml @@ -0,0 +1,18 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +patches: + - path: patch.yaml + +configMapGenerator: + - name: env-injector-files + namespace: kserve-e2e-perf + files: + - sitecustomize.py + - profiler_config.yaml + +generatorOptions: + disableNameSuffixHash: true diff --git a/overlays/psap-rhaiis-h200/patch.yaml b/overlays/psap-rhaiis-h200/patch.yaml new file mode 100644 index 0000000..beb1c97 --- /dev/null +++ b/overlays/psap-rhaiis-h200/patch.yaml @@ -0,0 +1,15 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: env-injector + namespace: vllm-profiler +spec: + template: + spec: + containers: + - name: webhook + env: + - name: TARGET_NAMESPACE + value: "kserve-e2e-perf" + - name: TARGET_LABELS + value: "vllm-profiler/enabled=true" diff --git a/overlays/psap-rhaiis-h200/profiler_config.yaml b/overlays/psap-rhaiis-h200/profiler_config.yaml new file mode 120000 index 0000000..e24091d --- /dev/null +++ b/overlays/psap-rhaiis-h200/profiler_config.yaml @@ -0,0 +1 @@ +../../profiler_config.yaml \ No newline at end of file diff --git a/overlays/psap-rhaiis-h200/sitecustomize.py b/overlays/psap-rhaiis-h200/sitecustomize.py new file mode 120000 index 0000000..884d1e5 --- /dev/null +++ b/overlays/psap-rhaiis-h200/sitecustomize.py @@ -0,0 +1 @@ +../../sitecustomize.py \ No newline at end of file diff --git a/profiler_config.yaml b/profiler_config.yaml index 75aa63d..58457f3 100644 --- a/profiler_config.yaml +++ b/profiler_config.yaml @@ -7,7 +7,7 @@ # "100-150" - Profile calls 100 through 150 # "50-100,200-300" - Profile calls 50-100 AND 200-300 # "0-50" - Profile first 50 calls -profiling_ranges: "100-150" +profiling_ranges: "1050-1060" # Torch profiler activities to record # Available: CPU, CUDA @@ -34,10 +34,13 @@ output: # Set to false to skip trace export (only print table) export_chrome_trace: true - # Output file path (supports environment variable substitution) - # Use {pid} as placeholder for process ID - # Use {rank} for tensor parallel rank (if available) - file_pattern: "/tmp/trace_pid{pid}.json" + # Output file path (supports placeholders) + # {pid} - process ID + # {rank} - tensor parallel rank (if available) + # {range} - profiling range (e.g., "100_150" for range 100-150) + # {counter} - range counter (0 for first range, 1 for second, etc.) + # {datetime} - timestamp in YYYYMMDD_HHMMSS format + file_pattern: "/tmp/trace_{range}_{counter}_{datetime}.json" # Table output settings table: diff --git a/sitecustomize.py b/sitecustomize.py index a7434fa..6b9f11b 100644 --- a/sitecustomize.py +++ b/sitecustomize.py @@ -180,12 +180,23 @@ def _parse_ranges(self, ranges_str: str) -> List[Tuple[int, int]]: print(f"[profiler-config] Warning: Invalid range '{range_str}': {e}") return ranges - def get_output_filename(self, pid: Optional[int] = None, rank: Optional[int] = None) -> str: + def get_output_filename(self, pid: Optional[int] = None, rank: Optional[int] = None, + range_start: Optional[int] = None, range_end: Optional[int] = None, + counter: int = 0) -> str: """Generate output filename with substitutions.""" + from datetime import datetime + filename = self.output_file_pattern filename = filename.replace('{pid}', str(pid or os.getpid())) if rank is not None: filename = filename.replace('{rank}', str(rank)) + if range_start is not None and range_end is not None: + filename = filename.replace('{range}', f"{range_start}_{range_end}") + if '{counter}' in filename: + filename = filename.replace('{counter}', str(counter)) + if '{datetime}' in filename: + datetime_str = datetime.now().strftime('%Y%m%d_%H%M%S') + filename = filename.replace('{datetime}', datetime_str) return filename @@ -307,9 +318,42 @@ def wrapped_func(*args, **kwargs): # Optionally export Chrome trace file if _config.export_chrome_trace: - output_file = _config.get_output_filename() + import gzip + import shutil + + # Generate filename with range and counter + output_file = _config.get_output_filename( + range_start=start, + range_end=end, + counter=current_range_idx + ) + + # Export uncompressed trace first prof.export_chrome_trace(output_file) print(f"[profiler] Exported trace to: {output_file}") + + # Compress the trace file + compressed_file = output_file + ".gz" + try: + with open(output_file, 'rb') as f_in: + with gzip.open(compressed_file, 'wb', compresslevel=9) as f_out: + shutil.copyfileobj(f_in, f_out) + + # Get file sizes for reporting + import os as _os + original_size = _os.path.getsize(output_file) + compressed_size = _os.path.getsize(compressed_file) + ratio = (1 - compressed_size / original_size) * 100 + + print(f"[profiler] Compressed trace to: {compressed_file}") + print(f"[profiler] Compression: {original_size:,} -> {compressed_size:,} bytes ({ratio:.1f}% reduction)") + + # Remove uncompressed file to save space + _os.remove(output_file) + print(f"[profiler] Removed uncompressed file: {output_file}") + except Exception as e: + print(f"[profiler] Warning: Failed to compress trace file: {e}") + print(f"[profiler] Uncompressed trace available at: {output_file}") else: print(f"[profiler] Chrome trace export disabled (export_chrome_trace=false)")