diff --git a/.dockerignore b/.dockerignore index 3196d2dd..5d0a434f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,67 @@ +# Environment files .env .env.* .venv/ +venv/ +env/ + +# Python cache +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python + +# Development tools +.git/ +.gitignore +.github/ +.aider*/ +.cursor/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Documentation +docs/ +site/ +*.md +!README.md + +# Logs +logs/ +*.log + +# Database +*.db +*.sqlite3 + +# Testing +.pytest_cache/ +.coverage +htmlcov/ + +# Build artifacts +build/ +dist/ +*.egg-info/ + +# Docker +docker-compose.yml +Dockerfile +.dockerignore + +# Examples and development scripts +examples/ +scripts/ + +# Static files (will be collected in container) +staticfiles/ + +# Misc +.DS_Store +*.pyc diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml new file mode 100644 index 00000000..eb0a8a2f --- /dev/null +++ b/.github/workflows/deploy-docs.yml @@ -0,0 +1,58 @@ +name: Deploy Documentation + +on: + push: + branches: + - main + - container_setup + paths: + - 'docs/**' + - 'mkdocs.yml' + - '.github/workflows/deploy-docs.yml' + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install dependencies + run: | + pip install mkdocs-material + pip install mkdocs-minify-plugin + + - name: Build MkDocs site + run: mkdocs build + + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: ./site + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index a9030879..4152a59d 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,6 @@ db_backup.sqlite3 # vllm vllm/ + +# mkdocs +site/ diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 8798d340..00000000 --- a/Dockerfile +++ /dev/null @@ -1,49 +0,0 @@ -# Use Python 3.11 as base image -FROM python:3.11-slim - -# Set environment variables -ENV PYTHONUNBUFFERED=1 -ENV PYTHONDONTWRITEBYTECODE=1 -ENV POETRY_VERSION=1.7.1 - -# Set work directory -WORKDIR /app - -# Install system dependencies -RUN apt-get update && apt-get install -y \ - build-essential \ - libpq-dev \ - && rm -rf /var/lib/apt/lists/* - -# Install Poetry -RUN pip install --no-cache-dir poetry==${POETRY_VERSION} - -# Install uvicorn and gunicorn -RUN pip install --no-cache-dir uvicorn gunicorn - -# Copy project files -COPY . . - -# Install dependencies -RUN poetry config virtualenvs.create false \ - && poetry install --only main --no-interaction --no-ansi - -# Set DJANGO_SETTINGS_MODULE env var specifically for this RUN command -# Collect static files -RUN export DJANGO_SETTINGS_MODULE=inference_gateway.settings - -# Create necessary directories -RUN mkdir -p /var/log/inference-service - -# Run the application with standard Uvicorn worker and explicit parameters (Workaround) -CMD ["gunicorn", \ - "inference_gateway.asgi:application", \ - "-k", "uvicorn.workers.UvicornWorker", \ - "-b", "0.0.0.0:7000", \ - "--workers", "5", \ - "--threads", "4", \ - "--timeout", "1800", \ - "--log-level", "debug", \ - "--access-logfile", "/var/log/inference-service/backend_gateway.access.log", \ - "--error-logfile", "/var/log/inference-service/backend_gateway.error.log", \ - "--capture-output"] diff --git a/README.md b/README.md index c6caa3a7..7c5f65df 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,10 @@ [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) ![Build](https://github.com/auroraGPT-ANL/inference-gateway/workflows/Django/badge.svg) -# Inference Gateway for FIRST toolkit +# FIRST FIRST (Federated Inference Resource Scheduling Toolkit) is a system that enables LLM (Large Language Model) inference as a service, allowing secure, remote execution of LLMs through an [OpenAI](https://platform.openai.com/docs/overview)-compatible API. FIRST's Inference Gateway is a RESTful API that validates and authorizes inference requests to scientific computing clusters using [Globus Auth](https://www.globus.org/globus-auth-service) and [Globus Compute](https://www.globus.org/compute). -## Table of Contents - -- [System Architecture](#system-architecture) -- [Prerequisites](#prerequisites) -- [Setup Overview](#setup-overview) -- [Gateway Setup](#gateway-setup) - - [Installation (Docker or Bare Metal)](#installation-docker-or-bare-metal) - - [Register Globus Application](#register-globus-application) - - [Configure Environment (.env)](#configure-environment-env) - - [Initialize Gateway Database](#initialize-gateway-database) -- [Inference Backend Setup (Remote/Local)](#inference-backend-setup-remotelocal) - - [Create Virtual Environment](#virtual-python-environment) - - [Install Inference Server (e.g., vLLM) and Globus Compute](#install-inference-server-eg-vllm-and-globus-compute) - - [Register Globus Compute Functions](#register-globus-compute-functions) - - [Configure and Start a Globus Compute Endpoint](#configure-and-start-a-globus-compute-endpoint) -- [Connecting Gateway and Backend](#connecting-gateway-and-backend) - - [Update Fixtures](#update-fixtures) - - [Load Fixtures](#load-fixtures) -- [Starting the Services](#starting-the-services) - - [Gateway (Docker or Bare Metal)](#gateway-docker-or-bare-metal) - - [Inference Backend (Globus Compute Endpoint)](#inference-backend-globus-compute-endpoint) -- [Verifying the Setup](#verifying-the-setup) -- [Benchmarking](#benchmarking) -- [Production Considerations (Nginx)](#production-considerations-nginx) -- [Monitoring](#monitoring) -- [Troubleshooting](#troubleshooting) - ## System Architecture ![System Architecture](./first_architecture.png) @@ -42,734 +15,48 @@ The Inference Gateway consists of several components: - **Globus Compute Endpoints**: Remote execution framework on HPC clusters (or local machines). - **Inference Server Backend**: (e.g., [vLLM](https://docs.vllm.ai/en/latest/)) High-performance inference service for LLMs running alongside the Globus Compute Endpoint. -## Prerequisites - -- [Python](https://www.python.org/) 3.11+ -- [PostgreSQL](https://www.postgresql.org/docs/) Server (included in the Docker deployment) -- [Poetry](https://python-poetry.org/docs/#installation) -- [Docker](https://docs.docker.com/) and [Docker Compose](https://docs.docker.com/compose/) (Recommended for Gateway deployment) -- [Globus Account](https://www.globus.org/) -- Access to a compute resource (HPC cluster or a local machine with sufficient resources for the chosen inference server and models) - -## Setup Overview - -The setup involves two main parts: -1. **Gateway Setup**: Installing and configuring the central API gateway service. -2. **Inference Backend Setup**: Setting up the inference server (like vLLM) and the Globus Compute components on the machine(s) where models will run. - -These parts can be done in parallel, but configuration details from each are needed to link them. - -## Gateway Setup - -This section covers setting up the central Django application. - -### Installation (Docker or Bare Metal) - -Clone the repository first: -```bash -git clone https://github.com/auroraGPT-ANL/inference-gateway.git -cd inference-gateway -``` - -**Option 1: Docker Deployment (Recommended)** - -```bash -# Create necessary directories needed by docker-compose.yml -mkdir -p logs prometheus -# Create a basic prometheus config if you don't have one -# echo "global:\n scrape_interval: 15s\nscrape_configs:\n - job_name: 'prometheus'\n static_configs:\n - targets: ['localhost:9090']" > prometheus/prometheus.yml - -# Configuration is done via the .env file (see next steps) -``` -*See [Starting the Services](#starting-the-services) for how to run this after configuration.* -*See `docker-compose.yml` for details on included services (Postgres, Redis, optional monitoring).* - -**Option 2: Bare Metal Setup / Local Development (need a PostgreSQL server)** - -```bash -# Set up Python environment with Poetry -poetry config virtualenvs.in-project true -poetry env use python3.11 -poetry install - -# Activate the environment -poetry shell -# Can also use 'source .venv/bin/activate' - -# Ensure PostgreSQL server is running and accessible. -# Configuration is done via the .env file (see next steps) -``` - -### Register Two Globus Applications - -**Service API Application** - -To handle authorization within the API, the Gateway needs to be registered as a Globus Service API application: - -1. Visit [developers.globus.org](https://app.globus.org/settings/developers) and sign in. -2. Under **Register an...**, click on **Register a service API ...**. -3. Select **none of the above - create a new project** or select one of your existing projects. -4. Complete the new project form (not needed if you selected an existing project). -5. Complete the registration form: - * Set **App Name** (e.g., "My Inference Gateway"). - * Add **Redirect URIs**. For local development with the default Django server (`runserver`), use `http://localhost:8000/complete/globus/`. For production, use `https:///complete/globus/`. - * You can leave the check boxes to their default setting. - * Set **Privacy Policy** and **Terms & Conditions** URLs if applicable. -6. After registration, a **Client UUID** will be assigned to your Globus application. Generate a **Client Secret** by clicking on the **Add Client Secret** button on the right-hand side. **You will need both for the `.env` configuration.** The UUID will be for `GLOBUS_APPLICATION_ID`, and the secret will be for `GLOBUS_APPLICATION_SECRET`. - -**Add a Globus Scope to your Service API Application** - -A scope is needed for users to generate an access token to access the inference service. First export the API client credentials in a terminal: -```bash -export CLIENT_ID="" -export CLIENT_SECRET="" -``` - -Make a request to Globus Auth to attach an `action_all` scope to your API client. The `curl` command below will also embed a dependent Globus Group scope to the main scope, which is needed for the API to query the user's Group memberships during the authorization process. -```bash -curl -X POST -s --user $CLIENT_ID:$CLIENT_SECRET \ - https://auth.globus.org/v2/api/clients/$CLIENT_ID/scopes \ - -H "Content-Type: application/json" \ - -d '{ - "scope": { - "name": "Action Provider - all", - "description": "Access to inference service.", - "scope_suffix": "action_all", - "dependent_scopes": [ - { - "scope": "73320ffe-4cb4-4b25-a0a3-83d53d59ce4f", - "optional": false, - "requires_refresh_token": true - } - ] - } - }' -``` - -To verify that the scope was successfully created, query the details of your API client, and look for the UUID in the `scopes` field: -```bash -curl -s --user $CLIENT_ID:$CLIENT_SECRET https://auth.globus.org/v2/api/clients/$CLIENT_ID -``` - -Query the details of your newly created scope (you shoud see `73320ffe-4cb4-4b25-a0a3-83d53d59ce4f` in the `dependent_scopes` field): -```bash -export SCOPE_ID="" -curl -s --user $CLIENT_ID:$CLIENT_SECRET https://auth.globus.org/v2/api/clients/$CLIENT_ID/scopes/$SCOPE_ID -``` - -**Service Account Application** - -To handle the communication between the Gateway API and the compute resources (the Inference Backend), you need to create a Globus **Service Account application**. This application represents the Globus identity that will own the Globus Compute endpoints. - -1. Visit [developers.globus.org](https://app.globus.org/settings/developers) and sign in. -2. Under **Projects**, click on project used to register your Service API application from the previous step. -3. Click on **Add an App**. -4. Select **Register a service account ...**. -5. Complete the registration form: - * Set **App Name** (e.g., "My Inference Endpoints"). - * Set **Privacy Policy** and **Terms & Conditions** URLs if applicable. -6. After registration, a **Client UUID** will be assigned to your Globus application. Generate a **Client Secret** by clicking on the **Add Client Secret** button on the right-hand side. **You will need both for the `.env` configuration.** The UUID will be for `POLARIS_ENDPOINT_ID`, and the secret will be for `POLARIS_ENDPOINT_SECRET`. - -### Configure Environment (.env) - -Create a `.env` file in the project root (`inference-gateway/`). This file is used by both Docker and bare-metal setups (if using `python-dotenv`). - -```dotenv -# --- Core Django Settings --- -SECRET_KEY="" # Can be generate with Django, e.g. python -c 'from django.core.management.utils import get_random_secret_key; print(get_random_secret_key())' -DEBUG=True # Set to False for production -ALLOWED_HOSTS="localhost,127.0.0.1" # Add your gateway domain/IP for production - -# --- Globus Credentials (from the "Register Globus Application" step) --- -# Client ID and Secret of the Globus Service API application -GLOBUS_APPLICATION_ID="" -GLOBUS_APPLICATION_SECRET="" -# Optional: Restrict access to specific Globus Groups (space-separated UUIDs) -# GLOBUS_GROUPS=" " -# Optional: Enforce specific Identity Provider usage (JSON string) -# AUTHORIZED_IDPS='{"Your Institution Domain": "your-institution-uuid"}' -# AUTHORIZED_GROUPS_PER_IDP='{"Institution Domain": "coma-separated list of group uuids"}' -# Optional: Enforce Globus high assurance policies (space-separated UUIDs) -# GLOBUS_POLICIES="" -# Client ID and Secret of the Globus Service Account application -POLARIS_ENDPOINT_ID="" -POLARIS_ENDPOINT_SECRET="" - -# --- CLI Authentication Helper --- -# Public Client ID used by the inference-auth-token.py script for user authentication -CLI_AUTH_CLIENT_ID="58fdd3bc-e1c3-4ce5-80ea-8d6b87cfb944" # Default public client, replace if needed -# Optional: Comma-separated list of allowed domains for CLI login (e.g., "anl.gov,alcf.anl.gov") -# CLI_ALLOWED_DOMAINS="anl.gov,alcf.anl.gov" -# Optional: Override token storage directory for CLI script -# CLI_TOKEN_DIR="~/.globus/my_custom_token_dir" -# Optional: Override App Name used by CLI script -# CLI_APP_NAME="my_custom_cli_app" - - -# --- Database Credentials --- -# Used by Django Gateway, Postgres container, postgres-exporter -POSTGRES_DB="inferencegateway" -POSTGRES_USER="inferencedev" -POSTGRES_PASSWORD="inferencedevpwd" # CHANGE THIS for production -# Hostname: Use "postgres" for Docker-compose networking. -# Use "localhost" for bare-metal if DB is local. -# Use "host.docker.internal" if Gateway runs in Docker but DB runs on the host machine. -PGHOST="postgres" # Important: Use "postgres" for Docker -PGPORT=5432 -PGUSER="dataportaldev" -PGPASSWORD="inferencedevpwd" # CHANGE THIS for production -PGDATABASE="inferencegateway" - -# --- Redis --- Used for caching, async tasks -# Use "redis" for Docker-compose networking. -# Use "localhost" (or relevant hostname) for bare-metal. -REDIS_URL="redis://redis:6379/0" - -# --- Gateway Specific Settings --- -MAX_BATCHES_PER_USER=2 # Max concurrent batch jobs allowed per user - -# --- Streaming Configuration --- -# Internal streaming server configuration (required for streaming functionality) -STREAMING_SERVER_HOST="localhost:8080" # Host and port of your internal streaming server -INTERNAL_STREAMING_SECRET="your-internal-streaming-secret-key" # Secret key for internal streaming authentication -# Both /chat/completions and /completions endpoints support streaming via the 'stream' parameter -# However, 'stream_options' parameter is only supported in /completions endpoint - -# --- Optional: Grafana Admin Credentials (for Docker setup) --- -# GF_SECURITY_ADMIN_USER=admin -# GF_SECURITY_ADMIN_PASSWORD=admin - -# QSTAT ENDPOINTS -# SOPHIA_QSTAT_ENDPOINT_UUID="" -# SOPHIA_QSTAT_FUNCTION_UUID="" -``` - -**Important**: Securely store all of your credentials and secrets, especially in production. The `CLI_AUTH_CLIENT_ID` is typically a public client and doesn't need to be kept secret. - -### Initialize Gateway Database - -Once you have configured the `.env` file, initialize the Gateway's database schame. - -**Option 1: Docker (also works with Podman)** - -First, build and run the Gateway's containers (there should be 7 in total): - -```bash -docker-compose -f docker-compose.yml up -d -``` - -Initialize the database: -```bash -docker-compose -f docker-compose.yml exec inference-gateway python manage.py makemigrations -docker-compose -f docker-compose.yml exec inference-gateway python manage.py migrate -``` - -**Option 2: Bare Metal** - -If you are not using Docker, you must have a PostgreSQL service running. -```bash -# Ensure DB connection vars are exported in your shell or use python-dotenv -# (e.g., export PGHOST=localhost POSTGRES_USER=...) -python manage.py makemigrations -python manage.py migrate -``` - -## Inference Backend Setup (Remote/Local) - -This section covers setting up the components on the machine where the AI models will actually run (e.g., an HPC compute node, a powerful workstation). - -### Virtual Python Environment - -All of the instruction below must be done within a Python virtual environment. Make sure to use a **virtual environment with the same Python version as the one used to deploy Gateway API** (Python 3.11 in this example). This will avoid version mismatch errors when using Globus Compute. - -```bash -# Example 1 with conda -conda create -n vllm-env python=3.11 -y -conda activate vllm-env - -# Example 2 with python venv -python3.11 -m venv vllm-env -source vllm-env/bin/activate # On Windows use `vllm-env\Scripts\activate` -``` - -### Install Inference Server (e.g., vLLM) and Globus Compute - -Choose and install an inference serving framework. vLLM is recommended for performance with many transformer models: - -```bash -# Make sure you have activated your virtual environment - -# Basic vLLM installation from source -git clone https://github.com/vllm-project/vllm.git -cd vllm -pip install -e . -# For specific hardware acceleration (CUDA, ROCm), follow official docs: -# https://docs.vllm.ai/en/latest/getting_started/installation.html -``` - -Install the Globus Compute Endpoint software and the Globus Compute SDK: -```bash -pip install globus-compute-sdk globus-compute-endpoint -``` - -### Register Globus Compute Functions - -The Gateway interacts with the inference server via functions registered with Globus Compute. You need to register: - -1. **Inference Function**: Wraps the call to your inference server (e.g., vLLM OpenAI-compatible endpoint). -2. **Status Function (Optional but Recommended)**: Queries the cluster scheduler (e.g., PBS `qstat`) and node status to aid federated routing. - -**Important:** When registering Globus Compute functions and endpoints, you need to explicitly tie the function/endpoint identity back to your Globus **Service Account** application (not the Service API application). Do this by exporting the client ID (value of `POLARIS_ENDPOINT_ID` in `.env`) and Secret (value of `POLARIS_ENDPOINT_SECRET` in `.env`) as environment variables, **before** running the registration script or configuring the endpoint: -```bash -export GLOBUS_COMPUTE_CLIENT_ID="" -export GLOBUS_COMPUTE_CLIENT_SECRET="" -``` - -Now, register the necessary functions: - -```bash -# Ensure you are in the correct Python environment - -# Navigate to the `compute-functions` directory in your local clone of the repository. -cd path/to/inference-gateway/compute-functions - -# Register the vLLM inference function (modify the script if needed) -# See compute-functions/vllm_register_function.py -python vllm_register_function.py -# Note the output Function UUID (e.g., ) -# Output example: -# Function registered with UUID - .... -# The UUID is stored in vllm_register_function_sophia_multiple_models.txt. - -# Register the qstat/status function (modify script for your scheduler if needed) -python qstat_register_function.py -# Note the output Function UUID (e.g., ) - -# (Register other functions like batch processing if needed) -``` - -**Important**: Keep track of the Function UUIDs generated. - -### Configure a Globus Compute Endpoint - -This endpoint runs on the backend machine, listens for tasks from Globus Compute, and executes the registered functions. - -```bash -# Ensure you are in the correct Python environment - -# Configure a new endpoint (follow prompts) -# Here we use "my-compute-endpoint" for the name but you can choose another name (e.g. local-vllm) -globus-compute-endpoint configure my-compute-endpoint - -# This creates a configuration directory, e.g., ~/.globus_compute/my-compute-endpoint/ -# Edit the config.yaml inside that directory. -``` - -### Note on HPC Configuration Examples: -See [`local-vllm-endpoint.yaml`](./compute-endpoints/local-vllm-endpoint.yaml) for a configuration example that submits tasks on local hardware. For an example of submitting inference tasks through a cluster scheduler like PBS (specifically on ALCF's Sophia cluster), see [`sophia-vllm-config-template-v2.0.yaml`](./compute-endpoints/sophia-villm-config-template-v2.0.yaml). Adapting cluster examples requires understanding the specific HPC environment (scheduler, modules, file paths, etc.). -Refer to [Globus Compute Endpoint Docs](https://globus-compute.readthedocs.io/en/latest/endpoints.html) for all options. - -If you adopted the above configuration examples, make sure to edit the following: - -* `allowed_functions`: Make sure the function UUIDs (one per line) are the ones you registered in the [Register Globus Compute Functions](#register-globus-compute-functions) section. -* `worker_init`: *Note:* For local setups, you'll typically activate your environment directly (e.g., `source vllm-env/bin/activate`). For cluster setups like the Sophia example, you might use a shared setup script (e.g., `source inference-gateway/compute-endpoints/common_setup.sh`). Ensure you point to the correct setup script or activation command for your environment. - -**After configuring `config.yaml`:** - -```bash -# Start the endpoint -globus-compute-endpoint start my-compute-endpoint - -# Note the Endpoint UUID displayed after starting. -# You can always recover the UUID by typing `globus-compute-endpoint list` -``` - -**Keep track of the Endpoint UUID.** - -## Connecting Gateway and Backend - -### Update Fixtures - -Edit the relevant fixtures file in the Gateway project directory (`inference-gateway/fixtures/`). - -For federated access, edit `federated_endpoints.json`. - -**Example: `fixtures/federated_endpoints.json`** - -```json -[ - { - "model": "resource_server.federatedendpoint", - "pk": 1, - "fields": { - "name": "OPT 125M (Federated)", - "slug": "federated-opt-125m", - "target_model_name": "facebook/opt-125m", // Model users request - "description": "Federated access point for the facebook/opt-125m model.", - "targets": [ - { - "cluster": "local", - "framework": "vllm", - "model": "facebook/opt-125m", // Model served by this specific target - "endpoint_slug": "local-vllm-facebook-opt-125m", // Unique ID for this target - "endpoint_uuid": "", - "function_uuid": "", - "api_port": 8001 // Port your local vLLM server runs on - } - // Add more targets here for the same model on different clusters/frameworks - // Example: A target on an HPC cluster like Sophia - // { - // "cluster": "sophia", - // "framework": "vllm", - // "model": "facebook/opt-125m", // Assuming OPT-125m is also deployed there - // "endpoint_slug": "sophia-vllm-facebook-opt-125m", - // "endpoint_uuid": "", - // "function_uuid": "", - // "api_port": 8000 // Port on Sophia's compute node - // } - ] - } - } - // Add more FederatedEndpoint entries for other models -] -``` - -For standard, non-federated access, edit `endpoints.json`. - -**Example: `fixtures/endpoints.json`** - -```json -[ - { - "model": "resource_server.endpoint", - "pk": 1, // Or next available primary key - "fields": { - "endpoint_slug": "local-vllm-facebook-opt-125m", // Example for local OPT-125m - "cluster": "local", // Or wherever your compute endpoint is hosted - "framework": "vllm", - "model": "facebook/opt-125m", // Model served by this endpoint - "api_port": 8001, // Example port for local vLLM server - "endpoint_uuid": "", - "function_uuid": "", - "batch_endpoint_uuid": "", - "batch_function_uuid": "", - "allowed_globus_groups": "" // Optional: Restrict this target further - } - } - // Add more Endpoint entries for other models/clusters -] -``` - -Replace placeholders (`<...>`) with the UUIDs and details from the previous steps. - -### Load Fixtures - -Load the updated fixture file into the Gateway database. - -**Option 1: Docker:** - -Make sure the fixture `json` files are updated within the running container. You can do this by editing the files directly within the container: -```bash -docker-compose -f docker-compose.yml exec inference-gateway /bin/bash -# The edit the /app/fixtures/endpoints.json (or federated_endpoints.json) file -``` - -or by transfering the edited local file directly into the container: -```bash -docker cp fixtures/endpoints.json inference-gateway_inference-gateway_1:/app/fixtures/ -# Or: docker cp fixtures/federated_endpoints.json inference-gateway_inference-gateway_1:/app/fixtures/ -``` - -Load the fixtures into the database: -```bash -docker-compose -f docker-compose.yml exec inference-gateway python manage.py loaddata fixtures/endpoints.json -# Or: docker-compose ... exec inference-gateway python manage.py loaddata fixtures/federated_endpoints.json -``` - -**Option 2: Bare Metal:** -```bash -python manage.py loaddata fixtures/endpoints.json -# Or: python manage.py loaddata fixtures/federated_endpoints.json -``` - -## Starting the Services - -### Gateway (Docker or Bare Metal) - -**Docker:** -```bash -# Start all services defined in docker-compose.yml (if not already running) -docker-compose -f docker-compose.yml up --build -d -``` - -**Bare Metal (Development):** -```bash -# Ensure DB connection vars are set -poetry shell -python manage.py runserver 0.0.0.0:8000 # Or your preferred port -``` - -**Bare Metal (Production with Gunicorn):** -```bash -# Ensure environment variables are set -poetry shell -poetry run gunicorn \ - inference_gateway.asgi:application \ - -k uvicorn.workers.UvicornWorker \ - -b 0.0.0.0:8000 \ - --workers 5 \ - --log-level info - # Add other Gunicorn flags as needed (--threads, --timeout, log files etc.) -``` - -### Inference Backend (Globus Compute Endpoint) - -Ensure the Globus Compute endpoint is running on the backend machine: - -```bash -# On the HPC login node / backend machine -globus-compute-endpoint start -``` - -Verify the associated inference server (e.g., vLLM) is started by the endpoint's worker initialization or job submission process. - -## Verifying the Setup - -Once both the Gateway and at least one Backend Compute Endpoint (with its inference server) are running, you can send a test request. You'll need a valid Globus authentication token obtained for the gateway's scope. - -### Setting up ngrok for Testing Streaming (Optional) - -For testing streaming functionality especially when working with remote endpoints, you can use ngrok to create a secure tunnel to your local gateway: - -1. **Install ngrok**: Visit [ngrok.com](https://ngrok.com/) and follow the installation instructions for your platform. - -2. **Start ngrok tunnel**: Start ngrok to create a tunnel to your local gateway (assuming it's running on port 8000): - ```bash - ngrok http 8000 - ``` - This will display a public URL (e.g., `https://abcd1234.ngrok.io`) that tunnels to your local gateway. - -3. **Test streaming with Python script**: Here's an example Python script for testing streaming functionality: - - ```python - import sys - import json - import time - import openai - - if len(sys.argv) != 2: - print("Usage: python test_streaming.py ") - sys.exit(1) - - access_token = sys.argv[1] - - # Configure the OpenAI client to connect to your gateway via ngrok - client = openai.OpenAI( - # Replace with your ngrok URL or direct gateway URL - base_url="https://your-ngrok-url.ngrok.io/resource_server/sophia/vllm/v1", - # Use your Globus access token - api_key=access_token - ) - - # Define your prompt - messages = [ - {"role": "user", "content": "Explain what is the best way to learn python"} - ] - - start_time = time.time() - total_chars = 0 - - print("πŸ€– AI Response: ", end="") - - # Call the chat completions endpoint with stream=True - try: - stream = client.chat.completions.create( - model="openai/gpt-oss-20b", # Replace with your deployed model - messages=messages, - stream=True - ) - - # Iterate over the stream to get the response chunks in real-time - for chunk in stream: - # Extract the content from the chunk (standard OpenAI format) - if chunk.choices[0].delta.content is not None: - content = chunk.choices[0].delta.content - # Print the content without a newline and flush the buffer - print(content, end="", flush=True) - total_chars += len(content) - - # Print a final newline character after the stream is complete - print() - - except Exception as e: - print(f"\n❌ Error: {e}") - - end_time = time.time() - latency = end_time - start_time - print(f"\n⏱️ Streaming Latency: {latency:.2f} seconds") - print(f"πŸ“¦ Throughput: {total_chars/latency:.2f} chars/sec") - ``` - - Save this as `test_streaming.py` and run it with your access token: - - ```bash - python test_streaming.py $MY_TOKEN - ``` - -4. **Get a Token using the Helper Script**: - * Ensure your `.env` file has `GLOBUS_APPLICATION_ID` (for the gateway) and `CLI_AUTH_CLIENT_ID` (for the helper script, a default public one is provided) set. You also need to configure `CLI_ALLOWED_DOMAINS` to target your specific identity provider (e.g. your institution's SSO). By default, without specifying `CLI_ALLOWED_DOMAINS`, the only allowed providers are `anl.gov` and `alcf.anl.gov`. - * **Authenticate (First time or if tokens expire):** Run the authentication script. This will open a browser window for Globus login. - ```bash - python inference-auth-token.py authenticate - ``` - This stores refresh and access tokens locally (typically in `~/.globus/app/...`). - * **Force Re-authentication:** If you need to change Globus accounts or encounter permission errors possibly related to expired sessions or policy changes, log out via [https://app.globus.org/logout](https://app.globus.org/logout) and force re-authentication: - ```bash - python inference-auth-token.py authenticate --force - ``` - * **Get Access Token:** Retrieve your current, valid access token: - ```bash - export MY_TOKEN=$(python inference-auth-token.py get_access_token) - echo "Token stored in MY_TOKEN environment variable." - # echo $MY_TOKEN # Uncomment to view the token directly - ``` - This command automatically uses the stored refresh token to get a new access token if the current one is expired. - - * **Token Validity:** Access tokens are valid for 48 hours. Refresh tokens allow getting new access tokens without re-logging in via browser, but they expire after 6 months of inactivity. Some institutions or policies might enforce re-authentication more frequently (e.g., weekly). - - -2. **Send Request using cURL**: You can adjust the model name and payload as appropriate. - - Example with a federated Globus Compute endpoint (assuming `federated_endpoints.json` was used): - - ```bash - # Example using the federated endpoint for OPT-125m - curl -X POST http://127.0.0.1:8000/resource_server/v1/chat/completions \\ - -H "Authorization: Bearer $MY_TOKEN" \\ - -H "Content-Type: application/json" \\ - -d '{ - "model": "facebook/opt-125m", - "messages": [ - {"role": "user", "content": "Explain the concept of Globus Compute in simple terms."} - ], - "max_tokens": 150 - }' - ``` - - Example with a standard, non-federated Globus Compute endpoint (assuming `endpoints.json` was used): - - ```bash - # Example targeting a specific vLLM endpoint on the 'local' cluster for OPT-125m - curl -X POST http://127.0.0.1:8000/resource_server/local/vllm/v1/chat/completions \\ - -H "Authorization: Bearer $MY_TOKEN" \\ - -H "Content-Type: application/json" \\ - -d '{ - "model": "facebook/opt-125m", - "messages": [ - {"role": "user", "content": "Explain the concept of Globus Compute in simple terms."} - ], - "max_tokens": 150 - }' - ``` - -A successful response will be a JSON object containing the model's completion. - -## Benchmarking - -A benchmark script is provided in `examples/load-testing/benchmark-serving.py` (adapted from vLLM's benchmarks) to test the performance of your deployed endpoints. - -1. **Download Dataset**: The script often uses the ShareGPT dataset. You can download it manually or let the script attempt to download it. A common location to place it might be `examples/load-testing/ShareGPT_V3_unfiltered_cleaned_split.json`. - ```bash - # Example: Download ShareGPT dataset (check the script for the exact URL if needed) - wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json -P examples/load-testing/ - ``` - -2. **Run Benchmark**: Execute the script, pointing it to your gateway's API endpoint. - ```bash - # Example: Benchmark local federated endpoint for opt-125m - python examples/load-testing/benchmark-serving.py \ - --backend vllm \ - --model facebook/opt-125m \ - --base-url http://127.0.0.1:8000/resource_server/v1/chat/completions \ - --dataset-name sharegpt \ - --dataset-path examples/load-testing/ShareGPT_V3_unfiltered_cleaned_split.json \ - --output-file benchmark_results_local_opt125m.jsonl \ - --num-prompts 100 # Adjust as needed - # Add --request-rate or --max-concurrency for more controlled tests - # Add --disable-ssl-verification if using self-signed certs locally - # Add --disable-stream if benchmarking a non-streaming endpoint - ``` - - You can adapt the `--base-url` to point to other OpenAI-compatible endpoints (like the OpenAI API itself, Anthropic, etc., potentially requiring environment variables like `OPENAI_API_KEY`). See the script's arguments (`--help`) for more options. - -## Production Considerations (Nginx) - -For production deployments (especially bare-metal), running Django/Gunicorn behind a reverse proxy like Nginx is highly recommended for: - -* **HTTPS/SSL Termination**: Securely handle TLS encryption. -* **Load Balancing**: Distribute requests across multiple Gunicorn workers/instances. -* **Serving Static Files**: Efficiently serve CSS, JS, images. -* **Security**: Add rate limiting, header checks, etc. -* **Hostname Routing**: Direct traffic based on domain names. - -**Example Nginx Configuration Snippet (`/etc/nginx/sites-available/inference_gateway`):** - -```nginx -upstream app_server { - # fail_timeout=0 means we always retry an upstream even if it failed - # to return a good HTTP response (in case the Gunicorn worker recovers). - server 127.0.0.1:8000 fail_timeout=0; - # Add more servers here if running multiple Gunicorn instances -} - -server { - listen 80; - # listen 443 ssl; - server_name your-gateway-domain.org; - - # ssl_certificate /path/to/your/cert.pem; - # ssl_certificate_key /path/to/your/key.pem; - - client_max_body_size 4G; - - location /static/ { - alias /path/to/inference-gateway/static/; - } - - location / { - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Host $http_host; - proxy_redirect off; - proxy_pass http://app_server; - } +## Documentation + +πŸ“š **Complete documentation is available at: [https://auroragpt-anl.github.io/inference-gateway/](https://auroragpt-anl.github.io/inference-gateway/)** + +### Quick Links + +- **[Administrator Guide](https://auroragpt-anl.github.io/inference-gateway/admin-guide/)** - Setup and deployment instructions + - [Docker Deployment](https://auroragpt-anl.github.io/inference-gateway/admin-guide/gateway-setup/docker/) + - [Bare Metal Setup](https://auroragpt-anl.github.io/inference-gateway/admin-guide/gateway-setup/bare-metal/) + - [Inference Backend Setup](https://auroragpt-anl.github.io/inference-gateway/admin-guide/inference-setup/) + +- **[User Guide](https://auroragpt-anl.github.io/inference-gateway/user-guide/)** - How to use the inference API + +- **[Example Deployment](https://github.com/argonne-lcf/inference-endpoints)** - ALCF production deployment + +## Citation + +If you use ALCF Inference Endpoints or the Federated Inference Resource Scheduling Toolkit (FIRST) in your research or workflows, please cite our paper: + +```bibtex +@inproceedings{10.1145/3731599.3767346, + author = {Tanikanti, Aditya and C\^{o}t\'{e}, Benoit and Guo, Yanfei and Chen, Le and Saint, Nickolaus and Chard, Ryan and Raffenetti, Ken and Thakur, Rajeev and Uram, Thomas and Foster, Ian and Papka, Michael E. and Vishwanath, Venkatram}, + title = {FIRST: Federated Inference Resource Scheduling Toolkit for Scientific AI Model Access}, + year = {2025}, + isbn = {9798400718717}, + publisher = {Association for Computing Machinery}, + address = {New York, NY, USA}, + url = {https://doi.org/10.1145/3731599.3767346}, + doi = {10.1145/3731599.3767346}, + abstract = {We present the Federated Inference Resource Scheduling Toolkit (FIRST), a framework enabling Inference-as-a-Service across distributed High-Performance Computing (HPC) clusters. FIRST provides cloud-like access to diverse AI models, like Large Language Models (LLMs), on existing HPC infrastructure. Leveraging Globus Auth and Globus Compute, the system allows researchers to run parallel inference workloads via an OpenAI-compliant API on private, secure environments. This cluster-agnostic API allows requests to be distributed across federated clusters, targeting numerous hosted models. FIRST supports multiple inference backends (e.g., vLLM), auto-scales resources, maintains "hot" nodes for low-latency execution, and offers both high-throughput batch and interactive modes. The framework addresses the growing demand for private, secure, and scalable AI inference in scientific workflows, allowing researchers to generate billions of tokens daily on-premises without relying on commercial cloud infrastructure.}, + booktitle = {Proceedings of the SC '25 Workshops of the International Conference for High Performance Computing, Networking, Storage and Analysis}, + pages = {52–60}, + numpages = {9}, + keywords = {Inference as a Service, High Performance Computing, Job Schedulers, Large Language Models, Globus, Scientific Computing}, + series = {SC Workshops '25} } ``` -*Remember to collect static files (`python manage.py collectstatic`) and configure Nginx to serve them from the specified `alias` path.* -*Consult Nginx documentation for details on SSL setup and other options.* - -## Monitoring - -Access the monitoring dashboard (if deployed via Docker Compose with monitoring enabled): - -- **Grafana**: http://localhost:3000 (default credentials: admin/admin) - Visualizes metrics. -- **Prometheus**: http://localhost:9090 - Collects metrics. - -The Grafana dashboard includes: -- Application metrics (request rates, latency, error rates) -- System metrics (CPU, memory, disk I/O via Node Exporter) -- Database metrics (connection counts, query performance via PostgreSQL Exporter) -- Custom Gateway metrics (inference rates, token counts - requires metrics endpoint in Gateway) - -## Example Usage Documentation +## Acknowledgements -For an example of how user-facing documentation might look for a deployed instance of this gateway, see the [ALCF Inference Endpoints documentation](https://github.com/argonne-lcf/inference-endpoints). +This work was supported by the U.S. Department of Energy, Office of Science, Office of Advanced Scientific Computing Research, under Contract No. DE-AC02-06CH11357. This research used resources of the Argonne Leadership Computing Facility, which is a DOE Office of Science User Facility. -## Troubleshooting +## License -* **Docker Nginx 404/502/504 Errors:** Verify `nginx_app.conf` mount and upstream definition, check `inference-gateway` logs. -* **Database Connection Errors:** Check `.env` variables (`PGHOST`, etc.) match context (Docker vs. Host vs. Bare-metal) and firewall/`pg_hba.conf` rules. -* **Globus Auth Errors**: Ensure Redirect URIs match in Globus Developer portal and `.env` credentials are correct. -* **Compute Endpoint Issues**: Check endpoint logs (`~/.globus_compute//endpoint.log`) for function execution errors, environment problems, or connection issues. -* **500 Server Errors on Gateway**: Check gateway logs (`docker-compose logs inference-gateway` or Gunicorn log files) for Python tracebacks. +This project is licensed under the Apache License 2.0 - see the [LICENSE](LICENSE) file for details. diff --git a/compute-endpoints/README.md b/compute-endpoints/README.md new file mode 100644 index 00000000..1d5ef6b5 --- /dev/null +++ b/compute-endpoints/README.md @@ -0,0 +1,334 @@ +# Globus Compute Endpoint Configurations + +This directory contains production-ready endpoint configurations and helper scripts for deploying vLLM inference servers with Globus Compute on HPC clusters. + +## Overview + +These examples are based on our production deployment at **Argonne Leadership Computing Facility (ALCF) Sophia cluster**. They demonstrate best practices for: + +- Multi-node model serving with Ray +- Dynamic environment management +- Production-grade logging and error handling +- Advanced vLLM configurations + +**Important**: These configurations are **cluster-specific** and must be adapted to your HPC environment. See [Adapting for Your Cluster](#adapting-for-your-cluster) below. + +## Files + +### Helper Scripts + +#### `launch_vllm_model.sh` +Modular vLLM launcher script with comprehensive features: + +- **Single and multi-node support**: Automatically detects and configures Ray for pipeline parallelism +- **Dynamic vLLM configuration**: Command-line arguments for all major vLLM parameters +- **Health monitoring**: Startup verification with timeout and retry logic +- **Flexible deployment**: Works with PBS, Slurm, or local execution + +**Usage:** +```bash +source launch_vllm_model.sh \ + --model-name meta-llama/Meta-Llama-3.1-8B-Instruct \ + --vllm-version v0.11.0 \ + --tensor-parallel 8 \ + --max-model-len 8192 \ + --enable-chunked-prefill \ + --enable-prefix-caching \ + --gpu-memory-util 0.95 +``` + +Run with `--help` for full documentation. + +#### `sophia_env_setup_with_ray.sh` +Environment setup script for ALCF Sophia cluster: + +- **Dynamic version selection**: Automatically loads correct conda environment based on `VLLM_VERSION` +- **Module management**: ALCF-specific modules (conda, gcc, spack-pe-base) +- **Ray cluster management**: Automated multi-node Ray setup using PBS nodefile +- **Comprehensive utilities**: Cleanup, monitoring, and troubleshooting functions + +**Key Functions:** +- `setup_environment()` - Initialize conda environment and exports +- `setup_ray_cluster()` - Setup multi-node Ray cluster for large models +- `start_model()` - Start vLLM with retry logic and health checks +- `cleanup_python_processes()` - Clean up zombie processes +- `stop_ray()` - Gracefully stop Ray cluster + +**Usage:** +```bash +source sophia_env_setup_with_ray.sh +setup_environment +setup_ray_cluster # For multi-node only +``` + +### Endpoint Configuration Examples + +#### Single-Node Configurations + +**`local-vllm-endpoint.yaml`** +- Basic local/workstation deployment +- Uses `LocalProvider` (no job scheduler) +- Includes inline environment setup and retry logic +- Good starting point for development + +**`sophia-vllm-singlenode-example.yaml`** +- Production single-node deployment on ALCF Sophia +- 1 node, 8 GPUs (tensor parallelism) +- Uses `launch_vllm_model.sh` for robust deployment +- PBS scheduling with optimized settings + +#### Multi-Node Configurations + +**`sophia-vllm-multinode-example.yaml`** +- Production multi-node deployment for large models (70B-405B) +- 4 nodes, 32 GPUs (TP=8, PP=4) +- Automatic Ray cluster setup via `launch_vllm_model.sh` +- PBS scheduling with multi-node allocation + +**`sophia-vllm-toolcalling-example.yaml`** +- Single-node deployment with tool calling support +- Custom chat templates for function calling +- Llama 4 models with pythonic tool parser + +#### Specialized Configurations + +**`sophia-vllm-batch-template.yaml`** +- Batch processing endpoint +- Lower idle timeout for ephemeral jobs +- Minimal blocks for cost efficiency + +**`pbs-qstat-example.yaml`** +- Job scheduler monitoring endpoint +- Runs on login node (no GPU) +- Provides cluster status information to gateway + +## Architecture + +### Production Setup Flow + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Globus Compute Endpoint Configuration (YAML) β”‚ +β”‚ β”‚ +β”‚ worker_init: | β”‚ +β”‚ source launch_vllm_model.sh \ β”‚ +β”‚ --model-name ... \ β”‚ +β”‚ --vllm-version v0.11.0 β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ launch_vllm_model.sh β”‚ +β”‚ β”‚ +β”‚ 1. Sources sophia_env_setup_with_ray.sh β”‚ +β”‚ 2. Calls setup_environment() β”‚ +β”‚ 3. Detects single vs multi-node mode β”‚ +β”‚ 4. Sets up Ray cluster if needed β”‚ +β”‚ 5. Builds vLLM command from arguments β”‚ +β”‚ 6. Calls start_model() with retry logic β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ sophia_env_setup_with_ray.sh β”‚ +β”‚ β”‚ +β”‚ - Loads modules (conda, gcc, spack) β”‚ +β”‚ - Activates correct conda environment β”‚ +β”‚ - Sets HF cache, proxy, NCCL settings β”‚ +β”‚ - Manages Ray cluster (if multi-node) β”‚ +β”‚ - Monitors model startup β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +### Multi-Node Ray Setup + +For large models requiring pipeline parallelism: + +1. `PBS_NODEFILE` parsed to identify head and worker nodes +2. Ray head started on first node +3. Ray workers started on remaining nodes +4. vLLM launched with Ray backend and appropriate TP/PP settings + +## Adapting for Your Cluster + +### 1. Environment Setup Script + +Create your own based on `sophia_env_setup_with_ray.sh`: + +**Required Changes:** +```bash +# Proxy settings (or remove if not needed) +export HTTP_PROXY="your-proxy:port" + +# Module system +module use /your/module/path +module load your-conda-module + +# Conda environments +CONDA_ENV="/your/path/to/vllm-v0.11.0-env/" + +# HuggingFace cache +export HF_HOME='/your/model/cache/' + +# Network interface (run 'ifconfig' on compute node) +export NCCL_SOCKET_IFNAME='your-interface' # e.g., 'ib0', 'eth0', 'ens0' + +# Node resources +export RAY_NUM_CPUS=64 # Adjust for your nodes +export RAY_NUM_GPUS=8 # Adjust for your nodes +``` + +### 2. Launcher Script (Optional) + +`launch_vllm_model.sh` is fairly generic. You may need to: + +- Update default environment setup script path (line 242-246) +- Adjust SSL certificate paths +- Modify default parameter values + +### 3. Endpoint YAML Configuration + +**For PBS:** +```yaml +provider: + type: PBSProProvider + account: your_project + queue: your_queue + scheduler_options: | + #PBS -l your:cluster:options +``` + +**For Slurm:** +```yaml +provider: + type: SlurmProvider + account: your_account + partition: your_partition + scheduler_options: | + #SBATCH --your-options +``` + +### 4. Cluster-Specific Checklist + +| Item | Check Method | Example | +|------|--------------|---------| +| **Network Interface** | `ifconfig` on compute node | `infinibond0`, `ib0`, `eth0` | +| **Module System** | `module avail` | Path to conda module | +| **File System** | Shared storage path | `/scratch`, `/home`, `/gpfs` | +| **Scheduler** | PBS or Slurm | Queue names, account codes | +| **GPU Allocation** | Scheduler syntax | `ngpus=8`, `--gpus-per-node=8` | +| **Proxy** | Required for internet | `http://proxy:3128` or none | +| **SSL Certs** | For HTTPS vLLM | Path or disable | + +## Quick Start + +### 1. Copy and Customize Scripts + +```bash +# Copy environment setup script +cp sophia_env_setup_with_ray.sh your_cluster_env_setup.sh + +# Edit with your cluster-specific settings +vim your_cluster_env_setup.sh +``` + +### 2. Update Launcher Script Reference + +```bash +# Edit launch_vllm_model.sh line 242 +ENV_SETUP_SCRIPT="${ENV_SETUP_SCRIPT:-/path/to/your_cluster_env_setup.sh}" +``` + +### 3. Create Endpoint Configuration + +```bash +# Start with single-node example +cp sophia-vllm-singlenode-example.yaml my-cluster-endpoint.yaml + +# Edit with your settings +vim my-cluster-endpoint.yaml +``` + +### 4. Configure and Start Endpoint + +```bash +# Configure endpoint +globus-compute-endpoint configure my-endpoint + +# Copy your YAML to the config directory +cp my-cluster-endpoint.yaml ~/.globus_compute/my-endpoint/config.yaml + +# Start endpoint +globus-compute-endpoint start my-endpoint +``` + +## Troubleshooting + +### Common Issues + +**Module not found:** +```bash +module avail # Check available modules +module use /correct/path # Add correct module path +``` + +**Conda activation fails:** +```bash +conda init bash +source ~/.bashrc +conda env list # Verify environment exists +``` + +**NCCL errors (multi-GPU):** +```bash +ifconfig # Find correct network interface +export NCCL_SOCKET_IFNAME='correct-interface' +``` + +**Ray cluster issues:** +```bash +cat $PBS_NODEFILE # Verify nodes allocated +mpiexec -n 2 hostname # Test node communication +ray status # Check Ray cluster state +``` + +**vLLM startup timeout:** +```bash +# Check logs in endpoint directory +tail -f ~/.globus_compute/my-endpoint/endpoint.log + +# Check vLLM logs +tail -f vllm.log + +# Check PBS/Slurm job logs +qstat -f # PBS +squeue -j # Slurm +``` + +## Best Practices + +1. **Test locally first**: Use `local-vllm-endpoint.yaml` as a starting point +2. **Start small**: Deploy a small model (e.g., `facebook/opt-125m`) for testing +3. **Check logs**: Always review `endpoint.log` and `vllm.log` for errors +4. **Monitor resources**: Use `qstat`/`squeue` to verify job allocation +5. **Keep nodes warm**: Set `min_blocks > 0` for production to reduce cold start latency +6. **Use versioned environments**: Create separate conda envs for each vLLM version + +## Additional Resources + +- **Documentation**: [Globus Compute Setup Guide](../docs/admin-guide/inference-setup/globus-compute.md) +- **Compute Functions**: See `../compute-functions/` for function registration scripts +- **Gateway Configuration**: See `../fixtures/` for endpoint registration examples + +## Support + +For ALCF-specific issues, contact ALCF Support. + +For FIRST Gateway issues, open an issue on [GitHub](https://github.com/auroraGPT-ANL/inference-gateway/issues). + +For Globus Compute issues, see [Globus Compute Documentation](https://globus-compute.readthedocs.io/). + +--- + +**Note**: These configurations represent a production deployment at a specific HPC facility. Your mileage may vary depending on your cluster's architecture, scheduler, and policies. Always test thoroughly before production deployment. + diff --git a/compute-endpoints/common_setup.sh b/compute-endpoints/common_setup.sh deleted file mode 100644 index 18e9c36d..00000000 --- a/compute-endpoints/common_setup.sh +++ /dev/null @@ -1,325 +0,0 @@ -#!/bin/bash -############################## -# Environment Setup Function # -############################## -setup_environment() { - echo "Setting up the environment..." - - # Set proxy configurations - export HTTP_PROXY="http://proxy.alcf.anl.gov:3128" - export HTTPS_PROXY="http://proxy.alcf.anl.gov:3128" - export http_proxy="http://proxy.alcf.anl.gov:3128" - export https_proxy="http://proxy.alcf.anl.gov:3128" - export ftp_proxy="http://proxy.alcf.anl.gov:3128" - - # Load modules and activate the conda environment - source /etc/profile.d/modules.sh - source /etc/profile # Initialize environment properly - module use /soft/modulefiles - module load conda - - # Source the Conda initialization script directly - #conda activate /eagle/argonne_tpc/inference-gateway/envs/vllmv0.8.2/ - - conda activate /eagle/argonne_tpc/inference-gateway/envs/vllmv0.8.5.post1 - # Set environment variables - export HF_DATASETS_CACHE='/eagle/argonne_tpc/model_weights/' - export HF_HOME='/eagle/argonne_tpc/model_weights/' - export HF_HUB_CACHE='/eagle/argonne_tpc/model_weights/hub' - export RAY_TMPDIR='/tmp' - export NCCL_SOCKET_IFNAME='infinibond0' - export OMP_NUM_THREADS=4 - export VLLM_IMAGE_FETCH_TIMEOUT=60 - export HF_TOKEN='' - export USE_FASTSAFETENSOR='true' - ulimit -c unlimited - - # Set path to common setup script - export COMMON_SETUP_SCRIPT='/home/openinference_svc/sophia_common_scripts.sh' - - #export PATH="/home/raffenet/software/hydra-4.2.3/bin/mpiexec:$PATH" - - echo "Environment setup complete." -} - -setup_environment_sglang() { - echo "Setting up the environment..." - - # Set proxy configurations - export HTTP_PROXY="http://proxy.alcf.anl.gov:3128" - export HTTPS_PROXY="http://proxy.alcf.anl.gov:3128" - export http_proxy="http://proxy.alcf.anl.gov:3128" - export https_proxy="http://proxy.alcf.anl.gov:3128" - export ftp_proxy="http://proxy.alcf.anl.gov:3128" - - # Load modules and activate the conda environment - source /etc/profile.d/modules.sh - source /etc/profile # Initialize environment properly - module use /soft/modulefiles - module load conda - - # Source the Conda initialization script directly - conda activate /eagle/argonne_tpc/inference-gateway/envs/sglangv0.4.4/ - - # Set environment variables - export HF_DATASETS_CACHE='/eagle/argonne_tpc/model_weights/' - export HF_HOME='/eagle/argonne_tpc/model_weights/' - export RAY_TMPDIR='/tmp' - export NCCL_SOCKET_IFNAME='infinibond0' - export OMP_NUM_THREADS=4 - export VLLM_IMAGE_FETCH_TIMEOUT=60 - export HF_TOKEN='' - - # Set path to common setup script - export COMMON_SETUP_SCRIPT='/home/openinference_svc/sophia_common_scripts.sh' - - echo "Environment setup complete." -} - -setup_environment_infinity(){ - echo "Setting up the environment..." - - # Set proxy configurations - export HTTP_PROXY="http://proxy.alcf.anl.gov:3128" - export HTTPS_PROXY="http://proxy.alcf.anl.gov:3128" - export http_proxy="http://proxy.alcf.anl.gov:3128" - export https_proxy="http://proxy.alcf.anl.gov:3128" - export ftp_proxy="http://proxy.alcf.anl.gov:3128" - - # Load modules and activate the conda environment - source /etc/profile.d/modules.sh - source /etc/profile # Initialize environment properly - module use /soft/modulefiles - module load conda - - # Source the Conda initialization script directly - conda activate /eagle/argonne_tpc/inference-gateway/infinityv0.0.71 - - # Set environment variables - export HF_DATASETS_CACHE='/eagle/argonne_tpc/model_weights/' - export HF_HOME='/eagle/argonne_tpc/model_weights/' - export NCCL_SOCKET_IFNAME='bond0' - export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True - ulimit -c unlimited - - # Set path to common setup script - export COMMON_SETUP_SCRIPT='/home/openinference_svc/sophia_common_scripts.sh' - - #export PATH="/home/raffenet/software/hydra-4.2.3/bin/mpiexec:$PATH" - - echo "Infinity Environment setup complete." -} - -################################ -# Cleanup Python Processes # -################################ -cleanup_python_processes() { - echo "Cleaning up existing Python processes..." - - # Define patterns to kill - local patterns=("vllm serve" "multiprocessing.spawn" "multiprocessing.resource_tracker") - - for pattern in "${patterns[@]}"; do - pids=$(pgrep -f "$pattern") - for pid in $pids; do - echo "Killing process $pid ($pattern)" - kill -9 "$pid" 2>/dev/null || true - done - done - - # Kill all Python processes owned by the current user - pids=$(pgrep -u "$USER" python) - for pid in $pids; do - echo "Killing Python process $pid" - kill -9 "$pid" 2>/dev/null || true - done - - sleep 5 # Give some time for processes to terminate - - # Force kill any remaining processes (second pass just in case) - for pattern in "${patterns[@]}"; do - pids=$(pgrep -f "$pattern") - for pid in $pids; do - echo "Force killing process $pid ($pattern)" - kill -9 "$pid" 2>/dev/null || true - done - done - - pids=$(pgrep -u "$USER" python) - for pid in $pids; do - echo "Force killing Python process $pid" - kill -9 "$pid" 2>/dev/null || true - done - - echo "Cleanup complete." -} - - -################################ -# Start Model Function # -################################ - -################################ -# Start Model Function # -################################ -start_model() { - local model_name="$1" - local command="$2" - local log_file="$3" - local -n attempt_counter_ref="$4" # Pass by reference for attempt counter - local max_attempts=2 - local timeout=3600 # Default timeout (can be parameterized) - - while [ "$attempt_counter_ref" -lt "$max_attempts" ]; do - attempt_counter_ref=$((attempt_counter_ref + 1)) - echo "Starting $model_name (Attempt $attempt_counter_ref of $max_attempts)" - - # Start the model in the background - log_dir="$(dirname "$log_file")" - # Create the directory if it doesn’t exist - mkdir -p "$log_dir" - - # Create an empty file if it doesn't already exist - touch "$log_file" - > "$log_file" - - nohup bash -c "$command" > "$log_file" 2>&1 & - local pid=$! - - local start_time=$(date +%s) - while true; do - if [[ -f "$log_file" ]] && grep -q "INFO: Application startup complete." "$log_file"; then - echo "$model_name started successfully" - return 0 - fi - - local current_time=$(date +%s) - local elapsed_time=$((current_time - start_time)) - - if [ "$elapsed_time" -ge "$timeout" ]; then - echo "Timeout reached for $model_name. Killing process." - kill -9 "$pid" 2>/dev/null || true - break - fi - - sleep 5 - done - - echo "Failed to start $model_name. Cleaning up and retrying..." | tee -a error_log.txt - cleanup_python_processes - done - - echo "Failed to start $model_name after $max_attempts attempts" | tee -a error_log.txt - exit 1 -} - - -################################ -# Ray Management Functions # -################################ - -# Function to stop Ray -stop_ray() { - echo "Stopping Ray on $(hostname)" - ray stop -f -} - - -# Function to start Ray head node -start_ray_head() { - set -x # Enable command echo for debugging - - current_node=$(hostname) - ray_port=6379 - - echo "Starting Ray head on $current_node" - # Start Ray head node - ray start --num-cpus=64 --num-gpus=8 --head --port=$ray_port - # Wait for Ray head to be up - echo "Waiting for Ray head to be up..." - until ray status &>/dev/null; do - sleep 5 - echo "Waiting for Ray head..." - done - echo "ray status: $(ray status)" - echo "Ray head node is up." -} - -# Function to start Ray worker node -start_ray_worker() { - set -x # Enable command echo for debugging - - current_node=$(hostname) - ray_port=6379 - - # Debug: Echo OMP_NUM_THREADS and hostname - echo "OMP_NUM_THREADS on $current_node: $OMP_NUM_THREADS" - - echo "Starting Ray worker on $current_node, connecting to $RAY_HEAD_IP:$ray_port" - # Start Ray worker node - ray start --num-gpus=8 --num-cpus=64 --address=$RAY_HEAD_IP:$ray_port - - # Wait for Ray worker to be up - echo "Waiting for Ray worker to be up..." - until ray status &>/dev/null; do - sleep 5 - echo "Waiting for Ray worker..." - done - - echo "ray status: $(ray status)" - echo "Ray worker node is up." -} - - - -# Function to verify Ray cluster status -verify_ray_cluster() { - local expected_nodes="$1" - local attempts=0 - local max_retries=2 - - while [ $attempts -le $max_retries ]; do - echo "Verifying Ray cluster status (Attempt $((attempts + 1)))..." - - # Extract active nodes count from ray status - actual_nodes=$(ray status | awk ' - /^Node status/ { in_active=0 } - /^Active:/ { in_active=1; next } - /^Pending:/ { in_active=0 } - in_active && /^ *[0-9]+ node_/ { count++ } - END { print count } - ') - - if [ "$actual_nodes" -eq "$expected_nodes" ]; then - echo "Ray cluster has the expected number of nodes: $actual_nodes" - return 0 - else - echo "Ray cluster has $actual_nodes active nodes, expected $expected_nodes." - if [ $attempts -lt $max_retries ]; then - echo "Attempting to restart Ray cluster..." - - # Stop Ray on all nodes - echo "Stopping Ray on all nodes..." - mpiexec -n "$expected_nodes" -hostfile "$PBS_NODEFILE" bash -c "source $COMMON_SETUP_SCRIPT; setup_environment; stop_ray" - - # Start Ray head node - echo "Starting Ray head node..." - mpiexec -n 1 -host "$head_node" bash -l -c "source $COMMON_SETUP_SCRIPT; setup_environment; start_ray_head" - - # Start Ray worker nodes - echo "Starting Ray worker nodes..." - for worker in "${worker_nodes[@]}"; do - echo "Starting Ray worker on $worker" - mpiexec -n 1 -host "$worker" bash -l -c "source $COMMON_SETUP_SCRIPT; setup_environment; start_ray_worker" - done - - # Allow some time for Ray to initialize - sleep 10 - fi - fi - attempts=$((attempts + 1)) - done - - echo "Ray cluster verification failed after $max_retries retries. Exiting." - exit 1 -} \ No newline at end of file diff --git a/compute-endpoints/launch_vllm_model.sh b/compute-endpoints/launch_vllm_model.sh new file mode 100755 index 00000000..fac3c89e --- /dev/null +++ b/compute-endpoints/launch_vllm_model.sh @@ -0,0 +1,449 @@ +#!/bin/bash +################################################################################ +# vLLM Model Launcher Script +# Version: 2.0 +# Description: Launch vLLM models with flexible arguments for single/multi-node +################################################################################ + +# Usage information +usage() { + cat << EOF +Usage: $0 [OPTIONS] + +Required Arguments: + --model-name MODEL HuggingFace model name (e.g., meta-llama/Meta-Llama-3.1-8B-Instruct) + +Optional Arguments: + --vllm-version VERSION vLLM version (default: v0.11.0, e.g., v0.8.2, v0.8.5.post1) + --tensor-parallel SIZE Tensor parallel size (default: 8) + --pipeline-parallel SIZE Pipeline parallel size (default: 1, requires Ray if >1) + --multi Force multi-node mode with Ray (flag) + --cuda-visible-devices IDS Set CUDA_VISIBLE_DEVICES (e.g., "0,1,2,3") + --port PORT Port to listen on (default: 8000) + --host HOST Host to bind to (default: 127.0.0.1) + --max-model-len LENGTH Maximum model context length (default: auto) + --max-num-seqs SEQS Maximum number of sequences (default: auto) + --gpu-memory-util RATIO GPU memory utilization (default: 0.95) + --chat-template PATH Path to custom chat template file + --trust-remote-code Trust remote code (flag) + --enable-chunked-prefill Enable chunked prefill (flag) + --enable-prefix-caching Enable prefix caching (flag) + --enable-auto-tool-choice Enable auto tool choice (flag) + --tool-call-parser PARSER Tool call parser (e.g., llama4_json, llama4_pythonic) + --served-model-name NAME Override served model name + --ssl-keyfile PATH Path to SSL key file (default: ~/certificates/mykey.key) + --ssl-certfile PATH Path to SSL cert file (default: ~/certificates/mycert.crt) + --disable-log-requests Disable request logging (flag) + --disable-log-stats Disable stats logging (flag) + --extra-args "ARGS" Additional vLLM arguments as a quoted string + --max-attempts NUM Maximum startup attempts (default: 2) + --timeout SECONDS Startup timeout per attempt (default: 3600) + --framework NAME Framework identifier for logging (default: vllm) + --cluster NAME Cluster name for logging (default: sophia) + +Examples: + # Single node, 8 GPUs + $0 --model-name meta-llama/Meta-Llama-3.1-8B-Instruct --tensor-parallel 8 + + # Single node using specific GPUs + $0 --model-name meta-llama/Meta-Llama-3.1-8B-Instruct \\ + --tensor-parallel 4 --cuda-visible-devices "0,1,2,3" + + # Force multi-node mode with Ray (even on single node) + $0 --model-name meta-llama/Meta-Llama-3.1-70B-Instruct \\ + --tensor-parallel 8 --multi + + # Multi-node with pipeline parallelism (auto-enables Ray) + $0 --model-name meta-llama/Meta-Llama-3.1-405B-Instruct \\ + --tensor-parallel 8 --pipeline-parallel 4 \\ + --max-model-len 16384 + + # With custom chat template and tool calling + $0 --model-name meta-llama/Llama-4-Scout-17B-16E-Instruct \\ + --tensor-parallel 8 \\ + --chat-template /eagle/argonne_tpc/model_weights/chat-templates/tool_chat_template_llama4_pythonic.jinja \\ + --enable-auto-tool-choice \\ + --tool-call-parser llama4_pythonic \\ + --trust-remote-code + + # Using older vLLM version + $0 --model-name meta-llama/Meta-Llama-3.1-8B-Instruct \\ + --vllm-version v0.8.5.post1 \\ + --tensor-parallel 8 + +EOF + exit 1 +} + +################################ +# Default Configuration # +################################ +MODEL_NAME="" +VLLM_VERSION="v0.11.0" +TENSOR_PARALLEL=8 +PIPELINE_PARALLEL=1 +FORCE_MULTI_NODE=false +CUDA_VISIBLE_DEVICES_ARG="" +PORT=8000 +HOST="127.0.0.1" +MAX_MODEL_LEN="" +MAX_NUM_SEQS="" +GPU_MEMORY_UTIL=0.95 +CHAT_TEMPLATE="" +TRUST_REMOTE_CODE=false +ENABLE_CHUNKED_PREFILL=false +ENABLE_PREFIX_CACHING=false +ENABLE_AUTO_TOOL_CHOICE=false +TOOL_CALL_PARSER="" +SERVED_MODEL_NAME="" +SSL_KEYFILE="${HOME}/certificates/mykey.key" +SSL_CERTFILE="${HOME}/certificates/mycert.crt" +DISABLE_LOG_REQUESTS=false +DISABLE_LOG_STATS=false +EXTRA_ARGS="" +MAX_ATTEMPTS=2 +TIMEOUT=3600 +FRAMEWORK="vllm" +CLUSTER="sophia" + +################################ +# Parse Arguments # +################################ +while [[ $# -gt 0 ]]; do + case $1 in + --model-name) + MODEL_NAME="$2" + shift 2 + ;; + --vllm-version) + VLLM_VERSION="$2" + shift 2 + ;; + --tensor-parallel) + TENSOR_PARALLEL="$2" + shift 2 + ;; + --pipeline-parallel) + PIPELINE_PARALLEL="$2" + shift 2 + ;; + --multi) + FORCE_MULTI_NODE=true + shift + ;; + --cuda-visible-devices) + CUDA_VISIBLE_DEVICES_ARG="$2" + shift 2 + ;; + --port) + PORT="$2" + shift 2 + ;; + --host) + HOST="$2" + shift 2 + ;; + --max-model-len) + MAX_MODEL_LEN="$2" + shift 2 + ;; + --max-num-seqs) + MAX_NUM_SEQS="$2" + shift 2 + ;; + --gpu-memory-util) + GPU_MEMORY_UTIL="$2" + shift 2 + ;; + --chat-template) + CHAT_TEMPLATE="$2" + shift 2 + ;; + --trust-remote-code) + TRUST_REMOTE_CODE=true + shift + ;; + --enable-chunked-prefill) + ENABLE_CHUNKED_PREFILL=true + shift + ;; + --enable-prefix-caching) + ENABLE_PREFIX_CACHING=true + shift + ;; + --enable-auto-tool-choice) + ENABLE_AUTO_TOOL_CHOICE=true + shift + ;; + --tool-call-parser) + TOOL_CALL_PARSER="$2" + shift 2 + ;; + --served-model-name) + SERVED_MODEL_NAME="$2" + shift 2 + ;; + --ssl-keyfile) + SSL_KEYFILE="$2" + shift 2 + ;; + --ssl-certfile) + SSL_CERTFILE="$2" + shift 2 + ;; + --disable-log-requests) + DISABLE_LOG_REQUESTS=true + shift + ;; + --disable-log-stats) + DISABLE_LOG_STATS=true + shift + ;; + --extra-args) + EXTRA_ARGS="$2" + shift 2 + ;; + --max-attempts) + MAX_ATTEMPTS="$2" + shift 2 + ;; + --timeout) + TIMEOUT="$2" + shift 2 + ;; + --framework) + FRAMEWORK="$2" + shift 2 + ;; + --cluster) + CLUSTER="$2" + shift 2 + ;; + -h|--help) + usage + ;; + *) + echo "Unknown option: $1" + usage + ;; + esac +done + +# Validate required arguments +if [ -z "$MODEL_NAME" ]; then + echo "ERROR: --model-name is required" + usage +fi + +################################ +# Source Environment Setup # +################################ +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ENV_SETUP_SCRIPT="${ENV_SETUP_SCRIPT:-${SCRIPT_DIR}/sophia_env_setup_with_ray.sh}" + +if [ ! -f "$ENV_SETUP_SCRIPT" ]; then + # Try alternate location + ENV_SETUP_SCRIPT="/home/openinference_svc/sophia_env_setup_with_ray.sh" +fi + +if [ ! -f "$ENV_SETUP_SCRIPT" ]; then + echo "ERROR: Environment setup script not found: $ENV_SETUP_SCRIPT" + exit 1 +fi + +echo "Sourcing environment setup script: $ENV_SETUP_SCRIPT" +source "$ENV_SETUP_SCRIPT" + +# Set vLLM version before environment setup +export VLLM_VERSION="$VLLM_VERSION" + +# Setup environment +setup_environment + +################################ +# Determine Execution Mode # +################################ +MULTI_NODE=false + +# Multi-node mode is enabled if: +# 1. User explicitly specifies --multi flag +# 2. Pipeline parallelism is used (PP > 1) +if [ "$FORCE_MULTI_NODE" = true ]; then + MULTI_NODE=true + echo "Multi-node mode explicitly enabled (--multi flag)" +elif [ "$PIPELINE_PARALLEL" -gt 1 ]; then + MULTI_NODE=true + echo "Pipeline parallelism detected (PP=$PIPELINE_PARALLEL), multi-node mode required" +fi + +echo "Execution mode: $([ "$MULTI_NODE" = true ] && echo "MULTI-NODE (Ray)" || echo "SINGLE-NODE (multiprocessing)")" + +################################ +# Setup Ray if Multi-Node # +################################ +if [ "$MULTI_NODE" = true ]; then + echo "Setting up Ray cluster for multi-node execution..." + + if ! setup_ray_cluster; then + echo "ERROR: Failed to setup Ray cluster" + exit 1 + fi + + # Use Ray backend for multi-node or pipeline parallelism + DISTRIBUTED_BACKEND="ray" +else + # Use multiprocessing backend for single-node tensor parallelism + DISTRIBUTED_BACKEND="mp" +fi + +################################ +# Build vLLM Command # +################################ +echo "Building vLLM command..." + +# Start with base command +VLLM_CMD="vllm serve ${MODEL_NAME}" +VLLM_CMD="${VLLM_CMD} --host ${HOST}" +VLLM_CMD="${VLLM_CMD} --port ${PORT}" +VLLM_CMD="${VLLM_CMD} --tensor-parallel-size ${TENSOR_PARALLEL}" + +if [ "$PIPELINE_PARALLEL" -gt 1 ]; then + VLLM_CMD="${VLLM_CMD} --pipeline-parallel-size ${PIPELINE_PARALLEL}" +fi + +if [ -n "$DISTRIBUTED_BACKEND" ]; then + VLLM_CMD="${VLLM_CMD} --distributed-executor-backend ${DISTRIBUTED_BACKEND}" +fi + +if [ -n "$MAX_MODEL_LEN" ]; then + VLLM_CMD="${VLLM_CMD} --max-model-len ${MAX_MODEL_LEN}" +fi + +if [ -n "$MAX_NUM_SEQS" ]; then + VLLM_CMD="${VLLM_CMD} --max-num-seqs ${MAX_NUM_SEQS}" +fi + +VLLM_CMD="${VLLM_CMD} --gpu-memory-utilization ${GPU_MEMORY_UTIL}" + +if [ "$TRUST_REMOTE_CODE" = true ]; then + VLLM_CMD="${VLLM_CMD} --trust-remote-code" +fi + +if [ "$ENABLE_CHUNKED_PREFILL" = true ]; then + VLLM_CMD="${VLLM_CMD} --enable-chunked-prefill" +fi + +if [ "$ENABLE_PREFIX_CACHING" = true ]; then + VLLM_CMD="${VLLM_CMD} --enable-prefix-caching" +fi + +if [ "$ENABLE_AUTO_TOOL_CHOICE" = true ]; then + VLLM_CMD="${VLLM_CMD} --enable-auto-tool-choice" +fi + +if [ -n "$TOOL_CALL_PARSER" ]; then + VLLM_CMD="${VLLM_CMD} --tool-call-parser ${TOOL_CALL_PARSER}" +fi + +if [ -n "$CHAT_TEMPLATE" ]; then + if [ -f "$CHAT_TEMPLATE" ]; then + VLLM_CMD="${VLLM_CMD} --chat-template ${CHAT_TEMPLATE}" + else + echo "WARNING: Chat template file not found: $CHAT_TEMPLATE" + fi +fi + +if [ -n "$SERVED_MODEL_NAME" ]; then + VLLM_CMD="${VLLM_CMD} --served-model-name ${SERVED_MODEL_NAME}" +fi + +if [ "$DISABLE_LOG_REQUESTS" = true ]; then + VLLM_CMD="${VLLM_CMD} --disable-log-requests" +fi + +if [ "$DISABLE_LOG_STATS" = true ]; then + VLLM_CMD="${VLLM_CMD} --disable-log-stats" +fi + +# SSL certificates +if [ -f "$SSL_KEYFILE" ] && [ -f "$SSL_CERTFILE" ]; then + VLLM_CMD="${VLLM_CMD} --ssl-keyfile ${SSL_KEYFILE}" + VLLM_CMD="${VLLM_CMD} --ssl-certfile ${SSL_CERTFILE}" +else + echo "WARNING: SSL certificates not found, running without SSL" +fi + +# Add extra arguments +if [ -n "$EXTRA_ARGS" ]; then + VLLM_CMD="${VLLM_CMD} ${EXTRA_ARGS}" +fi + +################################ +# Setup Logging # +################################ +# Create clean model name for log file +MODEL_NAME_CLEAN=$(echo "$MODEL_NAME" | sed 's/[^a-zA-Z0-9._-]/_/g') +LOG_FILE="${PWD}/logfile_${CLUSTER}-${FRAMEWORK}-${MODEL_NAME_CLEAN}_$(hostname).log" + +echo "==========================================" +echo "vLLM Launch Configuration" +echo "==========================================" +echo "Model: $MODEL_NAME" +echo "vLLM Version: $VLLM_VERSION" +echo "Tensor Parallel: $TENSOR_PARALLEL" +echo "Pipeline Parallel: $PIPELINE_PARALLEL" +echo "Distributed Backend: ${DISTRIBUTED_BACKEND:-multiprocessing}" +echo "Host: $HOST" +echo "Port: $PORT" +echo "Max Model Length: ${MAX_MODEL_LEN:-auto}" +echo "GPU Memory Util: $GPU_MEMORY_UTIL" +if [ -n "$CUDA_VISIBLE_DEVICES_ARG" ]; then + echo "CUDA Visible Devices: $CUDA_VISIBLE_DEVICES_ARG" +fi +echo "Log File: $LOG_FILE" +echo "==========================================" +echo "Command: $VLLM_CMD" +echo "==========================================" + +################################ +# Set CUDA_VISIBLE_DEVICES # +################################ +if [ -n "$CUDA_VISIBLE_DEVICES_ARG" ]; then + export CUDA_VISIBLE_DEVICES="$CUDA_VISIBLE_DEVICES_ARG" + echo "Set CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES" +fi + +################################ +# Start the Model # +################################ +retry_counter=0 + +echo "Starting model with retry logic (max attempts: $MAX_ATTEMPTS)..." + +while true; do + if start_model "$MODEL_NAME" "$VLLM_CMD" "$LOG_FILE" retry_counter "$MAX_ATTEMPTS" "$TIMEOUT"; then + echo "==========================================" + echo "Model started successfully!" + echo "==========================================" + echo "Model: $MODEL_NAME" + echo "Endpoint: https://$(hostname):${PORT}" + echo "Log: $LOG_FILE" + echo "==========================================" + + # Keep the script running to maintain the model process + echo "Model is running. Press Ctrl+C to stop." + wait + exit 0 + else + echo "Failed to start model on attempt $retry_counter" + if [ $retry_counter -ge $MAX_ATTEMPTS ]; then + echo "==========================================" + echo "ERROR: Failed to start model after $MAX_ATTEMPTS attempts" + echo "==========================================" + echo "Check log file: $LOG_FILE" + exit 1 + fi + echo "Retrying..." + fi +done + diff --git a/compute-endpoints/local-vllm-endpoint.yaml b/compute-endpoints/local-vllm-endpoint.yaml index 180707eb..f0400840 100644 --- a/compute-endpoints/local-vllm-endpoint.yaml +++ b/compute-endpoints/local-vllm-endpoint.yaml @@ -11,7 +11,7 @@ engine: type: SimpleLauncher init_blocks: 1 max_blocks: 1 - min_blocks: 0 + min_blocks: 1 worker_init: | # Activate the conda environment # conda activate /opt/anaconda3/envs/inference-gateway-py3.11.9-env @@ -112,4 +112,4 @@ engine: # independently after starting the background process. # sleep infinity allowed_functions: - - b9779cbe-c6d3-45ee-b68d-9012530cfa82 \ No newline at end of file + - $VLLM_FUNCTION_UUID \ No newline at end of file diff --git a/compute-endpoints/pbs-qstat-example.yaml b/compute-endpoints/pbs-qstat-example.yaml new file mode 100644 index 00000000..73c94729 --- /dev/null +++ b/compute-endpoints/pbs-qstat-example.yaml @@ -0,0 +1,16 @@ +amqp_port: 443 +display_name: sophia-qstat-parser-endpoint +engine: + max_retries_on_system_failure: 2 + max_workers_per_node: 2 + address: + type: address_by_interface + ifname: bond0.2210 + provider: + init_blocks: 1 + max_blocks: 1 + min_blocks: 1 + type: LocalProvider + type: GlobusComputeEngine +allowed_functions: + - $QSTAT_FUNCTION_UUID # Replace with the actual function UUID \ No newline at end of file diff --git a/compute-endpoints/sophia-villm-config-template-v2.0.yaml b/compute-endpoints/sophia-villm-config-template-v2.0.yaml deleted file mode 100644 index bb0099c6..00000000 --- a/compute-endpoints/sophia-villm-config-template-v2.0.yaml +++ /dev/null @@ -1,56 +0,0 @@ -amqp_port: 443 -display_name: sophia-vllm-llama3.1-artiller-test-endpoint -engine: - type: GlobusComputeEngine - max_retries_on_system_failure: 2 - max_workers_per_node: 100 - job_status_kwargs: - max_idletime: 86400 - address: - type: address_by_interface - ifname: ens10f0 - provider: - type: PBSProProvider - launcher: - type: SimpleLauncher - account: argonne_tpc - select_options: ngpus=8 - scheduler_options: '#PBS -l filesystems=home:eagle' - queue: 'by-node' - init_blocks: 0 - max_blocks: 2 - min_blocks: 0 - nodes_per_block: 1 - walltime: 24:00:00 - worker_init: | - # single_model_launch.sh - # Source the common script - source /home/openinference_svc/sophia_common_scripts.sh # Replace with the actual path to common.sh - # Setup the environment - setup_environment - # Define model parameters - model_name="Meta-Llama-3.1-8B-Instruct" - model_command="vllm serve meta-llama/Meta-Llama-3.1-8B-Instruct --host 127.0.0.1 --port 8000 \ - --tensor-parallel-size 8 --gpu-memory-utilization 0.95 \ - --disable-log-requests --enable-chunked-prefill \ - --enable-prefix-caching --multi-step-stream-outputs False \ - --served-model-name meta-llama/Meta-Llama-3.1-8B-Instruct/artillery-test \ - --ssl-keyfile ~/certificates/mykey.key --ssl-certfile ~/certificates/mycert.crt" - log_file="$PWD/logfile_sophia_vllm_$(model_name)_$(hostname).log" - - # Initialize retry counter for the model - retry_counter_model_1=0 - - # Start the model - # Loop to start models and restart if any model fails - while true; do - echo "Starting models sequence..." - # Start first model - if ! start_model "$model_name_1" "$model_command_1" "$logfile_1" retry_counter_model_1; then - continue # Restart from the beginning if this fails - fi - echo "All models started successfully." - break # Exit the loop if all models start successfully - done -allowed_functions: - - 3073ed77-6a17-4e85-826a-e1dca5309e01 \ No newline at end of file diff --git a/compute-endpoints/sophia-vllm-batch-template.yaml b/compute-endpoints/sophia-vllm-batch-template.yaml index 270a832e..fc1c7325 100644 --- a/compute-endpoints/sophia-vllm-batch-template.yaml +++ b/compute-endpoints/sophia-vllm-batch-template.yaml @@ -32,4 +32,4 @@ engine: setup_environment allowed_functions: - - d91563c3-34ed-4d6a-975a-e4827f6b73ae \ No newline at end of file + - $VLLM_BATCH_FUNCTION_UUID \ No newline at end of file diff --git a/compute-endpoints/sophia-vllm-config-multinode-template.yaml b/compute-endpoints/sophia-vllm-config-multinode-template.yaml deleted file mode 100644 index 07b5f21c..00000000 --- a/compute-endpoints/sophia-vllm-config-multinode-template.yaml +++ /dev/null @@ -1,107 +0,0 @@ -amqp_port: 443 -display_name: sophia-vllm-meta-llama-3.1-405b-instruct -engine: - type: GlobusComputeEngine - max_retries_on_system_failure: 1 - max_workers_per_node: 100 - job_status_kwargs: - max_idletime: 86400 - address: - type: address_by_interface - ifname: ens10f0 - provider: - type: PBSProProvider - launcher: - type: SimpleLauncher - account: argonne_tpc - select_options: ngpus=8 - scheduler_options: '#PBS -l filesystems=home:eagle' - queue: 'by-node' - init_blocks: 0 - max_blocks: 1 - min_blocks: 0 - nodes_per_block: 4 - walltime: 24:00:00 - worker_init: | - # Source the common script - source /home/openinference_svc/sophia_common_scripts.sh - - # Setup the environment - setup_environment - - # Read nodes from PBS_NODEFILE - nodes=($(sort -u "$PBS_NODEFILE")) - num_nodes=${#nodes[@]} - - # Get the current node's hostname (assumed to be the head node) - head_node=$(hostname | sed 's/.lab.alcf.anl.gov//') - - echo "Nodes: ${nodes[@]}" - echo "Head node: $head_node" - - # Get the IP address of the head node - RAY_HEAD_IP=$(getent hosts "$head_node" | awk '{ print $1 }') - echo "Ray head IP: $RAY_HEAD_IP" - - # Export variables for use in functions - export head_node - export RAY_HEAD_IP - export HOST_IP="$RAY_HEAD_IP" - export RAY_ADDRESS="$RAY_HEAD_IP:6379" - - # Define worker nodes (exclude head node) - worker_nodes=() - for node in "${nodes[@]}"; do - short_node=$(echo "$node" | sed 's/.lab.alcf.anl.gov//') - if [ "$short_node" != "$head_node" ]; then - worker_nodes+=("$short_node") - fi - done - - echo "Worker nodes: ${worker_nodes[@]}" - - # Stop Ray on all nodes using mpiexec - echo "Stopping any existing Ray processes on all nodes..." - mpiexec -n "$num_nodes" -hostfile "$PBS_NODEFILE" bash -c "source $COMMON_SETUP_SCRIPT; setup_environment; stop_ray; cleanup_python_processes;" - - # Start Ray head node - echo "Starting Ray head node..." - mpiexec -n 1 -host "$head_node" bash -l -c "source $COMMON_SETUP_SCRIPT; export RAY_HEAD_IP=$RAY_HEAD_IP; setup_environment; start_ray_head" - - echo "Starting Ray worker nodes..." - for worker in "${worker_nodes[@]}"; do - echo "Starting Ray worker on $worker" - mpiexec -n 1 -host "$worker" bash -l -c "source $COMMON_SETUP_SCRIPT; export RAY_HEAD_IP=$RAY_HEAD_IP; setup_environment; start_ray_worker" - done - - # Verify Ray cluster status - echo "Verifying Ray cluster status..." - verify_ray_cluster "$num_nodes" - - echo "Ray cluster setup complete." - - # Define model parameters - model_name="Meta-Llama-3.1-405B-Instruct" - model_command="vllm serve meta-llama/Meta-Llama-3.1-405B-Instruct --host 127.0.0.1 --port 8000 \ - --tensor-parallel-size 8 --pipeline-parallel-size 4 --max-model-len 16384 \ - --disable-log-stats --enable-prefix-caching --enable-chunked-prefill --multi-step-stream-outputs False \ - --trust-remote-code --gpu-memory-utilization 0.95 --disable-log-requests \ - --ssl-keyfile ~/certificates/mykey.key --ssl-certfile ~/certificates/mycert.crt" - log_file="$PWD/logfile_sophia-vllm-{model_name}_$(hostname).log" - - # Initialize retry counter for the model - retry_counter_model_1=0 - - # Start the model - while true; do - echo "Starting models sequence..." - if ! start_model "$model_name" "$model_command" "$log_file" retry_counter_model_1; then - continue # Restart from the beginning if this fails - fi - echo "All models started successfully." - break - done - -# Limit the functions UUID that can be execute -allowed_functions: - - 3073ed77-6a17-4e85-826a-e1dca5309e01 \ No newline at end of file diff --git a/compute-endpoints/sophia-vllm-multinode-example.yaml b/compute-endpoints/sophia-vllm-multinode-example.yaml new file mode 100644 index 00000000..c7926b81 --- /dev/null +++ b/compute-endpoints/sophia-vllm-multinode-example.yaml @@ -0,0 +1,44 @@ +amqp_port: 443 +display_name: sophia-vllm-llama3.1-405b-instruct-example +engine: + type: GlobusComputeEngine + max_retries_on_system_failure: 1 + max_workers_per_node: 100 + job_status_kwargs: + max_idletime: 86400 + address: + type: address_by_interface + ifname: ens10f0 + provider: + type: PBSProProvider + launcher: + type: SimpleLauncher + account: argonne_tpc + select_options: ngpus=8 + scheduler_options: '#PBS -l filesystems=home:eagle' + queue: 'by-node' + init_blocks: 0 + max_blocks: 1 + min_blocks: 0 + nodes_per_block: 4 + walltime: 24:00:00 + worker_init: | + # Multi-node setup with Ray for large model (TP=8, PP=4) + source /home/openinference_svc/launch_vllm_model.sh \ + --model-name meta-llama/Meta-Llama-3.1-405B-Instruct \ + --vllm-version v0.11.0 \ + --tensor-parallel 8 \ + --pipeline-parallel 4 \ + --max-model-len 16384 \ + --enable-prefix-caching \ + --enable-chunked-prefill \ + --trust-remote-code \ + --disable-log-requests \ + --disable-log-stats \ + --gpu-memory-util 0.95 \ + --framework vllm \ + --cluster sophia + +allowed_functions: + - $VLLM_FUNCTION_UUID + diff --git a/compute-endpoints/sophia-vllm-singlenode-example.yaml b/compute-endpoints/sophia-vllm-singlenode-example.yaml new file mode 100644 index 00000000..ce3f3f0c --- /dev/null +++ b/compute-endpoints/sophia-vllm-singlenode-example.yaml @@ -0,0 +1,42 @@ +amqp_port: 443 +display_name: sophia-vllm-llama3.1-8b-instruct-example +engine: + type: GlobusComputeEngine + max_retries_on_system_failure: 2 + max_workers_per_node: 100 + job_status_kwargs: + max_idletime: 86400 + address: + type: address_by_interface + ifname: ens10f0 + provider: + type: PBSProProvider + launcher: + type: SimpleLauncher + account: argonne_tpc + select_options: ngpus=8 + scheduler_options: '#PBS -l filesystems=home:eagle' + queue: 'by-node' + init_blocks: 0 + max_blocks: 2 + min_blocks: 0 + nodes_per_block: 1 + walltime: 24:00:00 + worker_init: | + # Source and launch using the new modular scripts + source /home/openinference_svc/launch_vllm_model.sh \ + --model-name meta-llama/Meta-Llama-3.1-8B-Instruct \ + --vllm-version v0.11.0 \ + --tensor-parallel 8 \ + --max-model-len 8192 \ + --enable-chunked-prefill \ + --enable-prefix-caching \ + --trust-remote-code \ + --disable-log-requests \ + --gpu-memory-util 0.95 \ + --framework vllm \ + --cluster sophia + +allowed_functions: + - $VLLM_FUNCTION_UUID + diff --git a/compute-endpoints/sophia-vllm-toolcalling-example.yaml b/compute-endpoints/sophia-vllm-toolcalling-example.yaml new file mode 100644 index 00000000..018b0537 --- /dev/null +++ b/compute-endpoints/sophia-vllm-toolcalling-example.yaml @@ -0,0 +1,44 @@ +amqp_port: 443 +display_name: sophia-vllm-llama4-scout-toolcalling-example +engine: + type: GlobusComputeEngine + max_retries_on_system_failure: 2 + max_workers_per_node: 100 + job_status_kwargs: + max_idletime: 86400 + address: + type: address_by_interface + ifname: ens10f0 + provider: + type: PBSProProvider + launcher: + type: SimpleLauncher + account: argonne_tpc + select_options: ngpus=8 + scheduler_options: '#PBS -l filesystems=home:eagle' + queue: 'by-node' + init_blocks: 0 + max_blocks: 2 + min_blocks: 0 + nodes_per_block: 1 + walltime: 24:00:00 + worker_init: | + # Single-node with tool calling support (Llama 4 models) + source /home/openinference_svc/launch_vllm_model.sh \ + --model-name meta-llama/Llama-4-Scout-17B-16E-Instruct \ + --vllm-version v0.11.0 \ + --tensor-parallel 8 \ + --max-model-len 32768 \ + --max-num-seqs 8 \ + --trust-remote-code \ + --enable-auto-tool-choice \ + --tool-call-parser llama4_pythonic \ + --chat-template /eagle/argonne_tpc/model_weights/chat-templates/tool_chat_template_llama4_pythonic.jinja \ + --disable-log-requests \ + --gpu-memory-util 0.95 \ + --framework vllm \ + --cluster sophia + +allowed_functions: + - $VLLM_FUNCTION_UUID + diff --git a/compute-endpoints/sophia_env_setup_with_ray.sh b/compute-endpoints/sophia_env_setup_with_ray.sh new file mode 100755 index 00000000..27e82483 --- /dev/null +++ b/compute-endpoints/sophia_env_setup_with_ray.sh @@ -0,0 +1,535 @@ +#!/bin/bash +################################################################################ +# Sophia Environment Setup Script with Ray Support +# Version: 2.0 +# Description: Environment setup and helper functions for vLLM on Sophia +################################################################################ + +################################ +# Environment Setup Function # +################################ +setup_environment() { + echo "==========================================" + echo "Setting up Sophia environment..." + echo "==========================================" + + # Proxy configuration for ALCF + export HTTP_PROXY="http://proxy.alcf.anl.gov:3128" + export HTTPS_PROXY="http://proxy.alcf.anl.gov:3128" + export http_proxy=$HTTP_PROXY + export https_proxy=$HTTPS_PROXY + export ftp_proxy=$HTTP_PROXY + + # Load required modules + echo "Loading modules..." + module use /soft/modulefiles + module load conda + module load spack-pe-base + module load gcc + + # Determine conda environment based on VLLM_VERSION + if [ -z "$VLLM_VERSION" ]; then + VLLM_VERSION="v0.11.0" + echo "VLLM_VERSION not set, using default: $VLLM_VERSION" + fi + + # Map version to conda environment + case "$VLLM_VERSION" in + v0.8.2) + CONDA_ENV="/eagle/argonne_tpc/inference-gateway/envs/vllmv0.8.2/" + ;; + v0.8.5*) + CONDA_ENV="/eagle/argonne_tpc/inference-gateway/envs/vllmv0.8.5.post1/" + ;; + v0.10.1) + CONDA_ENV="/eagle/argonne_tpc/inference-gateway/envs/vllmv0.10.1/" + ;; + v0.11.0) + CONDA_ENV="/eagle/argonne_tpc/inference-gateway/envs/vllmv0.11.0/" + ;; + *) + echo "WARNING: Unknown VLLM_VERSION '$VLLM_VERSION', defaulting to v0.11.0" + CONDA_ENV="/eagle/argonne_tpc/inference-gateway/envs/vllmv0.11.0/" + ;; + esac + + echo "Activating conda environment: $CONDA_ENV" + conda activate "$CONDA_ENV" + + # Verify activation + if [ $? -ne 0 ]; then + echo "ERROR: Failed to activate conda environment: $CONDA_ENV" + return 1 + fi + + echo "Active conda environment: $CONDA_DEFAULT_ENV" + + # HuggingFace cache and token configuration + export HF_DATASETS_CACHE='/eagle/argonne_tpc/model_weights/' + export HF_HOME='/eagle/argonne_tpc/model_weights/' + export HF_HUB_CACHE='/eagle/argonne_tpc/model_weights/hub' + export TRANSFORMERS_OFFLINE=1 + export HF_TOKEN=${HF_TOKEN} # Replace with your actual HuggingFace token + + # Ray configuration + export RAY_TMPDIR='/tmp' + + # NCCL configuration for multi-GPU/multi-node + export NCCL_SOCKET_IFNAME='infinibond0' + + # Threading configuration + export OMP_NUM_THREADS=4 + + # vLLM-specific settings + export VLLM_LOG_LEVEL=WARN + export USE_FASTSAFETENSOR=true + export VLLM_IMAGE_FETCH_TIMEOUT=60 + export VLLM_USE_RAY_COMPILED_DAG_CHANNEL_TYPE=shm + + # Increase core dump size for debugging + ulimit -c unlimited + + # Internal secrets for streaming + export INTERNAL_STREAMING_SECRET=${INTERNAL_STREAMING_SECRET} # Replace with your actual internal streaming secret + + # Export path to this script for nested calls + export COMMON_SETUP_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" + + echo "Environment setup complete." + echo " - Python: $(which python)" + echo " - vLLM version: $VLLM_VERSION" + echo " - Conda env: $CONDA_DEFAULT_ENV" + echo "==========================================" + + return 0 +} + +################################ +# Cleanup Python Processes # +################################ +cleanup_python_processes() { + echo "Cleaning up existing Python processes..." + + # Define patterns to kill + local patterns=("vllm serve" "vllm.entrypoints" "multiprocessing.spawn" "multiprocessing.resource_tracker" "ray::") + + for pattern in "${patterns[@]}"; do + pids=$(pgrep -f "$pattern" 2>/dev/null) + if [ -n "$pids" ]; then + for pid in $pids; do + echo " Killing process $pid (pattern: $pattern)" + kill -9 "$pid" 2>/dev/null || true + done + fi + done + + # Give processes time to terminate + sleep 2 + + echo "Cleanup complete." +} + +################################ +# Ray Cluster Management # +################################ + +# Function to stop Ray +stop_ray() { + echo "Stopping Ray on $(hostname)..." + ray stop -f 2>/dev/null || true + + # Cleanup Ray temporary files + if [ -n "$RAY_TMPDIR" ] && [ -d "$RAY_TMPDIR" ]; then + rm -rf "$RAY_TMPDIR"/ray_* 2>/dev/null || true + fi + + sleep 2 + echo "Ray stopped on $(hostname)" +} + +# Function to start Ray head node +start_ray_head() { + local ray_port=${RAY_PORT:-6379} + local num_cpus=${RAY_NUM_CPUS:-64} + local num_gpus=${RAY_NUM_GPUS:-8} + + echo "----------------------------------------" + echo "Starting Ray head node on $(hostname)" + echo " Port: $ray_port" + echo " CPUs: $num_cpus" + echo " GPUs: $num_gpus" + echo "----------------------------------------" + + # Stop any existing Ray instance + stop_ray + + # Start Ray head node + ray start \ + --head \ + --port=$ray_port \ + --num-cpus=$num_cpus \ + --num-gpus=$num_gpus \ + --include-dashboard=false \ + --disable-usage-stats + + if [ $? -ne 0 ]; then + echo "ERROR: Failed to start Ray head node" + return 1 + fi + + # Wait for Ray to be ready + echo "Waiting for Ray head to be ready..." + local max_wait=30 + local count=0 + until ray status &>/dev/null; do + sleep 2 + count=$((count + 2)) + if [ $count -ge $max_wait ]; then + echo "ERROR: Ray head did not start within ${max_wait}s" + return 1 + fi + echo " Waiting... (${count}s)" + done + + echo "Ray head node is ready." + ray status + echo "----------------------------------------" + + return 0 +} + +# Function to start Ray worker node +start_ray_worker() { + local ray_head_address="${RAY_HEAD_ADDRESS:-}" + local num_cpus=${RAY_NUM_CPUS:-64} + local num_gpus=${RAY_NUM_GPUS:-8} + + if [ -z "$ray_head_address" ]; then + echo "ERROR: RAY_HEAD_ADDRESS not set for worker node" + return 1 + fi + + echo "----------------------------------------" + echo "Starting Ray worker node on $(hostname)" + echo " Head address: $ray_head_address" + echo " CPUs: $num_cpus" + echo " GPUs: $num_gpus" + echo "----------------------------------------" + + # Stop any existing Ray instance + stop_ray + + # Start Ray worker node + ray start \ + --address=$ray_head_address \ + --num-cpus=$num_cpus \ + --num-gpus=$num_gpus \ + --disable-usage-stats + + if [ $? -ne 0 ]; then + echo "ERROR: Failed to start Ray worker node" + return 1 + fi + + # Wait for Ray to connect + echo "Waiting for Ray worker to connect..." + local max_wait=30 + local count=0 + until ray status &>/dev/null; do + sleep 2 + count=$((count + 2)) + if [ $count -ge $max_wait ]; then + echo "ERROR: Ray worker did not connect within ${max_wait}s" + return 1 + fi + echo " Waiting... (${count}s)" + done + + echo "Ray worker node connected." + echo "----------------------------------------" + + return 0 +} + +# Function to setup Ray cluster (multi-node) +setup_ray_cluster() { + echo "==========================================" + echo "Setting up Ray cluster for multi-node execution" + echo "==========================================" + + # Get node information from PBS + if [ -z "$PBS_NODEFILE" ]; then + echo "ERROR: PBS_NODEFILE not set. Ray cluster setup requires PBS environment." + return 1 + fi + + # Read nodes from PBS nodefile + mapfile -t all_nodes < "$PBS_NODEFILE" + + if [ ${#all_nodes[@]} -eq 0 ]; then + echo "ERROR: No nodes found in PBS_NODEFILE" + return 1 + fi + + # Get unique nodes (PBS nodefile may have duplicates for multi-core nodes) + unique_nodes=($(printf "%s\n" "${all_nodes[@]}" | sort -u)) + + echo "Detected ${#unique_nodes[@]} unique nodes:" + printf " %s\n" "${unique_nodes[@]}" + + # First node is head, rest are workers + local head_node="${unique_nodes[0]}" + local worker_nodes=("${unique_nodes[@]:1}") + + echo "Head node: $head_node" + echo "Worker nodes: ${worker_nodes[*]}" + + # Set Ray configuration + export RAY_PORT=6379 + export RAY_NUM_CPUS=64 + export RAY_NUM_GPUS=8 + + # Start Ray head node + echo "Starting Ray head on $head_node..." + if [ "$(hostname)" = "$head_node" ]; then + # We're on the head node, start directly + start_ray_head + else + # Start remotely via mpiexec + mpiexec -n 1 -host "$head_node" bash -l -c " + source '$COMMON_SETUP_SCRIPT' + setup_environment + start_ray_head + " + fi + + if [ $? -ne 0 ]; then + echo "ERROR: Failed to start Ray head node" + return 1 + fi + + # Get Ray head address + export RAY_HEAD_ADDRESS="${head_node}:${RAY_PORT}" + echo "Ray head address: $RAY_HEAD_ADDRESS" + + # Start Ray workers if we have multiple nodes + if [ ${#worker_nodes[@]} -gt 0 ]; then + echo "Starting Ray workers on ${#worker_nodes[@]} nodes..." + + for worker in "${worker_nodes[@]}"; do + echo " Starting Ray worker on $worker..." + if [ "$(hostname)" = "$worker" ]; then + # We're on this worker node, start directly + start_ray_worker + else + # Start remotely via mpiexec + mpiexec -n 1 -host "$worker" bash -l -c " + source '$COMMON_SETUP_SCRIPT' + setup_environment + export RAY_HEAD_ADDRESS='$RAY_HEAD_ADDRESS' + start_ray_worker + " + fi + + if [ $? -ne 0 ]; then + echo "WARNING: Failed to start Ray worker on $worker" + fi + done + else + echo "Single-node Ray cluster (head only)" + fi + + # Allow Ray cluster to stabilize + sleep 5 + + # Verify cluster status + echo "==========================================" + echo "Ray Cluster Status:" + ray status + echo "==========================================" + + # Check if we have the expected number of nodes + local expected_nodes=${#unique_nodes[@]} + local actual_nodes=$(ray status 2>/dev/null | grep -c "node_" || echo "0") + + if [ "$actual_nodes" -ge "$expected_nodes" ]; then + echo "Ray cluster setup successful: $actual_nodes nodes active" + return 0 + else + echo "WARNING: Ray cluster may not have all nodes active" + echo " Expected: $expected_nodes" + echo " Actual: $actual_nodes" + echo "Continuing anyway..." + return 0 + fi +} + +################################ +# Model Startup Function # +################################ +start_model() { + local model_name="$1" + local command="$2" + local log_file="$3" + local -n attempt_counter_ref="$4" # Pass by reference + local max_attempts="${5:-2}" + local timeout="${6:-3600}" + + echo "==========================================" + echo "Starting model: $model_name" + echo "Log file: $log_file" + echo "Max attempts: $max_attempts" + echo "Timeout: ${timeout}s" + echo "==========================================" + + # Increment attempt counter + attempt_counter_ref=$((attempt_counter_ref + 1)) + + if [ $attempt_counter_ref -gt $max_attempts ]; then + echo "ERROR: Max attempts ($max_attempts) exceeded" + return 1 + fi + + echo "Attempt $attempt_counter_ref of $max_attempts" + + # Ensure log directory exists + local log_dir="$(dirname "$log_file")" + mkdir -p "$log_dir" + + # Clear previous log + > "$log_file" + + # Start the model in background + echo "Executing: $command" + nohup bash -c "$command" > "$log_file" 2>&1 & + local pid=$! + + echo "Process started with PID: $pid" + echo "Monitoring startup (timeout: ${timeout}s)..." + + local start_time=$(date +%s) + local last_log_time=$start_time + + while true; do + # Check if process is still running + if ! kill -0 "$pid" 2>/dev/null; then + echo "ERROR: Process $pid exited unexpectedly" + echo "Last 30 lines of log:" + tail -n 30 "$log_file" + return 1 + fi + + # Check for successful startup message + if [ -f "$log_file" ] && grep -q "INFO: Application startup complete." "$log_file"; then + echo "SUCCESS: Model started successfully (PID: $pid)" + echo "==========================================" + return 0 + fi + + # Check for common error patterns + if [ -f "$log_file" ]; then + if grep -qi "error\|exception\|failed" "$log_file" 2>/dev/null; then + local current_time=$(date +%s) + # Only log errors periodically to avoid spam + if [ $((current_time - last_log_time)) -ge 10 ]; then + echo "Detected errors in log (process still running):" + grep -i "error\|exception\|failed" "$log_file" | tail -5 + last_log_time=$current_time + fi + fi + fi + + # Check timeout + local current_time=$(date +%s) + local elapsed_time=$((current_time - start_time)) + + if [ "$elapsed_time" -ge "$timeout" ]; then + echo "ERROR: Timeout reached after ${timeout}s" + echo "Killing process $pid..." + kill -9 "$pid" 2>/dev/null || true + echo "Last 50 lines of log:" + tail -n 50 "$log_file" + return 1 + fi + + # Progress indicator + if [ $((elapsed_time % 30)) -eq 0 ] && [ $elapsed_time -gt 0 ]; then + echo " Still waiting... (${elapsed_time}s elapsed)" + fi + + sleep 5 + done +} + +################################ +# Utility Functions # +################################ + +# Function to check if Ray is running +is_ray_running() { + ray status &>/dev/null + return $? +} + +# Function to get Ray cluster info +get_ray_info() { + if is_ray_running; then + echo "Ray Cluster Information:" + ray status + else + echo "Ray is not running" + return 1 + fi +} + +# Function to display environment info +show_environment_info() { + echo "==========================================" + echo "Environment Information" + echo "==========================================" + echo "Hostname: $(hostname)" + echo "Python: $(which python)" + echo "Python version: $(python --version 2>&1)" + echo "Conda env: ${CONDA_DEFAULT_ENV:-not set}" + echo "vLLM version: ${VLLM_VERSION:-not set}" + echo "HF_HOME: ${HF_HOME:-not set}" + echo "Ray status: $(is_ray_running && echo "running" || echo "not running")" + echo "==========================================" +} + +################################ +# Main Script Logic # +################################ + +# If script is executed directly (not sourced), show help +if [ "${BASH_SOURCE[0]}" = "${0}" ]; then + echo "==========================================" + echo "Sophia Environment Setup Script" + echo "==========================================" + echo "" + echo "This script should be sourced, not executed directly." + echo "" + echo "Usage:" + echo " source $0" + echo " setup_environment" + echo "" + echo "Available functions:" + echo " - setup_environment : Setup conda environment and exports" + echo " - setup_ray_cluster : Setup multi-node Ray cluster" + echo " - start_model : Start vLLM model with retry logic" + echo " - cleanup_python_processes : Kill all Python/vLLM processes" + echo " - stop_ray : Stop Ray on current node" + echo " - show_environment_info : Display environment details" + echo "" + echo "Environment variables:" + echo " - VLLM_VERSION : vLLM version to use (default: v0.11.0)" + echo " - RAY_NUM_CPUS : CPUs per Ray node (default: 64)" + echo " - RAY_NUM_GPUS : GPUs per Ray node (default: 8)" + echo " - RAY_PORT : Ray head port (default: 6379)" + echo "" + exit 0 +fi + +echo "Sophia environment setup script loaded successfully" +echo "Run 'setup_environment' to initialize the environment" + diff --git a/compute-functions/infinity_register_function.py b/compute-functions/infinity_register_function.py deleted file mode 100644 index 3e545a22..00000000 --- a/compute-functions/infinity_register_function.py +++ /dev/null @@ -1,85 +0,0 @@ -import globus_compute_sdk -import random - -def infinity_inference_function(parameters): - import socket - import os - import time - import requests - import json - - # Determine the hostname - hostname = socket.gethostname() - os.environ['no_proxy'] = f"localhost,{hostname},127.0.0.1" - - # Get the API key from environment variable - api_key = os.getenv("OPENAI_API_KEY", "random_api_key") - - # Prepare headers - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {api_key}" - } - - print("parameters", parameters) - - # Determine the endpoint based on the URL parameter - openai_endpoint = parameters['model_params'].pop('openai_endpoint') - - # Determine the port based on the URL parameter - api_port = parameters['model_params'].pop('api_port') - - base_url = f"http://127.0.0.1:{api_port}/" - url = base_url + openai_endpoint - - # Prepare the payload - payload = parameters['model_params'].copy() - - start_time = time.time() - - # Make the POST request - response = requests.post(url, headers=headers, json=payload, verify=False) - - end_time = time.time() - response_time = end_time - start_time - - # Check if the request was successful - if response.status_code == 200: - completion = response.json() - else: - raise Exception(f"API request failed with status code: {response.status_code} {response}") - - # Extract usage information - usage = completion.get('usage', {}) - total_num_tokens = usage.get('total_tokens', 0) - - throughput = total_num_tokens / response_time if response_time > 0 else 0 - - metrics = { - 'response_time': response_time, - 'throughput_tokens_per_second': throughput - } - - # Combine the API response and metrics - output = {**completion, **metrics} - json_output = json.dumps(output, indent=4) - - return json_output - -# Creating Globus Compute client -gcc = globus_compute_sdk.Client() - -# # Register the function -COMPUTE_FUNCTION_ID = gcc.register_function(infinity_inference_function) - -# # Write function UUID in a file -uuid_file_name = "infinity_register_function_sophia_multiple_models.txt" -with open(uuid_file_name, "w") as file: - file.write(COMPUTE_FUNCTION_ID) - file.write("\n") -file.close() - -# # End of script -print("Function registered with UUID -", COMPUTE_FUNCTION_ID) -print("The UUID is stored in " + uuid_file_name + ".") -print("") \ No newline at end of file diff --git a/compute-functions/llamacpp_register_function.py b/compute-functions/llamacpp_register_function.py deleted file mode 100644 index 1691cd1f..00000000 --- a/compute-functions/llamacpp_register_function.py +++ /dev/null @@ -1,55 +0,0 @@ -# Import packages -import globus_compute_sdk - -# Define Globus Compute function -def llamacpp_inference (parameters): - import socket - import json - import os - import time - import requests - - # Determine the hostname - hostname = socket.gethostname() - os.environ['no_proxy'] = "localhost,{hostname}".format(hostname=hostname) - # Construct the base_url - base_url = f"http://localhost:8080/completion" - # Get the API key from environment variable - api_key = os.getenv("OPENAI_API_KEY") - if not api_key: - api_key="random_api_key" - - start_time = time.time() - print("parameters",parameters) - response = requests.post(base_url, data=json.dumps(parameters['model_params'])) - end_time = time.time() - response_time = end_time - start_time - - # Convert the response to a JSON-formatted string - json_response = response.json() - json_response['response_time'] = response_time - - # Print the JSON response for debugging - print(json_response) - return json_response - -# Creating Globus Compute client -gcc = globus_compute_sdk.Client() - -# Register the function -COMPUTE_FUNCTION_ID = gcc.register_function(llamacpp_inference) - -# Write function UUID in a file -uuid_file_name = "llamacpp_function_uuid.txt" -with open(uuid_file_name, "w") as file: - file.write(COMPUTE_FUNCTION_ID) - file.write("\n") -file.close() - -# End of script -print("Function registered with UUID -", COMPUTE_FUNCTION_ID) -print("The UUID is stored in " + uuid_file_name + ".") -print("") - -#Test function -#llamacpp_inference({'model_params': {'model': 'meta-llama3-70b-instruct', 'temperature': 0.2, 'max_tokens': 150, 'prompt': 'List all proteins that interact with RAD51'}}) \ No newline at end of file diff --git a/compute-functions/vllm_register_function.py b/compute-functions/vllm_register_function.py deleted file mode 100644 index 7e55a565..00000000 --- a/compute-functions/vllm_register_function.py +++ /dev/null @@ -1,177 +0,0 @@ -import globus_compute_sdk -import random - -def vllm_inference_function(parameters): - import socket - import os - import time - import requests - import json - from requests.exceptions import RequestException - - try: - # Determine the hostname - hostname = socket.gethostname() - os.environ['no_proxy'] = f"localhost,{hostname},127.0.0.1" - - # Get the API key from environment variable - api_key = os.getenv("OPENAI_API_KEY", "random_api_key") - - # Prepare headers - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {api_key}" - } - - print("parameters", parameters) - - # Determine the endpoint based on the URL parameter - openai_endpoint = parameters['model_params'].pop('openai_endpoint') - - # Determine the port based on the URL parameter - api_port = parameters['model_params'].pop('api_port') - - base_url = f"https://127.0.0.1:{api_port}/v1/" - url = base_url + openai_endpoint - - # Prepare the payload - payload = parameters['model_params'].copy() - - start_time = time.time() - - # Make the POST request - response = requests.post(url, headers=headers, json=payload, verify=False) - - end_time = time.time() - response_time = end_time - start_time - - # Initialize metrics - metrics = { - 'response_time': response_time, - 'throughput_tokens_per_second': 0 - } - - # Handle different response scenarios - if response.status_code == 200: - try: - # Try to parse JSON response - completion = response.json() - - # Extract usage information if available - usage = completion.get('usage', {}) - total_num_tokens = usage.get('total_tokens', 0) - metrics['throughput_tokens_per_second'] = total_num_tokens / response_time if response_time > 0 else 0 - - # Return the response even if empty - output = {**completion, **metrics} - return json.dumps(output, indent=4) - except json.JSONDecodeError: - # If response is not JSON but status is 200, return the raw text - return json.dumps({ - 'completion': response.text, - **metrics - }, indent=4) - else: - # For non-200 responses, raise an exception with detailed error information - error_msg = f"API request failed with status code: {response.status_code}\n" - error_msg += f"Response text: {response.text}\n" - error_msg += f"Response headers: {dict(response.headers)}" - raise Exception(error_msg) - - except RequestException as e: - # Handle network-related errors - error_msg = f"Network error occurred: {str(e)}" - if 'start_time' in locals(): - error_msg += f"\nResponse time: {time.time() - start_time}" - raise Exception(error_msg) - except Exception as e: - # Handle any other unexpected errors - error_msg = f"Unexpected error of type {type(e).__name__}: {str(e)}" - if 'start_time' in locals(): - error_msg += f"\nResponse time: {time.time() - start_time}" - raise Exception(error_msg) - -# Creating Globus Compute client -gcc = globus_compute_sdk.Client() - -# # Register the function -COMPUTE_FUNCTION_ID = gcc.register_function(vllm_inference_function) - -# # Write function UUID in a file -uuid_file_name = "vllm_register_function_sophia_multiple_models.txt" -with open(uuid_file_name, "w") as file: - file.write(COMPUTE_FUNCTION_ID) - file.write("\n") -file.close() - -# # End of script -print("Function registered with UUID -", COMPUTE_FUNCTION_ID) -print("The UUID is stored in " + uuid_file_name + ".") -print("") - -# Example calls - -# List of sample prompts -# prompts = [ -# "Explain the concept of machine learning in simple terms.", -# "What are the main differences between Python and JavaScript?", -# "Write a short story about a robot learning to paint.", -# "Describe the process of photosynthesis.", -# "What are the key features of a good user interface design?" -# ] - - -# # # Chat completion example -# chat_out = vllm_inference_function({ -# 'model_params': { -# 'openai_endpoint': 'chat/completions', -# 'model': 'meta-llama/Meta-Llama-3-8B-Instruct', -# 'api_port': 8001, -# "messages": [{"role": "user", "content": random.choice(prompts)}], -# 'logprobs': True -# } -# }) -# print("Chat Completion Output for meta-llama/Meta-Llama-3-8B-Instruct") -# print(chat_out) - - - -# # # Chat completion example -# chat_out = vllm_inference_function({ -# 'model_params': { -# 'openai_endpoint': 'chat/completions', -# 'model': 'meta-llama/Meta-Llama-3-70B-Instruct', -# 'api_port': 8000, -# "messages": [{"role": "user", "content": random.choice(prompts)}], -# 'logprobs': True -# } -# }) -# print("Chat Completion Output for meta-llama/Meta-Llama-3-70B-Instruct") -# print(chat_out) - -# # # Chat completion example -# chat_out = vllm_inference_function({ -# 'model_params': { -# 'openai_endpoint': 'chat/completions', -# 'model': 'mistralai/Mistral-7B-Instruct-v0.3', -# 'api_port': 8002, -# "messages": [{"role": "user", "content": random.choice(prompts)}], -# 'logprobs': True -# } -# }) -# print("Chat Completion Output for meta-llama/Meta-Llama-3-8B-Instruct") -# print(chat_out) - -# # Text completion example -# text_out = vllm_inference_function({ -# 'model_params': { -# 'openai_endpoint': 'completions', -# 'model': 'meta-llama/Meta-Llama-3-8B-Instruct', -# 'temperature': 0.2, -# 'max_tokens': 150, -# 'prompt': "List all proteins that interact with RAD51", -# 'logprobs': True -# } -# }) -# print("\nText Completion Output:") -# print(text_out) \ No newline at end of file diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile new file mode 100644 index 00000000..557b17c7 --- /dev/null +++ b/deploy/docker/Dockerfile @@ -0,0 +1,45 @@ +# Use Python 3.12 as base image +FROM python:3.12-slim + +# Set environment variables +ENV PYTHONUNBUFFERED=1 +ENV PYTHONDONTWRITEBYTECODE=1 +ENV DJANGO_SETTINGS_MODULE=inference_gateway.settings + +# Set work directory +WORKDIR /app + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + build-essential \ + libpq-dev \ + && rm -rf /var/lib/apt/lists/* + +# Copy requirements file first for better layer caching +COPY requirements.txt . + +# Install Python dependencies +RUN pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir -r requirements.txt + +# Copy project files +COPY . . + +# Create necessary directories +RUN mkdir -p /app/staticfiles + +# Expose port +EXPOSE 7000 + +# Run the application with Uvicorn worker +CMD ["gunicorn", \ + "inference_gateway.asgi:application", \ + "-k", "uvicorn.workers.UvicornWorker", \ + "-b", "0.0.0.0:7000", \ + "--workers", "5", \ + "--threads", "4", \ + "--timeout", "1800", \ + "--log-level", "debug", \ + "--access-logfile", "-", \ + "--error-logfile", "-", \ + "--capture-output"] diff --git a/deploy/docker/README.md b/deploy/docker/README.md new file mode 100644 index 00000000..a20d18c8 --- /dev/null +++ b/deploy/docker/README.md @@ -0,0 +1,127 @@ +# Docker Deployment + +This directory contains all the necessary files for deploying the FIRST Inference Gateway using Docker. + +## Files + +- `docker-compose.yml` - Docker Compose configuration for multi-container deployment +- `Dockerfile` - Container image definition for the gateway application +- `nginx.conf` - Nginx reverse proxy configuration +- `env.example` - Example environment variables file + +## Prerequisites + +- Docker Engine 20.10+ +- Docker Compose 2.0+ + +## Quick Start + +**Note**: All commands should be run from this directory (`deploy/docker/`) + +1. Copy the example environment file: + ```bash + cp env.example .env + ``` + +2. Edit `.env` with your configuration: + - Set a strong `SECRET_KEY` (generate with: `python -c "from django.core.management.utils import get_random_secret_key; print(get_random_secret_key())"`) + - **For development**: Keep `RUNNING_AUTOMATED_TEST_SUITE=True` to skip Globus policy checks + - **For production**: Set `RUNNING_AUTOMATED_TEST_SUITE=False` and configure: + - Globus credentials (`GLOBUS_APPLICATION_ID`, `GLOBUS_APPLICATION_SECRET`, etc.) + - Globus policies (`GLOBUS_POLICIES`) + - Authorized IDP domains (`AUTHORIZED_IDP_DOMAINS`) + - Set `LOG_TO_STDOUT=True` so application logs are visible via `docker-compose logs` + - Adjust database credentials if needed + - Update `ALLOWED_HOSTS` for your domain + +3. Create required directories (if they don't exist): + ```bash + mkdir -p ../../staticfiles + ``` + +4. Start the services: + ```bash + docker-compose up -d + ``` + +5. Initialize the database: + ```bash + docker-compose exec inference-gateway python manage.py migrate + ``` + +6. Create a superuser (optional): + ```bash + docker-compose exec inference-gateway python manage.py createsuperuser + ``` + +7. Collect static files: + ```bash + docker-compose exec inference-gateway python manage.py collectstatic --noinput + ``` + +## Accessing the Gateway + +- Gateway API: http://localhost:8000 +- Admin panel: http://localhost:8000/admin + +## Managing the Services + +- **View logs**: `docker-compose logs -f` +- **Stop services**: `docker-compose down` +- **Rebuild after code changes**: `docker-compose up -d --build` +- **View running containers**: `docker-compose ps` + +## Troubleshooting + +### Application Not Starting / "Globus High Assurance Policy" Error +If the container exits immediately with no logs, or you see an error about Globus policies: +```bash +docker-compose logs inference-gateway +``` + +**Solution**: Ensure your `.env` file has `RUNNING_AUTOMATED_TEST_SUITE=True` for development, or configure proper Globus credentials for production. + +### Database Connection Issues +If you see database connection errors, ensure PostgreSQL is fully initialized: +```bash +docker-compose logs postgres +``` + +### Permission Issues +If you encounter permission issues with volumes, check that the directories exist: +```bash +ls -la ../../staticfiles +``` + +### View Application Logs +To see detailed application logs: +```bash +# Follow all logs +docker-compose logs -f + +# View only gateway logs +docker-compose logs -f inference-gateway + +# Check if container is running +docker-compose ps +``` + +### Rebuilding from Scratch +To completely reset the deployment: +```bash +docker-compose down -v # Warning: This deletes all data! +docker-compose up -d --build +``` + +## Production Considerations + +For production deployments: +1. Set `DEBUG=False` in `.env` +2. Use a strong, randomly generated `SECRET_KEY` +3. Configure proper `ALLOWED_HOSTS` +4. Set up SSL/TLS termination (use nginx with SSL certificates) +5. Configure proper backup strategies for PostgreSQL data +6. Use Docker secrets for sensitive credentials + +For detailed instructions, see the [Docker Deployment Guide](../../docs/admin-guide/gateway-setup/docker.md). + diff --git a/deploy/docker/docker-compose.yml b/deploy/docker/docker-compose.yml new file mode 100644 index 00000000..f6897746 --- /dev/null +++ b/deploy/docker/docker-compose.yml @@ -0,0 +1,50 @@ +version: '3.8' + +services: + nginx: + image: nginx:latest + ports: + - "8000:80" + volumes: + - ./nginx.conf:/etc/nginx/conf.d/default.conf:ro + - ../../staticfiles:/app/staticfiles:ro + depends_on: + - inference-gateway + networks: + - gateway + + inference-gateway: + build: + context: ../.. + dockerfile: deploy/docker/Dockerfile + env_file: + - .env + volumes: + - ../../staticfiles:/app/staticfiles + depends_on: + - postgres + - redis + networks: + - gateway + + postgres: + image: postgres:15 + container_name: postgres + env_file: + - .env + volumes: + - postgres_data:/var/lib/postgresql/data + networks: + - gateway + + redis: + image: redis:7 + networks: + - gateway + +volumes: + postgres_data: + +networks: + gateway: + driver: bridge \ No newline at end of file diff --git a/deploy/docker/nginx.conf b/deploy/docker/nginx.conf new file mode 100644 index 00000000..2c67d8ed --- /dev/null +++ b/deploy/docker/nginx.conf @@ -0,0 +1,40 @@ +upstream app_server { + # The internal hostname and port of the Gunicorn/Django service + server inference-gateway:7000; +} + +server { + listen 80; + server_name localhost; # Or your specific domain + + # Root location - proxy all other requests to the Django app + location / { + proxy_pass http://app_server; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_redirect off; + + # Increase timeouts for potentially long-running requests (adjust as needed) + proxy_connect_timeout 600s; + proxy_send_timeout 600s; + proxy_read_timeout 600s; + send_timeout 600s; + + # Buffer settings (adjust as needed) + proxy_buffer_size 128k; + proxy_buffers 4 256k; + proxy_busy_buffers_size 256k; + } + + # Serve static files directly from the volume mount + location /static/ { + alias /app/staticfiles/; + expires 30d; # Add caching headers for static files + add_header Cache-Control "public"; + } +} diff --git a/deploy/kubernetes/README.md b/deploy/kubernetes/README.md new file mode 100644 index 00000000..b0d58e77 --- /dev/null +++ b/deploy/kubernetes/README.md @@ -0,0 +1,23 @@ +# Kubernetes Deployment + +## Coming Soon + +Kubernetes deployment manifests and Helm charts for FIRST Inference Gateway will be available here. + +### Planned Components + +- Deployment manifests for the gateway application +- StatefulSet for PostgreSQL +- Service definitions +- Ingress configuration +- ConfigMaps and Secrets management +- Helm chart for easy deployment +- Horizontal Pod Autoscaling configuration +- Resource limits and requests + +### Stay Tuned + +Check back soon or watch the repository for updates on Kubernetes deployment support. + +For now, please use the [Docker deployment](../docker/) option for containerized deployments. + diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 060661c5..00000000 --- a/docker-compose.yml +++ /dev/null @@ -1,110 +0,0 @@ -version: '3.8' - -services: - nginx: - image: nginx:latest - ports: - - "8000:80" - volumes: - - ./nginx_app.conf:/etc/nginx/conf.d/default.conf:ro - - ./staticfiles:/app/staticfiles:ro - depends_on: - - inference-gateway - networks: - - monitoring - - inference-gateway: - build: - context: . - dockerfile: Dockerfile - env_file: - - .env - volumes: - - ./staticfiles:/app/staticfiles - - ./logs:/var/log/inference-service - depends_on: - - postgres - - redis - networks: - - monitoring - - postgres: - image: postgres:15 - container_name: postgres - env_file: - - .env - volumes: - - postgres_data:/var/lib/postgresql/data - networks: - - monitoring - - redis: - image: redis:7 - networks: - - monitoring - - prometheus: - image: prom/prometheus:latest - container_name: prometheus - ports: - - "9090:9090" - volumes: - - ./prometheus:/etc/prometheus - command: - - '--config.file=/etc/prometheus/prometheus.yml' - networks: - - monitoring - - grafana: - image: grafana/grafana:latest - container_name: grafana - ports: - - "3000:3000" - volumes: - - grafana_data:/var/lib/grafana - env_file: - - .env - restart: unless-stopped - depends_on: - - prometheus - networks: - - monitoring - - node-exporter: - image: prom/node-exporter:latest - container_name: node-exporter - ports: - - "9100:9100" - volumes: - - /proc:/host/proc:ro - - /sys:/host/sys:ro - - /:/rootfs:ro - command: - - '--path.procfs=/host/proc' - - '--path.sysfs=/host/sys' - - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)' - restart: unless-stopped - networks: - - monitoring - - postgres-exporter: - image: prometheuscommunity/postgres-exporter:latest - container_name: postgres-exporter - ports: - - "9187:9187" - environment: - - DATA_SOURCE_NAME=postgresql://${PGUSER}:${PGPASSWORD}@postgres:5432/${PGDATABASE}?sslmode=disable - restart: unless-stopped - depends_on: - - postgres - networks: - - monitoring - -volumes: - prometheus_data: - grafana_data: - postgres_data: - -networks: - monitoring: - driver: bridge \ No newline at end of file diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md new file mode 100644 index 00000000..9d2803dc --- /dev/null +++ b/docs/CONTRIBUTING.md @@ -0,0 +1,214 @@ +# Contributing to Documentation + +This guide explains how to contribute to the FIRST Inference Gateway documentation. + +## Documentation Structure + +The documentation is built using [MkDocs](https://www.mkdocs.org/) with the [Material theme](https://squidfunk.github.io/mkdocs-material/). + +``` +docs/ +β”œβ”€β”€ index.md # Homepage +β”œβ”€β”€ admin-guide/ # Administrator documentation +β”‚ β”œβ”€β”€ index.md +β”‚ β”œβ”€β”€ gateway-setup/ +β”‚ β”‚ β”œβ”€β”€ docker.md +β”‚ β”‚ β”œβ”€β”€ bare-metal.md +β”‚ β”‚ └── configuration.md +β”‚ β”œβ”€β”€ inference-setup/ +β”‚ β”‚ β”œβ”€β”€ index.md +β”‚ β”‚ β”œβ”€β”€ direct-api.md +β”‚ β”‚ β”œβ”€β”€ local-vllm.md +β”‚ β”‚ └── globus-compute.md +β”‚ β”œβ”€β”€ deployment/ +β”‚ β”‚ β”œβ”€β”€ kubernetes.md +β”‚ β”‚ └── production.md +β”‚ └── monitoring.md +β”œβ”€β”€ user-guide/ # User documentation +β”‚ β”œβ”€β”€ index.md +β”‚ β”œβ”€β”€ authentication.md +β”‚ β”œβ”€β”€ requests.md +β”‚ β”œβ”€β”€ batch.md +β”‚ └── examples.md +└── reference/ # Reference materials + β”œβ”€β”€ citation.md + β”œβ”€β”€ api.md + └── config.md +``` + +## Local Development + +### Install Dependencies + +```bash +pip install -r requirements-docs.txt +``` + +Or: + +```bash +pip install mkdocs-material mkdocs-minify-plugin +``` + +### Build Documentation + +```bash +mkdocs build +``` + +This creates the `site/` directory with static HTML files. + +### Serve Locally + +```bash +mkdocs serve +``` + +Then visit: http://127.0.0.1:8000 + +The site will automatically reload when you save changes. + +## Writing Documentation + +### Markdown Files + +All documentation is written in Markdown with support for: + +- Standard Markdown syntax +- [Material for MkDocs extensions](https://squidfunk.github.io/mkdocs-material/reference/) +- Code syntax highlighting +- Admonitions (notes, warnings, tips) +- Tables +- Mermaid diagrams + +### Admonitions + +```markdown +!!! note "Optional Title" + This is a note + +!!! warning + This is a warning + +!!! tip + This is a tip + +!!! danger + This is dangerous! +``` + +### Code Blocks + +````markdown +```python +def hello(): + print("Hello, World!") +``` + +```bash +mkdocs serve +``` +```` + +### Mermaid Diagrams + +````markdown +```mermaid +graph LR + A[User] --> B[Gateway] + B --> C[Backend] +``` +```` + +### Internal Links + +```markdown +[Link text](path/to/file.md) +[Link to section](path/to/file.md#section-name) +``` + +## Navigation + +Navigation is configured in `mkdocs.yml`: + +```yaml +nav: + - Home: index.md + - Administrator Guide: + - Overview: admin-guide/index.md + - Gateway Setup: + - Docker: admin-guide/gateway-setup/docker.md +``` + +## Deployment + +### Automatic Deployment + +Documentation automatically deploys to GitHub Pages when you push to `main`: + +1. Make your changes to files in `docs/` +2. Commit and push: + ```bash + git add docs/ + git commit -m "docs: update documentation" + git push origin main + ``` +3. GitHub Actions builds and deploys automatically +4. View at: https://auroragpt-anl.github.io/inference-gateway/ + +### Manual Deployment + +You can also manually trigger deployment: + +1. Go to Actions tab on GitHub +2. Select "Deploy Documentation" workflow +3. Click "Run workflow" + +## Style Guide + +### Headings + +- Use sentence case for headings +- One H1 (`#`) per page (the page title) +- Use hierarchical heading levels (don't skip levels) + +### Code Examples + +- Always include language identifier for syntax highlighting +- Add comments to explain complex code +- Test code examples before committing + +### File Names + +- Use lowercase with hyphens: `my-file.md` +- Be descriptive: `docker-deployment.md` not `docker.md` + +### Writing Style + +- Use clear, concise language +- Write in second person ("you") for instructions +- Use active voice +- Include examples wherever possible +- Add context before diving into details + +## Contributing Process + +1. Fork the repository +2. Create a branch: `git checkout -b docs/my-improvement` +3. Make your changes +4. Test locally: `mkdocs serve` +5. Commit with clear message: `git commit -m "docs: improve docker guide"` +6. Push and create a Pull Request + +## Questions? + +- Open an issue on GitHub +- Check existing documentation +- Review MkDocs Material documentation + +## Resources + +- [MkDocs Documentation](https://www.mkdocs.org/) +- [Material for MkDocs](https://squidfunk.github.io/mkdocs-material/) +- [Markdown Guide](https://www.markdownguide.org/) + diff --git a/docs/admin-guide/deployment/kubernetes.md b/docs/admin-guide/deployment/kubernetes.md new file mode 100644 index 00000000..cb9385e7 --- /dev/null +++ b/docs/admin-guide/deployment/kubernetes.md @@ -0,0 +1,65 @@ +# Kubernetes Deployment + +!!! warning "Coming Soon" + Kubernetes deployment manifests and Helm charts are currently under development. + +## Planned Features + +The Kubernetes deployment will include: + +- **Helm Chart**: Easy installation and configuration management +- **High Availability**: Multi-replica deployments with load balancing +- **Auto-Scaling**: Horizontal Pod Autoscaler based on metrics +- **StatefulSets**: For PostgreSQL and Redis persistence +- **Ingress Configuration**: HTTPS/TLS termination and routing +- **Secrets Management**: Kubernetes secrets for sensitive data +- **ConfigMaps**: Environment-specific configuration +- **Health Probes**: Liveness and readiness checks +- **Resource Limits**: CPU and memory management +- **Monitoring Integration**: Prometheus and Grafana + +## Current Status + +We are actively working on: + +1. Creating Kubernetes manifests for all components +2. Developing a Helm chart for simplified deployment +3. Testing on various Kubernetes distributions (EKS, GKE, OpenShift) +4. Documentation and best practices + +## Alternative: Docker Deployment + +For now, please use one of these deployment methods: + +- [Docker Deployment](../gateway-setup/docker.md) - Containerized deployment with Docker Compose +- [Bare Metal Setup](../gateway-setup/bare-metal.md) - Direct installation on servers + +## Get Notified + +To be notified when Kubernetes support is available: + +- :star: Star the [GitHub repository](https://github.com/auroraGPT-ANL/inference-gateway) +- Watch the repository for releases +- Check the [releases page](https://github.com/auroraGPT-ANL/inference-gateway/releases) + +## Contribute + +Interested in helping with Kubernetes deployment? + +- Check open issues tagged with `kubernetes` +- Submit a pull request +- Share your deployment configurations + +## Contact + +For enterprise Kubernetes deployments or consulting: + +- Open an issue on GitHub +- Contact the development team + +--- + +**Last Updated**: November 2025 + +Check back soon for updates! + diff --git a/docs/admin-guide/deployment/production.md b/docs/admin-guide/deployment/production.md new file mode 100644 index 00000000..df1402a1 --- /dev/null +++ b/docs/admin-guide/deployment/production.md @@ -0,0 +1,535 @@ +# Production Best Practices + +This guide covers best practices for deploying FIRST Inference Gateway in production environments. + +## Security + +### Authentication & Authorization + +#### Globus Group Restrictions + +Restrict access to specific Globus groups: + +```dotenv +GLOBUS_GROUPS="group-uuid-1 group-uuid-2" +``` + +#### Identity Provider Restrictions + +Limit to specific institutions: + +```dotenv +AUTHORIZED_IDPS='{"University Name": "idp-uuid"}' +AUTHORIZED_GROUPS_PER_IDP='{"University Name": "group-uuid-1,group-uuid-2"}' +``` + +#### High Assurance Policies + +Require MFA and other security policies: + +```dotenv +GLOBUS_POLICIES="policy-uuid-1 policy-uuid-2" +``` + +### Secrets Management + +**Never** store secrets in code or version control. + +#### Use Environment Files + +```bash +# .env (add to .gitignore) +SECRET_KEY="..." +POSTGRES_PASSWORD="..." +``` + +#### Docker Secrets + +```yaml +services: + gateway: + secrets: + - db_password + - globus_secret + +secrets: + db_password: + file: ./secrets/db_password.txt + globus_secret: + file: ./secrets/globus_secret.txt +``` + +#### Vault Integration + +For enterprise deployments, integrate with HashiCorp Vault or similar. + +### HTTPS/TLS + +Always use HTTPS in production. + +#### Let's Encrypt with Certbot + +```bash +sudo certbot --nginx -d yourdomain.com +``` + +#### Custom Certificates + +```nginx +server { + listen 443 ssl http2; + ssl_certificate /path/to/cert.pem; + ssl_certificate_key /path/to/key.pem; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers HIGH:!aNULL:!MD5; +} +``` + +### Firewall Configuration + +```bash +# Ubuntu/Debian +sudo ufw allow 80/tcp +sudo ufw allow 443/tcp +sudo ufw deny 8000/tcp # Don't expose Django directly + +# CentOS/RHEL +sudo firewall-cmd --permanent --add-service=http +sudo firewall-cmd --permanent --add-service=https +sudo firewall-cmd --reload +``` + +## Performance + +### Database Optimization + +#### Connection Pooling + +```python +# settings.py +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql', + 'CONN_MAX_AGE': 600, # Persistent connections + 'OPTIONS': { + 'connect_timeout': 10, + } + } +} +``` + +#### Indexes + +Ensure proper indexes on frequently queried fields: + +```python +python manage.py dbshell +CREATE INDEX idx_endpoint_slug ON resource_server_endpoint(endpoint_slug); +CREATE INDEX idx_created_at ON resource_server_listendpointslog(created_at); +``` + +### Caching + +#### Redis Configuration + +```python +# settings.py +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.redis.RedisCache', + 'LOCATION': 'redis://redis:6379/0', + 'OPTIONS': { + 'CLIENT_CLASS': 'django_redis.client.DefaultClient', + 'CONNECTION_POOL_KWARGS': { + 'max_connections': 50 + } + } + } +} +``` + +### Gunicorn Configuration + +#### Worker Calculation + +```python +workers = (2 * CPU_cores) + 1 +``` + +For a 16-core machine: + +```python +workers = (2 * 16) + 1 = 33 +``` + +#### Production Config + +```python +# gunicorn_asgi.config.py +import multiprocessing + +bind = "0.0.0.0:8000" +workers = multiprocessing.cpu_count() * 2 + 1 +worker_class = "uvicorn.workers.UvicornWorker" +worker_connections = 1000 +timeout = 120 +keepalive = 5 +max_requests = 1000 +max_requests_jitter = 50 +``` + +### Nginx Optimization + +```nginx +upstream gateway { + least_conn; # Load balancing algorithm + server 127.0.0.1:8000 max_fails=3 fail_timeout=30s; + server 127.0.0.1:8001 max_fails=3 fail_timeout=30s; + keepalive 64; +} + +server { + listen 443 ssl http2; + + # Gzip compression + gzip on; + gzip_types text/plain text/css application/json application/javascript; + gzip_min_length 1000; + + # Client body size + client_max_body_size 100M; + client_body_buffer_size 1M; + + # Timeouts + proxy_connect_timeout 600s; + proxy_send_timeout 600s; + proxy_read_timeout 600s; + send_timeout 600s; + + # Buffering + proxy_buffering off; # Important for streaming + proxy_request_buffering off; + + # Headers + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + + location /static/ { + alias /path/to/staticfiles/; + expires 30d; + add_header Cache-Control "public, immutable"; + } + + location / { + proxy_pass http://gateway; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } +} +``` + +## Monitoring + +### Application Monitoring + +#### Prometheus Metrics + +Add to `docker-compose.yml`: + +```yaml +services: + prometheus: + image: prom/prometheus + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml + - prometheus_data:/prometheus + ports: + - "9090:9090" +``` + +#### Grafana Dashboards + +```yaml +services: + grafana: + image: grafana/grafana + volumes: + - grafana_data:/var/lib/grafana + ports: + - "3000:3000" + environment: + - GF_SECURITY_ADMIN_PASSWORD=secure_password +``` + +### Log Aggregation + +#### Structured Logging + +```python +# logging_config.py +LOGGING = { + 'version': 1, + 'formatters': { + 'json': { + 'class': 'pythonjsonlogger.jsonlogger.JsonFormatter', + 'format': '%(asctime)s %(name)s %(levelname)s %(message)s' + } + }, + 'handlers': { + 'file': { + 'class': 'logging.handlers.RotatingFileHandler', + 'filename': 'logs/gateway.log', + 'maxBytes': 10485760, # 10MB + 'backupCount': 10, + 'formatter': 'json' + } + } +} +``` + +#### ELK Stack Integration + +For large deployments, consider Elasticsearch + Logstash + Kibana. + +### Health Checks + +#### Kubernetes Probes + +```yaml +livenessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 30 + periodSeconds: 10 + +readinessProbe: + httpGet: + path: /ready + port: 8000 + initialDelaySeconds: 5 + periodSeconds: 5 +``` + +#### Custom Health Endpoint + +Create a health check view in Django to verify database, Redis, and Globus Compute connectivity. + +## Backup & Recovery + +### Database Backups + +#### Automated Backups + +```bash +#!/bin/bash +# backup_db.sh + +DATE=$(date +%Y%m%d_%H%M%S) +BACKUP_DIR="/backups/postgres" +BACKUP_FILE="$BACKUP_DIR/backup_$DATE.sql.gz" + +pg_dump -h localhost -U inferencedev inferencegateway | gzip > $BACKUP_FILE + +# Keep only last 30 days +find $BACKUP_DIR -name "backup_*.sql.gz" -mtime +30 -delete +``` + +Add to crontab: + +```bash +0 2 * * * /path/to/backup_db.sh +``` + +#### Point-in-Time Recovery + +Configure PostgreSQL for WAL archiving: + +```ini +# postgresql.conf +wal_level = replica +archive_mode = on +archive_command = 'cp %p /backup/wal/%f' +``` + +### Configuration Backups + +```bash +# Backup environment and fixtures +tar -czf config_backup_$(date +%Y%m%d).tar.gz \ + .env \ + fixtures/ \ + nginx_app.conf \ + gunicorn_asgi.config.py +``` + +## Scaling + +### Horizontal Scaling + +#### Multiple Gateway Instances + +```nginx +upstream gateway { + server gateway1:8000; + server gateway2:8000; + server gateway3:8000; +} +``` + +#### Session Affinity + +For stateful sessions: + +```nginx +upstream gateway { + ip_hash; + server gateway1:8000; + server gateway2:8000; +} +``` + +### Database Scaling + +#### Read Replicas + +```python +# settings.py +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql', + 'HOST': 'primary.db.internal', + }, + 'replica': { + 'ENGINE': 'django.db.backends.postgresql', + 'HOST': 'replica.db.internal', + } +} + +DATABASE_ROUTERS = ['path.to.ReplicaRouter'] +``` + +#### Connection Pooling (PgBouncer) + +```ini +# pgbouncer.ini +[databases] +inferencegateway = host=localhost port=5432 dbname=inferencegateway + +[pgbouncer] +pool_mode = transaction +max_client_conn = 1000 +default_pool_size = 20 +``` + +### Inference Backend Scaling + +#### Federated Endpoints + +Deploy multiple Globus Compute endpoints and configure federated routing for automatic load balancing. + +#### Auto-Scaling + +Configure Globus Compute endpoints to auto-scale based on demand: + +```yaml +engine: + provider: + min_blocks: 1 + max_blocks: 20 +``` + +## Maintenance + +### Zero-Downtime Deployments + +#### Blue-Green Deployment + +1. Deploy new version alongside old +2. Switch traffic to new version +3. Monitor for issues +4. Decommission old version + +#### Rolling Updates + +```bash +# Update one instance at a time +for server in gateway1 gateway2 gateway3; do + ssh $server "cd /app && git pull && systemctl restart gateway" + sleep 60 # Allow time to stabilize +done +``` + +### Database Migrations + +Always test migrations in staging first: + +```bash +# Backup before migrating +./backup_db.sh + +# Run migration +python manage.py migrate + +# If issues occur, restore backup +psql -h localhost -U inferencedev inferencegateway < backup.sql +``` + +## Disaster Recovery + +### Disaster Recovery Plan + +1. **Recovery Time Objective (RTO)**: 2 hours +2. **Recovery Point Objective (RPO)**: 1 hour + +### Backup Strategy + +- **Hourly**: Database transaction logs +- **Daily**: Full database backup +- **Weekly**: Complete system backup (config, logs, data) +- **Monthly**: Archived to off-site storage + +### Failover Procedures + +Document step-by-step procedures for: + +1. Gateway failure β†’ Switch to backup gateway +2. Database failure β†’ Promote read replica +3. Complete site failure β†’ Activate DR site + +## Checklist + +### Pre-Production + +- [ ] All secrets are externalized +- [ ] HTTPS/TLS configured +- [ ] Firewall rules applied +- [ ] DEBUG=False +- [ ] Strong passwords set +- [ ] Database backed up +- [ ] Monitoring configured +- [ ] Log aggregation set up +- [ ] Health checks working +- [ ] Load testing completed +- [ ] Disaster recovery plan documented + +### Post-Deployment + +- [ ] Monitor logs for errors +- [ ] Verify all endpoints responding +- [ ] Check database performance +- [ ] Test authentication flow +- [ ] Verify Globus Compute connectivity +- [ ] Run integration tests +- [ ] Document any issues + +## Additional Resources + +- [Django Security Best Practices](https://docs.djangoproject.com/en/stable/topics/security/) +- [Nginx Performance Tuning](https://www.nginx.com/blog/tuning-nginx/) +- [PostgreSQL Performance Tips](https://wiki.postgresql.org/wiki/Performance_Optimization) +- [Monitoring Guide](../monitoring.md) + diff --git a/docs/admin-guide/gateway-setup/bare-metal.md b/docs/admin-guide/gateway-setup/bare-metal.md new file mode 100644 index 00000000..30d5a0de --- /dev/null +++ b/docs/admin-guide/gateway-setup/bare-metal.md @@ -0,0 +1,414 @@ +# Bare Metal Setup + +This guide covers installing the FIRST Inference Gateway directly on your server without Docker. + +## Prerequisites + +- Linux server (Ubuntu 20.04+, CentOS 8+, or similar) +- Python 3.12 or later +- PostgreSQL 13 or later +- Redis 6 or later +- Poetry (Python dependency manager) +- Sudo access for system packages +- At least 4GB RAM + +## Step 1: Install System Dependencies + +### Ubuntu/Debian + +```bash +sudo apt update +sudo apt install -y python3.12 python3.12-dev python3.12-venv \ + postgresql postgresql-contrib redis-server \ + build-essential libpq-dev git curl +``` + +### CentOS/RHEL + +```bash +sudo dnf install -y python3.12 python3.12-devel \ + postgresql postgresql-server redis \ + gcc gcc-c++ make libpq-devel git +``` + +## Step 2: Install Poetry + +```bash +curl -sSL https://install.python-poetry.org | python3 - + +# Add to PATH +export PATH="$HOME/.local/bin:$PATH" + +# Verify installation +poetry --version +``` + +## Step 3: Clone and Setup Project + +```bash +git clone https://github.com/auroraGPT-ANL/inference-gateway.git +cd inference-gateway + +# Configure Poetry to create venv in project +poetry config virtualenvs.in-project true + +# Set Python version +poetry env use python3.12 + +# Install dependencies +poetry install + +# Activate environment +poetry shell +``` + +## Step 4: Configure PostgreSQL + +### Initialize PostgreSQL (if first time) + +```bash +# Ubuntu/Debian (usually auto-initialized) +sudo systemctl start postgresql +sudo systemctl enable postgresql + +# CentOS/RHEL +sudo postgresql-setup --initdb +sudo systemctl start postgresql +sudo systemctl enable postgresql +``` + +### Create Database and User + +```bash +sudo -u postgres psql + +# In PostgreSQL shell: +CREATE DATABASE inferencegateway; +CREATE USER inferencedev WITH PASSWORD 'your-secure-password'; +ALTER ROLE inferencedev SET client_encoding TO 'utf8'; +ALTER ROLE inferencedev SET default_transaction_isolation TO 'read committed'; +ALTER ROLE inferencedev SET timezone TO 'UTC'; +GRANT ALL PRIVILEGES ON DATABASE inferencegateway TO inferencedev; +\q +``` + +### Configure PostgreSQL Authentication + +Edit `/etc/postgresql/*/main/pg_hba.conf` (path may vary): + +``` +# Add this line (adjust for your security needs) +host inferencegateway inferencedev 127.0.0.1/32 md5 +``` + +Restart PostgreSQL: + +```bash +sudo systemctl restart postgresql +``` + +## Step 5: Configure Redis + +Start and enable Redis: + +```bash +sudo systemctl start redis +sudo systemctl enable redis + +# Verify it's running +redis-cli ping +# Should return: PONG +``` + +## Step 6: Configure Environment + +Create a `.env` file from the [example environment file](../../../env.example) and customize the `.env` file following the instructions found in the example file: +```bash +cp env.example .env +``` + +Make sure you include all of the Globus UUIDs and secrets generated during the [Globus setup](../globus-setup/index.md) stage. You can generate the `SECRET_KEY` variable with the following Django command (if installed): +```bash +python -c 'from django.core.management.utils import get_random_secret_key; print(get_random_secret_key())' +``` + +!!! warning "Production Security" + For production deployments: + + - Set `RUNNING_AUTOMATED_TEST_SUITE=False` + - Set `DEBUG=False` + - Use secure passwords and secrets + - Add your domain to `ALLOWED_HOSTS` or use "*" if appropriate + - Add at least one Globus High Assurance policy (`GLOBUS_POLICIES`) + - Set authorized IDP domains (`AUTHORIZED_IDP_DOMAINS`) to match the policy + +## Step 7: Initialize Database + +```bash +# Make sure you're in the poetry shell +poetry shell + +# Run migrations +python manage.py makemigrations +python manage.py migrate + +# Create superuser +python manage.py createsuperuser + +# Collect static files +python manage.py collectstatic --noinput +``` + +## Step 8: Test the Gateway + +Run development server: + +```bash +python manage.py runserver 0.0.0.0:8000 +``` + +Test in another terminal: + +```bash +curl http://localhost:8000/ +``` + +## Step 9: Setup Production Server (Gunicorn) + +### Install Gunicorn (already included in poetry dependencies) + +Create a systemd service file: + +```bash +sudo nano /etc/systemd/system/inference-gateway.service +``` + +Add the following: + +```ini +[Unit] +Description=FIRST Inference Gateway +After=network.target postgresql.service redis.service + +[Service] +Type=notify +User=your-username +Group=your-username +WorkingDirectory=/path/to/inference-gateway +Environment="PATH=/path/to/inference-gateway/.venv/bin" +EnvironmentFile=/path/to/inference-gateway/.env +ExecStart=/path/to/inference-gateway/.venv/bin/gunicorn \ + inference_gateway.asgi:application \ + -k uvicorn.workers.UvicornWorker \ + -b 0.0.0.0:8000 \ + --workers 4 \ + --log-level info \ + --access-logfile /path/to/inference-gateway/logs/access.log \ + --error-logfile /path/to/inference-gateway/logs/error.log + +[Install] +WantedBy=multi-user.target +``` + +### Start and Enable Service + +```bash +# Create logs directory +mkdir -p logs + +# Reload systemd +sudo systemctl daemon-reload + +# Start service +sudo systemctl start inference-gateway + +# Enable on boot +sudo systemctl enable inference-gateway + +# Check status +sudo systemctl status inference-gateway +``` + +## Step 10: Setup Nginx (Recommended) + +### Install Nginx + +```bash +# Ubuntu/Debian +sudo apt install nginx + +# CentOS/RHEL +sudo dnf install nginx +``` + +### Configure Nginx + +Create site configuration: + +```bash +sudo nano /etc/nginx/sites-available/inference-gateway +``` + +Add the following: + +```nginx +upstream inference_gateway { + server 127.0.0.1:8000 fail_timeout=0; +} + +server { + listen 80; + server_name your-domain.com; + client_max_body_size 100M; + + location /static/ { + alias /path/to/inference-gateway/staticfiles/; + } + + location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + proxy_redirect off; + proxy_buffering off; + proxy_pass http://inference_gateway; + } +} +``` + +Enable the site: + +```bash +# Ubuntu/Debian +sudo ln -s /etc/nginx/sites-available/inference-gateway /etc/nginx/sites-enabled/ +sudo nginx -t +sudo systemctl restart nginx + +# CentOS/RHEL +sudo ln -s /etc/nginx/sites-available/inference-gateway /etc/nginx/conf.d/ +sudo nginx -t +sudo systemctl restart nginx +``` + +### Setup SSL with Let's Encrypt + +```bash +# Ubuntu/Debian +sudo apt install certbot python3-certbot-nginx + +# CentOS/RHEL +sudo dnf install certbot python3-certbot-nginx + +# Get certificate +sudo certbot --nginx -d your-domain.com +``` + +## Step 11: Configure Firewall + +```bash +# Ubuntu/Debian (UFW) +sudo ufw allow 80/tcp +sudo ufw allow 443/tcp +sudo ufw enable + +# CentOS/RHEL (firewalld) +sudo firewall-cmd --permanent --add-service=http +sudo firewall-cmd --permanent --add-service=https +sudo firewall-cmd --reload +``` + +## Maintenance + +### View Logs + +```bash +# Application logs +tail -f logs/error.log +tail -f logs/access.log + +# System service logs +sudo journalctl -u inference-gateway -f +``` + +### Restart Service + +```bash +sudo systemctl restart inference-gateway +``` + +### Update Application + +```bash +cd /path/to/inference-gateway +git pull origin main +poetry install +python manage.py migrate +python manage.py collectstatic --noinput +sudo systemctl restart inference-gateway +``` + +## Troubleshooting + +### Service won't start + +Check logs: + +```bash +sudo journalctl -u inference-gateway -n 50 +``` + +Check configuration: + +```bash +poetry shell +python manage.py check +``` + +### Database connection errors + +Verify PostgreSQL is running: + +```bash +sudo systemctl status postgresql +``` + +Test connection: + +```bash +psql -h localhost -U inferencedev -d inferencegateway +``` + +### Permission errors + +Ensure the service user owns the files: + +```bash +sudo chown -R your-username:your-username /path/to/inference-gateway +``` + +### Nginx errors + +Check nginx error log: + +```bash +sudo tail -f /var/log/nginx/error.log +``` + +Test configuration: + +```bash +sudo nginx -t +``` + +## Next Steps + +- [Configure Inference Backends](../inference-setup/index.md) +- [Production Best Practices](../deployment/production.md) +- [Monitoring Setup](../monitoring.md) + +## Additional Resources + +- [Configuration Reference](configuration.md) +- [Gunicorn Documentation](https://docs.gunicorn.org/) +- [Nginx Documentation](https://nginx.org/en/docs/) + diff --git a/docs/admin-guide/gateway-setup/configuration.md b/docs/admin-guide/gateway-setup/configuration.md new file mode 100644 index 00000000..da25e694 --- /dev/null +++ b/docs/admin-guide/gateway-setup/configuration.md @@ -0,0 +1,403 @@ +# Configuration Reference + +This page documents all environment variables and configuration options for the FIRST Inference Gateway. + +## Environment Variables + +All configuration is done through environment variables, typically stored in a `.env` file. + +### Core Django Settings + +| Variable | Required | Default | Description | +|----------|----------|---------|-------------| +| `SECRET_KEY` | Yes | - | Django secret key for cryptographic signing | +| `DEBUG` | No | `False` | Enable debug mode (never use in production) | +| `ALLOWED_HOSTS` | Yes | - | Comma-separated list of allowed hostnames | +| `RUNNING_AUTOMATED_TEST_SUITE` | No | `False` | Set to `True` to skip Globus High Assurance policy checks (development/testing only) | +| `LOG_TO_STDOUT` | No | `False` | Set to `True` to output logs to stdout (useful for Docker) | + +!!! danger "Security Warning" + - Never use `DEBUG=True` in production! This exposes sensitive information. + - Never use `RUNNING_AUTOMATED_TEST_SUITE=True` in production! This disables important security checks. + +### Globus Authentication + +| Variable | Required | Default | Description | +|----------|----------|---------|-------------| +| `GLOBUS_APPLICATION_ID` | Yes | - | Service API application client UUID | +| `GLOBUS_APPLICATION_SECRET` | Yes | - | Service API application client secret | +| `SERVICE_ACCOUNT_ID` | Yes | - | Service Account application client UUID | +| `SERVICE_ACCOUNT_SECRET` | Yes | - | Service Account application client secret | +| `GLOBUS_GROUPS` | No | - | Space-separated UUIDs of allowed Globus groups | +| `AUTHORIZED_IDPS` | No | - | JSON string of authorized identity providers | +| `AUTHORIZED_GROUPS_PER_IDP` | No | - | JSON string of groups per IDP | +| `GLOBUS_POLICIES` | No | - | Space-separated policy UUIDs | + +Example with group restrictions: + +```dotenv +GLOBUS_GROUPS="uuid-1 uuid-2 uuid-3" +``` + +Example with IDP restrictions: + +```dotenv +AUTHORIZED_IDPS='{"University": "uuid-here"}' +AUTHORIZED_GROUPS_PER_IDP='{"University": "group-uuid-1,group-uuid-2"}' +``` + +### Database Configuration + +| Variable | Required | Default | Description | +|----------|----------|---------|-------------| +| `POSTGRES_DB` | Yes | - | Database name | +| `POSTGRES_USER` | Yes | - | Database user | +| `POSTGRES_PASSWORD` | Yes | - | Database password | +| `PGHOST` | Yes | - | Database host (`postgres` for Docker, `localhost` for bare metal) | +| `PGPORT` | No | `5432` | Database port | +| `PGUSER` | Yes | - | Database user (can be same as POSTGRES_USER) | +| `PGPASSWORD` | Yes | - | Database password | +| `PGDATABASE` | Yes | - | Database name | + +!!! tip "Docker Networking" + When using Docker Compose, set `PGHOST=postgres` to use the container name. + For bare metal, use `PGHOST=localhost` or the actual hostname. + +### Redis Configuration + +| Variable | Required | Default | Description | +|----------|----------|---------|-------------| +| `REDIS_URL` | Yes | - | Redis connection URL | + +Examples: + +```dotenv +# Docker +REDIS_URL="redis://redis:6379/0" + +# Bare metal +REDIS_URL="redis://localhost:6379/0" + +# With password +REDIS_URL="redis://:password@localhost:6379/0" +``` + +### Gateway Settings + +| Variable | Required | Default | Description | +|----------|----------|---------|-------------| +| `MAX_BATCHES_PER_USER` | No | `2` | Maximum concurrent batch jobs per user | +| `STREAMING_SERVER_HOST` | No | - | Internal streaming server host:port | +| `INTERNAL_STREAMING_SECRET` | No | - | Secret for internal streaming authentication | + +### CLI Authentication Helper + +| Variable | Required | Default | Description | +|----------|----------|---------|-------------| +| `CLI_ALLOWED_DOMAINS` | No | - | Comma-separated domains for CLI login | +| `CLI_TOKEN_DIR` | No | `~/.globus/app` | Token storage directory | +| `CLI_APP_NAME` | No | `inference_auth` | App name for token storage | + +Example: + +```dotenv +CLI_ALLOWED_DOMAINS="anl.gov,alcf.anl.gov,university.edu" +``` + +### Metis (Direct API) Configuration + +For direct OpenAI-compatible API connections: + +| Variable | Required | Default | Description | +|----------|----------|---------|-------------| +| `METIS_STATUS_URL` | No | - | URL to status manifest JSON | +| `METIS_API_TOKENS` | No | - | JSON object of endpoint_id -> API token | + +Example: + +```dotenv +METIS_STATUS_URL="https://example.com/status.json" +METIS_API_TOKENS='{"openai-prod": "sk-...", "anthropic-prod": "sk-ant-..."}' +``` + +### Monitoring (Optional) + +| Variable | Required | Default | Description | +|----------|----------|---------|-------------| +| `GF_SECURITY_ADMIN_USER` | No | `admin` | Grafana admin username | +| `GF_SECURITY_ADMIN_PASSWORD` | No | `admin` | Grafana admin password | + +### Qstat Endpoints (Optional) + +For HPC cluster status monitoring: + +| Variable | Required | Default | Description | +|----------|----------|---------|-------------| +| `SOPHIA_QSTAT_ENDPOINT_UUID` | No | - | Endpoint UUID for qstat function | +| `SOPHIA_QSTAT_FUNCTION_UUID` | No | - | Function UUID for qstat | + +## Fixture Configuration + +Fixtures are JSON files that define available endpoints and models. + +### Endpoint Fixture Format + +Located at `fixtures/endpoints.json`: + +```json +[ + { + "model": "resource_server.endpoint", + "pk": 1, + "fields": { + "endpoint_slug": "local-vllm-opt-125m", + "cluster": "local", + "framework": "vllm", + "model": "facebook/opt-125m", + "api_port": 8001, + "endpoint_uuid": "", + "function_uuid": "", + "batch_endpoint_uuid": "", + "batch_function_uuid": "", + "allowed_globus_groups": "" + } + } +] +``` + +### Federated Endpoint Fixture Format + +Located at `fixtures/federated_endpoints.json`: + +```json +[ + { + "model": "resource_server.federatedendpoint", + "pk": 1, + "fields": { + "name": "OPT 125M (Federated)", + "slug": "federated-opt-125m", + "target_model_name": "facebook/opt-125m", + "description": "Federated access point", + "targets": [ + { + "cluster": "local", + "framework": "vllm", + "model": "facebook/opt-125m", + "endpoint_slug": "local-vllm-opt-125m", + "endpoint_uuid": "", + "function_uuid": "", + "api_port": 8001 + } + ] + } + } +] +``` + +## Django Settings + +Advanced settings in `inference_gateway/settings.py`: + +### CORS Settings + +```python +CORS_ALLOW_ALL_ORIGINS = False # Set True for development only +CORS_ALLOWED_ORIGINS = [ + "https://yourdomain.com", +] +``` + +### Logging Configuration + +Located in `logging_config.py`: + +```python +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'handlers': { + 'file': { + 'level': 'INFO', + 'class': 'logging.handlers.RotatingFileHandler', + 'filename': 'logs/django_info.log', + 'maxBytes': 1024 * 1024 * 15, # 15MB + 'backupCount': 10, + }, + }, + # ... more configuration +} +``` + +### Cache Configuration + +```python +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.redis.RedisCache', + 'LOCATION': os.environ.get('REDIS_URL'), + } +} +``` + +## Gunicorn Configuration + +For production deployments, configure Gunicorn in `gunicorn_asgi.config.py`: + +```python +bind = "0.0.0.0:8000" +workers = 4 # Adjust based on CPU cores +worker_class = "uvicorn.workers.UvicornWorker" +timeout = 120 +keepalive = 5 +``` + +### Worker Calculation + +Recommended formula: + +``` +workers = (2 * CPU_cores) + 1 +``` + +For a 4-core machine: +``` +workers = (2 * 4) + 1 = 9 +``` + +## Nginx Configuration + +Example production configuration: + +```nginx +upstream inference_gateway { + server 127.0.0.1:8000 fail_timeout=0; +} + +server { + listen 443 ssl http2; + server_name yourdomain.com; + + # SSL Configuration + ssl_certificate /etc/letsencrypt/live/yourdomain.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/yourdomain.com/privkey.pem; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers HIGH:!aNULL:!MD5; + + # File upload size limit + client_max_body_size 100M; + + # Timeouts + proxy_connect_timeout 600s; + proxy_send_timeout 600s; + proxy_read_timeout 600s; + + location /static/ { + alias /path/to/staticfiles/; + expires 30d; + add_header Cache-Control "public, immutable"; + } + + location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + proxy_redirect off; + proxy_buffering off; + proxy_pass http://inference_gateway; + } +} + +# Redirect HTTP to HTTPS +server { + listen 80; + server_name yourdomain.com; + return 301 https://$server_name$request_uri; +} +``` + +## Environment-Specific Configurations + +### Development + +```dotenv +DEBUG=True +ALLOWED_HOSTS="localhost,127.0.0.1" +SECRET_KEY="dev-secret-key-not-secure" +PGHOST="localhost" +REDIS_URL="redis://localhost:6379/0" +``` + +### Staging + +```dotenv +DEBUG=False +ALLOWED_HOSTS="staging.yourdomain.com" +SECRET_KEY="" +PGHOST="postgres-staging.internal" +REDIS_URL="redis://redis-staging.internal:6379/0" +``` + +### Production + +```dotenv +DEBUG=False +ALLOWED_HOSTS="yourdomain.com,api.yourdomain.com" +SECRET_KEY="" +PGHOST="postgres-prod.internal" +REDIS_URL="redis://redis-prod.internal:6379/0" + +# Enable security features +SECURE_SSL_REDIRECT=True +SESSION_COOKIE_SECURE=True +CSRF_COOKIE_SECURE=True +``` + +## Secrets Management + +### Using Docker Secrets + +```yaml +services: + inference-gateway: + secrets: + - globus_secret + - db_password + +secrets: + globus_secret: + file: ./secrets/globus_secret.txt + db_password: + file: ./secrets/db_password.txt +``` + +### Using Environment Files + +```bash +# .env.local (gitignored) +source .env.production +export POSTGRES_PASSWORD="" +export GLOBUS_APPLICATION_SECRET="" +``` + +## Validation + +Check your configuration: + +```bash +# Django check +python manage.py check + +# Database connectivity +python manage.py dbshell + +# Show current settings (dev only!) +python manage.py diffsettings +``` + +## Next Steps + +- [Docker Deployment](docker.md) +- [Bare Metal Setup](bare-metal.md) +- [Production Best Practices](../deployment/production.md) + diff --git a/docs/admin-guide/gateway-setup/docker.md b/docs/admin-guide/gateway-setup/docker.md new file mode 100644 index 00000000..78c55c35 --- /dev/null +++ b/docs/admin-guide/gateway-setup/docker.md @@ -0,0 +1,209 @@ +# Docker Deployment + +This guide shows you how to deploy the FIRST Inference Gateway using Docker and Docker Compose. + +## Prerequisites + +- Docker Desktop 4.29+ (or Docker Engine 24+) with Docker Compose v2 +- Git +- Globus Account and registered applications +- At least 4GB RAM available for containers + +## Step 1: Clone the Repository + +```bash +git clone https://github.com/auroraGPT-ANL/inference-gateway.git +cd inference-gateway +``` + +## Step 2: Configure Environment + +Create a `.env` file from the [example environment file](../../../env.example) and customize the `.env` file following the instructions found in the example file: +```bash +cp env.example .env +``` + +Make sure you include all of the Globus UUIDs and secrets generated during the [Globus setup](../globus-setup/index.md) stage. You can generate the `SECRET_KEY` variable with the following Django command (if installed): +```bash +python -c 'from django.core.management.utils import get_random_secret_key; print(get_random_secret_key())' +``` + +!!! warning "Production Security" + For production deployments: + + - Set `RUNNING_AUTOMATED_TEST_SUITE=False` + - Set `DEBUG=False` + - Use secure passwords and secrets + - Add your domain to `ALLOWED_HOSTS` or use "*" if appropriate + - Add at least one Globus High Assurance policy (`GLOBUS_POLICIES`) + - Set authorized IDP domains (`AUTHORIZED_IDP_DOMAINS`) to match the policy + - Consider using secrets management (e.g., Docker secrets) + +## Step 3: Start the Services +```bash +cd deploy/docker +docker-compose up -d --build +``` + +The `docker-compose.yml` includes: + +### Core Services + +- **inference-gateway**: Django API application (internal port 8000) +- **postgres**: PostgreSQL 15 database (internal port 5432) +- **redis**: Redis 7 cache (internal port 6379) +- **nginx**: Reverse proxy (internal port 80 exposed to localhost port 8000) + +### Optional Services + +You can add these to your compose file: + +- **prometheus**: Metrics collection +- **grafana**: Visualization dashboard + +Verify that the core-service containers are running: +```bash +docker-compose ps +``` + +## Step 4: Initialize the Database + +Run migrations: + +```bash +docker-compose exec inference-gateway python manage.py makemigrations +docker-compose exec inference-gateway python manage.py migrate +``` + +## Step 5: Test the Gateway + +Check that the gateway is running: +```bash +curl http://localhost:8000/resource_server/whoami +``` + +If everything is running, the command should give you the following error: +```bash +Missing ('Authorization': 'Bearer ') in request headers. +``` + +## Common Commands + +### View logs + +```bash +# All services +docker-compose logs -f + +# Specific service +docker-compose logs -f inference-gateway +``` + +### Restart services + +```bash +# All services +docker-compose restart + +# Specific service +docker-compose restart inference-gateway +``` + +### Stop services + +```bash +docker-compose down +``` + +### Stop and remove volumes (clean slate) + +```bash +docker-compose down -v +``` + +### Access container shell + +```bash +docker-compose exec inference-gateway /bin/bash +``` + +### Run Django management commands + +```bash +docker-compose exec inference-gateway python manage.py +``` + +## Updating the Deployment + +Pull latest changes: + +```bash +git pull origin main +docker-compose build +docker-compose up -d +docker-compose exec inference-gateway python manage.py migrate +``` + +## Troubleshooting + +### Gateway won't start + +Check logs: + +```bash +docker-compose logs inference-gateway +``` + +Common issues: + +- Missing environment variables +- Database connection failed +- Port 8000 already in use + +### Database connection errors + +Verify PostgreSQL is running: + +```bash +docker-compose ps postgres +``` + +Check database logs: + +```bash +docker-compose logs postgres +``` + +### Can't access admin panel + +Ensure you created a superuser: + +```bash +docker-compose exec inference-gateway python manage.py createsuperuser +``` + +### 502 Bad Gateway from Nginx + +Vefiry that the gateway container is running: + +```bash +docker-compose ps inference-gateway +``` + +Verify nginx configuration: + +```bash +docker-compose exec nginx nginx -t +``` + +## Next Steps + +- [Configure Inference Backends](../inference-setup/index.md) +- [Production Best Practices](../deployment/production.md) +- [Monitoring Setup](../monitoring.md) + +## Additional Resources + +- [Docker Compose Documentation](https://docs.docker.com/compose/) +- [Configuration Reference](configuration.md) +- [User Guide](../../user-guide/index.md) diff --git a/docs/admin-guide/globus-setup/index.md b/docs/admin-guide/globus-setup/index.md new file mode 100644 index 00000000..1f5e858f --- /dev/null +++ b/docs/admin-guide/globus-setup/index.md @@ -0,0 +1,155 @@ +# Globus Setup + +This guide covers creating a Globus project, registering Globus Applications, and optionally creating Globus groups and policies to control access to the service. **Securely store all of your UUIDs and secrets along the way**, you will need them at later stages. + +## Step 1: Create Globus Project + +A Globus project will store all of the required applications and policies. To create one: + +1. Visit [https://app.globus.org/settings/developers](https://app.globus.org/settings/developers) +2. Click on **Add Project** (should be on the top-right corner) +3. Fill the form (e.g. Project Name: Inference) + - Set **Project Name**: e.g., Inference + - Set **Contact Email**: Your email + - Set **Project Admins**: Add at least one of your Globus identity + - Click on **Submit** + +## Step 2: Register Service API Application + +This application is at the core of the API's authorization layer. It communicates with the [Globus Auth](https://www.globus.org/globus-auth-service) service to introspect incoming access tokens, and extracts the list of Globus Group memberships of each user. To register the application: + +1. Visit [https://app.globus.org/settings/developers](https://app.globus.org/settings/developers) +2. Click on **Register a service API ...** +3. Select your Globus project +4. Fill the form: + - Set **App Name**: e.g., Inference Service API + - Set **Redirect URIs**: `http://localhost:8000` + - You can use the default value for all other fields + - Click on **Register App** +5. You should now be able to see your application details, including your **Client UUID** +6. Click on **Add Client Secret** + - Add description (e.g., "inference token introspection") + - Click on **Generate Secret** + +### Add Scope to Service API Application + +To allow your Service API application to introspect incoming access tokens, you need a Globus scope that is specifically tied to your inference service. First, export your Service API client credentials into environment variables: +```bash +CLIENT_ID="" +CLIENT_SECRET="" +``` + +Define your scope name (e.g., My Inference Scope) and description (e.g., Access to my inference service): +```bash +SCOPE_NAME="" +SCOPE_DESCRIPTION="" +``` + +Execute the following command to create an `action_all` scope to your client. Make sure you keep the `73320ffe-4cb4-4b25-a0a3-83d53d59ce4f` dependent scope, which will allow your Service API client to access the Globus Group memberships of your users from their access tokens. +```bash +curl -X POST -s --user $CLIENT_ID:$CLIENT_SECRET \ + https://auth.globus.org/v2/api/clients/$CLIENT_ID/scopes \ + -H "Content-Type: application/json" \ + -d '{ + "scope": { + "name": $SCOPE_NAME, + "description": $SCOPE_DESCRIPTION, + "scope_suffix": "action_all", + "dependent_scopes": [ + { + "scope": "73320ffe-4cb4-4b25-a0a3-83d53d59ce4f", + "optional": false, + "requires_refresh_token": true + } + ] + } + }' +``` + +Verify that the scope was properly created (look for the `scopes` field in the response): +```bash +curl -s --user $CLIENT_ID:$CLIENT_SECRET https://auth.globus.org/v2/api/clients/$CLIENT_ID +``` + +Look at the details of your scope: +```bash +SCOPE_ID="" +curl -s --user $CLIENT_ID:$CLIENT_SECRET \ + https://auth.globus.org/v2/api/clients/$CLIENT_ID/scopes/$SCOPE_ID +``` + +Store your scope UUID, you will need it in a later stage. + +## Step 3: Register Service Account Application + +To handle the communication between the Gateway API and the compute resources (the Inference Backend), you need to create a Globus **Service Account application**. This application represents the Globus identity that will own the [Globus Compute](https://www.globus.org/compute) endpoints. + +1. Visit [https://app.globus.org/settings/developers](https://app.globus.org/settings/developers) +2. Click on **Register a service account ...** +3. Select your Globus project +4. Fill the form: + - Set **App Name**: e.g., My Inference Endpoints + - Set **Privacy Policy** and **Terms & Conditions** URLs if applicable. + - Click on **Register App** +5. You should now be able to see your application details, including your **Client UUID** +6. Click on **Add Client Secret** + - Add description (e.g., "inference compute endpoints") + - Click on **Generate Secret** + +## Step 4: Register Public Auth Client + +To provide users with an easy mechanism to get access tokens for the inference service, you can create a public client that will handle all communications with Globus to authenticate users: + +1. Visit [https://app.globus.org/settings/developers](https://app.globus.org/settings/developers) +2. Click on **Register a thick client ...** +3. Select your Globus project +4. Fill the form: + - Set **App Name**: e.g., Public User Auth Client + - Set **Redirect URIs**: `https://auth.globus.org/v2/web/auth-code` + - You can use the default value for all other fields + - Click on **Register App** +5. You should now be able to see your application details, including your **Client UUID**. This type of client has no secret. + +## [Optional] Step 5: Create a Globus High-Assurance Policy + +If you want to restrict access to your service based on institution domains, you can enforce a [Globus High Assurance](https://docs.globus.org/guides/overviews/security/high-assurance-overview/#user_authentication_and_access) Policy. This should be used if you only want to authorize specific identity providers (e.g., alcf.anl.gov, your-university.edu, etc.). To create such a policy: + +1. Visit [https://app.globus.org/settings/developers](https://app.globus.org/settings/developers) +2. Click on your Globus project +3. Click on the **Policies** tab +4. Click on **Add a Policy** +5. Fill the form: + - Set **Display Name**: e.g., My Inference High Assurance Policy + - Set **Description**: e.g., My policy to restrict access + - [IMPORTANT] Check the **High Assurance** check box + - Add list of authorized domains in **Included Domains** (one per line) + - Click on **Create Policy** +6. You should now be able to see your policy details, including your **Policy ID**. + +## [Optional] Step 6: Create Globus Groups + +If you want to further restrict access down to specific users (i.e., specific Globus identities), or if you want to implement role-base access within your inference service, you can create [Globus Groups](https://docs.globus.org/guides/tutorials/manage-identities/manage-groups/). To do so: + +1. Visit [https://app.globus.org/groups](https://app.globus.org/groups) +2. Click on **Create new group** (should be on the top-right corner) +3. Fill the form: + - Set **Group Name** and **Description** + - Make sure you set the group visibility to your needs + - Click on **Create Group** +4. You should now be able to see your group overview details, including your **Group UUID**. + +To add members to a specific group: +1. Visit [https://app.globus.org/groups](https://app.globus.org/groups) +2. Click on the targetted group +3. Click on **Members** tab +4. Click on **Invite Others** +5. Search for the Globus identity (identity UUID, email, or username) +6. Select appropriate role (typically **Member** for service users) +7. Click on **Send invitation** to send an invitation email to the user + +Alternatively, if you have the Globus CLI installed, you can add users directly without sending them an invitation email: +```bash +GROUP_ID="" +USER_ID="" +globus group member add $GROUP_ID $USER_ID +``` \ No newline at end of file diff --git a/docs/admin-guide/index.md b/docs/admin-guide/index.md new file mode 100644 index 00000000..1e2663f5 --- /dev/null +++ b/docs/admin-guide/index.md @@ -0,0 +1,174 @@ +# Administrator Guide + +Welcome to the FIRST Inference Gateway Administrator Guide. This guide will help you deploy and configure the gateway for your organization. + +## Overview + +Setting up FIRST involves two main components: + +1. **Gateway Setup**: The central API service that handles authentication and routing +2. **Inference Backend Setup**: The actual inference servers where models run + +These can be deployed independently and connected together through configuration. + +## Prerequisites + +Before you begin, ensure you have: + +- Python 3.12 or later +- Docker and Docker Compose (for Docker deployment) +- PostgreSQL Server (or use Docker) +- Globus Account +- Access to compute resources (for inference backends) + +## Globus Applications + +Globus applications are required to operate the service and manage the authentication and authorization layer. + +[Globus Guide β†’](globus-setup/index.md) + +## Deployment Architecture + +Choose your deployment approach: + +### Gateway Deployment Options + +**Docker Deployment (Recommended)** + +- Quick setup with Docker Compose +- **Pros**: Easy to deploy, includes all dependencies, portable +- **Cons**: Requires Docker knowledge +- **[Docker Guide β†’](gateway-setup/docker.md)** + +**Bare Metal Deployment** + +- Direct installation on your server infrastructure +- **Pros**: More control, better performance, easier debugging +- **Cons**: Manual dependency management +- **[Bare Metal Guide β†’](gateway-setup/bare-metal.md)** + +### Inference Backend Options + +**Globus Compute + vLLM** _(Recommended for Production)_ + +- Deploy vLLM on HPC clusters with Globus Compute for remote execution +- **Best for**: Multi-cluster, federated deployments, HPC environments +- **[Globus Compute Setup β†’](inference-setup/globus-compute.md)** + +**Local vLLM** + +- Run vLLM inference server locally without Globus Compute +- **Best for**: Single-node deployments, development +- **[Local vLLM Setup β†’](inference-setup/local-vllm.md)** + +**Direct API Connection** + +- Connect to existing OpenAI-compatible APIs (OpenAI, Anthropic, etc.) +- **Best for**: Simple setup, using commercial APIs +- **[Direct API Setup β†’](inference-setup/direct-api.md)** + +## Setup Workflow + +### Phase 1: Gateway Setup + +1. Choose your deployment method (Docker or Bare Metal) +2. Register Globus applications +3. Configure environment variables +4. Initialize the database +5. Start the gateway service + +### Phase 2: Inference Backend Setup + +1. Choose your backend type +2. Install required software (vLLM, Globus Compute, etc.) +3. Configure the backend +4. Register endpoints/functions +5. Test the connection + +### Phase 3: Connect Gateway and Backend + +1. Update fixture files with backend details +2. Load fixtures into the gateway database +3. Verify end-to-end functionality + +## Common Patterns + +### Pattern 1: Quick Local Development + +```mermaid +graph LR + A[Gateway] --> B[Local vLLM] + B --> C[Small Model
OPT-125M] +``` + +**Use**: Development and testing + +**Setup Time**: ~15 minutes + +**Resources**: 1 GPU or CPU + +### Pattern 2: Production Single Cluster + +```mermaid +graph LR + A[Gateway] --> B[Globus Compute] + B --> C[HPC Cluster] + C --> D[Multiple Models] +``` + +**Use**: Production deployment on single HPC cluster + +**Setup Time**: ~2 hours + +**Resources**: HPC cluster access + +### Pattern 3: Federated Multi-Cluster + +```mermaid +graph LR + A[Gateway] --> B[Cluster 1
Globus Compute] + A --> C[Cluster 2
Globus Compute] + A --> D[Cluster 3
Globus Compute] +``` + +**Use**: Maximum availability and resource pooling + +**Setup Time**: ~4 hours + +**Resources**: Multiple HPC clusters + +## Next Steps + +Ready to get started? Choose your path: + +- **Quick Start**: [Docker Deployment](gateway-setup/docker.md) +- **Full Setup**: [Bare Metal Deployment](gateway-setup/bare-metal.md) +- **Backend Setup**: [Inference Backend Overview](inference-setup/index.md) + +## Production Examples + +### ALCF Sophia Cluster + +We provide production-ready examples from our deployment at **Argonne Leadership Computing Facility (ALCF)** Sophia cluster: + +- **Modular launcher scripts** with automatic Ray setup for multi-node models +- **Environment management** with dynamic version selection +- **Production configurations** for single-node and multi-node deployments (up to 405B parameter models) +- **Advanced features**: chunked prefill, prefix caching, tool calling + +These examples are located in `compute-endpoints/` and `compute-functions/` directories and should be adapted for your specific HPC environment. + +!!! example "See ALCF Examples" + View the complete ALCF Sophia production setup in the [Globus Compute Guide](inference-setup/globus-compute.md#alcf-sophia-production-example) including: + + - `sophia_env_setup_with_ray.sh` - Environment and Ray cluster management + - `launch_vllm_model.sh` - Flexible vLLM launcher with multi-node support + - Example YAML configurations for various model sizes + +## Additional Resources + +- [Configuration Reference](gateway-setup/configuration.md) +- [Production Best Practices](deployment/production.md) +- [Monitoring & Troubleshooting](monitoring.md) +- [Kubernetes Deployment](deployment/kubernetes.md) (Coming Soon) + diff --git a/docs/admin-guide/inference-setup/direct-api.md b/docs/admin-guide/inference-setup/direct-api.md new file mode 100644 index 00000000..9d60c66e --- /dev/null +++ b/docs/admin-guide/inference-setup/direct-api.md @@ -0,0 +1,383 @@ +# Direct API Connection + +This guide shows you how to connect the FIRST Gateway to existing OpenAI-compatible APIs without running any local inference infrastructure. + +## Overview + +The Direct API backend allows you to: + +- Proxy requests to commercial APIs (OpenAI, Anthropic, etc.) +- Add Globus authentication to existing APIs +- Manage API keys centrally +- Route between multiple API providers + +## Architecture + +```mermaid +graph LR + A[User] -->|Globus Token| B[FIRST Gateway] + B -->|API Key| C[OpenAI API] + B -->|API Key| D[Anthropic API] + B -->|API Key| E[Custom API] +``` + +## Prerequisites + +- FIRST Gateway deployed and running +- API keys from your providers +- A way to host a status manifest (static file or endpoint) + +## Step 1: Create Status Manifest + +The gateway uses a status manifest to discover available endpoints. Create a JSON file: + +```json +{ + "openai-gpt4": { + "status": "Live", + "model": "OpenAI GPT-4", + "description": "GPT-4 models via OpenAI API", + "experts": [ + "gpt-4", + "gpt-4-turbo", + "gpt-4o", + "gpt-4o-mini" + ], + "url": "https://api.openai.com/v1", + "endpoint_id": "openai-production" + }, + "anthropic-claude": { + "status": "Live", + "model": "Anthropic Claude", + "description": "Claude models via Anthropic API", + "experts": [ + "claude-3-opus-20240229", + "claude-3-sonnet-20240229", + "claude-3-haiku-20240307" + ], + "url": "https://api.anthropic.com/v1", + "endpoint_id": "anthropic-production" + } +} +``` + +### Manifest Field Descriptions + +| Field | Required | Description | +|-------|----------|-------------| +| `status` | Yes | "Live", "Offline", or "Maintenance" | +| `model` | Yes | Human-readable model description | +| `description` | Yes | Detailed description | +| `experts` | Yes | Array of model identifiers | +| `url` | Yes | Base URL for the API | +| `endpoint_id` | Yes | Unique identifier (used for API key mapping) | + +## Step 2: Host the Status Manifest + +### Option A: Static File Server + +```bash +# Simple Python HTTP server +mkdir -p /var/www/metis +cp status.json /var/www/metis/ +cd /var/www/metis +python3 -m http.server 8055 +``` + +### Option B: Nginx + +```nginx +server { + listen 80; + server_name status.yourdomain.com; + + location / { + root /var/www/metis; + add_header Content-Type application/json; + add_header Access-Control-Allow-Origin *; + } +} +``` + +### Option C: S3/Cloud Storage + +Upload `status.json` to a public S3 bucket or equivalent: + +```bash +# AWS S3 +aws s3 cp status.json s3://your-bucket/status.json --acl public-read + +# Access via: https://your-bucket.s3.amazonaws.com/status.json +``` + +### Option D: Local for Docker (Development Only) + +For local Docker testing: + +```bash +# On host machine +mkdir -p deploy/docker/examples +cat > deploy/docker/examples/metis-status.json << 'EOF' +{ + "openai-gateway": { + "status": "Live", + "model": "OpenAI Pass-through", + "description": "Routes to OpenAI's GPT models", + "experts": ["gpt-4o-mini", "gpt-4"], + "url": "https://api.openai.com/v1", + "endpoint_id": "openai-production" + } +} +EOF + +# Serve it +python3 -m http.server 8055 --directory deploy/docker/examples +``` + +Then use `http://host.docker.internal:8055/metis-status.json` in your Docker `.env`. + +## Step 3: Configure Gateway + +Add these to your gateway's `.env` file: + +```dotenv +# URL to your status manifest +METIS_STATUS_URL="http://your-server:8055/status.json" + +# API keys mapped to endpoint_id from the manifest +METIS_API_TOKENS='{"openai-production": "sk-proj-...", "anthropic-production": "sk-ant-..."}' +``` + +### Environment Variable Format + +**METIS_STATUS_URL**: Direct URL to your JSON manifest + +**METIS_API_TOKENS**: JSON object where: + +- Keys are `endpoint_id` values from your manifest +- Values are the API keys for those services + +Example with multiple providers: + +```dotenv +METIS_API_TOKENS='{ + "openai-production": "sk-proj-abc123...", + "anthropic-production": "sk-ant-xyz789...", + "custom-api": "custom-key-here" +}' +``` + +## Step 4: Restart Gateway + +### Docker + +```bash +docker-compose up -d inference-gateway +``` + +### Bare Metal + +```bash +sudo systemctl restart inference-gateway +``` + +## Step 5: Test the Connection + +Get a Globus token: + +```bash +export TOKEN=$(python inference-auth-token.py get_access_token) +``` + +Test OpenAI endpoint: + +```bash +curl -X POST http://localhost:8000/resource_server/metis/api/v1/chat/completions \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "Hello from FIRST!"}], + "stream": false + }' +``` + +Expected response: + +```json +{ + "id": "chatcmpl-...", + "object": "chat.completion", + "created": 1234567890, + "model": "gpt-4o-mini", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": "Hello! How can I assist you today?" + }, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 15, + "total_tokens": 25 + } +} +``` + +## Advanced Configuration + +### Multiple Endpoints Per Provider + +```json +{ + "openai-us-east": { + "status": "Live", + "model": "OpenAI US East", + "experts": ["gpt-4"], + "url": "https://api.openai.com/v1", + "endpoint_id": "openai-us-east" + }, + "openai-eu-west": { + "status": "Live", + "model": "OpenAI EU West", + "experts": ["gpt-4"], + "url": "https://api.openai.com/v1", + "endpoint_id": "openai-eu-west" + } +} +``` + +### Custom API Headers + +For APIs requiring additional headers, you can extend the gateway code or use environment variables. Contact your administrator for custom integration. + +### Load Balancing + +The gateway automatically load-balances across all "Live" endpoints for the same model. + +### Failover + +If one endpoint returns an error, the gateway automatically tries the next available endpoint. + +## Monitoring + +### Check Endpoint Status + +The gateway periodically fetches the status manifest. View logs: + +```bash +# Docker +docker-compose logs -f inference-gateway | grep metis + +# Bare metal +tail -f logs/django_info.log | grep metis +``` + +### Track API Usage + +Monitor usage through: + +- Gateway access logs +- Provider API dashboards (OpenAI, Anthropic) +- Custom usage tracking (implement in gateway) + +## Cost Management + +### Setting Budgets + +Configure per-user or per-group budgets in your application logic or via API key restrictions at the provider level. + +### Rate Limiting + +The gateway supports rate limiting per user/group. Configure in Django admin or via settings. + +## Security Considerations + +### API Key Security + +!!! danger "Protect Your API Keys" + - Never commit API keys to version control + - Use environment variables or secrets management + - Rotate keys regularly + - Use separate keys for dev/staging/production + +### Status Manifest Security + +If your manifest contains sensitive information: + +- Serve over HTTPS +- Implement authentication (basic auth, token) +- Restrict IP access via firewall + +### Access Control + +Restrict which Globus groups can access which APIs: + +```dotenv +GLOBUS_GROUPS="allowed-group-uuid-1 allowed-group-uuid-2" +``` + +## Troubleshooting + +### Gateway can't fetch status manifest + +Check connectivity: + +```bash +curl http://your-server:8055/status.json +``` + +Verify `METIS_STATUS_URL` is correct and accessible from the gateway. + +### Authentication errors from provider + +- Verify API key is correct +- Check key hasn't expired +- Ensure key has required permissions +- Check provider status page + +### Model not found + +Ensure the model name matches exactly what's in the `experts` array of your manifest. + +### Rate limiting errors + +- Check provider rate limits +- Implement gateway-side rate limiting +- Consider upgrading provider plan + +## Example: Adding Azure OpenAI + +```json +{ + "azure-openai": { + "status": "Live", + "model": "Azure OpenAI", + "description": "GPT models via Azure OpenAI Service", + "experts": [ + "gpt-4", + "gpt-35-turbo" + ], + "url": "https://your-resource.openai.azure.com/openai/deployments/your-deployment", + "endpoint_id": "azure-openai-prod" + } +} +``` + +Azure requires additional configuration - consult Azure OpenAI documentation. + +## Next Steps + +- [User Guide](../../user-guide/index.md) - How to use the API +- [Monitoring](../monitoring.md) - Set up monitoring and alerts +- [Local vLLM](local-vllm.md) - Add local inference capabilities +- [Production Best Practices](../deployment/production.md) + +## Additional Resources + +- [OpenAI API Documentation](https://platform.openai.com/docs/api-reference) +- [Anthropic API Documentation](https://docs.anthropic.com/claude/reference) +- [Gateway Configuration Reference](../gateway-setup/configuration.md) + diff --git a/docs/admin-guide/inference-setup/globus-compute.md b/docs/admin-guide/inference-setup/globus-compute.md new file mode 100644 index 00000000..57e18ace --- /dev/null +++ b/docs/admin-guide/inference-setup/globus-compute.md @@ -0,0 +1,1022 @@ +# Globus Compute + vLLM Setup + +This guide shows you how to deploy vLLM on HPC clusters or remote servers using Globus Compute for federated inference. + +## Overview + +This is the recommended approach for: + +- Multi-cluster federated deployments +- HPC environments with job schedulers (PBS, Slurm) +- Organizations requiring high availability +- Remote execution with secure authentication + +## Architecture + +```mermaid +graph TB + A[User] -->|Globus Token| B[FIRST Gateway] + B -->|Globus Compute| C[HPC Cluster 1] + B -->|Globus Compute| D[HPC Cluster 2] + B -->|Globus Compute| E[Local Server] + C -->|Job Scheduler| F[vLLM + Model] + D -->|Job Scheduler| G[vLLM + Model] + E -->|Direct| H[vLLM + Model] +``` + +## Prerequisites + +- FIRST Gateway deployed with Service Account application credentials +- Access to compute resources (HPC cluster or powerful workstation) +- GPU resources for running models +- Python 3.12+ (same version as gateway) +- Globus Compute SDK and Endpoint software + +## Part 1: Setup on Compute Resource + +This is done on the machine(s) where models will run. + +### Step 1: Create Python Environment + +!!! warning "Version Match" + Use the **same Python version** as your gateway to avoid compatibility issues. + +```bash +# Using conda (recommended for HPC) +conda create -n vllm-env python=3.12 -y +conda activate vllm-env + +# OR using venv +python3.12 -m venv vllm-env +source vllm-env/bin/activate +``` + +### Step 2: Install vLLM + +```bash +# For CUDA 12.1 (default) +pip install vllm + +# For specific CUDA version +pip install vllm-cu118 # CUDA 11.8 + +# From source (for latest features) +git clone https://github.com/vllm-project/vllm.git +cd vllm +pip install -e . +``` + +### Step 3: Install Globus Compute + +```bash +pip install globus-compute-sdk globus-compute-endpoint +``` + +### Step 4: Export Service Account Credentials + +These are from your Gateway's Service Account application: + +```bash +export GLOBUS_COMPUTE_CLIENT_ID="" +export GLOBUS_COMPUTE_CLIENT_SECRET="" +``` + +Add to your `~/.bashrc` or `~/.bash_profile` for persistence: + +```bash +echo 'export GLOBUS_COMPUTE_CLIENT_ID="your-id"' >> ~/.bashrc +echo 'export GLOBUS_COMPUTE_CLIENT_SECRET="your-secret"' >> ~/.bashrc +``` + +### Step 5: Register Globus Compute Functions + +Navigate to the gateway repository's compute-functions directory: + +```bash +cd /path/to/inference-gateway/compute-functions +``` + +#### Register Inference Function + +```bash +python vllm_register_function_with_streaming.py +``` + +Output: + +``` +Function registered with UUID: 12345678-1234-1234-1234-123456789abc +The UUID is stored in vllm_register_function_streaming.txt +``` + +**Save this Function UUID** - you'll need it later. + +#### Register Status Function (Optional but Recommended) + +For HPC clusters with job schedulers, you can register a qstat function to monitor cluster status. + +```bash +python qstat_register_function.py +``` + +Save the Function UUID from the output. + +**Configure a qstat endpoint** on your HPC login node: + +```bash +globus-compute-endpoint configure qstat-endpoint +``` + +Edit `~/.globus_compute/qstat-endpoint/config.yaml`: + +```yaml +display_name: qstat-parser-endpoint +engine: + type: GlobusComputeEngine + max_retries_on_system_failure: 2 + max_workers_per_node: 2 + provider: + type: LocalProvider + init_blocks: 1 + max_blocks: 1 + min_blocks: 1 + +allowed_functions: + - # UUID from qstat_register_function.py +``` + +Start the qstat endpoint: + +```bash +globus-compute-endpoint start qstat-endpoint +``` + +Add the qstat endpoint configuration to your gateway's `.env`: + +```dotenv +SOPHIA_QSTAT_ENDPOINT_UUID="" +SOPHIA_QSTAT_FUNCTION_UUID="" +``` + +#### Register Batch Function (Optional) + +For batch processing support: + +```bash +python vllm_batch_function.py +``` + +Save the Function UUID. + +### Step 6: Configure Globus Compute Endpoint + +Create a new endpoint: + +```bash +globus-compute-endpoint configure my-inference-endpoint +``` + +This creates `~/.globus_compute/my-inference-endpoint/config.yaml`. + +#### For Local/Workstation Deployment + +Edit `config.yaml`: + +```yaml +display_name: My Inference Endpoint +engine: + type: GlobusComputeEngine + provider: + type: LocalProvider + init_blocks: 1 + max_blocks: 1 + min_blocks: 0 + +# Allow only your registered functions +allowed_functions: + - 12345678-1234-1234-1234-123456789abc # Your vLLM function UUID + +# Activate your environment and start vLLM server +worker_init: | + source /path/to/vllm-env/bin/activate + # OR: conda activate vllm-env + + # Start vLLM server in background + nohup vllm serve facebook/opt-125m \ + --host 0.0.0.0 \ + --port 8000 \ + --gpu-memory-utilization 0.9 \ + > vllm.log 2>&1 & + + # Wait for server to be ready + sleep 30 +``` + +#### For HPC with PBS (e.g., ALCF Sophia) + +```yaml +display_name: Sophia vLLM Endpoint +engine: + type: GlobusComputeEngine + provider: + type: PBSProProvider + account: YourProjectAccount + queue: demand + nodes_per_block: 1 + init_blocks: 1 + max_blocks: 4 + min_blocks: 0 + walltime: "06:00:00" + scheduler_options: | + #PBS -l filesystems=home:eagle + #PBS -l place=scatter + worker_init: | + module load conda + conda activate /path/to/vllm-env + + # Start vLLM server in background + nohup vllm serve facebook/opt-125m \ + --host 0.0.0.0 \ + --port 8000 \ + --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.9 \ + > vllm.log 2>&1 & + + # Wait for server to be ready + sleep 30 + +# GPU allocation +max_workers_per_node: 1 + +# Allow only your registered functions +allowed_functions: + - 12345678-1234-1234-1234-123456789abc +``` + +!!! note "Advanced Configuration Examples from ALCF Sophia Cluster" + For production deployments with advanced features, see the example configurations in `compute-endpoints/`. These examples are based on our deployment at **Argonne Leadership Computing Facility (ALCF) Sophia cluster** and should be adapted to your specific HPC environment: + + **Configuration Files:** + + - `sophia-vllm-singlenode-example.yaml` - Single-node deployment with optimized settings + - `sophia-vllm-multinode-example.yaml` - Multi-node deployment for large models (70B+) + - `sophia-vllm-toolcalling-example.yaml` - Configuration with tool calling support + - `pbs-qstat-example.yaml` - PBS job scheduler monitoring endpoint + + **Helper Scripts:** + + - `launch_vllm_model.sh` - Modular vLLM launcher with automatic Ray setup, retry logic, and health monitoring + - `sophia_env_setup_with_ray.sh` - Environment setup script for ALCF Sophia (loads modules, conda envs, sets proxy, configures Ray) + + These examples demonstrate a production-ready setup with: + + - Dynamic vLLM version selection + - Multi-node Ray cluster management + - Advanced vLLM parameters (chunked prefill, prefix caching, tool calling) + - Comprehensive logging and error handling + - Retry logic with timeout management + + !!! warning "Cluster-Specific Adaptation Required" + The Sophia scripts are **specific to ALCF infrastructure** and must be adapted for your cluster: + + - Module names and paths + - Conda environment locations + - File system paths (e.g., `/eagle/argonne_tpc/`) + - Proxy settings + - Network interface names (e.g., `infinibond0`, `ens10f0`) + - Job scheduler options and queue names + - SSL certificate paths + + Use these as **templates** to create your own cluster-specific scripts. + +#### For HPC with Slurm + +```yaml +display_name: Slurm vLLM Endpoint +engine: + type: GlobusComputeEngine + provider: + type: SlurmProvider + account: your_account + partition: gpu + nodes_per_block: 1 + init_blocks: 1 + max_blocks: 4 + walltime: "06:00:00" + scheduler_options: | + #SBATCH --gpus-per-node=1 + #SBATCH --mem=64G + worker_init: | + source /path/to/vllm-env/bin/activate + + # Start vLLM server in background + nohup vllm serve facebook/opt-125m \ + --host 0.0.0.0 \ + --port 8000 \ + --gpu-memory-utilization 0.9 \ + > vllm.log 2>&1 & + + # Wait for server to be ready + sleep 30 + +allowed_functions: + - 12345678-1234-1234-1234-123456789abc +``` + +### Step 7: Start the Endpoint + +```bash +globus-compute-endpoint start my-inference-endpoint +``` + +Output: + +``` +Starting endpoint; registered ID: abcdef12-3456-7890-abcd-ef1234567890 +``` + +**Save this Endpoint UUID** - you'll need it for gateway configuration. + +Verify it's running: + +```bash +globus-compute-endpoint list +``` + +View logs: + +```bash +globus-compute-endpoint log my-inference-endpoint +``` + +--- + +## ALCF Sophia Production Example + +This section shows how we deploy vLLM at **Argonne Leadership Computing Facility (ALCF)** on the Sophia cluster. These are production examples that you should adapt to your own HPC environment. + +### Architecture Overview + +Our production setup uses a modular approach: + +1. **Environment Setup Script** (`sophia_env_setup_with_ray.sh`) - Manages conda environments, modules, Ray cluster setup +2. **Model Launcher** (`launch_vllm_model.sh`) - Flexible vLLM launcher with multi-node support +3. **Endpoint Configurations** (YAML files) - PBS-specific settings for different model sizes + +### Environment Setup Script + +The `sophia_env_setup_with_ray.sh` provides: + +- **Dynamic version selection**: Automatically loads correct conda environment based on `VLLM_VERSION` +- **Module management**: Loads ALCF-specific modules (conda, gcc, spack) +- **Proxy configuration**: Sets up HTTP/HTTPS proxies for ALCF network +- **Ray cluster management**: Automated multi-node Ray setup for pipeline parallelism +- **Retry logic**: Robust model startup with monitoring and timeout handling + +**Key Functions:** + +```bash +# Load the script (done automatically by launch_vllm_model.sh) +source /home/openinference_svc/sophia_env_setup_with_ray.sh + +# Setup conda environment and exports +setup_environment + +# Setup multi-node Ray cluster (for large models) +setup_ray_cluster + +# Start model with retry logic +start_model "model-name" "vllm serve ..." "logfile.log" retry_counter 2 3600 + +# Cleanup processes +cleanup_python_processes + +# Stop Ray +stop_ray +``` + +### Model Launcher Script + +The `launch_vllm_model.sh` provides a unified interface for launching models: + +```bash +# Single-node example (8 GPUs, tensor parallelism) +source /home/openinference_svc/launch_vllm_model.sh \ + --model-name meta-llama/Meta-Llama-3.1-8B-Instruct \ + --vllm-version v0.11.0 \ + --tensor-parallel 8 \ + --max-model-len 8192 \ + --enable-chunked-prefill \ + --enable-prefix-caching \ + --trust-remote-code \ + --gpu-memory-util 0.95 \ + --framework vllm \ + --cluster sophia + +# Multi-node example (4 nodes, 32 GPUs, TP=8 PP=4) +source /home/openinference_svc/launch_vllm_model.sh \ + --model-name meta-llama/Meta-Llama-3.1-405B-Instruct \ + --vllm-version v0.11.0 \ + --tensor-parallel 8 \ + --pipeline-parallel 4 \ + --max-model-len 16384 \ + --enable-prefix-caching \ + --gpu-memory-util 0.95 +``` + +**The launcher automatically:** + +- Sources `sophia_env_setup_with_ray.sh` +- Calls `setup_environment()` +- Detects single vs multi-node based on `--pipeline-parallel` or `--multi` flag +- Sets up Ray cluster if needed +- Builds and executes vLLM command +- Monitors startup with retry logic +- Handles errors and logging + +### Example Endpoint Configurations + +#### Single-Node (8 GPUs) + +From `sophia-vllm-singlenode-example.yaml`: + +```yaml +display_name: sophia-vllm-llama3.1-8b-instruct +engine: + type: GlobusComputeEngine + provider: + type: PBSProProvider + account: argonne_tpc + select_options: ngpus=8 + queue: 'by-node' + nodes_per_block: 1 + max_blocks: 2 + walltime: 24:00:00 + scheduler_options: '#PBS -l filesystems=home:eagle' + worker_init: | + # Use the modular launcher script + source /home/openinference_svc/launch_vllm_model.sh \ + --model-name meta-llama/Meta-Llama-3.1-8B-Instruct \ + --vllm-version v0.11.0 \ + --tensor-parallel 8 \ + --max-model-len 8192 \ + --enable-chunked-prefill \ + --enable-prefix-caching \ + --trust-remote-code \ + --gpu-memory-util 0.95 \ + --framework vllm \ + --cluster sophia + +allowed_functions: + - # Replace with your function UUID +``` + +#### Multi-Node (4 nodes, 32 GPUs) + +From `sophia-vllm-multinode-example.yaml`: + +```yaml +display_name: sophia-vllm-llama3.1-405b-instruct +engine: + type: GlobusComputeEngine + provider: + type: PBSProProvider + account: argonne_tpc + select_options: ngpus=8 + queue: 'by-node' + nodes_per_block: 4 # 4 nodes + walltime: 24:00:00 + scheduler_options: '#PBS -l filesystems=home:eagle' + worker_init: | + # Multi-node with pipeline parallelism (requires Ray) + source /home/openinference_svc/launch_vllm_model.sh \ + --model-name meta-llama/Meta-Llama-3.1-405B-Instruct \ + --vllm-version v0.11.0 \ + --tensor-parallel 8 \ + --pipeline-parallel 4 \ + --max-model-len 16384 \ + --enable-prefix-caching \ + --enable-chunked-prefill \ + --trust-remote-code \ + --gpu-memory-util 0.95 \ + --framework vllm \ + --cluster sophia + +allowed_functions: + - # Replace with your function UUID +``` + +### Adapting for Your Cluster + +To adapt these scripts for your HPC environment: + +#### 1. Create Your Environment Setup Script + +Copy `sophia_env_setup_with_ray.sh` and modify: + +```bash +# Change proxy settings (or remove if not needed) +export HTTP_PROXY="your-proxy:port" + +# Change module loading +module use /your/module/path +module load your-conda-module + +# Change conda environment paths +case "$VLLM_VERSION" in + v0.11.0) + CONDA_ENV="/your/path/to/vllm-env/" + ;; +esac + +# Change HuggingFace cache paths +export HF_HOME='/your/model/cache/path/' + +# Change network interface for NCCL (check with `ifconfig`) +export NCCL_SOCKET_IFNAME='your-network-interface' # e.g., 'ib0', 'eth0' + +# Adjust Ray node resources +export RAY_NUM_CPUS=64 # CPUs per node +export RAY_NUM_GPUS=8 # GPUs per node +``` + +#### 2. Adapt the Launcher Script (Optional) + +The `launch_vllm_model.sh` is fairly generic, but you may need to: + +- Update the default environment setup script path (line 242) +- Adjust SSL certificate paths if using HTTPS +- Modify default values for your hardware + +#### 3. Update Endpoint YAML + +```yaml +# Change PBS/Slurm provider settings +provider: + type: PBSProProvider # or SlurmProvider + account: your_project_account + queue: your_queue_name + scheduler_options: | + #PBS -l your:scheduler:options + # Adjust for your cluster's requirements + +# Update worker_init paths +worker_init: | + source /your/path/to/your_env_setup.sh \ + --model-name your/model \ + --your-specific-flags + +# Adjust resource allocation +select_options: ngpus=4 # GPUs per node +nodes_per_block: 1 # Nodes per job +max_workers_per_node: 1 # Workers per node +``` + +#### 4. Key Cluster-Specific Items to Check + +| Item | ALCF Sophia Example | What to Change | +|------|---------------------|----------------| +| Network Interface | `infinibond0`, `ens10f0` | Run `ifconfig` on compute node | +| File System | `/eagle/argonne_tpc/` | Your shared file system path | +| Module System | `module use /soft/modulefiles` | Your cluster's module path | +| Conda Path | `/eagle/argonne_tpc/inference-gateway/envs/` | Your conda env location | +| Proxy | `http://proxy.alcf.anl.gov:3128` | Your proxy or remove | +| SSL Certs | `~/certificates/mykey.key` | Your SSL cert path or disable | +| PBS Options | `#PBS -l filesystems=home:eagle` | Your scheduler requirements | +| Queue Name | `by-node`, `demand` | Your cluster's queue names | + +### Troubleshooting ALCF Sophia Examples + +Common issues when adapting: + +**Module not found:** +```bash +# Check available modules +module avail + +# Update module paths in your env setup script +module use /correct/module/path +``` + +**Conda environment not activating:** +```bash +# Verify conda init +conda init bash +source ~/.bashrc + +# Check environment exists +conda env list + +# Update paths in setup_environment() function +``` + +**NCCL network errors:** +```bash +# Check network interfaces +ifconfig + +# Update NCCL_SOCKET_IFNAME in env setup script +export NCCL_SOCKET_IFNAME='correct-interface-name' +``` + +**Ray cluster not starting:** +```bash +# Check PBS nodefile exists +echo $PBS_NODEFILE +cat $PBS_NODEFILE + +# Verify nodes can communicate +mpiexec -n 2 hostname + +# Check Ray ports are open (default 6379) +``` + +--- + +## Part 2: Configure Gateway + +Now configure the gateway to use your Globus Compute endpoint. + +### Step 1: Create Endpoint Fixture + +On your gateway machine, edit `fixtures/endpoints.json`: + +#### Single Endpoint Configuration + +```json +[ + { + "model": "resource_server.endpoint", + "pk": 1, + "fields": { + "endpoint_slug": "local-vllm-opt-125m", + "cluster": "local", + "framework": "vllm", + "model": "facebook/opt-125m", + "api_port": 8000, + "endpoint_uuid": "abcdef12-3456-7890-abcd-ef1234567890", + "function_uuid": "12345678-1234-1234-1234-123456789abc", + "batch_endpoint_uuid": "", + "batch_function_uuid": "", + "allowed_globus_groups": "" + } + } +] +``` + +#### Multiple Clusters Configuration + +```json +[ + { + "model": "resource_server.endpoint", + "pk": 1, + "fields": { + "endpoint_slug": "local-vllm-opt-125m", + "cluster": "local", + "framework": "vllm", + "model": "facebook/opt-125m", + "api_port": 8000, + "endpoint_uuid": "local-endpoint-uuid", + "function_uuid": "local-function-uuid", + "allowed_globus_groups": "" + } + }, + { + "model": "resource_server.endpoint", + "pk": 2, + "fields": { + "endpoint_slug": "sophia-vllm-opt-125m", + "cluster": "sophia", + "framework": "vllm", + "model": "facebook/opt-125m", + "api_port": 8000, + "endpoint_uuid": "sophia-endpoint-uuid", + "function_uuid": "sophia-function-uuid", + "allowed_globus_groups": "" + } + } +] +``` + +### Step 2: Federated Endpoints (Optional but Recommended) + +For automatic load balancing and failover, use federated endpoints. + +Edit `fixtures/federated_endpoints.json`: + +```json +[ + { + "model": "resource_server.federatedendpoint", + "pk": 1, + "fields": { + "name": "OPT-125M (Federated)", + "slug": "federated-opt-125m", + "target_model_name": "facebook/opt-125m", + "description": "Federated access to OPT-125M across multiple clusters", + "targets": [ + { + "cluster": "local", + "framework": "vllm", + "model": "facebook/opt-125m", + "endpoint_slug": "local-vllm-opt-125m", + "endpoint_uuid": "local-endpoint-uuid", + "function_uuid": "local-function-uuid", + "api_port": 8000 + }, + { + "cluster": "sophia", + "framework": "vllm", + "model": "facebook/opt-125m", + "endpoint_slug": "sophia-vllm-opt-125m", + "endpoint_uuid": "sophia-endpoint-uuid", + "function_uuid": "sophia-function-uuid", + "api_port": 8000 + } + ] + } + } +] +``` + +### Step 3: Load Fixtures + +```bash +# Docker +docker-compose exec inference-gateway python manage.py loaddata fixtures/endpoints.json +docker-compose exec inference-gateway python manage.py loaddata fixtures/federated_endpoints.json + +# Bare Metal +python manage.py loaddata fixtures/endpoints.json +python manage.py loaddata fixtures/federated_endpoints.json +``` + +## Part 3: Testing + +### Test Globus Compute Endpoint + +From your gateway machine, test that you can reach the endpoint: + +```python +from globus_compute_sdk import Client + +gcc = Client() +endpoint_id = "abcdef12-3456-7890-abcd-ef1234567890" +function_id = "12345678-1234-1234-1234-123456789abc" + +# Simple test function +def hello(): + return "Hello from Globus Compute!" + +# Register and run +func_uuid = gcc.register_function(hello) +task = gcc.run(endpoint_id=endpoint_id, function_id=func_uuid) +print(gcc.get_result(task)) +``` + +### Test vLLM via Gateway + +First, download the authentication helper script: + +```bash +# Download the authentication helper script +wget https://raw.githubusercontent.com/argonne-lcf/inference-endpoints/refs/heads/main/inference_auth_token.py + +# Authenticate with your Globus account +python inference_auth_token.py authenticate +``` + +Get a Globus token: + +```bash +export TOKEN=$(python inference_auth_token.py get_access_token) +``` + +Test non-federated endpoint: + +```bash +curl -X POST http://localhost:8000/resource_server/local/vllm/v1/chat/completions \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "facebook/opt-125m", + "messages": [{"role": "user", "content": "What is Globus Compute?"}], + "max_tokens": 100 + }' +``` + +Test federated endpoint: + +```bash +curl -X POST http://localhost:8000/resource_server/v1/chat/completions \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "facebook/opt-125m", + "messages": [{"role": "user", "content": "What is Globus Compute?"}], + "max_tokens": 100 + }' +``` + +## Advanced Configuration + +### Hot Nodes / Keep-Alive + +For low-latency inference, keep compute nodes warm: + +```yaml +# In config.yaml +engine: + provider: + min_blocks: 1 # Always keep 1 node running + init_blocks: 1 # Start with 1 node +``` + +### Multi-Node vLLM + +For very large models (70B+), use tensor parallelism across multiple GPUs/nodes: + +See `compute-endpoints/sophia-vllm-multinode-example.yaml` for configuration examples. + +### Custom Inference Scripts + +The registered functions use scripts in `compute-functions/`. You can customize: + +- `launch_vllm_model.sh`: How vLLM is started +- `vllm_register_function_with_streaming.py`: The inference function logic + +### Batch Processing + +Enable batch processing by registering and configuring batch functions: + +```bash +python vllm_batch_function.py +``` + +Add batch UUIDs to your endpoint fixture: + +```json +{ + "batch_endpoint_uuid": "batch-endpoint-uuid", + "batch_function_uuid": "batch-function-uuid" +} +``` + +## Monitoring + +### Endpoint Status + +```bash +# Check endpoint status +globus-compute-endpoint status my-inference-endpoint + +# View Globus Compute endpoint logs +tail -f ~/.globus_compute/my-inference-endpoint/endpoint.log + +# Check task submission logs +ls -la ~/.globus_compute/my-inference-endpoint/submit_tasks/ +tail -f ~/.globus_compute/my-inference-endpoint/submit_tasks/*.submit.log + +# View vLLM server logs (from worker_init nohup) +tail -f vllm.log +``` + +### Job Scheduler Monitoring + +```bash +# PBS +qstat -u $USER + +# Slurm +squeue -u $USER +``` + +### Gateway Logs + +```bash +# Docker +docker-compose logs -f inference-gateway + +# Bare Metal +tail -f logs/django_info.log +``` + +## Troubleshooting + +### Endpoint won't start + +Check logs: + +```bash +globus-compute-endpoint log my-inference-endpoint +``` + +Common issues: + +- Service account credentials not set +- Function UUIDs not in `allowed_functions` +- Python environment activation fails +- Job scheduler configuration incorrect + +### Function execution fails + +- Verify environment can run vLLM: Test manually on compute node +- Check function is allowed in `config.yaml` +- Ensure Service Account has permissions +- Check job scheduler logs + +### Model not loading + +- Verify GPU allocation in scheduler options +- Check VRAM requirements +- Ensure model path/name is correct +- Check Hugging Face token if needed + +### Gateway can't connect + +- Verify endpoint is running: `globus-compute-endpoint list` +- Check endpoint UUID in fixtures is correct +- Ensure Service Account credentials match in both places +- Check network connectivity (firewall, VPN) + +## HPC-Specific Considerations + +### ALCF Systems (Polaris, Sophia) + +- Use `/lus/eagle` for model cache +- Load CUDA modules in `worker_init` +- Set appropriate walltime +- Use `#PBS -l filesystems=home:eagle` + +### Other HPC Centers + +- Check local documentation for: + - GPU allocation syntax + - File system paths + - Module system + - Queue/partition names + - Accounting project codes + +## Scaling + +### Auto-Scaling + +Configure `max_blocks` to allow automatic scaling: + +```yaml +engine: + provider: + init_blocks: 1 + min_blocks: 1 + max_blocks: 10 # Scale up to 10 nodes +``` + +### Load Balancing + +Federated endpoints automatically load balance across all "Live" targets. + +### Geographic Distribution + +Deploy endpoints at different locations and configure federated routing for optimal latency. + +## Security + +### Function Allowlisting + +Always specify `allowed_functions` in `config.yaml`: + +```yaml +allowed_functions: + - 12345678-1234-1234-1234-123456789abc # Only your functions +``` + +### Group-Based Access + +Restrict endpoints to specific Globus groups: + +```json +{ + "allowed_globus_groups": "group-uuid-1,group-uuid-2" +} +``` + +### Network Security + +- Globus Compute uses HTTPS and mutual TLS +- No inbound ports need to be opened on compute resources +- All communication is outbound from endpoint to Globus services + +## Next Steps + +- [Production Best Practices](../deployment/production.md) +- [Monitoring & Troubleshooting](../monitoring.md) +- [User Guide](../../user-guide/index.md) + +## Additional Resources + +- [Globus Compute Documentation](https://globus-compute.readthedocs.io/) +- [vLLM Documentation](https://docs.vllm.ai/) +- [Example Configurations](../../../compute-endpoints/) +- [Function Registration Scripts](../../../compute-functions/) + diff --git a/docs/admin-guide/inference-setup/index.md b/docs/admin-guide/inference-setup/index.md new file mode 100644 index 00000000..1450fa10 --- /dev/null +++ b/docs/admin-guide/inference-setup/index.md @@ -0,0 +1,223 @@ +# Inference Backend Setup + +The FIRST Inference Gateway supports multiple types of inference backends. Choose the approach that best fits your use case. + +## Backend Options + +### 1. Globus Compute + vLLM (Recommended) + +Deploy vLLM on HPC clusters or multiple servers with Globus Compute for remote execution and federated routing. + +**Best for:** + +- Multi-cluster deployments +- HPC environments +- Federated inference across organizations +- Production deployments requiring high availability + +**Pros:** + +- Federated routing across clusters +- Automatic failover +- HPC job scheduler integration +- Scales across organizations +- Secure, authenticated remote execution + +**Cons:** + +- More complex setup +- Requires Globus Compute knowledge +- Additional configuration overhead + +[β†’ Setup Globus Compute + vLLM](globus-compute.md) + +--- + +### 2. Local vLLM Setup + +Run vLLM inference server locally without Globus Compute. + +**Best for:** + +- Single-server deployments +- Development and testing with local models +- Controlled environments where Globus Compute isn't needed +- Simple architectures + +**Pros:** + +- Full control over models and data +- No external dependencies +- Simple architecture +- Lower latency for local requests + +**Cons:** + +- Requires GPU resources +- Single point of failure +- Manual scaling +- No federated routing + +[β†’ Setup Local vLLM](local-vllm.md) + +--- + +### 3. Direct API Connection + +Connect to existing OpenAI-compatible APIs without any local inference infrastructure. + +**Best for:** + +- Quick testing and development +- Using commercial API services (OpenAI, Anthropic, etc.) +- Organizations without GPU resources +- Proxying/adding authentication to existing APIs + +**Pros:** + +- Fastest setup (5-10 minutes) +- No local compute resources needed +- Scales with the provider +- Multiple models immediately available + +**Cons:** + +- Costs per API call +- Data leaves your infrastructure +- Dependent on third-party service + +[β†’ Setup Direct API Connection](direct-api.md) + +--- + +## Comparison Matrix + +| Feature | Globus Compute + vLLM | Local vLLM | Direct API | +|---------|----------------------|------------|------------| +| Setup Time | 2-4 hours | 30-60 min | 5-10 min | +| GPU Required | Yes | Yes | No | +| Data Privacy | Local | Local | External | +| Multi-Cluster | Yes | No | No | +| Failover | Automatic | Manual | Provider | +| Cost | Infrastructure | Infrastructure | Pay-per-use | +| HPC Integration | Yes | No | No | +| Complexity | High | Medium | Low | + +## Decision Tree + +```mermaid +graph TD + A[Choose Backend] --> B{Have GPU?} + B -->|No| C[Direct API] + B -->|Yes| D{Multiple Clusters?} + D -->|No| E{Need HPC Integration?} + D -->|Yes| F[Globus Compute + vLLM] + E -->|No| G[Local vLLM] + E -->|Yes| F +``` + +## Combined Approach + +You can use multiple backends simultaneously! The gateway supports: + +- **Multiple Direct API connections**: Route to OpenAI, Anthropic, etc. +- **Mix Local and Remote**: Some models local, some via Globus Compute +- **Federated Endpoints**: Route same model across multiple clusters + +Example combined setup: + +```json +{ + "endpoints": [ + { + "cluster": "openai", + "model": "gpt-4", + "type": "direct_api" + }, + { + "cluster": "local", + "model": "llama-3-8b", + "type": "vllm" + }, + { + "cluster": "sophia", + "model": "llama-3-70b", + "type": "globus_compute" + } + ] +} +``` + +## Prerequisites by Backend Type + +### All Backends + +- FIRST Gateway already deployed +- Access to gateway configuration +- Admin access to load fixtures + +### Direct API + +- API keys from providers +- Status manifest endpoint (can be static file) + +### Local vLLM + +- GPU with sufficient VRAM for your model +- Python 3.12+ +- Ability to run vLLM server + +### Globus Compute + vLLM + +- All Local vLLM requirements plus: +- Globus Service Account application +- Access to HPC cluster (optional) +- Ability to configure Globus Compute endpoints + +## Setup Workflow + +### Phase 1: Choose and Setup Backend + +Follow the guide for your chosen backend type. + +### Phase 2: Register with Gateway + +Update fixture files (`fixtures/endpoints.json` or `fixtures/federated_endpoints.json`) with your backend details. + +### Phase 3: Load Configuration + +```bash +# Docker +docker-compose exec inference-gateway python manage.py loaddata fixtures/endpoints.json + +# Bare metal +python manage.py loaddata fixtures/endpoints.json +``` + +### Phase 4: Test + +Send a test request to verify the connection: + +```bash +curl -X POST http://localhost:8000/resource_server/v1/chat/completions \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "your-model-name", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` + +## Next Steps + +Choose your backend setup guide: + +- [Globus Compute + vLLM](globus-compute.md) _(Recommended for Production)_ +- [Local vLLM Setup](local-vllm.md) +- [Direct API Connection](direct-api.md) + +After setup: + +- [Production Best Practices](../deployment/production.md) +- [Monitoring & Troubleshooting](../monitoring.md) + diff --git a/docs/admin-guide/inference-setup/local-vllm.md b/docs/admin-guide/inference-setup/local-vllm.md new file mode 100644 index 00000000..4db606d6 --- /dev/null +++ b/docs/admin-guide/inference-setup/local-vllm.md @@ -0,0 +1,471 @@ +# Local vLLM Setup + +This guide shows you how to run vLLM inference server locally and connect it to the FIRST Gateway **without** Globus Compute. + +## Overview + +This setup is ideal for: + +- Single-server deployments +- Development and testing +- Scenarios where Globus Compute overhead isn't needed +- Direct control over the inference process + +## Architecture + +```mermaid +graph LR + A[User] -->|Globus Token| B[FIRST Gateway] + B -->|HTTP| C[vLLM Server] + C -->|GPU| D[Model] +``` + +## Prerequisites + +- NVIDIA GPU with sufficient VRAM for your model +- CUDA 11.8 or later +- Python 3.12+ +- Docker (optional, for containerized vLLM) + +## Step 1: Install vLLM + +### Option A: Install from Source (Recommended) + +```bash +# Create virtual environment +python3.12 -m venv vllm-env +source vllm-env/bin/activate + +# Clone and install vLLM +git clone https://github.com/vllm-project/vllm.git +cd vllm +pip install -e . + +# Install additional dependencies +pip install openai # For testing +``` + +### Option B: Install via pip + +```bash +python3.12 -m venv vllm-env +source vllm-env/bin/activate + +pip install vllm + +# For specific CUDA version +pip install vllm # CUDA 12.1 by default +# OR +pip install vllm-cu118 # For CUDA 11.8 +``` + +### Option C: Use Docker + +```bash +docker pull vllm/vllm-openai:latest +``` + +## Step 2: Download a Model + +Choose a model based on your GPU VRAM: + +| Model | VRAM Required | Performance | +|-------|---------------|-------------| +| facebook/opt-125m | ~1GB | Fast, good for testing | +| meta-llama/Llama-2-7b-chat-hf | ~14GB | Good quality | +| meta-llama/Meta-Llama-3-8B-Instruct | ~16GB | Better quality | +| meta-llama/Llama-2-13b-chat-hf | ~26GB | High quality | +| meta-llama/Llama-2-70b-chat-hf | ~140GB | Best quality (multi-GPU) | + +### Using Hugging Face + +```bash +# Login to Hugging Face (for gated models like Llama) +pip install huggingface-hub +huggingface-cli login + +# Models will auto-download on first use +# Or pre-download: +huggingface-cli download meta-llama/Meta-Llama-3-8B-Instruct +``` + +## Step 3: Start vLLM Server + +### Basic Start + +```bash +source vllm-env/bin/activate + +python -m vllm.entrypoints.openai.api_server \ + --model facebook/opt-125m \ + --host 0.0.0.0 \ + --port 8001 +``` + +### Production Configuration + +```bash +python -m vllm.entrypoints.openai.api_server \ + --model meta-llama/Meta-Llama-3-8B-Instruct \ + --host 0.0.0.0 \ + --port 8001 \ + --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.9 \ + --max-model-len 4096 \ + --dtype auto \ + --enable-prefix-caching +``` + +### Multi-GPU Configuration + +```bash +# For 4 GPUs +python -m vllm.entrypoints.openai.api_server \ + --model meta-llama/Llama-2-70b-chat-hf \ + --host 0.0.0.0 \ + --port 8001 \ + --tensor-parallel-size 4 \ + --gpu-memory-utilization 0.9 +``` + +### Using Docker + +```bash +docker run --gpus all \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + -p 8001:8000 \ + --env "HUGGING_FACE_HUB_TOKEN=" \ + vllm/vllm-openai:latest \ + --model facebook/opt-125m \ + --host 0.0.0.0 \ + --port 8000 +``` + +## Step 4: Test vLLM Server + +```bash +curl http://localhost:8001/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "facebook/opt-125m", + "messages": [ + {"role": "user", "content": "Hello!"} + ] + }' +``` + +## Step 5: Create systemd Service (Optional) + +For production deployments, run vLLM as a system service: + +```bash +sudo nano /etc/systemd/system/vllm.service +``` + +Add: + +```ini +[Unit] +Description=vLLM Inference Server +After=network.target + +[Service] +Type=simple +User=your-username +WorkingDirectory=/home/your-username +Environment="PATH=/home/your-username/vllm-env/bin" +ExecStart=/home/your-username/vllm-env/bin/python -m vllm.entrypoints.openai.api_server \ + --model meta-llama/Meta-Llama-3-8B-Instruct \ + --host 0.0.0.0 \ + --port 8001 \ + --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.9 +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target +``` + +Enable and start: + +```bash +sudo systemctl daemon-reload +sudo systemctl enable vllm +sudo systemctl start vllm +sudo systemctl status vllm +``` + +## Step 6: Configure Gateway + +### Update Gateway Environment + +Edit your gateway's `.env`: + +```dotenv +# Add if using local vLLM without Globus Compute +LOCAL_VLLM_URL="http://localhost:8001" +``` + +### Create Endpoint Fixture + +Create or edit `fixtures/endpoints.json` in your gateway directory: + +```json +[ + { + "model": "resource_server.endpoint", + "pk": 1, + "fields": { + "endpoint_slug": "local-vllm-opt-125m", + "cluster": "local", + "framework": "vllm", + "model": "facebook/opt-125m", + "api_port": 8001, + "endpoint_uuid": "", + "function_uuid": "", + "batch_endpoint_uuid": "", + "batch_function_uuid": "", + "allowed_globus_groups": "" + } + } +] +``` + +For a Llama model: + +```json +[ + { + "model": "resource_server.endpoint", + "pk": 2, + "fields": { + "endpoint_slug": "local-vllm-llama3-8b", + "cluster": "local", + "framework": "vllm", + "model": "meta-llama/Meta-Llama-3-8B-Instruct", + "api_port": 8001, + "endpoint_uuid": "", + "function_uuid": "", + "batch_endpoint_uuid": "", + "batch_function_uuid": "", + "allowed_globus_groups": "" + } + } +] +``` + +### Load Fixture + +```bash +# Docker +docker-compose exec inference-gateway python manage.py loaddata fixtures/endpoints.json + +# Bare metal +python manage.py loaddata fixtures/endpoints.json +``` + +## Step 7: Test End-to-End + +Get a Globus token: + +```bash +export TOKEN=$(python inference-auth-token.py get_access_token) +``` + +Test via gateway: + +```bash +curl -X POST http://localhost:8000/resource_server/local/vllm/v1/chat/completions \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "facebook/opt-125m", + "messages": [{"role": "user", "content": "Explain machine learning in one sentence"}], + "max_tokens": 50 + }' +``` + +## Performance Tuning + +### GPU Memory Optimization + +```bash +# Use less GPU memory (if OOM errors) +--gpu-memory-utilization 0.8 + +# Use more GPU memory (if you have headroom) +--gpu-memory-utilization 0.95 +``` + +### Batch Processing + +```bash +# Increase batch size for throughput +--max-num-batched-tokens 8192 +--max-num-seqs 256 +``` + +### Context Length + +```bash +# Reduce for better throughput +--max-model-len 2048 + +# Increase for longer contexts +--max-model-len 8192 +``` + +### Quantization + +```bash +# Use 4-bit quantization (AWQ) +--quantization awq +--model TheBloke/Llama-2-7B-Chat-AWQ + +# Use 8-bit quantization (GPTQ) +--quantization gptq +--model TheBloke/Llama-2-7B-Chat-GPTQ +``` + +## Monitoring + +### vLLM Metrics + +vLLM exposes Prometheus metrics at `http://localhost:8001/metrics`: + +```bash +curl http://localhost:8001/metrics +``` + +### GPU Monitoring + +```bash +# Watch GPU usage +watch -n 1 nvidia-smi + +# More detailed stats +nvidia-smi dmon +``` + +### Log Monitoring + +```bash +# systemd service logs +sudo journalctl -u vllm -f + +# Or direct output if running in terminal +python -m vllm.entrypoints.openai.api_server ... | tee vllm.log +``` + +## Troubleshooting + +### Out of Memory (OOM) Errors + +```bash +# Reduce GPU memory usage +--gpu-memory-utilization 0.7 + +# Reduce context length +--max-model-len 2048 + +# Use quantization +--quantization awq + +# Use smaller model +``` + +### Slow Response Times + +- Check GPU utilization with `nvidia-smi` +- Increase `--gpu-memory-utilization` if GPU memory is underutilized +- Enable `--enable-prefix-caching` for repeated prompts +- Use tensor parallelism for multi-GPU setups + +### Model Not Found + +```bash +# Pre-download model +huggingface-cli download meta-llama/Meta-Llama-3-8B-Instruct + +# Or set cache directory +export HF_HOME=/path/to/cache +``` + +### CUDA Errors + +```bash +# Check CUDA version +nvidia-smi + +# Install matching vLLM version +pip install vllm-cu118 # For CUDA 11.8 +``` + +### Connection Refused from Gateway + +- Verify vLLM is running: `curl http://localhost:8001/health` +- Check firewall settings +- Ensure correct port in fixture configuration +- Verify host is `0.0.0.0` not `localhost` + +## Running Multiple Models + +You can run multiple vLLM instances on different ports: + +```bash +# Terminal 1 - OPT-125M +python -m vllm.entrypoints.openai.api_server \ + --model facebook/opt-125m \ + --port 8001 + +# Terminal 2 - Llama-2-7B +python -m vllm.entrypoints.openai.api_server \ + --model meta-llama/Llama-2-7b-chat-hf \ + --port 8002 +``` + +Add both to your fixtures: + +```json +[ + { + "model": "resource_server.endpoint", + "pk": 1, + "fields": { + "endpoint_slug": "local-vllm-opt-125m", + "cluster": "local", + "framework": "vllm", + "model": "facebook/opt-125m", + "api_port": 8001, + ... + } + }, + { + "model": "resource_server.endpoint", + "pk": 2, + "fields": { + "endpoint_slug": "local-vllm-llama2-7b", + "cluster": "local", + "framework": "vllm", + "model": "meta-llama/Llama-2-7b-chat-hf", + "api_port": 8002, + ... + } + } +] +``` + +## Next Steps + +- [Production Best Practices](../deployment/production.md) +- [Monitoring Setup](../monitoring.md) +- [User Guide](../../user-guide/index.md) +- Upgrade to [Globus Compute + vLLM](globus-compute.md) for federated deployment + +## Additional Resources + +- [vLLM Documentation](https://docs.vllm.ai/) +- [Hugging Face Models](https://huggingface.co/models) +- [NVIDIA GPU Documentation](https://docs.nvidia.com/cuda/) + diff --git a/docs/admin-guide/monitoring.md b/docs/admin-guide/monitoring.md new file mode 100644 index 00000000..3a1b3def --- /dev/null +++ b/docs/admin-guide/monitoring.md @@ -0,0 +1,330 @@ +# Monitoring & Troubleshooting + +This guide covers monitoring the FIRST Inference Gateway and troubleshooting common issues. + +## Monitoring + +### Application Logs + +#### Docker Deployment + +```bash +# View all logs +docker-compose logs -f + +# View gateway logs only +docker-compose logs -f inference-gateway + +# Last 100 lines +docker-compose logs --tail=100 inference-gateway +``` + +#### Bare Metal Deployment + +```bash +# Application logs +tail -f logs/django_info.log + +# Gunicorn logs +tail -f logs/backend_gateway.error.log +tail -f logs/backend_gateway.access.log + +# Systemd service logs +sudo journalctl -u inference-gateway -f +``` + +### Database Monitoring + +```bash +# Connection stats +psql -h localhost -U inferencedev -d inferencegateway -c "SELECT * FROM pg_stat_activity;" + +# Table sizes +psql -h localhost -U inferencedev -d inferencegateway -c " +SELECT schemaname,tablename,pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) +FROM pg_tables WHERE schemaname='public' ORDER BY pg_total_relation_size(schemaname||'.'||tablename) DESC;" +``` + +### Redis Monitoring + +```bash +# Connect to Redis CLI +redis-cli + +# In Redis CLI: +INFO +DBSIZE +MONITOR # Watch commands in real-time +``` + +### Globus Compute Endpoints + +```bash +# List endpoints +globus-compute-endpoint list + +# Check status +globus-compute-endpoint status my-endpoint + +# View logs +globus-compute-endpoint log my-endpoint -n 100 + +# Follow logs +tail -f ~/.globus_compute/my-endpoint/endpoint.log +``` + +## Common Issues + +### Gateway Won't Start + +**Symptoms**: Container/service fails to start + +**Check**: + +```bash +# Docker +docker-compose logs inference-gateway + +# Bare metal +sudo journalctl -u inference-gateway -n 50 +python manage.py check +``` + +**Common Causes**: + +- Missing environment variables +- Database connection failure +- Port already in use +- Syntax error in settings + +### Database Connection Errors + +**Symptoms**: `OperationalError: could not connect to server` + +**Solutions**: + +```bash +# Verify PostgreSQL is running +sudo systemctl status postgresql +docker-compose ps postgres + +# Test connection +psql -h localhost -U inferencedev -d inferencegateway + +# Check pg_hba.conf +sudo nano /etc/postgresql/*/main/pg_hba.conf + +# Restart PostgreSQL +sudo systemctl restart postgresql +``` + +### Authentication Failures + +**Symptoms**: 401 Unauthorized, Globus token errors + +**Solutions**: + +1. Verify Globus application credentials in `.env` +2. Check scope was created successfully: + ```bash + curl -s --user $CLIENT_ID:$CLIENT_SECRET \ + https://auth.globus.org/v2/api/clients/$CLIENT_ID + ``` +3. Force re-authentication: + ```bash + python inference-auth-token.py authenticate --force + ``` +4. Verify redirect URIs match in Globus app settings + +### Globus Compute Errors + +**Symptoms**: Function execution failures, timeout errors + +**Solutions**: + +```bash +# Check endpoint is running +globus-compute-endpoint list + +# Restart endpoint +globus-compute-endpoint restart my-endpoint + +# View detailed logs +globus-compute-endpoint log my-endpoint -n 200 + +# Verify function UUID is allowed +cat ~/.globus_compute/my-endpoint/config.yaml +``` + +### Model Not Found + +**Symptoms**: `Model 'xxx' not found` errors + +**Solutions**: + +1. Verify fixture was loaded: + ```bash + python manage.py dumpdata resource_server.endpoint + ``` +2. Check model name matches exactly in fixture +3. Reload fixtures: + ```bash + python manage.py loaddata fixtures/endpoints.json + ``` + +### Slow Response Times + +**Causes**: + +- Cold start (first request to endpoint) +- GPU not available +- Model loading time +- Network latency + +**Solutions**: + +1. Enable hot nodes (min_blocks > 0 in Globus Compute config) +2. Monitor GPU usage: `nvidia-smi` +3. Check vLLM logs for bottlenecks +4. Increase Gunicorn timeout: + ```python + timeout = 300 + ``` + +### Out of Memory Errors + +**Symptoms**: OOM kills, CUDA out of memory + +**Solutions**: + +```bash +# vLLM: Reduce GPU memory usage +--gpu-memory-utilization 0.7 + +# vLLM: Use quantization +--quantization awq + +# vLLM: Reduce context length +--max-model-len 2048 + +# System: Add swap space +sudo fallocate -l 32G /swapfile +sudo chmod 600 /swapfile +sudo mkswap /swapfile +sudo swapon /swapfile +``` + +## Health Checks + +### Manual Health Checks + +```bash +# Gateway health +curl http://localhost:8000/ + +# vLLM health +curl http://localhost:8001/health + +# Database connectivity +python manage.py dbshell + +# Redis connectivity +redis-cli ping +``` + +### Automated Health Monitoring + +Create a health check script: + +```bash +#!/bin/bash +# health_check.sh + +# Check gateway +if curl -s http://localhost:8000/ > /dev/null; then + echo "βœ“ Gateway is healthy" +else + echo "βœ— Gateway is down" + systemctl restart inference-gateway +fi + +# Check database +if psql -h localhost -U inferencedev -d inferencegateway -c "SELECT 1;" > /dev/null 2>&1; then + echo "βœ“ Database is healthy" +else + echo "βœ— Database is down" +fi +``` + +Add to crontab: + +```bash +*/5 * * * * /path/to/health_check.sh >> /var/log/health_check.log 2>&1 +``` + +## Performance Metrics + +### Key Metrics to Monitor + +1. **Request Rate**: Requests per second +2. **Latency**: Response time (p50, p95, p99) +3. **Error Rate**: Percentage of failed requests +4. **Queue Depth**: Pending Globus Compute tasks +5. **GPU Utilization**: GPU memory and compute usage +6. **Database Connections**: Active connections +7. **Cache Hit Rate**: Redis cache effectiveness + +### Prometheus Metrics + +If using Prometheus, key metrics to track: + +``` +# Request metrics +http_requests_total +http_request_duration_seconds + +# Globus Compute metrics +globus_compute_tasks_submitted +globus_compute_tasks_completed +globus_compute_tasks_failed + +# System metrics +process_cpu_seconds_total +process_resident_memory_bytes +``` + +## Troubleshooting Checklist + +When issues occur, work through this checklist: + +- [ ] Check application logs +- [ ] Verify all services are running +- [ ] Test database connectivity +- [ ] Check Redis connectivity +- [ ] Verify Globus Compute endpoints are online +- [ ] Test authentication flow +- [ ] Check network connectivity +- [ ] Review recent configuration changes +- [ ] Check disk space +- [ ] Monitor resource usage (CPU, RAM, GPU) + +## Getting Help + +If you're still stuck: + +1. **Check documentation**: Review the relevant setup guides +2. **Search issues**: Look for similar issues on [GitHub](https://github.com/auroraGPT-ANL/inference-gateway/issues) +3. **Enable debug logging**: Set `DEBUG=True` temporarily +4. **Collect information**: + - Version information + - Error messages and stack traces + - Configuration (sanitize secrets!) + - Relevant log excerpts +5. **Open an issue**: Provide all collected information + +## Additional Resources + +- [Production Best Practices](deployment/production.md) +- [Configuration Reference](gateway-setup/configuration.md) +- [Globus Compute Documentation](https://globus-compute.readthedocs.io/) + diff --git a/docs/first_architecture.png b/docs/first_architecture.png new file mode 100644 index 00000000..b8078a45 Binary files /dev/null and b/docs/first_architecture.png differ diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..b612b856 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,64 @@ +# FIRST + +Welcome to the documentation for the **Federated Inference Resource Scheduling Toolkit (FIRST)**. FIRST enables LLM inference as a service across distributed HPC clusters through an OpenAI-compatible API. + +## What is FIRST? + +FIRST (Federated Inference Resource Scheduling Toolkit) is a system that allows secure, remote execution of Large Language Models through an OpenAI-compatible API. It validates and authorizes inference requests to scientific computing clusters using Globus Auth and Globus Compute. + +## System Architecture + +![System Architecture](first_architecture.png) + +The Inference Gateway consists of several components: + +- **API Gateway**: Django-based REST/Ninja API that handles authorization and request routing +- **Globus Auth**: Authentication and authorization service +- **Globus Compute Endpoints**: Remote execution framework on HPC clusters (or local machines) +- **Inference Server Backend**: High-performance inference service for LLMs (e.g., vLLM) + +## Quick Links + +### For Administrators + +- **[Docker Deployment](admin-guide/gateway-setup/docker.md)** - Fast-track Docker deployment in under 10 minutes +- **[Bare Metal Setup](admin-guide/gateway-setup/bare-metal.md)** - Complete installation on your own infrastructure +- **[Inference Backend](admin-guide/inference-setup/index.md)** - Connect to OpenAI API, local vLLM, or Globus Compute +- **[Kubernetes](admin-guide/deployment/kubernetes.md)** - Deploy on Kubernetes clusters (Coming Soon) + +### For Users + +- **[User Guide](user-guide/index.md)** - Complete guide for authentication and making requests +- **[API Reference](reference/api.md)** - API endpoint documentation +- **[Examples](user-guide/index.md#using-the-openai-python-sdk)** - Code examples and tutorials + +## Key Features + +- **Federated Access**: Route requests across multiple HPC clusters automatically +- **OpenAI-Compatible**: Works with existing OpenAI SDK and tools +- **Secure**: Globus Auth integration with group-based access control +- **High Performance**: Support for vLLM and other optimized inference backends +- **Flexible**: Deploy via Docker, bare metal, or Kubernetes +- **Scalable**: Auto-scaling and resource management for HPC environments + +## Example Deployment + +For a production example, see the [ALCF Inference Endpoints](https://github.com/argonne-lcf/inference-endpoints) documentation. + +## Getting Help + +- **GitHub**: [Report issues or contribute](https://github.com/auroraGPT-ANL/inference-gateway) +- **Citation**: [Research Paper](reference/citation.md) +- **API Reference**: [Complete API documentation](reference/api.md) + +## License + +This project is licensed under the Apache License 2.0. See the [LICENSE](https://github.com/auroraGPT-ANL/inference-gateway/blob/main/LICENSE) file for details. + +--- + +!!! tip "Quick Start Paths" + - **Just want to try it out?** β†’ [Docker Quickstart](admin-guide/gateway-setup/docker.md) + - **Need full control?** β†’ [Bare Metal Setup](admin-guide/gateway-setup/bare-metal.md) + - **Want to use the API?** β†’ [User Guide](user-guide/index.md) + diff --git a/docs/reference/api.md b/docs/reference/api.md new file mode 100644 index 00000000..2f82093a --- /dev/null +++ b/docs/reference/api.md @@ -0,0 +1,47 @@ +# API Reference + +The FIRST Inference Gateway provides an OpenAI-compatible API. + +## Base URL + +``` +http://your-gateway-domain:8000/resource_server +``` + +## Authentication + +All requests require a Globus access token in the Authorization header: + +```http +Authorization: Bearer +``` + +## Endpoints + +### Chat Completions + +```http +POST /v1/chat/completions +POST /{cluster}/{framework}/v1/chat/completions +``` + +### Completions + +```http +POST /v1/completions +POST /{cluster}/{framework}/v1/completions +``` + +### Batch Processing + +```http +POST /v1/batches +GET /v1/batches/{batch_id} +``` + +For detailed API documentation, refer to the [OpenAI API Reference](https://platform.openai.com/docs/api-reference) as FIRST follows the same schema. + +## Request Parameters + +See the [User Guide](../user-guide/index.md#request-parameters) for detailed parameter documentation. + diff --git a/docs/reference/citation.md b/docs/reference/citation.md new file mode 100644 index 00000000..c6776ea3 --- /dev/null +++ b/docs/reference/citation.md @@ -0,0 +1,39 @@ +# Citation + +If you use ALCF Inference Endpoints or the Federated Inference Resource Scheduling Toolkit (FIRST) in your research or workflows, please cite our paper: + +## BibTeX + +```bibtex +@inproceedings{10.1145/3731599.3767346, + author = {Tanikanti, Aditya and C\^{o}t\'{e}, Benoit and Guo, Yanfei and Chen, Le and Saint, Nickolaus and Chard, Ryan and Raffenetti, Ken and Thakur, Rajeev and Uram, Thomas and Foster, Ian and Papka, Michael E. and Vishwanath, Venkatram}, + title = {FIRST: Federated Inference Resource Scheduling Toolkit for Scientific AI Model Access}, + year = {2025}, + isbn = {9798400718717}, + publisher = {Association for Computing Machinery}, + address = {New York, NY, USA}, + url = {https://doi.org/10.1145/3731599.3767346}, + doi = {10.1145/3731599.3767346}, + abstract = {We present the Federated Inference Resource Scheduling Toolkit (FIRST), a framework enabling Inference-as-a-Service across distributed High-Performance Computing (HPC) clusters. FIRST provides cloud-like access to diverse AI models, like Large Language Models (LLMs), on existing HPC infrastructure. Leveraging Globus Auth and Globus Compute, the system allows researchers to run parallel inference workloads via an OpenAI-compliant API on private, secure environments. This cluster-agnostic API allows requests to be distributed across federated clusters, targeting numerous hosted models. FIRST supports multiple inference backends (e.g., vLLM), auto-scales resources, maintains "hot" nodes for low-latency execution, and offers both high-throughput batch and interactive modes. The framework addresses the growing demand for private, secure, and scalable AI inference in scientific workflows, allowing researchers to generate billions of tokens daily on-premises without relying on commercial cloud infrastructure.}, + booktitle = {Proceedings of the SC '25 Workshops of the International Conference for High Performance Computing, Networking, Storage and Analysis}, + pages = {52–60}, + numpages = {9}, + keywords = {Inference as a Service, High Performance Computing, Job Schedulers, Large Language Models, Globus, Scientific Computing}, + series = {SC Workshops '25} +} +``` + +## Acknowledgements + +This work was supported by the U.S. Department of Energy, Office of Science, Office of Advanced Scientific Computing Research, under Contract No. DE-AC02-06CH11357. + +This research used resources of the Argonne Leadership Computing Facility, which is a DOE Office of Science User Facility. + +## Related Publications + +For more information about FIRST and related work: + +- [Research Paper](https://doi.org/10.1145/3731599.3767346) +- [ALCF Inference Endpoints](https://github.com/argonne-lcf/inference-endpoints) +- [GitHub Repository](https://github.com/auroraGPT-ANL/inference-gateway) + diff --git a/docs/reference/config.md b/docs/reference/config.md new file mode 100644 index 00000000..7d69ce47 --- /dev/null +++ b/docs/reference/config.md @@ -0,0 +1,4 @@ +# Configuration Options + +For a complete list of configuration options, see the [Configuration Reference](../admin-guide/gateway-setup/configuration.md). + diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md new file mode 100644 index 00000000..3f56c678 --- /dev/null +++ b/docs/user-guide/index.md @@ -0,0 +1,347 @@ +# Usage Guide + +This guide explains how to use the FIRST Inference Gateway to access Large Language Models through an OpenAI-compatible API. + +## Authentication + +The Gateway uses Globus Auth for authentication. You need a valid access token to make requests. + +### Getting Your Access Token + +Use the provided authentication script: + +#### First-Time Setup + +```bash +# Authenticate (opens browser for Globus login) +python inference-auth-token.py authenticate +``` + +This stores refresh and access tokens locally (typically in `~/.globus/app/...`). + +#### Getting an Access Token + +```bash +# Retrieve your current valid access token +export MY_TOKEN=$(python inference-auth-token.py get_access_token) +echo "Token stored in MY_TOKEN environment variable." +``` + +The script automatically refreshes expired tokens using your stored refresh token. + +#### Force Re-authentication + +If you need to change accounts or encounter permission errors: + +```bash +# Log out from Globus +# Visit: https://app.globus.org/logout + +# Force re-authentication +python inference-auth-token.py authenticate --force +``` + +#### Token Validity + +- **Access tokens**: Valid for 48 hours +- **Refresh tokens**: Valid for 6 months of inactivity +- Some institutions may enforce more frequent re-authentication (e.g., weekly) + +--- + +## Making Inference Requests + +The Gateway provides an OpenAI-compatible API with two main endpoints: + +### Chat Completions Endpoint + +For conversational interactions: + +**Federated endpoint (routes across multiple backends):** + +```bash +curl -X POST http://127.0.0.1:8000/resource_server/v1/chat/completions \ + -H "Authorization: Bearer $MY_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "facebook/opt-125m", + "messages": [ + {"role": "user", "content": "Explain the concept of Globus Compute in simple terms."} + ], + "max_tokens": 150 + }' +``` + +**Specific backend (targets a particular cluster/framework):** + +```bash +curl -X POST http://127.0.0.1:8000/resource_server/local/vllm/v1/chat/completions \ + -H "Authorization: Bearer $MY_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "facebook/opt-125m", + "messages": [ + {"role": "user", "content": "Explain the concept of Globus Compute in simple terms."} + ], + "max_tokens": 150 + }' +``` + +### Completions Endpoint + +For text completion: + +```bash +curl -X POST http://127.0.0.1:8000/resource_server/v1/completions \ + -H "Authorization: Bearer $MY_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "facebook/opt-125m", + "prompt": "The future of AI is", + "max_tokens": 100 + }' +``` + +### Streaming Responses + +Both endpoints support streaming. Set `stream: true` in your request: + +**Python example with streaming:** + +```python +import openai + +client = openai.OpenAI( + base_url="http://localhost:8000/resource_server/v1", + api_key=MY_TOKEN # Your Globus access token +) + +stream = client.chat.completions.create( + model="facebook/opt-125m", + messages=[{"role": "user", "content": "Explain quantum computing"}], + stream=True +) + +for chunk in stream: + if chunk.choices[0].delta.content is not None: + print(chunk.choices[0].delta.content, end="", flush=True) +``` + +### Testing Streaming with ngrok + +For testing streaming with remote endpoints, use ngrok to create a secure tunnel: + +1. **Install ngrok**: Visit [ngrok.com](https://ngrok.com/) + +2. **Start tunnel**: +```bash +ngrok http 8000 +``` + +3. **Update your test script** to use the ngrok URL: +```python +client = openai.OpenAI( + base_url="https://your-ngrok-url.ngrok.io/resource_server/v1", + api_key=access_token +) +``` + +--- + +## Using the OpenAI Python SDK + +The Gateway is fully compatible with the OpenAI Python SDK: + +```python +import openai + +# Configure client to point to the Gateway +client = openai.OpenAI( + base_url="http://localhost:8000/resource_server/v1", + api_key=MY_TOKEN # Your Globus access token +) + +# Make a request +response = client.chat.completions.create( + model="meta-llama/Meta-Llama-3-8B-Instruct", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is machine learning?"} + ], + max_tokens=200 +) + +print(response.choices[0].message.content) +``` + +--- + +## Available Models + +To see available models, contact your Gateway administrator or check the deployment's model catalog. + +Common model naming conventions: +- `facebook/opt-125m`, `facebook/opt-1.3b`, etc. +- `meta-llama/Meta-Llama-3-8B-Instruct` +- `openai/gpt-4o-mini` (if configured with OpenAI backend) + +The exact models available depend on your deployment's configuration. + +--- + +## Batch Processing + +For large-scale inference workloads, the Gateway supports batch processing: + +```bash +curl -X POST http://127.0.0.1:8000/resource_server/v1/batches \ + -H "Authorization: Bearer $MY_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "input_file_path": "/path/to/input.jsonl", + "output_folder_path": "/path/to/output/", + "model": "meta-llama/Meta-Llama-3-8B-Instruct" + }' +``` + +**Input file format (JSONL):** + +```jsonl +{"messages": [{"role": "user", "content": "What is AI?"}], "max_tokens": 100} +{"messages": [{"role": "user", "content": "Explain ML"}], "max_tokens": 100} +``` + +**Check batch status:** + +```bash +curl -X GET http://127.0.0.1:8000/resource_server/v1/batches/ \ + -H "Authorization: Bearer $MY_TOKEN" +``` + +--- + +## Benchmarking + +The repository includes a benchmark script for performance testing: + +### Download Dataset + +```bash +wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json -P examples/load-testing/ +``` + +### Run Benchmark + +```bash +python examples/load-testing/benchmark-serving.py \ + --backend vllm \ + --model facebook/opt-125m \ + --base-url http://127.0.0.1:8000/resource_server/v1/chat/completions \ + --dataset-name sharegpt \ + --dataset-path examples/load-testing/ShareGPT_V3_unfiltered_cleaned_split.json \ + --output-file benchmark_results.jsonl \ + --num-prompts 100 +``` + +**Additional options:** +- `--request-rate`: Control request rate (requests per second) +- `--max-concurrency`: Limit concurrent requests +- `--disable-ssl-verification`: For testing with self-signed certificates +- `--disable-stream`: Test non-streaming mode + +--- + +## Request Parameters + +### Common Parameters + +| Parameter | Type | Description | +|-----------|------|-------------| +| `model` | string | Model identifier (required) | +| `messages` | array | List of message objects (chat endpoint) | +| `prompt` | string | Text prompt (completions endpoint) | +| `max_tokens` | integer | Maximum tokens to generate | +| `temperature` | float | Sampling temperature (0.0-2.0) | +| `top_p` | float | Nucleus sampling parameter | +| `stream` | boolean | Enable streaming responses | +| `stop` | string/array | Stop sequences | + +### Example with Advanced Parameters + +```bash +curl -X POST http://127.0.0.1:8000/resource_server/v1/chat/completions \ + -H "Authorization: Bearer $MY_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "meta-llama/Meta-Llama-3-8B-Instruct", + "messages": [{"role": "user", "content": "Write a story"}], + "max_tokens": 500, + "temperature": 0.7, + "top_p": 0.9, + "stream": false, + "stop": ["\n\n"] + }' +``` + +--- + +## Error Handling + +The Gateway returns standard HTTP status codes: + +| Code | Meaning | +|------|---------| +| 200 | Success | +| 400 | Bad Request (invalid parameters) | +| 401 | Unauthorized (invalid or missing token) | +| 403 | Forbidden (insufficient permissions) | +| 404 | Not Found (model or endpoint not available) | +| 429 | Too Many Requests (rate limited) | +| 500 | Internal Server Error | +| 503 | Service Unavailable (backend offline) | + +**Error response format:** + +```json +{ + "error": { + "message": "Model not found", + "type": "invalid_request_error", + "code": "model_not_found" + } +} +``` + +--- + +## Best Practices + +1. **Token Management** + - Store tokens securely + - Refresh tokens before they expire + - Never commit tokens to version control + +2. **Request Optimization** + - Use appropriate `max_tokens` values + - Implement retry logic with exponential backoff + - Use streaming for long responses + +3. **Rate Limiting** + - Respect rate limits + - Implement client-side throttling + - Use batch processing for bulk operations + +4. **Error Handling** + - Handle network errors gracefully + - Implement timeout logic + - Log errors for debugging + +--- + +## Support + +For issues, questions, or feature requests: +- Check the [GitHub repository](https://github.com/auroraGPT-ANL/inference-gateway) +- Contact your Gateway administrator +- Review the [Administrator Setup Guide](../admin-guide/index.md) for deployment issues + diff --git a/env.example b/env.example new file mode 100644 index 00000000..847e793c --- /dev/null +++ b/env.example @@ -0,0 +1,72 @@ +# --- Core Django Settings --- +SECRET_KEY="" +DEBUG=True + +# --- Allowed hosts --- +# Optional: one per line (default to "*") +ALLOWED_HOSTS=" +localhost +127.0.0.1 +" + +# --- Test Mode (set to True to skip Globus policy checks and bypass rate-limiting during development) --- +RUNNING_AUTOMATED_TEST_SUITE=False + +# --- Logging --- +# Set to True to send application logs to stdout/stderr (recommended for Docker) +LOG_TO_STDOUT=True + +# --- Globus Credentials --- +# Required +GLOBUS_APPLICATION_ID="" +GLOBUS_APPLICATION_SECRET="" +SERVICE_ACCOUNT_ID="" +SERVICE_ACCOUNT_SECRET="" + +# --- Globus Policies --- +# Required: comma-separated list of policy IDs (at least one must be provided) +# Example: GLOBUS_POLICIES="policy-id-1,policy-id-2" +GLOBUS_POLICIES="" + +# --- Globus Groups --- +# Optional: comma-separated list of group IDs +GLOBUS_GROUPS="" + +# --- Authorized Identity Provider Domains (one per line) --- +# Required: one per line (cannot be empty, must match Globus policy) +# Example: AUTHORIZED_IDP_DOMAINS=" +# anl.gov, +# uchicago.edu +# " +AUTHORIZED_IDP_DOMAINS="" + +# --- Authorized Groups Per IDP --- +# Optional: JSON format with domains (keys: domains, values: comma-separated list of group IDs) +# Example: AUTHORIZED_GROUPS_PER_IDP=' +# { +# "uchicago.edu": "groupd-id-1" +# } +# ' +AUTHORIZED_GROUPS_PER_IDP='{}' + +# --- Postgres Database --- +POSTGRES_DB="inferencegateway" +POSTGRES_USER="inferencedev" +POSTGRES_PASSWORD="change-this-password" +PGHOST="postgres" +PGPORT=5432 +PGUSER="inferencedev" +PGPASSWORD="change-this-password" +PGDATABASE="inferencegateway" + +# --- Redis Cache --- +REDIS_URL="redis://redis:6379/0" +USE_REDIS_CACHE=true + +# --- Gateway Specific Settings --- +MAX_BATCHES_PER_USER=2 +RATE_LIMIT_PER_SEC_PER_USER=10 +STREAMING_SERVER_HOST="localhost:8080" +INTERNAL_STREAMING_SECRET="your-internal-streaming-secret-key" + + diff --git a/generate_auth_token.py b/generate_auth_token.py deleted file mode 100644 index 2a2dc803..00000000 --- a/generate_auth_token.py +++ /dev/null @@ -1,36 +0,0 @@ -import globus_sdk -import requests -import json - -# Globus ID and Scope -# =================== - -# Define your Globus thick-client ID -# https://app.globus.org/settings/developers -# This needs to be created by users (they can also re-use one of the existing clients) -auth_client_id = "58fdd3bc-e1c3-4ce5-80ea-8d6b87cfb944" - -# Define the inference-gateway resource service scope -# This will be publicly available to users -gateway_client_id = "681c10cc-f684-4540-bcd7-0b4df3bc26ef" -gateway_scope = f"https://auth.globus.org/scopes/{gateway_client_id}/action_all" - -# Authentication and Access Token -# =============================== - -# Start an Auth client with the vLLM scope -auth_client = globus_sdk.NativeAppAuthClient(auth_client_id) -auth_client.oauth2_start_flow(requested_scopes=gateway_scope) - -# Authenticate with your Globus account -authorize_url = auth_client.oauth2_get_authorize_url() -print(f"Please go to this URL and login:\n\n{authorize_url}\n") - -# Get the code from the command line -auth_code = input("Enter the code you just received: ") - -# Exchange the code for an access token -# Collect access token to vLLM service -token_response = auth_client.oauth2_exchange_code_for_tokens(auth_code) -access_token = token_response.by_resource_server[gateway_client_id]["access_token"] -print(f"Access token: {access_token}") \ No newline at end of file diff --git a/inference-auth-token.py b/inference-auth-token.py deleted file mode 100644 index 9daf56e0..00000000 --- a/inference-auth-token.py +++ /dev/null @@ -1,172 +0,0 @@ -import globus_sdk -from globus_sdk.login_flows import LocalServerLoginFlowManager # Needed to access globus_sdk.gare -import os -from dotenv import load_dotenv -import json # For parsing domains list -import sys # Import sys to redirect print output - -# Load .env variables first -load_dotenv() - -# --- Configuration --- -# Globus UserApp name (can remain constant or be made configurable if needed via .env) -APP_NAME = os.getenv("CLI_APP_NAME", "inference_app") # Default to original name - -# Public client ID for this CLI authentication flow (reads from .env, falls back to original default) -CLI_AUTH_CLIENT_ID = os.getenv("CLI_AUTH_CLIENT_ID", "58fdd3bc-e1c3-4ce5-80ea-8d6b87cfb944") - -# Inference gateway Application Client ID (reads from .env, MUST BE SET) -GATEWAY_CLIENT_ID = os.getenv("GLOBUS_APPLICATION_ID") -if not GATEWAY_CLIENT_ID: - raise Exception("GLOBUS_APPLICATION_ID must be set in the .env file.") - -# Define the Service API scope -GATEWAY_SCOPE = f"https://auth.globus.org/scopes/{GATEWAY_CLIENT_ID}/action_all" - -# Path where access and refresh tokens are stored (uses original logic based on CLI_AUTH_CLIENT_ID and APP_NAME) -# Ensure the parent directory exists -TOKENS_DIR = os.path.expanduser(f"~/.globus/app/{CLI_AUTH_CLIENT_ID}/{APP_NAME}") -os.makedirs(TOKENS_DIR, exist_ok=True) # Create dir if it doesn't exist -TOKENS_PATH = os.path.join(TOKENS_DIR, "tokens.json") # Original path construction - - -# Allowed identity provider domains (reads from .env, falls back to original default) -# Example: CLI_ALLOWED_DOMAINS="anl.gov,alcf.anl.gov" -allowed_domains_str = os.getenv("CLI_ALLOWED_DOMAINS", "anl.gov,alcf.anl.gov") # Default to original list -ALLOWED_DOMAINS = [domain.strip() for domain in allowed_domains_str.split(',') if domain.strip()] - -# Globus authorizer parameters to point to specific identity providers -if ALLOWED_DOMAINS: - GA_PARAMS = globus_sdk.gare.GlobusAuthorizationParameters(session_required_single_domain=ALLOWED_DOMAINS) -else: - GA_PARAMS = None - print("INFO: No domain restrictions specified for login.", file=sys.stderr) # To stderr - - -# --- Original Logic --- - -# Error handler to guide user through specific identity providers -class DomainBasedErrorHandler: - def __call__(self, app, error): - print(f"Encountered error '{error}', initiating login...", file=sys.stderr) # To stderr - app.login(auth_params=GA_PARAMS) - - -# Get refresh authorizer object -def get_auth_object(force=False): - """ - Create a Globus UserApp with the inference service scope - and trigger the authentication process. If authentication - has already happened, existing tokens will be reused. - Uses default token storage mechanisms. - """ - - # Create Globus user application - # NOTE: Not specifying token_storage uses the default behavior which - # should align with the standard path defined in TOKENS_PATH. - app = globus_sdk.UserApp( - APP_NAME, - client_id=CLI_AUTH_CLIENT_ID, - scope_requirements={GATEWAY_CLIENT_ID: [GATEWAY_SCOPE]}, - config=globus_sdk.GlobusAppConfig( - request_refresh_tokens=True, - token_validation_error_handler=DomainBasedErrorHandler() - ), - ) - - # Force re-login if required - if force: - # Original script just calls login, which handles clearing old tokens internally - print("INFO: Forcing new login (clearing existing tokens implicitly)...", file=sys.stderr) # To stderr - app.login(auth_params=GA_PARAMS) - - # Authenticate using your Globus account or reuse existing tokens - auth = app.get_authorizer(GATEWAY_CLIENT_ID) - - # Return the Globus refresh token authorizer - return auth - - -# Get access token -def get_access_token(): - """ - Load existing tokens, refresh the access token if necessary, - and return the valid access token. If there is no token stored - in the default location, or if the refresh token is expired following - inactivity, an authentication will be triggered. - """ - - # Get authorizer object and authenticate if need be - auth = get_auth_object() # force=False - - # Make sure the stored access token if valid, and refresh otherwise - try: - auth.ensure_valid_token() - except globus_sdk.AuthAPIError as e: - print(f"Error ensuring token validity: {e}", file=sys.stderr) # To stderr - print("Attempting re-authentication...", file=sys.stderr) # To stderr - auth = get_auth_object(force=True) # Force re-auth if refresh fails - auth.ensure_valid_token() # Try again - - # Return the access token - return auth.access_token - - -# If this file is executed as a script ... -if __name__ == "__main__": - - # Imports - import argparse - - # Exception to raise in case of errors - class InferenceAuthError(Exception): - pass - - # Constant - AUTHENTICATE_ACTION = "authenticate" - GET_ACCESS_TOKEN_ACTION = "get_access_token" - - # Define possible arguments - parser = argparse.ArgumentParser(description="Authenticate with Globus and get access tokens for the Inference Gateway.") - parser.add_argument('action', choices=[AUTHENTICATE_ACTION, GET_ACCESS_TOKEN_ACTION], help="Action to perform: 'authenticate' or 'get_access_token'.") - parser.add_argument("-f", "--force", action="store_true", help="Force re-authentication, ignoring existing tokens.") - args = parser.parse_args() - - # Authentication - if args.action == AUTHENTICATE_ACTION: - print("Attempting authentication...", file=sys.stderr) # To stderr - try: - # Authenticate using Globus account (original logic) - _ = get_auth_object(force=args.force) - # Info message to stderr - print(f"Authentication successful. Tokens stored/updated in default location derived from Client ID and App Name (likely near {TOKENS_PATH}).", file=sys.stderr) - except Exception as e: - # Error message to stderr - print(f"Authentication failed: {e}", file=sys.stderr) - raise InferenceAuthError(f"Authentication process failed. Details: {e}") - - # Get token - elif args.action == GET_ACCESS_TOKEN_ACTION: - - # Make sure tokens exist (original check based on constructed path) - if not os.path.isfile(TOKENS_PATH): - # Error message to stderr - print('Access token file not found at expected location. Please authenticate first by running:', file=sys.stderr) - print('python inference-auth_token.py authenticate', file=sys.stderr) - exit(1) - - # Make sure no force flag was provided (original check) - if args.force: - # argparse handles printing error to stderr and exiting - parser.error("The --force flag should be used with the 'authenticate' action, not 'get_access_token'. To force re-authentication, run: python inference_auth_token.py authenticate --force") - - try: - # Load tokens, refresh token if necessary, and print access token - access_token = get_access_token() - print(access_token) # <<< Print the token itself to STDOUT - except Exception as e: - # Error message to stderr - print(f"Failed to get access token: {e}", file=sys.stderr) - print("You might need to re-authenticate:", file=sys.stderr) - print("python inference_auth_token.py authenticate --force", file=sys.stderr) - raise InferenceAuthError(f"Failed to retrieve access token. Details: {e}") \ No newline at end of file diff --git a/inference_gateway/apps.py b/inference_gateway/apps.py index 1945d6c1..e7ed0c29 100644 --- a/inference_gateway/apps.py +++ b/inference_gateway/apps.py @@ -27,7 +27,7 @@ def ready(self): raise ImproperlyConfigured("AUTHORIZED_IDP_DOMAINS cannot be empty.") # Recover the Globus policy - client = globus_sdk.ConfidentialAppAuthClient(settings.POLARIS_ENDPOINT_ID, settings.POLARIS_ENDPOINT_SECRET) + client = globus_sdk.ConfidentialAppAuthClient(settings.SERVICE_ACCOUNT_ID, settings.SERVICE_ACCOUNT_SECRET) token_response = client.oauth2_client_credentials_tokens() globus_auth_token = token_response.by_resource_server["auth.globus.org"]["access_token"] auth_client = globus_sdk.AuthClient(authorizer=globus_sdk.AccessTokenAuthorizer(globus_auth_token)) diff --git a/inference_gateway/settings.py b/inference_gateway/settings.py index 1e0c3ece..f0d67430 100644 --- a/inference_gateway/settings.py +++ b/inference_gateway/settings.py @@ -34,8 +34,8 @@ # Globus App credentials GLOBUS_APPLICATION_ID = os.getenv("GLOBUS_APPLICATION_ID") GLOBUS_APPLICATION_SECRET = os.getenv("GLOBUS_APPLICATION_SECRET") -POLARIS_ENDPOINT_ID = os.getenv("POLARIS_ENDPOINT_ID") -POLARIS_ENDPOINT_SECRET = os.getenv("POLARIS_ENDPOINT_SECRET") +SERVICE_ACCOUNT_ID = os.getenv("SERVICE_ACCOUNT_ID") +SERVICE_ACCOUNT_SECRET = os.getenv("SERVICE_ACCOUNT_SECRET") GLOBUS_GROUP_MANAGER_ID = os.getenv("GLOBUS_GROUP_MANAGER_ID", "") GLOBUS_GROUP_MANAGER_SECRET = os.getenv("GLOBUS_GROUP_MANAGER_SECRET", "") @@ -119,14 +119,13 @@ domains_string = [domain for domain in AUTHORIZED_IDP_DOMAINS if not domain in AUTHORIZED_GROUPS_PER_IDP] AUTHORIZED_IDP_DOMAINS_STRING = ", ".join(domains_string) + ", or providers with approved projects" -# THIS SHOULD BE CHANGED -ALLOWED_HOSTS = ["*"] +# Allowed hosts +ALLOWED_HOSTS = textfield_to_strlist(os.getenv("ALLOWED_HOSTS", "*")) APPEND_SLASH = False # Application definition - INSTALLED_APPS = [ 'django.contrib.admin', 'django.contrib.auth', diff --git a/logging_config.py b/logging_config.py index 86b35ede..7a826617 100644 --- a/logging_config.py +++ b/logging_config.py @@ -1,13 +1,62 @@ import os -# Define the location of the log files (same as in gunicorn_asgi.config.py) +# Determine logging behaviour for different environments environment = os.getenv("ENV", "production") -if environment == "development": - accesslog = "./logs/backend_gateway.access.log" - errorlog = "./logs/backend_gateway.error.log" +log_to_stdout = os.getenv("LOG_TO_STDOUT", "false").lower() in ("true", "1", "t") + +handlers = {} +logger_handlers = { + "uvicorn.error": None, + "uvicorn.access": None, + "resource_server_async": None, + "utils": None, +} + +if log_to_stdout: + handlers["default"] = { + "formatter": "default", + "class": "logging.StreamHandler", + "stream": "ext://sys.stderr", + } + handlers["access"] = { + "formatter": "access", + "class": "logging.StreamHandler", + "stream": "ext://sys.stdout", + } + + logger_handlers["uvicorn.error"] = ["default"] + logger_handlers["uvicorn.access"] = ["access"] + logger_handlers["resource_server_async"] = ["default"] + logger_handlers["utils"] = ["default"] + root_handlers = ["default"] else: - accesslog = "/var/log/inference-service/backend_gateway.access.log" - errorlog = "/var/log/inference-service/backend_gateway.error.log" + if environment == "development": + accesslog = "./logs/backend_gateway.access.log" + errorlog = "./logs/backend_gateway.error.log" + else: + accesslog = "/var/log/inference-service/backend_gateway.access.log" + errorlog = "/var/log/inference-service/backend_gateway.error.log" + + handlers["default"] = { + "formatter": "default", + "class": "logging.FileHandler", + "filename": errorlog, + } + handlers["access"] = { + "formatter": "access", + "class": "logging.FileHandler", + "filename": accesslog, + } + handlers["console"] = { + "formatter": "default", + "class": "logging.StreamHandler", + } + + logger_handlers["uvicorn.error"] = ["default"] + logger_handlers["uvicorn.access"] = ["access"] + logger_handlers["resource_server_async"] = ["default", "console"] + logger_handlers["utils"] = ["default", "console"] + root_handlers = ["default", "console"] LOGGING_CONFIG = { @@ -25,30 +74,27 @@ "datefmt": "%Y-%m-%d %H:%M:%S", }, }, - "handlers": { - "default": { - "formatter": "default", - "class": "logging.FileHandler", - "filename": errorlog, + "handlers": handlers, + "loggers": { + "uvicorn.error": {"handlers": logger_handlers["uvicorn.error"], "level": "INFO"}, + "uvicorn.access": { + "handlers": logger_handlers["uvicorn.access"], + "level": "INFO", + "propagate": False, }, - "access": { - "formatter": "access", - "class": "logging.FileHandler", - "filename": accesslog, + "resource_server_async": { + "handlers": logger_handlers["resource_server_async"], + "level": "INFO", + "propagate": False, }, - "console": { - "formatter": "default", - "class": "logging.StreamHandler", + "utils": { + "handlers": logger_handlers["utils"], + "level": "INFO", + "propagate": False, }, }, - "loggers": { - "uvicorn.error": {"handlers": ["default"], "level": "INFO"}, - "uvicorn.access": {"handlers": ["access"], "level": "INFO", "propagate": False}, - "resource_server_async": {"handlers": ["default", "console"], "level": "INFO", "propagate": False}, - "utils": {"handlers": ["default", "console"], "level": "INFO", "propagate": False}, - }, "root": { "level": "INFO", - "handlers": ["default", "console"], + "handlers": root_handlers, }, } \ No newline at end of file diff --git a/materialized_views.sql b/materialized_views.sql deleted file mode 100644 index 5dfaaeb9..00000000 --- a/materialized_views.sql +++ /dev/null @@ -1,82 +0,0 @@ --- Materialized Views Export --- Exported on Tue Mar 18 12:20:16 AM CDT 2025 - -DROP MATERIALIZED VIEW IF EXISTS public.mv_model_throughput CASCADE; -CREATE MATERIALIZED VIEW public.mv_model_throughput AS - WITH parsed_throughput AS ( SELECT e.model, (json_extract_path_text((r.result)::json, VARIADIC ARRAY['throughput_tokens_per_second'::text]))::numeric AS throughput FROM (resource_server_log r JOIN resource_server_endpoint e ON (((e.endpoint_slug)::text = (r.endpoint_slug)::text))) WHERE ((r.response_status = 200) AND (r.result IS NOT NULL) AND (r.result <> ''::text) AND (((r.result)::json ->> 'throughput_tokens_per_second'::text) IS NOT NULL)) ) SELECT parsed_throughput.model, round(avg(parsed_throughput.throughput), 2) AS avg_throughput, count(*) AS successful_requests_with_throughput FROM parsed_throughput WHERE (parsed_throughput.throughput IS NOT NULL) GROUP BY parsed_throughput.model ORDER BY parsed_throughput.model;; - -DROP MATERIALIZED VIEW IF EXISTS public.mv_model_throughput_new CASCADE; -CREATE MATERIALIZED VIEW public.mv_model_throughput_new AS - WITH parsed_throughput AS ( SELECT e.model, (json_extract_path_text((r.result)::json, VARIADIC ARRAY['throughput_tokens_per_second'::text]))::numeric AS throughput FROM (resource_server_log r JOIN resource_server_endpoint e ON (((e.endpoint_slug)::text = (r.endpoint_slug)::text))) WHERE ((r.response_status = 200) AND (r.result IS NOT NULL) AND (r.result <> ''::text) AND (((r.result)::json ->> 'throughput_tokens_per_second'::text) IS NOT NULL)) ) SELECT parsed_throughput.model, round(avg(parsed_throughput.throughput), 2) AS avg_throughput, count(*) AS successful_requests_with_throughput FROM parsed_throughput WHERE (parsed_throughput.throughput IS NOT NULL) GROUP BY parsed_throughput.model ORDER BY parsed_throughput.model;; - -DROP MATERIALIZED VIEW IF EXISTS public.mv_model_requests CASCADE; -CREATE MATERIALIZED VIEW public.mv_model_requests AS - SELECT e.model, count(*) AS total_requests, count(*) FILTER (WHERE ((r.response_status = 200) OR (r.response_status IS NULL))) AS successful_requests, count(*) FILTER (WHERE (NOT ((r.response_status = 200) OR (r.response_status IS NULL)))) AS failed_requests FROM (resource_server_log r JOIN resource_server_endpoint e ON (((e.endpoint_slug)::text = (r.endpoint_slug)::text))) GROUP BY e.model;; - -DROP MATERIALIZED VIEW IF EXISTS public.mv_overall_stats CASCADE; -CREATE MATERIALIZED VIEW public.mv_overall_stats AS - SELECT count(*) AS total_requests, count(*) FILTER (WHERE ((resource_server_log.response_status = 200) OR (resource_server_log.response_status IS NULL))) AS successful_requests, count(*) FILTER (WHERE (NOT ((resource_server_log.response_status = 200) OR (resource_server_log.response_status IS NULL)))) AS failed_requests, count(DISTINCT resource_server_log.username) AS total_users FROM resource_server_log;; - -DROP MATERIALIZED VIEW IF EXISTS public.mv_user_details CASCADE; -CREATE MATERIALIZED VIEW public.mv_user_details AS - SELECT DISTINCT resource_server_log.name, resource_server_log.username FROM resource_server_log;; - -DROP MATERIALIZED VIEW IF EXISTS public.mv_model_latency CASCADE; -CREATE MATERIALIZED VIEW public.mv_model_latency AS - SELECT e.model, avg((r.timestamp_response - r.timestamp_receive)) AS avg_latency FROM (resource_server_log r JOIN resource_server_endpoint e ON (((e.endpoint_slug)::text = (r.endpoint_slug)::text))) WHERE ((r.response_status = 200) OR (r.response_status IS NULL)) GROUP BY e.model;; - -DROP MATERIALIZED VIEW IF EXISTS public.mv_users_per_model CASCADE; -CREATE MATERIALIZED VIEW public.mv_users_per_model AS - SELECT e.model, count(DISTINCT r.username) AS user_count FROM (resource_server_log r JOIN resource_server_endpoint e ON (((e.endpoint_slug)::text = (r.endpoint_slug)::text))) GROUP BY e.model;; - -DROP MATERIALIZED VIEW IF EXISTS public.mv_weekly_usage CASCADE; -CREATE MATERIALIZED VIEW public.mv_weekly_usage AS - SELECT date_trunc('week'::text, resource_server_log.timestamp_receive) AS week_start, count(*) AS request_count FROM resource_server_log GROUP BY (date_trunc('week'::text, resource_server_log.timestamp_receive)) ORDER BY (date_trunc('week'::text, resource_server_log.timestamp_receive));; - -DROP MATERIALIZED VIEW IF EXISTS public.mv_daily_usage_2_weeks CASCADE; -CREATE MATERIALIZED VIEW public.mv_daily_usage_2_weeks AS - SELECT date_trunc('day'::text, resource_server_log.timestamp_receive) AS day, count(*) AS request_count FROM resource_server_log WHERE (resource_server_log.timestamp_receive >= (CURRENT_DATE - '14 days'::interval)) GROUP BY (date_trunc('day'::text, resource_server_log.timestamp_receive)) ORDER BY (date_trunc('day'::text, resource_server_log.timestamp_receive));; - -DROP MATERIALIZED VIEW IF EXISTS public.mv_requests_per_user CASCADE; -CREATE MATERIALIZED VIEW public.mv_requests_per_user AS - SELECT resource_server_log.name, resource_server_log.username, count(*) AS total_requests, count(*) FILTER (WHERE ((resource_server_log.response_status = 200) OR (resource_server_log.response_status IS NULL))) AS successful_requests, count(*) FILTER (WHERE (NOT ((resource_server_log.response_status = 200) OR (resource_server_log.response_status IS NULL)))) AS failed_requests FROM resource_server_log GROUP BY resource_server_log.name, resource_server_log.username;; - -DROP MATERIALIZED VIEW IF EXISTS public.mv_monthly_usage CASCADE; -CREATE MATERIALIZED VIEW public.mv_monthly_usage AS - SELECT date_trunc('month'::text, resource_server_log.timestamp_receive) AS month_start, - count(*) AS request_count - FROM resource_server_log - GROUP BY (date_trunc('month'::text, resource_server_log.timestamp_receive)) - ORDER BY (date_trunc('month'::text, resource_server_log.timestamp_receive));; - -DROP MATERIALIZED VIEW IF EXISTS public.mv_total_token_counts CASCADE; - -CREATE MATERIALIZED VIEW public.mv_total_token_counts AS -SELECT - -- Sum prompt_tokens, converting the JSON text value to bigint - SUM((r.result::jsonb -> 'usage' ->> 'prompt_tokens')::bigint) AS total_prompt_tokens, - - -- Sum completion_tokens, converting the JSON text value to bigint - SUM((r.result::jsonb -> 'usage' ->> 'completion_tokens')::bigint) AS total_completion_tokens, - - -- Optionally, sum the total_tokens provided in the usage object as well - SUM((r.result::jsonb -> 'usage' ->> 'total_tokens')::bigint) AS grand_total_tokens -FROM - resource_server_log r -WHERE - -- Filter for successful requests - r.response_status = 200 - -- Ensure the result field is not null or empty - AND r.result IS NOT NULL - AND r.result <> '' - -- Crucially, ensure the 'usage' key exists and is a JSON object - AND jsonb_typeof(r.result::jsonb -> 'usage') = 'object' - -- Add checks to ensure the token fields exist within 'usage' and contain valid numeric strings - -- This prevents errors if the JSON structure is missing keys or has non-numeric values - AND r.result::jsonb -> 'usage' ->> 'prompt_tokens' ~ '^[0-9]+$' - AND r.result::jsonb -> 'usage' ->> 'completion_tokens' ~ '^[0-9]+$' - -- Optional check for total_tokens if you are summing it directly - AND r.result::jsonb -> 'usage' ->> 'total_tokens' ~ '^[0-9]+$'; - --- Example command to refresh the view (run periodically) --- REFRESH MATERIALIZED VIEW public.mv_total_token_counts; diff --git a/materialized_views_batch.sql b/materialized_views_batch.sql deleted file mode 100644 index d949ec77..00000000 --- a/materialized_views_batch.sql +++ /dev/null @@ -1,135 +0,0 @@ --- a. Total Number of batch jobs -DROP MATERIALIZED VIEW IF EXISTS public.mv_batch_total_jobs CASCADE; -CREATE MATERIALIZED VIEW public.mv_batch_total_jobs AS -SELECT - COUNT(*) AS total_batch_jobs, - COUNT(*) FILTER (WHERE status = 'completed') AS completed_batch_jobs, - COUNT(*) FILTER (WHERE status = 'failed') AS failed_batch_jobs, - COUNT(*) FILTER (WHERE status = 'pending') AS pending_batch_jobs, - COUNT(*) FILTER (WHERE status = 'running') AS running_batch_jobs -FROM resource_server_batch; - --- b. Total number of successful batch requests (based on num_responses) -DROP MATERIALIZED VIEW IF EXISTS public.mv_batch_successful_requests CASCADE; -CREATE MATERIALIZED VIEW public.mv_batch_successful_requests AS -SELECT - SUM( - (json_extract_path_text((result)::json, VARIADIC ARRAY['metrics', 'num_responses']))::numeric - ) AS total_successful_requests -FROM resource_server_batch -WHERE status = 'completed' -AND result IS NOT NULL -AND result <> '' -AND (result)::json -> 'metrics' -> 'num_responses' IS NOT NULL; - --- c. Batch Requests per model -DROP MATERIALIZED VIEW IF EXISTS public.mv_batch_requests_per_model CASCADE; -CREATE MATERIALIZED VIEW public.mv_batch_requests_per_model AS -SELECT - model, - COUNT(*) AS total_jobs, - COUNT(*) FILTER (WHERE status = 'completed') AS completed_jobs, - SUM( - CASE - WHEN status = 'completed' AND result IS NOT NULL AND result <> '' AND - (result)::json -> 'metrics' -> 'num_responses' IS NOT NULL - THEN (json_extract_path_text((result)::json, VARIADIC ARRAY['metrics', 'num_responses']))::numeric - ELSE 0 - END - ) AS total_requests -FROM resource_server_batch -GROUP BY model -ORDER BY model; - --- d. Total number of batch users -DROP MATERIALIZED VIEW IF EXISTS public.mv_batch_unique_users CASCADE; -CREATE MATERIALIZED VIEW public.mv_batch_unique_users AS -SELECT - COUNT(DISTINCT username) AS unique_users, - array_agg(DISTINCT username) AS user_list -FROM resource_server_batch; - --- e. Total number of tokens processed -DROP MATERIALIZED VIEW IF EXISTS public.mv_batch_total_tokens CASCADE; -CREATE MATERIALIZED VIEW public.mv_batch_total_tokens AS -SELECT - SUM( - (json_extract_path_text((result)::json, VARIADIC ARRAY['metrics', 'total_tokens']))::numeric - ) AS total_tokens_processed -FROM resource_server_batch -WHERE status = 'completed' -AND result IS NOT NULL -AND result <> '' -AND (result)::json -> 'metrics' -> 'total_tokens' IS NOT NULL; - --- f. Average latency per model -DROP MATERIALIZED VIEW IF EXISTS public.mv_batch_avg_latency CASCADE; -CREATE MATERIALIZED VIEW public.mv_batch_avg_latency AS -SELECT - model, - AVG( - (json_extract_path_text((result)::json, VARIADIC ARRAY['metrics', 'response_time']))::numeric - ) AS avg_response_time_sec -FROM resource_server_batch -WHERE status = 'completed' -AND result IS NOT NULL -AND result <> '' -AND (result)::json -> 'metrics' -> 'response_time' IS NOT NULL -GROUP BY model -ORDER BY model; - --- g. Average throughput per model -DROP MATERIALIZED VIEW IF EXISTS public.mv_batch_avg_throughput CASCADE; -CREATE MATERIALIZED VIEW public.mv_batch_avg_throughput AS -SELECT - model, - AVG( - (json_extract_path_text((result)::json, VARIADIC ARRAY['metrics', 'throughput_tokens_per_second']))::numeric - ) AS avg_throughput_tokens_per_sec -FROM resource_server_batch -WHERE status = 'completed' -AND result IS NOT NULL -AND result <> '' -AND (result)::json -> 'metrics' -> 'throughput_tokens_per_second' IS NOT NULL -GROUP BY model -ORDER BY model; - --- Daily batch usage (all time) -DROP MATERIALIZED VIEW IF EXISTS public.mv_batch_daily_usage CASCADE; -CREATE MATERIALIZED VIEW public.mv_batch_daily_usage AS -SELECT - date_trunc('day'::text, created_at) AS day, - COUNT(*) AS batch_count, - COUNT(*) FILTER (WHERE status = 'completed') AS completed_count, - COUNT(*) FILTER (WHERE status = 'failed') AS failed_count, - SUM( - CASE - WHEN status = 'completed' AND result IS NOT NULL AND result <> '' AND - (result)::json -> 'metrics' -> 'num_responses' IS NOT NULL - THEN (json_extract_path_text((result)::json, VARIADIC ARRAY['metrics', 'num_responses']))::numeric - ELSE 0 - END - ) AS total_requests -FROM resource_server_batch -GROUP BY date_trunc('day'::text, created_at) -ORDER BY date_trunc('day'::text, created_at); - --- Monthly batch usage (all time) -DROP MATERIALIZED VIEW IF EXISTS public.mv_batch_monthly_usage CASCADE; -CREATE MATERIALIZED VIEW public.mv_batch_monthly_usage AS -SELECT - date_trunc('month'::text, created_at) AS month_start, - COUNT(*) AS batch_count, - COUNT(*) FILTER (WHERE status = 'completed') AS completed_count, - COUNT(*) FILTER (WHERE status = 'failed') AS failed_count, - SUM( - CASE - WHEN status = 'completed' AND result IS NOT NULL AND result <> '' AND - (result)::json -> 'metrics' -> 'num_responses' IS NOT NULL - THEN (json_extract_path_text((result)::json, VARIADIC ARRAY['metrics', 'num_responses']))::numeric - ELSE 0 - END - ) AS total_requests_in_completed -FROM resource_server_batch -GROUP BY date_trunc('month'::text, created_at) -ORDER BY date_trunc('month'::text, created_at); \ No newline at end of file diff --git a/metrics_processing.sql b/metrics_processing.sql deleted file mode 100644 index 72af5890..00000000 --- a/metrics_processing.sql +++ /dev/null @@ -1,250 +0,0 @@ --- ============================================================================= --- Request Metrics Batch Processing Setup --- ============================================================================= --- This script sets up asynchronous batch processing for request metrics --- instead of using expensive synchronous triggers. --- --- Prerequisites: --- - metrics_processed column added to resource_server_async_requestlog table --- - Old trigger trg_upsert_requestmetrics removed --- - Old function fn_upsert_requestmetrics removed --- --- Deployment: --- 1. Run this script: psql -d yourdb -f batch_processing_setup.sql --- 2. Backfill existing data: SELECT backfill_existing_metrics(1000); --- 3. Set up cron job: */30 * * * * psql -d yourdb -c "SELECT process_unprocessed_metrics(500);" --- ============================================================================= - --- Create index for fast unprocessed lookup -CREATE INDEX IF NOT EXISTS idx_requestlog_unprocessed -ON resource_server_async_requestlog (metrics_processed, timestamp_compute_response) -WHERE metrics_processed = FALSE AND timestamp_compute_response IS NOT NULL; - -<<<<<<< HEAD --- Other indexes -CREATE INDEX idx_accesslog_status_code ON resource_server_async_accesslog(status_code); -CREATE INDEX idx_requestlog_result_tokens -ON resource_server_async_requestlog(id) -WHERE result ~ '"total_tokens"\s*:\s*\d+'; - -======= ->>>>>>> model_upgrades --- Create lightweight trigger to mark records for processing -CREATE OR REPLACE FUNCTION fn_mark_for_processing() RETURNS trigger AS $$ -BEGIN - -- Mark for processing when we get response data - IF NEW.timestamp_compute_response IS NOT NULL AND - NEW.result IS NOT NULL AND - NEW.result != '' THEN - NEW.metrics_processed := FALSE; - END IF; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - -CREATE TRIGGER trg_mark_for_processing -BEFORE INSERT OR UPDATE OF result, timestamp_compute_response -ON resource_server_async_requestlog -FOR EACH ROW EXECUTE FUNCTION fn_mark_for_processing(); - --- Main batch processor function -CREATE OR REPLACE FUNCTION process_unprocessed_metrics(batch_size INTEGER DEFAULT 1000) -RETURNS INTEGER AS $$ -DECLARE - processed_count INTEGER := 0; - batch_ids UUID[]; -BEGIN - -- Get batch of unprocessed IDs - SELECT ARRAY( - SELECT rl.id - FROM resource_server_async_requestlog rl - JOIN resource_server_async_accesslog al ON rl.access_log_id = al.id - WHERE rl.metrics_processed = FALSE - AND rl.timestamp_compute_response IS NOT NULL - AND rl.result IS NOT NULL - AND rl.result != '' -<<<<<<< HEAD - AND rl.result ~ '"total_tokens"\s*:\s*\d+' -- Pre-filter with regex - AND EXISTS ( - SELECT 1 FROM resource_server_async_accesslog al - WHERE al.id = rl.access_log_id - AND al.status_code BETWEEN 200 AND 299 - ) -======= - AND al.status_code >= 200 - AND al.status_code < 300 - AND rl.result ~ '"total_tokens"\s*:\s*\d+' ->>>>>>> model_upgrades - ORDER BY rl.timestamp_compute_response - LIMIT batch_size - FOR UPDATE SKIP LOCKED -- Prevent conflicts with multiple workers - ) INTO batch_ids; - - -- Exit if nothing to process - IF array_length(batch_ids, 1) IS NULL THEN - RETURN 0; - END IF; - - -- Process the batch - insert/update metrics - INSERT INTO resource_server_async_requestmetrics - (request_id, cluster, framework, model, status_code, - prompt_tokens, completion_tokens, total_tokens, - response_time_sec, throughput_tokens_per_sec, - timestamp_compute_request, timestamp_compute_response, created_at) - SELECT - rl.id, - rl.cluster, - rl.framework, - rl.model, - al.status_code, - COALESCE((regexp_match(rl.result, '"prompt_tokens"\s*:\s*(\d+)'))[1]::bigint, NULL), - COALESCE((regexp_match(rl.result, '"completion_tokens"\s*:\s*(\d+)'))[1]::bigint, NULL), - COALESCE((regexp_match(rl.result, '"total_tokens"\s*:\s*(\d+)'))[1]::bigint, NULL), - EXTRACT(EPOCH FROM (rl.timestamp_compute_response - rl.timestamp_compute_request)), - COALESCE((regexp_match(rl.result, '"throughput_tokens_per_second"\s*:\s*([0-9.]+)'))[1]::double precision, NULL), - rl.timestamp_compute_request, - rl.timestamp_compute_response, - NOW() - FROM resource_server_async_requestlog rl - JOIN resource_server_async_accesslog al ON rl.access_log_id = al.id - WHERE rl.id = ANY(batch_ids) - ON CONFLICT (request_id) DO UPDATE SET - cluster = EXCLUDED.cluster, - framework = EXCLUDED.framework, - model = EXCLUDED.model, - status_code = EXCLUDED.status_code, - prompt_tokens = EXCLUDED.prompt_tokens, - completion_tokens = EXCLUDED.completion_tokens, - total_tokens = EXCLUDED.total_tokens, - response_time_sec = EXCLUDED.response_time_sec, - throughput_tokens_per_sec = EXCLUDED.throughput_tokens_per_sec, - timestamp_compute_request = EXCLUDED.timestamp_compute_request, - timestamp_compute_response = EXCLUDED.timestamp_compute_response; - - -- Mark batch as processed - UPDATE resource_server_async_requestlog - SET metrics_processed = TRUE - WHERE id = ANY(batch_ids); - - GET DIAGNOSTICS processed_count = ROW_COUNT; - RETURN processed_count; -END; -$$ LANGUAGE plpgsql; - --- Monitoring function to check unprocessed records -CREATE OR REPLACE FUNCTION get_unprocessed_metrics_count() -RETURNS TABLE( - total_unprocessed BIGINT, - oldest_unprocessed TIMESTAMP WITH TIME ZONE, - newest_unprocessed TIMESTAMP WITH TIME ZONE -) AS $$ -BEGIN - RETURN QUERY - SELECT - COUNT(*), - MIN(rl.timestamp_compute_response), - MAX(rl.timestamp_compute_response) - FROM resource_server_async_requestlog rl - JOIN resource_server_async_accesslog al ON rl.access_log_id = al.id - WHERE rl.metrics_processed = FALSE - AND rl.timestamp_compute_response IS NOT NULL - AND rl.result IS NOT NULL - AND rl.result != '' - AND al.status_code >= 200 - AND al.status_code < 300 - AND rl.result ~ '"total_tokens"\s*:\s*\d+'; -END; -$$ LANGUAGE plpgsql; - --- One-time backfill function for existing data -CREATE OR REPLACE FUNCTION backfill_existing_metrics(batch_size INTEGER DEFAULT 5000) -RETURNS INTEGER AS $$ -DECLARE - total_processed INTEGER := 0; - batch_processed INTEGER; -BEGIN - RAISE NOTICE 'Starting backfill process...'; - - -- First, mark all existing records as unprocessed if they have the required data - UPDATE resource_server_async_requestlog - SET metrics_processed = FALSE - WHERE metrics_processed IS NULL - AND timestamp_compute_response IS NOT NULL - AND result IS NOT NULL - AND result != ''; - - RAISE NOTICE 'Marked existing records as unprocessed. Processing in batches of %...', batch_size; - - -- Process in batches to avoid overwhelming the system - LOOP - SELECT process_unprocessed_metrics(batch_size) INTO batch_processed; - total_processed := total_processed + batch_processed; - - -- Log progress - IF batch_processed > 0 THEN - RAISE NOTICE 'Processed % records (total: %)', batch_processed, total_processed; - END IF; - - -- Exit when no more to process - EXIT WHEN batch_processed = 0; - - -- Small delay to prevent overwhelming the system - PERFORM pg_sleep(0.1); - END LOOP; - - RAISE NOTICE 'Backfill complete. Total processed: %', total_processed; - RETURN total_processed; -END; -$$ LANGUAGE plpgsql; - --- Monitoring view for easy status checking -CREATE OR REPLACE VIEW v_metrics_processing_status AS -SELECT - COUNT(*) FILTER (WHERE metrics_processed = FALSE) as unprocessed_count, - COUNT(*) FILTER (WHERE metrics_processed = TRUE) as processed_count, - COUNT(*) as total_count, - ROUND( - (COUNT(*) FILTER (WHERE metrics_processed = TRUE)::NUMERIC / - NULLIF(COUNT(*), 0) * 100), 2 - ) as processed_percentage, - MIN(timestamp_compute_response) FILTER (WHERE metrics_processed = FALSE) as oldest_unprocessed, - MAX(timestamp_compute_response) FILTER (WHERE metrics_processed = FALSE) as newest_unprocessed -FROM resource_server_async_requestlog -WHERE timestamp_compute_response IS NOT NULL; - --- ============================================================================= --- USAGE INSTRUCTIONS --- ============================================================================= -/* - -AFTER RUNNING THIS SCRIPT: - -1. Backfill existing data (run once): - SELECT backfill_existing_metrics(1000); - -2. Set up cron job for ongoing processing: - Add to crontab: crontab -e - */30 * * * * psql -d yourdb -c "SELECT process_unprocessed_metrics(500);" >/dev/null 2>&1 - -3. Monitor the system: - SELECT * FROM v_metrics_processing_status; - SELECT * FROM get_unprocessed_metrics_count(); - -4. Manual processing if needed: - SELECT process_unprocessed_metrics(1000); - -5. For high-volume periods, run multiple workers: - - SKIP LOCKED prevents conflicts between workers - - Adjust batch_size based on system capacity (100-2000) - -PERFORMANCE TUNING: -- For light load: Run every 1-2 minutes with batch_size=200 -- For heavy load: Run every 30 seconds with batch_size=500-1000 -- For very heavy load: Run multiple workers with different batch sizes - -MONITORING QUERIES: -- Current backlog: SELECT unprocessed_count FROM v_metrics_processing_status; -- Processing lag: SELECT oldest_unprocessed FROM v_metrics_processing_status; -- Success rate: SELECT processed_percentage FROM v_metrics_processing_status; - -*/ \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000..f29b7318 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,101 @@ +site_name: FIRST Documentation +site_description: Federated Inference Resource Scheduling Toolkit - Documentation for deploying and using LLM inference as a service +site_author: Argonne National Laboratory +site_url: https://auroragpt-anl.github.io/inference-gateway/ + +repo_name: auroraGPT-ANL/inference-gateway +repo_url: https://github.com/auroraGPT-ANL/inference-gateway +edit_uri: edit/main/docs/ + +theme: + name: material + palette: + # Light mode + - media: "(prefers-color-scheme: light)" + scheme: default + primary: indigo + accent: indigo + toggle: + icon: material/brightness-7 + name: Switch to dark mode + # Dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + primary: indigo + accent: indigo + toggle: + icon: material/brightness-4 + name: Switch to light mode + font: + text: Roboto + code: Roboto Mono + features: + - navigation.tabs + - navigation.sections + - navigation.expand + - navigation.top + - navigation.tracking + - search.suggest + - search.highlight + - content.code.copy + - content.code.annotate + icon: + repo: fontawesome/brands/github + +plugins: + - search + - minify: + minify_html: true + +markdown_extensions: + - admonition + - pymdownx.details + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format + - pymdownx.highlight: + anchor_linenums: true + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.tabbed: + alternate_style: true + - tables + - attr_list + - md_in_html + - toc: + permalink: true + +nav: + - Home: index.md + - Administrator Guide: + - Overview: admin-guide/index.md + - Gateway Setup: + - Docker Deployment: admin-guide/gateway-setup/docker.md + - Bare Metal Setup: admin-guide/gateway-setup/bare-metal.md + - Configuration Reference: admin-guide/gateway-setup/configuration.md + - Inference Backend Setup: + - Overview: admin-guide/inference-setup/index.md + - Globus Compute + vLLM: admin-guide/inference-setup/globus-compute.md + - Local vLLM Setup: admin-guide/inference-setup/local-vllm.md + - Direct API Connection: admin-guide/inference-setup/direct-api.md + - Deployment: + - Kubernetes: admin-guide/deployment/kubernetes.md + - Production Best Practices: admin-guide/deployment/production.md + - Monitoring & Troubleshooting: admin-guide/monitoring.md + - User Guide: user-guide/index.md + - Reference: + - Citation: reference/citation.md + - API Endpoints: reference/api.md + - Configuration Options: reference/config.md + +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/auroraGPT-ANL/inference-gateway + - icon: fontawesome/solid/book + link: https://doi.org/10.1145/3731599.3767346 + +copyright: Copyright © 2025 Argonne National Laboratory - Apache 2.0 License + diff --git a/pgloader.load b/pgloader.load deleted file mode 100644 index 63eb5b63..00000000 --- a/pgloader.load +++ /dev/null @@ -1,18 +0,0 @@ -LOAD DATABASE - FROM sqlite:///path/db.sqlite3 - INTO postgresql://$(PGUSER)@$(PGHOST):$(PGPORT)/$(PGDATABASE) - -WITH include drop, create tables, create indexes, reset sequences, batch rows = 50000, batch size = 100MB, prefetch rows = 5000, data only - -SET work_mem to '128MB', - maintenance_work_mem to '512MB', - search_path to 'public' - -CAST - type text to varchar, - type bigint to integer, - type bigserial to serial, - type boolean to boolean, - type integer to integer, - type smallint to smallint -; diff --git a/pyproject.toml b/pyproject.toml index da218a9c..48e9207c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ django-filter = "^24.3" psycopg = "^3.2.2" psycopg-pool = "^3.2.3" websockets = "^13.0.1" +httpx = "^0.27.2" django-ninja = "^1.3.0" asyncache = "^0.3.1" globus-compute-common = "^0.7.1" @@ -37,6 +38,8 @@ requests = "^2.32.4" redis = "^6.3.0" django-redis = "^6.0.0" hiredis = "^3.2.1" +mkdocs-material = "^9.7.0" +mkdocs-minify-plugin = "^0.8.0" [build-system] requires = ["poetry-core"] diff --git a/requirements.txt b/requirements.txt index f46fdcf2..5e8d8293 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,8 @@ annotated-types==0.7.0 ; python_full_version >= "3.12.6" and python_full_version asgiref==3.10.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" asyncache==0.3.1 ; python_full_version >= "3.12.6" and python_version < "4.0" attrs==25.4.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" +babel==2.17.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" +backrefs==6.0.1 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" cachetools==5.5.2 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" certifi==2025.10.5 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" cffi==2.0.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" and platform_python_implementation != "PyPy" @@ -9,6 +11,7 @@ charset-normalizer==3.4.4 ; python_full_version >= "3.12.6" and python_full_vers click==8.3.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" colorama==0.4.6 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" cryptography==46.0.3 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" +csscompressor==0.9.5 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" dill==0.3.9 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" django-cors-headers==4.9.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" django-filter==24.3 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" @@ -18,20 +21,36 @@ django==5.2.8 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0 djangorestframework==3.16.1 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" drf-spectacular==0.27.2 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" exceptiongroup==1.3.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" +ghp-import==2.1.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" globus-compute-common==0.7.1 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" globus-compute-sdk==3.16.1 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" globus-sdk==3.65.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" gunicorn==23.0.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" h11==0.16.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" hiredis==3.3.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" +htmlmin2==0.1.13 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" idna==3.11 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" inflection==0.5.1 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" +httpx==0.27.2 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" +jinja2==3.1.6 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" +jsmin==3.0.1 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" jsonschema-specifications==2025.9.1 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" jsonschema==4.25.1 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" markdown-it-py==4.0.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" +markdown==3.10 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" +markupsafe==3.0.3 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" mdurl==0.1.2 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" +mergedeep==1.3.4 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" +mkdocs-get-deps==0.2.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" +mkdocs-material-extensions==1.3.1 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" +mkdocs-material==9.7.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" +mkdocs-minify-plugin==0.8.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" +mkdocs==1.6.1 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" packaging==25.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" +paginate==0.5.7 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" +pathspec==0.12.1 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" pika==1.3.2 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" +platformdirs==4.5.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" psutil==5.9.8 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" psycopg-pool==3.2.7 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" psycopg==3.2.12 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" @@ -40,7 +59,10 @@ pydantic-core==2.41.5 ; python_full_version >= "3.12.6" and python_full_version pydantic==2.12.4 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" pygments==2.19.2 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" pyjwt==2.10.1 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" +pymdown-extensions==10.17.1 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" +python-dateutil==2.9.0.post0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" python-dotenv==1.2.1 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" +pyyaml-env-tag==1.1 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" pyyaml==6.0.3 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" redis==6.4.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" referencing==0.37.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" @@ -48,6 +70,7 @@ requests==2.32.5 ; python_full_version >= "3.12.6" and python_full_version < "4. rich==14.2.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" rpds-py==0.28.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" shellingham==1.5.4 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" +six==1.17.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" sqlparse==0.5.3 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" tblib==1.7.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" texttable==1.7.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" @@ -58,4 +81,11 @@ tzdata==2025.2 ; python_full_version >= "3.12.6" and python_full_version < "4.0. uritemplate==4.2.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" urllib3==2.5.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" uvicorn==0.30.6 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" +watchdog==6.0.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" websockets==13.1 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" +# Documentation dependencies +mkdocs>=1.5.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" +mkdocs-material>=9.4.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" +mkdocs-minify-plugin>=0.7.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" +pymdown-extensions>=10.0 ; python_full_version >= "3.12.6" and python_full_version < "4.0.0" + diff --git a/reset_cursor.py b/reset_cursor.py deleted file mode 100644 index df87f406..00000000 --- a/reset_cursor.py +++ /dev/null @@ -1,46 +0,0 @@ -import psycopg2 -from psycopg2 import sql -import os - -# Fetch database connection settings from environment variables -db_name = os.getenv('PGDATABASE', 'default_db_name') -db_user = os.getenv('PGUSER', 'default_user') -db_password = os.getenv('PGPASSWORD', '') # This will be empty since .pgpass will handle the password -db_host = os.getenv('PGHOST', 'localhost') -db_port = os.getenv('PGPORT', '5432') - -# Connect to the PostgreSQL database -conn = psycopg2.connect( - dbname=db_name, user=db_user, password=db_password, host=db_host, port=db_port -) -cur = conn.cursor() - -# Fetch all tables in the public schema -cur.execute(""" - SELECT table_name - FROM information_schema.tables - WHERE table_schema = 'public' AND table_type = 'BASE TABLE'; -""") -tables = cur.fetchall() - -# Reset sequences for all tables -for table in tables: - table_name = table[0] - - # Try to reset the sequence for the primary key column - try: - reset_query = sql.SQL(""" - SELECT setval(pg_get_serial_sequence(%s, 'id'), COALESCE(MAX(id)+1, 1), false) - FROM {} - """).format(sql.Identifier(table_name)) - - cur.execute(reset_query, (table_name,)) - print(f"Reset sequence for table {table_name}") - - except Exception as e: - print(f"Could not reset sequence for table {table_name}: {e}") - -# Commit the changes and close the connection -conn.commit() -cur.close() -conn.close() diff --git a/utils/globus_utils.py b/utils/globus_utils.py index 493a0f72..64caf5ab 100644 --- a/utils/globus_utils.py +++ b/utils/globus_utils.py @@ -37,8 +37,8 @@ def get_compute_client_from_globus_app() -> globus_sdk.GlobusHTTPResponse: try: return Client( app=globus_sdk.ClientApp( - client_id=settings.POLARIS_ENDPOINT_ID, - client_secret=settings.POLARIS_ENDPOINT_SECRET + client_id=settings.SERVICE_ACCOUNT_ID, + client_secret=settings.SERVICE_ACCOUNT_SECRET ) ) except Exception as e: