Skip to content
CI Tests

CI Tests #619

Sign in to view logs

CI Tests

CI Tests #619

Workflow file for this run

.github/workflows/ci_tests.yml at 13ef66e
# CI Tests
# These should only be run on main, because they access secrets
name: CI Tests
on:
workflow_dispatch:
push:
branches: [main]
schedule:
# * is a special character in YAML so we quote this string
# Run at 1030 UTC every day
- cron: '30 10 * * *'
permissions:
id-token: write
contents: read
jobs:
general:
runs-on: gpu-runner
strategy:
fail-fast: false # Don't cancel all on first failure
matrix:
python-version: ["3.9", "3.10", "3.11", "3.12"]
steps:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Install Rust
shell: bash
run: |
curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain 1.75.0
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
- name: Show GPUs
run: |
nvidia-smi
- name: Update Ubuntu
run: |
sudo apt-get update
sudo apt-get -y upgrade
- name: Ensure NVIDIA SDK available
run: |
sudo apt-get -y install cuda-toolkit
echo "/usr/local/cuda-12.6/bin" >> $GITHUB_PATH
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install pytest
pip install -e .[all,test]
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
- name: GPU pip installs
run: |
pip install accelerate
CMAKE_ARGS="-DGGML_CUDA=on" pip install "llama-cpp-python!=0.2.58,!=0.2.75,!=0.2.84"
- name: Check GPU available
run: |
python -c "import torch; assert torch.cuda.is_available()"
- name: 'Az CLI login'
uses: azure/login@v2
with:
client-id: ${{ secrets.AZURE_CLIENT_ID }}
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
- name: 'Run Azure CLI commands'
run: |
az account show
az group list
- name: Test with pytest
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
# Configure endpoints for Azure OpenAI
AZUREAI_CHAT_ENDPOINT: ${{ vars.AZUREAI_CHAT_ENDPOINT }}
AZUREAI_CHAT_MODEL: ${{ vars.AZUREAI_CHAT_MODEL }}
AZUREAI_COMPLETION_ENDPOINT: ${{ vars.AZUREAI_COMPLETION_ENDPOINT }}
AZUREAI_COMPLETION_MODEL: ${{ vars.AZUREAI_COMPLETION_MODEL }}
# Configure endpoints for Azure AI Studio
AZURE_AI_STUDIO_PHI3_ENDPOINT: ${{ vars.AZURE_AI_STUDIO_PHI3_ENDPOINT }}
AZURE_AI_STUDIO_PHI3_DEPLOYMENT: ${{ vars.AZURE_AI_STUDIO_PHI3_DEPLOYMENT }}
AZURE_AI_STUDIO_PHI3_KEY: ${{ secrets.AZURE_AI_STUDIO_PHI3_KEY }}
AZURE_AI_STUDIO_MISTRAL_CHAT_ENDPOINT: ${{ vars.AZURE_AI_STUDIO_MISTRAL_CHAT_ENDPOINT }}
AZURE_AI_STUDIO_MISTRAL_CHAT_DEPLOYMENT: ${{ vars.AZURE_AI_STUDIO_MISTRAL_CHAT_DEPLOYMENT }}
AZURE_AI_STUDIO_MISTRAL_CHAT_KEY: ${{ secrets.AZURE_AI_STUDIO_MISTRAL_CHAT_KEY }}
AZURE_AI_STUDIO_LLAMA3_CHAT_ENDPOINT: ${{ vars.AZURE_AI_STUDIO_LLAMA3_CHAT_ENDPOINT }}
AZURE_AI_STUDIO_LLAMA3_CHAT_DEPLOYMENT: ${{ vars.AZURE_AI_STUDIO_LLAMA3_CHAT_DEPLOYMENT }}
AZURE_AI_STUDIO_LLAMA3_CHAT_KEY: ${{ secrets.AZURE_AI_STUDIO_LLAMA3_CHAT_KEY }}
run: |
pytest --cov=guidance --cov-report=xml --cov-report=term-missing \
-vv \
./tests/need_credentials
- name: Upload coverage reports to Codecov
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
minimal-model-specific-tests:
runs-on: gpu-runner
strategy:
fail-fast: false # Don't cancel all on first failure
matrix:
python-version: ["3.9", "3.10", "3.11", "3.12"]
model:
- llamacpp_gemma2_9b_cpu
- transformers_gemma2_9b_cpu
- transformers_gemma2_9b_gpu
- transformers_llama3_8b_cpu
# - transformers_llama3_8b_gpu Does not reliably work on GPU build machines
steps:
- uses: actions/checkout@v4
- name: Install Rust
shell: bash
run: |
curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain 1.75.0
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Show GPUs
run: |
nvidia-smi
- name: Update Ubuntu
run: |
sudo apt-get update
sudo apt-get -y upgrade
- name: Ensure NVIDIA SDK available
run: |
sudo apt-get -y install cuda-toolkit
echo "/usr/local/cuda-12.6/bin" >> $GITHUB_PATH
- name: Install dependencies
shell: bash
run: |
python -m pip install --upgrade pip
pip install pytest
pip install -e .[schemas,test,bench]
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
- name: Other dependencies
run: |
pip install sentencepiece
echo "=============================="
pip uninstall -y transformers
pip install "transformers!=4.43.0,!=4.43.1,!=4.43.2,!=4.43.3" # Issue 965
- name: GPU pip installs
run: |
pip install accelerate
pip uninstall -y llama-cpp-python
echo "======================"
nvcc --version
echo "======================"
CMAKE_ARGS="-DGGML_CUDA=on" pip install "llama-cpp-python!=0.2.58,!=0.2.75"
- name: Check GPU available
run: |
python -c "import torch; assert torch.cuda.is_available()"
# Note that we run a minimal set of tests here
- name: Run minimal tests for ${{ matrix.model }}
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
run: |
pytest -vv --cov=guidance --cov-report=xml --cov-report=term-missing \
--selected_model ${{ matrix.model }} \
./tests/model_integration/test_model.py \
./tests/model_specific/
- name: Upload coverage reports to Codecov
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
remote-endpoint-tests-linux-python:
strategy:
fail-fast: false # Don't cancel all on first failure
matrix:
python-version: ["3.9", "3.10", "3.11", "3.12"]
model:
- "azure_guidance"
runs-on: Large_Linux
steps:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
shell: bash
run: |
python -m pip install --upgrade pip
pip install pytest
pip install -e .[test]
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
- name: Run tests (except server)
shell: bash
env:
AZUREAI_GUIDANCE_ENABLED_URL: ${{ vars.AZUREAI_GUIDANCE_ENABLED_URL }}
AZUREAI_GUIDANCE_ENABLED_URL_KEY: ${{ secrets.AZUREAI_GUIDANCE_ENABLED_URL_KEY }}
run: |
pytest --cov=guidance --cov-report=xml --cov-report=term-missing \
--selected_model ${{ matrix.model }} \
./tests/unit ./tests/model_integration ./tests/model_specific
- name: Upload coverage reports to Codecov
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}