Core tests with LLMs #20
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow will install Python dependencies and run tests with a variety of Python versions | |
| # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions | |
| name: Core tests with LLMs | |
| on: | |
| schedule: | |
| - cron: "0 0 * * 0" # weekly on Sunday at midnight UTC | |
| workflow_dispatch: # allows manual triggering of the workflow | |
| # pull_request_target: | |
| # branches: ["main"] | |
| # paths: | |
| # - "autogen/**" | |
| # - "test/**" | |
| # - "notebook/agentchat_auto_feedback_from_code_execution.ipynb" | |
| # - "notebook/agentchat_function_call.ipynb" | |
| # - "notebook/agentchat_groupchat_finite_state_machine.ipynb" | |
| # - ".github/workflows/core-llm-test.yml" | |
| # - "pyproject.toml" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| {} | |
| jobs: | |
| test: | |
| strategy: | |
| matrix: | |
| llm: ["openai", "openai-realtime", "gemini", "gemini-realtime", "anthropic", "deepseek", "ollama", "bedrock", "cerebras"] | |
| python-version: ["3.10"] | |
| os: [ubuntu-latest] | |
| fail-fast: false | |
| runs-on: ${{ matrix.os }} | |
| environment: openai1 | |
| services: | |
| redis: | |
| image: redis@sha256:17a235ee842cd21d8bcb7f11a49fb177cbeaf5537996304a0089e94412de2ed5 | |
| ports: | |
| - 6379:6379 | |
| options: --entrypoint redis-server | |
| steps: | |
| - name: Get User Permission | |
| id: checkAccess | |
| uses: actions-cool/check-user-permission@c21884f3dda18dafc2f8b402fe807ccc9ec1aa5e # v2.4.0 | |
| with: | |
| require: write | |
| username: ${{ github.triggering_actor }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Check User Permission | |
| if: steps.checkAccess.outputs.require-result == 'false' | |
| run: | | |
| echo "${GITHUB_TRIGGERING_ACTOR} does not have permissions on this repo." | |
| echo "Current permission level is ${STEPS_CHECKACCESS_OUTPUTS_USER_PERMISSION}" | |
| echo "Job originally triggered by ${GITHUB_ACTOR}" | |
| exit 1 | |
| env: | |
| STEPS_CHECKACCESS_OUTPUTS_USER_PERMISSION: ${{ steps.checkAccess.outputs.user-permission }} | |
| # checkout to pr branch | |
| - name: Checkout | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.ref }} | |
| persist-credentials: false | |
| - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0 | |
| with: | |
| version: "latest" | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install packages for ${{ matrix.llm }} | |
| run: | | |
| docker --version | |
| uv pip install --system -e ".[test,redis,${{ matrix.llm }}]" | |
| - name: Install openai if testing gemini-realtime | |
| if: ${{ matrix.llm == 'gemini-realtime' }} | |
| run: | | |
| uv pip install --system -e ".[openai]" | |
| - name: Create OAI_CONFIG_LIST from Secret | |
| run: | | |
| cat > OAI_CONFIG_LIST << 'EOF' | |
| ${{ secrets.OAI_CONFIG_LIST }} | |
| EOF | |
| shell: bash | |
| - name: LLM tests using ${{ matrix.llm }} | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} | |
| AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} | |
| AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} | |
| OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} | |
| run: | | |
| llm_fixed="${{ matrix.llm }}" # Use the matrix variable | |
| llm_fixed="${llm_fixed//-/_}" # Replace hyphen with underscore | |
| bash scripts/test-core-llm.sh -m "$llm_fixed" | |
| - name: Show coverage report | |
| run: bash scripts/show-coverage-report.sh | |
| - name: Upload coverage to Codecov | |
| if: ${{ !contains(github.ref, 'gh-readonly-queue/') }} | |
| uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4 | |
| with: | |
| files: ./coverage.xml | |
| flags: core-llm, ${{ matrix.llm }}, ${{ matrix.os }}, ${{ matrix.python-version }} | |
| fail_ci_if_error: true | |
| token: ${{ secrets.CODECOV_TOKEN }} |