ci: Parallelize integration and smoke test shards

.github/skills/smoke-tests/SKILL.md

@@ -19,6 +19,7 @@ Both forward extra arguments after `--` to the runner (`test/smoke/test/index.js
 | Option | Description |
 |--------|-------------|
 | `-g <pattern>` (alias `-f`) | Grep filter on test/suite titles (mocha `grep`). |
+| `--group <all\|core\|chat>` | Run all tests (default), non-chat tests, or Chat, Agents Window, and Copilot CLI tests. Cannot be combined with `-g`. |
 | `--build <path>` | Run against a packaged build instead of the compiled-from-source dev build. |
 | `--tracing` | Capture Playwright traces (and screenshots on failure). |
 | `--web` | Run the browser smoke tests instead of Electron. |

.github/workflows/pr-darwin-test.yml

@@ -19,31 +19,34 @@ on:
       integration_tests:
         type: boolean
         default: true
+      agent_host_e2e_tests:
+        type: boolean
+        default: false
       smoke_tests:
         type: boolean
         default: true
+      smoke_test_group:
+        type: string
+        default: all
+      network_proxy_smoke_tests:
+        type: boolean
+        default: false
 
 jobs:
   macOS-test:
     name: ${{ inputs.job_name }}
     runs-on: macos-26-xlarge
     env:
-      ARTIFACT_NAME: ${{ (inputs.electron_tests && 'electron') || (inputs.browser_tests && 'browser') || (inputs.remote_tests && 'remote') || 'unknown' }}${{ (inputs.unit_tests && !inputs.integration_tests && '-unit') || (!inputs.unit_tests && !inputs.integration_tests && inputs.smoke_tests && '-smoke') || '' }}
+      ARTIFACT_NAME: ${{ (inputs.electron_tests && 'electron') || (inputs.browser_tests && 'browser') || (inputs.remote_tests && 'remote') || 'unknown' }}${{ (inputs.unit_tests && !inputs.integration_tests && '-unit') || (!inputs.unit_tests && !inputs.integration_tests && inputs.smoke_tests && '-smoke') || '' }}${{ (inputs.agent_host_e2e_tests && '-agent-host-e2e') || (inputs.network_proxy_smoke_tests && '-network-proxy') || (inputs.smoke_tests && inputs.smoke_test_group == 'chat' && '-chat') || '' }}
       NPM_ARCH: arm64
       VSCODE_ARCH: arm64
+      VSCODE_SKIP_PRELAUNCH: '1'
     steps:
       - name: Checkout microsoft/vscode
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           lfs: true
 
-      - name: Detect Agent Host E2E changes
-        if: ${{ inputs.electron_tests && inputs.integration_tests }}
-        id: agent-host-e2e-changes
-        uses: ./.github/actions/detect-agent-host-e2e-changes
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-
       - name: Setup Node.js
         uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
@@ -129,6 +132,13 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: 🧪 Run Agent Host E2E tests
+        if: ${{ inputs.agent_host_e2e_tests }}
+        timeout-minutes: 15
+        run: node scripts/test-agent-host-e2e.ts --tfs "Agent Host E2E"
+        env:
+          VSCODE_SKIP_PRELAUNCH: '1'
+
       - name: Compile browser integration test runner
         if: ${{ inputs.browser_tests && inputs.integration_tests }}
         working-directory: test/integration/browser
@@ -174,7 +184,7 @@ jobs:
           DEBUG: "*browser*"
 
       - name: Compile extensions for integration tests & smoke tests
-        if: ${{ inputs.integration_tests || inputs.smoke_tests }}
+        if: ${{ inputs.integration_tests || inputs.smoke_tests || inputs.network_proxy_smoke_tests }}
         run: |
           set -e
           npm run gulp \
@@ -201,7 +211,7 @@ jobs:
         timeout-minutes: 20
         run: ./scripts/test-integration.sh --tfs "Integration Tests"
         env:
-          VSCODE_SKIP_AGENT_HOST_E2E: ${{ steps.agent-host-e2e-changes.outputs.affected == 'false' && '1' || '0' }}
+          VSCODE_SKIP_AGENT_HOST_E2E: '1'
           VSCODE_SKIP_PRELAUNCH: '1'
 
       - name: 🧪 Run integration tests (Browser, Webkit)
@@ -217,36 +227,36 @@ jobs:
           VSCODE_SKIP_PRELAUNCH: '1'
 
       - name: Compile smoke tests
-        if: ${{ inputs.smoke_tests }}
+        if: ${{ inputs.smoke_tests || inputs.network_proxy_smoke_tests }}
         working-directory: test/smoke
         run: npm run compile
 
       - name: Compile Copilot Chat extension for smoke tests
-        if: ${{ inputs.smoke_tests }}
+        if: ${{ (inputs.smoke_tests && !inputs.browser_tests) || inputs.network_proxy_smoke_tests }}
         working-directory: extensions/copilot
         run: npm run compile
 
       - name: Diagnostics before smoke test run
-        if: ${{ inputs.smoke_tests && always() }}
+        if: ${{ (inputs.smoke_tests || inputs.network_proxy_smoke_tests) && always() }}
         run: ps -ef
         continue-on-error: true
 
       - name: 🧪 Run smoke tests (Electron)
         if: ${{ inputs.electron_tests && inputs.smoke_tests }}
         timeout-minutes: 20
-        run: npm run smoketest-no-compile -- --tracing
+        run: npm run smoketest-no-compile -- --tracing --group ${{ inputs.smoke_test_group }}
 
       - name: Install Squid for network-isolated smoke tests
-        if: ${{ inputs.electron_tests && inputs.smoke_tests }}
+        if: ${{ inputs.electron_tests && inputs.network_proxy_smoke_tests }}
         run: brew list squid >/dev/null 2>&1 || brew install squid
 
       - name: 🧪 Run Agents Window smoke tests through macOS PAC proxy
-        if: ${{ inputs.electron_tests && inputs.smoke_tests }}
+        if: ${{ inputs.electron_tests && inputs.network_proxy_smoke_tests }}
         timeout-minutes: 40
         run: bash test/smoke/scripts/run-agents-window-network-proxy.sh
 
       - name: 🧪 Run Agents Window smoke tests through Kerberos-authenticated macOS PAC proxy
-        if: ${{ inputs.electron_tests && inputs.smoke_tests }}
+        if: ${{ inputs.electron_tests && inputs.network_proxy_smoke_tests }}
         timeout-minutes: 40
         run: bash test/smoke/scripts/run-agents-window-network-proxy.sh --kerberos
 
@@ -261,7 +271,7 @@ jobs:
         run: npm run smoketest-no-compile -- --remote --tracing
 
       - name: Diagnostics after smoke test run
-        if: ${{ inputs.smoke_tests && always() }}
+        if: ${{ (inputs.smoke_tests || inputs.network_proxy_smoke_tests) && always() }}
         run: ps -ef
         continue-on-error: true
 

.github/workflows/pr-linux-test.yml

@@ -19,31 +19,31 @@ on:
       integration_tests:
         type: boolean
         default: true
+      agent_host_e2e_tests:
+        type: boolean
+        default: false
       smoke_tests:
         type: boolean
         default: true
+      smoke_test_group:
+        type: string
+        default: all
 
 jobs:
   linux-test:
     name: ${{ inputs.job_name }}
     runs-on: ubuntu-24.04
     env:
-      ARTIFACT_NAME: ${{ (inputs.electron_tests && 'electron') || (inputs.browser_tests && 'browser') || (inputs.remote_tests && 'remote') || 'unknown' }}${{ (inputs.unit_tests && !inputs.integration_tests && '-unit') || (!inputs.unit_tests && !inputs.integration_tests && inputs.smoke_tests && '-smoke') || '' }}
+      ARTIFACT_NAME: ${{ (inputs.electron_tests && 'electron') || (inputs.browser_tests && 'browser') || (inputs.remote_tests && 'remote') || 'unknown' }}${{ (inputs.unit_tests && !inputs.integration_tests && '-unit') || (!inputs.unit_tests && !inputs.integration_tests && inputs.smoke_tests && '-smoke') || '' }}${{ (inputs.agent_host_e2e_tests && '-agent-host-e2e') || (inputs.smoke_tests && inputs.smoke_test_group == 'chat' && '-chat') || '' }}
       NPM_ARCH: x64
       VSCODE_ARCH: x64
+      VSCODE_SKIP_PRELAUNCH: '1'
     steps:
       - name: Checkout microsoft/vscode
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           lfs: true
 
-      - name: Detect Agent Host E2E changes
-        if: ${{ inputs.electron_tests && inputs.integration_tests }}
-        id: agent-host-e2e-changes
-        uses: ./.github/actions/detect-agent-host-e2e-changes
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-
       - name: Setup Node.js
         uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
@@ -329,6 +329,14 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: 🧪 Run Agent Host E2E tests
+        if: ${{ inputs.agent_host_e2e_tests }}
+        timeout-minutes: 15
+        run: node scripts/test-agent-host-e2e.ts --tfs "Agent Host E2E"
+        env:
+          DISPLAY: ":10"
+          VSCODE_SKIP_PRELAUNCH: '1'
+
       - name: Compile browser integration test runner
         if: ${{ inputs.browser_tests && inputs.integration_tests }}
         working-directory: test/integration/browser
@@ -403,7 +411,7 @@ jobs:
         run: ./scripts/test-integration.sh --tfs "Integration Tests"
         env:
           DISPLAY: ":10"
-          VSCODE_SKIP_AGENT_HOST_E2E: ${{ steps.agent-host-e2e-changes.outputs.affected == 'false' && '1' || '0' }}
+          VSCODE_SKIP_AGENT_HOST_E2E: '1'
           VSCODE_SKIP_PRELAUNCH: '1'
 
       - name: 🧪 Run integration tests (Browser, Chromium)
@@ -425,7 +433,7 @@ jobs:
         run: npm run compile
 
       - name: Compile Copilot Chat extension for smoke tests
-        if: ${{ inputs.smoke_tests }}
+        if: ${{ inputs.smoke_tests && !inputs.browser_tests }}
         working-directory: extensions/copilot
         run: npm run compile
 
@@ -442,7 +450,7 @@ jobs:
       # https://github.com/anthropics/claude-agent-sdk-typescript for the SDK
       # resolution order.
       - name: Remove musl Claude binary on glibc Linux
-        if: ${{ inputs.smoke_tests }}
+        if: ${{ inputs.smoke_tests && !inputs.browser_tests }}
         run: rm -rf node_modules/@anthropic-ai/claude-agent-sdk-linux-x64-musl node_modules/@anthropic-ai/claude-agent-sdk-linux-arm64-musl
 
       - name: Diagnostics before smoke test run (processes, max_user_watches, number of opened file handles)
@@ -457,7 +465,7 @@ jobs:
       - name: 🧪 Run smoke tests (Electron)
         if: ${{ inputs.electron_tests && inputs.smoke_tests }}
         timeout-minutes: 20
-        run: npm run smoketest-no-compile -- --tracing
+        run: npm run smoketest-no-compile -- --tracing --group ${{ inputs.smoke_test_group }}
         env:
           DISPLAY: ":10"
 

.github/workflows/pr-win32-test.yml

@@ -19,31 +19,31 @@ on:
       integration_tests:
         type: boolean
         default: true
+      agent_host_e2e_tests:
+        type: boolean
+        default: false
       smoke_tests:
         type: boolean
         default: true
+      smoke_test_group:
+        type: string
+        default: all
 
 jobs:
   windows-test:
     name: ${{ inputs.job_name }}
     runs-on: [ self-hosted, 1ES.Pool=1es-vscode-oss-windows-2022-x64, "JobId=windows-test-${{ inputs.job_name }}-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" ]
     env:
-      ARTIFACT_NAME: ${{ (inputs.electron_tests && 'electron') || (inputs.browser_tests && 'browser') || (inputs.remote_tests && 'remote') || 'unknown' }}${{ (inputs.unit_tests && !inputs.integration_tests && '-unit') || (!inputs.unit_tests && !inputs.integration_tests && inputs.smoke_tests && '-smoke') || '' }}
+      ARTIFACT_NAME: ${{ (inputs.electron_tests && 'electron') || (inputs.browser_tests && 'browser') || (inputs.remote_tests && 'remote') || 'unknown' }}${{ (inputs.unit_tests && !inputs.integration_tests && '-unit') || (!inputs.unit_tests && !inputs.integration_tests && inputs.smoke_tests && '-smoke') || '' }}${{ (inputs.agent_host_e2e_tests && '-agent-host-e2e') || (inputs.smoke_tests && inputs.smoke_test_group == 'chat' && '-chat') || '' }}
       NPM_ARCH: x64
       VSCODE_ARCH: x64
+      VSCODE_SKIP_PRELAUNCH: '1'
     steps:
       - name: Checkout microsoft/vscode
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           lfs: true
 
-      - name: Detect Agent Host E2E changes
-        if: ${{ inputs.electron_tests && inputs.integration_tests }}
-        id: agent-host-e2e-changes
-        uses: ./.github/actions/detect-agent-host-e2e-changes
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-
       - name: Setup Node.js
         uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
@@ -139,6 +139,14 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: 🧪 Run Agent Host E2E tests
+        if: ${{ inputs.agent_host_e2e_tests }}
+        timeout-minutes: 15
+        shell: pwsh
+        run: node scripts/test-agent-host-e2e.ts --tfs "Agent Host E2E"
+        env:
+          VSCODE_SKIP_PRELAUNCH: '1'
+
       - name: Compile browser integration test runner
         if: ${{ inputs.browser_tests && inputs.integration_tests }}
         working-directory: test/integration/browser
@@ -226,7 +234,7 @@ jobs:
         shell: pwsh
         run: .\scripts\test-integration.bat --tfs "Integration Tests"
         env:
-          VSCODE_SKIP_AGENT_HOST_E2E: ${{ steps.agent-host-e2e-changes.outputs.affected == 'false' && '1' || '0' }}
+          VSCODE_SKIP_AGENT_HOST_E2E: '1'
           VSCODE_SKIP_PRELAUNCH: '1'
 
       - name: 🧪 Run integration tests (Browser, Chromium)
@@ -257,7 +265,7 @@ jobs:
         run: npm run compile
 
       - name: Compile Copilot Chat extension for smoke tests
-        if: ${{ inputs.smoke_tests }}
+        if: ${{ inputs.smoke_tests && !inputs.browser_tests }}
         working-directory: extensions/copilot
         run: npm run compile
 
@@ -271,7 +279,7 @@ jobs:
         if: ${{ inputs.electron_tests && inputs.smoke_tests }}
         timeout-minutes: 20
         shell: pwsh
-        run: npm run smoketest-no-compile -- --tracing
+        run: npm run smoketest-no-compile -- --tracing --group ${{ inputs.smoke_test_group }}
 
       - name: 🧪 Run smoke tests (Browser, Chromium)
         if: ${{ inputs.browser_tests && inputs.smoke_tests }}