diff --git a/.github/skills/smoke-tests/SKILL.md b/.github/skills/smoke-tests/SKILL.md index 01c227c14e18df..1018f8e354a625 100644 --- a/.github/skills/smoke-tests/SKILL.md +++ b/.github/skills/smoke-tests/SKILL.md @@ -19,6 +19,7 @@ Both forward extra arguments after `--` to the runner (`test/smoke/test/index.js | Option | Description | |--------|-------------| | `-g ` (alias `-f`) | Grep filter on test/suite titles (mocha `grep`). | +| `--group ` | Run all tests (default), non-chat tests, or Chat, Agents Window, and Copilot CLI tests. Cannot be combined with `-g`. | | `--build ` | Run against a packaged build instead of the compiled-from-source dev build. | | `--tracing` | Capture Playwright traces (and screenshots on failure). | | `--web` | Run the browser smoke tests instead of Electron. | diff --git a/.github/workflows/pr-darwin-test.yml b/.github/workflows/pr-darwin-test.yml index 4f41f59687bfb3..cf9875a6d64223 100644 --- a/.github/workflows/pr-darwin-test.yml +++ b/.github/workflows/pr-darwin-test.yml @@ -19,31 +19,34 @@ on: integration_tests: type: boolean default: true + agent_host_e2e_tests: + type: boolean + default: false smoke_tests: type: boolean default: true + smoke_test_group: + type: string + default: all + network_proxy_smoke_tests: + type: boolean + default: false jobs: macOS-test: name: ${{ inputs.job_name }} runs-on: macos-26-xlarge env: - ARTIFACT_NAME: ${{ (inputs.electron_tests && 'electron') || (inputs.browser_tests && 'browser') || (inputs.remote_tests && 'remote') || 'unknown' }}${{ (inputs.unit_tests && !inputs.integration_tests && '-unit') || (!inputs.unit_tests && !inputs.integration_tests && inputs.smoke_tests && '-smoke') || '' }} + ARTIFACT_NAME: ${{ (inputs.electron_tests && 'electron') || (inputs.browser_tests && 'browser') || (inputs.remote_tests && 'remote') || 'unknown' }}${{ (inputs.unit_tests && !inputs.integration_tests && '-unit') || (!inputs.unit_tests && !inputs.integration_tests && inputs.smoke_tests && '-smoke') || '' }}${{ (inputs.agent_host_e2e_tests && '-agent-host-e2e') || (inputs.network_proxy_smoke_tests && '-network-proxy') || (inputs.smoke_tests && inputs.smoke_test_group == 'chat' && '-chat') || '' }} NPM_ARCH: arm64 VSCODE_ARCH: arm64 + VSCODE_SKIP_PRELAUNCH: '1' steps: - name: Checkout microsoft/vscode uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: lfs: true - - name: Detect Agent Host E2E changes - if: ${{ inputs.electron_tests && inputs.integration_tests }} - id: agent-host-e2e-changes - uses: ./.github/actions/detect-agent-host-e2e-changes - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: Setup Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: @@ -129,6 +132,13 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: 🧪 Run Agent Host E2E tests + if: ${{ inputs.agent_host_e2e_tests }} + timeout-minutes: 15 + run: node scripts/test-agent-host-e2e.ts --tfs "Agent Host E2E" + env: + VSCODE_SKIP_PRELAUNCH: '1' + - name: Compile browser integration test runner if: ${{ inputs.browser_tests && inputs.integration_tests }} working-directory: test/integration/browser @@ -174,7 +184,7 @@ jobs: DEBUG: "*browser*" - name: Compile extensions for integration tests & smoke tests - if: ${{ inputs.integration_tests || inputs.smoke_tests }} + if: ${{ inputs.integration_tests || inputs.smoke_tests || inputs.network_proxy_smoke_tests }} run: | set -e npm run gulp \ @@ -201,7 +211,7 @@ jobs: timeout-minutes: 20 run: ./scripts/test-integration.sh --tfs "Integration Tests" env: - VSCODE_SKIP_AGENT_HOST_E2E: ${{ steps.agent-host-e2e-changes.outputs.affected == 'false' && '1' || '0' }} + VSCODE_SKIP_AGENT_HOST_E2E: '1' VSCODE_SKIP_PRELAUNCH: '1' - name: 🧪 Run integration tests (Browser, Webkit) @@ -217,36 +227,36 @@ jobs: VSCODE_SKIP_PRELAUNCH: '1' - name: Compile smoke tests - if: ${{ inputs.smoke_tests }} + if: ${{ inputs.smoke_tests || inputs.network_proxy_smoke_tests }} working-directory: test/smoke run: npm run compile - name: Compile Copilot Chat extension for smoke tests - if: ${{ inputs.smoke_tests }} + if: ${{ (inputs.smoke_tests && !inputs.browser_tests) || inputs.network_proxy_smoke_tests }} working-directory: extensions/copilot run: npm run compile - name: Diagnostics before smoke test run - if: ${{ inputs.smoke_tests && always() }} + if: ${{ (inputs.smoke_tests || inputs.network_proxy_smoke_tests) && always() }} run: ps -ef continue-on-error: true - name: 🧪 Run smoke tests (Electron) if: ${{ inputs.electron_tests && inputs.smoke_tests }} timeout-minutes: 20 - run: npm run smoketest-no-compile -- --tracing + run: npm run smoketest-no-compile -- --tracing --group ${{ inputs.smoke_test_group }} - name: Install Squid for network-isolated smoke tests - if: ${{ inputs.electron_tests && inputs.smoke_tests }} + if: ${{ inputs.electron_tests && inputs.network_proxy_smoke_tests }} run: brew list squid >/dev/null 2>&1 || brew install squid - name: 🧪 Run Agents Window smoke tests through macOS PAC proxy - if: ${{ inputs.electron_tests && inputs.smoke_tests }} + if: ${{ inputs.electron_tests && inputs.network_proxy_smoke_tests }} timeout-minutes: 40 run: bash test/smoke/scripts/run-agents-window-network-proxy.sh - name: 🧪 Run Agents Window smoke tests through Kerberos-authenticated macOS PAC proxy - if: ${{ inputs.electron_tests && inputs.smoke_tests }} + if: ${{ inputs.electron_tests && inputs.network_proxy_smoke_tests }} timeout-minutes: 40 run: bash test/smoke/scripts/run-agents-window-network-proxy.sh --kerberos @@ -261,7 +271,7 @@ jobs: run: npm run smoketest-no-compile -- --remote --tracing - name: Diagnostics after smoke test run - if: ${{ inputs.smoke_tests && always() }} + if: ${{ (inputs.smoke_tests || inputs.network_proxy_smoke_tests) && always() }} run: ps -ef continue-on-error: true diff --git a/.github/workflows/pr-linux-test.yml b/.github/workflows/pr-linux-test.yml index 1eb53e5de9f00f..860055d4a660b1 100644 --- a/.github/workflows/pr-linux-test.yml +++ b/.github/workflows/pr-linux-test.yml @@ -19,31 +19,31 @@ on: integration_tests: type: boolean default: true + agent_host_e2e_tests: + type: boolean + default: false smoke_tests: type: boolean default: true + smoke_test_group: + type: string + default: all jobs: linux-test: name: ${{ inputs.job_name }} runs-on: ubuntu-24.04 env: - ARTIFACT_NAME: ${{ (inputs.electron_tests && 'electron') || (inputs.browser_tests && 'browser') || (inputs.remote_tests && 'remote') || 'unknown' }}${{ (inputs.unit_tests && !inputs.integration_tests && '-unit') || (!inputs.unit_tests && !inputs.integration_tests && inputs.smoke_tests && '-smoke') || '' }} + ARTIFACT_NAME: ${{ (inputs.electron_tests && 'electron') || (inputs.browser_tests && 'browser') || (inputs.remote_tests && 'remote') || 'unknown' }}${{ (inputs.unit_tests && !inputs.integration_tests && '-unit') || (!inputs.unit_tests && !inputs.integration_tests && inputs.smoke_tests && '-smoke') || '' }}${{ (inputs.agent_host_e2e_tests && '-agent-host-e2e') || (inputs.smoke_tests && inputs.smoke_test_group == 'chat' && '-chat') || '' }} NPM_ARCH: x64 VSCODE_ARCH: x64 + VSCODE_SKIP_PRELAUNCH: '1' steps: - name: Checkout microsoft/vscode uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: lfs: true - - name: Detect Agent Host E2E changes - if: ${{ inputs.electron_tests && inputs.integration_tests }} - id: agent-host-e2e-changes - uses: ./.github/actions/detect-agent-host-e2e-changes - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: Setup Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: @@ -329,6 +329,14 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: 🧪 Run Agent Host E2E tests + if: ${{ inputs.agent_host_e2e_tests }} + timeout-minutes: 15 + run: node scripts/test-agent-host-e2e.ts --tfs "Agent Host E2E" + env: + DISPLAY: ":10" + VSCODE_SKIP_PRELAUNCH: '1' + - name: Compile browser integration test runner if: ${{ inputs.browser_tests && inputs.integration_tests }} working-directory: test/integration/browser @@ -403,7 +411,7 @@ jobs: run: ./scripts/test-integration.sh --tfs "Integration Tests" env: DISPLAY: ":10" - VSCODE_SKIP_AGENT_HOST_E2E: ${{ steps.agent-host-e2e-changes.outputs.affected == 'false' && '1' || '0' }} + VSCODE_SKIP_AGENT_HOST_E2E: '1' VSCODE_SKIP_PRELAUNCH: '1' - name: 🧪 Run integration tests (Browser, Chromium) @@ -425,7 +433,7 @@ jobs: run: npm run compile - name: Compile Copilot Chat extension for smoke tests - if: ${{ inputs.smoke_tests }} + if: ${{ inputs.smoke_tests && !inputs.browser_tests }} working-directory: extensions/copilot run: npm run compile @@ -442,7 +450,7 @@ jobs: # https://github.com/anthropics/claude-agent-sdk-typescript for the SDK # resolution order. - name: Remove musl Claude binary on glibc Linux - if: ${{ inputs.smoke_tests }} + if: ${{ inputs.smoke_tests && !inputs.browser_tests }} run: rm -rf node_modules/@anthropic-ai/claude-agent-sdk-linux-x64-musl node_modules/@anthropic-ai/claude-agent-sdk-linux-arm64-musl - name: Diagnostics before smoke test run (processes, max_user_watches, number of opened file handles) @@ -457,7 +465,7 @@ jobs: - name: 🧪 Run smoke tests (Electron) if: ${{ inputs.electron_tests && inputs.smoke_tests }} timeout-minutes: 20 - run: npm run smoketest-no-compile -- --tracing + run: npm run smoketest-no-compile -- --tracing --group ${{ inputs.smoke_test_group }} env: DISPLAY: ":10" diff --git a/.github/workflows/pr-win32-test.yml b/.github/workflows/pr-win32-test.yml index 9edb6eb1343f32..9fc8db7095ac6d 100644 --- a/.github/workflows/pr-win32-test.yml +++ b/.github/workflows/pr-win32-test.yml @@ -19,31 +19,31 @@ on: integration_tests: type: boolean default: true + agent_host_e2e_tests: + type: boolean + default: false smoke_tests: type: boolean default: true + smoke_test_group: + type: string + default: all jobs: windows-test: name: ${{ inputs.job_name }} runs-on: [ self-hosted, 1ES.Pool=1es-vscode-oss-windows-2022-x64, "JobId=windows-test-${{ inputs.job_name }}-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" ] env: - ARTIFACT_NAME: ${{ (inputs.electron_tests && 'electron') || (inputs.browser_tests && 'browser') || (inputs.remote_tests && 'remote') || 'unknown' }}${{ (inputs.unit_tests && !inputs.integration_tests && '-unit') || (!inputs.unit_tests && !inputs.integration_tests && inputs.smoke_tests && '-smoke') || '' }} + ARTIFACT_NAME: ${{ (inputs.electron_tests && 'electron') || (inputs.browser_tests && 'browser') || (inputs.remote_tests && 'remote') || 'unknown' }}${{ (inputs.unit_tests && !inputs.integration_tests && '-unit') || (!inputs.unit_tests && !inputs.integration_tests && inputs.smoke_tests && '-smoke') || '' }}${{ (inputs.agent_host_e2e_tests && '-agent-host-e2e') || (inputs.smoke_tests && inputs.smoke_test_group == 'chat' && '-chat') || '' }} NPM_ARCH: x64 VSCODE_ARCH: x64 + VSCODE_SKIP_PRELAUNCH: '1' steps: - name: Checkout microsoft/vscode uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: lfs: true - - name: Detect Agent Host E2E changes - if: ${{ inputs.electron_tests && inputs.integration_tests }} - id: agent-host-e2e-changes - uses: ./.github/actions/detect-agent-host-e2e-changes - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: Setup Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: @@ -139,6 +139,14 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: 🧪 Run Agent Host E2E tests + if: ${{ inputs.agent_host_e2e_tests }} + timeout-minutes: 15 + shell: pwsh + run: node scripts/test-agent-host-e2e.ts --tfs "Agent Host E2E" + env: + VSCODE_SKIP_PRELAUNCH: '1' + - name: Compile browser integration test runner if: ${{ inputs.browser_tests && inputs.integration_tests }} working-directory: test/integration/browser @@ -226,7 +234,7 @@ jobs: shell: pwsh run: .\scripts\test-integration.bat --tfs "Integration Tests" env: - VSCODE_SKIP_AGENT_HOST_E2E: ${{ steps.agent-host-e2e-changes.outputs.affected == 'false' && '1' || '0' }} + VSCODE_SKIP_AGENT_HOST_E2E: '1' VSCODE_SKIP_PRELAUNCH: '1' - name: 🧪 Run integration tests (Browser, Chromium) @@ -257,7 +265,7 @@ jobs: run: npm run compile - name: Compile Copilot Chat extension for smoke tests - if: ${{ inputs.smoke_tests }} + if: ${{ inputs.smoke_tests && !inputs.browser_tests }} working-directory: extensions/copilot run: npm run compile @@ -271,7 +279,7 @@ jobs: if: ${{ inputs.electron_tests && inputs.smoke_tests }} timeout-minutes: 20 shell: pwsh - run: npm run smoketest-no-compile -- --tracing + run: npm run smoketest-no-compile -- --tracing --group ${{ inputs.smoke_test_group }} - name: 🧪 Run smoke tests (Browser, Chromium) if: ${{ inputs.browser_tests && inputs.smoke_tests }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index f196afce554fb7..16ea06fcc11d95 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -106,6 +106,21 @@ jobs: - name: Check cyclic dependencies run: node build/lib/checkCyclicDependencies.ts out-build + agent-host-e2e-changes: + name: Agent Host E2E Changes + runs-on: [ self-hosted, 1ES.Pool=1es-vscode-oss-ubuntu-22.04-x64, "JobId=agent-host-e2e-changes-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" ] + outputs: + affected: ${{ steps.detect.outputs.affected }} + steps: + - name: Checkout microsoft/vscode + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Detect Agent Host E2E changes + id: detect + uses: ./.github/actions/detect-agent-host-e2e-changes + with: + token: ${{ secrets.GITHUB_TOKEN }} + linux-cli-tests: name: Linux uses: ./.github/workflows/pr-linux-cli-test.yml @@ -131,6 +146,19 @@ jobs: unit_tests: false smoke_tests: false + linux-agent-host-e2e-tests: + name: Linux + needs: agent-host-e2e-changes + if: needs.agent-host-e2e-changes.outputs.affected == 'true' + uses: ./.github/workflows/pr-linux-test.yml + with: + job_name: Agent-Host-E2E + electron_tests: true + agent_host_e2e_tests: true + unit_tests: false + integration_tests: false + smoke_tests: false + linux-electron-smoke-tests: name: Linux uses: ./.github/workflows/pr-linux-test.yml @@ -139,6 +167,17 @@ jobs: electron_tests: true unit_tests: false integration_tests: false + smoke_test_group: core + + linux-electron-chat-smoke-tests: + name: Linux + uses: ./.github/workflows/pr-linux-test.yml + with: + job_name: Electron-Smoke-Chat + electron_tests: true + unit_tests: false + integration_tests: false + smoke_test_group: chat linux-browser-tests: name: Linux @@ -172,6 +211,19 @@ jobs: unit_tests: false smoke_tests: false + macos-agent-host-e2e-tests: + name: macOS + needs: agent-host-e2e-changes + if: needs.agent-host-e2e-changes.outputs.affected == 'true' + uses: ./.github/workflows/pr-darwin-test.yml + with: + job_name: Agent-Host-E2E + electron_tests: true + agent_host_e2e_tests: true + unit_tests: false + integration_tests: false + smoke_tests: false + macos-electron-smoke-tests: name: macOS uses: ./.github/workflows/pr-darwin-test.yml @@ -180,6 +232,28 @@ jobs: electron_tests: true unit_tests: false integration_tests: false + smoke_test_group: core + + macos-electron-chat-smoke-tests: + name: macOS + uses: ./.github/workflows/pr-darwin-test.yml + with: + job_name: Electron-Smoke-Chat + electron_tests: true + unit_tests: false + integration_tests: false + smoke_test_group: chat + + macos-electron-network-proxy-smoke-tests: + name: macOS + uses: ./.github/workflows/pr-darwin-test.yml + with: + job_name: Electron-Smoke-Network-Proxy + electron_tests: true + network_proxy_smoke_tests: true + unit_tests: false + integration_tests: false + smoke_tests: false macos-browser-tests: name: macOS @@ -213,6 +287,19 @@ jobs: unit_tests: false smoke_tests: false + windows-agent-host-e2e-tests: + name: Windows + needs: agent-host-e2e-changes + if: needs.agent-host-e2e-changes.outputs.affected == 'true' + uses: ./.github/workflows/pr-win32-test.yml + with: + job_name: Agent-Host-E2E + electron_tests: true + agent_host_e2e_tests: true + unit_tests: false + integration_tests: false + smoke_tests: false + windows-electron-smoke-tests: name: Windows uses: ./.github/workflows/pr-win32-test.yml @@ -221,6 +308,17 @@ jobs: electron_tests: true unit_tests: false integration_tests: false + smoke_test_group: core + + windows-electron-chat-smoke-tests: + name: Windows + uses: ./.github/workflows/pr-win32-test.yml + with: + job_name: Electron-Smoke-Chat + electron_tests: true + unit_tests: false + integration_tests: false + smoke_test_group: chat windows-browser-tests: name: Windows diff --git a/extensions/vscode-test-resolver/src/extension.ts b/extensions/vscode-test-resolver/src/extension.ts index 3cc55e003e8391..1270d09ae82d77 100644 --- a/extensions/vscode-test-resolver/src/extension.ts +++ b/extensions/vscode-test-resolver/src/extension.ts @@ -147,7 +147,7 @@ export function activate(context: vscode.ExtensionContext) { } const { updateUrl, commit, quality, serverDataFolderName, serverApplicationName, dataFolderName } = getProductConfiguration(); - const commandArgs = ['--host=127.0.0.1', '--port=0', '--disable-telemetry', '--disable-experiments', '--use-host-proxy', '--accept-server-license-terms']; + const commandArgs = ['--host=127.0.0.1', '--port=0', '--disable-telemetry', '--disable-experiments', '--use-host-proxy', '--accept-server-license-terms', '--force-disable-user-env']; const env = getNewEnv(); const remoteDataDir = process.env['TESTRESOLVER_DATA_FOLDER'] || path.join(os.homedir(), `${serverDataFolderName || dataFolderName}-testresolver`); const logsDir = process.env['TESTRESOLVER_LOGS_FOLDER']; diff --git a/src/vs/platform/agentHost/test/node/e2e/README.md b/src/vs/platform/agentHost/test/node/e2e/README.md index fee63fbf569091..a80b492c0c3c56 100644 --- a/src/vs/platform/agentHost/test/node/e2e/README.md +++ b/src/vs/platform/agentHost/test/node/e2e/README.md @@ -202,7 +202,7 @@ npm run test-agent-host-e2e -- --jobs 2 The complete-suite runner starts one test process per entrypoint and runs up to four concurrently. `AGENT_HOST_E2E_JOBS` or `--jobs` can lower the worker count. Each process's output is printed as one block when it completes, and any Mocha failure details are repeated after the final suite summary so failures remain easy to find. Recording and snapshot-update modes remain per-provider commands so they never make concurrent writes or real CAPI requests. -Pull request Electron jobs run the complete suite only when the changed files can affect the Agent Host, its shared platform dependencies, provider SDK versions, build infrastructure, or the E2E harness. The classification happens inside each already-allocated Electron runner so Linux, macOS, and Windows jobs remain parallel. When no relevant files changed, CI sets `VSCODE_SKIP_AGENT_HOST_E2E=1`; `test-integration.sh` and `test-integration.bat` then skip this suite while continuing with every other integration test. +Pull request CI runs the complete suite only when the changed files can affect the Agent Host, its shared platform dependencies, provider SDK versions, build infrastructure, or the E2E harness. A lightweight job classifies the change once, then dedicated Linux, macOS, and Windows jobs run the suite in parallel with the remaining Electron integration tests. When no relevant files changed, the dedicated jobs are skipped. The regular Electron integration jobs set `VSCODE_SKIP_AGENT_HOST_E2E=1` to avoid running the suite twice. Provider availability: diff --git a/test/automation/src/playwrightBrowser.ts b/test/automation/src/playwrightBrowser.ts index deeb7b07d214bb..523434b29955ce 100644 --- a/test/automation/src/playwrightBrowser.ts +++ b/test/automation/src/playwrightBrowser.ts @@ -45,6 +45,7 @@ async function launchServer(options: LaunchOptions) { '--disable-telemetry', '--disable-experiments', '--disable-workspace-trust', + '--force-disable-user-env', `--port=${port++}`, '--enable-smoke-test-driver', '--accept-server-license-terms', diff --git a/test/integration/browser/src/index.ts b/test/integration/browser/src/index.ts index a61b482678b011..a77e0f5566075a 100644 --- a/test/integration/browser/src/index.ts +++ b/test/integration/browser/src/index.ts @@ -191,7 +191,7 @@ async function launchServer(browserType: BrowserType, browserChannel: BrowserCha ...process.env }; - const serverArgs = ['--enable-proposed-api', '--disable-telemetry', '--disable-experiments', '--server-data-dir', userDataDir, '--accept-server-license-terms', '--disable-workspace-trust', '--enable-smoke-test-driver']; + const serverArgs = ['--enable-proposed-api', '--disable-telemetry', '--disable-experiments', '--server-data-dir', userDataDir, '--accept-server-license-terms', '--disable-workspace-trust', '--enable-smoke-test-driver', '--force-disable-user-env']; let serverLocation: string; if (process.env.VSCODE_REMOTE_SERVER_PATH) { diff --git a/test/smoke/test/index.js b/test/smoke/test/index.js index d9115a3b2b478c..1de9c2227ae678 100644 --- a/test/smoke/test/index.js +++ b/test/smoke/test/index.js @@ -13,17 +13,25 @@ const minimist = require('minimist'); const [, , ...args] = process.argv; const opts = minimist(args, { boolean: ['web', 'fail-zero'], - string: ['f', 'g'] + string: ['f', 'g', 'group'] }); const suite = opts['web'] ? 'Browser Smoke Tests' : 'Desktop Smoke Tests'; +const smokeTestGroup = opts['group'] || 'all'; +if (!['all', 'core', 'chat'].includes(smokeTestGroup)) { + throw new Error(`Unknown smoke test group '${smokeTestGroup}'. Expected all, core, or chat.`); +} +if (smokeTestGroup !== 'all' && (opts['f'] || opts['g'])) { + throw new Error('Smoke test groups cannot be combined with a grep filter.'); +} const options = { color: true, timeout: 2 * 60 * 1000, slow: 30 * 1000, - grep: opts['f'] || opts['g'], - failZero: opts['fail-zero'] + grep: opts['f'] || opts['g'] || (smokeTestGroup !== 'all' ? 'Chat|Agents Window|Copilot CLI' : undefined), + invert: smokeTestGroup === 'core', + failZero: opts['fail-zero'] || smokeTestGroup !== 'all' }; if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY || process.env.GITHUB_WORKSPACE) { diff --git a/test/unit/browser/index.js b/test/unit/browser/index.js index 7789165f19e2ae..e6bfe06698ab89 100644 --- a/test/unit/browser/index.js +++ b/test/unit/browser/index.js @@ -91,7 +91,7 @@ const withReporter = (function () { { const testResultsRoot = process.env.BUILD_ARTIFACTSTAGINGDIRECTORY || process.env.GITHUB_WORKSPACE; return (browserType, runner) => { - new mocha.reporters.Spec(runner); + new mocha.reporters.Dot(runner); new MochaJUnitReporter(runner, { reporterOptions: { testsuitesTitle: `${args.tfs} ${process.platform}`,