diff --git a/.github/workflows/llmobs.yml b/.github/workflows/llmobs.yml index 14774cb80f7..27aad8ee9e2 100644 --- a/.github/workflows/llmobs.yml +++ b/.github/workflows/llmobs.yml @@ -72,3 +72,24 @@ jobs: uses: ./.github/actions/testagent/logs with: suffix: llmobs-${{ github.job }} + + aws-sdk: + runs-on: ubuntu-latest + env: + PLUGINS: aws-sdk + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/testagent/start + - uses: ./.github/actions/node/setup + - uses: ./.github/actions/install + - uses: ./.github/actions/node/18 + - run: yarn test:llmobs:plugins:ci + shell: bash + - uses: ./.github/actions/node/latest + - run: yarn test:llmobs:plugins:ci + shell: bash + - uses: codecov/codecov-action@v3 + - if: always() + uses: ./.github/actions/testagent/logs + with: + suffix: llmobs-${{ github.job }} diff --git a/README.md b/README.md index f8a761ca117..66f70b3de42 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # `dd-trace`: Node.js APM Tracer Library [![npm v5](https://img.shields.io/npm/v/dd-trace/latest?color=blue&label=dd-trace%40v5&logo=npm)](https://www.npmjs.com/package/dd-trace) -[![npm v4](https://img.shields.io/npm/v/dd-trace/latest-node16?color=blue&label=dd-trace%40v4&logo=npm)](https://www.npmjs.com/package/dd-trace/v/latest-node16) [![codecov](https://codecov.io/gh/DataDog/dd-trace-js/branch/master/graph/badge.svg)](https://codecov.io/gh/DataDog/dd-trace-js) Bits the dog JavaScript @@ -25,16 +24,16 @@ Most of the documentation for `dd-trace` is available on these webpages: | Release Line | Latest Version | Node.js | [SSI](https://docs.datadoghq.com/tracing/trace_collection/automatic_instrumentation/single-step-apm/?tab=linuxhostorvm) | [K8s Injection](https://docs.datadoghq.com/tracing/trace_collection/library_injection_local/?tab=kubernetes) |Status |Initial Release | End of Life | | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | -| [`v1`](https://github.com/DataDog/dd-trace-js/tree/v1.x) | ![npm v1](https://img.shields.io/npm/v/dd-trace/legacy-v1?color=white&label=%20&style=flat-square) | `>= v12` | NO | NO | **End of Life** | 2021-07-13 | 2022-02-25 | -| [`v2`](https://github.com/DataDog/dd-trace-js/tree/v2.x) | ![npm v2](https://img.shields.io/npm/v/dd-trace/latest-node12?color=white&label=%20&style=flat-square) | `>= v12` | NO | NO | **End of Life** | 2022-01-28 | 2023-08-15 | -| [`v3`](https://github.com/DataDog/dd-trace-js/tree/v3.x) | ![npm v3](https://img.shields.io/npm/v/dd-trace/latest-node14?color=white&label=%20&style=flat-square) | `>= v14` | NO | YES | **End of Life** | 2022-08-15 | 2024-05-15 | -| [`v4`](https://github.com/DataDog/dd-trace-js/tree/v4.x) | ![npm v4](https://img.shields.io/npm/v/dd-trace/latest-node16?color=white&label=%20&style=flat-square) | `>= v16` | YES | YES | **Maintenance** | 2023-05-12 | 2025-01-11 | +| [`v1`](https://github.com/DataDog/dd-trace-js/tree/v1.x) | ![npm v1](https://img.shields.io/npm/v/dd-trace/legacy-v1?color=white&label=%20&style=flat-square) | `>= v12` | NO | NO | **EOL** | 2021-07-13 | 2022-02-25 | +| [`v2`](https://github.com/DataDog/dd-trace-js/tree/v2.x) | ![npm v2](https://img.shields.io/npm/v/dd-trace/latest-node12?color=white&label=%20&style=flat-square) | `>= v12` | NO | NO | **EOL** | 2022-01-28 | 2023-08-15 | +| [`v3`](https://github.com/DataDog/dd-trace-js/tree/v3.x) | ![npm v3](https://img.shields.io/npm/v/dd-trace/latest-node14?color=white&label=%20&style=flat-square) | `>= v14` | NO | YES | **EOL** | 2022-08-15 | 2024-05-15 | +| [`v4`](https://github.com/DataDog/dd-trace-js/tree/v4.x) | ![npm v4](https://img.shields.io/npm/v/dd-trace/latest-node16?color=white&label=%20&style=flat-square) | `>= v16` | YES | YES | **EOL** | 2023-05-12 | 2025-01-11 | | [`v5`](https://github.com/DataDog/dd-trace-js/tree/v5.x) | ![npm v5](https://img.shields.io/npm/v/dd-trace/latest?color=white&label=%20&style=flat-square) | `>= v18` | YES | YES | **Current** | 2024-01-11 | Unknown | +* EOL = End-of-life * SSI = Single-Step Install -We currently maintain two release lines, namely `v5`, and `v4`. -Features and bug fixes that are merged are released to the `v5` line and, if appropriate, also `v4`. +We currently maintain one release line, namely `v5`. For any new projects it is recommended to use the `v5` release line: @@ -43,20 +42,22 @@ $ npm install dd-trace $ yarn add dd-trace ``` -However, existing projects that already use the `v4` release line, or projects that need to support EOL versions of Node.js, may continue to use these release lines. +Existing projects that need to use EOL versions of Node.js may continue to use these older release lines. This is done by specifying the version when installing the package. ```sh -$ npm install dd-trace@4 -$ yarn add dd-trace@4 +$ npm install dd-trace@4 # or whatever version you need +$ yarn add dd-trace@4 # or whatever version you need ``` +Note, however, that the end-of-life release lines are no longer maintained and will not receive updates. + Any backwards-breaking functionality that is introduced into the library will result in an increase of the major version of the library and therefore a new release line. Such releases are kept to a minimum to reduce the pain of upgrading the library. When a new release line is introduced the previous release line then enters maintenance mode where it will receive updates for the next year. Once that year is up the release line enters End of Life and will not receive new updates. -The library also follows the Node.js LTS lifecycle wherein new release lines drop compatibility with Node.js versions that reach end of life (with the maintenance release line still receiving updates for a year). +The library also follows the Node.js LTS lifecycle wherein new release lines drop compatibility with Node.js versions that reach end-of-life (with the maintenance release line still receiving updates for a year). For more information about library versioning and compatibility, see the [NodeJS Compatibility Requirements](https://docs.datadoghq.com/tracing/trace_collection/compatibility/nodejs/#releases) page. diff --git a/docs/test.ts b/docs/test.ts index 2c2cbea332e..c353e90b6ca 100644 --- a/docs/test.ts +++ b/docs/test.ts @@ -136,7 +136,10 @@ tracer.init({ redactionEnabled: true, redactionNamePattern: 'password', redactionValuePattern: 'bearer', - telemetryVerbosity: 'OFF' + telemetryVerbosity: 'OFF', + stackTrace: { + enabled: true + } } }); diff --git a/index.d.ts b/index.d.ts index 8984d02f81a..8d3fdf24ded 100644 --- a/index.d.ts +++ b/index.d.ts @@ -2233,7 +2233,17 @@ declare namespace tracer { /** * Specifies the verbosity of the sent telemetry. Default 'INFORMATION' */ - telemetryVerbosity?: string + telemetryVerbosity?: string, + + /** + * Configuration for stack trace reporting + */ + stackTrace?: { + /** Whether to enable stack trace reporting. + * @default true + */ + enabled?: boolean, + } } export namespace llmobs { diff --git a/integration-tests/automatic-log-submission.spec.js b/integration-tests/automatic-log-submission.spec.js index eade717dcf1..e8d005de538 100644 --- a/integration-tests/automatic-log-submission.spec.js +++ b/integration-tests/automatic-log-submission.spec.js @@ -12,9 +12,6 @@ const { } = require('./helpers') const { FakeCiVisIntake } = require('./ci-visibility-intake') const webAppServer = require('./ci-visibility/web-app-server') -const { NODE_MAJOR } = require('../version') - -const cucumberVersion = NODE_MAJOR <= 16 ? '9' : 'latest' describe('test visibility automatic log submission', () => { let sandbox, cwd, receiver, childProcess, webAppPort @@ -23,7 +20,7 @@ describe('test visibility automatic log submission', () => { before(async () => { sandbox = await createSandbox([ 'mocha', - `@cucumber/cucumber@${cucumberVersion}`, + '@cucumber/cucumber', 'jest', 'winston', 'chai@4' diff --git a/integration-tests/cucumber/cucumber.spec.js b/integration-tests/cucumber/cucumber.spec.js index b46205fcb05..ebda279f8c6 100644 --- a/integration-tests/cucumber/cucumber.spec.js +++ b/integration-tests/cucumber/cucumber.spec.js @@ -3,7 +3,6 @@ const { exec } = require('child_process') const getPort = require('get-port') -const semver = require('semver') const { assert } = require('chai') const { @@ -42,12 +41,12 @@ const { DI_DEBUG_ERROR_PREFIX, DI_DEBUG_ERROR_FILE_SUFFIX, DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX, - DI_DEBUG_ERROR_LINE_SUFFIX + DI_DEBUG_ERROR_LINE_SUFFIX, + TEST_RETRY_REASON } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') -const isOldNode = semver.satisfies(process.version, '<=16') -const versions = ['7.0.0', isOldNode ? '9' : 'latest'] +const versions = ['7.0.0', 'latest'] const runTestsCommand = './node_modules/.bin/cucumber-js ci-visibility/features/*.feature' const runTestsWithCoverageCommand = './node_modules/nyc/bin/nyc.js -r=text-summary ' + @@ -844,15 +843,13 @@ versions.forEach(version => { it('retries new tests', (done) => { const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) // cucumber.ci-visibility/features/farewell.feature.Say whatever will be considered new receiver.setKnownTests( @@ -884,6 +881,9 @@ versions.forEach(version => { retriedTests.length ) assert.equal(retriedTests.length, NUM_RETRIES_EFD) + retriedTests.forEach(test => { + assert.propertyVal(test.meta, TEST_RETRY_REASON, 'efd') + }) // Test name does not change newTests.forEach(test => { assert.equal(test.meta[TEST_NAME], 'Say whatever') @@ -907,15 +907,13 @@ versions.forEach(version => { it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => { const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -928,8 +926,12 @@ versions.forEach(version => { const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true' ) - // new tests are not detected - assert.equal(newTests.length, 0) + // new tests are detected but not retried + assert.equal(newTests.length, 1) + const retriedTests = tests.filter(test => + test.meta[TEST_IS_RETRY] === 'true' + ) + assert.equal(retriedTests.length, 0) }) // cucumber.ci-visibility/features/farewell.feature.Say whatever will be considered new receiver.setKnownTests({ @@ -957,15 +959,13 @@ versions.forEach(version => { it('retries flaky tests and sets exit code to 0 as long as one attempt passes', (done) => { const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) // Tests in "cucumber.ci-visibility/features-flaky/flaky.feature" will be considered new receiver.setKnownTests({}) @@ -1014,15 +1014,13 @@ versions.forEach(version => { it('does not retry tests that are skipped', (done) => { const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) // "cucumber.ci-visibility/features/farewell.feature.Say whatever" will be considered new // "cucumber.ci-visibility/features/greetings.feature.Say skip" will be considered new @@ -1066,15 +1064,13 @@ versions.forEach(version => { it('does not run EFD if the known tests request fails', (done) => { const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) receiver.setKnownTestsResponseCode(500) receiver.setKnownTests({}) @@ -1108,16 +1104,14 @@ versions.forEach(version => { it('bails out of EFD if the percentage of new tests is too high', (done) => { const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 0 - } + }, + known_tests_enabled: true }) // tests in cucumber.ci-visibility/features/farewell.feature will be considered new receiver.setKnownTests( @@ -1160,20 +1154,70 @@ versions.forEach(version => { }) }) + it('disables early flake detection if known tests should not be requested', (done) => { + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + }, + known_tests_enabled: false + }) + // cucumber.ci-visibility/features/farewell.feature.Say whatever will be considered new + receiver.setKnownTests( + { + cucumber: { + 'ci-visibility/features/farewell.feature': ['Say farewell'], + 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo', 'Say skip'] + } + } + ) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + // no new tests detected + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 0) + // no retries + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + }) + + childProcess = exec( + runTestsCommand, + { + cwd, + env: envVars, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + if (version !== '7.0.0') { // EFD in parallel mode only supported from cucumber>=11 context('parallel mode', () => { it('retries new tests', (done) => { const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) // cucumber.ci-visibility/features/farewell.feature.Say whatever will be considered new receiver.setKnownTests( @@ -1231,15 +1275,13 @@ versions.forEach(version => { it('retries flaky tests and sets exit code to 0 as long as one attempt passes', (done) => { const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) // Tests in "cucumber.ci-visibility/features-flaky/flaky.feature" will be considered new receiver.setKnownTests({}) @@ -1293,16 +1335,14 @@ versions.forEach(version => { it('bails out of EFD if the percentage of new tests is too high', (done) => { const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 0 - } + }, + known_tests_enabled: true }) // tests in cucumber.ci-visibility/features/farewell.feature will be considered new receiver.setKnownTests( @@ -1350,15 +1390,13 @@ versions.forEach(version => { it('does not retry tests that are skipped', (done) => { const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) // "cucumber.ci-visibility/features/farewell.feature.Say whatever" will be considered new // "cucumber.ci-visibility/features/greetings.feature.Say skip" will be considered new @@ -1909,5 +1947,54 @@ versions.forEach(version => { }) }) }) + + context('known tests without early flake detection', () => { + it('detects new tests without retrying them', (done) => { + receiver.setSettings({ + early_flake_detection: { + enabled: false + }, + known_tests_enabled: true + }) + // cucumber.ci-visibility/features/farewell.feature.Say whatever will be considered new + receiver.setKnownTests( + { + cucumber: { + 'ci-visibility/features/farewell.feature': ['Say farewell'], + 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo', 'Say skip'] + } + } + ) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + // new tests detected but not retried + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 1) + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + }) + + childProcess = exec( + runTestsCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + }) }) }) diff --git a/integration-tests/cypress/cypress.spec.js b/integration-tests/cypress/cypress.spec.js index 0a6f5f065f9..d1fda8baa23 100644 --- a/integration-tests/cypress/cypress.spec.js +++ b/integration-tests/cypress/cypress.spec.js @@ -35,7 +35,8 @@ const { TEST_SUITE, TEST_CODE_OWNERS, TEST_SESSION_NAME, - TEST_LEVEL_EVENT_TYPES + TEST_LEVEL_EVENT_TYPES, + TEST_RETRY_REASON } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') @@ -1019,15 +1020,13 @@ moduleTypes.forEach(({ context('early flake detection', () => { it('retries new tests', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) receiver.setKnownTests({ @@ -1051,6 +1050,10 @@ moduleTypes.forEach(({ const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') assert.equal(retriedTests.length, NUM_RETRIES_EFD) + retriedTests.forEach((retriedTest) => { + assert.equal(retriedTest.meta[TEST_RETRY_REASON], 'efd') + }) + newTests.forEach(newTest => { assert.equal(newTest.resource, 'cypress/e2e/spec.cy.js.context passes') }) @@ -1092,15 +1095,13 @@ moduleTypes.forEach(({ it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) receiver.setKnownTests({ @@ -1123,8 +1124,12 @@ moduleTypes.forEach(({ const tests = events.filter(event => event.type === 'test').map(event => event.content) assert.equal(tests.length, 2) + // new tests are detected but not retried const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') - assert.equal(newTests.length, 0) + assert.equal(newTests.length, 1) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) const testSession = events.find(event => event.type === 'test_session_end').content assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) @@ -1154,15 +1159,13 @@ moduleTypes.forEach(({ it('does not retry tests that are skipped', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) receiver.setKnownTests({}) @@ -1211,15 +1214,13 @@ moduleTypes.forEach(({ it('does not run EFD if the known tests request fails', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) receiver.setKnownTestsResponseCode(500) @@ -1264,6 +1265,70 @@ moduleTypes.forEach(({ }).catch(done) }) }) + + it('disables early flake detection if known tests should not be requested', (done) => { + receiver.setSettings({ + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + }, + known_tests_enabled: false + }) + + receiver.setKnownTests({ + cypress: { + 'cypress/e2e/spec.cy.js': [ + // 'context passes', // This test will be considered new + 'other context fails' + ] + } + }) + + const { + NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress + ...restEnvVars + } = getCiVisEvpProxyConfig(receiver.port) + + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + assert.equal(tests.length, 2) + + // new tests are not detected + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 0) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + }) + + const specToRun = 'cypress/e2e/spec.cy.js' + childProcess = exec( + version === 'latest' ? testCommand : `${testCommand} --spec ${specToRun}`, + { + cwd, + env: { + ...restEnvVars, + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: specToRun, + DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED: 'false' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise.then(() => { + done() + }).catch(done) + }) + }) }) context('flaky test retries', () => { @@ -1511,5 +1576,65 @@ moduleTypes.forEach(({ }).catch(done) }) }) + + context('known tests without early flake detection', () => { + it('detects new tests without retrying them', (done) => { + receiver.setSettings({ + known_tests_enabled: true + }) + + receiver.setKnownTests({ + cypress: { + 'cypress/e2e/spec.cy.js': [ + // 'context passes', // This test will be considered new + 'other context fails' + ] + } + }) + + const { + NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress + ...restEnvVars + } = getCiVisEvpProxyConfig(receiver.port) + + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + assert.equal(tests.length, 2) + + // new tests are detected but not retried + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 1) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + }) + + const specToRun = 'cypress/e2e/spec.cy.js' + childProcess = exec( + version === 'latest' ? testCommand : `${testCommand} --spec ${specToRun}`, + { + cwd, + env: { + ...restEnvVars, + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: specToRun, + DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED: 'false' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise.then(() => { + done() + }).catch(done) + }) + }) + }) }) }) diff --git a/integration-tests/init.spec.js b/integration-tests/init.spec.js index fc274fb1480..d9738a8160b 100644 --- a/integration-tests/init.spec.js +++ b/integration-tests/init.spec.js @@ -7,7 +7,6 @@ const { } = require('./helpers') const path = require('path') const fs = require('fs') -const { DD_MAJOR } = require('../version') const DD_INJECTION_ENABLED = 'tracing' const DD_INJECT_FORCE = 'true' @@ -104,13 +103,13 @@ function testRuntimeVersionChecks (arg, filename) { it('should not initialize the tracer', () => doTest(`Aborting application instrumentation due to incompatible_runtime. Found incompatible runtime nodejs ${process.versions.node}, Supported runtimes: nodejs \ ->=${DD_MAJOR === 4 ? '16' : '18'}. +>=18. false `, ...telemetryAbort)) it('should initialize the tracer, if DD_INJECT_FORCE', () => doTestForced(`Aborting application instrumentation due to incompatible_runtime. Found incompatible runtime nodejs ${process.versions.node}, Supported runtimes: nodejs \ ->=${DD_MAJOR === 4 ? '16' : '18'}. +>=18. DD_INJECT_FORCE enabled, allowing unsupported runtimes and continuing. Application instrumentation bootstrapping complete true diff --git a/integration-tests/jest/jest.spec.js b/integration-tests/jest/jest.spec.js index ac604d96b5e..784ea393e5a 100644 --- a/integration-tests/jest/jest.spec.js +++ b/integration-tests/jest/jest.spec.js @@ -30,6 +30,7 @@ const { TEST_NAME, JEST_DISPLAY_NAME, TEST_EARLY_FLAKE_ABORT_REASON, + TEST_RETRY_REASON, TEST_SOURCE_START, TEST_CODE_OWNERS, TEST_SESSION_NAME, @@ -1609,16 +1610,14 @@ describe('jest CommonJS', () => { }) const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { @@ -1652,6 +1651,9 @@ describe('jest CommonJS', () => { retriedTests.length ) assert.equal(retriedTests.length, NUM_RETRIES_EFD) + retriedTests.forEach(test => { + assert.propertyVal(test.meta, TEST_RETRY_REASON, 'efd') + }) // Test name does not change newTests.forEach(test => { assert.equal(test.meta[TEST_NAME], 'ci visibility 2 can report tests 2') @@ -1682,16 +1684,14 @@ describe('jest CommonJS', () => { } }) receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': 3 }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const parameterizedTestFile = 'test-parameterized.js' @@ -1757,16 +1757,14 @@ describe('jest CommonJS', () => { } }) receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': 3 }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -1779,8 +1777,12 @@ describe('jest CommonJS', () => { const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true' ) - // new tests are not detected - assert.equal(newTests.length, 0) + // new tests are detected but not retried + assert.equal(newTests.length, 1) + const retriedTests = tests.filter(test => + test.meta[TEST_IS_RETRY] === 'true' + ) + assert.equal(retriedTests.length, 0) }) childProcess = exec( @@ -1809,16 +1811,14 @@ describe('jest CommonJS', () => { const NUM_RETRIES_EFD = 5 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -1875,16 +1875,14 @@ describe('jest CommonJS', () => { const NUM_RETRIES_EFD = 5 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -1931,16 +1929,14 @@ describe('jest CommonJS', () => { receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': 3 }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) // Tests from ci-visibility/test/skipped-and-todo-test will be considered new receiver.setKnownTests({ @@ -1999,16 +1995,14 @@ describe('jest CommonJS', () => { const NUM_RETRIES_EFD = 5 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -2051,16 +2045,14 @@ describe('jest CommonJS', () => { const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -2127,16 +2119,14 @@ describe('jest CommonJS', () => { const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -2183,16 +2173,14 @@ describe('jest CommonJS', () => { const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 1 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -2235,16 +2223,14 @@ describe('jest CommonJS', () => { }) const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -2301,6 +2287,66 @@ describe('jest CommonJS', () => { }).catch(done) }) }) + + it('disables early flake detection if known tests should not be requested', (done) => { + receiver.setSettings({ + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': 3 + } + }, + known_tests_enabled: false + }) + + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new + receiver.setKnownTests({ + jest: { + 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const oldTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test.js' + ) + oldTests.forEach(test => { + assert.notProperty(test.meta, TEST_IS_NEW) + }) + assert.equal(oldTests.length, 1) + const newTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test-2.js' + ) + newTests.forEach(test => { + assert.notProperty(test.meta, TEST_IS_NEW) + }) + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { ...getCiVisEvpProxyConfig(receiver.port), TESTS_TO_RUN: 'test/ci-visibility-test' }, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) }) context('flaky test retries', () => { @@ -2797,4 +2843,66 @@ describe('jest CommonJS', () => { }) }) }) + + context('known tests without early flake detection', () => { + it('detects new tests without retrying them', (done) => { + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new + receiver.setKnownTests({ + jest: { + 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] + } + }) + receiver.setSettings({ + early_flake_detection: { + enabled: false + }, + known_tests_enabled: true + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + // no other tests are considered new + const oldTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test.js' + ) + oldTests.forEach(test => { + assert.notProperty(test.meta, TEST_IS_NEW) + }) + assert.equal(oldTests.length, 1) + + const newTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test-2.js' + ) + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // no test has been retried + assert.equal(retriedTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { ...getCiVisEvpProxyConfig(receiver.port), TESTS_TO_RUN: 'test/ci-visibility-test' }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + }) }) diff --git a/integration-tests/mocha/mocha.spec.js b/integration-tests/mocha/mocha.spec.js index a7c23b067df..21e7670d077 100644 --- a/integration-tests/mocha/mocha.spec.js +++ b/integration-tests/mocha/mocha.spec.js @@ -40,7 +40,8 @@ const { DI_DEBUG_ERROR_PREFIX, DI_DEBUG_ERROR_FILE_SUFFIX, DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX, - DI_DEBUG_ERROR_LINE_SUFFIX + DI_DEBUG_ERROR_LINE_SUFFIX, + TEST_RETRY_REASON } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') @@ -1141,16 +1142,14 @@ describe('mocha CommonJS', function () { }) const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { @@ -1184,6 +1183,9 @@ describe('mocha CommonJS', function () { retriedTests.length ) assert.equal(retriedTests.length, NUM_RETRIES_EFD) + retriedTests.forEach(test => { + assert.propertyVal(test.meta, TEST_RETRY_REASON, 'efd') + }) // Test name does not change newTests.forEach(test => { assert.equal(test.meta[TEST_NAME], 'ci visibility 2 can report tests 2') @@ -1220,16 +1222,14 @@ describe('mocha CommonJS', function () { } }) receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': 3 }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -1298,16 +1298,14 @@ describe('mocha CommonJS', function () { } }) receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': 3 }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -1320,8 +1318,12 @@ describe('mocha CommonJS', function () { const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true' ) - // new tests are not detected - assert.equal(newTests.length, 0) + // new tests are detected but not retried + assert.equal(newTests.length, 1) + const retriedTests = tests.filter(test => + test.meta[TEST_IS_RETRY] === 'true' + ) + assert.equal(retriedTests.length, 0) }) childProcess = exec( @@ -1339,6 +1341,7 @@ describe('mocha CommonJS', function () { stdio: 'inherit' } ) + childProcess.on('exit', () => { eventsPromise.then(() => { done() @@ -1352,16 +1355,14 @@ describe('mocha CommonJS', function () { const NUM_RETRIES_EFD = 5 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -1421,16 +1422,14 @@ describe('mocha CommonJS', function () { const NUM_RETRIES_EFD = 5 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -1472,16 +1471,14 @@ describe('mocha CommonJS', function () { it('handles spaces in test names', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': 3 }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) // Tests from ci-visibility/test/skipped-and-todo-test will be considered new receiver.setKnownTests({ @@ -1541,16 +1538,14 @@ describe('mocha CommonJS', function () { const NUM_RETRIES_EFD = 5 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -1595,16 +1590,14 @@ describe('mocha CommonJS', function () { const NUM_RETRIES_EFD = 3 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -1668,16 +1661,14 @@ describe('mocha CommonJS', function () { const NUM_RETRIES_EFD = 5 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 0 - } + }, + known_tests_enabled: true }) // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new receiver.setKnownTests({ @@ -1732,16 +1723,14 @@ describe('mocha CommonJS', function () { const NUM_RETRIES_EFD = 5 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -1770,6 +1759,7 @@ describe('mocha CommonJS', function () { // Test name does not change retriedTests.forEach(test => { assert.equal(test.meta[TEST_NAME], 'fail occasionally fails') + assert.equal(test.meta[TEST_RETRY_REASON], 'efd') }) }) @@ -1787,22 +1777,21 @@ describe('mocha CommonJS', function () { }).catch(done) }) }) + it('retries new tests when using the programmatic API', (done) => { // Tests from ci-visibility/test/occasionally-failing-test will be considered new receiver.setKnownTests({}) const NUM_RETRIES_EFD = 5 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 100 - } + }, + known_tests_enabled: true }) const eventsPromise = receiver @@ -1855,20 +1844,19 @@ describe('mocha CommonJS', function () { }).catch(done) }) }) + it('bails out of EFD if the percentage of new tests is too high', (done) => { const NUM_RETRIES_EFD = 5 receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 0 - } + }, + known_tests_enabled: true }) // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new receiver.setKnownTests({ @@ -1917,6 +1905,71 @@ describe('mocha CommonJS', function () { }) }) }) + + it('disables early flake detection if known tests should not be requested', (done) => { + receiver.setSettings({ + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': 3 + } + }, + known_tests_enabled: false + }) + // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new + receiver.setKnownTests({ + mocha: { + 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + const oldTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test.js' + ) + oldTests.forEach(test => { + assert.notProperty(test.meta, TEST_IS_NEW) + }) + assert.equal(oldTests.length, 1) + const newTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test-2.js' + ) + newTests.forEach(test => { + assert.notProperty(test.meta, TEST_IS_NEW) + }) + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test/ci-visibility-test.js', + './test/ci-visibility-test-2.js' + ]) + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) }) context('auto test retries', () => { @@ -2399,4 +2452,72 @@ describe('mocha CommonJS', function () { }) }) }) + + context('known tests without early flake detection', () => { + it('detects new tests without retrying them', (done) => { + receiver.setSettings({ + early_flake_detection: { + enabled: false + }, + known_tests_enabled: true + }) + receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] }) + // Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new + receiver.setKnownTests({ + mocha: { + 'ci-visibility/test/ci-visibility-test.js': ['ci visibility can report tests'] + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + // no other tests are considered new + const oldTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test.js' + ) + oldTests.forEach(test => { + assert.notProperty(test.meta, TEST_IS_NEW) + }) + assert.equal(oldTests.length, 1) + + const newTests = tests.filter(test => + test.meta[TEST_SUITE] === 'ci-visibility/test/ci-visibility-test-2.js' + ) + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // no test has been retried + assert.equal(retriedTests.length, 0) + }) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './test/ci-visibility-test.js', + './test/ci-visibility-test-2.js' + ]) + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + }) }) diff --git a/integration-tests/playwright/playwright.spec.js b/integration-tests/playwright/playwright.spec.js index 3f6a49e01b7..691a09b4d13 100644 --- a/integration-tests/playwright/playwright.spec.js +++ b/integration-tests/playwright/playwright.spec.js @@ -24,7 +24,8 @@ const { TEST_SUITE, TEST_CODE_OWNERS, TEST_SESSION_NAME, - TEST_LEVEL_EVENT_TYPES + TEST_LEVEL_EVENT_TYPES, + TEST_RETRY_REASON } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') @@ -252,15 +253,13 @@ versions.forEach((version) => { context('early flake detection', () => { it('retries new tests', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) receiver.setKnownTests( @@ -303,6 +302,10 @@ versions.forEach((version) => { assert.equal(retriedTests.length, NUM_RETRIES_EFD) + retriedTests.forEach(test => { + assert.propertyVal(test.meta, TEST_RETRY_REASON, 'efd') + }) + // all but one has been retried assert.equal(retriedTests.length, newTests.length - 1) }) @@ -326,15 +329,13 @@ versions.forEach((version) => { it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) receiver.setKnownTests( @@ -366,12 +367,12 @@ versions.forEach((version) => { const newTests = tests.filter(test => test.resource.endsWith('should work with passing tests') ) + // new tests are detected but not retried newTests.forEach(test => { - assert.notProperty(test.meta, TEST_IS_NEW) + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') }) const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') - assert.equal(retriedTests.length, 0) }) @@ -395,15 +396,13 @@ versions.forEach((version) => { it('does not retry tests that are skipped', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) receiver.setKnownTests( @@ -467,15 +466,13 @@ versions.forEach((version) => { it('does not run EFD if the known tests request fails', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) receiver.setKnownTestsResponseCode(500) @@ -515,6 +512,74 @@ versions.forEach((version) => { .catch(done) }) }) + + it('disables early flake detection if known tests should not be requested', (done) => { + receiver.setSettings({ + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + }, + known_tests_enabled: false + }) + + receiver.setKnownTests( + { + playwright: { + 'landing-page-test.js': [ + // it will be considered new + // 'highest-level-describe leading and trailing spaces should work with passing tests', + 'highest-level-describe leading and trailing spaces should work with skipped tests', + 'highest-level-describe leading and trailing spaces should work with fixme', + 'highest-level-describe leading and trailing spaces should work with annotated tests' + ], + 'skipped-suite-test.js': [ + 'should work with fixme root' + ], + 'todo-list-page-test.js': [ + 'playwright should work with failing tests', + 'should work with fixme root' + ] + } + } + ) + + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const newTests = tests.filter(test => + test.resource.endsWith('should work with passing tests') + ) + newTests.forEach(test => { + assert.notProperty(test.meta, TEST_IS_NEW) + }) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + }) + + childProcess = exec( + './node_modules/.bin/playwright test -c playwright.config.js', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + PW_BASE_URL: `http://localhost:${webAppPort}` + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise.then(() => done()).catch(done) + }) + }) }) } @@ -716,5 +781,72 @@ versions.forEach((version) => { }).catch(done) }) }) + + if (version === 'latest') { + context('known tests without early flake detection', () => { + it('detects new tests without retrying them', (done) => { + receiver.setSettings({ + known_tests_enabled: true + }) + + receiver.setKnownTests( + { + playwright: { + 'landing-page-test.js': [ + // it will be considered new + // 'highest-level-describe leading and trailing spaces should work with passing tests', + 'highest-level-describe leading and trailing spaces should work with skipped tests', + 'highest-level-describe leading and trailing spaces should work with fixme', + 'highest-level-describe leading and trailing spaces should work with annotated tests' + ], + 'skipped-suite-test.js': [ + 'should work with fixme root' + ], + 'todo-list-page-test.js': [ + 'playwright should work with failing tests', + 'should work with fixme root' + ] + } + } + ) + + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const newTests = tests.filter(test => + test.resource.endsWith('should work with passing tests') + ) + // new tests detected but no retries + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + }) + + childProcess = exec( + './node_modules/.bin/playwright test -c playwright.config.js', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + PW_BASE_URL: `http://localhost:${webAppPort}` + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise.then(() => done()).catch(done) + }) + }) + }) + } }) }) diff --git a/integration-tests/selenium/selenium.spec.js b/integration-tests/selenium/selenium.spec.js index 50fc9d19568..74738967c9a 100644 --- a/integration-tests/selenium/selenium.spec.js +++ b/integration-tests/selenium/selenium.spec.js @@ -16,9 +16,6 @@ const { TEST_IS_RUM_ACTIVE, TEST_TYPE } = require('../../packages/dd-trace/src/plugins/util/test') -const { NODE_MAJOR } = require('../../version') - -const cucumberVersion = NODE_MAJOR <= 16 ? '9' : 'latest' const webAppServer = require('../ci-visibility/web-app-server') @@ -36,7 +33,7 @@ versionRange.forEach(version => { sandbox = await createSandbox([ 'mocha', 'jest', - `@cucumber/cucumber@${cucumberVersion}`, + '@cucumber/cucumber', 'chai@v4', `selenium-webdriver@${version}` ]) diff --git a/integration-tests/vitest/vitest.spec.js b/integration-tests/vitest/vitest.spec.js index eb2fe21ba78..eb53b395202 100644 --- a/integration-tests/vitest/vitest.spec.js +++ b/integration-tests/vitest/vitest.spec.js @@ -29,7 +29,8 @@ const { DI_DEBUG_ERROR_PREFIX, DI_DEBUG_ERROR_FILE_SUFFIX, DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX, - DI_DEBUG_ERROR_LINE_SUFFIX + DI_DEBUG_ERROR_LINE_SUFFIX, + TEST_RETRY_REASON } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') @@ -421,15 +422,13 @@ versions.forEach((version) => { context('early flake detection', () => { it('retries new tests', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) receiver.setKnownTests({ @@ -469,10 +468,15 @@ versions.forEach((version) => { 'early flake detection does not retry if the test is skipped' ]) const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') - assert.equal(newTests.length, 12) // 4 executions of the three new tests + // 4 executions of the 3 new tests + 1 new skipped test (not retried) + assert.equal(newTests.length, 13) const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') - assert.equal(retriedTests.length, 9) // 3 retries of the three new tests + assert.equal(retriedTests.length, 9) // 3 retries of the 3 new tests + + retriedTests.forEach(test => { + assert.equal(test.meta[TEST_RETRY_REASON], 'efd') + }) // exit code should be 0 and test session should be reported as passed, // even though there are some failing executions @@ -507,15 +511,13 @@ versions.forEach((version) => { it('fails if all the attempts fail', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) receiver.setKnownTests({ @@ -550,10 +552,11 @@ versions.forEach((version) => { 'early flake detection does not retry if the test is skipped' ]) const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') - assert.equal(newTests.length, 8) // 4 executions of the two new tests + // 4 executions of the 2 new tests + 1 new skipped test (not retried) + assert.equal(newTests.length, 9) const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') - assert.equal(retriedTests.length, 6) // 3 retries of the two new tests + assert.equal(retriedTests.length, 6) // 3 retries of the 2 new tests // the multiple attempts did not result in a single pass, // so the test session should be reported as failed @@ -588,16 +591,14 @@ versions.forEach((version) => { it('bails out of EFD if the percentage of new tests is too high', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD }, faulty_session_threshold: 0 - } + }, + known_tests_enabled: true }) receiver.setKnownTests({ @@ -628,9 +629,7 @@ versions.forEach((version) => { env: { ...getCiVisAgentlessConfig(receiver.port), TEST_DIR: 'ci-visibility/vitest-tests/early-flake-detection*', - NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init', - DD_TRACE_DEBUG: '1', - DD_TRACE_LOG_LEVEL: 'error' + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init' }, stdio: 'pipe' } @@ -646,15 +645,13 @@ versions.forEach((version) => { it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) receiver.setKnownTests({ @@ -662,7 +659,7 @@ versions.forEach((version) => { 'ci-visibility/vitest-tests/early-flake-detection.mjs': [ // 'early flake detection can retry tests that eventually pass', // will be considered new // 'early flake detection can retry tests that always pass', // will be considered new - // 'early flake detection does not retry if the test is skipped', // skipped so not retried + // 'early flake detection does not retry if the test is skipped', // will be considered new 'early flake detection does not retry if it is not new' ] } @@ -682,8 +679,10 @@ versions.forEach((version) => { 'early flake detection does not retry if it is not new', 'early flake detection does not retry if the test is skipped' ]) + + // new tests are detected but not retried const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') - assert.equal(newTests.length, 0) + assert.equal(newTests.length, 3) const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') assert.equal(retriedTests.length, 0) @@ -718,15 +717,13 @@ versions.forEach((version) => { it('does not run EFD if the known tests request fails', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) receiver.setKnownTestsResponseCode(500) @@ -781,15 +778,13 @@ versions.forEach((version) => { it('works when the cwd is not the repository root', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: true, slow_test_retries: { '5s': NUM_RETRIES_EFD } - } + }, + known_tests_enabled: true }) receiver.setKnownTests({ @@ -837,11 +832,21 @@ versions.forEach((version) => { it('works with repeats config when EFD is disabled', (done) => { receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, early_flake_detection: { enabled: false + }, + known_tests_enabled: true + }) + + receiver.setKnownTests({ + vitest: { + 'ci-visibility/vitest-tests/early-flake-detection.mjs': [ + // 'early flake detection can retry tests that eventually pass', // will be considered new + // 'early flake detection can retry tests that always pass', // will be considered new + // 'early flake detection can retry tests that eventually fail', // will be considered new + // 'early flake detection does not retry if the test is skipped', // will be considered new + 'early flake detection does not retry if it is not new' + ] } }) @@ -864,13 +869,14 @@ versions.forEach((version) => { 'early flake detection does not retry if the test is skipped' ]) const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') - assert.equal(newTests.length, 0) // no new test detected + // all but one are considered new + assert.equal(newTests.length, 7) const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') assert.equal(retriedTests.length, 4) // 2 repetitions on 2 tests // vitest reports the test as failed if any of the repetitions fail, so we'll follow that - // TODO: we might want to improve htis + // TODO: we might want to improve this const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') assert.equal(failedTests.length, 3) @@ -900,6 +906,77 @@ versions.forEach((version) => { }).catch(done) }) }) + + it('disables early flake detection if known tests should not be requested', (done) => { + receiver.setSettings({ + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + }, + known_tests_enabled: false + }) + + receiver.setKnownTests({ + vitest: { + 'ci-visibility/vitest-tests/early-flake-detection.mjs': [ + // 'early flake detection can retry tests that eventually pass', // will be considered new + // 'early flake detection can retry tests that always pass', // will be considered new + // 'early flake detection does not retry if the test is skipped', // will be considered new + 'early flake detection does not retry if it is not new' + ] + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(test => test.content) + + assert.equal(tests.length, 4) + + assert.includeMembers(tests.map(test => test.meta[TEST_NAME]), [ + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that always pass', + 'early flake detection does not retry if it is not new', + 'early flake detection does not retry if the test is skipped' + ]) + + // new tests are not detected and not retried + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 0) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + + const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(failedTests.length, 1) + const testSessionEvent = events.find(event => event.type === 'test_session_end').content + assert.equal(testSessionEvent.meta[TEST_STATUS], 'fail') + }) + + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TEST_DIR: 'ci-visibility/vitest-tests/early-flake-detection*', + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', (exitCode) => { + eventsPromise.then(() => { + assert.equal(exitCode, 1) + done() + }).catch(done) + }) + }) }) // dynamic instrumentation only supported from >=2.0.0 @@ -1150,5 +1227,76 @@ versions.forEach((version) => { }) }) } + + context('known tests without early flake detection', () => { + it('detects new tests without retrying them', (done) => { + receiver.setSettings({ + early_flake_detection: { + enabled: false + }, + known_tests_enabled: true + }) + + receiver.setKnownTests({ + vitest: { + 'ci-visibility/vitest-tests/early-flake-detection.mjs': [ + // 'early flake detection can retry tests that eventually pass', // will be considered new + // 'early flake detection can retry tests that always pass', // will be considered new + // 'early flake detection does not retry if the test is skipped', // will be considered new + 'early flake detection does not retry if it is not new' + ] + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(test => test.content) + + assert.equal(tests.length, 4) + + assert.includeMembers(tests.map(test => test.meta[TEST_NAME]), [ + 'early flake detection can retry tests that eventually pass', + 'early flake detection can retry tests that always pass', + 'early flake detection does not retry if it is not new', + 'early flake detection does not retry if the test is skipped' + ]) + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + // all but one are considered new + assert.equal(newTests.length, 3) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + + const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(failedTests.length, 1) + + const testSessionEvent = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSessionEvent.meta, TEST_STATUS, 'fail') + assert.notProperty(testSessionEvent.meta, TEST_EARLY_FLAKE_ENABLED) + }) + + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TEST_DIR: 'ci-visibility/vitest-tests/early-flake-detection*', + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', (exitCode) => { + eventsPromise.then(() => { + assert.equal(exitCode, 1) + done() + }).catch(done) + }) + }) + }) }) }) diff --git a/package.json b/package.json index 7ac70ae0cb3..1d658ab59bb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dd-trace", - "version": "5.32.0", + "version": "5.33.0", "description": "Datadog APM tracing client for JavaScript", "main": "index.js", "typings": "index.d.ts", @@ -86,7 +86,7 @@ "@datadog/native-iast-rewriter": "2.6.1", "@datadog/native-iast-taint-tracking": "3.2.0", "@datadog/native-metrics": "^3.1.0", - "@datadog/pprof": "5.4.1", + "@datadog/pprof": "5.5.0", "@datadog/sketches-js": "^2.1.0", "@isaacs/ttlcache": "^1.4.1", "@opentelemetry/api": ">=1.0.0 <1.9.0", diff --git a/packages/datadog-instrumentations/src/aws-sdk.js b/packages/datadog-instrumentations/src/aws-sdk.js index a82092927d7..f645eb18f7c 100644 --- a/packages/datadog-instrumentations/src/aws-sdk.js +++ b/packages/datadog-instrumentations/src/aws-sdk.js @@ -155,6 +155,8 @@ function getMessage (request, error, result) { } function getChannelSuffix (name) { + // some resource identifiers have spaces between ex: bedrock runtime + name = name.replaceAll(' ', '') return [ 'cloudwatchlogs', 'dynamodb', @@ -168,7 +170,7 @@ function getChannelSuffix (name) { 'sqs', 'states', 'stepfunctions', - 'bedrock runtime' + 'bedrockruntime' ].includes(name) ? name : 'default' diff --git a/packages/datadog-instrumentations/src/cucumber.js b/packages/datadog-instrumentations/src/cucumber.js index a3a5ae105fd..639f955cc56 100644 --- a/packages/datadog-instrumentations/src/cucumber.js +++ b/packages/datadog-instrumentations/src/cucumber.js @@ -70,6 +70,7 @@ let earlyFlakeDetectionNumRetries = 0 let earlyFlakeDetectionFaultyThreshold = 0 let isEarlyFlakeDetectionFaulty = false let isFlakyTestRetriesEnabled = false +let isKnownTestsEnabled = false let numTestRetries = 0 let knownTests = [] let skippedSuites = [] @@ -292,7 +293,7 @@ function wrapRun (pl, isLatestVersion) { } let isNew = false let isEfdRetry = false - if (isEarlyFlakeDetectionEnabled && status !== 'skip') { + if (isKnownTestsEnabled && status !== 'skip') { const numRetries = numRetriesByPickleId.get(this.pickle.id) isNew = numRetries !== undefined @@ -394,13 +395,15 @@ function getWrappedStart (start, frameworkVersion, isParallel = false, isCoordin isSuitesSkippingEnabled = configurationResponse.libraryConfig?.isSuitesSkippingEnabled isFlakyTestRetriesEnabled = configurationResponse.libraryConfig?.isFlakyTestRetriesEnabled numTestRetries = configurationResponse.libraryConfig?.flakyTestRetriesCount + isKnownTestsEnabled = configurationResponse.libraryConfig?.isKnownTestsEnabled - if (isEarlyFlakeDetectionEnabled) { + if (isKnownTestsEnabled) { const knownTestsResponse = await getChannelPromise(knownTestsCh) if (!knownTestsResponse.err) { knownTests = knownTestsResponse.knownTests } else { isEarlyFlakeDetectionEnabled = false + isKnownTestsEnabled = false } } @@ -437,7 +440,7 @@ function getWrappedStart (start, frameworkVersion, isParallel = false, isCoordin pickleByFile = isCoordinator ? getPickleByFileNew(this) : getPickleByFile(this) - if (isEarlyFlakeDetectionEnabled) { + if (isKnownTestsEnabled) { const isFaulty = getIsFaultyEarlyFlakeDetection( Object.keys(pickleByFile), knownTests.cucumber || {}, @@ -445,6 +448,7 @@ function getWrappedStart (start, frameworkVersion, isParallel = false, isCoordin ) if (isFaulty) { isEarlyFlakeDetectionEnabled = false + isKnownTestsEnabled = false isEarlyFlakeDetectionFaulty = true } } @@ -533,7 +537,7 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa let isNew = false - if (isEarlyFlakeDetectionEnabled) { + if (isKnownTestsEnabled) { isNew = isNewTest(testSuitePath, pickle.name) if (isNew) { numRetriesByPickleId.set(pickle.id, 0) @@ -678,14 +682,14 @@ function getWrappedParseWorkerMessage (parseWorkerMessageFunction, isNewVersion) const { status } = getStatusFromResultLatest(worstTestStepResult) let isNew = false - if (isEarlyFlakeDetectionEnabled) { + if (isKnownTestsEnabled) { isNew = isNewTest(pickle.uri, pickle.name) } const testFileAbsolutePath = pickle.uri const finished = pickleResultByFile[testFileAbsolutePath] - if (isNew) { + if (isEarlyFlakeDetectionEnabled && isNew) { const testFullname = `${pickle.uri}:${pickle.name}` let testStatuses = newTestsByTestFullname.get(testFullname) if (!testStatuses) { @@ -839,7 +843,8 @@ addHook({ ) // EFD in parallel mode only supported in >=11.0.0 shimmer.wrap(adapterPackage.ChildProcessAdapter.prototype, 'startWorker', startWorker => function () { - if (isEarlyFlakeDetectionEnabled) { + if (isKnownTestsEnabled) { + this.options.worldParameters._ddIsEarlyFlakeDetectionEnabled = isEarlyFlakeDetectionEnabled this.options.worldParameters._ddKnownTests = knownTests this.options.worldParameters._ddEarlyFlakeDetectionNumRetries = earlyFlakeDetectionNumRetries } @@ -862,9 +867,12 @@ addHook({ 'initialize', initialize => async function () { await initialize.apply(this, arguments) - isEarlyFlakeDetectionEnabled = !!this.options.worldParameters._ddKnownTests - if (isEarlyFlakeDetectionEnabled) { + isKnownTestsEnabled = !!this.options.worldParameters._ddKnownTests + if (isKnownTestsEnabled) { knownTests = this.options.worldParameters._ddKnownTests + } + isEarlyFlakeDetectionEnabled = !!this.options.worldParameters._ddIsEarlyFlakeDetectionEnabled + if (isEarlyFlakeDetectionEnabled) { earlyFlakeDetectionNumRetries = this.options.worldParameters._ddEarlyFlakeDetectionNumRetries } } diff --git a/packages/datadog-instrumentations/src/jest.js b/packages/datadog-instrumentations/src/jest.js index 7a1001d11f3..bc01fecc150 100644 --- a/packages/datadog-instrumentations/src/jest.js +++ b/packages/datadog-instrumentations/src/jest.js @@ -69,6 +69,7 @@ let earlyFlakeDetectionNumRetries = 0 let earlyFlakeDetectionFaultyThreshold = 30 let isEarlyFlakeDetectionFaulty = false let hasFilteredSkippableSuites = false +let isKnownTestsEnabled = false const sessionAsyncResource = new AsyncResource('bound-anonymous-fn') @@ -138,17 +139,19 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { this.isFlakyTestRetriesEnabled = this.testEnvironmentOptions._ddIsFlakyTestRetriesEnabled this.flakyTestRetriesCount = this.testEnvironmentOptions._ddFlakyTestRetriesCount this.isDiEnabled = this.testEnvironmentOptions._ddIsDiEnabled + this.isKnownTestsEnabled = this.testEnvironmentOptions._ddIsKnownTestsEnabled - if (this.isEarlyFlakeDetectionEnabled) { - const hasKnownTests = !!knownTests.jest - earlyFlakeDetectionNumRetries = this.testEnvironmentOptions._ddEarlyFlakeDetectionNumRetries + if (this.isKnownTestsEnabled) { try { + const hasKnownTests = !!knownTests.jest + earlyFlakeDetectionNumRetries = this.testEnvironmentOptions._ddEarlyFlakeDetectionNumRetries this.knownTestsForThisSuite = hasKnownTests ? (knownTests.jest[this.testSuite] || []) : this.getKnownTestsForSuite(this.testEnvironmentOptions._ddKnownTests) } catch (e) { // If there has been an error parsing the tests, we'll disable Early Flake Deteciton this.isEarlyFlakeDetectionEnabled = false + this.isKnownTestsEnabled = false } } @@ -228,7 +231,7 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { asyncResources.set(event.test, asyncResource) const testName = getJestTestName(event.test) - if (this.isEarlyFlakeDetectionEnabled) { + if (this.isKnownTestsEnabled) { const originalTestName = removeEfdStringFromTestName(testName) isNewTest = retriedTestsToNumAttempts.has(originalTestName) if (isNewTest) { @@ -254,24 +257,26 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { }) } if (event.name === 'add_test') { - if (this.isEarlyFlakeDetectionEnabled) { + if (this.isKnownTestsEnabled) { const testName = this.getTestNameFromAddTestEvent(event, state) const isNew = !this.knownTestsForThisSuite?.includes(testName) const isSkipped = event.mode === 'todo' || event.mode === 'skip' if (isNew && !isSkipped && !retriedTestsToNumAttempts.has(testName)) { retriedTestsToNumAttempts.set(testName, 0) - // Retrying snapshots has proven to be problematic, so we'll skip them for now - // We'll still detect new tests, but we won't retry them. - // TODO: do not bail out of EFD with the whole test suite - if (this.getHasSnapshotTests()) { - log.warn('Early flake detection is disabled for suites with snapshots') - return - } - for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) { - if (this.global.test) { - this.global.test(addEfdStringToTestName(event.testName, retryIndex), event.fn, event.timeout) - } else { - log.error('Early flake detection could not retry test because global.test is undefined') + if (this.isEarlyFlakeDetectionEnabled) { + // Retrying snapshots has proven to be problematic, so we'll skip them for now + // We'll still detect new tests, but we won't retry them. + // TODO: do not bail out of EFD with the whole test suite + if (this.getHasSnapshotTests()) { + log.warn('Early flake detection is disabled for suites with snapshots') + return + } + for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) { + if (this.global.test) { + this.global.test(addEfdStringToTestName(event.testName, retryIndex), event.fn, event.timeout) + } else { + log.error('Early flake detection could not retry test because global.test is undefined') + } } } } @@ -286,7 +291,7 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { event.test.fn = originalTestFns.get(event.test) // We'll store the test statuses of the retries - if (this.isEarlyFlakeDetectionEnabled) { + if (this.isKnownTestsEnabled) { const testName = getJestTestName(event.test) const originalTestName = removeEfdStringFromTestName(testName) const isNewTest = retriedTestsToNumAttempts.has(originalTestName) @@ -483,12 +488,13 @@ function cliWrapper (cli, jestVersion) { isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries earlyFlakeDetectionFaultyThreshold = libraryConfig.earlyFlakeDetectionFaultyThreshold + isKnownTestsEnabled = libraryConfig.isKnownTestsEnabled } } catch (err) { log.error('Jest library configuration error', err) } - if (isEarlyFlakeDetectionEnabled) { + if (isKnownTestsEnabled) { const knownTestsPromise = new Promise((resolve) => { onDone = resolve }) @@ -504,6 +510,7 @@ function cliWrapper (cli, jestVersion) { } else { // We disable EFD if there has been an error in the known tests request isEarlyFlakeDetectionEnabled = false + isKnownTestsEnabled = false } } catch (err) { log.error('Jest known tests error', err) @@ -821,6 +828,7 @@ addHook({ _ddIsFlakyTestRetriesEnabled, _ddFlakyTestRetriesCount, _ddIsDiEnabled, + _ddIsKnownTestsEnabled, ...restOfTestEnvironmentOptions } = testEnvironmentOptions @@ -848,17 +856,19 @@ addHook({ const testPaths = await getTestPaths.apply(this, arguments) const [{ rootDir, shard }] = arguments - if (isEarlyFlakeDetectionEnabled) { + if (isKnownTestsEnabled) { const projectSuites = testPaths.tests.map(test => getTestSuitePath(test.path, test.context.config.rootDir)) const isFaulty = getIsFaultyEarlyFlakeDetection(projectSuites, knownTests.jest || {}, earlyFlakeDetectionFaultyThreshold) if (isFaulty) { log.error('Early flake detection is disabled because the number of new suites is too high.') isEarlyFlakeDetectionEnabled = false + isKnownTestsEnabled = false const testEnvironmentOptions = testPaths.tests[0]?.context?.config?.testEnvironmentOptions // Project config is shared among all tests, so we can modify it here if (testEnvironmentOptions) { testEnvironmentOptions._ddIsEarlyFlakeDetectionEnabled = false + testEnvironmentOptions._ddIsKnownTestsEnabled = false } isEarlyFlakeDetectionFaulty = true } @@ -929,6 +939,11 @@ addHook({ return runtimePackage }) +/* +* This hook does two things: +* - Pass known tests to the workers. +* - Receive trace, coverage and logs payloads from the workers. +*/ addHook({ name: 'jest-worker', versions: ['>=24.9.0'], @@ -936,7 +951,7 @@ addHook({ }, (childProcessWorker) => { const ChildProcessWorker = childProcessWorker.default shimmer.wrap(ChildProcessWorker.prototype, 'send', send => function (request) { - if (!isEarlyFlakeDetectionEnabled) { + if (!isKnownTestsEnabled) { return send.apply(this, arguments) } const [type] = request diff --git a/packages/datadog-instrumentations/src/mocha/main.js b/packages/datadog-instrumentations/src/mocha/main.js index 2e796a71371..afa7bfe0fc4 100644 --- a/packages/datadog-instrumentations/src/mocha/main.js +++ b/packages/datadog-instrumentations/src/mocha/main.js @@ -201,6 +201,7 @@ function getExecutionConfiguration (runner, isParallel, onFinishRequest) { if (err) { config.knownTests = [] config.isEarlyFlakeDetectionEnabled = false + config.isKnownTestsEnabled = false } else { config.knownTests = knownTests } @@ -222,12 +223,13 @@ function getExecutionConfiguration (runner, isParallel, onFinishRequest) { config.isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled config.earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries config.earlyFlakeDetectionFaultyThreshold = libraryConfig.earlyFlakeDetectionFaultyThreshold + config.isKnownTestsEnabled = libraryConfig.isKnownTestsEnabled // ITR and auto test retries are not supported in parallel mode yet config.isSuitesSkippingEnabled = !isParallel && libraryConfig.isSuitesSkippingEnabled config.isFlakyTestRetriesEnabled = !isParallel && libraryConfig.isFlakyTestRetriesEnabled config.flakyTestRetriesCount = !isParallel && libraryConfig.flakyTestRetriesCount - if (config.isEarlyFlakeDetectionEnabled) { + if (config.isKnownTestsEnabled) { knownTestsCh.publish({ onDone: mochaRunAsyncResource.bind(onReceivedKnownTests) }) @@ -273,7 +275,7 @@ addHook({ }) getExecutionConfiguration(runner, false, () => { - if (config.isEarlyFlakeDetectionEnabled) { + if (config.isKnownTestsEnabled) { const testSuites = this.files.map(file => getTestSuitePath(file, process.cwd())) const isFaulty = getIsFaultyEarlyFlakeDetection( testSuites, @@ -283,6 +285,7 @@ addHook({ if (isFaulty) { config.isEarlyFlakeDetectionEnabled = false config.isEarlyFlakeDetectionFaulty = true + config.isKnownTestsEnabled = false } } if (getCodeCoverageCh.hasSubscribers) { @@ -537,7 +540,7 @@ addHook({ this.once('end', getOnEndHandler(true)) getExecutionConfiguration(this, true, () => { - if (config.isEarlyFlakeDetectionEnabled) { + if (config.isKnownTestsEnabled) { const testSuites = files.map(file => getTestSuitePath(file, process.cwd())) const isFaulty = getIsFaultyEarlyFlakeDetection( testSuites, @@ -545,6 +548,7 @@ addHook({ config.earlyFlakeDetectionFaultyThreshold ) if (isFaulty) { + config.isKnownTestsEnabled = false config.isEarlyFlakeDetectionEnabled = false config.isEarlyFlakeDetectionFaulty = true } @@ -569,7 +573,7 @@ addHook({ const { BufferedWorkerPool } = BufferedWorkerPoolPackage shimmer.wrap(BufferedWorkerPool.prototype, 'run', run => async function (testSuiteAbsolutePath, workerArgs) { - if (!testStartCh.hasSubscribers || !config.isEarlyFlakeDetectionEnabled) { + if (!testStartCh.hasSubscribers || !config.isKnownTestsEnabled) { return run.apply(this, arguments) } @@ -584,6 +588,7 @@ addHook({ { ...workerArgs, _ddEfdNumRetries: config.earlyFlakeDetectionNumRetries, + _ddIsEfdEnabled: config.isEarlyFlakeDetectionEnabled, _ddKnownTests: { mocha: { [testPath]: testSuiteKnownTests diff --git a/packages/datadog-instrumentations/src/mocha/utils.js b/packages/datadog-instrumentations/src/mocha/utils.js index 97b5f2d1209..30710ab645b 100644 --- a/packages/datadog-instrumentations/src/mocha/utils.js +++ b/packages/datadog-instrumentations/src/mocha/utils.js @@ -349,12 +349,14 @@ function getOnPendingHandler () { // Hook to add retries to tests if EFD is enabled function getRunTestsWrapper (runTests, config) { return function (suite, fn) { - if (config.isEarlyFlakeDetectionEnabled) { + if (config.isKnownTestsEnabled) { // by the time we reach `this.on('test')`, it is too late. We need to add retries here suite.tests.forEach(test => { if (!test.isPending() && isNewTest(test, config.knownTests)) { test._ddIsNew = true - retryTest(test, config.earlyFlakeDetectionNumRetries) + if (config.isEarlyFlakeDetectionEnabled) { + retryTest(test, config.earlyFlakeDetectionNumRetries) + } } }) } diff --git a/packages/datadog-instrumentations/src/mocha/worker.js b/packages/datadog-instrumentations/src/mocha/worker.js index 63670ba5db2..56a9dc75270 100644 --- a/packages/datadog-instrumentations/src/mocha/worker.js +++ b/packages/datadog-instrumentations/src/mocha/worker.js @@ -25,10 +25,12 @@ addHook({ }, (Mocha) => { shimmer.wrap(Mocha.prototype, 'run', run => function () { if (this.options._ddKnownTests) { - // EFD is enabled if there's a list of known tests - config.isEarlyFlakeDetectionEnabled = true + // If there are known tests, it means isKnownTestsEnabled should be true + config.isKnownTestsEnabled = true + config.isEarlyFlakeDetectionEnabled = this.options._ddIsEfdEnabled config.knownTests = this.options._ddKnownTests config.earlyFlakeDetectionNumRetries = this.options._ddEfdNumRetries + delete this.options._ddIsEfdEnabled delete this.options._ddKnownTests delete this.options._ddEfdNumRetries } diff --git a/packages/datadog-instrumentations/src/playwright.js b/packages/datadog-instrumentations/src/playwright.js index 4eab55b1797..9cc7d64cd1c 100644 --- a/packages/datadog-instrumentations/src/playwright.js +++ b/packages/datadog-instrumentations/src/playwright.js @@ -35,6 +35,7 @@ const STATUS_TO_TEST_STATUS = { } let remainingTestsByFile = {} +let isKnownTestsEnabled = false let isEarlyFlakeDetectionEnabled = false let earlyFlakeDetectionNumRetries = 0 let isFlakyTestRetriesEnabled = false @@ -418,6 +419,7 @@ function runnerHook (runnerExport, playwrightVersion) { try { const { err, libraryConfig } = await getChannelPromise(libraryConfigurationCh) if (!err) { + isKnownTestsEnabled = libraryConfig.isKnownTestsEnabled isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries isFlakyTestRetriesEnabled = libraryConfig.isFlakyTestRetriesEnabled @@ -425,19 +427,22 @@ function runnerHook (runnerExport, playwrightVersion) { } } catch (e) { isEarlyFlakeDetectionEnabled = false + isKnownTestsEnabled = false log.error('Playwright session start error', e) } - if (isEarlyFlakeDetectionEnabled && semver.gte(playwrightVersion, MINIMUM_SUPPORTED_VERSION_EFD)) { + if (isKnownTestsEnabled && semver.gte(playwrightVersion, MINIMUM_SUPPORTED_VERSION_EFD)) { try { const { err, knownTests: receivedKnownTests } = await getChannelPromise(knownTestsCh) if (!err) { knownTests = receivedKnownTests } else { isEarlyFlakeDetectionEnabled = false + isKnownTestsEnabled = false } } catch (err) { isEarlyFlakeDetectionEnabled = false + isKnownTestsEnabled = false log.error('Playwright known tests error', err) } } @@ -553,7 +558,7 @@ addHook({ async function newCreateRootSuite () { const rootSuite = await oldCreateRootSuite.apply(this, arguments) - if (!isEarlyFlakeDetectionEnabled) { + if (!isKnownTestsEnabled) { return rootSuite } const newTests = rootSuite @@ -562,7 +567,7 @@ addHook({ newTests.forEach(newTest => { newTest._ddIsNew = true - if (newTest.expectedStatus !== 'skipped') { + if (isEarlyFlakeDetectionEnabled && newTest.expectedStatus !== 'skipped') { const fileSuite = getSuiteType(newTest, 'file') const projectSuite = getSuiteType(newTest, 'project') for (let repeatEachIndex = 0; repeatEachIndex < earlyFlakeDetectionNumRetries; repeatEachIndex++) { diff --git a/packages/datadog-instrumentations/src/vitest.js b/packages/datadog-instrumentations/src/vitest.js index b3f2a9af8b8..ebde98b4789 100644 --- a/packages/datadog-instrumentations/src/vitest.js +++ b/packages/datadog-instrumentations/src/vitest.js @@ -25,6 +25,7 @@ const isEarlyFlakeDetectionFaultyCh = channel('ci:vitest:is-early-flake-detectio const taskToAsync = new WeakMap() const taskToStatuses = new WeakMap() const newTasks = new WeakSet() +let isRetryReasonEfd = false const switchedStatuses = new WeakSet() const sessionAsyncResource = new AsyncResource('bound-anonymous-fn') @@ -44,14 +45,16 @@ function getProvidedContext () { _ddIsEarlyFlakeDetectionEnabled, _ddIsDiEnabled, _ddKnownTests: knownTests, - _ddEarlyFlakeDetectionNumRetries: numRepeats + _ddEarlyFlakeDetectionNumRetries: numRepeats, + _ddIsKnownTestsEnabled: isKnownTestsEnabled } = globalThis.__vitest_worker__.providedContext return { isDiEnabled: _ddIsDiEnabled, isEarlyFlakeDetectionEnabled: _ddIsEarlyFlakeDetectionEnabled, knownTests, - numRepeats + numRepeats, + isKnownTestsEnabled } } catch (e) { log.error('Vitest workers could not parse provided context, so some features will not work.') @@ -59,7 +62,8 @@ function getProvidedContext () { isDiEnabled: false, isEarlyFlakeDetectionEnabled: false, knownTests: {}, - numRepeats: 0 + numRepeats: 0, + isKnownTestsEnabled: false } } } @@ -153,6 +157,7 @@ function getSortWrapper (sort) { let isEarlyFlakeDetectionEnabled = false let earlyFlakeDetectionNumRetries = 0 let isEarlyFlakeDetectionFaulty = false + let isKnownTestsEnabled = false let isDiEnabled = false let knownTests = {} @@ -164,18 +169,20 @@ function getSortWrapper (sort) { isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries isDiEnabled = libraryConfig.isDiEnabled + isKnownTestsEnabled = libraryConfig.isKnownTestsEnabled } } catch (e) { isFlakyTestRetriesEnabled = false isEarlyFlakeDetectionEnabled = false isDiEnabled = false + isKnownTestsEnabled = false } if (isFlakyTestRetriesEnabled && !this.ctx.config.retry && flakyTestRetriesCount > 0) { this.ctx.config.retry = flakyTestRetriesCount } - if (isEarlyFlakeDetectionEnabled) { + if (isKnownTestsEnabled) { const knownTestsResponse = await getChannelPromise(knownTestsCh) if (!knownTestsResponse.err) { knownTests = knownTestsResponse.knownTests @@ -192,13 +199,15 @@ function getSortWrapper (sort) { }) if (isEarlyFlakeDetectionFaulty) { isEarlyFlakeDetectionEnabled = false - log.warn('Early flake detection is disabled because the number of new tests is too high.') + isKnownTestsEnabled = false + log.warn('New test detection is disabled because the number of new tests is too high.') } else { // TODO: use this to pass session and module IDs to the worker, instead of polluting process.env // Note: setting this.ctx.config.provide directly does not work because it's cached try { const workspaceProject = this.ctx.getCoreWorkspaceProject() - workspaceProject._provided._ddKnownTests = knownTests.vitest + workspaceProject._provided._ddIsKnownTestsEnabled = isKnownTestsEnabled + workspaceProject._provided._ddKnownTests = knownTests.vitest || {} workspaceProject._provided._ddIsEarlyFlakeDetectionEnabled = isEarlyFlakeDetectionEnabled workspaceProject._provided._ddEarlyFlakeDetectionNumRetries = earlyFlakeDetectionNumRetries } catch (e) { @@ -207,6 +216,7 @@ function getSortWrapper (sort) { } } else { isEarlyFlakeDetectionEnabled = false + isKnownTestsEnabled = false } } @@ -295,17 +305,21 @@ addHook({ const { knownTests, isEarlyFlakeDetectionEnabled, + isKnownTestsEnabled, numRepeats } = getProvidedContext() - if (isEarlyFlakeDetectionEnabled) { + if (isKnownTestsEnabled) { isNewTestCh.publish({ knownTests, testSuiteAbsolutePath: task.file.filepath, testName, onDone: (isNew) => { if (isNew) { - task.repeats = numRepeats + if (isEarlyFlakeDetectionEnabled) { + isRetryReasonEfd = task.repeats !== numRepeats + task.repeats = numRepeats + } newTasks.add(task) taskToStatuses.set(task, []) } @@ -344,11 +358,12 @@ addHook({ let isNew = false const { + isKnownTestsEnabled, isEarlyFlakeDetectionEnabled, isDiEnabled } = getProvidedContext() - if (isEarlyFlakeDetectionEnabled) { + if (isKnownTestsEnabled) { isNew = newTasks.has(task) } @@ -431,6 +446,7 @@ addHook({ testName, testSuiteAbsolutePath: task.file.filepath, isRetry: numAttempt > 0 || numRepetition > 0, + isRetryReasonEfd, isNew, mightHitProbe: isDiEnabled && numAttempt > 0 }) @@ -576,7 +592,11 @@ addHook({ if (result) { const { state, duration, errors } = result if (state === 'skip') { // programmatic skip - testSkipCh.publish({ testName: getTestName(task), testSuiteAbsolutePath: task.file.filepath }) + testSkipCh.publish({ + testName: getTestName(task), + testSuiteAbsolutePath: task.file.filepath, + isNew: newTasks.has(task) + }) } else if (state === 'pass' && !isSwitchedStatus) { if (testAsyncResource) { testAsyncResource.runInAsyncScope(() => { @@ -602,7 +622,11 @@ addHook({ } } } else { // test.skip or test.todo - testSkipCh.publish({ testName: getTestName(task), testSuiteAbsolutePath: task.file.filepath }) + testSkipCh.publish({ + testName: getTestName(task), + testSuiteAbsolutePath: task.file.filepath, + isNew: newTasks.has(task) + }) } }) diff --git a/packages/datadog-plugin-aws-sdk/src/services/bedrockruntime/index.js b/packages/datadog-plugin-aws-sdk/src/services/bedrockruntime/index.js new file mode 100644 index 00000000000..c123c02fa65 --- /dev/null +++ b/packages/datadog-plugin-aws-sdk/src/services/bedrockruntime/index.js @@ -0,0 +1,16 @@ +const CompositePlugin = require('../../../../dd-trace/src/plugins/composite') +const BedrockRuntimeTracing = require('./tracing') +const BedrockRuntimeLLMObsPlugin = require('../../../../dd-trace/src/llmobs/plugins/bedrockruntime') +class BedrockRuntimePlugin extends CompositePlugin { + static get id () { + return 'bedrockruntime' + } + + static get plugins () { + return { + llmobs: BedrockRuntimeLLMObsPlugin, + tracing: BedrockRuntimeTracing + } + } +} +module.exports = BedrockRuntimePlugin diff --git a/packages/datadog-plugin-aws-sdk/src/services/bedrockruntime/tracing.js b/packages/datadog-plugin-aws-sdk/src/services/bedrockruntime/tracing.js new file mode 100644 index 00000000000..9d7d0fb1ac7 --- /dev/null +++ b/packages/datadog-plugin-aws-sdk/src/services/bedrockruntime/tracing.js @@ -0,0 +1,63 @@ +'use strict' + +const BaseAwsSdkPlugin = require('../../base') +const { parseModelId, extractRequestParams, extractTextAndResponseReason } = require('./utils') + +const enabledOperations = ['invokeModel'] + +class BedrockRuntime extends BaseAwsSdkPlugin { + static get id () { return 'bedrockruntime' } + + isEnabled (request) { + const operation = request.operation + if (!enabledOperations.includes(operation)) { + return false + } + + return super.isEnabled(request) + } + + generateTags (params, operation, response) { + const { modelProvider, modelName } = parseModelId(params.modelId) + + const requestParams = extractRequestParams(params, modelProvider) + const textAndResponseReason = extractTextAndResponseReason(response, modelProvider, modelName) + + const tags = buildTagsFromParams(requestParams, textAndResponseReason, modelProvider, modelName, operation) + + return tags + } +} + +function buildTagsFromParams (requestParams, textAndResponseReason, modelProvider, modelName, operation) { + const tags = {} + + // add request tags + tags['resource.name'] = operation + tags['aws.bedrock.request.model'] = modelName + tags['aws.bedrock.request.model_provider'] = modelProvider.toLowerCase() + tags['aws.bedrock.request.prompt'] = requestParams.prompt + tags['aws.bedrock.request.temperature'] = requestParams.temperature + tags['aws.bedrock.request.top_p'] = requestParams.topP + tags['aws.bedrock.request.top_k'] = requestParams.topK + tags['aws.bedrock.request.max_tokens'] = requestParams.maxTokens + tags['aws.bedrock.request.stop_sequences'] = requestParams.stopSequences + tags['aws.bedrock.request.input_type'] = requestParams.inputType + tags['aws.bedrock.request.truncate'] = requestParams.truncate + tags['aws.bedrock.request.stream'] = requestParams.stream + tags['aws.bedrock.request.n'] = requestParams.n + + // add response tags + if (modelName.includes('embed')) { + tags['aws.bedrock.response.embedding_length'] = textAndResponseReason.message.length + } + if (textAndResponseReason.choiceId) { + tags['aws.bedrock.response.choices.id'] = textAndResponseReason.choiceId + } + tags['aws.bedrock.response.choices.text'] = textAndResponseReason.message + tags['aws.bedrock.response.choices.finish_reason'] = textAndResponseReason.finishReason + + return tags +} + +module.exports = BedrockRuntime diff --git a/packages/datadog-plugin-aws-sdk/src/services/bedrockruntime.js b/packages/datadog-plugin-aws-sdk/src/services/bedrockruntime/utils.js similarity index 72% rename from packages/datadog-plugin-aws-sdk/src/services/bedrockruntime.js rename to packages/datadog-plugin-aws-sdk/src/services/bedrockruntime/utils.js index ef4efe76291..8bcb6a6f592 100644 --- a/packages/datadog-plugin-aws-sdk/src/services/bedrockruntime.js +++ b/packages/datadog-plugin-aws-sdk/src/services/bedrockruntime/utils.js @@ -1,7 +1,17 @@ 'use strict' -const BaseAwsSdkPlugin = require('../base') -const log = require('../../../dd-trace/src/log') +const log = require('../../../../dd-trace/src/log') + +const MODEL_TYPE_IDENTIFIERS = [ + 'foundation-model/', + 'custom-model/', + 'provisioned-model/', + 'imported-module/', + 'prompt/', + 'endpoint/', + 'inference-profile/', + 'default-prompt-router/' +] const PROVIDER = { AI21: 'AI21', @@ -13,44 +23,6 @@ const PROVIDER = { MISTRAL: 'MISTRAL' } -const enabledOperations = ['invokeModel'] - -class BedrockRuntime extends BaseAwsSdkPlugin { - static get id () { return 'bedrock runtime' } - - isEnabled (request) { - const operation = request.operation - if (!enabledOperations.includes(operation)) { - return false - } - - return super.isEnabled(request) - } - - generateTags (params, operation, response) { - let tags = {} - let modelName = '' - let modelProvider = '' - const modelMeta = params.modelId.split('.') - if (modelMeta.length === 2) { - [modelProvider, modelName] = modelMeta - modelProvider = modelProvider.toUpperCase() - } else { - [, modelProvider, modelName] = modelMeta - modelProvider = modelProvider.toUpperCase() - } - - const shouldSetChoiceIds = modelProvider === PROVIDER.COHERE && !modelName.includes('embed') - - const requestParams = extractRequestParams(params, modelProvider) - const textAndResponseReason = extractTextAndResponseReason(response, modelProvider, modelName, shouldSetChoiceIds) - - tags = buildTagsFromParams(requestParams, textAndResponseReason, modelProvider, modelName, operation) - - return tags - } -} - class Generation { constructor ({ message = '', finishReason = '', choiceId = '' } = {}) { // stringify message as it could be a single generated message as well as a list of embeddings @@ -65,6 +37,7 @@ class RequestParams { prompt = '', temperature = undefined, topP = undefined, + topK = undefined, maxTokens = undefined, stopSequences = [], inputType = '', @@ -72,11 +45,11 @@ class RequestParams { stream = '', n = undefined } = {}) { - // TODO: set a truncation limit to prompt // stringify prompt as it could be a single prompt as well as a list of message objects this.prompt = typeof prompt === 'string' ? prompt : JSON.stringify(prompt) || '' this.temperature = temperature !== undefined ? temperature : undefined this.topP = topP !== undefined ? topP : undefined + this.topK = topK !== undefined ? topK : undefined this.maxTokens = maxTokens !== undefined ? maxTokens : undefined this.stopSequences = stopSequences || [] this.inputType = inputType || '' @@ -86,11 +59,53 @@ class RequestParams { } } +function parseModelId (modelId) { + // Best effort to extract the model provider and model name from the bedrock model ID. + // modelId can be a 1/2 period-separated string or a full AWS ARN, based on the following formats: + // 1. Base model: "{model_provider}.{model_name}" + // 2. Cross-region model: "{region}.{model_provider}.{model_name}" + // 3. Other: Prefixed by AWS ARN "arn:aws{+region?}:bedrock:{region}:{account-id}:" + // a. Foundation model: ARN prefix + "foundation-model/{region?}.{model_provider}.{model_name}" + // b. Custom model: ARN prefix + "custom-model/{model_provider}.{model_name}" + // c. Provisioned model: ARN prefix + "provisioned-model/{model-id}" + // d. Imported model: ARN prefix + "imported-module/{model-id}" + // e. Prompt management: ARN prefix + "prompt/{prompt-id}" + // f. Sagemaker: ARN prefix + "endpoint/{model-id}" + // g. Inference profile: ARN prefix + "{application-?}inference-profile/{model-id}" + // h. Default prompt router: ARN prefix + "default-prompt-router/{prompt-id}" + // If model provider cannot be inferred from the modelId formatting, then default to "custom" + modelId = modelId.toLowerCase() + if (!modelId.startsWith('arn:aws')) { + const modelMeta = modelId.split('.') + if (modelMeta.length < 2) { + return { modelProvider: 'custom', modelName: modelMeta[0] } + } + return { modelProvider: modelMeta[modelMeta.length - 2], modelName: modelMeta[modelMeta.length - 1] } + } + + for (const identifier of MODEL_TYPE_IDENTIFIERS) { + if (!modelId.includes(identifier)) { + continue + } + modelId = modelId.split(identifier).pop() + if (['foundation-model/', 'custom-model/'].includes(identifier)) { + const modelMeta = modelId.split('.') + if (modelMeta.length < 2) { + return { modelProvider: 'custom', modelName: modelId } + } + return { modelProvider: modelMeta[modelMeta.length - 2], modelName: modelMeta[modelMeta.length - 1] } + } + return { modelProvider: 'custom', modelName: modelId } + } + + return { modelProvider: 'custom', modelName: 'custom' } +} + function extractRequestParams (params, provider) { const requestBody = JSON.parse(params.body) const modelId = params.modelId - switch (provider) { + switch (provider.toUpperCase()) { case PROVIDER.AI21: { let userPrompt = requestBody.prompt if (modelId.includes('jamba')) { @@ -176,11 +191,11 @@ function extractRequestParams (params, provider) { } } -function extractTextAndResponseReason (response, provider, modelName, shouldSetChoiceIds) { +function extractTextAndResponseReason (response, provider, modelName) { const body = JSON.parse(Buffer.from(response.body).toString('utf8')) - + const shouldSetChoiceIds = provider.toUpperCase() === PROVIDER.COHERE && !modelName.includes('embed') try { - switch (provider) { + switch (provider.toUpperCase()) { case PROVIDER.AI21: { if (modelName.includes('jamba')) { const generations = body.choices || [] @@ -262,34 +277,11 @@ function extractTextAndResponseReason (response, provider, modelName, shouldSetC return new Generation() } -function buildTagsFromParams (requestParams, textAndResponseReason, modelProvider, modelName, operation) { - const tags = {} - - // add request tags - tags['resource.name'] = operation - tags['aws.bedrock.request.model'] = modelName - tags['aws.bedrock.request.model_provider'] = modelProvider - tags['aws.bedrock.request.prompt'] = requestParams.prompt - tags['aws.bedrock.request.temperature'] = requestParams.temperature - tags['aws.bedrock.request.top_p'] = requestParams.topP - tags['aws.bedrock.request.max_tokens'] = requestParams.maxTokens - tags['aws.bedrock.request.stop_sequences'] = requestParams.stopSequences - tags['aws.bedrock.request.input_type'] = requestParams.inputType - tags['aws.bedrock.request.truncate'] = requestParams.truncate - tags['aws.bedrock.request.stream'] = requestParams.stream - tags['aws.bedrock.request.n'] = requestParams.n - - // add response tags - if (modelName.includes('embed')) { - tags['aws.bedrock.response.embedding_length'] = textAndResponseReason.message.length - } - if (textAndResponseReason.choiceId) { - tags['aws.bedrock.response.choices.id'] = textAndResponseReason.choiceId - } - tags['aws.bedrock.response.choices.text'] = textAndResponseReason.message - tags['aws.bedrock.response.choices.finish_reason'] = textAndResponseReason.finishReason - - return tags +module.exports = { + Generation, + RequestParams, + parseModelId, + extractRequestParams, + extractTextAndResponseReason, + PROVIDER } - -module.exports = BedrockRuntime diff --git a/packages/datadog-plugin-aws-sdk/test/bedrock.spec.js b/packages/datadog-plugin-aws-sdk/test/bedrock.spec.js deleted file mode 100644 index 0990f25e198..00000000000 --- a/packages/datadog-plugin-aws-sdk/test/bedrock.spec.js +++ /dev/null @@ -1,238 +0,0 @@ -'use strict' - -const agent = require('../../dd-trace/test/plugins/agent') -const nock = require('nock') -const { setup } = require('./spec_helpers') - -const serviceName = 'bedrock-service-name-test' - -const PROVIDER = { - AI21: 'AI21', - AMAZON: 'AMAZON', - ANTHROPIC: 'ANTHROPIC', - COHERE: 'COHERE', - META: 'META', - MISTRAL: 'MISTRAL' -} - -describe('Plugin', () => { - describe('aws-sdk (bedrock)', function () { - setup() - - withVersions('aws-sdk', ['@aws-sdk/smithy-client', 'aws-sdk'], '>=3', (version, moduleName) => { - let AWS - let bedrockRuntimeClient - - const bedrockRuntimeClientName = - moduleName === '@aws-sdk/smithy-client' ? '@aws-sdk/client-bedrock-runtime' : 'aws-sdk' - describe('with configuration', () => { - before(() => { - return agent.load('aws-sdk') - }) - - before(done => { - const requireVersion = version === '3.0.0' ? '3.422.0' : '>=3.422.0' - AWS = require(`../../../versions/${bedrockRuntimeClientName}@${requireVersion}`).get() - bedrockRuntimeClient = new AWS.BedrockRuntimeClient( - { endpoint: 'http://127.0.0.1:4566', region: 'us-east-1', ServiceId: serviceName } - ) - done() - }) - - after(async () => { - nock.cleanAll() - return agent.close({ ritmReset: false }) - }) - - const prompt = 'What is the capital of France?' - const temperature = 0.5 - const topP = 1 - const topK = 1 - const maxTokens = 512 - - const models = [ - { - provider: PROVIDER.AMAZON, - modelId: 'amazon.titan-text-lite-v1', - userPrompt: prompt, - requestBody: { - inputText: prompt, - textGenerationConfig: { - temperature, - topP, - maxTokenCount: maxTokens - } - }, - response: { - inputTextTokenCount: 7, - results: { - inputTextTokenCount: 7, - results: [ - { - tokenCount: 35, - outputText: '\n' + - 'Paris is the capital of France. France is a country that is located in Western Europe. ' + - 'Paris is one of the most populous cities in the European Union. ', - completionReason: 'FINISH' - } - ] - } - } - }, - { - provider: PROVIDER.AI21, - modelId: 'ai21.jamba-1-5-mini-v1', - userPrompt: prompt, - requestBody: { - messages: [ - { - role: 'user', - content: prompt - } - ], - max_tokens: maxTokens, - temperature, - top_p: topP, - top_k: topK - }, - response: { - id: 'req_0987654321', - choices: [ - { - index: 0, - message: { - role: 'assistant', - content: 'The capital of France is Paris.' - }, - finish_reason: 'stop' - } - ], - usage: { - prompt_tokens: 10, - completion_tokens: 7, - total_tokens: 17 - } - } - }, - { - provider: PROVIDER.ANTHROPIC, - modelId: 'anthropic.claude-v2', - userPrompt: `\n\nHuman:${prompt}\n\nAssistant:`, - requestBody: { - prompt: `\n\nHuman:${prompt}\n\nAssistant:`, - temperature, - top_p: topP, - top_k: topK, - max_tokens_to_sample: maxTokens - }, - response: { - type: 'completion', - completion: ' Paris is the capital of France.', - stop_reason: 'stop_sequence', - stop: '\n\nHuman:' - } - }, - { - provider: PROVIDER.COHERE, - modelId: 'cohere.command-light-text-v14', - userPrompt: prompt, - requestBody: { - prompt, - temperature, - p: topP, - k: topK, - max_tokens: maxTokens - }, - response: { - id: '91c65da4-e2cd-4930-a4a9-f5c68c8a137c', - generations: [ - { - id: 'c040d384-ad9c-4d15-8c2f-f36fbfb0eb55', - text: ' The capital of France is Paris. \n', - finish_reason: 'COMPLETE' - } - ], - prompt: 'What is the capital of France?' - } - - }, - { - provider: PROVIDER.META, - modelId: 'meta.llama3-70b-instruct-v1', - userPrompt: prompt, - requestBody: { - prompt, - temperature, - top_p: topP, - max_gen_len: maxTokens - }, - response: { - generation: '\n\nThe capital of France is Paris.', - prompt_token_count: 10, - generation_token_count: 7, - stop_reason: 'stop' - } - }, - { - provider: PROVIDER.MISTRAL, - modelId: 'mistral.mistral-7b-instruct-v0', - userPrompt: prompt, - requestBody: { - prompt, - max_tokens: maxTokens, - temperature, - top_p: topP, - top_k: topK - }, - response: { - outputs: [ - { - text: 'The capital of France is Paris.', - stop_reason: 'stop' - } - ] - } - } - ] - - models.forEach(model => { - it(`should invoke model for provider:${model.provider}`, done => { - const request = { - body: JSON.stringify(model.requestBody), - contentType: 'application/json', - accept: 'application/json', - modelId: model.modelId - } - - const response = JSON.stringify(model.response) - - nock('http://127.0.0.1:4566') - .post(`/model/${model.modelId}/invoke`) - .reply(200, response) - - const command = new AWS.InvokeModelCommand(request) - - agent.use(traces => { - const span = traces[0][0] - expect(span.meta).to.include({ - 'aws.operation': 'invokeModel', - 'aws.bedrock.request.model': model.modelId.split('.')[1], - 'aws.bedrock.request.model_provider': model.provider, - 'aws.bedrock.request.prompt': model.userPrompt - }) - expect(span.metrics).to.include({ - 'aws.bedrock.request.temperature': temperature, - 'aws.bedrock.request.top_p': topP, - 'aws.bedrock.request.max_tokens': maxTokens - }) - }).then(done).catch(done) - - bedrockRuntimeClient.send(command, (err) => { - if (err) return done(err) - }) - }) - }) - }) - }) - }) -}) diff --git a/packages/datadog-plugin-aws-sdk/test/bedrockruntime.spec.js b/packages/datadog-plugin-aws-sdk/test/bedrockruntime.spec.js new file mode 100644 index 00000000000..4885af36f85 --- /dev/null +++ b/packages/datadog-plugin-aws-sdk/test/bedrockruntime.spec.js @@ -0,0 +1,79 @@ +'use strict' + +const agent = require('../../dd-trace/test/plugins/agent') +const nock = require('nock') +const { setup } = require('./spec_helpers') +const { models, modelConfig } = require('./fixtures/bedrockruntime') + +const serviceName = 'bedrock-service-name-test' + +describe('Plugin', () => { + describe('aws-sdk (bedrockruntime)', function () { + setup() + + withVersions('aws-sdk', ['@aws-sdk/smithy-client', 'aws-sdk'], '>=3', (version, moduleName) => { + let AWS + let bedrockRuntimeClient + + const bedrockRuntimeClientName = + moduleName === '@aws-sdk/smithy-client' ? '@aws-sdk/client-bedrock-runtime' : 'aws-sdk' + describe('with configuration', () => { + before(() => { + return agent.load('aws-sdk') + }) + + before(done => { + const requireVersion = version === '3.0.0' ? '3.422.0' : '>=3.422.0' + AWS = require(`../../../versions/${bedrockRuntimeClientName}@${requireVersion}`).get() + bedrockRuntimeClient = new AWS.BedrockRuntimeClient( + { endpoint: 'http://127.0.0.1:4566', region: 'us-east-1', ServiceId: serviceName } + ) + done() + }) + + after(async () => { + nock.cleanAll() + return agent.close({ ritmReset: false }) + }) + + models.forEach(model => { + it(`should invoke model for provider:${model.provider}`, done => { + const request = { + body: JSON.stringify(model.requestBody), + contentType: 'application/json', + accept: 'application/json', + modelId: model.modelId + } + + const response = JSON.stringify(model.response) + + nock('http://127.0.0.1:4566') + .post(`/model/${model.modelId}/invoke`) + .reply(200, response) + + const command = new AWS.InvokeModelCommand(request) + + agent.use(traces => { + const span = traces[0][0] + expect(span.meta).to.include({ + 'aws.operation': 'invokeModel', + 'aws.bedrock.request.model': model.modelId.split('.')[1], + 'aws.bedrock.request.model_provider': model.provider.toLowerCase(), + 'aws.bedrock.request.prompt': model.userPrompt + }) + expect(span.metrics).to.include({ + 'aws.bedrock.request.temperature': modelConfig.temperature, + 'aws.bedrock.request.top_p': modelConfig.topP, + 'aws.bedrock.request.max_tokens': modelConfig.maxTokens + }) + }).then(done).catch(done) + + bedrockRuntimeClient.send(command, (err) => { + if (err) return done(err) + }) + }) + }) + }) + }) + }) +}) diff --git a/packages/datadog-plugin-aws-sdk/test/fixtures/bedrockruntime.js b/packages/datadog-plugin-aws-sdk/test/fixtures/bedrockruntime.js new file mode 100644 index 00000000000..39b5ef8b963 --- /dev/null +++ b/packages/datadog-plugin-aws-sdk/test/fixtures/bedrockruntime.js @@ -0,0 +1,171 @@ +'use strict' + +const bedrockruntime = {} + +const PROVIDER = { + AI21: 'AI21', + AMAZON: 'AMAZON', + ANTHROPIC: 'ANTHROPIC', + COHERE: 'COHERE', + META: 'META', + MISTRAL: 'MISTRAL' +} + +const prompt = 'What is the capital of France?' +const temperature = 0.5 +const topP = 1 +const topK = 1 +const maxTokens = 512 + +bedrockruntime.models = [ + { + provider: PROVIDER.AMAZON, + modelId: 'amazon.titan-text-lite-v1', + userPrompt: prompt, + requestBody: { + inputText: prompt, + textGenerationConfig: { + temperature, + topP, + maxTokenCount: maxTokens + } + }, + response: { + inputTextTokenCount: 7, + results: { + inputTextTokenCount: 7, + results: [ + { + tokenCount: 35, + outputText: '\n' + + 'Paris is the capital of France. France is a country that is located in Western Europe. ' + + 'Paris is one of the most populous cities in the European Union. ', + completionReason: 'FINISH' + } + ] + } + } + }, + { + provider: PROVIDER.AI21, + modelId: 'ai21.jamba-1-5-mini-v1', + userPrompt: prompt, + requestBody: { + messages: [ + { + role: 'user', + content: prompt + } + ], + max_tokens: maxTokens, + temperature, + top_p: topP, + top_k: topK + }, + response: { + id: 'req_0987654321', + choices: [ + { + index: 0, + message: { + role: 'assistant', + content: 'The capital of France is Paris.' + }, + finish_reason: 'stop' + } + ], + usage: { + prompt_tokens: 10, + completion_tokens: 7, + total_tokens: 17 + } + } + }, + { + provider: PROVIDER.ANTHROPIC, + modelId: 'anthropic.claude-v2', + userPrompt: `\n\nHuman:${prompt}\n\nAssistant:`, + requestBody: { + prompt: `\n\nHuman:${prompt}\n\nAssistant:`, + temperature, + top_p: topP, + top_k: topK, + max_tokens_to_sample: maxTokens + }, + response: { + type: 'completion', + completion: ' Paris is the capital of France.', + stop_reason: 'stop_sequence', + stop: '\n\nHuman:' + } + }, + { + provider: PROVIDER.COHERE, + modelId: 'cohere.command-light-text-v14', + userPrompt: prompt, + requestBody: { + prompt, + temperature, + p: topP, + k: topK, + max_tokens: maxTokens + }, + response: { + id: '91c65da4-e2cd-4930-a4a9-f5c68c8a137c', + generations: [ + { + id: 'c040d384-ad9c-4d15-8c2f-f36fbfb0eb55', + text: ' The capital of France is Paris. \n', + finish_reason: 'COMPLETE' + } + ], + prompt: 'What is the capital of France?' + } + + }, + { + provider: PROVIDER.META, + modelId: 'meta.llama3-70b-instruct-v1', + userPrompt: prompt, + requestBody: { + prompt, + temperature, + top_p: topP, + max_gen_len: maxTokens + }, + response: { + generation: '\n\nThe capital of France is Paris.', + prompt_token_count: 10, + generation_token_count: 7, + stop_reason: 'stop' + } + }, + { + provider: PROVIDER.MISTRAL, + modelId: 'mistral.mistral-7b-instruct-v0', + userPrompt: prompt, + requestBody: { + prompt, + max_tokens: maxTokens, + temperature, + top_p: topP, + top_k: topK + }, + response: { + outputs: [ + { + text: 'The capital of France is Paris.', + stop_reason: 'stop' + } + ] + } + } +] +bedrockruntime.modelConfig = { + temperature, + topP, + topK, + maxTokens +} + +module.exports = bedrockruntime diff --git a/packages/datadog-plugin-cassandra-driver/test/integration-test/client.spec.js b/packages/datadog-plugin-cassandra-driver/test/integration-test/client.spec.js index da680b7fa25..b23376bb3df 100644 --- a/packages/datadog-plugin-cassandra-driver/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-cassandra-driver/test/integration-test/client.spec.js @@ -7,10 +7,6 @@ const { spawnPluginIntegrationTestProc } = require('../../../../integration-tests/helpers') const { assert } = require('chai') -const { NODE_MAJOR } = require('../../../../version') - -// newer packages are not supported on older node versions -const range = NODE_MAJOR < 16 ? '<3' : '>=4.4.0' describe('esm', () => { let agent @@ -18,7 +14,7 @@ describe('esm', () => { let sandbox // test against later versions because server.mjs uses newer package syntax - withVersions('cassandra-driver', 'cassandra-driver', range, version => { + withVersions('cassandra-driver', 'cassandra-driver', '>=4.4.0', version => { before(async function () { this.timeout(20000) sandbox = await createSandbox([`'cassandra-driver@${version}'`], false, [ diff --git a/packages/datadog-plugin-cucumber/src/index.js b/packages/datadog-plugin-cucumber/src/index.js index 16cca8b6b59..7454c87560b 100644 --- a/packages/datadog-plugin-cucumber/src/index.js +++ b/packages/datadog-plugin-cucumber/src/index.js @@ -26,7 +26,8 @@ const { TEST_MODULE, TEST_MODULE_ID, TEST_SUITE, - CUCUMBER_IS_PARALLEL + CUCUMBER_IS_PARALLEL, + TEST_RETRY_REASON } = require('../../dd-trace/src/plugins/util/test') const { RESOURCE_NAME } = require('../../../ext/tags') const { COMPONENT, ERROR_MESSAGE } = require('../../dd-trace/src/constants') @@ -321,6 +322,7 @@ class CucumberPlugin extends CiPlugin { span.setTag(TEST_IS_NEW, 'true') if (isEfdRetry) { span.setTag(TEST_IS_RETRY, 'true') + span.setTag(TEST_RETRY_REASON, 'efd') } } diff --git a/packages/datadog-plugin-cucumber/test/index.spec.js b/packages/datadog-plugin-cucumber/test/index.spec.js index a43a2a53509..863d5703063 100644 --- a/packages/datadog-plugin-cucumber/test/index.spec.js +++ b/packages/datadog-plugin-cucumber/test/index.spec.js @@ -1,7 +1,6 @@ 'use strict' const path = require('path') const { PassThrough } = require('stream') -const semver = require('semver') const proxyquire = require('proxyquire').noPreserveCache() const nock = require('nock') @@ -24,7 +23,6 @@ const { TEST_SOURCE_START } = require('../../dd-trace/src/plugins/util/test') -const { NODE_MAJOR } = require('../../../version') const { version: ddTraceVersion } = require('../../../package.json') const runCucumber = (version, Cucumber, requireName, featureName, testName) => { @@ -56,8 +54,6 @@ describe('Plugin', function () { let Cucumber this.timeout(10000) withVersions('cucumber', '@cucumber/cucumber', (version, _, specificVersion) => { - if (NODE_MAJOR <= 16 && semver.satisfies(specificVersion, '>=10')) return - afterEach(() => { // > If you want to run tests multiple times, you may need to clear Node's require cache // before subsequent calls in whichever manner best suits your needs. diff --git a/packages/datadog-plugin-cypress/src/cypress-plugin.js b/packages/datadog-plugin-cypress/src/cypress-plugin.js index 2ed62070fda..31d4d282f64 100644 --- a/packages/datadog-plugin-cypress/src/cypress-plugin.js +++ b/packages/datadog-plugin-cypress/src/cypress-plugin.js @@ -31,7 +31,8 @@ const { TEST_EARLY_FLAKE_ENABLED, getTestSessionName, TEST_SESSION_NAME, - TEST_LEVEL_EVENT_TYPES + TEST_LEVEL_EVENT_TYPES, + TEST_RETRY_REASON } = require('../../dd-trace/src/plugins/util/test') const { isMarkedAsUnskippable } = require('../../datadog-plugin-jest/src/util') const { ORIGIN_KEY, COMPONENT } = require('../../dd-trace/src/constants') @@ -112,7 +113,7 @@ function getCypressCommand (details) { function getLibraryConfiguration (tracer, testConfiguration) { return new Promise(resolve => { if (!tracer._tracer._exporter?.getLibraryConfiguration) { - return resolve({ err: new Error('CI Visibility was not initialized correctly') }) + return resolve({ err: new Error('Test Optimization was not initialized correctly') }) } tracer._tracer._exporter.getLibraryConfiguration(testConfiguration, (err, libraryConfig) => { @@ -124,7 +125,7 @@ function getLibraryConfiguration (tracer, testConfiguration) { function getSkippableTests (tracer, testConfiguration) { return new Promise(resolve => { if (!tracer._tracer._exporter?.getSkippableSuites) { - return resolve({ err: new Error('CI Visibility was not initialized correctly') }) + return resolve({ err: new Error('Test Optimization was not initialized correctly') }) } tracer._tracer._exporter.getSkippableSuites(testConfiguration, (err, skippableTests, correlationId) => { resolve({ @@ -139,7 +140,7 @@ function getSkippableTests (tracer, testConfiguration) { function getKnownTests (tracer, testConfiguration) { return new Promise(resolve => { if (!tracer._tracer._exporter?.getKnownTests) { - return resolve({ err: new Error('CI Visibility was not initialized correctly') }) + return resolve({ err: new Error('Test Optimization was not initialized correctly') }) } tracer._tracer._exporter.getKnownTests(testConfiguration, (err, knownTests) => { resolve({ @@ -203,6 +204,7 @@ class CypressPlugin { this.isSuitesSkippingEnabled = false this.isCodeCoverageEnabled = false this.isEarlyFlakeDetectionEnabled = false + this.isKnownTestsEnabled = false this.earlyFlakeDetectionNumRetries = 0 this.testsToSkip = [] this.skippedTests = [] @@ -232,13 +234,15 @@ class CypressPlugin { isEarlyFlakeDetectionEnabled, earlyFlakeDetectionNumRetries, isFlakyTestRetriesEnabled, - flakyTestRetriesCount + flakyTestRetriesCount, + isKnownTestsEnabled } } = libraryConfigurationResponse this.isSuitesSkippingEnabled = isSuitesSkippingEnabled this.isCodeCoverageEnabled = isCodeCoverageEnabled this.isEarlyFlakeDetectionEnabled = isEarlyFlakeDetectionEnabled this.earlyFlakeDetectionNumRetries = earlyFlakeDetectionNumRetries + this.isKnownTestsEnabled = isKnownTestsEnabled if (isFlakyTestRetriesEnabled) { this.cypressConfig.retries.runMode = flakyTestRetriesCount } @@ -354,7 +358,7 @@ class CypressPlugin { this.frameworkVersion = getCypressVersion(details) this.rootDir = getRootDir(details) - if (this.isEarlyFlakeDetectionEnabled) { + if (this.isKnownTestsEnabled) { const knownTestsResponse = await getKnownTests( this.tracer, this.testConfiguration @@ -362,6 +366,7 @@ class CypressPlugin { if (knownTestsResponse.err) { log.error('Cypress known tests response error', knownTestsResponse.err) this.isEarlyFlakeDetectionEnabled = false + this.isKnownTestsEnabled = false } else { // We use TEST_FRAMEWORK_NAME for the name of the module this.knownTestsByTestSuite = knownTestsResponse.knownTests[TEST_FRAMEWORK_NAME] @@ -567,6 +572,9 @@ class CypressPlugin { cypressTestStatus = CYPRESS_STATUS_TO_TEST_STATUS[cypressTest.attempts[attemptIndex].state] if (attemptIndex > 0) { finishedTest.testSpan.setTag(TEST_IS_RETRY, 'true') + if (finishedTest.isEfdRetry) { + finishedTest.testSpan.setTag(TEST_RETRY_REASON, 'efd') + } } } if (cypressTest.displayError) { @@ -618,7 +626,8 @@ class CypressPlugin { const suitePayload = { isEarlyFlakeDetectionEnabled: this.isEarlyFlakeDetectionEnabled, knownTestsForSuite: this.knownTestsByTestSuite?.[testSuite] || [], - earlyFlakeDetectionNumRetries: this.earlyFlakeDetectionNumRetries + earlyFlakeDetectionNumRetries: this.earlyFlakeDetectionNumRetries, + isKnownTestsEnabled: this.isKnownTestsEnabled } if (this.testSuiteSpan) { @@ -703,13 +712,15 @@ class CypressPlugin { this.activeTestSpan.setTag(TEST_IS_NEW, 'true') if (isEfdRetry) { this.activeTestSpan.setTag(TEST_IS_RETRY, 'true') + this.activeTestSpan.setTag(TEST_RETRY_REASON, 'efd') } } const finishedTest = { testName, testStatus, finishTime: this.activeTestSpan._getTime(), // we store the finish time here - testSpan: this.activeTestSpan + testSpan: this.activeTestSpan, + isEfdRetry } if (this.finishedTestsByFile[testSuite]) { this.finishedTestsByFile[testSuite].push(finishedTest) diff --git a/packages/datadog-plugin-cypress/src/support.js b/packages/datadog-plugin-cypress/src/support.js index 8900f2695fb..6e31e9e45a1 100644 --- a/packages/datadog-plugin-cypress/src/support.js +++ b/packages/datadog-plugin-cypress/src/support.js @@ -1,5 +1,6 @@ /* eslint-disable */ let isEarlyFlakeDetectionEnabled = false +let isKnownTestsEnabled = false let knownTestsForSuite = [] let suiteTests = [] let earlyFlakeDetectionNumRetries = 0 @@ -33,7 +34,7 @@ function retryTest (test, suiteTests) { const oldRunTests = Cypress.mocha.getRunner().runTests Cypress.mocha.getRunner().runTests = function (suite, fn) { - if (!isEarlyFlakeDetectionEnabled) { + if (!isKnownTestsEnabled) { return oldRunTests.apply(this, arguments) } // We copy the new tests at the beginning of the suite run (runTests), so that they're run @@ -41,7 +42,9 @@ Cypress.mocha.getRunner().runTests = function (suite, fn) { suite.tests.forEach(test => { if (!test._ddIsNew && !test.isPending() && isNewTest(test)) { test._ddIsNew = true - retryTest(test, suite.tests) + if (isEarlyFlakeDetectionEnabled) { + retryTest(test, suite.tests) + } } }) @@ -67,6 +70,7 @@ before(function () { }).then((suiteConfig) => { if (suiteConfig) { isEarlyFlakeDetectionEnabled = suiteConfig.isEarlyFlakeDetectionEnabled + isKnownTestsEnabled = suiteConfig.isKnownTestsEnabled knownTestsForSuite = suiteConfig.knownTestsForSuite earlyFlakeDetectionNumRetries = suiteConfig.earlyFlakeDetectionNumRetries } diff --git a/packages/datadog-plugin-fetch/src/index.js b/packages/datadog-plugin-fetch/src/index.js index 44173a561ca..943a1908ddb 100644 --- a/packages/datadog-plugin-fetch/src/index.js +++ b/packages/datadog-plugin-fetch/src/index.js @@ -9,7 +9,7 @@ class FetchPlugin extends HttpClientPlugin { bindStart (ctx) { const req = ctx.req const options = new URL(req.url) - const headers = options.headers = Object.fromEntries(req.headers.entries()) + options.headers = Object.fromEntries(req.headers.entries()) options.method = req.method @@ -17,9 +17,9 @@ class FetchPlugin extends HttpClientPlugin { const store = super.bindStart(ctx) - for (const name in headers) { + for (const name in options.headers) { if (!req.headers.has(name)) { - req.headers.set(name, headers[name]) + req.headers.set(name, options.headers[name]) } } diff --git a/packages/datadog-plugin-fetch/test/index.spec.js b/packages/datadog-plugin-fetch/test/index.spec.js index 1d322de04a4..1d20d375d79 100644 --- a/packages/datadog-plugin-fetch/test/index.spec.js +++ b/packages/datadog-plugin-fetch/test/index.spec.js @@ -14,7 +14,9 @@ const HTTP_RESPONSE_HEADERS = tags.HTTP_RESPONSE_HEADERS const SERVICE_NAME = DD_MAJOR < 3 ? 'test-http-client' : 'test' const describe = globalThis.fetch ? globalThis.describe : globalThis.describe.skip -describe('Plugin', () => { +describe('Plugin', function () { + this.timeout(0) + let express let fetch let appListener @@ -215,102 +217,6 @@ describe('Plugin', () => { }) }) - it('should skip injecting if the Authorization header contains an AWS signature', done => { - const app = express() - - app.get('/', (req, res) => { - try { - expect(req.get('x-datadog-trace-id')).to.be.undefined - expect(req.get('x-datadog-parent-id')).to.be.undefined - - res.status(200).send() - - done() - } catch (e) { - done(e) - } - }) - - appListener = server(app, port => { - fetch(`http://localhost:${port}/`, { - headers: { - Authorization: 'AWS4-HMAC-SHA256 ...' - } - }) - }) - }) - - it('should skip injecting if one of the Authorization headers contains an AWS signature', done => { - const app = express() - - app.get('/', (req, res) => { - try { - expect(req.get('x-datadog-trace-id')).to.be.undefined - expect(req.get('x-datadog-parent-id')).to.be.undefined - - res.status(200).send() - - done() - } catch (e) { - done(e) - } - }) - - appListener = server(app, port => { - fetch(`http://localhost:${port}/`, { - headers: { - Authorization: ['AWS4-HMAC-SHA256 ...'] - } - }) - }) - }) - - it('should skip injecting if the X-Amz-Signature header is set', done => { - const app = express() - - app.get('/', (req, res) => { - try { - expect(req.get('x-datadog-trace-id')).to.be.undefined - expect(req.get('x-datadog-parent-id')).to.be.undefined - - res.status(200).send() - - done() - } catch (e) { - done(e) - } - }) - - appListener = server(app, port => { - fetch(`http://localhost:${port}/`, { - headers: { - 'X-Amz-Signature': 'abc123' - } - }) - }) - }) - - it('should skip injecting if the X-Amz-Signature query param is set', done => { - const app = express() - - app.get('/', (req, res) => { - try { - expect(req.get('x-datadog-trace-id')).to.be.undefined - expect(req.get('x-datadog-parent-id')).to.be.undefined - - res.status(200).send() - - done() - } catch (e) { - done(e) - } - }) - - appListener = server(app, port => { - fetch(`http://localhost:${port}/?X-Amz-Signature=abc123`) - }) - }) - it('should handle connection errors', done => { let error diff --git a/packages/datadog-plugin-http/src/client.js b/packages/datadog-plugin-http/src/client.js index d4c105d2508..2bc408e648b 100644 --- a/packages/datadog-plugin-http/src/client.js +++ b/packages/datadog-plugin-http/src/client.js @@ -59,6 +59,11 @@ class HttpClientPlugin extends ClientPlugin { } if (this.shouldInjectTraceHeaders(options, uri)) { + // Clone the headers object in case an upstream lib has a reference to the original headers + // Implemented due to aws-sdk issue where request signing is broken if we mutate the headers + // Explained further in: + // https://github.com/open-telemetry/opentelemetry-js-contrib/issues/1609#issuecomment-1826167348 + options.headers = Object.assign({}, options.headers) this.tracer.inject(span, HTTP_HEADERS, options.headers) } @@ -72,10 +77,6 @@ class HttpClientPlugin extends ClientPlugin { } shouldInjectTraceHeaders (options, uri) { - if (hasAmazonSignature(options) && !this.config.enablePropagationWithAmazonHeaders) { - return false - } - if (!this.config.propagationFilter(uri)) { return false } @@ -212,31 +213,6 @@ function getHooks (config) { return { request } } -function hasAmazonSignature (options) { - if (!options) { - return false - } - - if (options.headers) { - const headers = Object.keys(options.headers) - .reduce((prev, next) => Object.assign(prev, { - [next.toLowerCase()]: options.headers[next] - }), {}) - - if (headers['x-amz-signature']) { - return true - } - - if ([].concat(headers.authorization).some(startsWith('AWS4-HMAC-SHA256'))) { - return true - } - } - - const search = options.search || options.path - - return search && search.toLowerCase().indexOf('x-amz-signature=') !== -1 -} - function extractSessionDetails (options) { if (typeof options === 'string') { return new URL(options).host @@ -248,8 +224,4 @@ function extractSessionDetails (options) { return { host, port } } -function startsWith (searchString) { - return value => String(value).startsWith(searchString) -} - module.exports = HttpClientPlugin diff --git a/packages/datadog-plugin-http/test/client.spec.js b/packages/datadog-plugin-http/test/client.spec.js index 42f4c8436f8..ff2d220d0cd 100644 --- a/packages/datadog-plugin-http/test/client.spec.js +++ b/packages/datadog-plugin-http/test/client.spec.js @@ -446,97 +446,24 @@ describe('Plugin', () => { }) }) - it('should skip injecting if the Authorization header contains an AWS signature', done => { - const app = express() - - app.get('/', (req, res) => { - try { - expect(req.get('x-datadog-trace-id')).to.be.undefined - expect(req.get('x-datadog-parent-id')).to.be.undefined - - res.status(200).send() - - done() - } catch (e) { - done(e) - } - }) - - appListener = server(app, port => { - const req = http.request({ - port, - headers: { - Authorization: 'AWS4-HMAC-SHA256 ...' - } - }) + it('should inject tracing header into request without mutating the header', done => { + // ensures that the tracer clones request headers instead of mutating. + // Fixes aws-sdk InvalidSignatureException, more info: + // https://github.com/open-telemetry/opentelemetry-js-contrib/issues/1609#issuecomment-1826167348 - req.end() - }) - }) - - it('should skip injecting if one of the Authorization headers contains an AWS signature', done => { const app = express() - app.get('/', (req, res) => { - try { - expect(req.get('x-datadog-trace-id')).to.be.undefined - expect(req.get('x-datadog-parent-id')).to.be.undefined - - res.status(200).send() - - done() - } catch (e) { - done(e) - } - }) - - appListener = server(app, port => { - const req = http.request({ - port, - headers: { - Authorization: ['AWS4-HMAC-SHA256 ...'] - } - }) - - req.end() - }) - }) - - it('should skip injecting if the X-Amz-Signature header is set', done => { - const app = express() + const originalHeaders = { + Authorization: 'AWS4-HMAC-SHA256 ...' + } app.get('/', (req, res) => { try { - expect(req.get('x-datadog-trace-id')).to.be.undefined - expect(req.get('x-datadog-parent-id')).to.be.undefined - - res.status(200).send() - - done() - } catch (e) { - done(e) - } - }) - - appListener = server(app, port => { - const req = http.request({ - port, - headers: { - 'X-Amz-Signature': 'abc123' - } - }) - - req.end() - }) - }) - - it('should skip injecting if the X-Amz-Signature query param is set', done => { - const app = express() + expect(req.get('x-datadog-trace-id')).to.be.a('string') + expect(req.get('x-datadog-parent-id')).to.be.a('string') - app.get('/', (req, res) => { - try { - expect(req.get('x-datadog-trace-id')).to.be.undefined - expect(req.get('x-datadog-parent-id')).to.be.undefined + expect(originalHeaders['x-datadog-trace-id']).to.be.undefined + expect(originalHeaders['x-datadog-parent-id']).to.be.undefined res.status(200).send() @@ -549,7 +476,7 @@ describe('Plugin', () => { appListener = server(app, port => { const req = http.request({ port, - path: '/?X-Amz-Signature=abc123' + headers: originalHeaders }) req.end() @@ -1093,50 +1020,6 @@ describe('Plugin', () => { }) }) - describe('with config enablePropagationWithAmazonHeaders enabled', () => { - let config - - beforeEach(() => { - config = { - enablePropagationWithAmazonHeaders: true - } - - return agent.load('http', config) - .then(() => { - http = require(pluginToBeLoaded) - express = require('express') - }) - }) - - it('should inject tracing header into AWS signed request', done => { - const app = express() - - app.get('/', (req, res) => { - try { - expect(req.get('x-datadog-trace-id')).to.be.a('string') - expect(req.get('x-datadog-parent-id')).to.be.a('string') - - res.status(200).send() - - done() - } catch (e) { - done(e) - } - }) - - appListener = server(app, port => { - const req = http.request({ - port, - headers: { - Authorization: 'AWS4-HMAC-SHA256 ...' - } - }) - - req.end() - }) - }) - }) - describe('with validateStatus configuration', () => { let config diff --git a/packages/datadog-plugin-jest/src/index.js b/packages/datadog-plugin-jest/src/index.js index 751cbef790b..f82899f20d1 100644 --- a/packages/datadog-plugin-jest/src/index.js +++ b/packages/datadog-plugin-jest/src/index.js @@ -23,7 +23,8 @@ const { JEST_DISPLAY_NAME, TEST_IS_RUM_ACTIVE, TEST_BROWSER_DRIVER, - getFormattedError + getFormattedError, + TEST_RETRY_REASON } = require('../../dd-trace/src/plugins/util/test') const { COMPONENT } = require('../../dd-trace/src/constants') const id = require('../../dd-trace/src/id') @@ -167,6 +168,7 @@ class JestPlugin extends CiPlugin { config._ddIsFlakyTestRetriesEnabled = this.libraryConfig?.isFlakyTestRetriesEnabled ?? false config._ddFlakyTestRetriesCount = this.libraryConfig?.flakyTestRetriesCount config._ddIsDiEnabled = this.libraryConfig?.isDiEnabled ?? false + config._ddIsKnownTestsEnabled = this.libraryConfig?.isKnownTestsEnabled ?? false }) }) @@ -410,6 +412,7 @@ class JestPlugin extends CiPlugin { extraTags[TEST_IS_NEW] = 'true' if (isEfdRetry) { extraTags[TEST_IS_RETRY] = 'true' + extraTags[TEST_RETRY_REASON] = 'efd' } } diff --git a/packages/datadog-plugin-mocha/src/index.js b/packages/datadog-plugin-mocha/src/index.js index bea9400b083..f4c9b063328 100644 --- a/packages/datadog-plugin-mocha/src/index.js +++ b/packages/datadog-plugin-mocha/src/index.js @@ -30,7 +30,8 @@ const { TEST_SUITE, MOCHA_IS_PARALLEL, TEST_IS_RUM_ACTIVE, - TEST_BROWSER_DRIVER + TEST_BROWSER_DRIVER, + TEST_RETRY_REASON } = require('../../dd-trace/src/plugins/util/test') const { COMPONENT } = require('../../dd-trace/src/constants') const { @@ -421,6 +422,7 @@ class MochaPlugin extends CiPlugin { extraTags[TEST_IS_NEW] = 'true' if (isEfdRetry) { extraTags[TEST_IS_RETRY] = 'true' + extraTags[TEST_RETRY_REASON] = 'efd' } } diff --git a/packages/datadog-plugin-next/test/index.spec.js b/packages/datadog-plugin-next/test/index.spec.js index caec28e3b1a..3fa35e4e280 100644 --- a/packages/datadog-plugin-next/test/index.spec.js +++ b/packages/datadog-plugin-next/test/index.spec.js @@ -9,15 +9,8 @@ const { execSync, spawn } = require('child_process') const agent = require('../../dd-trace/test/plugins/agent') const { writeFileSync, readdirSync } = require('fs') const { satisfies } = require('semver') -const { DD_MAJOR, NODE_MAJOR } = require('../../../version') const { rawExpectedSchema } = require('./naming') -const BUILD_COMMAND = NODE_MAJOR < 18 - ? 'yarn exec next build' - : 'NODE_OPTIONS=--openssl-legacy-provider yarn exec next build' -let VERSIONS_TO_TEST = NODE_MAJOR < 18 ? '>=11.1 <13.2' : '>=11.1' -VERSIONS_TO_TEST = DD_MAJOR >= 4 ? VERSIONS_TO_TEST : '>=9.5 <11.1' - describe('Plugin', function () { let server let port @@ -26,7 +19,7 @@ describe('Plugin', function () { const satisfiesStandalone = version => satisfies(version, '>=12.0.0') // TODO: Figure out why 10.x tests are failing. - withVersions('next', 'next', VERSIONS_TO_TEST, version => { + withVersions('next', 'next', '>=11.1', version => { const pkg = require(`../../../versions/next@${version}/node_modules/next/package.json`) const startServer = ({ withConfig, standalone }, schemaVersion = 'v0', defaultToGlobalService = false) => { @@ -110,7 +103,7 @@ describe('Plugin', function () { } // building in-process makes tests fail for an unknown reason - execSync(BUILD_COMMAND, { + execSync('NODE_OPTIONS=--openssl-legacy-provider yarn exec next build', { cwd, env: { ...process.env, diff --git a/packages/datadog-plugin-next/test/integration-test/client.spec.js b/packages/datadog-plugin-next/test/integration-test/client.spec.js index 5bd4825ce93..841e9402584 100644 --- a/packages/datadog-plugin-next/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-next/test/integration-test/client.spec.js @@ -8,31 +8,21 @@ const { spawnPluginIntegrationTestProc } = require('../../../../integration-tests/helpers') const { assert } = require('chai') -const { NODE_MAJOR } = require('../../../../version') const hookFile = 'dd-trace/loader-hook.mjs' -const BUILD_COMMAND = NODE_MAJOR < 18 - ? 'yarn exec next build' - : 'NODE_OPTIONS=--openssl-legacy-provider yarn exec next build' -const NODE_OPTIONS = NODE_MAJOR < 18 - ? `--loader=${hookFile} --require dd-trace/init` - : `--loader=${hookFile} --require dd-trace/init --openssl-legacy-provider` - -const VERSIONS_TO_TEST = NODE_MAJOR < 18 ? '>=11.1 <13.2' : '>=11.1' - describe('esm', () => { let agent let proc let sandbox // match versions tested with unit tests - withVersions('next', 'next', VERSIONS_TO_TEST, version => { + withVersions('next', 'next', '>=11.1', version => { before(async function () { // next builds slower in the CI, match timeout with unit tests this.timeout(120 * 1000) sandbox = await createSandbox([`'next@${version}'`, 'react@^18.2.0', 'react-dom@^18.2.0'], false, ['./packages/datadog-plugin-next/test/integration-test/*'], - BUILD_COMMAND) + 'NODE_OPTIONS=--openssl-legacy-provider yarn exec next build') }) after(async () => { @@ -50,7 +40,7 @@ describe('esm', () => { it('is instrumented', async () => { proc = await spawnPluginIntegrationTestProc(sandbox.folder, 'server.mjs', agent.port, undefined, { - NODE_OPTIONS + NODE_OPTIONS: `--loader=${hookFile} --require dd-trace/init --openssl-legacy-provider` }) return curlAndAssertMessage(agent, proc, ({ headers, payload }) => { assert.propertyVal(headers, 'host', `127.0.0.1:${agent.port}`) diff --git a/packages/datadog-plugin-openai/test/integration-test/client.spec.js b/packages/datadog-plugin-openai/test/integration-test/client.spec.js index a68613f47fd..22339e35e5b 100644 --- a/packages/datadog-plugin-openai/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-openai/test/integration-test/client.spec.js @@ -8,7 +8,8 @@ const { } = require('../../../../integration-tests/helpers') const { assert } = require('chai') -describe('esm', () => { +// TODO(sabrenner): re-enable once issues with mocking OpenAI calls are resolved +describe.skip('esm', () => { let agent let proc let sandbox diff --git a/packages/datadog-plugin-playwright/src/index.js b/packages/datadog-plugin-playwright/src/index.js index 941f779ff54..8fd8ac6fef0 100644 --- a/packages/datadog-plugin-playwright/src/index.js +++ b/packages/datadog-plugin-playwright/src/index.js @@ -15,7 +15,8 @@ const { TEST_IS_NEW, TEST_IS_RETRY, TEST_EARLY_FLAKE_ENABLED, - TELEMETRY_TEST_SESSION + TELEMETRY_TEST_SESSION, + TEST_RETRY_REASON } = require('../../dd-trace/src/plugins/util/test') const { RESOURCE_NAME } = require('../../../ext/tags') const { COMPONENT } = require('../../dd-trace/src/constants') @@ -144,6 +145,7 @@ class PlaywrightPlugin extends CiPlugin { span.setTag(TEST_IS_NEW, 'true') if (isEfdRetry) { span.setTag(TEST_IS_RETRY, 'true') + span.setTag(TEST_RETRY_REASON, 'efd') } } if (isRetry) { diff --git a/packages/datadog-plugin-vitest/src/index.js b/packages/datadog-plugin-vitest/src/index.js index 5b8bc9e865e..c4f94548f10 100644 --- a/packages/datadog-plugin-vitest/src/index.js +++ b/packages/datadog-plugin-vitest/src/index.js @@ -17,7 +17,8 @@ const { TEST_SOURCE_START, TEST_IS_NEW, TEST_EARLY_FLAKE_ENABLED, - TEST_EARLY_FLAKE_ABORT_REASON + TEST_EARLY_FLAKE_ABORT_REASON, + TEST_RETRY_REASON } = require('../../dd-trace/src/plugins/util/test') const { COMPONENT } = require('../../dd-trace/src/constants') const { @@ -60,7 +61,14 @@ class VitestPlugin extends CiPlugin { onDone(isFaulty) }) - this.addSub('ci:vitest:test:start', ({ testName, testSuiteAbsolutePath, isRetry, isNew, mightHitProbe }) => { + this.addSub('ci:vitest:test:start', ({ + testName, + testSuiteAbsolutePath, + isRetry, + isNew, + mightHitProbe, + isRetryReasonEfd + }) => { const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) const store = storage.getStore() @@ -73,6 +81,9 @@ class VitestPlugin extends CiPlugin { if (isNew) { extraTags[TEST_IS_NEW] = 'true' } + if (isRetryReasonEfd) { + extraTags[TEST_RETRY_REASON] = 'efd' + } const span = this.startTestSpan( testName, @@ -147,7 +158,7 @@ class VitestPlugin extends CiPlugin { } }) - this.addSub('ci:vitest:test:skip', ({ testName, testSuiteAbsolutePath }) => { + this.addSub('ci:vitest:test:skip', ({ testName, testSuiteAbsolutePath, isNew }) => { const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) const testSpan = this.startTestSpan( testName, @@ -156,7 +167,8 @@ class VitestPlugin extends CiPlugin { { [TEST_SOURCE_FILE]: testSuite, [TEST_SOURCE_START]: 1, // we can't get the proper start line in vitest - [TEST_STATUS]: 'skip' + [TEST_STATUS]: 'skip', + ...(isNew ? { [TEST_IS_NEW]: 'true' } : {}) } ) this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'test', { diff --git a/packages/dd-trace/src/appsec/iast/analyzers/cookie-analyzer.js b/packages/dd-trace/src/appsec/iast/analyzers/cookie-analyzer.js index a898a0a379c..836908f36e4 100644 --- a/packages/dd-trace/src/appsec/iast/analyzers/cookie-analyzer.js +++ b/packages/dd-trace/src/appsec/iast/analyzers/cookie-analyzer.js @@ -54,15 +54,15 @@ class CookieAnalyzer extends Analyzer { return super._checkOCE(context, value) } - _getLocation (value) { + _getLocation (value, callSiteFrames) { if (!value) { - return super._getLocation() + return super._getLocation(value, callSiteFrames) } if (value.location) { return value.location } - const location = super._getLocation(value) + const location = super._getLocation(value, callSiteFrames) value.location = location return location } diff --git a/packages/dd-trace/src/appsec/iast/analyzers/vulnerability-analyzer.js b/packages/dd-trace/src/appsec/iast/analyzers/vulnerability-analyzer.js index f79e7a44f71..1cb244dbbdc 100644 --- a/packages/dd-trace/src/appsec/iast/analyzers/vulnerability-analyzer.js +++ b/packages/dd-trace/src/appsec/iast/analyzers/vulnerability-analyzer.js @@ -1,12 +1,15 @@ 'use strict' const { storage } = require('../../../../../datadog-core') -const { getFirstNonDDPathAndLine } = require('../path-line') -const { addVulnerability } = require('../vulnerability-reporter') -const { getIastContext } = require('../iast-context') +const { getNonDDCallSiteFrames } = require('../path-line') +const { getIastContext, getIastStackTraceId } = require('../iast-context') const overheadController = require('../overhead-controller') const { SinkIastPlugin } = require('../iast-plugin') -const { getOriginalPathAndLineFromSourceMap } = require('../taint-tracking/rewriter') +const { + addVulnerability, + getVulnerabilityCallSiteFrames, + replaceCallSiteFromSourceMap +} = require('../vulnerability-reporter') class Analyzer extends SinkIastPlugin { constructor (type) { @@ -28,12 +31,24 @@ class Analyzer extends SinkIastPlugin { } _reportEvidence (value, context, evidence) { - const location = this._getLocation(value) + const callSiteFrames = getVulnerabilityCallSiteFrames() + const nonDDCallSiteFrames = getNonDDCallSiteFrames(callSiteFrames, this._getExcludedPaths()) + + const location = this._getLocation(value, nonDDCallSiteFrames) + if (!this._isExcluded(location)) { - const locationSourceMap = this._replaceLocationFromSourceMap(location) + const originalLocation = this._getOriginalLocation(location) const spanId = context && context.rootSpan && context.rootSpan.context().toSpanId() - const vulnerability = this._createVulnerability(this._type, evidence, spanId, locationSourceMap) - addVulnerability(context, vulnerability) + const stackId = getIastStackTraceId(context) + const vulnerability = this._createVulnerability( + this._type, + evidence, + spanId, + originalLocation, + stackId + ) + + addVulnerability(context, vulnerability, nonDDCallSiteFrames) } } @@ -49,24 +64,25 @@ class Analyzer extends SinkIastPlugin { return { value } } - _getLocation () { - return getFirstNonDDPathAndLine(this._getExcludedPaths()) + _getLocation (value, callSiteFrames) { + return callSiteFrames[0] } - _replaceLocationFromSourceMap (location) { - if (location) { - const { path, line, column } = getOriginalPathAndLineFromSourceMap(location) - if (path) { - location.path = path - } - if (line) { - location.line = line - } - if (column) { - location.column = column - } + _getOriginalLocation (location) { + const locationFromSourceMap = replaceCallSiteFromSourceMap(location) + const originalLocation = {} + + if (locationFromSourceMap?.path) { + originalLocation.path = locationFromSourceMap.path + } + if (locationFromSourceMap?.line) { + originalLocation.line = locationFromSourceMap.line } - return location + if (locationFromSourceMap?.column) { + originalLocation.column = locationFromSourceMap.column + } + + return originalLocation } _getExcludedPaths () {} @@ -102,12 +118,13 @@ class Analyzer extends SinkIastPlugin { return overheadController.hasQuota(overheadController.OPERATIONS.REPORT_VULNERABILITY, context) } - _createVulnerability (type, evidence, spanId, location) { + _createVulnerability (type, evidence, spanId, location, stackId) { if (type && evidence) { const _spanId = spanId || 0 return { type, evidence, + stackId, location: { spanId: _spanId, ...location diff --git a/packages/dd-trace/src/appsec/iast/iast-context.js b/packages/dd-trace/src/appsec/iast/iast-context.js index 6d697dcf978..77c757fff8a 100644 --- a/packages/dd-trace/src/appsec/iast/iast-context.js +++ b/packages/dd-trace/src/appsec/iast/iast-context.js @@ -9,6 +9,17 @@ function getIastContext (store, topContext) { return iastContext } +function getIastStackTraceId (iastContext) { + if (!iastContext) return 0 + + if (!iastContext.stackTraceId) { + iastContext.stackTraceId = 0 + } + + iastContext.stackTraceId += 1 + return iastContext.stackTraceId +} + /* TODO Fix storage problem when the close event is called without finish event to remove `topContext` references We have to save the context in two places, because @@ -51,6 +62,7 @@ module.exports = { getIastContext, saveIastContext, cleanIastContext, + getIastStackTraceId, IAST_CONTEXT_KEY, IAST_TRANSACTION_ID } diff --git a/packages/dd-trace/src/appsec/iast/path-line.js b/packages/dd-trace/src/appsec/iast/path-line.js index bf7c3eb2d84..1163bb8d604 100644 --- a/packages/dd-trace/src/appsec/iast/path-line.js +++ b/packages/dd-trace/src/appsec/iast/path-line.js @@ -3,12 +3,10 @@ const path = require('path') const process = require('process') const { calculateDDBasePath } = require('../../util') -const { getCallSiteList } = require('../stack_trace') const pathLine = { - getFirstNonDDPathAndLine, getNodeModulesPaths, getRelativePath, - getFirstNonDDPathAndLineFromCallsites, // Exported only for test purposes + getNonDDCallSiteFrames, calculateDDBasePath, // Exported only for test purposes ddBasePath: calculateDDBasePath(__dirname) // Only for test purposes } @@ -25,22 +23,24 @@ const EXCLUDED_PATH_PREFIXES = [ 'async_hooks' ] -function getFirstNonDDPathAndLineFromCallsites (callsites, externallyExcludedPaths) { - if (callsites) { - for (let i = 0; i < callsites.length; i++) { - const callsite = callsites[i] - const filepath = callsite.getFileName() - if (!isExcluded(callsite, externallyExcludedPaths) && filepath.indexOf(pathLine.ddBasePath) === -1) { - return { - path: getRelativePath(filepath), - line: callsite.getLineNumber(), - column: callsite.getColumnNumber(), - isInternal: !path.isAbsolute(filepath) - } - } +function getNonDDCallSiteFrames (callSiteFrames, externallyExcludedPaths) { + if (!callSiteFrames) { + return [] + } + + const result = [] + + for (const callsite of callSiteFrames) { + const filepath = callsite.file + if (!isExcluded(callsite, externallyExcludedPaths) && filepath.indexOf(pathLine.ddBasePath) === -1) { + callsite.path = getRelativePath(filepath) + callsite.isInternal = !path.isAbsolute(filepath) + + result.push(callsite) } } - return null + + return result } function getRelativePath (filepath) { @@ -48,8 +48,8 @@ function getRelativePath (filepath) { } function isExcluded (callsite, externallyExcludedPaths) { - if (callsite.isNative()) return true - const filename = callsite.getFileName() + if (callsite.isNative) return true + const filename = callsite.file if (!filename) { return true } @@ -73,10 +73,6 @@ function isExcluded (callsite, externallyExcludedPaths) { return false } -function getFirstNonDDPathAndLine (externallyExcludedPaths) { - return getFirstNonDDPathAndLineFromCallsites(getCallSiteList(), externallyExcludedPaths) -} - function getNodeModulesPaths (...paths) { const nodeModulesPaths = [] diff --git a/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/index.js b/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/index.js index d704743dde4..88af720a285 100644 --- a/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/index.js +++ b/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/index.js @@ -84,6 +84,7 @@ class VulnerabilityFormatter { const formattedVulnerability = { type: vulnerability.type, hash: vulnerability.hash, + stackId: vulnerability.stackId, evidence: this.formatEvidence(vulnerability.type, vulnerability.evidence, sourcesIndexes, sources), location: { spanId: vulnerability.location.spanId diff --git a/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js b/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js index 05aea14cf02..4adc636e5af 100644 --- a/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js +++ b/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js @@ -6,6 +6,8 @@ const { IAST_ENABLED_TAG_KEY, IAST_JSON_TAG_KEY } = require('./tags') const standalone = require('../standalone') const { SAMPLING_MECHANISM_APPSEC } = require('../../constants') const { keepTrace } = require('../../priority_sampler') +const { reportStackTrace, getCallsiteFrames, canReportStackTrace, STACK_TRACE_NAMESPACES } = require('../stack_trace') +const { getOriginalPathAndLineFromSourceMap } = require('./taint-tracking/rewriter') const VULNERABILITIES_KEY = 'vulnerabilities' const VULNERABILITY_HASHES_MAX_SIZE = 1000 @@ -15,39 +17,60 @@ const RESET_VULNERABILITY_CACHE_INTERVAL = 60 * 60 * 1000 // 1 hour let tracer let resetVulnerabilityCacheTimer let deduplicationEnabled = true +let stackTraceEnabled = true +let stackTraceMaxDepth +let maxStackTraces -function addVulnerability (iastContext, vulnerability) { - if (vulnerability?.evidence && vulnerability?.type && vulnerability?.location) { - if (deduplicationEnabled && isDuplicatedVulnerability(vulnerability)) return +function canAddVulnerability (vulnerability) { + const hasRequiredFields = vulnerability?.evidence && vulnerability?.type && vulnerability?.location + if (!hasRequiredFields) return false - VULNERABILITY_HASHES.set(`${vulnerability.type}${vulnerability.hash}`, true) + const isDuplicated = deduplicationEnabled && isDuplicatedVulnerability(vulnerability) - let span = iastContext?.rootSpan + return !isDuplicated +} - if (!span && tracer) { - span = tracer.startSpan('vulnerability', { - type: 'vulnerability' - }) +function addVulnerability (iastContext, vulnerability, callSiteFrames) { + if (!canAddVulnerability(vulnerability)) return - vulnerability.location.spanId = span.context().toSpanId() + VULNERABILITY_HASHES.set(`${vulnerability.type}${vulnerability.hash}`, true) - span.addTags({ - [IAST_ENABLED_TAG_KEY]: 1 - }) - } + let span = iastContext?.rootSpan - if (!span) return + if (!span && tracer) { + span = tracer.startSpan('vulnerability', { + type: 'vulnerability' + }) - keepTrace(span, SAMPLING_MECHANISM_APPSEC) - standalone.sample(span) + vulnerability.location.spanId = span.context().toSpanId() - if (iastContext?.rootSpan) { - iastContext[VULNERABILITIES_KEY] = iastContext[VULNERABILITIES_KEY] || [] - iastContext[VULNERABILITIES_KEY].push(vulnerability) - } else { - sendVulnerabilities([vulnerability], span) - span.finish() - } + span.addTags({ + [IAST_ENABLED_TAG_KEY]: 1 + }) + } + + if (!span) return + + keepTrace(span, SAMPLING_MECHANISM_APPSEC) + standalone.sample(span) + + if (stackTraceEnabled && canReportStackTrace(span, maxStackTraces, STACK_TRACE_NAMESPACES.IAST)) { + const originalCallSiteList = callSiteFrames.map(callsite => replaceCallSiteFromSourceMap(callsite)) + + reportStackTrace( + span, + vulnerability.stackId, + originalCallSiteList, + STACK_TRACE_NAMESPACES.IAST + ) + } + + if (iastContext?.rootSpan) { + iastContext[VULNERABILITIES_KEY] = iastContext[VULNERABILITIES_KEY] || [] + iastContext[VULNERABILITIES_KEY].push(vulnerability) + } else { + sendVulnerabilities([vulnerability], span) + span.finish() } } @@ -94,8 +117,34 @@ function isDuplicatedVulnerability (vulnerability) { return VULNERABILITY_HASHES.get(`${vulnerability.type}${vulnerability.hash}`) } +function getVulnerabilityCallSiteFrames () { + return getCallsiteFrames(stackTraceMaxDepth) +} + +function replaceCallSiteFromSourceMap (callsite) { + if (callsite) { + const { path, line, column } = getOriginalPathAndLineFromSourceMap(callsite) + if (path) { + callsite.file = path + callsite.path = path + } + if (line) { + callsite.line = line + } + if (column) { + callsite.column = column + } + } + + return callsite +} + function start (config, _tracer) { deduplicationEnabled = config.iast.deduplicationEnabled + stackTraceEnabled = config.iast.stackTrace.enabled + stackTraceMaxDepth = config.appsec.stackTrace.maxDepth + maxStackTraces = config.appsec.stackTrace.maxStackTraces + vulnerabilitiesFormatter.setRedactVulnerabilities( config.iast.redactionEnabled, config.iast.redactionNamePattern, @@ -114,6 +163,8 @@ function stop () { module.exports = { addVulnerability, sendVulnerabilities, + getVulnerabilityCallSiteFrames, + replaceCallSiteFromSourceMap, clearCache, start, stop diff --git a/packages/dd-trace/src/appsec/rasp/utils.js b/packages/dd-trace/src/appsec/rasp/utils.js index a454a71b8c6..17875c48c7b 100644 --- a/packages/dd-trace/src/appsec/rasp/utils.js +++ b/packages/dd-trace/src/appsec/rasp/utils.js @@ -1,7 +1,7 @@ 'use strict' const web = require('../../plugins/util/web') -const { reportStackTrace } = require('../stack_trace') +const { getCallsiteFrames, reportStackTrace, canReportStackTrace } = require('../stack_trace') const { getBlockingAction } = require('../blocking') const log = require('../../log') @@ -30,13 +30,18 @@ class DatadogRaspAbortError extends Error { function handleResult (actions, req, res, abortController, config) { const generateStackTraceAction = actions?.generate_stack - if (generateStackTraceAction && config.appsec.stackTrace.enabled) { - const rootSpan = web.root(req) + + const { enabled, maxDepth, maxStackTraces } = config.appsec.stackTrace + + const rootSpan = web.root(req) + + if (generateStackTraceAction && enabled && canReportStackTrace(rootSpan, maxStackTraces)) { + const frames = getCallsiteFrames(maxDepth) + reportStackTrace( rootSpan, generateStackTraceAction.stack_id, - config.appsec.stackTrace.maxDepth, - config.appsec.stackTrace.maxStackTraces + frames ) } diff --git a/packages/dd-trace/src/appsec/stack_trace.js b/packages/dd-trace/src/appsec/stack_trace.js index ea49ed1e877..53fc0e27811 100644 --- a/packages/dd-trace/src/appsec/stack_trace.js +++ b/packages/dd-trace/src/appsec/stack_trace.js @@ -6,11 +6,18 @@ const ddBasePath = calculateDDBasePath(__dirname) const LIBRARY_FRAMES_BUFFER = 20 +const STACK_TRACE_NAMESPACES = { + RASP: 'exploit', + IAST: 'vulnerability' +} + function getCallSiteList (maxDepth = 100) { const previousPrepareStackTrace = Error.prepareStackTrace const previousStackTraceLimit = Error.stackTraceLimit let callsiteList - Error.stackTraceLimit = maxDepth + // Since some frames will be discarded because they come from tracer codebase, a buffer is added + // to the limit in order to get as close as `maxDepth` number of frames. + Error.stackTraceLimit = maxDepth + LIBRARY_FRAMES_BUFFER try { Error.prepareStackTrace = function (_, callsites) { @@ -30,7 +37,10 @@ function filterOutFramesFromLibrary (callSiteList) { return callSiteList.filter(callSite => !callSite.getFileName()?.startsWith(ddBasePath)) } -function getFramesForMetaStruct (callSiteList, maxDepth = 32) { +function getCallsiteFrames (maxDepth = 32, callSiteListGetter = getCallSiteList) { + if (maxDepth < 1) maxDepth = Infinity + + const callSiteList = callSiteListGetter(maxDepth) const filteredFrames = filterOutFramesFromLibrary(callSiteList) const half = filteredFrames.length > maxDepth ? Math.round(maxDepth / 2) : Infinity @@ -45,46 +55,46 @@ function getFramesForMetaStruct (callSiteList, maxDepth = 32) { line: callSite.getLineNumber(), column: callSite.getColumnNumber(), function: callSite.getFunctionName(), - class_name: callSite.getTypeName() + class_name: callSite.getTypeName(), + isNative: callSite.isNative() }) } return indexedFrames } -function reportStackTrace (rootSpan, stackId, maxDepth, maxStackTraces, callSiteListGetter = getCallSiteList) { +function reportStackTrace (rootSpan, stackId, frames, namespace = STACK_TRACE_NAMESPACES.RASP) { if (!rootSpan) return + if (!Array.isArray(frames)) return - if (maxStackTraces < 1 || (rootSpan.meta_struct?.['_dd.stack']?.exploit?.length ?? 0) < maxStackTraces) { - // Since some frames will be discarded because they come from tracer codebase, a buffer is added - // to the limit in order to get as close as `maxDepth` number of frames. - if (maxDepth < 1) maxDepth = Infinity - const callSiteList = callSiteListGetter(maxDepth + LIBRARY_FRAMES_BUFFER) - if (!Array.isArray(callSiteList)) return + if (!rootSpan.meta_struct) { + rootSpan.meta_struct = {} + } - if (!rootSpan.meta_struct) { - rootSpan.meta_struct = {} - } + if (!rootSpan.meta_struct['_dd.stack']) { + rootSpan.meta_struct['_dd.stack'] = {} + } - if (!rootSpan.meta_struct['_dd.stack']) { - rootSpan.meta_struct['_dd.stack'] = {} - } + if (!rootSpan.meta_struct['_dd.stack'][namespace]) { + rootSpan.meta_struct['_dd.stack'][namespace] = [] + } - if (!rootSpan.meta_struct['_dd.stack'].exploit) { - rootSpan.meta_struct['_dd.stack'].exploit = [] - } + rootSpan.meta_struct['_dd.stack'][namespace].push({ + id: stackId, + language: 'nodejs', + frames + }) +} - const frames = getFramesForMetaStruct(callSiteList, maxDepth) +function canReportStackTrace (rootSpan, maxStackTraces, namespace = STACK_TRACE_NAMESPACES.RASP) { + if (!rootSpan) return false - rootSpan.meta_struct['_dd.stack'].exploit.push({ - id: stackId, - language: 'nodejs', - frames - }) - } + return maxStackTraces < 1 || (rootSpan.meta_struct?.['_dd.stack']?.[namespace]?.length ?? 0) < maxStackTraces } module.exports = { - getCallSiteList, - reportStackTrace + getCallsiteFrames, + reportStackTrace, + canReportStackTrace, + STACK_TRACE_NAMESPACES } diff --git a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js index 3ad1a11e027..3cbd64afbc2 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js +++ b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js @@ -87,9 +87,8 @@ class CiVisibilityExporter extends AgentInfoExporter { shouldRequestKnownTests () { return !!( - this._config.isEarlyFlakeDetectionEnabled && this._canUseCiVisProtocol && - this._libraryConfig?.isEarlyFlakeDetectionEnabled + this._libraryConfig?.isKnownTestsEnabled ) } @@ -197,7 +196,8 @@ class CiVisibilityExporter extends AgentInfoExporter { earlyFlakeDetectionNumRetries, earlyFlakeDetectionFaultyThreshold, isFlakyTestRetriesEnabled, - isDiEnabled + isDiEnabled, + isKnownTestsEnabled } = remoteConfiguration return { isCodeCoverageEnabled, @@ -209,7 +209,8 @@ class CiVisibilityExporter extends AgentInfoExporter { earlyFlakeDetectionFaultyThreshold, isFlakyTestRetriesEnabled: isFlakyTestRetriesEnabled && this._config.isFlakyTestRetriesEnabled, flakyTestRetriesCount: this._config.flakyTestRetriesCount, - isDiEnabled: isDiEnabled && this._config.isTestDynamicInstrumentationEnabled + isDiEnabled: isDiEnabled && this._config.isTestDynamicInstrumentationEnabled, + isKnownTestsEnabled } } diff --git a/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js b/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js index e39770dea82..26d818bcdd2 100644 --- a/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js +++ b/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js @@ -93,7 +93,8 @@ function getLibraryConfiguration ({ require_git: requireGit, early_flake_detection: earlyFlakeDetectionConfig, flaky_test_retries_enabled: isFlakyTestRetriesEnabled, - di_enabled: isDiEnabled + di_enabled: isDiEnabled, + known_tests_enabled: isKnownTestsEnabled } } } = JSON.parse(res) @@ -103,13 +104,14 @@ function getLibraryConfiguration ({ isSuitesSkippingEnabled, isItrEnabled, requireGit, - isEarlyFlakeDetectionEnabled: earlyFlakeDetectionConfig?.enabled ?? false, + isEarlyFlakeDetectionEnabled: isKnownTestsEnabled && (earlyFlakeDetectionConfig?.enabled ?? false), earlyFlakeDetectionNumRetries: earlyFlakeDetectionConfig?.slow_test_retries?.['5s'] || DEFAULT_EARLY_FLAKE_DETECTION_NUM_RETRIES, earlyFlakeDetectionFaultyThreshold: earlyFlakeDetectionConfig?.faulty_session_threshold ?? DEFAULT_EARLY_FLAKE_DETECTION_ERROR_THRESHOLD, isFlakyTestRetriesEnabled, - isDiEnabled: isDiEnabled && isFlakyTestRetriesEnabled + isDiEnabled: isDiEnabled && isFlakyTestRetriesEnabled, + isKnownTestsEnabled } log.debug(() => `Remote settings: ${JSON.stringify(settings)}`) diff --git a/packages/dd-trace/src/config.js b/packages/dd-trace/src/config.js index 8dd63cccdf6..f529bc635e2 100644 --- a/packages/dd-trace/src/config.js +++ b/packages/dd-trace/src/config.js @@ -497,6 +497,7 @@ class Config { this._setValue(defaults, 'iast.redactionValuePattern', null) this._setValue(defaults, 'iast.requestSampling', 30) this._setValue(defaults, 'iast.telemetryVerbosity', 'INFORMATION') + this._setValue(defaults, 'iast.stackTrace.enabled', true) this._setValue(defaults, 'injectionEnabled', []) this._setValue(defaults, 'isAzureFunction', false) this._setValue(defaults, 'isCiVisibility', false) @@ -622,6 +623,7 @@ class Config { DD_IAST_REDACTION_VALUE_PATTERN, DD_IAST_REQUEST_SAMPLING, DD_IAST_TELEMETRY_VERBOSITY, + DD_IAST_STACK_TRACE_ENABLED, DD_INJECTION_ENABLED, DD_INSTRUMENTATION_TELEMETRY_ENABLED, DD_INSTRUMENTATION_CONFIG_ID, @@ -787,6 +789,7 @@ class Config { } this._envUnprocessed['iast.requestSampling'] = DD_IAST_REQUEST_SAMPLING this._setString(env, 'iast.telemetryVerbosity', DD_IAST_TELEMETRY_VERBOSITY) + this._setBoolean(env, 'iast.stackTrace.enabled', DD_IAST_STACK_TRACE_ENABLED) this._setArray(env, 'injectionEnabled', DD_INJECTION_ENABLED) this._setBoolean(env, 'isAzureFunction', getIsAzureFunction()) this._setBoolean(env, 'isGCPFunction', getIsGCPFunction()) @@ -976,6 +979,7 @@ class Config { this._optsUnprocessed['iast.requestSampling'] = options.iast?.requestSampling } this._setString(opts, 'iast.telemetryVerbosity', options.iast && options.iast.telemetryVerbosity) + this._setBoolean(opts, 'iast.stackTrace.enabled', options.iast?.stackTrace?.enabled) this._setBoolean(opts, 'isCiVisibility', options.isCiVisibility) this._setBoolean(opts, 'legacyBaggageEnabled', options.legacyBaggageEnabled) this._setBoolean(opts, 'llmobs.agentlessEnabled', options.llmobs?.agentlessEnabled) diff --git a/packages/dd-trace/src/llmobs/plugins/bedrockruntime.js b/packages/dd-trace/src/llmobs/plugins/bedrockruntime.js new file mode 100644 index 00000000000..cf74fb15981 --- /dev/null +++ b/packages/dd-trace/src/llmobs/plugins/bedrockruntime.js @@ -0,0 +1,59 @@ +const BaseLLMObsPlugin = require('./base') +const { storage } = require('../../../../datadog-core') +const llmobsStore = storage('llmobs') + +const { + extractRequestParams, + extractTextAndResponseReason, + parseModelId +} = require('../../../../datadog-plugin-aws-sdk/src/services/bedrockruntime/utils') + +const enabledOperations = ['invokeModel'] + +class BedrockRuntimeLLMObsPlugin extends BaseLLMObsPlugin { + constructor () { + super(...arguments) + + this.addSub('apm:aws:request:complete:bedrockruntime', ({ response }) => { + const request = response.request + const operation = request.operation + // avoids instrumenting other non supported runtime operations + if (!enabledOperations.includes(operation)) { + return + } + const { modelProvider, modelName } = parseModelId(request.params.modelId) + + // avoids instrumenting non llm type + if (modelName.includes('embed')) { + return + } + const span = storage.getStore()?.span + this.setLLMObsTags({ request, span, response, modelProvider, modelName }) + }) + } + + setLLMObsTags ({ request, span, response, modelProvider, modelName }) { + const parent = llmobsStore.getStore()?.span + this._tagger.registerLLMObsSpan(span, { + parent, + modelName: modelName.toLowerCase(), + modelProvider: modelProvider.toLowerCase(), + kind: 'llm', + name: 'bedrock-runtime.command' + }) + + const requestParams = extractRequestParams(request.params, modelProvider) + const textAndResponseReason = extractTextAndResponseReason(response, modelProvider, modelName) + + // add metadata tags + this._tagger.tagMetadata(span, { + temperature: parseFloat(requestParams.temperature) || 0.0, + max_tokens: parseInt(requestParams.maxTokens) || 0 + }) + + // add I/O tags + this._tagger.tagLLMIO(span, requestParams.prompt, textAndResponseReason.message) + } +} + +module.exports = BedrockRuntimeLLMObsPlugin diff --git a/packages/dd-trace/src/plugins/ci_plugin.js b/packages/dd-trace/src/plugins/ci_plugin.js index 60c1c59a9bc..287d3e6d55d 100644 --- a/packages/dd-trace/src/plugins/ci_plugin.js +++ b/packages/dd-trace/src/plugins/ci_plugin.js @@ -158,6 +158,7 @@ module.exports = class CiPlugin extends Plugin { if (err) { log.error('Known tests could not be fetched. %s', err.message) this.libraryConfig.isEarlyFlakeDetectionEnabled = false + this.libraryConfig.isKnownTestsEnabled = false } onDone({ err, knownTests }) }) diff --git a/packages/dd-trace/src/plugins/util/inferred_proxy.js b/packages/dd-trace/src/plugins/util/inferred_proxy.js index 54fe2cb761b..83628084ead 100644 --- a/packages/dd-trace/src/plugins/util/inferred_proxy.js +++ b/packages/dd-trace/src/plugins/util/inferred_proxy.js @@ -2,7 +2,6 @@ const log = require('../../log') const tags = require('../../../../../ext/tags') const RESOURCE_NAME = tags.RESOURCE_NAME -const HTTP_ROUTE = tags.HTTP_ROUTE const SPAN_KIND = tags.SPAN_KIND const SPAN_TYPE = tags.SPAN_TYPE const HTTP_URL = tags.HTTP_URL @@ -54,7 +53,6 @@ function createInferredProxySpan (headers, childOf, tracer, context) { [SPAN_TYPE]: 'web', [HTTP_METHOD]: proxyContext.method, [HTTP_URL]: proxyContext.domainName + proxyContext.path, - [HTTP_ROUTE]: proxyContext.path, stage: proxyContext.stage } } diff --git a/packages/dd-trace/src/plugins/util/test.js b/packages/dd-trace/src/plugins/util/test.js index d8aab1a44da..2d8ce1a1d33 100644 --- a/packages/dd-trace/src/plugins/util/test.js +++ b/packages/dd-trace/src/plugins/util/test.js @@ -59,6 +59,7 @@ const TEST_IS_NEW = 'test.is_new' const TEST_IS_RETRY = 'test.is_retry' const TEST_EARLY_FLAKE_ENABLED = 'test.early_flake.enabled' const TEST_EARLY_FLAKE_ABORT_REASON = 'test.early_flake.abort_reason' +const TEST_RETRY_REASON = 'test.retry_reason' const CI_APP_ORIGIN = 'ciapp-test' @@ -145,6 +146,7 @@ module.exports = { TEST_IS_RETRY, TEST_EARLY_FLAKE_ENABLED, TEST_EARLY_FLAKE_ABORT_REASON, + TEST_RETRY_REASON, getTestEnvironmentMetadata, getTestParametersString, finishAllTraceSpans, diff --git a/packages/dd-trace/test/appsec/iast/analyzers/hardcoded-password-analyzer.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/hardcoded-password-analyzer.spec.js index e20c83ef33d..16fe264328c 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/hardcoded-password-analyzer.spec.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/hardcoded-password-analyzer.spec.js @@ -10,6 +10,7 @@ const Config = require('../../../../src/config') const hardcodedPasswordAnalyzer = require('../../../../src/appsec/iast/analyzers/hardcoded-password-analyzer') const iast = require('../../../../src/appsec/iast') +const vulnerabilityReporter = require('../../../../src/appsec/iast/vulnerability-reporter') const ruleId = 'hardcoded-password' const samples = [ @@ -131,6 +132,7 @@ describe('Hardcoded Password Analyzer', () => { afterEach(() => { iast.disable() + vulnerabilityReporter.clearCache() }) afterEach(() => { diff --git a/packages/dd-trace/test/appsec/iast/analyzers/hardcoded-secret-analyzer.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/hardcoded-secret-analyzer.spec.js index 67d00a8b53a..b65aed0a614 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/hardcoded-secret-analyzer.spec.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/hardcoded-secret-analyzer.spec.js @@ -11,6 +11,7 @@ const { NameAndValue, ValueOnly } = require('../../../../src/appsec/iast/analyze const hardcodedSecretAnalyzer = require('../../../../src/appsec/iast/analyzers/hardcoded-secret-analyzer') const { suite } = require('./resources/hardcoded-secrets-suite.json') const iast = require('../../../../src/appsec/iast') +const vulnerabilityReporter = require('../../../../src/appsec/iast/vulnerability-reporter') describe('Hardcoded Secret Analyzer', () => { describe('unit test', () => { @@ -101,6 +102,7 @@ describe('Hardcoded Secret Analyzer', () => { afterEach(() => { iast.disable() + vulnerabilityReporter.clearCache() }) afterEach(() => { diff --git a/packages/dd-trace/test/appsec/iast/analyzers/vulnerability-analyzer.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/vulnerability-analyzer.spec.js index b47fb95b81b..cdb7e8cc4e2 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/vulnerability-analyzer.spec.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/vulnerability-analyzer.spec.js @@ -6,25 +6,20 @@ const proxyquire = require('proxyquire') describe('vulnerability-analyzer', () => { const VULNERABLE_VALUE = 'VULNERABLE_VALUE' const VULNERABILITY = 'VULNERABILITY' - const VULNERABILITY_LOCATION = { path: 'VULNERABILITY_LOCATION', line: 11 } - const VULNERABILITY_LOCATION_FROM_SOURCEMAP = { path: 'VULNERABILITY_LOCATION_FROM_SOURCEMAP', line: 42 } + const VULNERABILITY_LOCATION_FROM_SOURCEMAP = { path: 'VULNERABILITY_LOCATION_FROM_SOURCEMAP', line: 42, column: 21 } const ANALYZER_TYPE = 'TEST_ANALYZER' const SPAN_ID = '123456' let VulnerabilityAnalyzer let vulnerabilityReporter let overheadController - let pathLine let iastContextHandler - let rewriter beforeEach(() => { vulnerabilityReporter = { createVulnerability: sinon.stub().returns(VULNERABILITY), - addVulnerability: sinon.stub() - } - pathLine = { - getFirstNonDDPathAndLine: sinon.stub().returns(VULNERABILITY_LOCATION) + addVulnerability: sinon.stub(), + replaceCallSiteFromSourceMap: sinon.stub().returns(VULNERABILITY_LOCATION_FROM_SOURCEMAP) } overheadController = { hasQuota: sinon.stub() @@ -32,16 +27,11 @@ describe('vulnerability-analyzer', () => { iastContextHandler = { getIastContext: sinon.stub() } - rewriter = { - getOriginalPathAndLineFromSourceMap: sinon.stub().returns(VULNERABILITY_LOCATION_FROM_SOURCEMAP) - } VulnerabilityAnalyzer = proxyquire('../../../../src/appsec/iast/analyzers/vulnerability-analyzer', { '../vulnerability-reporter': vulnerabilityReporter, - '../path-line': pathLine, '../overhead-controller': overheadController, - '../iast-context': iastContextHandler, - '../taint-tracking/rewriter': rewriter + '../iast-context': iastContextHandler }) }) @@ -120,16 +110,17 @@ describe('vulnerability-analyzer', () => { context, { type: 'TEST_ANALYZER', + stackId: 1, evidence: { value: 'VULNERABLE_VALUE' }, location: { spanId: '123456', - path: 'VULNERABILITY_LOCATION_FROM_SOURCEMAP', - line: 42 + ...VULNERABILITY_LOCATION_FROM_SOURCEMAP }, hash: 5975567724 - } + }, + sinon.match.array ) }) @@ -160,7 +151,6 @@ describe('vulnerability-analyzer', () => { VulnerabilityAnalyzer = proxyquire('../../../../src/appsec/iast/analyzers/vulnerability-analyzer', { '../vulnerability-reporter': vulnerabilityReporter, - '../path-line': pathLine, '../overhead-controller': overheadController, '../iast-context': iastContextHandler, '../iast-plugin': { @@ -285,7 +275,7 @@ describe('vulnerability-analyzer', () => { ANALYZER_TYPE, { value: 'test' }, SPAN_ID, - VULNERABILITY_LOCATION + VULNERABILITY_LOCATION_FROM_SOURCEMAP ) }) }) diff --git a/packages/dd-trace/test/appsec/iast/path-line.spec.js b/packages/dd-trace/test/appsec/iast/path-line.spec.js index 11905bcb880..eee98c31ef9 100644 --- a/packages/dd-trace/test/appsec/iast/path-line.spec.js +++ b/packages/dd-trace/test/appsec/iast/path-line.spec.js @@ -2,27 +2,16 @@ const proxyquire = require('proxyquire') const path = require('path') const os = require('os') const { expect } = require('chai') +const { getCallsiteFrames } = require('../../../src/appsec/stack_trace') class CallSiteMock { constructor (fileName, lineNumber, columnNumber = 0) { - this.fileName = fileName - this.lineNumber = lineNumber - this.columnNumber = columnNumber + this.file = fileName + this.line = lineNumber + this.column = columnNumber } - getLineNumber () { - return this.lineNumber - } - - getColumnNumber () { - return this.columnNumber - } - - getFileName () { - return this.fileName - } - - isNative () { + get isNative () { return false } } @@ -50,13 +39,6 @@ describe('path-line', function () { }) }) - describe('getFirstNonDDPathAndLine', () => { - it('call does not fail', () => { - const obj = pathLine.getFirstNonDDPathAndLine() - expect(obj).to.not.be.null - }) - }) - describe('calculateDDBasePath', () => { it('/node_modules/dd-trace', () => { const basePath = path.join(rootPath, 'node_modules', 'dd-trace', 'packages', path.sep) @@ -78,18 +60,21 @@ describe('path-line', function () { }) }) - describe('getFirstNonDDPathAndLineFromCallsites', () => { + describe('getNonDDCallSiteFrames', () => { describe('does not fail', () => { it('with null parameter', () => { - pathLine.getFirstNonDDPathAndLineFromCallsites(null) + const result = pathLine.getNonDDCallSiteFrames(null) + expect(result).to.be.an('array').that.is.empty }) it('with empty list parameter', () => { - pathLine.getFirstNonDDPathAndLineFromCallsites([]) + const result = pathLine.getNonDDCallSiteFrames([]) + expect(result).to.be.an('array').that.is.empty }) it('without parameter', () => { - pathLine.getFirstNonDDPathAndLineFromCallsites() + const result = pathLine.getNonDDCallSiteFrames() + expect(result).to.be.an('array').that.is.empty }) }) @@ -110,52 +95,65 @@ describe('path-line', function () { pathLine.ddBasePath = prevDDBasePath }) - it('should return first non DD library when two stack are in dd-trace files and the next is the client line', - () => { - const callsites = [] - const expectedFirstFileOutOfDD = path.join('first', 'file', 'out', 'of', 'dd.js') - const firstFileOutOfDD = path.join(PROJECT_PATH, expectedFirstFileOutOfDD) - const firstFileOutOfDDLineNumber = 13 + it('should return all no DD entries when multiple stack frames are present', () => { + const callsites = [] + const expectedFilePaths = [ + path.join('first', 'file', 'out', 'of', 'dd.js'), + path.join('second', 'file', 'out', 'of', 'dd.js') + ] + const firstFileOutOfDD = path.join(PROJECT_PATH, expectedFilePaths[0]) + const secondFileOutOfDD = path.join(PROJECT_PATH, expectedFilePaths[1]) - callsites.push(new CallSiteMock(PATH_AND_LINE_PATH, PATH_AND_LINE_LINE)) - callsites.push(new CallSiteMock(path.join(DD_BASE_PATH, 'other', 'file', 'in', 'dd.js'), 89)) - callsites.push(new CallSiteMock(path.join(DD_BASE_PATH, 'other', 'file', 'in', 'dd.js'), 5)) - callsites.push(new CallSiteMock(firstFileOutOfDD, firstFileOutOfDDLineNumber, 42)) - const pathAndLine = pathLine.getFirstNonDDPathAndLineFromCallsites(callsites) - expect(pathAndLine.path).to.be.equals(expectedFirstFileOutOfDD) - expect(pathAndLine.line).to.be.equals(firstFileOutOfDDLineNumber) - expect(pathAndLine.column).to.be.equals(42) - }) + callsites.push(new CallSiteMock(PATH_AND_LINE_PATH, PATH_AND_LINE_LINE)) + callsites.push(new CallSiteMock(path.join(DD_BASE_PATH, 'other', 'file', 'in', 'dd.js'), 89)) + callsites.push(new CallSiteMock(firstFileOutOfDD, 13, 42)) + callsites.push(new CallSiteMock(secondFileOutOfDD, 20, 15)) + + const results = pathLine.getNonDDCallSiteFrames(callsites) + + expect(results).to.have.lengthOf(2) + + expect(results[0].path).to.be.equals(expectedFilePaths[0]) + expect(results[0].line).to.be.equals(13) + expect(results[0].column).to.be.equals(42) + + expect(results[1].path).to.be.equals(expectedFilePaths[1]) + expect(results[1].line).to.be.equals(20) + expect(results[1].column).to.be.equals(15) + }) - it('should return null when all stack is in dd trace', () => { + it('should return an empty array when all stack frames are in dd trace', () => { const callsites = [] callsites.push(new CallSiteMock(PATH_AND_LINE_PATH, PATH_AND_LINE_LINE)) callsites.push(new CallSiteMock(path.join(DD_BASE_PATH, 'other', 'file', 'in', 'dd.js'), 89)) - callsites.push(new CallSiteMock(path.join(DD_BASE_PATH, 'other', 'file', 'in', 'dd.js'), 5)) - const pathAndLine = pathLine.getFirstNonDDPathAndLineFromCallsites(callsites) - expect(pathAndLine).to.be.null + callsites.push(new CallSiteMock(path.join(DD_BASE_PATH, 'another', 'file', 'in', 'dd.js'), 5)) + + const results = pathLine.getNonDDCallSiteFrames(callsites) + expect(results).to.be.an('array').that.is.empty }) DIAGNOSTICS_CHANNEL_PATHS.forEach((dcPath) => { - it(`should not return ${dcPath} path`, () => { + it(`should exclude ${dcPath} from the results`, () => { const callsites = [] - const expectedFirstFileOutOfDD = path.join('first', 'file', 'out', 'of', 'dd.js') - const firstFileOutOfDD = path.join(PROJECT_PATH, expectedFirstFileOutOfDD) - const firstFileOutOfDDLineNumber = 13 + const expectedFilePath = path.join('first', 'file', 'out', 'of', 'dd.js') + const firstFileOutOfDD = path.join(PROJECT_PATH, expectedFilePath) + callsites.push(new CallSiteMock(PATH_AND_LINE_PATH, PATH_AND_LINE_LINE)) callsites.push(new CallSiteMock(path.join(DD_BASE_PATH, 'other', 'file', 'in', 'dd.js'), 89)) callsites.push(new CallSiteMock(dcPath, 25)) - callsites.push(new CallSiteMock(path.join(DD_BASE_PATH, 'other', 'file', 'in', 'dd.js'), 5)) - callsites.push(new CallSiteMock(firstFileOutOfDD, firstFileOutOfDDLineNumber, 42)) - const pathAndLine = pathLine.getFirstNonDDPathAndLineFromCallsites(callsites) - expect(pathAndLine.path).to.be.equals(expectedFirstFileOutOfDD) - expect(pathAndLine.line).to.be.equals(firstFileOutOfDDLineNumber) - expect(pathAndLine.column).to.be.equals(42) + callsites.push(new CallSiteMock(firstFileOutOfDD, 13, 42)) + + const results = pathLine.getNonDDCallSiteFrames(callsites) + expect(results).to.have.lengthOf(1) + + expect(results[0].path).to.be.equals(expectedFilePath) + expect(results[0].line).to.be.equals(13) + expect(results[0].column).to.be.equals(42) }) }) }) - describe('dd-trace is in other directory', () => { + describe('dd-trace is in another directory', () => { const PROJECT_PATH = path.join(tmpdir, 'project-path') const DD_BASE_PATH = path.join(tmpdir, 'dd-tracer-path') const PATH_AND_LINE_PATH = path.join(DD_BASE_PATH, 'packages', @@ -173,37 +171,30 @@ describe('path-line', function () { pathLine.ddBasePath = previousDDBasePath }) - it('two in dd-trace files and the next is the client line', () => { + it('should return all non-DD entries', () => { const callsites = [] - const expectedFilePath = path.join('first', 'file', 'out', 'of', 'dd.js') - const firstFileOutOfDD = path.join(PROJECT_PATH, expectedFilePath) - const firstFileOutOfDDLineNumber = 13 + const expectedFilePaths = [ + path.join('first', 'file', 'out', 'of', 'dd.js'), + path.join('second', 'file', 'out', 'of', 'dd.js') + ] + const firstFileOutOfDD = path.join(PROJECT_PATH, expectedFilePaths[0]) + const secondFileOutOfDD = path.join(PROJECT_PATH, expectedFilePaths[1]) + callsites.push(new CallSiteMock(PATH_AND_LINE_PATH, PATH_AND_LINE_LINE)) callsites.push(new CallSiteMock(path.join(DD_BASE_PATH, 'other', 'file', 'in', 'dd.js'), 89)) - callsites.push(new CallSiteMock(path.join(DD_BASE_PATH, 'other', 'file', 'in', 'dd.js'), 5)) - callsites.push(new CallSiteMock(firstFileOutOfDD, firstFileOutOfDDLineNumber, 42)) - const pathAndLine = pathLine.getFirstNonDDPathAndLineFromCallsites(callsites) - expect(pathAndLine.path).to.be.equals(expectedFilePath) - expect(pathAndLine.line).to.be.equals(firstFileOutOfDDLineNumber) - expect(pathAndLine.column).to.be.equals(42) - }) + callsites.push(new CallSiteMock(firstFileOutOfDD, 13, 42)) + callsites.push(new CallSiteMock(secondFileOutOfDD, 20, 15)) - DIAGNOSTICS_CHANNEL_PATHS.forEach((dcPath) => { - it(`should not return ${dcPath} path`, () => { - const callsites = [] - const expectedFilePath = path.join('first', 'file', 'out', 'of', 'dd.js') - const firstFileOutOfDD = path.join(PROJECT_PATH, expectedFilePath) - const firstFileOutOfDDLineNumber = 13 - callsites.push(new CallSiteMock(PATH_AND_LINE_PATH, PATH_AND_LINE_LINE)) - callsites.push(new CallSiteMock(path.join(DD_BASE_PATH, 'other', 'file', 'in', 'dd.js'), 89)) - callsites.push(new CallSiteMock(dcPath, 25)) - callsites.push(new CallSiteMock(path.join(DD_BASE_PATH, 'other', 'file', 'in', 'dd.js'), 5)) - callsites.push(new CallSiteMock(firstFileOutOfDD, firstFileOutOfDDLineNumber, 42)) - const pathAndLine = pathLine.getFirstNonDDPathAndLineFromCallsites(callsites) - expect(pathAndLine.path).to.be.equals(expectedFilePath) - expect(pathAndLine.line).to.be.equals(firstFileOutOfDDLineNumber) - expect(pathAndLine.column).to.be.equals(42) - }) + const results = pathLine.getNonDDCallSiteFrames(callsites) + expect(results).to.have.lengthOf(2) + + expect(results[0].path).to.be.equals(expectedFilePaths[0]) + expect(results[0].line).to.be.equals(13) + expect(results[0].column).to.be.equals(42) + + expect(results[1].path).to.be.equals(expectedFilePaths[1]) + expect(results[1].line).to.be.equals(20) + expect(results[1].column).to.be.equals(15) }) }) }) @@ -221,6 +212,7 @@ describe('path-line', function () { e.stack Error.prepareStackTrace = previousPrepareStackTrace Error.stackTraceLimit = previousStackTraceLimit + return callsiteList } @@ -228,11 +220,13 @@ describe('path-line', function () { const basePath = pathLine.ddBasePath pathLine.ddBasePath = path.join('test', 'base', 'path') - const list = getCallSiteInfo() - const firstNonDDPath = pathLine.getFirstNonDDPathAndLineFromCallsites(list) + const list = getCallsiteFrames(32, getCallSiteInfo) + const firstNonDDPath = pathLine.getNonDDCallSiteFrames(list)[0] + + const expectedPath = path.join('node_modules', firstNonDDPath.path) + const nodeModulesPaths = pathLine.getNodeModulesPaths(firstNonDDPath.path) - const nodeModulesPaths = pathLine.getNodeModulesPaths(__filename) - expect(nodeModulesPaths[0]).to.eq(path.join('node_modules', process.cwd(), firstNonDDPath.path)) + expect(nodeModulesPaths[0]).to.equal(expectedPath) pathLine.ddBasePath = basePath }) diff --git a/packages/dd-trace/test/appsec/iast/utils.js b/packages/dd-trace/test/appsec/iast/utils.js index 01274dd954e..5597788bd9d 100644 --- a/packages/dd-trace/test/appsec/iast/utils.js +++ b/packages/dd-trace/test/appsec/iast/utils.js @@ -3,12 +3,14 @@ const fs = require('fs') const os = require('os') const path = require('path') +const { assert } = require('chai') const agent = require('../../plugins/agent') const axios = require('axios') const iast = require('../../../src/appsec/iast') const Config = require('../../../src/config') const vulnerabilityReporter = require('../../../src/appsec/iast/vulnerability-reporter') +const { getWebSpan } = require('../utils') function testInRequest (app, tests) { let http @@ -161,6 +163,10 @@ function checkVulnerabilityInRequest (vulnerability, occurrencesAndLocation, cb, .use(traces => { expect(traces[0][0].metrics['_dd.iast.enabled']).to.be.equal(1) expect(traces[0][0].meta).to.have.property('_dd.iast.json') + + const span = getWebSpan(traces) + assert.property(span.meta_struct, '_dd.stack') + const vulnerabilitiesTrace = JSON.parse(traces[0][0].meta['_dd.iast.json']) expect(vulnerabilitiesTrace).to.not.be.null const vulnerabilitiesCount = new Map() diff --git a/packages/dd-trace/test/appsec/iast/vulnerability-reporter.spec.js b/packages/dd-trace/test/appsec/iast/vulnerability-reporter.spec.js index 2ebe646a2d8..9cf28bdac32 100644 --- a/packages/dd-trace/test/appsec/iast/vulnerability-reporter.spec.js +++ b/packages/dd-trace/test/appsec/iast/vulnerability-reporter.spec.js @@ -28,12 +28,12 @@ describe('vulnerability-reporter', () => { describe('with rootSpan', () => { let iastContext = { - rootSpan: true + rootSpan: {} } afterEach(() => { iastContext = { - rootSpan: true + rootSpan: {} } }) @@ -47,27 +47,27 @@ describe('vulnerability-reporter', () => { it('should create vulnerability array if it does not exist', () => { addVulnerability(iastContext, - vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, 888)) + vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, 888), []) expect(iastContext).to.have.property('vulnerabilities') expect(iastContext.vulnerabilities).to.be.an('array') }) it('should deduplicate same vulnerabilities', () => { addVulnerability(iastContext, - vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, -555)) + vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, -555), []) addVulnerability(iastContext, - vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, 888)) + vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, 888), []) addVulnerability(iastContext, - vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, 123)) + vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, 123), []) expect(iastContext.vulnerabilities).to.have.length(1) }) it('should add in the context evidence properties', () => { addVulnerability(iastContext, - vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, 888)) + vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, 888), []) addVulnerability(iastContext, vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'md5' }, - -123, { path: 'path.js', line: 12 })) + -123, { path: 'path.js', line: 12 }), []) expect(iastContext.vulnerabilities).to.have.length(2) expect(iastContext).to.have.nested.property('vulnerabilities.0.type', 'INSECURE_HASHING') expect(iastContext).to.have.nested.property('vulnerabilities.0.evidence.value', 'sha1') @@ -106,7 +106,17 @@ describe('vulnerability-reporter', () => { } start({ iast: { - deduplicationEnabled: true + deduplicationEnabled: true, + stackTrace: { + enabled: true + } + }, + appsec: { + stackTrace: { + enabled: true, + maxStackTraces: 2, + maxDepth: 42 + } } }, fakeTracer) }) @@ -119,15 +129,15 @@ describe('vulnerability-reporter', () => { it('should create span on the fly', () => { const vulnerability = vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, undefined, - { path: 'filename.js', line: 73 }) - addVulnerability(undefined, vulnerability) + { path: 'filename.js', line: 73 }, 1) + addVulnerability(undefined, vulnerability, []) expect(fakeTracer.startSpan).to.have.been.calledOnceWithExactly('vulnerability', { type: 'vulnerability' }) expect(onTheFlySpan.addTags.firstCall).to.have.been.calledWithExactly({ '_dd.iast.enabled': 1 }) expect(onTheFlySpan.addTags.secondCall).to.have.been.calledWithExactly({ '_dd.iast.json': '{"sources":[],"vulnerabilities":[{"type":"INSECURE_HASHING","hash":3410512655,' + - '"evidence":{"value":"sha1"},"location":{"spanId":42,"path":"filename.js","line":73}}]}' + '"stackId":1,"evidence":{"value":"sha1"},"location":{"spanId":42,"path":"filename.js","line":73}}]}' }) expect(prioritySampler.setPriority) .to.have.been.calledOnceWithExactly(onTheFlySpan, USER_KEEP, SAMPLING_MECHANISM_APPSEC) @@ -138,12 +148,108 @@ describe('vulnerability-reporter', () => { const vulnerability = vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, undefined, { path: 'filename.js', line: 73 }) - addVulnerability(undefined, vulnerability) + addVulnerability(undefined, vulnerability, []) expect(vulnerability.location.spanId).to.be.equal(42) }) }) }) + describe('with maxStackTraces limit', () => { + let iastContext, vulnerability, callSiteFrames + + beforeEach(() => { + iastContext = { + rootSpan: { + meta_struct: { + '_dd.stack': {} + } + } + } + vulnerability = vulnerabilityAnalyzer._createVulnerability( + 'INSECURE_HASHING', + { value: 'sha1' }, + 888, + { path: 'test.js', line: 1 } + ) + callSiteFrames = [{ + getFileName: () => 'test.js', + getLineNumber: () => 1 + }] + }) + + afterEach(() => { + stop() + }) + + it('should report stack trace when under maxStackTraces limit', () => { + start({ + iast: { + deduplicationEnabled: true, + stackTrace: { + enabled: true + } + }, + appsec: { + stackTrace: { + enabled: true, + maxStackTraces: 2, + maxDepth: 42 + } + } + }) + addVulnerability(iastContext, vulnerability, callSiteFrames) + + expect(iastContext.rootSpan.meta_struct['_dd.stack'].vulnerability).to.have.length(1) + }) + + it('should not report stack trace when at maxStackTraces limit', () => { + start({ + iast: { + deduplicationEnabled: true, + stackTrace: { + enabled: true + } + }, + appsec: { + stackTrace: { + enabled: true, + maxStackTraces: 1, + maxDepth: 42 + } + } + }) + iastContext.rootSpan.meta_struct['_dd.stack'].vulnerability = ['existing_stack'] + + addVulnerability(iastContext, vulnerability, callSiteFrames) + + expect(iastContext.rootSpan.meta_struct['_dd.stack'].vulnerability).to.have.length(1) + expect(iastContext.rootSpan.meta_struct['_dd.stack'].vulnerability[0]).to.equal('existing_stack') + }) + + it('should always report stack trace when maxStackTraces is 0', () => { + start({ + iast: { + deduplicationEnabled: true, + stackTrace: { + enabled: true + } + }, + appsec: { + stackTrace: { + enabled: true, + maxStackTraces: 0, + maxDepth: 42 + } + } + }) + iastContext.rootSpan.meta_struct['_dd.stack'].vulnerability = ['stack1', 'stack2'] + + addVulnerability(iastContext, vulnerability, callSiteFrames) + + expect(iastContext.rootSpan.meta_struct['_dd.stack'].vulnerability).to.have.length(3) + }) + }) + describe('sendVulnerabilities', () => { let span let context @@ -161,7 +267,17 @@ describe('vulnerability-reporter', () => { } start({ iast: { - deduplicationEnabled: true + deduplicationEnabled: true, + stackTrace: { + enabled: true + } + }, + appsec: { + stackTrace: { + enabled: true, + maxStackTraces: 2, + maxDepth: 42 + } } }) }) @@ -187,7 +303,7 @@ describe('vulnerability-reporter', () => { it('should send one with one vulnerability', () => { const iastContext = { rootSpan: span } addVulnerability(iastContext, - vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, 888)) + vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, 888), []) sendVulnerabilities(iastContext.vulnerabilities, span) expect(span.addTags).to.have.been.calledOnceWithExactly({ '_dd.iast.json': '{"sources":[],"vulnerabilities":[{"type":"INSECURE_HASHING","hash":3254801297,' + @@ -199,7 +315,7 @@ describe('vulnerability-reporter', () => { it('should send only valid vulnerabilities', () => { const iastContext = { rootSpan: span } addVulnerability(iastContext, - vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, 888)) + vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, 888), []) iastContext.vulnerabilities.push({ invalid: 'vulnerability' }) sendVulnerabilities(iastContext.vulnerabilities, span) expect(span.addTags).to.have.been.calledOnceWithExactly({ @@ -227,7 +343,8 @@ describe('vulnerability-reporter', () => { } addVulnerability( iastContext, - vulnerabilityAnalyzer._createVulnerability('SQL_INJECTION', evidence1, 888, { path: 'filename.js', line: 88 }) + vulnerabilityAnalyzer._createVulnerability('SQL_INJECTION', evidence1, 888, { path: 'filename.js', line: 88 }), + [] ) const evidence2 = { @@ -246,7 +363,8 @@ describe('vulnerability-reporter', () => { } addVulnerability( iastContext, - vulnerabilityAnalyzer._createVulnerability('SQL_INJECTION', evidence2, 888, { path: 'filename.js', line: 99 }) + vulnerabilityAnalyzer._createVulnerability('SQL_INJECTION', evidence2, 888, { path: 'filename.js', line: 99 }), + [] ) sendVulnerabilities(iastContext.vulnerabilities, span) @@ -286,7 +404,8 @@ describe('vulnerability-reporter', () => { } addVulnerability( iastContext, - vulnerabilityAnalyzer._createVulnerability('SQL_INJECTION', evidence1, 888, { path: 'filename.js', line: 88 }) + vulnerabilityAnalyzer._createVulnerability('SQL_INJECTION', evidence1, 888, { path: 'filename.js', line: 88 }), + [] ) const evidence2 = { @@ -305,7 +424,8 @@ describe('vulnerability-reporter', () => { } addVulnerability( iastContext, - vulnerabilityAnalyzer._createVulnerability('SQL_INJECTION', evidence2, 888, { path: 'filename.js', line: 99 }) + vulnerabilityAnalyzer._createVulnerability('SQL_INJECTION', evidence2, 888, { path: 'filename.js', line: 99 }), + [] ) sendVulnerabilities(iastContext.vulnerabilities, span) @@ -329,11 +449,11 @@ describe('vulnerability-reporter', () => { it('should send once with multiple vulnerabilities', () => { const iastContext = { rootSpan: span } addVulnerability(iastContext, vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, - 888, { path: '/path/to/file1.js', line: 1 })) + 888, { path: '/path/to/file1.js', line: 1 }), []) addVulnerability(iastContext, vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'md5' }, 1, - { path: '/path/to/file2.js', line: 1 })) + { path: '/path/to/file2.js', line: 1 }), []) addVulnerability(iastContext, vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'md5' }, -5, - { path: '/path/to/file3.js', line: 3 })) + { path: '/path/to/file3.js', line: 3 }), []) sendVulnerabilities(iastContext.vulnerabilities, span) expect(span.addTags).to.have.been.calledOnceWithExactly({ '_dd.iast.json': '{"sources":[],"vulnerabilities":[' + @@ -357,7 +477,7 @@ describe('vulnerability-reporter', () => { const iastContext = { rootSpan: span } addVulnerability(iastContext, vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, 888, - { path: 'filename.js', line: 88 })) + { path: 'filename.js', line: 88 }), []) sendVulnerabilities(iastContext.vulnerabilities, span) expect(span.addTags).to.have.been.calledOnceWithExactly({ '_dd.iast.json': '{"sources":[],"vulnerabilities":[{"type":"INSECURE_HASHING","hash":3410512691,' + @@ -370,10 +490,10 @@ describe('vulnerability-reporter', () => { const iastContext = { rootSpan: span } addVulnerability(iastContext, vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, 888, - { path: 'filename.js', line: 88 })) + { path: 'filename.js', line: 88 }), []) addVulnerability(iastContext, vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, 888, - { path: 'filename.js', line: 88 })) + { path: 'filename.js', line: 88 }), []) sendVulnerabilities(iastContext.vulnerabilities, span) expect(span.addTags).to.have.been.calledOnceWithExactly({ '_dd.iast.json': '{"sources":[],"vulnerabilities":[{"type":"INSECURE_HASHING","hash":3410512691,' + @@ -385,14 +505,24 @@ describe('vulnerability-reporter', () => { it('should not deduplicate vulnerabilities if not enabled', () => { start({ iast: { - deduplicationEnabled: false + deduplicationEnabled: false, + stackTrace: { + enabled: true + } + }, + appsec: { + stackTrace: { + enabled: true, + maxStackTraces: 2, + maxDepth: 42 + } } }) const iastContext = { rootSpan: span } addVulnerability(iastContext, vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', - { value: 'sha1' }, 888, { path: 'filename.js', line: 88 })) + { value: 'sha1' }, 888, { path: 'filename.js', line: 88 }), []) addVulnerability(iastContext, vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', - { value: 'sha1' }, 888, { path: 'filename.js', line: 88 })) + { value: 'sha1' }, 888, { path: 'filename.js', line: 88 }), []) sendVulnerabilities(iastContext.vulnerabilities, span) expect(span.addTags).to.have.been.calledOnceWithExactly({ '_dd.iast.json': '{"sources":[],"vulnerabilities":[{"type":"INSECURE_HASHING","hash":3410512691,' + @@ -411,7 +541,7 @@ describe('vulnerability-reporter', () => { appsecStandalone.configure({ appsec: { standalone: { enabled: true } } }) const iastContext = { rootSpan: span } addVulnerability(iastContext, - vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, 999)) + vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, 999), []) sendVulnerabilities(iastContext.vulnerabilities, span) @@ -429,7 +559,7 @@ describe('vulnerability-reporter', () => { appsecStandalone.configure({ appsec: {} }) const iastContext = { rootSpan: span } addVulnerability(iastContext, - vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, 999)) + vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, 999), []) sendVulnerabilities(iastContext.vulnerabilities, span) @@ -477,18 +607,18 @@ describe('vulnerability-reporter', () => { const MAX = 1000 const vulnerabilityToRepeatInTheNext = vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, 888, - { path: 'filename.js', line: 0 }) - addVulnerability(iastContext, vulnerabilityToRepeatInTheNext) + { path: 'filename.js', line: 0 }, 1) + addVulnerability(iastContext, vulnerabilityToRepeatInTheNext, []) for (let i = 1; i <= MAX; i++) { addVulnerability(iastContext, vulnerabilityAnalyzer._createVulnerability('INSECURE_HASHING', { value: 'sha1' }, 888, - { path: 'filename.js', line: i })) + { path: 'filename.js', line: i }), []) } sendVulnerabilities(iastContext.vulnerabilities, span) expect(span.addTags).to.have.been.calledOnce const nextIastContext = { rootSpan: span } - addVulnerability(nextIastContext, vulnerabilityToRepeatInTheNext) + addVulnerability(nextIastContext, vulnerabilityToRepeatInTheNext, []) sendVulnerabilities(nextIastContext.vulnerabilities, span) expect(span.addTags).to.have.been.calledTwice }) @@ -496,7 +626,17 @@ describe('vulnerability-reporter', () => { it('should set timer to clear cache every hour if deduplication is enabled', () => { const config = { iast: { - deduplicationEnabled: true + deduplicationEnabled: true, + stackTrace: { + enabled: true + } + }, + appsec: { + stackTrace: { + enabled: true, + maxStackTraces: 2, + maxDepth: 42 + } } } start(config) @@ -506,7 +646,17 @@ describe('vulnerability-reporter', () => { it('should not set timer to clear cache every hour if deduplication is not enabled', () => { const config = { iast: { - deduplicationEnabled: false + deduplicationEnabled: false, + stackTrace: { + enabled: true + } + }, + appsec: { + stackTrace: { + enabled: true, + maxStackTraces: 2, + maxDepth: 42 + } } } start(config) @@ -516,7 +666,17 @@ describe('vulnerability-reporter', () => { it('should unset timer to clear cache every hour', () => { const config = { iast: { - deduplicationEnabled: true + deduplicationEnabled: true, + stackTrace: { + enabled: true + } + }, + appsec: { + stackTrace: { + enabled: true, + maxStackTraces: 2, + maxDepth: 42 + } } } start(config) @@ -541,7 +701,17 @@ describe('vulnerability-reporter', () => { iast: { redactionEnabled: true, redactionNamePattern: null, - redactionValuePattern: null + redactionValuePattern: null, + stackTrace: { + enabled: true + } + }, + appsec: { + stackTrace: { + enabled: true, + maxStackTraces: 2, + maxDepth: 42 + } } } start(config) diff --git a/packages/dd-trace/test/appsec/index.next.plugin.spec.js b/packages/dd-trace/test/appsec/index.next.plugin.spec.js index 38cac8f375c..de711c5ff94 100644 --- a/packages/dd-trace/test/appsec/index.next.plugin.spec.js +++ b/packages/dd-trace/test/appsec/index.next.plugin.spec.js @@ -8,22 +8,15 @@ const { writeFileSync } = require('fs') const { satisfies } = require('semver') const path = require('path') -const { DD_MAJOR, NODE_MAJOR } = require('../../../../version') const agent = require('../plugins/agent') -const BUILD_COMMAND = NODE_MAJOR < 18 - ? 'yarn exec next build' - : 'NODE_OPTIONS=--openssl-legacy-provider yarn exec next build' -let VERSIONS_TO_TEST = NODE_MAJOR < 18 ? '>=11.1 <13.2' : '>=11.1' -VERSIONS_TO_TEST = DD_MAJOR >= 4 ? VERSIONS_TO_TEST : '>=9.5 <11.1' - describe('test suite', () => { let server let port const satisfiesStandalone = version => satisfies(version, '>=12.0.0') - withVersions('next', 'next', VERSIONS_TO_TEST, version => { + withVersions('next', 'next', '>=11.1', version => { const realVersion = require(`../../../../versions/next@${version}`).version() function initApp (appName) { @@ -58,7 +51,7 @@ describe('test suite', () => { } // building in-process makes tests fail for an unknown reason - execSync(BUILD_COMMAND, { + execSync('NODE_OPTIONS=--openssl-legacy-provider yarn exec next build', { cwd, env: { ...process.env, diff --git a/packages/dd-trace/test/appsec/rasp/utils.js b/packages/dd-trace/test/appsec/rasp/utils.js index 0d8a3e076a4..b8834afb468 100644 --- a/packages/dd-trace/test/appsec/rasp/utils.js +++ b/packages/dd-trace/test/appsec/rasp/utils.js @@ -1,17 +1,7 @@ 'use strict' const { assert } = require('chai') - -function getWebSpan (traces) { - for (const trace of traces) { - for (const span of trace) { - if (span.type === 'web') { - return span - } - } - } - throw new Error('web span not found') -} +const { getWebSpan } = require('../utils') function checkRaspExecutedAndNotThreat (agent, checkRuleEval = true) { return agent.use((traces) => { @@ -39,7 +29,6 @@ function checkRaspExecutedAndHasThreat (agent, ruleId, ruleEvalCount = 1) { } module.exports = { - getWebSpan, checkRaspExecutedAndNotThreat, checkRaspExecutedAndHasThreat } diff --git a/packages/dd-trace/test/appsec/rasp/utils.spec.js b/packages/dd-trace/test/appsec/rasp/utils.spec.js index 255f498a117..6a74c07444d 100644 --- a/packages/dd-trace/test/appsec/rasp/utils.spec.js +++ b/packages/dd-trace/test/appsec/rasp/utils.spec.js @@ -44,7 +44,42 @@ describe('RASP - utils.js', () => { web.root.returns(rootSpan) utils.handleResult(result, req, undefined, undefined, config) - sinon.assert.calledOnceWithExactly(stackTrace.reportStackTrace, rootSpan, stackId, 42, 2) + sinon.assert.calledOnceWithExactly(stackTrace.reportStackTrace, rootSpan, stackId, sinon.match.array) + }) + + it('should not report stack trace when max stack traces limit is reached', () => { + const req = {} + const rootSpan = { + meta_struct: { + '_dd.stack': { + exploit: ['stack1', 'stack2'] + } + } + } + const result = { + generate_stack: { + stack_id: 'stackId' + } + } + + web.root.returns(rootSpan) + + utils.handleResult(result, req, undefined, undefined, config) + sinon.assert.notCalled(stackTrace.reportStackTrace) + }) + + it('should not report stack trace when rootSpan is null', () => { + const req = {} + const result = { + generate_stack: { + stack_id: 'stackId' + } + } + + web.root.returns(null) + + utils.handleResult(result, req, undefined, undefined, config) + sinon.assert.notCalled(stackTrace.reportStackTrace) }) it('should not report stack trace when no action is present in waf result', () => { diff --git a/packages/dd-trace/test/appsec/stack_trace.spec.js b/packages/dd-trace/test/appsec/stack_trace.spec.js index 1ac2ca4db5e..406944c0381 100644 --- a/packages/dd-trace/test/appsec/stack_trace.spec.js +++ b/packages/dd-trace/test/appsec/stack_trace.spec.js @@ -3,11 +3,11 @@ const { assert } = require('chai') const path = require('path') -const { reportStackTrace } = require('../../src/appsec/stack_trace') +const { reportStackTrace, getCallsiteFrames } = require('../../src/appsec/stack_trace') describe('Stack trace reporter', () => { describe('frame filtering', () => { - it('should filer out frames from library', () => { + it('should filter out frames from library', () => { const callSiteList = Array(10).fill().map((_, i) => ( { @@ -15,7 +15,8 @@ describe('Stack trace reporter', () => { getLineNumber: () => i, getColumnNumber: () => i, getFunctionName: () => `libraryFunction${i}`, - getTypeName: () => `LibraryClass${i}` + getTypeName: () => `LibraryClass${i}`, + isNative: () => false } )).concat( Array(10).fill().map((_, i) => ( @@ -24,7 +25,8 @@ describe('Stack trace reporter', () => { getLineNumber: () => i, getColumnNumber: () => i, getFunctionName: () => `function${i}`, - getTypeName: () => `Class${i}` + getTypeName: () => `Class${i}`, + isNative: () => false } )) ).concat([ @@ -33,7 +35,8 @@ describe('Stack trace reporter', () => { getLineNumber: () => null, getColumnNumber: () => null, getFunctionName: () => null, - getTypeName: () => null + getTypeName: () => null, + isNative: () => false } ]) @@ -44,7 +47,8 @@ describe('Stack trace reporter', () => { line: i, column: i, function: `function${i}`, - class_name: `Class${i}` + class_name: `Class${i}`, + isNative: false } )) .concat([ @@ -54,15 +58,17 @@ describe('Stack trace reporter', () => { line: null, column: null, function: null, - class_name: null + class_name: null, + isNative: false } ]) const rootSpan = {} const stackId = 'test_stack_id' const maxDepth = 32 - const maxStackTraces = 2 - reportStackTrace(rootSpan, stackId, maxDepth, maxStackTraces, () => callSiteList) + const frames = getCallsiteFrames(maxDepth, () => callSiteList) + + reportStackTrace(rootSpan, stackId, frames) assert.deepEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].frames, expectedFrames) }) @@ -75,16 +81,16 @@ describe('Stack trace reporter', () => { getLineNumber: () => i, getColumnNumber: () => i, getFunctionName: () => `function${i}`, - getTypeName: () => `type${i}` + getTypeName: () => `type${i}`, + isNative: () => false } )) it('should not fail if no root span is passed', () => { const rootSpan = undefined const stackId = 'test_stack_id' - const maxDepth = 32 try { - reportStackTrace(rootSpan, stackId, maxDepth, 2, () => callSiteList) + reportStackTrace(rootSpan, stackId, callSiteList) } catch (e) { assert.fail() } @@ -101,11 +107,14 @@ describe('Stack trace reporter', () => { line: i, column: i, function: `function${i}`, - class_name: `type${i}` + class_name: `type${i}`, + isNative: false } )) - reportStackTrace(rootSpan, stackId, maxDepth, 2, () => callSiteList) + const frames = getCallsiteFrames(maxDepth, () => callSiteList) + + reportStackTrace(rootSpan, stackId, frames) assert.strictEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].id, stackId) assert.strictEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].language, 'nodejs') @@ -127,11 +136,14 @@ describe('Stack trace reporter', () => { line: i, column: i, function: `function${i}`, - class_name: `type${i}` + class_name: `type${i}`, + isNative: false } )) - reportStackTrace(rootSpan, stackId, maxDepth, 2, () => callSiteList) + const frames = getCallsiteFrames(maxDepth, () => callSiteList) + + reportStackTrace(rootSpan, stackId, frames) assert.strictEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].id, stackId) assert.strictEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].language, 'nodejs') @@ -157,11 +169,14 @@ describe('Stack trace reporter', () => { line: i, column: i, function: `function${i}`, - class_name: `type${i}` + class_name: `type${i}`, + isNative: false } )) - reportStackTrace(rootSpan, stackId, maxDepth, 2, () => callSiteList) + const frames = getCallsiteFrames(maxDepth, () => callSiteList) + + reportStackTrace(rootSpan, stackId, frames) assert.strictEqual(rootSpan.meta_struct['_dd.stack'].exploit[1].id, stackId) assert.strictEqual(rootSpan.meta_struct['_dd.stack'].exploit[1].language, 'nodejs') @@ -169,24 +184,6 @@ describe('Stack trace reporter', () => { assert.property(rootSpan.meta_struct, 'another_tag') }) - it('should not report stack trace when the maximum has been reached', () => { - const rootSpan = { - meta_struct: { - '_dd.stack': { - exploit: [callSiteList, callSiteList] - }, - another_tag: [] - } - } - const stackId = 'test_stack_id' - const maxDepth = 32 - - reportStackTrace(rootSpan, stackId, maxDepth, 2, () => callSiteList) - - assert.equal(rootSpan.meta_struct['_dd.stack'].exploit.length, 2) - assert.property(rootSpan.meta_struct, 'another_tag') - }) - it('should add stack trace when the max stack trace is 0', () => { const rootSpan = { meta_struct: { @@ -199,7 +196,9 @@ describe('Stack trace reporter', () => { const stackId = 'test_stack_id' const maxDepth = 32 - reportStackTrace(rootSpan, stackId, maxDepth, 0, () => callSiteList) + const frames = getCallsiteFrames(maxDepth, () => callSiteList) + + reportStackTrace(rootSpan, stackId, frames) assert.equal(rootSpan.meta_struct['_dd.stack'].exploit.length, 3) assert.property(rootSpan.meta_struct, 'another_tag') @@ -217,7 +216,9 @@ describe('Stack trace reporter', () => { const stackId = 'test_stack_id' const maxDepth = 32 - reportStackTrace(rootSpan, stackId, maxDepth, -1, () => callSiteList) + const frames = getCallsiteFrames(maxDepth, () => callSiteList) + + reportStackTrace(rootSpan, stackId, frames) assert.equal(rootSpan.meta_struct['_dd.stack'].exploit.length, 3) assert.property(rootSpan.meta_struct, 'another_tag') @@ -230,9 +231,7 @@ describe('Stack trace reporter', () => { } } const stackId = 'test_stack_id' - const maxDepth = 32 - const maxStackTraces = 2 - reportStackTrace(rootSpan, stackId, maxDepth, maxStackTraces, () => undefined) + reportStackTrace(rootSpan, stackId, undefined) assert.property(rootSpan.meta_struct, 'another_tag') assert.notProperty(rootSpan.meta_struct, '_dd.stack') }) @@ -245,7 +244,8 @@ describe('Stack trace reporter', () => { getLineNumber: () => i, getColumnNumber: () => i, getFunctionName: () => `function${i}`, - getTypeName: () => `type${i}` + getTypeName: () => `type${i}`, + isNative: () => false } )) @@ -260,11 +260,14 @@ describe('Stack trace reporter', () => { line: i, column: i, function: `function${i}`, - class_name: `type${i}` + class_name: `type${i}`, + isNative: false } )) - reportStackTrace(rootSpan, stackId, maxDepth, 2, () => callSiteList) + const frames = getCallsiteFrames(maxDepth, () => callSiteList) + + reportStackTrace(rootSpan, stackId, frames) assert.deepEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].frames, expectedFrames) }) @@ -279,7 +282,8 @@ describe('Stack trace reporter', () => { getLineNumber: () => 314, getColumnNumber: () => 271, getFunctionName: () => 'libraryFunction', - getTypeName: () => 'libraryType' + getTypeName: () => 'libraryType', + isNative: () => false } ].concat(Array(120).fill().map((_, i) => ( { @@ -287,7 +291,8 @@ describe('Stack trace reporter', () => { getLineNumber: () => i, getColumnNumber: () => i, getFunctionName: () => `function${i}`, - getTypeName: () => `type${i}` + getTypeName: () => `type${i}`, + isNative: () => false } )).concat([ { @@ -295,7 +300,8 @@ describe('Stack trace reporter', () => { getLineNumber: () => 271, getColumnNumber: () => 314, getFunctionName: () => 'libraryFunction', - getTypeName: () => 'libraryType' + getTypeName: () => 'libraryType', + isNative: () => false } ])) const expectedFrames = [0, 1, 2, 118, 119].map(i => ( @@ -305,11 +311,14 @@ describe('Stack trace reporter', () => { line: i, column: i, function: `function${i}`, - class_name: `type${i}` + class_name: `type${i}`, + isNative: false } )) - reportStackTrace(rootSpan, stackId, maxDepth, 2, () => callSiteListWithLibraryFrames) + const frames = getCallsiteFrames(maxDepth, () => callSiteListWithLibraryFrames) + + reportStackTrace(rootSpan, stackId, frames) assert.deepEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].frames, expectedFrames) }) @@ -325,11 +334,14 @@ describe('Stack trace reporter', () => { line: i, column: i, function: `function${i}`, - class_name: `type${i}` + class_name: `type${i}`, + isNative: false } )) - reportStackTrace(rootSpan, stackId, maxDepth, 2, () => callSiteList) + const frames = getCallsiteFrames(maxDepth, () => callSiteList) + + reportStackTrace(rootSpan, stackId, frames) assert.deepEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].frames, expectedFrames) }) @@ -345,11 +357,14 @@ describe('Stack trace reporter', () => { line: i, column: i, function: `function${i}`, - class_name: `type${i}` + class_name: `type${i}`, + isNative: false } )) - reportStackTrace(rootSpan, stackId, maxDepth, 2, () => callSiteList) + const frames = getCallsiteFrames(maxDepth, () => callSiteList) + + reportStackTrace(rootSpan, stackId, frames) assert.deepEqual(rootSpan.meta_struct['_dd.stack'].exploit[0].frames, expectedFrames) }) diff --git a/packages/dd-trace/test/appsec/utils.js b/packages/dd-trace/test/appsec/utils.js new file mode 100644 index 00000000000..ec9f22ad283 --- /dev/null +++ b/packages/dd-trace/test/appsec/utils.js @@ -0,0 +1,16 @@ +'use strict' + +function getWebSpan (traces) { + for (const trace of traces) { + for (const span of trace) { + if (span.type === 'web') { + return span + } + } + } + throw new Error('web span not found') +} + +module.exports = { + getWebSpan +} diff --git a/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js b/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js index 7b09f8fba2d..26dd5a7a611 100644 --- a/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js +++ b/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js @@ -151,6 +151,7 @@ describe('CI Visibility Exporter', () => { }) ciVisibilityExporter._resolveCanUseCiVisProtocol(true) }) + it('should request the API after EVP proxy is resolved', (done) => { const scope = nock(`http://localhost:${port}`) .post('/api/v2/libraries/tests/services/setting') @@ -160,7 +161,8 @@ describe('CI Visibility Exporter', () => { itr_enabled: true, require_git: false, code_coverage: true, - tests_skipping: true + tests_skipping: true, + known_tests_enabled: false } } })) @@ -649,34 +651,39 @@ describe('CI Visibility Exporter', () => { }) describe('getKnownTests', () => { - context('if early flake detection is disabled', () => { - it('should resolve immediately to undefined', (done) => { - const scope = nock(`http://localhost:${port}`) + context('if known tests is disabled', () => { + it('should resolve to undefined', (done) => { + const knownTestsScope = nock(`http://localhost:${port}`) .post('/api/v2/ci/libraries/tests') .reply(200) - const ciVisibilityExporter = new CiVisibilityExporter({ port, isEarlyFlakeDetectionEnabled: false }) + const ciVisibilityExporter = new CiVisibilityExporter({ + port + }) ciVisibilityExporter._resolveCanUseCiVisProtocol(true) + ciVisibilityExporter._libraryConfig = { isKnownTestsEnabled: false } ciVisibilityExporter.getKnownTests({}, (err, knownTests) => { expect(err).to.be.null expect(knownTests).to.eql(undefined) - expect(scope.isDone()).not.to.be.true + expect(knownTestsScope.isDone()).not.to.be.true done() }) }) }) - context('if early flake detection is enabled but can not use CI Visibility protocol', () => { + + context('if known tests is enabled but can not use CI Visibility protocol', () => { it('should not request known tests', (done) => { const scope = nock(`http://localhost:${port}`) .post('/api/v2/ci/libraries/tests') .reply(200) - const ciVisibilityExporter = new CiVisibilityExporter({ port, isEarlyFlakeDetectionEnabled: true }) + const ciVisibilityExporter = new CiVisibilityExporter({ port }) ciVisibilityExporter._resolveCanUseCiVisProtocol(false) - ciVisibilityExporter._libraryConfig = { isEarlyFlakeDetectionEnabled: true } + ciVisibilityExporter._libraryConfig = { isKnownTestsEnabled: true } + ciVisibilityExporter.getKnownTests({}, (err) => { expect(err).to.be.null expect(scope.isDone()).not.to.be.true @@ -684,7 +691,8 @@ describe('CI Visibility Exporter', () => { }) }) }) - context('if early flake detection is enabled and can use CI Vis Protocol', () => { + + context('if known tests is enabled and can use CI Vis Protocol', () => { it('should request known tests', (done) => { const scope = nock(`http://localhost:${port}`) .post('/api/v2/ci/libraries/tests') @@ -701,10 +709,10 @@ describe('CI Visibility Exporter', () => { } })) - const ciVisibilityExporter = new CiVisibilityExporter({ port, isEarlyFlakeDetectionEnabled: true }) + const ciVisibilityExporter = new CiVisibilityExporter({ port }) ciVisibilityExporter._resolveCanUseCiVisProtocol(true) - ciVisibilityExporter._libraryConfig = { isEarlyFlakeDetectionEnabled: true } + ciVisibilityExporter._libraryConfig = { isKnownTestsEnabled: true } ciVisibilityExporter.getKnownTests({}, (err, knownTests) => { expect(err).to.be.null expect(knownTests).to.eql({ @@ -717,20 +725,22 @@ describe('CI Visibility Exporter', () => { done() }) }) + it('should return an error if the request fails', (done) => { const scope = nock(`http://localhost:${port}`) .post('/api/v2/ci/libraries/tests') .reply(500) - const ciVisibilityExporter = new CiVisibilityExporter({ port, isEarlyFlakeDetectionEnabled: true }) + const ciVisibilityExporter = new CiVisibilityExporter({ port }) ciVisibilityExporter._resolveCanUseCiVisProtocol(true) - ciVisibilityExporter._libraryConfig = { isEarlyFlakeDetectionEnabled: true } + ciVisibilityExporter._libraryConfig = { isKnownTestsEnabled: true } ciVisibilityExporter.getKnownTests({}, (err) => { expect(err).not.to.be.null expect(scope.isDone()).to.be.true done() }) }) + it('should accept gzip if the exporter is gzip compatible', (done) => { let requestHeaders = {} const scope = nock(`http://localhost:${port}`) @@ -754,10 +764,10 @@ describe('CI Visibility Exporter', () => { 'content-encoding': 'gzip' }) - const ciVisibilityExporter = new CiVisibilityExporter({ port, isEarlyFlakeDetectionEnabled: true }) + const ciVisibilityExporter = new CiVisibilityExporter({ port }) ciVisibilityExporter._resolveCanUseCiVisProtocol(true) - ciVisibilityExporter._libraryConfig = { isEarlyFlakeDetectionEnabled: true } + ciVisibilityExporter._libraryConfig = { isKnownTestsEnabled: true } ciVisibilityExporter._isGzipCompatible = true ciVisibilityExporter.getKnownTests({}, (err, knownTests) => { expect(err).to.be.null @@ -772,6 +782,7 @@ describe('CI Visibility Exporter', () => { done() }) }) + it('should not accept gzip if the exporter is gzip incompatible', (done) => { let requestHeaders = {} const scope = nock(`http://localhost:${port}`) @@ -793,11 +804,10 @@ describe('CI Visibility Exporter', () => { }) }) - const ciVisibilityExporter = new CiVisibilityExporter({ port, isEarlyFlakeDetectionEnabled: true }) + const ciVisibilityExporter = new CiVisibilityExporter({ port }) ciVisibilityExporter._resolveCanUseCiVisProtocol(true) - ciVisibilityExporter._libraryConfig = { isEarlyFlakeDetectionEnabled: true } - + ciVisibilityExporter._libraryConfig = { isKnownTestsEnabled: true } ciVisibilityExporter._isGzipCompatible = false ciVisibilityExporter.getKnownTests({}, (err, knownTests) => { diff --git a/packages/dd-trace/test/config.spec.js b/packages/dd-trace/test/config.spec.js index 6bf7bf32e98..1b43a7859b2 100644 --- a/packages/dd-trace/test/config.spec.js +++ b/packages/dd-trace/test/config.spec.js @@ -265,6 +265,7 @@ describe('Config', () => { expect(config).to.have.nested.property('iast.redactionNamePattern', null) expect(config).to.have.nested.property('iast.redactionValuePattern', null) expect(config).to.have.nested.property('iast.telemetryVerbosity', 'INFORMATION') + expect(config).to.have.nested.property('iast.stackTrace.enabled', true) expect(config).to.have.nested.property('installSignature.id', null) expect(config).to.have.nested.property('installSignature.time', null) expect(config).to.have.nested.property('installSignature.type', null) @@ -330,6 +331,7 @@ describe('Config', () => { { name: 'iast.redactionValuePattern', value: null, origin: 'default' }, { name: 'iast.requestSampling', value: 30, origin: 'default' }, { name: 'iast.telemetryVerbosity', value: 'INFORMATION', origin: 'default' }, + { name: 'iast.stackTrace.enabled', value: true, origin: 'default' }, { name: 'injectionEnabled', value: [], origin: 'default' }, { name: 'isCiVisibility', value: false, origin: 'default' }, { name: 'isEarlyFlakeDetectionEnabled', value: false, origin: 'default' }, @@ -509,6 +511,7 @@ describe('Config', () => { process.env.DD_IAST_REDACTION_NAME_PATTERN = 'REDACTION_NAME_PATTERN' process.env.DD_IAST_REDACTION_VALUE_PATTERN = 'REDACTION_VALUE_PATTERN' process.env.DD_IAST_TELEMETRY_VERBOSITY = 'DEBUG' + process.env.DD_IAST_STACK_TRACE_ENABLED = 'false' process.env.DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED = 'true' process.env.DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED = 'true' process.env.DD_PROFILING_ENABLED = 'true' @@ -623,6 +626,7 @@ describe('Config', () => { expect(config).to.have.nested.property('iast.redactionNamePattern', 'REDACTION_NAME_PATTERN') expect(config).to.have.nested.property('iast.redactionValuePattern', 'REDACTION_VALUE_PATTERN') expect(config).to.have.nested.property('iast.telemetryVerbosity', 'DEBUG') + expect(config).to.have.nested.property('iast.stackTrace.enabled', false) expect(config).to.have.deep.property('installSignature', { id: '68e75c48-57ca-4a12-adfc-575c4b05fcbe', type: 'k8s_single_step', @@ -674,6 +678,7 @@ describe('Config', () => { { name: 'iast.redactionValuePattern', value: 'REDACTION_VALUE_PATTERN', origin: 'env_var' }, { name: 'iast.requestSampling', value: '40', origin: 'env_var' }, { name: 'iast.telemetryVerbosity', value: 'DEBUG', origin: 'env_var' }, + { name: 'iast.stackTrace.enabled', value: false, origin: 'env_var' }, { name: 'instrumentation_config_id', value: 'abcdef123', origin: 'env_var' }, { name: 'injectionEnabled', value: ['profiler'], origin: 'env_var' }, { name: 'isGCPFunction', value: false, origin: 'env_var' }, @@ -872,7 +877,10 @@ describe('Config', () => { redactionEnabled: false, redactionNamePattern: 'REDACTION_NAME_PATTERN', redactionValuePattern: 'REDACTION_VALUE_PATTERN', - telemetryVerbosity: 'DEBUG' + telemetryVerbosity: 'DEBUG', + stackTrace: { + enabled: false + } }, appsec: { standalone: { @@ -948,6 +956,7 @@ describe('Config', () => { expect(config).to.have.nested.property('iast.redactionNamePattern', 'REDACTION_NAME_PATTERN') expect(config).to.have.nested.property('iast.redactionValuePattern', 'REDACTION_VALUE_PATTERN') expect(config).to.have.nested.property('iast.telemetryVerbosity', 'DEBUG') + expect(config).to.have.nested.property('iast.stackTrace.enabled', false) expect(config).to.have.deep.nested.property('sampler', { sampleRate: 0.5, rateLimit: 1000, @@ -1002,6 +1011,7 @@ describe('Config', () => { { name: 'iast.redactionValuePattern', value: 'REDACTION_VALUE_PATTERN', origin: 'code' }, { name: 'iast.requestSampling', value: 50, origin: 'code' }, { name: 'iast.telemetryVerbosity', value: 'DEBUG', origin: 'code' }, + { name: 'iast.stackTrace.enabled', value: false, origin: 'code' }, { name: 'peerServiceMapping', value: { d: 'dd' }, origin: 'code' }, { name: 'plugins', value: false, origin: 'code' }, { name: 'port', value: '6218', origin: 'code' }, @@ -1224,6 +1234,7 @@ describe('Config', () => { process.env.DD_IAST_COOKIE_FILTER_PATTERN = '.*' process.env.DD_IAST_REDACTION_NAME_PATTERN = 'name_pattern_to_be_overriden_by_options' process.env.DD_IAST_REDACTION_VALUE_PATTERN = 'value_pattern_to_be_overriden_by_options' + process.env.DD_IAST_STACK_TRACE_ENABLED = 'true' process.env.DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED = 'true' process.env.DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED = 'true' process.env.DD_LLMOBS_ML_APP = 'myMlApp' @@ -1304,7 +1315,10 @@ describe('Config', () => { cookieFilterPattern: '.{10,}', dbRowsToTaint: 3, redactionNamePattern: 'REDACTION_NAME_PATTERN', - redactionValuePattern: 'REDACTION_VALUE_PATTERN' + redactionValuePattern: 'REDACTION_VALUE_PATTERN', + stackTrace: { + enabled: false + } }, remoteConfig: { pollInterval: 42 @@ -1379,6 +1393,7 @@ describe('Config', () => { expect(config).to.have.nested.property('iast.redactionEnabled', true) expect(config).to.have.nested.property('iast.redactionNamePattern', 'REDACTION_NAME_PATTERN') expect(config).to.have.nested.property('iast.redactionValuePattern', 'REDACTION_VALUE_PATTERN') + expect(config).to.have.nested.property('iast.stackTrace.enabled', false) expect(config).to.have.nested.property('llmobs.mlApp', 'myOtherMlApp') expect(config).to.have.nested.property('llmobs.agentlessEnabled', false) }) @@ -1416,7 +1431,10 @@ describe('Config', () => { redactionEnabled: false, redactionNamePattern: 'REDACTION_NAME_PATTERN', redactionValuePattern: 'REDACTION_VALUE_PATTERN', - telemetryVerbosity: 'DEBUG' + telemetryVerbosity: 'DEBUG', + stackTrace: { + enabled: false + } }, experimental: { appsec: { @@ -1450,7 +1468,10 @@ describe('Config', () => { redactionEnabled: true, redactionNamePattern: 'IGNORED_REDACTION_NAME_PATTERN', redactionValuePattern: 'IGNORED_REDACTION_VALUE_PATTERN', - telemetryVerbosity: 'OFF' + telemetryVerbosity: 'OFF', + stackTrace: { + enabled: true + } } } }) @@ -1499,7 +1520,10 @@ describe('Config', () => { redactionEnabled: false, redactionNamePattern: 'REDACTION_NAME_PATTERN', redactionValuePattern: 'REDACTION_VALUE_PATTERN', - telemetryVerbosity: 'DEBUG' + telemetryVerbosity: 'DEBUG', + stackTrace: { + enabled: false + } }) }) diff --git a/packages/dd-trace/test/llmobs/plugins/aws-sdk/bedrockruntime.spec.js b/packages/dd-trace/test/llmobs/plugins/aws-sdk/bedrockruntime.spec.js new file mode 100644 index 00000000000..42a902f1ba8 --- /dev/null +++ b/packages/dd-trace/test/llmobs/plugins/aws-sdk/bedrockruntime.spec.js @@ -0,0 +1,117 @@ +'use strict' + +const agent = require('../../../plugins/agent') + +const nock = require('nock') +const { expectedLLMObsLLMSpanEvent, deepEqualWithMockValues, MOCK_ANY } = require('../../util') +const { models, modelConfig } = require('../../../../../datadog-plugin-aws-sdk/test/fixtures/bedrockruntime') +const chai = require('chai') +const LLMObsAgentProxySpanWriter = require('../../../../src/llmobs/writers/spans/agentProxy') + +chai.Assertion.addMethod('deepEqualWithMockValues', deepEqualWithMockValues) + +const serviceName = 'bedrock-service-name-test' + +describe('Plugin', () => { + describe('aws-sdk (bedrockruntime)', function () { + before(() => { + process.env.AWS_SECRET_ACCESS_KEY = '0000000000/00000000000000000000000000000' + process.env.AWS_ACCESS_KEY_ID = '00000000000000000000' + }) + + after(() => { + delete process.env.AWS_SECRET_ACCESS_KEY + delete process.env.AWS_ACCESS_KEY_ID + }) + + withVersions('aws-sdk', ['@aws-sdk/smithy-client', 'aws-sdk'], '>=3', (version, moduleName) => { + let AWS + let bedrockRuntimeClient + + const bedrockRuntimeClientName = + moduleName === '@aws-sdk/smithy-client' ? '@aws-sdk/client-bedrock-runtime' : 'aws-sdk' + + describe('with configuration', () => { + before(() => { + sinon.stub(LLMObsAgentProxySpanWriter.prototype, 'append') + + // reduce errors related to too many listeners + process.removeAllListeners('beforeExit') + LLMObsAgentProxySpanWriter.prototype.append.reset() + + return agent.load('aws-sdk', {}, { + llmobs: { + mlApp: 'test' + } + }) + }) + + before(done => { + const requireVersion = version === '3.0.0' ? '3.422.0' : '>=3.422.0' + AWS = require(`../../../../../../versions/${bedrockRuntimeClientName}@${requireVersion}`).get() + bedrockRuntimeClient = new AWS.BedrockRuntimeClient( + { endpoint: 'http://127.0.0.1:4566', region: 'us-east-1', ServiceId: serviceName } + ) + done() + }) + + afterEach(() => { + nock.cleanAll() + LLMObsAgentProxySpanWriter.prototype.append.reset() + }) + + after(() => { + sinon.restore() + return agent.close({ ritmReset: false, wipe: true }) + }) + + models.forEach(model => { + it(`should invoke model for provider:${model.provider}`, done => { + const request = { + body: JSON.stringify(model.requestBody), + contentType: 'application/json', + accept: 'application/json', + modelId: model.modelId + } + + const response = JSON.stringify(model.response) + + nock('http://127.0.0.1:4566') + .post(`/model/${model.modelId}/invoke`) + .reply(200, response) + + const command = new AWS.InvokeModelCommand(request) + + agent.use(traces => { + const span = traces[0][0] + const spanEvent = LLMObsAgentProxySpanWriter.prototype.append.getCall(0).args[0] + const expected = expectedLLMObsLLMSpanEvent({ + span, + spanKind: 'llm', + name: 'bedrock-runtime.command', + inputMessages: [ + { content: model.userPrompt } + ], + outputMessages: MOCK_ANY, + tokenMetrics: { input_tokens: 0, output_tokens: 0, total_tokens: 0 }, + modelName: model.modelId.split('.')[1].toLowerCase(), + modelProvider: model.provider.toLowerCase(), + metadata: { + temperature: modelConfig.temperature, + max_tokens: modelConfig.maxTokens + }, + tags: { ml_app: 'test', language: 'javascript' } + }) + + expect(spanEvent).to.deepEqualWithMockValues(expected) + }).then(done).catch(done) + + bedrockRuntimeClient.send(command, (err) => { + if (err) return done(err) + }) + }) + }) + }) + }) + }) +}) diff --git a/packages/dd-trace/test/plugins/util/inferred_proxy.spec.js b/packages/dd-trace/test/plugins/util/inferred_proxy.spec.js index 78a8443c91c..0a02c149336 100644 --- a/packages/dd-trace/test/plugins/util/inferred_proxy.spec.js +++ b/packages/dd-trace/test/plugins/util/inferred_proxy.spec.js @@ -81,7 +81,6 @@ describe('Inferred Proxy Spans', function () { expect(spans[0].meta).to.have.property('http.url', 'example.com/test') expect(spans[0].meta).to.have.property('http.method', 'GET') expect(spans[0].meta).to.have.property('http.status_code', '200') - expect(spans[0].meta).to.have.property('http.route', '/test') expect(spans[0].meta).to.have.property('span.kind', 'internal') expect(spans[0].meta).to.have.property('component', 'aws-apigateway') expect(spans[0].meta).to.have.property('_dd.inferred_span', '1') @@ -130,7 +129,6 @@ describe('Inferred Proxy Spans', function () { expect(spans[0].meta).to.have.property('http.url', 'example.com/test') expect(spans[0].meta).to.have.property('http.method', 'GET') expect(spans[0].meta).to.have.property('http.status_code', '500') - expect(spans[0].meta).to.have.property('http.route', '/test') expect(spans[0].meta).to.have.property('span.kind', 'internal') expect(spans[0].meta).to.have.property('component', 'aws-apigateway') expect(spans[0].error).to.be.equal(1) diff --git a/yarn.lock b/yarn.lock index 243c4088334..83e7cd846ce 100644 --- a/yarn.lock +++ b/yarn.lock @@ -436,10 +436,10 @@ node-addon-api "^6.1.0" node-gyp-build "^3.9.0" -"@datadog/pprof@5.4.1": - version "5.4.1" - resolved "https://registry.yarnpkg.com/@datadog/pprof/-/pprof-5.4.1.tgz#08c9bcf5d8efb2eeafdfc9f5bb5402f79fb41266" - integrity sha512-IvpL96e/cuh8ugP5O8Czdup7XQOLHeIDgM5pac5W7Lc1YzGe5zTtebKFpitvb1CPw1YY+1qFx0pWGgKP2kOfHg== +"@datadog/pprof@5.5.0": + version "5.5.0" + resolved "https://registry.yarnpkg.com/@datadog/pprof/-/pprof-5.5.0.tgz#48fff2d70c5d2975e1f7a2b00b45160d89cdeb06" + integrity sha512-+53v76BDLr6o9MWC8dj7FIhnUwNGeCxPwJcT2ZlioyKWHJqpbPQ0Pc92visXg/QI4s6Vpz7mZbThvD2kIe57Ng== dependencies: delay "^5.0.0" node-gyp-build "<4.0"