Skip to content

Commit

Permalink
[test optimization] [SDTEST-1355] Fix DI issues with auto test retries (
Browse files Browse the repository at this point in the history 
#5072)

Co-authored-by: Thomas Watson <[email protected]>
  • Loading branch information
@juan-fernandez @watson
juan-fernandez and watson committed Jan 22, 2025
1 parent eef8fb7 commit 98f5cd9
Show file tree
Hide file tree
Showing 18 changed files with 583 additions and 428 deletions.
56 changes: 30 additions & 26 deletions integration-tests/cucumber/cucumber.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,10 @@ const {
TEST_SESSION_NAME,
TEST_LEVEL_EVENT_TYPES,
DI_ERROR_DEBUG_INFO_CAPTURED,
DI_DEBUG_ERROR_FILE,
DI_DEBUG_ERROR_SNAPSHOT_ID,
DI_DEBUG_ERROR_LINE
DI_DEBUG_ERROR_PREFIX,
DI_DEBUG_ERROR_FILE_SUFFIX,
DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX,
DI_DEBUG_ERROR_LINE_SUFFIX
} = require('../../packages/dd-trace/src/plugins/util/test')
const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')

Expand Down Expand Up @@ -1559,10 +1560,12 @@ versions.forEach(version => {
assert.equal(retriedTests.length, 1)
const [retriedTest] = retriedTests

assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE)
assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID)
const hasDebugTags = Object.keys(retriedTest.meta)
.some(property =>
property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED
)

assert.isFalse(hasDebugTags)
})
const logsPromise = receiver
.gatherPayloadsMaxTimeout(({ url }) => url === logsEndpoint, (payloads) => {
Expand Down Expand Up @@ -1602,11 +1605,12 @@ versions.forEach(version => {

assert.equal(retriedTests.length, 1)
const [retriedTest] = retriedTests
const hasDebugTags = Object.keys(retriedTest.meta)
.some(property =>
property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED
)

assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE)
assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID)
assert.isFalse(hasDebugTags)
})
const logsPromise = receiver
.gatherPayloadsMaxTimeout(({ url }) => url === logsEndpoint, (payloads) => {
Expand Down Expand Up @@ -1655,15 +1659,17 @@ versions.forEach(version => {
const [retriedTest] = retriedTests

assert.propertyVal(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
assert.propertyVal(
retriedTest.meta,
DI_DEBUG_ERROR_FILE,
'ci-visibility/features-di/support/sum.js'

assert.isTrue(
retriedTest.meta[`${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_FILE_SUFFIX}`]
.endsWith('ci-visibility/features-di/support/sum.js')
)
assert.equal(retriedTest.metrics[DI_DEBUG_ERROR_LINE], 4)
assert.exists(retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID])
assert.equal(retriedTest.metrics[`${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_LINE_SUFFIX}`], 4)

const snapshotIdKey = `${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX}`
assert.exists(retriedTest.meta[snapshotIdKey])

snapshotIdByTest = retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID]
snapshotIdByTest = retriedTest.meta[snapshotIdKey]
spanIdByTest = retriedTest.span_id.toString()
traceIdByTest = retriedTest.trace_id.toString()
})
Expand Down Expand Up @@ -1733,14 +1739,12 @@ versions.forEach(version => {
assert.equal(retriedTests.length, 1)
const [retriedTest] = retriedTests

assert.propertyVal(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
assert.propertyVal(
retriedTest.meta,
DI_DEBUG_ERROR_FILE,
'ci-visibility/features-di/support/sum.js'
)
assert.equal(retriedTest.metrics[DI_DEBUG_ERROR_LINE], 4)
assert.exists(retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID])
const hasDebugTags = Object.keys(retriedTest.meta)
.some(property =>
property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED
)

assert.isFalse(hasDebugTags)
})
const logsPromise = receiver
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/logs'), (payloads) => {
Expand Down
52 changes: 26 additions & 26 deletions integration-tests/jest/jest.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,10 @@ const {
TEST_SESSION_NAME,
TEST_LEVEL_EVENT_TYPES,
DI_ERROR_DEBUG_INFO_CAPTURED,
DI_DEBUG_ERROR_FILE,
DI_DEBUG_ERROR_SNAPSHOT_ID,
DI_DEBUG_ERROR_LINE
DI_DEBUG_ERROR_PREFIX,
DI_DEBUG_ERROR_FILE_SUFFIX,
DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX,
DI_DEBUG_ERROR_LINE_SUFFIX
} = require('../../packages/dd-trace/src/plugins/util/test')
const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants')
Expand Down Expand Up @@ -2426,11 +2427,12 @@ describe('jest CommonJS', () => {
assert.equal(retriedTests.length, 1)
const [retriedTest] = retriedTests

assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE)
assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID)
const hasDebugTags = Object.keys(retriedTest.meta)
.some(property => property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED)

assert.isFalse(hasDebugTags)
})

const logsPromise = receiver
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/logs'), (payloads) => {
if (payloads.length > 0) {
Expand Down Expand Up @@ -2472,10 +2474,10 @@ describe('jest CommonJS', () => {
assert.equal(retriedTests.length, 1)
const [retriedTest] = retriedTests

assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE)
assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID)
const hasDebugTags = Object.keys(retriedTest.meta)
.some(property => property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED)

assert.isFalse(hasDebugTags)
})
const logsPromise = receiver
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/logs'), (payloads) => {
Expand Down Expand Up @@ -2522,15 +2524,17 @@ describe('jest CommonJS', () => {
const [retriedTest] = retriedTests

assert.propertyVal(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
assert.propertyVal(
retriedTest.meta,
DI_DEBUG_ERROR_FILE,
'ci-visibility/dynamic-instrumentation/dependency.js'

assert.isTrue(
retriedTest.meta[`${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_FILE_SUFFIX}`]
.endsWith('ci-visibility/dynamic-instrumentation/dependency.js')
)
assert.equal(retriedTest.metrics[DI_DEBUG_ERROR_LINE], 4)
assert.exists(retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID])
assert.equal(retriedTest.metrics[`${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_LINE_SUFFIX}`], 4)

snapshotIdByTest = retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID]
const snapshotIdKey = `${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX}`
assert.exists(retriedTest.meta[snapshotIdKey])

snapshotIdByTest = retriedTest.meta[snapshotIdKey]
spanIdByTest = retriedTest.span_id.toString()
traceIdByTest = retriedTest.trace_id.toString()

Expand Down Expand Up @@ -2603,14 +2607,10 @@ describe('jest CommonJS', () => {
assert.equal(retriedTests.length, 1)
const [retriedTest] = retriedTests

assert.propertyVal(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
assert.propertyVal(
retriedTest.meta,
DI_DEBUG_ERROR_FILE,
'ci-visibility/dynamic-instrumentation/dependency.js'
)
assert.equal(retriedTest.metrics[DI_DEBUG_ERROR_LINE], 4)
assert.exists(retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID])
const hasDebugTags = Object.keys(retriedTest.meta)
.some(property => property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED)

assert.isFalse(hasDebugTags)
})
const logsPromise = receiver
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/logs'), (payloads) => {
Expand Down
51 changes: 25 additions & 26 deletions integration-tests/mocha/mocha.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,10 @@ const {
TEST_LEVEL_EVENT_TYPES,
TEST_EARLY_FLAKE_ABORT_REASON,
DI_ERROR_DEBUG_INFO_CAPTURED,
DI_DEBUG_ERROR_FILE,
DI_DEBUG_ERROR_SNAPSHOT_ID,
DI_DEBUG_ERROR_LINE
DI_DEBUG_ERROR_PREFIX,
DI_DEBUG_ERROR_FILE_SUFFIX,
DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX,
DI_DEBUG_ERROR_LINE_SUFFIX
} = require('../../packages/dd-trace/src/plugins/util/test')
const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants')
Expand Down Expand Up @@ -2166,10 +2167,10 @@ describe('mocha CommonJS', function () {
assert.equal(retriedTests.length, 1)
const [retriedTest] = retriedTests

assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE)
assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID)
const hasDebugTags = Object.keys(retriedTest.meta)
.some(property => property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED)

assert.isFalse(hasDebugTags)
})

const logsPromise = receiver
Expand Down Expand Up @@ -2217,10 +2218,10 @@ describe('mocha CommonJS', function () {
assert.equal(retriedTests.length, 1)
const [retriedTest] = retriedTests

assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE)
assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID)
const hasDebugTags = Object.keys(retriedTest.meta)
.some(property => property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED)

assert.isFalse(hasDebugTags)
})

const logsPromise = receiver
Expand Down Expand Up @@ -2273,15 +2274,17 @@ describe('mocha CommonJS', function () {
const [retriedTest] = retriedTests

assert.propertyVal(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
assert.propertyVal(
retriedTest.meta,
DI_DEBUG_ERROR_FILE,
'ci-visibility/dynamic-instrumentation/dependency.js'
assert.isTrue(
retriedTest.meta[`${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_FILE_SUFFIX}`]
.endsWith('ci-visibility/dynamic-instrumentation/dependency.js')
)
assert.equal(retriedTest.metrics[DI_DEBUG_ERROR_LINE], 4)
assert.exists(retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID])
assert.equal(retriedTest.metrics[`${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_LINE_SUFFIX}`], 4)

const snapshotIdKey = `${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX}`

assert.exists(retriedTest.meta[snapshotIdKey])

snapshotIdByTest = retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID]
snapshotIdByTest = retriedTest.meta[snapshotIdKey]
spanIdByTest = retriedTest.span_id.toString()
traceIdByTest = retriedTest.trace_id.toString()

Expand Down Expand Up @@ -2358,14 +2361,10 @@ describe('mocha CommonJS', function () {
assert.equal(retriedTests.length, 1)
const [retriedTest] = retriedTests

assert.propertyVal(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
assert.propertyVal(
retriedTest.meta,
DI_DEBUG_ERROR_FILE,
'ci-visibility/dynamic-instrumentation/dependency.js'
)
assert.equal(retriedTest.metrics[DI_DEBUG_ERROR_LINE], 4)
assert.exists(retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID])
const hasDebugTags = Object.keys(retriedTest.meta)
.some(property => property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED)

assert.isFalse(hasDebugTags)
})
const logsPromise = receiver
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/logs'), (payloads) => {
Expand Down
58 changes: 31 additions & 27 deletions integration-tests/vitest/vitest.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,16 @@ const {
TEST_EARLY_FLAKE_ABORT_REASON,
TEST_SUITE,
DI_ERROR_DEBUG_INFO_CAPTURED,
DI_DEBUG_ERROR_FILE,
DI_DEBUG_ERROR_LINE,
DI_DEBUG_ERROR_SNAPSHOT_ID
DI_DEBUG_ERROR_PREFIX,
DI_DEBUG_ERROR_FILE_SUFFIX,
DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX,
DI_DEBUG_ERROR_LINE_SUFFIX
} = require('../../packages/dd-trace/src/plugins/util/test')
const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')

const NUM_RETRIES_EFD = 3

const versions = ['1.6.0', 'latest']
const versions = ['latest']

const linePctMatchRegex = /Lines\s+:\s+([\d.]+)%/

Expand Down Expand Up @@ -920,10 +921,12 @@ versions.forEach((version) => {
assert.equal(retriedTests.length, 1)
const [retriedTest] = retriedTests

assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE)
assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID)
const hasDebugTags = Object.keys(retriedTest.meta)
.some(property =>
property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED
)

assert.isFalse(hasDebugTags)
})

const logsPromise = receiver
Expand Down Expand Up @@ -968,11 +971,12 @@ versions.forEach((version) => {

assert.equal(retriedTests.length, 1)
const [retriedTest] = retriedTests
const hasDebugTags = Object.keys(retriedTest.meta)
.some(property =>
property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED
)

assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE)
assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID)
assert.isFalse(hasDebugTags)
})

const logsPromise = receiver
Expand Down Expand Up @@ -1023,15 +1027,17 @@ versions.forEach((version) => {
const [retriedTest] = retriedTests

assert.propertyVal(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
assert.propertyVal(
retriedTest.meta,
DI_DEBUG_ERROR_FILE,
'ci-visibility/vitest-tests/bad-sum.mjs'

assert.isTrue(
retriedTest.meta[`${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_FILE_SUFFIX}`]
.endsWith('ci-visibility/vitest-tests/bad-sum.mjs')
)
assert.equal(retriedTest.metrics[DI_DEBUG_ERROR_LINE], 4)
assert.exists(retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID])
assert.equal(retriedTest.metrics[`${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_LINE_SUFFIX}`], 4)

const snapshotIdKey = `${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX}`
assert.exists(retriedTest.meta[snapshotIdKey])

snapshotIdByTest = retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID]
snapshotIdByTest = retriedTest.meta[snapshotIdKey]
spanIdByTest = retriedTest.span_id.toString()
traceIdByTest = retriedTest.trace_id.toString()

Expand Down Expand Up @@ -1107,14 +1113,12 @@ versions.forEach((version) => {
assert.equal(retriedTests.length, 1)
const [retriedTest] = retriedTests

assert.propertyVal(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
assert.propertyVal(
retriedTest.meta,
DI_DEBUG_ERROR_FILE,
'ci-visibility/vitest-tests/bad-sum.mjs'
)
assert.equal(retriedTest.metrics[DI_DEBUG_ERROR_LINE], 4)
assert.exists(retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID])
const hasDebugTags = Object.keys(retriedTest.meta)
.some(property =>
property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED
)

assert.isFalse(hasDebugTags)
})

const logsPromise = receiver
Expand Down
Loading

0 comments on commit 98f5cd9

Please sign in to comment.