Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[test optimization] [SDTEST-1355] Fix DI issues with auto test retries #5072

Merged
merged 14 commits into from
Jan 14, 2025
Merged

[test optimization] [SDTEST-1355] Fix DI issues with auto test retries #5072

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
6ad15c3
fix sync issues
juan-fernandez Jan 8, 2025
25a1877
remove redundant listener
juan-fernandez Jan 8, 2025
f55d1e4
clean up
juan-fernandez Jan 8, 2025
0f248df
fix stack index
juan-fernandez Jan 9, 2025
05f8598
fix jest tests
juan-fernandez Jan 9, 2025
e340f7a
fix mocha
juan-fernandez Jan 9, 2025
0ff41ea
fix cucumber
juan-fernandez Jan 9, 2025
6cce993
fix vitest
juan-fernandez Jan 9, 2025
410c980
fix vitest, now for real
juan-fernandez Jan 10, 2025
aebda9d
clean up probes
juan-fernandez Jan 10, 2025
923fa0d
fix unit test
juan-fernandez Jan 10, 2025
4a296b8
fix misused function
juan-fernandez Jan 10, 2025
101facb
fix order
juan-fernandez Jan 10, 2025
6b2a867
Apply suggestions from code review
juan-fernandez Jan 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 30 additions & 26 deletions integration-tests/cucumber/cucumber.spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,10 @@ const {
TEST_SESSION_NAME,
TEST_LEVEL_EVENT_TYPES,
DI_ERROR_DEBUG_INFO_CAPTURED,
DI_DEBUG_ERROR_FILE,
DI_DEBUG_ERROR_SNAPSHOT_ID,
DI_DEBUG_ERROR_LINE
DI_DEBUG_ERROR_PREFIX,
DI_DEBUG_ERROR_FILE_SUFFIX,
DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX,
DI_DEBUG_ERROR_LINE_SUFFIX
} = require('../../packages/dd-trace/src/plugins/util/test')
const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')

Expand Down Expand Up @@ -1559,10 +1560,12 @@ versions.forEach(version => {
assert.equal(retriedTests.length, 1)
const [retriedTest] = retriedTests

assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE)
assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID)
const hasDebugTags = Object.keys(retriedTest.meta)
.some(property =>
property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED
)

assert.isFalse(hasDebugTags)
})
const logsPromise = receiver
.gatherPayloadsMaxTimeout(({ url }) => url === logsEndpoint, (payloads) => {
Expand Down Expand Up @@ -1602,11 +1605,12 @@ versions.forEach(version => {

assert.equal(retriedTests.length, 1)
const [retriedTest] = retriedTests
const hasDebugTags = Object.keys(retriedTest.meta)
.some(property =>
property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED
)

assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE)
assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID)
assert.isFalse(hasDebugTags)
})
const logsPromise = receiver
.gatherPayloadsMaxTimeout(({ url }) => url === logsEndpoint, (payloads) => {
Expand Down Expand Up @@ -1655,15 +1659,17 @@ versions.forEach(version => {
const [retriedTest] = retriedTests

assert.propertyVal(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
assert.propertyVal(
retriedTest.meta,
DI_DEBUG_ERROR_FILE,
'ci-visibility/features-di/support/sum.js'

assert.isTrue(
retriedTest.meta[`${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_FILE_SUFFIX}`]
.endsWith('ci-visibility/features-di/support/sum.js')
)
assert.equal(retriedTest.metrics[DI_DEBUG_ERROR_LINE], 4)
assert.exists(retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID])
assert.equal(retriedTest.metrics[`${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_LINE_SUFFIX}`], 4)

const snapshotIdKey = `${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX}`
assert.exists(retriedTest.meta[snapshotIdKey])

snapshotIdByTest = retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID]
snapshotIdByTest = retriedTest.meta[snapshotIdKey]
spanIdByTest = retriedTest.span_id.toString()
traceIdByTest = retriedTest.trace_id.toString()
})
Expand Down Expand Up @@ -1733,14 +1739,12 @@ versions.forEach(version => {
assert.equal(retriedTests.length, 1)
const [retriedTest] = retriedTests

assert.propertyVal(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
assert.propertyVal(
retriedTest.meta,
DI_DEBUG_ERROR_FILE,
'ci-visibility/features-di/support/sum.js'
)
assert.equal(retriedTest.metrics[DI_DEBUG_ERROR_LINE], 4)
assert.exists(retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID])
const hasDebugTags = Object.keys(retriedTest.meta)
.some(property =>
property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED
)

assert.isFalse(hasDebugTags)
})
const logsPromise = receiver
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/logs'), (payloads) => {
Expand Down
52 changes: 26 additions & 26 deletions integration-tests/jest/jest.spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,10 @@ const {
TEST_SESSION_NAME,
TEST_LEVEL_EVENT_TYPES,
DI_ERROR_DEBUG_INFO_CAPTURED,
DI_DEBUG_ERROR_FILE,
DI_DEBUG_ERROR_SNAPSHOT_ID,
DI_DEBUG_ERROR_LINE
DI_DEBUG_ERROR_PREFIX,
DI_DEBUG_ERROR_FILE_SUFFIX,
DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX,
DI_DEBUG_ERROR_LINE_SUFFIX
} = require('../../packages/dd-trace/src/plugins/util/test')
const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants')
Expand Down Expand Up @@ -2426,11 +2427,12 @@ describe('jest CommonJS', () => {
assert.equal(retriedTests.length, 1)
const [retriedTest] = retriedTests

assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE)
assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID)
const hasDebugTags = Object.keys(retriedTest.meta)
.some(property => property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED)

assert.isFalse(hasDebugTags)
})

const logsPromise = receiver
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/logs'), (payloads) => {
if (payloads.length > 0) {
Expand Down Expand Up @@ -2472,10 +2474,10 @@ describe('jest CommonJS', () => {
assert.equal(retriedTests.length, 1)
const [retriedTest] = retriedTests

assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE)
assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID)
const hasDebugTags = Object.keys(retriedTest.meta)
.some(property => property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED)

assert.isFalse(hasDebugTags)
})
const logsPromise = receiver
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/logs'), (payloads) => {
Expand Down Expand Up @@ -2522,15 +2524,17 @@ describe('jest CommonJS', () => {
const [retriedTest] = retriedTests

assert.propertyVal(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
assert.propertyVal(
retriedTest.meta,
DI_DEBUG_ERROR_FILE,
'ci-visibility/dynamic-instrumentation/dependency.js'

assert.isTrue(
retriedTest.meta[`${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_FILE_SUFFIX}`]
.endsWith('ci-visibility/dynamic-instrumentation/dependency.js')
)
assert.equal(retriedTest.metrics[DI_DEBUG_ERROR_LINE], 4)
assert.exists(retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID])
assert.equal(retriedTest.metrics[`${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_LINE_SUFFIX}`], 4)

snapshotIdByTest = retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID]
const snapshotIdKey = `${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX}`
assert.exists(retriedTest.meta[snapshotIdKey])

snapshotIdByTest = retriedTest.meta[snapshotIdKey]
spanIdByTest = retriedTest.span_id.toString()
traceIdByTest = retriedTest.trace_id.toString()

Expand Down Expand Up @@ -2603,14 +2607,10 @@ describe('jest CommonJS', () => {
assert.equal(retriedTests.length, 1)
const [retriedTest] = retriedTests

assert.propertyVal(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
assert.propertyVal(
retriedTest.meta,
DI_DEBUG_ERROR_FILE,
'ci-visibility/dynamic-instrumentation/dependency.js'
)
assert.equal(retriedTest.metrics[DI_DEBUG_ERROR_LINE], 4)
assert.exists(retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID])
const hasDebugTags = Object.keys(retriedTest.meta)
.some(property => property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED)

assert.isFalse(hasDebugTags)
})
const logsPromise = receiver
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/logs'), (payloads) => {
Expand Down
51 changes: 25 additions & 26 deletions integration-tests/mocha/mocha.spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,10 @@ const {
TEST_LEVEL_EVENT_TYPES,
TEST_EARLY_FLAKE_ABORT_REASON,
DI_ERROR_DEBUG_INFO_CAPTURED,
DI_DEBUG_ERROR_FILE,
DI_DEBUG_ERROR_SNAPSHOT_ID,
DI_DEBUG_ERROR_LINE
DI_DEBUG_ERROR_PREFIX,
DI_DEBUG_ERROR_FILE_SUFFIX,
DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX,
DI_DEBUG_ERROR_LINE_SUFFIX
} = require('../../packages/dd-trace/src/plugins/util/test')
const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants')
Expand Down Expand Up @@ -2166,10 +2167,10 @@ describe('mocha CommonJS', function () {
assert.equal(retriedTests.length, 1)
const [retriedTest] = retriedTests

assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE)
assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID)
const hasDebugTags = Object.keys(retriedTest.meta)
.some(property => property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED)

assert.isFalse(hasDebugTags)
})

const logsPromise = receiver
Expand Down Expand Up @@ -2217,10 +2218,10 @@ describe('mocha CommonJS', function () {
assert.equal(retriedTests.length, 1)
const [retriedTest] = retriedTests

assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE)
assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID)
const hasDebugTags = Object.keys(retriedTest.meta)
.some(property => property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED)

assert.isFalse(hasDebugTags)
})

const logsPromise = receiver
Expand Down Expand Up @@ -2273,15 +2274,17 @@ describe('mocha CommonJS', function () {
const [retriedTest] = retriedTests

assert.propertyVal(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
assert.propertyVal(
retriedTest.meta,
DI_DEBUG_ERROR_FILE,
'ci-visibility/dynamic-instrumentation/dependency.js'
assert.isTrue(
retriedTest.meta[`${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_FILE_SUFFIX}`]
.endsWith('ci-visibility/dynamic-instrumentation/dependency.js')
)
assert.equal(retriedTest.metrics[DI_DEBUG_ERROR_LINE], 4)
assert.exists(retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID])
assert.equal(retriedTest.metrics[`${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_LINE_SUFFIX}`], 4)

const snapshotIdKey = `${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX}`

assert.exists(retriedTest.meta[snapshotIdKey])

snapshotIdByTest = retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID]
snapshotIdByTest = retriedTest.meta[snapshotIdKey]
spanIdByTest = retriedTest.span_id.toString()
traceIdByTest = retriedTest.trace_id.toString()

Expand Down Expand Up @@ -2358,14 +2361,10 @@ describe('mocha CommonJS', function () {
assert.equal(retriedTests.length, 1)
const [retriedTest] = retriedTests

assert.propertyVal(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
assert.propertyVal(
retriedTest.meta,
DI_DEBUG_ERROR_FILE,
'ci-visibility/dynamic-instrumentation/dependency.js'
)
assert.equal(retriedTest.metrics[DI_DEBUG_ERROR_LINE], 4)
assert.exists(retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID])
const hasDebugTags = Object.keys(retriedTest.meta)
.some(property => property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED)

assert.isFalse(hasDebugTags)
})
const logsPromise = receiver
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/logs'), (payloads) => {
Expand Down
58 changes: 31 additions & 27 deletions integration-tests/vitest/vitest.spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,16 @@ const {
TEST_EARLY_FLAKE_ABORT_REASON,
TEST_SUITE,
DI_ERROR_DEBUG_INFO_CAPTURED,
DI_DEBUG_ERROR_FILE,
DI_DEBUG_ERROR_LINE,
DI_DEBUG_ERROR_SNAPSHOT_ID
DI_DEBUG_ERROR_PREFIX,
DI_DEBUG_ERROR_FILE_SUFFIX,
DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX,
DI_DEBUG_ERROR_LINE_SUFFIX
} = require('../../packages/dd-trace/src/plugins/util/test')
const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')

const NUM_RETRIES_EFD = 3

const versions = ['1.6.0', 'latest']
const versions = ['latest']

const linePctMatchRegex = /Lines\s+:\s+([\d.]+)%/

Expand Down Expand Up @@ -920,10 +921,12 @@ versions.forEach((version) => {
assert.equal(retriedTests.length, 1)
const [retriedTest] = retriedTests

assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE)
assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID)
const hasDebugTags = Object.keys(retriedTest.meta)
.some(property =>
property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED
)

assert.isFalse(hasDebugTags)
})

const logsPromise = receiver
Expand Down Expand Up @@ -968,11 +971,12 @@ versions.forEach((version) => {

assert.equal(retriedTests.length, 1)
const [retriedTest] = retriedTests
const hasDebugTags = Object.keys(retriedTest.meta)
.some(property =>
property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED
)

assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE)
assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE)
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID)
assert.isFalse(hasDebugTags)
})

const logsPromise = receiver
Expand Down Expand Up @@ -1023,15 +1027,17 @@ versions.forEach((version) => {
const [retriedTest] = retriedTests

assert.propertyVal(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
assert.propertyVal(
retriedTest.meta,
DI_DEBUG_ERROR_FILE,
'ci-visibility/vitest-tests/bad-sum.mjs'

assert.isTrue(
retriedTest.meta[`${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_FILE_SUFFIX}`]
.endsWith('ci-visibility/vitest-tests/bad-sum.mjs')
)
assert.equal(retriedTest.metrics[DI_DEBUG_ERROR_LINE], 4)
assert.exists(retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID])
assert.equal(retriedTest.metrics[`${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_LINE_SUFFIX}`], 4)

const snapshotIdKey = `${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX}`
assert.exists(retriedTest.meta[snapshotIdKey])

snapshotIdByTest = retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID]
snapshotIdByTest = retriedTest.meta[snapshotIdKey]
spanIdByTest = retriedTest.span_id.toString()
traceIdByTest = retriedTest.trace_id.toString()

Expand Down Expand Up @@ -1107,14 +1113,12 @@ versions.forEach((version) => {
assert.equal(retriedTests.length, 1)
const [retriedTest] = retriedTests

assert.propertyVal(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
assert.propertyVal(
retriedTest.meta,
DI_DEBUG_ERROR_FILE,
'ci-visibility/vitest-tests/bad-sum.mjs'
)
assert.equal(retriedTest.metrics[DI_DEBUG_ERROR_LINE], 4)
assert.exists(retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID])
const hasDebugTags = Object.keys(retriedTest.meta)
.some(property =>
property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED
)

assert.isFalse(hasDebugTags)
})

const logsPromise = receiver
Expand Down
Loading
Loading