[test optimization] Support for @fast-check/jest (#6632)

Author: juan-fernandez

integration-tests/ci-visibility/jest-fast-check/jest-fast-check.js

@@ -0,0 +1,9 @@
+'use strict'
+
+const { test, fc } = require('@fast-check/jest')
+
+describe('fast check', () => {
+  test.prop([fc.string(), fc.string(), fc.string()])('will not include seed', (a, b, c) => {
+    expect([a, b, c]).toContain(b)
+  })
+})

integration-tests/ci-visibility/jest-fast-check/jest-no-fast-check.js

@@ -0,0 +1,7 @@
+'use strict'
+
+describe('fast check with seed', () => {
+  it('should include seed (with seed=12)', () => {
+    expect(1 + 2).toEqual(3)
+  })
+})

integration-tests/jest/jest.spec.js

@@ -91,7 +91,8 @@ describe('jest CommonJS', () => {
       '@happy-dom/jest-environment',
       'office-addin-mock',
       'winston',
-      'jest-image-snapshot'
+      'jest-image-snapshot',
+      '@fast-check/jest'
     ], true)
     cwd = sandbox.folder
     startupTestFile = path.join(cwd, testFile)
@@ -4487,7 +4488,7 @@ describe('jest CommonJS', () => {
     })
   })
 
-  describe('winston mocking', () => {
+  context('winston mocking', () => {
     it('should allow winston to be mocked and verify createLogger is called', async () => {
       childProcess = exec(
         runTestsWithCoverageCommand,
@@ -4505,4 +4506,58 @@ describe('jest CommonJS', () => {
       assert.equal(code, 0, `Jest should pass but failed with code ${code}`)
     })
   })
+
+  context('fast-check', () => {
+    it('should remove seed from the test name if @fast-check/jest is used in the test', async () => {
+      const eventsPromise = receiver
+        .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => {
+          const events = payloads.flatMap(({ payload }) => payload.events)
+          const tests = events.filter(event => event.type === 'test').map(event => event.content)
+          assert.equal(tests.length, 1)
+          assert.equal(tests[0].meta[TEST_NAME], 'fast check will not include seed')
+        })
+
+      childProcess = exec(
+        runTestsWithCoverageCommand,
+        {
+          cwd,
+          env: {
+            ...getCiVisAgentlessConfig(receiver.port),
+            TESTS_TO_RUN: 'jest-fast-check/jest-fast-check',
+          }
+        }
+      )
+
+      await Promise.all([
+        once(childProcess, 'exit'),
+        eventsPromise
+      ])
+    })
+
+    it('should not remove seed if @fast-check/jest is not used', async () => {
+      const eventsPromise = receiver
+        .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => {
+          const events = payloads.flatMap(({ payload }) => payload.events)
+          const tests = events.filter(event => event.type === 'test').map(event => event.content)
+          assert.equal(tests.length, 1)
+          assert.equal(tests[0].meta[TEST_NAME], 'fast check with seed should include seed (with seed=12)')
+        })
+
+      childProcess = exec(
+        runTestsWithCoverageCommand,
+        {
+          cwd,
+          env: {
+            ...getCiVisAgentlessConfig(receiver.port),
+            TESTS_TO_RUN: 'jest-fast-check/jest-no-fast-check',
+          }
+        }
+      )
+
+      await Promise.all([
+        once(childProcess, 'exit'),
+        eventsPromise
+      ])
+    })
+  })
 })

packages/datadog-instrumentations/src/jest.js

@@ -89,6 +89,7 @@ const attemptToFixRetriedTestsStatuses = new Map()
 const wrappedWorkers = new WeakSet()
 const testSuiteMockedFiles = new Map()
 const testsToBeRetried = new Set()
+const testSuiteAbsolutePathsWithFastCheck = new Set()
 
 const BREAKPOINT_HIT_GRACE_PERIOD_MS = 200
 
@@ -304,9 +305,13 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
       }
     }
 
+    getShouldStripSeedFromTestName () {
+      return testSuiteAbsolutePathsWithFastCheck.has(this.testSuiteAbsolutePath)
+    }
+
     // At the `add_test` event we don't have the test object yet, so we can't use it
     getTestNameFromAddTestEvent (event, state) {
-      const describeSuffix = getJestTestName(state.currentDescribeBlock)
+      const describeSuffix = getJestTestName(state.currentDescribeBlock, this.getShouldStripSeedFromTestName())
       return describeSuffix ? `${describeSuffix} ${event.testName}` : event.testName
     }
 
@@ -329,7 +334,7 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
         })
       }
       if (event.name === 'test_start') {
-        const testName = getJestTestName(event.test)
+        const testName = getJestTestName(event.test, this.getShouldStripSeedFromTestName())
         if (testsToBeRetried.has(testName)) {
           // This is needed because we're trying tests with the same name
           this.resetSnapshotState()
@@ -491,7 +496,7 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
         let failedAllTests = false
         let isAttemptToFix = false
         if (this.isTestManagementTestsEnabled) {
-          const testName = getJestTestName(event.test)
+          const testName = getJestTestName(event.test, this.getShouldStripSeedFromTestName())
           isAttemptToFix = this.testManagementTestsForThisSuite?.attemptToFix?.includes(testName)
           if (isAttemptToFix) {
             if (attemptToFixRetriedTestsStatuses.has(testName)) {
@@ -519,7 +524,7 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
         let isEfdRetry = false
         // We'll store the test statuses of the retries
         if (this.isKnownTestsEnabled) {
-          const testName = getJestTestName(event.test)
+          const testName = getJestTestName(event.test, this.getShouldStripSeedFromTestName())
           const isNewTest = retriedTestsToNumAttempts.has(testName)
           if (isNewTest) {
             if (newTestsTestStatuses.has(testName)) {
@@ -579,16 +584,17 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
         }
       }
       if (event.name === 'test_skip' || event.name === 'test_todo') {
+        const testName = getJestTestName(event.test, this.getShouldStripSeedFromTestName())
         testSkippedCh.publish({
           test: {
-            name: getJestTestName(event.test),
+            name: testName,
             suite: this.testSuite,
             testSourceFile: this.testSourceFile,
             displayName: this.displayName,
             frameworkVersion: jestVersion,
             testStartLine: getTestLineStart(event.test.asyncError, this.testSuite)
           },
-          isDisabled: this.testManagementTestsForThisSuite?.disabled?.includes(getJestTestName(event.test))
+          isDisabled: this.testManagementTestsForThisSuite?.disabled?.includes(testName)
         })
       }
     }
@@ -1300,6 +1306,10 @@ addHook({
       // To bypass jest's own require engine
       return this._requireCoreModule(moduleName)
     }
+    // This means that `@fast-check/jest` is used in the test file.
+    if (moduleName === '@fast-check/jest') {
+      testSuiteAbsolutePathsWithFastCheck.add(this._testPath)
+    }
     return requireModuleOrMock.apply(this, arguments)
   })
 

packages/datadog-plugin-jest/src/util.js

@@ -41,16 +41,24 @@ function getFormattedJestTestParameters (testParameters) {
   return formattedParameters
 }
 
+// Support for `@fast-check/jest`: this library modifies the test name to include the seed
+// A test name that keeps changing breaks some Test Optimization's features.
+const SEED_SUFFIX_RE = /\s*\(with seed=-?\d+\)\s*$/i
 // https://github.com/facebook/jest/blob/3e38157ad5f23fb7d24669d24fae8ded06a7ab75/packages/jest-circus/src/utils.ts#L396
-function getJestTestName (test) {
+function getJestTestName (test, shouldStripSeed = false) {
   const titles = []
   let parent = test
   do {
     titles.unshift(parent.name)
   } while ((parent = parent.parent))
 
   titles.shift() // remove TOP_DESCRIBE_BLOCK_NAME
-  return titles.join(' ')
+
+  const testName = titles.join(' ')
+  if (shouldStripSeed) {
+    return testName.replace(SEED_SUFFIX_RE, '')
+  }
+  return testName
 }
 
 function isMarkedAsUnskippable (test) {

packages/dd-trace/test/plugins/versions/package.json

@@ -30,6 +30,7 @@
     "@elastic/transport": "9.1.2",
     "@fastify/cookie": "11.0.2",
     "@fastify/multipart": "9.2.1",
+    "@fast-check/jest": "2.1.1",
     "@google-cloud/pubsub": "5.2.0",
     "@google-cloud/vertexai": "1.10.0",
     "@graphql-tools/executor": "1.4.9",