[test optimization] [SDTEST-1529] Add quarantined tests logic (#5236)

DataDog · Feb 17, 2025 · 366368a · 366368a
1 parent b599fab
commit 366368a
Show file tree

Hide file tree

Showing 36 changed files with 1,408 additions and 73 deletions.
diff --git a/integration-tests/ci-visibility-intake.js b/integration-tests/ci-visibility-intake.js
@@ -12,31 +12,43 @@ const DEFAULT_SETTINGS = {
   code_coverage: true,
   tests_skipping: true,
   itr_enabled: true,
+  require_git: false,
   early_flake_detection: {
     enabled: false,
     slow_test_retries: {
       '5s': 3
     }
+  },
+  flaky_test_retries_enabled: false,
+  di_enabled: false,
+  known_tests_enabled: false,
+  test_management: {
+    enabled: false
   }
 }
 
 const DEFAULT_SUITES_TO_SKIP = []
 const DEFAULT_GIT_UPLOAD_STATUS = 200
-const DEFAULT_KNOWN_TESTS_UPLOAD_STATUS = 200
+const DEFAULT_KNOWN_TESTS_RESPONSE_STATUS = 200
 const DEFAULT_INFO_RESPONSE = {
   endpoints: ['/evp_proxy/v2', '/debugger/v1/input']
 }
 const DEFAULT_CORRELATION_ID = '1234'
 const DEFAULT_KNOWN_TESTS = ['test-suite1.js.test-name1', 'test-suite2.js.test-name2']
 
+const DEFAULT_QUARANTINED_TESTS = {}
+const DEFAULT_QUARANTINED_TESTS_RESPONSE_STATUS = 200
+
 let settings = DEFAULT_SETTINGS
 let suitesToSkip = DEFAULT_SUITES_TO_SKIP
 let gitUploadStatus = DEFAULT_GIT_UPLOAD_STATUS
 let infoResponse = DEFAULT_INFO_RESPONSE
 let correlationId = DEFAULT_CORRELATION_ID
 let knownTests = DEFAULT_KNOWN_TESTS
-let knownTestsStatusCode = DEFAULT_KNOWN_TESTS_UPLOAD_STATUS
+let knownTestsStatusCode = DEFAULT_KNOWN_TESTS_RESPONSE_STATUS
 let waitingTime = 0
+let quarantineResponse = DEFAULT_QUARANTINED_TESTS
+let quarantineResponseStatusCode = DEFAULT_QUARANTINED_TESTS_RESPONSE_STATUS
 
 class FakeCiVisIntake extends FakeAgent {
   setKnownTestsResponseCode (statusCode) {
@@ -71,6 +83,14 @@ class FakeCiVisIntake extends FakeAgent {
     waitingTime = newWaitingTime
   }
 
+  setQuarantinedTests (newQuarantinedTests) {
+    quarantineResponse = newQuarantinedTests
+  }
+
+  setQuarantinedTestsResponseCode (newStatusCode) {
+    quarantineResponseStatusCode = newStatusCode
+  }
+
   async start () {
     const app = express()
     app.use(bodyParser.raw({ limit: Infinity, type: 'application/msgpack' }))
@@ -219,6 +239,25 @@ class FakeCiVisIntake extends FakeAgent {
       })
     })
 
+    app.post([
+      '/api/v2/test/libraries/test-management/tests',
+      '/evp_proxy/:version/api/v2/test/libraries/test-management/tests'
+    ], (req, res) => {
+      res.setHeader('content-type', 'application/json')
+      const data = JSON.stringify({
+        data: {
+          attributes: {
+            modules: quarantineResponse
+          }
+        }
+      })
+      res.status(quarantineResponseStatusCode).send(data)
+      this.emit('message', {
+        headers: req.headers,
+        url: req.url
+      })
+    })
+
     return new Promise((resolve, reject) => {
       const timeoutObj = setTimeout(() => {
         reject(new Error('Intake timed out starting up'))
@@ -237,8 +276,10 @@ class FakeCiVisIntake extends FakeAgent {
     settings = DEFAULT_SETTINGS
     suitesToSkip = DEFAULT_SUITES_TO_SKIP
     gitUploadStatus = DEFAULT_GIT_UPLOAD_STATUS
-    knownTestsStatusCode = DEFAULT_KNOWN_TESTS_UPLOAD_STATUS
+    knownTestsStatusCode = DEFAULT_KNOWN_TESTS_RESPONSE_STATUS
     infoResponse = DEFAULT_INFO_RESPONSE
+    quarantineResponseStatusCode = DEFAULT_QUARANTINED_TESTS_RESPONSE_STATUS
+    quarantineResponse = DEFAULT_QUARANTINED_TESTS
     this.removeAllListeners()
     if (this.waitingTimeoutId) {
       clearTimeout(this.waitingTimeoutId)

diff --git a/integration-tests/ci-visibility/features-quarantine/quarantine.feature b/integration-tests/ci-visibility/features-quarantine/quarantine.feature
@@ -0,0 +1,4 @@
+Feature: Quarantine
+  Scenario: Say quarantine
+    When the greeter says quarantine
+    Then I should have heard "quarantine"
diff --git a/integration-tests/ci-visibility/features-quarantine/support/steps.js b/integration-tests/ci-visibility/features-quarantine/support/steps.js
@@ -0,0 +1,10 @@
+const assert = require('assert')
+const { When, Then } = require('@cucumber/cucumber')
+
+Then('I should have heard {string}', function (expectedResponse) {
+  assert.equal(this.whatIHeard, 'fail')
+})
+
+When('the greeter says quarantine', function () {
+  this.whatIHeard = 'quarantine'
+})
diff --git a/integration-tests/ci-visibility/playwright-tests-quarantine/quarantine-test.js b/integration-tests/ci-visibility/playwright-tests-quarantine/quarantine-test.js
@@ -0,0 +1,13 @@
+const { test, expect } = require('@playwright/test')
+
+test.beforeEach(async ({ page }) => {
+  await page.goto(process.env.PW_BASE_URL)
+})
+
+test.describe('quarantine', () => {
+  test('should quarantine failed test', async ({ page }) => {
+    await expect(page.locator('.hello-world')).toHaveText([
+      'Hello Warld'
+    ])
+  })
+})
diff --git a/integration-tests/ci-visibility/quarantine/test-quarantine-1.js b/integration-tests/ci-visibility/quarantine/test-quarantine-1.js
@@ -0,0 +1,11 @@
+const { expect } = require('chai')
+
+describe('quarantine tests', () => {
+  it('can quarantine a test', () => {
+    expect(1 + 2).to.equal(4)
+  })
+
+  it('can pass normally', () => {
+    expect(1 + 2).to.equal(3)
+  })
+})
diff --git a/integration-tests/ci-visibility/quarantine/test-quarantine-2.js b/integration-tests/ci-visibility/quarantine/test-quarantine-2.js
@@ -0,0 +1,11 @@
+const { expect } = require('chai')
+
+describe('quarantine tests 2', () => {
+  it('can quarantine a test', () => {
+    expect(1 + 2).to.equal(3)
+  })
+
+  it('can pass normally', () => {
+    expect(1 + 2).to.equal(3)
+  })
+})
diff --git a/integration-tests/ci-visibility/run-jest.js b/integration-tests/ci-visibility/run-jest.js
@@ -31,8 +31,12 @@ if (process.env.COLLECT_COVERAGE_FROM) {
 jest.runCLI(
   options,
   options.projects
-).then(() => {
+).then((results) => {
   if (process.send) {
     process.send('finished')
   }
+  if (process.env.SHOULD_CHECK_RESULTS) {
+    const exitCode = results.results.success ? 0 : 1
+    process.exit(exitCode)
+  }
 })
diff --git a/integration-tests/ci-visibility/run-mocha.js b/integration-tests/ci-visibility/run-mocha.js
@@ -12,11 +12,14 @@ if (process.env.TESTS_TO_RUN) {
   mocha.addFile(require.resolve('./test/ci-visibility-test.js'))
   mocha.addFile(require.resolve('./test/ci-visibility-test-2.js'))
 }
-mocha.run(() => {
+mocha.run((failures) => {
   if (process.send) {
     process.send('finished')
   }
-}).on('end', () => {
+  if (process.env.SHOULD_CHECK_RESULTS && failures > 0) {
+    process.exit(1)
+  }
+}).on('end', (res) => {
   // eslint-disable-next-line
   console.log('end event: can add event listeners to mocha')
 })
diff --git a/integration-tests/ci-visibility/vitest-tests/test-quarantine.mjs b/integration-tests/ci-visibility/vitest-tests/test-quarantine.mjs
@@ -0,0 +1,11 @@
+import { describe, test, expect } from 'vitest'
+
+describe('quarantine tests', () => {
+  test('can quarantine a test', () => {
+    expect(1 + 2).to.equal(4)
+  })
+
+  test('can pass normally', () => {
+    expect(1 + 2).to.equal(3)
+  })
+})
diff --git a/integration-tests/cucumber/cucumber.spec.js b/integration-tests/cucumber/cucumber.spec.js
@@ -43,7 +43,9 @@ const {
   DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX,
   DI_DEBUG_ERROR_LINE_SUFFIX,
   TEST_RETRY_REASON,
-  DD_TEST_IS_USER_PROVIDED_SERVICE
+  DD_TEST_IS_USER_PROVIDED_SERVICE,
+  TEST_MANAGEMENT_ENABLED,
+  TEST_MANAGEMENT_IS_QUARANTINED
 } = require('../../packages/dd-trace/src/plugins/util/test')
 const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
 
@@ -2029,5 +2031,94 @@ versions.forEach(version => {
         }).catch(done)
       })
     })
+
+    context('quarantine', () => {
+      beforeEach(() => {
+        receiver.setQuarantinedTests({
+          cucumber: {
+            suites: {
+              'ci-visibility/features-quarantine/quarantine.feature': {
+                tests: {
+                  'Say quarantine': {
+                    properties: {
+                      quarantined: true
+                    }
+                  }
+                }
+              }
+            }
+          }
+        })
+      })
+
+      const getTestAssertions = (isQuarantining) =>
+        receiver
+          .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
+            const events = payloads.flatMap(({ payload }) => payload.events)
+            const failedTest = events.find(event => event.type === 'test').content
+            const testSession = events.find(event => event.type === 'test_session_end').content
+
+            if (isQuarantining) {
+              assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true')
+            } else {
+              assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED)
+            }
+
+            assert.equal(failedTest.resource, 'ci-visibility/features-quarantine/quarantine.feature.Say quarantine')
+
+            assert.equal(failedTest.meta[TEST_STATUS], 'fail')
+            if (isQuarantining) {
+              assert.propertyVal(failedTest.meta, TEST_MANAGEMENT_IS_QUARANTINED, 'true')
+            } else {
+              assert.notProperty(failedTest.meta, TEST_MANAGEMENT_IS_QUARANTINED)
+            }
+          })
+
+      const runTest = (done, isQuarantining, extraEnvVars) => {
+        const testAssertionsPromise = getTestAssertions(isQuarantining)
+
+        childProcess = exec(
+          './node_modules/.bin/cucumber-js ci-visibility/features-quarantine/*.feature',
+          {
+            cwd,
+            env: {
+              ...getCiVisAgentlessConfig(receiver.port),
+              ...extraEnvVars
+            },
+            stdio: 'inherit'
+          }
+        )
+
+        childProcess.on('exit', exitCode => {
+          testAssertionsPromise.then(() => {
+            if (isQuarantining) {
+              // even though a test fails, the exit code is 1 because the test is quarantined
+              assert.equal(exitCode, 0)
+            } else {
+              assert.equal(exitCode, 1)
+            }
+            done()
+          }).catch(done)
+        })
+      }
+
+      it('can quarantine tests', (done) => {
+        receiver.setSettings({ test_management: { enabled: true } })
+
+        runTest(done, true)
+      })
+
+      it('fails if quarantine is not enabled', (done) => {
+        receiver.setSettings({ test_management: { enabled: false } })
+
+        runTest(done, false)
+      })
+
+      it('does not enable quarantine tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => {
+        receiver.setSettings({ test_management: { enabled: true } })
+
+        runTest(done, false, { DD_TEST_MANAGEMENT_ENABLED: '0' })
+      })
+    })
   })
 })
diff --git a/integration-tests/cypress-esm-config.mjs b/integration-tests/cypress-esm-config.mjs
@@ -1,7 +1,7 @@
 import cypress from 'cypress'
 
 async function runCypress () {
-  await cypress.run({
+  const results = await cypress.run({
     config: {
       defaultCommandTimeout: 1000,
       e2e: {
@@ -39,6 +39,9 @@ async function runCypress () {
       screenshotOnRunFailure: false
     }
   })
+  if (results.totalFailed !== 0) {
+    process.exit(1)
+  }
 }
 
 runCypress()