test(mutation): [Hackweek] experiment with mutation testing

.github/workflows/mutation.yml

@@ -0,0 +1,195 @@
+name: 'Mutation Testing'
+on:
+  push:
+    branches:
+      - lms/test-hackweek-mutation-testing
+  workflow_dispatch:
+    inputs:
+      commit:
+        description: If the commit you want to test isn't the head of a branch, provide its SHA here
+        required: false
+  schedule:
+    # run every sunday at midnight:
+    - cron: '0 0 * * 0'
+
+# Cancel in progress workflows on pull_requests.
+# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  HEAD_COMMIT: ${{ github.event.inputs.commit || github.sha }}
+
+  # WARNING: this disables cross os caching as ~ and
+  # github.workspace evaluate to differents paths
+  CACHED_DEPENDENCY_PATHS: |
+    ${{ github.workspace }}/node_modules
+    ${{ github.workspace }}/packages/*/node_modules
+    ${{ github.workspace }}/dev-packages/*/node_modules
+    ~/.cache/mongodb-binaries/
+
+  # DEPENDENCY_CACHE_KEY: can't be set here because we don't have access to yarn.lock
+
+  # WARNING: this disables cross os caching as ~ and
+  # github.workspace evaluate to differents paths
+  # packages/utils/cjs and packages/utils/esm: Symlinks to the folders inside of `build`, needed for tests
+  CACHED_BUILD_PATHS: |
+    ${{ github.workspace }}/dev-packages/*/build
+    ${{ github.workspace }}/packages/*/build
+    ${{ github.workspace }}/packages/ember/*.d.ts
+    ${{ github.workspace }}/packages/gatsby/*.d.ts
+    ${{ github.workspace }}/packages/core/src/version.ts
+    ${{ github.workspace }}/packages/utils/cjs
+    ${{ github.workspace }}/packages/utils/esm
+
+  BUILD_CACHE_KEY: build-cache-${{ github.event.inputs.commit || github.sha }}
+  BUILD_CACHE_TARBALL_KEY: tarball-${{ github.event.inputs.commit || github.sha }}
+
+  # GH will use the first restore-key it finds that matches
+  # So it will start by looking for one from the same branch, else take the newest one it can find elsewhere
+  # We want to prefer the cache from the current develop branch, if we don't find any on the current branch
+  NX_CACHE_RESTORE_KEYS: |
+    nx-Linux-${{ github.ref }}-${{ github.event.inputs.commit || github.sha }}
+    nx-Linux-${{ github.ref }}
+    nx-Linux
+
+jobs:
+  job_get_metadata:
+    name: Get Metadata
+    runs-on: ubuntu-20.04
+    permissions:
+      pull-requests: read
+    steps:
+      - name: Check out current commit
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.HEAD_COMMIT }}
+          # We need to check out not only the fake merge commit between the PR and the base branch which GH creates, but
+          # also its parents, so that we can pull the commit message from the head commit of the PR
+          fetch-depth: 2
+
+      - name: Get metadata
+        id: get_metadata
+        # We need to try a number of different options for finding the head commit, because each kind of trigger event
+        # stores it in a different location
+        run: |
+          COMMIT_SHA=$(git rev-parse --short ${{ github.event.head_commit.id || env.HEAD_COMMIT }})
+          echo "COMMIT_SHA=$COMMIT_SHA" >> $GITHUB_ENV
+          echo "COMMIT_MESSAGE=$(git log -n 1 --pretty=format:%s $COMMIT_SHA)" >> $GITHUB_ENV
+
+    outputs:
+      commit_label: '${{ env.COMMIT_SHA }}: ${{ env.COMMIT_MESSAGE }}'
+
+  job_build:
+    name: Build
+    needs: job_get_metadata
+    runs-on: ubuntu-20.04
+    timeout-minutes: 15
+    steps:
+      - name: 'Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})'
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.HEAD_COMMIT }}
+
+      - name: Set up Node
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: 'package.json'
+
+      - name: Install Dependencies
+        uses: ./.github/actions/install-dependencies
+        id: install_dependencies
+
+      - name: Check build cache
+        uses: actions/cache@v4
+        id: cache_built_packages
+        with:
+          path: ${{ env.CACHED_BUILD_PATHS }}
+          key: ${{ env.BUILD_CACHE_KEY }}
+
+      - name: NX cache
+        uses: actions/cache@v4
+        with:
+          path: .nxcache
+          key: nx-Linux-${{ github.ref }}-${{ env.HEAD_COMMIT || github.sha }}
+          restore-keys:
+            ${{ env.NX_CACHE_RESTORE_KEYS || 'nx-never-restore'}}
+
+      - name: Build packages
+        run: yarn build
+
+      - name: Generate Sentry Trace
+        id: step_sentry_trace
+        run: |
+          node ./scripts/stryker/start-trace.mjs
+          echo output=${{ env.GITHUB_OUTPUT }}
+
+    outputs:
+      dependency_cache_key: ${{ steps.install_dependencies.outputs.cache_key }}
+      sentry_trace: ${{ steps.step_sentry_trace.outputs.sentry_trace }}
+      baggage: ${{ steps.step_sentry_trace.outputs.baggage }}
+
+  job_mutation_test:
+    name: Mutation Test (${{ matrix.package }})
+    needs: [job_get_metadata, job_build]
+    timeout-minutes: 180
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        package: [
+          'angular',
+          'astro',
+          'aws-serverless',
+          'browser',
+          'core',
+          'gatsby',
+          'google-cloud-serverless',
+          'nextjs',
+          'node',
+          'svelte',
+          'sveltekit',
+          'utils'
+        ]
+    steps:
+      - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.HEAD_COMMIT }}
+
+      - name: Set up Node
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: 'package.json'
+
+      - name: Restore caches
+        uses: ./.github/actions/restore-cache
+        env:
+          DEPENDENCY_CACHE_KEY: ${{ needs.job_build.outputs.dependency_cache_key }}
+
+      - name: NX cache
+        uses: actions/cache@v4
+        with:
+          path: .nxcache
+          key: nx-Linux-${{ github.ref }}-${{ env.HEAD_COMMIT || github.sha }}
+          restore-keys:
+            ${{ env.NX_CACHE_RESTORE_KEYS || 'nx-never-restore'}}
+
+      - name: Set trace propagation data
+        run: |
+          echo "SENTRY_MUT_SENTRY_TRACE=${{ needs.job_build.outputs.sentry_trace }}" >> $GITHUB_ENV
+          echo "SENTRY_MUT_BAGGAGE=${{ needs.job_build.outputs.baggage }}" >> $GITHUB_ENV
+
+      - name: Run mutation test
+        run: yarn nx run @sentry/${{ matrix.package }}:test:mutation
+        env:
+          NODE_VERSION: ${{ matrix.node }}
+
+      - name: Upload Results
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-reports ${{ matrix.package }}
+          path: |
+            ${{ github.workspace }}/packages/**/mutation.json
+            ${{ github.workspace }}/packages/**/mutation.html

.gitignore

@@ -10,6 +10,7 @@ build/
 # various integration test builds
 dist/
 coverage/
+reports/
 scratch/
 *.js.map
 *.pyc
@@ -42,6 +43,8 @@ local.log
 
 .rpt2_cache
 
+.stryker-tmp
+
 lint-results.json
 
 # legacy

dev-packages/jest-config/README.md

@@ -0,0 +1,3 @@
+# `@sentry-internal/jest-config`
+
+A package containing the base Jest config for the sentry-javascript SDK repo

dev-packages/jest-config/package.json

@@ -0,0 +1,26 @@
+{
+  "private": true,
+  "version": "8.26.0",
+  "name": "@sentry-internal/jest-config",
+  "author": "Sentry",
+  "license": "MIT",
+  "main": "./src/jest.config.js",
+  "files": [
+    "src"
+  ],
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "require": {
+        "default": "./src/jest.config.js"
+      }
+    }
+  },
+  "sideEffects": false,
+  "engines": {
+    "node": ">=14.18"
+  },
+  "volta": {
+    "extends": "../../package.json"
+  }
+}

dev-packages/jest-config/src/jest.config.js

@@ -0,0 +1,29 @@
+module.exports = {
+  // this is the package root, even when tests are being run at the repo level
+  rootDir: process.cwd(),
+  collectCoverage: true,
+  transform: {
+    '^.+\\.ts$': 'ts-jest',
+    '^.+\\.tsx$': 'ts-jest',
+  },
+  coverageDirectory: '<rootDir>/coverage',
+  moduleFileExtensions: ['js', 'ts', 'tsx'],
+  testMatch: ['<rootDir>/**/*.test.ts', '<rootDir>/**/*.test.tsx'],
+  moduleNameMapper: {
+    '^axios$': require.resolve('axios'),
+  },
+  globals: {
+    'ts-jest': {
+      tsconfig: '<rootDir>/tsconfig.test.json',
+    },
+    __DEBUG_BUILD__: true,
+  },
+  testPathIgnorePatterns: ['<rootDir>/build/', '<rootDir>/node_modules/', '<rootDir>/.stryker-tmp/'],
+
+  // On CI, we do not need the pretty CLI output, as it makes logs harder to parse
+  ...(process.env.CI
+    ? {
+        coverageReporters: ['json', 'lcov', 'clover'],
+      }
+    : {}),
+};

dev-packages/stryker-config/README.md

@@ -0,0 +1,3 @@
+# `@sentry-internal/stryker-config`
+
+A package containing the base Stryker config for the sentry-javascript SDK repo

dev-packages/stryker-config/package.json

@@ -0,0 +1,26 @@
+{
+  "private": true,
+  "version": "8.26.0",
+  "name": "@sentry-internal/stryker-config",
+  "author": "Sentry",
+  "license": "MIT",
+  "files": [
+    "src"
+  ],
+  "exports": {
+    "./package.json": "./package.json",
+    "./config": {
+      "import": {
+        "default": "./src/stryker.config.mjs"
+      }
+    }
+  },
+  "sideEffects": false,
+  "engines": {
+    "node": ">=14.18"
+  },
+  "volta": {
+    "extends": "../../package.json"
+  },
+  "type": "module"
+}

dev-packages/stryker-config/src/stryker.config.mjs

@@ -0,0 +1,25 @@
+import child_process from 'child_process';
+
+// STRYKER_DASHBOARD_API_KEY env variable needs to be set to upload to dashboard
+
+/** @type {import('@stryker-mutator/api/core').PartialStrykerOptions} */
+const config = {
+  packageManager: 'yarn',
+  reporters: ['html', 'clear-text', 'progress', 'json'],
+  testRunner: 'vitest',
+  coverageAnalysis: 'perTest',
+  ignoreStatic: true,
+  dashboard: {
+    // TODO: change to getsentry/sentry-javascript if we ship this into production
+    project: 'github.com/Lms24/sentry-javascript-test-fork',
+    version: child_process.execSync('git rev-parse HEAD').toString().trim(),
+  },
+  clearTextReporter: {
+    logTests: false,
+    reportMutants: false,
+    reportScoreTable: true,
+    skipFull: false,
+  },
+};
+
+export default config;

dev-packages/stryker-runner/README.md

@@ -0,0 +1,3 @@
+# `@sentry-internal/stryker-runner`
+
+A runner to run stryker with Sentry instrumentation to catch errors during test runs, as well as performance data

dev-packages/stryker-runner/package.json

@@ -0,0 +1,33 @@
+{
+  "private": true,
+  "version": "8.26.0",
+  "name": "@sentry-internal/stryker-runner",
+  "author": "Sentry",
+  "license": "MIT",
+  "files": [
+    "build"
+  ],
+  "exports": {
+    "./package.json": "./package.json"
+  },
+  "engines": {
+    "node": ">=14.18"
+  },
+  "volta": {
+    "extends": "../../package.json"
+  },
+  "dependencies": {
+    "@stryker-mutator/core": "^8.5.0",
+    "typescript": "^4.9.5"
+  },
+  "devDependencies": {
+    "@stryker-mutator/api": "^8.5.0"
+  },
+  "scripts": {
+    "build": "tsc",
+    "build:watch": "tsc --watch",
+    "start": "node ./build/esm/run.js"
+  },
+  "type": "module",
+  "bin": "./build/esm/run.js"
+}