evergreen.yml

#                        🧞    Genny   🧞‍♀️
#
# This file defines how this project is built by evergreen at
#
#   https://evergreen.mongodb.com/waterfall/genny
#
# 1. Indent using 2 spaces. Structures start on indents (e.g. dashes aren't
#    indented). Don't quote unnecessarily.
#
# 2.  Use `f_` prefix for function names and `t_` prefix for task names.
#
#     This allows
#
#     1.  grep for all usage of a function/task without colliding with
#         inline shell etc.
#     2.  grep for definition of a function `FOO` by grepping for `f_FOO:`
#         (with a colon)
#
# 3.  Put comments in yaml not in inline shell.
#
# 4.  Keep yaml anchors/aliases to a minimum and always prefix with `a_`
#     for similar grep-ability. Favor functions and tasks with
#     dependencies.
#
# 5.  Always use functions for all task-bodies and pre/post-bodies rather
#     than inlines. This keeps the overall logic separate from the
#     implementation.
#
# 6.  Maintain one blank line between each unit (each function, task,
#     variant etc). Two lines between each section.
#
# 7.  Don't change the order of the sections. Goal is to start with "entry
#     points" (variants) and then define children in breadth-first order.
#
# 8.  Functions that are non-obvious must have a comment with the existing
#     comment style.
#
# 9.  Any function that requires parameters or `${}`-style expansions must
#     document those parameters appropriately such that callers don't need
#     to read the function definition to figure out what they mean.
#
#                                                                       🧞‍♀️

##                ⚡️ Variants ⚡️

ignore:
  - "*.md" # don't schedule tests if a commit only changes markdown files

buildvariants:
  - name: amazon2
    display_name: Amazon Linux 2 x64
    modules: [mongo, mothra]
    run_on:
      - amazon2-build
    tasks:
      - name: tg_compile_and_test_with_server_on_amz
      - name: t_compile_and_dry_run_asan

  - name: amazon2_arm64
    display_name: Amazon Linux 2 arm64
    modules: [mongo, mothra]
    run_on:
      - amazon2-arm64-large
    tasks:
      - name: tg_compile_and_test_with_server_on_amz
      - name: t_compile_and_dry_run_asan

  - name: amazon2023
    display_name: Amazon Linux 2023 x64
    modules: [mongo, mothra]
    run_on:
      - amazon2023.0-large
    tasks:
      - name: t_compile_and_dry_run
      - name: t_compile_and_dry_run_asan

  - name: amazon2023_arm64
    display_name: Amazon Linux 2023 arm64
    modules: [mongo, mothra]
    run_on:
      - amazon2023.0-arm64-large
    tasks:
      - name: t_compile_and_dry_run
      - name: t_compile_and_dry_run_asan

  - name: rhel70
    display_name: RHEL 7
    modules: [mongo, mothra]
    run_on:
      - rhel70
    tasks:
      - name: tg_compile

  - name: rhel8
    display_name: RHEL 8
    modules: [mongo, mothra]
    run_on:
      - rhel80-test
    tasks:
      - name: tg_compile

  - name: ubuntu2204
    display_name: Ubuntu 22.04
    modules: [mongo, mothra]
    run_on:
      - ubuntu2204-large
    tasks:
      - name: tg_compile

  - name: macos-1100
    display_name: macOS Big Sur x64
    modules: [mongo, mothra]
    run_on:
      - macos-1100
    tasks:
      - name: tg_compile_and_test_with_server_on_macos

  - name: macos-1100-arm64
    display_name: macOS Big Sur arm64
    modules: [mongo, mothra]
    run_on:
      - macos-1100-arm64
    tasks:
      - name: tg_compile_and_test_with_server_on_macos

  - name: validate-genny-tasks
    display_name: Validate Genny Tasks
    modules: [dsi, PrivateWorkloads]
    run_on:
      - amazon2-arm64-latest-small
    tasks:
      - name: validate_genny_tasks

##                ⚡️ Tasks ⚡️

tasks:
  - name: t_compile
    commands:
      - func: f_fetch_source
      - func: f_compile

  - name: t_compile_and_dry_run
    commands:
      - func: f_fetch_source
      - func: f_compile
      - func: f_dry_run_variant_auto_tasks
      - func: f_dry_run_workloads

  - name: t_compile_and_dry_run_asan
    commands:
      - func: f_fetch_source
      - func: f_compile_asan
      - func: f_dry_run_variant_auto_tasks
      - func: f_dry_run_workloads_asan

  - name: t_python_test
    commands:
      - func: f_python_test

  - name: t_lint_workloads
    commands:
      - func: f_lint_workloads

  - name: t_check_generated_docs
    commands:
      - func: f_check_generated_docs

  - name: t_lint_python
    commands:
      - func: f_fetch_source
      - func: f_lint_python

  - name: t_cmake_test
    commands:
      - func: f_cmake_test

  - name: t_cmake_test_ignore_flaky
    commands:
      - func: f_cmake_test_ignore_flaky

  - name: t_benchmark_test
    commands:
      - func: f_benchmark_test

  # It's not possible to do a standalone version due to
  # some tests' reliance on an apm callback that doesn't
  # work on standalone. (Specifically, since read concern
  # is irrelevant on a standalone, recent driver versions
  # simply omit it.)
  - name: t_integration_test_single_node_replset
    commands:
      - command: subprocess.exec
        params:
          binary: ./src/genny/run-genny
          args:
            - -v
            - resmoke-test
            - --suites
            - ../../src/genny/src/resmokeconfig/genny_single_node_replset.yml

  - name: t_integration_test_three_node_replset
    commands:
      - command: subprocess.exec
        params:
          binary: ./src/genny/run-genny
          args:
            - -v
            - resmoke-test
            - --suites
            - ../../src/genny/src/resmokeconfig/genny_three_node_replset.yml

  - name: t_integration_test_sharded
    commands:
      - command: subprocess.exec
        params:
          binary: ./src/genny/run-genny
          args:
            - -v
            - resmoke-test
            - --suites
            - ../../src/genny/src/resmokeconfig/genny_sharded.yml

  # Must run this task last to avoid polluting test results with tests generated
  # by the new actor.
  - name: t_test_create_new_actor_script
    commands:
      - func: f_run_create_new_actor_script
      - func: f_compile
      - func: f_dry_run_new_actor_script # Test the generated yaml file is valid.
      - command: subprocess.exec
        params:
          binary: ./src/genny/run-genny
          args:
            - -v
            - resmoke-test
            - --create-new-actor-test-suite

  - name: validate_genny_tasks
    commands:
      - func: f_fetch_source
      - func: f_validate_genny_tasks

##                ⚡️ Task Groups ⚡️

task_groups:
  - &a_compile_and_test
    name: tg_compile_and_test
    max_hosts: 1
    teardown_task:
      # Attaching results is prohibited in "teardown_group".
      # So we call it in "teardown_task" but make the result file optional.
      - func: f_report_test_results
    tasks:
      # Rely on implicit dependencies in task_groups where tasks are executed
      # in the order they're defined. Tasks with explicit "depends_on" tasks
      # may conflict with task_group's implicit dependency, causing
      # unexpected behavior.
      #
      # For this task group, the implicit dependency is that t_*_test has to
      # run after t_compile_and_dry_run.
      - t_compile_and_dry_run
      - t_python_test
      # - t_lint_python
      - t_lint_workloads
      - t_check_generated_docs
      - t_cmake_test

  - !!merge <<: *a_compile_and_test
    name: tg_compile_and_benchmark
    tasks:
      - t_compile_and_dry_run
      - t_benchmark_test

  - !!merge <<: *a_compile_and_test
    name: tg_compile_and_test_with_server_on_amz
    tasks:
      - t_compile
      - t_python_test
      # - t_lint_python
      - t_lint_workloads
      - t_check_generated_docs
      - t_cmake_test
      - t_integration_test_single_node_replset
      - t_integration_test_three_node_replset
      - t_integration_test_sharded
      - t_test_create_new_actor_script # Must run this task last.

  - !!merge <<: *a_compile_and_test
    name: tg_compile_and_test_with_server_on_macos
    tasks:
      - t_compile_and_dry_run
      - t_python_test
      - t_cmake_test_ignore_flaky
      - t_integration_test_single_node_replset
      - t_integration_test_three_node_replset
      - t_integration_test_sharded
      - t_test_create_new_actor_script # Must run this task last.

  - name: tg_compile
    max_hosts: 1
    tasks:
      - t_compile_and_dry_run

##                ⚡️ Functions ⚡️

functions:
  f_fetch_source:
    - command: manifest.load
    - command: git.get_project
      params:
        directory: src/genny
        revisions:
          mongo: e61bf27c2f6a83fed36e5a13c008a32d563babe2

  ##
  # Compile the project in src/build.
  #
  # We always remove the toolchain and curator directories because hosts
  # may have incomplete or invalid toolchain directories from
  # previous (unsuccessful) builds in the gennytoolchain project.
  ##
  f_compile:
    - command: subprocess.exec
      params:
        binary: ./src/genny/run-genny
        args:
          - -v
          - install

  f_compile_asan:
    - command: subprocess.exec
      params:
        binary: ./src/genny/run-genny
        args:
          - -v
          - install
          - --sanitizer=asan

  ##
  # Runs schedule_variant_auto_tasks as a dry run
  # to verify AutoRun logic in all workload YAML files.
  ##
  f_dry_run_variant_auto_tasks:
    - command: subprocess.exec
      params:
        binary: ./src/genny/run-genny
        args:
          - auto-tasks
          - --tasks
          - variant_tasks
          - --dry-run

  ##
  # Runs each workload as a dry run
  # Requires f_compile to have been run first.
  ##
  f_dry_run_workloads:
    - command: subprocess.exec
      params:
        binary: ./src/genny/run-genny
        args:
          - -v
          - dry-run-workloads

  ##
  # Runs each workload as a dry run
  # Requires f_compile_asan to have been run first.
  ##
  f_dry_run_workloads_asan:
    - command: subprocess.exec
      params:
        binary: ./src/genny/run-genny
        args:
          - -v
          - dry-run-workloads

  ##
  # Dry-runs workload generated by f_run_create_new_actor_script
  # Requires f_compile to have been run first.
  ##
  f_dry_run_new_actor_script:
    - command: subprocess.exec
      params:
        binary: ./src/genny/run-genny
        args:
          - -v
          - dry-run-workloads
          - --workload
          - ./src/genny/src/workloads/docs/SelfTestActor.yml

  ##
  # Runs tests via ctest.
  # Requires f_compile to have been run first.
  ##
  f_cmake_test:
    - command: subprocess.exec
      params:
        binary: ./src/genny/run-genny
        args:
          - -v
          - cmake-test

  ##
  # Runs tests via ctest, ignoring flaky tests (see EVG-19909).
  # Requires f_compile to have been run first.
  # Feel free to update regex_clude per the latest test results
  ##
  f_cmake_test_ignore_flaky:
    - command: subprocess.exec
      params:
        binary: ./src/genny/run-genny
        args:
          - -v
          - cmake-test
          - --regex-exclude
          - "IgnoredOnMacOs"

  ##
  # Runs benchmarks via ctest.
  # Requires f_compile to have been run first.
  ##
  f_benchmark_test:
    - command: subprocess.exec
      params:
        binary: ./src/genny/run-genny
        args:
          - -v
          - benchmark-test

  ##
  # Runs tests for the create_new_actor script.
  ##
  f_run_create_new_actor_script:
    - command: subprocess.exec
      params:
        binary: ./src/genny/run-genny
        args:
          - -v
          - create-new-actor
          - SelfTestActor

  ##
  # Runs python unit tests.
  ##
  f_python_test:
    - command: subprocess.exec
      params:
        binary: ./src/genny/run-genny
        args:
          - -v
          # TODO: rename to pytest
          - self-test

  ##
  # Runs the YAML linter for Genny workloads
  ##
  f_lint_workloads:
    - command: subprocess.exec
      params:
        binary: ./src/genny/run-genny
        args:
          - -v
          - lint-yaml
          - --format
  ##
  # Lint python
  ##
  f_lint_python:
    - command: subprocess.exec
      params:
        binary: ./src/genny/run-genny
        args:
          - -v
          - lint-python

  f_check_generated_docs:
    - command: shell.exec
      params:
        script: |
          cd src/genny

          ./run-genny generate-docs

          # Check for changes
          if git diff --quiet; then
              echo 'No changes in the repository'
              exit 0
          else
              echo 'Workload documentation needs to be regenerated. Please run \`./run-genny generate-docs\` and commit the changes.'
              exit 1
          fi

  ##
  # Reports test results to evergreen API.
  ##
  f_report_test_results:
    - command: shell.exec
      params:
        script: |
          pwd
          find . -iname '*.xml'
          ls build/XUnitXML/*.xml
          cat build/XUnitXML/*.xml
    - command: attach.xunit_results
      params:
        file: build/XUnitXML/*.xml

  ##
  # Validate that the auto-generated genny tasks are up to date.
  ##
  f_validate_genny_tasks:
    - command: shell.exec
      params:
        working_dir: src/genny
        script: |
          set -eu
          versions=("master" "8.0" "7.0" "6.0" "5.0")

          # Move original files to backup locations
          for version in $versions; do
            mv evergreen/system_perf/$version/genny_tasks.yml evergreen/system_perf/$version/genny_tasks.bak.yml
          done

          # Regenerate the genny task files
          ./run-genny auto-tasks-local --dsi-path ../modules/dsi --private-workloads-path ../modules/PrivateWorkloads

          # Validate that the files are identical
          for version in $versions; do
            original="evergreen/system_perf/$version/genny_tasks.bak.yml"
            new="evergreen/system_perf/$version/genny_tasks.yml"
            exitCode=0
            diff $original $new || exitCode=1

            if [ $exitCode -ne 0 ]; then
              echo -e '\033[31mThe auto-generated genny task files are not up to date!\033[0m'
              echo -e '\033[31mThis typically happens if you are adding a new genny test,\033[0m'
              echo -e '\033[31mupdate the AutoRun section of a test, or someone added a new variant in DSI and did not regenerate the files.\033[0m'
              echo -e '\033[31mTo resolve this issue, please run ./run-genny auto-tasks-local and commit the new files in the evergreen folder.\033[0m'
              exit 1
            fi
          done

##                ⚡️ Modules ⚡️

modules:
  - name: mongo
    owner: 10gen
    repo: mongo
    # This prefix is relative to the path in git.get_project
    # I.e., src/genny/../../src = ./src
    prefix: ../../src
    branch: master
  - name: mothra
    owner: 10gen
    repo: mothra
    branch: main
  - name: dsi
    owner: 10gen
    repo: dsi
    prefix: ../modules
    branch: master
    auto_update: true
  - name: PrivateWorkloads
    owner: 10gen
    repo: PrivateWorkloads
    prefix: ../modules
    branch: production
    auto_update: true