DAG-1992 Generate only burn_in tasks when --burn-in is passed (#47)

Author: MikhailShchatko

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.6.0 - 2022-08-26
+* Generate only burn_in tasks when --burn-in is passed.
+
 ## 0.5.3 - 2022-08-19
 * Distribute tests without historic runtime data evenly between subsuites.
 

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "mongo-task-generator"
-version = "0.5.3"
+version = "0.6.0"
 repository = "https://github.com/mongodb/mongo-task-generator"
 authors = ["Decision Automation Group <[email protected]>"]
 edition = "2018"

Cargo.toml

@@ -157,6 +157,49 @@ of the task definition. When this tag is present, both the extra setup steps and
 of multiversion sub-tasks will be preformed. In order to only perform the extra setup steps
 the `"no_version_combinations"` tag should also be included.
 
+### Burn in tests and burn in tags
+
+Newly added or modified tests might become flaky. In order to avoid that, those tests can be run
+continuously multiple times in a row to see if the results are consistent. This process is called
+burn-in.
+
+`burn_in_tests_gen` task is used to generate burn-in tasks on the same buildvariant the task is
+added to. The [example](https://github.com/mongodb/mongo/blob/81c41bdfdc56f05973fae70e80e80919f18f50c9/etc/evergreen_yml_components/definitions.yml#L3252-L3256)
+of task configuration:
+
+```yaml
+- <<: *gen_task_template
+  name: burn_in_tests_gen
+  tags: []
+  commands:
+  - func: "generate resmoke tasks"
+```
+
+`burn_in_tags_gen` task is used to generate separate burn-in buildvariants. This way we can burn-in
+on the requested buildvariant as well as the other, additional buildvariants to ensure there is no
+difference between them.
+
+The [example](https://github.com/mongodb/mongo/blob/81c41bdfdc56f05973fae70e80e80919f18f50c9/etc/evergreen_yml_components/definitions.yml#L4317-L4321)
+of task configuration:
+
+```yaml
+- <<: *gen_task_template
+  name: burn_in_tags_gen
+  tags: []
+  commands:
+  - func: "generate resmoke tasks"
+```
+
+`burn_in_tag_buildvariants` buildvariant expansion is used to configure base buildvariant names.
+Base buildvariant names should be delimited by spaces. The [example](https://github.com/mongodb/mongo/blob/81c41bdfdc56f05973fae70e80e80919f18f50c9/etc/evergreen.yml#L1257)
+of `burn_in_tag_buildvariants` buildvariant expansion:
+
+```yaml
+burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-inmem enterprise-rhel-80-64-bit-multiversion
+```
+
+Burn-in related tasks are generated when `--burn-in` is passed.
+
 ## Working with generated tasks
 
 A generated tasks is typically composed of a number of related sub-tasks. Because evergreen does
@@ -221,6 +264,11 @@ if the default value does not apply.
   store your resmokeconfig is a different directory, you can adjust this value:
   `python buildscripts/resmoke.py --configDir=path/to/resmokeconfig`.
 * **target-directory**: Directory to write generated configuration to. This defaults to `generated_resmoke_config`.
+* **burn-in**: Whether to generate burn_in related tasks. If specified only burn_in tasks will be
+  generated.
+* **burn-in-tests-command**: How to invoke the burn_in_tests command. The burn_in_tests command is
+  used to discover modified or added tests and the tasks they being run on. It defaults to
+  `python buildscripts/burn_in_tests.py`.
 
 ## Usage help
 
@@ -232,6 +280,10 @@ USAGE:
     mongo-task-generator [OPTIONS] --expansion-file <EXPANSION_FILE>
 
 OPTIONS:
+        --burn-in
+            Generate burn_in related tasks
+        --burn-in-tests-command <BURN_IN_TESTS_COMMAND>
+            Command to invoke burn_in_tests [default: "python buildscripts/burn_in_tests.py"]
         --evg-auth-file <EVG_AUTH_FILE>
             File with information on how to authenticate against the evergreen API [default:
             ~/.evergreen.yml]
@@ -240,7 +292,7 @@ OPTIONS:
         --expansion-file <EXPANSION_FILE>
             File containing expansions that impact task generation
         --generate-sub-tasks-config <GENERATE_SUB_TASKS_CONFIG>
-            File containing configuration for generating sub-tasks            
+            File containing configuration for generating sub-tasks
     -h, --help
             Print help information
         --resmoke-command <RESMOKE_COMMAND>

docs/generating_tasks.md

@@ -128,6 +128,8 @@ pub struct ExecutionConfiguration<'a> {
     pub config_location: &'a str,
     /// Should burn_in tasks be generated.
     pub gen_burn_in: bool,
+    /// Command to execute burn_in_tests.
+    pub burn_in_tests_command: &'a str,
 }
 
 /// Collection of services needed to execution.
@@ -208,7 +210,7 @@ impl Dependencies {
             execution_config.gen_burn_in,
         ));
 
-        let burn_in_discovery = Arc::new(BurnInProxy::new());
+        let burn_in_discovery = Arc::new(BurnInProxy::new(execution_config.burn_in_tests_command));
         let burn_in_service = Arc::new(BurnInServiceImpl::new(
             burn_in_discovery,
             gen_resmoke_task_service,
@@ -467,8 +469,8 @@ impl GenerateTasksService for GenerateTasksServiceImpl {
             for task in &build_variant.tasks {
                 // Burn in tasks could be different for each build variant, so we will always
                 // handle them.
-                if task.name == BURN_IN_TESTS {
-                    if self.gen_burn_in {
+                if self.gen_burn_in {
+                    if task.name == BURN_IN_TESTS {
                         thread_handles.push(create_burn_in_worker(
                             deps,
                             task_map.clone(),
@@ -477,11 +479,8 @@ impl GenerateTasksService for GenerateTasksServiceImpl {
                             generated_tasks.clone(),
                         ));
                     }
-                    continue;
-                }
 
-                if task.name == BURN_IN_TAGS {
-                    if self.gen_burn_in {
+                    if task.name == BURN_IN_TAGS {
                         for base_bv_name in self
                             .evg_config_utils
                             .lookup_and_split_by_whitespace_build_variant_expansion(
@@ -501,6 +500,11 @@ impl GenerateTasksService for GenerateTasksServiceImpl {
                             ));
                         }
                     }
+
+                    continue;
+                }
+
+                if task.name == BURN_IN_TESTS || task.name == BURN_IN_TAGS {
                     continue;
                 }
 

src/lib.rs

@@ -16,6 +16,7 @@ use tracing_subscriber::fmt::format;
 const DEFAULT_EVG_AUTH_FILE: &str = "~/.evergreen.yml";
 const DEFAULT_EVG_PROJECT_FILE: &str = "etc/evergreen.yml";
 const DEFAULT_RESMOKE_COMMAND: &str = "python buildscripts/resmoke.py";
+const DEFAULT_BURN_IN_TESTS_COMMAND: &str = "python buildscripts/burn_in_tests.py";
 const DEFAULT_TARGET_DIRECTORY: &str = "generated_resmoke_config";
 
 /// Expansions from evergreen to determine settings for how task should be generated.
@@ -84,6 +85,10 @@ struct Args {
     /// Generate burn_in related tasks.
     #[clap(long)]
     burn_in: bool,
+
+    /// Command to invoke burn_in_tests.
+    #[clap(long, default_value = DEFAULT_BURN_IN_TESTS_COMMAND)]
+    burn_in_tests_command: String,
 }
 
 /// Configure logging for the command execution.
@@ -116,6 +121,7 @@ async fn main() {
         generating_task: &evg_expansions.task_name,
         config_location: &evg_expansions.config_location(),
         gen_burn_in: args.burn_in,
+        burn_in_tests_command: &args.burn_in_tests_command,
     };
     let deps = Dependencies::new(execution_config).unwrap();
 

src/main.rs

@@ -36,11 +36,27 @@ pub trait BurnInDiscovery: Send + Sync {
     fn discover_tasks(&self, build_variant: &str) -> Result<Vec<DiscoveredTask>>;
 }
 
-pub struct BurnInProxy {}
+pub struct BurnInProxy {
+    /// Primary command to invoke burn_in_tests (usually `python`).
+    burn_in_tests_cmd: String,
+    /// Script to invoke burn_in_tests.
+    burn_in_tests_script: Vec<String>,
+}
 
 impl BurnInProxy {
-    pub fn new() -> Self {
-        BurnInProxy {}
+    /// Create a new `BurnInProxy` instance.
+    ///
+    /// # Arguments
+    ///
+    /// * `burn_in_tests_cmd` - Command to invoke resmoke.
+    pub fn new(burn_in_tests_cmd: &str) -> Self {
+        let cmd_parts: Vec<_> = burn_in_tests_cmd.split(' ').collect();
+        let cmd = cmd_parts[0];
+        let script = cmd_parts[1..].iter().map(|s| s.to_string()).collect();
+        Self {
+            burn_in_tests_cmd: cmd.to_string(),
+            burn_in_tests_script: script,
+        }
     }
 }
 
@@ -55,13 +71,15 @@ impl BurnInDiscovery for BurnInProxy {
     ///
     /// A list of tasks/tests that were discovered by burn_in_tests.
     fn discover_tasks(&self, build_variant: &str) -> Result<Vec<DiscoveredTask>> {
-        let cmd = vec![
-            "python",
-            "buildscripts/burn_in_tests.py",
-            "--build-variant",
-            build_variant,
-            "--yaml",
-        ];
+        let mut cmd = vec![self.burn_in_tests_cmd.as_str()];
+        cmd.append(
+            &mut self
+                .burn_in_tests_script
+                .iter()
+                .map(|s| s.as_str())
+                .collect(),
+        );
+        cmd.append(&mut vec!["--build-variant", build_variant, "--yaml"]);
         let start = Instant::now();
 
         let cmd_output = run_command(&cmd)?;

src/resmoke/burn_in_proxy.rs

@@ -30,3 +30,38 @@ fn test_end2end_execution() {
     let files = std::fs::read_dir(tmp_dir_path).unwrap();
     assert_eq!(2647, files.into_iter().collect::<Vec<_>>().len());
 }
+
+#[test]
+fn test_end2end_burn_in_execution() {
+    let mut cmd = Command::cargo_bin("mongo-task-generator").unwrap();
+    let tmp_dir = TempDir::new("generated_resmoke_config").unwrap();
+
+    cmd.args(&[
+        "--target-directory",
+        tmp_dir.path().to_str().unwrap(),
+        "--expansion-file",
+        "tests/data/sample_expansions.yml",
+        "--evg-project-file",
+        "tests/data/evergreen.yml",
+        "--evg-auth-file",
+        "tests/data/sample_evergreen_auth.yml",
+        "--resmoke-command",
+        "python3 tests/mocks/resmoke.py",
+        "--use-task-split-fallback",
+        "--generate-sub-tasks-config",
+        "tests/data/sample_generate_subtasks_config.yml",
+        "--burn-in",
+        "--burn-in-tests-command",
+        "python3 tests/mocks/burn_in_tests.py",
+    ])
+    .assert()
+    .success();
+
+    let tmp_dir_path = tmp_dir.path();
+    assert!(tmp_dir_path.exists());
+
+    let files = std::fs::read_dir(tmp_dir_path).unwrap();
+    // Only one file `evergreen_config.json` should be generated.
+    // That means non-burn-in tasks are NOT generated.
+    assert_eq!(1, files.into_iter().collect::<Vec<_>>().len());
+}

tests/integration_test.rs

@@ -0,0 +1,22 @@
+"""Mock of burn_in_tests.py for testing task generation."""
+def burn_in_discovery():
+    print("""
+discovered_tasks:
+- task_name: jsCore
+  test_list:
+  - tests/data/tests/test_0.js
+- task_name: sharding_jscore_passthrough
+  test_list:
+  - tests/data/tests/test_0.js
+- task_name: replica_sets_jscore_passthrough
+  test_list:
+  - tests/data/tests/test_0.js
+    """)
+
+
+def main():
+    burn_in_discovery()
+
+
+if __name__ == '__main__':
+    main()