DAG-1940 Generate tasks separately for Windows, MacOS, Linux distro groups (#43)

Author: MikhailShchatko

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.5.0 - 2022-08-01
+
+* Generate tasks separately for Windows, MacOS, Linux distro groups.
+
 ## 0.4.7 - 2022-07-14
 
 * Add support for burn_in_tags generation.

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "mongo-task-generator"
-version = "0.4.7"
+version = "0.5.0"
 repository = "https://github.com/mongodb/mongo-task-generator"
 authors = ["Decision Automation Group <[email protected]>"]
 edition = "2018"

Cargo.toml

@@ -1,4 +1,5 @@
 use std::collections::{HashMap, HashSet};
+use std::vec;
 
 use anyhow::{bail, Result};
 use lazy_static::lazy_static;
@@ -7,7 +8,9 @@ use shrub_rs::models::commands::EvgCommand::Function;
 use shrub_rs::models::params::ParamValue;
 use shrub_rs::models::{commands::FunctionCall, task::EvgTask, variant::BuildVariant};
 
-use crate::evergreen_names::{ENTERPRISE_MODULE, GENERATE_RESMOKE_TASKS, IS_FUZZER};
+use crate::evergreen_names::{
+    ENTERPRISE_MODULE, GENERATE_RESMOKE_TASKS, IS_FUZZER, LINUX, MACOS, WINDOWS,
+};
 use crate::utils::task_name::remove_gen_suffix;
 
 lazy_static! {
@@ -229,6 +232,17 @@ pub trait EvgConfigUtils: Sync + Send {
     ///
     /// true if given build variant includes the enterprise module.
     fn is_enterprise_build_variant(&self, build_variant: &BuildVariant) -> bool;
+
+    /// Infer platform that build variant will be running on.
+    ///
+    /// # Arguments
+    ///
+    /// * `build_variant` - Build variant to query.
+    ///
+    /// # Returns
+    ///
+    /// Linux, or Mac, or Windows platform that build variant will be running on.
+    fn infer_build_variant_platform(&self, build_variant: &BuildVariant) -> String;
 }
 
 /// Service for utilities to help interpret evergreen configuration.
@@ -581,6 +595,33 @@ impl EvgConfigUtils for EvgConfigUtilsImpl {
             false
         }
     }
+
+    /// Infer platform that build variant will run on.
+    ///
+    /// # Arguments
+    ///
+    /// * `build_variant` - Build variant to query.
+    ///
+    /// # Returns
+    ///
+    /// linux, or mac, or windows platform that build variant will run on.
+    fn infer_build_variant_platform(&self, build_variant: &BuildVariant) -> String {
+        let distro = build_variant
+            .run_on
+            .as_ref()
+            .unwrap_or(&vec!["".to_string()])
+            .first()
+            .unwrap_or(&"".to_string())
+            .to_lowercase();
+
+        if distro.contains(MACOS) {
+            MACOS.to_string()
+        } else if distro.contains(WINDOWS) {
+            WINDOWS.to_string()
+        } else {
+            LINUX.to_string()
+        }
+    }
 }
 
 /// Get the shrub function make the 'generate resmoke task' call in the given task.
@@ -1271,4 +1312,27 @@ mod tests {
 
         assert!(!evg_config_utils.is_enterprise_build_variant(&build_variant));
     }
+
+    // tests for infer_build_variant_platform
+    #[rstest]
+    #[case(Some(vec!["rhel80-small".to_string()]), "linux".to_string())]
+    #[case(Some(vec!["windows-vsCurrent-small".to_string()]), "windows".to_string())]
+    #[case(Some(vec!["macos-1100".to_string()]), "macos".to_string())]
+    #[case(Some(vec!["rhel80-small".to_string(), "macos-1100".to_string()]), "linux".to_string())]
+    #[case(Some(vec![]), "linux".to_string())]
+    fn test_infer_build_variant_platform(
+        #[case] distros: Option<Vec<String>>,
+        #[case] platform: String,
+    ) {
+        let build_variant = BuildVariant {
+            run_on: distros,
+            ..Default::default()
+        };
+        let evg_config_utils = EvgConfigUtilsImpl::new();
+
+        assert_eq!(
+            evg_config_utils.infer_build_variant_platform(&build_variant),
+            platform
+        );
+    }
 }

src/evergreen/evg_config_utils.rs

@@ -111,3 +111,11 @@ pub const MULTIVERSION_LAST_CONTINUOUS: &str = "last_continuous";
 // Distros
 /// Name of compile task distro for burn_in_tags.
 pub const COMPILE_TASK_DISTRO: &str = "rhel80-xlarge";
+
+// Distro group names
+/// Windows distro group name.
+pub const WINDOWS: &str = "windows";
+/// MacOS distro group name.
+pub const MACOS: &str = "macos";
+/// Linux distro group name.
+pub const LINUX: &str = "linux";

src/evergreen_names.rs

@@ -461,6 +461,9 @@ impl GenerateTasksService for GenerateTasksServiceImpl {
             let is_enterprise = self
                 .evg_config_utils
                 .is_enterprise_build_variant(build_variant);
+            let platform = self
+                .evg_config_utils
+                .infer_build_variant_platform(build_variant);
             for task in &build_variant.tasks {
                 // Burn in tasks could be different for each build variant, so we will always
                 // handle them.
@@ -502,7 +505,7 @@ impl GenerateTasksService for GenerateTasksServiceImpl {
                 }
 
                 // Skip tasks that have already been seen.
-                let task_name = lookup_task_name(is_enterprise, &task.name);
+                let task_name = lookup_task_name(is_enterprise, &task.name, &platform);
                 if seen_tasks.contains(&task_name) {
                     continue;
                 }
@@ -553,13 +556,24 @@ impl GenerateTasksService for GenerateTasksServiceImpl {
 
             Some(self.gen_fuzzer_service.generate_fuzzer_task(&params)?)
         } else {
-            event!(Level::INFO, "Generating resmoke task: {}", task_def.name);
             let is_enterprise = self
                 .evg_config_utils
                 .is_enterprise_build_variant(build_variant);
-            let params = self
-                .config_extraction_service
-                .task_def_to_resmoke_params(task_def, is_enterprise)?;
+            let platform = self
+                .evg_config_utils
+                .infer_build_variant_platform(build_variant);
+            event!(
+                Level::INFO,
+                "Generating resmoke task: {}, is_enterprise: {}, platform: {}",
+                task_def.name,
+                is_enterprise,
+                platform
+            );
+            let params = self.config_extraction_service.task_def_to_resmoke_params(
+                task_def,
+                is_enterprise,
+                Some(platform),
+            )?;
             Some(
                 self.gen_resmoke_service
                     .generate_resmoke_task(&params, &build_variant.name)
@@ -593,6 +607,9 @@ impl GenerateTasksService for GenerateTasksServiceImpl {
             let is_enterprise = self
                 .evg_config_utils
                 .is_enterprise_build_variant(build_variant);
+            let platform = self
+                .evg_config_utils
+                .infer_build_variant_platform(build_variant);
             let mut gen_config = GeneratedConfig::new();
             let mut generating_tasks = vec![];
             let large_distro_name = self
@@ -617,7 +634,7 @@ impl GenerateTasksService for GenerateTasksServiceImpl {
                 let task_name = if task.name == BURN_IN_TESTS {
                     format!("{}-{}", BURN_IN_PREFIX, build_variant.name)
                 } else {
-                    lookup_task_name(is_enterprise, &task.name)
+                    lookup_task_name(is_enterprise, &task.name, &platform)
                 };
 
                 if let Some(generated_task) = generated_tasks.get(&task_name) {
@@ -687,20 +704,17 @@ impl GenerateTasksService for GenerateTasksServiceImpl {
 /// # Arguments
 ///
 /// * `is_enterprise` - Whether the task is for an enterprise build variant.
-/// * `task` - Evergreen definition of task.
+/// * `task` - Name of task.
+/// * `platform` - Platform that task will run on.
 ///
 /// # Returns
 ///
 /// Name to use for task.
-fn lookup_task_name(is_enterprise: bool, task_name: &str) -> String {
-    if task_name == BURN_IN_TESTS {
-        return task_name.to_string();
-    }
-
+fn lookup_task_name(is_enterprise: bool, task_name: &str, platform: &str) -> String {
     if is_enterprise {
-        format!("{}-{}", task_name, ENTERPRISE_MODULE)
+        format!("{}-{}-{}", task_name, platform, ENTERPRISE_MODULE)
     } else {
-        task_name.to_string()
+        format!("{}-{}", task_name, platform)
     }
 }
 
@@ -735,7 +749,8 @@ fn create_task_worker(
             .unwrap();
 
         let is_enterprise = evg_config_utils.is_enterprise_build_variant(&build_variant);
-        let task_name = lookup_task_name(is_enterprise, &task_def.name);
+        let platform = evg_config_utils.infer_build_variant_platform(&build_variant);
+        let task_name = lookup_task_name(is_enterprise, &task_def.name, &platform);
 
         if let Some(generated_task) = generated_task {
             let mut generated_tasks = generated_tasks.lock().unwrap();
@@ -933,15 +948,16 @@ mod tests {
 
     // tests for lookup_task_name.
     #[rstest]
-    #[case(false, "my_task", "my_task")]
-    #[case(true, "my_task", "my_task-enterprise")]
+    #[case(false, "my_task", "my_platform", "my_task-my_platform")]
+    #[case(true, "my_task", "my_platform", "my_task-my_platform-enterprise")]
     fn test_lookup_task_name_should_use_enterprise_when_specified(
         #[case] is_enterprise: bool,
         #[case] task_name: &str,
+        #[case] platform: &str,
         #[case] expected_task_name: &str,
     ) {
         assert_eq!(
-            lookup_task_name(is_enterprise, task_name),
+            lookup_task_name(is_enterprise, task_name, platform),
             expected_task_name.to_string()
         );
     }
@@ -1048,6 +1064,10 @@ mod tests {
         fn is_enterprise_build_variant(&self, _build_variant: &BuildVariant) -> bool {
             todo!()
         }
+
+        fn infer_build_variant_platform(&self, _build_variant: &BuildVariant) -> String {
+            todo!()
+        }
     }
 
     struct MockResmokeConfigActorService {}
@@ -1083,6 +1103,7 @@ mod tests {
             &self,
             _task_def: &EvgTask,
             _is_enterprise: bool,
+            _platform: Option<String>,
         ) -> Result<ResmokeGenParams> {
             todo!()
         }

src/lib.rs

@@ -37,6 +37,8 @@ pub trait ConfigExtractionService: Sync + Send {
     /// # Arguments
     ///
     /// * `task-def` - Task definition of task to generate.
+    /// * `is_enterprise` - Is this being generated for an enterprise build variant.
+    /// * `platform` - Platform that task will run on.
     ///
     /// # Returns
     ///
@@ -45,6 +47,7 @@ pub trait ConfigExtractionService: Sync + Send {
         &self,
         task_def: &EvgTask,
         is_enterprise: bool,
+        platform: Option<String>,
     ) -> Result<ResmokeGenParams>;
 }
 
@@ -160,6 +163,7 @@ impl ConfigExtractionService for ConfigExtractionServiceImpl {
             config_location: self.config_location.clone(),
             dependencies: self.determine_task_dependencies(task_def),
             is_enterprise,
+            platform: Some(evg_config_utils.infer_build_variant_platform(build_variant)),
         })
     }
 
@@ -169,6 +173,7 @@ impl ConfigExtractionService for ConfigExtractionServiceImpl {
     ///
     /// * `task_def` - Task definition of task to generate.
     /// * `is_enterprise` - Is this being generated for an enterprise build variant.
+    /// * `platform` - Platform that task will run on.
     ///
     /// # Returns
     ///
@@ -177,6 +182,7 @@ impl ConfigExtractionService for ConfigExtractionServiceImpl {
         &self,
         task_def: &EvgTask,
         is_enterprise: bool,
+        platform: Option<String>,
     ) -> Result<ResmokeGenParams> {
         let task_name = remove_gen_suffix(&task_def.name).to_string();
         let suite = self.evg_config_utils.find_suite_name(task_def).to_string();
@@ -209,6 +215,7 @@ impl ConfigExtractionService for ConfigExtractionServiceImpl {
             dependencies: self.determine_task_dependencies(task_def),
             is_enterprise,
             pass_through_vars: self.evg_config_utils.get_gen_task_vars(task_def),
+            platform,
         })
     }
 }

src/services/config_extraction.rs

@@ -169,7 +169,7 @@ impl BurnInServiceImpl {
         for (index, test) in discovered_task.test_list.iter().enumerate() {
             let mut params = self
                 .config_extraction_service
-                .task_def_to_resmoke_params(task_def, false)?;
+                .task_def_to_resmoke_params(task_def, false, None)?;
             update_resmoke_params_for_burn_in(&mut params, test);
 
             if params.generate_multiversion_combos {
@@ -238,6 +238,7 @@ impl BurnInServiceImpl {
             origin_suite: origin_suite.clone(),
             mv_exclude_tags: suite_info.multiversion_tags.clone(),
             is_enterprise: false,
+            platform: None,
         };
 
         self.gen_resmoke_task_service.build_resmoke_sub_task(
@@ -513,6 +514,7 @@ mod tests {
             &self,
             _task_def: &EvgTask,
             _is_enterprise: bool,
+            _platform: Option<String>,
         ) -> Result<ResmokeGenParams> {
             Ok(ResmokeGenParams {
                 generate_multiversion_combos: self.is_multiversion,

src/task_types/burn_in_tests.rs

@@ -57,6 +57,8 @@ pub struct FuzzerGenTaskParams {
     pub dependencies: Vec<String>,
     /// Is this task for enterprise builds.
     pub is_enterprise: bool,
+    /// Name of platform the task will run on.
+    pub platform: Option<String>,
 }
 
 impl FuzzerGenTaskParams {
@@ -273,6 +275,7 @@ fn build_fuzzer_sub_task(
         Some(sub_task_index),
         params.num_tasks as usize,
         params.is_enterprise,
+        params.platform.as_deref(),
     );
 
     let mut commands = vec![];

src/task_types/fuzzer_tasks.rs

@@ -148,7 +148,13 @@ impl WriteConfigActorImpl {
 
                 let filename = format!(
                     "{}.yml",
-                    name_generated_task(&s.name, s.index, total_tasks, s.is_enterprise)
+                    name_generated_task(
+                        &s.name,
+                        s.index,
+                        total_tasks,
+                        s.is_enterprise,
+                        s.platform.as_deref()
+                    )
                 );
                 let mut path = PathBuf::from(&self.target_dir);
                 path.push(filename);
@@ -182,7 +188,13 @@ impl WriteConfigActorImpl {
                 let misc_config = origin_config.with_new_tests(None, Some(&test_list));
                 let filename = format!(
                     "{}.yml",
-                    name_generated_task(&s.name, s.index, total_tasks, s.is_enterprise)
+                    name_generated_task(
+                        &s.name,
+                        s.index,
+                        total_tasks,
+                        s.is_enterprise,
+                        s.platform.as_deref()
+                    )
                 );
                 let mut path = PathBuf::from(&self.target_dir);
                 path.push(filename);