From ba3002ebc74a090122336074bd8ae28e2926a31c Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Fri, 18 Oct 2024 08:44:13 -0400
Subject: [PATCH] teuthology/suite: merge base_config with other fragments

Presently the code tries to merge the base_config when the worker starts
running. There's no need to construct it this way and it prevents sharing the
"defaults" with the fragment merging infrastructure. It also prevents
overriding defaults like:

    kernel
        branch: wip-pdonnell-i66704
        client:
            branch: wip-pdonnell-i66704
            flavor: default
            kdb: 1
            sha1: 745cacd8f31e50d7f3b6039bbd8c9a8dfc07bf03
        flavor: default
        kdb: 1
        sha1: 745cacd8f31e50d7f3b6039bbd8c9a8dfc07bf03

A YAML fragment set kernel.client but it cannot delete the changes to
kernel.(branch|flavor|kdb|sha1) because there's no way to remove YAML elements
via a deep merge.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 teuthology/suite/merge.py          |  4 +++-
 teuthology/suite/run.py            | 15 +--------------
 teuthology/suite/test/test_run_.py |  3 +--
 3 files changed, 5 insertions(+), 17 deletions(-)

diff --git a/teuthology/suite/merge.py b/teuthology/suite/merge.py
index f13058e47..e3bf78c5a 100644
--- a/teuthology/suite/merge.py
+++ b/teuthology/suite/merge.py
@@ -115,12 +115,14 @@ def config_merge(configs, suite_name=None, **kwargs):
     the entire job (config) from the list.
     """
     seed = kwargs.setdefault('seed', 1)
+    base_config = kwargs.setdefault('base_config', {})
     if not isinstance(seed, int):
         log.debug("no valid seed input: using 1")
         seed = 1
     log.debug("configuring Lua randomseed to %d", seed)
     L.execute(f'local math = require"math"; math.randomseed({seed});')
     new_script = L.eval('new_script')
+    base_config_yaml = yaml.safe_load(str(base_config))
     yaml_cache = {}
     for desc, paths in configs:
         log.debug("merging config %s", desc)
@@ -128,7 +130,7 @@ def config_merge(configs, suite_name=None, **kwargs):
         if suite_name is not None:
             desc = combine_path(suite_name, desc)
 
-        yaml_complete_obj = {}
+        yaml_complete_obj = copy.deepcopy(base_config_yaml)
         deep_merge(yaml_complete_obj, dict(TEUTHOLOGY_TEMPLATE))
         for path in paths:
             if path not in yaml_cache:
diff --git a/teuthology/suite/run.py b/teuthology/suite/run.py
index cc6859b73..ba72a4334 100644
--- a/teuthology/suite/run.py
+++ b/teuthology/suite/run.py
@@ -9,7 +9,6 @@
 
 from humanfriendly import format_timespan
 
-from tempfile import NamedTemporaryFile
 from teuthology import repo_utils
 
 from teuthology.config import config, JobConfig
@@ -639,17 +638,10 @@ def schedule_suite(self):
             filter_out=self.args.filter_out,
             filter_all=self.args.filter_all,
             filter_fragments=self.args.filter_fragments,
+            base_config=self.base_config,
             seed=self.args.seed,
             suite_name=suite_name))
 
-        # create, but do not write, the temp file here, so it can be
-        # added to the args in collect_jobs, but not filled until
-        # any backtracking is done
-        base_yaml_path = NamedTemporaryFile(
-            prefix='schedule_suite_', delete=False
-        ).name
-        self.base_yaml_paths.insert(0, base_yaml_path)
-
         # compute job limit in respect of --sleep-before-teardown
         job_limit = self.args.limit or 0
         sleep_before_teardown = int(self.args.sleep_before_teardown or 0)
@@ -714,9 +706,6 @@ def schedule_suite(self):
                     dry_run=self.args.dry_run,
                 )
 
-        with open(base_yaml_path, 'w+b') as base_yaml:
-            base_yaml.write(str(self.base_config).encode())
-
         if jobs_to_schedule:
             self.write_rerun_memo()
 
@@ -728,8 +717,6 @@ def schedule_suite(self):
 
         self.schedule_jobs(jobs_missing_packages, jobs_to_schedule, name)
 
-        os.remove(base_yaml_path)
-
         count = len(jobs_to_schedule)
         missing_count = len(jobs_missing_packages)
         total_count = count
diff --git a/teuthology/suite/test/test_run_.py b/teuthology/suite/test/test_run_.py
index 0fad56ddf..2b8f81e33 100644
--- a/teuthology/suite/test/test_run_.py
+++ b/teuthology/suite/test/test_run_.py
@@ -5,7 +5,7 @@
 import yaml
 
 from datetime import datetime, timedelta, timezone
-from mock import patch, call, ANY
+from mock import patch, call
 from io import StringIO
 from io import BytesIO
 
@@ -287,7 +287,6 @@ def test_successful_schedule(
                 '--description',
                 os.path.join(self.args.suite, build_matrix_desc),
                 '--',
-                ANY, # base config
                 '-'
             ],
             stdin=yaml.dump(y),