diff --git a/teuthology/suite/merge.py b/teuthology/suite/merge.py index f13058e47..0e109af02 100644 --- a/teuthology/suite/merge.py +++ b/teuthology/suite/merge.py @@ -5,6 +5,7 @@ from types import MappingProxyType import yaml +from teuthology.config import JobConfig from teuthology.suite.build_matrix import combine_path from teuthology.suite.util import strip_fragment_path from teuthology.misc import deep_merge @@ -115,6 +116,7 @@ def config_merge(configs, suite_name=None, **kwargs): the entire job (config) from the list. """ seed = kwargs.setdefault('seed', 1) + base_config = kwargs.setdefault('base_config', JobConfig()) if not isinstance(seed, int): log.debug("no valid seed input: using 1") seed = 1 @@ -128,7 +130,7 @@ def config_merge(configs, suite_name=None, **kwargs): if suite_name is not None: desc = combine_path(suite_name, desc) - yaml_complete_obj = {} + yaml_complete_obj = copy.deepcopy(base_config.to_dict()) deep_merge(yaml_complete_obj, dict(TEUTHOLOGY_TEMPLATE)) for path in paths: if path not in yaml_cache: diff --git a/teuthology/suite/run.py b/teuthology/suite/run.py index 9b2b6fa70..ba72a4334 100644 --- a/teuthology/suite/run.py +++ b/teuthology/suite/run.py @@ -9,7 +9,6 @@ from humanfriendly import format_timespan -from tempfile import NamedTemporaryFile from teuthology import repo_utils from teuthology.config import config, JobConfig @@ -167,13 +166,16 @@ def choose_kernel(self): # Put together a stanza specifying the kernel hash if self.args.kernel_branch == 'distro': kernel_hash = 'distro' + kernel_branch = 'distro' # Skip the stanza if '-k none' is given elif self.args.kernel_branch is None or \ self.args.kernel_branch.lower() == 'none': kernel_hash = None + kernel_branch = None else: + kernel_branch = self.args.kernel_branch kernel_hash = util.get_gitbuilder_hash( - 'kernel', self.args.kernel_branch, 'default', + 'kernel', kernel_branch, 'default', self.args.machine_type, self.args.distro, self.args.distro_version, ) @@ -189,7 +191,7 @@ def choose_kernel(self): if kernel_hash: log.info("kernel sha1: {hash}".format(hash=kernel_hash)) - kernel_dict = dict(kernel=dict(kdb=kdb, sha1=kernel_hash)) + kernel_dict = dict(kernel=dict(branch=kernel_branch, kdb=kdb, sha1=kernel_hash)) if kernel_hash != 'distro': kernel_dict['kernel']['flavor'] = 'default' else: @@ -622,6 +624,9 @@ def schedule_suite(self): log.debug('Suite %s in %s' % (suite_name, suite_path)) log.debug(f"subset = {self.args.subset}") log.debug(f"no_nested_subset = {self.args.no_nested_subset}") + if self.args.dry_run: + log.debug("Base job config:\n%s" % self.base_config) + configs = build_matrix(suite_path, subset=self.args.subset, no_nested_subset=self.args.no_nested_subset, @@ -633,20 +638,10 @@ def schedule_suite(self): filter_out=self.args.filter_out, filter_all=self.args.filter_all, filter_fragments=self.args.filter_fragments, + base_config=self.base_config, seed=self.args.seed, suite_name=suite_name)) - if self.args.dry_run: - log.debug("Base job config:\n%s" % self.base_config) - - # create, but do not write, the temp file here, so it can be - # added to the args in collect_jobs, but not filled until - # any backtracking is done - base_yaml_path = NamedTemporaryFile( - prefix='schedule_suite_', delete=False - ).name - self.base_yaml_paths.insert(0, base_yaml_path) - # compute job limit in respect of --sleep-before-teardown job_limit = self.args.limit or 0 sleep_before_teardown = int(self.args.sleep_before_teardown or 0) @@ -711,9 +706,6 @@ def schedule_suite(self): dry_run=self.args.dry_run, ) - with open(base_yaml_path, 'w+b') as base_yaml: - base_yaml.write(str(self.base_config).encode()) - if jobs_to_schedule: self.write_rerun_memo() @@ -725,8 +717,6 @@ def schedule_suite(self): self.schedule_jobs(jobs_missing_packages, jobs_to_schedule, name) - os.remove(base_yaml_path) - count = len(jobs_to_schedule) missing_count = len(jobs_missing_packages) total_count = count diff --git a/teuthology/suite/test/test_run_.py b/teuthology/suite/test/test_run_.py index 0fad56ddf..c97e26e5c 100644 --- a/teuthology/suite/test/test_run_.py +++ b/teuthology/suite/test/test_run_.py @@ -1,3 +1,4 @@ +import logging import os import pytest import requests @@ -14,6 +15,7 @@ from teuthology.suite import run from teuthology.util.time import TIMESTAMP_FMT +log = logging.getLogger(__name__) class TestRun(object): klass = run.Run @@ -270,14 +272,35 @@ def test_successful_schedule( [call('ceph_sha1', 'default', 'ubuntu', '14.04', 'machine_type')], ) y = { - 'teuthology': { - 'fragments_dropped': [], - 'meta': {}, - 'postmerge': [] - }, 'field1': 'val1', 'field2': 'val2' } + teuthology_keys = [ + 'branch', + 'machine_type', + 'name', + 'os_type', + 'os_version', + 'overrides', + 'priority', + 'repo', + 'seed', + 'sha1', + 'sleep_before_teardown', + 'suite', + 'suite_branch', + 'suite_relpath', + 'suite_repo', + 'suite_sha1', + 'tasks', + 'teuthology_branch', + 'teuthology_sha1', + 'timestamp', + 'user', + 'teuthology', + ] + for t in teuthology_keys: + y[t] = ANY expected_job = dict( yaml=y, sha1='ceph_sha1', @@ -287,16 +310,23 @@ def test_successful_schedule( '--description', os.path.join(self.args.suite, build_matrix_desc), '--', - ANY, # base config '-' ], - stdin=yaml.dump(y), + stdin=ANY, desc=os.path.join(self.args.suite, build_matrix_desc), ) m_schedule_jobs.assert_has_calls( [call([], [expected_job], runobj.name)], ) + args = m_schedule_jobs.call_args.args + log.debug("args =\n%s", args) + jobargs = args[1][0] + stdin_yaml = yaml.safe_load(jobargs['stdin']) + for k in y: + assert y[k] == stdin_yaml[k] + for k in teuthology_keys: + assert k in stdin_yaml m_write_rerun_memo.assert_called_once_with() @patch('teuthology.suite.util.find_git_parents') diff --git a/teuthology/task/kernel.py b/teuthology/task/kernel.py index 59242d31b..22cd1a69c 100644 --- a/teuthology/task/kernel.py +++ b/teuthology/task/kernel.py @@ -33,7 +33,7 @@ log = logging.getLogger(__name__) -CONFIG_DEFAULT = {'branch': 'main'} +CONFIG_DEFAULT = {'branch': 'distro', 'sha1': 'distro'} TIMEOUT_DEFAULT = 300 VERSION_KEYS = ['branch', 'tag', 'sha1', 'deb', 'rpm', 'koji', 'koji_task']