From eb72c12b31560dad49caff2e532472e8920f38d5 Mon Sep 17 00:00:00 2001 From: Max K Date: Fri, 7 Jun 2024 00:51:37 +0200 Subject: [PATCH 1/4] CI: Minor fixes in ci scripts --- tests/ci/ci_settings.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/ci/ci_settings.py b/tests/ci/ci_settings.py index 62e7826dac58..7807cc7ac10c 100644 --- a/tests/ci/ci_settings.py +++ b/tests/ci/ci_settings.py @@ -211,12 +211,15 @@ def apply( ): res[job] = job_config + add_parents = [] for job in list(res): parent_jobs = CI_CONFIG.get_job_parents(job) for parent_job in parent_jobs: if parent_job not in res: + add_parents.append(parent_job) print(f"Job [{job}] requires [{parent_job}] - add") - res[parent_job] = job_configs[parent_job] + for job in add_parents: + res[job] = job_configs[job] for job, job_config in res.items(): batches = [] From 0deb862c93824146cf9012f95fa247e459c3683d Mon Sep 17 00:00:00 2001 From: Max K Date: Fri, 7 Jun 2024 01:22:47 +0200 Subject: [PATCH 2/4] Re-enable Fast test in MQ --- tests/ci/ci.py | 6 +++++- tests/ci/ci_settings.py | 6 ++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/ci/ci.py b/tests/ci/ci.py index 55a18a2f335e..ec6e84dea8c0 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -442,7 +442,11 @@ def _configure_jobs( # filter jobs in accordance with ci settings job_configs = ci_settings.apply( - job_configs, pr_info.is_release, is_pr=pr_info.is_pr, labels=pr_info.labels + job_configs, + pr_info.is_release, + is_pr=pr_info.is_pr, + is_mq=pr_info.is_merge_queue, + labels=pr_info.labels, ) # check jobs in ci cache diff --git a/tests/ci/ci_settings.py b/tests/ci/ci_settings.py index 7807cc7ac10c..83d4ddb42114 100644 --- a/tests/ci/ci_settings.py +++ b/tests/ci/ci_settings.py @@ -134,6 +134,7 @@ def _check_if_selected( job_config: JobConfig, is_release: bool, is_pr: bool, + is_mq: bool, labels: Iterable[str], ) -> bool: # type: ignore #too-many-return-statements if self.do_not_test: @@ -189,7 +190,7 @@ def _check_if_selected( if job_config.release_only and not is_release: return False - elif job_config.pr_only and not is_pr: + elif job_config.pr_only and not is_pr and not is_mq: return False return not to_deny @@ -199,6 +200,7 @@ def apply( job_configs: Dict[str, JobConfig], is_release: bool, is_pr: bool, + is_mq: bool, labels: Iterable[str], ) -> Dict[str, JobConfig]: """ @@ -207,7 +209,7 @@ def apply( res = {} for job, job_config in job_configs.items(): if self._check_if_selected( - job, job_config, is_release=is_release, is_pr=is_pr, labels=labels + job, job_config, is_release=is_release, is_pr=is_pr, is_mq=is_mq, labels=labels ): res[job] = job_config From 8f26f77505a08197f236f6e2cc069cd4111d71ec Mon Sep 17 00:00:00 2001 From: Max K Date: Fri, 7 Jun 2024 01:35:39 +0200 Subject: [PATCH 3/4] fix unit test --- tests/ci/test_ci_options.py | 58 +++++++++++++++++++++++++++---------- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/tests/ci/test_ci_options.py b/tests/ci/test_ci_options.py index c00cd0b92160..e6fa03896495 100644 --- a/tests/ci/test_ci_options.py +++ b/tests/ci/test_ci_options.py @@ -167,19 +167,19 @@ def test_options_applied(self): ) jobs_configs = {job: JobConfig() for job in _TEST_JOB_LIST} - jobs_configs[ - "fuzzers" - ].run_by_label = ( + jobs_configs["fuzzers"].run_by_label = ( "TEST_LABEL" # check "fuzzers" appears in the result due to the label ) - jobs_configs[ - "Integration tests (asan)" - ].release_only = ( + jobs_configs["Integration tests (asan)"].release_only = ( True # still must be included as it's set with include keywords ) filtered_jobs = list( ci_options.apply( - jobs_configs, is_release=False, is_pr=True, labels=["TEST_LABEL"] + jobs_configs, + is_release=False, + is_pr=True, + is_mq=False, + labels=["TEST_LABEL"], ) ) self.assertCountEqual( @@ -212,7 +212,9 @@ def test_options_applied_2(self): jobs_configs["fuzzers"].run_by_label = "TEST_LABEL" # no settings are set filtered_jobs = list( - CiSettings().apply(jobs_configs, is_release=False, is_pr=True, labels=[]) + CiSettings().apply( + jobs_configs, is_release=False, is_pr=False, is_mq=True, labels=[] + ) ) self.assertCountEqual( filtered_jobs, @@ -220,9 +222,21 @@ def test_options_applied_2(self): "Fast test", ], ) - filtered_jobs = list( - CiSettings().apply(jobs_configs, is_release=True, is_pr=False, labels=[]) + CiSettings().apply( + jobs_configs, is_release=False, is_pr=True, is_mq=False, labels=[] + ) + ) + self.assertCountEqual( + filtered_jobs, + [ + "Fast test", + ], + ) + filtered_jobs = list( + CiSettings().apply( + jobs_configs, is_release=True, is_pr=False, is_mq=False, labels=[] + ) ) self.assertCountEqual( filtered_jobs, @@ -240,7 +254,11 @@ def test_options_applied_3(self): # no settings are set filtered_jobs = list( ci_settings.apply( - jobs_configs, is_release=False, is_pr=True, labels=["TEST_LABEL"] + jobs_configs, + is_release=False, + is_pr=True, + is_mq=False, + labels=["TEST_LABEL"], ) ) self.assertCountEqual( @@ -253,7 +271,11 @@ def test_options_applied_3(self): ci_settings.include_keywords = ["Fast"] filtered_jobs = list( ci_settings.apply( - jobs_configs, is_release=True, is_pr=False, labels=["TEST_LABEL"] + jobs_configs, + is_release=True, + is_pr=False, + is_mq=False, + labels=["TEST_LABEL"], ) ) self.assertCountEqual( @@ -271,13 +293,17 @@ def test_options_applied_4(self): self.assertCountEqual(ci_options.include_keywords, ["analyzer"]) self.assertIsNone(ci_options.exclude_keywords) jobs_configs = {job: JobConfig() for job in _TEST_JOB_LIST} - jobs_configs[ - "fuzzers" - ].run_by_label = "TEST_LABEL" # check "fuzzers" does not appears in the result + jobs_configs["fuzzers"].run_by_label = ( + "TEST_LABEL" # check "fuzzers" does not appears in the result + ) jobs_configs["Integration tests (asan)"].release_only = True filtered_jobs = list( ci_options.apply( - jobs_configs, is_release=False, is_pr=True, labels=["TEST_LABEL"] + jobs_configs, + is_release=False, + is_pr=True, + is_mq=False, + labels=["TEST_LABEL"], ) ) self.assertCountEqual( From 367d41e7f042137d4a25b55c740ba3835b5d5435 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 6 Jun 2024 23:41:25 +0000 Subject: [PATCH 4/4] Automatic style fix --- tests/ci/ci_settings.py | 7 ++++++- tests/ci/test_ci_options.py | 14 +++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/tests/ci/ci_settings.py b/tests/ci/ci_settings.py index 83d4ddb42114..f25344c77019 100644 --- a/tests/ci/ci_settings.py +++ b/tests/ci/ci_settings.py @@ -209,7 +209,12 @@ def apply( res = {} for job, job_config in job_configs.items(): if self._check_if_selected( - job, job_config, is_release=is_release, is_pr=is_pr, is_mq=is_mq, labels=labels + job, + job_config, + is_release=is_release, + is_pr=is_pr, + is_mq=is_mq, + labels=labels, ): res[job] = job_config diff --git a/tests/ci/test_ci_options.py b/tests/ci/test_ci_options.py index e6fa03896495..608889328030 100644 --- a/tests/ci/test_ci_options.py +++ b/tests/ci/test_ci_options.py @@ -167,10 +167,14 @@ def test_options_applied(self): ) jobs_configs = {job: JobConfig() for job in _TEST_JOB_LIST} - jobs_configs["fuzzers"].run_by_label = ( + jobs_configs[ + "fuzzers" + ].run_by_label = ( "TEST_LABEL" # check "fuzzers" appears in the result due to the label ) - jobs_configs["Integration tests (asan)"].release_only = ( + jobs_configs[ + "Integration tests (asan)" + ].release_only = ( True # still must be included as it's set with include keywords ) filtered_jobs = list( @@ -293,9 +297,9 @@ def test_options_applied_4(self): self.assertCountEqual(ci_options.include_keywords, ["analyzer"]) self.assertIsNone(ci_options.exclude_keywords) jobs_configs = {job: JobConfig() for job in _TEST_JOB_LIST} - jobs_configs["fuzzers"].run_by_label = ( - "TEST_LABEL" # check "fuzzers" does not appears in the result - ) + jobs_configs[ + "fuzzers" + ].run_by_label = "TEST_LABEL" # check "fuzzers" does not appears in the result jobs_configs["Integration tests (asan)"].release_only = True filtered_jobs = list( ci_options.apply(