From c9f7b09168d15454138caa7583734d6540238e09 Mon Sep 17 00:00:00 2001 From: Marcelo Robert Santos Date: Tue, 4 Feb 2025 15:09:20 -0300 Subject: [PATCH] wip renames processed_issues to issue_dicts separates processed_tests from decide_test_in_filter --- .../kernelCI_app/helpers/hardwareDetails.py | 56 +++++++++++++------ backend/kernelCI_app/utils.py | 4 +- .../views/hardwareDetailsBootsView.py | 7 ++- .../views/hardwareDetailsSummaryView.py | 45 ++++++++++++--- .../views/hardwareDetailsTestsView.py | 7 ++- .../kernelCI_app/views/hardwareDetailsView.py | 34 +++++++++-- 6 files changed, 120 insertions(+), 33 deletions(-) diff --git a/backend/kernelCI_app/helpers/hardwareDetails.py b/backend/kernelCI_app/helpers/hardwareDetails.py index 2c2bf89a..cc335c3e 100644 --- a/backend/kernelCI_app/helpers/hardwareDetails.py +++ b/backend/kernelCI_app/helpers/hardwareDetails.py @@ -534,7 +534,7 @@ def handle_test_summary( *, record: Dict, task: TestSummary, - processed_issues: Dict, + issue_dict: Dict, processed_archs: Dict[str, TestArchSummaryItem], ) -> None: status = record["status"] @@ -576,10 +576,12 @@ def handle_test_summary( processed_archs[arch_key] = arch_summary setattr(arch_summary.status, status, getattr(arch_summary.status, status) + 1) - process_issue(record=record, task_issues_dict=processed_issues, issue_from="test") + process_issue(record=record, task_issues_dict=issue_dict, issue_from="test") -def handle_build_history(*, record: Dict, tree_idx: int, builds: List[HardwareBuildHistoryItem]) -> None: +def handle_build_history( + *, record: Dict, tree_idx: int, builds: List[HardwareBuildHistoryItem] +) -> None: build = get_build_typed(record=record, tree_idx=tree_idx) builds.append(build) @@ -588,7 +590,7 @@ def handle_build_summary( *, record: Dict, builds_summary: BuildSummary, - processed_issues: Dict, + issue_dict: Dict, tree_index: int, ) -> None: build: HardwareBuildHistoryItem = get_build_typed(record, tree_idx=tree_index) @@ -630,7 +632,7 @@ def handle_build_summary( ): builds_summary.architectures[arch].compilers.append(compiler) - process_issue(record=record, task_issues_dict=processed_issues, issue_from="build") + process_issue(record=record, task_issues_dict=issue_dict, issue_from="build") # deprecated, use handle_build_history and handle_build_summary separately instead, with typing @@ -752,13 +754,35 @@ def decide_if_is_build_in_filter( return is_build_not_processed and not is_build_filtered_out_result +def get_processed_issue_key(*, record: Dict) -> str: + issue_id = record["incidents__issue__id"] + issue_id_key = issue_id if issue_id is not None else UNKNOWN_STRING + + issue_version = record["incidents__issue__version"] + issue_version_key = str(issue_version) if issue_version is not None else '' + + processed_issue_key = ( + record["id"] + + issue_id_key + + issue_version_key + ) + return processed_issue_key + + +def is_issue_processed(*, record: Dict, processed_issues: Set[str]) -> bool: + processed_issue_key = get_processed_issue_key(record=record) + is_issue_processed_result = processed_issue_key in processed_issues + return is_issue_processed_result + + +def is_test_processed(*, record: Dict, processed_tests: Set[str]) -> bool: + is_test_processed_result = record["id"] in processed_tests + return is_test_processed_result + + def decide_if_is_test_in_filter( *, instance, test_type: PossibleTestType, record: Dict, processed_tests: Set[str] ) -> bool: - is_test_processed = record["id"] in processed_tests - if is_test_processed: - return False - test_filter_pass = True status = record["status"] @@ -906,17 +930,17 @@ def format_issue_summary_for_response( builds_summary: BuildSummary, boots_summary: TestSummary, tests_summary: TestSummary, - processed_issues: Dict, + issue_dicts: Dict, ) -> None: builds_summary.issues = convert_issues_dict_to_list_typed( - issues_dict=processed_issues["build"]["issues"] + issues_dict_list=issue_dicts["build"]["issues"] ) boots_summary.issues = convert_issues_dict_to_list_typed( - issues_dict=processed_issues["boot"]["issues"] + issues_dict_list=issue_dicts["boot"]["issues"] ) tests_summary.issues = convert_issues_dict_to_list_typed( - issues_dict=processed_issues["test"]["issues"] + issues_dict_list=issue_dicts["test"]["issues"] ) - builds_summary.unknown_issues = processed_issues["build"]["failedWithUnknownIssues"] - boots_summary.unknown_issues = processed_issues["boot"]["failedWithUnknownIssues"] - tests_summary.unknown_issues = processed_issues["test"]["failedWithUnknownIssues"] + builds_summary.unknown_issues = issue_dicts["build"]["failedWithUnknownIssues"] + boots_summary.unknown_issues = issue_dicts["boot"]["failedWithUnknownIssues"] + tests_summary.unknown_issues = issue_dicts["test"]["failedWithUnknownIssues"] diff --git a/backend/kernelCI_app/utils.py b/backend/kernelCI_app/utils.py index 3b409039..3212b5d5 100644 --- a/backend/kernelCI_app/utils.py +++ b/backend/kernelCI_app/utils.py @@ -30,9 +30,9 @@ def convert_issues_dict_to_list(issues_dict: Dict[str, Issue]) -> List[Issue]: return list(issues_dict.values()) -def convert_issues_dict_to_list_typed(*, issues_dict: Dict) -> List[Issue]: +def convert_issues_dict_to_list_typed(*, issues_dict_list: Dict) -> List[Issue]: issues: List[Issue] = [] - for issue in issues_dict.values(): + for issue in issues_dict_list.values(): issues.append( Issue( id=issue["id"], diff --git a/backend/kernelCI_app/views/hardwareDetailsBootsView.py b/backend/kernelCI_app/views/hardwareDetailsBootsView.py index 9293cc3b..bf9f1372 100644 --- a/backend/kernelCI_app/views/hardwareDetailsBootsView.py +++ b/backend/kernelCI_app/views/hardwareDetailsBootsView.py @@ -13,6 +13,7 @@ get_trees_with_selected_commit, get_validated_current_tree, handle_test_history, + is_test_processed, unstable_parse_post_body, ) from kernelCI_app.typeModels.commonDetails import ( @@ -54,6 +55,10 @@ def _process_test(self, record: Dict) -> None: if not is_record_boot: return + is_test_processed_result = is_test_processed(record=record, processed_tests=self.processed_tests) + if (is_test_processed_result): + return + should_process_test = decide_if_is_test_in_filter( instance=self, test_type="boot", @@ -61,12 +66,12 @@ def _process_test(self, record: Dict) -> None: processed_tests=self.processed_tests, ) - self.processed_tests.add(record["id"]) if should_process_test: handle_test_history( record=record, task=self.boots, ) + self.processed_tests.add(record["id"]) def _sanitize_records( self, records, trees: List[Tree], is_all_selected: bool diff --git a/backend/kernelCI_app/views/hardwareDetailsSummaryView.py b/backend/kernelCI_app/views/hardwareDetailsSummaryView.py index 54616064..ccb71d51 100644 --- a/backend/kernelCI_app/views/hardwareDetailsSummaryView.py +++ b/backend/kernelCI_app/views/hardwareDetailsSummaryView.py @@ -17,11 +17,15 @@ get_filter_options, get_hardware_details_data, get_hardware_trees_data, + get_processed_issue_key, get_trees_with_selected_commit, get_validated_current_tree, handle_build_summary, handle_test_summary, handle_tree_status_summary, + is_issue_processed, + is_test_processed, + process_issue, set_trees_status_summary, format_issue_summary_for_response, unstable_parse_post_body, @@ -66,9 +70,15 @@ def __init__(self): self.processed_builds = set() self.processed_tests = set() + self.processed_issues: Dict[str, Set[str]] = { + "build": set(), + "boot": set(), + "test": set(), + } + self.processed_compatibles: Set[str] = set() - self.processed_issues = { + self.issue_dicts = { "build": { "issues": {}, "failedWithUnknownIssues": 0, @@ -112,6 +122,12 @@ def _process_test(self, record: Dict) -> None: test_type_key: PossibleTestType = "boot" if is_record_boot else "test" task = self.boots_summary if is_record_boot else self.tests_summary + is_test_processed_result = is_test_processed( + record=record, processed_tests=self.processed_tests + ) + is_issue_processed_result = is_issue_processed( + record=record, processed_issues=self.processed_issues[test_type_key] + ) should_process_test = decide_if_is_test_in_filter( instance=self, test_type=test_type_key, @@ -119,14 +135,24 @@ def _process_test(self, record: Dict) -> None: processed_tests=self.processed_tests, ) - self.processed_tests.add(record["id"]) - if should_process_test: + if should_process_test and not is_issue_processed_result and is_test_processed_result: + process_issue( + record=record, + task_issues_dict=self.issue_dicts[test_type_key], + issue_from="test", + ) + processed_issue_key = get_processed_issue_key(record=record) + print("🚀 ~ processed_issue_key:", processed_issue_key) + self.processed_issues[test_type_key].add(processed_issue_key) + + if should_process_test and not is_test_processed_result: handle_test_summary( record=record, task=task, - processed_issues=self.processed_issues.get(test_type_key), + issue_dict=self.issue_dicts.get(test_type_key), processed_archs=self.processed_architectures[test_type_key], ) + self.processed_tests.add(record["id"]) def _process_build(self, record: Dict, tree_index: int) -> None: build = get_build(record, tree_index) @@ -144,7 +170,7 @@ def _process_build(self, record: Dict, tree_index: int) -> None: handle_build_summary( record=record, builds_summary=self.builds_summary, - processed_issues=self.processed_issues["build"], + issue_dict=self.issue_dicts["build"], tree_index=tree_index, ) @@ -200,7 +226,7 @@ def _format_processing_for_response(self, hardware_id: str) -> None: builds_summary=self.builds_summary, boots_summary=self.boots_summary, tests_summary=self.tests_summary, - processed_issues=self.processed_issues, + issue_dicts=self.issue_dicts, ) # Using post to receive a body request @@ -221,10 +247,11 @@ def post(self, request, hardware_id) -> Response: }, status=HTTPStatus.BAD_REQUEST, ) - except (ValueError, TypeError): + except (ValueError, TypeError) as e: return Response( data={ - "error": "startTimeStamp and endTimeStamp must be a Unix Timestamp" + "error": "startTimeStamp and endTimeStamp must be a Unix Timestamp", + "exception": str(e), }, status=HTTPStatus.BAD_REQUEST, ) @@ -291,7 +318,7 @@ def post(self, request, hardware_id) -> Response: ), tests=HardwareTestLocalFilters( issues=list(self.unfiltered_test_issues), - platforms=list(self.unfiltered_test_platforms) + platforms=list(self.unfiltered_test_platforms), ), ), common=HardwareCommon( diff --git a/backend/kernelCI_app/views/hardwareDetailsTestsView.py b/backend/kernelCI_app/views/hardwareDetailsTestsView.py index 5aa2b46e..5b15308f 100644 --- a/backend/kernelCI_app/views/hardwareDetailsTestsView.py +++ b/backend/kernelCI_app/views/hardwareDetailsTestsView.py @@ -13,6 +13,7 @@ get_trees_with_selected_commit, get_validated_current_tree, handle_test_history, + is_test_processed, unstable_parse_post_body, ) from kernelCI_app.typeModels.commonDetails import ( @@ -52,6 +53,10 @@ def _process_test(self, record: Dict) -> None: if is_record_boot: return + is_test_processed_result = is_test_processed(record=record, processed_tests=self.processed_tests) + if (is_test_processed_result): + return + should_process_test = decide_if_is_test_in_filter( instance=self, test_type="test", @@ -59,12 +64,12 @@ def _process_test(self, record: Dict) -> None: processed_tests=self.processed_tests, ) - self.processed_tests.add(record["id"]) if should_process_test: handle_test_history( record=record, task=self.tests, ) + self.processed_tests.add(record["id"]) def _sanitize_records( self, records: List[Dict], trees: List[Tree], is_all_selected: bool diff --git a/backend/kernelCI_app/views/hardwareDetailsView.py b/backend/kernelCI_app/views/hardwareDetailsView.py index 3870b6e4..e151a68c 100644 --- a/backend/kernelCI_app/views/hardwareDetailsView.py +++ b/backend/kernelCI_app/views/hardwareDetailsView.py @@ -17,12 +17,16 @@ get_filter_options, get_hardware_details_data, get_hardware_trees_data, + get_processed_issue_key, get_trees_with_selected_commit, get_validated_current_tree, handle_build, handle_test_or_boot, handle_tree_status_summary, + is_issue_processed, + is_test_processed, mutate_properties_to_list, + process_issue, set_trees_status_summary, unstable_parse_post_body, ) @@ -70,6 +74,12 @@ def __init__(self): self.processed_builds = set() self.processed_tests = set() + self.processed_issues: Dict[str, Set[str]] = { + "build": set(), + "boot": set(), + "test": set(), + } + self.processed_compatibles: Set[str] = set() self.builds = { @@ -95,18 +105,34 @@ def __init__(self): def _process_test(self, record: Dict) -> None: is_record_boot = is_boot(record["path"]) - test_filter_key: PossibleTestType = "boot" if is_record_boot else "test" + test_type_key: PossibleTestType = "boot" if is_record_boot else "test" + task = self.boots if is_record_boot else self.tests + is_test_processed_result = is_test_processed( + record=record, processed_tests=self.processed_tests + ) + is_issue_processed_result = is_issue_processed( + record=record, processed_issues=self.processed_issues[test_type_key] + ) should_process_test = decide_if_is_test_in_filter( instance=self, - test_type=test_filter_key, + test_type=test_type_key, record=record, processed_tests=self.processed_tests, ) - if should_process_test: + if should_process_test and not is_issue_processed_result and is_test_processed_result: + process_issue( + record=record, + task_issues_dict=task, + issue_from="test", + ) + processed_issue_key = get_processed_issue_key(record=record) + self.processed_issues[test_type_key].add(processed_issue_key) + + if should_process_test and not is_test_processed_result: + handle_test_or_boot(record=record, task=task) self.processed_tests.add(record["id"]) - handle_test_or_boot(record, self.boots if is_record_boot else self.tests) def _process_build(self, record: Dict, tree_index: int) -> None: build = get_build(record, tree_index)