diff --git a/backend/kernelCI_app/urls.py b/backend/kernelCI_app/urls.py index fa4cfc98..44f8017a 100644 --- a/backend/kernelCI_app/urls.py +++ b/backend/kernelCI_app/urls.py @@ -11,6 +11,8 @@ "tree//tests/", views.TreeTestsView.as_view(), name="treeTests" ), path("build//status-count", views.BuildStatusCountView.as_view(), name="buildStatusCount"), + path("tree//full", + views.TreeDetailsSlow.as_view(), name="TreeDetailsSlow"), path("build/", views.BuildDetails.as_view(), name="buildDetails"), path("build//tests", views.BuildTests.as_view(), name="buildTests"), path("tests/test/", views.TestDetails.as_view(), name="testDetails"), diff --git a/backend/kernelCI_app/utils.py b/backend/kernelCI_app/utils.py index d0f309ad..a28f2e87 100644 --- a/backend/kernelCI_app/utils.py +++ b/backend/kernelCI_app/utils.py @@ -1,4 +1,5 @@ import json +from typing import Union from django.utils import timezone from datetime import timedelta import re @@ -6,6 +7,37 @@ DEFAULT_QUERY_TIME_INTERVAL = {'days': 7} +def extract_platform(misc_environment: Union[str, dict, None]): + parsedEnvMisc = None + if isinstance(misc_environment, dict): + parsedEnvMisc = misc_environment + elif misc_environment is None: + return "unknown" + else: + parsedEnvMisc = json.loads(misc_environment) + platform = parsedEnvMisc.get("platform") + if platform: + return platform + print("unknown platform in misc_environment", misc_environment) + return "unknown" + + +# TODO misc is not stable and should be used as a POC only +def extract_error_message(misc: Union[str, dict, None]): + parsedEnv = None + if misc is None: + return "unknown error" + elif isinstance(misc, dict): + parsedEnv = misc + else: + parsedEnv = json.loads(misc) + error_message = parsedEnv.get("error_msg") + if error_message: + return error_message + print("unknown error_msg in misc", misc) + return "unknown error" + + def getQueryTimeInterval(**kwargs): if not kwargs: return timezone.now() - timedelta(**DEFAULT_QUERY_TIME_INTERVAL) diff --git a/backend/kernelCI_app/views/treeDetailsSlowView.py b/backend/kernelCI_app/views/treeDetailsSlowView.py new file mode 100644 index 00000000..3b16c835 --- /dev/null +++ b/backend/kernelCI_app/views/treeDetailsSlowView.py @@ -0,0 +1,202 @@ +from django.http import JsonResponse +from django.views import View +from kernelCI_app.utils import extract_error_message, extract_platform +from django.db import connection + + +class TreeDetailsSlow(View): + def get(self, request, commit_hash: str | None): + origin_param = request.GET.get("origin") + git_url_param = request.GET.get("git_url") + git_branch_param = request.GET.get("git_branch") + query = """ + SELECT + tests.build_id AS tests_build_id, + tests.id AS tests_id, + tests.origin AS tests_origin, + tests.environment_comment AS tests_enviroment_comment, + tests.environment_misc AS tests_enviroment_misc, + tests.path AS tests_path, + tests.comment AS tests_comment, + tests.log_url AS tests_log_url, + tests.status AS tests_status, + tests.waived AS tests_waived, + tests.start_time AS tests_start_time, + tests.duration AS tests_duration, + tests.number_value AS tests_number_value, + tests.misc AS tests_misc, + builds_filter.* + FROM + ( + SELECT + builds.checkout_id AS builds_checkout_id, + builds.id AS builds_id, + builds.comment AS builds_comment, + builds.start_time AS builds_start_time, + builds.duration AS builds_duration, + builds.architecture AS builds_architecture, + builds.command AS builds_command, + builds.compiler AS builds_compiler, + builds.config_name AS builds_config_name, + builds.config_url AS builds_config_url, + builds.log_url AS builds_log_url, + builds.valid AS builds_valid, + tree_head.* + FROM + ( + SELECT + checkouts.id AS checkout_id + FROM + checkouts + WHERE + checkouts.git_commit_hash = %(commit_hash)s AND + checkouts.git_repository_url = %(git_url_param)s AND + checkouts.git_repository_branch = %(git_branch_param)s AND + checkouts.origin = %(origin_param)s + ) AS tree_head + LEFT JOIN builds + ON tree_head.checkout_id = builds.checkout_id + WHERE + builds.origin = %(origin_param)s + ) AS builds_filter + LEFT JOIN tests + ON builds_filter.builds_id = tests.build_id + WHERE + tests.origin = %(origin_param)s + """ + with connection.cursor() as cursor: + cursor.execute( + query, + { + "commit_hash": commit_hash, + "origin_param": origin_param, + "git_url_param": git_url_param, + "git_branch_param": git_branch_param, + }, + ) + rows = cursor.fetchall() + + testHistory = [] + testStatusSummary = {} + testConfigs = {} + testPlatformsWithErrors = set() + testFailReasons = {} + testArchSummary = {} + bootHistory = [] + bootStatusSummary = {} + bootConfigs = {} + bootPlatformsFailing = set() + bootFailReasons = {} + bootArchSummary = {} + + tempColumnDict = { + "tests_id": 1, + "tests_origin": 2, + "tests_enviroment_comment": 3, + "tests_enviroment_misc": 4, + "tests_path": 5, + "tests_comment": 6, + "tests_log_url": 7, + "tests_status": 8, + "tests_waived": 9, + "tests_start_time": 10, + "tests_duration": 11, + "tests_number_value": 12, + "tests_misc": 13, + "builds_checkout_id": 14, + "builds_id": 15, + "builds_comment": 16, + "builds_start_time": 17, + "builds_duration": 18, + "builds_architecture": 19, + "builds_command": 20, + "builds_compiler": 21, + "builds_config_name": 22, + "builds_config_url": 23, + "builds_log_url": 24, + "builds_valid": 25, + } + t = tempColumnDict + for r in rows: + path = r[t["tests_path"]] + testId = r[t["tests_id"]] + testStatus = r[t["tests_status"]] + buildConfig = r[t["builds_config_name"]] + buildArch = r[t["builds_architecture"]] + buildCompiler = r[t["builds_compiler"]] + + testPlatform = extract_platform(r[t["tests_misc"]]) + testError = extract_error_message(r[t["tests_misc"]]) + + # Test history for boot and non boot + historyItem = { + "id": testId, + "status": testStatus, + "path": path, + "duration": r[t["tests_duration"]], + "startTime": r[t["tests_start_time"]], + } + if path.startswith("boot"): + bootHistory.append(historyItem) + bootStatusSummary[testStatus] = bootStatusSummary.get( + testStatus, 0) + 1 + + archKey = "%s-%s" % (buildArch, buildCompiler) + archSummary = bootArchSummary.get(archKey, { + "arch": buildArch, + "compiler": buildCompiler, + "status": {} + }) + archSummary["status"][testStatus] = archSummary["status"].get(testStatus, 0) + 1 + bootArchSummary[archKey] = archSummary + + configSummary = bootConfigs.get(buildConfig, {}) + configSummary[testStatus] = configSummary.get( + testStatus, 0) + 1 + bootConfigs[buildConfig] = configSummary + + if testStatus == "ERROR" or testStatus == "FAIL" or testStatus == "MISS": + bootPlatformsFailing.add(testPlatform) + bootFailReasons[testError] = bootFailReasons.get( + testError, 0) + 1 + else: + testHistory.append(historyItem) + testStatusSummary[testStatus] = testStatusSummary.get( + testStatus, 0) + 1 + + archKey = "%s-%s" % (buildArch, buildCompiler) + archSummary = testArchSummary.get(archKey, { + "arch": buildArch, + "compiler": buildCompiler, + "status": {} + }) + archSummary["status"][testStatus] = archSummary["status"].get(testStatus, 0) + 1 + testArchSummary[archKey] = archSummary + + configSummary = testConfigs.get(buildConfig, {}) + configSummary[testStatus] = configSummary.get( + testStatus, 0) + 1 + testConfigs[buildConfig] = configSummary + + if testStatus == "ERROR" or testStatus == "FAIL" or testStatus == "MISS": + testPlatformsWithErrors.add(testPlatform) + testFailReasons[testError] = testFailReasons.get( + testError, 0) + 1 + + return JsonResponse( + { + "bootArchSummary": list(bootArchSummary.values()), + "testArchSummary": list(testArchSummary.values()), + "bootFailReasons": bootFailReasons, + "testFailReasons": testFailReasons, + "testPlatformsWithErrors": list(testPlatformsWithErrors), + "bootPlatformsFailing": list(bootPlatformsFailing), + "testConfigs": testConfigs, + "bootConfigs": bootConfigs, + "testStatusSummary": testStatusSummary, + "bootStatusSummary": bootStatusSummary, + "bootHistory": bootHistory, + "testHistory": testHistory + }, + safe=False + ) diff --git a/backend/kernelCI_app/views/treeTestsView.py b/backend/kernelCI_app/views/treeTestsView.py index d12ec0e0..0d61d10d 100644 --- a/backend/kernelCI_app/views/treeTestsView.py +++ b/backend/kernelCI_app/views/treeTestsView.py @@ -1,10 +1,9 @@ from collections import defaultdict -from typing_extensions import Union from django.http import HttpResponseBadRequest, JsonResponse from django.views import View -from kernelCI_app.utils import FilterParams, InvalidComparisonOP, getErrorResponseBody +from kernelCI_app.utils import (FilterParams, InvalidComparisonOP, + extract_error_message, extract_platform, getErrorResponseBody) from kernelCI_app.models import Checkouts -import json class TreeTestsView(View): @@ -12,35 +11,6 @@ class TreeTestsView(View): # use the standardized field when that gets available valid_filter_fields = ['status', 'duration'] - def extract_platform(self, misc_environment: Union[str, dict, None]): - parsedEnvMisc = None - if isinstance(misc_environment, dict): - parsedEnvMisc = misc_environment - elif misc_environment is None: - return "unknown" - else: - parsedEnvMisc = json.loads(misc_environment) - platform = parsedEnvMisc.get("platform") - if platform: - return platform - print("unknown platform in misc_environment", misc_environment) - return "unknown" - - # TODO misc is not stable and should be used as a POC only - def extract_error_message(self, misc: Union[str, dict, None]): - parsedEnv = None - if misc is None: - return "unknown error" - elif isinstance(misc, dict): - parsedEnv = misc - else: - parsedEnv = json.loads(misc) - error_message = parsedEnv.get("error_msg") - if error_message: - return error_message - print("unknown error_msg in misc", misc) - return "unknown error" - def __paramsFromBootAndFilters(self, commit_hash, path_param, filter_params, params): if path_param: # TODO: 'boot' and 'boot.', right now is only using 'boot.' @@ -56,7 +26,8 @@ def __paramsFromBootAndFilters(self, commit_hash, path_param, filter_params, par value = f['value'] value_is_list = isinstance(value, list) if value_is_list: - placeholder = ",".join(['%s' for i in value]) if value_is_list else '%s' + placeholder = ",".join( + ['%s' for i in value]) if value_is_list else '%s' path_filter += f" AND t.{field} IN ({placeholder})" params.extend(value) else: @@ -151,8 +122,8 @@ def get(self, request, commit_hash: str | None): compilersPerArchitecture[record.architecture].add( record.compiler) platformsWithError.add( - self.extract_platform(record.environment_misc)) - currentErrorMessage = self.extract_error_message(record.misc) + extract_platform(record.environment_misc)) + currentErrorMessage = extract_error_message(record.misc) errorMessageCounts[currentErrorMessage] += 1 errorMessageCounts[currentErrorMessage] += 1 for architecture in compilersPerArchitecture: