feat: add filters to megaquery

backend/kernelCI_app/utils.py

@@ -7,6 +7,13 @@
 DEFAULT_QUERY_TIME_INTERVAL = {'days': 7}
 
 
+def toIntOrDefault(value, default):
+    try:
+        return int(value)
+    except ValueError:
+        return default
+
+
 def extract_platform(misc_environment: Union[str, dict, None]):
     parsedEnvMisc = None
     if isinstance(misc_environment, dict):
@@ -105,11 +112,13 @@ def create_filters_from_req(self, request):
 
     def add_filter(self, field, value, comparison_op):
         self.validate_comparison_op(comparison_op)
-        self.filters.append({'field': field, 'value': value, 'comparison_op': comparison_op})
+        self.filters.append({'field': field, 'value': value,
+                            'comparison_op': comparison_op})
 
     def validate_comparison_op(self, op):
         if op not in self.comparison_ops.keys():
-            raise InvalidComparisonOP(f'Filter with invalid comparison operator `{op}` found`')
+            raise InvalidComparisonOP(
+                f'Filter with invalid comparison operator `{op}` found`')
 
     def get_comparison_op(self, filter, op_type='orm'):
         idx = self.comparison_op_type_idx[op_type]

backend/kernelCI_app/views/treeDetailsSlowView.py

@@ -1,14 +1,50 @@
-from django.http import JsonResponse
+from django.http import JsonResponse, HttpResponseBadRequest
 from django.views import View
-from kernelCI_app.utils import extract_error_message, extract_platform
+from kernelCI_app.utils import (FilterParams, extract_error_message,
+                                extract_platform, InvalidComparisonOP, getErrorResponseBody, toIntOrDefault)
 from django.db import connection
 
 
 class TreeDetailsSlow(View):
+    def __processFilters(self, request):
+        filterTestStatus = ["FAIL", "MISS", "PASS", "DONE", "ERROR", "SKIP"]
+        filterTestDurationMin, filterTestDurationMax = None, None
+        filterBootStatus = ["FAIL", "MISS", "PASS", "DONE", "ERROR", "SKIP"]
+        filterBootDurationMin, filterBootDurationMax = None, None
+
+        try:
+            filter_params = FilterParams(request)
+            for f in filter_params.filters:
+                field = f["field"]
+                value = f["value"]
+                operation = f["comparison_op"]
+                if field == "boot.status":
+                    filterBootStatus = value
+                elif field == "boot.duration":
+                    if operation == "lte":
+                        filterBootDurationMax = toIntOrDefault(value, None)
+                    else:
+                        filterBootDurationMin = toIntOrDefault(value, None)
+                if field == "test.status":
+                    filterTestStatus = value
+                elif field == "test.duration":
+                    if operation == "lte":
+                        filterTestDurationMax = toIntOrDefault(value, None)
+                    else:
+                        filterTestDurationMin = toIntOrDefault(value, None)
+        except InvalidComparisonOP as e:
+            return HttpResponseBadRequest(getErrorResponseBody(str(e)))
+        return (filterTestStatus, filterTestDurationMin, filterTestDurationMax, filterBootStatus,
+                filterBootDurationMin, filterBootDurationMax)
+
     def get(self, request, commit_hash: str | None):
         origin_param = request.GET.get("origin")
         git_url_param = request.GET.get("git_url")
         git_branch_param = request.GET.get("git_branch")
+
+        (filterTestStatus, filterTestDurationMin, filterTestDurationMax, filterBootStatus,
+         filterBootDurationMin, filterBootDurationMax) = self.__processFilters(request)
+
         query = """
         SELECT
                 tests.build_id AS tests_build_id,
@@ -65,15 +101,12 @@ def get(self, request, commit_hash: str | None):
             tests.origin = %(origin_param)s
         """
         with connection.cursor() as cursor:
-            cursor.execute(
-                query,
-                {
-                    "commit_hash": commit_hash,
-                    "origin_param": origin_param,
-                    "git_url_param": git_url_param,
-                    "git_branch_param": git_branch_param,
-                },
-            )
+            cursor.execute(query, {
+                "commit_hash": commit_hash,
+                "origin_param": origin_param,
+                "git_url_param": git_url_param,
+                "git_branch_param": git_branch_param,
+            })
             rows = cursor.fetchall()
 
         testHistory = []
@@ -121,22 +154,34 @@ def get(self, request, commit_hash: str | None):
             path = r[t["tests_path"]]
             testId = r[t["tests_id"]]
             testStatus = r[t["tests_status"]]
+            testDuration = r[t["tests_duration"]]
             buildConfig = r[t["builds_config_name"]]
             buildArch = r[t["builds_architecture"]]
             buildCompiler = r[t["builds_compiler"]]
 
-            testPlatform = extract_platform(r[t["tests_misc"]])
+            testPlatform = extract_platform(r[t["tests_enviroment_misc"]])
             testError = extract_error_message(r[t["tests_misc"]])
 
             #  Test history for boot and non boot
             historyItem = {
                 "id": testId,
                 "status": testStatus,
                 "path": path,
-                "duration": r[t["tests_duration"]],
+                "duration": testDuration,
                 "startTime": r[t["tests_start_time"]],
             }
             if path.startswith("boot"):
+                if testStatus not in filterBootStatus:
+                    continue
+                if (
+                    filterBootDurationMax is not None
+                    and (testStatus is None or toIntOrDefault(testStatus, 0) > filterBootDurationMax)
+                ) and (
+                    filterBootDurationMin is not None
+                    and (testStatus is None or toIntOrDefault(testStatus, 0) < filterBootDurationMin)
+                ):
+                    continue
+
                 bootHistory.append(historyItem)
                 bootStatusSummary[testStatus] = bootStatusSummary.get(
                     testStatus, 0) + 1
@@ -147,7 +192,8 @@ def get(self, request, commit_hash: str | None):
                     "compiler": buildCompiler,
                     "status": {}
                 })
-                archSummary["status"][testStatus] = archSummary["status"].get(testStatus, 0) + 1
+                archSummary["status"][testStatus] = archSummary["status"].get(
+                    testStatus, 0) + 1
                 bootArchSummary[archKey] = archSummary
 
                 configSummary = bootConfigs.get(buildConfig, {})
@@ -160,6 +206,16 @@ def get(self, request, commit_hash: str | None):
                     bootFailReasons[testError] = bootFailReasons.get(
                         testError, 0) + 1
             else:
+                if testStatus not in filterTestStatus:
+                    continue
+                if (
+                    filterTestDurationMax is not None
+                    and (testStatus is None or toIntOrDefault(testStatus, 0) > filterTestDurationMax)
+                ) and (
+                    filterTestDurationMin is not None
+                    and (testStatus is None or toIntOrDefault(testStatus, 0) < filterTestDurationMin)
+                ):
+                    continue
                 testHistory.append(historyItem)
                 testStatusSummary[testStatus] = testStatusSummary.get(
                     testStatus, 0) + 1
@@ -170,7 +226,8 @@ def get(self, request, commit_hash: str | None):
                     "compiler": buildCompiler,
                     "status": {}
                 })
-                archSummary["status"][testStatus] = archSummary["status"].get(testStatus, 0) + 1
+                archSummary["status"][testStatus] = archSummary["status"].get(
+                    testStatus, 0) + 1
                 testArchSummary[archKey] = archSummary
 
                 configSummary = testConfigs.get(buildConfig, {})

dashboard/src/utils/filters.ts

@@ -2,8 +2,14 @@ import { TTreeDetailsFilter } from '@/types/tree/TreeDetails';
 
 // TODO: We can improve this idea and replace mapFilterToReq entirely
 const requestFilters = {
-  boot: ['boot.status', 'boot.duration_[gte]', 'boot.duration_[lte]'],
-  test: ['test.status', 'test.duration_[gte]', 'test.duration_[lte]'],
+  test: [
+    'test.status',
+    'test.duration_[gte]',
+    'test.duration_[lte]',
+    'boot.status',
+    'boot.duration_[gte]',
+    'boot.duration_[lte]',
+  ],
   treeDetails: [
     'treeDetails.config_name',
     'treeDetails.architecture',
@@ -30,7 +36,11 @@ export const getTargetFilter = (
 
     const splitted = k.split('.');
     const field = splitted[splitted.length - 1];
-    acc[field] = v;
+    if (target == 'test') {
+      acc[k] = v;
+    } else {
+      acc[field] = v;
+    }
   });
 
   return acc;