feat: add query level cache logic

backend/kernelCI/settings.py

@@ -196,7 +196,7 @@ def __getitem__(self, item):
 
 CORS_ALLOW_ALL_ORIGINS = False
 
-CACHE_TIMEOUT = int(get_json_env_var("CACHE_TIMEOUT", "600"))
+CACHE_TIMEOUT = int(get_json_env_var("CACHE_TIMEOUT", "180"))
 
 if DEBUG:
     CORS_ALLOWED_ORIGIN_REGEXES = [

backend/kernelCI_app/apps.py

@@ -1,6 +1,11 @@
 from django.apps import AppConfig
+from kernelCI_app.cache import runCacheInvalidator
 
 
 class KernelciAppConfig(AppConfig):
     default_auto_field = "django.db.models.BigAutoField"
     name = "kernelCI_app"
+
+    def ready(self) -> None:
+        runCacheInvalidator()
+        return super().ready()

backend/kernelCI_app/cache.py

@@ -0,0 +1,83 @@
+from django.core.cache import cache
+from django.db import connection
+from threading import Thread
+from django.conf import settings
+
+NOTIFY_CHANNEL = "teste"
+timeout = settings.CACHE_TIMEOUT
+
+__commit_lookup = {}
+__build_lookup = {}
+__test_lookup = {}
+
+
+def __createCacheParamsHash(params: dict):
+    params_list = list(params.items())
+    params_list.sort(key=lambda x: x[0])
+    params_string = ",".join([str(i[1]) for i in params_list])
+    return hash(params_string)
+
+
+def setQueryCache(key, params, rows, commit_hash=None, build_id=None, test_id=None,
+                  timeout=timeout):
+    params_hash = __createCacheParamsHash(params)
+    hash_key = "%s-%s" % (key, params_hash)
+
+    __addToLookup(hash_key, commit_hash, __commit_lookup)
+    __addToLookup(hash_key, build_id, __build_lookup)
+    __addToLookup(hash_key, test_id, __test_lookup)
+
+    return cache.set(hash_key, rows, timeout)
+
+
+def getQueryCache(key, params: dict):
+    params_hash = __createCacheParamsHash(params)
+    return cache.get("%s-%s" % (key, params_hash))
+
+
+def runCacheInvalidator():
+    cacheWorker = Thread(target=__listenWorker)
+    cacheWorker.start()
+
+
+def __invalidateCommit(commit):
+    if commit in __commit_lookup:
+        cache.delete_many(__commit_lookup[commit])
+
+
+def __invalidateBuild(commit):
+    if commit in __build_lookup:
+        cache.delete_many(__build_lookup[commit])
+
+
+def __invalidateTest(commit):
+    if commit in __test_lookup:
+        cache.delete_many(__test_lookup[commit])
+
+
+def __processUpdate(value: str):
+    if value.startswith("commit:"):
+        __invalidateCommit(value.split(":", maxsplit=2)[1])
+    elif value.startswith("build:"):
+        __invalidateBuild(value.split(":", maxsplit=2)[1])
+    elif value.startswith("test:"):
+        __invalidateTest(value.split(":", maxsplit=2)[1])
+
+
+def __listenWorker():
+    try:
+        with connection.cursor() as c:
+            c.execute("LISTEN %s" % NOTIFY_CHANNEL)
+            for n in c.connection.notifies():
+                if n.channel == NOTIFY_CHANNEL:
+                    __processUpdate(n.payload)
+    except Exception as ex:
+        print(ex)
+
+
+def __addToLookup(cacheKey, propertyKey, lookup):
+    if propertyKey is None:
+        return
+    if lookup[propertyKey] is None:
+        lookup[propertyKey] = set()
+    lookup[propertyKey].append(cacheKey)

backend/kernelCI_app/urls.py

@@ -37,7 +37,7 @@ def viewCache(view):
          name="treeDetails"
          ),
     path("tree/<str:commit_hash>/full",
-         viewCache(views.TreeDetailsSlow),
+         views.TreeDetailsSlow.as_view(),
          name="TreeDetailsSlow"
          ),
     path("tree/<str:commit_hash>/commits",

backend/kernelCI_app/views/treeDetailsSlowView.py

@@ -9,6 +9,7 @@
     toIntOrDefault,
     create_issue
 )
+from kernelCI_app.cache import (getQueryCache, setQueryCache)
 from django.db import connection
 
 
@@ -298,88 +299,92 @@ def __processNonBootsTest(self, currentRowData):
             self.testFailReasons[testError] = self.testFailReasons.get(testError, 0) + 1
 
     def get(self, request, commit_hash: str | None):
+        cache_key = "treeDetailsSlow"
         origin_param = request.GET.get("origin")
         git_url_param = request.GET.get("git_url")
         git_branch_param = request.GET.get("git_branch")
         self.__processFilters(request)
 
-        # Right now this query is only using for showing test data so it is doing inner joins
-        # in case it is needed for builds data they should become left join and the logic should be updated
-        query = """
-        SELECT
-                tests.build_id AS tests_build_id,
-                tests.id AS tests_id,
-                tests.origin AS tests_origin,
-                tests.environment_comment AS tests_environment_comment,
-                tests.environment_misc AS tests_environment_misc,
-                tests.path AS tests_path,
-                tests.comment AS tests_comment,
-                tests.log_url AS tests_log_url,
-                tests.status AS tests_status,
-                tests.waived AS tests_waived,
-                tests.start_time AS tests_start_time,
-                tests.duration AS tests_duration,
-                tests.number_value AS tests_number_value,
-                tests.misc AS tests_misc,
-                builds_filter.*,
-                incidents.id,
-                incidents.present,
-                issues.id,
-                issues.comment,
-                issues.report_url
-        FROM
-            (
-                SELECT
-                    builds.checkout_id AS builds_checkout_id,
-                    builds.id AS builds_id,
-                    builds.comment AS builds_comment,
-                    builds.start_time AS builds_start_time,
-                    builds.duration AS builds_duration,
-                    builds.architecture AS builds_architecture,
-                    builds.command AS builds_command,
-                    builds.compiler AS builds_compiler,
-                    builds.config_name AS builds_config_name,
-                    builds.config_url AS builds_config_url,
-                    builds.log_url AS builds_log_url,
-                    builds.valid AS builds_valid,
-                    tree_head.*
-                FROM
-                    (
-                        SELECT
-                            checkouts.id AS checkout_id
-                        FROM
-                            checkouts
-                        WHERE
-                            checkouts.git_commit_hash = %(commit_hash)s AND
-                            checkouts.git_repository_url = %(git_url_param)s AND
-                            checkouts.git_repository_branch = %(git_branch_param)s AND
-                            checkouts.origin = %(origin_param)s
-                    ) AS tree_head
-                INNER JOIN builds
-                    ON tree_head.checkout_id = builds.checkout_id
-                WHERE
-                    builds.origin = %(origin_param)s
-            ) AS builds_filter
-        INNER JOIN tests
-            ON builds_filter.builds_id = tests.build_id
-        LEFT JOIN incidents
-            ON tests.id = incidents.test_id
-        LEFT JOIN issues
-            ON incidents.issue_id = issues.id
-        WHERE
-            tests.origin = %(origin_param)s
-        """
-        with connection.cursor() as cursor:
-            cursor.execute(
-                query,
-                {
-                    "commit_hash": commit_hash,
-                    "origin_param": origin_param,
-                    "git_url_param": git_url_param,
-                    "git_branch_param": git_branch_param,
-                },
-            )
-            rows = cursor.fetchall()
+        params = {
+            "commit_hash": commit_hash,
+            "origin_param": origin_param,
+            "git_url_param": git_url_param,
+            "git_branch_param": git_branch_param,
+        }
+
+        rows = getQueryCache(cache_key, params)
+
+        if rows is None:
+            # Right now this query is only using for showing test data so it is doing inner joins
+            # if it is needed for builds data they should become left join and the logic should be updated
+            query = """
+            SELECT
+                    tests.build_id AS tests_build_id,
+                    tests.id AS tests_id,
+                    tests.origin AS tests_origin,
+                    tests.environment_comment AS tests_enviroment_comment,
+                    tests.environment_misc AS tests_enviroment_misc,
+                    tests.path AS tests_path,
+                    tests.comment AS tests_comment,
+                    tests.log_url AS tests_log_url,
+                    tests.status AS tests_status,
+                    tests.waived AS tests_waived,
+                    tests.start_time AS tests_start_time,
+                    tests.duration AS tests_duration,
+                    tests.number_value AS tests_number_value,
+                    tests.misc AS tests_misc,
+                    builds_filter.*,
+                    incidents.id,
+                    incidents.present,
+                    issues.id,
+                    issues.comment,
+                    issues.report_url
+            FROM
+                (
+                    SELECT
+                        builds.checkout_id AS builds_checkout_id,
+                        builds.id AS builds_id,
+                        builds.comment AS builds_comment,
+                        builds.start_time AS builds_start_time,
+                        builds.duration AS builds_duration,
+                        builds.architecture AS builds_architecture,
+                        builds.command AS builds_command,
+                        builds.compiler AS builds_compiler,
+                        builds.config_name AS builds_config_name,
+                        builds.config_url AS builds_config_url,
+                        builds.log_url AS builds_log_url,
+                        builds.valid AS builds_valid,
+                        tree_head.*
+                    FROM
+                        (
+                            SELECT
+                                checkouts.id AS checkout_id
+                            FROM
+                                checkouts
+                            WHERE
+                                checkouts.git_commit_hash = %(commit_hash)s AND
+                                checkouts.git_repository_url = %(git_url_param)s AND
+                                checkouts.git_repository_branch = %(git_branch_param)s AND
+                                checkouts.origin = %(origin_param)s
+                        ) AS tree_head
+                    INNER JOIN builds
+                        ON tree_head.checkout_id = builds.checkout_id
+                    WHERE
+                        builds.origin = %(origin_param)s
+                ) AS builds_filter
+            INNER JOIN tests
+                ON builds_filter.builds_id = tests.build_id
+            LEFT JOIN incidents
+                ON tests.id = incidents.test_id
+            LEFT JOIN issues
+                ON incidents.issue_id = issues.id
+            WHERE
+                tests.origin = %(origin_param)s
+            """
+            with connection.cursor() as cursor:
+                cursor.execute(query, params)
+                rows = cursor.fetchall()
+                setQueryCache(cache_key, params, rows)
 
         for currentRow in rows:
             currentRowData = self.__getCurrentRowData(currentRow)

backend/kernelCI_app/views/treeView.py

@@ -10,7 +10,13 @@
 
 class TreeView(View):
     def get(self, request):
-        origin = request.GET.get('origin', DEFAULT_ORIGIN)
+        origin_param = request.GET.get('origin', DEFAULT_ORIGIN)
+        interval_param = getQueryTimeInterval().timestamp()
+
+        params = {
+            "origin_param": origin_param,
+            "interval_param": interval_param
+        }
 
         checkouts = Checkouts.objects.raw(
             """
@@ -29,7 +35,7 @@ def get(self, request):
                         FROM
                             checkouts
                         WHERE
-                            origin = %s AND start_time  >= TO_TIMESTAMP(%s)
+                            origin = %(origin_param)s AND start_time  >= TO_TIMESTAMP(%(interval_param)s)
                         ORDER BY
                             start_time DESC
                     ) AS selection_sorted
@@ -94,9 +100,8 @@ def get(self, request):
             ORDER BY
                 checkouts.start_time DESC;
             ;
-            """, [origin, getQueryTimeInterval().timestamp()]
+            """, params
         )
-
         serializer = TreeSerializer(checkouts, many=True)
         resp = JsonResponse(serializer.data, safe=False)
         return resp

dashboard/src/api/Tree.tsx

@@ -35,7 +35,7 @@ const fetchTreeFastCheckoutData = async (
 export const useTreeTableFast = (): UseQueryResult<TreeFastPathResponse> => {
   const { origin } = useSearch({ from: '/tree' });
   return useQuery({
-    queryKey: ['treeTableFAst', origin],
+    queryKey: ['treeTableFast', origin],
     queryFn: () => fetchTreeFastCheckoutData(origin),
   });
 };