diff --git a/backend/kernelCI/settings.py b/backend/kernelCI/settings.py index a87b9b9c..80e6c395 100644 --- a/backend/kernelCI/settings.py +++ b/backend/kernelCI/settings.py @@ -196,7 +196,7 @@ def __getitem__(self, item): CORS_ALLOW_ALL_ORIGINS = False -CACHE_TIMEOUT = int(get_json_env_var("CACHE_TIMEOUT", "600")) +CACHE_TIMEOUT = int(get_json_env_var("CACHE_TIMEOUT", "180")) if DEBUG: CORS_ALLOWED_ORIGIN_REGEXES = [ diff --git a/backend/kernelCI_app/apps.py b/backend/kernelCI_app/apps.py index 77c34f89..37362c61 100644 --- a/backend/kernelCI_app/apps.py +++ b/backend/kernelCI_app/apps.py @@ -1,6 +1,11 @@ from django.apps import AppConfig +from kernelCI_app.cache import runCacheInvalidator class KernelciAppConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "kernelCI_app" + + def ready(self) -> None: + runCacheInvalidator() + return super().ready() diff --git a/backend/kernelCI_app/cache.py b/backend/kernelCI_app/cache.py new file mode 100644 index 00000000..baac4cba --- /dev/null +++ b/backend/kernelCI_app/cache.py @@ -0,0 +1,83 @@ +from django.core.cache import cache +from django.db import connection +from threading import Thread +from django.conf import settings + +NOTIFY_CHANNEL = "teste" +timeout = settings.CACHE_TIMEOUT + +__commit_lookup = {} +__build_lookup = {} +__test_lookup = {} + + +def __createCacheParamsHash(params: dict): + params_list = list(params.items()) + params_list.sort(key=lambda x: x[0]) + params_string = ",".join([str(i[1]) for i in params_list]) + return hash(params_string) + + +def setQueryCache(key, params, rows, commit_hash=None, build_id=None, test_id=None, + timeout=timeout): + params_hash = __createCacheParamsHash(params) + hash_key = "%s-%s" % (key, params_hash) + + __addToLookup(hash_key, commit_hash, __commit_lookup) + __addToLookup(hash_key, build_id, __build_lookup) + __addToLookup(hash_key, test_id, __test_lookup) + + return cache.set(hash_key, rows, timeout) + + +def getQueryCache(key, params: dict): + params_hash = __createCacheParamsHash(params) + return cache.get("%s-%s" % (key, params_hash)) + + +def runCacheInvalidator(): + cacheWorker = Thread(target=__listenWorker) + cacheWorker.start() + + +def __invalidateCommit(commit): + if commit in __commit_lookup: + cache.delete_many(__commit_lookup[commit]) + + +def __invalidateBuild(commit): + if commit in __build_lookup: + cache.delete_many(__build_lookup[commit]) + + +def __invalidateTest(commit): + if commit in __test_lookup: + cache.delete_many(__test_lookup[commit]) + + +def __processUpdate(value: str): + if value.startswith("commit:"): + __invalidateCommit(value.split(":", maxsplit=2)[1]) + elif value.startswith("build:"): + __invalidateBuild(value.split(":", maxsplit=2)[1]) + elif value.startswith("test:"): + __invalidateTest(value.split(":", maxsplit=2)[1]) + + +def __listenWorker(): + try: + with connection.cursor() as c: + c.execute("LISTEN %s" % NOTIFY_CHANNEL) + for n in c.connection.notifies(): + if n.channel == NOTIFY_CHANNEL: + __processUpdate(n.payload) + except Exception as ex: + print(ex) + + +def __addToLookup(cacheKey, propertyKey, lookup): + if propertyKey is None: + return + if lookup[propertyKey] is None: + lookup[propertyKey] = set() + lookup[propertyKey].append(cacheKey) diff --git a/backend/kernelCI_app/urls.py b/backend/kernelCI_app/urls.py index ff3d39eb..913f07ff 100644 --- a/backend/kernelCI_app/urls.py +++ b/backend/kernelCI_app/urls.py @@ -37,7 +37,7 @@ def viewCache(view): name="treeDetails" ), path("tree//full", - viewCache(views.TreeDetailsSlow), + views.TreeDetailsSlow.as_view(), name="TreeDetailsSlow" ), path("tree//commits", diff --git a/backend/kernelCI_app/views/treeDetailsSlowView.py b/backend/kernelCI_app/views/treeDetailsSlowView.py index 03cceeea..98b63d9e 100644 --- a/backend/kernelCI_app/views/treeDetailsSlowView.py +++ b/backend/kernelCI_app/views/treeDetailsSlowView.py @@ -9,6 +9,7 @@ toIntOrDefault, create_issue ) +from kernelCI_app.cache import (getQueryCache, setQueryCache) from django.db import connection @@ -298,88 +299,92 @@ def __processNonBootsTest(self, currentRowData): self.testFailReasons[testError] = self.testFailReasons.get(testError, 0) + 1 def get(self, request, commit_hash: str | None): + cache_key = "treeDetailsSlow" origin_param = request.GET.get("origin") git_url_param = request.GET.get("git_url") git_branch_param = request.GET.get("git_branch") self.__processFilters(request) - # Right now this query is only using for showing test data so it is doing inner joins - # in case it is needed for builds data they should become left join and the logic should be updated - query = """ - SELECT - tests.build_id AS tests_build_id, - tests.id AS tests_id, - tests.origin AS tests_origin, - tests.environment_comment AS tests_environment_comment, - tests.environment_misc AS tests_environment_misc, - tests.path AS tests_path, - tests.comment AS tests_comment, - tests.log_url AS tests_log_url, - tests.status AS tests_status, - tests.waived AS tests_waived, - tests.start_time AS tests_start_time, - tests.duration AS tests_duration, - tests.number_value AS tests_number_value, - tests.misc AS tests_misc, - builds_filter.*, - incidents.id, - incidents.present, - issues.id, - issues.comment, - issues.report_url - FROM - ( - SELECT - builds.checkout_id AS builds_checkout_id, - builds.id AS builds_id, - builds.comment AS builds_comment, - builds.start_time AS builds_start_time, - builds.duration AS builds_duration, - builds.architecture AS builds_architecture, - builds.command AS builds_command, - builds.compiler AS builds_compiler, - builds.config_name AS builds_config_name, - builds.config_url AS builds_config_url, - builds.log_url AS builds_log_url, - builds.valid AS builds_valid, - tree_head.* - FROM - ( - SELECT - checkouts.id AS checkout_id - FROM - checkouts - WHERE - checkouts.git_commit_hash = %(commit_hash)s AND - checkouts.git_repository_url = %(git_url_param)s AND - checkouts.git_repository_branch = %(git_branch_param)s AND - checkouts.origin = %(origin_param)s - ) AS tree_head - INNER JOIN builds - ON tree_head.checkout_id = builds.checkout_id - WHERE - builds.origin = %(origin_param)s - ) AS builds_filter - INNER JOIN tests - ON builds_filter.builds_id = tests.build_id - LEFT JOIN incidents - ON tests.id = incidents.test_id - LEFT JOIN issues - ON incidents.issue_id = issues.id - WHERE - tests.origin = %(origin_param)s - """ - with connection.cursor() as cursor: - cursor.execute( - query, - { - "commit_hash": commit_hash, - "origin_param": origin_param, - "git_url_param": git_url_param, - "git_branch_param": git_branch_param, - }, - ) - rows = cursor.fetchall() + params = { + "commit_hash": commit_hash, + "origin_param": origin_param, + "git_url_param": git_url_param, + "git_branch_param": git_branch_param, + } + + rows = getQueryCache(cache_key, params) + + if rows is None: + # Right now this query is only using for showing test data so it is doing inner joins + # if it is needed for builds data they should become left join and the logic should be updated + query = """ + SELECT + tests.build_id AS tests_build_id, + tests.id AS tests_id, + tests.origin AS tests_origin, + tests.environment_comment AS tests_enviroment_comment, + tests.environment_misc AS tests_enviroment_misc, + tests.path AS tests_path, + tests.comment AS tests_comment, + tests.log_url AS tests_log_url, + tests.status AS tests_status, + tests.waived AS tests_waived, + tests.start_time AS tests_start_time, + tests.duration AS tests_duration, + tests.number_value AS tests_number_value, + tests.misc AS tests_misc, + builds_filter.*, + incidents.id, + incidents.present, + issues.id, + issues.comment, + issues.report_url + FROM + ( + SELECT + builds.checkout_id AS builds_checkout_id, + builds.id AS builds_id, + builds.comment AS builds_comment, + builds.start_time AS builds_start_time, + builds.duration AS builds_duration, + builds.architecture AS builds_architecture, + builds.command AS builds_command, + builds.compiler AS builds_compiler, + builds.config_name AS builds_config_name, + builds.config_url AS builds_config_url, + builds.log_url AS builds_log_url, + builds.valid AS builds_valid, + tree_head.* + FROM + ( + SELECT + checkouts.id AS checkout_id + FROM + checkouts + WHERE + checkouts.git_commit_hash = %(commit_hash)s AND + checkouts.git_repository_url = %(git_url_param)s AND + checkouts.git_repository_branch = %(git_branch_param)s AND + checkouts.origin = %(origin_param)s + ) AS tree_head + INNER JOIN builds + ON tree_head.checkout_id = builds.checkout_id + WHERE + builds.origin = %(origin_param)s + ) AS builds_filter + INNER JOIN tests + ON builds_filter.builds_id = tests.build_id + LEFT JOIN incidents + ON tests.id = incidents.test_id + LEFT JOIN issues + ON incidents.issue_id = issues.id + WHERE + tests.origin = %(origin_param)s + """ + with connection.cursor() as cursor: + cursor.execute(query, params) + rows = cursor.fetchall() + setQueryCache(cache_key, params, rows) for currentRow in rows: currentRowData = self.__getCurrentRowData(currentRow)