From bf749ee657cdb3b6b4a3cd1edf492b39313da1b6 Mon Sep 17 00:00:00 2001
From: David Liu <david@cs.toronto.edu>
Date: Thu, 21 Dec 2023 11:57:37 -0500
Subject: [PATCH] Track test environment setup in redis

---
 Changelog.md                       |  1 +
 client/autotest_client/__init__.py | 13 ++++++++++++-
 server/autotest_server/__init__.py | 12 ++++++++++--
 3 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/Changelog.md b/Changelog.md
index a1ea8152..0a66ac87 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -5,6 +5,7 @@ All notable changes to this project will be documented here.
 
 - Fix bug that prevented test results from being returned when a feedback file could not be found (#458)
 - Add support for Python 3.11 and 3.12 (#467)
+- Track test environment setup status and report errors when running tests if environment setup is in progress or raised an error (#468)
 
 ## [v2.3.1]
 - Fix a bug that prevented test file from being copied from a zip file to another location on disk (#426)
diff --git a/client/autotest_client/__init__.py b/client/autotest_client/__init__.py
index 5ac845fe..dc2ea3a8 100644
--- a/client/autotest_client/__init__.py
+++ b/client/autotest_client/__init__.py
@@ -213,6 +213,17 @@ def update_settings(settings_id, user):
 @app.route("/settings/<settings_id>/test", methods=["PUT"])
 @authorize
 def run_tests(settings_id, user):
+    test_settings = json.loads(REDIS_CONNECTION.hget("autotest:settings", key=settings_id))
+    env_status = test_settings.get("_env_status")
+    if env_status == "setup":
+        raise Exception("Setting up test environment. Please try again later.")
+    elif env_status == "error":
+        msg = "Settings Error"
+        settings_error = test_settings.get("_error", "")
+        if settings_error:
+            msg += f": {settings_error}"
+        raise Exception(msg)
+
     test_data = request.json["test_data"]
     categories = request.json["categories"]
     high_priority = request.json.get("request_high_priority")
@@ -221,7 +232,7 @@ def run_tests(settings_id, user):
 
     timeout = 0
 
-    for settings_ in settings(settings_id)["testers"]:
+    for settings_ in test_settings["testers"]:
         for data in settings_["test_data"]:
             timeout += data["timeout"]
 
diff --git a/server/autotest_server/__init__.py b/server/autotest_server/__init__.py
index 2d9e6816..4d067f9b 100644
--- a/server/autotest_server/__init__.py
+++ b/server/autotest_server/__init__.py
@@ -204,7 +204,8 @@ def _run_test_specs(
                 timeout_expired = None
                 timeout = test_data.get("timeout")
                 try:
-                    env_vars = {**os.environ, **_get_env_vars(test_username), **settings["_env"]}
+                    env = settings.get("_env", {})
+                    env_vars = {**os.environ, **_get_env_vars(test_username), **env}
                     env_vars = _update_env_vars(env_vars, test_env_vars)
                     proc = subprocess.Popen(
                         args,
@@ -216,7 +217,7 @@ def _run_test_specs(
                         stdin=subprocess.PIPE,
                         preexec_fn=set_rlimits_before_test,
                         universal_newlines=True,
-                        env={**os.environ, **env_vars, **settings["_env"]},
+                        env={**os.environ, **env_vars, **env},
                     )
                     try:
                         settings_json = json.dumps({**settings, "test_data": test_data})
@@ -359,6 +360,11 @@ def ignore_missing_dir_error(
 
 
 def update_test_settings(user, settings_id, test_settings, file_url):
+    test_settings["_user"] = user
+    test_settings["_last_access"] = int(time.time())
+    test_settings["_env_status"] = "setup"
+    redis_connection().hset("autotest:settings", key=settings_id, value=json.dumps(test_settings))
+
     try:
         settings_dir = os.path.join(TEST_SCRIPT_DIR, str(settings_id))
 
@@ -391,8 +397,10 @@ def update_test_settings(user, settings_id, test_settings, file_url):
             test_settings["testers"][i] = tester_settings
         test_settings["_files"] = files_dir
         test_settings.pop("_error", None)
+        test_settings["_env_status"] = "ready"
     except Exception as e:
         test_settings["_error"] = str(e)
+        test_settings["_env_status"] = "error"
         raise
     finally:
         test_settings["_user"] = user