#290: Updated code to read ci.json instead of testconfig (#318)

* Added new function that reads ci.json using Pydantic, removed old testconfig and added unit test case

Author: sgn4sangar

doc/changes/changes_3.4.1.md

@@ -4,3 +4,4 @@
 
 - #309: Relaxed exasol-bucketfs from "^2.0.0" to ">=1,<3"
 - #289: Removed an obsolete param test-language of run-db-test
+- #290: Removed usage of testconfig by reading ci.json instead

exasol/slc/internal/tasks/test/test_runner_db_test_base_task.py

@@ -1,3 +1,4 @@
+import json
 import pathlib
 from collections.abc import Generator
 from typing import Any, Optional
@@ -54,6 +55,11 @@
     UploadExportedContainer,
 )
 from exasol.slc.internal.tasks.upload.language_definition import LanguageDefinition
+from exasol.slc.models.flavor_ci_model import (
+    FlavorCiConfig,
+    TestConfig,
+    TestSet,
+)
 from exasol.slc.models.run_db_test_result import RunDBTestsInTestConfigResult
 
 
@@ -196,9 +202,10 @@ def _get_uploaded_container_name(self) -> str:
     def run_test(
         self, test_environment_info: EnvironmentInfo, uploaded_container_name: str
     ) -> Generator[RunDBTestsInTestConfig, Any, RunDBTestsInTestConfigResult]:
-        test_config = self.read_test_config()
-        generic_language_tests = self.get_generic_language_tests(test_config)
-        test_folders = self.get_test_folders(test_config)
+        ci_json_file = pathlib.Path(self.flavor_path).joinpath("ci.json")
+        ci_test_cfg = read_ci_json(ci_json_file)
+        generic_language_tests = self.get_generic_language_tests(ci_test_cfg)
+        test_folders = self.get_test_folders(ci_test_cfg)
         database_credentials = self.get_database_credentials()
         # "myudfs/containers/" + self.export_info.name + ".tar.gz"
         language_definition = LanguageDefinition(
@@ -238,8 +245,12 @@ def get_result_status(status):
 
     def get_test_folders(self, test_config):
         test_folders = []
-        if test_config["test_folders"] != "":
-            test_folders = test_config["test_folders"].split(" ")
+        for test_set in test_config.test_sets:
+            test_set_goal = str(test_set.goal)
+            luigi_param = str(TestRunnerDBTestBaseTask.release_goal)
+            if test_set_goal.lower() == luigi_param.lower():
+                for test_folder in test_set.folders:
+                    test_folders.append(test_folder)
         if self.tests_specified_in_parameters():
             test_folders = self.test_folders
         return test_folders
@@ -253,25 +264,20 @@ def tests_specified_in_parameters(self):
 
     def get_generic_language_tests(self, test_config):
         generic_language_tests = []
-        if test_config["generic_language_tests"] != "":
-            generic_language_tests = test_config["generic_language_tests"].split(" ")
+        for test_set in test_config.test_sets:
+            test_set_goal = str(test_set.goal)
+            luigi_param = str(TestRunnerDBTestBaseTask.release_goal)
+            if test_set_goal.lower() == luigi_param.lower():
+                for gen_lang_test in test_set.generic_language_tests:
+                    generic_language_tests.append(gen_lang_test)
         if self.tests_specified_in_parameters():
             generic_language_tests = self.generic_language_tests
         return generic_language_tests
 
-    def read_test_config(self):
-        with (
-            pathlib.Path(self.flavor_path)
-            .joinpath("flavor_base")
-            .joinpath("testconfig")
-            .open("r") as file
-        ):
-            test_config_str = file.read()
-            test_config = {}
-            for line in test_config_str.splitlines():
-                if not line.startswith("#") and not line == "":
-                    split = line.split("=")
-                    key = split[0]
-                    value = "=".join(split[1:])
-                    test_config[key] = value
-        return test_config
+
+def read_ci_json(ci_json_file: pathlib.Path):
+    if ci_json_file.exists() and ci_json_file.is_file():
+        with ci_json_file.open("r") as json_file:
+            ci_json_str = json_file.read()
+            flavor_ci_cfg = FlavorCiConfig.model_validate_json(ci_json_str)
+            return flavor_ci_cfg.test_config

exasol/slc/models/flavor_ci_model.py

@@ -0,0 +1,25 @@
+from typing import List, Optional
+
+from pydantic import BaseModel
+
+from exasol.slc.models.accelerator import Accelerator
+
+
+class TestSet(BaseModel):
+    name: str
+    files: list[str]
+    folders: list[str]
+    goal: str
+    generic_language_tests: list[str]
+    test_runner: Optional[str] = None
+    accelerator: Accelerator = Accelerator.NONE
+
+
+class TestConfig(BaseModel):
+    default_test_runner: str
+    test_sets: list[TestSet]
+
+
+class FlavorCiConfig(BaseModel):
+    build_runner: str
+    test_config: TestConfig

test/resources/flavors/real-test-flavor/ci.json

@@ -0,0 +1,15 @@
+{
+    "build_runner": "24.04",
+    "test_config": {
+        "default_test_runner": "24.04",
+        "test_sets": [
+            {
+                "name": "python",
+                "goal": "rel",
+                "files": [],
+                "folders": ["selftest"],
+                "generic_language_tests": []
+            }
+        ]
+    }
+}

test/resources/flavors/real-test-flavor/real_flavor_base/testconfig

@@ -0,0 +1,15 @@
+{
+    "build_runner": "24.04",
+    "test_config": {
+        "default_test_runner": "24.04",
+        "test_sets": [
+            {
+                "name": "python",
+                "goal": "rel",
+                "files": [],
+                "folders": ["selftest"],
+                "generic_language_tests": []
+            }
+        ]
+    }
+}

test/resources/flavors/test-flavor spaces/real-test-flavor/ci.json

@@ -1,4 +1,6 @@
 import inspect
+import json
+from pathlib import Path
 
 import pytest
 from exasol_integration_test_docker_environment.testing.api_consistency_utils import (  # type: ignore
@@ -9,6 +11,10 @@
 )
 
 from exasol.slc import api
+from exasol.slc.internal.tasks.test.test_runner_db_test_base_task import (
+    read_ci_json,
+)
+from exasol.slc.models.flavor_ci_model import FlavorCiConfig, TestConfig, TestSet
 from exasol.slc.tool import commands
 
 IGNORE_LIST = ["compression_strategy", "accelerator"]
@@ -83,3 +89,34 @@ def test_same_functions():
     )
 
     assert click_command_names == api_function_names
+
+
+def test_parse_ci_json(tmp_path: Path):
+    ci_json_data = {
+        "build_runner": "24.04",
+        "test_config": {
+            "default_test_runner": "24.04",
+            "test_sets": [
+                {
+                    "name": "python",
+                    "files": [],
+                    "folders": ["fld00", "fld01"],
+                    "goal": "rel",
+                    "generic_language_tests": ["py3.11", "py3.12"],
+                },
+                {
+                    "name": "python",
+                    "files": [],
+                    "folders": ["fld10", "fld11"],
+                    "goal": "rel",
+                    "generic_language_tests": ["py3.13", "py3.14"],
+                },
+            ],
+        },
+    }
+    json_file_path = tmp_path / "temp_data.json"
+    with open(json_file_path, "w") as json_file:
+        json.dump(ci_json_data, json_file)
+    actual_res = read_ci_json(json_file_path)
+    expected_res = FlavorCiConfig.model_validate(ci_json_data)
+    assert actual_res == expected_res.test_config