Add per-entry ctime/mtime

src/aerovaldb/jsondb/cache.py

@@ -129,7 +129,7 @@ async def _read_json(self, file_path: str | Path) -> str:
                 return await f.read()
 
         with open(abspath, "r") as f:
-            return f.read().replace("\n", "")
+            return f.read()
 
     def _get(self, abspath: str) -> str:
         """Returns an element from the cache."""
@@ -140,7 +140,7 @@ def _get(self, abspath: str) -> str:
 
     def _put(self, abspath: str, *, json: str, modified: float):
         self._cache[abspath] = {
-            "json": "".join(json.split(r"\n")),
+            "json": json,
             "last_modified": os.path.getmtime(abspath),
         }
         while self.size > self._max_size:

src/aerovaldb/jsondb/jsonfiledb.py

@@ -208,9 +208,9 @@ async def _get_version(self, project: str, experiment: str) -> Version:
                 version = Version("0.0.1")
             finally:
                 return version
-        except simplejson.JSONDecodeError:
-            # Work around for https://github.com/metno/aerovaldb/issues/28
-            return Version("0.14.0")
+        # except simplejson.JSONDecodeError:
+        #    # Work around for https://github.com/metno/aerovaldb/issues/28
+        #    return Version("0.14.0")
 
         try:
             version_str = config["exp_info"]["pyaerocom_version"]
@@ -503,7 +503,7 @@ def _get_uri_for_file(self, file_path: str) -> str:
             route_arg_names = extract_substitutions(route)
 
             try:
-                all_args = parse_formatted_string(template, f"./{file_path}")
+                all_args = parse_formatted_string(template, f"./{file_path}")  # type: ignore
 
                 route_args = {k: v for k, v in all_args.items() if k in route_arg_names}
                 kwargs = {

src/aerovaldb/sqlitedb/sqlitedb.py

@@ -25,9 +25,8 @@ class AerovalSqliteDB(AerovalDB):
 
     # When creating a table it works to extract the substitution template
     # names from the route, as this constitutes all arguments. For the ones
-    # which have extra arguments (currently only time) the following table
-    # defines the override. Currently this only applies to map which has
-    # an extra time argument.
+    # which have extra arguments the following table defines the override.
+    # Currently this only applies to map which has an extra time argument.
     ROUTE_COLUMN_NAME_OVERRIDE = {
         ROUTE_MAP: (
             "project",
@@ -144,12 +143,35 @@ def _initialize_db(self):
 
             cur.execute(
                 f"""
-                CREATE TABLE IF NOT EXISTS {table_name}({column_names},json TEXT,
+                CREATE TABLE IF NOT EXISTS {table_name}(
+                    {column_names},
+                    ctime TEXT,
+                    mtime TEXT,
+                    json TEXT,
 
                 UNIQUE({column_names}))
                 """
             )
 
+            cur.execute(
+                f"""
+                CREATE TRIGGER IF NOT EXISTS insert_Timestamp_Trigger_{table_name}
+                AFTER INSERT ON {table_name}
+                BEGIN
+                   UPDATE {table_name} SET ctime =STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW') WHERE ROWID = NEW.ROWID;
+                END;
+                """
+            )
+            cur.execute(
+                f"""
+                CREATE TRIGGER IF NOT EXISTS update_Timestamp_Trigger_{table_name}
+                AFTER UPDATE On {table_name}
+                BEGIN
+                   UPDATE {table_name} SET mtime = STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW') WHERE ROWID = NEW.ROWID;
+                END;
+                """
+            )
+
         self._con.commit()
 
     def _get_column_list_and_substitution_list(self, kwargs: dict) -> tuple[str, str]:
@@ -200,7 +222,7 @@ async def _get(self, route, route_args, *args, **kwargs):
                 raise FileNotFoundError("Object not found")
             for r in fetched:
                 for k in r.keys():
-                    if k == "json":
+                    if k in ("json", "ctime", "mtime"):
                         continue
                     if not (k in route_args | kwargs) and r[k] is not None:
                         break
@@ -247,6 +269,10 @@ async def _put(self, obj, route, route_args, *args, **kwargs):
             """,
             route_args | kwargs,
         )
+
+        self._set_metadata_by_key(
+            "last_modified_by", f"aerovaldb_{aerovaldb.__version__}"
+        )
         self._con.commit()
 
     @async_and_sync
@@ -276,8 +302,6 @@ async def get_by_uri(
     async def put_by_uri(self, obj, uri: str):
         route, route_args, kwargs = parse_uri(uri)
 
-        # if isinstance(obj, str):
-        #    obj = "".join(obj.split(r"\n"))
         await self._put(obj, route, route_args, **kwargs)
 
     def list_all(self):
@@ -303,9 +327,6 @@ def list_all(self):
                     else:
                         kwargs[k] = r[k]
 
-                # route_args = {k: v for k, v in r.items() if k != "json" and k in arg_names}
-                # kwargs = {k: v for k, v in r.items() if k != "json" and not (k in arg_names)}
-
                 route = build_uri(route, route_args, kwargs)
                 yield route
 

src/aerovaldb/utils.py

@@ -48,14 +48,11 @@ def parse_formatted_string(template: str, s: str) -> dict:
     pattern = "".join(tokens)
 
     # Use our pattern to match the given string, raise if it doesn't match
-    matches = re.match(pattern, s)
-    if not matches:
+    if not (match := re.match(pattern, s)):
         raise Exception("Format string did not match")
 
     # Return a dict with all of our keywords and their values
-    result = {x: matches.group(x) for x in keywords}
-
-    return result
+    return {x: match.group(x) for x in keywords}
 
 
 def parse_uri(uri: str) -> tuple[str, dict[str, str], dict[str, str]]:

tests/test_aerovaldb.py

@@ -286,23 +286,13 @@ def tmpdb(tmp_path, dbtype: str) -> aerovaldb.AerovalDB:
 
 
 @pytest.mark.asyncio
-@pytest.mark.parametrize(
-    "resource",
-    (
-        pytest.param(
-            "json_files:./tests/test-db/json",
-        ),
-        pytest.param(
-            "sqlitedb:./tests/test-db/sqlite/test.sqlite",
-        ),
-    ),
-)
+@TESTDB_PARAMETRIZATION
 @GET_PARAMETRIZATION
-async def test_getter(resource: str, fun: str, args: list, kwargs: dict, expected):
+async def test_getter(testdb: str, fun: str, args: list, kwargs: dict, expected):
     """
     This test tests that data is read as expected from a static, fixed database.
     """
-    with aerovaldb.open(resource, use_async=True) as db:
+    with aerovaldb.open(testdb, use_async=True) as db:
         f = getattr(db, fun)
 
         if kwargs is not None:
@@ -313,18 +303,10 @@ async def test_getter(resource: str, fun: str, args: list, kwargs: dict, expecte
         assert data["path"] == expected
 
 
-@pytest.mark.parametrize(
-    "resource",
-    (
-        pytest.param(
-            "json_files:./tests/test-db/json",
-        ),
-        pytest.param("sqlitedb:./tests/test-db/sqlite/test.sqlite"),
-    ),
-)
+@TESTDB_PARAMETRIZATION
 @GET_PARAMETRIZATION
-def test_getter_sync(resource: str, fun: str, args: list, kwargs: dict, expected):
-    with aerovaldb.open(resource, use_async=False) as db:
+def test_getter_sync(testdb: str, fun: str, args: list, kwargs: dict, expected):
+    with aerovaldb.open(testdb, use_async=False) as db:
         f = getattr(db, fun)
 
         if kwargs is not None: