From 01d0aec6b655ae5716e007bc4ee9f2bb2c772dbd Mon Sep 17 00:00:00 2001 From: thorbjoernl <51087536+thorbjoernl@users.noreply.github.com> Date: Wed, 14 Aug 2024 10:20:57 +0200 Subject: [PATCH] Add per-entry ctime/mtime --- src/aerovaldb/jsondb/cache.py | 4 +-- src/aerovaldb/jsondb/jsonfiledb.py | 8 +++--- src/aerovaldb/sqlitedb/sqlitedb.py | 41 ++++++++++++++++++++++-------- src/aerovaldb/utils.py | 7 ++--- tests/test_aerovaldb.py | 30 +++++----------------- 5 files changed, 45 insertions(+), 45 deletions(-) diff --git a/src/aerovaldb/jsondb/cache.py b/src/aerovaldb/jsondb/cache.py index 91ad610..9ddcb72 100644 --- a/src/aerovaldb/jsondb/cache.py +++ b/src/aerovaldb/jsondb/cache.py @@ -129,7 +129,7 @@ async def _read_json(self, file_path: str | Path) -> str: return await f.read() with open(abspath, "r") as f: - return f.read().replace("\n", "") + return f.read() def _get(self, abspath: str) -> str: """Returns an element from the cache.""" @@ -140,7 +140,7 @@ def _get(self, abspath: str) -> str: def _put(self, abspath: str, *, json: str, modified: float): self._cache[abspath] = { - "json": "".join(json.split(r"\n")), + "json": json, "last_modified": os.path.getmtime(abspath), } while self.size > self._max_size: diff --git a/src/aerovaldb/jsondb/jsonfiledb.py b/src/aerovaldb/jsondb/jsonfiledb.py index 5f6371d..71a5e3c 100644 --- a/src/aerovaldb/jsondb/jsonfiledb.py +++ b/src/aerovaldb/jsondb/jsonfiledb.py @@ -208,9 +208,9 @@ async def _get_version(self, project: str, experiment: str) -> Version: version = Version("0.0.1") finally: return version - except simplejson.JSONDecodeError: - # Work around for https://github.com/metno/aerovaldb/issues/28 - return Version("0.14.0") + # except simplejson.JSONDecodeError: + # # Work around for https://github.com/metno/aerovaldb/issues/28 + # return Version("0.14.0") try: version_str = config["exp_info"]["pyaerocom_version"] @@ -503,7 +503,7 @@ def _get_uri_for_file(self, file_path: str) -> str: route_arg_names = extract_substitutions(route) try: - all_args = parse_formatted_string(template, f"./{file_path}") + all_args = parse_formatted_string(template, f"./{file_path}") # type: ignore route_args = {k: v for k, v in all_args.items() if k in route_arg_names} kwargs = { diff --git a/src/aerovaldb/sqlitedb/sqlitedb.py b/src/aerovaldb/sqlitedb/sqlitedb.py index bc8274a..32611f9 100644 --- a/src/aerovaldb/sqlitedb/sqlitedb.py +++ b/src/aerovaldb/sqlitedb/sqlitedb.py @@ -25,9 +25,8 @@ class AerovalSqliteDB(AerovalDB): # When creating a table it works to extract the substitution template # names from the route, as this constitutes all arguments. For the ones - # which have extra arguments (currently only time) the following table - # defines the override. Currently this only applies to map which has - # an extra time argument. + # which have extra arguments the following table defines the override. + # Currently this only applies to map which has an extra time argument. ROUTE_COLUMN_NAME_OVERRIDE = { ROUTE_MAP: ( "project", @@ -144,12 +143,35 @@ def _initialize_db(self): cur.execute( f""" - CREATE TABLE IF NOT EXISTS {table_name}({column_names},json TEXT, + CREATE TABLE IF NOT EXISTS {table_name}( + {column_names}, + ctime TEXT, + mtime TEXT, + json TEXT, UNIQUE({column_names})) """ ) + cur.execute( + f""" + CREATE TRIGGER IF NOT EXISTS insert_Timestamp_Trigger_{table_name} + AFTER INSERT ON {table_name} + BEGIN + UPDATE {table_name} SET ctime =STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW') WHERE ROWID = NEW.ROWID; + END; + """ + ) + cur.execute( + f""" + CREATE TRIGGER IF NOT EXISTS update_Timestamp_Trigger_{table_name} + AFTER UPDATE On {table_name} + BEGIN + UPDATE {table_name} SET mtime = STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW') WHERE ROWID = NEW.ROWID; + END; + """ + ) + self._con.commit() def _get_column_list_and_substitution_list(self, kwargs: dict) -> tuple[str, str]: @@ -200,7 +222,7 @@ async def _get(self, route, route_args, *args, **kwargs): raise FileNotFoundError("Object not found") for r in fetched: for k in r.keys(): - if k == "json": + if k in ("json", "ctime", "mtime"): continue if not (k in route_args | kwargs) and r[k] is not None: break @@ -247,6 +269,10 @@ async def _put(self, obj, route, route_args, *args, **kwargs): """, route_args | kwargs, ) + + self._set_metadata_by_key( + "last_modified_by", f"aerovaldb_{aerovaldb.__version__}" + ) self._con.commit() @async_and_sync @@ -276,8 +302,6 @@ async def get_by_uri( async def put_by_uri(self, obj, uri: str): route, route_args, kwargs = parse_uri(uri) - # if isinstance(obj, str): - # obj = "".join(obj.split(r"\n")) await self._put(obj, route, route_args, **kwargs) def list_all(self): @@ -303,9 +327,6 @@ def list_all(self): else: kwargs[k] = r[k] - # route_args = {k: v for k, v in r.items() if k != "json" and k in arg_names} - # kwargs = {k: v for k, v in r.items() if k != "json" and not (k in arg_names)} - route = build_uri(route, route_args, kwargs) yield route diff --git a/src/aerovaldb/utils.py b/src/aerovaldb/utils.py index b195c67..324ef53 100644 --- a/src/aerovaldb/utils.py +++ b/src/aerovaldb/utils.py @@ -48,14 +48,11 @@ def parse_formatted_string(template: str, s: str) -> dict: pattern = "".join(tokens) # Use our pattern to match the given string, raise if it doesn't match - matches = re.match(pattern, s) - if not matches: + if not (match := re.match(pattern, s)): raise Exception("Format string did not match") # Return a dict with all of our keywords and their values - result = {x: matches.group(x) for x in keywords} - - return result + return {x: match.group(x) for x in keywords} def parse_uri(uri: str) -> tuple[str, dict[str, str], dict[str, str]]: diff --git a/tests/test_aerovaldb.py b/tests/test_aerovaldb.py index adb08b7..418de92 100644 --- a/tests/test_aerovaldb.py +++ b/tests/test_aerovaldb.py @@ -286,23 +286,13 @@ def tmpdb(tmp_path, dbtype: str) -> aerovaldb.AerovalDB: @pytest.mark.asyncio -@pytest.mark.parametrize( - "resource", - ( - pytest.param( - "json_files:./tests/test-db/json", - ), - pytest.param( - "sqlitedb:./tests/test-db/sqlite/test.sqlite", - ), - ), -) +@TESTDB_PARAMETRIZATION @GET_PARAMETRIZATION -async def test_getter(resource: str, fun: str, args: list, kwargs: dict, expected): +async def test_getter(testdb: str, fun: str, args: list, kwargs: dict, expected): """ This test tests that data is read as expected from a static, fixed database. """ - with aerovaldb.open(resource, use_async=True) as db: + with aerovaldb.open(testdb, use_async=True) as db: f = getattr(db, fun) if kwargs is not None: @@ -313,18 +303,10 @@ async def test_getter(resource: str, fun: str, args: list, kwargs: dict, expecte assert data["path"] == expected -@pytest.mark.parametrize( - "resource", - ( - pytest.param( - "json_files:./tests/test-db/json", - ), - pytest.param("sqlitedb:./tests/test-db/sqlite/test.sqlite"), - ), -) +@TESTDB_PARAMETRIZATION @GET_PARAMETRIZATION -def test_getter_sync(resource: str, fun: str, args: list, kwargs: dict, expected): - with aerovaldb.open(resource, use_async=False) as db: +def test_getter_sync(testdb: str, fun: str, args: list, kwargs: dict, expected): + with aerovaldb.open(testdb, use_async=False) as db: f = getattr(db, fun) if kwargs is not None: