From 29650dae4794a2dd43af7549ade6f20809aab42f Mon Sep 17 00:00:00 2001 From: zack jensen Date: Mon, 18 Nov 2024 12:20:57 -0700 Subject: [PATCH 01/17] feat: cache queries to speed up _get_id calls Added: - Function to create an integer for query_string/param input - _QUERY_CACHE object to hold results from all queries --- src/plexosdb/sqlite.py | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/src/plexosdb/sqlite.py b/src/plexosdb/sqlite.py index b3d2839..87c1765 100644 --- a/src/plexosdb/sqlite.py +++ b/src/plexosdb/sqlite.py @@ -27,6 +27,7 @@ class PlexosSQLite: DB_FILENAME = "plexos.db" _conn: sqlite3.Connection + _QUERY_CACHE: dict[int, list[tuple]] = {} def __init__( self, @@ -1041,6 +1042,32 @@ def execute_query(self, query: str, params=None) -> None: _ = conn.execute(query, params) if params else conn.execute(query) return + @staticmethod + def _query_hash(query_string: str, params: tuple|dict|None = None) -> int: + """ + Create a hash int for a query string and params dictionary + Parameters + ---------- + query_str + String to get passed to the database connector. + params + Tuple or dict for passing + + Returns + ------- + Int + likely unique integer for given query_string and params object + """ + + if params is None: + return hash(query_string) + if isinstance(params, dict): + return hash((query_string, str(params))) + if isinstance(params, list): + return hash((query_string, *params)) + return hash((query_string, params)) + + def query(self, query_string: str, params=None) -> list[tuple]: """Execute of query to the database. @@ -1063,9 +1090,18 @@ def query(self, query_string: str, params=None) -> list[tuple]: This function could be slow depending the complexity of the query passed. """ + query_key = self._query_hash(query_string, params) + if query_key in self._QUERY_CACHE: + return self._QUERY_CACHE[query_key] + with self._conn as conn: res = conn.execute(query_string, params) if params else conn.execute(query_string) - return res.fetchall() + ret = res.fetchall() + + if ret: + self._QUERY_CACHE[query_key] = ret + + return ret def ingest_from_records(self, tag: str, record_data: Sequence): """Insert elements from xml to database.""" From d32720d54d0d66c6f54393ad41f0f8482ca1884c Mon Sep 17 00:00:00 2001 From: zack jensen Date: Mon, 18 Nov 2024 12:21:21 -0700 Subject: [PATCH 02/17] docs: unused parameter --- src/plexosdb/sqlite.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/plexosdb/sqlite.py b/src/plexosdb/sqlite.py index 87c1765..2105236 100644 --- a/src/plexosdb/sqlite.py +++ b/src/plexosdb/sqlite.py @@ -1081,8 +1081,6 @@ def query(self, query_string: str, params=None) -> list[tuple]: String to get passed to the database connector. params Tuple or dict for passing - fetchone - Return firstrow Note ---- From 0c67e83f4b9138c3de52bf7c2102119f9c6681e2 Mon Sep 17 00:00:00 2001 From: zack jensen Date: Mon, 18 Nov 2024 13:05:22 -0700 Subject: [PATCH 03/17] fix: only cache ids Some queries do change throught the process of using plexosdb from r2x, get_id should not change during a run --- src/plexosdb/sqlite.py | 75 ++++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/src/plexosdb/sqlite.py b/src/plexosdb/sqlite.py index 2105236..77011e0 100644 --- a/src/plexosdb/sqlite.py +++ b/src/plexosdb/sqlite.py @@ -27,7 +27,6 @@ class PlexosSQLite: DB_FILENAME = "plexos.db" _conn: sqlite3.Connection - _QUERY_CACHE: dict[int, list[tuple]] = {} def __init__( self, @@ -38,6 +37,7 @@ def __init__( super().__init__() self._conn = sqlite3.connect(":memory:") self._sqlite_config() + self._QUERY_CACHE: dict[int, int] = {} if create_collations: self._create_collations() self._create_table_schema() @@ -757,6 +757,7 @@ def _get_id( parent_class_name: ClassEnum | None = None, child_class_name: ClassEnum | None = None, category_name: str | None = None, + use_cache: bool = True, ) -> int: """Return the ID for a given table and object name combination. @@ -838,6 +839,9 @@ def _get_id( if conditions else f" WHERE {table_name}.name = :object_name" ) + query_key = self._query_hash(query, params) + if query_key in self._QUERY_CACHE and use_cache: + return self._QUERY_CACHE[query_key] result = self.query(query, params) @@ -848,7 +852,12 @@ def _get_id( if len(result) > 1: msg = f"Multiple ids returned for {object_name} and {class_name}. Try passing addtional filters" raise ValueError(msg) - return result[0][0] # Get first element of tuple + + ret: int = result[0][0] # Get first element of tuple + + self._QUERY_CACHE[query_key] = ret + + return ret def get_membership_id( self, @@ -1042,32 +1051,6 @@ def execute_query(self, query: str, params=None) -> None: _ = conn.execute(query, params) if params else conn.execute(query) return - @staticmethod - def _query_hash(query_string: str, params: tuple|dict|None = None) -> int: - """ - Create a hash int for a query string and params dictionary - Parameters - ---------- - query_str - String to get passed to the database connector. - params - Tuple or dict for passing - - Returns - ------- - Int - likely unique integer for given query_string and params object - """ - - if params is None: - return hash(query_string) - if isinstance(params, dict): - return hash((query_string, str(params))) - if isinstance(params, list): - return hash((query_string, *params)) - return hash((query_string, params)) - - def query(self, query_string: str, params=None) -> list[tuple]: """Execute of query to the database. @@ -1088,17 +1071,10 @@ def query(self, query_string: str, params=None) -> list[tuple]: This function could be slow depending the complexity of the query passed. """ - query_key = self._query_hash(query_string, params) - if query_key in self._QUERY_CACHE: - return self._QUERY_CACHE[query_key] - with self._conn as conn: res = conn.execute(query_string, params) if params else conn.execute(query_string) ret = res.fetchall() - if ret: - self._QUERY_CACHE[query_key] = ret - return ret def ingest_from_records(self, tag: str, record_data: Sequence): @@ -1269,3 +1245,32 @@ def _create_collations(self) -> None: """Add collate function for helping search enums.""" self._conn.create_collation("NOSPACE", no_space) return + + @staticmethod + def _query_hash(query_string: str, params: tuple | dict | None = None) -> int: + """ + Create a hash int for a query string and params dictionary. + + Parameters + ---------- + query_str + String to get passed to the database connector. + params + Tuple or dict for passing + + Returns + ------- + Int + likely unique integer for given query_string and params object + """ + if params is None: + return hash(query_string) + if isinstance(params, dict): + return hash((query_string, str(params))) + if isinstance(params, list): + return hash((query_string, *params)) + return hash((query_string, params)) + + def clear_id_cache(self): + """Clear the cache of id's returned by _get_id.""" + self._QUERY_CACHE.clear() From 288c05e579d44436ac4d5b3ea4ccebfea767448d Mon Sep 17 00:00:00 2001 From: zack jensen Date: Mon, 18 Nov 2024 17:18:00 -0700 Subject: [PATCH 04/17] feat: cache flag added - Added flag to turn cache on/off - Added clear_id_cache method - Moved all cache methods to bottom of class --- src/plexosdb/sqlite.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/plexosdb/sqlite.py b/src/plexosdb/sqlite.py index 77011e0..868921c 100644 --- a/src/plexosdb/sqlite.py +++ b/src/plexosdb/sqlite.py @@ -38,6 +38,7 @@ def __init__( self._conn = sqlite3.connect(":memory:") self._sqlite_config() self._QUERY_CACHE: dict[int, int] = {} + if create_collations: self._create_collations() self._create_table_schema() @@ -757,7 +758,6 @@ def _get_id( parent_class_name: ClassEnum | None = None, child_class_name: ClassEnum | None = None, category_name: str | None = None, - use_cache: bool = True, ) -> int: """Return the ID for a given table and object name combination. @@ -840,7 +840,7 @@ def _get_id( else f" WHERE {table_name}.name = :object_name" ) query_key = self._query_hash(query, params) - if query_key in self._QUERY_CACHE and use_cache: + if query_key in self._QUERY_CACHE: return self._QUERY_CACHE[query_key] result = self.query(query, params) @@ -1272,5 +1272,5 @@ def _query_hash(query_string: str, params: tuple | dict | None = None) -> int: return hash((query_string, params)) def clear_id_cache(self): - """Clear the cache of id's returned by _get_id.""" + """Clear the cache for the _get_id method.""" self._QUERY_CACHE.clear() From c805239aaf881f8b2428a946ef218319b44ba6ab Mon Sep 17 00:00:00 2001 From: zack jensen Date: Mon, 18 Nov 2024 17:18:58 -0700 Subject: [PATCH 05/17] fix: clear cache between unit tests --- tests/test_plexosdb_sqlite.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_plexosdb_sqlite.py b/tests/test_plexosdb_sqlite.py index 25c5291..f2ed3be 100644 --- a/tests/test_plexosdb_sqlite.py +++ b/tests/test_plexosdb_sqlite.py @@ -156,6 +156,7 @@ def test_get_collection_id(db): @pytest.mark.get_functions def test_get_object_id(db): + db.clear_id_cache() gen_01_name = "gen1" gen_id = db.add_object( gen_01_name, ClassEnum.Generator, CollectionEnum.Generators, description="Test Gen" @@ -189,6 +190,7 @@ def test_get_object_id(db): @pytest.mark.get_functions def test_get_memberships(db): + db.clear_id_cache() # Test Node node_name = "Node 1" node_id = db.add_object(node_name, ClassEnum.Node, CollectionEnum.Nodes, description="Test Node") From deaab287d75b8bb67be2837bfcc99d0d09941633 Mon Sep 17 00:00:00 2001 From: zack jensen Date: Wed, 20 Nov 2024 20:34:21 -0700 Subject: [PATCH 06/17] feat: enforce uniqueness on objects - Enforce unique name/class for all objects - Enforce unqiue object ids for all objects - Raise ValueError if an object is added without a unique name/class --- src/plexosdb/schema.sql | 3 ++- src/plexosdb/sqlite.py | 22 ++++++++++++++-------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/plexosdb/schema.sql b/src/plexosdb/schema.sql index b2192a4..ebc76b2 100644 --- a/src/plexosdb/schema.sql +++ b/src/plexosdb/schema.sql @@ -231,7 +231,7 @@ CREATE TABLE `t_category` CREATE TABLE `t_object` ( - `object_id` INTEGER, + `object_id` INTEGER UNIQUE, `class_id` INT NULL, `name` VARCHAR(512) NULL COLLATE NOCASE, `category_id` INT NULL, @@ -241,6 +241,7 @@ CREATE TABLE `t_object` `X` INT NULL, `Y` INT NULL, `Z` INT NULL, + UNIQUE (`class_id`, `name`) CONSTRAINT PK_t_object PRIMARY KEY (`object_id`) ); diff --git a/src/plexosdb/sqlite.py b/src/plexosdb/sqlite.py index 868921c..9dd7cba 100644 --- a/src/plexosdb/sqlite.py +++ b/src/plexosdb/sqlite.py @@ -461,14 +461,17 @@ def add_object( params = (object_name, class_id, category_id, str(uuid.uuid4()), description) placeholders = ", ".join("?" * len(params)) - with self._conn as conn: - cursor = conn.cursor() - cursor.execute( - f"INSERT INTO {Schema.Objects.name}(name, class_id, category_id, GUID, description) " - f"VALUES({placeholders})", - params, - ) - object_id = cursor.lastrowid + try: + with self._conn as conn: + cursor = conn.cursor() + cursor.execute( + f"INSERT INTO {Schema.Objects.name}(name, class_id, category_id, GUID, description) " + f"VALUES({placeholders})", + params, + ) + object_id = cursor.lastrowid + except sqlite3.IntegrityError as err: + raise ValueError(err) if object_id is None: raise TypeError("Could not fetch the last row of the insert. Check query format.") @@ -1096,8 +1099,11 @@ def ingest_from_records(self, tag: str, record_data: Sequence): try: with self._conn as conn: conn.execute(ingestion_sql, record) + except sqlite3.IntegrityError as err: + raise ValueError(err) except sqlite3.Error as err: raise err + logger.trace("Finished ingesting {}", tag) return From f9a4ce163b07c0dc96d9a8a11834a9cbc1b4c582 Mon Sep 17 00:00:00 2001 From: zack jensen Date: Wed, 20 Nov 2024 20:47:19 -0700 Subject: [PATCH 07/17] fix: unqiue names in test xml file --- tests/data/plexosdb.xml | 2 +- tests/test_plexosdb_sqlite.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/data/plexosdb.xml b/tests/data/plexosdb.xml index 6870ce6..358b8d3 100644 --- a/tests/data/plexosdb.xml +++ b/tests/data/plexosdb.xml @@ -82,7 +82,7 @@ 3 2 - SolarPV01 + SolarPV02 97 40d15a07-8ccc-460e-919a-ec8e211899a8 diff --git a/tests/test_plexosdb_sqlite.py b/tests/test_plexosdb_sqlite.py index f2ed3be..c6d7b2e 100644 --- a/tests/test_plexosdb_sqlite.py +++ b/tests/test_plexosdb_sqlite.py @@ -80,10 +80,6 @@ def test_check_id_exists(db): assert isinstance(system_check, bool) assert not system_check - # Check that returns ValueError if multiple object founds - with pytest.raises(ValueError): - _ = db.check_id_exists(Schema.Objects, "SolarPV01", class_name=ClassEnum.Generator) - @pytest.mark.get_functions def test_get_id(db): From 4f86ae330ac2564fb23cc64dce48d571eaaf6260 Mon Sep 17 00:00:00 2001 From: zack jensen Date: Wed, 20 Nov 2024 20:49:03 -0700 Subject: [PATCH 08/17] fix: create temporary database xml --- tests/test_plexosdb_sqlite.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/test_plexosdb_sqlite.py b/tests/test_plexosdb_sqlite.py index c6d7b2e..f2b74c1 100644 --- a/tests/test_plexosdb_sqlite.py +++ b/tests/test_plexosdb_sqlite.py @@ -1,7 +1,9 @@ import pytest +import shutil import xml.etree.ElementTree as ET # noqa: N817 from plexosdb.enums import ClassEnum, CollectionEnum, Schema from plexosdb.sqlite import PlexosSQLite +from collections.abc import Generator DB_FILENAME = "plexosdb.xml" @@ -12,8 +14,13 @@ def db_empty() -> "PlexosSQLite": @pytest.fixture -def db(data_folder) -> PlexosSQLite: - return PlexosSQLite(xml_fname=data_folder.joinpath(DB_FILENAME)) +def db(data_folder) -> Generator[PlexosSQLite, None, None]: + xml_fname = data_folder.joinpath(DB_FILENAME) + xml_copy = data_folder.joinpath(f"copy_{DB_FILENAME}") + shutil.copy(xml_fname, xml_copy) + db = PlexosSQLite(xml_fname=xml_copy) + yield db + xml_copy.unlink() def test_database_initialization(db): @@ -186,7 +193,6 @@ def test_get_object_id(db): @pytest.mark.get_functions def test_get_memberships(db): - db.clear_id_cache() # Test Node node_name = "Node 1" node_id = db.add_object(node_name, ClassEnum.Node, CollectionEnum.Nodes, description="Test Node") From a64377bb4910c91e052293aced8018ce05016e8b Mon Sep 17 00:00:00 2001 From: zack jensen Date: Wed, 20 Nov 2024 20:49:25 -0700 Subject: [PATCH 09/17] fix: update object id test for uniqueness --- tests/test_plexosdb_sqlite.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/tests/test_plexosdb_sqlite.py b/tests/test_plexosdb_sqlite.py index f2b74c1..373f1df 100644 --- a/tests/test_plexosdb_sqlite.py +++ b/tests/test_plexosdb_sqlite.py @@ -159,19 +159,9 @@ def test_get_collection_id(db): @pytest.mark.get_functions def test_get_object_id(db): - db.clear_id_cache() - gen_01_name = "gen1" - gen_id = db.add_object( - gen_01_name, ClassEnum.Generator, CollectionEnum.Generators, description="Test Gen" - ) - assert gen_id - - gen_id_get = db.get_object_id(gen_01_name, class_name=ClassEnum.Generator) - assert gen_id == gen_id_get - - # Add generator with same name different category gen_01_name = "gen1" category_name = "PV Gens" + gen_id = db.add_object( gen_01_name, ClassEnum.Generator, @@ -179,8 +169,22 @@ def test_get_object_id(db): description="Test Gen", category_name=category_name, ) + assert gen_id + + gen_id_get = db.get_object_id(gen_01_name, class_name=ClassEnum.Generator) + assert gen_id == gen_id_get + + # Add generator with same name different category with pytest.raises(ValueError): - _ = db.get_object_id(gen_01_name, class_name=ClassEnum.Generator) + gen_01_name = "gen1" + gen_id = db.add_object( + gen_01_name, + ClassEnum.Generator, + CollectionEnum.Generators, + description="Test Gen", + ) + # for a given class, all names should be unique + _ = db.get_object_id(gen_01_name, class_name=ClassEnum.Generator) max_rank = db.get_category_max_id(ClassEnum.Generator) assert max_rank == 2 # Data has ranks 0, 1. 2 is with the new category From 886a4996a9ec8fe76382cf9e7cc40981f6fd9ca9 Mon Sep 17 00:00:00 2001 From: zack jensen Date: Thu, 21 Nov 2024 11:24:24 -0700 Subject: [PATCH 10/17] test: remove unused code in test_object_id --- tests/test_plexosdb_sqlite.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_plexosdb_sqlite.py b/tests/test_plexosdb_sqlite.py index 373f1df..81a8698 100644 --- a/tests/test_plexosdb_sqlite.py +++ b/tests/test_plexosdb_sqlite.py @@ -174,17 +174,14 @@ def test_get_object_id(db): gen_id_get = db.get_object_id(gen_01_name, class_name=ClassEnum.Generator) assert gen_id == gen_id_get - # Add generator with same name different category + # Add generator with same name and no category with pytest.raises(ValueError): - gen_01_name = "gen1" gen_id = db.add_object( gen_01_name, ClassEnum.Generator, CollectionEnum.Generators, description="Test Gen", ) - # for a given class, all names should be unique - _ = db.get_object_id(gen_01_name, class_name=ClassEnum.Generator) max_rank = db.get_category_max_id(ClassEnum.Generator) assert max_rank == 2 # Data has ranks 0, 1. 2 is with the new category From 595823da6b599782a7322443c199750a0978f434 Mon Sep 17 00:00:00 2001 From: zack jensen Date: Thu, 21 Nov 2024 11:40:09 -0700 Subject: [PATCH 11/17] test: create temporary directory --- tests/test_plexosdb_sqlite.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_plexosdb_sqlite.py b/tests/test_plexosdb_sqlite.py index 81a8698..8e2bffe 100644 --- a/tests/test_plexosdb_sqlite.py +++ b/tests/test_plexosdb_sqlite.py @@ -4,6 +4,8 @@ from plexosdb.enums import ClassEnum, CollectionEnum, Schema from plexosdb.sqlite import PlexosSQLite from collections.abc import Generator +from pathlib import Path +from tempfile import TemporaryDirectory DB_FILENAME = "plexosdb.xml" @@ -14,11 +16,12 @@ def db_empty() -> "PlexosSQLite": @pytest.fixture -def db(data_folder) -> Generator[PlexosSQLite, None, None]: - xml_fname = data_folder.joinpath(DB_FILENAME) - xml_copy = data_folder.joinpath(f"copy_{DB_FILENAME}") +def db(data_folder: Path) -> Generator[PlexosSQLite, None, None]: + tmp_dir = TemporaryDirectory() # dir=data_folder.parent) + xml_fname = data_folder / DB_FILENAME + xml_copy = Path(tmp_dir.name) / f"copy_{DB_FILENAME}" shutil.copy(xml_fname, xml_copy) - db = PlexosSQLite(xml_fname=xml_copy) + db = PlexosSQLite(xml_fname=str(xml_copy)) yield db xml_copy.unlink() From ae6efdf0525fc4276e94a16bf21f1649b1fe1602 Mon Sep 17 00:00:00 2001 From: zack jensen Date: Thu, 21 Nov 2024 11:48:39 -0700 Subject: [PATCH 12/17] fix: remove ValueError --- src/plexosdb/sqlite.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/plexosdb/sqlite.py b/src/plexosdb/sqlite.py index 9dd7cba..bd320cd 100644 --- a/src/plexosdb/sqlite.py +++ b/src/plexosdb/sqlite.py @@ -446,6 +446,11 @@ def add_object( ----- By default, we add all objects to the system membership. + Raises + ------ + sqlite.IntegrityError + if an object is inserted without a unique name/class pair + Returns ------- int @@ -461,17 +466,14 @@ def add_object( params = (object_name, class_id, category_id, str(uuid.uuid4()), description) placeholders = ", ".join("?" * len(params)) - try: - with self._conn as conn: - cursor = conn.cursor() - cursor.execute( - f"INSERT INTO {Schema.Objects.name}(name, class_id, category_id, GUID, description) " - f"VALUES({placeholders})", - params, - ) - object_id = cursor.lastrowid - except sqlite3.IntegrityError as err: - raise ValueError(err) + with self._conn as conn: + cursor = conn.cursor() + cursor.execute( + f"INSERT INTO {Schema.Objects.name}(name, class_id, category_id, GUID, description) " + f"VALUES({placeholders})", + params, + ) + object_id = cursor.lastrowid if object_id is None: raise TypeError("Could not fetch the last row of the insert. Check query format.") From a94d8e36547607f429690896b637b2b120a4e592 Mon Sep 17 00:00:00 2001 From: zack jensen Date: Thu, 21 Nov 2024 11:52:54 -0700 Subject: [PATCH 13/17] fix: IntegrityError in test_sqlite --- tests/test_plexosdb_sqlite.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_plexosdb_sqlite.py b/tests/test_plexosdb_sqlite.py index 8e2bffe..286b037 100644 --- a/tests/test_plexosdb_sqlite.py +++ b/tests/test_plexosdb_sqlite.py @@ -3,6 +3,7 @@ import xml.etree.ElementTree as ET # noqa: N817 from plexosdb.enums import ClassEnum, CollectionEnum, Schema from plexosdb.sqlite import PlexosSQLite +from sqlite3 import IntegrityError from collections.abc import Generator from pathlib import Path from tempfile import TemporaryDirectory @@ -178,7 +179,7 @@ def test_get_object_id(db): assert gen_id == gen_id_get # Add generator with same name and no category - with pytest.raises(ValueError): + with pytest.raises(IntegrityError): gen_id = db.add_object( gen_01_name, ClassEnum.Generator, From 96f73b822e6569bf3d86faa266ae31b59f97f86b Mon Sep 17 00:00:00 2001 From: zack jensen Date: Thu, 21 Nov 2024 15:36:27 -0700 Subject: [PATCH 14/17] refactor: simplify id cache --- src/plexosdb/sqlite.py | 40 ++++++---------------------------------- 1 file changed, 6 insertions(+), 34 deletions(-) diff --git a/src/plexosdb/sqlite.py b/src/plexosdb/sqlite.py index bd320cd..84a80e6 100644 --- a/src/plexosdb/sqlite.py +++ b/src/plexosdb/sqlite.py @@ -37,7 +37,7 @@ def __init__( super().__init__() self._conn = sqlite3.connect(":memory:") self._sqlite_config() - self._QUERY_CACHE: dict[int, int] = {} + self._QUERY_CACHE: dict[tuple, int] = {} if create_collations: self._create_collations() @@ -802,6 +802,11 @@ def _get_id( "object_name": object_name, } + # tuple that should be unique for any id returned + query_key = (table.name, object_name, class_name, parent_class_name, child_class_name) + if query_key in self._QUERY_CACHE: + return self._QUERY_CACHE[query_key] + query = f"SELECT {column_name} FROM `{table_name}`" conditions = [] join_clauses = [] @@ -844,10 +849,6 @@ def _get_id( if conditions else f" WHERE {table_name}.name = :object_name" ) - query_key = self._query_hash(query, params) - if query_key in self._QUERY_CACHE: - return self._QUERY_CACHE[query_key] - result = self.query(query, params) if not result: @@ -1253,32 +1254,3 @@ def _create_collations(self) -> None: """Add collate function for helping search enums.""" self._conn.create_collation("NOSPACE", no_space) return - - @staticmethod - def _query_hash(query_string: str, params: tuple | dict | None = None) -> int: - """ - Create a hash int for a query string and params dictionary. - - Parameters - ---------- - query_str - String to get passed to the database connector. - params - Tuple or dict for passing - - Returns - ------- - Int - likely unique integer for given query_string and params object - """ - if params is None: - return hash(query_string) - if isinstance(params, dict): - return hash((query_string, str(params))) - if isinstance(params, list): - return hash((query_string, *params)) - return hash((query_string, params)) - - def clear_id_cache(self): - """Clear the cache for the _get_id method.""" - self._QUERY_CACHE.clear() From ccdfe929bb8d7906ab59388d8ee5f7dee12eeca0 Mon Sep 17 00:00:00 2001 From: zack jensen Date: Thu, 21 Nov 2024 15:36:53 -0700 Subject: [PATCH 15/17] refactor: remove unnecessary exception --- src/plexosdb/sqlite.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/plexosdb/sqlite.py b/src/plexosdb/sqlite.py index 84a80e6..5be818e 100644 --- a/src/plexosdb/sqlite.py +++ b/src/plexosdb/sqlite.py @@ -1102,8 +1102,6 @@ def ingest_from_records(self, tag: str, record_data: Sequence): try: with self._conn as conn: conn.execute(ingestion_sql, record) - except sqlite3.IntegrityError as err: - raise ValueError(err) except sqlite3.Error as err: raise err From 02e7f0a4f8176388cf6c85c4390550bf89cfcdb2 Mon Sep 17 00:00:00 2001 From: zack jensen Date: Thu, 21 Nov 2024 15:55:46 -0700 Subject: [PATCH 16/17] refactor: use pytest builtin tmp path --- tests/test_plexosdb_sqlite.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_plexosdb_sqlite.py b/tests/test_plexosdb_sqlite.py index 286b037..2708058 100644 --- a/tests/test_plexosdb_sqlite.py +++ b/tests/test_plexosdb_sqlite.py @@ -6,7 +6,6 @@ from sqlite3 import IntegrityError from collections.abc import Generator from pathlib import Path -from tempfile import TemporaryDirectory DB_FILENAME = "plexosdb.xml" @@ -17,10 +16,9 @@ def db_empty() -> "PlexosSQLite": @pytest.fixture -def db(data_folder: Path) -> Generator[PlexosSQLite, None, None]: - tmp_dir = TemporaryDirectory() # dir=data_folder.parent) +def db(data_folder: Path, tmp_path: Path) -> Generator[PlexosSQLite, None, None]: xml_fname = data_folder / DB_FILENAME - xml_copy = Path(tmp_dir.name) / f"copy_{DB_FILENAME}" + xml_copy = tmp_path / f"copy_{DB_FILENAME}" shutil.copy(xml_fname, xml_copy) db = PlexosSQLite(xml_fname=str(xml_copy)) yield db From a3200a514a10dad15b221ae49d0191f710eff17a Mon Sep 17 00:00:00 2001 From: zack jensen Date: Fri, 22 Nov 2024 12:14:25 -0700 Subject: [PATCH 17/17] bump version: 0.0.7 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 65780b7..b9c800a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "plexosdb" -version = "0.0.6" +version = "0.0.7" readme = "README.md" license = {file = "LICENSE.txt"} keywords = []