From cbbde23f12769c8038effd8af504c4354e1ccb8f Mon Sep 17 00:00:00 2001 From: pesap Date: Thu, 10 Oct 2024 16:28:19 -0600 Subject: [PATCH 1/5] feat: Adding simple query for objects --- src/plexosdb/queries/simple_object_query.sql | 75 ++++++++++++++++++++ src/plexosdb/sqlite.py | 2 +- 2 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 src/plexosdb/queries/simple_object_query.sql diff --git a/src/plexosdb/queries/simple_object_query.sql b/src/plexosdb/queries/simple_object_query.sql new file mode 100644 index 0000000..d37f84f --- /dev/null +++ b/src/plexosdb/queries/simple_object_query.sql @@ -0,0 +1,75 @@ +-- Query to get all the objects and nested objects +-- We do not include scenario tags on nested objects. +WITH scenario_cte as ( +SELECT + obj.name, + obj.object_id, + tag.data_id, + mem.collection_id, + cat.name AS category_name +FROM + t_membership AS mem +LEFT JOIN t_tag AS tag ON + tag.object_id = mem.child_object_id +LEFT JOIN t_object AS obj ON + mem.child_object_id = obj.object_id +LEFT JOIN t_category AS cat ON + obj.category_id = cat.category_id +WHERE + mem.collection_id in + ( + SELECT + collection_id + FROM + t_collection + LEFT JOIN t_class ON + t_class.class_id = t_collection.parent_class_id + where + t_class.name <> 'System' + ) + ) +SELECT + obj.name, + obj.object_id, + m.collection_id, + m.membership_id, + prop.name, + data.value AS value, + date_from.date as date_from, + date_to.date as date_to, + text.value as text_name, + text.class_id as text_class_id, + tag.data_id as tag_data_id, + tag_object.name as tag_object, + tag.object_id as tag_object_id, + tag_class.name as tag_class_name, + scenario.name AS scenario, + scenario.object_id AS scenario_obj_id, + scenario.data_id AS scenario_data_id, + scenario.collection_id AS scenario_collection_id +FROM + t_membership m +LEFT JOIN t_object AS obj ON + m.child_object_id = obj.object_id +LEFT JOIN t_data data ON + m.membership_id = data.membership_id +LEFT JOIN t_property as prop ON + data.property_id = prop.property_id +LEFT JOIN t_date_from AS date_from ON + data.data_id = date_from.data_id +LEFT JOIN t_date_to AS date_to ON + data.data_id = date_to.data_id +LEFT JOIN t_tag as tag ON + data.data_id = tag.data_id +LEFT JOIN scenario_cte as scenario ON + scenario.data_id = data.data_id + and tag.data_id = scenario.data_id +LEFT JOIN t_text as text on + text.data_id = data.data_id +LEFT JOIN t_object as tag_object ON + tag_object.object_id = tag.object_id +LEFT JOIN t_class as tag_class ON + tag_class.class_id = tag_object.class_id +WHERE + (tag_class.name ISNULL + or tag_class.name <> 'Scenario') diff --git a/src/plexosdb/sqlite.py b/src/plexosdb/sqlite.py index f198264..5efef32 100644 --- a/src/plexosdb/sqlite.py +++ b/src/plexosdb/sqlite.py @@ -1178,7 +1178,7 @@ def _sqlite_config(self): """Call all sqlite configuration prior schema creation.""" with self._conn as conn: conn.execute("PRAGMA synchronous = OFF") # Make it asynchronous - # conn.execute("PRAGMA journal_mode = OFF") # Make it asynchronous + conn.execute("PRAGMA journal_mode = OFF") # Make it asynchronous def _properties_to_sql_ingest( self, component_properties: list[dict], memberships: dict, property_ids: dict From 4ff1e2ab71032f67e288d5482dc0ecd2c8193905 Mon Sep 17 00:00:00 2001 From: pesap Date: Fri, 11 Oct 2024 02:04:12 -0600 Subject: [PATCH 2/5] fix: Cleaning the query --- src/plexosdb/queries/simple_object_query.sql | 93 ++++++++++++-------- 1 file changed, 57 insertions(+), 36 deletions(-) diff --git a/src/plexosdb/queries/simple_object_query.sql b/src/plexosdb/queries/simple_object_query.sql index d37f84f..923cafc 100644 --- a/src/plexosdb/queries/simple_object_query.sql +++ b/src/plexosdb/queries/simple_object_query.sql @@ -1,9 +1,9 @@ -- Query to get all the objects and nested objects -- We do not include scenario tags on nested objects. -WITH scenario_cte as ( +WITH scenario_cte AS ( SELECT - obj.name, - obj.object_id, + object.name, + object.object_id, tag.data_id, mem.collection_id, cat.name AS category_name @@ -11,10 +11,10 @@ FROM t_membership AS mem LEFT JOIN t_tag AS tag ON tag.object_id = mem.child_object_id -LEFT JOIN t_object AS obj ON - mem.child_object_id = obj.object_id +LEFT JOIN t_object AS object ON + mem.child_object_id = object.object_id LEFT JOIN t_category AS cat ON - obj.category_id = cat.category_id + object.category_id = cat.category_id WHERE mem.collection_id in ( @@ -29,46 +29,67 @@ WHERE ) ) SELECT - obj.name, - obj.object_id, - m.collection_id, - m.membership_id, - prop.name, - data.value AS value, - date_from.date as date_from, - date_to.date as date_to, - text.value as text_name, - text.class_id as text_class_id, - tag.data_id as tag_data_id, - tag_object.name as tag_object, - tag.object_id as tag_object_id, - tag_class.name as tag_class_name, - scenario.name AS scenario, - scenario.object_id AS scenario_obj_id, - scenario.data_id AS scenario_data_id, - scenario.collection_id AS scenario_collection_id + parent_class.name AS parent_class_name, + child_class.name AS child_class_name, + parent_object.object_id AS parent_obj_id, + object.object_id AS object_id, + object.name AS name, + category.name AS category, + property.name AS property_name, + unit.value AS property_unit, + data.value AS property_value, + IFNULL(band.band_id, 1) AS band, + date_from.date AS date_from, + date_to.date AS date_to, + text.value AS text, + text_class.name AS text_class_name, + tag.data_id AS tag_data_id, + tag_object.name AS tag_object, + tag.object_id AS tag_object_id, + tag_class.name AS tag_class_name, + action.action_symbol as action, + scenario.name AS scenario + -- scenario.object_id AS scenario_obj_id, + -- scenario.data_id AS scenario_data_id, + -- scenario.collection_id AS scenario_collection_id FROM - t_membership m -LEFT JOIN t_object AS obj ON - m.child_object_id = obj.object_id -LEFT JOIN t_data data ON - m.membership_id = data.membership_id -LEFT JOIN t_property as prop ON - data.property_id = prop.property_id + t_membership membership +LEFT JOIN t_object AS object ON + membership.child_object_id = object.object_id +LEFT JOIN t_category AS category ON + object.category_id = category.category_id +LEFT JOIN t_object AS parent_object ON + membership.parent_object_id = parent_object.object_id +LEFT JOIN t_class AS child_class ON + membership.child_class_id = child_class.class_id +LEFT JOIN t_class AS parent_class ON + membership.parent_class_id = parent_class.class_id +LEFT JOIN t_data AS data ON + membership.membership_id = data.membership_id +LEFT JOIN t_band AS band on + data.data_id = band.band_id +LEFT JOIN t_property AS property ON + data.property_id = property.property_id +LEFT JOIN t_unit AS unit ON + property.unit_id = unit.unit_id LEFT JOIN t_date_from AS date_from ON data.data_id = date_from.data_id LEFT JOIN t_date_to AS date_to ON data.data_id = date_to.data_id -LEFT JOIN t_tag as tag ON +LEFT JOIN t_tag AS tag ON data.data_id = tag.data_id -LEFT JOIN scenario_cte as scenario ON +LEFT JOIN t_action AS action ON + tag.action_id = action.action_id +LEFT JOIN scenario_cte AS scenario ON scenario.data_id = data.data_id and tag.data_id = scenario.data_id -LEFT JOIN t_text as text on +LEFT JOIN t_text AS text on text.data_id = data.data_id -LEFT JOIN t_object as tag_object ON +LEFT JOIN t_class AS text_class on + text.class_id = text_class.class_id +LEFT JOIN t_object AS tag_object ON tag_object.object_id = tag.object_id -LEFT JOIN t_class as tag_class ON +LEFT JOIN t_class AS tag_class ON tag_class.class_id = tag_object.class_id WHERE (tag_class.name ISNULL From 7bd829cd7f87d8b3ec1192704487344b96906d35 Mon Sep 17 00:00:00 2001 From: pesap Date: Fri, 18 Oct 2024 01:26:10 -0600 Subject: [PATCH 3/5] feat: Added new format for simple_object_query and added collate for noncase search --- src/plexosdb/queries/simple_object_query.sql | 21 ++++++++++---------- src/plexosdb/schema.sql | 4 +++- src/plexosdb/sqlite.py | 10 ++++++++-- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/plexosdb/queries/simple_object_query.sql b/src/plexosdb/queries/simple_object_query.sql index 923cafc..5d60576 100644 --- a/src/plexosdb/queries/simple_object_query.sql +++ b/src/plexosdb/queries/simple_object_query.sql @@ -43,15 +43,14 @@ SELECT date_to.date AS date_to, text.value AS text, text_class.name AS text_class_name, - tag.data_id AS tag_data_id, - tag_object.name AS tag_object, - tag.object_id AS tag_object_id, - tag_class.name AS tag_class_name, + MAX(CASE WHEN tag_class.name == 'Timeslice' THEN tag_object.name END) AS tag_timeslice, + MAX(CASE WHEN tag_class.name == 'Timeslice' THEN tag_object.object_id END) AS tag_timeslice_objet_id, + MAX(CASE WHEN tag_class.name == 'Data File' THEN tag_object.name END) AS tag_datafile, + MAX(CASE WHEN tag_class.name == 'Data File' THEN tag_object.object_id END) AS tag_datafile_objet_id, + MAX(CASE WHEN tag_class.name == 'Variable' THEN tag_object.name END) AS tag_variable, + MAX(CASE WHEN tag_class.name == 'Variable' THEN tag_object.object_id END) AS tag_variable_objet_id, action.action_symbol as action, scenario.name AS scenario - -- scenario.object_id AS scenario_obj_id, - -- scenario.data_id AS scenario_data_id, - -- scenario.collection_id AS scenario_collection_id FROM t_membership membership LEFT JOIN t_object AS object ON @@ -67,7 +66,7 @@ LEFT JOIN t_class AS parent_class ON LEFT JOIN t_data AS data ON membership.membership_id = data.membership_id LEFT JOIN t_band AS band on - data.data_id = band.band_id + data.data_id = band.data_id LEFT JOIN t_property AS property ON data.property_id = property.property_id LEFT JOIN t_unit AS unit ON @@ -91,6 +90,6 @@ LEFT JOIN t_object AS tag_object ON tag_object.object_id = tag.object_id LEFT JOIN t_class AS tag_class ON tag_class.class_id = tag_object.class_id -WHERE - (tag_class.name ISNULL - or tag_class.name <> 'Scenario') +GROUP BY + membership.membership_id, + data.data_id diff --git a/src/plexosdb/schema.sql b/src/plexosdb/schema.sql index 11ad044..b2192a4 100644 --- a/src/plexosdb/schema.sql +++ b/src/plexosdb/schema.sql @@ -233,7 +233,7 @@ CREATE TABLE `t_object` ( `object_id` INTEGER, `class_id` INT NULL, - `name` VARCHAR(512) NULL, + `name` VARCHAR(512) NULL COLLATE NOCASE, `category_id` INT NULL, `description` VARCHAR(255) NULL, `GUID` CHAR(36) NOT NULL, @@ -245,6 +245,8 @@ CREATE TABLE `t_object` PRIMARY KEY (`object_id`) ); + + CREATE TABLE `t_memo_object` ( `object_id` INT NOT NULL, diff --git a/src/plexosdb/sqlite.py b/src/plexosdb/sqlite.py index 5efef32..b3d2839 100644 --- a/src/plexosdb/sqlite.py +++ b/src/plexosdb/sqlite.py @@ -28,11 +28,17 @@ class PlexosSQLite: DB_FILENAME = "plexos.db" _conn: sqlite3.Connection - def __init__(self, xml_fname: str | None = None, xml_handler: XMLHandler | None = None) -> None: + def __init__( + self, + xml_fname: str | None = None, + xml_handler: XMLHandler | None = None, + create_collations: bool = True, + ) -> None: super().__init__() self._conn = sqlite3.connect(":memory:") self._sqlite_config() - self._create_collations() + if create_collations: + self._create_collations() self._create_table_schema() self._populate_database(xml_fname=xml_fname, xml_handler=xml_handler) From f3a852a53a65e3edde7f7573bc5bb4e8f93814ac Mon Sep 17 00:00:00 2001 From: pesap Date: Mon, 21 Oct 2024 11:15:26 -0600 Subject: [PATCH 4/5] fix: Adding proper indentation --- src/plexosdb/queries/simple_object_query.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plexosdb/queries/simple_object_query.sql b/src/plexosdb/queries/simple_object_query.sql index 5d60576..37d7e10 100644 --- a/src/plexosdb/queries/simple_object_query.sql +++ b/src/plexosdb/queries/simple_object_query.sql @@ -6,7 +6,7 @@ SELECT object.object_id, tag.data_id, mem.collection_id, - cat.name AS category_name + cat.name AS category_name FROM t_membership AS mem LEFT JOIN t_tag AS tag ON From 2c7530180f81279663ff59ace618ae5a1bcaf8a1 Mon Sep 17 00:00:00 2001 From: pesap Date: Mon, 21 Oct 2024 11:20:45 -0600 Subject: [PATCH 5/5] bumpversion --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index dc8755f..c3e4ceb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "plexosdb" -version = "0.0.4" +version = "0.0.5" readme = "README.md" license = {file = "LICENSE.txt"} keywords = []