diff --git a/polars_db_process/README.rst b/polars_db_process/README.rst new file mode 100644 index 000000000..8c4cabd6f --- /dev/null +++ b/polars_db_process/README.rst @@ -0,0 +1,100 @@ +======================= +Polars Database Process +======================= + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:3fb8a401fe8c3d73e23b477915bbd9ff0b2bcb331711726a4fd5be280ad53d5d + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Alpha-red.png + :target: https://odoo-community.org/page/development-status + :alt: Alpha +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fserver--backend-lightgray.png?logo=github + :target: https://github.com/OCA/server-backend/tree/18.0/polars_db_process + :alt: OCA/server-backend +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/server-backend-18-0/server-backend-18-0-polars_db_process + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/server-backend&target_branch=18.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +From a db query, this module allows to transform data in Polars +dataframe and process them according to rules in order to: + +- filter data and display +- obtain another dataframe with only the expected data to use in Odoo + +A such dataframe can help to prepare data in order to be used to +create/update or import + +For that you need to transform/arrange data to the same way + +.. IMPORTANT:: + This is an alpha version, the data model and design can change at any time without warning. + Only for development or testing purpose, do not use in production. + `More details on development status `_ + +**Table of contents** + +.. contents:: + :local: + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +------- + +* Akretion + +Contributors +------------ + +- Akretion + + - David BEAL david.beal@akretion.com + +Maintainers +----------- + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +.. |maintainer-bealdav| image:: https://github.com/bealdav.png?size=40px + :target: https://github.com/bealdav + :alt: bealdav + +Current `maintainer `__: + +|maintainer-bealdav| + +This module is part of the `OCA/server-backend `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/polars_db_process/__init__.py b/polars_db_process/__init__.py new file mode 100644 index 000000000..aee8895e7 --- /dev/null +++ b/polars_db_process/__init__.py @@ -0,0 +1,2 @@ +from . import models +from . import wizards diff --git a/polars_db_process/__manifest__.py b/polars_db_process/__manifest__.py new file mode 100644 index 000000000..4c340892c --- /dev/null +++ b/polars_db_process/__manifest__.py @@ -0,0 +1,36 @@ +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +{ + "name": "Polars Database Process", + "version": "18.0.1.0.0", + "summary": "Allow to create a Polars dataframe from db.query and " + "check it and process it according to rules", + "category": "Reporting", + "license": "AGPL-3", + "author": "Akretion, Odoo Community Association (OCA)", + "development_status": "Alpha", + "website": "https://github.com/OCA/server-backend", + "maintainers": ["bealdav"], + "depends": [ + "polars_process", + ], + "external_dependencies": { + "python": [ + "connectorx", + ] + }, + "data": [ + "security/ir.model.access.xml", + "wizards/df_process_wiz.xml", + "views/model_map.xml", + "views/df_field.xml", + "views/df_source.xml", + "views/db_config.xml", + "data/action.xml", + "data/demo.xml", # TODO remove + ], + "demo": [ + "data/demo.xml", + ], + "installable": True, +} diff --git a/polars_db_process/data/action.xml b/polars_db_process/data/action.xml new file mode 100644 index 000000000..8858573ca --- /dev/null +++ b/polars_db_process/data/action.xml @@ -0,0 +1,10 @@ + + + + Remove ir_model_data created from there + + + code + env['model.map']._remove_uidstring_related_records() + + diff --git a/polars_db_process/data/chinook.sqlite b/polars_db_process/data/chinook.sqlite new file mode 100644 index 000000000..38a98b391 Binary files /dev/null and b/polars_db_process/data/chinook.sqlite differ diff --git a/polars_db_process/data/demo.xml b/polars_db_process/data/demo.xml new file mode 100644 index 000000000..8bfd46431 --- /dev/null +++ b/polars_db_process/data/demo.xml @@ -0,0 +1,13 @@ + + + Chinook + sqlite://.../polars_db_schema/tests/files/chinook.sqlite + + + + + Chinook Customers + + diff --git a/polars_db_process/data/df/customers.sql b/polars_db_process/data/df/customers.sql new file mode 100644 index 000000000..f61c1d67c --- /dev/null +++ b/polars_db_process/data/df/customers.sql @@ -0,0 +1,4 @@ +--{"model": "res.partner", "db_conf": "Chinook"} +SELECT LastName AS name, State AS state, PostalCode AS zip, Email AS mail +-- , Company, Address, City, Country, Phone, Fax, SupportRepId +FROM customers; diff --git a/polars_db_process/models/__init__.py b/polars_db_process/models/__init__.py new file mode 100644 index 000000000..fc31db880 --- /dev/null +++ b/polars_db_process/models/__init__.py @@ -0,0 +1,3 @@ +from . import df_source +from . import db_config +from . import model_map diff --git a/polars_db_process/models/db_config.py b/polars_db_process/models/db_config.py new file mode 100644 index 000000000..b07f330ec --- /dev/null +++ b/polars_db_process/models/db_config.py @@ -0,0 +1,47 @@ +import connectorx as cx + +from odoo import _, exceptions, fields, models + +HELP = """ +String connexion samples: + +postgres://user:PASSWORD@server:port/database +mssql://user:PASSWORD@server:port/db.encrypt=true&trusted_connection=false +sqlite:///home/user/path/test.db +mysql://user:PASSWORD@server:port/database +oracle://user:PASSWORD@server:port/database +""" + + +class DbConfig(models.Model): + _name = "db.config" + _description = "External db.configuration" + _rec_name = "name" + _order = "name" + _rec_names_search = ["name"] + + name = fields.Char(required=True) + string_connexion = fields.Char(required=True, help=HELP) + password = fields.Char(help="Not required for Sqlite") + + def _get_connexion(self): + return self.string_connexion.replace("PASSWORD", self.password or "") + + def test_connexion(self): + res = self._read_sql("SELECT 1") + if len(res): + # Not invalid in reality + raise exceptions.ValidationError(_("Connexion OK !")) + + def _read_sql(self, query): + try: + return cx.read_sql(self._get_connexion(), query, return_type="polars") + except RuntimeError as err: + raise exceptions.ValidationError(err) from err + except TimeoutError as err: + raise exceptions.ValidationError(err) from err + except Exception as err: + raise exceptions.ValidationError(err) from err + + def _set_uidstring_module_name(self): + return "polars" diff --git a/polars_db_process/models/df_source.py b/polars_db_process/models/df_source.py new file mode 100644 index 000000000..17dabf215 --- /dev/null +++ b/polars_db_process/models/df_source.py @@ -0,0 +1,82 @@ +from pathlib import Path + +from odoo import fields, models +from odoo.modules.module import get_module_path +from odoo.tools.safe_eval import safe_eval + +MODULE = __name__[12 : __name__.index(".", 13)] + +HELP = """Supported files: .xlsx and .sql +Sql files may contains a comment on first line captured by File Parameters field +to be mapped automatically with related objects, i.e:\n +-- {'model': 'product.product', 'db_conf': mydb, 'code': 'my_delivery_address'} +""" + +PARAMS = """{'model': False, 'code': False, 'db_conf': False} +# 'model/code' to guess model.map', db_conf' name to guess db.config +""" + + +class DfSource(models.Model): + _inherit = "df.source" + + name = fields.Char(help=HELP) + query = fields.Char() + params = fields.Char( + string="File Parameters", + default=PARAMS, + readonly=True, + help="Coming from sql files", + ) + db_conf_id = fields.Many2one( + comodel_name="db.config", help="Database Configuration" + ) + + def _reset_process(self): + res = super()._reset_process() + mapp = self.model_map_id + if mapp and mapp.action == "import": + mapp._remove_uidstring_related_records() + return res + + def _file_hook(self, file, db_confs, model_map): + "Map sql file with the right Odoo model via model_map and the right db.config" + vals = super()._file_hook(file, db_confs, model_map) + if ".sql" in file: + content = self._get_file(file).decode("utf-8") + contents = content.split("\n") + if contents: + vals["query"] = content + # TODO: improve + # we only detect first line + meta = safe_eval(contents[0].replace("--", "")) + if meta: + vals["params"] = meta + if model_map: + if meta.get("code"): + vals["model_map_id"] = model_map["code"].get(meta["code"]) + elif meta.get("model"): + vals["model_map_id"] = model_map["model"].get(meta["model"]) + if meta.get("db_conf"): + vals["db_conf_id"] = db_confs.get(meta["db_conf"]) or False + return vals + + def _populate(self): + chinook = self.env.ref(f"{MODULE}.sqlite_chinook") + if chinook: + # TODO fix + # Demo behavior only + path = Path(get_module_path(MODULE)) / "data/chinook.sqlite" + chinook.string_connexion = f"sqlite://{str(path)}" + return super()._populate() + + def _get_modules_w_df_files(self): + res = super()._get_modules_w_df_files() + res.update( + { + "polars_db_process": { + "xmlid": "polars_db_process.contact_chinook", + } + } + ) + return res diff --git a/polars_db_process/models/model_map.py b/polars_db_process/models/model_map.py new file mode 100644 index 000000000..d968d3f1b --- /dev/null +++ b/polars_db_process/models/model_map.py @@ -0,0 +1,34 @@ +from odoo import models + + +class ModelMap(models.Model): + _inherit = "model.map" + + def _remove_uidstring_related_records(self): + active_ids = self.env.context.get("active_ids") + if active_ids: + self = self.browse(self.env.context.get("active_ids")) + for rec in self: + im_data = self.env["ir.model.data"].search( + [ + ("model", "=", rec.model_id.model), + ("module", "=", self._get_uidstring_module_name()), + ] + ) + if im_data: + record_ids = [int(x.split(",")[1]) for x in im_data.mapped("reference")] + self.env[rec.model_id.model].browse(record_ids).unlink() + return True + + def _get_uidstring_module_name(self): + # TODO move to static method + return "polars" + + def _get_touchy_fields_to_import(self): + """inherit me + Some fields may break your process and could be benefit + of a specific process. We have to know them + i.e. {"res.partner": ["vat"]} + """ + # TODO move to static method + return {} diff --git a/polars_db_process/pyproject.toml b/polars_db_process/pyproject.toml new file mode 100644 index 000000000..4231d0ccc --- /dev/null +++ b/polars_db_process/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["whool"] +build-backend = "whool.buildapi" diff --git a/polars_db_process/readme/CONTRIBUTORS.md b/polars_db_process/readme/CONTRIBUTORS.md new file mode 100644 index 000000000..7997db4f3 --- /dev/null +++ b/polars_db_process/readme/CONTRIBUTORS.md @@ -0,0 +1,3 @@ +- Akretion + + - David BEAL diff --git a/polars_db_process/readme/DESCRIPTION.md b/polars_db_process/readme/DESCRIPTION.md new file mode 100644 index 000000000..b71632d8c --- /dev/null +++ b/polars_db_process/readme/DESCRIPTION.md @@ -0,0 +1,8 @@ +From a db query, this module allows to transform data in Polars dataframe and process them according to rules in order to: + +- filter data and display +- obtain another dataframe with only the expected data to use in Odoo + +A such dataframe can help to prepare data in order to be used to create/update or import + +For that you need to transform/arrange data to the same way diff --git a/polars_db_process/security/ir.model.access.xml b/polars_db_process/security/ir.model.access.xml new file mode 100644 index 000000000..9cc3d07de --- /dev/null +++ b/polars_db_process/security/ir.model.access.xml @@ -0,0 +1,11 @@ + + + Database Config + + + + + + + + diff --git a/polars_db_process/static/description/index.html b/polars_db_process/static/description/index.html new file mode 100644 index 000000000..469438868 --- /dev/null +++ b/polars_db_process/static/description/index.html @@ -0,0 +1,442 @@ + + + + + +Polars Database Process + + + +
+

Polars Database Process

+ + +

Alpha License: AGPL-3 OCA/server-backend Translate me on Weblate Try me on Runboat

+

From a db query, this module allows to transform data in Polars +dataframe and process them according to rules in order to:

+
    +
  • filter data and display
    • +
  • obtain another dataframe with only the expected data to use in Odoo
    • +
+

A such dataframe can help to prepare data in order to be used to +create/update or import

+

For that you need to transform/arrange data to the same way

+
+

Important

+

This is an alpha version, the data model and design can change at any time without warning. +Only for development or testing purpose, do not use in production. +More details on development status

+
+

Table of contents

+ +
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • Akretion
    • +
+
+
+

Contributors

+ +
+
+

Maintainers

+

This module is maintained by the OCA.

+ +Odoo Community Association + +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

Current maintainer:

+

bealdav

+

This module is part of the OCA/server-backend project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/polars_db_process/tests/__init__.py b/polars_db_process/tests/__init__.py new file mode 100644 index 000000000..d9b96c4fa --- /dev/null +++ b/polars_db_process/tests/__init__.py @@ -0,0 +1 @@ +from . import test_module diff --git a/polars_db_process/tests/test_module.py b/polars_db_process/tests/test_module.py new file mode 100644 index 000000000..ca14680e0 --- /dev/null +++ b/polars_db_process/tests/test_module.py @@ -0,0 +1,7 @@ +from odoo.tests.common import TransactionCase + + +class TestModule(TransactionCase): + @classmethod + def setUpClass(cls): + super().setUpClass() diff --git a/polars_db_process/views/db_config.xml b/polars_db_process/views/db_config.xml new file mode 100644 index 000000000..21443bb5e --- /dev/null +++ b/polars_db_process/views/db_config.xml @@ -0,0 +1,59 @@ + + + db.config + +
+
+
+ + + + + + + + + + + +
+ PASSWORD in string connexion'll be replaced by password field +
+ + +
+ + + + + db.config + + + + + + + + + Database Config + db.config + list,form + db-config + + + + diff --git a/polars_db_process/views/df_field.xml b/polars_db_process/views/df_field.xml new file mode 100644 index 000000000..238507479 --- /dev/null +++ b/polars_db_process/views/df_field.xml @@ -0,0 +1 @@ + diff --git a/polars_db_process/views/df_source.xml b/polars_db_process/views/df_source.xml new file mode 100644 index 000000000..6fae3bc79 --- /dev/null +++ b/polars_db_process/views/df_source.xml @@ -0,0 +1,62 @@ + + + df.source + + + + + + + + + + + + + + + df.source + + + + + + + Remove data for model map action on 'import' + + + {'accepted_file_extensions': '.xlsx,.sql'} + + + query and not db_conf_id + + + + + + df.source + + + + + + + + + + + diff --git a/polars_db_process/views/model_map.xml b/polars_db_process/views/model_map.xml new file mode 100644 index 000000000..238507479 --- /dev/null +++ b/polars_db_process/views/model_map.xml @@ -0,0 +1 @@ + diff --git a/polars_db_process/wizards/__init__.py b/polars_db_process/wizards/__init__.py new file mode 100644 index 000000000..9c2ab917e --- /dev/null +++ b/polars_db_process/wizards/__init__.py @@ -0,0 +1 @@ +from . import df_process_wiz diff --git a/polars_db_process/wizards/df_process_wiz.py b/polars_db_process/wizards/df_process_wiz.py new file mode 100644 index 000000000..6f3b099a5 --- /dev/null +++ b/polars_db_process/wizards/df_process_wiz.py @@ -0,0 +1,117 @@ +import logging + +from odoo import _, exceptions, models + +logger = logging.getLogger(__name__) + + +class DfProcessWiz(models.TransientModel): + _inherit = "df.process.wiz" + + def _pre_process(self): + res = super()._pre_process() + if not self.file: + self._pre_process_query() + return res + + def _pre_process_query(self): + "You may inherit to set your own behavior" + if not self.df_source_id.db_conf_id: + raise exceptions.ValidationError( + _("Missing database configuration in your df source ") + ) + self._process_query() + + def _process_query(self): + self.ensure_one() + df = self.df_source_id.db_conf_id._read_sql(self.df_source_id.query) + if self.model_map_id: + if self.model_map_id.action == "import": + self._odoo_data_import(df) + + def _odoo_data_import(self, df): + model = self.model_map_id.model_id.model + vals_list = df.to_dicts() + mapper = {} + touchy = {} + if model == "product.product": + categs = { + x.name: x.res_id + for x in self._get_ir_model_data("product.category", "cat-") + } + if model == "res.partner" and "livr.sql" in self.df_source_id.name: + parents = { + x.name: x.res_id + for x in self._get_ir_model_data("res.partner", "societe") + } + cpt = 0 + for vals in vals_list: + # if cpt == 200: + # self.env.cr.commit() + # logger.info("200 created") + # cpt = 0 + touchy_model = ( + self.env["model.map"]._get_touchy_fields_to_import().get(model) + ) + if touchy_model: + for key in ( + self.env["model.map"]._get_touchy_fields_to_import().get(model) + ): + if key in vals: + touchy[key] = vals.pop(key) + uidstring = vals.pop("id") + nvals = { + x: val for x, val in vals.items() if x in self.env[model]._fields.keys() + } + if "categ_id" in nvals: + nvals["categ_id"] = categs.get(nvals["categ_id"]) or 1 + if "parent_id" in nvals: + # here for product.category + # TODO move this specific behavior elsewhere + if model == "res.partner": + nvals["parent_id"] = parents.get(nvals["parent_id"]) + if model == "product.category": + nvals["parent_id"] = mapper.get(nvals["parent_id"]) + rec = self.env[model].create(nvals) + if model == "product.category": + mapper[uidstring] = rec.id + self._set_unique_idstring(uidstring, rec, model) + self._process_touchy_fields(rec, touchy) + cpt += 1 + + def _set_unique_idstring(self, uidstring, record, model): + """Create Unique Id String also know as XmlId in the Odoo world, + even if not really xml ;-)""" + self.env["ir.model.data"].create( + { + "res_id": record.id, + "model": model, + "module": self.model_map_id._get_uidstring_module_name(), + "name": uidstring, + "noupdate": False, + } + ) + + def _get_ir_model_data(self, model, string, module=None): + module = module or self.env["db.config"]._set_uidstring_module_name() + return self.env["ir.model.data"].search( + [ + ("module", "=", module), + ("model", "=", model), + ("name", "ilike", f"%{string}%"), + ] + ) + + def _process_touchy_fields(self, record, touchy): + """Override Suggestion: + self._touchy_fields_fallback(record, touchy) + or any other alternative + """ + + def _touchy_fields_fallback(self, record, touchy): + for key in touchy: + try: + record[key] = touchy[key] + except Exception: + logger.warning(f"\n\n\n\n\nCarefull here {touchy[key]}") + continue diff --git a/polars_db_process/wizards/df_process_wiz.xml b/polars_db_process/wizards/df_process_wiz.xml new file mode 100644 index 000000000..238507479 --- /dev/null +++ b/polars_db_process/wizards/df_process_wiz.xml @@ -0,0 +1 @@ + diff --git a/polars_db_schema/README.rst b/polars_db_schema/README.rst new file mode 100644 index 000000000..7eb218475 --- /dev/null +++ b/polars_db_schema/README.rst @@ -0,0 +1,88 @@ +================ +Polars Db Schema +================ + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:0e6c26c886f0fc18c585b743314981ec58966dd913b8a469530e62e313327de2 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Alpha-red.png + :target: https://odoo-community.org/page/development-status + :alt: Alpha +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fserver--backend-lightgray.png?logo=github + :target: https://github.com/OCA/server-backend/tree/18.0/polars_db_schema + :alt: OCA/server-backend +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/server-backend-18-0/server-backend-18-0-polars_db_schema + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/server-backend&target_branch=18.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +Introspect external database + +.. image:: https://raw.githubusercontent.com/OCA/server-backend/18.0/polars_db_schema/static/description/figure1.png + +Use case: you want discover an external database extracting data from +only relevant tables and columns + +.. IMPORTANT:: + This is an alpha version, the data model and design can change at any time without warning. + Only for development or testing purpose, do not use in production. + `More details on development status `_ + +**Table of contents** + +.. contents:: + :local: + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +------- + +* Akretion + +Contributors +------------ + +- Akretion + + - David BEAL david.beal@akretion.com + +Maintainers +----------- + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +This module is part of the `OCA/server-backend `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/polars_db_schema/__init__.py b/polars_db_schema/__init__.py new file mode 100644 index 000000000..0650744f6 --- /dev/null +++ b/polars_db_schema/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/polars_db_schema/__manifest__.py b/polars_db_schema/__manifest__.py new file mode 100644 index 000000000..17d6046e0 --- /dev/null +++ b/polars_db_schema/__manifest__.py @@ -0,0 +1,23 @@ +{ + "name": "Polars Db Schema", + "version": "18.0.1.0.0", + "author": "Akretion, Odoo Community Association (OCA)", + "development_status": "Alpha", + "website": "https://github.com/OCA/server-backend", + "license": "AGPL-3", + "depends": [ + "polars_db_process", + ], + "external_dependencies": {"python": []}, + "data": [ + "security/ir.model.access.xml", + "views/db_table.xml", + "views/db_config.xml", + "views/db_type.xml", + "data/db_type_postgresql.xml", + "data/db_type_sqlite.xml", + "data/db_type_sql_server.xml", + ], + "demo": [], + "installable": True, +} diff --git a/polars_db_schema/data/db_type_postgresql.xml b/polars_db_schema/data/db_type_postgresql.xml new file mode 100644 index 000000000..cd5106736 --- /dev/null +++ b/polars_db_schema/data/db_type_postgresql.xml @@ -0,0 +1,14 @@ + + + PostgreSQL + postgresql + bytea + +-- This query order tables by rows count + +SELECT relname AS name, n_live_tup AS row_count + FROM pg_stat_user_tables +ORDER BY row_count DESC; + + + diff --git a/polars_db_schema/data/db_type_sql_server.xml b/polars_db_schema/data/db_type_sql_server.xml new file mode 100644 index 000000000..408fb774e --- /dev/null +++ b/polars_db_schema/data/db_type_sql_server.xml @@ -0,0 +1,51 @@ + + + SQL Server + mssql + Binary + -- This query comes from https://dataedo.com/kb/query/sql-server/list-of-foreign-keys-with-columns + +SELECT lower(fk_tab.name) as foreign_table, + '>-' as rel, + lower(pk_tab.name) as primary_table, + fk_cols.constraint_column_id as no, + fk_col.name as fk_column_name, + ' = ' as [join], + pk_col.name as pk_column_name, + fk.name as fk_constraint_name +FROM sys.foreign_keys fk + inner join sys.tables fk_tab + on fk_tab.object_id = fk.parent_object_id + inner join sys.tables pk_tab + on pk_tab.object_id = fk.referenced_object_id + inner join sys.foreign_key_columns fk_cols + on fk_cols.constraint_object_id = fk.object_id + inner join sys.columns fk_col + on fk_col.column_id = fk_cols.parent_column_id + and fk_col.object_id = fk_tab.object_id + inner join sys.columns pk_col + on pk_col.column_id = fk_cols.referenced_column_id + and pk_col.object_id = pk_tab.object_id +ORDER BY schema_name(fk_tab.schema_id) + '.' + fk_tab.name, + schema_name(pk_tab.schema_id) + '.' + pk_tab.name, + fk_cols.constraint_column_id + + -- This query order tables by rows count + +SELECT lower(sOBJ.name) AS 'name', SUM(sPTN.Rows) AS row_count +FROM + sys.objects AS sOBJ + INNER JOIN sys.partitions AS sPTN + ON sOBJ.object_id = sPTN.object_id +WHERE + sOBJ.type = 'U' + AND sOBJ.is_ms_shipped = 0x0 + AND index_id < 2 -- 0:Heap, 1:Clustered +GROUP BY + sOBJ.schema_id, sOBJ.name +ORDER BY row_count desc + + + diff --git a/polars_db_schema/data/db_type_sqlite.xml b/polars_db_schema/data/db_type_sqlite.xml new file mode 100644 index 000000000..6b03fb941 --- /dev/null +++ b/polars_db_schema/data/db_type_sqlite.xml @@ -0,0 +1,12 @@ + + + Sqlite + sqlite + BLOB + +-- This query order tables by rows count + +SELECT tbl, stat FROM sqlite_stat1 + + + diff --git a/polars_db_schema/models/__init__.py b/polars_db_schema/models/__init__.py new file mode 100644 index 000000000..0ee0fe9c7 --- /dev/null +++ b/polars_db_schema/models/__init__.py @@ -0,0 +1,4 @@ +from . import db_config +from . import db_type +from . import db_table +from . import db_source diff --git a/polars_db_schema/models/db_config.py b/polars_db_schema/models/db_config.py new file mode 100644 index 000000000..ad624adb3 --- /dev/null +++ b/polars_db_schema/models/db_config.py @@ -0,0 +1,127 @@ +from collections import defaultdict + +import polars as pl + +from odoo import exceptions, fields, models +from odoo.tools.safe_eval import safe_eval + + +class DbConfig(models.Model): + _inherit = "db.config" + + db_type_id = fields.Many2one(comodel_name="db.type") + db_table_ids = fields.One2many(comodel_name="db.table", inverse_name="db_config_id") + row_count_query = fields.Text(related="db_type_id.row_count_query") + table_filter = fields.Char( + help="Remove tables with a name matching like sql expression" + ) + table_sort = fields.Selection( + selection=[("row_count", "Count"), ("alias", "Alias"), ("odoo", "Odoo")], + default="row_count", + ) + manually_entries = fields.Text( + readonly=True, + help="Odoo matching models, alias and display. " "Can be backup in your module", + ) + + def get_db_metadata(self): + self.ensure_one() + foreign, entries = {}, {} + if self.db_type_id.foreign_key_query: + foreign = self._get_foreign_keys(self._get_aliases()) + if self.row_count_query: + sql = self.row_count_query + if self.table_filter: + sql = sql.replace( + "WHERE", f"WHERE name NOT like '{self.table_filter}' AND" + ) + df = self._read_sql(sql) + if self.db_type_id.code == "sqlite": + # Sqlite has weird information schema structure + # we need a little hack + df = sqlite(df) + df = df.filter(pl.col("row_count") > 0).with_columns( + # add m2o foreign key + db_config_id=pl.lit(self.id) + ) + df = self._filter_df(df) + self.env["db.table"].search([("db_config_id", "=", self.id)]).unlink() + if self.manually_entries: + entries = safe_eval(self.manually_entries) + vals_list = [] + for row in df.to_dicts(): + name = row.get("name") + if name in foreign: + row["foreign_keys"] = "\n".join(foreign[name]) + if entries: + if name in entries.get("odoo_model"): + row["odoo_model"] = entries["odoo_model"][name] + if name in entries.get("alias"): + row["alias"] = entries["alias"][name] + vals_list.append(row) + self.env["db.table"].create(vals_list) + + def _get_foreign_keys(self, aliases): + foreign = defaultdict(list) + df = self._read_sql(self.db_type_id.foreign_key_query) + mdicts = df.to_dicts() + cols = ["primary_table", "foreign_table", "fk_column_name"] + if mdicts and any([x for x in cols if x not in mdicts[0].keys()]): + raise exceptions.ValidationError( + f"Missing one of these columns {cols} in the query" + ) + for mdict in mdicts: + primary_table = aliases.get(mdict["primary_table"]) + primary_table = ( + aliases.get(mdict["primary_table"]) or mdict["primary_table"] + ) + foreign[mdict["foreign_table"]].append( + f"{mdict['fk_column_name']} = {primary_table}.{mdict['fk_column_name']}" + ) + return foreign + + def _get_aliases(self, reverse=False): + self.ensure_one() + aliases = {x.name: x.alias for x in self.db_table_ids if x.alias} + if reverse: + return {value: key for key, value in aliases.items()} + return aliases + + def _save_manually_entered_data(self): + def get_dict_format(column): + res = ", ".join( + [f"'{x.name}': '{x[column]}'" for x in self.db_table_ids if x[column]] + ) + if res: + return safe_eval(f"{ {res} }".replace('"', "")) + return + + mdict = {} + for mvar in ("odoo_model", "alias", "display"): + sub_dict = get_dict_format(mvar) + if sub_dict: + mdict[mvar] = sub_dict + self.manually_entries = str(mdict).replace("}, '", "},\n'") + + def _filter_df(self, df): + "You may want ignore some tables: inherit me" + return df + + +def sqlite(df): + "Extract row_count info from 'stat' column" + + def extract_first_part(value): + values = value.split(" ") + return values and int(values[0]) or int(value) + + return ( + df.with_columns( + pl.col("stat").map_elements(extract_first_part, return_dtype=pl.Int32) + ) + # rename columns + .rename({"tbl": "name", "stat": "row_count"}) + # stat columns store extra info leading to duplicate lines, + # then make it unique + .unique(maintain_order=True) + ) diff --git a/polars_db_schema/models/db_source.py b/polars_db_schema/models/db_source.py new file mode 100644 index 000000000..24b8632fb --- /dev/null +++ b/polars_db_schema/models/db_source.py @@ -0,0 +1,16 @@ +from odoo import models + + +class DfSource(models.Model): + _inherit = "df.source" + + # def _get_modules_w_df_files(self): + # res = super()._get_modules_w_df_files() + # res.update( + # { + # "polars_db_schema": { + # "xmlid": "migr.contact", + # } + # } + # ) + # return res diff --git a/polars_db_schema/models/db_table.py b/polars_db_schema/models/db_table.py new file mode 100644 index 000000000..2452133a1 --- /dev/null +++ b/polars_db_schema/models/db_table.py @@ -0,0 +1,140 @@ +import base64 +import io + +import connectorx as cx +import polars as pl + +from odoo import _, exceptions, fields, models + + +class DbTable(models.Model): + _name = "db.table" + _description = "Access to database tables" + _order = "row_count DESC" + + name = fields.Char(required=True, readonly=True, help="Name of the table") + row_count = fields.Integer( + string="Count", readonly=True, help="Number of rows contained in the table" + ) + xlsx = fields.Binary(string="File", attachment=False, readonly=True) + db_config_id = fields.Many2one(comodel_name="db.config", readonly=True) + filename = fields.Char(readonly=True) + foreign_keys = fields.Text(readonly=True, help="Foreign keys towards other tables") + alias = fields.Char(help="Used to make SQL query easier to read") + odoo_model = fields.Char(help="Odoo matching model") + display = fields.Char( + help="Fields to combinate (separated by comma) to give a " + "user friendly representation of the record" + ) + sql = fields.Text( + string="Relevant Columns", + readonly=True, + help="Columns with variable data over rows", + ) + unique = fields.Text( + string="Unique Values", + readonly=True, + help="Columns with the same value over rows.\n" + "It could be useless to extract data from these columns,\n" + "because they're probably unused by the application", + ) + + def get_metadata_info(self): + self.ensure_one() + query = f"SELECT * FROM {self.name}" + connexion = self.db_config_id._get_connexion() + df = cx.read_sql(connexion, query, return_type="polars") + # i.e. Binary types should be ignored because of display considerations + excluded_types = self.db_config_id.db_type_id.excluded_types.split("\n") + cols = [x[0] for x in df.schema.items() if str(x[1]) not in excluded_types] + relevant_cols = [] + unique = {} + key_cols, relations = "", "" + # Search columns with non unique value in rows + for col in cols: + # TODO improve it + # Some database have dirty column names: :-( + conditions = [x for x in (" ", "*", "-") if x in col] + if any(conditions): + continue + query = f"SELECT distinct {col} FROM self" + res = df.sql(query) + if len(res) > 1: + relevant_cols.append(f"{self.alias or self.name}.{col}") + else: + # column has the same value on any rows + # we prefer ignore them + unique[col] = res.to_series()[0] + self.unique = f"{unique}" + if self.foreign_keys: + joint = [ + x.split(".")[0].split(" = ")[1] for x in self.foreign_keys.split("\n") + ] + count = {x: joint.count(x) for x in set(joint)} + print(joint, count) + # breakpoint() # import pdb; pdb.set_trace() + foreign_list = [ + # table, foreign=othertable.colname + # x[0], x[1][0] x[1][1] + (self.name, x.split(" = ")) + for x in self.foreign_keys.split("\n") + ] + key_cols = ", ".join([x[1][1] for x in foreign_list]) + "," + aliases = self.db_config_id._get_aliases() + aliases_rev = self.db_config_id._get_aliases(reverse=True) + relations = "\n\t".join( + [ + f"LEFT JOIN {aliases_rev.get(x[1][1].split('.')[0], x[0])} " + f"{x[1][1].split('.')[0]} ON {aliases.get(x[0], x[0])}" + f".{x[1][0]} = {x[1][1]}" + for x in foreign_list + ] + ) + print(relations) + if relevant_cols: + self.sql = f"""SELECT {key_cols} {', '. join(relevant_cols)} +FROM {self.name} {self.alias or ''}\n\t{relations};\n""" + + # WARNING Thread virtual real time limit (151/120s) reached. + # Dumping stacktrace of limit exceeding threads before reloading + + def write(self, vals): + res = super().write(vals) + if "odoo_model" in vals or "alias" in vals or "display" in vals: + for conf in self.mapped("db_config_id"): + conf._save_manually_entered_data() + return res + + def get_spreadsheet(self): + self.ensure_one() + if not self.sql: + self.get_metadata_info() + if not self.sql: + raise exceptions.ValidationError( + _( + "There is no column with variable data in this table: " + "check Uniques Values column" + ) + ) + df = cx.read_sql( + self.db_config_id._get_connexion(), self.sql, return_type="polars" + ) + excel_stream = io.BytesIO() + vals = {"workbook": excel_stream} + vals.update(self._get_spreadsheet_settings()) + df.write_excel(**vals) + excel_stream.seek(0) + self.filename = f"{self.name}.xlsx" + self.xlsx = base64.encodebytes(excel_stream.read()) + + def _get_spreadsheet_settings(self): + return { + "position": "A1", + "table_style": "Table Style Light 16", + "dtype_formats": {pl.Date: "dd/mm/yyyy"}, + "float_precision": 6, + "header_format": {"bold": True, "font_color": "#702963"}, + "freeze_panes": "A2", + "autofit": True, + } diff --git a/polars_db_schema/models/db_type.py b/polars_db_schema/models/db_type.py new file mode 100644 index 000000000..389fc6f27 --- /dev/null +++ b/polars_db_schema/models/db_type.py @@ -0,0 +1,16 @@ +from odoo import fields, models + + +class DbType(models.Model): + _name = "db.type" + _description = "Constant parameters about database" + + name = fields.Char(required=True) + code = fields.Char(required=True) + row_count_query = fields.Text( + required=True, help="SQL code to find how many records contains each table" + ) + foreign_key_query = fields.Text(help="Help to instrospect relations between tables") + excluded_types = fields.Text( + help="Column types to ignore for better introspection (set 1 data by line)" + ) diff --git a/polars_db_schema/pyproject.toml b/polars_db_schema/pyproject.toml new file mode 100644 index 000000000..4231d0ccc --- /dev/null +++ b/polars_db_schema/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["whool"] +build-backend = "whool.buildapi" diff --git a/polars_db_schema/readme/CONTRIBUTORS.md b/polars_db_schema/readme/CONTRIBUTORS.md new file mode 100644 index 000000000..dbb2754b0 --- /dev/null +++ b/polars_db_schema/readme/CONTRIBUTORS.md @@ -0,0 +1,2 @@ +- Akretion + - David BEAL diff --git a/polars_db_schema/readme/DESCRIPTION.md b/polars_db_schema/readme/DESCRIPTION.md new file mode 100644 index 000000000..2832ea239 --- /dev/null +++ b/polars_db_schema/readme/DESCRIPTION.md @@ -0,0 +1,5 @@ +Introspect external database + +.. image:: ../static/description/figure1.png + +Use case: you want discover an external database extracting data from only relevant tables and columns diff --git a/polars_db_schema/security/ir.model.access.xml b/polars_db_schema/security/ir.model.access.xml new file mode 100644 index 000000000..cd5d12469 --- /dev/null +++ b/polars_db_schema/security/ir.model.access.xml @@ -0,0 +1,20 @@ + + + Database Table + + + + + + + + + Database Type + + + + + + + + diff --git a/polars_db_schema/static/description/figure1.png b/polars_db_schema/static/description/figure1.png new file mode 100644 index 000000000..c180b6135 Binary files /dev/null and b/polars_db_schema/static/description/figure1.png differ diff --git a/polars_db_schema/static/description/index.html b/polars_db_schema/static/description/index.html new file mode 100644 index 000000000..6cdf1aac1 --- /dev/null +++ b/polars_db_schema/static/description/index.html @@ -0,0 +1,435 @@ + + + + + +Polars Db Schema + + + +
+

Polars Db Schema

+ + +

Alpha License: AGPL-3 OCA/server-backend Translate me on Weblate Try me on Runboat

+

Introspect external database

+https://raw.githubusercontent.com/OCA/server-backend/18.0/polars_db_schema/static/description/figure1.png +

Use case: you want discover an external database extracting data from +only relevant tables and columns

+
+

Important

+

This is an alpha version, the data model and design can change at any time without warning. +Only for development or testing purpose, do not use in production. +More details on development status

+
+

Table of contents

+ +
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • Akretion
    • +
+
+
+

Contributors

+ +
+
+

Maintainers

+

This module is maintained by the OCA.

+ +Odoo Community Association + +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

This module is part of the OCA/server-backend project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/polars_db_schema/views/db_config.xml b/polars_db_schema/views/db_config.xml new file mode 100644 index 000000000..b67d3a12a --- /dev/null +++ b/polars_db_schema/views/db_config.xml @@ -0,0 +1,39 @@ + + + db.config + + + +