diff --git a/import_helper_base/README.rst b/import_helper_base/README.rst
new file mode 100644
index 0000000..0cb9bf7
--- /dev/null
+++ b/import_helper_base/README.rst
@@ -0,0 +1,8 @@
+==================
+Import Helper Base
+==================
+
+This is a technical module that contains common code for several import helper modules:
+
+* partner_import_helper
+* product_import_helper
diff --git a/import_helper_base/__init__.py b/import_helper_base/__init__.py
new file mode 100644
index 0000000..5cb1c49
--- /dev/null
+++ b/import_helper_base/__init__.py
@@ -0,0 +1 @@
+from . import wizards
diff --git a/import_helper_base/__manifest__.py b/import_helper_base/__manifest__.py
new file mode 100644
index 0000000..e42c92b
--- /dev/null
+++ b/import_helper_base/__manifest__.py
@@ -0,0 +1,22 @@
+# Copyright 2023 Akretion France (http://www.akretion.com/)
+# @author: Alexis de Lattre
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
+
+{
+ "name": "Import Helper Base",
+ "version": "18.0.1.0.0",
+ "category": "Extra Tools",
+ "license": "AGPL-3",
+ "summary": "Common code for all import helper modules",
+ "author": "Akretion",
+ "website": "https://github.com/Akretion/odoo-import-helper",
+ "depends": [
+ "base",
+ ],
+ "external_dependencies": {"python": ["pycountry", "openai", "unidecode"]},
+ "data": [
+ "security/ir.model.access.csv",
+ "wizards/import_helper_view.xml",
+ ],
+ "installable": True,
+}
diff --git a/import_helper_base/pyproject.toml b/import_helper_base/pyproject.toml
new file mode 100644
index 0000000..4231d0c
--- /dev/null
+++ b/import_helper_base/pyproject.toml
@@ -0,0 +1,3 @@
+[build-system]
+requires = ["whool"]
+build-backend = "whool.buildapi"
diff --git a/import_helper_base/security/ir.model.access.csv b/import_helper_base/security/ir.model.access.csv
new file mode 100644
index 0000000..0bba804
--- /dev/null
+++ b/import_helper_base/security/ir.model.access.csv
@@ -0,0 +1,2 @@
+id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink
+access_import_helper_full,Full access on import.helper wizard,model_import_helper,base.group_user,1,1,1,1
diff --git a/import_helper_base/tests/__init__.py b/import_helper_base/tests/__init__.py
new file mode 100644
index 0000000..28ecdfb
--- /dev/null
+++ b/import_helper_base/tests/__init__.py
@@ -0,0 +1 @@
+from . import test_import_helper
diff --git a/import_helper_base/tests/test_import_helper.py b/import_helper_base/tests/test_import_helper.py
new file mode 100644
index 0000000..52788d8
--- /dev/null
+++ b/import_helper_base/tests/test_import_helper.py
@@ -0,0 +1,45 @@
+# Copyright 2023 Akretion France (http://www.akretion.com/)
+# @author: Alexis de Lattre
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
+
+from odoo.tests.common import TransactionCase
+
+
+class TestBaseImportHelper(TransactionCase):
+ @classmethod
+ def setUpClass(cls):
+ super().setUpClass()
+ cls.env = cls.env(context=dict(cls.env.context, tracking_disable=True))
+ # create data
+
+ def test_match_country(self):
+ iho = self.env["import.helper"]
+ speedy = iho._prepare_speedy()
+ country_id = iho._match_country(
+ {"country_name": "fr"}, "country_name", "res.partner", "country_id", speedy
+ )
+ self.assertEqual(country_id, self.env.ref("base.fr").id)
+ country_id = iho._match_country(
+ {"country_name": "FRA"}, "country_name", "res.partner", "country_id", speedy
+ )
+ self.assertEqual(country_id, self.env.ref("base.fr").id)
+ country_id = iho._match_country(
+ {"country_name": "France"},
+ "country_name",
+ "res.partner",
+ "country_id",
+ speedy,
+ )
+ self.assertEqual(country_id, self.env.ref("base.fr").id)
+ country_id = iho._match_country(
+ {"country_name": "U.S.A."},
+ "country_name",
+ "res.partner",
+ "country_id",
+ speedy,
+ )
+ self.assertEqual(country_id, self.env.ref("base.us").id)
+ # country_id = iho._match_country(
+ # {"country_name": "EspaƱa"}, "country_name", "res.partner",
+ # "country_id", speedy)
+ # self.assertEqual(country_id, self.env.ref('base.es').id)
diff --git a/import_helper_base/wizards/__init__.py b/import_helper_base/wizards/__init__.py
new file mode 100644
index 0000000..d07f7a6
--- /dev/null
+++ b/import_helper_base/wizards/__init__.py
@@ -0,0 +1 @@
+from . import import_helper
diff --git a/import_helper_base/wizards/import_helper.py b/import_helper_base/wizards/import_helper.py
new file mode 100644
index 0000000..8ce0fae
--- /dev/null
+++ b/import_helper_base/wizards/import_helper.py
@@ -0,0 +1,326 @@
+# Copyright 2023 Akretion France (http://www.akretion.com/)
+# @author: Alexis de Lattre
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
+
+import logging
+import re
+from collections import defaultdict
+from datetime import datetime
+
+from unidecode import unidecode
+
+from odoo import _, api, fields, models, tools
+from odoo.exceptions import UserError
+
+logger = logging.getLogger(__name__)
+
+try:
+ import pycountry
+except ImportError:
+ logger.debug("Cannot import pycountry")
+try:
+ from openai import OpenAI
+except ImportError:
+ logger.debug("Cannot import openai")
+
+
+class ImportHelper(models.TransientModel):
+ _name = "import.helper"
+ _description = "Helper to import data in Odoo"
+
+ logs = fields.Html(readonly=True)
+
+ @api.model
+ def _prepare_speedy(self, aiengine="chatgpt"):
+ logger.debug("Start to prepare import speedy")
+ speedy = {
+ # country is used both for partner and product
+ "country": {
+ "name2code": {
+ "usa": "US",
+ "etatsunis": "US",
+ "grandebretagne": "GB",
+ "angleterre": "GB",
+ },
+ "code2id": {},
+ "id2code": {}, # used to check iban and vat number prefixes
+ "code2name": {}, # used in log messages
+ },
+ "aiengine": aiengine,
+ "field2label": {},
+ "logs": {},
+ # 'logs' is a dict {'res.partner': [], 'product.product': []}
+ # where the value is a list of dict :
+ # {'msg': 'Checksum IBAN wrong',
+ # 'value': 'FR9879834739',
+ # 'vals': vals, # used to get the line
+ # (and display_name if partner has been created)
+ # 'field': 'res.partner,email',
+ # 'reset': True, # True if the data is NOT imported in Odoo
+ # }
+ }
+ cyd = speedy["country"]
+ code2to3 = {}
+ for country in pycountry.countries:
+ code2to3[country.alpha_2] = country.alpha_3
+ for country in self.env["res.country"].search_read([], ["code", "name"]):
+ cyd["code2id"][country["code"]] = country["id"]
+ cyd["id2code"][country["id"]] = country["code"]
+ cyd["code2name"][country["code"]] = country["name"]
+ code3 = code2to3.get(country["code"])
+ if code3:
+ cyd["code2id"][code3] = country["id"]
+ cyd["code2name"][code3] = country["name"]
+ for lang in self.env["res.lang"].search([]):
+ logger.info("Working on lang %s", lang.code)
+ for country in (
+ self.env["res.country"]
+ .with_context(lang=lang.code)
+ .search_read([], ["code", "name"])
+ ):
+ country_name_match = self._prepare_country_name_match(country["name"])
+ cyd["name2code"][country_name_match] = country["code"]
+ if aiengine == "chatgpt":
+ openai_api_key = tools.config.get("openai_api_key", False)
+ if not openai_api_key:
+ raise UserError(
+ _(
+ "Missing entry openai_api_key in the Odoo server "
+ "configuration file."
+ )
+ )
+ speedy["openai_client"] = OpenAI(api_key=openai_api_key)
+ speedy["openai_tokens"] = 0
+ return speedy
+
+ @api.model
+ def _prepare_country_name_match(self, country_name):
+ assert country_name
+ country_name_match = unidecode(country_name).lower()
+ country_name_match = "".join(re.findall(r"[a-z]+", country_name_match))
+ assert country_name_match
+ return country_name_match
+
+ def _match_country(self, vals, country_key, model, country_field_name, speedy):
+ assert isinstance(model, str)
+ country_name = vals[country_key]
+ log = {
+ "value": country_name,
+ "vals": vals,
+ "field": f"{model},{country_field_name}",
+ }
+ cyd = speedy["country"]
+ if len(country_name) in (2, 3):
+ country_code = country_name.upper()
+ if country_code in cyd["code2id"]:
+ logger.info(
+ "Country name '%s' is an ISO country code (%s)",
+ country_name,
+ cyd["code2name"][country_code],
+ )
+ country_id = cyd["code2id"][country_code]
+ return country_id
+ country_name_match = self._prepare_country_name_match(country_name)
+ if country_name_match in cyd["name2code"]:
+ country_code = cyd["name2code"][country_name_match]
+ logger.info(
+ "Country '%s' matched on country %s (%s)",
+ country_name,
+ cyd["code2name"][country_code],
+ country_code,
+ )
+ country_id = cyd["code2id"][country_code]
+ return country_id
+ logger.info(
+ "No direct match for country '%s': now asking ChatGPT.", country_name
+ )
+ # ask ChatGPT !
+ content = f"""ISO country code of "{country_name}", nothing else"""
+ logger.debug("ChatGPT question: %s", content)
+ chat_completion = speedy["openai_client"].chat.completions.create(
+ model="gpt-3.5-turbo",
+ messages=[{"role": "user", "content": content}],
+ temperature=0,
+ )
+
+ # print the chat completion
+ tokens = chat_completion.usage.total_tokens
+ logger.debug("%d tokens have been used", tokens)
+ speedy["openai_tokens"] += tokens
+ answer = chat_completion.choices[0].message.content
+ if answer:
+ answer = answer.strip()
+ logger.info("ChatGPT answer: %s", answer)
+ if len(answer) == 2:
+ country_code = answer.upper()
+ if country_code in cyd["code2id"]:
+ cyd_code2n_cntry = cyd["code2name"][country_code]
+ logger.info(
+ "ChatGPT matched country '%s' to %s (%s)",
+ country_name,
+ cyd_code2n_cntry,
+ country_code,
+ )
+ speedy["logs"][model].append(
+ dict(
+ log,
+ msg=(
+ "Country name could not be found in Odoo. "
+ f"ChatGPT said ISO code was '{country_code}', "
+ f"which matched to '{cyd_code2n_cntry}'"
+ ),
+ )
+ )
+ country_id = cyd["code2id"][country_code]
+ cyd["name2code"][country_name_match] = country_code
+ return country_id
+ else:
+ speedy["logs"][model].append(
+ dict(
+ log,
+ msg=(
+ "Country name could not be found in Odoo. "
+ f"ChatGPT said ISO code was '{country_code}', "
+ "which didn't match to any country"
+ ),
+ ),
+ reset=True,
+ )
+ else:
+ speedy["logs"][model].append(
+ dict(
+ log,
+ msg="ChatGPT didn't answer a 2 letter country code "
+ f"but '{answer}'",
+ reset=True,
+ )
+ )
+ else:
+ logger.warning("No answer from chatGPT")
+ speedy["logs"][model].append(
+ dict(log, msg="No answer from chatGPT", reset=True)
+ )
+ return False
+
+ def _field_label(self, field, speedy):
+ if field not in speedy["field2label"]:
+ field_split = field.split(",")
+ ofield = self.env["ir.model.fields"].search(
+ [
+ ("model", "=", field_split[0]),
+ ("name", "=", field_split[1]),
+ ],
+ limit=1,
+ )
+ if ofield:
+ speedy["field2label"][field] = ofield.field_description
+ else:
+ speedy["field2label"][field] = f"{field_split[1]} ({field_split[0]})"
+ return speedy["field2label"][field]
+
+ def _convert_logs2html(self, speedy):
+ html = (
+ 'For the logs in red, '
+ "the data was not imported in Odoo
"
+ )
+ if speedy.get("aiengine") == "chatgpt":
+ html += "{} OpenAI tokens where used
".format(
+ speedy["openai_tokens"]
+ )
+ for obj_name, log_list in speedy["logs"].items():
+ obj_rec = self.env["ir.model"].search([("model", "=", obj_name)], limit=1)
+ assert obj_rec
+ html += f'{obj_rec.name}
'
+ line2logs = defaultdict(list)
+ field2logs = defaultdict(list)
+ for log in log_list:
+ if log["vals"].get("line"):
+ line2logs[log["vals"]["line"]].append(log)
+ if log.get("field"):
+ field2logs[log["field"]].append(log)
+ html += 'Logs per line
'
+ for line, logs in line2logs.items():
+ log_labels = []
+ for log in logs:
+ log_labels.append(
+ '{}: {} - {}'.format(
+ log.get("reset") and "red" or "black",
+ self._field_label(log["field"], speedy),
+ log["value"],
+ log["msg"],
+ )
+ )
+ h3 = f"Line {line}"
+ if log["vals"].get("id"):
+ h3 += f": {log['vals']['display_name']} (ID {log['vals']['id']})"
+ html += "{}
\n
".format(
+ h3, "\n".join(log_labels)
+ )
+ html += 'Logs per field
'
+ for field, logs in field2logs.items():
+ log_labels = []
+ for log in logs:
+ line_label = f"Line {log['vals'].get('line', 'unknown')}"
+ if log["vals"].get("id"):
+ line_label += " (%(display_name)s ID %(id)d)" % {
+ "display_name": log["vals"]["display_name"],
+ "id": log["vals"]["id"],
+ }
+ log_labels.append(
+ (
+ '{}: ' "{} - {}"
+ ).format(
+ log.get("reset") and "red" or "black",
+ line_label,
+ log["value"],
+ log["msg"],
+ )
+ )
+ html += (
+ f"{self._field_label(field, speedy)}
\n"
+ "{'\n'.join(log_labels)}
"
+ )
+ return html
+
+ def _result_action(self, speedy):
+ action = {
+ "name": "Result",
+ "type": "ir.actions.act_window",
+ "res_model": "import.helper",
+ "view_mode": "form",
+ "target": "new",
+ "context": dict(
+ self._context, default_logs=self._convert_logs2html(speedy)
+ ),
+ }
+ return action
+
+ def _prepare_create_date(self, vals, speedy):
+ create_date = vals.get("create_date")
+ create_date_dt = False
+ if isinstance(create_date, str) and len(create_date) == 10:
+ try:
+ create_date_dt = datetime.strptime(create_date, "%Y-%m-%d")
+ except Exception as e:
+ speedy["logs"].append(
+ {
+ "msg": f"Failed to convert '{create_date}' to datetime: {e}",
+ "value": vals["create_date"],
+ "vals": vals,
+ "field": "product.product,create_date",
+ "reset": True,
+ }
+ )
+ elif isinstance(create_date, datetime):
+ create_date_dt = create_date
+ if create_date_dt and create_date_dt.date() > fields.Date.context_today(self):
+ speedy["logs"].append(
+ {
+ "msg": f"create_date {create_date_dt} cannot be in the future",
+ "value": create_date,
+ "vals": vals,
+ "field": "product.product,create_date",
+ "reset": True,
+ }
+ )
+ return create_date_dt
diff --git a/import_helper_base/wizards/import_helper_view.xml b/import_helper_base/wizards/import_helper_view.xml
new file mode 100644
index 0000000..c30db50
--- /dev/null
+++ b/import_helper_base/wizards/import_helper_view.xml
@@ -0,0 +1,21 @@
+
+
+
+
+ import.helper
+
+
+
+
+
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..54bcb3c
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,4 @@
+# generated from manifests external_dependencies
+openai
+pycountry
+unidecode