From af98af3f4eb8d456e214a3574c548d7ea2140837 Mon Sep 17 00:00:00 2001 From: Alexis de Lattre Date: Sat, 7 Oct 2023 00:33:03 +0200 Subject: [PATCH 1/6] Add module product_import_helper Move code from partner_import_helper to import_helper_base --- import_helper_base/README.rst | 8 + import_helper_base/__init__.py | 1 + import_helper_base/__manifest__.py | 21 +++ .../security/ir.model.access.csv | 2 + import_helper_base/wizards/__init__.py | 1 + .../wizards/import_show_logs.py | 142 ++++++++++++++++++ .../wizards/import_show_logs_view.xml | 21 +++ 7 files changed, 196 insertions(+) create mode 100644 import_helper_base/README.rst create mode 100644 import_helper_base/__init__.py create mode 100644 import_helper_base/__manifest__.py create mode 100644 import_helper_base/security/ir.model.access.csv create mode 100644 import_helper_base/wizards/__init__.py create mode 100644 import_helper_base/wizards/import_show_logs.py create mode 100644 import_helper_base/wizards/import_show_logs_view.xml diff --git a/import_helper_base/README.rst b/import_helper_base/README.rst new file mode 100644 index 0000000..0cb9bf7 --- /dev/null +++ b/import_helper_base/README.rst @@ -0,0 +1,8 @@ +================== +Import Helper Base +================== + +This is a technical module that contains common code for several import helper modules: + +* partner_import_helper +* product_import_helper diff --git a/import_helper_base/__init__.py b/import_helper_base/__init__.py new file mode 100644 index 0000000..5cb1c49 --- /dev/null +++ b/import_helper_base/__init__.py @@ -0,0 +1 @@ +from . import wizards diff --git a/import_helper_base/__manifest__.py b/import_helper_base/__manifest__.py new file mode 100644 index 0000000..33cb935 --- /dev/null +++ b/import_helper_base/__manifest__.py @@ -0,0 +1,21 @@ +# Copyright 2023 Akretion France (http://www.akretion.com/) +# @author: Alexis de Lattre +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +{ + 'name': 'Import Helper Base', + 'version': '16.0.1.0.0', + 'category': 'Extra Tools', + 'license': 'AGPL-3', + 'summary': 'Common code for all import helper modules', + 'author': 'Akretion', + 'website': 'https://github.com/akretion/odoo-import-helper', + 'depends': [ + 'base', + ], + 'data': [ + 'security/ir.model.access.csv', + 'wizards/import_show_logs_view.xml', + ], + 'installable': True, +} diff --git a/import_helper_base/security/ir.model.access.csv b/import_helper_base/security/ir.model.access.csv new file mode 100644 index 0000000..6415e08 --- /dev/null +++ b/import_helper_base/security/ir.model.access.csv @@ -0,0 +1,2 @@ +id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink +access_import_show_logs,Full access on import.show.logs wizard,model_import_show_logs,base.group_user,1,1,1,1 diff --git a/import_helper_base/wizards/__init__.py b/import_helper_base/wizards/__init__.py new file mode 100644 index 0000000..bec5909 --- /dev/null +++ b/import_helper_base/wizards/__init__.py @@ -0,0 +1 @@ +from . import import_show_logs diff --git a/import_helper_base/wizards/import_show_logs.py b/import_helper_base/wizards/import_show_logs.py new file mode 100644 index 0000000..a8ea130 --- /dev/null +++ b/import_helper_base/wizards/import_show_logs.py @@ -0,0 +1,142 @@ +# Copyright 2023 Akretion France (http://www.akretion.com/) +# @author: Alexis de Lattre +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +from odoo import api, fields, models, tools, _ +from odoo.exceptions import UserError +from collections import defaultdict +from datetime import datetime + +import logging +logger = logging.getLogger(__name__) + +try: + import openai +except ImportError: + logger.debug('Cannot import openai') + +class ImportShowLogs(models.TransientModel): + _name = "import.show.logs" + _description = "Pop-up to show warnings after import" + + logs = fields.Html(readonly=True) + + @api.model + def _import_speedy(self, chatgpt=False): + logger.debug('Start to prepare import speedy') + speedy = { + 'chatgpt': chatgpt, + 'field2label': {}, + 'logs': [], + # 'logs' should contain a list of dict : + # {'msg': 'Checksum IBAN wrong', + # 'value': 'FR9879834739', + # 'vals': vals, # used to get the line + # (and display_name if partner has been created) + # 'field': 'res.partner,email', + # 'reset': True, # True if the data is NOT imported in Odoo + # } + } + if chatgpt: + openai_api_key = tools.config.get('openai_api_key', False) + if not openai_api_key: + raise UserError(_( + "Missing entry openai_api_key in the Odoo server configuration file.")) + openai.api_key = openai_api_key + speedy['openai_tokens'] = 0 + return speedy + + def _field_label(self, field, speedy): + if field not in speedy['field2label']: + field_split = field.split(',') + ofield = self.env['ir.model.fields'].search([ + ('model', '=', field_split[0]), + ('name', '=', field_split[1]), + ], limit=1) + if ofield: + speedy['field2label'][field] = ofield.field_description + else: + speedy['field2label'][field] = '%s (%s)' % ( + field_split[1], field_split[0]) + return speedy['field2label'][field] + + def _import_logs2html(self, speedy): + line2logs = defaultdict(list) + field2logs = defaultdict(list) + for log in speedy['logs']: + if log['vals'].get('line'): + line2logs[log['vals']['line']].append(log) + if log.get('field'): + field2logs[log['field']].append(log) + html = '

For the logs in red, the data was not imported in Odoo
' + if speedy.get('chatgpt'): + html += '%d OpenAI tokens where used

' % speedy['openai_tokens'] + html += '

Logs per line

' + for line, logs in line2logs.items(): + log_labels = [] + for log in logs: + log_labels.append( + '
  • %s: %s - %s
    • ' % ( + log.get('reset') and 'red' or 'black', + self._field_label(log['field'], speedy), + log['value'], + log['msg'], + )) + h3 = 'Line %s' % line + if log['vals'].get('id'): + h3 += ': %s (ID %d)' % (log['vals']['display_name'], log['vals']['id']) + html += '

    %s

    \n

      %s

    ' % (h3, '\n'.join(log_labels)) + html += '

    Logs per field

    ' + for field, logs in field2logs.items(): + log_labels = [] + for log in logs: + line_label = 'Line %s' % log['vals'].get('line', 'unknown') + if log['vals'].get('id'): + line_label += ' (%s ID %d)' % (log['vals']['display_name'], log['vals']['id']) + log_labels.append( + '
  • %s: %s - %s
    • ' % ( + log.get('reset') and 'red' or 'black', + line_label, + log['value'], + log['msg'], + )) + html += '

    %s

    \n

      %s

    ' % ( + self._field_label(field, speedy), '\n'.join(log_labels)) + return html + + def _import_result_action(self, speedy): + action = { + 'name': 'Result', + 'type': 'ir.actions.act_window', + 'res_model': 'import.show.logs', + 'view_mode': 'form', + 'target': 'new', + 'context': dict(self._context, default_logs=self._import_logs2html(speedy)), + } + return action + + def _prepare_create_date(self, vals, speedy): + create_date = vals.get('create_date') + create_date_dt = False + if isinstance(create_date, str) and len(create_date) == 10: + try: + create_date_dt = datetime.strptime(create_date, '%Y-%m-%d') + except Exception as e: + speedy['logs'].append({ + 'msg': "Failed to convert '%s' to datetime: %s" % (create_date, e), + 'value': vals['create_date'], + 'vals': vals, + 'field': 'product.product,create_date', + 'reset': True, + }) + elif isinstance(create_date, datetime): + create_date_dt = create_date + if create_date_dt and create_date_dt.date() > fields.Date.context_today(self): + speedy['logs'].append({ + 'msg': 'create_date %s cannot be in the future' % create_date_dt, + 'value': create_date, + 'vals': vals, + 'field': 'product.product,create_date', + 'reset': True, + }) + return create_date_dt diff --git a/import_helper_base/wizards/import_show_logs_view.xml b/import_helper_base/wizards/import_show_logs_view.xml new file mode 100644 index 0000000..eaf1735 --- /dev/null +++ b/import_helper_base/wizards/import_show_logs_view.xml @@ -0,0 +1,21 @@ + + + + + import.show.logs + +
    + + + +
    +
    +
    +     +     +     From 7fac1ee820a35841072d48e9512011273c63d679 Mon Sep 17 00:00:00 2001 From: Alexis de Lattre Date: Mon, 15 Jan 2024 18:09:58 +0100 Subject: [PATCH 2/6] Adapt to changes in openai lib --- import_helper_base/wizards/import_show_logs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/import_helper_base/wizards/import_show_logs.py b/import_helper_base/wizards/import_show_logs.py index a8ea130..a06ea8e 100644 --- a/import_helper_base/wizards/import_show_logs.py +++ b/import_helper_base/wizards/import_show_logs.py @@ -11,7 +11,7 @@ logger = logging.getLogger(__name__) try: - import openai + from openai import OpenAI except ImportError: logger.debug('Cannot import openai') @@ -42,7 +42,7 @@ def _import_speedy(self, chatgpt=False): if not openai_api_key: raise UserError(_( "Missing entry openai_api_key in the Odoo server configuration file.")) - openai.api_key = openai_api_key + speedy['openai_client'] = OpenAI(api_key=openai_api_key) speedy['openai_tokens'] = 0 return speedy From b771dddb330a71b286f5fe4edbfa5db783217743 Mon Sep 17 00:00:00 2001 From: Alexis de Lattre Date: Tue, 16 Jan 2024 00:35:22 +0100 Subject: [PATCH 3/6] import_helper module: full code reorganisation, to make it easier to import products and suppliers at the same time --- import_helper_base/__manifest__.py | 2 +- .../security/ir.model.access.csv | 2 +- import_helper_base/wizards/__init__.py | 2 +- .../{import_show_logs.py => import_helper.py} | 106 +++++++++--------- ...w_logs_view.xml => import_helper_view.xml} | 4 +- 5 files changed, 61 insertions(+), 55 deletions(-) rename import_helper_base/wizards/{import_show_logs.py => import_helper.py} (55%) rename import_helper_base/wizards/{import_show_logs_view.xml => import_helper_view.xml} (84%) diff --git a/import_helper_base/__manifest__.py b/import_helper_base/__manifest__.py index 33cb935..b6f40d2 100644 --- a/import_helper_base/__manifest__.py +++ b/import_helper_base/__manifest__.py @@ -15,7 +15,7 @@ ], 'data': [ 'security/ir.model.access.csv', - 'wizards/import_show_logs_view.xml', + 'wizards/import_helper_view.xml', ], 'installable': True, } diff --git a/import_helper_base/security/ir.model.access.csv b/import_helper_base/security/ir.model.access.csv index 6415e08..0bba804 100644 --- a/import_helper_base/security/ir.model.access.csv +++ b/import_helper_base/security/ir.model.access.csv @@ -1,2 +1,2 @@ id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink -access_import_show_logs,Full access on import.show.logs wizard,model_import_show_logs,base.group_user,1,1,1,1 +access_import_helper_full,Full access on import.helper wizard,model_import_helper,base.group_user,1,1,1,1 diff --git a/import_helper_base/wizards/__init__.py b/import_helper_base/wizards/__init__.py index bec5909..d07f7a6 100644 --- a/import_helper_base/wizards/__init__.py +++ b/import_helper_base/wizards/__init__.py @@ -1 +1 @@ -from . import import_show_logs +from . import import_helper diff --git a/import_helper_base/wizards/import_show_logs.py b/import_helper_base/wizards/import_helper.py similarity index 55% rename from import_helper_base/wizards/import_show_logs.py rename to import_helper_base/wizards/import_helper.py index a06ea8e..e810686 100644 --- a/import_helper_base/wizards/import_show_logs.py +++ b/import_helper_base/wizards/import_helper.py @@ -15,20 +15,22 @@ except ImportError: logger.debug('Cannot import openai') -class ImportShowLogs(models.TransientModel): - _name = "import.show.logs" - _description = "Pop-up to show warnings after import" + +class ImportHelper(models.TransientModel): + _name = "import.helper" + _description = "Helper to import data in Odoo" logs = fields.Html(readonly=True) @api.model - def _import_speedy(self, chatgpt=False): + def _prepare_speedy(self, aiengine='chatgpt'): logger.debug('Start to prepare import speedy') speedy = { - 'chatgpt': chatgpt, + 'aiengine': aiengine, 'field2label': {}, - 'logs': [], - # 'logs' should contain a list of dict : + 'logs': {}, + # 'logs' is a dict {'res.partner': [], 'product.product': []} + # where the value is a list of dict : # {'msg': 'Checksum IBAN wrong', # 'value': 'FR9879834739', # 'vals': vals, # used to get the line @@ -37,7 +39,7 @@ def _import_speedy(self, chatgpt=False): # 'reset': True, # True if the data is NOT imported in Odoo # } } - if chatgpt: + if aiengine == 'chatgpt': openai_api_key = tools.config.get('openai_api_key', False) if not openai_api_key: raise UserError(_( @@ -60,58 +62,62 @@ def _field_label(self, field, speedy): field_split[1], field_split[0]) return speedy['field2label'][field] - def _import_logs2html(self, speedy): - line2logs = defaultdict(list) - field2logs = defaultdict(list) - for log in speedy['logs']: - if log['vals'].get('line'): - line2logs[log['vals']['line']].append(log) - if log.get('field'): - field2logs[log['field']].append(log) + def _convert_logs2html(self, speedy): html = '

    For the logs in red, the data was not imported in Odoo
    ' - if speedy.get('chatgpt'): + if speedy.get('aiengine') == 'chatgpt': html += '%d OpenAI tokens where used

    ' % speedy['openai_tokens'] - html += '

    Logs per line

    ' - for line, logs in line2logs.items(): - log_labels = [] - for log in logs: - log_labels.append( - '
  • %s: %s - %s
    • ' % ( - log.get('reset') and 'red' or 'black', - self._field_label(log['field'], speedy), - log['value'], - log['msg'], - )) - h3 = 'Line %s' % line - if log['vals'].get('id'): - h3 += ': %s (ID %d)' % (log['vals']['display_name'], log['vals']['id']) - html += '

    %s

    \n

      %s

    ' % (h3, '\n'.join(log_labels)) - html += '

    Logs per field

    ' - for field, logs in field2logs.items(): - log_labels = [] - for log in logs: - line_label = 'Line %s' % log['vals'].get('line', 'unknown') + for obj_name, log_list in speedy['logs'].items(): + obj_rec = self.env['ir.model'].search([('model', '=', obj_name)], limit=1) + assert obj_rec + html += '

    %s

    ' % obj_rec.name + line2logs = defaultdict(list) + field2logs = defaultdict(list) + for log in log_list: + if log['vals'].get('line'): + line2logs[log['vals']['line']].append(log) + if log.get('field'): + field2logs[log['field']].append(log) + html += '

    Logs per line

    ' + for line, logs in line2logs.items(): + log_labels = [] + for log in logs: + log_labels.append( + '
  • %s: %s - %s
    • ' % ( + log.get('reset') and 'red' or 'black', + self._field_label(log['field'], speedy), + log['value'], + log['msg'], + )) + h3 = 'Line %s' % line if log['vals'].get('id'): - line_label += ' (%s ID %d)' % (log['vals']['display_name'], log['vals']['id']) - log_labels.append( - '
  • %s: %s - %s
    • ' % ( - log.get('reset') and 'red' or 'black', - line_label, - log['value'], - log['msg'], - )) - html += '

    %s

    \n

      %s

    ' % ( - self._field_label(field, speedy), '\n'.join(log_labels)) + h3 += ': %s (ID %d)' % (log['vals']['display_name'], log['vals']['id']) + html += '

    %s

    \n

      %s

    ' % (h3, '\n'.join(log_labels)) + html += '

    Logs per field

    ' + for field, logs in field2logs.items(): + log_labels = [] + for log in logs: + line_label = 'Line %s' % log['vals'].get('line', 'unknown') + if log['vals'].get('id'): + line_label += ' (%s ID %d)' % (log['vals']['display_name'], log['vals']['id']) + log_labels.append( + '
  • %s: %s - %s
    • ' % ( + log.get('reset') and 'red' or 'black', + line_label, + log['value'], + log['msg'], + )) + html += '

    %s

    \n

      %s

    ' % ( + self._field_label(field, speedy), '\n'.join(log_labels)) return html - def _import_result_action(self, speedy): + def _result_action(self, speedy): action = { 'name': 'Result', 'type': 'ir.actions.act_window', - 'res_model': 'import.show.logs', + 'res_model': 'import.helper', 'view_mode': 'form', 'target': 'new', - 'context': dict(self._context, default_logs=self._import_logs2html(speedy)), + 'context': dict(self._context, default_logs=self._convert_logs2html(speedy)), } return action diff --git a/import_helper_base/wizards/import_show_logs_view.xml b/import_helper_base/wizards/import_helper_view.xml similarity index 84% rename from import_helper_base/wizards/import_show_logs_view.xml rename to import_helper_base/wizards/import_helper_view.xml index eaf1735..c30db50 100644 --- a/import_helper_base/wizards/import_show_logs_view.xml +++ b/import_helper_base/wizards/import_helper_view.xml @@ -5,8 +5,8 @@ License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). --> - - import.show.logs + + import.helper
    From e32cb881d05b434563e0aebb9e839d61164fea41 Mon Sep 17 00:00:00 2001 From: Alexis de Lattre Date: Tue, 24 Sep 2024 20:21:24 +0000 Subject: [PATCH 4/6] product_import_helper: remove dep on account_product_fiscal_classification move code (and tests) for country match from partner_import_helper to import_helper_base product_import_helper: add keys origin_country_name and hs_code_code --- import_helper_base/__manifest__.py | 1 + import_helper_base/tests/__init__.py | 1 + .../tests/test_import_helper.py | 33 ++++++ import_helper_base/wizards/import_helper.py | 100 ++++++++++++++++++ 4 files changed, 135 insertions(+) create mode 100644 import_helper_base/tests/__init__.py create mode 100644 import_helper_base/tests/test_import_helper.py diff --git a/import_helper_base/__manifest__.py b/import_helper_base/__manifest__.py index b6f40d2..f215e46 100644 --- a/import_helper_base/__manifest__.py +++ b/import_helper_base/__manifest__.py @@ -13,6 +13,7 @@ 'depends': [ 'base', ], + "external_dependencies": {"python" : ["pycountry", "openai"]}, 'data': [ 'security/ir.model.access.csv', 'wizards/import_helper_view.xml', diff --git a/import_helper_base/tests/__init__.py b/import_helper_base/tests/__init__.py new file mode 100644 index 0000000..28ecdfb --- /dev/null +++ b/import_helper_base/tests/__init__.py @@ -0,0 +1 @@ +from . import test_import_helper diff --git a/import_helper_base/tests/test_import_helper.py b/import_helper_base/tests/test_import_helper.py new file mode 100644 index 0000000..cd0a0f6 --- /dev/null +++ b/import_helper_base/tests/test_import_helper.py @@ -0,0 +1,33 @@ +# Copyright 2023 Akretion France (http://www.akretion.com/) +# @author: Alexis de Lattre +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). + +from odoo.tests.common import TransactionCase + + +class TestBaseImportHelper(TransactionCase): + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.env = cls.env(context=dict(cls.env.context, tracking_disable=True)) + # create data + + def test_match_country(self): + iho = self.env['import.helper'] + speedy = iho._prepare_speedy() + country_id = iho._match_country( + {"country_name": "fr"}, "country_name", "res.partner", "country_id", speedy) + self.assertEqual(country_id, self.env.ref('base.fr').id) + country_id = iho._match_country( + {"country_name": "FRA"}, "country_name", "res.partner", "country_id", speedy) + self.assertEqual(country_id, self.env.ref('base.fr').id) + country_id = iho._match_country( + {"country_name": "France"}, "country_name", "res.partner", "country_id", speedy) + self.assertEqual(country_id, self.env.ref('base.fr').id) + country_id = iho._match_country( + {"country_name": "U.S.A."}, "country_name", "res.partner", "country_id", speedy) + self.assertEqual(country_id, self.env.ref('base.us').id) + # country_id = iho._match_country( + # {"country_name": "España"}, "country_name", "res.partner", "country_id", speedy) + # self.assertEqual(country_id, self.env.ref('base.es').id) diff --git a/import_helper_base/wizards/import_helper.py b/import_helper_base/wizards/import_helper.py index e810686..a6ca541 100644 --- a/import_helper_base/wizards/import_helper.py +++ b/import_helper_base/wizards/import_helper.py @@ -6,10 +6,16 @@ from odoo.exceptions import UserError from collections import defaultdict from datetime import datetime +from unidecode import unidecode +import re import logging logger = logging.getLogger(__name__) +try: + import pycountry +except ImportError: + logger.debug('Cannot import pycountry') try: from openai import OpenAI except ImportError: @@ -26,6 +32,18 @@ class ImportHelper(models.TransientModel): def _prepare_speedy(self, aiengine='chatgpt'): logger.debug('Start to prepare import speedy') speedy = { + # country is used both for partner and product + 'country': { + 'name2code': { + "usa": "US", + "etatsunis": "US", + "grandebretagne": "GB", + "angleterre": "GB", + }, + 'code2id': {}, + 'id2code': {}, # used to check iban and vat number prefixes + 'code2name': {}, # used in log messages + }, 'aiengine': aiengine, 'field2label': {}, 'logs': {}, @@ -39,6 +57,23 @@ def _prepare_speedy(self, aiengine='chatgpt'): # 'reset': True, # True if the data is NOT imported in Odoo # } } + cyd = speedy['country'] + code2to3 = {} + for country in pycountry.countries: + code2to3[country.alpha_2] = country.alpha_3 + for country in self.env['res.country'].search_read([], ['code', 'name']): + cyd['code2id'][country['code']] = country['id'] + cyd['id2code'][country['id']] = country['code'] + cyd['code2name'][country['code']] = country['name'] + code3 = code2to3.get(country['code']) + if code3: + cyd['code2id'][code3] = country['id'] + cyd['code2name'][code3] = country['name'] + for lang in self.env['res.lang'].search([]): + logger.info('Working on lang %s', lang.code) + for country in self.env['res.country'].with_context(lang=lang.code).search_read([], ['code', 'name']): + country_name_match = self._prepare_country_name_match(country['name']) + cyd['name2code'][country_name_match] = country['code'] if aiengine == 'chatgpt': openai_api_key = tools.config.get('openai_api_key', False) if not openai_api_key: @@ -48,6 +83,71 @@ def _prepare_speedy(self, aiengine='chatgpt'): speedy['openai_tokens'] = 0 return speedy + @api.model + def _prepare_country_name_match(self, country_name): + assert country_name + country_name_match = unidecode(country_name).lower() + country_name_match = ''.join(re.findall(r'[a-z]+', country_name_match)) + assert country_name_match + return country_name_match + + def _match_country(self, vals, country_key, model, country_field_name, speedy): + assert isinstance(model, str) + country_name = vals[country_key] + log = { + 'value': country_name, + 'vals': vals, + 'field': f'{model},{country_field_name}', + } + cyd = speedy['country'] + if len(country_name) in (2, 3): + country_code = country_name.upper() + if country_code in cyd['code2id']: + logger.info("Country name '%s' is an ISO country code (%s)", country_name, cyd['code2name'][country_code]) + country_id = cyd['code2id'][country_code] + return country_id + country_name_match = self._prepare_country_name_match(country_name) + if country_name_match in cyd['name2code']: + country_code = cyd['name2code'][country_name_match] + logger.info("Country '%s' matched on country %s (%s)", country_name, cyd['code2name'][country_code], country_code) + country_id = cyd['code2id'][country_code] + return country_id + logger.info("No direct match for country '%s': now asking ChatGPT.", country_name) + # ask ChatGPT ! + content = """ISO country code of "%s", nothing else""" % country_name + logger.debug('ChatGPT question: %s', content) + chat_completion = speedy['openai_client'].chat.completions.create( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": content}], + temperature=0, + ) + + # print the chat completion + tokens = chat_completion.usage.total_tokens + logger.debug("%d tokens have been used", tokens) + speedy["openai_tokens"] += tokens + answer = chat_completion.choices[0].message.content + if answer: + answer = answer.strip() + logger.info('ChatGPT answer: %s', answer) + if len(answer) == 2: + country_code = answer.upper() + if country_code in cyd['code2id']: + logger.info("ChatGPT matched country '%s' to %s (%s)", country_name, cyd['code2name'][country_code], country_code) + speedy['logs'][model].append(dict(log, msg="Country name could not be found in Odoo. ChatGPT said ISO code was '%s', which matched to '%s'" % (country_code, cyd['code2name'][country_code]))) + country_id = cyd['code2id'][country_code] + cyd['name2code'][country_name_match] = country_code + return country_id + else: + speedy['logs'][model].append(dict(log, msg="Country name could not be found in Odoo. ChatGPT said ISO code was '%s', which didn't match to any country" % country_code), reset=True) + else: + speedy['logs'][model].append( + dict(log, msg="ChatGPT didn't answer a 2 letter country code but '%s'" % answer, reset=True)) + else: + logger.warning('No answer from chatGPT') + speedy['logs'][model].append(dict(log, msg='No answer from chatGPT', reset=True)) + return False + def _field_label(self, field, speedy): if field not in speedy['field2label']: field_split = field.split(',') From 63b921f2d9919b5e68e8a1dffee0153059865dad Mon Sep 17 00:00:00 2001 From: David Beal Date: Tue, 19 Nov 2024 08:59:52 +0100 Subject: [PATCH 5/6] PREcommit --- import_helper_base/__manifest__.py | 32 +- import_helper_base/pyproject.toml | 3 + .../tests/test_import_helper.py | 31 +- import_helper_base/wizards/import_helper.py | 339 +++++++++++------- requirements.txt | 3 + 5 files changed, 249 insertions(+), 159 deletions(-) create mode 100644 import_helper_base/pyproject.toml create mode 100644 requirements.txt diff --git a/import_helper_base/__manifest__.py b/import_helper_base/__manifest__.py index f215e46..4ab6e54 100644 --- a/import_helper_base/__manifest__.py +++ b/import_helper_base/__manifest__.py @@ -3,20 +3,20 @@ # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). { - 'name': 'Import Helper Base', - 'version': '16.0.1.0.0', - 'category': 'Extra Tools', - 'license': 'AGPL-3', - 'summary': 'Common code for all import helper modules', - 'author': 'Akretion', - 'website': 'https://github.com/akretion/odoo-import-helper', - 'depends': [ - 'base', - ], - "external_dependencies": {"python" : ["pycountry", "openai"]}, - 'data': [ - 'security/ir.model.access.csv', - 'wizards/import_helper_view.xml', - ], - 'installable': True, + "name": "Import Helper Base", + "version": "16.0.1.0.0", + "category": "Extra Tools", + "license": "AGPL-3", + "summary": "Common code for all import helper modules", + "author": "Akretion", + "website": "https://github.com/Akretion/odoo-import-helper", + "depends": [ + "base", + ], + "external_dependencies": {"python": ["pycountry", "openai"]}, + "data": [ + "security/ir.model.access.csv", + "wizards/import_helper_view.xml", + ], + "installable": True, } diff --git a/import_helper_base/pyproject.toml b/import_helper_base/pyproject.toml new file mode 100644 index 0000000..4231d0c --- /dev/null +++ b/import_helper_base/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["whool"] +build-backend = "whool.buildapi" diff --git a/import_helper_base/tests/test_import_helper.py b/import_helper_base/tests/test_import_helper.py index cd0a0f6..2fe06e7 100644 --- a/import_helper_base/tests/test_import_helper.py +++ b/import_helper_base/tests/test_import_helper.py @@ -6,7 +6,6 @@ class TestBaseImportHelper(TransactionCase): - @classmethod def setUpClass(cls): super().setUpClass() @@ -14,20 +13,32 @@ def setUpClass(cls): # create data def test_match_country(self): - iho = self.env['import.helper'] + iho = self.env["import.helper"] speedy = iho._prepare_speedy() country_id = iho._match_country( - {"country_name": "fr"}, "country_name", "res.partner", "country_id", speedy) - self.assertEqual(country_id, self.env.ref('base.fr').id) + {"country_name": "fr"}, "country_name", "res.partner", "country_id", speedy + ) + self.assertEqual(country_id, self.env.ref("base.fr").id) country_id = iho._match_country( - {"country_name": "FRA"}, "country_name", "res.partner", "country_id", speedy) - self.assertEqual(country_id, self.env.ref('base.fr').id) + {"country_name": "FRA"}, "country_name", "res.partner", "country_id", speedy + ) + self.assertEqual(country_id, self.env.ref("base.fr").id) country_id = iho._match_country( - {"country_name": "France"}, "country_name", "res.partner", "country_id", speedy) - self.assertEqual(country_id, self.env.ref('base.fr').id) + {"country_name": "France"}, + "country_name", + "res.partner", + "country_id", + speedy, + ) + self.assertEqual(country_id, self.env.ref("base.fr").id) country_id = iho._match_country( - {"country_name": "U.S.A."}, "country_name", "res.partner", "country_id", speedy) - self.assertEqual(country_id, self.env.ref('base.us').id) + {"country_name": "U.S.A."}, + "country_name", + "res.partner", + "country_id", + speedy, + ) + self.assertEqual(country_id, self.env.ref("base.us").id) # country_id = iho._match_country( # {"country_name": "España"}, "country_name", "res.partner", "country_id", speedy) # self.assertEqual(country_id, self.env.ref('base.es').id) diff --git a/import_helper_base/wizards/import_helper.py b/import_helper_base/wizards/import_helper.py index a6ca541..b2fb2f1 100644 --- a/import_helper_base/wizards/import_helper.py +++ b/import_helper_base/wizards/import_helper.py @@ -2,24 +2,26 @@ # @author: Alexis de Lattre # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). -from odoo import api, fields, models, tools, _ -from odoo.exceptions import UserError +import logging +import re from collections import defaultdict from datetime import datetime + from unidecode import unidecode -import re -import logging +from odoo import _, api, fields, models, tools +from odoo.exceptions import UserError + logger = logging.getLogger(__name__) try: import pycountry except ImportError: - logger.debug('Cannot import pycountry') + logger.debug("Cannot import pycountry") try: from openai import OpenAI except ImportError: - logger.debug('Cannot import openai') + logger.debug("Cannot import openai") class ImportHelper(models.TransientModel): @@ -29,65 +31,72 @@ class ImportHelper(models.TransientModel): logs = fields.Html(readonly=True) @api.model - def _prepare_speedy(self, aiengine='chatgpt'): - logger.debug('Start to prepare import speedy') + def _prepare_speedy(self, aiengine="chatgpt"): + logger.debug("Start to prepare import speedy") speedy = { # country is used both for partner and product - 'country': { - 'name2code': { + "country": { + "name2code": { "usa": "US", "etatsunis": "US", "grandebretagne": "GB", "angleterre": "GB", - }, - 'code2id': {}, - 'id2code': {}, # used to check iban and vat number prefixes - 'code2name': {}, # used in log messages }, - 'aiengine': aiengine, - 'field2label': {}, - 'logs': {}, - # 'logs' is a dict {'res.partner': [], 'product.product': []} - # where the value is a list of dict : - # {'msg': 'Checksum IBAN wrong', - # 'value': 'FR9879834739', - # 'vals': vals, # used to get the line - # (and display_name if partner has been created) - # 'field': 'res.partner,email', - # 'reset': True, # True if the data is NOT imported in Odoo - # } + "code2id": {}, + "id2code": {}, # used to check iban and vat number prefixes + "code2name": {}, # used in log messages + }, + "aiengine": aiengine, + "field2label": {}, + "logs": {}, + # 'logs' is a dict {'res.partner': [], 'product.product': []} + # where the value is a list of dict : + # {'msg': 'Checksum IBAN wrong', + # 'value': 'FR9879834739', + # 'vals': vals, # used to get the line + # (and display_name if partner has been created) + # 'field': 'res.partner,email', + # 'reset': True, # True if the data is NOT imported in Odoo + # } } - cyd = speedy['country'] + cyd = speedy["country"] code2to3 = {} for country in pycountry.countries: code2to3[country.alpha_2] = country.alpha_3 - for country in self.env['res.country'].search_read([], ['code', 'name']): - cyd['code2id'][country['code']] = country['id'] - cyd['id2code'][country['id']] = country['code'] - cyd['code2name'][country['code']] = country['name'] - code3 = code2to3.get(country['code']) + for country in self.env["res.country"].search_read([], ["code", "name"]): + cyd["code2id"][country["code"]] = country["id"] + cyd["id2code"][country["id"]] = country["code"] + cyd["code2name"][country["code"]] = country["name"] + code3 = code2to3.get(country["code"]) if code3: - cyd['code2id'][code3] = country['id'] - cyd['code2name'][code3] = country['name'] - for lang in self.env['res.lang'].search([]): - logger.info('Working on lang %s', lang.code) - for country in self.env['res.country'].with_context(lang=lang.code).search_read([], ['code', 'name']): - country_name_match = self._prepare_country_name_match(country['name']) - cyd['name2code'][country_name_match] = country['code'] - if aiengine == 'chatgpt': - openai_api_key = tools.config.get('openai_api_key', False) + cyd["code2id"][code3] = country["id"] + cyd["code2name"][code3] = country["name"] + for lang in self.env["res.lang"].search([]): + logger.info("Working on lang %s", lang.code) + for country in ( + self.env["res.country"] + .with_context(lang=lang.code) + .search_read([], ["code", "name"]) + ): + country_name_match = self._prepare_country_name_match(country["name"]) + cyd["name2code"][country_name_match] = country["code"] + if aiengine == "chatgpt": + openai_api_key = tools.config.get("openai_api_key", False) if not openai_api_key: - raise UserError(_( - "Missing entry openai_api_key in the Odoo server configuration file.")) - speedy['openai_client'] = OpenAI(api_key=openai_api_key) - speedy['openai_tokens'] = 0 + raise UserError( + _( + "Missing entry openai_api_key in the Odoo server configuration file." + ) + ) + speedy["openai_client"] = OpenAI(api_key=openai_api_key) + speedy["openai_tokens"] = 0 return speedy @api.model def _prepare_country_name_match(self, country_name): assert country_name country_name_match = unidecode(country_name).lower() - country_name_match = ''.join(re.findall(r'[a-z]+', country_name_match)) + country_name_match = "".join(re.findall(r"[a-z]+", country_name_match)) assert country_name_match return country_name_match @@ -95,28 +104,39 @@ def _match_country(self, vals, country_key, model, country_field_name, speedy): assert isinstance(model, str) country_name = vals[country_key] log = { - 'value': country_name, - 'vals': vals, - 'field': f'{model},{country_field_name}', - } - cyd = speedy['country'] + "value": country_name, + "vals": vals, + "field": f"{model},{country_field_name}", + } + cyd = speedy["country"] if len(country_name) in (2, 3): country_code = country_name.upper() - if country_code in cyd['code2id']: - logger.info("Country name '%s' is an ISO country code (%s)", country_name, cyd['code2name'][country_code]) - country_id = cyd['code2id'][country_code] + if country_code in cyd["code2id"]: + logger.info( + "Country name '%s' is an ISO country code (%s)", + country_name, + cyd["code2name"][country_code], + ) + country_id = cyd["code2id"][country_code] return country_id country_name_match = self._prepare_country_name_match(country_name) - if country_name_match in cyd['name2code']: - country_code = cyd['name2code'][country_name_match] - logger.info("Country '%s' matched on country %s (%s)", country_name, cyd['code2name'][country_code], country_code) - country_id = cyd['code2id'][country_code] + if country_name_match in cyd["name2code"]: + country_code = cyd["name2code"][country_name_match] + logger.info( + "Country '%s' matched on country %s (%s)", + country_name, + cyd["code2name"][country_code], + country_code, + ) + country_id = cyd["code2id"][country_code] return country_id - logger.info("No direct match for country '%s': now asking ChatGPT.", country_name) + logger.info( + "No direct match for country '%s': now asking ChatGPT.", country_name + ) # ask ChatGPT ! content = """ISO country code of "%s", nothing else""" % country_name - logger.debug('ChatGPT question: %s', content) - chat_completion = speedy['openai_client'].chat.completions.create( + logger.debug("ChatGPT question: %s", content) + chat_completion = speedy["openai_client"].chat.completions.create( model="gpt-3.5-turbo", messages=[{"role": "user", "content": content}], temperature=0, @@ -129,120 +149,173 @@ def _match_country(self, vals, country_key, model, country_field_name, speedy): answer = chat_completion.choices[0].message.content if answer: answer = answer.strip() - logger.info('ChatGPT answer: %s', answer) + logger.info("ChatGPT answer: %s", answer) if len(answer) == 2: country_code = answer.upper() - if country_code in cyd['code2id']: - logger.info("ChatGPT matched country '%s' to %s (%s)", country_name, cyd['code2name'][country_code], country_code) - speedy['logs'][model].append(dict(log, msg="Country name could not be found in Odoo. ChatGPT said ISO code was '%s', which matched to '%s'" % (country_code, cyd['code2name'][country_code]))) - country_id = cyd['code2id'][country_code] - cyd['name2code'][country_name_match] = country_code + if country_code in cyd["code2id"]: + logger.info( + "ChatGPT matched country '%s' to %s (%s)", + country_name, + cyd["code2name"][country_code], + country_code, + ) + speedy["logs"][model].append( + dict( + log, + msg="Country name could not be found in Odoo. ChatGPT said ISO code was '%s', which matched to '%s'" + % (country_code, cyd["code2name"][country_code]), + ) + ) + country_id = cyd["code2id"][country_code] + cyd["name2code"][country_name_match] = country_code return country_id else: - speedy['logs'][model].append(dict(log, msg="Country name could not be found in Odoo. ChatGPT said ISO code was '%s', which didn't match to any country" % country_code), reset=True) + speedy["logs"][model].append( + dict( + log, + msg="Country name could not be found in Odoo. ChatGPT said ISO code was '%s', which didn't match to any country" + % country_code, + ), + reset=True, + ) else: - speedy['logs'][model].append( - dict(log, msg="ChatGPT didn't answer a 2 letter country code but '%s'" % answer, reset=True)) + speedy["logs"][model].append( + dict( + log, + msg="ChatGPT didn't answer a 2 letter country code but '%s'" + % answer, + reset=True, + ) + ) else: - logger.warning('No answer from chatGPT') - speedy['logs'][model].append(dict(log, msg='No answer from chatGPT', reset=True)) + logger.warning("No answer from chatGPT") + speedy["logs"][model].append( + dict(log, msg="No answer from chatGPT", reset=True) + ) return False def _field_label(self, field, speedy): - if field not in speedy['field2label']: - field_split = field.split(',') - ofield = self.env['ir.model.fields'].search([ - ('model', '=', field_split[0]), - ('name', '=', field_split[1]), - ], limit=1) + if field not in speedy["field2label"]: + field_split = field.split(",") + ofield = self.env["ir.model.fields"].search( + [ + ("model", "=", field_split[0]), + ("name", "=", field_split[1]), + ], + limit=1, + ) if ofield: - speedy['field2label'][field] = ofield.field_description + speedy["field2label"][field] = ofield.field_description else: - speedy['field2label'][field] = '%s (%s)' % ( - field_split[1], field_split[0]) - return speedy['field2label'][field] + speedy["field2label"][field] = "%s (%s)" % ( + field_split[1], + field_split[0], + ) + return speedy["field2label"][field] def _convert_logs2html(self, speedy): html = '

    For the logs in red, the data was not imported in Odoo
    ' - if speedy.get('aiengine') == 'chatgpt': - html += '%d OpenAI tokens where used

    ' % speedy['openai_tokens'] - for obj_name, log_list in speedy['logs'].items(): - obj_rec = self.env['ir.model'].search([('model', '=', obj_name)], limit=1) + if speedy.get("aiengine") == "chatgpt": + html += ( + "%d OpenAI tokens where used

    " + % speedy["openai_tokens"] + ) + for obj_name, log_list in speedy["logs"].items(): + obj_rec = self.env["ir.model"].search([("model", "=", obj_name)], limit=1) assert obj_rec html += '

    %s

    ' % obj_rec.name line2logs = defaultdict(list) field2logs = defaultdict(list) for log in log_list: - if log['vals'].get('line'): - line2logs[log['vals']['line']].append(log) - if log.get('field'): - field2logs[log['field']].append(log) + if log["vals"].get("line"): + line2logs[log["vals"]["line"]].append(log) + if log.get("field"): + field2logs[log["field"]].append(log) html += '

    Logs per line

    ' for line, logs in line2logs.items(): log_labels = [] for log in logs: log_labels.append( - '
  • %s: %s - %s
    • ' % ( - log.get('reset') and 'red' or 'black', - self._field_label(log['field'], speedy), - log['value'], - log['msg'], - )) - h3 = 'Line %s' % line - if log['vals'].get('id'): - h3 += ': %s (ID %d)' % (log['vals']['display_name'], log['vals']['id']) - html += '

    %s

    \n

      %s

    ' % (h3, '\n'.join(log_labels)) + '
  • %s: %s - %s
    • ' + % ( + log.get("reset") and "red" or "black", + self._field_label(log["field"], speedy), + log["value"], + log["msg"], + ) + ) + h3 = "Line %s" % line + if log["vals"].get("id"): + h3 += ": %s (ID %d)" % ( + log["vals"]["display_name"], + log["vals"]["id"], + ) + html += "

    %s

    \n

      %s

    " % (h3, "\n".join(log_labels)) html += '

    Logs per field

    ' for field, logs in field2logs.items(): log_labels = [] for log in logs: - line_label = 'Line %s' % log['vals'].get('line', 'unknown') - if log['vals'].get('id'): - line_label += ' (%s ID %d)' % (log['vals']['display_name'], log['vals']['id']) + line_label = "Line %s" % log["vals"].get("line", "unknown") + if log["vals"].get("id"): + line_label += " (%s ID %d)" % ( + log["vals"]["display_name"], + log["vals"]["id"], + ) log_labels.append( - '
  • %s: %s - %s
    • ' % ( - log.get('reset') and 'red' or 'black', + '
  • %s: %s - %s
    • ' + % ( + log.get("reset") and "red" or "black", line_label, - log['value'], - log['msg'], - )) - html += '

    %s

    \n

      %s

    ' % ( - self._field_label(field, speedy), '\n'.join(log_labels)) + log["value"], + log["msg"], + ) + ) + html += "

    %s

    \n

      %s

    " % ( + self._field_label(field, speedy), + "\n".join(log_labels), + ) return html def _result_action(self, speedy): action = { - 'name': 'Result', - 'type': 'ir.actions.act_window', - 'res_model': 'import.helper', - 'view_mode': 'form', - 'target': 'new', - 'context': dict(self._context, default_logs=self._convert_logs2html(speedy)), - } + "name": "Result", + "type": "ir.actions.act_window", + "res_model": "import.helper", + "view_mode": "form", + "target": "new", + "context": dict( + self._context, default_logs=self._convert_logs2html(speedy) + ), + } return action def _prepare_create_date(self, vals, speedy): - create_date = vals.get('create_date') + create_date = vals.get("create_date") create_date_dt = False if isinstance(create_date, str) and len(create_date) == 10: try: - create_date_dt = datetime.strptime(create_date, '%Y-%m-%d') + create_date_dt = datetime.strptime(create_date, "%Y-%m-%d") except Exception as e: - speedy['logs'].append({ - 'msg': "Failed to convert '%s' to datetime: %s" % (create_date, e), - 'value': vals['create_date'], - 'vals': vals, - 'field': 'product.product,create_date', - 'reset': True, - }) + speedy["logs"].append( + { + "msg": "Failed to convert '%s' to datetime: %s" + % (create_date, e), + "value": vals["create_date"], + "vals": vals, + "field": "product.product,create_date", + "reset": True, + } + ) elif isinstance(create_date, datetime): create_date_dt = create_date if create_date_dt and create_date_dt.date() > fields.Date.context_today(self): - speedy['logs'].append({ - 'msg': 'create_date %s cannot be in the future' % create_date_dt, - 'value': create_date, - 'vals': vals, - 'field': 'product.product,create_date', - 'reset': True, - }) + speedy["logs"].append( + { + "msg": "create_date %s cannot be in the future" % create_date_dt, + "value": create_date, + "vals": vals, + "field": "product.product,create_date", + "reset": True, + } + ) return create_date_dt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..201e17f --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +# generated from manifests external_dependencies +openai +pycountry From d3f74ec217d8aaeea46366e156151dddfca67049 Mon Sep 17 00:00:00 2001 From: David Beal Date: Tue, 19 Nov 2024 10:54:26 +0100 Subject: [PATCH 6/6] [MIG] import_helper_base 18.0 --- import_helper_base/__manifest__.py | 4 +- .../tests/test_import_helper.py | 3 +- import_helper_base/wizards/import_helper.py | 83 ++++++++++--------- requirements.txt | 1 + 4 files changed, 49 insertions(+), 42 deletions(-) diff --git a/import_helper_base/__manifest__.py b/import_helper_base/__manifest__.py index 4ab6e54..e42c92b 100644 --- a/import_helper_base/__manifest__.py +++ b/import_helper_base/__manifest__.py @@ -4,7 +4,7 @@ { "name": "Import Helper Base", - "version": "16.0.1.0.0", + "version": "18.0.1.0.0", "category": "Extra Tools", "license": "AGPL-3", "summary": "Common code for all import helper modules", @@ -13,7 +13,7 @@ "depends": [ "base", ], - "external_dependencies": {"python": ["pycountry", "openai"]}, + "external_dependencies": {"python": ["pycountry", "openai", "unidecode"]}, "data": [ "security/ir.model.access.csv", "wizards/import_helper_view.xml", diff --git a/import_helper_base/tests/test_import_helper.py b/import_helper_base/tests/test_import_helper.py index 2fe06e7..52788d8 100644 --- a/import_helper_base/tests/test_import_helper.py +++ b/import_helper_base/tests/test_import_helper.py @@ -40,5 +40,6 @@ def test_match_country(self): ) self.assertEqual(country_id, self.env.ref("base.us").id) # country_id = iho._match_country( - # {"country_name": "España"}, "country_name", "res.partner", "country_id", speedy) + # {"country_name": "España"}, "country_name", "res.partner", + # "country_id", speedy) # self.assertEqual(country_id, self.env.ref('base.es').id) diff --git a/import_helper_base/wizards/import_helper.py b/import_helper_base/wizards/import_helper.py index b2fb2f1..8ce0fae 100644 --- a/import_helper_base/wizards/import_helper.py +++ b/import_helper_base/wizards/import_helper.py @@ -85,7 +85,8 @@ def _prepare_speedy(self, aiengine="chatgpt"): if not openai_api_key: raise UserError( _( - "Missing entry openai_api_key in the Odoo server configuration file." + "Missing entry openai_api_key in the Odoo server " + "configuration file." ) ) speedy["openai_client"] = OpenAI(api_key=openai_api_key) @@ -134,7 +135,7 @@ def _match_country(self, vals, country_key, model, country_field_name, speedy): "No direct match for country '%s': now asking ChatGPT.", country_name ) # ask ChatGPT ! - content = """ISO country code of "%s", nothing else""" % country_name + content = f"""ISO country code of "{country_name}", nothing else""" logger.debug("ChatGPT question: %s", content) chat_completion = speedy["openai_client"].chat.completions.create( model="gpt-3.5-turbo", @@ -153,17 +154,21 @@ def _match_country(self, vals, country_key, model, country_field_name, speedy): if len(answer) == 2: country_code = answer.upper() if country_code in cyd["code2id"]: + cyd_code2n_cntry = cyd["code2name"][country_code] logger.info( "ChatGPT matched country '%s' to %s (%s)", country_name, - cyd["code2name"][country_code], + cyd_code2n_cntry, country_code, ) speedy["logs"][model].append( dict( log, - msg="Country name could not be found in Odoo. ChatGPT said ISO code was '%s', which matched to '%s'" - % (country_code, cyd["code2name"][country_code]), + msg=( + "Country name could not be found in Odoo. " + f"ChatGPT said ISO code was '{country_code}', " + f"which matched to '{cyd_code2n_cntry}'" + ), ) ) country_id = cyd["code2id"][country_code] @@ -173,8 +178,11 @@ def _match_country(self, vals, country_key, model, country_field_name, speedy): speedy["logs"][model].append( dict( log, - msg="Country name could not be found in Odoo. ChatGPT said ISO code was '%s', which didn't match to any country" - % country_code, + msg=( + "Country name could not be found in Odoo. " + f"ChatGPT said ISO code was '{country_code}', " + "which didn't match to any country" + ), ), reset=True, ) @@ -182,8 +190,8 @@ def _match_country(self, vals, country_key, model, country_field_name, speedy): speedy["logs"][model].append( dict( log, - msg="ChatGPT didn't answer a 2 letter country code but '%s'" - % answer, + msg="ChatGPT didn't answer a 2 letter country code " + f"but '{answer}'", reset=True, ) ) @@ -207,23 +215,22 @@ def _field_label(self, field, speedy): if ofield: speedy["field2label"][field] = ofield.field_description else: - speedy["field2label"][field] = "%s (%s)" % ( - field_split[1], - field_split[0], - ) + speedy["field2label"][field] = f"{field_split[1]} ({field_split[0]})" return speedy["field2label"][field] def _convert_logs2html(self, speedy): - html = '

    For the logs in red, the data was not imported in Odoo
    ' + html = ( + '

    For the logs in red, ' + "the data was not imported in Odoo
    " + ) if speedy.get("aiengine") == "chatgpt": - html += ( - "%d OpenAI tokens where used

    " - % speedy["openai_tokens"] + html += "{} OpenAI tokens where used

    ".format( + speedy["openai_tokens"] ) for obj_name, log_list in speedy["logs"].items(): obj_rec = self.env["ir.model"].search([("model", "=", obj_name)], limit=1) assert obj_rec - html += '

    %s

    ' % obj_rec.name + html += f'

    {obj_rec.name}

    ' line2logs = defaultdict(list) field2logs = defaultdict(list) for log in log_list: @@ -236,43 +243,42 @@ def _convert_logs2html(self, speedy): log_labels = [] for log in logs: log_labels.append( - '
  • %s: %s - %s
    • ' - % ( + '
  • {}: {} - {}
    • '.format( log.get("reset") and "red" or "black", self._field_label(log["field"], speedy), log["value"], log["msg"], ) ) - h3 = "Line %s" % line + h3 = f"Line {line}" if log["vals"].get("id"): - h3 += ": %s (ID %d)" % ( - log["vals"]["display_name"], - log["vals"]["id"], - ) - html += "

    %s

    \n

      %s

    " % (h3, "\n".join(log_labels)) + h3 += f": {log['vals']['display_name']} (ID {log['vals']['id']})" + html += "

    {}

    \n

      {}

    ".format( + h3, "\n".join(log_labels) + ) html += '

    Logs per field

    ' for field, logs in field2logs.items(): log_labels = [] for log in logs: - line_label = "Line %s" % log["vals"].get("line", "unknown") + line_label = f"Line {log['vals'].get('line', 'unknown')}" if log["vals"].get("id"): - line_label += " (%s ID %d)" % ( - log["vals"]["display_name"], - log["vals"]["id"], - ) + line_label += " (%(display_name)s ID %(id)d)" % { + "display_name": log["vals"]["display_name"], + "id": log["vals"]["id"], + } log_labels.append( - '
  • %s: %s - %s
    • ' - % ( + ( + '
  • {}: ' "{} - {}
    • " + ).format( log.get("reset") and "red" or "black", line_label, log["value"], log["msg"], ) ) - html += "

    %s

    \n

      %s

    " % ( - self._field_label(field, speedy), - "\n".join(log_labels), + html += ( + f"

    {self._field_label(field, speedy)}

    \n

      " + "{'\n'.join(log_labels)}

    " ) return html @@ -298,8 +304,7 @@ def _prepare_create_date(self, vals, speedy): except Exception as e: speedy["logs"].append( { - "msg": "Failed to convert '%s' to datetime: %s" - % (create_date, e), + "msg": f"Failed to convert '{create_date}' to datetime: {e}", "value": vals["create_date"], "vals": vals, "field": "product.product,create_date", @@ -311,7 +316,7 @@ def _prepare_create_date(self, vals, speedy): if create_date_dt and create_date_dt.date() > fields.Date.context_today(self): speedy["logs"].append( { - "msg": "create_date %s cannot be in the future" % create_date_dt, + "msg": f"create_date {create_date_dt} cannot be in the future", "value": create_date, "vals": vals, "field": "product.product,create_date", diff --git a/requirements.txt b/requirements.txt index 201e17f..54bcb3c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ # generated from manifests external_dependencies openai pycountry +unidecode