From a38689fa5d455a3624d863b49ac2b993209e97a2 Mon Sep 17 00:00:00 2001 From: Doug Mills <110824173+dougmills-DIT@users.noreply.github.com> Date: Wed, 10 Jan 2024 14:33:34 +0000 Subject: [PATCH] Tp2000 819 importer refactor (#1015) * initial new parser commit * add new parsers and process for getting info from xml to dictionary for message * updated to using subclasses to find all parsers, then populate the object from the data provided * updated / added new parsers for all objects * updated parsers and linked to handlers (for dependency data, may refactor) * added validation + working on reporting issues * TP2000-844 Add suffix and indent columns to find commodities view (#885) * Add suffix and indent column * Update test * Show indent as at today * Update test * Refactor * Use optional type hint * TP2000-738 Display rule check progress (#887) * Display rule check progress in UI * Add test * Add test docstring * TP2000-849 Add active state filter to quota search (#886) * Add active state filter * Add test * Test invalid active state filter option * Rename fixture variable * TP2000-741 ME32 & ME16 rules checker failing (#874) * TP2000-477 Prepopulate geo exclusions (#889) * Pre-populate geo exclusions formset * Add conditions initial data, update tests * Fix tests * Fix tests * Add test for formset_add_or_delete * Add erga omnes prefix * Use regex * Tp2000 762 virus checker (#888) * adding docker network * virus check validator * skip virus check localy * removing variable * validator migration * PR comments and handler update * updating to make it obvious creds aren't real * removing dev file upoad handler * testing defaults * updated local config * TP2000-806 get envelope history (#890) * TP2000-860 Sqlite dump and upload failing (#892) * Skip migration during sqlite database creation. * Run Sqlite export process during out of hours. * updated exclusion mapping list as country has none (#895) * TP2000-861 Fix footnote delete button on measure edit view (#894) * Fix delete button * Add trailing slash to auto-generated URLs * Order indents to ensure access to latest version. (#897) * updating link data (WIP) * updating link data (WIP) * updating link data (WIP) * set parent import model (WIP) * set parent import model (WIP) * WIP - adding tests with example envelopes for the new importer * wip * wip * initial new parser commit * add new parsers and process for getting info from xml to dictionary for message * updated to using subclasses to find all parsers, then populate the object from the data provided * updated / added new parsers for all objects * updated parsers and linked to handlers (for dependency data, may refactor) * added validation + working on reporting issues * updating link data (WIP) * updating link data (WIP) * updating link data (WIP) * set parent import model (WIP) * set parent import model (WIP) * WIP - adding tests with example envelopes for the new importer * wip * wip * test updates * test updates * test updates * test updates * test updates * test updates * test updates * test updates * test updates * test updates * test updates * test updates * test updates * test updates * building out some UI for the new importer * building out some UI for the new importer * building out some UI for the new importer * building out some UI for the new importer * adding identity fields to all parsers * fix flaky tests * testing updates * testing updates * testing updates - finished update tests for all TARIC objects we use currently * testing updates - added add codes delete tests * testing updates - added add codes delete tests * testing updates - added certificate, commodity, footnote, geoarea, and measure delete tests * moved new parser / importer to its own app * finished adding delete tests and removed unused methods * added documentation * added documentation + small fixes * added documentation + small fixes * added documentation + small fixes * update comm code import objects * added documentation + small fixes * added documentation + small fixes * added documentation + small fixes * minor updates based on PR comments * minor updates based on PR comments * minor updates based on PR comments * minor updates based on PR comments * minor updates based on PR comments * minor updates based on PR comments * test fix for py 3.8 * test fix for py 3.8 * tidy up chunker a little and add tests * duplicated a bunch of files referenced in the importer app - anything thats not a model now also lives in the taric_parsers app. * duplicated a bunch of files referenced in the importer app - anything that's not a model now also lives in the taric_parsers app. * minor changes from PR review * updated issue_type for import error creation to a text choice * added tests to UI elements * altered ot absolute imports * altered exception raised on TaricXmlSourceBase * import tidy up - absolute imports * minor updates * minor updates * minor updates * minor updates * minor updates * minor updates * minor updates * minor updates * Updating documentation + PR feedback * Updating documentation + PR feedback * updates based on PR feedback * updates based on PR feedback * updates based on PR feedback * updates based on PR feedback * updates based on PR feedback * updates based on PR feedback * updates based on PR feedback * updates based on PR feedback * updates based on PR feedback * updates based on PR feedback * updates based on PR feedback * updates based on PR feedback * updates based on PR feedback * update pre-commit-hook format, was showing as invalid yaml - no functional change - just formatting * updates to readme, adding pre-commit info * updates to readme, adding pre-commit info * updates to readme, adding pre-commit info * updates to readme, adding pre-commit info * updates to readme, adding pre-commit info * rename to remove New prefix * rename to remove New prefix * PR updates * PR updates * PR updates * PR updates - adding in importerv2 to main comm code UI * PR updates : integrating UI - mid progress * minor UI update * fix unit tests mid way * interim commit * interim commit * interim commit * interim commit * fixed tests * fixed tests * fixed tests * add in missing migration * update NIG5 to not trigger when goods nomenclature also being deleted in the same transaction --------- Co-authored-by: Dale Cannon <118175145+dalecannon@users.noreply.github.com> Co-authored-by: Paul Pepper <85895113+paulpepper-trade@users.noreply.github.com> Co-authored-by: Edie Pearce Co-authored-by: Anthoni Gleeson --- .pre-commit-config.yaml | 12 +- README.rst | 89 ++ commodities/business_rules.py | 11 + commodities/models/dc.py | 44 +- .../tests/business_rules/test_NIG5_origin.py | 72 ++ commodities/tests/test_migrations.py | 13 +- common/jinja2/layouts/layout.jinja | 6 +- common/tests/test_views.py | 2 +- common/tests/util.py | 65 +- importer/chunker.py | 5 +- importer/forms.py | 103 +- importer/handlers.py | 77 +- importer/jinja2/eu-importer/details.jinja | 42 + .../jinja2/eu-importer/select-imports.jinja | 1 - .../jinja2/includes/taric_importer_list.jinja | 6 +- importer/migrations/0012_batchimporterror.py | 70 ++ .../0013_alter_importbatch_status.py | 29 + importer/models.py | 196 +++- importer/namespaces.py | 22 +- importer/tasks.py | 28 +- importer/tests/conftest.py | 6 +- importer/tests/test_chunker.py | 4 +- .../test_files/additional_code_CREATE.xml | 55 + importer/tests/test_forms.py | 30 +- importer/tests/test_nursery.py | 14 +- importer/tests/test_views.py | 1 + importer/urls.py | 5 + importer/validators.py | 13 + importer/views.py | 17 + pii-secret-exclude.txt | 1 + pyproject.toml | 4 +- reports/tests/test_report_utils.py | 5 +- settings/common.py | 12 + taric_parsers/__init__.py | 0 taric_parsers/admin.py | 1 + taric_parsers/apps.py | 5 + taric_parsers/chunker.py | 256 ++++ taric_parsers/forms.py | 125 ++ taric_parsers/importer.py | 1008 ++++++++++++++++ taric_parsers/importer_issue.py | 59 + .../jinja2/includes/importer_list.jinja | 24 + .../jinja2/includes/taric_importer_list.jinja | 55 + .../jinja2/taric_parser/create.jinja | 25 + .../jinja2/taric_parser/details.jinja | 42 + .../jinja2/taric_parser/importer_list.jinja | 31 + taric_parsers/jinja2/taric_parser/list.jinja | 7 + taric_parsers/migrations/__init__.py | 0 taric_parsers/namespaces.py | 185 +++ taric_parsers/parser_model_link.py | 23 + .../parsers/additional_code_parsers.py | 266 +++++ taric_parsers/parsers/certificate_parser.py | 191 +++ taric_parsers/parsers/commodity_parser.py | 345 ++++++ taric_parsers/parsers/footnote_parser.py | 189 +++ taric_parsers/parsers/geo_area_parser.py | 182 +++ taric_parsers/parsers/measure_parser.py | 1037 +++++++++++++++++ taric_parsers/parsers/mixins.py | 39 + taric_parsers/parsers/quota_parser.py | 575 +++++++++ taric_parsers/parsers/regulation_parser.py | 314 +++++ taric_parsers/parsers/taric_parser.py | 833 +++++++++++++ taric_parsers/taric_xml_source.py | 20 + taric_parsers/tasks.py | 172 +++ taric_parsers/tests/__init__.py | 0 .../additional_code_CREATE.xml | 55 + .../additional_code_DELETE.xml | 22 + .../additional_code_UPDATE.xml | 22 + .../additional_code_description_CREATE.xml | 92 ++ .../additional_code_description_DELETE.xml | 24 + .../additional_code_description_UPDATE.xml | 24 + ...ription_invalid_additional_code_CREATE.xml | 24 + ...itional_code_description_period_CREATE.xml | 92 ++ ...itional_code_description_period_DELETE.xml | 23 + ...itional_code_description_period_UPDATE.xml | 23 + ...tion_period_without_description_CREATE.xml | 57 + .../additional_code_invalid_type_CREATE.xml | 22 + .../additional_code_type_CREATE.xml | 39 + .../additional_code_type_DELETE.xml | 23 + .../additional_code_type_UPDATE.xml | 23 + ...dditional_code_type_description_CREATE.xml | 38 + ...dditional_code_type_description_DELETE.xml | 21 + ...dditional_code_type_description_UPDATE.xml | 21 + ...e_type_description_without_type_CREATE.xml | 21 + ...tional_code_type_no_description_CREATE.xml | 22 + ...ote_association_additional_code_CREATE.xml | 135 +++ ...ote_association_additional_code_DELETE.xml | 27 + ...ote_association_additional_code_UPDATE.xml | 27 + ...dditional_code_invalid_footnote_CREATE.xml | 81 ++ ...test_additional_code_description_parser.py | 164 +++ ...ditional_code_description_period_parser.py | 166 +++ .../test_additional_code_parser.py | 140 +++ ...additional_code_type_description_parser.py | 138 +++ .../test_additional_code_type_parser.py | 133 +++ ...note_association_additional_code_parser.py | 161 +++ .../importer_examples/certificate_CREATE.xml | 54 + .../importer_examples/certificate_DELETE.xml | 22 + .../importer_examples/certificate_UPDATE.xml | 22 + .../certificate_description_CREATE.xml | 90 ++ .../certificate_description_DELETE.xml | 23 + .../certificate_description_UPDATE.xml | 24 + .../certificate_description_period_CREATE.xml | 89 ++ .../certificate_description_period_DELETE.xml | 22 + .../certificate_description_period_UPDATE.xml | 22 + .../certificate_type_CREATE.xml | 38 + .../certificate_type_DELETE.xml | 22 + .../certificate_type_UPDATE.xml | 21 + .../certificate_type_description_CREATE.xml | 37 + .../certificate_type_description_DELETE.xml | 21 + .../certificate_type_description_UPDATE.xml | 21 + .../test_certificate_description_parser.py | 108 ++ ...t_certificate_description_period_parser.py | 109 ++ .../test_certificate_parser.py | 108 ++ ...est_certificate_type_description_parser.py | 95 ++ .../test_certificate_type_parser.py | 94 ++ ..._association_goods_nomenclature_CREATE.xml | 98 ++ ..._association_goods_nomenclature_DELETE.xml | 25 + ..._association_goods_nomenclature_UPDATE.xml | 25 + ..._goods_nomenclature_no_footnote_CREATE.xml | 80 ++ .../goods_nomenclature_CREATE.xml | 23 + .../goods_nomenclature_CREATE_then_DELETE.xml | 41 + .../goods_nomenclature_DELETE.xml | 23 + .../goods_nomenclature_UPDATE.xml | 23 + .../goods_nomenclature_description_DELETE.xml | 24 + .../goods_nomenclature_description_UPDATE.xml | 24 + ...enclature_description_no_period_CREATE.xml | 44 + ...enclature_description_no_period_UPDATE.xml | 24 + ...nomenclature_description_period_UPDATE.xml | 82 ++ ...lature_description_period_multi_CREATE.xml | 119 ++ ...clature_description_period_only_DELETE.xml | 23 + ...clature_description_period_only_UPDATE.xml | 23 + ...clature_description_with_period_CREATE.xml | 62 + .../goods_nomenclature_indent_CREATE.xml | 44 + .../goods_nomenclature_indent_DELETE.xml | 24 + .../goods_nomenclature_indent_UPDATE.xml | 24 + .../goods_nomenclature_origin_CREATE.xml | 77 ++ .../goods_nomenclature_origin_UPDATE.xml | 23 + .../goods_nomenclature_successor_CREATE.xml | 77 ++ .../goods_nomenclature_successor_DELETE.xml | 23 + .../goods_nomenclature_successor_UPDATE.xml | 23 + ...e_association_goods_nomenclature_parser.py | 166 +++ ...t_goods_nomenclature_description_parser.py | 210 ++++ ..._nomenclature_description_period_parser.py | 142 +++ .../test_goods_nomenclature_indent_parser.py | 127 ++ .../test_goods_nomenclature_origin_parser.py | 91 ++ .../test_goods_nomenclature_parser.py | 143 +++ ...est_goods_nomenclature_successor_parser.py | 99 ++ taric_parsers/tests/conftest.py | 116 ++ .../importer_examples/footnote_CREATE.xml | 76 ++ .../importer_examples/footnote_DELETE.xml | 23 + .../importer_examples/footnote_UPDATE.xml | 23 + .../footnote_description_CREATE.xml | 98 ++ .../footnote_description_DELETE.xml | 24 + .../footnote_description_UPDATE.xml | 24 + .../footnote_description_period_CREATE.xml | 98 ++ .../footnote_description_period_DELETE.xml | 23 + .../footnote_description_period_UPDATE.xml | 23 + .../footnote_type_CREATE.xml | 40 + .../footnote_type_DELETE.xml | 23 + .../footnote_type_UPDATE.xml | 23 + .../footnote_type_description_CREATE.xml | 40 + .../footnote_type_description_DELETE.xml | 22 + .../footnote_type_description_UPDATE.xml | 22 + .../footnote_type_no_description_CREATE.xml | 23 + .../test_footnote_description_parser.py | 102 ++ ...test_footnote_description_period_parser.py | 107 ++ .../footnote_parsers/test_footnote_parser.py | 103 ++ .../test_footnote_type_description_parser.py | 89 ++ .../test_footnote_type_parser.py | 112 ++ .../geographical_area_CREATE.xml | 24 + .../geographical_area_DELETE.xml | 24 + .../geographical_area_UPDATE.xml | 24 + .../geographical_area_description_CREATE.xml | 64 + .../geographical_area_description_DELETE.xml | 24 + .../geographical_area_description_UPDATE.xml | 24 + ...aphical_area_description_period_CREATE.xml | 64 + ...aphical_area_description_period_DELETE.xml | 24 + ...aphical_area_description_period_UPDATE.xml | 24 + .../geographical_membership_CREATE.xml | 64 + .../geographical_membership_DELETE.xml | 24 + .../geographical_membership_UPDATE.xml | 24 + ...st_geographical_area_description_parser.py | 102 ++ ...raphical_area_description_period_parser.py | 123 ++ .../test_geographical_area_parser.py | 114 ++ .../test_geographical_membership_parser.py | 102 ++ .../additional_code_CREATE.xml | 55 + .../additional_code_UPDATE.xml | 22 + .../additional_code_second_CREATE.xml | 22 + ...ditional_code_type_measure_type_CREATE.xml | 138 +++ ...ditional_code_type_measure_type_DELETE.xml | 23 + ...ditional_code_type_measure_type_UPDATE.xml | 23 + .../duty_expression_CREATE.xml | 42 + .../duty_expression_DELETE.xml | 25 + .../duty_expression_UPDATE.xml | 25 + .../duty_expression_description_CREATE.xml | 42 + .../duty_expression_description_DELETE.xml | 22 + .../duty_expression_description_UPDATE.xml | 22 + .../footnote_association_measure_CREATE.xml | 371 ++++++ .../footnote_association_measure_DELETE.xml | 22 + .../footnote_association_measure_UPDATE.xml | 22 + .../importer_examples/measure_CREATE.xml | 297 +++++ .../importer_examples/measure_DELETE.xml | 32 + .../importer_examples/measure_UPDATE.xml | 32 + .../measure_action_CREATE.xml | 39 + .../measure_action_DELETE.xml | 22 + .../measure_action_UPDATE.xml | 22 + .../measure_action_description_CREATE.xml | 39 + .../measure_action_description_DELETE.xml | 22 + .../measure_action_description_UPDATE.xml | 22 + .../measure_component_CREATE.xml | 484 ++++++++ .../measure_component_DELETE.xml | 25 + .../measure_component_UPDATE.xml | 25 + .../measure_condition_CREATE.xml | 708 +++++++++++ .../measure_condition_DELETE.xml | 30 + .../measure_condition_UPDATE.xml | 30 + .../measure_condition_code_CREATE.xml | 39 + .../measure_condition_code_DELETE.xml | 22 + .../measure_condition_code_UPDATE.xml | 22 + ...sure_condition_code_description_CREATE.xml | 39 + ...sure_condition_code_description_DELETE.xml | 22 + ...sure_condition_code_description_UPDATE.xml | 22 + .../measure_condition_component_CREATE.xml | 730 ++++++++++++ .../measure_condition_component_DELETE.xml | 24 + .../measure_condition_component_UPDATE.xml | 24 + ...sure_excluded_geographical_area_CREATE.xml | 316 +++++ ...sure_excluded_geographical_area_DELETE.xml | 22 + ...sure_excluded_geographical_area_UPDATE.xml | 22 + .../measure_series_CREATE.xml | 40 + .../measure_series_DELETE.xml | 23 + .../measure_series_UPDATE.xml | 23 + .../measure_series_no_description_CREATE.xml | 23 + .../importer_examples/measure_type_CREATE.xml | 83 ++ .../importer_examples/measure_type_DELETE.xml | 29 + .../importer_examples/measure_type_UPDATE.xml | 29 + .../measure_type_description_CREATE.xml | 83 ++ .../measure_type_description_DELETE.xml | 22 + .../measure_type_description_UPDATE.xml | 22 + ...measure_type_series_description_CREATE.xml | 40 + ...measure_type_series_description_DELETE.xml | 22 + ...measure_type_series_description_UPDATE.xml | 22 + .../importer_examples/measurement_CREATE.xml | 95 ++ .../importer_examples/measurement_DELETE.xml | 23 + .../importer_examples/measurement_UPDATE.xml | 23 + .../measurement_unit_CREATE.xml | 39 + .../measurement_unit_DELETE.xml | 22 + .../measurement_unit_UPDATE.xml | 22 + .../measurement_unit_description_CREATE.xml | 39 + .../measurement_unit_description_DELETE.xml | 22 + .../measurement_unit_description_UPDATE.xml | 22 + .../measurement_unit_qualifier_CREATE.xml | 39 + .../measurement_unit_qualifier_DELETE.xml | 22 + .../measurement_unit_qualifier_UPDATE.xml | 22 + ...ment_unit_qualifier_description_CREATE.xml | 39 + ...ment_unit_qualifier_description_DELETE.xml | 22 + ...ment_unit_qualifier_description_UPDATE.xml | 22 + .../monetary_unit_CREATE.xml | 40 + .../monetary_unit_DELETE.xml | 22 + .../monetary_unit_UPDATE.xml | 22 + .../monetary_unit_description_CREATE.xml | 40 + .../monetary_unit_description_DELETE.xml | 23 + .../monetary_unit_description_UPDATE.xml | 23 + ...dditional_code_type_measure_type_parser.py | 113 ++ ...test_duty_expression_description_parser.py | 99 ++ .../test_duty_expression_parser.py | 113 ++ ...est_footnote_association_measure_parser.py | 91 ++ .../test_measure_action_description_parser.py | 95 ++ .../test_measure_action_parser.py | 97 ++ .../test_measure_component_parser.py | 113 ++ ...asure_condition_code_description_parser.py | 107 ++ .../test_measure_condition_code_parser.py | 97 ++ ...test_measure_condition_component_parser.py | 111 ++ .../test_measure_condition_parser.py | 136 +++ ...asure_excluded_geographical_area_parser.py | 105 ++ .../measure_parsers/test_measure_parser.py | 178 +++ .../test_measure_type_description_parser.py | 95 ++ .../test_measure_type_parser.py | 131 +++ ..._measure_type_series_description_parser.py | 106 ++ .../test_measure_type_series_parser.py | 133 +++ .../test_measurement_parser.py | 102 ++ ...est_measurement_unit_description_parser.py | 93 ++ .../test_measurement_unit_parser.py | 96 ++ ...ement_unit_qualifier_description_parser.py | 112 ++ .../test_measurement_unit_qualifier_parser.py | 96 ++ .../test_monetary_unit_description_parser.py | 95 ++ .../test_monetary_unit_parser.py | 96 ++ .../quota_association_CREATE.xml | 206 ++++ .../quota_association_DELETE.xml | 24 + .../quota_association_UPDATE.xml | 24 + .../quota_balance_event_CREATE.xml | 206 ++++ .../quota_balance_event_DELETE.xml | 24 + .../quota_balance_event_UPDATE.xml | 24 + .../quota_blocking_period_CREATE.xml | 208 ++++ .../quota_blocking_period_DELETE.xml | 26 + .../quota_blocking_period_UPDATE.xml | 26 + ...ta_closed_and_transferred_event_CREATE.xml | 206 ++++ ...ta_closed_and_transferred_event_DELETE.xml | 24 + ...ta_closed_and_transferred_event_UPDATE.xml | 24 + .../quota_critical_event_CREATE.xml | 204 ++++ .../quota_critical_event_DELETE.xml | 22 + .../quota_critical_event_UPDATE.xml | 22 + .../quota_definition_CREATE.xml | 186 +++ .../quota_definition_DELETE.xml | 32 + .../quota_definition_UPDATE.xml | 32 + .../quota_exhaustion_event_CREATE.xml | 203 ++++ .../quota_exhaustion_event_DELETE.xml | 21 + .../quota_exhaustion_event_UPDATE.xml | 21 + .../quota_order_number_CREATE.xml | 23 + .../quota_order_number_DELETE.xml | 23 + .../quota_order_number_UPDATE.xml | 23 + .../quota_order_number_origin_CREATE.xml | 105 ++ .../quota_order_number_origin_DELETE.xml | 26 + .../quota_order_number_origin_UPDATE.xml | 26 + ...a_order_number_origin_exclusion_CREATE.xml | 123 ++ ...a_order_number_origin_exclusion_DELETE.xml | 22 + ...a_order_number_origin_exclusion_UPDATE.xml | 22 + .../quota_reopening_event_CREATE.xml | 203 ++++ .../quota_reopening_event_DELETE.xml | 21 + .../quota_reopening_event_UPDATE.xml | 21 + .../quota_suspension_CREATE.xml | 207 ++++ .../quota_suspension_DELETE.xml | 25 + .../quota_suspension_UPDATE.xml | 25 + .../quota_unblocking_event_CREATE.xml | 203 ++++ .../quota_unblocking_event_DELETE.xml | 21 + .../quota_unblocking_event_UPDATE.xml | 21 + .../quota_unsuspension_event_CREATE.xml | 203 ++++ .../quota_unsuspension_event_DELETE.xml | 21 + .../quota_unsuspension_event_UPDATE.xml | 21 + .../test_quota_association_parser.py | 100 ++ .../test_quota_balance_event_parser.py | 125 ++ .../test_quota_blocking_parser.py | 112 ++ ...ota_closed_and_transferred_event_parser.py | 144 +++ .../test_quota_critical_event_parser.py | 114 ++ .../test_quota_definition_parser.py | 181 +++ .../test_quota_exhaustion_event_parser.py | 106 ++ ...ta_order_number_origin_exclusion_parser.py | 108 ++ .../test_quota_order_number_origin_parser.py | 131 +++ .../test_quota_order_number_parser.py | 103 ++ .../test_quota_reopening_event_parser.py | 103 ++ .../test_quota_suspension_parser.py | 106 ++ .../test_quota_unblocking_event_parser.py | 105 ++ .../test_quota_unsuspension_event_parser.py | 106 ++ .../base_regulation_CREATE.xml | 69 ++ .../base_regulation_DELETE.xml | 33 + .../base_regulation_UPDATE.xml | 33 + .../fts_regulation_action_CREATE.xml | 117 ++ .../regulation_group_CREATE.xml | 39 + .../regulation_group_DELETE.xml | 22 + .../regulation_group_UPDATE.xml | 22 + .../regulation_group_description_CREATE.xml | 39 + .../regulation_group_description_DELETE.xml | 22 + .../regulation_group_description_UPDATE.xml | 22 + .../regulation_replacement_CREATE.xml | 222 ++++ .../test_base_regulation_parser.py | 182 +++ .../test_full_temporary_stop_action_parser.py | 76 ++ ...t_full_temporary_stop_regulation_parser.py | 98 ++ .../test_modification_regulation_parser.py | 111 ++ ...est_regulation_group_description_parser.py | 114 ++ .../test_regulation_group_parser.py | 118 ++ .../test_regulation_replacement_parser.py | 94 ++ .../tests/support/additional_code_CREATE.xml | 55 + taric_parsers/tests/support/broken.xml | 18 + taric_parsers/tests/support/dtd.xml | 7 + taric_parsers/tests/support/invalid.xml | 24 + taric_parsers/tests/support/invalid_type.txt | 1 + taric_parsers/tests/support/valid.xml | 24 + taric_parsers/tests/test_chunker.py | 167 +++ .../tests/test_for_generic_import_errors.py | 42 + taric_parsers/tests/test_forms.py | 60 + taric_parsers/tests/test_namespaces.py | 42 + .../tests/test_new_element_parser.py | 797 +++++++++++++ .../tests/test_new_parser_generics.py | 854 ++++++++++++++ taric_parsers/tests/test_parser_helper.py | 287 +++++ taric_parsers/tests/test_taric_xml_sources.py | 55 + .../tests/test_transaction_parser.py | 44 + taric_parsers/tests/test_views.py | 104 ++ taric_parsers/urls.py | 23 + taric_parsers/utils.py | 145 +++ taric_parsers/validators.py | 15 + taric_parsers/views.py | 77 ++ urls.py | 3 +- 377 files changed, 30711 insertions(+), 193 deletions(-) create mode 100644 importer/jinja2/eu-importer/details.jinja create mode 100644 importer/migrations/0012_batchimporterror.py create mode 100644 importer/migrations/0013_alter_importbatch_status.py create mode 100644 importer/tests/test_files/additional_code_CREATE.xml create mode 100644 importer/validators.py create mode 100644 taric_parsers/__init__.py create mode 100644 taric_parsers/admin.py create mode 100644 taric_parsers/apps.py create mode 100644 taric_parsers/chunker.py create mode 100644 taric_parsers/forms.py create mode 100644 taric_parsers/importer.py create mode 100644 taric_parsers/importer_issue.py create mode 100644 taric_parsers/jinja2/includes/importer_list.jinja create mode 100644 taric_parsers/jinja2/includes/taric_importer_list.jinja create mode 100644 taric_parsers/jinja2/taric_parser/create.jinja create mode 100644 taric_parsers/jinja2/taric_parser/details.jinja create mode 100644 taric_parsers/jinja2/taric_parser/importer_list.jinja create mode 100644 taric_parsers/jinja2/taric_parser/list.jinja create mode 100644 taric_parsers/migrations/__init__.py create mode 100644 taric_parsers/namespaces.py create mode 100644 taric_parsers/parser_model_link.py create mode 100644 taric_parsers/parsers/additional_code_parsers.py create mode 100644 taric_parsers/parsers/certificate_parser.py create mode 100644 taric_parsers/parsers/commodity_parser.py create mode 100644 taric_parsers/parsers/footnote_parser.py create mode 100644 taric_parsers/parsers/geo_area_parser.py create mode 100644 taric_parsers/parsers/measure_parser.py create mode 100644 taric_parsers/parsers/mixins.py create mode 100644 taric_parsers/parsers/quota_parser.py create mode 100644 taric_parsers/parsers/regulation_parser.py create mode 100644 taric_parsers/parsers/taric_parser.py create mode 100644 taric_parsers/taric_xml_source.py create mode 100644 taric_parsers/tasks.py create mode 100644 taric_parsers/tests/__init__.py create mode 100644 taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_CREATE.xml create mode 100644 taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_DELETE.xml create mode 100644 taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_UPDATE.xml create mode 100644 taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_CREATE.xml create mode 100644 taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_DELETE.xml create mode 100644 taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_UPDATE.xml create mode 100644 taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_invalid_additional_code_CREATE.xml create mode 100644 taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_period_CREATE.xml create mode 100644 taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_period_DELETE.xml create mode 100644 taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_period_UPDATE.xml create mode 100644 taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_period_without_description_CREATE.xml create mode 100644 taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_invalid_type_CREATE.xml create mode 100644 taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_CREATE.xml create mode 100644 taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_DELETE.xml create mode 100644 taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_UPDATE.xml create mode 100644 taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_description_CREATE.xml create mode 100644 taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_description_DELETE.xml create mode 100644 taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_description_UPDATE.xml create mode 100644 taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_description_without_type_CREATE.xml create mode 100644 taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_no_description_CREATE.xml create mode 100644 taric_parsers/tests/additional_code_parsers/importer_examples/footnote_association_additional_code_CREATE.xml create mode 100644 taric_parsers/tests/additional_code_parsers/importer_examples/footnote_association_additional_code_DELETE.xml create mode 100644 taric_parsers/tests/additional_code_parsers/importer_examples/footnote_association_additional_code_UPDATE.xml create mode 100644 taric_parsers/tests/additional_code_parsers/importer_examples/footnote_association_additional_code_invalid_footnote_CREATE.xml create mode 100644 taric_parsers/tests/additional_code_parsers/test_additional_code_description_parser.py create mode 100644 taric_parsers/tests/additional_code_parsers/test_additional_code_description_period_parser.py create mode 100644 taric_parsers/tests/additional_code_parsers/test_additional_code_parser.py create mode 100644 taric_parsers/tests/additional_code_parsers/test_additional_code_type_description_parser.py create mode 100644 taric_parsers/tests/additional_code_parsers/test_additional_code_type_parser.py create mode 100644 taric_parsers/tests/additional_code_parsers/test_footnote_association_additional_code_parser.py create mode 100644 taric_parsers/tests/certificate_parsers/importer_examples/certificate_CREATE.xml create mode 100644 taric_parsers/tests/certificate_parsers/importer_examples/certificate_DELETE.xml create mode 100644 taric_parsers/tests/certificate_parsers/importer_examples/certificate_UPDATE.xml create mode 100644 taric_parsers/tests/certificate_parsers/importer_examples/certificate_description_CREATE.xml create mode 100644 taric_parsers/tests/certificate_parsers/importer_examples/certificate_description_DELETE.xml create mode 100644 taric_parsers/tests/certificate_parsers/importer_examples/certificate_description_UPDATE.xml create mode 100644 taric_parsers/tests/certificate_parsers/importer_examples/certificate_description_period_CREATE.xml create mode 100644 taric_parsers/tests/certificate_parsers/importer_examples/certificate_description_period_DELETE.xml create mode 100644 taric_parsers/tests/certificate_parsers/importer_examples/certificate_description_period_UPDATE.xml create mode 100644 taric_parsers/tests/certificate_parsers/importer_examples/certificate_type_CREATE.xml create mode 100644 taric_parsers/tests/certificate_parsers/importer_examples/certificate_type_DELETE.xml create mode 100644 taric_parsers/tests/certificate_parsers/importer_examples/certificate_type_UPDATE.xml create mode 100644 taric_parsers/tests/certificate_parsers/importer_examples/certificate_type_description_CREATE.xml create mode 100644 taric_parsers/tests/certificate_parsers/importer_examples/certificate_type_description_DELETE.xml create mode 100644 taric_parsers/tests/certificate_parsers/importer_examples/certificate_type_description_UPDATE.xml create mode 100644 taric_parsers/tests/certificate_parsers/test_certificate_description_parser.py create mode 100644 taric_parsers/tests/certificate_parsers/test_certificate_description_period_parser.py create mode 100644 taric_parsers/tests/certificate_parsers/test_certificate_parser.py create mode 100644 taric_parsers/tests/certificate_parsers/test_certificate_type_description_parser.py create mode 100644 taric_parsers/tests/certificate_parsers/test_certificate_type_parser.py create mode 100644 taric_parsers/tests/commoditiy_parsers/importer_examples/footnote_association_goods_nomenclature_CREATE.xml create mode 100644 taric_parsers/tests/commoditiy_parsers/importer_examples/footnote_association_goods_nomenclature_DELETE.xml create mode 100644 taric_parsers/tests/commoditiy_parsers/importer_examples/footnote_association_goods_nomenclature_UPDATE.xml create mode 100644 taric_parsers/tests/commoditiy_parsers/importer_examples/footnote_association_goods_nomenclature_no_footnote_CREATE.xml create mode 100644 taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_CREATE.xml create mode 100644 taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_CREATE_then_DELETE.xml create mode 100644 taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_DELETE.xml create mode 100644 taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_UPDATE.xml create mode 100644 taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_DELETE.xml create mode 100644 taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_UPDATE.xml create mode 100644 taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_no_period_CREATE.xml create mode 100644 taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_no_period_UPDATE.xml create mode 100644 taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_period_UPDATE.xml create mode 100644 taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_period_multi_CREATE.xml create mode 100644 taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_period_only_DELETE.xml create mode 100644 taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_period_only_UPDATE.xml create mode 100644 taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_with_period_CREATE.xml create mode 100644 taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_indent_CREATE.xml create mode 100644 taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_indent_DELETE.xml create mode 100644 taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_indent_UPDATE.xml create mode 100644 taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_origin_CREATE.xml create mode 100644 taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_origin_UPDATE.xml create mode 100644 taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_successor_CREATE.xml create mode 100644 taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_successor_DELETE.xml create mode 100644 taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_successor_UPDATE.xml create mode 100644 taric_parsers/tests/commoditiy_parsers/test_footnote_association_goods_nomenclature_parser.py create mode 100644 taric_parsers/tests/commoditiy_parsers/test_goods_nomenclature_description_parser.py create mode 100644 taric_parsers/tests/commoditiy_parsers/test_goods_nomenclature_description_period_parser.py create mode 100644 taric_parsers/tests/commoditiy_parsers/test_goods_nomenclature_indent_parser.py create mode 100644 taric_parsers/tests/commoditiy_parsers/test_goods_nomenclature_origin_parser.py create mode 100644 taric_parsers/tests/commoditiy_parsers/test_goods_nomenclature_parser.py create mode 100644 taric_parsers/tests/commoditiy_parsers/test_goods_nomenclature_successor_parser.py create mode 100644 taric_parsers/tests/conftest.py create mode 100644 taric_parsers/tests/footnote_parsers/importer_examples/footnote_CREATE.xml create mode 100644 taric_parsers/tests/footnote_parsers/importer_examples/footnote_DELETE.xml create mode 100644 taric_parsers/tests/footnote_parsers/importer_examples/footnote_UPDATE.xml create mode 100644 taric_parsers/tests/footnote_parsers/importer_examples/footnote_description_CREATE.xml create mode 100644 taric_parsers/tests/footnote_parsers/importer_examples/footnote_description_DELETE.xml create mode 100644 taric_parsers/tests/footnote_parsers/importer_examples/footnote_description_UPDATE.xml create mode 100644 taric_parsers/tests/footnote_parsers/importer_examples/footnote_description_period_CREATE.xml create mode 100644 taric_parsers/tests/footnote_parsers/importer_examples/footnote_description_period_DELETE.xml create mode 100644 taric_parsers/tests/footnote_parsers/importer_examples/footnote_description_period_UPDATE.xml create mode 100644 taric_parsers/tests/footnote_parsers/importer_examples/footnote_type_CREATE.xml create mode 100644 taric_parsers/tests/footnote_parsers/importer_examples/footnote_type_DELETE.xml create mode 100644 taric_parsers/tests/footnote_parsers/importer_examples/footnote_type_UPDATE.xml create mode 100644 taric_parsers/tests/footnote_parsers/importer_examples/footnote_type_description_CREATE.xml create mode 100644 taric_parsers/tests/footnote_parsers/importer_examples/footnote_type_description_DELETE.xml create mode 100644 taric_parsers/tests/footnote_parsers/importer_examples/footnote_type_description_UPDATE.xml create mode 100644 taric_parsers/tests/footnote_parsers/importer_examples/footnote_type_no_description_CREATE.xml create mode 100644 taric_parsers/tests/footnote_parsers/test_footnote_description_parser.py create mode 100644 taric_parsers/tests/footnote_parsers/test_footnote_description_period_parser.py create mode 100644 taric_parsers/tests/footnote_parsers/test_footnote_parser.py create mode 100644 taric_parsers/tests/footnote_parsers/test_footnote_type_description_parser.py create mode 100644 taric_parsers/tests/footnote_parsers/test_footnote_type_parser.py create mode 100644 taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_CREATE.xml create mode 100644 taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_DELETE.xml create mode 100644 taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_UPDATE.xml create mode 100644 taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_description_CREATE.xml create mode 100644 taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_description_DELETE.xml create mode 100644 taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_description_UPDATE.xml create mode 100644 taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_description_period_CREATE.xml create mode 100644 taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_description_period_DELETE.xml create mode 100644 taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_description_period_UPDATE.xml create mode 100644 taric_parsers/tests/geo_area_parsers/importer_examples/geographical_membership_CREATE.xml create mode 100644 taric_parsers/tests/geo_area_parsers/importer_examples/geographical_membership_DELETE.xml create mode 100644 taric_parsers/tests/geo_area_parsers/importer_examples/geographical_membership_UPDATE.xml create mode 100644 taric_parsers/tests/geo_area_parsers/test_geographical_area_description_parser.py create mode 100644 taric_parsers/tests/geo_area_parsers/test_geographical_area_description_period_parser.py create mode 100644 taric_parsers/tests/geo_area_parsers/test_geographical_area_parser.py create mode 100644 taric_parsers/tests/geo_area_parsers/test_geographical_membership_parser.py create mode 100644 taric_parsers/tests/importer_examples/additional_code_CREATE.xml create mode 100644 taric_parsers/tests/importer_examples/additional_code_UPDATE.xml create mode 100644 taric_parsers/tests/importer_examples/additional_code_second_CREATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/additional_code_type_measure_type_CREATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/additional_code_type_measure_type_DELETE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/additional_code_type_measure_type_UPDATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/duty_expression_CREATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/duty_expression_DELETE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/duty_expression_UPDATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/duty_expression_description_CREATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/duty_expression_description_DELETE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/duty_expression_description_UPDATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/footnote_association_measure_CREATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/footnote_association_measure_DELETE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/footnote_association_measure_UPDATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_CREATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_DELETE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_UPDATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_action_CREATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_action_DELETE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_action_UPDATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_action_description_CREATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_action_description_DELETE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_action_description_UPDATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_component_CREATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_component_DELETE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_component_UPDATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_condition_CREATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_condition_DELETE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_condition_UPDATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_condition_code_CREATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_condition_code_DELETE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_condition_code_UPDATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_condition_code_description_CREATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_condition_code_description_DELETE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_condition_code_description_UPDATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_condition_component_CREATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_condition_component_DELETE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_condition_component_UPDATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_excluded_geographical_area_CREATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_excluded_geographical_area_DELETE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_excluded_geographical_area_UPDATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_series_CREATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_series_DELETE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_series_UPDATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_series_no_description_CREATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_type_CREATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_type_DELETE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_type_UPDATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_type_description_CREATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_type_description_DELETE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_type_description_UPDATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_type_series_description_CREATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_type_series_description_DELETE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measure_type_series_description_UPDATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measurement_CREATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measurement_DELETE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measurement_UPDATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_CREATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_DELETE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_UPDATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_description_CREATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_description_DELETE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_description_UPDATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_qualifier_CREATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_qualifier_DELETE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_qualifier_UPDATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_qualifier_description_CREATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_qualifier_description_DELETE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_qualifier_description_UPDATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/monetary_unit_CREATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/monetary_unit_DELETE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/monetary_unit_UPDATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/monetary_unit_description_CREATE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/monetary_unit_description_DELETE.xml create mode 100644 taric_parsers/tests/measure_parsers/importer_examples/monetary_unit_description_UPDATE.xml create mode 100644 taric_parsers/tests/measure_parsers/test_additional_code_type_measure_type_parser.py create mode 100644 taric_parsers/tests/measure_parsers/test_duty_expression_description_parser.py create mode 100644 taric_parsers/tests/measure_parsers/test_duty_expression_parser.py create mode 100644 taric_parsers/tests/measure_parsers/test_footnote_association_measure_parser.py create mode 100644 taric_parsers/tests/measure_parsers/test_measure_action_description_parser.py create mode 100644 taric_parsers/tests/measure_parsers/test_measure_action_parser.py create mode 100644 taric_parsers/tests/measure_parsers/test_measure_component_parser.py create mode 100644 taric_parsers/tests/measure_parsers/test_measure_condition_code_description_parser.py create mode 100644 taric_parsers/tests/measure_parsers/test_measure_condition_code_parser.py create mode 100644 taric_parsers/tests/measure_parsers/test_measure_condition_component_parser.py create mode 100644 taric_parsers/tests/measure_parsers/test_measure_condition_parser.py create mode 100644 taric_parsers/tests/measure_parsers/test_measure_excluded_geographical_area_parser.py create mode 100644 taric_parsers/tests/measure_parsers/test_measure_parser.py create mode 100644 taric_parsers/tests/measure_parsers/test_measure_type_description_parser.py create mode 100644 taric_parsers/tests/measure_parsers/test_measure_type_parser.py create mode 100644 taric_parsers/tests/measure_parsers/test_measure_type_series_description_parser.py create mode 100644 taric_parsers/tests/measure_parsers/test_measure_type_series_parser.py create mode 100644 taric_parsers/tests/measure_parsers/test_measurement_parser.py create mode 100644 taric_parsers/tests/measure_parsers/test_measurement_unit_description_parser.py create mode 100644 taric_parsers/tests/measure_parsers/test_measurement_unit_parser.py create mode 100644 taric_parsers/tests/measure_parsers/test_measurement_unit_qualifier_description_parser.py create mode 100644 taric_parsers/tests/measure_parsers/test_measurement_unit_qualifier_parser.py create mode 100644 taric_parsers/tests/measure_parsers/test_monetary_unit_description_parser.py create mode 100644 taric_parsers/tests/measure_parsers/test_monetary_unit_parser.py create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_association_CREATE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_association_DELETE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_association_UPDATE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_balance_event_CREATE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_balance_event_DELETE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_balance_event_UPDATE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_blocking_period_CREATE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_blocking_period_DELETE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_blocking_period_UPDATE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_closed_and_transferred_event_CREATE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_closed_and_transferred_event_DELETE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_closed_and_transferred_event_UPDATE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_critical_event_CREATE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_critical_event_DELETE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_critical_event_UPDATE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_definition_CREATE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_definition_DELETE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_definition_UPDATE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_exhaustion_event_CREATE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_exhaustion_event_DELETE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_exhaustion_event_UPDATE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_order_number_CREATE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_order_number_DELETE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_order_number_UPDATE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_order_number_origin_CREATE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_order_number_origin_DELETE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_order_number_origin_UPDATE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_order_number_origin_exclusion_CREATE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_order_number_origin_exclusion_DELETE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_order_number_origin_exclusion_UPDATE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_reopening_event_CREATE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_reopening_event_DELETE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_reopening_event_UPDATE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_suspension_CREATE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_suspension_DELETE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_suspension_UPDATE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_unblocking_event_CREATE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_unblocking_event_DELETE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_unblocking_event_UPDATE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_unsuspension_event_CREATE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_unsuspension_event_DELETE.xml create mode 100644 taric_parsers/tests/quota_parser/importer_examples/quota_unsuspension_event_UPDATE.xml create mode 100644 taric_parsers/tests/quota_parser/test_quota_association_parser.py create mode 100644 taric_parsers/tests/quota_parser/test_quota_balance_event_parser.py create mode 100644 taric_parsers/tests/quota_parser/test_quota_blocking_parser.py create mode 100644 taric_parsers/tests/quota_parser/test_quota_closed_and_transferred_event_parser.py create mode 100644 taric_parsers/tests/quota_parser/test_quota_critical_event_parser.py create mode 100644 taric_parsers/tests/quota_parser/test_quota_definition_parser.py create mode 100644 taric_parsers/tests/quota_parser/test_quota_exhaustion_event_parser.py create mode 100644 taric_parsers/tests/quota_parser/test_quota_order_number_origin_exclusion_parser.py create mode 100644 taric_parsers/tests/quota_parser/test_quota_order_number_origin_parser.py create mode 100644 taric_parsers/tests/quota_parser/test_quota_order_number_parser.py create mode 100644 taric_parsers/tests/quota_parser/test_quota_reopening_event_parser.py create mode 100644 taric_parsers/tests/quota_parser/test_quota_suspension_parser.py create mode 100644 taric_parsers/tests/quota_parser/test_quota_unblocking_event_parser.py create mode 100644 taric_parsers/tests/quota_parser/test_quota_unsuspension_event_parser.py create mode 100644 taric_parsers/tests/regulation_parsers/importer_examples/base_regulation_CREATE.xml create mode 100644 taric_parsers/tests/regulation_parsers/importer_examples/base_regulation_DELETE.xml create mode 100644 taric_parsers/tests/regulation_parsers/importer_examples/base_regulation_UPDATE.xml create mode 100644 taric_parsers/tests/regulation_parsers/importer_examples/fts_regulation_action_CREATE.xml create mode 100644 taric_parsers/tests/regulation_parsers/importer_examples/regulation_group_CREATE.xml create mode 100644 taric_parsers/tests/regulation_parsers/importer_examples/regulation_group_DELETE.xml create mode 100644 taric_parsers/tests/regulation_parsers/importer_examples/regulation_group_UPDATE.xml create mode 100644 taric_parsers/tests/regulation_parsers/importer_examples/regulation_group_description_CREATE.xml create mode 100644 taric_parsers/tests/regulation_parsers/importer_examples/regulation_group_description_DELETE.xml create mode 100644 taric_parsers/tests/regulation_parsers/importer_examples/regulation_group_description_UPDATE.xml create mode 100644 taric_parsers/tests/regulation_parsers/importer_examples/regulation_replacement_CREATE.xml create mode 100644 taric_parsers/tests/regulation_parsers/test_base_regulation_parser.py create mode 100644 taric_parsers/tests/regulation_parsers/test_full_temporary_stop_action_parser.py create mode 100644 taric_parsers/tests/regulation_parsers/test_full_temporary_stop_regulation_parser.py create mode 100644 taric_parsers/tests/regulation_parsers/test_modification_regulation_parser.py create mode 100644 taric_parsers/tests/regulation_parsers/test_regulation_group_description_parser.py create mode 100644 taric_parsers/tests/regulation_parsers/test_regulation_group_parser.py create mode 100644 taric_parsers/tests/regulation_parsers/test_regulation_replacement_parser.py create mode 100644 taric_parsers/tests/support/additional_code_CREATE.xml create mode 100644 taric_parsers/tests/support/broken.xml create mode 100644 taric_parsers/tests/support/dtd.xml create mode 100644 taric_parsers/tests/support/invalid.xml create mode 100644 taric_parsers/tests/support/invalid_type.txt create mode 100644 taric_parsers/tests/support/valid.xml create mode 100644 taric_parsers/tests/test_chunker.py create mode 100644 taric_parsers/tests/test_for_generic_import_errors.py create mode 100644 taric_parsers/tests/test_forms.py create mode 100644 taric_parsers/tests/test_namespaces.py create mode 100644 taric_parsers/tests/test_new_element_parser.py create mode 100644 taric_parsers/tests/test_new_parser_generics.py create mode 100644 taric_parsers/tests/test_parser_helper.py create mode 100644 taric_parsers/tests/test_taric_xml_sources.py create mode 100644 taric_parsers/tests/test_transaction_parser.py create mode 100644 taric_parsers/tests/test_views.py create mode 100644 taric_parsers/urls.py create mode 100644 taric_parsers/utils.py create mode 100644 taric_parsers/validators.py create mode 100644 taric_parsers/views.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c01024af2..3a659fdf1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,14 +1,13 @@ repos: - repo: https://github.com/asottile/add-trailing-comma - rev: v2.4.0 + rev: v3.1.0 hooks: - id: add-trailing-comma - repo: https://github.com/myint/autoflake.git - rev: v2.1.1 + rev: v2.2.1 hooks: - id: autoflake - args: - [ + args: [ "--in-place", "--remove-all-unused-imports", "--remove-unused-variable", @@ -22,8 +21,7 @@ repos: rev: v1.7.5 hooks: - id: docformatter - args: - [ + args: [ "--in-place", "--wrap-summaries=80", "--wrap-descriptions=80", @@ -34,7 +32,7 @@ repos: hooks: - id: black - repo: https://github.com/ikamensh/flynt/ - rev: '0.78' + rev: '1.0.1' hooks: - id: flynt - repo: https://github.com/uktrade/pii-secret-check-hooks diff --git a/README.rst b/README.rst index b4507596c..044607d81 100644 --- a/README.rst +++ b/README.rst @@ -134,6 +134,86 @@ To run tests use the following command: For more detailed information on running tests, see :doc:`testing` +Pre-commit hooks +---------------- + +This project uses pre-commit hooks to update formatting and identify potential sensitive data before +it is committed to the public repo. + +note: The python package pre-commit is a requirement within requirements-dev.txt and should be installed +to meet development requirements + +Install +~~~~~~~ + +To initially setup the pre-commit hooks you can run the following command. + +.. code:: sh + + $ pre-commit install + +Once installed, when committing it will first run all the predefined processes to clean up code formatting +and notify about any detected sensitive strings found that are not in pii exclude files. + +Note: the first commit or run of the pre-commit hooks after installing may take a few minutes for setup the +dependent packages for the first time. This is normal, and will be faster on subsequent commits. + +Update +~~~~~~ + +The packages used to perform the pre-commit process are regularly updated. Periodically its advised +you run the following command to keep the dependencies updated. + +.. code:: sh + + $ pre-commit autoupdate + +This will verify that the dependencies are updated based on requirements. + +Uninstall +~~~~~~~~~ + +The pre-commit hooks can be uninstalled with the following command + +.. code:: sh + + $ pre-commit uninstall + +Run the hooks without committing +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You may at times want to run the pre-commit hooks before committing. This can be done with +the following command. This command will run the hooks on all changed files. + +.. code:: sh + + $ pre-commit run + +If you would like to run the hooks over all files you can run the following command + +.. code:: sh + + $ pre-commit run -a + +or + +.. code:: sh + + $ pre-commit run --all-files + +Troubleshooting +~~~~~~~~~~~~~~~ + +If you encounter issues with the pre-commit hooks there are a number of things you can +clear the cached pre-committed files using this command: + +.. code:: sh + + $ pre-commit clean + +If that fails you can try updating the dependencies for the hooks + +If the above fails, uninstall and then install again. Dockerisation ------------- @@ -309,6 +389,15 @@ This command is broken into two stages: $ python manage.py run_import_batch +Using the TARIC parser (currently referenced importer v2) +----------------------------------------- + +There are no command line tools available for this tool. + +This tool is available as an importer alternative found within the web front end in the footer menu under "New TARIC parser". + +This tool addresses several short falls that the current importer has. + Using the exporter ------------------ diff --git a/commodities/business_rules.py b/commodities/business_rules.py index 2735b8e70..5ac61dcfb 100644 --- a/commodities/business_rules.py +++ b/commodities/business_rules.py @@ -206,6 +206,7 @@ def validate(self, origin): If it does not meet these criteria it's a validation fail """ + from commodities.models.orm import GoodsNomenclature from commodities.models.orm import GoodsNomenclatureOrigin if origin.new_goods_nomenclature.valid_between.lower < date(2021, 1, 1): @@ -224,6 +225,16 @@ def validate(self, origin): ): return + # verify the goods nomenclature exists in the same transaction, it will not if it has been deleted + if not ( + GoodsNomenclature.objects.approved_up_to_transaction( + origin.transaction, + ) + .filter(sid=origin.new_goods_nomenclature.sid) + .exists() + ): + return + raise self.violation( model=origin, message="Non top-level goods must have an origin specified. None remain if this origin is deleted", diff --git a/commodities/models/dc.py b/commodities/models/dc.py index 38ba18ce2..c803ce06b 100644 --- a/commodities/models/dc.py +++ b/commodities/models/dc.py @@ -425,7 +425,7 @@ def __post_init__(self) -> None: super().__post_init__() def get_potential_parents(self, commodity: Commodity) -> List[Commodity]: - """gets possible parents, for queries where a parent may be end-dated, + """Gets possible parents, for queries where a parent may be end-dated, and another parent could potentially be present to inherit the child.""" parent = self.get_parent(commodity) @@ -686,10 +686,11 @@ def _get_diff( """ Returns a snapshot diff for a given relation on a sinlge commodity. - For detailed overview on snapshot diffs, see the docs for the SnapshotDiff class. + For detailed overview on snapshot diffs, see the docs for the + SnapshotDiff class. - You can get one diff per commodity and relation type - (e.g. diff the children of '9999.20.00.00' or diff the siblings of '9999.30.20.10') + You can get one diff per commodity and relation type (e.g. diff the + children of '9999.20.00.00' or diff the siblings of '9999.30.20.10') """ if snapshot.moment.clock_type != self.moment.clock_type: raise ValueError("Cannot diff snapshots with different clock types.") @@ -838,19 +839,17 @@ def _get_snapshot_commodities( """ Returns the list of commodities than belong to a snapshot. - This method needs to be very efficient - - In particular, it should require the same number - of database round trips regardless of the level + This method needs to be very efficient - In particular, it should + require the same number of database round trips regardless of the level of the root commodity in the tree. - The solution is fetch all goods matching the snapshot moment - (incl. potentially multiple versions of each) - and then call a new util method get_latest_version, - which efficiently yields only the latest version - of each good from within the returned queryset. + The solution is fetch all goods matching the snapshot moment (incl. + potentially multiple versions of each) and then call a new util method + get_latest_version, which efficiently yields only the latest version of + each good from within the returned queryset. - We then efficiently find the commodities in our collection - that match the latest_version goods. + We then efficiently find the commodities in our collection that match + the latest_version goods. """ item_ids = {c.item_id for c in self.commodities if c.obj} goods = GoodsNomenclature.objects.approved_up_to_transaction( @@ -1338,10 +1337,8 @@ def _handle_hierarchy_side_effect( """ Updates or deletes a clashing ME32 measure. - If either measure validity period - is fully contained in the other's, - the only option is to delete one, - favoring the one with lower-level code. + If either measure validity period is fully contained in the other's, the + only option is to delete one, favoring the one with lower-level code. Otherwise we have an opportunity to cap the earlier measure. """ @@ -1526,13 +1523,12 @@ def as_at_date(self) -> date: """ Returns the threshold date for the commodity code change. - Dependent measures (or other dependent models with validity spans) - would be exclude those with effective end date before this date. + Dependent measures (or other dependent models with validity spans) would + be exclude those with effective end date before this date. - In the case of a commodity UPDATE or DELETE, - the threshold is the commodity's validity end date. - In the case of a commodity CREATE, - the threshold is the commodity's validity start date. + In the case of a commodity UPDATE or DELETE, the threshold is the + commodity's validity end date. In the case of a commodity CREATE, the + threshold is the commodity's validity start date. """ if self.update_type == UpdateType.UPDATE: return self.current.valid_between.upper diff --git a/commodities/tests/business_rules/test_NIG5_origin.py b/commodities/tests/business_rules/test_NIG5_origin.py index 5677c3bc8..7fdd867cc 100644 --- a/commodities/tests/business_rules/test_NIG5_origin.py +++ b/commodities/tests/business_rules/test_NIG5_origin.py @@ -138,3 +138,75 @@ def test_NIG5_origin_does_not_raise_violation_when_origin_still_exists(workbaske ) business_rules.NIG5_origin(editable_transaction).validate(origin_delete) + + +def test_NIG5_origin_does_not_raise_violation_when_both_origin_and_goods_nomenclature_are_deleted( + workbasket, +): + """ + When deleting an origin the violation does not trigger if the goods + nomenclature is also being deleted in the same transaction. + + This rule is only applicable when origins are deleted. + """ + # Setup data + published_workbasket = factories.PublishedWorkBasketFactory() + + approved_transaction = factories.ApprovedTransactionFactory.create( + workbasket=published_workbasket, + order=1, + ) + + approved_transaction_2 = factories.ApprovedTransactionFactory.create( + workbasket=published_workbasket, + order=2, + ) + + approved_transaction_3 = factories.ApprovedTransactionFactory.create( + workbasket=published_workbasket, + order=3, + ) + + goods_chapter = factories.GoodsNomenclatureFactory.create( + item_id="2000000000", + transaction=approved_transaction, + ) + + # This is the goods we are going to delete the origin from + goods = factories.GoodsNomenclatureFactory.create( + item_id="2000000010", + origin__derived_from_goods_nomenclature=goods_chapter, + indent__indent=1, + transaction=approved_transaction_2, + ) + + editable_workbasket = factories.WorkBasketFactory() + + editable_transaction = factories.TransactionFactory.create( + workbasket=editable_workbasket, + order=4, + ) + + origin_delete = ( + GoodsNomenclatureOrigin.objects.approved_up_to_transaction( + approved_transaction_2, + ) + .filter( + new_goods_nomenclature=goods, + derived_from_goods_nomenclature=goods_chapter, + ) + .first() + .new_version( + transaction=editable_transaction, + update_type=UpdateType.DELETE, + workbasket=editable_workbasket, + ) + ) + + goods_delete = goods.new_version( + transaction=editable_transaction, + update_type=UpdateType.DELETE, + workbasket=editable_workbasket, + ) + + business_rules.NIG5_origin(editable_transaction).validate(origin_delete) diff --git a/commodities/tests/test_migrations.py b/commodities/tests/test_migrations.py index 617108588..9f528c3ab 100644 --- a/commodities/tests/test_migrations.py +++ b/commodities/tests/test_migrations.py @@ -8,15 +8,16 @@ @pytest.mark.django_db() -def test_main_migration_works(migrator): +def test_main_migration_works(migrator, setup_content_types): """Ensures that the description date fix for TOPS-745 migration works.""" - # migrator.reset() # before migration old_state = migrator.apply_initial_migration( ("commodities", "0011_TOPS_745_migration_dependencies"), ) + setup_content_types(old_state.apps) + GoodsNomenclatureDescription = old_state.apps.get_model( "commodities", "GoodsNomenclatureDescription", @@ -83,16 +84,16 @@ def test_main_migration_works(migrator): 6, ) - migrator.reset() - @pytest.mark.django_db() -def test_main_migration_ignores_if_no_data(migrator): +def test_main_migration_ignores_if_no_data(migrator, setup_content_types): # before migration old_state = migrator.apply_initial_migration( ("commodities", "0011_TOPS_745_migration_dependencies"), ) + setup_content_types(old_state.apps) + GoodsNomenclatureDescription = old_state.apps.get_model( "commodities", "GoodsNomenclatureDescription", @@ -104,5 +105,3 @@ def test_main_migration_ignores_if_no_data(migrator): migrator.apply_tested_migration( ("commodities", "0012_TOPS_745_description_date_fix"), ) - - migrator.reset() diff --git a/common/jinja2/layouts/layout.jinja b/common/jinja2/layouts/layout.jinja index 3efa6a7b2..331d29103 100644 --- a/common/jinja2/layouts/layout.jinja +++ b/common/jinja2/layouts/layout.jinja @@ -83,7 +83,11 @@ }, { "href": url("import_batch-ui-list"), - "text": "Master importer" + "text": "Importer V1" + }, + { + "href": url("taric_parser_import_ui_list"), + "text": "Importer V2" }, { "href": url("reports:index"), diff --git a/common/tests/test_views.py b/common/tests/test_views.py index b6c244a50..dda2ffede 100644 --- a/common/tests/test_views.py +++ b/common/tests/test_views.py @@ -189,7 +189,7 @@ def test_index_displays_footer_links(valid_user_client): page = BeautifulSoup(str(response.content), "html.parser") a_tags = page.select("footer a") - assert len(a_tags) == 6 + assert len(a_tags) == 7 assert "Privacy policy" in a_tags[0].text assert ( a_tags[0].attrs["href"] diff --git a/common/tests/util.py b/common/tests/util.py index 2e7651c5c..3a42113da 100644 --- a/common/tests/util.py +++ b/common/tests/util.py @@ -1,6 +1,7 @@ import contextlib import importlib import json +import os from datetime import date from datetime import datetime from functools import lru_cache @@ -32,14 +33,18 @@ from commodities.models.orm import GoodsNomenclature from common.business_rules import BusinessRule +from common.models import Transaction from common.models.trackedmodel import TrackedModel -from common.models.transactions import Transaction from common.renderers import counter_generator from common.tariffs_api import Endpoints from common.tests import factories from common.util import TaricDateRange from common.util import get_accessor from common.util import get_field_tuple +from taric_parsers.importer import TaricImporter +from taric_parsers.taric_xml_source import TaricXMLFileSource +from workbaskets.models import WorkBasket +from workbaskets.validators import WorkflowStatus INTERDEPENDENT_IMPORT_IMPLEMENTED = True UPDATE_IMPORTER_IMPLEMENTED = True @@ -919,3 +924,61 @@ def wrap_numbers_over_max_digits(number: int, max_digits): # For negative numbers one digit is reserved for the sign. return number % -(10 ** (max_digits - 1)) + + +def get_test_xml_file(file_name, from_file): + path_to_current_file = os.path.realpath(from_file) + current_directory = os.path.split(path_to_current_file)[0] + return os.path.join(current_directory, "importer_examples", file_name) + + +def preload_import(file_name, from_file, approve_workbasket=False): + file_to_import = get_test_xml_file( + file_name, + from_file, + ) + + workbasket = factories.WorkBasketFactory.create(status=WorkflowStatus.EDITING) + import_batch = factories.ImportBatchFactory.create(workbasket=workbasket) + factories.UserFactory.create() + + importer = TaricImporter( + import_batch=import_batch, + taric_xml_source=TaricXMLFileSource(file_to_import), + ) + + importer.process_and_save_if_valid(workbasket) + + if importer.can_save() and approve_workbasket: + # force publish workbasket + queue_workbasket_for_test(workbasket.id) + elif not importer.can_save() and approve_workbasket: # there + assert ( + False + ), "Tried to approve workbasket but the import contains issues, check the import" + + return importer + + +def queue_workbasket_for_test(workbasket_id): + user = factories.UserFactory.create() + + workbasket = WorkBasket.objects.all().get(id=int(workbasket_id)) + + # force publish workbasket + workbasket.full_clean() + workbasket.approve(int(user.id), "REVISION_ONLY") + workbasket.status = WorkflowStatus.QUEUED + workbasket.save() + + assert ( + WorkBasket.objects.all().get(id=int(workbasket_id)).status + == WorkflowStatus.QUEUED + ) + assert ( + WorkBasket.objects.all() + .get(id=int(workbasket_id)) + .transactions.first() + .partition + == 2 + ) diff --git a/importer/chunker.py b/importer/chunker.py index c2c4c9d08..daaca1df1 100644 --- a/importer/chunker.py +++ b/importer/chunker.py @@ -15,10 +15,9 @@ from importer.namespaces import nsmap from importer.namespaces import xsd_schema_paths from importer.utils import build_dependency_tree +from settings import MAX_IMPORT_FILE_SIZE -MAX_FILE_SIZE = 1024 * 1024 * 50 # Will keep chunks roughly close to 50MB Tags = make_schema_dataclass(xsd_schema_paths) - logger = getLogger(__name__) @@ -263,7 +262,7 @@ def write_transaction_to_chunk( .replace(b"xmlns:ns1=", b"xmlns:ns2="), ) - if chunk.tell() > MAX_FILE_SIZE: + if chunk.tell() > MAX_IMPORT_FILE_SIZE: key = (record_code, chapter_heading) if chapter_heading else record_code close_chunk(chunk, batch, key) chunks_in_progress.pop(key) diff --git a/importer/forms.py b/importer/forms.py index a18877d55..8704ce357 100644 --- a/importer/forms.py +++ b/importer/forms.py @@ -1,4 +1,5 @@ import os +from typing import Sequence import lxml from crispy_forms_gds.helper import FormHelper @@ -21,11 +22,79 @@ from importer.management.commands.run_import_batch import run_batch from importer.models import ImportBatch from importer.namespaces import TARIC_RECORD_GROUPS +from taric_parsers.importer import run_batch as run_batch_v2 from workbaskets.models import WorkBasket from workbaskets.validators import WorkflowStatus from workbaskets.validators import tops_jira_number_validator +class ImporterV2FormMixin: + """Mixin for taric parser forms, providing common taric_file clean and + processing support.""" + + def process_file( + self, + file: InMemoryUploadedFile, + batch, + user, + workbasket_title: str, + record_group: Sequence[str] = None, + ): + """ + Create ImporterXmlChunk associate with `batch`, and schedule parser + execution against `batch` conditional upon a chunk havinfg been created. + + The function returns the number of chunks created by the chunker. + + Note that a zero chunk count can result, for instance, when an imported + file contains no entities of interest, as can happen when a TGB file + contains only non-400 record code elements. A value of 0 (zero) is + returned by this function in such cases. + """ + + chunk_count = chunk_taric(file, batch, record_group=record_group) + + if chunk_count: + run_batch_v2( + batch_id=batch.pk, + username=user.username, + workbasket_title=workbasket_title, + ) + return chunk_count + + def clean_taric_file(self): + """Perform validation checks against the uploaded file.""" + uploaded_taric_file = self.cleaned_data["taric_file"] + generic_error_message = "The selected file could not be uploaded - try again" + + mime_type = get_mime_type(uploaded_taric_file) + if mime_type not in ["text/xml", "application/xml"]: + raise ValidationError("The selected file must be XML") + + try: + xml_file = parse_xml(uploaded_taric_file) + except (lxml.etree.XMLSyntaxError, DTDForbidden) as e: + if settings.SENTRY_ENABLED: + capture_exception(e) + raise ValidationError(generic_error_message) + + with open(self.xsd_file) as xsd_file: + xmlschema = lxml.etree.XMLSchema(file=xsd_file) + + try: + xmlschema.assertValid(xml_file) + except lxml.etree.DocumentInvalid as e: + if settings.SENTRY_ENABLED: + capture_exception(e) + raise ValidationError(generic_error_message) + + # read() in an InMemoryUploadedFile returns an empty string the second time it is called + # calling seek(0) again fixes this + # https://code.djangoproject.com/ticket/7812 + uploaded_taric_file.seek(0) + return uploaded_taric_file + + class ImportFormMixin: """Mixin for importer forms, providing common taric_file clean and processing support.""" @@ -184,7 +253,7 @@ def save(self, user: User): return batch -class CommodityImportForm(ImportFormMixin, forms.Form): +class CommodityImportForm(ImporterV2FormMixin, forms.Form): """Form used to create new instances of ImportBatch via upload of a commodity code file.""" @@ -266,37 +335,31 @@ def save(self): """ file_name = os.path.splitext(self.cleaned_data["name"])[0] description = f"TARIC {file_name} commodity code changes" - workbasket = WorkBasket.objects.create( - title=self.cleaned_data["workbasket_title"], - reason=description, - author=self.request.user, - ) - workbasket.save() - batch = ImportBatch( + import_batch = ImportBatch( author=self.request.user, name=self.cleaned_data["name"], goods_import=True, - workbasket=workbasket, ) - # ensure at the start of the file stream + self.files["taric_file"].seek(0, os.SEEK_SET) - batch.taric_file.save( + import_batch.taric_file.save( self.files["taric_file"].name, ContentFile(self.files["taric_file"].read()), ) - batch.save() + + import_batch.save() chunk_count = self.process_file( self.files["taric_file"], - batch, + import_batch, self.request.user, - workbasket_id=batch.workbasket.id, + workbasket_title=description, + record_group=list(TARIC_RECORD_GROUPS["commodities"]), ) - if not chunk_count: - # No chunks to process so the import is considered done and - # succeeded. - batch.succeeded() - batch.save() - return batch + if chunk_count < 1: + import_batch.failed_empty() + import_batch.save() + + return import_batch diff --git a/importer/handlers.py b/importer/handlers.py index d4e53ceb5..54fa9989f 100644 --- a/importer/handlers.py +++ b/importer/handlers.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -import re from copy import deepcopy from dataclasses import dataclass from typing import Iterable @@ -19,49 +18,11 @@ from importer.utils import DispatchedObjectType from importer.utils import LinksType from importer.utils import generate_key +from taric_parsers.importer_issue import ImportIssueReportItem logger = logging.getLogger(__name__) -class ImportIssueReportItem: - """ - Class for in memory representation if an issue detected on import, the - status may change during the process so this will not be committed until the - import process is complete or has been found to have errors. - - params: - object_type: str, - String representation of the object type, as found in XML e.g. goods.nomenclature - related_object_type: str, - String representation of the related object type, as found in XML e.g. goods.nomenclature.description - related_object_identity_keys: dict, - Dictionary of identity names and values used to link the related object - related_cache_key: str, - The string expected to be used to cache the related object - description: str, - Description of the detected issue - """ - - def __init__( - self, - object_type: str, - related_object_type: str, - related_object_identity_keys: Iterable[str], - related_cache_key: str, - description: str, - ): - self.object_type = object_type - self.related_object_type = related_object_type - self.related_object_identity_keys = related_object_identity_keys - self.related_cache_key = related_cache_key - self.description = description - - def missing_object_method_name(self): - """Returns a string representing the related object data type (but - replaces full stops with underscores for readability.""" - return re.sub("\\.", "_", self.related_object_type) - - @dataclass class DependencyMappingData: """ @@ -384,7 +345,7 @@ def resolve_dependencies(self) -> bool: dependencies.update(set(dependency.dependency_keys) - resolved_dependencies) return True - def _get_missing_dependencies(self) -> list: + def _get_missing_dependencies(self) -> list[str]: """ Returns a list of dependencies that are not in the cache. @@ -408,8 +369,8 @@ def get_generic_link(self, model, kwargs): Raises DoesNotExist if no kwargs passed. - First attempts to retrieve the object PK from the cache (saves queries). If this - is not found a database query is made to find the object. + First attempts to retrieve the object PK from the cache (saves queries). + If this is not found a database query is made to find the object. returns tuple(Object: model, bool: From Cache) """ @@ -501,8 +462,8 @@ def pre_save(self, data: dict, links: dict) -> dict: """ Pre-processing before the object is saved to the database. - Generally this is used for adding the links to the object (as these cannot - be easily validated against the serializer). + Generally this is used for adding the links to the object (as these + cannot be easily validated against the serializer). Return the final dataset to be used when saving to the database. """ @@ -525,11 +486,13 @@ def build(self) -> Set[str]: """ Build up all the data for the object. - This method co-ordinates the attempts to fetch the dependent data as well as the linked - data. If at any point one of these steps fails an empty set returns (signifying failure). + This method co-ordinates the attempts to fetch the dependent data as + well as the linked data. If at any point one of these steps fails an + empty set returns (signifying failure). - if all steps are deemed successful the object is dispatched to the database automatically. - On success a set of all the keys for any objects used which may be in the cache is returned. + if all steps are deemed successful the object is dispatched to the + database automatically. On success a set of all the keys for any objects + used which may be in the cache is returned. """ if not self.dependency_keys and not self.links: self.dispatch() @@ -553,27 +516,27 @@ def get_import_issues(self): if not self.resolve_dependencies(): # generic error - can do better to resolve later - dep_missing_details = "" + dep_missing_details_dict = {} for key in self._get_missing_dependencies(): missing_dependency_data = self._get_dependency_key_data(key) - dep_missing_details += f" type: {missing_dependency_data.tag}, " + dep_missing_details = ( + f"dependency missing of type {missing_dependency_data.tag}, " + ) for index, field in enumerate( missing_dependency_data.identifying_fields, ): - dep_missing_details += ( - f"{field}:{missing_dependency_data.data[field]} " - ) - - dep_missing_details += "." + dep_missing_details_dict[field] = missing_dependency_data.data[ + field + ] self.import_issues.append( ImportIssueReportItem( self.tag, missing_dependency_data.tag, - missing_dependency_data.identifying_fields, + dep_missing_details_dict, key, dep_missing_details, ), diff --git a/importer/jinja2/eu-importer/details.jinja b/importer/jinja2/eu-importer/details.jinja new file mode 100644 index 000000000..a9cddd61b --- /dev/null +++ b/importer/jinja2/eu-importer/details.jinja @@ -0,0 +1,42 @@ +{% extends "layouts/layout.jinja" %} +{% from "components/table/macro.njk" import govukTable %} + +{% set page_title = "Details for batch import" %} + +{% block breadcrumb %} + {{ breadcrumbs(request, [ + {"text": "Find and edit import batches", "href": url("taric_parser_import_ui_list")}, + {"text": page_title} + ]) }} +{% endblock %} + +{% block content %} +
+
+

{{ page_title }} : {{ object.name }}

+ {% set table_rows = [] %} + {% for issue in issues %} + {{ table_rows.append([ + {"text": issue.issue_type}, + {"text": issue.object_type}, + {"text": issue.object_update_type_name}, + {"text": issue.related_object_type}, + {"text": issue.object_data_to_str}, + {"text": issue.description}, + ]) or "" }} + {% endfor %} + {{ govukTable({ + "head": [ + {"text": "Severity"}, + {"text": "Object"}, + {"text": "Update Type"}, + {"text": "Related Object"}, + {"text": "Object Data"}, + {"text": "Description"}, + ], + "rows": table_rows + }) }} + +
+
+{% endblock %} diff --git a/importer/jinja2/eu-importer/select-imports.jinja b/importer/jinja2/eu-importer/select-imports.jinja index 6f7da55af..6c01a9a25 100644 --- a/importer/jinja2/eu-importer/select-imports.jinja +++ b/importer/jinja2/eu-importer/select-imports.jinja @@ -41,7 +41,6 @@ ] %} - {% block breadcrumb %} {{ govukBreadcrumbs({ "items": [ diff --git a/importer/jinja2/includes/taric_importer_list.jinja b/importer/jinja2/includes/taric_importer_list.jinja index b396bffca..e2128a178 100644 --- a/importer/jinja2/includes/taric_importer_list.jinja +++ b/importer/jinja2/includes/taric_importer_list.jinja @@ -7,6 +7,10 @@ {{ create_status_tag(object, status_tag_generator) }} {%- endset -%} + {%- set details_link -%} + {{ object.name }} + {%- endset %} + {%- set goods_status_cell -%} {% if goods_status(object) == "editable_goods" %} @@ -38,7 +42,7 @@ {%- endset -%} {{ table_rows.append([ - {"text": object.name}, + {"text": details_link}, {"text": "{:%d %b %Y}".format(object.created_at)}, {"text": author}, {"text": import_status_cell}, diff --git a/importer/migrations/0012_batchimporterror.py b/importer/migrations/0012_batchimporterror.py new file mode 100644 index 000000000..ee50bee8a --- /dev/null +++ b/importer/migrations/0012_batchimporterror.py @@ -0,0 +1,70 @@ +# Generated by Django 3.2.23 on 2023-12-04 12:08 + +import django.db.models.deletion +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + dependencies = [ + ("importer", "0011_importbatch_goods_import"), + ] + + operations = [ + migrations.CreateModel( + name="BatchImportError", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("object_type", models.CharField(max_length=250)), + ("related_object_type", models.CharField(max_length=250)), + ( + "related_object_identity_keys", + models.JSONField(default=None, null=True), + ), + ("description", models.CharField(max_length=2000)), + ( + "issue_type", + models.CharField( + choices=[ + ("ERROR", "Error"), + ("WARNING", "warning"), + ("INFORMATION", "Information"), + ], + max_length=50, + ), + ), + ("object_data", models.JSONField(default=None, null=True)), + ( + "object_update_type", + models.PositiveSmallIntegerField( + blank=True, + choices=[(1, "Update"), (2, "Delete"), (3, "Create")], + db_index=True, + null=True, + ), + ), + ("transaction_id", models.CharField(default=None, max_length=50)), + ( + "batch", + models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + related_name="issues", + to="importer.importbatch", + ), + ), + ], + options={ + "abstract": False, + }, + ), + ] diff --git a/importer/migrations/0013_alter_importbatch_status.py b/importer/migrations/0013_alter_importbatch_status.py new file mode 100644 index 000000000..64457bec0 --- /dev/null +++ b/importer/migrations/0013_alter_importbatch_status.py @@ -0,0 +1,29 @@ +# Generated by Django 3.2.23 on 2024-01-10 09:52 + +import django_fsm +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("importer", "0012_batchimporterror"), + ] + + operations = [ + migrations.AlterField( + model_name="importbatch", + name="status", + field=django_fsm.FSMField( + choices=[ + ("IMPORTING", "Importing"), + ("SUCCEEDED", "Succeeded"), + ("FAILED", "Failed"), + ("FAILED_EMPTY", "Failed Empty"), + ], + db_index=True, + default="IMPORTING", + editable=False, + max_length=50, + ), + ), + ] diff --git a/importer/models.py b/importer/models.py index a50497c5c..ac5031c1a 100644 --- a/importer/models.py +++ b/importer/models.py @@ -8,8 +8,11 @@ from django_fsm import FSMField from django_fsm import transition +from common import validators from common.models import TimestampedMixin from importer.storages import CommodityImporterStorage +from importer.validators import ImportIssueType +from taric_parsers.importer_issue import ImportIssueReportItem from workbaskets.util import clear_workbasket from workbaskets.validators import WorkflowStatus @@ -54,6 +57,9 @@ class ImportBatchStatus(models.TextChoices): FAILED = "FAILED", "Failed" """The import process completed / terminated but finished in some error state.""" + FAILED_EMPTY = "FAILED_EMPTY", "Failed Empty" + """The import was not able to complete, because there were no changes to + import.""" class ImportBatch(TimestampedMixin): @@ -121,17 +127,22 @@ def succeeded(self): """The import process completed and was successful.""" logger.info(f"Transitioning status of import pk={self.pk} to SUCCEEDED.") - if ( - not self.workbasket.tracked_models.exists() - and self.workbasket.status == WorkflowStatus.EDITING - ): - # Successful imports with an empty workbasket are archived. + if self.workbasket: + if ( + not self.workbasket.tracked_models.exists() + and self.workbasket.status == WorkflowStatus.EDITING + ): + # Successful imports with an empty workbasket are archived. + logger.info( + f"Archiving empty workbasket pk={self.workbasket.pk} " + f"associated with SUCCEEDED import pk={self.pk}.", + ) + self.workbasket.archive() + self.workbasket.save() + else: logger.info( - f"Archiving empty workbasket pk={self.workbasket.pk} " - f"associated with SUCCEEDED import pk={self.pk}.", + f"No workbasket associated with import pk={self.pk}.", ) - self.workbasket.archive() - self.workbasket.save() @transition( field=status, @@ -142,21 +153,57 @@ def succeeded(self): def failed(self): """The import process completed with an error condition.""" logger.info(f"Transitioning status of import pk={self.pk} to FAILED.") - - if self.workbasket.tracked_models.exists(): + if self.workbasket: + if self.workbasket.tracked_models.exists(): + logger.info( + f"Clearing workbasket pk={self.workbasket.pk} contents " + f"associated with FAILED import pk={self.pk}.", + ) + clear_workbasket(self.workbasket) + + if self.workbasket.status == WorkflowStatus.EDITING: + logger.info( + f"Archiving workbasket pk={self.workbasket.pk} " + f"associated with FAILED import pk={self.pk}.", + ) + self.workbasket.archive() + self.workbasket.save() + else: logger.info( - f"Clearing workbasket pk={self.workbasket.pk} contents " - f"associated with FAILED import pk={self.pk}.", + f"FAILED import pk={self.pk}.", ) - clear_workbasket(self.workbasket) - if self.workbasket.status == WorkflowStatus.EDITING: - logger.info( - f"Archiving workbasket pk={self.workbasket.pk} " - f"associated with FAILED import pk={self.pk}.", - ) - self.workbasket.archive() - self.workbasket.save() + def can_transition_to_failed_empty(self): + """ + This check is used in the transition to failed empty, which should only + be used before a workbasket is created. + + This is in support of the importerV2 which runs validation before + committing anything to the database can can be used to indicate the + import did not error but failed due to lack of content. The importerV1 + does not have this same behaviour, and will create a workbasket + regardless, so use the failed transition in this case. + """ + + if self.workbasket: + return False + return True + + @transition( + field=status, + source=ImportBatchStatus.IMPORTING, + target=ImportBatchStatus.FAILED_EMPTY, + on_error=ImportBatchStatus.FAILED, + conditions=[can_transition_to_failed_empty], + custom={"label": "Failed Empty"}, + ) + def failed_empty(self): + """The import process did not find any changes to import.""" + logger.info(f"Transitioning status of import pk={self.pk} to FAILED_EMPTY.") + + logger.info( + f"FAILED_EMPTY import pk={self.pk}.", + ) @property def ready_chunks(self): @@ -179,6 +226,113 @@ def __repr__(self) -> str: ) +class BatchImportError(TimestampedMixin): + """ + Batch Import Error. + + This class is used to represent an error on import, and should be used to + inform and assist to the import process when things go wrong. + + Most of the fields are populated with data read from the XML on import + attempt, and are populated when a record cant be created or has issues. + + This class has a *-1 relationship with ImportBatch + + This object is used at the end of an import to iterate through found issues + and persist them, there are other examples of issues being created outside + the TARIC parsing process, a bad file for example but the main use is to + persist detailed information for the user to review. + """ + + # the XML tag of an object, if required. Could be empty if an issue is related to a more generic error or the object type cant be determined + object_type = models.CharField(max_length=250) + + # the XML tag of a related object, if required. Could be empty if an issue is related to a more generic error or the object + # type cant be determined or there is no related object to the object type + related_object_type = models.CharField(max_length=250) + + # A dictionary containing identity fields and values for an object related to the object being imported. This field will be populated typically if + # an issue was identified where the related object expected by the import does not exist. + related_object_identity_keys = models.JSONField(default=None, null=True) + + # Text description of the encountered issue + description = models.CharField(max_length=2000) + + # Issue type, either ERROR, WARNING or INFO (from ImportIssueType choices) + issue_type = models.CharField( + max_length=50, + choices=ImportIssueType.choices, + ) + + # The BatchImport the BatchImportError relates to + batch = models.ForeignKey( + ImportBatch, + on_delete=models.PROTECT, + related_name="issues", + ) + + # A dictionary of the values for the object where applicable. This field will be blank for generic errors not related to an object. + object_data = models.JSONField(default=None, null=True) + + # Update type in the TARIC entry that the issue relates to, this can be null for issues relating to the import and not a specific + # record but typically will be populated with the numeric value relating to the update type + object_update_type: validators.UpdateType = models.PositiveSmallIntegerField( + choices=validators.UpdateType.choices, + db_index=True, + blank=True, + null=True, + ) + + # If this is related to a transaction, the transaction ID will be recorded here. This will be the ID in the XML. + transaction_id = models.CharField(max_length=50, default=None) + + @property + def object_update_type_name(self): + if self.object_update_type: + for update_type, update_type_name in validators.UpdateType.choices: + if update_type == self.object_update_type: + return update_type_name + return "" + + @property + def object_data_to_str(self): + str = "" + if self.object_data: + for key, value in self.object_data.items(): + str += f"{key}: {value}\n" + + return str + + @classmethod + def create_from_import_issue_report_item( + cls, + issue: ImportIssueReportItem, + import_batch: ImportBatch, + ) -> None: + """ + Creates a BatchImportError instance from the provided information, + committed to the database. + + Args: + issue: ImportIssueReportItem, An object containing information to report within BatchImportError + import_batch: BatchImport, the batch object that the issue will be linked to + + Returns: + None + """ + cls.objects.create( + batch=import_batch, + object_type=issue.object_type, + related_object_type=issue.related_object_type, + related_object_identity_keys=issue.related_object_identity_keys, + description=issue.description, + issue_type=issue.issue_type, + object_update_type=issue.object_update_type, + object_data=issue.object_data, + transaction_id=issue.transaction_id, + ) + + class ImporterXMLChunk(TimestampedMixin): """A chunk of TARIC XML.""" diff --git a/importer/namespaces.py b/importer/namespaces.py index 60727b371..117a7cdac 100644 --- a/importer/namespaces.py +++ b/importer/namespaces.py @@ -26,29 +26,31 @@ ("env", PATH_XSD_ENVELOPE), ("oub", PATH_XSD_TARIC), ) - """ -Define additional groups in the below dictionary -for use as a `record_group` argument -to importer.chunker.chunk_taric. +Define additional groups in the below dictionary for use as a `record_group` +argument to importer.chunker.chunk_taric. -Check importer.forms.UploadTaricForm.save -for example usage when users check -the 'Commodities Only' box in /importers/create. +Check importer.forms.UploadTaricForm.save for example usage when users check the +'Commodities Only' box in /importers/create. -The only group defined at the moment is commodities, -which is easily extensible to additional record groups. +The only group defined at the moment is commodities, which is easily extensible +to additional record groups. """ TARIC_RECORD_GROUPS: Dict[str, Sequence[str]] = dict( # Record "40020" is excluded from the below record group # because we don't want to synchronize footnote associations # with external systems when we align commodity code changes. + # + # Record "40025" is excluded from the below record group + # because we don't want to synchronize nomenclature group memberships + # with external systems when we align commodity code changes. commodities=( "40000", "40005", "40010", "40015", - "40025", + # "40020", footnote associations + # "40025", nomenclature group memberships "40035", "40040", ), diff --git a/importer/tasks.py b/importer/tasks.py index e74c377ff..4e1c10a43 100644 --- a/importer/tasks.py +++ b/importer/tasks.py @@ -6,7 +6,9 @@ from typing import Sequence from common.celery import app -from importer import models +from importer.models import ImportBatch +from importer.models import ImporterChunkStatus +from importer.models import ImporterXMLChunk from importer.taric import process_taric_xml_stream from importer.utils import build_dependency_tree from workbaskets.models import get_partition_scheme @@ -32,7 +34,7 @@ def import_chunk( """ partition_scheme = get_partition_scheme(partition_scheme_setting) - chunk = models.ImporterXMLChunk.objects.get(pk=chunk_pk) + chunk = ImporterXMLChunk.objects.get(pk=chunk_pk) batch = chunk.batch logger.info( @@ -43,7 +45,7 @@ def import_chunk( chunk.chunk_number, ) - chunk.status = models.ImporterChunkStatus.RUNNING + chunk.status = ImporterChunkStatus.RUNNING chunk.save() try: @@ -61,15 +63,15 @@ def import_chunk( batch.failed() batch.save() - chunk.status = models.ImporterChunkStatus.ERRORED + chunk.status = ImporterChunkStatus.ERRORED chunk.save() raise e - chunk.status = models.ImporterChunkStatus.DONE + chunk.status = ImporterChunkStatus.DONE chunk.save() batch_errored_chunks = batch.chunks.filter( - status=models.ImporterChunkStatus.ERRORED, + status=ImporterChunkStatus.ERRORED, ) if not batch.ready_chunks.exists(): if not batch_errored_chunks: @@ -92,7 +94,7 @@ def import_chunk( def setup_chunk_task( - batch: models.ImportBatch, + batch: ImportBatch, workbasket_id: str, workbasket_status: str, partition_scheme_setting: str, @@ -116,7 +118,7 @@ def setup_chunk_task( get_partition_scheme(partition_scheme_setting) if batch.ready_chunks.filter( - record_code=record_code, status=models.ImporterChunkStatus.RUNNING, **kwargs + record_code=record_code, status=ImporterChunkStatus.RUNNING, **kwargs ).exists(): return @@ -129,7 +131,7 @@ def setup_chunk_task( if not chunk: return - chunk.status = models.ImporterChunkStatus.RUNNING + chunk.status = ImporterChunkStatus.RUNNING chunk.save() import_chunk.delay( chunk.pk, @@ -142,7 +144,7 @@ def setup_chunk_task( def find_and_run_next_batch_chunks( - batch: models.ImportBatch, + batch: ImportBatch, workbasket_id: str, workbasket_status: str, partition_scheme_setting: str, @@ -180,7 +182,7 @@ def find_and_run_next_batch_chunks( logger.info(f"BatchImport {batch.pk} finished.") if ( - batch.chunks.exclude(status=models.ImporterChunkStatus.DONE) + batch.chunks.exclude(status=ImporterChunkStatus.DONE) .defer("chunk_text") .exists() ): @@ -188,7 +190,7 @@ def find_and_run_next_batch_chunks( logger.info("Batch %s errored", batch) return - for dependent_batch in models.ImportBatch.objects.depends_on( + for dependent_batch in ImportBatch.objects.depends_on( batch, ).dependencies_finished(): logger.info("setting up tasks for %s", dependent_batch) @@ -216,7 +218,7 @@ def find_and_run_next_batch_chunks( dependency_tree = build_dependency_tree() record_codes = set( - batch.chunks.exclude(status=models.ImporterChunkStatus.DONE) + batch.chunks.exclude(status=ImporterChunkStatus.DONE) .values_list("record_code", flat=True) .distinct(), ) diff --git a/importer/tests/conftest.py b/importer/tests/conftest.py index c615d81a8..c5cfafdce 100644 --- a/importer/tests/conftest.py +++ b/importer/tests/conftest.py @@ -62,7 +62,7 @@ def __init__(self): @pytest.fixture -def handler_class(mock_serializer) -> Type[BaseHandler]: +def parser_class(mock_serializer) -> Type[BaseHandler]: class TestHandler(BaseHandler): serializer_class = mock_serializer tag = "test_handler" @@ -158,8 +158,8 @@ def handler_footnote_type_description_test_data( @pytest.fixture -def prepped_handler(object_nursery, handler_class, handler_test_data) -> BaseHandler: - return handler_class(handler_test_data, object_nursery) +def prepped_handler(object_nursery, parser_class, handler_test_data) -> BaseHandler: + return parser_class(handler_test_data, object_nursery) @pytest.fixture diff --git a/importer/tests/test_chunker.py b/importer/tests/test_chunker.py index 19d4e788c..832b1454c 100644 --- a/importer/tests/test_chunker.py +++ b/importer/tests/test_chunker.py @@ -13,7 +13,6 @@ from common.tests import factories from common.tests.util import generate_test_import_xml from importer import chunker -from importer.chunker import MAX_FILE_SIZE from importer.chunker import chunk_taric from importer.chunker import filter_transaction_records from importer.chunker import get_chapter_heading @@ -24,6 +23,7 @@ from importer.models import ImporterXMLChunk from importer.namespaces import TTags from importer.namespaces import nsmap +from settings import MAX_IMPORT_FILE_SIZE from .test_namespaces import get_snippet_transaction @@ -241,7 +241,7 @@ def test_write_transaction_to_chunk_exceed_max_file_size( ) chunks_in_progress = {} chunk = BytesIO() - chunk.seek(MAX_FILE_SIZE + 1) + chunk.seek(MAX_IMPORT_FILE_SIZE + 1) record_code = get_record_code(transaction) chapter_heading = get_chapter_heading(transaction) key = (record_code, chapter_heading) diff --git a/importer/tests/test_files/additional_code_CREATE.xml b/importer/tests/test_files/additional_code_CREATE.xml new file mode 100644 index 000000000..d81d37516 --- /dev/null +++ b/importer/tests/test_files/additional_code_CREATE.xml @@ -0,0 +1,55 @@ + + + + + + + 1 + 120 + 00 + 1 + 3 + + 5 + 3 + 2021-01-01 + 2021-12-31 + + + + + + + + 1 + 245 + 00 + 2 + 3 + + 1 + 5 + 3 + 2021-01-01 + + + + + + + + 1 + 120 + 05 + 3 + 3 + + 5 + zz + some description + + + + + + \ No newline at end of file diff --git a/importer/tests/test_forms.py b/importer/tests/test_forms.py index 2524a9544..031711f4e 100644 --- a/importer/tests/test_forms.py +++ b/importer/tests/test_forms.py @@ -4,6 +4,7 @@ import pytest from django.core.files.uploadedfile import SimpleUploadedFile +from django.http import HttpRequest from importer import forms from workbaskets.validators import WorkflowStatus @@ -13,6 +14,7 @@ pytestmark = pytest.mark.django_db +@pytest.mark.importer_v2 def test_upload_taric_form_valid_envelope_id(): with open(f"{TEST_FILES_PATH}/valid.xml", "rb") as upload_file: data = { @@ -31,11 +33,12 @@ def test_upload_taric_form_valid_envelope_id(): assert form.is_valid() +@pytest.mark.importer_v2 @pytest.mark.parametrize("file_name,", ("invalid_id", "dtd")) @patch("importer.forms.capture_exception") def test_upload_taric_form_invalid_envelope(capture_exception, file_name, settings): """Test that form returns generic validation error and sentry captures - exception when given xml file with invalid id or document type + xception when given xml file with invalid id or document type declaration.""" settings.SENTRY_ENABLED = True with open(f"{TEST_FILES_PATH}/{file_name}.xml", "rb") as upload_file: @@ -56,6 +59,7 @@ def test_upload_taric_form_invalid_envelope(capture_exception, file_name, settin capture_exception.assert_called_once() +@pytest.mark.importer_v2 def test_import_form_non_xml_file(): """Test that form returns incorrect file type validation error when passed a text file instead of xml.""" @@ -75,6 +79,7 @@ def test_import_form_non_xml_file(): # https://uktrade.atlassian.net/browse/TP2000-486 # We forgot to add `self` to process_file params and no tests caught it. +@pytest.mark.importer_v2 @patch("importer.forms.chunk_taric") @patch("importer.forms.run_batch") def test_upload_taric_form_save(run_batch, chunk_taric, superuser): @@ -104,13 +109,15 @@ def test_upload_taric_form_save(run_batch, chunk_taric, superuser): @patch("importer.forms.chunk_taric") -@patch("importer.forms.run_batch") +@patch("importer.forms.run_batch_v2") +@pytest.mark.importer_v2 def test_commodity_import_form_valid_envelope( run_batch, chunk_taric, superuser, importer_storage, ): + chunk_taric.return_value = 1 """Test that form is valid when given valid xml file.""" mock_request = MagicMock() @@ -139,19 +146,16 @@ def test_commodity_import_form_valid_envelope( ): batch = form.save() assert batch.name.find(file_data["taric_file"].name) != -1 - assert batch.goods_import == True - assert batch.split_job == False + assert batch.goods_import is True + assert batch.split_job is False assert batch.author.id == superuser.id - assert batch.workbasket.title == data["workbasket_title"] - assert ( - batch.workbasket.reason - == f'TARIC {file_data["taric_file"].name[:-4]} commodity code changes' - ) + assert batch.workbasket is None - run_batch.assert_called_once() + # run_batch.assert_called_once() chunk_taric.assert_called_once() +@pytest.mark.importer_v2 @pytest.mark.parametrize("file_name,", ("invalid_id", "dtd")) @patch("importer.forms.capture_exception") def test_commodity_import_form_invalid_envelope(capture_exception, file_name, settings): @@ -176,10 +180,12 @@ def test_commodity_import_form_invalid_envelope(capture_exception, file_name, se capture_exception.assert_called_once() +@pytest.mark.importer_v2 def test_commodity_import_form_non_xml_file(): """Test that form returns incorrect file type validation error when passed a text file instead of xml.""" with open(f"{TEST_FILES_PATH}/invalid_type.txt", "rb") as upload_file: + form_req = HttpRequest() file_data = { "taric_file": SimpleUploadedFile( upload_file.name, @@ -187,12 +193,14 @@ def test_commodity_import_form_non_xml_file(): content_type="text", ), } - form = forms.CommodityImportForm({}, file_data) + + form = forms.CommodityImportForm({}, file_data, request=form_req) assert not form.is_valid() assert "The selected file must be XML" in form.errors["taric_file"] +@pytest.mark.importer_v2 # https://uktrade.atlassian.net/browse/TP2000-571 def test_commodity_import_form_long_definition_description(superuser): """Tests that form is valid when provided with QuotaDefinition description diff --git a/importer/tests/test_nursery.py b/importer/tests/test_nursery.py index 8142caea6..75fefa8ac 100644 --- a/importer/tests/test_nursery.py +++ b/importer/tests/test_nursery.py @@ -8,8 +8,8 @@ from importer import nursery -def test_nursery_gets_handler_with_tag(object_nursery, handler_class): - assert object_nursery.get_handler(handler_class.tag) is handler_class +def test_nursery_gets_handler_with_tag(object_nursery, parser_class): + assert object_nursery.get_handler(parser_class.tag) is parser_class def test_nursery_throws_error_on_no_handler(object_nursery): @@ -19,12 +19,12 @@ def test_nursery_throws_error_on_no_handler(object_nursery): @pytest.mark.django_db def test_nursery_clears_cache( - handler_class, + parser_class, object_nursery, date_ranges, unapproved_transaction, ): - handler = handler_class( + handler = parser_class( { "data": { "sid": 1, @@ -35,7 +35,7 @@ def test_nursery_clears_cache( "upper": date_ranges.normal.upper, }, }, - "tag": handler_class.tag, + "tag": parser_class.tag, "transaction_id": unapproved_transaction.pk, }, object_nursery, @@ -53,8 +53,8 @@ def test_nursery_clears_cache( assert TestModel1.objects.get().sid == 1 -def test_nursery_caches_object(object_nursery, handler_class): - handler = handler_class( +def test_nursery_caches_object(object_nursery, parser_class): + handler = parser_class( { "data": {"sid": 1}, "tag": "some unique tag", diff --git a/importer/tests/test_views.py b/importer/tests/test_views.py index bb792323f..b6cf21a5c 100644 --- a/importer/tests/test_views.py +++ b/importer/tests/test_views.py @@ -194,6 +194,7 @@ def test_commodity_importer_import_new_success_redirect(mock_save, valid_user_cl assert response.status_code == 200 +@pytest.mark.importer_v2 @pytest.mark.parametrize( "file_name,error_msg", [ diff --git a/importer/urls.py b/importer/urls.py index 6497ffca3..2949a02ef 100644 --- a/importer/urls.py +++ b/importer/urls.py @@ -31,6 +31,11 @@ views.CommodityImportCreateSuccessView.as_view(), name="commodity_importer-ui-create-success", ), + path( + "commodity-importer//details/", + views.CommodityImportDetails.as_view(), + name="commodity_importer-ui-details", + ), path( "download-admin-envelope//", views.DownloadAdminTaricView.as_view(), diff --git a/importer/validators.py b/importer/validators.py new file mode 100644 index 000000000..c4c718294 --- /dev/null +++ b/importer/validators.py @@ -0,0 +1,13 @@ +from django.db import models + + +class ImportIssueType(models.TextChoices): + ERROR = "ERROR", "Error" + """An error occurred that prevents the processing of the import.""" + + WARNING = "WARNING", "warning" + """An issue was detected, but not severe enough to prevent import.""" + + INFORMATION = "INFORMATION", "Information" + """Information about the import that is of note but not effecting the + success of the import.""" diff --git a/importer/views.py b/importer/views.py index 5c12042f9..972f948c6 100644 --- a/importer/views.py +++ b/importer/views.py @@ -18,6 +18,7 @@ from importer.filters import ImportBatchFilter from importer.filters import TaricImportFilter from importer.goods_report import GoodsReporter +from importer.models import ImportBatch from importer.models import ImportBatchStatus from notifications.models import GoodsSuccessfulImportNotification from workbaskets.validators import WorkflowStatus @@ -149,6 +150,8 @@ def status_tag_generator(cls, import_batch: ImportBatchFilter) -> dict: elif import_batch.status == ImportBatchStatus.FAILED: return {"text": "FAILED", "tag_class": "status-badge-red"} + elif import_batch.status == ImportBatchStatus.FAILED_EMPTY: + return {"text": "EMPTY", "tag_class": "status-badge-grey"} if workbasket: if ( @@ -208,6 +211,20 @@ def form_valid(self, form): ) +class CommodityImportDetails(RequiresSuperuserMixin, DetailView): + """UI endpoint for viewing details of a TARIC parser import, and view + failures and errors.""" + + model = ImportBatch + queryset = ImportBatch.objects.all() + template_name = "eu-importer/details.jinja" + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context["issues"] = context["object"].issues.all() + return context + + class CommodityImportCreateSuccessView(DetailView): """Commodity code import success view.""" diff --git a/pii-secret-exclude.txt b/pii-secret-exclude.txt index 679b7b249..fb246ad85 100644 --- a/pii-secret-exclude.txt +++ b/pii-secret-exclude.txt @@ -1,4 +1,5 @@ importer/tests/test_files/invalid_type.txt +taric_parsers/tests/support/invalid_type.txt common/fixtures/test-user-fixtures.json docs/requirements.txt docs/source/CODE_OF_CONDUCT.rst diff --git a/pyproject.toml b/pyproject.toml index 1f4389e19..ec3ae3bda 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,4 +96,6 @@ norecursedirs = [ "run", "venv", ] -addopts = "-n=auto" +markers = [ + "importer_v2" +] diff --git a/reports/tests/test_report_utils.py b/reports/tests/test_report_utils.py index a5a953b60..235e4ec6a 100644 --- a/reports/tests/test_report_utils.py +++ b/reports/tests/test_report_utils.py @@ -4,7 +4,10 @@ from reports.reports.base import ReportBase from reports.reports.base_table import ReportBaseTable from reports.reports.blank_goods_nomenclature_descriptions import Report -from reports.utils import * +from reports.utils import get_child_classes +from reports.utils import get_report_by_slug +from reports.utils import get_reports +from reports.utils import get_template_by_type class TestUtils: diff --git a/settings/common.py b/settings/common.py index b72a5d7c2..2d2bdf543 100644 --- a/settings/common.py +++ b/settings/common.py @@ -114,6 +114,7 @@ "quotas.apps.QuotasConfig", "reports.apps.ReportsConfig", "regulations.apps.RegulationsConfig", + "taric_parsers.apps.TaricParsersConfig", ] TAMATO_APPS = [ @@ -322,11 +323,19 @@ }, } +# Importer settings NURSERY_CACHE_ENGINE = os.getenv( "NURSERY_CACHE_ENGINE", "importer.cache.memory.MemoryCacheEngine", ) +# Maximum import file size (50mb) in bytes. This is an arbitrary value extracted from the importer +# HMRC have stipulated that exported envelopes should not exceed 40mb. +# A typical import of 40mb may result in an envelope of 20mb or less due to +# the selective process of which changes the UK tariff needs. This value provides a significant margin of +# distance from hitting the export limit in these instances. +MAX_IMPORT_FILE_SIZE = 1024 * 1024 * 50 + # Settings about retrying uploads if the bucket or endpoint cannot be contacted. # Names correspond to celery settings for retrying tasks: # https://docs.celeryproject.org/en/master/userguide/tasks.html#automatic-retry-for-known-exceptions @@ -552,6 +561,9 @@ re.compile(r"(importer)\.tasks\..*"): { "queue": "importer", }, + re.compile(r"(taric_parsers)\.tasks\..*"): { + "queue": "importer", + }, re.compile(r"(exporter|notifications|publishing)\.tasks\..*"): { "queue": "standard", }, diff --git a/taric_parsers/__init__.py b/taric_parsers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/taric_parsers/admin.py b/taric_parsers/admin.py new file mode 100644 index 000000000..846f6b406 --- /dev/null +++ b/taric_parsers/admin.py @@ -0,0 +1 @@ +# Register your models here. diff --git a/taric_parsers/apps.py b/taric_parsers/apps.py new file mode 100644 index 000000000..517085a45 --- /dev/null +++ b/taric_parsers/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class TaricParsersConfig(AppConfig): + name = "taric_parsers" diff --git a/taric_parsers/chunker.py b/taric_parsers/chunker.py new file mode 100644 index 000000000..b49e4a757 --- /dev/null +++ b/taric_parsers/chunker.py @@ -0,0 +1,256 @@ +import os +import xml.etree.ElementTree as ET +from logging import getLogger +from tempfile import TemporaryFile +from typing import Optional +from typing import Sequence + +from django.core.files.uploadedfile import InMemoryUploadedFile +from django.template.loader import render_to_string + +from importer import models +from importer.models import BatchImportError +from importer.models import ImportIssueType +from settings import MAX_IMPORT_FILE_SIZE +from taric_parsers.namespaces import make_schema_dataclass +from taric_parsers.namespaces import xsd_schema_paths + +Tags = make_schema_dataclass(xsd_schema_paths) + +logger = getLogger(__name__) + + +def find_or_create_chunk( + chunks_in_progress: dict, + envelope_id: str, + record_code=None, + chapter_heading=None, +) -> TemporaryFile: + """ + Find or create the chunk currently being written to. If the chunk has to be + created write the initial envelope header for it. + + Chunks are eventually written to the database as Django models, however + handling the large strings and storing it in memory before updating is (I + believe) O(n!). Whereas storing tempfiles and simply appending to the files + before writing to the database is O(n). + """ + key = (record_code, chapter_heading) if chapter_heading else record_code + + try: + chunk = chunks_in_progress[key] + except KeyError: + chunk = TemporaryFile() + chunk.write( + render_to_string(template_name="common/taric/start_file.xml").encode(), + ) + chunk.write( + render_to_string( + template_name="common/taric/start_envelope.xml", + context={"envelope_id": envelope_id}, + ).encode(), + ) + chunks_in_progress[key] = chunk + + return chunk + + +def close_chunk(chunk: TemporaryFile, batch: models.ImportBatch, key): + """ + Write a chunk to the database. + + To close a chunk properly it must have the envelope closing tag added before + being read into the db. + """ + chunk.write( + render_to_string(template_name="common/taric/end_envelope.xml").encode(), + ) + chunk.seek(0) + if isinstance(key, tuple): + record_code, chapter_heading = key + else: + record_code = key + chapter_heading = None + + models.ImporterXMLChunk.objects.create( + batch=batch, + record_code=record_code, + chapter=chapter_heading, + chunk_number=batch.chunks.filter( + record_code=record_code, + chapter=chapter_heading, + ).count(), + chunk_text=chunk.read().decode(), + ) + chunk.close() + + logger.info( + "closed chunk with code %s and chapter %s", + record_code, + chapter_heading, + ) + + +def write_transaction_to_chunk( + transaction: ET.Element, + chunks_in_progress: dict, + batch: models.ImportBatch, + envelope_id: str, +): + """ + Write a given transaction to the relevant chunk. + + Finds the chunk to write to. If the batch is a split_job the chunk is based + on record code and possibly chapter heading (for commodities and measures). + If the batch is not a split job it simply uses the current or next chunk. + + If a chunk reaches the given size limit it is written to the database and a + new chunk started. + """ + chapter_heading = None + record_code = None + + chunk = find_or_create_chunk( + chunks_in_progress, + envelope_id, + record_code=record_code, + chapter_heading=chapter_heading, + ) + + chunk.write( + ET.tostring( + transaction, + ) # pythons XML doesn't write namespaces back correctly. + .replace(b" int: + """ + Parses a TARIC3 XML stream and breaks it into a batch of chunks. + + All chunks are written to the database. If the batch is intended to be split + on record code then the commodity codes are also sorted into the correct + order. + + Returns the number of chunks created and associated with `batch`. + """ + chunks_in_progress = {} + + # set file position to start of stream + taric3_file.seek(0, os.SEEK_SET) + xmlparser = ET.iterparse(taric3_file, ["start", "end"]) + + element_counter = 0 + envelope_id = None + for event, elem in xmlparser: + if event == "start" and elem.tag == Tags.ENV_ENVELOPE.qualified_name: + envelope_id = elem.get("id", taric3_file.name) + if event != "end" or elem.tag != Tags.ENV_TRANSACTION.qualified_name: + continue + + transaction = filter_transaction_records(elem, record_group) + + if transaction is None: + continue + + write_transaction_to_chunk(transaction, chunks_in_progress, batch, envelope_id) + transaction.clear() + + element_counter += 1 + if element_counter % 100000 == 0: + logger.info("%d transactions done", element_counter) + + chunk_count = len(chunks_in_progress) + + for key, chunk in chunks_in_progress.items(): + close_chunk(chunk, batch, key) + + if batch.split_job: + BatchImportError.objects.create( + batch=batch, + description="This Import has been flagged as a split job. Please review file size. Maximum import size is 50mb, " + "anything larger than this should be split before attempting import. This importer does not support " + "split jobs", + object_update_type=None, + issue_type=ImportIssueType.ERROR, + transaction_id="", + object_type="", + ) + raise Exception( + "Unexpected split job, split jobs are not compatible with importer v2. Please split files up before importing.", + ) + + return chunk_count diff --git a/taric_parsers/forms.py b/taric_parsers/forms.py new file mode 100644 index 000000000..a48b83359 --- /dev/null +++ b/taric_parsers/forms.py @@ -0,0 +1,125 @@ +from typing import Sequence + +from crispy_forms_gds.helper import FormHelper +from crispy_forms_gds.layout import Layout +from crispy_forms_gds.layout import Size +from crispy_forms_gds.layout import Submit +from django import forms +from django.conf import settings +from django.contrib.auth.models import User +from django.core.files.uploadedfile import InMemoryUploadedFile +from django.db import transaction + +from importer.models import ImportBatch +from importer.namespaces import TARIC_RECORD_GROUPS +from taric_parsers.chunker import chunk_taric +from taric_parsers.importer import run_batch + + +class TaricParserFormMixin: + """Mixin for taric parser forms, providing common taric_file clean and + processing support.""" + + def process_file( + self, + file: InMemoryUploadedFile, + batch, + user, + workbasket_title: str, + record_group: Sequence[str] = None, + ): + """ + Create ImporterXmlChunk associate with `batch`, and schedule parser + execution against `batch` conditional upon a chunk havinfg been created. + + The function returns the number of chunks created by the chunker. + + Note that a zero chunk count can result, for instance, when an imported + file contains no entities of interest, as can happen when a TGB file + contains only non-400 record code elements. A value of 0 (zero) is + returned by this function in such cases. + """ + + chunk_count = chunk_taric(file, batch, record_group=record_group) + + if chunk_count: + run_batch( + batch_id=batch.pk, + username=user.username, + workbasket_title=workbasket_title, + ) + return chunk_count + + +class UploadTaricForm(TaricParserFormMixin, forms.ModelForm): + """ + Generic TARIC file import form, used to import TARIC files containing any. + + type of entity - Additional Codes, Certificates, Footnotes, etc. + """ + + class Meta: + model = ImportBatch + fields = ["name"] + + taric_file = forms.FileField( + required=True, + help_text="TARIC3 XML file containing non-goods entities.", + ) + + commodities_only = forms.BooleanField( + required=False, + label="Commodities Only", + ) + + xsd_file = settings.PATH_XSD_TARIC + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.fields["name"].help_text = ( + "Import name also used to name the workbasket created by and " + "associated with the import." + ) + + self.helper = FormHelper() + self.helper.label_size = Size.SMALL + self.helper.legend_size = Size.SMALL + + self.helper.layout = Layout( + "name", + "commodities_only", + "taric_file", + Submit( + "submit", + "Upload", + data_module="govuk-button", + data_prevent_double_click="true", + ), + ) + + @transaction.atomic + def save(self, user: User): + import_batch = super().save(commit=False) + import_batch.goods_import = False + import_batch.author = user + import_batch.save() + + if self.cleaned_data["commodities_only"]: + record_group = list(TARIC_RECORD_GROUPS["commodities"]) + else: + record_group = None + + chunk_count = self.process_file( + self.files["taric_file"], + import_batch, + user, + record_group=record_group, + workbasket_title=f"Import for {self.cleaned_data['name']}", + ) + + if chunk_count < 1: + import_batch.failed_empty() + import_batch.save() + + return import_batch diff --git a/taric_parsers/importer.py b/taric_parsers/importer.py new file mode 100644 index 000000000..5d6553273 --- /dev/null +++ b/taric_parsers/importer.py @@ -0,0 +1,1008 @@ +from typing import Generator +from typing import List +from typing import Optional + +from bs4 import BeautifulSoup +from django.db import IntegrityError +from django.db import transaction + +from common import validators +from common.models import Transaction +from common.validators import UpdateType +from importer.models import BatchImportError +from importer.models import ImportBatch +from importer.models import ImportIssueType +from taric.models import Envelope +from taric_parsers.parsers.additional_code_parsers import ( # noqa + AdditionalCodeDescriptionParserV2, +) +from taric_parsers.parsers.additional_code_parsers import ( # noqa + AdditionalCodeDescriptionPeriodParserV2, +) +from taric_parsers.parsers.additional_code_parsers import AdditionalCodeParserV2 # noqa +from taric_parsers.parsers.additional_code_parsers import ( # noqa + AdditionalCodeTypeDescriptionParserV2, +) +from taric_parsers.parsers.additional_code_parsers import ( # noqa + AdditionalCodeTypeParserV2, +) +from taric_parsers.parsers.additional_code_parsers import ( # noqa + FootnoteAssociationAdditionalCodeParserV2, +) +from taric_parsers.parsers.certificate_parser import ( # noqa + CertificateDescriptionParserV2, +) +from taric_parsers.parsers.certificate_parser import ( # noqa + CertificateDescriptionPeriodParserV2, +) +from taric_parsers.parsers.certificate_parser import CertificateParserV2 # noqa +from taric_parsers.parsers.certificate_parser import ( # noqa + CertificateTypeDescriptionParserV2, +) +from taric_parsers.parsers.certificate_parser import CertificateTypeParserV2 # noqa +from taric_parsers.parsers.commodity_parser import ( # noqa + FootnoteAssociationGoodsNomenclatureParserV2, +) +from taric_parsers.parsers.commodity_parser import ( # noqa + GoodsNomenclatureDescriptionParserV2, +) +from taric_parsers.parsers.commodity_parser import ( # noqa + GoodsNomenclatureDescriptionPeriodParserV2, +) +from taric_parsers.parsers.commodity_parser import ( # noqa + GoodsNomenclatureIndentParserV2, +) +from taric_parsers.parsers.commodity_parser import ( # noqa + GoodsNomenclatureOriginParserV2, +) +from taric_parsers.parsers.commodity_parser import GoodsNomenclatureParserV2 # noqa +from taric_parsers.parsers.commodity_parser import ( # noqa + GoodsNomenclatureSuccessorParserV2, +) +from taric_parsers.parsers.footnote_parser import FootnoteDescriptionParserV2 # noqa +from taric_parsers.parsers.footnote_parser import ( # noqa + FootnoteDescriptionPeriodParserV2, +) +from taric_parsers.parsers.footnote_parser import FootnoteParserV2 # noqa +from taric_parsers.parsers.footnote_parser import ( # noqa + FootnoteTypeDescriptionParserV2, +) +from taric_parsers.parsers.footnote_parser import FootnoteTypeParserV2 # noqa +from taric_parsers.parsers.geo_area_parser import ( # noqa + GeographicalAreaDescriptionParserV2, +) +from taric_parsers.parsers.geo_area_parser import ( # noqa + GeographicalAreaDescriptionPeriodParserV2, +) +from taric_parsers.parsers.geo_area_parser import GeographicalAreaParserV2 # noqa +from taric_parsers.parsers.geo_area_parser import GeographicalMembershipParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + AdditionalCodeTypeMeasureTypeParserV2, +) +from taric_parsers.parsers.measure_parser import ( # noqa + DutyExpressionDescriptionParserV2, +) +from taric_parsers.parsers.measure_parser import DutyExpressionParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + FootnoteAssociationMeasureParserV2, +) +from taric_parsers.parsers.measure_parser import ( # noqa + MeasureActionDescriptionParserV2, +) +from taric_parsers.parsers.measure_parser import MeasureActionParserV2 # noqa +from taric_parsers.parsers.measure_parser import MeasureComponentParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + MeasureConditionCodeDescriptionParserV2, +) +from taric_parsers.parsers.measure_parser import MeasureConditionCodeParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + MeasureConditionComponentParserV2, +) +from taric_parsers.parsers.measure_parser import MeasureConditionParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + MeasureExcludedGeographicalAreaParserV2, +) +from taric_parsers.parsers.measure_parser import MeasurementParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + MeasurementUnitDescriptionParserV2, +) +from taric_parsers.parsers.measure_parser import MeasurementUnitParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + MeasurementUnitQualifierDescriptionParserV2, +) +from taric_parsers.parsers.measure_parser import ( # noqa + MeasurementUnitQualifierParserV2, +) +from taric_parsers.parsers.measure_parser import MeasureParserV2 # noqa +from taric_parsers.parsers.measure_parser import MeasureTypeDescriptionParserV2 # noqa +from taric_parsers.parsers.measure_parser import MeasureTypeParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + MeasureTypeSeriesDescriptionParserV2, +) +from taric_parsers.parsers.measure_parser import MeasureTypeSeriesParserV2 # noqa +from taric_parsers.parsers.measure_parser import MonetaryUnitDescriptionParserV2 # noqa +from taric_parsers.parsers.measure_parser import MonetaryUnitParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaAssociationParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaBalanceEventParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaBlockingParserV2 # noqa +from taric_parsers.parsers.quota_parser import ( # noqa + QuotaClosedAndTransferredEventParserV2, +) +from taric_parsers.parsers.quota_parser import QuotaCriticalEventParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaDefinitionParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaEventParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaExhaustionEventParserV2 # noqa +from taric_parsers.parsers.quota_parser import ( # noqa + QuotaOrderNumberOriginExclusionParserV2, +) +from taric_parsers.parsers.quota_parser import QuotaOrderNumberOriginParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaOrderNumberParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaReopeningEventParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaSuspensionParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaUnblockingEventParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaUnsuspensionEventParserV2 # noqa +from taric_parsers.parsers.regulation_parser import BaseRegulationParserV2 # noqa +from taric_parsers.parsers.regulation_parser import ( # noqa + FullTemporaryStopActionParserV2, +) +from taric_parsers.parsers.regulation_parser import ( # noqa + FullTemporaryStopRegulationParserV2, +) +from taric_parsers.parsers.regulation_parser import ( # noqa + ModificationRegulationParserV2, +) +from taric_parsers.parsers.regulation_parser import ( # noqa + RegulationGroupDescriptionParserV2, +) +from taric_parsers.parsers.regulation_parser import RegulationGroupParserV2 # noqa +from taric_parsers.parsers.regulation_parser import ( # noqa + RegulationReplacementParserV2, +) +from taric_parsers.parsers.taric_parser import BaseTaricParser # noqa +from taric_parsers.parsers.taric_parser import ImportIssueReportItem # noqa +from taric_parsers.parsers.taric_parser import MessageParser # noqa +from taric_parsers.parsers.taric_parser import ParserHelper # noqa +from taric_parsers.parsers.taric_parser import TransactionParser # noqa +from taric_parsers.taric_xml_source import TaricXMLSourceBase +from taric_parsers.tasks import parse_and_import +from taric_parsers.validators import ImportStatus + + +class TaricImporter: + """ + TARIC importer. This class is initialised with either a TARIC 3 file or a + TARIC 3 XML string. Subsequently, the XML is parsed and objects in memory + are created and validated. + + If issues with the import are identified, the issues are logged in the + database against the import report. If the import has no issues the importer + proceeds to commit the data to the database. + """ + + bs_taric3_file: BeautifulSoup + raw_xml: str + parsed_transactions: List[TransactionParser] + + def __init__( + self, + import_batch: ImportBatch, + taric_xml_source: TaricXMLSourceBase, + ): + """ + TaricImporter initializer. This class imports TARIC data into the TAP + database, or reports on data issues encountered. + + Args: + import_batch: ImportBatch + This object is used to link instances of ImportIssueReportItem + taric_xml_source: TaricXMLSourceBase + Path to a local xml file that should be imported. + """ + + self.parsed_transactions = [] + self.raw_xml = taric_xml_source.get_xml_string() + + self.bs_taric3_file = BeautifulSoup(self.raw_xml, "xml") + + self.import_batch = import_batch + self.workbasket = None + self.parse() + self.validate() + + def process_and_save_if_valid(self, workbasket): + self.workbasket = workbasket + # validate: check dependencies and data + + if self.can_save(): + self.populate_parent_attributes() + self.commit_data() + + return self.status + + def commit_issues(self): + # Store issues against import + for issue in self.issues(): + BatchImportError.create_from_import_issue_report_item( + issue, + self.import_batch, + ) + + def clear_issues(self): + # clears issues against import + for parsed_transaction in self.parsed_transactions: + for message in parsed_transaction.parsed_messages: + message.taric_object.issues = [] + + def ordered_transactions_and_messages( + self, + up_to_transaction=None, + ) -> Generator[TransactionParser, MessageParser, None]: + """ + Returns TransactionParser, Message until att TransactionParsers have + been iterated or the up_to_transaction matches the current transaction. + + Args: + up_to_transaction: TransactionParser (optional) + Only iterate up to the provided transaction, inclusive. + + Returns: + Generator[TransactionParser, MessageParser] for all matching provided criteria + """ + for parsed_transaction in self.parsed_transactions: + for message in parsed_transaction.parsed_messages: + yield transaction, message + + # note: breaks after iterating each MessageParser in transaction. + if transaction == up_to_transaction: + break + + def __find_parent_for_parser_object(self, taric_object: BaseTaricParser): + """ + Finds a parent object within the same import, matching the key identity + fields to the child object. + + This method should only be used when it has been confirmed that it is a child object. Using this + function on an object that is not a child object will result in an exception being raised. + + Args: + taric_object: (required) BaseTaricParser, the child object we want to resolve the parent for. + + Returns: + BaseTaricParser, The parsed object that matches identity fields and is not a child parser. + Exception: + Raised when there is no match for the parent object. + """ + + if not taric_object.is_child_object(): + raise Exception(f"Only call this method on child objects") + + for ( + parsed_transaction, + parsed_message, + ) in self.ordered_transactions_and_messages(): + possible_parent_taric_object = parsed_message.taric_object + if ( + not possible_parent_taric_object.is_child_object() + and possible_parent_taric_object.__class__.model + == taric_object.__class__.model + ): + match = True + + for ( + key, + value, + ) in taric_object.get_identity_fields_and_values_for_parent().items(): + if ( + hasattr(possible_parent_taric_object, key) + and getattr(possible_parent_taric_object, key) != value + ): + match = False + + if match: + return possible_parent_taric_object + + raise Exception(f"No parent matched for {taric_object.__class__.__name__}") + + def populate_parent_attributes(self): + """ + Populates parent attributes from child objects, generally this applies + to description and description period objects, where in the database + stores both parent and child in the same table. + + Returns: + None, changes are applied to the instances of parser classes stored in memory. + """ + # need to copy all child attributes to parent objects within the import only + for ( + parsed_transaction, + parsed_message, + ) in self.ordered_transactions_and_messages(): + # skip if the update has been flagged as not to import changes + if not parsed_message.taric_object.import_changes: + continue + + if parsed_message.taric_object.is_child_object(): + # We only need the parent to be present for creation, if it's an update it can be applied in isolation + if parsed_message.update_type != validators.UpdateType.UPDATE: + parent = self.__find_parent_for_parser_object( + parsed_message.taric_object, + ) + attributes = parsed_message.taric_object.model_attributes( + self.workbasket.transactions.last(), + False, + ) + for attribute_key in attributes.keys(): + setattr(parent, attribute_key, attributes[attribute_key]) + + @transaction.atomic + def commit_data(self): + """ + Commit the import to the database, iterating through each parsed + transaction and the parsed objects contained within. + + Returns: + None + """ + + envelope = Envelope.new_envelope() + + transaction_order = 1 + + for parsed_transaction in self.parsed_transactions: + # create transaction + transaction_inst = Transaction.objects.create( + composite_key=f"{envelope.envelope_id}{transaction_order}", + workbasket=self.workbasket, + order=transaction_order, + ) + + for message in parsed_transaction.parsed_messages: + if not message.taric_object.import_changes: + continue + + if message.taric_object.can_save_to_model(): + self.commit_changes_from_message( + message, + transaction_inst, + ) + + transaction_order += 1 + + if len(self.issues(ImportIssueType.ERROR)) > 0: + transaction.set_rollback(True) + + def commit_changes_from_message( + self, + message: MessageParser, + transaction: Transaction, + ): + """ + Commit the changes in a parsed message to the database. + + Args: + message: (Required) MessageParser, The message being committed to the database. + transaction: (Required) Transaction, The database transaction the changes are being committed to. + + Returns: + None + """ + try: + if message.update_type == validators.UpdateType.UPDATE: # Update + # find model based on identity key + model_instance = ( + message.taric_object.__class__.model.objects.approved_up_to_transaction( + transaction, + ) + .filter(**message.taric_object.model_query_parameters()) + .last() + ) + + # update model with all attributes from model + model_instance.new_version( + transaction=transaction, + workbasket=transaction.workbasket, + **message.taric_object.model_attributes(transaction), + ) + + elif message.update_type == validators.UpdateType.DELETE: # Delete + model_instances = message.taric_object.__class__.model.objects.approved_up_to_transaction( + transaction, + ).filter( + **message.taric_object.model_query_parameters(), + ) + + if model_instances.count() == 1: + # mark the model as deleted + model_instances.first().new_version( + transaction=transaction, + workbasket=transaction.workbasket, + update_type=message.taric_object.update_type, + ) + elif model_instances.count() != 1: + if model_instances.count() > 1: + msg = "Multiple models matching query detected, please review data and correct before proceeding with this import." + else: + msg = "No matches for this model detected in published data, please verify record exists before attempting a delete of the record" + + self.create_import_issue( + message, + related_tag="self", + related_identity_keys=message.taric_object.model_query_parameters(), + issue_type=ImportIssueType.ERROR, + message=msg, + ) + + elif message.update_type == validators.UpdateType.CREATE: # Create + message.taric_object.__class__.model.objects.create( + transaction=transaction, + **message.taric_object.model_attributes( + transaction, + include_non_taric_attributes=True, + ), + ) + except IntegrityError as e: + self.create_import_issue( + message, + "None", + {}, + f"Database Integrity error, review related issues to determine what went wrong {e}", + ) + + @property + def status(self): + """ + Status of the import, indicating the types of warnings / errors + encountered. + + Returns: + str, string representation of the outcome of the import. + - COMPLETED : Successful import, no issues or warnings detected + - COMPLETED_WITH_WARNINGS : Successful import, but with warnings + - FAILED : failed import, no data committed to the database. Issues recorded against the import. + """ + if len(self.issues(ImportIssueType.ERROR)) > 0: + return ImportStatus.FAILED + elif len(self.issues(ImportIssueType.WARNING)) > 0: + return ImportStatus.COMPLETED_WITH_WARNINGS + else: + return ImportStatus.COMPLETED + + def can_save(self): + """ + Indicates if the current import can be saved. + Note: If warnings have been detected, the import can still be completed. The warnings will be available to review. + + Returns: + boolean, indicating the state of the import, and its ability to save due to no issues being recorded. + """ + + if self.status not in [ImportStatus.FAILED, ImportStatus.EMPTY]: + return True + return False + + def is_empty(self): + """ + Indicates if the current import is empty. + + Returns: + boolean, indicating the emptiness of the import. + """ + + return self.status == ImportStatus.EMPTY + + def parse(self): + """ + Iterates XML transaction nodes, parses and creates parsed transaction + instances. This populates self.parsed_transactions. + + Returns: + None + """ + transactions = self.bs_taric3_file.find_all("env:transaction") + + for index, xml_transaction in enumerate(transactions): + self.parsed_transactions.append(TransactionParser(xml_transaction, index)) + + def find_child_objects_in_import( + self, + child_parser_class, + parent: BaseTaricParser, + last_transaction: TransactionParser, + ): + """ + + Args: + child_parser_class: (required) BaseTaricParser Class, The class to search for. + parent: : (required) BaseTaricParser, parent object that we are looking for children of + last_transaction: (required) TransactionParser, The last transaction to check, and all preceding it. + + Returns: + list of BaseTaricParser matching parent criteria + """ + result = [] + + for ( + parsed_transaction, + parsed_message, + ) in self.ordered_transactions_and_messages(last_transaction): + if isinstance(parsed_message.taric_object, child_parser_class): + # check identity fields + if parsed_message.taric_object.is_child_for(parent): + result.append(parsed_message.taric_object) + + return result + + def find_parent_in_import( + self, + child_parser: BaseTaricParser, + up_to_transaction: TransactionParser, + ): + """ + Will return the latest matching parent up to the provided transaction in + the current import if it exists. If there is no match against the + parent, None will be returned. + + Args: + child_parser: (required) BaseTaricParser, The child parser instance. + up_to_transaction: (required) TransactionParser, The transaction that contains the child object. + + Returns: + Parent parser or None + """ + + parent = None + for ( + parsed_transaction, + parsed_message, + ) in self.ordered_transactions_and_messages(up_to_transaction): + potential_parent = parsed_message.taric_object + # matching model and not child? + if ( + not potential_parent.is_child_object() + and potential_parent.model == child_parser.model + ): + # check key fields + if child_parser.is_child_for(potential_parent): + parent = potential_parent + + return parent + + def create_import_issue( + self, + parsed_message, + related_tag="", + related_identity_keys=None, + message="", + issue_type=ImportIssueType.ERROR, + ): + """ + Creates an import issue that will be recorded against the database at + the end of the import process regardless of the success or failure. + + Args: + parsed_message: (required) MessageParser, Parsed message from the XML import + related_tag: (optional) str, the XML tag for the parsed message + related_identity_keys: (optional) dict, a dictionary of keys defining the identity of the object. + message: (optional) str, A string describing the encountered issue / warning. + issue_type: (optional) str, from choices BatchImportErrorIssueType.choices (ERROR, WARNING and INFORMATION) + """ + + if related_identity_keys is None: + related_identity_keys = {} + + report_item = ImportIssueReportItem( + parsed_message.taric_object.xml_object_tag, + related_tag, + related_identity_keys, + message, + object_update_type=parsed_message.update_type, + object_data=parsed_message.taric_object.model_attributes( + Transaction.objects.approved().last(), + raise_import_issue_if_no_match=False, + json_compatible=True, + ), + transaction_id=parsed_message.transaction_id, + issue_type=issue_type, + ) + + parsed_message.taric_object.issues.append(report_item) + + def validate(self): + """ + Iterate through transactions and each taric model within, and verify + progressively from the first transaction onwards, but not looking + forwards for related objects, only each transaction backwards.0. + + This method should raise import issues with missing data where the child + object is not present + + Returns: + None, any outoput is appended as sisues + """ + + for parsed_transaction in self.parsed_transactions: + for parsed_message in parsed_transaction.parsed_messages: + # Check update type + self.validate_update_type_for(parsed_message, parsed_transaction) + + # get the child models + child_parser_classes = ParserHelper.get_child_parsers( + parsed_message.taric_object, + ) + + # No child classes, we are good to go here + if len(child_parser_classes) == 0: + continue + + # verify if a child of these types, linked to the current object exist in previous / current + # transactions + child_matches = {} + for child_parser_class in child_parser_classes: + object_matches = self.find_child_objects_in_import( + child_parser_class, + parsed_message.taric_object, + parsed_transaction, + ) + for object_match in object_matches: + if object_match.__class__.__name__ in child_matches.keys(): + child_matches[object_match.__class__.__name__] += 1 + else: + child_matches[object_match.__class__.__name__] = 1 + + for child_parser_class in child_parser_classes: + if child_parser_class.__name__ not in child_matches.keys(): + # This is where description periods can inherit from last create / update if the parsed message is an update to an existing object + if not parsed_message.can_populate_child_attrs_from_history(): + self.create_import_issue( + parsed_message, + child_parser_class.xml_object_tag, + {}, + f"Missing expected child object {child_parser_class.__name__}", + ) + + def validate_update_type_update(self, parsed_message, parsed_transaction): + """ + Validates a single parsed message that is an UPDATE to a taric object, + within a transaction and adds any detected issues as import issues + (ImportIssueReportItem) that are later stored in BatchImportError + objects, adn committed to the database. + + Args: + parsed_message: MessageParser + The message that requires validation + parsed_transaction: TransactionParser + The transaction that contains the message that requires validation + + Returns: + None + """ + if not parsed_message.taric_object.__class__.updates_allowed: + self.create_import_issue( + parsed_message, + "", + parsed_message.taric_object.model_query_parameters(), + f"Taric objects of type {parsed_message.taric_object.__class__.model.__name__} can't be updated", + ) + + # Check if updated, deleted object exists, else raise issue + model_instances = parsed_message.taric_object.__class__.model.objects.latest_approved().filter( + **parsed_message.taric_object.model_query_parameters(), + ) + + last_parsed_message_for_model = None + + for tmp_transaction in self.parsed_transactions: + if parsed_transaction.index == tmp_transaction.index: + break # done, don't want to process same transaction as the object we are looking at + + for message in tmp_transaction.parsed_messages: + # Check for match of identifying fields + if ( + message.taric_object.model_query_parameters() + == parsed_message.taric_object.model_query_parameters() + ): + if type(message.taric_object) is type(parsed_message.taric_object): + # We have a match + last_parsed_message_for_model = message + + change_valid = True + message = "" + + # If there are not any entries for this model, prior to this change : not valid + if not last_parsed_message_for_model and model_instances.count() == 0: + change_valid = False + message = ( + f"Identity keys do not match an existing object in database or import, cant apply update to a deleted or non existent object", + ) + # If the model has been deleted previously in the same envelope : not valid + elif ( + last_parsed_message_for_model + and last_parsed_message_for_model.update_type + == validators.UpdateType.DELETE + ): + change_valid = False + message = ( + f"Identity keys match a previous message in this import that deletes this object", + ) + + if not change_valid: + self.create_import_issue( + parsed_message, + "", + parsed_message.taric_object.model_query_parameters(), + message, + ) + + @staticmethod + def find_change_in_parsed_transaction( + parsed_transaction, + parser_class, + identity_fields, + ) -> Optional[BaseTaricParser]: + """ + Finds any changes to an object based on the parser class and the + identity fields provided. + + Args: + parsed_transaction: TransactionParser + The parsed transaction containing messages to check + parser_class: + The class (child of BaseTaricParser) that we are searching for + identity_fields: dict + A dictionary of keys and values that are used to identify a parsed message of type (parser_clas) + + Returns: + BaseTaricParser or child of : When a match is found + None : When no match is found + """ + for parsed_message in parsed_transaction.parsed_messages: + if parsed_message.taric_object is parser_class: + match = True + + for identity_field in identity_fields.keys(): + if ( + getattr(parsed_message.taric_object, identity_field) + != identity_fields[identity_field] + ): + match = False + + if match and len(identity_fields.keys()) > 0: + return parsed_message.taric_object + + return None + + def validate_update_type_delete(self, parsed_message, parsed_transaction): + """ + Validates a single parsed message that is an DELETE to a taric object, + within a transaction and adds any detected issues as import issues + (ImportIssueReportItem) that are later stored in BatchImportError + objects, adn committed to the database. + + Args: + parsed_message: MessageParser + The message that requires validation + parsed_transaction: TransactionParser + The transaction that contains the message that requires validation + + Returns: + None + """ + if not parsed_message.taric_object.__class__.deletes_allowed: + if parsed_message.taric_object.is_child_object: + # is the parent being deleted in the same transaction? + if self.find_change_in_parsed_transaction( + parsed_transaction, + ParserHelper.get_parser_by_model( + parsed_message.taric_object.__class__.model, + ), + parsed_message.taric_object.get_identity_fields_and_values_for_parent(), + ): + return + + parsed_message.taric_object.import_changes = False + + msg = f"Children of Taric objects of type {parsed_message.taric_object.__class__.model.__name__} can't be deleted directly. This change will not be imported." + issue_type = ImportIssueType.WARNING + else: + msg = f"Taric objects of type {parsed_message.taric_object.__class__.model.__name__} can't be deleted" + issue_type = ImportIssueType.ERROR + + self.create_import_issue( + parsed_message, + "", + parsed_message.taric_object.model_query_parameters(), + msg, + issue_type, + ) + + # Check if updated, deleted object exists, else raise issue + model_instances = parsed_message.taric_object.__class__.model.objects.latest_approved().filter( + **parsed_message.taric_object.model_query_parameters(), + ) + + last_parsed_message_for_model = None + + for tmp_transaction in self.parsed_transactions: + if parsed_transaction.index == tmp_transaction.index: + break # done, don't want to process same transaction as the object we are looking at + + for message in tmp_transaction.parsed_messages: + # Check for match of identifying fields + if ( + message.taric_object.model_query_parameters() + == parsed_message.taric_object.model_query_parameters() + ): + if type(message.taric_object) is type(parsed_message.taric_object): + # We have a match + last_parsed_message_for_model = message + + change_valid = True + message = "" + + # If there are not any entries for this model, prior to this change : not valid + if not last_parsed_message_for_model and model_instances.count() == 0: + change_valid = False + message = ( + f"Identity keys do not match an existing object in database or import, cant delete non existent object", + ) + # If the model has been deleted previously in the same envelope : not valid + elif ( + last_parsed_message_for_model + and last_parsed_message_for_model.update_type + == validators.UpdateType.DELETE + ): + change_valid = False + message = ( + f"Identity keys match a previous message in this import that deletes this object", + ) + + if not change_valid: + self.create_import_issue( + parsed_message, + "", + parsed_message.taric_object.model_query_parameters(), + message, + ) + + def validate_update_type_create(self, parsed_message, parsed_transaction): + """ + Validates a single parsed message that is an CREATE to a taric object, + within a transaction and adds any detected issues as import issues + (ImportIssueReportItem) that are later stored in BatchImportError + objects, adn committed to the database. + + Args: + parsed_message: MessageParser + The message that requires validation + parsed_transaction: TransactionParser + The transaction that contains the message that requires validation + + Returns: + None + """ + if parsed_message.taric_object.is_child_object(): + parent_parser_class = ParserHelper.get_parser_by_model( + parsed_message.taric_object.__class__.model, + ) + parent = self.find_parent_in_import( + parsed_message.taric_object, + parsed_transaction, + ) + + if parent is None: + self.create_import_issue( + parsed_message, + parent_parser_class.xml_object_tag, + parsed_message.taric_object.get_identity_fields_and_values_for_parent(), + f"Missing expected parent object {parent_parser_class.__name__}", + ) + + return None + + last_parsed_message_for_model = None + + for tmp_transaction in self.parsed_transactions: + if parsed_transaction.index == tmp_transaction.index: + break # done, don't want to process same transaction as the object we are looking at + + for message in tmp_transaction.parsed_messages: + # Check for match of identifying fields + if ( + message.taric_object.model_query_parameters() + == parsed_message.taric_object.model_query_parameters() + ): + if type(message.taric_object) is type( + parsed_message.taric_object, + ): + # We have a match + last_parsed_message_for_model = message + + # Check if record exists for identity keys + model_instances = ( + parsed_message.taric_object.__class__.model.objects.all().filter( + **parsed_message.taric_object.model_query_parameters(), + ) + ) + + # check for deletes + create_issue = False + if not parsed_message.taric_object.skip_identity_check: + if ( + model_instances.count() > 0 + and model_instances.last().update_type != UpdateType.DELETE + ): + create_issue = True + elif ( + last_parsed_message_for_model + and last_parsed_message_for_model.update_type != UpdateType.DELETE + ): + create_issue = True + + if create_issue: + self.create_import_issue( + parsed_message, + "", + parsed_message.taric_object.model_attributes( + Transaction.objects.approved().last(), + raise_import_issue_if_no_match=False, + json_compatible=True, + ), + f"Identity keys match existing non-deleted object in database (checking all published and unpublished data)", + ) + + def validate_update_type_for(self, parsed_message, parsed_transaction): + """ + Determines the appropriate method to apply validation for a + MessageParser instance. + + Args: + parsed_message: MessageParser + The instance we want to validate + parsed_transaction: TransactionParser + The transaction that contains the MessageParser instance + + Returns: + None + """ + if parsed_message.update_type == validators.UpdateType.CREATE: # Create + self.validate_update_type_create(parsed_message, parsed_transaction) + if parsed_message.update_type == validators.UpdateType.UPDATE: # Update + self.validate_update_type_update(parsed_message, parsed_transaction) + if parsed_message.update_type == validators.UpdateType.DELETE: # Delete + self.validate_update_type_delete(parsed_message, parsed_transaction) + + def issues(self, filter_by_issue_type: str = None) -> List[ImportIssueReportItem]: + """ + Get issues identified during import. + + Args: + filter_by_issue_type: (optional) str, either ERROR or WARNING + + Returns: + list[ImportIssueReportItem], recorded issues during import + """ + issues = [] + for parsed_transaction in self.parsed_transactions: + for message in parsed_transaction.parsed_messages: + for issue in message.taric_object.issues: + if filter_by_issue_type: + if issue.issue_type == filter_by_issue_type: + issues.append(issue) + else: + issues.append(issue) + + return issues + + +def run_batch( + batch_id: int, + username: str, + workbasket_title: str, +): + import_batch = ImportBatch.objects.get(pk=batch_id) + + parse_and_import.delay( + chunk_pk=import_batch.chunks.first().pk, + workbasket_title=workbasket_title, + username=username, + ) diff --git a/taric_parsers/importer_issue.py b/taric_parsers/importer_issue.py new file mode 100644 index 000000000..54726d827 --- /dev/null +++ b/taric_parsers/importer_issue.py @@ -0,0 +1,59 @@ +import re + +from importer.validators import ImportIssueType + + +class ImportIssueReportItem: + """ + Class for in memory representation if an issue detected on import, the + status may change during the process so this will not be committed until the + import process is complete or has been found to have errors. + + params: + object_type: str, + String representation of the object type, as found in XML e.g. goods.nomenclature + related_object_type: str, + String representation of the related object type, as found in XML e.g. goods.nomenclature.description + related_object_identity_keys: dict, + Dictionary of identity names and values used to link the related object + related_cache_key: str, + The string expected to be used to cache the related object + description: str, + Description of the detected issue + """ + + def __init__( + self, + object_type: str, + related_object_type: str, + related_object_identity_keys: dict = None, + description: str = None, + issue_type: str = ImportIssueType.ERROR, + object_update_type: int = None, + object_data: dict = None, + transaction_id: int = 0, + ): + self.object_type = object_type + self.related_object_type = related_object_type + self.related_object_identity_keys = related_object_identity_keys + self.description = description + self.issue_type = issue_type + self.object_update_type = object_update_type + self.object_data = object_data + self.transaction_id = transaction_id + + def __str__(self): + result = ( + f"{self.issue_type}: {self.description}\n" + f" {self.object_type} > {self.related_object_type}\n" + f" link_data: {self.related_object_identity_keys}" + ) + return result + + def __repr__(self): + return self.__str__() + + def missing_object_method_name(self): + """Returns a string representing the related object data type (but + replaces full stops with underscores for readability.""" + return re.sub("\\.", "_", self.related_object_type) diff --git a/taric_parsers/jinja2/includes/importer_list.jinja b/taric_parsers/jinja2/includes/importer_list.jinja new file mode 100644 index 000000000..8c51a142f --- /dev/null +++ b/taric_parsers/jinja2/includes/importer_list.jinja @@ -0,0 +1,24 @@ +{% set table_rows = [] %} +{% for object in object_list %} + {{ table_rows.append([ + {"text": object.pk}, + {"text": object.name}, + {"text": "{:%d %b %Y}".format(object.created_at)}, + {"text": object.chunks_waiting}, + {"text": object.chunks_running}, + {"text": object.chunks_done}, + {"text": object.chunks_errored}, + ]) or "" }} +{% endfor %} +{{ govukTable({ + "head": [ + {"text": "ID"}, + {"text": "Name"}, + {"text": "Start date"}, + {"text": "Chunks waiting"}, + {"text": "Chunks running"}, + {"text": "Chunks done"}, + {"text": "Chunks errored"}, + ], + "rows": table_rows +}) }} diff --git a/taric_parsers/jinja2/includes/taric_importer_list.jinja b/taric_parsers/jinja2/includes/taric_importer_list.jinja new file mode 100644 index 000000000..4527293a9 --- /dev/null +++ b/taric_parsers/jinja2/includes/taric_importer_list.jinja @@ -0,0 +1,55 @@ +{% set table_rows = [] %} + +{% for object in object_list %} + {%- set import_status_cell -%} + {{object.status}} + {%- endset -%} + + {%- set goods_status_cell -%} + {% if goods_status(object) == "editable_goods" %} +
+ + + + +
+ {% elif goods_status(object) == "no_goods" %} + 0 goods items + {%- else -%} + + {%- endif -%} + {%- endset -%} + + {%- set author -%} + {% if object.author %} + {% if object.author.get_full_name() %} + {{ object.author.get_full_name() }} + {% elif object.author.email %} + {{ object.author.email }} + {% else %} + {{ object.author.username }} + {% endif %} + {% else %} + - + {% endif %} + {%- endset -%} + + {{ table_rows.append([ + {"text": object.name}, + {"text": "{:%d %b %Y}".format(object.created_at)}, + {"text": author}, + {"text": import_status_cell}, + {"html": goods_status_cell, "classes": "goods-status " ~ goods_status(object)}, + ]) or "" }} +{% endfor %} + +{{ govukTable({ + "head": [ + {"text": "Taric ID number"}, + {"text": "Date added"}, + {"text": "Uploaded by"}, + {"text": "Importer status"}, + {"text": ""}, + ], + "rows": table_rows +}) }} diff --git a/taric_parsers/jinja2/taric_parser/create.jinja b/taric_parsers/jinja2/taric_parser/create.jinja new file mode 100644 index 000000000..403269b29 --- /dev/null +++ b/taric_parsers/jinja2/taric_parser/create.jinja @@ -0,0 +1,25 @@ +{% extends "layouts/layout.jinja" %} + +{% from "components/button/macro.njk" import govukButton %} +{% from "components/input/macro.njk" import govukInput %} +{% from "components/table/macro.njk" import govukTable %} + +{% set page_title = "Create a new import batch" %} + +{% block breadcrumb %} + {{ breadcrumbs(request, [ + {"text": "Find and edit import batches", "href": url("taric_parser_import_ui_list")}, + {"text": page_title} + ]) + }} +{% endblock %} + +{% block content %} +
+
+

{{ page_title }}

+ + {{ crispy(form, context={"csrf_token": csrf_token, "request": request}) }} +
+
+{% endblock %} diff --git a/taric_parsers/jinja2/taric_parser/details.jinja b/taric_parsers/jinja2/taric_parser/details.jinja new file mode 100644 index 000000000..a9cddd61b --- /dev/null +++ b/taric_parsers/jinja2/taric_parser/details.jinja @@ -0,0 +1,42 @@ +{% extends "layouts/layout.jinja" %} +{% from "components/table/macro.njk" import govukTable %} + +{% set page_title = "Details for batch import" %} + +{% block breadcrumb %} + {{ breadcrumbs(request, [ + {"text": "Find and edit import batches", "href": url("taric_parser_import_ui_list")}, + {"text": page_title} + ]) }} +{% endblock %} + +{% block content %} +
+
+

{{ page_title }} : {{ object.name }}

+ {% set table_rows = [] %} + {% for issue in issues %} + {{ table_rows.append([ + {"text": issue.issue_type}, + {"text": issue.object_type}, + {"text": issue.object_update_type_name}, + {"text": issue.related_object_type}, + {"text": issue.object_data_to_str}, + {"text": issue.description}, + ]) or "" }} + {% endfor %} + {{ govukTable({ + "head": [ + {"text": "Severity"}, + {"text": "Object"}, + {"text": "Update Type"}, + {"text": "Related Object"}, + {"text": "Object Data"}, + {"text": "Description"}, + ], + "rows": table_rows + }) }} + +
+
+{% endblock %} diff --git a/taric_parsers/jinja2/taric_parser/importer_list.jinja b/taric_parsers/jinja2/taric_parser/importer_list.jinja new file mode 100644 index 000000000..87f7ef17f --- /dev/null +++ b/taric_parsers/jinja2/taric_parser/importer_list.jinja @@ -0,0 +1,31 @@ +{% set table_rows = [] %} +{% for object in object_list %} +{% set details_link -%} +{{ object.name }} +{%- endset %} +{%- set workbasket_linked_id -%} + {{ object.workbasket_id }} + {%- endset -%} +{{ table_rows.append([ + {"text": object.pk}, + {"text": details_link}, + {"text": object.status}, + {"text": object.import_issues_error_count}, + {"text": object.import_issues_warning_count}, + {"text": workbasket_linked_id}, + ]) or "" }} +{% endfor %} +{{ govukTable({ + "head": [ + {"text": "ID"}, + {"text": "Name"}, + {"text": "Status"}, + {"text": "Errors"}, + {"text": "Warnings"}, + {"text": "workbasket link"}, + ], + "rows": table_rows +}) }} diff --git a/taric_parsers/jinja2/taric_parser/list.jinja b/taric_parsers/jinja2/taric_parser/list.jinja new file mode 100644 index 000000000..f8da6ae93 --- /dev/null +++ b/taric_parsers/jinja2/taric_parser/list.jinja @@ -0,0 +1,7 @@ +{% set object_type = "import batch" %} +{% set object_type_plural = "import batches" %} +{% set form_url = "taric_parser_import_ui_list" %} +{% set list_include = "taric_parser/importer_list.jinja" %} + + +{%- include "layouts/list.jinja" -%} diff --git a/taric_parsers/migrations/__init__.py b/taric_parsers/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/taric_parsers/namespaces.py b/taric_parsers/namespaces.py new file mode 100644 index 000000000..a8004508e --- /dev/null +++ b/taric_parsers/namespaces.py @@ -0,0 +1,185 @@ +"""Provides dataclasses and config classes for xml elements and the taric +schema.""" + +import re +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from dataclasses import field +from dataclasses import make_dataclass +from typing import Dict +from typing import Iterator +from typing import Sequence +from typing import Tuple +from typing import TypeVar +from typing import Union + +from common.xml.namespaces import SEED_MESSAGE +from common.xml.namespaces import nsmap +from settings.common import PATH_XSD_ENVELOPE +from settings.common import PATH_XSD_TARIC + +TTag = TypeVar("TTag", bound="Tag") +TTags = TypeVar("TTags", bound="SchemaTagsBase") + +# pattern used to identify a string as a regex or regular string +RE_PATTERN_TEST = re.compile(r"[^A-Za-z\.\_]") + +xsd_schema_paths: Tuple[Tuple[str, any], Tuple[str, any]] = ( + ("env", PATH_XSD_ENVELOPE), + ("oub", PATH_XSD_TARIC), +) +""" +Define additional groups in the below dictionary for use as a `record_group` +argument to importer.chunker.chunk_taric. + +Check importer.forms.UploadTaricForm.save for example usage when users check the +'Commodities Only' box in /importers/create. + +The only group defined at the moment is commodities, which is easily extensible +to additional record groups. +""" +TARIC_RECORD_GROUPS: Dict[str, Sequence[str]] = dict( + # Record "40020" is excluded from the below record group + # because we don't want to synchronize footnote associations + # with external systems when we align commodity code changes. + # + # Record "40025" is excluded from the below record group + # because we don't want to synchronize nomenclature group memberships + # with external systems when we align commodity code changes. + commodities=( + "40000", + "40005", + "40010", + "40015", + # "40020", footnote associations + # "40025", nomenclature group memberships + "40035", + "40040", + ), +) + + +@dataclass +class Tag: + """ + A dataclass for xml element tags. + + :py:attr:`name` corresponds to the name attribute of the Element element in the XML + Schema. + + :py:attr:`prefix` reflects namespace prefixes defined in the taric3 and envelope + xsd-s. + + :py:attr:`nsmap` this is a prefix-namespace mapping in the format required by + xml.etree.ElementTree + """ + + name: str + prefix: str = field(default=SEED_MESSAGE) + nsmap: Dict[str, str] = field(default_factory=lambda: nsmap) + + @property + def namespace(self) -> str: + """Returns the namespace for the tag.""" + return self.nsmap.get(self.prefix) + + @property + def qualified_name(self) -> str: + """Returns a fully qualified element tag.""" + ns = self.namespace + + if ns is None: + return self.name + + return f"{{{ns}}}{self.name}" + + @property + def prefixed_name(self) -> str: + """Returns the prefixed element tag.""" + if self.prefix is None: + return self.name + + return f"{self.prefix}:{self.name}" + + @property + def is_pattern(self) -> bool: + """Returns true if the tag name is a regex pattern.""" + return RE_PATTERN_TEST.search(self.name) is not None + + @property + def pattern(self): + """Returns a compiled regex pattern.""" + if self.is_pattern is False: + return self.qualified_name + + return re.compile(re.escape(f"{{{self.namespace}}}") + self.name) + + def iter(self, parent: ET.Element) -> Iterator[ET.Element]: + """Returns an iterator of descendants of the parent matching this tag's + name.""" + qname = self.qualified_name + return (el for el in parent.iter() if el.tag == qname) + + def first(self, parent: ET.Element) -> ET.Element: + """Returns the first descendant of the parent matching this tag's + name.""" + try: + return next(self.iter(parent)) + except StopIteration: + return + + def __eq__(self, tag: Union[str, TTag]) -> bool: + """Returns true if the qualified names of the two tags are equal.""" + is_pattern = self.is_pattern + + if isinstance(tag, Tag): + tag_qualified_name = tag.qualified_name + else: + tag_qualified_name = tag + + if is_pattern is True: + return self.pattern.search(tag_qualified_name) is not None + + return self.qualified_name == tag_qualified_name + + def __str__(self): + """Returns a string representation of the tag.""" + return self.qualified_name + + +@dataclass +class SchemaTagsBase: + """Provides a base dataclass for schema element tag definitions.""" + + XS_ELEMENT = Tag("element", prefix="xs") + + +def make_schema_dataclass(xsd_schema_paths: Dict[str, str]) -> TTags: + """Returns a dynamic dataclass with taric schema element tag definitions.""" + schema_tags = dict() + + for prefix, path in xsd_schema_paths: + iterator = ET.iterparse(path, events=["start", "end"]) + + for event, elem in iterator: + if ( + event == "start" + and elem.tag == SchemaTagsBase.XS_ELEMENT.qualified_name + ): + name = elem.get("name") + + if name is None: + continue + + attr = name.replace(".", "_") + attr = f"{prefix}_{attr}".upper() + + tag = Tag(name, prefix=prefix) + schema_tags[attr] = tag + + Tags = make_dataclass( + "TaricSchemaTags", + schema_tags.keys(), + bases=(SchemaTagsBase,), + ) + return Tags(**schema_tags) diff --git a/taric_parsers/parser_model_link.py b/taric_parsers/parser_model_link.py new file mode 100644 index 000000000..1548a3738 --- /dev/null +++ b/taric_parsers/parser_model_link.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from typing import List + + +class ModelLinkField: + def __init__(self, parser_field_name, object_field_name): + self.parser_field_name = parser_field_name + self.object_field_name = object_field_name + + +class ModelLink: + def __init__( + self, + model, + fields: List[ModelLinkField], + xml_tag_name: str, + optional=False, + ): + self.model = model + self.fields = fields + self.xml_tag_name = xml_tag_name + self.optional = optional diff --git a/taric_parsers/parsers/additional_code_parsers.py b/taric_parsers/parsers/additional_code_parsers.py new file mode 100644 index 000000000..771cccdc0 --- /dev/null +++ b/taric_parsers/parsers/additional_code_parsers.py @@ -0,0 +1,266 @@ +from datetime import date + +from additional_codes.models import AdditionalCode +from additional_codes.models import AdditionalCodeDescription +from additional_codes.models import AdditionalCodeType +from additional_codes.models import FootnoteAssociationAdditionalCode +from footnotes.models import Footnote +from taric_parsers.parser_model_link import ModelLink +from taric_parsers.parser_model_link import ModelLinkField +from taric_parsers.parsers.mixins import ChildPeriod +from taric_parsers.parsers.mixins import ValidityMixin +from taric_parsers.parsers.mixins import Writable +from taric_parsers.parsers.taric_parser import BaseTaricParser + + +class AdditionalCodeTypeParserV2(ValidityMixin, Writable, BaseTaricParser): + model = AdditionalCodeType + model_links = [] + + value_mapping = { + "additional_code_type_id": "sid", + "validity_start_date": "valid_between_lower", + "validity_end_date": "valid_between_upper", + } + + record_code = "120" + subrecord_code = "00" + + xml_object_tag = "additional.code.type" + + identity_fields = ["sid"] + + sid: str = None + valid_between_lower: date = None + valid_between_upper: date = None + application_code: str = None + allow_update_without_children = True + + +class AdditionalCodeTypeDescriptionParserV2(Writable, BaseTaricParser): + model = AdditionalCodeType + parent_parser = AdditionalCodeTypeParserV2 + + model_links = [ + ModelLink( + AdditionalCodeType, + [ + ModelLinkField("sid", "sid"), + ], + "additional.code.type", + ), + ] + + value_mapping = { + "additional_code_type_id": "sid", + } + + record_code = "120" + subrecord_code = "05" + + xml_object_tag = "additional.code.type.description" + + identity_fields = ["sid"] + + deletes_allowed = False + + sid: str = None + description: str = None + + +class AdditionalCodeParserV2(Writable, BaseTaricParser): + model = AdditionalCode + + model_links = [ + ModelLink( + AdditionalCodeType, + [ + ModelLinkField("type__sid", "sid"), + ], + "additional.code.type", + ), + ] + + value_mapping = { + "additional_code": "code", + "additional_code_sid": "sid", + "additional_code_type_id": "type__sid", + "validity_start_date": "valid_between_lower", + "validity_end_date": "valid_between_upper", + } + + record_code = "245" + subrecord_code = "00" + + xml_object_tag = "additional.code" + + identity_fields = ["sid"] + + sid: int = None + type__sid: str = None + code: str = None + valid_between_lower: date = None + valid_between_upper: date = None + + +class AdditionalCodeDescriptionParserV2(Writable, BaseTaricParser): + model = AdditionalCodeDescription + + model_links = [ + ModelLink( + AdditionalCode, + [ + ModelLinkField("described_additionalcode__sid", "sid"), + ModelLinkField("described_additionalcode__code", "code"), + ModelLinkField("described_additionalcode__type__sid", "type__sid"), + ], + "additional.code", + ), + ] + + value_mapping = { + "additional_code_description_period_sid": "sid", + "additional_code_sid": "described_additionalcode__sid", + "additional_code_type_id": "described_additionalcode__type__sid", + "additional_code": "described_additionalcode__code", + "validity_start_date": "validity_start", + } + + record_code = "245" + subrecord_code = "10" + + xml_object_tag = "additional.code.description" + + identity_fields = [ + "described_additionalcode__sid", + "described_additionalcode__type__sid", + "described_additionalcode__code", + ] + + sid: int = None + # language_id: str = None + described_additionalcode__sid: int = None + described_additionalcode__type__sid: str = None + described_additionalcode__code: str = None + description: str = None + validity_start: date = None + allow_update_without_children = True + + +class AdditionalCodeDescriptionPeriodParserV2( + Writable, + BaseTaricParser, + ChildPeriod, +): + model = AdditionalCodeDescription + parent_parser = AdditionalCodeDescriptionParserV2 + + model_links = [ + ModelLink( + AdditionalCode, + [ + ModelLinkField("described_additionalcode__sid", "sid"), + ModelLinkField("described_additionalcode__code", "code"), + ModelLinkField("described_additionalcode__type__sid", "type__sid"), + ], + "additional.code", + ), + ModelLink( + AdditionalCodeDescription, + [ + ModelLinkField("sid", "sid"), + ], + "additional.code.description", + ), + ] + + value_mapping = { + "additional_code_description_period_sid": "sid", + "additional_code_sid": "described_additionalcode__sid", + "additional_code_type_id": "described_additionalcode__type__sid", + "additional_code": "described_additionalcode__code", + "validity_start_date": "validity_start", + } + + record_code = "245" + subrecord_code = "05" + + xml_object_tag = "additional.code.description.period" + + identity_fields = ["sid"] + + deletes_allowed = False + + sid: int = None + described_additionalcode__sid: int = None + described_additionalcode__type__sid: str = None + described_additionalcode__code: str = None + validity_start: date = None + + +class FootnoteAssociationAdditionalCodeParserV2( + ValidityMixin, + Writable, + BaseTaricParser, +): + model = FootnoteAssociationAdditionalCode + + model_links = [ + ModelLink( + Footnote, + [ + ModelLinkField( + "associated_footnote__footnote_type__footnote_type_id", + "footnote_type__footnote_type_id", + ), + ModelLinkField("associated_footnote__footnote_id", "footnote_id"), + ], + "footnote", + ), + ModelLink( + AdditionalCodeType, + [ + ModelLinkField("additional_code__type__sid", "sid"), + ], + "additional.code.type", + ), + ModelLink( + AdditionalCode, + [ + ModelLinkField("additional_code__code", "code"), + ModelLinkField("additional_code__sid", "sid"), + ], + "additional.code", + ), + ] + + value_mapping = { + "additional_code_sid": "additional_code__sid", + "footnote_type_id": "associated_footnote__footnote_type__footnote_type_id", + "footnote_id": "associated_footnote__footnote_id", + "additional_code_type_id": "additional_code__type__sid", + "additional_code": "additional_code__code", + "validity_start_date": "valid_between_lower", + "validity_end_date": "valid_between_upper", + } + + record_code = "245" + subrecord_code = "15" + + xml_object_tag = "footnote.association.additional.code" + + identity_fields = [ + "additional_code__sid", + "additional_code__code", + "additional_code__type__sid", + "associated_footnote__footnote_type__footnote_type_id", + "associated_footnote__footnote_id", + ] + + additional_code__sid: int = None + additional_code__code: str = None + additional_code__type__sid: str = None + associated_footnote__footnote_type__footnote_type_id: int = None + associated_footnote__footnote_id: str = None + valid_between_lower: date = None + valid_between_upper: date = None diff --git a/taric_parsers/parsers/certificate_parser.py b/taric_parsers/parsers/certificate_parser.py new file mode 100644 index 000000000..2f2b3b525 --- /dev/null +++ b/taric_parsers/parsers/certificate_parser.py @@ -0,0 +1,191 @@ +from datetime import date + +from certificates.models import Certificate +from certificates.models import CertificateDescription +from certificates.models import CertificateType +from taric_parsers.parser_model_link import ModelLink +from taric_parsers.parser_model_link import ModelLinkField +from taric_parsers.parsers.mixins import ChildPeriod +from taric_parsers.parsers.mixins import ValidityMixin +from taric_parsers.parsers.mixins import Writable +from taric_parsers.parsers.taric_parser import BaseTaricParser + + +class CertificateTypeParserV2(ValidityMixin, Writable, BaseTaricParser): + model = CertificateType + record_code = "110" + subrecord_code = "00" + allow_update_without_children = True + + xml_object_tag = "certificate.type" + + value_mapping = { + "certificate_type_code": "sid", + "validity_start_date": "valid_between_lower", + "validity_end_date": "valid_between_upper", + } + + model_links = [] + + identity_fields = ["sid"] + + sid: str = None + valid_between_lower: date = None + valid_between_upper: date = None + + +class CertificateTypeDescriptionParserV2(Writable, BaseTaricParser): + model = CertificateType + parent_parser = CertificateTypeParserV2 + + model_links = [ + ModelLink( + CertificateType, + [ + ModelLinkField("sid", "sid"), + ], + "certificate.type", + ), + ] + + value_mapping = { + "certificate_type_code": "sid", + } + + deletes_allowed = False + + record_code = "110" + subrecord_code = "05" + + xml_object_tag = "certificate.type.description" + + identity_fields = ["sid"] + + sid: str = None + description: str = None + + +class CertificateParserV2(ValidityMixin, Writable, BaseTaricParser): + model = Certificate + + model_links = [ + ModelLink( + CertificateType, + [ + ModelLinkField("certificate_type__sid", "sid"), + ], + "certificate.type", + ), + ] + + value_mapping = { + "certificate_code": "sid", + "certificate_type_code": "certificate_type__sid", + "validity_start_date": "valid_between_lower", + "validity_end_date": "valid_between_upper", + } + + record_code = "205" + subrecord_code = "00" + + xml_object_tag = "certificate" + + identity_fields = ["sid"] + + sid: int = None + certificate_type__sid: str = None + valid_between_lower: date = None + valid_between_upper: date = None + + +class CertificateDescriptionParserV2(Writable, BaseTaricParser): + model = CertificateDescription + allow_update_without_children = True + model_links = [ + ModelLink( + CertificateType, + [ + ModelLinkField("described_certificate__certificate_type__sid", "sid"), + ], + "certificate.type", + ), + ModelLink( + Certificate, + [ + ModelLinkField("described_certificate__sid", "sid"), + ], + "certificate", + ), + ] + + record_code = "205" + subrecord_code = "10" + + xml_object_tag = "certificate.description" + + value_mapping = { + "certificate_description_period_sid": "sid", + "certificate_type_code": "described_certificate__certificate_type__sid", + "certificate_code": "described_certificate__sid", + } + + identity_fields = ["sid", "described_certificate__sid"] + + sid: int = None + described_certificate__certificate_type__sid: str = None + described_certificate__sid: str = None + description: str = None + + +class CertificateDescriptionPeriodParserV2( + Writable, + BaseTaricParser, + ChildPeriod, +): + model = CertificateDescription + parent_parser = CertificateDescriptionParserV2 + + model_links = [ + ModelLink( + CertificateType, + [ + ModelLinkField("described_certificate__certificate_type__sid", "sid"), + ], + "certificate.type", + ), + ModelLink( + Certificate, + [ + ModelLinkField("described_certificate__sid", "sid"), + ], + "certificate", + ), + ModelLink( + CertificateDescription, + [ + ModelLinkField("sid", "sid"), + ], + "certificate.description", + ), + ] + + value_mapping = { + "certificate_description_period_sid": "sid", + "certificate_type_code": "described_certificate__certificate_type__sid", + "certificate_code": "described_certificate__sid", + "validity_start_date": "validity_start", + } + + record_code = "205" + subrecord_code = "05" + + xml_object_tag = "certificate.description.period" + + identity_fields = ["sid"] + + deletes_allowed = False + + sid: int = None + described_certificate__certificate_type__sid: str = None + described_certificate__sid: str = None + validity_start: date = None diff --git a/taric_parsers/parsers/commodity_parser.py b/taric_parsers/parsers/commodity_parser.py new file mode 100644 index 000000000..cee5f3533 --- /dev/null +++ b/taric_parsers/parsers/commodity_parser.py @@ -0,0 +1,345 @@ +from datetime import date + +from commodities.models import FootnoteAssociationGoodsNomenclature +from commodities.models import GoodsNomenclature +from commodities.models import GoodsNomenclatureDescription +from commodities.models import GoodsNomenclatureIndent +from commodities.models import GoodsNomenclatureOrigin +from commodities.models import GoodsNomenclatureSuccessor +from footnotes.models import Footnote +from taric_parsers.parser_model_link import ModelLink +from taric_parsers.parser_model_link import ModelLinkField +from taric_parsers.parsers.mixins import ChildPeriod +from taric_parsers.parsers.mixins import ValidityMixin +from taric_parsers.parsers.mixins import Writable +from taric_parsers.parsers.taric_parser import BaseTaricParser + + +class GoodsNomenclatureParserV2(Writable, BaseTaricParser): + model = GoodsNomenclature + + model_links = [] + value_mapping = { + "goods_nomenclature_sid": "sid", + "goods_nomenclature_item_id": "item_id", + "producline_suffix": "suffix", + "validity_start_date": "valid_between_lower", + "validity_end_date": "valid_between_upper", + "statistical_indicator": "statistical", + } + + record_code = "400" + subrecord_code = "00" + + xml_object_tag = "goods.nomenclature" + + identity_fields = ["sid", "item_id", "suffix"] + + sid: int = None + item_id: str = None + suffix: int = None + valid_between_lower: date = None + valid_between_upper: date = None + statistical: int = None + + +class GoodsNomenclatureOriginParserV2(Writable, BaseTaricParser): + model = GoodsNomenclatureOrigin + + model_links = [ + ModelLink( + GoodsNomenclature, + [ + ModelLinkField("new_goods_nomenclature__sid", "sid"), + ModelLinkField("new_goods_nomenclature__item_id", "item_id"), + ModelLinkField("new_goods_nomenclature__suffix", "suffix"), + ], + "goods.nomenclature", + ), + ModelLink( + GoodsNomenclature, + [ + ModelLinkField("derived_from_goods_nomenclature__item_id", "item_id"), + ModelLinkField("derived_from_goods_nomenclature__suffix", "suffix"), + ], + "goods.nomenclature", + ), + ] + + value_mapping = { + "goods_nomenclature_sid": "new_goods_nomenclature__sid", + "goods_nomenclature_item_id": "new_goods_nomenclature__item_id", + "derived_goods_nomenclature_sid": "derived_from_goods_nomenclature__sid", + "derived_goods_nomenclature_item_id": "derived_from_goods_nomenclature__item_id", + "productline_suffix": "new_goods_nomenclature__suffix", + "derived_productline_suffix": "derived_from_goods_nomenclature__suffix", + } + + record_code = "400" + subrecord_code = "35" + + xml_object_tag = "goods.nomenclature.origin" + + identity_fields = [ + "new_goods_nomenclature__sid", + "new_goods_nomenclature__item_id", + "new_goods_nomenclature__suffix", + "derived_from_goods_nomenclature__item_id", + "derived_from_goods_nomenclature__suffix", + ] + + updates_allowed = False + + new_goods_nomenclature__sid: int = None + new_goods_nomenclature__item_id: str = None + new_goods_nomenclature__suffix: int = None + derived_from_goods_nomenclature__item_id: str = None + derived_from_goods_nomenclature__suffix: int = None + + +class GoodsNomenclatureSuccessorParserV2(Writable, BaseTaricParser): + model = GoodsNomenclatureSuccessor + + model_links = [ + ModelLink( + GoodsNomenclature, + [ + ModelLinkField("replaced_goods_nomenclature__sid", "sid"), + ModelLinkField("replaced_goods_nomenclature__item_id", "item_id"), + ModelLinkField("replaced_goods_nomenclature__suffix", "suffix"), + ], + "goods.nomenclature", + ), + ModelLink( + GoodsNomenclature, + [ + ModelLinkField("absorbed_into_goods_nomenclature__item_id", "item_id"), + ModelLinkField("absorbed_into_goods_nomenclature__suffix", "suffix"), + ], + "goods.nomenclature", + ), + ] + + value_mapping = { + "goods_nomenclature_sid": "replaced_goods_nomenclature__sid", + "goods_nomenclature_item_id": "replaced_goods_nomenclature__item_id", + "productline_suffix": "replaced_goods_nomenclature__suffix", + "absorbed_goods_nomenclature_item_id": "absorbed_into_goods_nomenclature__item_id", + "absorbed_productline_suffix": "absorbed_into_goods_nomenclature__suffix", + } + + record_code = "400" + subrecord_code = "40" + + xml_object_tag = "goods.nomenclature.successor" + + identity_fields = [ + "replaced_goods_nomenclature__sid", + "replaced_goods_nomenclature__item_id", + "replaced_goods_nomenclature__suffix", + "absorbed_into_goods_nomenclature__item_id", + "absorbed_into_goods_nomenclature__suffix", + ] + + updates_allowed = False + + replaced_goods_nomenclature__sid: int = None + replaced_goods_nomenclature__item_id: str = None + replaced_goods_nomenclature__suffix: int = None + absorbed_into_goods_nomenclature__item_id: str = None + absorbed_into_goods_nomenclature__suffix: int = None + + +class GoodsNomenclatureDescriptionParserV2(Writable, BaseTaricParser): + model = GoodsNomenclatureDescription + + model_links = [ + ModelLink( + GoodsNomenclature, + [ + ModelLinkField("described_goods_nomenclature__sid", "sid"), + ModelLinkField("described_goods_nomenclature__item_id", "item_id"), + ModelLinkField("described_goods_nomenclature__suffix", "suffix"), + ], + "goods.nomenclature", + ), + ] + + # from field name (coming from XML) : to field name on this object + value_mapping = { + "goods_nomenclature_description_period_sid": "sid", + "goods_nomenclature_sid": "described_goods_nomenclature__sid", + "goods_nomenclature_item_id": "described_goods_nomenclature__item_id", + "productline_suffix": "described_goods_nomenclature__suffix", + } + + record_code = "400" + subrecord_code = "15" + + xml_object_tag = "goods.nomenclature.description" + + identity_fields = [ + "described_goods_nomenclature__sid", + "described_goods_nomenclature__item_id", + "described_goods_nomenclature__suffix", + ] + + sid: int = None + # language_id: str = None + described_goods_nomenclature__sid: int = None + described_goods_nomenclature__item_id: str = None + described_goods_nomenclature__suffix: int = None + description: str = None + allow_update_without_children = True + skip_identity_check = True + + +class GoodsNomenclatureDescriptionPeriodParserV2( + Writable, + BaseTaricParser, + ChildPeriod, +): + model = GoodsNomenclatureDescription + parent_parser = GoodsNomenclatureDescriptionParserV2 + + model_links = [ + ModelLink( + GoodsNomenclature, + [ + ModelLinkField("described_goods_nomenclature__sid", "sid"), + ModelLinkField("described_goods_nomenclature__item_id", "item_id"), + ModelLinkField("described_goods_nomenclature__suffix", "suffix"), + ], + "goods.nomenclature", + ), + ModelLink( + GoodsNomenclatureDescription, + [ + ModelLinkField("sid", "sid"), + ], + "goods.nomenclature.description", + ), + ] + + value_mapping = { + "goods_nomenclature_description_period_sid": "sid", + "goods_nomenclature_sid": "described_goods_nomenclature__sid", + "goods_nomenclature_item_id": "described_goods_nomenclature__item_id", + "productline_suffix": "described_goods_nomenclature__suffix", + "validity_start_date": "validity_start", + } + + deletes_allowed = False + + identity_fields = ["sid"] + + record_code = "400" + subrecord_code = "10" + + xml_object_tag = "goods.nomenclature.description.period" + + sid: int = None + described_goods_nomenclature__sid: int = None + described_goods_nomenclature__item_id: str = None + described_goods_nomenclature__suffix: int = None + validity_start: date = None + + +class GoodsNomenclatureIndentParserV2(Writable, BaseTaricParser): + model = GoodsNomenclatureIndent + + model_links = [ + ModelLink( + GoodsNomenclature, + [ + ModelLinkField("indented_goods_nomenclature__sid", "sid"), + ModelLinkField("indented_goods_nomenclature__item_id", "item_id"), + ModelLinkField("indented_goods_nomenclature__suffix", "suffix"), + ], + "goods.nomenclature", + ), + ] + + value_mapping = { + "goods_nomenclature_indent_sid": "sid", + "goods_nomenclature_sid": "indented_goods_nomenclature__sid", + "goods_nomenclature_item_id": "indented_goods_nomenclature__item_id", + "productline_suffix": "indented_goods_nomenclature__suffix", + "validity_start_date": "validity_start", + "number_indents": "indent", + } + + record_code = "400" + subrecord_code = "05" + + xml_object_tag = "goods.nomenclature.indents" + + identity_fields = ["sid"] + + sid: int = None + indented_goods_nomenclature__sid: int = None + indented_goods_nomenclature__item_id: str = None + indented_goods_nomenclature__suffix: int = None + validity_start: date = None + indent: int = None + + +class FootnoteAssociationGoodsNomenclatureParserV2( + ValidityMixin, + Writable, + BaseTaricParser, +): + model = FootnoteAssociationGoodsNomenclature + + model_links = [ + ModelLink( + GoodsNomenclature, + [ + ModelLinkField("goods_nomenclature__sid", "sid"), + ModelLinkField("goods_nomenclature__item_id", "item_id"), + ModelLinkField("goods_nomenclature__suffix", "suffix"), + ], + "goods.nomenclature", + ), + ModelLink( + Footnote, + [ + ModelLinkField("associated_footnote__footnote_id", "footnote_id"), + ModelLinkField( + "associated_footnote__footnote_type__footnote_type_id", + "footnote_type__footnote_type_id", + ), + ], + "footnote", + ), + ] + + value_mapping = { + "goods_nomenclature_sid": "goods_nomenclature__sid", + "footnote_type": "associated_footnote__footnote_type__footnote_type_id", + "footnote_id": "associated_footnote__footnote_id", + "validity_start_date": "valid_between_lower", + "validity_end_date": "valid_between_upper", + "goods_nomenclature_item_id": "goods_nomenclature__item_id", + "productline_suffix": "goods_nomenclature__suffix", + } + + record_code = "400" + subrecord_code = "20" + + xml_object_tag = "footnote.association.goods.nomenclature" + + identity_fields = [ + "goods_nomenclature__sid", + "goods_nomenclature__item_id", + "goods_nomenclature__suffix", + "associated_footnote__footnote_id", + ] + + goods_nomenclature__sid: int = None + goods_nomenclature__item_id: str = None + goods_nomenclature__suffix: int = None + associated_footnote__footnote_type__footnote_type_id: int = None + associated_footnote__footnote_id: int = None + valid_between_lower: date = None + valid_between_upper: date = None diff --git a/taric_parsers/parsers/footnote_parser.py b/taric_parsers/parsers/footnote_parser.py new file mode 100644 index 000000000..e760c6069 --- /dev/null +++ b/taric_parsers/parsers/footnote_parser.py @@ -0,0 +1,189 @@ +from datetime import date + +from footnotes.models import Footnote +from footnotes.models import FootnoteDescription +from footnotes.models import FootnoteType +from taric_parsers.parser_model_link import ModelLink # noqa +from taric_parsers.parser_model_link import ModelLinkField +from taric_parsers.parsers.mixins import ChildPeriod +from taric_parsers.parsers.mixins import ValidityMixin +from taric_parsers.parsers.mixins import Writable +from taric_parsers.parsers.taric_parser import BaseTaricParser + + +class FootnoteTypeParserV2(Writable, BaseTaricParser): + model = FootnoteType + + model_links = [] + + value_mapping = { + "validity_start_date": "valid_between_lower", + "validity_end_date": "valid_between_upper", + } + + record_code = "100" + subrecord_code = "00" + + xml_object_tag = "footnote.type" + + identity_fields = [ + "footnote_type_id", + ] + + allow_update_without_children = True + footnote_type_id: str = None + valid_between_lower: date = None + valid_between_upper: date = None + application_code: str = None + + +class FootnoteTypeDescriptionParserV2(Writable, BaseTaricParser): + model = FootnoteType + parent_parser = FootnoteTypeParserV2 + + model_links = [ + ModelLink( + FootnoteType, + [ + ModelLinkField("footnote_type_id", "footnote_type_id"), + ], + "footnote.type", + ), + ] + + record_code = "100" + subrecord_code = "05" + + xml_object_tag = "footnote.type.description" + + identity_fields = [ + "footnote_type_id", + ] + + footnote_type_id: str = None + # language_id: str = None + description: str = None + + +class FootnoteParserV2(ValidityMixin, Writable, BaseTaricParser): + model = Footnote + + model_links = [ + ModelLink( + FootnoteType, + [ + ModelLinkField("footnote_type__footnote_type_id", "footnote_type_id"), + ], + "footnote.type", + ), + ] + + value_mapping = { + "validity_start_date": "valid_between_lower", + "validity_end_date": "valid_between_upper", + "footnote_type_id": "footnote_type__footnote_type_id", + } + + record_code = "200" + subrecord_code = "00" + + xml_object_tag = "footnote" + + identity_fields = [ + "footnote_id", + ] + + footnote_type__footnote_type_id: str = None + footnote_id: str = None + valid_between_lower: date = None + valid_between_upper: date = None + + +class FootnoteDescriptionParserV2(Writable, BaseTaricParser): + model = FootnoteDescription + + model_links = [ + ModelLink( + Footnote, + [ + ModelLinkField("described_footnote__footnote_id", "footnote_id"), + ModelLinkField( + "described_footnote__footnote_type__footnote_type_id", + "footnote_type__footnote_type_id", + ), + ], + "footnote", + ), + ] + + value_mapping = { + "footnote_description_period_sid": "sid", + "footnote_type_id": "described_footnote__footnote_type__footnote_type_id", + "footnote_id": "described_footnote__footnote_id", + } + + record_code = "200" + subrecord_code = "10" + + xml_object_tag = "footnote.description" + + identity_fields = [ + "described_footnote__footnote_type__footnote_type_id", + "described_footnote__footnote_id", + ] + + allow_update_without_children = True + + sid: int = None + described_footnote__footnote_type__footnote_type_id: str = None + described_footnote__footnote_id: str = None + description: str = None + + +class FootnoteDescriptionPeriodParserV2(Writable, BaseTaricParser, ChildPeriod): + model = FootnoteDescription + parent_parser = FootnoteDescriptionParserV2 + + model_links = [ + ModelLink( + Footnote, + [ + ModelLinkField("described_footnote__footnote_id", "footnote_id"), + ModelLinkField( + "described_footnote__footnote_type__footnote_type_id", + "footnote_type__footnote_type_id", + ), + ], + "footnote", + ), + ModelLink( + FootnoteDescription, + [ + ModelLinkField("sid", "sid"), + ], + "footnote.description", + ), + ] + + value_mapping = { + "footnote_description_period_sid": "sid", + "footnote_type_id": "described_footnote__footnote_type__footnote_type_id", + "footnote_id": "described_footnote__footnote_id", + "validity_start_date": "validity_start", + } + + record_code = "200" + subrecord_code = "05" + + xml_object_tag = "footnote.description.period" + + identity_fields = [ + "sid", + ] + + deletes_allowed = False + + sid: int = None + described_footnote__footnote_type__footnote_type_id: str = None + described_footnote__footnote_id: str = None + validity_start: date = None diff --git a/taric_parsers/parsers/geo_area_parser.py b/taric_parsers/parsers/geo_area_parser.py new file mode 100644 index 000000000..59554ddfa --- /dev/null +++ b/taric_parsers/parsers/geo_area_parser.py @@ -0,0 +1,182 @@ +from datetime import date + +from geo_areas.models import GeographicalArea +from geo_areas.models import GeographicalAreaDescription +from geo_areas.models import GeographicalMembership +from taric_parsers.parser_model_link import ModelLink +from taric_parsers.parser_model_link import ModelLinkField +from taric_parsers.parsers.mixins import ChildPeriod +from taric_parsers.parsers.mixins import ValidityMixin +from taric_parsers.parsers.mixins import Writable +from taric_parsers.parsers.taric_parser import BaseTaricParser + + +class GeographicalAreaParserV2(ValidityMixin, Writable, BaseTaricParser): + model = GeographicalArea + + model_links = [ + ModelLink( + GeographicalArea, + [ + ModelLinkField("parent__sid", "sid"), + ], + "geographical.area", + True, + ), + ] + + value_mapping = { + "geographical_area_sid": "sid", + "geographical_area_id": "area_id", + "validity_start_date": "valid_between_lower", + "validity_end_date": "valid_between_upper", + "geographical_code": "area_code", + "parent_geographical_area_group_sid": "parent__sid", + } + + record_code = "250" + subrecord_code = "00" + + xml_object_tag = "geographical.area" + + identity_fields = [ + "sid", + ] + + sid: int = None + area_id: str = None + valid_between_lower: date = None + valid_between_upper: date = None + area_code: int = None + parent__sid: int = None + + +class GeographicalAreaDescriptionParserV2(Writable, BaseTaricParser): + model = GeographicalAreaDescription + + model_links = [ + ModelLink( + GeographicalArea, + [ + ModelLinkField("described_geographicalarea__sid", "sid"), + ModelLinkField("described_geographicalarea__area_id", "area_id"), + ], + "geographical.area", + ), + ] + + value_mapping = { + "geographical_area_description_period_sid": "sid", + "geographical_area_sid": "described_geographicalarea__sid", + "geographical_area_id": "described_geographicalarea__area_id", + } + + record_code = "250" + subrecord_code = "10" + + xml_object_tag = "geographical.area.description" + + identity_fields = [ + "sid", + "described_geographicalarea__sid", + "described_geographicalarea__area_id", + ] + + allow_update_without_children = True + + sid: int = None + described_geographicalarea__sid: int = None + described_geographicalarea__area_id: str = None + description: str = None + + +class GeographicalAreaDescriptionPeriodParserV2( + Writable, + BaseTaricParser, + ChildPeriod, +): + model = GeographicalAreaDescription + parent_parser = GeographicalAreaDescriptionParserV2 + + model_links = [ + ModelLink( + GeographicalArea, + [ + ModelLinkField("described_geographicalarea__sid", "sid"), + ModelLinkField("described_geographicalarea__area_id", "area_id"), + ], + "geographical.area", + ), + ModelLink( + GeographicalAreaDescription, + [ + ModelLinkField("sid", "sid"), + ], + "geographical.area.description", + ), + ] + + value_mapping = { + "geographical_area_description_period_sid": "sid", + "geographical_area_sid": "described_geographicalarea__sid", + "geographical_area_id": "described_geographicalarea__area_id", + "validity_start_date": "validity_start", + } + + record_code = "250" + subrecord_code = "05" + + xml_object_tag = "geographical.area.description.period" + + identity_fields = [ + "sid", + ] + + deletes_allowed = False + sid: int = None + described_geographicalarea__sid: int = None + described_geographicalarea__area_id: str = None + validity_start: date = None + + +class GeographicalMembershipParserV2(ValidityMixin, Writable, BaseTaricParser): + model = GeographicalMembership + + model_links = [ + ModelLink( + GeographicalArea, + [ + ModelLinkField("member__sid", "sid"), + ], + "geographical.area", + ), + ModelLink( + GeographicalArea, + [ + ModelLinkField("geo_group__sid", "sid"), + ], + "geographical.area", + ), + ] + + value_mapping = { + "geographical_area_sid": "member__sid", + "geographical_area_group_sid": "geo_group__sid", + "validity_start_date": "valid_between_lower", + "validity_end_date": "valid_between_upper", + } + + record_code = "250" + subrecord_code = "15" + + xml_object_tag = "geographical.membership" + + identity_fields = [ + "member__sid", + "geo_group__sid", + ] + + member__sid: int = None + geo_group__sid: int = None + valid_between_lower: date = None + valid_between_upper: date = None diff --git a/taric_parsers/parsers/measure_parser.py b/taric_parsers/parsers/measure_parser.py new file mode 100644 index 000000000..ff7b36f6c --- /dev/null +++ b/taric_parsers/parsers/measure_parser.py @@ -0,0 +1,1037 @@ +from datetime import date + +from additional_codes.models import AdditionalCode +from additional_codes.models import AdditionalCodeType +from certificates.models import Certificate +from commodities.models import GoodsNomenclature +from footnotes.models import Footnote +from geo_areas.models import GeographicalArea +from measures.models import AdditionalCodeTypeMeasureType +from measures.models import DutyExpression +from measures.models import FootnoteAssociationMeasure +from measures.models import Measure +from measures.models import MeasureAction +from measures.models import MeasureComponent +from measures.models import MeasureCondition +from measures.models import MeasureConditionCode +from measures.models import MeasureConditionComponent +from measures.models import MeasureExcludedGeographicalArea +from measures.models import Measurement +from measures.models import MeasurementUnit +from measures.models import MeasurementUnitQualifier +from measures.models import MeasureType +from measures.models import MeasureTypeSeries +from measures.models import MonetaryUnit +from quotas.models import QuotaOrderNumber +from regulations.models import Regulation +from taric_parsers.parser_model_link import ModelLink +from taric_parsers.parser_model_link import ModelLinkField +from taric_parsers.parsers.mixins import ValidityMixin +from taric_parsers.parsers.mixins import Writable +from taric_parsers.parsers.taric_parser import BaseTaricParser + + +class MeasureTypeSeriesParserV2(ValidityMixin, Writable, BaseTaricParser): + model = MeasureTypeSeries + record_code = "140" + subrecord_code = "00" + + xml_object_tag = "measure.type.series" + + model_links = [] + + value_mapping = { + "measure_type_series_id": "sid", + "validity_start_date": "valid_between_lower", + "validity_end_date": "valid_between_upper", + } + + identity_fields = [ + "sid", + ] + + allow_update_without_children = True + + sid: str = None + valid_between_lower: date = None + valid_between_upper: date = None + measure_type_combination: int = None + + +class MeasureTypeSeriesDescriptionParserV2(Writable, BaseTaricParser): + model = MeasureTypeSeries + parent_parser = MeasureTypeSeriesParserV2 + + model_links = [ + ModelLink( + MeasureTypeSeries, + [ModelLinkField("sid", "sid")], + "measure.type.series", + ), + ] + + value_mapping = { + "measure_type_series_id": "sid", + } + + record_code = "140" + subrecord_code = "05" + + xml_object_tag = "measure.type.series.description" + + identity_fields = [ + "sid", + ] + + deletes_allowed = False + sid: str = None + description: str = None + + +class MeasurementUnitParserV2(ValidityMixin, Writable, BaseTaricParser): + model = MeasurementUnit + record_code = "210" + subrecord_code = "00" + + xml_object_tag = "measurement.unit" + + identity_fields = [ + "code", + ] + + value_mapping = { + "measurement_unit_code": "code", + "validity_start_date": "valid_between_lower", + "validity_end_date": "valid_between_upper", + } + + allow_update_without_children = True + + code: str = None + valid_between_lower: date = None + valid_between_upper: date = None + + +class MeasurementUnitDescriptionParserV2(Writable, BaseTaricParser): + model = MeasurementUnit + parent_parser = MeasurementUnitParserV2 + + model_links = [ + ModelLink( + MeasurementUnit, + [ + ModelLinkField("code", "code"), + ], + "measurement.unit", + ), + ] + + value_mapping = { + "measurement_unit_code": "code", + } + + record_code = "210" + subrecord_code = "05" + + xml_object_tag = "measurement.unit.description" + + identity_fields = [ + "code", + ] + + code: str = None + description: str = None + + +class MeasurementUnitQualifierParserV2( + ValidityMixin, + Writable, + BaseTaricParser, +): + model = MeasurementUnitQualifier + + value_mapping = { + "measurement_unit_qualifier_code": "code", + "validity_start_date": "valid_between_lower", + "validity_end_date": "valid_between_upper", + } + + record_code = "215" + subrecord_code = "00" + + xml_object_tag = "measurement.unit.qualifier" + + identity_fields = [ + "code", + ] + + allow_update_without_children = True + + code: str = None + valid_between_lower: date = None + valid_between_upper: date = None + + +class MeasurementUnitQualifierDescriptionParserV2(Writable, BaseTaricParser): + model = MeasurementUnitQualifier + parent_parser = MeasurementUnitQualifierParserV2 + + model_links = [ + ModelLink( + MeasurementUnitQualifier, + [ + ModelLinkField("code", "code"), + ], + "measurement.unit.qualifier", + ), + ] + + value_mapping = { + "measurement_unit_qualifier_code": "code", + } + + record_code = "215" + subrecord_code = "05" + + xml_object_tag = "measurement.unit.qualifier.description" + + identity_fields = [ + "code", + ] + + deletes_allowed = False + code: str = None + description: str = None + + +class MeasurementParserV2(ValidityMixin, Writable, BaseTaricParser): + model = Measurement + + model_links = [ + ModelLink( + MeasurementUnit, + [ + ModelLinkField("measurement_unit__code", "code"), + ], + "measurement.unit", + ), + ModelLink( + MeasurementUnitQualifier, + [ + ModelLinkField("measurement_unit_qualifier__code", "code"), + ], + "measurement.unit.qualifier", + ), + ] + + value_mapping = { + "measurement_unit_code": "measurement_unit__code", + "measurement_unit_qualifier_code": "measurement_unit_qualifier__code", + "validity_start_date": "valid_between_lower", + "validity_end_date": "valid_between_upper", + } + + record_code = "220" + subrecord_code = "00" + + xml_object_tag = "measurement" + + identity_fields = [ + "measurement_unit__code", + "measurement_unit_qualifier__code", + ] + + measurement_unit__code: str = None + measurement_unit_qualifier__code: str = None + valid_between_lower: date = None + valid_between_upper: date = None + + +class MonetaryUnitParserV2(ValidityMixin, Writable, BaseTaricParser): + model = MonetaryUnit + record_code = "225" + subrecord_code = "00" + + xml_object_tag = "monetary.unit" + + model_links = [] + + value_mapping = { + "monetary_unit_code": "code", + "validity_start_date": "valid_between_lower", + "validity_end_date": "valid_between_upper", + } + + identity_fields = [ + "code", + ] + + allow_update_without_children = True + + code: str = None + valid_between_lower: date = None + valid_between_upper: date = None + + +class MonetaryUnitDescriptionParserV2(Writable, BaseTaricParser): + model = MonetaryUnit + parent_parser = MonetaryUnitParserV2 + + model_links = [ + ModelLink( + MonetaryUnit, + [ + ModelLinkField("code", "code"), + ], + "monetary.unit", + ), + ] + + value_mapping = { + "monetary_unit_code": "code", + } + + record_code = "225" + subrecord_code = "05" + + xml_object_tag = "monetary.unit.description" + + identity_fields = [ + "code", + ] + + deletes_allowed = False + + code: str = None + description: str = None + + +class DutyExpressionParserV2(ValidityMixin, Writable, BaseTaricParser): + model = DutyExpression + + record_code = "230" + subrecord_code = "00" + + xml_object_tag = "duty.expression" + + identity_fields = [ + "sid", + "measurement_unit_applicability_code", + "monetary_unit_applicability_code", + ] + + model_links = [] + + value_mapping = { + "duty_expression_id": "sid", + "validity_start_date": "valid_between_lower", + "validity_end_date": "valid_between_upper", + } + + allow_update_without_children = True + + sid: int = None + valid_between_lower: date = None + valid_between_upper: date = None + duty_amount_applicability_code: int = None + measurement_unit_applicability_code: int = None + monetary_unit_applicability_code: int = None + + +class DutyExpressionDescriptionParserV2(Writable, BaseTaricParser): + model = DutyExpression + parent_parser = DutyExpressionParserV2 + + model_links = [ + ModelLink( + DutyExpression, + [ + ModelLinkField("sid", "sid"), + ], + "duty.expression", + ), + ] + + value_mapping = { + "duty_expression_id": "sid", + } + + record_code = "230" + subrecord_code = "05" + + xml_object_tag = "duty.expression.description" + + identity_fields = [ + "sid", + ] + + deletes_allowed = False + sid: int = None + description: str = None + + +class MeasureTypeParserV2(ValidityMixin, Writable, BaseTaricParser): + model = MeasureType + model_links = [ + ModelLink( + MeasureTypeSeries, + [ + ModelLinkField("measure_type_series__sid", "sid"), + ], + "measure.type.series", + ), + ] + + record_code = "235" + subrecord_code = "00" + + xml_object_tag = "measure.type" + + value_mapping = { + "measure_type_id": "sid", + "measure_component_applicable_code": "measure_component_applicability_code", + "origin_dest_code": "origin_destination_code", + "measure_type_series_id": "measure_type_series__sid", + "validity_start_date": "valid_between_lower", + "validity_end_date": "valid_between_upper", + } + + identity_fields = [ + "sid", + ] + + allow_update_without_children = True + + sid: str = None + trade_movement_code: int = None + priority_code: int = None + measure_component_applicability_code: int = None + origin_destination_code: int = None + order_number_capture_code: int = None + measure_explosion_level: int = None + measure_type_series__sid: str = None + valid_between_lower: date = None + valid_between_upper: date = None + + +class MeasureTypeDescriptionParserV2(Writable, BaseTaricParser): + model = MeasureType + parent_parser = MeasureTypeParserV2 + + model_links = [ + ModelLink( + MeasureType, + [ + ModelLinkField("sid", "sid"), + ], + "measure.type", + ), + ] + + value_mapping = { + "measure_type_id": "sid", + } + + record_code = "235" + subrecord_code = "05" + + xml_object_tag = "measure.type.description" + + identity_fields = [ + "sid", + ] + + deletes_allowed = False + sid: str = None + description: str = None + + +class AdditionalCodeTypeMeasureTypeParserV2( + ValidityMixin, + Writable, + BaseTaricParser, +): + model = AdditionalCodeTypeMeasureType + model_links = [ + ModelLink( + MeasureType, + [ + ModelLinkField("measure_type__sid", "sid"), + ], + "measure.type", + ), + ModelLink( + AdditionalCodeType, + [ + ModelLinkField("additional_code_type__sid", "sid"), + ], + "additional.code.type", + ), + ] + + value_mapping = { + "measure_type_id": "measure_type__sid", + "additional_code_type_id": "additional_code_type__sid", + "validity_start_date": "valid_between_lower", + "validity_end_date": "valid_between_upper", + } + + record_code = "240" + subrecord_code = "00" + + xml_object_tag = "additional.code.type.measure.type" + + identity_fields = [ + "measure_type__sid", + "additional_code_type__sid", + ] + + measure_type__sid: str = None + additional_code_type__sid: str = None + valid_between_lower: date = None + valid_between_upper: date = None + + +class MeasureConditionCodeParserV2(ValidityMixin, Writable, BaseTaricParser): + model = MeasureConditionCode + record_code = "350" + subrecord_code = "00" + + xml_object_tag = "measure.condition.code" + + value_mapping = { + "condition_code": "code", + "validity_start_date": "valid_between_lower", + "validity_end_date": "valid_between_upper", + } + + identity_fields = [ + "code", + ] + + allow_update_without_children = True + + code: str = None + valid_between_lower: date = None + valid_between_upper: date = None + + +class MeasureConditionCodeDescriptionParserV2(Writable, BaseTaricParser): + model = MeasureConditionCode + parent_parser = MeasureConditionCodeParserV2 + + model_links = [ + ModelLink( + MeasureConditionCode, + [ + ModelLinkField("code", "code"), + ], + "measure.condition.code", + ), + ] + + value_mapping = { + "condition_code": "code", + } + + record_code = "350" + subrecord_code = "05" + + xml_object_tag = "measure.condition.code.description" + + identity_fields = [ + "code", + ] + + deletes_allowed = False + code: str = None + description: str = None + + +class MeasureActionParserV2(ValidityMixin, Writable, BaseTaricParser): + model = MeasureAction + + model_links = [] + + record_code = "355" + subrecord_code = "00" + + xml_object_tag = "measure.action" + + value_mapping = { + "action_code": "code", + "validity_start_date": "valid_between_lower", + "validity_end_date": "valid_between_upper", + } + + identity_fields = [ + "code", + ] + + allow_update_without_children = True + + code: str = None + valid_between_lower: date = None + valid_between_upper: date = None + + +class MeasureActionDescriptionParserV2(Writable, BaseTaricParser): + model = MeasureAction + parent_parser = MeasureActionParserV2 + + model_links = [ + ModelLink( + MeasureAction, + [ + ModelLinkField("code", "code"), + ], + "measure.action", + ), + ] + + value_mapping = { + "action_code": "code", + } + + record_code = "355" + subrecord_code = "05" + + xml_object_tag = "measure.action.description" + + identity_fields = [ + "code", + ] + + deletes_allowed = False + code: str = None + description: str = None + + +class MeasureParserV2(ValidityMixin, Writable, BaseTaricParser): + model = Measure + model_links = [ + ModelLink( + MeasureType, + [ + ModelLinkField("measure_type__sid", "sid"), + ], + "measure.type", + ), + ModelLink( + GeographicalArea, + [ + ModelLinkField("geographical_area__area_id", "area_id"), + ModelLinkField("geographical_area__sid", "sid"), + ], + "geographical.area", + ), + ModelLink( + GoodsNomenclature, + [ + ModelLinkField("goods_nomenclature__item_id", "item_id"), + ModelLinkField("goods_nomenclature__sid", "sid"), + ], + "goods.nomenclature", + ), + ModelLink( + AdditionalCode, + [ + ModelLinkField("additional_code__code", "code"), + ModelLinkField("additional_code__sid", "sid"), + ModelLinkField("additional_code__type__sid", "type__sid"), + ], + "additional.code", + True, + ), + ModelLink( + QuotaOrderNumber, + [ + ModelLinkField("order_number__order_number", "order_number"), + ], + "additional.code.type", + True, # optional - can be blank + ), + ModelLink( + Regulation, + [ + ModelLinkField("generating_regulation__role_type", "role_type"), + ModelLinkField("generating_regulation__regulation_id", "regulation_id"), + ], + "regulation", + ), + ModelLink( + Regulation, + [ + ModelLinkField("terminating_regulation__role_type", "role_type"), + ModelLinkField( + "terminating_regulation__regulation_id", + "regulation_id", + ), + ], + "regulation", + ), + ] + + value_mapping = { + "measure_sid": "sid", + "justification_regulation_role": "terminating_regulation__role_type", + "justification_regulation_id": "terminating_regulation__regulation_id", + "measure_type": "measure_type__sid", + "geographical_area_sid": "geographical_area__sid", + "geographical_area": "geographical_area__area_id", + "goods_nomenclature_item_id": "goods_nomenclature__item_id", + "additional_code_type": "additional_code__type__sid", + "additional_code": "additional_code__code", + "additional_code_sid": "additional_code__sid", + "ordernumber": "order_number__order_number", + "reduction_indicator": "reduction", + "validity_start_date": "valid_between_lower", + "validity_end_date": "valid_between_upper", + "measure_generating_regulation_role": "generating_regulation__role_type", + "measure_generating_regulation_id": "generating_regulation__regulation_id", + "stopped_flag": "stopped", + "goods_nomenclature_sid": "goods_nomenclature__sid", + } + + record_code = "430" + subrecord_code = "00" + + xml_object_tag = "measure" + + identity_fields = [ + "sid", + ] + + sid: int = None + measure_type__sid: str = None + geographical_area__area_id: str = None + geographical_area__sid: int = None + goods_nomenclature__item_id: str = None + goods_nomenclature__sid: int = None + additional_code__type__sid: str = None + additional_code__code: str = None + additional_code__sid: int = None + order_number__order_number: str = None + reduction: int = None + valid_between_lower: date = None + valid_between_upper: date = None + generating_regulation__role_type: int = None + generating_regulation__regulation_id: str = None + terminating_regulation__role_type: int = None + terminating_regulation__regulation_id: str = None + stopped: bool = None + + +class MeasureComponentParserV2(Writable, BaseTaricParser): + model = MeasureComponent + model_links = [ + ModelLink( + Measure, + [ + ModelLinkField("component_measure__sid", "sid"), + ], + "measure", + ), + ModelLink( + DutyExpression, + [ + ModelLinkField("duty_expression__sid", "sid"), + ], + "duty.expression", + ), + ModelLink( + MonetaryUnit, + [ + ModelLinkField("monetary_unit__code", "code"), + ], + "monetary.unit", + True, + ), + ModelLink( + Measurement, + [ + ModelLinkField( + "component_measurement__measurement_unit__code", + "measurement_unit__code", + ), + ModelLinkField( + "component_measurement__measurement_unit_qualifier__code", + "measurement_unit_qualifier__code", + ), + ], + "measurement", + True, + ), + ] + + value_mapping = { + "measure_sid": "component_measure__sid", + "duty_expression_id": "duty_expression__sid", + "monetary_unit_code": "monetary_unit__code", + "measurement_unit_code": "component_measurement__measurement_unit__code", + "measurement_unit_qualifier_code": "component_measurement__measurement_unit_qualifier__code", + } + + record_code = "430" + subrecord_code = "05" + + xml_object_tag = "measure.component" + + identity_fields = [ + "component_measure__sid", + "duty_expression__sid", + ] + + component_measure__sid: int = None + duty_expression__sid: int = None + duty_amount: float = None + monetary_unit__code: str = None + component_measurement__measurement_unit__code: str = None + component_measurement__measurement_unit_qualifier__code: str = None + + +class MeasureConditionParserV2(Writable, BaseTaricParser): + model = MeasureCondition + model_links = [ + ModelLink( + Measure, + [ + ModelLinkField("dependent_measure__sid", "sid"), + ], + "measure", + ), + ModelLink( + MeasureConditionCode, + [ + ModelLinkField("condition_code__code", "code"), + ], + "measure.condition.code", + ), + ModelLink( + MonetaryUnit, + [ + ModelLinkField("monetary_unit__code", "code"), + ], + "monetary.unit", + ), + ModelLink( + Measurement, + [ + ModelLinkField( + "condition_measurement__measurement_unit__code", + "measurement_unit__code", + ), + ModelLinkField( + "condition_measurement__measurement_unit_qualifier__code", + "measurement_unit_qualifier__code", + ), + ], + "measurement", + ), + ModelLink( + MeasureAction, + [ + ModelLinkField("action__code", "code"), + ], + "measure.action", + ), + ModelLink( + Certificate, + [ + ModelLinkField("required_certificate__sid", "sid"), + ModelLinkField( + "required_certificate__certificate_type__sid", + "certificate_type__sid", + ), + ], + "certificate", + ), + ] + + record_code = "430" + subrecord_code = "10" + + xml_object_tag = "measure.condition" + + value_mapping = { + "measure_condition_sid": "sid", + "measure_sid": "dependent_measure__sid", + "condition_code": "condition_code__code", + "condition_duty_amount": "duty_amount", + "condition_monetary_unit_code": "monetary_unit__code", + "condition_measurement_unit_code": "condition_measurement__measurement_unit__code", + "condition_measurement_unit_qualifier_code": "condition_measurement__measurement_unit_qualifier__code", + "action_code": "action__code", + "certificate_type_code": "required_certificate__certificate_type__sid", + "certificate_code": "required_certificate__sid", + } + + identity_fields = [ + "sid", + ] + + sid: int = None + dependent_measure__sid: int = None + condition_code__code: str = None + component_sequence_number: int = None + duty_amount: float = None + monetary_unit__code: str = None + condition_measurement__measurement_unit__code: str = None + condition_measurement__measurement_unit_qualifier__code: str = None + action__code: str = None + required_certificate__certificate_type__sid: str = None + required_certificate__sid: str = None + + +class MeasureConditionComponentParserV2(Writable, BaseTaricParser): + model = MeasureConditionComponent + model_links = [ + ModelLink( + MeasureCondition, + [ + ModelLinkField("condition__sid", "sid"), + ], + "measure.condition", + ), + ModelLink( + DutyExpression, + [ + ModelLinkField("duty_expression__sid", "sid"), + ], + "duty.expression", + ), + ModelLink( + MonetaryUnit, + [ + ModelLinkField("monetary_unit__code", "code"), + ], + "monetary.unit", + ), + ModelLink( + Measurement, + [ + ModelLinkField( + "component_measurement__measurement_unit__code", + "measurement_unit__code", + ), + ModelLinkField( + "component_measurement__measurement_unit_qualifier__code", + "measurement_unit_qualifier__code", + ), + ], + "measurement", + ), + ] + + record_code = "430" + subrecord_code = "11" + + xml_object_tag = "measure.condition.component" + + value_mapping = { + "measure_condition_sid": "condition__sid", + "duty_expression_id": "duty_expression__sid", + "monetary_unit_code": "monetary_unit__code", + "measurement_unit_code": "component_measurement__measurement_unit__code", + "measurement_unit_qualifier_code": "component_measurement__measurement_unit_qualifier__code", + } + + identity_fields = [ + "condition__sid", + "component_measurement__measurement_unit__code", + "component_measurement__measurement_unit_qualifier__code", + ] + + condition__sid: int = None + duty_expression__sid: int = None + duty_amount: float = None + monetary_unit__code: str = None + component_measurement__measurement_unit__code: str = None + component_measurement__measurement_unit_qualifier__code: str = None + + +class MeasureExcludedGeographicalAreaParserV2(Writable, BaseTaricParser): + model = MeasureExcludedGeographicalArea + model_links = [ + ModelLink( + Measure, + [ + ModelLinkField("modified_measure__sid", "sid"), + ], + "measure", + ), + ModelLink( + GeographicalArea, + [ + ModelLinkField("excluded_geographical_area__area_id", "area_id"), + ModelLinkField("excluded_geographical_area__sid", "sid"), + ], + "geographical.area", + ), + ] + + record_code = "430" + subrecord_code = "15" + + xml_object_tag = "measure.excluded.geographical.area" + + value_mapping = { + "measure_sid": "modified_measure__sid", + "excluded_geographical_area": "excluded_geographical_area__area_id", + "geographical_area_sid": "excluded_geographical_area__sid", + } + + identity_fields = [ + "modified_measure__sid", + "excluded_geographical_area__sid", + ] + + modified_measure__sid: int = None + excluded_geographical_area__area_id: str = None + excluded_geographical_area__sid: int = None + + +class FootnoteAssociationMeasureParserV2(Writable, BaseTaricParser): + model = FootnoteAssociationMeasure + model_links = [ + ModelLink( + Measure, + [ + ModelLinkField("footnoted_measure__sid", "sid"), + ], + "measure", + ), + ModelLink( + Footnote, + [ + ModelLinkField("associated_footnote__footnote_id", "footnote_id"), + ModelLinkField( + "associated_footnote__footnote_type__footnote_type_id", + "footnote_type__footnote_type_id", + ), + ], + "footnote", + ), + ] + + value_mapping = { + "measure_sid": "footnoted_measure__sid", + "footnote_type_id": "associated_footnote__footnote_type__footnote_type_id", + "footnote_id": "associated_footnote__footnote_id", + } + + record_code = "430" + subrecord_code = "20" + + xml_object_tag = "footnote.association.measure" + + identity_fields = [ + "footnoted_measure__sid", + "associated_footnote__footnote_id", + ] + + updates_allowed = False + + footnoted_measure__sid: int = None + associated_footnote__footnote_type__footnote_type_id: str = None + associated_footnote__footnote_id: str = None diff --git a/taric_parsers/parsers/mixins.py b/taric_parsers/parsers/mixins.py new file mode 100644 index 000000000..1035fbaff --- /dev/null +++ b/taric_parsers/parsers/mixins.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from datetime import date + +from common.validators import UpdateType + + +class ValidityMixin: + """Parse validity start and end dates.""" + + valid_between_lower: date = None + valid_between_upper: date = None + + +class ValidityStartMixin: + """Parse validity start date.""" + + validity_start: date = None + + +class Writable: + update_type: str + + def commit_to_database(self): + kwargs = { + "update_type": UpdateType.CREATE, + "transaction": transaction, + "order_number": quota_order_number, + "geographical_area": resolve_geo_area(order_number["origin"]), + "valid_between": valid_between, + } + + +class ChildPeriod: + def parent_attributes(self): + return { + "sid": self.sid, + "validity_start": self.validity_start, + } diff --git a/taric_parsers/parsers/quota_parser.py b/taric_parsers/parsers/quota_parser.py new file mode 100644 index 000000000..6a5caaa0f --- /dev/null +++ b/taric_parsers/parsers/quota_parser.py @@ -0,0 +1,575 @@ +import json +from datetime import date +from datetime import datetime + +from geo_areas.models import GeographicalArea +from measures.models import MeasurementUnit +from measures.models import MeasurementUnitQualifier +from measures.models import MonetaryUnit +from quotas.models import QuotaAssociation +from quotas.models import QuotaBlocking +from quotas.models import QuotaDefinition +from quotas.models import QuotaEvent +from quotas.models import QuotaOrderNumber +from quotas.models import QuotaOrderNumberOrigin +from quotas.models import QuotaOrderNumberOriginExclusion +from quotas.models import QuotaSuspension +from taric_parsers.parser_model_link import ModelLink +from taric_parsers.parser_model_link import ModelLinkField +from taric_parsers.parsers.taric_parser import BaseTaricParser + + +class QuotaOrderNumberParserV2(BaseTaricParser): + model = QuotaOrderNumber + + value_mapping = { + "quota_order_number_sid": "sid", + "quota_order_number_id": "order_number", + "validity_start_date": "valid_between_lower", + "validity_end_date": "valid_between_upper", + } + + non_taric_additional_fields = [ + "mechanism", + "category", + ] + + model_links = [] + + record_code = "360" + subrecord_code = "00" + + xml_object_tag = "quota.order.number" + + identity_fields = [ + "sid", + ] + + sid: int = None + order_number: str = None + valid_between_lower: date = None + valid_between_upper: date = None + + # non taric properties + mechanism: int = 0 # default + category: int = 1 # default + + +class QuotaOrderNumberOriginParserV2(BaseTaricParser): + model = QuotaOrderNumberOrigin + + value_mapping = { + "quota_order_number_origin_sid": "sid", + "quota_order_number_sid": "order_number__sid", + "validity_start_date": "valid_between_lower", + "validity_end_date": "valid_between_upper", + "geographical_area_id": "geographical_area__area_id", + "geographical_area_sid": "geographical_area__sid", + } + + model_links = [ + ModelLink( + QuotaOrderNumber, + [ + ModelLinkField("order_number__sid", "sid"), + ], + "quota.order.number", + ), + ModelLink( + GeographicalArea, + [ + ModelLinkField("geographical_area__area_id", "area_id"), + ModelLinkField("geographical_area__sid", "sid"), + ], + "geographical.area", + ), + ] + + xml_object_tag = "quota.order.number.origin" + record_code = "360" + subrecord_code = "10" + + identity_fields = [ + "sid", + ] + + sid: int = None + order_number__sid: int = None + geographical_area__area_id: str = None + valid_between_lower: date = None + valid_between_upper: date = None + geographical_area__sid: int = None + + +class QuotaOrderNumberOriginExclusionParserV2(BaseTaricParser): + model = QuotaOrderNumberOriginExclusion + + model_links = [ + ModelLink( + QuotaOrderNumberOrigin, + [ + ModelLinkField("origin__sid", "sid"), + ], + "quota.order.number.origin", + ), + ModelLink( + GeographicalArea, + [ + ModelLinkField("excluded_geographical_area__sid", "sid"), + ], + "geographical.area", + ), + ] + + value_mapping = { + "quota_order_number_origin_sid": "origin__sid", + "excluded_geographical_area_sid": "excluded_geographical_area__sid", + } + + xml_object_tag = "quota.order.number.origin.exclusions" + record_code = "360" + subrecord_code = "15" + + identity_fields = [ + "origin__sid", + "excluded_geographical_area__sid", + ] + + origin__sid: int = None + excluded_geographical_area__sid: int = None + + +class QuotaDefinitionParserV2(BaseTaricParser): + model = QuotaDefinition + + model_links = [ + ModelLink( + QuotaOrderNumber, + [ + ModelLinkField("order_number__order_number", "order_number"), + ModelLinkField("order_number__sid", "sid"), + ], + "quota.order.number", + ), + ModelLink( + MonetaryUnit, + [ + ModelLinkField("monetary_unit__code", "code"), + ], + "monetary.unit", + True, # optional + ), + ModelLink( + MeasurementUnit, + [ + ModelLinkField("measurement_unit__code", "code"), + ], + "measurement.unit", + True, # optional + ), + ModelLink( + MeasurementUnitQualifier, + [ + ModelLinkField("measurement_unit_qualifier__code", "code"), + ], + "measurement.unit.qualifier", + True, # optional + ), + ] + + xml_object_tag = "quota.definition" + record_code = "370" + subrecord_code = "00" + + value_mapping = { + "quota_definition_sid": "sid", + "quota_order_number_id": "order_number__order_number", + "validity_start_date": "valid_between_lower", + "validity_end_date": "valid_between_upper", + "quota_order_number_sid": "order_number__sid", + "monetary_unit_code": "monetary_unit__code", + "measurement_unit_code": "measurement_unit__code", + "measurement_unit_qualifier_code": "measurement_unit_qualifier__code", + "critical_state": "quota_critical", + "critical_threshold": "quota_critical_threshold", + } + + identity_fields = [ + "sid", + "order_number__sid", + ] + + sid: int = None + order_number__order_number: str = None + valid_between_lower: date = None + valid_between_upper: date = None + order_number__sid: int = None + volume: int = None + initial_volume: float = None + monetary_unit__code: str = None + measurement_unit__code: str = None + measurement_unit_qualifier__code: str = None + maximum_precision: int = None + quota_critical: bool = None + quota_critical_threshold: int = None + description: str = None + + +class QuotaAssociationParserV2(BaseTaricParser): + model = QuotaAssociation + model_links = [ + ModelLink( + QuotaDefinition, + [ + ModelLinkField("main_quota__sid", "sid"), + ], + "quota.definition", + ), + ModelLink( + QuotaDefinition, + [ + ModelLinkField("sub_quota__sid", "sid"), + ], + "quota.definition", + ), + ] + + value_mapping = { + "main_quota_definition_sid": "main_quota__sid", + "sub_quota_definition_sid": "sub_quota__sid", + "relation_type": "sub_quota_relation_type", + } + + record_code = "370" + subrecord_code = "05" + + xml_object_tag = "quota.association" + + identity_fields = [ + "main_quota__sid", + "sub_quota__sid", + "sub_quota_relation_type", + ] + + main_quota__sid: int = None + sub_quota__sid: int = None + sub_quota_relation_type: str = None + coefficient: float = None + + +class QuotaSuspensionParserV2(BaseTaricParser): + model = QuotaSuspension + + model_links = [ + ModelLink( + QuotaDefinition, + [ + ModelLinkField("quota_definition__sid", "sid"), + ], + "quota.definition", + ), + ] + + value_mapping = { + "suspension_start_date": "valid_between_lower", + "suspension_end_date": "valid_between_upper", + "quota_suspension_period_sid": "sid", + "quota_definition_sid": "quota_definition__sid", + } + + record_code = "370" + subrecord_code = "15" + + xml_object_tag = "quota.suspension.period" + + identity_fields = [ + "sid", + ] + + sid: int = None + quota_definition__sid: int = None + valid_between_lower: date = None + valid_between_upper: date = None + description: str = None + + +class QuotaBlockingParserV2(BaseTaricParser): + model = QuotaBlocking + + xml_object_tag = "quota.blocking.period" + record_code = "370" + subrecord_code = "10" + + value_mapping = { + "quota_blocking_period_sid": "sid", + "quota_definition_sid": "quota_definition__sid", + "blocking_start_date": "valid_between_lower", + "blocking_end_date": "valid_between_upper", + } + + model_links = [ + ModelLink( + QuotaDefinition, + [ + ModelLinkField("quota_definition__sid", "sid"), + ], + "quota.definition", + ), + ] + + identity_fields = [ + "sid", + ] + + sid: int = None + quota_definition__sid: int = None + valid_between_lower: date = None + valid_between_upper: date = None + blocking_period_type: int = None + description: str = None + + +class QuotaEventParserV2(BaseTaricParser): + model = QuotaEvent + + model_links = [ + ModelLink( + QuotaDefinition, + [ + ModelLinkField("quota_definition__sid", "sid"), + ], + "quota.definition", + ), + ] + + data_fields = [] + + record_code = "375" + subrecord_code = "subrecord_code" + + xml_object_tag = "parent.quota.event" + + identity_fields = [ + "quota_definition__sid", + "occurrence_timestamp", + ] + + quota_definition__sid: str = None + occurrence_timestamp: datetime = None + + @property + def data(self): + data_result = {} + for field in self.__class__.data_fields: + data_result[field.replace("_", ".")] = getattr(self, field) + + return json.dumps(data_result) + + +class QuotaBalanceEventParserV2(QuotaEventParserV2): + xml_object_tag = "quota.balance.event" + subrecord_code = "00" + + value_mapping = { + "quota_definition_sid": "quota_definition__sid", + } + + data_fields = [ + "new_balance", + "old_balance", + "imported_amount", + "last_import_date_in_allocation", + ] + + identity_fields = [ + "quota_definition__sid", + "occurrence_timestamp", + ] + + updates_allowed = False + + # data fields + new_balance: str = None + old_balance: str = None + imported_amount: str = None + last_import_date_in_allocation: str = None + + # fields + quota_definition__sid: int = None + occurrence_timestamp: datetime = None + + +class QuotaUnblockingEventParserV2(QuotaEventParserV2): + subrecord_code = "05" + + value_mapping = { + "quota_definition_sid": "quota_definition__sid", + } + + xml_object_tag = "quota.unblocking.event" + + identity_fields = [ + "quota_definition__sid", + "occurrence_timestamp", + ] + + data_fields = [ + "unblocking_date", + ] + + updates_allowed = False + + # data fields + unblocking_date: str = None + + # fields + quota_definition__sid: int = None + occurrence_timestamp: datetime = None + + +class QuotaCriticalEventParserV2(QuotaEventParserV2): + subrecord_code = "10" + + value_mapping = { + "quota_definition_sid": "quota_definition__sid", + } + + data_fields = [ + "critical_state", + "critical_state_change_date", + ] + + xml_object_tag = "quota.critical.event" + + identity_fields = [ + "quota_definition__sid", + "occurrence_timestamp", + ] + + updates_allowed = False + + # data fields + critical_state: str = None + critical_state_change_date: str = None + + # fields + quota_definition__sid: int = None + occurrence_timestamp: datetime = None + + +class QuotaExhaustionEventParserV2(QuotaEventParserV2): + subrecord_code = "15" + + value_mapping = { + "quota_definition_sid": "quota_definition__sid", + } + + data_fields = [ + "exhaustion_date", + ] + + xml_object_tag = "quota.exhaustion.event" + + identity_fields = [ + "quota_definition__sid", + "occurrence_timestamp", + ] + + updates_allowed = False + + # data fields + exhaustion_date: str = None + + # fields + quota_definition__sid: int = None + occurrence_timestamp: datetime = None + + +class QuotaReopeningEventParserV2(QuotaEventParserV2): + subrecord_code = "20" + + value_mapping = { + "quota_definition_sid": "quota_definition__sid", + } + + data_fields = [ + "reopening_date", + ] + + xml_object_tag = "quota.reopening.event" + + identity_fields = [ + "quota_definition__sid", + "occurrence_timestamp", + ] + + updates_allowed = False + + # data fields + reopening_date: str = None + + # fields + quota_definition__sid: int = None + occurrence_timestamp: datetime = None + + +class QuotaUnsuspensionEventParserV2(QuotaEventParserV2): + subrecord_code = "25" + + value_mapping = { + "quota_definition_sid": "quota_definition__sid", + } + + data_fields = [ + "unsuspension_date", + ] + + xml_object_tag = "quota.unsuspension.event" + + identity_fields = [ + "quota_definition__sid", + "occurrence_timestamp", + ] + + updates_allowed = False + + # data fields + unsuspension_date: str = None + + # fields + quota_definition__sid: int = None + occurrence_timestamp: datetime = None + + +class QuotaClosedAndTransferredEventParserV2(QuotaEventParserV2): + subrecord_code = "30" + + value_mapping = { + "quota_definition_sid": "quota_definition__sid", + } + + data_fields = [ + "quota_closed", + "transferred_amount", + "transfer_date", + "target_quota_definition_sid", + ] + + xml_object_tag = "quota.closed.and.transferred.event" + + identity_fields = [ + "quota_definition__sid", + "occurrence_timestamp", + ] + + updates_allowed = False + + # data fields + quota_closed: str = None + transferred_amount: str = None + transfer_date: str = None + target_quota_definition_sid: str = None + + # fields + quota_definition__sid: int = None + occurrence_timestamp: datetime = None diff --git a/taric_parsers/parsers/regulation_parser.py b/taric_parsers/parsers/regulation_parser.py new file mode 100644 index 000000000..3fd52e3df --- /dev/null +++ b/taric_parsers/parsers/regulation_parser.py @@ -0,0 +1,314 @@ +from datetime import date + +from regulations.models import Amendment +from regulations.models import Group +from regulations.models import Regulation +from regulations.models import Replacement +from regulations.models import Suspension +from taric_parsers.parser_model_link import ModelLink +from taric_parsers.parser_model_link import ModelLinkField +from taric_parsers.parsers.mixins import ValidityMixin +from taric_parsers.parsers.mixins import Writable +from taric_parsers.parsers.taric_parser import BaseTaricParser + + +class RegulationGroupParserV2(ValidityMixin, Writable, BaseTaricParser): + model = Group + + record_code = "150" + subrecord_code = "00" + + xml_object_tag = "regulation.group" + + model_links = [] + + value_mapping = { + "regulation_group_id": "group_id", + "validity_start_date": "valid_between_lower", + "validity_end_date": "valid_between_upper", + } + + identity_fields = [ + "group_id", + ] + + allow_update_without_children = True + + group_id: str = None + valid_between_lower: date = None + valid_between_upper: date = None + + +class RegulationGroupDescriptionParserV2(Writable, BaseTaricParser): + model = Group + parent_parser = RegulationGroupParserV2 + + model_links = [ + ModelLink( + Group, + [ + ModelLinkField("group_id", "group_id"), + ], + "group", + ), + ] + + value_mapping = { + "regulation_group_id": "group_id", + } + + record_code = "150" + subrecord_code = "05" + + xml_object_tag = "regulation.group.description" + + identity_fields = [ + "group_id", + ] + + deletes_allowed = False + group_id: str = None + description: str = None + + +class BaseRegulationParserV2(ValidityMixin, Writable, BaseTaricParser): + model = Regulation + + model_links = [ + ModelLink( + Group, + [ + ModelLinkField("regulation_group__group_id", "group_id"), + ], + "group", + ), + ] + + value_mapping = { + "officialjournal_number": "official_journal_number", + "officialjournal_page": "official_journal_page", + "published_date": "published_at", + "base_regulation_id": "regulation_id", + "base_regulation_role": "role_type", + "regulation_group_id": "regulation_group__group_id", + "validity_start_date": "valid_between_lower", + "validity_end_date": "valid_between_upper", + "stopped_flag": "stopped", + "approved_flag": "approved", + } + + record_code = "285" + subrecord_code = "00" + + xml_object_tag = "base.regulation" + + identity_fields = [ + "regulation_id", + "role_type", + ] + + role_type: int = None + regulation_id: str = None + published_at: date = None + official_journal_number: str = None + official_journal_page: int = None + valid_between_lower: date = None + valid_between_upper: date = None + effective_end_date: date = None + community_code: int = None + regulation_group__group_id: str = None + replacement_indicator: int = None + stopped: bool = None + information_text: str = None + approved: bool = None + + +class ModificationRegulationParserV2(ValidityMixin, Writable, BaseTaricParser): + model = Amendment + + record_code = "290" + subrecord_code = "00" + + xml_object_tag = "modification.regulation" + + value_mapping = { + "modification_regulation_role": "enacting_regulation__role_type", + "modification_regulation_id": "enacting_regulation__regulation_id", + "published_date": "enacting_regulation__published_at", + "officialjournal_number": "enacting_regulation__official_journal_number", + "officialjournal_page": "enacting_regulation__official_journal_page", + "validity_start_date": "enacting_regulation__valid_between_lower", + "validity_end_date": "enacting_regulation__valid_between_upper", + "effective_end_date": "enacting_regulation__effective_end_date", + "base_regulation_role": "target_regulation__role_type", + "base_regulation_id": "target_regulation__regulation_id", + "replacement_indicator": "enacting_regulation__replacement_indicator", + "stopped_flag": "enacting_regulation__stopped", + "information_text": "enacting_regulation__information_text", + "approved_flag": "enacting_regulation__approved", + } + + identity_fields = [ + "enacting_regulation__role_type", + "enacting_regulation__regulation_id", + "target_regulation__regulation_id", + "target_regulation__role_type", + ] + + enacting_regulation__role_type: int = None + enacting_regulation__regulation_id: str = None + enacting_regulation__published_at: date = None + enacting_regulation__official_journal_number: str = None + enacting_regulation__official_journal_page: int = None + enacting_regulation__valid_between_lower: date = None + enacting_regulation__valid_between_upper: date = None + enacting_regulation__effective_end_date: date = None + target_regulation__role_type: int = None + target_regulation__regulation_id: str = None + enacting_regulation__replacement_indicator: int = None + enacting_regulation__stopped: bool = None + enacting_regulation__information_text: str = None + enacting_regulation__approved: bool = None + + +class FullTemporaryStopRegulationParserV2( + ValidityMixin, + Writable, + BaseTaricParser, +): + """This handler creates both a base regulation with provided properties, and + a suspension.""" + + value_mapping = { + "full_temporary_stop_regulation_role": "enacting_regulation__role_type", + "full_temporary_stop_regulation_id": "enacting_regulation__regulation_id", + "published_date": "enacting_regulation__published_at", + "officialjournal_number": "enacting_regulation__official_journal_number", + "officialjournal_page": "enacting_regulation__official_journal_page", + "validity_start_date": "enacting_regulation__valid_between_lower", + "validity_end_date": "enacting_regulation__valid_between_upper", + "effective_enddate": "effective_end_date", + "replacement_indicator": "enacting_regulation__replacement_indicator", + "information_text": "enacting_regulation__information_text", + "approved_flag": "enacting_regulation__approved", + } + + model = Suspension + record_code = "300" + subrecord_code = "00" + + xml_object_tag = "full.temporary.stop.regulation" + + identity_fields = [ + "enacting_regulation__role_type", + "enacting_regulation__regulation_id", + ] + + enacting_regulation__role_type: int = None + enacting_regulation__regulation_id: str = None + enacting_regulation__published_at: date = None + enacting_regulation__official_journal_number: str = None + enacting_regulation__official_journal_page: int = None + enacting_regulation__valid_between_lower: date = None + enacting_regulation__valid_between_upper: date = None + effective_end_date: date = None + enacting_regulation__replacement_indicator: int = None + enacting_regulation__information_text: str = None + enacting_regulation__approved: bool = None + + +class FullTemporaryStopActionParserV2(Writable, BaseTaricParser): + model = Suspension + + model_links = [ + ModelLink( + Regulation, + [ + ModelLinkField("enacting_regulation__role_type", "role_type"), + ModelLinkField("enacting_regulation__regulation_id", "regulation_id"), + ], + "base.regulation", + ), + ModelLink( + Regulation, + [ + ModelLinkField("target_regulation__role_type", "role_type"), + ModelLinkField("target_regulation__regulation_id", "regulation_id"), + ], + "base.regulation", + ), + ] + + value_mapping = { + "fts_regulation_role": "enacting_regulation__role_type", + "fts_regulation_id": "enacting_regulation__regulation_id", + "stopped_regulation_role": "target_regulation__role_type", + "stopped_regulation_id": "target_regulation__regulation_id", + } + + record_code = "305" + subrecord_code = "00" + + xml_object_tag = "fts.regulation.action" + + identity_fields = [ + "enacting_regulation__role_type", + "enacting_regulation__regulation_id", + "target_regulation__regulation_id", + "target_regulation__role_type", + ] + + enacting_regulation__role_type: str = None + enacting_regulation__regulation_id: str = None + target_regulation__role_type: str = None + target_regulation__regulation_id: str = None + + +class RegulationReplacementParserV2(Writable, BaseTaricParser): + model = Replacement + + model_links = [ + ModelLink( + Regulation, + [ + ModelLinkField("enacting_regulation__role_type", "role_type"), + ModelLinkField("enacting_regulation__regulation_id", "regulation_id"), + ], + "base.regulation", + ), + ModelLink( + Regulation, + [ + ModelLinkField("target_regulation__role_type", "role_type"), + ModelLinkField("target_regulation__regulation_id", "regulation_id"), + ], + "base.regulation", + ), + ] + + value_mapping = { + "replacing_regulation_role": "enacting_regulation__role_type", + "replacing_regulation_id": "enacting_regulation__regulation_id", + "replaced_regulation_role": "target_regulation__role_type", + "replaced_regulation_id": "target_regulation__regulation_id", + } + + record_code = "305" + subrecord_code = "00" + + xml_object_tag = "regulation.replacement" + + identity_fields = [ + "enacting_regulation__role_type", + "enacting_regulation__regulation_id", + "target_regulation__regulation_id", + "target_regulation__role_type", + ] + + enacting_regulation__role_type: int = None + enacting_regulation__regulation_id: str = None + target_regulation__role_type: int = None + target_regulation__regulation_id: str = None + measure_type_id: str = None + geographical_area_id: str = None + chapter_heading: str = None diff --git a/taric_parsers/parsers/taric_parser.py b/taric_parsers/parsers/taric_parser.py new file mode 100644 index 000000000..9f0b93134 --- /dev/null +++ b/taric_parsers/parsers/taric_parser.py @@ -0,0 +1,833 @@ +from __future__ import annotations + +from datetime import date +from datetime import datetime +from typing import List +from typing import get_type_hints + +import bs4 +from bs4 import NavigableString + +from common import validators +from common.models import Transaction +from common.util import TaricDateRange +from common.validators import UpdateType +from quotas.models import QuotaEvent +from taric_parsers.importer_issue import ImportIssueReportItem +from taric_parsers.parser_model_link import ModelLink + +# import all parsers +EXCLUDED_PARSER_PROPERTIES = [ + "__annotations__", + "__doc__", + "__module__", + "issues", + "model", + "model_links", + "parent_parser", + "value_mapping", + "xml_object_tag", + "valid_between_lower", + "valid_between_upper", + "sequence_number", + "update_type_name", + "links_valid", + "transaction_id", + "import_changes", +] + +EXCLUDED_FIELDS_FOR_POPULATION = [ + "language_id", + "antidumping_regulation_role", + "related_antidumping_regulation_id", + "complete_abrogation_regulation_role", + "complete_abrogation_regulation_id", + "explicit_abrogation_regulation_role", + "explicit_abrogation_regulation_id", + "export_refund_nomenclature_sid", + "meursing_table_plan_id", +] + + +class TransactionParser: + """ + Responsible for representing a parsed TARIC transaciton. + + A transmission contains multiple messages. + + Parsing a transmission will result in an ordered list of parsed messages + (taric objects we need to try and import) + """ + + def __init__(self, transaction: bs4.Tag, index): + """ + Responsible for representing a parsed TARIC transaction. + + Args: + transaction: (required) bs4.Tag, The containing XML tag that contains the transaction data + index: (required) + """ + self.parsed_messages: List[MessageParser] = [] + self.taric_objects = [] + self.index = index + + self.messages_xml_tags = transaction.find_all("env:app.message") + + for message_xml in self.messages_xml_tags: + self.parsed_messages.append(MessageParser(message_xml)) + + for message in self.parsed_messages: + self.taric_objects.append(message.taric_object) + + +class MessageParser: + """Responsible for representing a parsed TARIC message.""" + + def __init__(self, message: bs4.Tag): + """ + Initialise a message parser instance. + + This object represents a parsed TARIC3 message - the container for all object types, e.g. goods nomenclature + + Args: + message: (required) bs4.Tag, The containing XML tag that contains the message data + """ + + self.data = {} + self.message = message + self.transaction_id = self.message.find("oub:transaction.id").text + self.record_code = self.message.find("oub:record.code").text + self.subrecord_code = self.message.find("oub:subrecord.code").text + self.sequence_number = self.message.find("oub:record.sequence.number").text + self.update_type = int(self.message.find("oub:update.type").text) + self.object_type = "" + sibling = self.message.find("oub:update.type").next_sibling + + # get object type + while self.object_type == "": + if sibling is None: + break + elif isinstance(sibling, NavigableString): + sibling = sibling.next_sibling + continue + elif isinstance(sibling, bs4.Tag): + self.object_type = sibling.name + break + + for update_type in UpdateType: + if update_type.value == self.update_type: + self.update_type_name = update_type.name.lower() + break + + self._populate_data_dict(self.update_type, self.update_type_name) + self.taric_object = self._construct_taric_object() + + def can_populate_child_attrs_from_history(self): + """ + Determines if an object can be updated without all child parsers + present. + + Returns: + bool, indicates if the parsed object can be updated without all children present + """ + + if self.update_type != validators.UpdateType.CREATE: + # child objects allowed to be populated from history on updates + return self.taric_object.__class__.allow_update_without_children + + return False + + def _populate_data_dict(self, update_type, update_type_name): + """Iterates through properties in the object tag and returns a + dictionary of those properties.""" + + # also set update type and string representation + + self.data = { + "update_type": update_type, + "update_type_name": update_type_name, + } + + for tag in self.message.find(self.object_type).children: + if isinstance(tag, NavigableString): + continue + elif isinstance(tag, bs4.Tag): + data_property_name = self._parse_tag_name(tag.name, self.object_type) + self.data[data_property_name] = tag.text + + return + + def _construct_taric_object(self): + parser_cls = ParserHelper.get_parser_by_tag(self.object_type) + parser = parser_cls() + + parser.populate( + self.transaction_id, + self.record_code, + self.subrecord_code, + self.sequence_number, + self.data, + ) + + return parser + + def _parse_tag_name(self, tag_name, object_type): + parsed_tag_name = tag_name + + # anything left will be full stop seperated, this needs to change to underscores + parsed_tag_name = parsed_tag_name.replace(".", "_") + + return parsed_tag_name + + +class BaseTaricParser: + """ + This object represents a TARIC3 parsed model, this is the object that does a + lot of the work, connecting the TARIC3 data and makes ORM models. There are + a few key concepts worth documenting below. + + value_mapping : + This dictionary (that may change to an array of models at some point) has key : value pairs that are used to + map TARIC3 is values from the XML to a subclass of this model and at the same time, change the name of the + field if needed. Values that don't need mapping to a new name just import with that name (but replacing full + stops with underscores) + + the value mapping allows at processing time, for values to be renamed in preparation to + represent the ORM fields. The structure is: + + { + taric3_xml_field_name: parser_field_name + } + + Where the left side (key) is the name of the field in XML, and the right side (value) is the + destination field on the parser. + + Additional Note: + Some values represent a foreign key, for example : additional_code__sid + + note the double underscore - like filtering in django ORM, this represents a relationship + + If a field name has double underscore, firstly the property e.g. 'additional_code' must exist on the destination + model in the django ORM, it will be checked. and secondly, the field after the first double underscore must + exist on the related model, if not it will not import. + """ + + transaction_id: int = None + record_code: str = None + subrecord_code: str = None + xml_object_tag: str = None + update_type: int = None + update_type_name: str = None + links_valid: bool = None + value_mapping = {} + model_links = [] + parent_parser = None + data_fields = [] + issues = [] + parent_handler = None + non_taric_additional_fields = [] + model = None + identity_fields = [] + import_changes = True + + # Properties to explicitly define behaviour + allow_update_without_children = False + updates_allowed = True + deletes_allowed = True + skip_identity_check = False + + def __init__(self): + """Initialises a blank instance of BaseTaricParser, with default + values.""" + self.issues = [] + self.sequence_number = None + self.model = None + + def links(self) -> list[ModelLink]: + """ + Get defined links to other models. + + Returns: + list[ModelLink], A list of model link objects defined on the mdoel + """ + if self.model_links is None: + raise Exception( + f"No model defined for {self.__class__.__name__}, is this correct?", + ) + + return self.model_links + + def model_query_parameters(self) -> dict: + """ + Get the models query parameters, to search for it, and parent objects in + the import data and in the database. + + Returns: + dict, a dictionary of the parsers identity fields, and the corresponding values. + """ + query_args = {} + for identity_field in self.identity_fields: + query_arg_value = getattr(self, identity_field) + + if query_arg_value is None or query_arg_value == "": + raise Exception( + f"No value for identity field {identity_field} for object : {self.__class__.__name__}", + ) + + query_args[identity_field] = query_arg_value + + if len(query_args.keys()) == 0: + raise Exception( + f"No arguments present for object : {self.__class__.__name__}", + ) + + return query_args + + def missing_child_attributes(self): + """ + When a parent object that has children for example a description that + also contains description period properties is not fully populated, for + example the period is not present in the import, and the description is + being created (not updated) there is no way to populate the period. + + This method is designed to highlight this issue + """ + + # check if the object has children, if not - return false + child_parsers = ParserHelper.get_child_parsers(self) + if len(child_parsers) == 0: + return None + + # if it does, get the fields that should be populated by a child relationship and return true, and add a import + # issue to the object. + result = {} + + for child_parser in child_parsers: + field_sets = child_parser.identity_fields_for_parent() + for child_field in field_sets.keys(): + parent_field = field_sets[child_field] + + if not hasattr(self, parent_field): + # Guard clause + raise Exception( + f"Field referenced by child {child_parser.__name__} : {child_field} " + f"does not exist on parent {self.__class__.__name__} : {parent_field}", + ) + + # Include attribute in response if empty + if getattr(self, parent_field) is None: + if str(child_parser.__name__) in result.keys(): + result[str(child_parser.__name__)].append(child_field) + else: + result[str(child_parser.__name__)] = [child_field] + + return result + + def is_child_for(self, potential_parent) -> bool: + """ + Returns a boolean to indicate of the current is associated with the + potential parent. + + Args: + potential_parent: (required) BaseTaricParser, A parsed object that needs to be checked against. + + Returns: + bool, indicating if the potential_parent matches the identity keys the child has. + """ + if ( + potential_parent.is_child_object() + or potential_parent.__class__.model != self.__class__.model + ): + return False + + identity_fields = self.get_identity_fields_and_values_for_parent() + + # guard clause + if len(identity_fields.keys()) == 0: + raise Exception("No parent identity fields presented") + + match = True + for identity_field in identity_fields.keys(): + if ( + getattr(potential_parent, identity_field) + != identity_fields[identity_field] + ): + match = False + + return match + + def get_identity_fields_and_values_for_parent(self): + """Return a dict of values to be used to query the parent, using the + child values.""" + result = {} + + for value in self.identity_fields_for_parent().values(): + result[value] = getattr(self, value) + + return result + + @classmethod + def identity_fields_for_parent(cls, include_optional=False) -> dict: + """ + Returns a dictionary of identity keys and values that will link the + child to the parent. + + Args: + include_optional: bool, if the relationship is optional it will not be included if include_optional is false + + Returns: + dict, a dictionary of field mappings between child and parent parser objects + """ + + # guard clauses + if cls.parent_parser is None: + raise Exception(f"Model {cls.__name__} has no parent parser") + + if cls.model_links is None or cls.model_links == []: + raise Exception( + f"Model {cls.__name__} appears to have a parent parser but no model links", + ) + + matched_parent_class = False + for link in cls.model_links: + if link.model == cls.parent_parser.model: + matched_parent_class = True + if not matched_parent_class: + raise Exception( + f"Model {cls.__name__} appears to not have a model links to the parent model", + ) + + key_fields = {} + for model_link in cls.model_links: + # match link to target model for parser - then we can extract the key fields we need to match for the object + if model_link.model == cls.model: + for model_link_field in model_link.fields: + if not model_link.optional or ( + model_link.optional and include_optional + ): + key_fields[ + model_link_field.parser_field_name + ] = model_link_field.object_field_name + + return key_fields + + def populate( + self, + transaction_id: int, + record_code: str, + subrecord_code: str, + sequence_number: int, + data: dict, + ): + """ + Populate the parser from the defined properties and data. + + Args: + transaction_id: (required) int, The transaction ID from TARIC XML + record_code: (required) str, record code from TARIC XML + subrecord_code: (required) str, subrecord code from TARIC XML + sequence_number: (required) int, sequence number from TARIC XML + data: (required) dict, data from TARIC XML + + Returns: + None, all actions are performed on the current object + """ + # standard data + self.transaction_id = transaction_id + if self.record_code != record_code: + raise Exception( + f"Record code mismatch : expected : {self.record_code}, got : {record_code} - data: {data}, Type: {self.__class__.__name__}", + ) + if self.subrecord_code != subrecord_code: + raise Exception( + f"Sub-record code mismatch : expected : {self.subrecord_code}, got : {subrecord_code} - data: {data}, Type: {self.__class__.__name__}", + ) + + self.sequence_number = sequence_number + + # model specific data + for data_item_key in data.keys(): + # some fields like language_id need to be skipped - we only care about en + if data_item_key in EXCLUDED_FIELDS_FOR_POPULATION: + continue + + mapped_data_item_key = data_item_key + if data_item_key in self.value_mapping: + mapped_data_item_key = self.value_mapping[data_item_key] + + if hasattr(self, mapped_data_item_key): + field_data_raw = data[data_item_key] + + if mapped_data_item_key == "update_type": + field_data_type = int + elif mapped_data_item_key == "update_type_name": + field_data_type = str + else: + field_data_type = get_type_hints(self)[mapped_data_item_key] + + field_data_typed = None + + # convert string objects to the correct types, based on parser class annotations + + if field_data_type == str: + field_data_typed = str(field_data_raw) + elif field_data_type == date: + if field_data_raw is not None: + field_data_typed = datetime.strptime( + field_data_raw, + "%Y-%m-%d", + ).date() + elif field_data_type == datetime: + if field_data_raw is not None: + field_data_typed = datetime.fromisoformat(field_data_raw) + elif field_data_type == int: + field_data_typed = int(field_data_raw) + elif field_data_type == float: + field_data_typed = float(field_data_raw) + elif field_data_type == bool: + if field_data_raw in ["1", "Y"]: + field_data_typed = True + elif field_data_raw in ["0", "N"]: + field_data_typed = False + else: + raise Exception( + f"data value for bool : {field_data_raw} not handled, should only be 1 or 0, {mapped_data_item_key}", + ) + else: + raise Exception( + f"data type {field_data_type.__name__} not handled, does the handler have the correct data type?", + ) + + setattr(self, mapped_data_item_key, field_data_typed) + else: + raise Exception( + f"{self.xml_object_tag} {self.__class__.__name__} does not have a {mapped_data_item_key} attribute, and " + f"can't assign value {data[data_item_key]}", + ) + + if hasattr(self, "valid_between_lower") and hasattr( + self, + "valid_between_upper", + ): + if self.valid_between_upper: + self.valid_between = TaricDateRange( + self.valid_between_lower, + self.valid_between_upper, + ) + else: + self.valid_between = TaricDateRange(self.valid_between_lower) + + def get_linked_model( + self, + fields_and_values: dict, + related_model, + transaction: Transaction, + ): + """ + Get the linked model from the existing database records. + + Args: + fields_and_values: dict, dictionary of fields and properties that are used to query the database + related_model: TrackedModel class, The tracked model that is to be queried + transaction: Transaction, The transaction (in the database) that the query can perform up to. + + Returns: + TrackedModel, when matched + None, When not matched + + Exception: + When multiple models are matched which is invalid and should not happen normally + """ + models = related_model.objects.approved_up_to_transaction(transaction).filter( + **fields_and_values, + ) + + if models.count() == 1: + return models.first() + elif models.count() > 1: + filtered_models = [] + for model in models: + if hasattr(model, "valid_between"): + # Check if this record is current + if date.today() in model.valid_between: + filtered_models.append(model) + + elif hasattr(model, "validity_start"): + # check for latest + if ( + len(filtered_models) > 0 + and model.validity_start > filtered_models[0].validity_start + ): + filtered_models = [model] + elif len(filtered_models) == 0: + filtered_models = [model] + + if len(filtered_models) == 1: + return filtered_models[0] + + raise Exception( + f"multiple models matched query for {related_model.__name__} using {fields_and_values}, please check data and query", + ) + else: + return None + + def model_attributes( + self, + transaction: Transaction, + raise_import_issue_if_no_match=True, + include_non_taric_attributes=False, + json_compatible=False, + ) -> dict: + """ + Returns a dictionary of model attributes, for use in populating database + models, and other uses. + + Args: + transaction: (required) Transaction, The transaction used to search the database up to for models + raise_import_issue_if_no_match: (optional) bool, Flag to indicate if an import issue should be created if a related model cant be matched. + include_non_taric_attributes: (optional) bool, flag to indicate if output should include non TARIC attributes. There are some edge cases where this is needed. + json_compatible: (optional) bool, flat to indicate if the output should be JSON serializable + + Returns: + dict, Dictionary of populated attributes for the model + """ + additional_excluded_variable_names = [] + + model_attributes = {} + + # resolve links to other models + for link in self.model_links: + # check all fields link to the same property / linked model + property_list = [] + for field in link.fields: + additional_excluded_variable_names.append(field.parser_field_name) + property_list.append(field.parser_field_name.split("__")[0]) + + property_list = list(dict.fromkeys(property_list)) + + # if an exception raises from the two checks below, it indicates that there is an issue with the parser + # in some way, these circumstances should not occur if the parsers are good and well-formed . + if len(property_list) > 1: + raise Exception( + f"multiple properties for link : {self.__class__.__name__} : {property_list}", + ) + elif len(property_list) == 0: + raise Exception(f"no properties for link : {self.__class__.__name__}") + + fields_and_values = {} + for field in link.fields: + fields_and_values[field.object_field_name] = getattr( + self, + field.parser_field_name, + ) + + linked_model = self.get_linked_model( + fields_and_values, + link.model, + transaction, + ) + + if linked_model: + # There are cases where this will not return a value, which is fine when the linked + # model is also in the same transaction + if json_compatible: + model_attributes[property_list[0]] = str(linked_model) + else: + model_attributes[property_list[0]] = linked_model + + elif raise_import_issue_if_no_match and not link.optional: + report_item = ImportIssueReportItem( + self.xml_object_tag, + ParserHelper.get_parser_by_model(link.model).xml_object_tag, + fields_and_values, + f"Missing expected linked object {ParserHelper.get_parser_by_model(link.model).__name__}", + object_update_type=self.update_type, + object_data={}, + transaction_id=self.transaction_id, + ) + + self.issues.append(report_item) + + for model_field in vars(self).keys(): + if ( + model_field + in EXCLUDED_PARSER_PROPERTIES + additional_excluded_variable_names + ): + continue + + # Only append non data fields to the model, data fields are used to collect and attach via a defined + # column in json format - mainly used for quota events + if model_field not in self.data_fields: + if hasattr(self.__class__.model, model_field): + # if we are storing attributes to a JSON field, we need to break apart the TaricDateRange as it's not JSON serializable as it is + if json_compatible: + if isinstance(getattr(self, model_field), TaricDateRange): + model_attributes[model_field] = { + "start": str(getattr(self, model_field).lower), + "end": str(getattr(self, model_field).upper), + } + elif isinstance(getattr(self, model_field), date): + model_attributes[model_field] = str( + getattr(self, model_field), + ) + else: + model_attributes[model_field] = getattr(self, model_field) + else: + model_attributes[model_field] = getattr(self, model_field) + else: + raise Exception( + f"Error creating model {self.__class__.model.__name__}, model does not have an attribute {model_field}", + ) + + # QuotaEvents have the subrecord code recorded in the database table to distinguish the type + if self.__class__.model == QuotaEvent: + model_attributes["subrecord_code"] = self.subrecord_code + + # there are instances where no taric fields have defaults, and if not populated will cause an exception when + # data is written to the database. non_taric_attribute is a property available in each of the parsers and can be + # populated with these values, and will receive a default value from the parser model on creation + if include_non_taric_attributes: + for non_taric_attribute in self.non_taric_additional_fields: + model_attributes[non_taric_attribute] = getattr( + self, + non_taric_attribute, + ) + + # finally, if this model; has a parent, remove sid, code and group_id which will be linked to the model it should be updating + if self.parent_parser: + for field in ["sid", "code", "group_id"]: + if field in model_attributes.keys() and field in self.identity_fields: + del model_attributes[field] + + return model_attributes + + def can_save_to_model(self) -> bool: + """ + Can the model be saved to the database. + + This method checks that the parser that represents a TARIC object can be saved to a TAP model, checking parent_parser and update type. + + A child parser can be saved directly to the parent as an update. Creates and deletes need the accompanying parent to action. + + Returns: + bool, value indicating if the model can be saved + """ + if self.parent_parser: + if self.update_type == validators.UpdateType.UPDATE: # update + return True + return False + return True + + def is_child_object(self): + """ + Is the object a child object, meaning does it have a parent it needs to + append attributes to. + + Returns: + bool, boolean indicating it is or is not a child object in the TAP database model + """ + return self.parent_parser is not None + + +class ParserHelper: + @staticmethod + def get_parser_by_model(model): + """ + Gets the corresponding parser for the presented model. + + Args: + model: TrackedModel Class, The class to get the parser class for + + Returns: + BaseTaricParser, The parser class used to represent the presented model + + Exception: + If no matching parser class is found. + """ + # get all classes that can represent an imported taric object + classes = ParserHelper.get_parser_classes() + + # iterate through classes and find the one that matches the tag or error + for cls in classes: + if cls.model == model and cls.parent_parser is None: + return cls + + raise Exception( + f"No parser class found for parsing {model.__name__}. Have you imported all required parser models? ", + ) + + @staticmethod + def get_child_parsers(parser: BaseTaricParser): + """ + Returns a list of child parsers associated with the presented parser. + + Args: + parser: (required) BaseTaricParser, The parent parser class, to get the children for. + + Returns: + list, list containing all child parsers associated with the provided model + """ + result = [] + + parser_classes = ParserHelper.get_parser_classes() + for parser_class in parser_classes: + if parser_class.parent_parser and parser_class.parent_parser == type( + parser, + ): + result.append(parser_class) + + return result + + @staticmethod + def get_parser_by_tag(object_type: str): + """ + Returns the child parsers associated with the presented XML tag, from + the TARIC spec. + + Args: + object_type: (required) str, string representing the XML tag + + Returns: + BaseTaricParser Class, class matching the provided tag + + Exception: + raises if there is no match + """ + # get all classes that can represent an imported taric object + classes = ParserHelper.get_parser_classes() + + # iterate through classes and find the one that matches the tag or error + for cls in classes: + if cls.xml_object_tag == object_type: + return cls + + raise Exception(f"No parser class matching {object_type}") + + @staticmethod + def get_parser_classes() -> list[BaseTaricParser]: + """ + List of parser classes. + + Returns: + list[BaseTaricParser], list of all classes that parse TARIC objects + """ + return ParserHelper.subclasses_for(BaseTaricParser) + + @staticmethod + def subclasses_for(cls) -> list: + """ + Recursive, Returns all subclasses of the provided class. + + Args: + cls: class, Any class, that you need all the subclasses for. + + Returns: + list, list of classes + """ + all_subclasses = [] + + for subclass in cls.__subclasses__(): + all_subclasses.append(subclass) + all_subclasses.extend(ParserHelper.subclasses_for(subclass)) + + return all_subclasses diff --git a/taric_parsers/taric_xml_source.py b/taric_parsers/taric_xml_source.py new file mode 100644 index 000000000..8a198abd5 --- /dev/null +++ b/taric_parsers/taric_xml_source.py @@ -0,0 +1,20 @@ +class TaricXMLSourceBase: + def get_xml_string(self): + raise NotImplementedError("Implement on child class") + + +class TaricXMLFileSource(TaricXMLSourceBase): + def __init__(self, file_path: str): + self.file_path = file_path + + def get_xml_string(self): + with open(self.file_path, "r") as file: + return file.read() + + +class TaricXMLStringSource(TaricXMLSourceBase): + def __init__(self, xml_string: str): + self.xml_string = xml_string + + def get_xml_string(self): + return self.xml_string diff --git a/taric_parsers/tasks.py b/taric_parsers/tasks.py new file mode 100644 index 000000000..0b6d08870 --- /dev/null +++ b/taric_parsers/tasks.py @@ -0,0 +1,172 @@ +from logging import getLogger +from typing import Sequence + +from django.contrib.auth.models import User + +import taric_parsers.importer +from common.celery import app +from importer.models import BatchImportError +from importer.models import ImportBatch +from importer.models import ImporterChunkStatus +from importer.models import ImporterXMLChunk +from importer.models import ImportIssueType +from taric_parsers.taric_xml_source import TaricXMLStringSource +from workbaskets.models import WorkBasket +from workbaskets.models import get_partition_scheme + +logger = getLogger(__name__) + + +@app.task +def parse_and_import( + chunk_pk: int, + username: str, + workbasket_title: str, +): + """ + Task for importing an XML chunk into the database. + + This task must ensure the chunks workbasket_status reflects the import + process, whether it is currently running, errored or done. Once complete it + is also responsible for finding and setting up the next chunk tasks. + """ + + chunk = ImporterXMLChunk.objects.get(pk=chunk_pk) + batch = chunk.batch + + logger.info( + "RUNNING CHUNK Batch: %s Record code: %s Chapter heading: %s Chunk number: %d", + batch.name, + chunk.record_code, + chunk.chapter, + chunk.chunk_number, + ) + + chunk.status = ImporterChunkStatus.RUNNING + chunk.save() + + try: + importer = taric_parsers.importer.TaricImporter( + import_batch=batch, + taric_xml_source=TaricXMLStringSource(chunk.chunk_text), + ) + + if importer.can_save(): + # at this point we can create the workbasket and have a high degree of confidence that the import will complete. + import_user = User.objects.all().get(username=username, is_active=True) + workbasket = WorkBasket.objects.create( + title=workbasket_title, + author=import_user, + ) + + # need to associate workbasket with the ImportBatch + batch.workbasket = workbasket + batch.save() + + importer.process_and_save_if_valid(workbasket) + elif importer.is_empty(): + importer.clear_issues() + else: + importer.commit_issues() + + if len(importer.issues(filter_by_issue_type=ImportIssueType.ERROR)) > 0: + chunk.status = ImporterChunkStatus.ERRORED + else: + chunk.status = ImporterChunkStatus.DONE + + chunk.save() + + except Exception as e: + batch.failed() + batch.save() + + chunk.status = ImporterChunkStatus.ERRORED + chunk.save() + + import_error = BatchImportError( + object_type="Exception at Chunk Creation", + related_object_type="", + related_object_identity_keys="", + description=str(e), + batch=batch, + object_update_type=None, + object_data=None, + transaction_id=0, + issue_type=ImportIssueType.ERROR, + ) + + import_error.save() + + raise e + + batch_errored_chunks = batch.chunks.filter( + status=ImporterChunkStatus.ERRORED, + ) + + if not batch.ready_chunks.exists(): + if not batch_errored_chunks: + if batch.chunks.count() == 0: + # This indicates that there was not any data to import. The batch should. + # be flagged with status FAILED_EMPTY. + batch.failed_empty() + else: + # This was batch's last chunk requiring processing, and it has no + # chunks with status ERRORED, so transition batch to SUCCEEDED. + batch.succeeded() + else: + # This was batch's last chunk requiring processing, and it did have + # chunks with status ERRORED, so transition batch to ERRORED. + batch.failed() + batch.save() + + +def setup_new_chunk_task( + batch: ImportBatch, + workbasket_id: str, + workbasket_status: str, + partition_scheme_setting: str, + username: str, + record_code: str = None, + record_group: Sequence[str] = None, + **kwargs, +): + """ + Setup new task to be run for the given chunk. + + Once a task is made it is important that the task is not duplicated. To stop + this the system checks the chunk status. If the status is `RUNNING`, + `ERRORED` or `DONE` the task is not setup. If the status is `WAITING` then + the status is updated to `RUNNING` and the task is set up. + """ + + # Call get_partition_scheme before invoking celery so that it can raise ImproperlyConfigured if + # partition_scheme_setting is invalid. + + get_partition_scheme(partition_scheme_setting) + + if batch.ready_chunks.filter( + record_code=record_code, + status=ImporterChunkStatus.RUNNING, + **kwargs, + ).exists(): + return + + chunk = ( + batch.ready_chunks.filter(record_code=record_code, **kwargs) + .order_by("chunk_number") + .first() + ) + + if not chunk: + return + + chunk.status = ImporterChunkStatus.RUNNING + chunk.save() + parse_and_import.delay( + chunk.pk, + workbasket_id, + workbasket_status, + partition_scheme_setting, + username, + record_group=record_group, + ) diff --git a/taric_parsers/tests/__init__.py b/taric_parsers/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_CREATE.xml b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_CREATE.xml new file mode 100644 index 000000000..d81d37516 --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_CREATE.xml @@ -0,0 +1,55 @@ + + + + + + + 1 + 120 + 00 + 1 + 3 + + 5 + 3 + 2021-01-01 + 2021-12-31 + + + + + + + + 1 + 245 + 00 + 2 + 3 + + 1 + 5 + 3 + 2021-01-01 + + + + + + + + 1 + 120 + 05 + 3 + 3 + + 5 + zz + some description + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_DELETE.xml b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_DELETE.xml new file mode 100644 index 000000000..e7cc074b8 --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_DELETE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 245 + 00 + 2 + 2 + + 1 + 5 + 3 + 2021-01-11 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_UPDATE.xml b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_UPDATE.xml new file mode 100644 index 000000000..0e26874c7 --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_UPDATE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 245 + 00 + 2 + 1 + + 1 + 5 + 3 + 2021-01-11 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_CREATE.xml b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_CREATE.xml new file mode 100644 index 000000000..3ab2963cf --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_CREATE.xml @@ -0,0 +1,92 @@ + + + + + + + 1 + 120 + 00 + 1 + 3 + + 9 + 7 + 2021-01-01 + 2021-12-31 + + + + + + + + 1 + 120 + 05 + 2 + 3 + + 9 + zz + some description + + + + + + + + 1 + 245 + 00 + 2 + 3 + + 1 + 9 + 2 + 2021-01-01 + + + + + + + + 1 + 245 + 05 + 3 + 3 + + 5 + 1 + 9 + 2 + 2021-01-01 + + + + + + + + 1 + 245 + 10 + 4 + 3 + + 5 + zz + 1 + 9 + 2 + some description + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_DELETE.xml b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_DELETE.xml new file mode 100644 index 000000000..50fee259d --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_DELETE.xml @@ -0,0 +1,24 @@ + + + + + + + 1 + 245 + 10 + 4 + 2 + + 5 + zz + 1 + 9 + 2 + some description change + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_UPDATE.xml b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_UPDATE.xml new file mode 100644 index 000000000..c72e22d13 --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_UPDATE.xml @@ -0,0 +1,24 @@ + + + + + + + 1 + 245 + 10 + 4 + 1 + + 5 + zz + 1 + 9 + 2 + some description change + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_invalid_additional_code_CREATE.xml b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_invalid_additional_code_CREATE.xml new file mode 100644 index 000000000..905062ecc --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_invalid_additional_code_CREATE.xml @@ -0,0 +1,24 @@ + + + + + + + 1 + 245 + 10 + 1 + 3 + + 5 + zz + 1 + 12 + 111 + some description + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_period_CREATE.xml b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_period_CREATE.xml new file mode 100644 index 000000000..42c4c31b8 --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_period_CREATE.xml @@ -0,0 +1,92 @@ + + + + + + + 1 + 120 + 00 + 1 + 3 + + 4 + 3 + 2021-01-01 + 2021-12-31 + + + + + + + + 1 + 120 + 05 + 2 + 3 + + 4 + zz + some description + + + + + + + + 1 + 245 + 00 + 3 + 3 + + 1 + 4 + 3 + 2021-01-01 + + + + + + + + 1 + 245 + 05 + 4 + 3 + + 5 + 1 + 4 + 3 + 2021-01-01 + + + + + + + + 1 + 245 + 10 + 5 + 3 + + 5 + zz + 1 + 4 + 3 + some description + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_period_DELETE.xml b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_period_DELETE.xml new file mode 100644 index 000000000..3512c4da3 --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_period_DELETE.xml @@ -0,0 +1,23 @@ + + + + + + + 1 + 245 + 05 + 4 + 2 + + 5 + 1 + 4 + 3 + 2021-01-11 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_period_UPDATE.xml b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_period_UPDATE.xml new file mode 100644 index 000000000..5cf1efb3a --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_period_UPDATE.xml @@ -0,0 +1,23 @@ + + + + + + + 1 + 245 + 05 + 4 + 1 + + 5 + 1 + 4 + 3 + 2021-01-11 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_period_without_description_CREATE.xml b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_period_without_description_CREATE.xml new file mode 100644 index 000000000..b7daaaef3 --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_description_period_without_description_CREATE.xml @@ -0,0 +1,57 @@ + + + + + + + 1 + 120 + 00 + 1 + 3 + + 5 + 3 + 2021-01-01 + 2021-12-31 + + + + + + + + 1 + 245 + 00 + 2 + 3 + + 1 + 5 + 3 + 2021-01-01 + + + + + + + + 1 + 245 + 05 + 3 + 3 + + 5 + 1 + 5 + 3 + 2021-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_invalid_type_CREATE.xml b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_invalid_type_CREATE.xml new file mode 100644 index 000000000..13d5184f3 --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_invalid_type_CREATE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 245 + 00 + 1 + 3 + + 1 + 12 + 111 + 2021-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_CREATE.xml b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_CREATE.xml new file mode 100644 index 000000000..7c2ed545f --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_CREATE.xml @@ -0,0 +1,39 @@ + + + + + + + 1 + 120 + 05 + 2 + 3 + + 1 + zz + some description + + + + + + + + 1 + 120 + 00 + 1 + 3 + + 1 + 111 + 2021-01-01 + 2021-12-31 + + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_DELETE.xml b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_DELETE.xml new file mode 100644 index 000000000..fedb81e96 --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_DELETE.xml @@ -0,0 +1,23 @@ + + + + + + + 1 + 120 + 00 + 1 + 2 + + 1 + 111 + 2021-01-11 + 2021-12-31 + + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_UPDATE.xml b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_UPDATE.xml new file mode 100644 index 000000000..6fad12874 --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_UPDATE.xml @@ -0,0 +1,23 @@ + + + + + + + 1 + 120 + 00 + 1 + 1 + + 1 + 111 + 2021-01-11 + 2021-12-31 + + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_description_CREATE.xml b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_description_CREATE.xml new file mode 100644 index 000000000..2902c032f --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_description_CREATE.xml @@ -0,0 +1,38 @@ + + + + + + + 1 + 120 + 00 + 1 + 3 + + 5 + 3 + 2021-01-01 + 2021-12-31 + + + + + + + + 1 + 120 + 05 + 3 + 3 + + 5 + zz + some description + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_description_DELETE.xml b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_description_DELETE.xml new file mode 100644 index 000000000..253c5a0ef --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_description_DELETE.xml @@ -0,0 +1,21 @@ + + + + + + + 1 + 120 + 05 + 2 + 2 + + 5 + zz + some description that changed + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_description_UPDATE.xml b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_description_UPDATE.xml new file mode 100644 index 000000000..3bfaefe97 --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_description_UPDATE.xml @@ -0,0 +1,21 @@ + + + + + + + 1 + 120 + 05 + 2 + 1 + + 5 + zz + some description that changed + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_description_without_type_CREATE.xml b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_description_without_type_CREATE.xml new file mode 100644 index 000000000..c9381a07b --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_description_without_type_CREATE.xml @@ -0,0 +1,21 @@ + + + + + + + 1 + 120 + 05 + 1 + 3 + + 12 + zz + some description + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_no_description_CREATE.xml b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_no_description_CREATE.xml new file mode 100644 index 000000000..743921895 --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/importer_examples/additional_code_type_no_description_CREATE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 120 + 00 + 1 + 3 + + 5 + 111 + 2021-01-01 + 2021-12-31 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/additional_code_parsers/importer_examples/footnote_association_additional_code_CREATE.xml b/taric_parsers/tests/additional_code_parsers/importer_examples/footnote_association_additional_code_CREATE.xml new file mode 100644 index 000000000..58e834885 --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/importer_examples/footnote_association_additional_code_CREATE.xml @@ -0,0 +1,135 @@ + + + + + + + 1 + 120 + 00 + 1 + 3 + + 4 + 1 + 2021-01-01 + 2021-12-31 + + + + + + + + 1 + 120 + 05 + 1 + 3 + + 4 + zz + some description + + + + + + + + 1 + 245 + 00 + 2 + 3 + + 1 + 4 + 7 + 2021-01-01 + + + + + + + + + + 1 + 100 + 05 + 3 + 3 + + 7 + ZZ + Some description + + + + + + + + 1 + 100 + 00 + 4 + 3 + + 7 + 2021-01-01 + 2021-12-01 + 9 + + + + + + + + 1 + 200 + 00 + 5 + 3 + + 7 + 4 + 2021-01-01 + 2022-01-01 + + + + + + + + + + 2 + 245 + 15 + 6z + 3 + + 1 + 7 + 4 + 2021-01-01 + 2021-01-01 + 4 + 7 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/additional_code_parsers/importer_examples/footnote_association_additional_code_DELETE.xml b/taric_parsers/tests/additional_code_parsers/importer_examples/footnote_association_additional_code_DELETE.xml new file mode 100644 index 000000000..825951291 --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/importer_examples/footnote_association_additional_code_DELETE.xml @@ -0,0 +1,27 @@ + + + + + + + 2 + 245 + 15 + 6z + 2 + + 1 + 7 + 4 + 2021-01-11 + 2022-01-01 + 4 + 7 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/additional_code_parsers/importer_examples/footnote_association_additional_code_UPDATE.xml b/taric_parsers/tests/additional_code_parsers/importer_examples/footnote_association_additional_code_UPDATE.xml new file mode 100644 index 000000000..cd1171e30 --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/importer_examples/footnote_association_additional_code_UPDATE.xml @@ -0,0 +1,27 @@ + + + + + + + 2 + 245 + 15 + 6z + 1 + + 1 + 7 + 4 + 2021-01-11 + 2022-01-01 + 4 + 7 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/additional_code_parsers/importer_examples/footnote_association_additional_code_invalid_footnote_CREATE.xml b/taric_parsers/tests/additional_code_parsers/importer_examples/footnote_association_additional_code_invalid_footnote_CREATE.xml new file mode 100644 index 000000000..2ab77360b --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/importer_examples/footnote_association_additional_code_invalid_footnote_CREATE.xml @@ -0,0 +1,81 @@ + + + + + + + 1 + 120 + 00 + 1 + 3 + + 7 + 4 + 2021-01-01 + 2021-12-31 + + + + + + + + 1 + 245 + 00 + 2 + 3 + + 1 + 7 + 4 + 2021-01-01 + + + + + + + + 1 + 245 + 15 + 3 + 3 + + 1 + 7 + 9 + 2021-01-01 + 2021-01-01 + 7 + 4 + + + + + + + + 1 + 100 + 00 + 4 + 3 + + 7 + 2021-01-01 + 2021-01-01 + 4 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/additional_code_parsers/test_additional_code_description_parser.py b/taric_parsers/tests/additional_code_parsers/test_additional_code_description_parser.py new file mode 100644 index 000000000..99abd6472 --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/test_additional_code_description_parser.py @@ -0,0 +1,164 @@ +from datetime import date + +import pytest + +from common.tests.util import preload_import +from common.validators import UpdateType +from taric_parsers.parsers.additional_code_parsers import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestAdditionalCodeDescriptionParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + + """ + + target_parser_class = AdditionalCodeDescriptionParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "additional_code_description_period_sid": "123", + "additional_code_sid": "123", + "additional_code_type_id": "A", + "additional_code": "123", + "description": "some description", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.sid == 123 + assert target.described_additionalcode__sid == 123 + assert target.described_additionalcode__type__sid == "A" + assert target.described_additionalcode__code == "123" + assert target.description == "some description" + + def test_import_create(self, superuser): + importer = preload_import("additional_code_description_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + assert len(importer.parsed_transactions[0].parsed_messages) == 5 + + target_message = importer.parsed_transactions[0].parsed_messages[4] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target_taric_object = target_message.taric_object + + for message in importer.parsed_transactions[0].parsed_messages: + assert len(message.taric_object.issues) == 0 + + assert AdditionalCode.objects.all().count() == 1 + assert AdditionalCodeType.objects.all().count() == 1 + assert AdditionalCodeDescription.objects.all().count() == 1 + + assert importer.can_save() + assert type(target_taric_object) == AdditionalCodeDescriptionParserV2 + assert target_taric_object.sid == 5 + assert target_taric_object.described_additionalcode__sid == 1 + assert target_taric_object.described_additionalcode__type__sid == "9" + assert target_taric_object.described_additionalcode__code == "2" + assert target_taric_object.description == "some description" + assert target_taric_object.validity_start == date(2021, 1, 1) + + assert importer.issues() == [] + + def test_import_update(self, superuser): + preload_import("additional_code_description_CREATE.xml", __file__, True) + importer = preload_import("additional_code_description_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + target_taric_object = target_message.taric_object + + assert importer.can_save() + assert type(target_taric_object) == AdditionalCodeDescriptionParserV2 + assert target_taric_object.sid == 5 + assert target_taric_object.described_additionalcode__sid == 1 + assert target_taric_object.described_additionalcode__type__sid == "9" + assert target_taric_object.described_additionalcode__code == "2" + assert target_taric_object.description == "some description change" + assert ( + target_taric_object.validity_start is None + ) # No data in update for period + + assert importer.issues() == [] + + assert AdditionalCodeDescription.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("additional_code_description_CREATE.xml", __file__, True) + importer = preload_import("additional_code_description_DELETE.xml", __file__) + + assert importer.can_save() + assert importer.issues() == [] + assert AdditionalCodeDescription.objects.all().count() == 2 + assert ( + AdditionalCodeDescription.objects.all().last().update_type + == UpdateType.DELETE + ) + + def test_import_invalid_additional_code(self, superuser): + importer = preload_import( + "additional_code_description_invalid_additional_code_CREATE.xml", + __file__, + ) + + assert len(importer.parsed_transactions) == 1 + assert len(importer.parsed_transactions[0].parsed_messages) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + assert ( + target_message.record_code == AdditionalCodeDescriptionParserV2.record_code + ) + assert ( + target_message.subrecord_code + == AdditionalCodeDescriptionParserV2.subrecord_code + ) + assert type(target_message.taric_object) == AdditionalCodeDescriptionParserV2 + + target_taric_object = target_message.taric_object + assert target_taric_object.sid == 5 + assert target_taric_object.described_additionalcode__sid == 1 + assert target_taric_object.described_additionalcode__type__sid == "12" + assert target_taric_object.described_additionalcode__code == "111" + assert target_taric_object.description == "some description" + + assert len(importer.issues()) == 1 + + assert ( + "ERROR: Missing expected child object AdditionalCodeDescriptionPeriodParserV2" + in str(importer.issues()[0]) + ) + assert ( + "additional.code.description > additional.code.description.period" + in str(importer.issues()[0]) + ) + assert "link_data: {}" in str(importer.issues()[0]) diff --git a/taric_parsers/tests/additional_code_parsers/test_additional_code_description_period_parser.py b/taric_parsers/tests/additional_code_parsers/test_additional_code_description_period_parser.py new file mode 100644 index 000000000..a9468648b --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/test_additional_code_description_period_parser.py @@ -0,0 +1,166 @@ +from datetime import date + +import pytest + +from additional_codes.models import AdditionalCodeDescription +from common.tests.util import preload_import +from taric_parsers.parsers.additional_code_parsers import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestAdditionalCodeDescriptionPeriodParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + """ + + target_parser_class = AdditionalCodeDescriptionPeriodParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "additional_code_description_period_sid": "123", + "additional_code_sid": "123", + "additional_code_type_id": "A", + "additional_code": "123", + "validity_start_date": "2023-01-22", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.sid == 123 + assert target.validity_start == date(2023, 1, 22) + assert target.described_additionalcode__sid == 123 + assert target.described_additionalcode__type__sid == "A" + assert target.described_additionalcode__code == "123" + + def test_import(self, superuser): + importer = preload_import( + "additional_code_description_period_CREATE.xml", + __file__, + ) + + assert len(importer.parsed_transactions) == 1 + assert len(importer.parsed_transactions[0].parsed_messages) == 5 + + target_message = importer.parsed_transactions[0].parsed_messages[3] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target_taric_object = target_message.taric_object + assert target_taric_object.sid == 5 + assert target_taric_object.described_additionalcode__sid == 1 + assert target_taric_object.described_additionalcode__type__sid == "4" + assert target_taric_object.described_additionalcode__code == "3" + assert target_taric_object.validity_start == date(2021, 1, 1) + + assert importer.issues() == [] + + def test_import_update(self): + preload_import("additional_code_description_period_CREATE.xml", __file__, True) + importer = preload_import( + "additional_code_description_period_UPDATE.xml", + __file__, + ) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + target_taric_object = target_message.taric_object + + assert importer.can_save() + + assert target_taric_object.sid == 5 + assert target_taric_object.described_additionalcode__sid == 1 + assert target_taric_object.described_additionalcode__type__sid == "4" + assert target_taric_object.described_additionalcode__code == "3" + assert target_taric_object.validity_start == date(2021, 1, 11) + + assert importer.issues() == [] + + assert AdditionalCodeDescription.objects.all().count() == 2 + + def test_import_delete(self): + preload_import("additional_code_description_period_CREATE.xml", __file__, True) + importer = preload_import( + "additional_code_description_period_DELETE.xml", + __file__, + ) + + assert importer.can_save() + + assert len(importer.issues()) == 1 + + assert ( + "Taric objects of type AdditionalCodeDescription can't be deleted" + in str(importer.issues()[0]) + ) + + def test_import_no_description(self): + importer = preload_import( + "additional_code_description_period_without_description_CREATE.xml", + __file__, + ) + + assert len(importer.parsed_transactions) == 1 + assert len(importer.parsed_transactions[0].parsed_messages) == 3 + + target_message = importer.parsed_transactions[0].parsed_messages[2] + + assert ( + target_message.record_code + == AdditionalCodeDescriptionPeriodParserV2.record_code + ) + assert ( + target_message.subrecord_code + == AdditionalCodeDescriptionPeriodParserV2.subrecord_code + ) + assert ( + type(target_message.taric_object) == AdditionalCodeDescriptionPeriodParserV2 + ) + + target = target_message.taric_object + assert target.sid == 5 + assert target.described_additionalcode__sid == 1 + assert target.described_additionalcode__type__sid == "5" + assert target.described_additionalcode__code == "3" + assert target.validity_start == date(2021, 1, 1) + + assert len(importer.issues()) == 2 + + assert ( + str(importer.issues()[0]) + == "ERROR: Missing expected child object AdditionalCodeTypeDescriptionParserV2\n " + "additional.code.type > additional.code.type.description\n " + "link_data: {}" + ) + + assert ( + str(importer.issues()[1]) + == "ERROR: Missing expected parent object AdditionalCodeDescriptionParserV2\n" + " additional.code.description.period > additional.code.description\n" + " link_data: {'sid': 5}" + ) diff --git a/taric_parsers/tests/additional_code_parsers/test_additional_code_parser.py b/taric_parsers/tests/additional_code_parsers/test_additional_code_parser.py new file mode 100644 index 000000000..6345e7b5a --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/test_additional_code_parser.py @@ -0,0 +1,140 @@ +from datetime import date + +import pytest + +from additional_codes.models import AdditionalCode +from common.tests.util import preload_import +from common.validators import UpdateType +from taric_parsers.parsers.additional_code_parsers import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestAdditionalCodeParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + """ + + target_parser_class = AdditionalCodeParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "additional_code_sid": 123, + "additional_code_type_id": "A", + "additional_code": "123", + "validity_start_date": "2023-01-22", + "validity_end_date": "2024-01-22", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.sid == 123 + assert target.type__sid == "A" + assert target.code == "123" + assert target.valid_between_lower == date(2023, 1, 22) + assert target.valid_between_upper == date(2024, 1, 22) + + def test_import_success(self, superuser): + importer = preload_import("additional_code_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + assert len(importer.parsed_transactions[0].parsed_messages) == 3 + + target_message = importer.parsed_transactions[0].parsed_messages[1] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + taric_object = target_message.taric_object + assert taric_object.sid == 1 + assert taric_object.valid_between_lower == date(2021, 1, 1) + assert taric_object.valid_between_upper is None + assert taric_object.type__sid == "5" + assert taric_object.code == "3" + + # check for issues + assert importer.issues() == [] + + def test_import_update(self, superuser): + preload_import("additional_code_CREATE.xml", __file__, True) + importer = preload_import("additional_code_UPDATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + assert len(importer.parsed_transactions[0].parsed_messages) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + taric_object = target_message.taric_object + assert taric_object.sid == 1 + assert taric_object.valid_between_lower == date(2021, 1, 11) + assert taric_object.valid_between_upper is None + assert taric_object.type__sid == "5" + assert taric_object.code == "3" + + # check for issues + assert importer.issues() == [] + + def test_import_delete(self, superuser): + preload_import("additional_code_CREATE.xml", __file__, True) + importer = preload_import("additional_code_DELETE.xml", __file__) + + # check for issues + assert importer.issues() == [] + assert importer.can_save() + assert AdditionalCode.objects.all().count() == 2 + assert AdditionalCode.objects.all().last().update_type == UpdateType.DELETE + + def test_import_invalid_type(self, superuser): + importer = preload_import("additional_code_invalid_type_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + assert len(importer.parsed_transactions[0].parsed_messages) == 1 + assert ( + importer.parsed_transactions[0].parsed_messages[0].record_code + == AdditionalCodeParserV2.record_code + ) + assert ( + importer.parsed_transactions[0].parsed_messages[0].subrecord_code + == AdditionalCodeTypeParserV2.subrecord_code + ) + assert ( + type(importer.parsed_transactions[0].parsed_messages[0].taric_object) + == AdditionalCodeParserV2 + ) + + taric_object = importer.parsed_transactions[0].parsed_messages[0].taric_object + assert taric_object.sid == 1 + assert taric_object.valid_between_lower == date(2021, 1, 1) + assert taric_object.valid_between_upper is None + assert taric_object.type__sid == "12" + assert taric_object.code == "111" + + # check for issues + assert len(importer.issues()) == 2 diff --git a/taric_parsers/tests/additional_code_parsers/test_additional_code_type_description_parser.py b/taric_parsers/tests/additional_code_parsers/test_additional_code_type_description_parser.py new file mode 100644 index 000000000..2ffbac105 --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/test_additional_code_type_description_parser.py @@ -0,0 +1,138 @@ +import pytest + +from common.tests.util import preload_import +from taric_parsers.parsers.additional_code_parsers import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestAdditionalCodeTypeDescriptionParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + """ + + target_parser_class = AdditionalCodeTypeDescriptionParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "additional_code_type_id": "B", + "description": "some description", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + assert target.sid == "B" + assert target.description == "some description" + + def test_import(self, superuser): + importer = preload_import( + "additional_code_type_description_CREATE.xml", + __file__, + ) + + assert len(importer.parsed_transactions) == 1 + assert len(importer.parsed_transactions[0].parsed_messages) == 2 + + target_message = importer.parsed_transactions[0].parsed_messages[1] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target_taric_object = target_message.taric_object + assert target_taric_object.sid == "5" + assert target_taric_object.description == "some description" + + assert importer.issues() == [] + + def test_import_update(self, superuser): + preload_import("additional_code_type_description_CREATE.xml", __file__, True) + importer = preload_import( + "additional_code_type_description_UPDATE.xml", + __file__, + ) + + assert len(importer.parsed_transactions) == 1 + assert len(importer.parsed_transactions[0].parsed_messages) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target_taric_object = target_message.taric_object + assert target_taric_object.sid == "5" + assert target_taric_object.description == "some description that changed" + + assert importer.issues() == [] + + def test_import_delete(self, superuser): + preload_import("additional_code_type_description_CREATE.xml", __file__, True) + importer = preload_import( + "additional_code_type_description_DELETE.xml", + __file__, + ) + + assert importer.can_save() + + # check for issues + assert len(importer.issues()) == 1 + + assert ( + "Children of Taric objects of type AdditionalCodeType can't be deleted directly" + in str(importer.issues()[0]) + ) + + def test_import_invalid_type(self, superuser): + importer = preload_import( + "additional_code_type_description_without_type_CREATE.xml", + __file__, + ) + + assert len(importer.parsed_transactions) == 1 + assert len(importer.parsed_transactions[0].parsed_messages) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[0] + assert ( + target_message.record_code + == AdditionalCodeTypeDescriptionParserV2.record_code + ) + assert ( + target_message.subrecord_code + == AdditionalCodeTypeDescriptionParserV2.subrecord_code + ) + assert ( + type(target_message.taric_object) == AdditionalCodeTypeDescriptionParserV2 + ) + + target_taric_object = target_message.taric_object + assert target_taric_object.sid == "12" + assert target_taric_object.description == "some description" + + assert len(importer.issues()) == 1 + assert ( + str(target_taric_object.issues[0]) + == "ERROR: Missing expected parent object AdditionalCodeTypeParserV2\n" + " additional.code.type.description > additional.code.type\n" + " link_data: {'sid': '12'}" + ) diff --git a/taric_parsers/tests/additional_code_parsers/test_additional_code_type_parser.py b/taric_parsers/tests/additional_code_parsers/test_additional_code_type_parser.py new file mode 100644 index 000000000..4a74a92a0 --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/test_additional_code_type_parser.py @@ -0,0 +1,133 @@ +from datetime import date + +import pytest + +from additional_codes.models import AdditionalCodeType +from common.tests.util import preload_import +from common.validators import UpdateType +from taric_parsers.parsers.additional_code_parsers import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestAdditionalCodeTypeParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + """ + + target_parser_class = AdditionalCodeTypeParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "additional_code_type_id": "123", + "validity_start_date": "2023-01-22", + "validity_end_date": "2024-01-22", + "application_code": "123", + "meursing_table_plan_id": "123", # this property is not imported + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + assert target.sid == "123" + assert target.valid_between_lower == date(2023, 1, 22) + assert target.valid_between_upper == date(2024, 1, 22) + assert target.application_code == "123" + + def test_import(self, superuser): + importer = preload_import("additional_code_type_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + assert len(importer.parsed_transactions[0].parsed_messages) == 2 + + target_message = importer.parsed_transactions[0].parsed_messages[1] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + # check properties + target_taric_object = target_message.taric_object + assert target_taric_object.sid == "1" + assert target_taric_object.valid_between_lower == date(2021, 1, 1) + assert target_taric_object.valid_between_upper == date(2021, 12, 31) + assert target_taric_object.application_code == "111" + + assert importer.issues() == [] + + def test_import_update(self, superuser): + preload_import("additional_code_type_CREATE.xml", __file__, True) + importer = preload_import("additional_code_type_UPDATE.xml", __file__) + + assert importer.issues() == [] + + assert AdditionalCodeType.objects.all().count() == 2 + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + # check properties + target_taric_object = target_message.taric_object + + assert target_taric_object.valid_between_lower == date(2021, 1, 11) + assert target_taric_object.valid_between_upper == date(2021, 12, 31) + + def test_import_delete(self, superuser): + preload_import("additional_code_type_CREATE.xml", __file__, True) + importer = preload_import("additional_code_type_DELETE.xml", __file__) + + # check for issues + assert importer.issues() == [] + assert importer.can_save() + assert AdditionalCodeType.objects.all().count() == 2 + assert AdditionalCodeType.objects.all().last().update_type == UpdateType.DELETE + + def test_import_no_description(self, superuser): + importer = preload_import( + "additional_code_type_no_description_CREATE.xml", + __file__, + ) + + assert len(importer.parsed_transactions) == 1 + assert len(importer.parsed_transactions[0].parsed_messages) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + # check properties + target_taric_object = target_message.taric_object + assert target_taric_object.sid == "5" + assert target_taric_object.valid_between_lower == date(2021, 1, 1) + assert target_taric_object.valid_between_upper == date(2021, 12, 31) + assert target_taric_object.application_code == "111" + + assert len(importer.issues()) == 1 + + assert ( + str(importer.issues()[0]) + == "ERROR: Missing expected child object AdditionalCodeTypeDescriptionParserV2\n " + "additional.code.type > additional.code.type.description\n " + "link_data: {}" + ) diff --git a/taric_parsers/tests/additional_code_parsers/test_footnote_association_additional_code_parser.py b/taric_parsers/tests/additional_code_parsers/test_footnote_association_additional_code_parser.py new file mode 100644 index 000000000..c9df524f2 --- /dev/null +++ b/taric_parsers/tests/additional_code_parsers/test_footnote_association_additional_code_parser.py @@ -0,0 +1,161 @@ +from datetime import date + +import pytest + +from additional_codes.models import * +from common.tests.util import preload_import +from common.validators import UpdateType +from taric_parsers.parsers.additional_code_parsers import * +from taric_parsers.parsers.footnote_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestAdditionalCodeTypeParserV2: + """ + Example XML: + + .. code-block: XML + + + + + + + + + + + + + + + """ + + target_parser_class = FootnoteAssociationAdditionalCodeParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "additional_code_sid": "111", + "footnote_type_id": "5", + "footnote_id": "555", + "validity_start_date": "2023-01-22", + "validity_end_date": "2024-01-22", + "additional_code": "666", + "additional_code_type_id": "Z", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.additional_code__sid == 111 + assert target.associated_footnote__footnote_type__footnote_type_id == 5 + assert target.associated_footnote__footnote_id == "555" + assert target.valid_between_lower == date(2023, 1, 22) + assert target.valid_between_upper == date(2024, 1, 22) + assert target.additional_code__code == "666" + assert target.additional_code__type__sid == "Z" + + def test_import(self, superuser): + importer = preload_import( + "footnote_association_additional_code_CREATE.xml", + __file__, + ) + + assert len(importer.parsed_transactions) == 3 + + target_message = importer.parsed_transactions[2].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + # check properties + target_taric_object = target_message.taric_object + assert target_taric_object.additional_code__sid == 1 + assert ( + target_taric_object.associated_footnote__footnote_type__footnote_type_id + == 7 + ) + assert target_taric_object.associated_footnote__footnote_id == "4" + assert target_taric_object.valid_between_upper == date(2021, 1, 1) + assert target_taric_object.valid_between_lower == date(2021, 1, 1) + assert target_taric_object.additional_code__type__sid == "4" + assert target_taric_object.additional_code__code == "7" + + assert importer.issues() == [] + + def test_import_update(self, superuser): + preload_import( + "footnote_association_additional_code_CREATE.xml", + __file__, + True, + ) + importer = preload_import( + "footnote_association_additional_code_UPDATE.xml", + __file__, + ) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + # check properties + target_taric_object = target_message.taric_object + assert target_taric_object.additional_code__sid == 1 + assert ( + target_taric_object.associated_footnote__footnote_type__footnote_type_id + == 7 + ) + assert target_taric_object.associated_footnote__footnote_id == "4" + assert target_taric_object.valid_between_upper == date(2022, 1, 1) + assert target_taric_object.valid_between_lower == date(2021, 1, 11) + assert target_taric_object.additional_code__type__sid == "4" + assert target_taric_object.additional_code__code == "7" + + assert importer.issues() == [] + + def test_import_delete(self, superuser): + preload_import( + "footnote_association_additional_code_CREATE.xml", + __file__, + True, + ) + importer = preload_import( + "footnote_association_additional_code_DELETE.xml", + __file__, + ) + # check for issues + assert importer.issues() == [] + assert importer.can_save() + assert FootnoteAssociationAdditionalCode.objects.all().count() == 2 + assert ( + FootnoteAssociationAdditionalCode.objects.all().last().update_type + == UpdateType.DELETE + ) + + def test_import_invalid_footnote(self, superuser): + importer = preload_import( + "footnote_association_additional_code_invalid_footnote_CREATE.xml", + __file__, + ) + + assert len(importer.issues()) == 2 + assert ( + str(importer.issues()[0]) + == "ERROR: Missing expected child object AdditionalCodeTypeDescriptionParserV2\n" + " additional.code.type > additional.code.type.description\n" + " link_data: {}" + ) + assert ( + str(importer.issues()[1]) + == "ERROR: Missing expected child object FootnoteTypeDescriptionParserV2\n" + " footnote.type > footnote.type.description\n" + " link_data: {}" + ) diff --git a/taric_parsers/tests/certificate_parsers/importer_examples/certificate_CREATE.xml b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_CREATE.xml new file mode 100644 index 000000000..36c181470 --- /dev/null +++ b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_CREATE.xml @@ -0,0 +1,54 @@ + + + + + + + 1 + 110 + 00 + 1 + 3 + + A + 2021-01-01 + 2021-12-31 + + + + + + + + 1 + 110 + 05 + 1 + 3 + + A + EN + some description + + + + + + + + 1 + 205 + 00 + 1 + 3 + + A + 123 + 2021-01-01 + 2021-12-31 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/certificate_parsers/importer_examples/certificate_DELETE.xml b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_DELETE.xml new file mode 100644 index 000000000..abae027cd --- /dev/null +++ b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_DELETE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 205 + 00 + 1 + 2 + + A + 123 + 2021-01-11 + 2021-12-31 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/certificate_parsers/importer_examples/certificate_UPDATE.xml b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_UPDATE.xml new file mode 100644 index 000000000..bc8c82d6a --- /dev/null +++ b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_UPDATE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 205 + 00 + 1 + 1 + + A + 123 + 2021-01-11 + 2021-12-31 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/certificate_parsers/importer_examples/certificate_description_CREATE.xml b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_description_CREATE.xml new file mode 100644 index 000000000..2c68ef15f --- /dev/null +++ b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_description_CREATE.xml @@ -0,0 +1,90 @@ + + + + + + + 1 + 110 + 00 + 1 + 3 + + A + 2021-01-01 + 2021-12-31 + + + + + + + + 1 + 110 + 05 + 2 + 3 + + A + EN + some description + + + + + + + + 1 + 205 + 00 + 3 + 3 + + A + 123 + 2021-01-01 + 2021-12-31 + + + + + + + + 1 + 205 + 05 + 4 + 3 + + 8 + A + 123 + 2021-01-01 + + + + + + + + 1 + 205 + 10 + 5 + 3 + + A + 8 + EN + 123 + This is a description + + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/certificate_parsers/importer_examples/certificate_description_DELETE.xml b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_description_DELETE.xml new file mode 100644 index 000000000..e7a90c1a5 --- /dev/null +++ b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_description_DELETE.xml @@ -0,0 +1,23 @@ + + + + + + + 1 + 205 + 10 + 5 + 2 + + A + 8 + EN + 123 + This is a description + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/certificate_parsers/importer_examples/certificate_description_UPDATE.xml b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_description_UPDATE.xml new file mode 100644 index 000000000..04a315a7c --- /dev/null +++ b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_description_UPDATE.xml @@ -0,0 +1,24 @@ + + + + + + + 1 + 205 + 10 + 5 + 1 + + A + 8 + EN + 123 + This is a description with changes + + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/certificate_parsers/importer_examples/certificate_description_period_CREATE.xml b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_description_period_CREATE.xml new file mode 100644 index 000000000..0a1e42852 --- /dev/null +++ b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_description_period_CREATE.xml @@ -0,0 +1,89 @@ + + + + + + + 1 + 110 + 00 + 1 + 3 + + A + 2021-01-01 + 2021-12-31 + + + + + + + + 1 + 110 + 05 + 2 + 3 + + A + EN + some description + + + + + + + + 1 + 205 + 00 + 3 + 3 + + A + 123 + 2021-01-01 + 2021-12-31 + + + + + + + + 1 + 205 + 05 + 4 + 3 + + 9 + 123 + A + 2021-12-31 + + + + + + + + 1 + 205 + 10 + 5 + 3 + + A + 9 + EN + 123 + This is a description + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/certificate_parsers/importer_examples/certificate_description_period_DELETE.xml b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_description_period_DELETE.xml new file mode 100644 index 000000000..a5057cee2 --- /dev/null +++ b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_description_period_DELETE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 205 + 05 + 4 + 2 + + 9 + 123 + A + 2021-11-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/certificate_parsers/importer_examples/certificate_description_period_UPDATE.xml b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_description_period_UPDATE.xml new file mode 100644 index 000000000..ba483c8cb --- /dev/null +++ b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_description_period_UPDATE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 205 + 05 + 4 + 1 + + 9 + 123 + A + 2021-11-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/certificate_parsers/importer_examples/certificate_type_CREATE.xml b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_type_CREATE.xml new file mode 100644 index 000000000..5aa6906e7 --- /dev/null +++ b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_type_CREATE.xml @@ -0,0 +1,38 @@ + + + + + + + 1 + 110 + 00 + 1 + 3 + + A + 2021-01-01 + 2021-12-31 + + + + + + + + 1 + 110 + 05 + 1 + 3 + + A + EN + some description + + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/certificate_parsers/importer_examples/certificate_type_DELETE.xml b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_type_DELETE.xml new file mode 100644 index 000000000..a7af1ebe8 --- /dev/null +++ b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_type_DELETE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 110 + 00 + 1 + 2 + + A + 2021-01-01 + 2021-12-31 + + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/certificate_parsers/importer_examples/certificate_type_UPDATE.xml b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_type_UPDATE.xml new file mode 100644 index 000000000..1d3c6caad --- /dev/null +++ b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_type_UPDATE.xml @@ -0,0 +1,21 @@ + + + + + + + 1 + 110 + 00 + 1 + 1 + + A + 2021-01-11 + 2021-12-31 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/certificate_parsers/importer_examples/certificate_type_description_CREATE.xml b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_type_description_CREATE.xml new file mode 100644 index 000000000..823fa9395 --- /dev/null +++ b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_type_description_CREATE.xml @@ -0,0 +1,37 @@ + + + + + + + 1 + 110 + 00 + 1 + 3 + + A + 2021-01-01 + 2021-12-31 + + + + + + + + 1 + 110 + 05 + 1 + 3 + + A + EN + some description + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/certificate_parsers/importer_examples/certificate_type_description_DELETE.xml b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_type_description_DELETE.xml new file mode 100644 index 000000000..bcf2883f8 --- /dev/null +++ b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_type_description_DELETE.xml @@ -0,0 +1,21 @@ + + + + + + + 1 + 110 + 05 + 1 + 2 + + A + EN + some description with changes + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/certificate_parsers/importer_examples/certificate_type_description_UPDATE.xml b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_type_description_UPDATE.xml new file mode 100644 index 000000000..ad69e737f --- /dev/null +++ b/taric_parsers/tests/certificate_parsers/importer_examples/certificate_type_description_UPDATE.xml @@ -0,0 +1,21 @@ + + + + + + + 1 + 110 + 05 + 1 + 1 + + A + EN + some description with changes + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/certificate_parsers/test_certificate_description_parser.py b/taric_parsers/tests/certificate_parsers/test_certificate_description_parser.py new file mode 100644 index 000000000..1999e72ae --- /dev/null +++ b/taric_parsers/tests/certificate_parsers/test_certificate_description_parser.py @@ -0,0 +1,108 @@ +import pytest + +from common.tests.util import preload_import +from common.validators import UpdateType + +# note : need to import these objects to make them available to the parser +from taric_parsers.parsers.certificate_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestCertificateDescriptionParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + """ + + target_parser_class = CertificateDescriptionParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "certificate_description_period_sid": 555, + "language_id": "EN", # gets ignored, but will come in from import + "certificate_type_code": "666", + "certificate_code": "777", + "description": "this is a description", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.sid == 555 + assert target.described_certificate__certificate_type__sid == "666" + assert target.described_certificate__sid == "777" + assert target.description == "this is a description" + + def test_import(self, superuser): + importer = preload_import("certificate_description_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + assert len(importer.parsed_transactions[0].parsed_messages) == 5 + + target_message = importer.parsed_transactions[0].parsed_messages[4] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + assert target.sid == 8 + assert target.described_certificate__certificate_type__sid == "A" + assert target.described_certificate__sid == "123" + assert target.description == "This is a description" + + assert importer.issues() == [] + + def test_import_update(self, superuser): + preload_import("certificate_description_CREATE.xml", __file__, True) + importer = preload_import("certificate_description_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + target = target_message.taric_object + + assert importer.issues() == [] + + assert target.sid == 8 + assert target.described_certificate__certificate_type__sid == "A" + assert target.described_certificate__sid == "123" + assert target.description == "This is a description with changes" + + def test_import_delete(self): + preload_import( + "certificate_description_CREATE.xml", + __file__, + True, + ) + importer = preload_import( + "certificate_description_DELETE.xml", + __file__, + ) + # check for issues + assert importer.issues() == [] + assert importer.can_save() + assert CertificateDescription.objects.all().count() == 2 + assert ( + CertificateDescription.objects.all().last().update_type == UpdateType.DELETE + ) diff --git a/taric_parsers/tests/certificate_parsers/test_certificate_description_period_parser.py b/taric_parsers/tests/certificate_parsers/test_certificate_description_period_parser.py new file mode 100644 index 000000000..5cc05689c --- /dev/null +++ b/taric_parsers/tests/certificate_parsers/test_certificate_description_period_parser.py @@ -0,0 +1,109 @@ +from datetime import date + +import pytest + +from certificates.models import CertificateDescription +from common.tests.util import preload_import + +# note : need to import these objects to make them available to the parser +from taric_parsers.parsers.certificate_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestCertificateDescriptionPeriodParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + """ + + target_parser_class = CertificateDescriptionPeriodParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "certificate_description_period_sid": "123", + "certificate_type_code": "A", + "certificate_code": "BBB", + "validity_start_date": "2021-01-01", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.sid == 123 + assert target.described_certificate__certificate_type__sid == "A" + assert target.described_certificate__sid == "BBB" + assert target.validity_start == date(2021, 1, 1) + + def test_import(self, superuser): + importer = preload_import("certificate_description_period_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + assert len(importer.parsed_transactions[0].parsed_messages) == 5 + + target_message = importer.parsed_transactions[0].parsed_messages[3] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target_taric_object = target_message.taric_object + assert target_taric_object.sid == 9 + assert target_taric_object.described_certificate__certificate_type__sid == "A" + assert target_taric_object.described_certificate__sid == "123" + assert target_taric_object.validity_start == date(2021, 12, 31) + + target = CertificateDescription.objects.all().last() + assert target.validity_start == target_taric_object.validity_start + assert target.sid == int(target_taric_object.sid) + + assert importer.issues() == [] + + def test_import_update(self, superuser): + preload_import("certificate_description_period_CREATE.xml", __file__, True) + importer = preload_import("certificate_description_period_UPDATE.xml", __file__) + + assert importer.issues() == [] + + target_message = importer.parsed_transactions[0].parsed_messages[0] + target_taric_object = target_message.taric_object + assert target_taric_object.sid == 9 + assert target_taric_object.described_certificate__certificate_type__sid == "A" + assert target_taric_object.described_certificate__sid == "123" + assert target_taric_object.validity_start == date(2021, 11, 1) + + def test_import_delete(self): + preload_import("certificate_description_period_CREATE.xml", __file__, True) + importer = preload_import( + "certificate_description_period_DELETE.xml", + __file__, + ) + + assert importer.can_save() + + assert len(importer.issues()) == 1 + + assert ( + "Children of Taric objects of type CertificateDescription can't be deleted directly" + in str(importer.issues()[0]) + ) diff --git a/taric_parsers/tests/certificate_parsers/test_certificate_parser.py b/taric_parsers/tests/certificate_parsers/test_certificate_parser.py new file mode 100644 index 000000000..b9e5a11a5 --- /dev/null +++ b/taric_parsers/tests/certificate_parsers/test_certificate_parser.py @@ -0,0 +1,108 @@ +from datetime import date + +import pytest + +from certificates.models import Certificate +from common.tests.util import preload_import +from common.validators import UpdateType + +# note : need to import these objects to make them available to the parser +from taric_parsers.parsers.certificate_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestCertificateParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + """ + + target_parser_class = CertificateParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "certificate_code": "456", + "certificate_type_code": "891", + "validity_start_date": "2023-01-22", + "validity_end_date": "2024-01-22", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.sid == 456 + assert target.certificate_type__sid == "891" + assert target.valid_between_lower == date(2023, 1, 22) + assert target.valid_between_upper == date(2024, 1, 22) + + def test_import(self, superuser): + importer = preload_import("certificate_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + assert len(importer.parsed_transactions[0].parsed_messages) == 3 + + target_message = importer.parsed_transactions[0].parsed_messages[2] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target_taric_object = target_message.taric_object + assert target_taric_object.sid == 123 + assert target_taric_object.certificate_type__sid == "A" + assert target_taric_object.valid_between_lower == date(2021, 1, 1) + assert target_taric_object.valid_between_upper == date(2021, 12, 31) + + assert importer.issues() == [] + + def test_import_update(self, superuser): + preload_import("certificate_CREATE.xml", __file__, True) + importer = preload_import("certificate_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + target_taric_object = target_message.taric_object + + assert target_taric_object.sid == 123 + assert target_taric_object.certificate_type__sid == "A" + assert target_taric_object.valid_between_lower == date(2021, 1, 11) + assert target_taric_object.valid_between_upper == date(2021, 12, 31) + + assert importer.issues() == [] + + def test_import_delete(self): + preload_import( + "certificate_CREATE.xml", + __file__, + True, + ) + importer = preload_import( + "certificate_DELETE.xml", + __file__, + ) + # check for issues + assert importer.issues() == [] + assert importer.can_save() + assert Certificate.objects.all().count() == 2 + assert Certificate.objects.all().last().update_type == UpdateType.DELETE diff --git a/taric_parsers/tests/certificate_parsers/test_certificate_type_description_parser.py b/taric_parsers/tests/certificate_parsers/test_certificate_type_description_parser.py new file mode 100644 index 000000000..c2d38a4fc --- /dev/null +++ b/taric_parsers/tests/certificate_parsers/test_certificate_type_description_parser.py @@ -0,0 +1,95 @@ +import pytest + +from common.tests.util import preload_import + +# note : need to import these objects to make them available to the parser +from taric_parsers.parsers.certificate_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestCertificateTypeDescriptionParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + """ + + target_parser_class = CertificateTypeDescriptionParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "certificate_type_code": "123", + "language_id": "EN", + "description": "Some description", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.sid == "123" + assert target.description == "Some description" + + def test_import(self, superuser): + importer = preload_import("certificate_type_description_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + assert len(importer.parsed_transactions[0].parsed_messages) == 2 + + target_message = importer.parsed_transactions[0].parsed_messages[1] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + assert target.sid == "A" + assert target.description == "some description" + assert importer.issues() == [] + + def test_import_update(self, superuser): + preload_import("certificate_type_description_CREATE.xml", __file__, True) + importer = preload_import("certificate_type_description_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + target = target_message.taric_object + + assert target.sid == "A" + assert target.description == "some description with changes" + + assert importer.issues() == [] + + def test_import_delete(self, superuser): + preload_import("certificate_type_description_CREATE.xml", __file__, True) + importer = preload_import("certificate_type_description_DELETE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + target = target_message.taric_object + + assert target.sid == "A" + assert target.description == "some description with changes" + + assert len(importer.issues()) == 1 + + assert ( + "Children of Taric objects of type CertificateType can't be deleted directly" + in str(importer.issues()[0]) + ) diff --git a/taric_parsers/tests/certificate_parsers/test_certificate_type_parser.py b/taric_parsers/tests/certificate_parsers/test_certificate_type_parser.py new file mode 100644 index 000000000..e53ed9dd8 --- /dev/null +++ b/taric_parsers/tests/certificate_parsers/test_certificate_type_parser.py @@ -0,0 +1,94 @@ +from datetime import date + +import pytest + +from certificates.models import CertificateType +from common.tests.util import preload_import +from common.validators import UpdateType +from taric_parsers.parsers.certificate_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestCertificateTypeParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + """ + + target_parser_class = CertificateTypeParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "certificate_type_code": "123", + "validity_start_date": "2023-01-22", + "validity_end_date": "2024-01-22", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.sid == "123" + assert target.valid_between_lower == date(2023, 1, 22) + assert target.valid_between_upper == date(2024, 1, 22) + + def test_import(self, superuser): + importer = preload_import("certificate_type_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + assert len(importer.parsed_transactions[0].parsed_messages) == 2 + + target_message = importer.parsed_transactions[0].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target_taric_object = target_message.taric_object + assert target_taric_object.sid == "A" + assert target_taric_object.valid_between_lower == date(2021, 1, 1) + assert target_taric_object.valid_between_upper == date(2021, 12, 31) + + assert importer.issues() == [] + + def test_import_update(self, superuser): + preload_import("certificate_type_CREATE.xml", __file__, True) + importer = preload_import("certificate_type_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target_taric_object = target_message.taric_object + assert target_taric_object.sid == "A" + assert target_taric_object.valid_between_lower == date(2021, 1, 11) + assert target_taric_object.valid_between_upper == date(2021, 12, 31) + + assert importer.issues() == [] + + def test_import_delete(self, superuser): + preload_import("certificate_type_CREATE.xml", __file__, True) + importer = preload_import("certificate_type_DELETE.xml", __file__) + + assert importer.issues() == [] + + assert importer.can_save() + assert CertificateType.objects.all().count() == 2 + assert CertificateType.objects.all().last().update_type == UpdateType.DELETE diff --git a/taric_parsers/tests/commoditiy_parsers/importer_examples/footnote_association_goods_nomenclature_CREATE.xml b/taric_parsers/tests/commoditiy_parsers/importer_examples/footnote_association_goods_nomenclature_CREATE.xml new file mode 100644 index 000000000..27af59e2f --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/importer_examples/footnote_association_goods_nomenclature_CREATE.xml @@ -0,0 +1,98 @@ + + + + + + + 1 + 400 + 00 + 1 + 3 + + 1 + 0100000000 + 10 + 2022-01-01 + 0 + + + + + + + + 1 + 100 + 05 + 2 + 3 + + 3 + ZZ + Some description + + + + + + + + 1 + 100 + 00 + 3 + 3 + + 3 + 2021-01-01 + 2021-12-01 + 9 + + + + + + + + 1 + 200 + 00 + 4 + 3 + + 3 + 9 + 2021-01-01 + 2021-01-01 + + + + + + + + + + 1 + 400 + 20 + 1 + 3 + + 1 + 0100000000 + 10 + 2022-01-01 + 2023-01-01 + 3 + 9 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/commoditiy_parsers/importer_examples/footnote_association_goods_nomenclature_DELETE.xml b/taric_parsers/tests/commoditiy_parsers/importer_examples/footnote_association_goods_nomenclature_DELETE.xml new file mode 100644 index 000000000..633f78529 --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/importer_examples/footnote_association_goods_nomenclature_DELETE.xml @@ -0,0 +1,25 @@ + + + + + + + 1 + 400 + 20 + 1 + 2 + + 1 + 0100000000 + 10 + 2022-01-01 + 2023-01-01 + 3 + 9 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/commoditiy_parsers/importer_examples/footnote_association_goods_nomenclature_UPDATE.xml b/taric_parsers/tests/commoditiy_parsers/importer_examples/footnote_association_goods_nomenclature_UPDATE.xml new file mode 100644 index 000000000..980b8126b --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/importer_examples/footnote_association_goods_nomenclature_UPDATE.xml @@ -0,0 +1,25 @@ + + + + + + + 1 + 400 + 20 + 1 + 1 + + 1 + 0100000000 + 10 + 2022-01-11 + 2023-01-01 + 3 + 9 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/commoditiy_parsers/importer_examples/footnote_association_goods_nomenclature_no_footnote_CREATE.xml b/taric_parsers/tests/commoditiy_parsers/importer_examples/footnote_association_goods_nomenclature_no_footnote_CREATE.xml new file mode 100644 index 000000000..684d27daa --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/importer_examples/footnote_association_goods_nomenclature_no_footnote_CREATE.xml @@ -0,0 +1,80 @@ + + + + + + + 1 + 400 + 00 + 1 + 3 + + 1 + 0100000000 + 10 + 2022-01-01 + 0 + + + + + + + + 1 + 100 + 05 + 2 + 3 + + 3 + ZZ + Some description + + + + + + + + 1 + 100 + 00 + 3 + 3 + + 3 + 2021-01-01 + 2021-12-01 + 9 + + + + + + + + + + 1 + 400 + 20 + 1 + 3 + + 1 + 0100000000 + 10 + 2022-01-01 + 2023-01-01 + 3 + 9 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_CREATE.xml b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_CREATE.xml new file mode 100644 index 000000000..828997d34 --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_CREATE.xml @@ -0,0 +1,23 @@ + + + + + + + 1 + 400 + 00 + 1 + 3 + + 1 + 0100000000 + 10 + 2021-01-01 + 0 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_CREATE_then_DELETE.xml b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_CREATE_then_DELETE.xml new file mode 100644 index 000000000..db9d0acdb --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_CREATE_then_DELETE.xml @@ -0,0 +1,41 @@ + + + + + + + 1 + 400 + 00 + 1 + 3 + + 1 + 0100000000 + 10 + 2021-01-01 + 0 + + + + + + + + + + 1 + 400 + 00 + 1 + 2 + + 1 + 0100000000 + 10 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_DELETE.xml b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_DELETE.xml new file mode 100644 index 000000000..91c54218a --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_DELETE.xml @@ -0,0 +1,23 @@ + + + + + + + 1 + 400 + 00 + 1 + 2 + + 1 + 0100000000 + 10 + 2021-01-02 + 0 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_UPDATE.xml b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_UPDATE.xml new file mode 100644 index 000000000..1fefbd92f --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_UPDATE.xml @@ -0,0 +1,23 @@ + + + + + + + 1 + 400 + 00 + 1 + 1 + + 1 + 0100000000 + 10 + 2021-01-02 + 0 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_DELETE.xml b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_DELETE.xml new file mode 100644 index 000000000..b004ced34 --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_DELETE.xml @@ -0,0 +1,24 @@ + + + + + + + 1 + 400 + 15 + 1 + 2 + + 7 + 1 + 0100000000 + ZZ + 10 + Some Description that changed + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_UPDATE.xml b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_UPDATE.xml new file mode 100644 index 000000000..fd866195c --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_UPDATE.xml @@ -0,0 +1,24 @@ + + + + + + + 1 + 400 + 15 + 1 + 1 + + 7 + 1 + 0100000000 + ZZ + 10 + Some Description that changed + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_no_period_CREATE.xml b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_no_period_CREATE.xml new file mode 100644 index 000000000..9832d2c6b --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_no_period_CREATE.xml @@ -0,0 +1,44 @@ + + + + + + + 1 + 400 + 00 + 1 + 3 + + 7 + 0102000000 + 10 + 2021-01-01 + 0 + + + + + + + + + + 1 + 400 + 15 + 2 + 3 + + 9 + 7 + 0102000000 + ZZ + 10 + Some Description + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_no_period_UPDATE.xml b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_no_period_UPDATE.xml new file mode 100644 index 000000000..73be59607 --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_no_period_UPDATE.xml @@ -0,0 +1,24 @@ + + + + + + + 1 + 400 + 15 + 1 + 1 + + 7 + 1 + 0100000000 + ZZ + 10 + Some Description Changed + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_period_UPDATE.xml b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_period_UPDATE.xml new file mode 100644 index 000000000..661121014 --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_period_UPDATE.xml @@ -0,0 +1,82 @@ + + + + + + + 1 + 400 + 00 + 1 + 3 + + 1 + 0100000000 + 10 + 2021-01-01 + 0 + + + + + + + + + + 1 + 400 + 15 + 1 + 3 + + 7 + 1 + 0100000000 + ZZ + 10 + Some Description + + + + + + + + 2 + 400 + 10 + 3 + 3 + + 7 + 1 + 0100000000 + 2021-01-01 + 10 + + + + + + + + + + 2 + 400 + 10 + 3 + 1 + + 7 + 1 + 0100000000 + 2021-01-02 + 10 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_period_multi_CREATE.xml b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_period_multi_CREATE.xml new file mode 100644 index 000000000..92bca0099 --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_period_multi_CREATE.xml @@ -0,0 +1,119 @@ + + + + + + + 1 + 400 + 00 + 1 + 3 + + 1 + 0100000000 + 10 + 2021-01-01 + 0 + + + + + + + + 1 + 400 + 00 + 1 + 3 + + 2 + 0102000000 + 10 + 2021-01-01 + 0 + + + + + + + + + + 1 + 400 + 15 + 1 + 3 + + 7 + 1 + 0100000000 + ZZ + 10 + Some Description + + + + + + + + 1 + 400 + 15 + 1 + 3 + + 7 + 2 + 0102000000 + ZZ + 10 + Some Description + + + + + + + + 2 + 400 + 10 + 3 + 3 + + 7 + 1 + 0100000000 + 2021-01-01 + 10 + + + + + + + + + + 2 + 400 + 10 + 3 + 1 + + 7 + 1 + 0100000000 + 2021-01-02 + 10 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_period_only_DELETE.xml b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_period_only_DELETE.xml new file mode 100644 index 000000000..51ff5dfd6 --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_period_only_DELETE.xml @@ -0,0 +1,23 @@ + + + + + + + 2 + 400 + 10 + 3 + 2 + + 7 + 1 + 0100000000 + 2021-01-03 + 10 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_period_only_UPDATE.xml b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_period_only_UPDATE.xml new file mode 100644 index 000000000..96e19c5bc --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_period_only_UPDATE.xml @@ -0,0 +1,23 @@ + + + + + + + 2 + 400 + 10 + 3 + 1 + + 7 + 1 + 0100000000 + 2021-01-03 + 10 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_with_period_CREATE.xml b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_with_period_CREATE.xml new file mode 100644 index 000000000..1c44d5d65 --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_description_with_period_CREATE.xml @@ -0,0 +1,62 @@ + + + + + + + 1 + 400 + 00 + 1 + 3 + + 1 + 0100000000 + 10 + 2021-01-01 + 0 + + + + + + + + + + 1 + 400 + 15 + 1 + 3 + + 7 + 1 + 0100000000 + ZZ + 10 + Some Description + + + + + + + + 2 + 400 + 10 + 3 + 3 + + 7 + 1 + 0100000000 + 2021-01-01 + 10 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_indent_CREATE.xml b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_indent_CREATE.xml new file mode 100644 index 000000000..a5b986951 --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_indent_CREATE.xml @@ -0,0 +1,44 @@ + + + + + + + 1 + 400 + 00 + 1 + 3 + + 1 + 0100000000 + 10 + 2021-01-01 + 0 + + + + + + + + + + 2 + 400 + 05 + 1 + 3 + + 9 + 1 + 0100000000 + 10 + 2021-01-01 + 1 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_indent_DELETE.xml b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_indent_DELETE.xml new file mode 100644 index 000000000..95a74d4e9 --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_indent_DELETE.xml @@ -0,0 +1,24 @@ + + + + + + + 2 + 400 + 05 + 1 + 2 + + 9 + 1 + 0100000000 + 10 + 2021-01-01 + 1 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_indent_UPDATE.xml b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_indent_UPDATE.xml new file mode 100644 index 000000000..2c3c4e1c9 --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_indent_UPDATE.xml @@ -0,0 +1,24 @@ + + + + + + + 2 + 400 + 05 + 1 + 1 + + 9 + 1 + 0100000000 + 10 + 2021-01-02 + 1 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_origin_CREATE.xml b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_origin_CREATE.xml new file mode 100644 index 000000000..dddacc1dd --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_origin_CREATE.xml @@ -0,0 +1,77 @@ + + + + + + + 1 + 400 + 00 + 1 + 3 + + 1 + 0100000000 + 10 + 2021-01-01 + 0 + + + + + + + + 1 + 400 + 00 + 2 + 3 + + 2 + 0101000000 + 10 + 2021-01-01 + 0 + + + + + + + + 1 + 400 + 00 + 3 + 3 + + 3 + 0102000000 + 10 + 2021-01-01 + 0 + + + + + + + + 1 + 400 + 35 + 3 + 3 + + 2 + 0101000000 + 10 + 0102000000 + 10 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_origin_UPDATE.xml b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_origin_UPDATE.xml new file mode 100644 index 000000000..05cafb2b9 --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_origin_UPDATE.xml @@ -0,0 +1,23 @@ + + + + + + + 1 + 400 + 35 + 3 + 1 + + 2 + 0101000000 + 10 + 0102000000 + 10 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_successor_CREATE.xml b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_successor_CREATE.xml new file mode 100644 index 000000000..e81463a30 --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_successor_CREATE.xml @@ -0,0 +1,77 @@ + + + + + + + 1 + 400 + 00 + 1 + 3 + + 1 + 0100000000 + 10 + 2021-01-01 + 0 + + + + + + + + 1 + 400 + 00 + 2 + 3 + + 2 + 0101000000 + 10 + 2021-01-01 + 0 + + + + + + + + 1 + 400 + 00 + 3 + 3 + + 3 + 0102000000 + 10 + 2021-01-01 + 0 + + + + + + + + 1 + 400 + 40 + 3 + 3 + + 2 + 0101000000 + 10 + 0102000000 + 10 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_successor_DELETE.xml b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_successor_DELETE.xml new file mode 100644 index 000000000..a722a78d2 --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_successor_DELETE.xml @@ -0,0 +1,23 @@ + + + + + + + 1 + 400 + 40 + 3 + 2 + + 2 + 0101000000 + 10 + 0102000000 + 10 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_successor_UPDATE.xml b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_successor_UPDATE.xml new file mode 100644 index 000000000..2b9866e00 --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/importer_examples/goods_nomenclature_successor_UPDATE.xml @@ -0,0 +1,23 @@ + + + + + + + 1 + 400 + 40 + 3 + 1 + + 2 + 0101000000 + 10 + 0102000000 + 10 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/commoditiy_parsers/test_footnote_association_goods_nomenclature_parser.py b/taric_parsers/tests/commoditiy_parsers/test_footnote_association_goods_nomenclature_parser.py new file mode 100644 index 000000000..07e0c4ce2 --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/test_footnote_association_goods_nomenclature_parser.py @@ -0,0 +1,166 @@ +from datetime import date + +import pytest + +# note : need to import these objects to make them available to the parser +from commodities.models import FootnoteAssociationGoodsNomenclature +from commodities.models import GoodsNomenclature +from common.tests.util import preload_import +from taric_parsers.parsers.commodity_parser import * +from taric_parsers.parsers.footnote_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestFootnoteAssociationGoodsNomenclatureParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + + + """ + + target_parser_class = FootnoteAssociationGoodsNomenclatureParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "goods_nomenclature_sid": "8", + "footnote_type": "8", + "footnote_id": "8", + "validity_start_date": "2022-01-01", + "validity_end_date": "2023-01-01", + "goods_nomenclature_item_id": "0100000000", + "productline_suffix": "10", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + assert target.goods_nomenclature__sid == 8 + assert target.associated_footnote__footnote_type__footnote_type_id == 8 + assert target.associated_footnote__footnote_id == 8 + assert target.valid_between_lower == date(2022, 1, 1) + assert target.valid_between_upper == date(2023, 1, 1) + assert target.goods_nomenclature__item_id == "0100000000" + assert target.goods_nomenclature__suffix == 10 + + def test_import(self, superuser): + importer = preload_import( + "footnote_association_goods_nomenclature_CREATE.xml", + __file__, + ) + + assert len(importer.parsed_transactions) == 2 + assert len(importer.parsed_transactions[0].parsed_messages) == 4 + assert len(importer.parsed_transactions[1].parsed_messages) == 1 + + target_message = importer.parsed_transactions[1].parsed_messages[0] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.goods_nomenclature__sid == 1 + assert target.associated_footnote__footnote_type__footnote_type_id == 3 + assert target.associated_footnote__footnote_id == 9 + assert target.valid_between_lower == date(2022, 1, 1) + assert target.valid_between_upper == date(2023, 1, 1) + assert target.goods_nomenclature__item_id == "0100000000" + assert target.goods_nomenclature__suffix == 10 + + assert FootnoteAssociationGoodsNomenclature.objects.all().count() == 1 + assert GoodsNomenclature.objects.all().count() == 1 + + assert len(importer.issues()) == 0 + + def test_import_update(self): + preload_import( + "footnote_association_goods_nomenclature_CREATE.xml", + __file__, + True, + ) + importer = preload_import( + "footnote_association_goods_nomenclature_UPDATE.xml", + __file__, + ) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + target = target_message.taric_object + + assert target.goods_nomenclature__sid == 1 + assert target.associated_footnote__footnote_type__footnote_type_id == 3 + assert target.associated_footnote__footnote_id == 9 + assert target.valid_between_lower == date(2022, 1, 11) + assert target.valid_between_upper == date(2023, 1, 1) + assert target.goods_nomenclature__item_id == "0100000000" + assert target.goods_nomenclature__suffix == 10 + + assert FootnoteAssociationGoodsNomenclature.objects.all().count() == 2 + + assert len(importer.issues()) == 0 + + def test_import_delete(self): + preload_import( + "footnote_association_goods_nomenclature_CREATE.xml", + __file__, + True, + ) + importer = preload_import( + "footnote_association_goods_nomenclature_DELETE.xml", + __file__, + ) + + assert len(importer.issues()) == 0 + assert importer.can_save() + + assert FootnoteAssociationGoodsNomenclature.objects.all().count() == 2 + + def test_import_failure_no_footnote(self, superuser): + importer = preload_import( + "footnote_association_goods_nomenclature_no_footnote_CREATE.xml", + __file__, + ) + + assert not importer.can_save() + + assert len(importer.parsed_transactions) == 2 + assert len(importer.parsed_transactions[1].parsed_messages) == 1 + + target_message = importer.parsed_transactions[1].parsed_messages[0] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + assert len(importer.issues()) == 2 + assert "ERROR: Missing expected linked object FootnoteParserV2\n" in str( + importer.issues()[0], + ) + assert ( + "ERROR: Database Integrity error, review related issues to determine what went wrong null value in column " + '"associated_footnote_id" of relation "commodities_footnoteassociationgoodsnomenclature" violates not-null ' + "constraint\n" in str(importer.issues()[1]) + ) diff --git a/taric_parsers/tests/commoditiy_parsers/test_goods_nomenclature_description_parser.py b/taric_parsers/tests/commoditiy_parsers/test_goods_nomenclature_description_parser.py new file mode 100644 index 000000000..7ed22add6 --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/test_goods_nomenclature_description_parser.py @@ -0,0 +1,210 @@ +from datetime import date + +import pytest + +# note : need to import these objects to make them available to the parser +from commodities.models import GoodsNomenclature +from commodities.models import GoodsNomenclatureDescription +from common.tests.util import preload_import +from taric_parsers.parsers.commodity_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestGoodsNomenclatureDescriptionParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + + """ + + target_parser_class = GoodsNomenclatureDescriptionParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "goods_nomenclature_description_period_sid": "7", + "goods_nomenclature_sid": "555", + "goods_nomenclature_item_id": "0100000000", + "language_id": "ZZ", + "productline_suffix": "10", + "description": "Some Description", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.sid == 7 + assert target.described_goods_nomenclature__sid == 555 + assert target.described_goods_nomenclature__item_id == "0100000000" + assert target.described_goods_nomenclature__suffix == 10 + assert target.description == "Some Description" + + def test_import(self, superuser): + importer = preload_import( + "goods_nomenclature_description_with_period_CREATE.xml", + __file__, + ) + + assert len(importer.parsed_transactions) == 2 + assert len(importer.parsed_transactions[1].parsed_messages) == 2 + + target_message = importer.parsed_transactions[1].parsed_messages[0] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.sid == 7 + assert target.described_goods_nomenclature__sid == 1 + assert target.described_goods_nomenclature__item_id == "0100000000" + assert target.description == "Some Description" + assert target.described_goods_nomenclature__suffix == 10 + + assert GoodsNomenclatureDescription.objects.all().count() == 1 + assert GoodsNomenclature.objects.all().count() == 1 + + assert len(importer.issues()) == 0 + + def test_import_update(self, superuser): + preload_import( + "goods_nomenclature_description_with_period_CREATE.xml", + __file__, + True, + ) + importer = preload_import("goods_nomenclature_description_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + target = target_message.taric_object + + assert target.sid == 7 + assert target.described_goods_nomenclature__sid == 1 + assert target.described_goods_nomenclature__item_id == "0100000000" + assert target.description == "Some Description that changed" + assert target.described_goods_nomenclature__suffix == 10 + + assert GoodsNomenclatureDescription.objects.all().count() == 2 + assert GoodsNomenclature.objects.all().count() == 1 + + assert len(importer.issues()) == 0 + + def test_import_delete(self, superuser): + preload_import( + "goods_nomenclature_description_with_period_CREATE.xml", + __file__, + True, + ) + importer = preload_import("goods_nomenclature_description_DELETE.xml", __file__) + + assert importer.can_save() + assert GoodsNomenclatureDescription.objects.all().count() == 2 + assert len(importer.issues()) == 0 + + def test_import_failure_no_period(self, superuser): + importer = preload_import( + "goods_nomenclature_description_no_period_CREATE.xml", + __file__, + ) + + assert not importer.can_save() + + assert len(importer.parsed_transactions) == 2 + assert len(importer.parsed_transactions[1].parsed_messages) == 1 + + target_message = importer.parsed_transactions[1].parsed_messages[0] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.sid == 9 + assert target.described_goods_nomenclature__sid == 7 + assert target.described_goods_nomenclature__item_id == "0102000000" + assert target.description == "Some Description" + assert target.described_goods_nomenclature__suffix == 10 + + assert ( + len(importer.parsed_transactions[1].parsed_messages[0].taric_object.issues) + == 1 + ) + + assert len(importer.issues()) == 1 + assert ( + "Missing expected child object GoodsNomenclatureDescriptionPeriodParserV2" + in str(importer.issues()[0]) + ) + assert ( + "goods.nomenclature.description > goods.nomenclature.description.period" + in str(importer.issues()[0]) + ) + + def test_import_successfully_gets_previous_period(self, superuser): + # preload data and approve + preload_import( + "goods_nomenclature_description_with_period_CREATE.xml", + __file__, + True, + ) + + # load data not approved + importer = preload_import( + "goods_nomenclature_description_no_period_UPDATE.xml", + __file__, + ) + + assert len(importer.parsed_transactions) == 1 + + assert importer.issues() == [] + + assert importer.can_save() + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.sid == 7 + assert target.described_goods_nomenclature__sid == 1 + assert target.described_goods_nomenclature__item_id == "0100000000" + assert target.description == "Some Description Changed" + assert target.described_goods_nomenclature__suffix == 10 + + assert len(importer.issues()) == 0 + + assert GoodsNomenclatureDescription.objects.all().count() == 2 + + last_imported_goods_description = ( + GoodsNomenclatureDescription.objects.all().order_by("pk").last() + ) + + assert last_imported_goods_description.description == "Some Description Changed" + assert last_imported_goods_description.sid == 7 + assert last_imported_goods_description.validity_start == date(2021, 1, 1) diff --git a/taric_parsers/tests/commoditiy_parsers/test_goods_nomenclature_description_period_parser.py b/taric_parsers/tests/commoditiy_parsers/test_goods_nomenclature_description_period_parser.py new file mode 100644 index 000000000..0e800426e --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/test_goods_nomenclature_description_period_parser.py @@ -0,0 +1,142 @@ +from datetime import date + +import pytest + +# note : need to import these objects to make them available to the parser +from commodities.models import GoodsNomenclatureDescription +from common.tests.util import preload_import +from taric_parsers.parsers.commodity_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestGoodsNomenclatureDescriptionPeriodParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + """ + + target_parser_class = GoodsNomenclatureDescriptionPeriodParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "goods_nomenclature_description_period_sid": "7", + "goods_nomenclature_sid": "555", + "goods_nomenclature_item_id": "0100000000", + "language_id": "ZZ", + "productline_suffix": "10", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.sid == 7 + assert target.described_goods_nomenclature__sid == 555 + assert target.described_goods_nomenclature__item_id == "0100000000" + assert target.described_goods_nomenclature__suffix == 10 + + def test_import_create_and_update_in_same_file(self, superuser): + importer = preload_import( + "goods_nomenclature_description_period_UPDATE.xml", + __file__, + ) + + assert len(importer.parsed_transactions) == 3 + + target_message = importer.parsed_transactions[2].parsed_messages[0] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + # check properties + target = target_message.taric_object + + assert target.sid == 7 + assert target.described_goods_nomenclature__sid == 1 + assert target.described_goods_nomenclature__item_id == "0100000000" + assert target.described_goods_nomenclature__suffix == 10 + + assert len(importer.issues()) == 0 + + assert GoodsNomenclatureDescription.objects.all().count() == 2 + assert GoodsNomenclatureDescription.objects.all().last().validity_start == date( + 2021, + 1, + 2, + ) + + def test_import_update(self, superuser): + preload_import( + "goods_nomenclature_description_period_UPDATE.xml", + __file__, + True, + ) + importer = preload_import( + "goods_nomenclature_description_period_only_UPDATE.xml", + __file__, + ) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + # check properties + target = target_message.taric_object + + assert target.sid == 7 + assert target.described_goods_nomenclature__sid == 1 + assert target.described_goods_nomenclature__item_id == "0100000000" + assert target.described_goods_nomenclature__suffix == 10 + + assert len(importer.issues()) == 0 + + assert GoodsNomenclatureDescription.objects.all().count() == 3 + assert GoodsNomenclatureDescription.objects.all().last().validity_start == date( + 2021, + 1, + 3, + ) + + def test_import_delete(self, superuser): + preload_import( + "goods_nomenclature_description_period_UPDATE.xml", + __file__, + True, + ) + importer = preload_import( + "goods_nomenclature_description_period_only_DELETE.xml", + __file__, + ) + + assert importer.can_save() + + assert len(importer.issues()) == 1 + assert ( + "Children of Taric objects of type GoodsNomenclatureDescription can't be deleted directly" + in str(importer.issues()[0]) + ) diff --git a/taric_parsers/tests/commoditiy_parsers/test_goods_nomenclature_indent_parser.py b/taric_parsers/tests/commoditiy_parsers/test_goods_nomenclature_indent_parser.py new file mode 100644 index 000000000..20b1e6e4e --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/test_goods_nomenclature_indent_parser.py @@ -0,0 +1,127 @@ +from datetime import date + +import pytest + +# note : need to import these objects to make them available to the parser +from commodities.models import GoodsNomenclatureIndent +from common.tests.util import preload_import +from taric_parsers.parsers.commodity_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestGoodsNomenclatureIndentParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + + """ + + target_parser_class = GoodsNomenclatureIndentParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "goods_nomenclature_indent_sid": "8", + "goods_nomenclature_sid": "555", + "goods_nomenclature_item_id": "0100000000", + "productline_suffix": "10", + "validity_start_date": "2022-01-01", + "number_indents": "2", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.sid == 8 + assert target.indented_goods_nomenclature__sid == 555 + assert target.validity_start == date(2022, 1, 1) + assert target.indent == 2 + assert target.indented_goods_nomenclature__item_id == "0100000000" + assert target.indented_goods_nomenclature__suffix == 10 + + def test_import(self, superuser): + importer = preload_import("goods_nomenclature_indent_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 2 + assert len(importer.parsed_transactions[0].parsed_messages) == 1 + assert len(importer.parsed_transactions[1].parsed_messages) == 1 + + target_message = importer.parsed_transactions[1].parsed_messages[0] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.sid == 9 + assert target.indented_goods_nomenclature__sid == 1 + assert target.validity_start == date(2021, 1, 1) + assert target.indent == 1 + assert target.indented_goods_nomenclature__item_id == "0100000000" + assert target.indented_goods_nomenclature__suffix == 10 + + assert GoodsNomenclatureIndent.objects.all().count() == 1 + + assert len(importer.issues()) == 0 + + def test_import_update(self, superuser): + preload_import("goods_nomenclature_indent_CREATE.xml", __file__, True) + importer = preload_import("goods_nomenclature_indent_UPDATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.sid == 9 + assert target.indented_goods_nomenclature__sid == 1 + assert target.validity_start == date(2021, 1, 2) + assert target.indent == 1 + assert target.indented_goods_nomenclature__item_id == "0100000000" + assert target.indented_goods_nomenclature__suffix == 10 + + assert GoodsNomenclatureIndent.objects.all().count() == 2 + + assert len(importer.issues()) == 0 + + latest_goods_indent = ( + GoodsNomenclatureIndent.objects.all().order_by("pk").last() + ) + + assert latest_goods_indent.validity_start == date(2021, 1, 2) + + def test_import_delete(self, superuser): + preload_import("goods_nomenclature_indent_CREATE.xml", __file__, True) + importer = preload_import("goods_nomenclature_indent_DELETE.xml", __file__) + + assert importer.can_save() + assert len(importer.issues()) == 0 + assert GoodsNomenclatureIndent.objects.all().count() == 2 diff --git a/taric_parsers/tests/commoditiy_parsers/test_goods_nomenclature_origin_parser.py b/taric_parsers/tests/commoditiy_parsers/test_goods_nomenclature_origin_parser.py new file mode 100644 index 000000000..666d2af7f --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/test_goods_nomenclature_origin_parser.py @@ -0,0 +1,91 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from commodities.models import GoodsNomenclatureOrigin +from common.tests.util import preload_import +from taric_parsers.parsers.commodity_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestGoodsNomenclatureOriginParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + """ + + target_parser_class = GoodsNomenclatureOriginParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "goods_nomenclature_sid": "555", + "goods_nomenclature_item_id": "0100000000", + "productline_suffix": "10", + "derived_productline_suffix": "10", + "derived_goods_nomenclature_item_id": "0101000000", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.new_goods_nomenclature__sid == 555 + assert target.new_goods_nomenclature__item_id == "0100000000" + assert target.new_goods_nomenclature__suffix == 10 + assert target.derived_from_goods_nomenclature__suffix == 10 + assert target.derived_from_goods_nomenclature__item_id == "0101000000" + + def test_import(self, superuser): + importer = preload_import("goods_nomenclature_origin_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + assert len(importer.parsed_transactions[0].parsed_messages) == 4 + + target_message = importer.parsed_transactions[0].parsed_messages[3] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + assert target.new_goods_nomenclature__sid == 2 + + assert target.new_goods_nomenclature__item_id == "0101000000" + assert target.new_goods_nomenclature__suffix == 10 + assert target.derived_from_goods_nomenclature__suffix == 10 + assert target.derived_from_goods_nomenclature__item_id == "0102000000" + + assert GoodsNomenclatureOrigin.objects.all().count() == 1 + + assert importer.issues() == [] + + def test_import_update_raises_issue(self, superuser): + preload_import("goods_nomenclature_origin_CREATE.xml", __file__, True) + importer = preload_import("goods_nomenclature_origin_UPDATE.xml", __file__) + + assert len(importer.issues()) == 1 + + assert "Taric objects of type GoodsNomenclatureOrigin can't be updated" in str( + importer.issues()[0], + ) diff --git a/taric_parsers/tests/commoditiy_parsers/test_goods_nomenclature_parser.py b/taric_parsers/tests/commoditiy_parsers/test_goods_nomenclature_parser.py new file mode 100644 index 000000000..79c74e85e --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/test_goods_nomenclature_parser.py @@ -0,0 +1,143 @@ +from datetime import date + +import pytest + +# note : need to import these objects to make them available to the parser +from commodities.models import GoodsNomenclature +from common.tests.util import preload_import +from taric_parsers.parsers.commodity_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestGoodsNomenclatureParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + + """ + + target_parser_class = GoodsNomenclatureParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "goods_nomenclature_sid": "555", + "goods_nomenclature_item_id": "0100000000", # gets ignored, but will come in from import + "producline_suffix": "10", + "validity_start_date": "2020-01-01", + "validity_end_date": "2020-12-01", + "statistical_indicator": "0", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.sid == 555 + assert target.item_id == "0100000000" + assert target.suffix == 10 + assert target.valid_between_lower == date(2020, 1, 1) + assert target.valid_between_upper == date(2020, 12, 1) + assert target.statistical == 0 + + def test_import(self, superuser): + importer = preload_import("goods_nomenclature_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + assert len(importer.parsed_transactions[0].parsed_messages) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target_taric_object = target_message.taric_object + assert target_taric_object.sid == 1 + assert target_taric_object.item_id == "0100000000" + assert target_taric_object.suffix == 10 + assert target_taric_object.valid_between_lower == date(2021, 1, 1) + assert target_taric_object.valid_between_upper is None + assert target_taric_object.statistical == 0 + + assert GoodsNomenclature.objects.all().count() == 1 + + assert len(importer.issues()) == 0 + + def test_import_then_delete(self, superuser): + importer = preload_import("goods_nomenclature_CREATE_then_DELETE.xml", __file__) + + assert len(importer.parsed_transactions) == 2 + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target_taric_object = target_message.taric_object + assert target_taric_object.sid == 1 + assert target_taric_object.item_id == "0100000000" + assert target_taric_object.suffix == 10 + assert target_taric_object.valid_between_lower == date(2021, 1, 1) + assert target_taric_object.valid_between_upper is None + assert target_taric_object.statistical == 0 + + assert GoodsNomenclature.objects.all().count() == 2 + assert GoodsNomenclature.objects.latest_approved().count() == 0 + + assert len(importer.issues()) == 0 + + def test_import_update(self, superuser): + preload_import("goods_nomenclature_CREATE.xml", __file__, True) + importer = preload_import("goods_nomenclature_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target_taric_object = target_message.taric_object + + assert target_taric_object.sid == 1 + assert target_taric_object.item_id == "0100000000" + assert target_taric_object.suffix == 10 + assert target_taric_object.valid_between_lower == date(2021, 1, 2) + assert target_taric_object.valid_between_upper is None + assert target_taric_object.statistical == 0 + + assert GoodsNomenclature.objects.all().count() == 2 + assert GoodsNomenclature.objects.latest_approved().count() == 1 + + assert len(importer.issues()) == 0 + + latest_gn = GoodsNomenclature.objects.all().order_by("pk").last() + + assert latest_gn.valid_between.lower == date(2021, 1, 2) + + def test_import_delete_raises_issue(self, superuser): + preload_import("goods_nomenclature_CREATE.xml", __file__, True) + importer = preload_import("goods_nomenclature_DELETE.xml", __file__) + + assert len(importer.issues()) == 0 + assert importer.can_save() + assert GoodsNomenclature.objects.all().count() == 2 + assert GoodsNomenclature.objects.latest_approved().count() == 1 diff --git a/taric_parsers/tests/commoditiy_parsers/test_goods_nomenclature_successor_parser.py b/taric_parsers/tests/commoditiy_parsers/test_goods_nomenclature_successor_parser.py new file mode 100644 index 000000000..1d62d57d2 --- /dev/null +++ b/taric_parsers/tests/commoditiy_parsers/test_goods_nomenclature_successor_parser.py @@ -0,0 +1,99 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from commodities.models import GoodsNomenclatureSuccessor +from common.tests.util import preload_import +from taric_parsers.parsers.commodity_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestGoodsNomenclatureSuccessorParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + """ + + target_parser_class = GoodsNomenclatureSuccessorParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "goods_nomenclature_sid": "555", + "goods_nomenclature_item_id": "0100000000", + "absorbed_goods_nomenclature_item_id": "0101000000", + "absorbed_productline_suffix": "10", + "productline_suffix": "10", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.replaced_goods_nomenclature__sid == 555 + assert target.replaced_goods_nomenclature__item_id == "0100000000" + assert target.replaced_goods_nomenclature__suffix == 10 + assert target.absorbed_into_goods_nomenclature__item_id == "0101000000" + assert target.absorbed_into_goods_nomenclature__suffix == 10 + + def test_import(self, superuser): + importer = preload_import("goods_nomenclature_successor_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + assert len(importer.parsed_transactions[0].parsed_messages) == 4 + + target_message = importer.parsed_transactions[0].parsed_messages[3] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.absorbed_into_goods_nomenclature__item_id == "0102000000" + assert target.absorbed_into_goods_nomenclature__suffix == 10 + assert target.replaced_goods_nomenclature__sid == 2 + assert target.replaced_goods_nomenclature__item_id == "0101000000" + assert target.replaced_goods_nomenclature__suffix == 10 + + assert GoodsNomenclatureSuccessor.objects.all().count() == 1 + + assert len(importer.issues()) == 0 + + def test_import_update_raises_issue(self, superuser): + preload_import("goods_nomenclature_successor_CREATE.xml", __file__, True) + importer = preload_import("goods_nomenclature_successor_UPDATE.xml", __file__) + + assert len(importer.issues()) == 1 + + assert ( + "Taric objects of type GoodsNomenclatureSuccessor can't be updated" + in str(importer.issues()[0]) + ) + + def test_import_delete_raises_issue(self, superuser): + preload_import("goods_nomenclature_successor_CREATE.xml", __file__, True) + importer = preload_import("goods_nomenclature_successor_DELETE.xml", __file__) + + assert len(importer.issues()) == 0 + assert importer.can_save() diff --git a/taric_parsers/tests/conftest.py b/taric_parsers/tests/conftest.py new file mode 100644 index 000000000..596c06396 --- /dev/null +++ b/taric_parsers/tests/conftest.py @@ -0,0 +1,116 @@ +import os +import shutil +from pathlib import Path +from typing import Sequence +from typing import Type + +import pytest +from django.forms.models import model_to_dict +from rest_framework import serializers +from rest_framework.serializers import ModelSerializer + +from common.serializers import TrackedModelSerializerMixin +from common.serializers import ValiditySerializerMixin +from common.tests import factories +from common.tests.models import TestModel1 +from common.tests.util import generate_test_import_xml +from importer.handlers import BaseHandler +from importer.namespaces import TARIC_RECORD_GROUPS +from importer.namespaces import Tag +from importer.namespaces import TTags +from importer.namespaces import make_schema_dataclass +from importer.namespaces import xsd_schema_paths +from importer.nursery import TariffObjectNursery +from importer.nursery import get_nursery + + +def get_project_root(): + return Path(__file__).parents[2] + + +@pytest.fixture +def object_nursery() -> TariffObjectNursery: + nursery = get_nursery() + yield nursery + nursery.cache.clear() + + +@pytest.fixture +def mock_serializer() -> Type[ModelSerializer]: + class TestSerializer(TrackedModelSerializerMixin, ValiditySerializerMixin): + sid = serializers.IntegerField() + + class Meta: + model = TestModel1 + exclude = ("version_group",) + + return TestSerializer + + +@pytest.fixture +def parser_class(mock_serializer) -> Type[BaseHandler]: + class TestHandler(BaseHandler): + serializer_class = mock_serializer + tag = "test_handler" + + return TestHandler + + +@pytest.fixture +def taric_schema_tags() -> TTags: + return make_schema_dataclass(xsd_schema_paths) + + +@pytest.fixture +def record_group() -> Sequence[str]: + return TARIC_RECORD_GROUPS["commodities"] + + +@pytest.fixture +def envelope_measure() -> bytes: + model = factories.MeasureFactory.create() + data = model_to_dict(model) + data.update( + { + "record_code": model.record_code, + "subrecord_code": model.subrecord_code, + "taric_template": "taric/measure.xml", + }, + ) + data["goods_nomenclature"] = {"item_id": model.goods_nomenclature.item_id} + return generate_test_import_xml([data]).read() + + +@pytest.fixture +def envelope_commodity() -> bytes: + model = factories.GoodsNomenclatureFactory.create() + data = model_to_dict(model) + data.update( + { + "record_code": model.record_code, + "subrecord_code": model.subrecord_code, + "taric_template": "taric/goods_nomenclature.xml", + }, + ) + return generate_test_import_xml([data]).read() + + +@pytest.fixture +def tag_name() -> Tag: + return Tag(r"quota.event") + + +@pytest.fixture +def tag_regex() -> Tag: + return Tag(r"quota.([a-z.]+).event") + + +@pytest.fixture +def example_goods_taric_file_location(): + root_path = get_project_root() + src = os.path.join(root_path, "importer/tests/test_files/goods.xml") + dst = os.path.join(root_path, "tmp/taric/goods.xml") + os.makedirs(os.path.dirname(dst), exist_ok=True) + shutil.copyfile(src, dst) + taric_file_location = dst + return taric_file_location diff --git a/taric_parsers/tests/footnote_parsers/importer_examples/footnote_CREATE.xml b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_CREATE.xml new file mode 100644 index 000000000..0465664b2 --- /dev/null +++ b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_CREATE.xml @@ -0,0 +1,76 @@ + + + + + + + 1 + 400 + 00 + 1 + 3 + + 1 + 0100000000 + 10 + 2022-01-01 + 0 + + + + + + + + 1 + 100 + 05 + 2 + 3 + + 3 + ZZ + Some description + + + + + + + + 1 + 100 + 00 + 3 + 3 + + 3 + 2021-01-01 + 2021-12-01 + 9 + + + + + + + + 1 + 200 + 00 + 4 + 3 + + 3 + 9 + 2021-01-01 + 2021-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/footnote_parsers/importer_examples/footnote_DELETE.xml b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_DELETE.xml new file mode 100644 index 000000000..e092c874f --- /dev/null +++ b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_DELETE.xml @@ -0,0 +1,23 @@ + + + + + + + 1 + 200 + 00 + 4 + 2 + + 3 + 9 + 2021-01-01 + 2021-01-22 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/footnote_parsers/importer_examples/footnote_UPDATE.xml b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_UPDATE.xml new file mode 100644 index 000000000..293bcbf68 --- /dev/null +++ b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_UPDATE.xml @@ -0,0 +1,23 @@ + + + + + + + 1 + 200 + 00 + 4 + 1 + + 3 + 9 + 2021-01-01 + 2021-01-22 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/footnote_parsers/importer_examples/footnote_description_CREATE.xml b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_description_CREATE.xml new file mode 100644 index 000000000..5b0d072ac --- /dev/null +++ b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_description_CREATE.xml @@ -0,0 +1,98 @@ + + + + + + + 1 + 100 + 05 + 1 + 3 + + 3 + ZZ + Some description + + + + + + + + 1 + 100 + 00 + 2 + 3 + + 3 + 2021-01-01 + 2021-12-01 + 9 + + + + + + + + + 1 + 200 + 00 + 4 + 3 + + 3 + 9 + 2021-01-01 + 2021-01-01 + + + + + + + + + + 1 + 200 + 10 + 4 + 3 + + 7 + zz + 3 + 9 + Some Description + + + + + + + + 1 + 200 + 05 + 4 + 3 + + 7 + 3 + 9 + 2021-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/footnote_parsers/importer_examples/footnote_description_DELETE.xml b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_description_DELETE.xml new file mode 100644 index 000000000..aff48e6f5 --- /dev/null +++ b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_description_DELETE.xml @@ -0,0 +1,24 @@ + + + + + + + 1 + 200 + 10 + 4 + 2 + + 7 + zz + 3 + 9 + Some Description that changed + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/footnote_parsers/importer_examples/footnote_description_UPDATE.xml b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_description_UPDATE.xml new file mode 100644 index 000000000..8654fe2a7 --- /dev/null +++ b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_description_UPDATE.xml @@ -0,0 +1,24 @@ + + + + + + + 1 + 200 + 10 + 4 + 1 + + 7 + zz + 3 + 9 + Some Description that changed + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/footnote_parsers/importer_examples/footnote_description_period_CREATE.xml b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_description_period_CREATE.xml new file mode 100644 index 000000000..5b0d072ac --- /dev/null +++ b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_description_period_CREATE.xml @@ -0,0 +1,98 @@ + + + + + + + 1 + 100 + 05 + 1 + 3 + + 3 + ZZ + Some description + + + + + + + + 1 + 100 + 00 + 2 + 3 + + 3 + 2021-01-01 + 2021-12-01 + 9 + + + + + + + + + 1 + 200 + 00 + 4 + 3 + + 3 + 9 + 2021-01-01 + 2021-01-01 + + + + + + + + + + 1 + 200 + 10 + 4 + 3 + + 7 + zz + 3 + 9 + Some Description + + + + + + + + 1 + 200 + 05 + 4 + 3 + + 7 + 3 + 9 + 2021-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/footnote_parsers/importer_examples/footnote_description_period_DELETE.xml b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_description_period_DELETE.xml new file mode 100644 index 000000000..3f07f7d83 --- /dev/null +++ b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_description_period_DELETE.xml @@ -0,0 +1,23 @@ + + + + + + + 1 + 200 + 05 + 4 + 2 + + 7 + 3 + 9 + 2021-01-22 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/footnote_parsers/importer_examples/footnote_description_period_UPDATE.xml b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_description_period_UPDATE.xml new file mode 100644 index 000000000..d5e97de11 --- /dev/null +++ b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_description_period_UPDATE.xml @@ -0,0 +1,23 @@ + + + + + + + 1 + 200 + 05 + 4 + 1 + + 7 + 3 + 9 + 2021-01-22 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/footnote_parsers/importer_examples/footnote_type_CREATE.xml b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_type_CREATE.xml new file mode 100644 index 000000000..2dce2a170 --- /dev/null +++ b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_type_CREATE.xml @@ -0,0 +1,40 @@ + + + + + + + 1 + 100 + 05 + 2 + 3 + + 3 + ZZ + Some description + + + + + + + + 1 + 100 + 00 + 3 + 3 + + 3 + 2021-01-01 + 2021-12-01 + 9 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/footnote_parsers/importer_examples/footnote_type_DELETE.xml b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_type_DELETE.xml new file mode 100644 index 000000000..0ab44a354 --- /dev/null +++ b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_type_DELETE.xml @@ -0,0 +1,23 @@ + + + + + + + 1 + 100 + 00 + 3 + 2 + + 3 + 2021-01-01 + 2021-12-22 + 9 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/footnote_parsers/importer_examples/footnote_type_UPDATE.xml b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_type_UPDATE.xml new file mode 100644 index 000000000..0be94d4b9 --- /dev/null +++ b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_type_UPDATE.xml @@ -0,0 +1,23 @@ + + + + + + + 1 + 100 + 00 + 3 + 1 + + 3 + 2021-01-01 + 2021-12-22 + 9 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/footnote_parsers/importer_examples/footnote_type_description_CREATE.xml b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_type_description_CREATE.xml new file mode 100644 index 000000000..4137c2605 --- /dev/null +++ b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_type_description_CREATE.xml @@ -0,0 +1,40 @@ + + + + + + + 1 + 100 + 05 + 1 + 3 + + 3 + ZZ + Some description + + + + + + + + 1 + 100 + 00 + 2 + 3 + + 3 + 2021-01-01 + 2021-12-01 + 9 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/footnote_parsers/importer_examples/footnote_type_description_DELETE.xml b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_type_description_DELETE.xml new file mode 100644 index 000000000..191c32d41 --- /dev/null +++ b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_type_description_DELETE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 100 + 05 + 1 + 1 + + 3 + ZZ + Some description that changed + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/footnote_parsers/importer_examples/footnote_type_description_UPDATE.xml b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_type_description_UPDATE.xml new file mode 100644 index 000000000..191c32d41 --- /dev/null +++ b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_type_description_UPDATE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 100 + 05 + 1 + 1 + + 3 + ZZ + Some description that changed + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/footnote_parsers/importer_examples/footnote_type_no_description_CREATE.xml b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_type_no_description_CREATE.xml new file mode 100644 index 000000000..4363e588e --- /dev/null +++ b/taric_parsers/tests/footnote_parsers/importer_examples/footnote_type_no_description_CREATE.xml @@ -0,0 +1,23 @@ + + + + + + + 1 + 100 + 00 + 3 + 3 + + 3 + 2021-01-01 + 2021-12-01 + 9 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/footnote_parsers/test_footnote_description_parser.py b/taric_parsers/tests/footnote_parsers/test_footnote_description_parser.py new file mode 100644 index 000000000..cfaa0b8c0 --- /dev/null +++ b/taric_parsers/tests/footnote_parsers/test_footnote_description_parser.py @@ -0,0 +1,102 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from footnotes.models import FootnoteDescription +from taric_parsers.parsers.footnote_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestFootnoteDescriptionParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + """ + + target_parser_class = FootnoteDescriptionParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "footnote_description_period_sid": "8", + "language_id": "zz", + "footnote_type_id": "7", + "footnote_id": "6", + "description": "Some Description", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.sid == 8 + assert target.described_footnote__footnote_type__footnote_type_id == "7" + assert target.described_footnote__footnote_id == "6" + assert target.description == "Some Description" + + def test_import(self, superuser): + importer = preload_import("footnote_description_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 2 + + target_message = importer.parsed_transactions[1].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.sid == 7 + assert target.described_footnote__footnote_type__footnote_type_id == "3" + assert target.described_footnote__footnote_id == "9" + assert target.description == "Some Description" + + assert len(importer.issues()) == 0 + + assert FootnoteDescription.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("footnote_description_CREATE.xml", __file__, True) + importer = preload_import("footnote_description_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + target = target_message.taric_object + + assert target.sid == 7 + assert target.described_footnote__footnote_type__footnote_type_id == "3" + assert target.described_footnote__footnote_id == "9" + assert target.description == "Some Description that changed" + + assert len(importer.issues()) == 0 + + assert FootnoteDescription.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("footnote_description_CREATE.xml", __file__, True) + importer = preload_import("footnote_description_DELETE.xml", __file__) + + assert len(importer.issues()) == 0 + assert importer.can_save() + + assert FootnoteDescription.objects.all().count() == 2 diff --git a/taric_parsers/tests/footnote_parsers/test_footnote_description_period_parser.py b/taric_parsers/tests/footnote_parsers/test_footnote_description_period_parser.py new file mode 100644 index 000000000..9149ee312 --- /dev/null +++ b/taric_parsers/tests/footnote_parsers/test_footnote_description_period_parser.py @@ -0,0 +1,107 @@ +from datetime import date + +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from footnotes.models import FootnoteDescription +from taric_parsers.parsers.footnote_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestFootnoteDescriptionPeriodParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + """ + + target_parser_class = FootnoteDescriptionPeriodParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "footnote_description_period_sid": "8", + "footnote_type_id": "7", + "footnote_id": "6", + "validity_start_date": "2022-01-01", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.sid == 8 + assert target.described_footnote__footnote_type__footnote_type_id == "7" + assert target.described_footnote__footnote_id == "6" + assert target.validity_start == date(2022, 1, 1) + + def test_import(self, superuser): + importer = preload_import("footnote_description_period_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 2 + + target_message = importer.parsed_transactions[1].parsed_messages[1] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.sid == 7 + assert target.described_footnote__footnote_type__footnote_type_id == "3" + assert target.described_footnote__footnote_id == "9" + assert target.validity_start == date(2021, 1, 1) + + assert len(importer.issues()) == 0 + + assert FootnoteDescription.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("footnote_description_period_CREATE.xml", __file__, True) + importer = preload_import("footnote_description_period_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target = target_message.taric_object + + assert target.sid == 7 + assert target.described_footnote__footnote_type__footnote_type_id == "3" + assert target.described_footnote__footnote_id == "9" + assert target.validity_start == date(2021, 1, 22) + + assert len(importer.issues()) == 0 + + assert FootnoteDescription.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("footnote_description_period_CREATE.xml", __file__, True) + importer = preload_import("footnote_description_period_DELETE.xml", __file__) + + assert importer.can_save() + + assert len(importer.issues()) == 1 + assert ( + "Children of Taric objects of type FootnoteDescription can't be deleted directly" + in str(importer.issues()[0]) + ) diff --git a/taric_parsers/tests/footnote_parsers/test_footnote_parser.py b/taric_parsers/tests/footnote_parsers/test_footnote_parser.py new file mode 100644 index 000000000..35a28d0e4 --- /dev/null +++ b/taric_parsers/tests/footnote_parsers/test_footnote_parser.py @@ -0,0 +1,103 @@ +from datetime import date + +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from footnotes.models import Footnote +from taric_parsers.parsers.footnote_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestFootnoteParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + """ + + target_parser_class = FootnoteParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "footnote_type_id": "7", + "footnote_id": "6", + "validity_start_date": "2021-01-01", + "validity_end_date": "2022-01-01", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.footnote_type__footnote_type_id == "7" + assert target.footnote_id == "6" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + + def test_import(self, superuser): + importer = preload_import("footnote_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[3] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + assert target.footnote_type__footnote_type_id == "3" + assert target.footnote_id == "9" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2021, 1, 1) + + assert Footnote.objects.all().count() == 1 + + assert len(importer.issues()) == 0 + + def test_import_update(self, superuser): + preload_import("footnote_CREATE.xml", __file__, True) + importer = preload_import("footnote_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target = target_message.taric_object + + assert target.footnote_type__footnote_type_id == "3" + assert target.footnote_id == "9" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2021, 1, 22) + + assert Footnote.objects.all().count() == 2 + + assert importer.issues() == [] + + def test_import_delete(self, superuser): + preload_import("footnote_CREATE.xml", __file__, True) + importer = preload_import("footnote_DELETE.xml", __file__) + + assert importer.issues() == [] + assert importer.can_save() + + assert Footnote.objects.all().count() == 2 diff --git a/taric_parsers/tests/footnote_parsers/test_footnote_type_description_parser.py b/taric_parsers/tests/footnote_parsers/test_footnote_type_description_parser.py new file mode 100644 index 000000000..29b00d1b2 --- /dev/null +++ b/taric_parsers/tests/footnote_parsers/test_footnote_type_description_parser.py @@ -0,0 +1,89 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from footnotes.models import FootnoteType +from taric_parsers.parsers.footnote_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestFootnoteTypeDescriptionParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + """ + + target_parser_class = FootnoteTypeDescriptionParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "footnote_type_id": "3", + "language_id": "zz", # gets ignored, but will come in from import + "description": "Some Description", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.footnote_type_id == "3" + assert target.description == "Some Description" + + def test_import(self, superuser): + importer = preload_import("footnote_type_description_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + assert target.footnote_type_id == "3" + assert target.description == "Some description" + + assert FootnoteType.objects.all().count() == 1 + assert len(importer.issues()) == 0 + + def test_import_update(self, superuser): + preload_import("footnote_type_description_CREATE.xml", __file__, True) + importer = preload_import("footnote_type_description_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + target = target_message.taric_object + + assert target.footnote_type_id == "3" + assert target.description == "Some description that changed" + + assert FootnoteType.objects.all().count() == 2 + assert importer.issues() == [] + + def test_import_delete(self, superuser): + preload_import("footnote_type_description_CREATE.xml", __file__, True) + importer = preload_import("footnote_type_description_DELETE.xml", __file__) + + assert FootnoteType.objects.all().count() == 2 + assert importer.issues() == [] + assert importer.can_save() diff --git a/taric_parsers/tests/footnote_parsers/test_footnote_type_parser.py b/taric_parsers/tests/footnote_parsers/test_footnote_type_parser.py new file mode 100644 index 000000000..285e7d141 --- /dev/null +++ b/taric_parsers/tests/footnote_parsers/test_footnote_type_parser.py @@ -0,0 +1,112 @@ +from datetime import date + +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from footnotes.models import FootnoteType +from taric_parsers.parsers.footnote_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestFootnoteTypeParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + """ + + target_parser_class = FootnoteTypeParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "footnote_type_id": "4", + "validity_start_date": "2021-01-01", + "validity_end_date": "2022-01-01", + "application_code": "7", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.footnote_type_id == "4" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + assert target.application_code == "7" + + def test_import(self, superuser): + importer = preload_import("footnote_type_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[1] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target_taric_object = target_message.taric_object + + assert target_taric_object.footnote_type_id == "3" + assert target_taric_object.valid_between_lower == date(2021, 1, 1) + assert target_taric_object.valid_between_upper == date(2021, 12, 1) + assert target_taric_object.application_code == "9" + + assert FootnoteType.objects.all().count() == 1 + + assert len(importer.issues()) == 0 + + def test_import_update(self, superuser): + preload_import("footnote_type_CREATE.xml", __file__, True) + importer = preload_import("footnote_type_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target_taric_object = target_message.taric_object + + assert target_taric_object.footnote_type_id == "3" + assert target_taric_object.valid_between_lower == date(2021, 1, 1) + assert target_taric_object.valid_between_upper == date(2021, 12, 22) + assert target_taric_object.application_code == "9" + + assert importer.issues() == [] + assert FootnoteType.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("footnote_type_CREATE.xml", __file__, True) + importer = preload_import("footnote_type_DELETE.xml", __file__) + + assert importer.issues() == [] + assert importer.can_save() + assert FootnoteType.objects.all().count() == 2 + + def test_import_no_description(self, superuser): + importer = preload_import("footnote_type_no_description_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + assert len(importer.parsed_transactions[0].parsed_messages) == 1 + + assert FootnoteType.objects.all().count() == 0 + + assert len(importer.issues()) == 1 diff --git a/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_CREATE.xml b/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_CREATE.xml new file mode 100644 index 000000000..4e431bfe1 --- /dev/null +++ b/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_CREATE.xml @@ -0,0 +1,24 @@ + + + + + + + 1 + 250 + 00 + 3 + 3 + + 8 + AB01 + 2021-01-01 + 2022-01-01 + 1 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_DELETE.xml b/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_DELETE.xml new file mode 100644 index 000000000..f839a0ab3 --- /dev/null +++ b/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_DELETE.xml @@ -0,0 +1,24 @@ + + + + + + + 1 + 250 + 00 + 3 + 2 + + 8 + AB01 + 2021-01-11 + 2022-01-01 + 1 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_UPDATE.xml b/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_UPDATE.xml new file mode 100644 index 000000000..cacbc891a --- /dev/null +++ b/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_UPDATE.xml @@ -0,0 +1,24 @@ + + + + + + + 1 + 250 + 00 + 3 + 1 + + 8 + AB01 + 2021-01-11 + 2022-01-01 + 1 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_description_CREATE.xml b/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_description_CREATE.xml new file mode 100644 index 000000000..ad3ff15a5 --- /dev/null +++ b/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_description_CREATE.xml @@ -0,0 +1,64 @@ + + + + + + + 1 + 250 + 00 + 3 + 3 + + 8 + AB01 + 2021-01-01 + 2022-01-01 + 1 + + + + + + + + + + 1 + 250 + 10 + 3 + 3 + + 3 + zz + 8 + AB01 + Some Description + + + + + + + + 1 + 250 + 05 + 3 + 3 + + 3 + 2022-01-01 + 8 + AB01 + + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_description_DELETE.xml b/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_description_DELETE.xml new file mode 100644 index 000000000..41ce7c3b9 --- /dev/null +++ b/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_description_DELETE.xml @@ -0,0 +1,24 @@ + + + + + + + 1 + 250 + 10 + 3 + 2 + + 3 + zz + 8 + AB01 + Some Description that changed + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_description_UPDATE.xml b/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_description_UPDATE.xml new file mode 100644 index 000000000..604a8f8d3 --- /dev/null +++ b/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_description_UPDATE.xml @@ -0,0 +1,24 @@ + + + + + + + 1 + 250 + 10 + 3 + 1 + + 3 + zz + 8 + AB01 + Some Description that changed + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_description_period_CREATE.xml b/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_description_period_CREATE.xml new file mode 100644 index 000000000..ad3ff15a5 --- /dev/null +++ b/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_description_period_CREATE.xml @@ -0,0 +1,64 @@ + + + + + + + 1 + 250 + 00 + 3 + 3 + + 8 + AB01 + 2021-01-01 + 2022-01-01 + 1 + + + + + + + + + + 1 + 250 + 10 + 3 + 3 + + 3 + zz + 8 + AB01 + Some Description + + + + + + + + 1 + 250 + 05 + 3 + 3 + + 3 + 2022-01-01 + 8 + AB01 + + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_description_period_DELETE.xml b/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_description_period_DELETE.xml new file mode 100644 index 000000000..df2d810c6 --- /dev/null +++ b/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_description_period_DELETE.xml @@ -0,0 +1,24 @@ + + + + + + + 1 + 250 + 05 + 3 + 2 + + 3 + 2022-01-11 + 8 + AB01 + + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_description_period_UPDATE.xml b/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_description_period_UPDATE.xml new file mode 100644 index 000000000..81d7236df --- /dev/null +++ b/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_area_description_period_UPDATE.xml @@ -0,0 +1,24 @@ + + + + + + + 1 + 250 + 05 + 3 + 1 + + 3 + 2022-01-11 + 8 + AB01 + + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_membership_CREATE.xml b/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_membership_CREATE.xml new file mode 100644 index 000000000..3c021d136 --- /dev/null +++ b/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_membership_CREATE.xml @@ -0,0 +1,64 @@ + + + + + + + 1 + 250 + 00 + 3 + 3 + + 8 + AB01 + 2021-01-01 + 2022-01-01 + 1 + + + + + + + + 1 + 250 + 00 + 3 + 3 + + 9 + XY01 + 2021-01-01 + 2022-01-01 + 1 + + + + + + + + + + 1 + 250 + 15 + 3 + 3 + + 9 + 8 + 2021-01-01 + 2022-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_membership_DELETE.xml b/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_membership_DELETE.xml new file mode 100644 index 000000000..0c98b51dc --- /dev/null +++ b/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_membership_DELETE.xml @@ -0,0 +1,24 @@ + + + + + + + 1 + 250 + 15 + 3 + 2 + + 9 + 8 + 2021-01-21 + 2022-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_membership_UPDATE.xml b/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_membership_UPDATE.xml new file mode 100644 index 000000000..08eb47c0c --- /dev/null +++ b/taric_parsers/tests/geo_area_parsers/importer_examples/geographical_membership_UPDATE.xml @@ -0,0 +1,24 @@ + + + + + + + 1 + 250 + 15 + 3 + 1 + + 9 + 8 + 2021-01-21 + 2022-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/geo_area_parsers/test_geographical_area_description_parser.py b/taric_parsers/tests/geo_area_parsers/test_geographical_area_description_parser.py new file mode 100644 index 000000000..52b472d66 --- /dev/null +++ b/taric_parsers/tests/geo_area_parsers/test_geographical_area_description_parser.py @@ -0,0 +1,102 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from geo_areas.models import GeographicalAreaDescription +from taric_parsers.parsers.geo_area_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestGeographicalAreaDescriptionParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + """ + + target_parser_class = GeographicalAreaDescriptionParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "geographical_area_description_period_sid": "8", + "language_id": "zz", + "geographical_area_sid": "7", + "geographical_area_id": "6", + "description": "Some Description", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.sid == 8 + assert target.described_geographicalarea__sid == 7 + assert target.described_geographicalarea__area_id == "6" + assert target.description == "Some Description" + + def test_import(self, superuser): + importer = preload_import("geographical_area_description_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 2 + + target_message = importer.parsed_transactions[1].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.sid == 3 + assert target.described_geographicalarea__sid == 8 + assert target.described_geographicalarea__area_id == "AB01" + assert target.description == "Some Description" + + assert importer.issues() == [] + + assert GeographicalAreaDescription.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("geographical_area_description_CREATE.xml", __file__, True) + importer = preload_import("geographical_area_description_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + target = target_message.taric_object + + assert target.sid == 3 + assert target.described_geographicalarea__sid == 8 + assert target.described_geographicalarea__area_id == "AB01" + assert target.description == "Some Description that changed" + + assert importer.issues() == [] + + assert GeographicalAreaDescription.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("geographical_area_description_CREATE.xml", __file__, True) + importer = preload_import("geographical_area_description_DELETE.xml", __file__) + + assert importer.issues() == [] + assert importer.can_save() + + assert GeographicalAreaDescription.objects.all().count() == 2 diff --git a/taric_parsers/tests/geo_area_parsers/test_geographical_area_description_period_parser.py b/taric_parsers/tests/geo_area_parsers/test_geographical_area_description_period_parser.py new file mode 100644 index 000000000..ff7f42688 --- /dev/null +++ b/taric_parsers/tests/geo_area_parsers/test_geographical_area_description_period_parser.py @@ -0,0 +1,123 @@ +from datetime import date + +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from geo_areas.models import GeographicalAreaDescription +from taric_parsers.parsers.geo_area_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestGeographicalAreaDescriptionPeriodParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + """ + + target_parser_class = GeographicalAreaDescriptionPeriodParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "geographical_area_description_period_sid": "8", + "geographical_area_sid": "7", + "validity_start_date": "2021-01-01", + "geographical_area_id": "6", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.sid == 8 + assert target.described_geographicalarea__sid == 7 + assert target.described_geographicalarea__area_id == "6" + assert target.validity_start == date(2021, 1, 1) + + def test_import(self, superuser): + importer = preload_import( + "geographical_area_description_period_CREATE.xml", + __file__, + ) + + assert len(importer.parsed_transactions) == 2 + + target_message = importer.parsed_transactions[1].parsed_messages[1] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.sid == 3 + assert target.described_geographicalarea__sid == 8 + assert target.described_geographicalarea__area_id == "AB01" + assert target.validity_start == date(2022, 1, 1) + + assert len(importer.issues()) == 0 + + assert GeographicalAreaDescription.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import( + "geographical_area_description_period_CREATE.xml", + __file__, + True, + ) + importer = preload_import( + "geographical_area_description_period_UPDATE.xml", + __file__, + ) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target = target_message.taric_object + + assert importer.issues() == [] + + assert target.sid == 3 + assert target.described_geographicalarea__sid == 8 + assert target.described_geographicalarea__area_id == "AB01" + assert target.validity_start == date(2022, 1, 11) + + assert GeographicalAreaDescription.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import( + "geographical_area_description_period_CREATE.xml", + __file__, + True, + ) + importer = preload_import( + "geographical_area_description_period_DELETE.xml", + __file__, + ) + + assert importer.can_save() + + assert len(importer.issues()) == 1 + assert ( + "Children of Taric objects of type GeographicalAreaDescription can't be deleted directly" + in str(importer.issues()[0]) + ) diff --git a/taric_parsers/tests/geo_area_parsers/test_geographical_area_parser.py b/taric_parsers/tests/geo_area_parsers/test_geographical_area_parser.py new file mode 100644 index 000000000..4e384a342 --- /dev/null +++ b/taric_parsers/tests/geo_area_parsers/test_geographical_area_parser.py @@ -0,0 +1,114 @@ +from datetime import date + +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from geo_areas.models import GeographicalArea +from taric_parsers.parsers.geo_area_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestGeographicalAreaParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + + """ + + target_parser_class = GeographicalAreaParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "geographical_area_sid": "8", + "geographical_area_id": "9", + "validity_start_date": "2021-01-01", + "validity_end_date": "2022-01-01", + "geographical_code": "5", + "parent_geographical_area_group_sid": "7", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + + assert target.sid == 8 + assert target.area_id == "9" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + assert target.area_code == 5 + assert target.parent__sid == 7 + + def test_import(self, superuser): + importer = preload_import("geographical_area_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.sid == 8 + assert target.area_id == "AB01" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + assert target.area_code == 1 + assert target.parent__sid is None + + assert len(importer.issues()) == 0 + + assert GeographicalArea.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("geographical_area_CREATE.xml", __file__, True) + importer = preload_import("geographical_area_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target = target_message.taric_object + + assert target.sid == 8 + assert target.area_id == "AB01" + assert target.valid_between_lower == date(2021, 1, 11) + assert target.valid_between_upper == date(2022, 1, 1) + assert target.area_code == 1 + assert target.parent__sid is None + + assert importer.issues() == [] + + assert GeographicalArea.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("geographical_area_CREATE.xml", __file__, True) + importer = preload_import("geographical_area_DELETE.xml", __file__) + + assert importer.can_save() + assert len(importer.issues()) == 0 + + assert GeographicalArea.objects.all().count() == 2 diff --git a/taric_parsers/tests/geo_area_parsers/test_geographical_membership_parser.py b/taric_parsers/tests/geo_area_parsers/test_geographical_membership_parser.py new file mode 100644 index 000000000..98143215f --- /dev/null +++ b/taric_parsers/tests/geo_area_parsers/test_geographical_membership_parser.py @@ -0,0 +1,102 @@ +from datetime import date + +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from geo_areas.models import GeographicalMembership +from taric_parsers.parsers.geo_area_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestGeographicalMembershipParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + """ + + target_parser_class = GeographicalMembershipParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "geographical_area_sid": "8", + "geographical_area_group_sid": "7", + "validity_start_date": "2021-01-01", + "validity_end_date": "2022-01-01", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.member__sid == 8 + assert target.geo_group__sid == 7 + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + + def test_import(self, superuser): + importer = preload_import("geographical_membership_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 2 + + target_message = importer.parsed_transactions[1].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.member__sid == 9 + assert target.geo_group__sid == 8 + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + + assert importer.issues() == [] + + assert GeographicalMembership.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("geographical_membership_CREATE.xml", __file__, True) + importer = preload_import("geographical_membership_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + target = target_message.taric_object + + assert target.member__sid == 9 + assert target.geo_group__sid == 8 + assert target.valid_between_lower == date(2021, 1, 21) + assert target.valid_between_upper == date(2022, 1, 1) + + assert importer.issues() == [] + + assert GeographicalMembership.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("geographical_membership_CREATE.xml", __file__, True) + importer = preload_import("geographical_membership_DELETE.xml", __file__) + + assert importer.issues() == [] + assert importer.can_save() + + assert GeographicalMembership.objects.all().count() == 2 diff --git a/taric_parsers/tests/importer_examples/additional_code_CREATE.xml b/taric_parsers/tests/importer_examples/additional_code_CREATE.xml new file mode 100644 index 000000000..d81d37516 --- /dev/null +++ b/taric_parsers/tests/importer_examples/additional_code_CREATE.xml @@ -0,0 +1,55 @@ + + + + + + + 1 + 120 + 00 + 1 + 3 + + 5 + 3 + 2021-01-01 + 2021-12-31 + + + + + + + + 1 + 245 + 00 + 2 + 3 + + 1 + 5 + 3 + 2021-01-01 + + + + + + + + 1 + 120 + 05 + 3 + 3 + + 5 + zz + some description + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/importer_examples/additional_code_UPDATE.xml b/taric_parsers/tests/importer_examples/additional_code_UPDATE.xml new file mode 100644 index 000000000..3f75d4aea --- /dev/null +++ b/taric_parsers/tests/importer_examples/additional_code_UPDATE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 245 + 00 + 2 + 1 + + 1 + 5 + 3 + 2021-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/importer_examples/additional_code_second_CREATE.xml b/taric_parsers/tests/importer_examples/additional_code_second_CREATE.xml new file mode 100644 index 000000000..792baea18 --- /dev/null +++ b/taric_parsers/tests/importer_examples/additional_code_second_CREATE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 245 + 00 + 2 + 3 + + 1 + 5 + 3 + 2021-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/additional_code_type_measure_type_CREATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/additional_code_type_measure_type_CREATE.xml new file mode 100644 index 000000000..a6b3f64e2 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/additional_code_type_measure_type_CREATE.xml @@ -0,0 +1,138 @@ + + + + + + + 1 + 140 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + 6 + + + + + + + + 1 + 140 + 05 + 3 + 3 + + A + zz + Some Description + + + + + + + + + + 1 + 235 + 00 + 3 + 3 + + ZZZ + 2021-01-01 + 2022-01-01 + 1 + 2 + 3 + 4 + 5 + 6 + A + + + + + + + + 1 + 235 + 05 + 3 + 3 + + ZZZ + ZZ + Some Description x + + + + + + + + + + 1 + 120 + 05 + 2 + 3 + + Z + zz + some description + + + + + + + + 1 + 120 + 00 + 1 + 3 + + Z + 111 + 2021-01-01 + 2021-12-31 + + + + + + + + + + 1 + 240 + 00 + 3 + 3 + + ZZZ + Z + 2021-01-01 + 2022-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/additional_code_type_measure_type_DELETE.xml b/taric_parsers/tests/measure_parsers/importer_examples/additional_code_type_measure_type_DELETE.xml new file mode 100644 index 000000000..e214fbde4 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/additional_code_type_measure_type_DELETE.xml @@ -0,0 +1,23 @@ + + + + + + + 1 + 240 + 00 + 3 + 2 + + ZZZ + Z + 2021-01-21 + 2022-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/additional_code_type_measure_type_UPDATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/additional_code_type_measure_type_UPDATE.xml new file mode 100644 index 000000000..9767835d8 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/additional_code_type_measure_type_UPDATE.xml @@ -0,0 +1,23 @@ + + + + + + + 1 + 240 + 00 + 3 + 1 + + ZZZ + Z + 2021-01-21 + 2022-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/duty_expression_CREATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/duty_expression_CREATE.xml new file mode 100644 index 000000000..6fee6ebaa --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/duty_expression_CREATE.xml @@ -0,0 +1,42 @@ + + + + + + + 1 + 230 + 00 + 3 + 3 + + 7 + 2021-01-01 + 2022-01-01 + 8 + 9 + 10 + + + + + + + + 1 + 230 + 05 + 3 + 3 + + 7 + ZZZ + Some Description s + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/duty_expression_DELETE.xml b/taric_parsers/tests/measure_parsers/importer_examples/duty_expression_DELETE.xml new file mode 100644 index 000000000..e5f53ca60 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/duty_expression_DELETE.xml @@ -0,0 +1,25 @@ + + + + + + + 1 + 230 + 00 + 3 + 2 + + 7 + 2021-01-21 + 2022-01-01 + 8 + 9 + 10 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/duty_expression_UPDATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/duty_expression_UPDATE.xml new file mode 100644 index 000000000..6fe241d5a --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/duty_expression_UPDATE.xml @@ -0,0 +1,25 @@ + + + + + + + 1 + 230 + 00 + 3 + 1 + + 7 + 2021-01-21 + 2022-01-01 + 8 + 9 + 10 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/duty_expression_description_CREATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/duty_expression_description_CREATE.xml new file mode 100644 index 000000000..6fee6ebaa --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/duty_expression_description_CREATE.xml @@ -0,0 +1,42 @@ + + + + + + + 1 + 230 + 00 + 3 + 3 + + 7 + 2021-01-01 + 2022-01-01 + 8 + 9 + 10 + + + + + + + + 1 + 230 + 05 + 3 + 3 + + 7 + ZZZ + Some Description s + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/duty_expression_description_DELETE.xml b/taric_parsers/tests/measure_parsers/importer_examples/duty_expression_description_DELETE.xml new file mode 100644 index 000000000..8f27175dd --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/duty_expression_description_DELETE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 230 + 05 + 3 + 2 + + 7 + ZZZ + Some Description that changed + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/duty_expression_description_UPDATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/duty_expression_description_UPDATE.xml new file mode 100644 index 000000000..19a091673 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/duty_expression_description_UPDATE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 230 + 05 + 3 + 1 + + 7 + ZZZ + Some Description that changed + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/footnote_association_measure_CREATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/footnote_association_measure_CREATE.xml new file mode 100644 index 000000000..a32db11d7 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/footnote_association_measure_CREATE.xml @@ -0,0 +1,371 @@ + + + + + + + 1 + 400 + 00 + 1 + 3 + + 1 + 0100000000 + 10 + 2021-01-01 + 0 + + + + + + + + + + 1 + 150 + 00 + 3 + 3 + + ABC + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 150 + 05 + 3 + 3 + + ABC + ZZ + Some Description x + + + + + + + + + + 1 + 285 + 00 + 3 + 3 + + 1 + Z0000001 + 2023-01-01 + ABCDE + 7 + 2023-01-01 + 1 + 7 + 0 + Some Info Text + 1 + ABC + 2021-01-01 + 2022-01-01 + + + + + + + + + + 1 + 400 + 15 + 1 + 3 + + 7 + 1 + 0100000000 + ZZ + 10 + Some Description + + + + + + + + 2 + 400 + 10 + 3 + 3 + + 7 + 1 + 0100000000 + 2021-01-01 + 10 + + + + + + + + + + 1 + 250 + 00 + 3 + 3 + + 8 + AB01 + 2021-01-01 + 2022-01-01 + 1 + + + + + + + + + + 1 + 250 + 10 + 3 + 3 + + 3 + zz + 8 + AB01 + Some Description + + + + + + + + 1 + 250 + 05 + 3 + 3 + + 3 + 2022-01-01 + 8 + AB01 + + + + + + + + + + 1 + 140 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + 6 + + + + + + + + 1 + 140 + 05 + 3 + 3 + + A + zz + Some Description + + + + + + + + + + 1 + 235 + 00 + 3 + 3 + + ZZZ + 2021-01-01 + 2022-01-01 + 1 + 2 + 3 + 4 + 5 + 6 + A + + + + + + + + 1 + 235 + 05 + 3 + 3 + + ZZZ + ZZ + Some Description x + + + + + + + + + + 1 + 430 + 00 + 3 + 3 + + 99 + ZZZ + AB01 + 0100000000 + 1 + 2021-01-01 + 2022-01-01 + 1 + Z0000001 + 1 + Z0000001 + 1 + 8 + + + + + + + + + + 1 + 100 + 05 + 2 + 3 + + 3 + ZZ + Some description + + + + + + + + 1 + 100 + 00 + 3 + 3 + + 3 + 2021-01-01 + 2021-12-01 + 9 + + + + + + + + 1 + 200 + 00 + 4 + 3 + + 3 + 9 + 2021-01-01 + 2021-01-01 + + + + + + + + + + 1 + 430 + 20 + 3 + 3 + + 99 + 3 + 9 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/footnote_association_measure_DELETE.xml b/taric_parsers/tests/measure_parsers/importer_examples/footnote_association_measure_DELETE.xml new file mode 100644 index 000000000..f9314aaa9 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/footnote_association_measure_DELETE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 430 + 20 + 3 + 2 + + 99 + 3 + 9 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/footnote_association_measure_UPDATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/footnote_association_measure_UPDATE.xml new file mode 100644 index 000000000..24b36d5f0 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/footnote_association_measure_UPDATE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 430 + 20 + 3 + 1 + + 99 + 3 + 9 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_CREATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_CREATE.xml new file mode 100644 index 000000000..4512b81bb --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_CREATE.xml @@ -0,0 +1,297 @@ + + + + + + + 1 + 400 + 00 + 1 + 3 + + 1 + 0100000000 + 10 + 2021-01-01 + 0 + + + + + + + + + + 1 + 150 + 00 + 3 + 3 + + ABC + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 150 + 05 + 3 + 3 + + ABC + ZZ + Some Description x + + + + + + + + + + 1 + 285 + 00 + 3 + 3 + + 1 + Z0000001 + 2023-01-01 + ABCDE + 7 + 2023-01-01 + 1 + 7 + 0 + Some Info Text + 1 + ABC + 2021-01-01 + 2022-01-01 + + + + + + + + + + 1 + 400 + 15 + 1 + 3 + + 7 + 1 + 0100000000 + ZZ + 10 + Some Description + + + + + + + + 2 + 400 + 10 + 3 + 3 + + 7 + 1 + 0100000000 + 2021-01-01 + 10 + + + + + + + + + + 1 + 250 + 00 + 3 + 3 + + 8 + AB01 + 2021-01-01 + 2022-01-01 + 1 + + + + + + + + + + 1 + 250 + 10 + 3 + 3 + + 3 + zz + 8 + AB01 + Some Description + + + + + + + + 1 + 250 + 05 + 3 + 3 + + 3 + 2022-01-01 + 8 + AB01 + + + + + + + + + + 1 + 140 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + 6 + + + + + + + + 1 + 140 + 05 + 3 + 3 + + A + zz + Some Description + + + + + + + + + + 1 + 235 + 00 + 3 + 3 + + ZZZ + 2021-01-01 + 2022-01-01 + 1 + 2 + 3 + 4 + 5 + 6 + A + + + + + + + + 1 + 235 + 05 + 3 + 3 + + ZZZ + ZZ + Some Description x + + + + + + + + + + 1 + 430 + 00 + 3 + 3 + + 99 + ZZZ + AB01 + 8 + 0100000000 + 1 + 2021-01-01 + 2022-01-01 + 1 + Z0000001 + 1 + Z0000001 + 1 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_DELETE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_DELETE.xml new file mode 100644 index 000000000..85498699e --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_DELETE.xml @@ -0,0 +1,32 @@ + + + + + + + 1 + 430 + 00 + 3 + 2 + + 99 + ZZZ + AB01 + 8 + 0100000000 + 1 + 2021-01-11 + 2022-01-01 + 1 + Z0000001 + 1 + Z0000001 + 1 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_UPDATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_UPDATE.xml new file mode 100644 index 000000000..e1fb07915 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_UPDATE.xml @@ -0,0 +1,32 @@ + + + + + + + 1 + 430 + 00 + 3 + 1 + + 99 + ZZZ + AB01 + 8 + 0100000000 + 1 + 2021-01-11 + 2022-01-01 + 1 + Z0000001 + 1 + Z0000001 + 1 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_action_CREATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_action_CREATE.xml new file mode 100644 index 000000000..719a23192 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_action_CREATE.xml @@ -0,0 +1,39 @@ + + + + + + + 1 + 355 + 00 + 3 + 3 + + ABC + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 355 + 05 + 3 + 3 + + ABC + zz + Some Description + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_action_DELETE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_action_DELETE.xml new file mode 100644 index 000000000..543f35ca1 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_action_DELETE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 355 + 00 + 3 + 2 + + ABC + 2021-01-22 + 2022-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_action_UPDATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_action_UPDATE.xml new file mode 100644 index 000000000..45a602c34 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_action_UPDATE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 355 + 00 + 3 + 1 + + ABC + 2021-01-22 + 2022-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_action_description_CREATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_action_description_CREATE.xml new file mode 100644 index 000000000..1cfc4de04 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_action_description_CREATE.xml @@ -0,0 +1,39 @@ + + + + + + + 1 + 355 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 355 + 05 + 3 + 3 + + A + zz + Some Description + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_action_description_DELETE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_action_description_DELETE.xml new file mode 100644 index 000000000..db1d40e2c --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_action_description_DELETE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 355 + 05 + 3 + 2 + + A + zz + Some Description Changed + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_action_description_UPDATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_action_description_UPDATE.xml new file mode 100644 index 000000000..41f4e1e35 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_action_description_UPDATE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 355 + 05 + 3 + 1 + + A + zz + Some Description Changed + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_component_CREATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_component_CREATE.xml new file mode 100644 index 000000000..c96d23874 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_component_CREATE.xml @@ -0,0 +1,484 @@ + + + + + + + 1 + 400 + 00 + 1 + 3 + + 1 + 0100000000 + 10 + 2021-01-01 + 0 + + + + + + + + + + 1 + 150 + 00 + 3 + 3 + + ABC + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 150 + 05 + 3 + 3 + + ABC + ZZ + Some Description x + + + + + + + + + + 1 + 285 + 00 + 3 + 3 + + 1 + Z0000001 + 2023-01-01 + ABCDE + 7 + 2023-01-01 + 1 + 7 + 0 + Some Info Text + 1 + ABC + 2021-01-01 + 2022-01-01 + + + + + + + + + + 1 + 400 + 15 + 1 + 3 + + 7 + 1 + 0100000000 + ZZ + 10 + Some Description + + + + + + + + 2 + 400 + 10 + 3 + 3 + + 7 + 1 + 0100000000 + 2021-01-01 + 10 + + + + + + + + + + 1 + 250 + 00 + 3 + 3 + + 8 + AB01 + 2021-01-01 + 2022-01-01 + 1 + + + + + + + + + + 1 + 250 + 10 + 3 + 3 + + 3 + zz + 8 + AB01 + Some Description + + + + + + + + 1 + 250 + 05 + 3 + 3 + + 3 + 2022-01-01 + 8 + AB01 + + + + + + + + + + 1 + 230 + 00 + 3 + 3 + + 7 + 2021-01-01 + 2022-01-01 + 8 + 9 + 10 + + + + + + + + 1 + 230 + 05 + 3 + 3 + + 7 + ZZZ + Some Description s + + + + + + + + + + 1 + 140 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + 6 + + + + + + + + 1 + 140 + 05 + 3 + 3 + + A + zz + Some Description + + + + + + + + + + 1 + 235 + 00 + 3 + 3 + + ZZZ + 2021-01-01 + 2022-01-01 + 1 + 2 + 3 + 4 + 5 + 6 + A + + + + + + + + 1 + 235 + 05 + 3 + 3 + + ZZZ + ZZ + Some Description x + + + + + + + + + + 1 + 430 + 00 + 3 + 3 + + 99 + ZZZ + AB01 + 8 + 0100000000 + 1 + 2021-01-01 + 2022-01-01 + 1 + Z0000001 + 1 + Z0000001 + 1 + + + + + + + + + + 1 + 225 + 00 + 3 + 3 + + ZZZ + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 225 + 05 + 3 + 3 + + ZZZ + zz + Some Description + + + + + + + + + + 1 + 210 + 00 + 3 + 3 + + XYZ + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 210 + 05 + 3 + 3 + + XYZ + zz + Some Description + + + + + + + + + + 1 + 215 + 00 + 3 + 3 + + F + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 215 + 05 + 3 + 3 + + F + ZZ + Some description + + + + + + + + 1 + 220 + 00 + 3 + 3 + + F + XYZ + 2021-01-01 + 2022-01-01 + + + + + + + + + + 1 + 430 + 05 + 3 + 3 + + 99 + 7 + 12.77 + ZZZ + XYZ + F + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_component_DELETE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_component_DELETE.xml new file mode 100644 index 000000000..924ffc7c9 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_component_DELETE.xml @@ -0,0 +1,25 @@ + + + + + + + 1 + 430 + 05 + 3 + 2 + + 99 + 7 + 17.5 + ZZZ + XYZ + F + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_component_UPDATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_component_UPDATE.xml new file mode 100644 index 000000000..a3128a7d2 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_component_UPDATE.xml @@ -0,0 +1,25 @@ + + + + + + + 1 + 430 + 05 + 3 + 1 + + 99 + 7 + 17.5 + ZZZ + XYZ + F + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_CREATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_CREATE.xml new file mode 100644 index 000000000..daa452f9e --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_CREATE.xml @@ -0,0 +1,708 @@ + + + + + + + 1 + 400 + 00 + 1 + 3 + + 1 + 0100000000 + 10 + 2021-01-01 + 0 + + + + + + + + + + 1 + 150 + 00 + 3 + 3 + + ABC + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 150 + 05 + 3 + 3 + + ABC + ZZ + Some Description x + + + + + + + + + + 1 + 285 + 00 + 3 + 3 + + 1 + Z0000001 + 2023-01-01 + ABCDE + 7 + 2023-01-01 + 1 + 7 + 0 + Some Info Text + 1 + ABC + 2021-01-01 + 2022-01-01 + + + + + + + + + + 1 + 400 + 15 + 1 + 3 + + 7 + 1 + 0100000000 + ZZ + 10 + Some Description + + + + + + + + 2 + 400 + 10 + 3 + 3 + + 7 + 1 + 0100000000 + 2021-01-01 + 10 + + + + + + + + + + 1 + 250 + 00 + 3 + 3 + + 8 + AB01 + 2021-01-01 + 2022-01-01 + 1 + + + + + + + + + + 1 + 250 + 10 + 3 + 3 + + 3 + zz + 8 + AB01 + Some Description + + + + + + + + 1 + 250 + 05 + 3 + 3 + + 3 + 2022-01-01 + 8 + AB01 + + + + + + + + + + 1 + 230 + 00 + 3 + 3 + + 7 + 2021-01-01 + 2022-01-01 + 8 + 9 + 10 + + + + + + + + 1 + 230 + 05 + 3 + 3 + + 7 + ZZZ + Some Description s + + + + + + + + + + 1 + 140 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + 6 + + + + + + + + 1 + 140 + 05 + 3 + 3 + + A + zz + Some Description + + + + + + + + + + 1 + 235 + 00 + 3 + 3 + + ZZZ + 2021-01-01 + 2022-01-01 + 1 + 2 + 3 + 4 + 5 + 6 + A + + + + + + + + 1 + 235 + 05 + 3 + 3 + + ZZZ + ZZ + Some Description x + + + + + + + + + + 1 + 430 + 00 + 3 + 3 + + 99 + ZZZ + AB01 + 8 + 0100000000 + 1 + 2021-01-01 + 2022-01-01 + 1 + Z0000001 + 1 + Z0000001 + 1 + + + + + + + + + + 1 + 225 + 00 + 3 + 3 + + ZZZ + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 225 + 05 + 3 + 3 + + ZZZ + zz + Some Description + + + + + + + + + + 1 + 210 + 00 + 3 + 3 + + XYZ + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 210 + 05 + 3 + 3 + + XYZ + zz + Some Description + + + + + + + + + + 1 + 215 + 00 + 3 + 3 + + F + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 215 + 05 + 3 + 3 + + F + ZZ + Some description + + + + + + + + + + 1 + 430 + 05 + 3 + 3 + + 99 + 7 + 12.77 + ZZZ + XYZ + F + + + + + + + + + + 1 + 350 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 350 + 05 + 3 + 3 + + A + ZZ + Some Description + + + + + + + + + + 1 + 355 + 00 + 3 + 3 + + ABC + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 355 + 05 + 3 + 3 + + ABC + zz + Some Description + + + + + + + + + + 1 + 110 + 00 + 1 + 3 + + A + 2021-01-01 + 2021-12-31 + + + + + + + + 1 + 110 + 05 + 1 + 3 + + A + EN + some description + + + + + + + + 1 + 205 + 00 + 1 + 3 + + A + 123 + 2021-01-01 + 2021-12-31 + + + + + + + + + + 1 + 215 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 215 + 05 + 3 + 3 + + A + ZZ + Some description + + + + + + + + + + 1 + 210 + 00 + 3 + 3 + + XXX + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 210 + 05 + 3 + 3 + + XXX + zz + Some Description + + + + + + + + + + 1 + 220 + 00 + 3 + 3 + + A + XXX + 2021-01-01 + 2022-01-01 + + + + + + + + + + 1 + 430 + 10 + 3 + 3 + + 5 + 99 + A + 5 + 12.77 + ZZZ + XXX + A + ABC + A + 123 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_DELETE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_DELETE.xml new file mode 100644 index 000000000..0bb076e1d --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_DELETE.xml @@ -0,0 +1,30 @@ + + + + + + + 1 + 430 + 10 + 3 + 2 + + 5 + 99 + A + 5 + 99.99 + ZZZ + XXX + A + ABC + A + 123 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_UPDATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_UPDATE.xml new file mode 100644 index 000000000..375c013c5 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_UPDATE.xml @@ -0,0 +1,30 @@ + + + + + + + 1 + 430 + 10 + 3 + 1 + + 5 + 99 + A + 5 + 99.99 + ZZZ + XXX + A + ABC + A + 123 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_code_CREATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_code_CREATE.xml new file mode 100644 index 000000000..cffb6446a --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_code_CREATE.xml @@ -0,0 +1,39 @@ + + + + + + + 1 + 350 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 350 + 05 + 3 + 3 + + A + ZZ + Some Description + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_code_DELETE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_code_DELETE.xml new file mode 100644 index 000000000..fd74700a7 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_code_DELETE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 350 + 00 + 3 + 2 + + A + 2021-01-11 + 2022-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_code_UPDATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_code_UPDATE.xml new file mode 100644 index 000000000..933316bb8 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_code_UPDATE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 350 + 00 + 3 + 1 + + A + 2021-01-11 + 2022-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_code_description_CREATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_code_description_CREATE.xml new file mode 100644 index 000000000..cffb6446a --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_code_description_CREATE.xml @@ -0,0 +1,39 @@ + + + + + + + 1 + 350 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 350 + 05 + 3 + 3 + + A + ZZ + Some Description + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_code_description_DELETE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_code_description_DELETE.xml new file mode 100644 index 000000000..a607a5738 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_code_description_DELETE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 350 + 05 + 3 + 2 + + A + ZZ + Some Description with changes + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_code_description_UPDATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_code_description_UPDATE.xml new file mode 100644 index 000000000..69c22ae2a --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_code_description_UPDATE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 350 + 05 + 3 + 1 + + A + ZZ + Some Description with changes + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_component_CREATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_component_CREATE.xml new file mode 100644 index 000000000..31d3cc03e --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_component_CREATE.xml @@ -0,0 +1,730 @@ + + + + + + + 1 + 400 + 00 + 1 + 3 + + 1 + 0100000000 + 10 + 2021-01-01 + 0 + + + + + + + + + + 1 + 150 + 00 + 3 + 3 + + ABC + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 150 + 05 + 3 + 3 + + ABC + ZZ + Some Description x + + + + + + + + + + 1 + 285 + 00 + 3 + 3 + + 1 + Z0000001 + 2023-01-01 + ABCDE + 7 + 2023-01-01 + 1 + 7 + 0 + Some Info Text + 1 + ABC + 2021-01-01 + 2022-01-01 + + + + + + + + + + 1 + 400 + 15 + 1 + 3 + + 7 + 1 + 0100000000 + ZZ + 10 + Some Description + + + + + + + + 2 + 400 + 10 + 3 + 3 + + 7 + 1 + 0100000000 + 2021-01-01 + 10 + + + + + + + + + + 1 + 250 + 00 + 3 + 3 + + 8 + AB01 + 2021-01-01 + 2022-01-01 + 1 + + + + + + + + + + 1 + 250 + 10 + 3 + 3 + + 3 + zz + 8 + AB01 + Some Description + + + + + + + + 1 + 250 + 05 + 3 + 3 + + 3 + 2022-01-01 + 8 + AB01 + + + + + + + + + + 1 + 230 + 00 + 3 + 3 + + 7 + 2021-01-01 + 2022-01-01 + 8 + 9 + 10 + + + + + + + + 1 + 230 + 05 + 3 + 3 + + 7 + ZZZ + Some Description s + + + + + + + + + + 1 + 140 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + 6 + + + + + + + + 1 + 140 + 05 + 3 + 3 + + A + zz + Some Description + + + + + + + + + + 1 + 235 + 00 + 3 + 3 + + ZZZ + 2021-01-01 + 2022-01-01 + 1 + 2 + 3 + 4 + 5 + 6 + A + + + + + + + + 1 + 235 + 05 + 3 + 3 + + ZZZ + ZZ + Some Description x + + + + + + + + + + 1 + 430 + 00 + 3 + 3 + + 99 + ZZZ + AB01 + 8 + 0100000000 + 1 + 2021-01-01 + 2022-01-01 + 1 + Z0000001 + 1 + Z0000001 + 1 + + + + + + + + + + 1 + 225 + 00 + 3 + 3 + + ZZZ + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 225 + 05 + 3 + 3 + + ZZZ + zz + Some Description + + + + + + + + + + 1 + 210 + 00 + 3 + 3 + + XYZ + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 210 + 05 + 3 + 3 + + XYZ + zz + Some Description + + + + + + + + + + 1 + 215 + 00 + 3 + 3 + + F + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 215 + 05 + 3 + 3 + + F + ZZ + Some description + + + + + + + + + + 1 + 430 + 05 + 3 + 3 + + 99 + 7 + 12.77 + ZZZ + XYZ + F + + + + + + + + + + 1 + 350 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 350 + 05 + 3 + 3 + + A + ZZ + Some Description + + + + + + + + + + 1 + 355 + 00 + 3 + 3 + + ABC + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 355 + 05 + 3 + 3 + + ABC + zz + Some Description + + + + + + + + + + 1 + 110 + 00 + 1 + 3 + + A + 2021-01-01 + 2021-12-31 + + + + + + + + 1 + 110 + 05 + 1 + 3 + + A + EN + some description + + + + + + + + 1 + 205 + 00 + 1 + 3 + + A + 123 + 2021-01-01 + 2021-12-31 + + + + + + + + + + 1 + 215 + 00 + 3 + 3 + + B + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 215 + 05 + 3 + 3 + + B + ZZ + Some description + + + + + + + + + + 1 + 210 + 00 + 3 + 3 + + XXX + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 210 + 05 + 3 + 3 + + XXX + zz + Some Description + + + + + + + + + + 1 + 220 + 00 + 3 + 3 + + B + XXX + 2021-01-01 + 2022-01-01 + + + + + + + + + + 1 + 430 + 10 + 3 + 3 + + 5 + 99 + A + 5 + 12.77 + ZZZ + XXX + B + ABC + A + 123 + + + + + + + + + + 1 + 430 + 11 + 3 + 3 + + 5 + 7 + 14.5 + ZZZ + XXX + B + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_component_DELETE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_component_DELETE.xml new file mode 100644 index 000000000..2dfdc9b83 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_component_DELETE.xml @@ -0,0 +1,24 @@ + + + + + + + 1 + 430 + 11 + 3 + 2 + + 5 + 7 + 99.99 + ZZZ + XXX + B + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_component_UPDATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_component_UPDATE.xml new file mode 100644 index 000000000..088a84fe2 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_condition_component_UPDATE.xml @@ -0,0 +1,24 @@ + + + + + + + 1 + 430 + 11 + 3 + 1 + + 5 + 7 + 99.99 + ZZZ + XXX + B + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_excluded_geographical_area_CREATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_excluded_geographical_area_CREATE.xml new file mode 100644 index 000000000..6947bfcc8 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_excluded_geographical_area_CREATE.xml @@ -0,0 +1,316 @@ + + + + + + + 1 + 400 + 00 + 1 + 3 + + 1 + 0100000000 + 10 + 2021-01-01 + 0 + + + + + + + + + + 1 + 150 + 00 + 3 + 3 + + ABC + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 150 + 05 + 3 + 3 + + ABC + ZZ + Some Description x + + + + + + + + + + 1 + 285 + 00 + 3 + 3 + + 1 + Z0000001 + 2023-01-01 + ABCDE + 7 + 2023-01-01 + 1 + 7 + 0 + Some Info Text + 1 + ABC + 2021-01-01 + 2022-01-01 + + + + + + + + + + 1 + 400 + 15 + 1 + 3 + + 7 + 1 + 0100000000 + ZZ + 10 + Some Description + + + + + + + + 2 + 400 + 10 + 3 + 3 + + 7 + 1 + 0100000000 + 2021-01-01 + 10 + + + + + + + + + + 1 + 250 + 00 + 3 + 3 + + 8 + AB01 + 2021-01-01 + 2022-01-01 + 1 + + + + + + + + + + 1 + 250 + 10 + 3 + 3 + + 3 + zz + 8 + AB01 + Some Description + + + + + + + + 1 + 250 + 05 + 3 + 3 + + 3 + 2022-01-01 + 8 + AB01 + + + + + + + + + + 1 + 140 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + 6 + + + + + + + + 1 + 140 + 05 + 3 + 3 + + A + zz + Some Description + + + + + + + + + + 1 + 235 + 00 + 3 + 3 + + ZZZ + 2021-01-01 + 2022-01-01 + 1 + 2 + 3 + 4 + 5 + 6 + A + + + + + + + + 1 + 235 + 05 + 3 + 3 + + ZZZ + ZZ + Some Description x + + + + + + + + + + 1 + 430 + 00 + 3 + 3 + + 99 + ZZZ + AB01 + 8 + 0100000000 + 1 + 2021-01-01 + 2022-01-01 + 1 + Z0000001 + 1 + Z0000001 + 1 + + + + + + + + + + 1 + 430 + 15 + 3 + 3 + + 99 + AB01 + 8 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_excluded_geographical_area_DELETE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_excluded_geographical_area_DELETE.xml new file mode 100644 index 000000000..866152e32 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_excluded_geographical_area_DELETE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 430 + 15 + 3 + 2 + + 99 + AB01 + 8 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_excluded_geographical_area_UPDATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_excluded_geographical_area_UPDATE.xml new file mode 100644 index 000000000..a0f8c9147 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_excluded_geographical_area_UPDATE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 430 + 15 + 3 + 1 + + 99 + AB01 + 8 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_series_CREATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_series_CREATE.xml new file mode 100644 index 000000000..386024c71 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_series_CREATE.xml @@ -0,0 +1,40 @@ + + + + + + + 1 + 140 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + 6 + + + + + + + + 1 + 140 + 05 + 3 + 3 + + A + zz + Some Description + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_series_DELETE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_series_DELETE.xml new file mode 100644 index 000000000..ae05a8787 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_series_DELETE.xml @@ -0,0 +1,23 @@ + + + + + + + 1 + 140 + 00 + 3 + 2 + + A + 2021-01-11 + 2022-01-01 + 6 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_series_UPDATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_series_UPDATE.xml new file mode 100644 index 000000000..cdcc70cb2 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_series_UPDATE.xml @@ -0,0 +1,23 @@ + + + + + + + 1 + 140 + 00 + 3 + 1 + + A + 2021-01-11 + 2022-01-01 + 6 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_series_no_description_CREATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_series_no_description_CREATE.xml new file mode 100644 index 000000000..33e9609c2 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_series_no_description_CREATE.xml @@ -0,0 +1,23 @@ + + + + + + + 1 + 140 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + 6 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_type_CREATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_type_CREATE.xml new file mode 100644 index 000000000..a815a5151 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_type_CREATE.xml @@ -0,0 +1,83 @@ + + + + + + + 1 + 140 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + 6 + + + + + + + + 1 + 140 + 05 + 3 + 3 + + A + zz + Some Description + + + + + + + + + + 1 + 235 + 00 + 3 + 3 + + ZZZ + 2021-01-01 + 2022-01-01 + 1 + 2 + 3 + 4 + 5 + 6 + A + + + + + + + + 1 + 235 + 05 + 3 + 3 + + ZZZ + ZZ + Some Description x + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_type_DELETE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_type_DELETE.xml new file mode 100644 index 000000000..19646ee71 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_type_DELETE.xml @@ -0,0 +1,29 @@ + + + + + + + 1 + 235 + 00 + 3 + 2 + + ZZZ + 2021-01-11 + 2022-01-01 + 1 + 2 + 3 + 4 + 5 + 6 + A + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_type_UPDATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_type_UPDATE.xml new file mode 100644 index 000000000..2cea7472c --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_type_UPDATE.xml @@ -0,0 +1,29 @@ + + + + + + + 1 + 235 + 00 + 3 + 1 + + ZZZ + 2021-01-11 + 2022-01-01 + 1 + 2 + 3 + 4 + 5 + 6 + A + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_type_description_CREATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_type_description_CREATE.xml new file mode 100644 index 000000000..a815a5151 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_type_description_CREATE.xml @@ -0,0 +1,83 @@ + + + + + + + 1 + 140 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + 6 + + + + + + + + 1 + 140 + 05 + 3 + 3 + + A + zz + Some Description + + + + + + + + + + 1 + 235 + 00 + 3 + 3 + + ZZZ + 2021-01-01 + 2022-01-01 + 1 + 2 + 3 + 4 + 5 + 6 + A + + + + + + + + 1 + 235 + 05 + 3 + 3 + + ZZZ + ZZ + Some Description x + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_type_description_DELETE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_type_description_DELETE.xml new file mode 100644 index 000000000..d014f7eb9 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_type_description_DELETE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 235 + 05 + 3 + 2 + + ZZZ + ZZ + Some Description with changes + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_type_description_UPDATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_type_description_UPDATE.xml new file mode 100644 index 000000000..c7715d273 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_type_description_UPDATE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 235 + 05 + 3 + 1 + + ZZZ + ZZ + Some Description with changes + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_type_series_description_CREATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_type_series_description_CREATE.xml new file mode 100644 index 000000000..386024c71 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_type_series_description_CREATE.xml @@ -0,0 +1,40 @@ + + + + + + + 1 + 140 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + 6 + + + + + + + + 1 + 140 + 05 + 3 + 3 + + A + zz + Some Description + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_type_series_description_DELETE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_type_series_description_DELETE.xml new file mode 100644 index 000000000..a932c8e24 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_type_series_description_DELETE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 140 + 05 + 3 + 2 + + A + zz + Some Description with changes + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measure_type_series_description_UPDATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measure_type_series_description_UPDATE.xml new file mode 100644 index 000000000..87c75993a --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measure_type_series_description_UPDATE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 140 + 05 + 3 + 1 + + A + zz + Some Description with changes + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measurement_CREATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measurement_CREATE.xml new file mode 100644 index 000000000..7ae746a3f --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measurement_CREATE.xml @@ -0,0 +1,95 @@ + + + + + + + 1 + 215 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 215 + 05 + 3 + 3 + + A + ZZ + Some description + + + + + + + + + + 1 + 210 + 00 + 3 + 3 + + XXX + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 210 + 05 + 3 + 3 + + XXX + zz + Some Description + + + + + + + + + + 1 + 220 + 00 + 3 + 3 + + A + XXX + 2021-01-01 + 2022-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measurement_DELETE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measurement_DELETE.xml new file mode 100644 index 000000000..152a8bef5 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measurement_DELETE.xml @@ -0,0 +1,23 @@ + + + + + + + 1 + 220 + 00 + 3 + 2 + + A + XXX + 2021-01-11 + 2022-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measurement_UPDATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measurement_UPDATE.xml new file mode 100644 index 000000000..925ea190c --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measurement_UPDATE.xml @@ -0,0 +1,23 @@ + + + + + + + 1 + 220 + 00 + 3 + 1 + + A + XXX + 2021-01-11 + 2022-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_CREATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_CREATE.xml new file mode 100644 index 000000000..d11a45276 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_CREATE.xml @@ -0,0 +1,39 @@ + + + + + + + 1 + 210 + 00 + 3 + 3 + + XXX + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 210 + 05 + 3 + 3 + + XXX + zz + Some Description + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_DELETE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_DELETE.xml new file mode 100644 index 000000000..bb41caaca --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_DELETE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 210 + 00 + 3 + 2 + + XXX + 2021-01-11 + 2022-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_UPDATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_UPDATE.xml new file mode 100644 index 000000000..bb958d1d9 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_UPDATE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 210 + 00 + 3 + 1 + + XXX + 2021-01-11 + 2022-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_description_CREATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_description_CREATE.xml new file mode 100644 index 000000000..d11a45276 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_description_CREATE.xml @@ -0,0 +1,39 @@ + + + + + + + 1 + 210 + 00 + 3 + 3 + + XXX + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 210 + 05 + 3 + 3 + + XXX + zz + Some Description + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_description_DELETE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_description_DELETE.xml new file mode 100644 index 000000000..61a5446d5 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_description_DELETE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 210 + 05 + 3 + 1 + + XXX + zz + Some Description with changes + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_description_UPDATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_description_UPDATE.xml new file mode 100644 index 000000000..61a5446d5 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_description_UPDATE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 210 + 05 + 3 + 1 + + XXX + zz + Some Description with changes + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_qualifier_CREATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_qualifier_CREATE.xml new file mode 100644 index 000000000..bd8b9018b --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_qualifier_CREATE.xml @@ -0,0 +1,39 @@ + + + + + + + 1 + 215 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 215 + 05 + 3 + 3 + + A + ZZ + Some description + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_qualifier_DELETE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_qualifier_DELETE.xml new file mode 100644 index 000000000..cbc47d2fc --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_qualifier_DELETE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 215 + 00 + 3 + 2 + + A + 2021-01-11 + 2022-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_qualifier_UPDATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_qualifier_UPDATE.xml new file mode 100644 index 000000000..576e76db9 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_qualifier_UPDATE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 215 + 00 + 3 + 1 + + A + 2021-01-11 + 2022-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_qualifier_description_CREATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_qualifier_description_CREATE.xml new file mode 100644 index 000000000..eeb5c8762 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_qualifier_description_CREATE.xml @@ -0,0 +1,39 @@ + + + + + + + 1 + 215 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 215 + 05 + 3 + 3 + + A + ZZ + Some Description + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_qualifier_description_DELETE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_qualifier_description_DELETE.xml new file mode 100644 index 000000000..ba2eb091a --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_qualifier_description_DELETE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 215 + 05 + 3 + 2 + + A + ZZ + Some Description with changes + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_qualifier_description_UPDATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_qualifier_description_UPDATE.xml new file mode 100644 index 000000000..2e249a80a --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/measurement_unit_qualifier_description_UPDATE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 215 + 05 + 3 + 1 + + A + ZZ + Some Description with changes + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/monetary_unit_CREATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/monetary_unit_CREATE.xml new file mode 100644 index 000000000..697d03bc9 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/monetary_unit_CREATE.xml @@ -0,0 +1,40 @@ + + + + + + + 1 + 225 + 00 + 3 + 3 + + ZZZ + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 225 + 05 + 3 + 3 + + ZZZ + zz + Some Description + + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/monetary_unit_DELETE.xml b/taric_parsers/tests/measure_parsers/importer_examples/monetary_unit_DELETE.xml new file mode 100644 index 000000000..5ff8a5319 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/monetary_unit_DELETE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 225 + 00 + 3 + 2 + + ZZZ + 2021-01-11 + 2022-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/monetary_unit_UPDATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/monetary_unit_UPDATE.xml new file mode 100644 index 000000000..cb7f4fe1e --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/monetary_unit_UPDATE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 225 + 00 + 3 + 1 + + ZZZ + 2021-01-11 + 2022-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/monetary_unit_description_CREATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/monetary_unit_description_CREATE.xml new file mode 100644 index 000000000..697d03bc9 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/monetary_unit_description_CREATE.xml @@ -0,0 +1,40 @@ + + + + + + + 1 + 225 + 00 + 3 + 3 + + ZZZ + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 225 + 05 + 3 + 3 + + ZZZ + zz + Some Description + + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/monetary_unit_description_DELETE.xml b/taric_parsers/tests/measure_parsers/importer_examples/monetary_unit_description_DELETE.xml new file mode 100644 index 000000000..1f9543ac3 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/monetary_unit_description_DELETE.xml @@ -0,0 +1,23 @@ + + + + + + + 1 + 225 + 05 + 3 + 2 + + ZZZ + zz + Some Description with changes + + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/importer_examples/monetary_unit_description_UPDATE.xml b/taric_parsers/tests/measure_parsers/importer_examples/monetary_unit_description_UPDATE.xml new file mode 100644 index 000000000..817bf3320 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/importer_examples/monetary_unit_description_UPDATE.xml @@ -0,0 +1,23 @@ + + + + + + + 1 + 225 + 05 + 3 + 1 + + ZZZ + zz + Some Description with changes + + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/measure_parsers/test_additional_code_type_measure_type_parser.py b/taric_parsers/tests/measure_parsers/test_additional_code_type_measure_type_parser.py new file mode 100644 index 000000000..882386dc2 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/test_additional_code_type_measure_type_parser.py @@ -0,0 +1,113 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from measures.models import AdditionalCodeTypeMeasureType +from taric_parsers.parsers.additional_code_parsers import * +from taric_parsers.parsers.geo_area_parser import * +from taric_parsers.parsers.measure_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestAdditionalCodeTypeMeasureTypeParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + """ + + target_parser_class = AdditionalCodeTypeMeasureTypeParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "measure_type_id": "A", + "additional_code_type_id": "Z", + "validity_start_date": "2021-01-01", + "validity_end_date": "2022-01-01", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.measure_type__sid == "A" + assert target.additional_code_type__sid == "Z" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + + def test_import(self, superuser): + importer = preload_import( + "additional_code_type_measure_type_CREATE.xml", + __file__, + ) + + assert len(importer.parsed_transactions) == 4 + + target_message = importer.parsed_transactions[3].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + # verify all properties + assert target.measure_type__sid == "ZZZ" + assert target.additional_code_type__sid == "Z" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + + assert len(importer.issues()) == 0 + + assert AdditionalCodeTypeMeasureType.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("additional_code_type_measure_type_CREATE.xml", __file__, True) + importer = preload_import( + "additional_code_type_measure_type_UPDATE.xml", + __file__, + ) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + target = target_message.taric_object + + # verify all properties + assert target.measure_type__sid == "ZZZ" + assert target.additional_code_type__sid == "Z" + assert target.valid_between_lower == date(2021, 1, 21) + assert target.valid_between_upper == date(2022, 1, 1) + + assert importer.issues() == [] + + assert AdditionalCodeTypeMeasureType.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("additional_code_type_measure_type_CREATE.xml", __file__, True) + importer = preload_import( + "additional_code_type_measure_type_DELETE.xml", + __file__, + ) + + assert importer.can_save() + assert importer.issues() == [] + + assert AdditionalCodeTypeMeasureType.objects.all().count() == 2 diff --git a/taric_parsers/tests/measure_parsers/test_duty_expression_description_parser.py b/taric_parsers/tests/measure_parsers/test_duty_expression_description_parser.py new file mode 100644 index 000000000..50e7d6e18 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/test_duty_expression_description_parser.py @@ -0,0 +1,99 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from measures.models import DutyExpression +from taric_parsers.parsers.measure_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestDutyExpressionDescriptionParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + """ + + target_parser_class = DutyExpressionDescriptionParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "duty_expression_id": "7", + "language_id": "2022-01-01", + "description": "Some Description", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + + assert target.sid == 7 + assert target.description == "Some Description" + + def test_import(self, superuser): + importer = preload_import("duty_expression_description_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[1] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.sid == 7 + assert target.description == "Some Description s" + + assert len(importer.issues()) == 0 + + assert DutyExpression.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("duty_expression_description_CREATE.xml", __file__, True) + importer = preload_import("duty_expression_description_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target = target_message.taric_object + + assert target.sid == 7 + assert target.description == "Some Description that changed" + + assert len(importer.issues()) == 0 + + assert DutyExpression.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("duty_expression_description_CREATE.xml", __file__, True) + importer = preload_import("duty_expression_description_DELETE.xml", __file__) + + # whilst there is an issue, it's only a warning, so it will be able to save + assert importer.can_save() + + assert len(importer.issues()) == 1 + + assert ( + "Children of Taric objects of type DutyExpression can't be deleted directly" + in str(importer.issues()[0]) + ) diff --git a/taric_parsers/tests/measure_parsers/test_duty_expression_parser.py b/taric_parsers/tests/measure_parsers/test_duty_expression_parser.py new file mode 100644 index 000000000..6c4aea654 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/test_duty_expression_parser.py @@ -0,0 +1,113 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from measures.models import DutyExpression +from taric_parsers.parsers.geo_area_parser import * +from taric_parsers.parsers.measure_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestDutyExpressionParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + + """ + + target_parser_class = DutyExpressionParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "duty_expression_id": "7", + "validity_start_date": "2021-01-01", + "validity_end_date": "2022-01-01", + "duty_amount_applicability_code": "8", + "measurement_unit_applicability_code": "9", + "monetary_unit_applicability_code": "10", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + + assert target.sid == 7 + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + assert target.duty_amount_applicability_code == 8 + assert target.measurement_unit_applicability_code == 9 + assert target.monetary_unit_applicability_code == 10 + + def test_import(self, superuser): + importer = preload_import("duty_expression_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.sid == 7 + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + assert target.duty_amount_applicability_code == 8 + assert target.measurement_unit_applicability_code == 9 + assert target.monetary_unit_applicability_code == 10 + + assert len(importer.issues()) == 0 + + assert DutyExpression.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("duty_expression_CREATE.xml", __file__, True) + importer = preload_import("duty_expression_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target = target_message.taric_object + + assert target.sid == 7 + assert target.valid_between_lower == date(2021, 1, 21) + assert target.valid_between_upper == date(2022, 1, 1) + assert target.duty_amount_applicability_code == 8 + assert target.measurement_unit_applicability_code == 9 + assert target.monetary_unit_applicability_code == 10 + + assert len(importer.issues()) == 0 + + assert DutyExpression.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("duty_expression_CREATE.xml", __file__, True) + importer = preload_import("duty_expression_DELETE.xml", __file__) + + assert len(importer.issues()) == 0 + assert importer.issues() == [] + + assert DutyExpression.objects.all().count() == 2 diff --git a/taric_parsers/tests/measure_parsers/test_footnote_association_measure_parser.py b/taric_parsers/tests/measure_parsers/test_footnote_association_measure_parser.py new file mode 100644 index 000000000..854d23174 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/test_footnote_association_measure_parser.py @@ -0,0 +1,91 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from measures.models import FootnoteAssociationMeasure +from taric_parsers.parsers.measure_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestFootnoteAssociationMeasureParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + """ + + target_parser_class = FootnoteAssociationMeasureParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "measure_sid": "1", + "footnote_type_id": "AA", + "footnote_id": "BBB", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.footnoted_measure__sid == 1 + assert target.associated_footnote__footnote_type__footnote_type_id == "AA" + assert target.associated_footnote__footnote_id == "BBB" + + def test_import(self, superuser): + importer = preload_import("footnote_association_measure_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 11 + + target_message = importer.parsed_transactions[10].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.footnoted_measure__sid == 99 + assert target.associated_footnote__footnote_type__footnote_type_id == "3" + assert target.associated_footnote__footnote_id == "9" + + assert len(importer.issues()) == 0 + + assert FootnoteAssociationMeasure.objects.all().count() == 1 + + def test_import_update_raises_issue(self, superuser): + preload_import("footnote_association_measure_CREATE.xml", __file__, True) + importer = preload_import("footnote_association_measure_UPDATE.xml", __file__) + + assert len(importer.issues()) == 1 + + assert ( + "Taric objects of type FootnoteAssociationMeasure can't be updated" + in str(importer.issues()[0]) + ) + + def test_import_delete(self, superuser): + preload_import("footnote_association_measure_CREATE.xml", __file__, True) + importer = preload_import("footnote_association_measure_DELETE.xml", __file__) + + assert len(importer.issues()) == 0 + assert importer.can_save() + + assert FootnoteAssociationMeasure.objects.all().count() == 2 diff --git a/taric_parsers/tests/measure_parsers/test_measure_action_description_parser.py b/taric_parsers/tests/measure_parsers/test_measure_action_description_parser.py new file mode 100644 index 000000000..91e3ef937 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/test_measure_action_description_parser.py @@ -0,0 +1,95 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from measures.models import MeasureAction +from taric_parsers.parsers.measure_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestMeasureActionDescriptionParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + """ + + target_parser_class = MeasureActionDescriptionParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "action_code": "01", + "language_id": "ZZ", + "description": "Some Description", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.code == "01" + assert target.description == "Some Description" + + def test_import(self, superuser): + importer = preload_import("measure_action_description_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[1] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.code == "A" + assert target.description == "Some Description" + + assert len(importer.issues()) == 0 + + assert MeasureAction.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("measure_action_description_CREATE.xml", __file__, True) + importer = preload_import("measure_action_description_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target = target_message.taric_object + + assert target.code == "A" + assert target.description == "Some Description Changed" + + assert importer.issues() == [] + + assert MeasureAction.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("measure_action_description_CREATE.xml", __file__, True) + importer = preload_import("measure_action_description_DELETE.xml", __file__) + + assert importer.can_save() + + assert ( + "Children of Taric objects of type MeasureAction can't be deleted directly" + in str(importer.issues()[0]) + ) diff --git a/taric_parsers/tests/measure_parsers/test_measure_action_parser.py b/taric_parsers/tests/measure_parsers/test_measure_action_parser.py new file mode 100644 index 000000000..fb7f406bd --- /dev/null +++ b/taric_parsers/tests/measure_parsers/test_measure_action_parser.py @@ -0,0 +1,97 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from measures.models import MeasureAction +from taric_parsers.parsers.geo_area_parser import * +from taric_parsers.parsers.measure_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestMeasureActionParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + """ + + target_parser_class = MeasureActionParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "action_code": "ABC", + "validity_start_date": "2021-01-01", + "validity_end_date": "2022-01-01", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.code == "ABC" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + + def test_import(self, superuser): + importer = preload_import("measure_action_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.code == "ABC" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + + assert len(importer.issues()) == 0 + + assert MeasureAction.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("measure_action_CREATE.xml", __file__, True) + importer = preload_import("measure_action_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target = target_message.taric_object + + assert target.code == "ABC" + assert target.valid_between_lower == date(2021, 1, 22) + assert target.valid_between_upper == date(2022, 1, 1) + + assert importer.issues() == [] + + assert MeasureAction.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("measure_action_CREATE.xml", __file__, True) + importer = preload_import("measure_action_DELETE.xml", __file__) + + assert importer.issues() == [] + assert importer.can_save() + + assert MeasureAction.objects.all().count() == 2 diff --git a/taric_parsers/tests/measure_parsers/test_measure_component_parser.py b/taric_parsers/tests/measure_parsers/test_measure_component_parser.py new file mode 100644 index 000000000..b84b8f909 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/test_measure_component_parser.py @@ -0,0 +1,113 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from measures.models import MeasureComponent +from taric_parsers.parsers.commodity_parser import * +from taric_parsers.parsers.geo_area_parser import * +from taric_parsers.parsers.measure_parser import * +from taric_parsers.parsers.regulation_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestMeasureComponentParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + + """ + + target_parser_class = MeasureComponentParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "measure_sid": "5", + "duty_expression_id": "4", + "duty_amount": "12.93", + "monetary_unit_code": "ABC", + "measurement_unit_code": "3", + "measurement_unit_qualifier_code": "2", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.component_measure__sid == 5 + assert target.duty_expression__sid == 4 + assert target.duty_amount == 12.93 + assert target.monetary_unit__code == "ABC" + assert target.component_measurement__measurement_unit__code == "3" + assert target.component_measurement__measurement_unit_qualifier__code == "2" + + def test_import(self, superuser): + importer = preload_import("measure_component_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 14 + + target_message = importer.parsed_transactions[13].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.component_measure__sid == 99 + assert target.duty_expression__sid == 7 + assert target.duty_amount == 12.77 + assert target.monetary_unit__code == "ZZZ" + assert target.component_measurement__measurement_unit__code == "XYZ" + assert target.component_measurement__measurement_unit_qualifier__code == "F" + + assert len(importer.issues()) == 0 + + assert MeasureComponent.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("measure_component_CREATE.xml", __file__, True) + importer = preload_import("measure_component_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + target = target_message.taric_object + + assert target.component_measure__sid == 99 + assert target.duty_expression__sid == 7 + assert target.duty_amount == 17.5 + assert target.monetary_unit__code == "ZZZ" + assert target.component_measurement__measurement_unit__code == "XYZ" + assert target.component_measurement__measurement_unit_qualifier__code == "F" + + assert len(importer.issues()) == 0 + + assert MeasureComponent.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("measure_component_CREATE.xml", __file__, True) + importer = preload_import("measure_component_DELETE.xml", __file__) + + assert len(importer.issues()) == 0 + assert importer.can_save() + + assert MeasureComponent.objects.all().count() == 2 diff --git a/taric_parsers/tests/measure_parsers/test_measure_condition_code_description_parser.py b/taric_parsers/tests/measure_parsers/test_measure_condition_code_description_parser.py new file mode 100644 index 000000000..14b23134f --- /dev/null +++ b/taric_parsers/tests/measure_parsers/test_measure_condition_code_description_parser.py @@ -0,0 +1,107 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from measures.models import MeasureConditionCode +from taric_parsers.parsers.measure_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestMeasureConditionCodeDescriptionParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + """ + + target_parser_class = MeasureConditionCodeDescriptionParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "condition_code": "A", + "language_id": "ZZ", + "description": "Some Description", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.code == "A" + assert target.description == "Some Description" + + def test_import(self, superuser): + importer = preload_import( + "measure_condition_code_description_CREATE.xml", + __file__, + ) + + assert len(importer.parsed_transactions) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[1] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + # verify all properties + assert target.code == "A" + assert target.description == "Some Description" + + assert len(importer.issues()) == 0 + + assert MeasureConditionCode.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("measure_condition_code_description_CREATE.xml", __file__, True) + importer = preload_import( + "measure_condition_code_description_UPDATE.xml", + __file__, + ) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + target = target_message.taric_object + + # verify all properties + assert target.code == "A" + assert target.description == "Some Description with changes" + + assert len(importer.issues()) == 0 + + assert MeasureConditionCode.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("measure_condition_code_description_CREATE.xml", __file__, True) + importer = preload_import( + "measure_condition_code_description_DELETE.xml", + __file__, + ) + + assert importer.can_save() + + assert len(importer.issues()) == 1 + + assert ( + "Children of Taric objects of type MeasureConditionCode can't be deleted directly" + in str(importer.issues()[0]) + ) diff --git a/taric_parsers/tests/measure_parsers/test_measure_condition_code_parser.py b/taric_parsers/tests/measure_parsers/test_measure_condition_code_parser.py new file mode 100644 index 000000000..d176bd200 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/test_measure_condition_code_parser.py @@ -0,0 +1,97 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from measures.models import MeasureConditionCode +from taric_parsers.parsers.geo_area_parser import * +from taric_parsers.parsers.measure_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestMeasureConditionCodeParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + """ + + target_parser_class = MeasureConditionCodeParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "condition_code": "A", + "validity_start_date": "2021-01-01", + "validity_end_date": "2022-01-01", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.code == "A" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + + def test_import(self, superuser): + importer = preload_import("measure_condition_code_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.code == "A" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + + assert len(importer.issues()) == 0 + + assert MeasureConditionCode.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("measure_condition_code_CREATE.xml", __file__, True) + importer = preload_import("measure_condition_code_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target = target_message.taric_object + + assert target.code == "A" + assert target.valid_between_lower == date(2021, 1, 11) + assert target.valid_between_upper == date(2022, 1, 1) + + assert len(importer.issues()) == 0 + + assert MeasureConditionCode.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("measure_condition_code_CREATE.xml", __file__, True) + importer = preload_import("measure_condition_code_DELETE.xml", __file__) + + assert len(importer.issues()) == 0 + assert importer.can_save() + + assert MeasureConditionCode.objects.all().count() == 2 diff --git a/taric_parsers/tests/measure_parsers/test_measure_condition_component_parser.py b/taric_parsers/tests/measure_parsers/test_measure_condition_component_parser.py new file mode 100644 index 000000000..5d8fc2fad --- /dev/null +++ b/taric_parsers/tests/measure_parsers/test_measure_condition_component_parser.py @@ -0,0 +1,111 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from measures.models import MeasureConditionComponent +from taric_parsers.parsers.measure_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestMeasureConditionComponentParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + + """ + + target_parser_class = MeasureConditionComponentParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "measure_condition_sid": "1", + "duty_expression_id": "77", + "duty_amount": "12.7", + "monetary_unit_code": "ABC", + "measurement_unit_code": "CDE", + "measurement_unit_qualifier_code": "X", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.condition__sid == 1 + assert target.duty_expression__sid == 77 + assert target.duty_amount == 12.7 + assert target.monetary_unit__code == "ABC" + assert target.component_measurement__measurement_unit__code == "CDE" + assert target.component_measurement__measurement_unit_qualifier__code == "X" + + def test_import(self, superuser): + importer = preload_import("measure_condition_component_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 22 + + target_message = importer.parsed_transactions[21].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.condition__sid == 5 + assert target.duty_expression__sid == 7 + assert target.duty_amount == 14.5 + assert target.monetary_unit__code == "ZZZ" + assert target.component_measurement__measurement_unit__code == "XXX" + assert target.component_measurement__measurement_unit_qualifier__code == "B" + + assert len(importer.issues()) == 0 + + assert MeasureConditionComponent.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("measure_condition_component_CREATE.xml", __file__, True) + importer = preload_import("measure_condition_component_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target = target_message.taric_object + + assert target.condition__sid == 5 + assert target.duty_expression__sid == 7 + assert target.duty_amount == 99.99 + assert target.monetary_unit__code == "ZZZ" + assert target.component_measurement__measurement_unit__code == "XXX" + assert target.component_measurement__measurement_unit_qualifier__code == "B" + + assert len(importer.issues()) == 0 + + assert MeasureConditionComponent.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("measure_condition_component_CREATE.xml", __file__, True) + importer = preload_import("measure_condition_component_DELETE.xml", __file__) + + assert len(importer.issues()) == 0 + assert importer.can_save() + + assert MeasureConditionComponent.objects.all().count() == 2 diff --git a/taric_parsers/tests/measure_parsers/test_measure_condition_parser.py b/taric_parsers/tests/measure_parsers/test_measure_condition_parser.py new file mode 100644 index 000000000..d9a4e5457 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/test_measure_condition_parser.py @@ -0,0 +1,136 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from measures.models import MeasureCondition +from taric_parsers.parsers.measure_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestMeasureConditionParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + + + + + + + """ + + target_parser_class = MeasureConditionParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "measure_condition_sid": "7", + "measure_sid": "8", + "condition_code": "AA", + "component_sequence_number": "3", + "condition_duty_amount": "12.5", + "condition_monetary_unit_code": "GBP", + "condition_measurement_unit_code": "ABC", + "condition_measurement_unit_qualifier_code": "A", + "action_code": "XYZ", + "certificate_type_code": "A", + "certificate_code": "CER", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.sid == 7 + assert target.dependent_measure__sid == 8 + assert target.condition_code__code == "AA" + assert target.component_sequence_number == 3 + assert target.duty_amount == 12.5 + assert target.monetary_unit__code == "GBP" + assert target.condition_measurement__measurement_unit__code == "ABC" + assert target.condition_measurement__measurement_unit_qualifier__code == "A" + assert target.action__code == "XYZ" + assert target.required_certificate__certificate_type__sid == "A" + assert target.required_certificate__sid == "CER" + + def test_import(self, superuser): + importer = preload_import("measure_condition_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 21 + + target_message = importer.parsed_transactions[20].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.sid == 5 + assert target.dependent_measure__sid == 99 + assert target.condition_code__code == "A" + assert target.component_sequence_number == 5 + assert target.duty_amount == 12.77 + assert target.monetary_unit__code == "ZZZ" + assert target.condition_measurement__measurement_unit__code == "XXX" + assert target.condition_measurement__measurement_unit_qualifier__code == "A" + assert target.action__code == "ABC" + assert target.required_certificate__certificate_type__sid == "A" + assert target.required_certificate__sid == "123" + + assert len(importer.issues()) == 0 + + assert MeasureCondition.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("measure_condition_CREATE.xml", __file__, True) + importer = preload_import("measure_condition_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target = target_message.taric_object + + assert target.sid == 5 + assert target.dependent_measure__sid == 99 + assert target.condition_code__code == "A" + assert target.component_sequence_number == 5 + assert target.duty_amount == 99.99 + assert target.monetary_unit__code == "ZZZ" + assert target.condition_measurement__measurement_unit__code == "XXX" + assert target.condition_measurement__measurement_unit_qualifier__code == "A" + assert target.action__code == "ABC" + assert target.required_certificate__certificate_type__sid == "A" + assert target.required_certificate__sid == "123" + + assert len(importer.issues()) == 0 + + assert MeasureCondition.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("measure_condition_CREATE.xml", __file__, True) + importer = preload_import("measure_condition_DELETE.xml", __file__) + + assert len(importer.issues()) == 0 + assert importer.can_save() + + assert MeasureCondition.objects.all().count() == 2 diff --git a/taric_parsers/tests/measure_parsers/test_measure_excluded_geographical_area_parser.py b/taric_parsers/tests/measure_parsers/test_measure_excluded_geographical_area_parser.py new file mode 100644 index 000000000..ce130a424 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/test_measure_excluded_geographical_area_parser.py @@ -0,0 +1,105 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from measures.models import MeasureExcludedGeographicalArea +from taric_parsers.parsers.measure_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestMeasureExcludedGeographicalAreaParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + """ + + target_parser_class = MeasureExcludedGeographicalAreaParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "measure_sid": "12", + "excluded_geographical_area": "ABCD", + "geographical_area_sid": "77", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.modified_measure__sid == 12 + assert target.excluded_geographical_area__area_id == "ABCD" + assert target.excluded_geographical_area__sid == 77 + + def test_import(self, superuser): + importer = preload_import( + "measure_excluded_geographical_area_CREATE.xml", + __file__, + ) + + assert len(importer.parsed_transactions) == 10 + + target_message = importer.parsed_transactions[9].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.modified_measure__sid == 99 + assert target.excluded_geographical_area__area_id == "AB01" + assert target.excluded_geographical_area__sid == 8 + + assert len(importer.issues()) == 0 + + assert MeasureExcludedGeographicalArea.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("measure_excluded_geographical_area_CREATE.xml", __file__, True) + importer = preload_import( + "measure_excluded_geographical_area_UPDATE.xml", + __file__, + ) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target = target_message.taric_object + + assert target.modified_measure__sid == 99 + assert target.excluded_geographical_area__area_id == "AB01" + assert target.excluded_geographical_area__sid == 8 + + assert len(importer.issues()) == 0 + + assert MeasureExcludedGeographicalArea.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("measure_excluded_geographical_area_CREATE.xml", __file__, True) + importer = preload_import( + "measure_excluded_geographical_area_DELETE.xml", + __file__, + ) + + assert len(importer.issues()) == 0 + assert importer.can_save() + + assert MeasureExcludedGeographicalArea.objects.all().count() == 2 diff --git a/taric_parsers/tests/measure_parsers/test_measure_parser.py b/taric_parsers/tests/measure_parsers/test_measure_parser.py new file mode 100644 index 000000000..29f612928 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/test_measure_parser.py @@ -0,0 +1,178 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from measures.models import Measure +from taric_parsers.parsers.additional_code_parsers import * +from taric_parsers.parsers.commodity_parser import * +from taric_parsers.parsers.footnote_parser import * +from taric_parsers.parsers.geo_area_parser import * +from taric_parsers.parsers.measure_parser import * +from taric_parsers.parsers.regulation_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestNewMeasureParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + + + + + + + + + + + + + + + """ + + target_parser_class = MeasureParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "measure_sid": "1", + "measure_type": "AA", + "geographical_area": "BB", + "geographical_area_sid": "99", + "goods_nomenclature_item_id": "1122334455", + "additional_code_type": "A", + "additional_code": "123", + "ordernumber": "012345", + "reduction_indicator": "2", + "validity_start_date": "2021-01-01", + "validity_end_date": "2022-01-01", + "measure_generating_regulation_role": "7", + "measure_generating_regulation_id": "ABCDEF", + "justification_regulation_role": "8", + "justification_regulation_id": "GHIJKL", + "stopped_flag": "1", + "goods_nomenclature_sid": "123", + "additional_code_sid": "234", + "export_refund_nomenclature_sid": "345", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.sid == 1 + assert target.measure_type__sid == "AA" + assert target.geographical_area__area_id == "BB" + assert target.geographical_area__sid == 99 + assert target.goods_nomenclature__item_id == "1122334455" + assert target.goods_nomenclature__sid == 123 + assert target.additional_code__type__sid == "A" + assert target.additional_code__code == "123" + assert target.additional_code__sid == 234 + assert target.order_number__order_number == "012345" + assert target.reduction == 2 + assert target.generating_regulation__role_type == 7 + assert target.generating_regulation__regulation_id == "ABCDEF" + assert target.terminating_regulation__role_type == 8 + assert target.terminating_regulation__regulation_id == "GHIJKL" + assert target.stopped is True + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + + def test_import(self, superuser): + importer = preload_import("measure_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 9 + + target_message = importer.parsed_transactions[8].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.sid == 99 + assert target.measure_type__sid == "ZZZ" + assert target.geographical_area__area_id == "AB01" + assert target.geographical_area__sid == 8 + assert target.goods_nomenclature__item_id == "0100000000" + assert target.goods_nomenclature__sid == 1 + assert target.additional_code__type__sid is None + assert target.additional_code__code is None + assert target.additional_code__sid is None + assert target.order_number__order_number is None + assert target.reduction is None + assert target.generating_regulation__role_type == 1 + assert target.generating_regulation__regulation_id == "Z0000001" + assert target.terminating_regulation__role_type == 1 + assert target.terminating_regulation__regulation_id == "Z0000001" + assert target.stopped is True + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + + assert len(importer.issues()) == 0 + + assert Measure.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("measure_CREATE.xml", __file__, True) + importer = preload_import("measure_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target = target_message.taric_object + + assert target.sid == 99 + assert target.measure_type__sid == "ZZZ" + assert target.geographical_area__area_id == "AB01" + assert target.geographical_area__sid == 8 + assert target.goods_nomenclature__item_id == "0100000000" + assert target.goods_nomenclature__sid == 1 + assert target.additional_code__type__sid is None + assert target.additional_code__code is None + assert target.additional_code__sid is None + assert target.order_number__order_number is None + assert target.reduction is None + assert target.generating_regulation__role_type == 1 + assert target.generating_regulation__regulation_id == "Z0000001" + assert target.terminating_regulation__role_type == 1 + assert target.terminating_regulation__regulation_id == "Z0000001" + assert target.stopped is True + assert target.valid_between_lower == date(2021, 1, 11) + assert target.valid_between_upper == date(2022, 1, 1) + + assert len(importer.issues()) == 0 + + assert Measure.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("measure_CREATE.xml", __file__, True) + importer = preload_import("measure_DELETE.xml", __file__) + + assert len(importer.issues()) == 0 + assert importer.can_save() + + assert Measure.objects.all().count() == 2 diff --git a/taric_parsers/tests/measure_parsers/test_measure_type_description_parser.py b/taric_parsers/tests/measure_parsers/test_measure_type_description_parser.py new file mode 100644 index 000000000..ed86c5574 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/test_measure_type_description_parser.py @@ -0,0 +1,95 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from measures.models import MeasureType +from taric_parsers.parsers.measure_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestMeasureTypeDescriptionParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + """ + + target_parser_class = MeasureTypeDescriptionParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "measure_type_id": "AAA", + "language_id": "ZZ", + "description": "Some Description", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.sid == "AAA" + assert target.description == "Some Description" + + def test_import(self, superuser): + importer = preload_import("measure_type_description_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 2 + + target_message = importer.parsed_transactions[1].parsed_messages[1] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.sid == "ZZZ" + assert target.description == "Some Description x" + + assert len(importer.issues()) == 0 + + assert MeasureType.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("measure_type_description_CREATE.xml", __file__, True) + importer = preload_import("measure_type_description_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + target = target_message.taric_object + + assert target.sid == "ZZZ" + assert target.description == "Some Description with changes" + + assert len(importer.issues()) == 0 + + assert MeasureType.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("measure_type_description_CREATE.xml", __file__, True) + importer = preload_import("measure_type_description_DELETE.xml", __file__) + + assert importer.can_save() + + assert len(importer.issues()) == 1 + assert ( + "Children of Taric objects of type MeasureType can't be deleted directly" + in str(importer.issues()[0]) + ) diff --git a/taric_parsers/tests/measure_parsers/test_measure_type_parser.py b/taric_parsers/tests/measure_parsers/test_measure_type_parser.py new file mode 100644 index 000000000..d43d34f27 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/test_measure_type_parser.py @@ -0,0 +1,131 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from measures.models import MeasureType +from taric_parsers.parsers.geo_area_parser import * +from taric_parsers.parsers.measure_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestMeasureTypeParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + + + + + + """ + + target_parser_class = MeasureTypeParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "measure_type_id": "ZZZ", + "trade_movement_code": "1", + "priority_code": "2", + "measure_component_applicable_code": "3", + "origin_dest_code": "4", + "order_number_capture_code": "5", + "measure_explosion_level": "6", + "measure_type_series_id": "7", + "validity_start_date": "2021-01-01", + "validity_end_date": "2022-01-01", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.sid == "ZZZ" + assert target.trade_movement_code == 1 + assert target.priority_code == 2 + assert target.measure_component_applicability_code == 3 + assert target.origin_destination_code == 4 + assert target.order_number_capture_code == 5 + assert target.measure_explosion_level == 6 + assert target.measure_type_series__sid == "7" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + + def test_import(self, superuser): + importer = preload_import("measure_type_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 2 + + target_message = importer.parsed_transactions[1].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.sid == "ZZZ" + assert target.trade_movement_code == 1 + assert target.priority_code == 2 + assert target.measure_component_applicability_code == 3 + assert target.origin_destination_code == 4 + assert target.order_number_capture_code == 5 + assert target.measure_explosion_level == 6 + assert target.measure_type_series__sid == "A" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + + assert len(importer.issues()) == 0 + + assert MeasureType.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("measure_type_CREATE.xml", __file__, True) + importer = preload_import("measure_type_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + target = target_message.taric_object + + assert target.sid == "ZZZ" + assert target.trade_movement_code == 1 + assert target.priority_code == 2 + assert target.measure_component_applicability_code == 3 + assert target.origin_destination_code == 4 + assert target.order_number_capture_code == 5 + assert target.measure_explosion_level == 6 + assert target.measure_type_series__sid == "A" + assert target.valid_between_lower == date(2021, 1, 11) + assert target.valid_between_upper == date(2022, 1, 1) + + assert len(importer.issues()) == 0 + + assert MeasureType.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("measure_type_CREATE.xml", __file__, True) + importer = preload_import("measure_type_DELETE.xml", __file__) + + assert len(importer.issues()) == 0 + assert importer.can_save() + + assert MeasureType.objects.all().count() == 2 diff --git a/taric_parsers/tests/measure_parsers/test_measure_type_series_description_parser.py b/taric_parsers/tests/measure_parsers/test_measure_type_series_description_parser.py new file mode 100644 index 000000000..9192b21fc --- /dev/null +++ b/taric_parsers/tests/measure_parsers/test_measure_type_series_description_parser.py @@ -0,0 +1,106 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from measures.models import MeasureTypeSeries +from taric_parsers.parsers.measure_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestMeasureTypeSeriesDescriptionParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + """ + + target_parser_class = MeasureTypeSeriesDescriptionParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "measure_type_series_id": "AB", + "language_id": "ZZ", + "description": "Some Description", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.sid == "AB" + assert target.description == "Some Description" + + def test_import(self, superuser): + importer = preload_import( + "measure_type_series_description_CREATE.xml", + __file__, + ) + + assert len(importer.parsed_transactions) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[1] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + # verify all properties + assert target.sid == "A" + assert target.description == "Some Description" + + assert len(importer.issues()) == 0 + + assert MeasureTypeSeries.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("measure_type_series_description_CREATE.xml", __file__, True) + importer = preload_import( + "measure_type_series_description_UPDATE.xml", + __file__, + ) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + target = target_message.taric_object + + # verify all properties + assert target.sid == "A" + assert target.description == "Some Description with changes" + + assert len(importer.issues()) == 0 + + assert MeasureTypeSeries.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("measure_type_series_description_CREATE.xml", __file__, True) + importer = preload_import( + "measure_type_series_description_DELETE.xml", + __file__, + ) + + assert importer.can_save() + + assert len(importer.issues()) == 1 + assert ( + "Children of Taric objects of type MeasureTypeSeries can't be deleted directly" + in str(importer.issues()[0]) + ) diff --git a/taric_parsers/tests/measure_parsers/test_measure_type_series_parser.py b/taric_parsers/tests/measure_parsers/test_measure_type_series_parser.py new file mode 100644 index 000000000..f0d0be109 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/test_measure_type_series_parser.py @@ -0,0 +1,133 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from measures.models import MeasureTypeSeries +from taric_parsers.parsers.geo_area_parser import * +from taric_parsers.parsers.measure_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestMeasureTypeSeriesParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + """ + + target_parser_class = MeasureTypeSeriesParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "measure_type_series_id": "A", + "validity_start_date": "2021-01-01", + "validity_end_date": "2022-01-01", + "measure_type_combination": "6", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.sid == "A" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + assert target.measure_type_combination == 6 + + def test_import(self, superuser): + importer = preload_import("measure_series_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.sid == "A" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + assert target.measure_type_combination == 6 + + assert len(importer.issues()) == 0 + + assert MeasureTypeSeries.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("measure_series_CREATE.xml", __file__, True) + importer = preload_import("measure_series_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target = target_message.taric_object + + assert target.sid == "A" + assert target.valid_between_lower == date(2021, 1, 11) + assert target.valid_between_upper == date(2022, 1, 1) + assert target.measure_type_combination == 6 + + assert len(importer.issues()) == 0 + + assert MeasureTypeSeries.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("measure_series_CREATE.xml", __file__, True) + importer = preload_import("measure_series_DELETE.xml", __file__) + + assert importer.issues() == [] + assert importer.can_save() + + assert MeasureTypeSeries.objects.all().count() == 2 + + def test_import_failure_no_description(self, superuser): + importer = preload_import("measure_series_no_description_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.sid == "A" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + assert target.measure_type_combination == 6 + + assert len(importer.issues()) == 1 + + issue_error_str = str(importer.issues()[0]) + + assert ( + "ERROR: Missing expected child object MeasureTypeSeriesDescriptionParserV2" + in issue_error_str + ) + assert ( + "measure.type.series > measure.type.series.description" in issue_error_str + ) + + assert importer.can_save() is False diff --git a/taric_parsers/tests/measure_parsers/test_measurement_parser.py b/taric_parsers/tests/measure_parsers/test_measurement_parser.py new file mode 100644 index 000000000..4256d856b --- /dev/null +++ b/taric_parsers/tests/measure_parsers/test_measurement_parser.py @@ -0,0 +1,102 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from measures.models import Measurement +from taric_parsers.parsers.geo_area_parser import * +from taric_parsers.parsers.measure_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestMeasurementParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + """ + + target_parser_class = MeasurementParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "measurement_unit_code": "ABC", + "measurement_unit_qualifier_code": "D", + "validity_start_date": "2021-01-01", + "validity_end_date": "2022-01-01", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.measurement_unit__code == "ABC" + assert target.measurement_unit_qualifier__code == "D" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + + def test_import(self, superuser): + importer = preload_import("measurement_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 3 + + target_message = importer.parsed_transactions[2].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.measurement_unit__code == "XXX" + assert target.measurement_unit_qualifier__code == "A" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + + assert len(importer.issues()) == 0 + + assert Measurement.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("measurement_CREATE.xml", __file__, True) + importer = preload_import("measurement_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target = target_message.taric_object + + assert target.measurement_unit__code == "XXX" + assert target.measurement_unit_qualifier__code == "A" + assert target.valid_between_lower == date(2021, 1, 11) + assert target.valid_between_upper == date(2022, 1, 1) + + assert len(importer.issues()) == 0 + + assert Measurement.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("measurement_CREATE.xml", __file__, True) + importer = preload_import("measurement_DELETE.xml", __file__) + + assert len(importer.issues()) == 0 + assert importer.can_save() + + assert Measurement.objects.all().count() == 2 diff --git a/taric_parsers/tests/measure_parsers/test_measurement_unit_description_parser.py b/taric_parsers/tests/measure_parsers/test_measurement_unit_description_parser.py new file mode 100644 index 000000000..6c5f64f9f --- /dev/null +++ b/taric_parsers/tests/measure_parsers/test_measurement_unit_description_parser.py @@ -0,0 +1,93 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from measures.models import MeasurementUnit +from taric_parsers.parsers.measure_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestMeasurementUnitDescriptionParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + """ + + target_parser_class = MeasurementUnitDescriptionParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "measurement_unit_code": "XXX", + "language_id": "ZZ", + "description": "Some Description", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.code == "XXX" + assert target.description == "Some Description" + + def test_import(self, superuser): + importer = preload_import("measurement_unit_description_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[1] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.code == "XXX" + assert target.description == "Some Description" + + assert len(importer.issues()) == 0 + + assert MeasurementUnit.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("measurement_unit_description_CREATE.xml", __file__, True) + importer = preload_import("measurement_unit_description_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target = target_message.taric_object + + assert target.code == "XXX" + assert target.description == "Some Description with changes" + + assert len(importer.issues()) == 0 + + assert MeasurementUnit.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("measurement_unit_description_CREATE.xml", __file__, True) + importer = preload_import("measurement_unit_description_DELETE.xml", __file__) + + assert len(importer.issues()) == 0 + assert importer.can_save() + + assert MeasurementUnit.objects.all().count() == 2 diff --git a/taric_parsers/tests/measure_parsers/test_measurement_unit_parser.py b/taric_parsers/tests/measure_parsers/test_measurement_unit_parser.py new file mode 100644 index 000000000..823ad8b04 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/test_measurement_unit_parser.py @@ -0,0 +1,96 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from measures.models import MeasurementUnit +from taric_parsers.parsers.geo_area_parser import * +from taric_parsers.parsers.measure_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestMeasurementUnitParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + """ + + target_parser_class = MeasurementUnitParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "measurement_unit_code": "XXX", + "validity_start_date": "2021-01-01", + "validity_end_date": "2022-01-01", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.code == "XXX" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + + def test_import(self, superuser): + importer = preload_import("measurement_unit_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.code == "XXX" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + + assert len(importer.issues()) == 0 + + assert MeasurementUnit.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("measurement_unit_CREATE.xml", __file__, True) + importer = preload_import("measurement_unit_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target = target_message.taric_object + + assert target.code == "XXX" + assert target.valid_between_lower == date(2021, 1, 11) + assert target.valid_between_upper == date(2022, 1, 1) + + assert len(importer.issues()) == 0 + + assert MeasurementUnit.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("measurement_unit_CREATE.xml", __file__, True) + importer = preload_import("measurement_unit_DELETE.xml", __file__) + + assert len(importer.issues()) == 0 + + assert MeasurementUnit.objects.all().count() == 2 diff --git a/taric_parsers/tests/measure_parsers/test_measurement_unit_qualifier_description_parser.py b/taric_parsers/tests/measure_parsers/test_measurement_unit_qualifier_description_parser.py new file mode 100644 index 000000000..142413206 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/test_measurement_unit_qualifier_description_parser.py @@ -0,0 +1,112 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from measures.models import MeasurementUnitQualifier +from taric_parsers.parsers.measure_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestMeasurementUnitQualifierDescriptionParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + """ + + target_parser_class = MeasurementUnitQualifierDescriptionParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "measurement_unit_qualifier_code": "A", + "language_id": "ZZ", + "description": "Some Description", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.code == "A" + assert target.description == "Some Description" + + def test_import(self, superuser): + importer = preload_import( + "measurement_unit_qualifier_description_CREATE.xml", + __file__, + ) + + assert len(importer.parsed_transactions) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[1] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.code == "A" + assert target.description == "Some Description" + + assert len(importer.issues()) == 0 + + assert MeasurementUnitQualifier.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import( + "measurement_unit_qualifier_description_CREATE.xml", + __file__, + True, + ) + importer = preload_import( + "measurement_unit_qualifier_description_UPDATE.xml", + __file__, + ) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target = target_message.taric_object + + assert target.code == "A" + assert target.description == "Some Description with changes" + + assert len(importer.issues()) == 0 + + assert MeasurementUnitQualifier.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import( + "measurement_unit_qualifier_description_CREATE.xml", + __file__, + True, + ) + importer = preload_import( + "measurement_unit_qualifier_description_DELETE.xml", + __file__, + ) + + assert len(importer.issues()) == 1 + + assert ( + "Children of Taric objects of type MeasurementUnitQualifier can't be deleted directly" + in str(importer.issues()[0]) + ) diff --git a/taric_parsers/tests/measure_parsers/test_measurement_unit_qualifier_parser.py b/taric_parsers/tests/measure_parsers/test_measurement_unit_qualifier_parser.py new file mode 100644 index 000000000..647dcd2e1 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/test_measurement_unit_qualifier_parser.py @@ -0,0 +1,96 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from measures.models import MeasurementUnitQualifier +from taric_parsers.parsers.geo_area_parser import * +from taric_parsers.parsers.measure_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestMeasurementUnitQualifierParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + """ + + target_parser_class = MeasurementUnitQualifierParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "measurement_unit_qualifier_code": "A", + "validity_start_date": "2021-01-01", + "validity_end_date": "2022-01-01", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.code == "A" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + + def test_import(self, superuser): + importer = preload_import("measurement_unit_qualifier_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.code == "A" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + + assert len(importer.issues()) == 0 + + assert MeasurementUnitQualifier.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("measurement_unit_qualifier_CREATE.xml", __file__, True) + importer = preload_import("measurement_unit_qualifier_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target = target_message.taric_object + + assert target.code == "A" + assert target.valid_between_lower == date(2021, 1, 11) + assert target.valid_between_upper == date(2022, 1, 1) + + assert len(importer.issues()) == 0 + + assert MeasurementUnitQualifier.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("measurement_unit_qualifier_CREATE.xml", __file__, True) + importer = preload_import("measurement_unit_qualifier_DELETE.xml", __file__) + + assert len(importer.issues()) == 0 + + assert MeasurementUnitQualifier.objects.all().count() == 2 diff --git a/taric_parsers/tests/measure_parsers/test_monetary_unit_description_parser.py b/taric_parsers/tests/measure_parsers/test_monetary_unit_description_parser.py new file mode 100644 index 000000000..8c891340b --- /dev/null +++ b/taric_parsers/tests/measure_parsers/test_monetary_unit_description_parser.py @@ -0,0 +1,95 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from measures.models import MonetaryUnit +from taric_parsers.parsers.measure_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestMonetaryUnitDescriptionParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + """ + + target_parser_class = MonetaryUnitDescriptionParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "monetary_unit_code": "ZZZ", + "language_id": "ZZ", + "description": "Some Description", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.code == "ZZZ" + assert target.description == "Some Description" + + def test_import(self, superuser): + importer = preload_import("monetary_unit_description_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[1] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.code == "ZZZ" + assert target.description == "Some Description" + + assert len(importer.issues()) == 0 + + assert MonetaryUnit.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("monetary_unit_description_CREATE.xml", __file__, True) + importer = preload_import("monetary_unit_description_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target = target_message.taric_object + + assert target.code == "ZZZ" + assert target.description == "Some Description with changes" + + assert len(importer.issues()) == 0 + + assert MonetaryUnit.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("monetary_unit_description_CREATE.xml", __file__, True) + importer = preload_import("monetary_unit_description_DELETE.xml", __file__) + + assert len(importer.issues()) == 1 + + assert ( + "Children of Taric objects of type MonetaryUnit can't be deleted directly" + in str(importer.issues()[0]) + ) diff --git a/taric_parsers/tests/measure_parsers/test_monetary_unit_parser.py b/taric_parsers/tests/measure_parsers/test_monetary_unit_parser.py new file mode 100644 index 000000000..825ec6984 --- /dev/null +++ b/taric_parsers/tests/measure_parsers/test_monetary_unit_parser.py @@ -0,0 +1,96 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from measures.models import MonetaryUnit +from taric_parsers.parsers.geo_area_parser import * +from taric_parsers.parsers.measure_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestMonetaryUnitParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + """ + + target_parser_class = MonetaryUnitParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "monetary_unit_code": "ZZZ", + "validity_start_date": "2021-01-01", + "validity_end_date": "2022-01-01", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.code == "ZZZ" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + + def test_import(self, superuser): + importer = preload_import("monetary_unit_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.code == "ZZZ" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + + assert len(importer.issues()) == 0 + + assert MonetaryUnit.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("monetary_unit_CREATE.xml", __file__, True) + importer = preload_import("monetary_unit_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target = target_message.taric_object + + assert target.code == "ZZZ" + assert target.valid_between_lower == date(2021, 1, 11) + assert target.valid_between_upper == date(2022, 1, 1) + + assert len(importer.issues()) == 0 + + assert MonetaryUnit.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("monetary_unit_CREATE.xml", __file__, True) + importer = preload_import("monetary_unit_DELETE.xml", __file__) + + assert len(importer.issues()) == 0 + + assert MonetaryUnit.objects.all().count() == 2 diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_association_CREATE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_association_CREATE.xml new file mode 100644 index 000000000..11393dff7 --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_association_CREATE.xml @@ -0,0 +1,206 @@ + + + + + + + 1 + 225 + 00 + 3 + 3 + + ZZZ + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 225 + 05 + 3 + 3 + + ZZZ + zz + Some Description + + + + + + + + + + 1 + 215 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 215 + 05 + 3 + 3 + + A + ZZ + Some description + + + + + + + + + + 1 + 210 + 00 + 3 + 3 + + XXX + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 210 + 05 + 3 + 3 + + XXX + zz + Some Description + + + + + + + + + + 19971104 + 360 + 00 + 1 + 3 + + 7 + 054515 + 2021-01-01 + + + + + + + + + + 19971104 + 370 + 00 + 1 + 3 + + 99 + 054515 + 7 + 2021-01-01 + 2022-01-01 + 1200 + 200 + XXX + A + 3 + 0 + 75 + Some Description + + + + + + + + 19971104 + 370 + 00 + 1 + 3 + + 100 + 054515 + 7 + 2023-01-01 + 2024-01-01 + 1200 + 200 + ZZZ + 3 + 0 + 75 + Some Description + + + + + + + + + + 19971104 + 370 + 05 + 1 + 3 + + 99 + 100 + EQ + 1.6 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_association_DELETE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_association_DELETE.xml new file mode 100644 index 000000000..ac997d90b --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_association_DELETE.xml @@ -0,0 +1,24 @@ + + + + + + + 19971104 + 370 + 05 + 1 + 2 + + 99 + 100 + EQ + 1.1 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_association_UPDATE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_association_UPDATE.xml new file mode 100644 index 000000000..2a3d46ebf --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_association_UPDATE.xml @@ -0,0 +1,24 @@ + + + + + + + 19971104 + 370 + 05 + 1 + 1 + + 99 + 100 + EQ + 1.1 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_balance_event_CREATE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_balance_event_CREATE.xml new file mode 100644 index 000000000..6c718db34 --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_balance_event_CREATE.xml @@ -0,0 +1,206 @@ + + + + + + + 1 + 225 + 00 + 3 + 3 + + ZZZ + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 225 + 05 + 3 + 3 + + ZZZ + zz + Some Description + + + + + + + + + + 1 + 215 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 215 + 05 + 3 + 3 + + A + ZZ + Some description + + + + + + + + + + 1 + 210 + 00 + 3 + 3 + + XXX + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 210 + 05 + 3 + 3 + + XXX + zz + Some Description + + + + + + + + + + 19971104 + 360 + 00 + 1 + 3 + + 7 + 054515 + 2021-01-01 + + + + + + + + + + 19971104 + 370 + 00 + 1 + 3 + + 99 + 054515 + 7 + 2021-01-01 + 2022-01-01 + 1200 + 200 + XXX + A + 3 + 0 + 75 + Some Description + + + + + + + + 19971104 + 370 + 00 + 1 + 3 + + 100 + 054515 + 7 + 2023-01-01 + 2024-01-01 + 1200 + 200 + ZZZ + 3 + 0 + 75 + Some Description + + + + + + + + + + 1 + 375 + 00 + 1 + 3 + + 100 + 2020-09-25T14:58:58 + 10 + 0 + 0 + 2021-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_balance_event_DELETE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_balance_event_DELETE.xml new file mode 100644 index 000000000..f9c728320 --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_balance_event_DELETE.xml @@ -0,0 +1,24 @@ + + + + + + + 1 + 375 + 00 + 1 + 2 + + 100 + 2020-09-25T14:58:58 + 10 + 0 + 0 + 2021-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_balance_event_UPDATE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_balance_event_UPDATE.xml new file mode 100644 index 000000000..eb3726cd6 --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_balance_event_UPDATE.xml @@ -0,0 +1,24 @@ + + + + + + + 1 + 375 + 00 + 1 + 1 + + 100 + 2020-09-25T14:58:58 + 10 + 0 + 0 + 2021-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_blocking_period_CREATE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_blocking_period_CREATE.xml new file mode 100644 index 000000000..75ff8a92c --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_blocking_period_CREATE.xml @@ -0,0 +1,208 @@ + + + + + + + 1 + 225 + 00 + 3 + 3 + + ZZZ + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 225 + 05 + 3 + 3 + + ZZZ + zz + Some Description + + + + + + + + + + 1 + 215 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 215 + 05 + 3 + 3 + + A + ZZ + Some description + + + + + + + + + + 1 + 210 + 00 + 3 + 3 + + XXX + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 210 + 05 + 3 + 3 + + XXX + zz + Some Description + + + + + + + + + + 19971104 + 360 + 00 + 1 + 3 + + 7 + 054515 + 2021-01-01 + + + + + + + + + + 19971104 + 370 + 00 + 1 + 3 + + 99 + 054515 + 7 + 2021-01-01 + 2022-01-01 + 1200 + 200 + XXX + A + 3 + 0 + 75 + Some Description + + + + + + + + 19971104 + 370 + 00 + 1 + 3 + + 100 + 054515 + 7 + 2023-01-01 + 2024-01-01 + 1200 + 200 + ZZZ + 3 + 0 + 75 + Some Description + + + + + + + + + + 19971104 + 370 + 10 + 1 + 3 + + 22 + 99 + 2021-01-01 + 2022-01-01 + 3 + Some Description + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_blocking_period_DELETE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_blocking_period_DELETE.xml new file mode 100644 index 000000000..81c4b9d10 --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_blocking_period_DELETE.xml @@ -0,0 +1,26 @@ + + + + + + + 19971104 + 370 + 10 + 1 + 2 + + 22 + 99 + 2021-01-11 + 2022-01-01 + 3 + Some Description with changes + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_blocking_period_UPDATE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_blocking_period_UPDATE.xml new file mode 100644 index 000000000..e6a03d31c --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_blocking_period_UPDATE.xml @@ -0,0 +1,26 @@ + + + + + + + 19971104 + 370 + 10 + 1 + 1 + + 22 + 99 + 2021-01-11 + 2022-01-01 + 3 + Some Description with changes + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_closed_and_transferred_event_CREATE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_closed_and_transferred_event_CREATE.xml new file mode 100644 index 000000000..73557ed98 --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_closed_and_transferred_event_CREATE.xml @@ -0,0 +1,206 @@ + + + + + + + 1 + 225 + 00 + 3 + 3 + + ZZZ + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 225 + 05 + 3 + 3 + + ZZZ + zz + Some Description + + + + + + + + + + 1 + 215 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 215 + 05 + 3 + 3 + + A + ZZ + Some description + + + + + + + + + + 1 + 210 + 00 + 3 + 3 + + XXX + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 210 + 05 + 3 + 3 + + XXX + zz + Some Description + + + + + + + + + + 19971104 + 360 + 00 + 1 + 3 + + 7 + 054515 + 2021-01-01 + + + + + + + + + + 19971104 + 370 + 00 + 1 + 3 + + 99 + 054515 + 7 + 2021-01-01 + 2022-01-01 + 1200 + 200 + XXX + A + 3 + 0 + 75 + Some Description + + + + + + + + 19971104 + 370 + 00 + 1 + 3 + + 100 + 054515 + 7 + 2023-01-01 + 2024-01-01 + 1200 + 200 + ZZZ + 3 + 0 + 75 + Some Description + + + + + + + + + + 1 + 375 + 30 + 1 + 3 + + 100 + 2020-09-25T14:58:58 + 2021-01-01 + ZZZ + 123.77 + 56 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_closed_and_transferred_event_DELETE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_closed_and_transferred_event_DELETE.xml new file mode 100644 index 000000000..53e4d5d75 --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_closed_and_transferred_event_DELETE.xml @@ -0,0 +1,24 @@ + + + + + + + 1 + 375 + 30 + 1 + 2 + + 100 + 2020-09-25T14:58:58 + 2021-01-01 + ZZZ + 123.77 + 56 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_closed_and_transferred_event_UPDATE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_closed_and_transferred_event_UPDATE.xml new file mode 100644 index 000000000..7f2f3e1b6 --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_closed_and_transferred_event_UPDATE.xml @@ -0,0 +1,24 @@ + + + + + + + 1 + 375 + 30 + 1 + 1 + + 100 + 2020-09-25T14:58:58 + 2021-01-01 + ZZZ + 123.77 + 56 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_critical_event_CREATE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_critical_event_CREATE.xml new file mode 100644 index 000000000..9a1576f88 --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_critical_event_CREATE.xml @@ -0,0 +1,204 @@ + + + + + + + 1 + 225 + 00 + 3 + 3 + + ZZZ + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 225 + 05 + 3 + 3 + + ZZZ + zz + Some Description + + + + + + + + + + 1 + 215 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 215 + 05 + 3 + 3 + + A + ZZ + Some description + + + + + + + + + + 1 + 210 + 00 + 3 + 3 + + XXX + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 210 + 05 + 3 + 3 + + XXX + zz + Some Description + + + + + + + + + + 19971104 + 360 + 00 + 1 + 3 + + 7 + 054515 + 2021-01-01 + + + + + + + + + + 19971104 + 370 + 00 + 1 + 3 + + 99 + 054515 + 7 + 2021-01-01 + 2022-01-01 + 1200 + 200 + XXX + A + 3 + 0 + 75 + Some Description + + + + + + + + 19971104 + 370 + 00 + 1 + 3 + + 100 + 054515 + 7 + 2023-01-01 + 2024-01-01 + 1200 + 200 + ZZZ + 3 + 0 + 75 + Some Description + + + + + + + + + + 1 + 375 + 10 + 1 + 3 + + 100 + 2020-09-25T14:58:58 + Y + 2021-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_critical_event_DELETE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_critical_event_DELETE.xml new file mode 100644 index 000000000..fc8dc86be --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_critical_event_DELETE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 375 + 10 + 1 + 2 + + 100 + 2020-09-25T14:58:58 + Y + 2021-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_critical_event_UPDATE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_critical_event_UPDATE.xml new file mode 100644 index 000000000..b0b5086c0 --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_critical_event_UPDATE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 375 + 10 + 1 + 1 + + 100 + 2020-09-25T14:58:58 + Y + 2021-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_definition_CREATE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_definition_CREATE.xml new file mode 100644 index 000000000..fc5da093d --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_definition_CREATE.xml @@ -0,0 +1,186 @@ + + + + + + + 1 + 225 + 00 + 3 + 3 + + ZZZ + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 225 + 05 + 3 + 3 + + ZZZ + zz + Some Description + + + + + + + + + + 1 + 215 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 215 + 05 + 3 + 3 + + A + ZZ + Some description + + + + + + + + + + 1 + 210 + 00 + 3 + 3 + + XXX + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 210 + 05 + 3 + 3 + + XXX + zz + Some Description + + + + + + + + + + 19971104 + 360 + 00 + 1 + 3 + + 7 + 054515 + 2021-01-01 + + + + + + + + + + 19971104 + 370 + 00 + 1 + 3 + + 99 + 054515 + 7 + 2021-01-01 + 2022-01-01 + 1200 + 200 + XXX + A + 3 + 0 + 75 + Some Description + + + + + + + + 19971104 + 370 + 00 + 1 + 3 + + 100 + 054515 + 7 + 2023-01-01 + 2024-01-01 + 1200 + 200 + ZZZ + 3 + 0 + 75 + Some Description + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_definition_DELETE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_definition_DELETE.xml new file mode 100644 index 000000000..554d8513b --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_definition_DELETE.xml @@ -0,0 +1,32 @@ + + + + + + + 19971104 + 370 + 00 + 1 + 2 + + 99 + 054515 + 7 + 2023-01-11 + 2024-01-01 + 1200 + 200 + ZZZ + 3 + 0 + 75 + Some Description with changes + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_definition_UPDATE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_definition_UPDATE.xml new file mode 100644 index 000000000..1b2c3e0a1 --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_definition_UPDATE.xml @@ -0,0 +1,32 @@ + + + + + + + 19971104 + 370 + 00 + 1 + 1 + + 100 + 054515 + 7 + 2023-01-11 + 2024-01-01 + 1200 + 200 + ZZZ + 3 + 0 + 75 + Some Description with changes + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_exhaustion_event_CREATE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_exhaustion_event_CREATE.xml new file mode 100644 index 000000000..a3bb20022 --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_exhaustion_event_CREATE.xml @@ -0,0 +1,203 @@ + + + + + + + 1 + 225 + 00 + 3 + 3 + + ZZZ + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 225 + 05 + 3 + 3 + + ZZZ + zz + Some Description + + + + + + + + + + 1 + 215 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 215 + 05 + 3 + 3 + + A + ZZ + Some description + + + + + + + + + + 1 + 210 + 00 + 3 + 3 + + XXX + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 210 + 05 + 3 + 3 + + XXX + zz + Some Description + + + + + + + + + + 19971104 + 360 + 00 + 1 + 3 + + 7 + 054515 + 2021-01-01 + + + + + + + + + + 19971104 + 370 + 00 + 1 + 3 + + 99 + 054515 + 7 + 2021-01-01 + 2022-01-01 + 1200 + 200 + XXX + A + 3 + 0 + 75 + Some Description + + + + + + + + 19971104 + 370 + 00 + 1 + 3 + + 100 + 054515 + 7 + 2023-01-01 + 2024-01-01 + 1200 + 200 + ZZZ + 3 + 0 + 75 + Some Description + + + + + + + + + + 1 + 375 + 15 + 1 + 3 + + 100 + 2020-09-25T14:58:58 + 2021-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_exhaustion_event_DELETE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_exhaustion_event_DELETE.xml new file mode 100644 index 000000000..9279a6ffc --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_exhaustion_event_DELETE.xml @@ -0,0 +1,21 @@ + + + + + + + 1 + 375 + 15 + 1 + 2 + + 100 + 2020-09-25T14:58:58 + 2021-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_exhaustion_event_UPDATE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_exhaustion_event_UPDATE.xml new file mode 100644 index 000000000..cba4ed783 --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_exhaustion_event_UPDATE.xml @@ -0,0 +1,21 @@ + + + + + + + 1 + 375 + 15 + 1 + 1 + + 100 + 2020-09-25T14:58:58 + 2021-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_order_number_CREATE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_order_number_CREATE.xml new file mode 100644 index 000000000..b99ab4fa9 --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_order_number_CREATE.xml @@ -0,0 +1,23 @@ + + + + + + + 19971104 + 360 + 00 + 1 + 3 + + 7 + 054515 + 2021-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_order_number_DELETE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_order_number_DELETE.xml new file mode 100644 index 000000000..65747988f --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_order_number_DELETE.xml @@ -0,0 +1,23 @@ + + + + + + + 19971104 + 360 + 00 + 1 + 2 + + 7 + 054515 + 2021-01-11 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_order_number_UPDATE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_order_number_UPDATE.xml new file mode 100644 index 000000000..652c23c75 --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_order_number_UPDATE.xml @@ -0,0 +1,23 @@ + + + + + + + 19971104 + 360 + 00 + 1 + 1 + + 7 + 054515 + 2021-01-11 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_order_number_origin_CREATE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_order_number_origin_CREATE.xml new file mode 100644 index 000000000..bdd2d2c38 --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_order_number_origin_CREATE.xml @@ -0,0 +1,105 @@ + + + + + + + 1 + 250 + 00 + 3 + 3 + + 8 + AB01 + 2021-01-01 + 2022-01-01 + 1 + + + + + + + + + + 1 + 250 + 10 + 3 + 3 + + 3 + zz + 8 + AB01 + Some Description + + + + + + + + 1 + 250 + 05 + 3 + 3 + + 3 + 2022-01-01 + 8 + AB01 + + + + + + + + + + 19971104 + 360 + 00 + 1 + 3 + + 7 + 054515 + 2021-01-01 + + + + + + + + + + 19971104 + 360 + 10 + 1 + 3 + + 123 + 7 + AB01 + 8 + 2021-01-01 + 2022-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_order_number_origin_DELETE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_order_number_origin_DELETE.xml new file mode 100644 index 000000000..7386a8c17 --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_order_number_origin_DELETE.xml @@ -0,0 +1,26 @@ + + + + + + + 19971104 + 360 + 10 + 1 + 2 + + 123 + 7 + AB01 + 8 + 2021-01-11 + 2022-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_order_number_origin_UPDATE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_order_number_origin_UPDATE.xml new file mode 100644 index 000000000..290603dae --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_order_number_origin_UPDATE.xml @@ -0,0 +1,26 @@ + + + + + + + 19971104 + 360 + 10 + 1 + 1 + + 123 + 7 + AB01 + 8 + 2021-01-11 + 2022-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_order_number_origin_exclusion_CREATE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_order_number_origin_exclusion_CREATE.xml new file mode 100644 index 000000000..53e610975 --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_order_number_origin_exclusion_CREATE.xml @@ -0,0 +1,123 @@ + + + + + + + 1 + 250 + 00 + 3 + 3 + + 8 + AB01 + 2021-01-01 + 2022-01-01 + 1 + + + + + + + + + + 1 + 250 + 10 + 3 + 3 + + 3 + zz + 8 + AB01 + Some Description + + + + + + + + 1 + 250 + 05 + 3 + 3 + + 3 + 2022-01-01 + 8 + AB01 + + + + + + + + + + 19971104 + 360 + 00 + 1 + 3 + + 7 + 054515 + 2021-01-01 + + + + + + + + + + 19971104 + 360 + 10 + 1 + 3 + + 123 + 7 + AB01 + 8 + 2021-01-01 + 2022-01-01 + + + + + + + + + + 19971104 + 360 + 15 + 1 + 3 + + 123 + 8 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_order_number_origin_exclusion_DELETE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_order_number_origin_exclusion_DELETE.xml new file mode 100644 index 000000000..6e30b395c --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_order_number_origin_exclusion_DELETE.xml @@ -0,0 +1,22 @@ + + + + + + + 19971104 + 360 + 15 + 1 + 2 + + 123 + 8 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_order_number_origin_exclusion_UPDATE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_order_number_origin_exclusion_UPDATE.xml new file mode 100644 index 000000000..1f2914f68 --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_order_number_origin_exclusion_UPDATE.xml @@ -0,0 +1,22 @@ + + + + + + + 19971104 + 360 + 15 + 1 + 1 + + 123 + 8 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_reopening_event_CREATE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_reopening_event_CREATE.xml new file mode 100644 index 000000000..353d64491 --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_reopening_event_CREATE.xml @@ -0,0 +1,203 @@ + + + + + + + 1 + 225 + 00 + 3 + 3 + + ZZZ + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 225 + 05 + 3 + 3 + + ZZZ + zz + Some Description + + + + + + + + + + 1 + 215 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 215 + 05 + 3 + 3 + + A + ZZ + Some description + + + + + + + + + + 1 + 210 + 00 + 3 + 3 + + XXX + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 210 + 05 + 3 + 3 + + XXX + zz + Some Description + + + + + + + + + + 19971104 + 360 + 00 + 1 + 3 + + 7 + 054515 + 2021-01-01 + + + + + + + + + + 19971104 + 370 + 00 + 1 + 3 + + 99 + 054515 + 7 + 2021-01-01 + 2022-01-01 + 1200 + 200 + XXX + A + 3 + 0 + 75 + Some Description + + + + + + + + 19971104 + 370 + 00 + 1 + 3 + + 100 + 054515 + 7 + 2023-01-01 + 2024-01-01 + 1200 + 200 + ZZZ + 3 + 0 + 75 + Some Description + + + + + + + + + + 1 + 375 + 20 + 1 + 3 + + 100 + 2020-09-25T14:58:58 + 2021-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_reopening_event_DELETE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_reopening_event_DELETE.xml new file mode 100644 index 000000000..cc33a29db --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_reopening_event_DELETE.xml @@ -0,0 +1,21 @@ + + + + + + + 1 + 375 + 20 + 1 + 2 + + 100 + 2020-09-25T14:58:58 + 2021-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_reopening_event_UPDATE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_reopening_event_UPDATE.xml new file mode 100644 index 000000000..5b7fcb754 --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_reopening_event_UPDATE.xml @@ -0,0 +1,21 @@ + + + + + + + 1 + 375 + 20 + 1 + 1 + + 100 + 2020-09-25T14:58:58 + 2021-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_suspension_CREATE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_suspension_CREATE.xml new file mode 100644 index 000000000..c0cfe1915 --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_suspension_CREATE.xml @@ -0,0 +1,207 @@ + + + + + + + 1 + 225 + 00 + 3 + 3 + + ZZZ + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 225 + 05 + 3 + 3 + + ZZZ + zz + Some Description + + + + + + + + + + 1 + 215 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 215 + 05 + 3 + 3 + + A + ZZ + Some description + + + + + + + + + + 1 + 210 + 00 + 3 + 3 + + XXX + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 210 + 05 + 3 + 3 + + XXX + zz + Some Description + + + + + + + + + + 19971104 + 360 + 00 + 1 + 3 + + 7 + 054515 + 2021-01-01 + + + + + + + + + + 19971104 + 370 + 00 + 1 + 3 + + 99 + 054515 + 7 + 2021-01-01 + 2022-01-01 + 1200 + 200 + XXX + A + 3 + 0 + 75 + Some Description + + + + + + + + 19971104 + 370 + 00 + 1 + 3 + + 100 + 054515 + 7 + 2023-01-01 + 2024-01-01 + 1200 + 200 + ZZZ + 3 + 0 + 75 + Some Description + + + + + + + + + + 19971104 + 370 + 15 + 1 + 3 + + 123 + 99 + 2021-01-01 + 2022-01-01 + Some Description + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_suspension_DELETE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_suspension_DELETE.xml new file mode 100644 index 000000000..c33726fd0 --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_suspension_DELETE.xml @@ -0,0 +1,25 @@ + + + + + + + 19971104 + 370 + 15 + 1 + 2 + + 123 + 99 + 2021-01-11 + 2022-01-01 + Some Description with changes + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_suspension_UPDATE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_suspension_UPDATE.xml new file mode 100644 index 000000000..bd2fa5be5 --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_suspension_UPDATE.xml @@ -0,0 +1,25 @@ + + + + + + + 19971104 + 370 + 15 + 1 + 1 + + 123 + 99 + 2021-01-11 + 2022-01-01 + Some Description with changes + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_unblocking_event_CREATE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_unblocking_event_CREATE.xml new file mode 100644 index 000000000..0f972503d --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_unblocking_event_CREATE.xml @@ -0,0 +1,203 @@ + + + + + + + 1 + 225 + 00 + 3 + 3 + + ZZZ + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 225 + 05 + 3 + 3 + + ZZZ + zz + Some Description + + + + + + + + + + 1 + 215 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 215 + 05 + 3 + 3 + + A + ZZ + Some description + + + + + + + + + + 1 + 210 + 00 + 3 + 3 + + XXX + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 210 + 05 + 3 + 3 + + XXX + zz + Some Description + + + + + + + + + + 19971104 + 360 + 00 + 1 + 3 + + 7 + 054515 + 2021-01-01 + + + + + + + + + + 19971104 + 370 + 00 + 1 + 3 + + 99 + 054515 + 7 + 2021-01-01 + 2022-01-01 + 1200 + 200 + XXX + A + 3 + 0 + 75 + Some Description + + + + + + + + 19971104 + 370 + 00 + 1 + 3 + + 100 + 054515 + 7 + 2023-01-01 + 2024-01-01 + 1200 + 200 + ZZZ + 3 + 0 + 75 + Some Description + + + + + + + + + + 1 + 375 + 05 + 1 + 3 + + 100 + 2020-09-25T14:58:58 + 2021-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_unblocking_event_DELETE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_unblocking_event_DELETE.xml new file mode 100644 index 000000000..0a4787fa8 --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_unblocking_event_DELETE.xml @@ -0,0 +1,21 @@ + + + + + + + 1 + 375 + 05 + 1 + 2 + + 100 + 2020-09-25T14:58:58 + 2021-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_unblocking_event_UPDATE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_unblocking_event_UPDATE.xml new file mode 100644 index 000000000..ff1923ff4 --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_unblocking_event_UPDATE.xml @@ -0,0 +1,21 @@ + + + + + + + 1 + 375 + 05 + 1 + 1 + + 100 + 2020-09-25T14:58:58 + 2021-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_unsuspension_event_CREATE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_unsuspension_event_CREATE.xml new file mode 100644 index 000000000..42c51cc9d --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_unsuspension_event_CREATE.xml @@ -0,0 +1,203 @@ + + + + + + + 1 + 225 + 00 + 3 + 3 + + ZZZ + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 225 + 05 + 3 + 3 + + ZZZ + zz + Some Description + + + + + + + + + + 1 + 215 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 215 + 05 + 3 + 3 + + A + ZZ + Some description + + + + + + + + + + 1 + 210 + 00 + 3 + 3 + + XXX + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 210 + 05 + 3 + 3 + + XXX + zz + Some Description + + + + + + + + + + 19971104 + 360 + 00 + 1 + 3 + + 7 + 054515 + 2021-01-01 + + + + + + + + + + 19971104 + 370 + 00 + 1 + 3 + + 99 + 054515 + 7 + 2021-01-01 + 2022-01-01 + 1200 + 200 + XXX + A + 3 + 0 + 75 + Some Description + + + + + + + + 19971104 + 370 + 00 + 1 + 3 + + 100 + 054515 + 7 + 2023-01-01 + 2024-01-01 + 1200 + 200 + ZZZ + 3 + 0 + 75 + Some Description + + + + + + + + + + 1 + 375 + 25 + 1 + 3 + + 100 + 2020-09-25T14:58:58 + 2021-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_unsuspension_event_DELETE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_unsuspension_event_DELETE.xml new file mode 100644 index 000000000..cbede4d0f --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_unsuspension_event_DELETE.xml @@ -0,0 +1,21 @@ + + + + + + + 1 + 375 + 25 + 1 + 2 + + 100 + 2020-09-25T14:58:58 + 2021-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/importer_examples/quota_unsuspension_event_UPDATE.xml b/taric_parsers/tests/quota_parser/importer_examples/quota_unsuspension_event_UPDATE.xml new file mode 100644 index 000000000..086a7fbfb --- /dev/null +++ b/taric_parsers/tests/quota_parser/importer_examples/quota_unsuspension_event_UPDATE.xml @@ -0,0 +1,21 @@ + + + + + + + 1 + 375 + 25 + 1 + 1 + + 100 + 2020-09-25T14:58:58 + 2021-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/quota_parser/test_quota_association_parser.py b/taric_parsers/tests/quota_parser/test_quota_association_parser.py new file mode 100644 index 000000000..65bbb2ed6 --- /dev/null +++ b/taric_parsers/tests/quota_parser/test_quota_association_parser.py @@ -0,0 +1,100 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from quotas.models import QuotaAssociation +from taric_parsers.parsers.measure_parser import * +from taric_parsers.parsers.quota_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestQuotaAssociationParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + """ + + target_parser_class = QuotaAssociationParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "main_quota_definition_sid": "12", + "sub_quota_definition_sid": "13", # gets ignored, but will come in from import + "relation_type": "ZZ", + "coefficient": "12.432", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.main_quota__sid == 12 + assert target.sub_quota__sid == 13 + assert target.sub_quota_relation_type == "ZZ" + assert target.coefficient == 12.432 + + def test_import(self, superuser): + importer = preload_import("quota_association_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 6 + + target_message = importer.parsed_transactions[5].parsed_messages[0] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + assert target.main_quota__sid == 99 + assert target.sub_quota__sid == 100 + assert target.sub_quota_relation_type == "EQ" + assert target.coefficient == 1.6 + + assert len(importer.issues()) == 0 + assert QuotaAssociation.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("quota_association_CREATE.xml", __file__, True) + importer = preload_import("quota_association_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target = target_message.taric_object + + assert target.main_quota__sid == 99 + assert target.sub_quota__sid == 100 + assert target.sub_quota_relation_type == "EQ" + assert target.coefficient == 1.1 + + assert len(importer.issues()) == 0 + + assert QuotaAssociation.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("quota_association_CREATE.xml", __file__, True) + importer = preload_import("quota_association_DELETE.xml", __file__) + + assert len(importer.issues()) == 0 + + assert QuotaAssociation.objects.all().count() == 2 diff --git a/taric_parsers/tests/quota_parser/test_quota_balance_event_parser.py b/taric_parsers/tests/quota_parser/test_quota_balance_event_parser.py new file mode 100644 index 000000000..659c63a79 --- /dev/null +++ b/taric_parsers/tests/quota_parser/test_quota_balance_event_parser.py @@ -0,0 +1,125 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from quotas.models import QuotaEvent +from taric_parsers.parsers.measure_parser import * +from taric_parsers.parsers.quota_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestQuotaBalanceEventParserV2: + """ + Could be one of any of the below: + + .. code:: XML + + + + + + + + + + + + + + + """ + + target_parser_class = QuotaBalanceEventParserV2 + + def test_it_handles_population_from_expected_data_structure_quota_balance_event( + self, + ): + expected_data_example = { + "quota_definition_sid": "123", + "occurrence_timestamp": "2020-09-25T14:58:58", + "old_balance": "234000", + "new_balance": "19000", + "imported_amount": "123123", + "last_import_date_in_allocation": "2021-07-04", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + assert target.occurrence_timestamp == datetime(2020, 9, 25, 14, 58, 58) + assert target.quota_definition__sid == 123 + assert target.new_balance == "19000" + assert target.old_balance == "234000" + assert target.imported_amount == "123123" + assert target.last_import_date_in_allocation == "2021-07-04" + + assert ( + target.data + == '{"new.balance": "19000", "old.balance": "234000", "imported.amount": "123123", ' + '"last.import.date.in.allocation": "2021-07-04"}' + ) + + def test_import(self, superuser): + importer = preload_import("quota_balance_event_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 6 + assert len(importer.parsed_transactions[5].parsed_messages) == 1 + + target_message = importer.parsed_transactions[5].parsed_messages[0] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.new_balance == "0" + assert target.old_balance == "10" + assert target.imported_amount == "0" + assert target.last_import_date_in_allocation == "2021-01-01" + assert target.quota_definition__sid == 100 + assert target.occurrence_timestamp == datetime(2020, 9, 25, 14, 58, 58) + + assert ( + target.data + == '{"new.balance": "0", "old.balance": "10", "imported.amount": "0", ' + '"last.import.date.in.allocation": "2021-01-01"}' + ) + + assert len(importer.issues()) == 0 + + assert ( + QuotaEvent.objects.all() + .filter(subrecord_code=self.target_parser_class.subrecord_code) + .count() + == 1 + ) + + def test_import_update(self, superuser): + # No Updates for this object type + preload_import("quota_balance_event_CREATE.xml", __file__, True) + importer = preload_import("quota_balance_event_UPDATE.xml", __file__) + + assert len(importer.issues()) == 1 + + assert "Taric objects of type QuotaEvent can't be updated" in str( + importer.issues()[0], + ) + + def test_import_delete(self, superuser): + # No Updates for this object type + preload_import("quota_balance_event_CREATE.xml", __file__, True) + importer = preload_import("quota_balance_event_DELETE.xml", __file__) + + assert len(importer.issues()) == 0 + + assert QuotaEvent.objects.all().count() == 2 diff --git a/taric_parsers/tests/quota_parser/test_quota_blocking_parser.py b/taric_parsers/tests/quota_parser/test_quota_blocking_parser.py new file mode 100644 index 000000000..fc641c00d --- /dev/null +++ b/taric_parsers/tests/quota_parser/test_quota_blocking_parser.py @@ -0,0 +1,112 @@ +from datetime import date + +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from quotas.models import QuotaBlocking +from taric_parsers.parsers.measure_parser import * +from taric_parsers.parsers.quota_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestQuotaBlockingParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + + """ + + target_parser_class = QuotaBlockingParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "quota_blocking_period_sid": "123", + "quota_definition_sid": "14", # gets ignored, but will come in from import + "blocking_start_date": "2020-01-01", + "blocking_end_date": "2020-12-01", + "blocking_period_type": "0", + "description": "Some Description", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + assert target.sid == 123 + assert target.quota_definition__sid == 14 + assert target.valid_between_lower == date(2020, 1, 1) + assert target.valid_between_upper == date(2020, 12, 1) + assert target.blocking_period_type == 0 + assert target.description == "Some Description" + + def test_import(self, superuser): + importer = preload_import("quota_blocking_period_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 6 + + target_message = importer.parsed_transactions[5].parsed_messages[0] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + assert target.sid == 22 + assert target.quota_definition__sid == 99 + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + assert target.blocking_period_type == 3 + assert target.description == "Some Description" + + assert QuotaBlocking.objects.all().count() == 1 + + assert len(importer.issues()) == 0 + + def test_import_update(self, superuser): + preload_import("quota_blocking_period_CREATE.xml", __file__, True) + importer = preload_import("quota_blocking_period_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target = target_message.taric_object + + assert target.sid == 22 + assert target.quota_definition__sid == 99 + assert target.valid_between_lower == date(2021, 1, 11) + assert target.valid_between_upper == date(2022, 1, 1) + assert target.blocking_period_type == 3 + assert target.description == "Some Description with changes" + + assert len(importer.issues()) == 0 + + assert QuotaBlocking.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("quota_blocking_period_CREATE.xml", __file__, True) + importer = preload_import("quota_blocking_period_DELETE.xml", __file__) + + assert len(importer.issues()) == 0 + + assert QuotaBlocking.objects.all().count() == 2 diff --git a/taric_parsers/tests/quota_parser/test_quota_closed_and_transferred_event_parser.py b/taric_parsers/tests/quota_parser/test_quota_closed_and_transferred_event_parser.py new file mode 100644 index 000000000..35107a7bf --- /dev/null +++ b/taric_parsers/tests/quota_parser/test_quota_closed_and_transferred_event_parser.py @@ -0,0 +1,144 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from quotas.models import QuotaEvent +from taric_parsers.parsers.measure_parser import * +from taric_parsers.parsers.quota_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestQuotaClosedAndTransferredEventParserV2: + """ + Could be one of any of the below: + + .. code:: XML + + + + + + + + + + + + + + + + + + """ + + target_parser_class = QuotaClosedAndTransferredEventParserV2 + + def test_it_handles_population_from_expected_data_structure_quota_balance_event( + self, + ): + expected_data_example = { + "quota_definition_sid": "123", + "occurrence_timestamp": "2020-09-25T14:58:58", + "transfer_date": "2021-07-04", + "quota_closed": "zzz", + "transferred_amount": "19000", + "target_quota_definition_sid": "123123", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.occurrence_timestamp == datetime(2020, 9, 25, 14, 58, 58) + assert target.quota_definition__sid == 123 + assert target.transfer_date == "2021-07-04" + assert target.quota_closed == "zzz" + assert target.transferred_amount == "19000" + assert target.target_quota_definition_sid == "123123" + + assert ( + target.data + == '{"quota.closed": "zzz", "transferred.amount": "19000", "transfer.date": "2021-07-04", ' + '"target.quota.definition.sid": "123123"}' + ) + + def test_import(self, superuser): + importer = preload_import( + "quota_closed_and_transferred_event_CREATE.xml", + __file__, + ) + + assert len(importer.parsed_transactions) == 6 + assert len(importer.parsed_transactions[5].parsed_messages) == 1 + + target_message = importer.parsed_transactions[5].parsed_messages[0] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.quota_definition__sid == 100 + assert target.occurrence_timestamp == datetime(2020, 9, 25, 14, 58, 58) + + assert target.transfer_date == "2021-01-01" + assert target.quota_closed == "ZZZ" + assert target.transferred_amount == "123.77" + assert target.target_quota_definition_sid == "56" + + assert ( + target.data == '{"quota.closed": "ZZZ", "transferred.amount": "123.77", ' + '"transfer.date": "2021-01-01", "target.quota.definition.sid": "56"}' + ) + + assert len(importer.issues()) == 0 + + assert ( + QuotaEvent.objects.all() + .filter(subrecord_code=self.target_parser_class.subrecord_code) + .count() + == 1 + ) + + def test_import_update(self, superuser): + preload_import( + "quota_closed_and_transferred_event_CREATE.xml", + __file__, + True, + ) + importer = preload_import( + "quota_closed_and_transferred_event_UPDATE.xml", + __file__, + ) + + assert len(importer.issues()) == 1 + + assert "Taric objects of type QuotaEvent can't be updated" in str( + importer.issues()[0], + ) + + def test_import_delete(self, superuser): + preload_import( + "quota_closed_and_transferred_event_CREATE.xml", + __file__, + True, + ) + importer = preload_import( + "quota_closed_and_transferred_event_DELETE.xml", + __file__, + ) + + assert len(importer.issues()) == 0 + + assert QuotaEvent.objects.all().count() == 2 diff --git a/taric_parsers/tests/quota_parser/test_quota_critical_event_parser.py b/taric_parsers/tests/quota_parser/test_quota_critical_event_parser.py new file mode 100644 index 000000000..f22fd50c9 --- /dev/null +++ b/taric_parsers/tests/quota_parser/test_quota_critical_event_parser.py @@ -0,0 +1,114 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from quotas.models import QuotaEvent +from taric_parsers.parsers.measure_parser import * +from taric_parsers.parsers.quota_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestQuotaEventParserV2: + """ + Could be one of any of the below: + + .. code:: XML + + + + + + + + + + + + + """ + + target_parser_class = QuotaCriticalEventParserV2 + + def test_it_handles_population_from_expected_data_structure_quota_balance_event( + self, + ): + expected_data_example = { + "quota_definition_sid": "123", + "occurrence_timestamp": "2020-09-25T14:58:58", + "critical_state": "ZZZ", + "critical_state_change_date": "2020-01-01", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.occurrence_timestamp == datetime(2020, 9, 25, 14, 58, 58) + assert target.quota_definition__sid == 123 + assert target.critical_state == "ZZZ" + assert target.critical_state_change_date == "2020-01-01" + + assert ( + target.data + == '{"critical.state": "ZZZ", "critical.state.change.date": "2020-01-01"}' + ) + + def test_import(self, superuser): + importer = preload_import("quota_critical_event_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 6 + assert len(importer.parsed_transactions[5].parsed_messages) == 1 + + target_message = importer.parsed_transactions[5].parsed_messages[0] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.quota_definition__sid == 100 + assert target.occurrence_timestamp == datetime(2020, 9, 25, 14, 58, 58) + assert target.critical_state == "Y" + assert target.critical_state_change_date == "2021-01-01" + + assert ( + target.data + == '{"critical.state": "Y", "critical.state.change.date": "2021-01-01"}' + ) + + assert len(importer.issues()) == 0 + + assert ( + QuotaEvent.objects.all() + .filter(subrecord_code=self.target_parser_class.subrecord_code) + .count() + == 1 + ) + + def test_import_update(self, superuser): + preload_import("quota_critical_event_CREATE.xml", __file__, True) + importer = preload_import("quota_critical_event_UPDATE.xml", __file__) + + assert len(importer.issues()) == 1 + + assert "Taric objects of type QuotaEvent can't be updated" in str( + importer.issues()[0], + ) + + def test_import_delete(self, superuser): + preload_import("quota_critical_event_CREATE.xml", __file__, True) + importer = preload_import("quota_critical_event_DELETE.xml", __file__) + + assert len(importer.issues()) == 0 + + assert QuotaEvent.objects.all().count() == 2 diff --git a/taric_parsers/tests/quota_parser/test_quota_definition_parser.py b/taric_parsers/tests/quota_parser/test_quota_definition_parser.py new file mode 100644 index 000000000..04bb4b590 --- /dev/null +++ b/taric_parsers/tests/quota_parser/test_quota_definition_parser.py @@ -0,0 +1,181 @@ +from datetime import date + +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from quotas.models import QuotaDefinition +from taric_parsers.parsers.measure_parser import * +from taric_parsers.parsers.quota_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestQuotaDefinitionParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + + + + + + + + + + """ + + target_parser_class = QuotaDefinitionParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "quota_definition_sid": "123", + "quota_order_number_id": "010101", + "validity_start_date": "2020-01-01", + "validity_end_date": "2020-12-01", + "quota_order_number_sid": "234", + "volume": "33000", + "initial_volume": "23000", + "monetary_unit_code": "ABC", + "measurement_unit_code": "EFG", + "measurement_unit_qualifier_code": "Z", + "maximum_precision": "7", + "critical_state": "0", + "critical_threshold": "7", + "description": "Some Description", + } + + target = self.target_parser_class() + + target.populate( + 1, + target.record_code, + target.subrecord_code, + 1, + expected_data_example, + ) + + assert target.sid == 123 + assert target.order_number__order_number == "010101" + assert target.valid_between_lower == date(2020, 1, 1) + assert target.valid_between_upper == date(2020, 12, 1) + assert target.order_number__sid == 234 + assert target.volume == 33000 + assert target.initial_volume == 23000 + assert target.monetary_unit__code == "ABC" + assert target.measurement_unit__code == "EFG" + assert target.measurement_unit_qualifier__code == "Z" + assert target.maximum_precision == 7 + assert target.quota_critical == False + assert target.quota_critical_threshold == 7 + assert target.description == "Some Description" + + def test_import(self, superuser): + importer = preload_import("quota_definition_CREATE.xml", __file__) + + # check - measurement unit variant + assert len(importer.parsed_transactions) == 5 + + target_message = importer.parsed_transactions[4].parsed_messages[0] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + # check properties + assert target.sid == 99 + assert target.order_number__order_number == "054515" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + assert target.order_number__sid == 7 + assert target.volume == 1200.0 + assert target.initial_volume == 200.0 + assert target.monetary_unit__code == None + assert target.measurement_unit__code == "XXX" + assert target.measurement_unit_qualifier__code == "A" + assert target.maximum_precision == 3 + assert target.quota_critical is False + assert target.quota_critical_threshold == 75 + assert target.description == "Some Description" + + # Check monetary unit variant + target_message = importer.parsed_transactions[4].parsed_messages[1] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + # check properties + assert target.sid == 100 + assert target.order_number__order_number == "054515" + assert target.valid_between_lower == date(2023, 1, 1) + assert target.valid_between_upper == date(2024, 1, 1) + assert target.order_number__sid == 7 + assert target.volume == 1200.0 + assert target.initial_volume == 200.0 + assert target.monetary_unit__code == "ZZZ" + assert target.measurement_unit__code is None + assert target.measurement_unit_qualifier__code is None + assert target.maximum_precision == 3 + assert target.quota_critical is False + assert target.quota_critical_threshold == 75 + assert target.description == "Some Description" + + assert len(importer.issues()) == 0 + + assert QuotaDefinition.objects.all().count() == 2 + + def test_import_update(self, superuser): + preload_import("quota_definition_CREATE.xml", __file__, True) + importer = preload_import("quota_definition_UPDATE.xml", __file__) + + # check - measurement unit variant + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target = target_message.taric_object + + # check properties + assert target.sid == 100 + assert target.order_number__order_number == "054515" + assert target.valid_between_lower == date(2023, 1, 11) + assert target.valid_between_upper == date(2024, 1, 1) + assert target.order_number__sid == 7 + assert target.volume == 1200.0 + assert target.initial_volume == 200.0 + assert target.monetary_unit__code == "ZZZ" + assert target.measurement_unit__code is None + assert target.measurement_unit_qualifier__code is None + assert target.maximum_precision == 3 + assert target.quota_critical is False + assert target.quota_critical_threshold == 75 + assert target.description == "Some Description with changes" + + assert importer.issues() == [] + + assert QuotaDefinition.objects.all().count() == 3 + + def test_import_delete(self, superuser): + preload_import("quota_definition_CREATE.xml", __file__, True) + importer = preload_import("quota_definition_DELETE.xml", __file__) + + assert importer.issues() == [] + + assert QuotaDefinition.objects.all().count() == 3 diff --git a/taric_parsers/tests/quota_parser/test_quota_exhaustion_event_parser.py b/taric_parsers/tests/quota_parser/test_quota_exhaustion_event_parser.py new file mode 100644 index 000000000..56722db65 --- /dev/null +++ b/taric_parsers/tests/quota_parser/test_quota_exhaustion_event_parser.py @@ -0,0 +1,106 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from quotas.models import QuotaEvent +from taric_parsers.parsers.measure_parser import * +from taric_parsers.parsers.quota_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestQuotaEventParserV2: + """ + Could be one of any of the below: + + .. code:: XML + + + + + + + + + + + """ + + target_parser_class = QuotaExhaustionEventParserV2 + + def test_it_handles_population_from_expected_data_structure_quota_balance_event( + self, + ): + expected_data_example = { + "quota_definition_sid": "123", + "occurrence_timestamp": "2020-09-25T14:58:58", + "exhaustion_date": "2020-01-01", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.occurrence_timestamp == datetime(2020, 9, 25, 14, 58, 58) + assert target.quota_definition__sid == 123 + assert target.exhaustion_date == "2020-01-01" + + assert target.data == '{"exhaustion.date": "2020-01-01"}' + + def test_import(self, superuser): + importer = preload_import( + "quota_exhaustion_event_CREATE.xml", + __file__, + ) + + assert len(importer.parsed_transactions) == 6 + assert len(importer.parsed_transactions[5].parsed_messages) == 1 + + target_message = importer.parsed_transactions[5].parsed_messages[0] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.quota_definition__sid == 100 + assert target.occurrence_timestamp == datetime(2020, 9, 25, 14, 58, 58) + assert target.exhaustion_date == "2021-01-01" + + assert target.data == '{"exhaustion.date": "2021-01-01"}' + + assert len(importer.issues()) == 0 + + assert ( + QuotaEvent.objects.all() + .filter(subrecord_code=self.target_parser_class.subrecord_code) + .count() + == 1 + ) + + def test_import_update(self, superuser): + preload_import("quota_exhaustion_event_CREATE.xml", __file__, True) + importer = preload_import("quota_exhaustion_event_UPDATE.xml", __file__) + + assert len(importer.issues()) == 1 + + assert "Taric objects of type QuotaEvent can't be updated" in str( + importer.issues()[0], + ) + + def test_import_delete(self, superuser): + preload_import("quota_exhaustion_event_CREATE.xml", __file__, True) + importer = preload_import("quota_exhaustion_event_DELETE.xml", __file__) + + assert len(importer.issues()) == 0 + + assert QuotaEvent.objects.all().count() == 2 diff --git a/taric_parsers/tests/quota_parser/test_quota_order_number_origin_exclusion_parser.py b/taric_parsers/tests/quota_parser/test_quota_order_number_origin_exclusion_parser.py new file mode 100644 index 000000000..fcddaee74 --- /dev/null +++ b/taric_parsers/tests/quota_parser/test_quota_order_number_origin_exclusion_parser.py @@ -0,0 +1,108 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from quotas.models import QuotaOrderNumberOriginExclusion +from taric_parsers.parsers.geo_area_parser import * +from taric_parsers.parsers.quota_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestQuotaOrderNumberOriginExclusionParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + """ + + target_parser_class = QuotaOrderNumberOriginExclusionParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "quota_order_number_origin_sid": "30", + "excluded_geographical_area_sid": "10", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.origin__sid == 30 + assert target.excluded_geographical_area__sid == 10 + + def test_import(self, superuser): + importer = preload_import( + "quota_order_number_origin_exclusion_CREATE.xml", + __file__, + ) + + assert len(importer.parsed_transactions) == 5 + + target_message = importer.parsed_transactions[4].parsed_messages[0] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + # check properties + target = target_message.taric_object + assert target.origin__sid == 123 + assert target.excluded_geographical_area__sid == 8 + + assert len(importer.issues()) == 0 + assert QuotaOrderNumberOriginExclusion.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import( + "quota_order_number_origin_exclusion_CREATE.xml", + __file__, + True, + ) + importer = preload_import( + "quota_order_number_origin_exclusion_UPDATE.xml", + __file__, + ) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + # check properties + target = target_message.taric_object + assert target.origin__sid == 123 + assert target.excluded_geographical_area__sid == 8 + + assert len(importer.issues()) == 0 + + assert QuotaOrderNumberOriginExclusion.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import( + "quota_order_number_origin_exclusion_CREATE.xml", + __file__, + True, + ) + importer = preload_import( + "quota_order_number_origin_exclusion_DELETE.xml", + __file__, + ) + + assert len(importer.issues()) == 0 + + assert QuotaOrderNumberOriginExclusion.objects.all().count() == 2 diff --git a/taric_parsers/tests/quota_parser/test_quota_order_number_origin_parser.py b/taric_parsers/tests/quota_parser/test_quota_order_number_origin_parser.py new file mode 100644 index 000000000..93e65d9b8 --- /dev/null +++ b/taric_parsers/tests/quota_parser/test_quota_order_number_origin_parser.py @@ -0,0 +1,131 @@ +from datetime import date + +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from quotas.models import QuotaOrderNumberOrigin +from taric_parsers.parsers.geo_area_parser import * +from taric_parsers.parsers.quota_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestQuotaOrderNumberOriginParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + + """ + + target_parser_class = QuotaOrderNumberOriginParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "quota_order_number_origin_sid": "555", + "quota_order_number_sid": "123", + "geographical_area_id": "ABCD", # gets ignored, but will come in from import + "validity_start_date": "2020-01-01", + "validity_end_date": "2020-12-01", + "geographical_area_sid": "1234", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.sid == 555 # converts "certificate_code" to sid + assert target.order_number__sid == 123 # converts "certificate_code" to sid + assert target.geographical_area__sid == 1234 + assert target.geographical_area__area_id == "ABCD" + assert target.valid_between_lower == date(2020, 1, 1) + assert target.valid_between_upper == date(2020, 12, 1) + + def test_import(self, superuser): + importer = preload_import( + "quota_order_number_origin_CREATE.xml", + __file__, + ) + + assert len(importer.parsed_transactions) == 4 + + target_message = importer.parsed_transactions[3].parsed_messages[0] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + # check properties + target = target_message.taric_object + + assert target.sid == 123 + assert target.order_number__sid == 7 + assert target.geographical_area__sid == 8 + assert target.geographical_area__area_id == "AB01" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + + assert len(importer.issues()) == 0 + assert QuotaOrderNumberOrigin.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import( + "quota_order_number_origin_CREATE.xml", + __file__, + True, + ) + importer = preload_import( + "quota_order_number_origin_UPDATE.xml", + __file__, + ) + target_message = importer.parsed_transactions[0].parsed_messages[0] + + # check properties + target = target_message.taric_object + + assert target.sid == 123 + assert target.order_number__sid == 7 + assert target.geographical_area__sid == 8 + assert target.geographical_area__area_id == "AB01" + assert target.valid_between_lower == date(2021, 1, 11) + assert target.valid_between_upper == date(2022, 1, 1) + + assert len(importer.issues()) == 0 + + assert QuotaOrderNumberOrigin.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import( + "quota_order_number_origin_CREATE.xml", + __file__, + True, + ) + importer = preload_import( + "quota_order_number_origin_DELETE.xml", + __file__, + ) + + assert len(importer.issues()) == 0 + + assert QuotaOrderNumberOrigin.objects.all().count() == 2 diff --git a/taric_parsers/tests/quota_parser/test_quota_order_number_parser.py b/taric_parsers/tests/quota_parser/test_quota_order_number_parser.py new file mode 100644 index 000000000..f7e95bb85 --- /dev/null +++ b/taric_parsers/tests/quota_parser/test_quota_order_number_parser.py @@ -0,0 +1,103 @@ +from datetime import date + +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from quotas.models import QuotaOrderNumber +from taric_parsers.parsers.quota_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestQuotaOrderNumberParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + """ + + target_parser_class = QuotaOrderNumberParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "quota_order_number_sid": "7", + "quota_order_number_id": "010000", + "validity_start_date": "2020-01-01", + "validity_end_date": "2020-12-01", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.sid == 7 + assert target.order_number == "010000" + assert target.valid_between_lower == date(2020, 1, 1) + assert target.valid_between_upper == date(2020, 12, 1) + + def test_import(self, superuser): + importer = preload_import("quota_order_number_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.sid == 7 + assert target.order_number == "054515" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper is None + + assert len(importer.issues()) == 0 + + assert QuotaOrderNumber.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("quota_order_number_CREATE.xml", __file__, True) + importer = preload_import("quota_order_number_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target = target_message.taric_object + + assert target.sid == 7 + assert target.order_number == "054515" + assert target.valid_between_lower == date(2021, 1, 11) + assert target.valid_between_upper is None + + assert len(importer.issues()) == 0 + + assert QuotaOrderNumber.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("quota_order_number_CREATE.xml", __file__, True) + importer = preload_import("quota_order_number_DELETE.xml", __file__) + + assert len(importer.issues()) == 0 + + assert QuotaOrderNumber.objects.all().count() == 2 diff --git a/taric_parsers/tests/quota_parser/test_quota_reopening_event_parser.py b/taric_parsers/tests/quota_parser/test_quota_reopening_event_parser.py new file mode 100644 index 000000000..1b71b595a --- /dev/null +++ b/taric_parsers/tests/quota_parser/test_quota_reopening_event_parser.py @@ -0,0 +1,103 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from quotas.models import QuotaEvent +from taric_parsers.parsers.measure_parser import * +from taric_parsers.parsers.quota_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestQuotaEventParserV2: + """ + Could be one of any of the below: + + .. code:: XML + + + + + + + + + + + """ + + target_parser_class = QuotaReopeningEventParserV2 + + def test_it_handles_population_from_expected_data_structure_quota_balance_event( + self, + ): + expected_data_example = { + "quota_definition_sid": "123", + "occurrence_timestamp": "2020-09-25T14:58:58", + "reopening_date": "2020-01-01", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.occurrence_timestamp == datetime(2020, 9, 25, 14, 58, 58) + assert target.quota_definition__sid == 123 + assert target.reopening_date == "2020-01-01" + + assert target.data == '{"reopening.date": "2020-01-01"}' + + def test_import(self, superuser): + importer = preload_import("quota_reopening_event_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 6 + assert len(importer.parsed_transactions[5].parsed_messages) == 1 + + target_message = importer.parsed_transactions[5].parsed_messages[0] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.quota_definition__sid == 100 + assert target.occurrence_timestamp == datetime(2020, 9, 25, 14, 58, 58) + assert target.reopening_date == "2021-01-01" + + assert target.data == '{"reopening.date": "2021-01-01"}' + + assert len(importer.issues()) == 0 + + assert ( + QuotaEvent.objects.all() + .filter(subrecord_code=self.target_parser_class.subrecord_code) + .count() + == 1 + ) + + def test_import_update(self, superuser): + preload_import("quota_reopening_event_CREATE.xml", __file__, True) + importer = preload_import("quota_reopening_event_UPDATE.xml", __file__) + + assert len(importer.issues()) == 1 + + assert "Taric objects of type QuotaEvent can't be updated" in str( + importer.issues()[0], + ) + + def test_import_delete(self, superuser): + preload_import("quota_reopening_event_CREATE.xml", __file__, True) + importer = preload_import("quota_reopening_event_DELETE.xml", __file__) + + assert len(importer.issues()) == 0 + + assert QuotaEvent.objects.all().count() == 2 diff --git a/taric_parsers/tests/quota_parser/test_quota_suspension_parser.py b/taric_parsers/tests/quota_parser/test_quota_suspension_parser.py new file mode 100644 index 000000000..61e7cc54f --- /dev/null +++ b/taric_parsers/tests/quota_parser/test_quota_suspension_parser.py @@ -0,0 +1,106 @@ +from datetime import date + +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from quotas.models import QuotaSuspension +from taric_parsers.parsers.measure_parser import * +from taric_parsers.parsers.quota_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestQuotaSuspensionParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + """ + + target_parser_class = QuotaSuspensionParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "quota_suspension_period_sid": "456", + "quota_definition_sid": "123", + "suspension_start_date": "2020-01-01", + "suspension_end_date": "2021-01-01", + "description": "Some Description", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.sid == 456 + assert target.quota_definition__sid == 123 + assert target.valid_between_lower == date(2020, 1, 1) + assert target.valid_between_upper == date(2021, 1, 1) + assert target.description == "Some Description" + + def test_import(self, superuser): + importer = preload_import("quota_suspension_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 6 + + target_message = importer.parsed_transactions[5].parsed_messages[0] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.sid == 123 + assert target.quota_definition__sid == 99 + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + assert target.description == "Some Description" + + assert len(importer.issues()) == 0 + assert QuotaSuspension.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import("quota_suspension_CREATE.xml", __file__, True) + importer = preload_import("quota_suspension_UPDATE.xml", __file__) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target = target_message.taric_object + + assert target.sid == 123 + assert target.quota_definition__sid == 99 + assert target.valid_between_lower == date(2021, 1, 11) + assert target.valid_between_upper == date(2022, 1, 1) + assert target.description == "Some Description with changes" + + assert len(importer.issues()) == 0 + assert QuotaSuspension.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import("quota_suspension_CREATE.xml", __file__, True) + importer = preload_import("quota_suspension_DELETE.xml", __file__) + + assert len(importer.issues()) == 0 + assert QuotaSuspension.objects.all().count() == 2 diff --git a/taric_parsers/tests/quota_parser/test_quota_unblocking_event_parser.py b/taric_parsers/tests/quota_parser/test_quota_unblocking_event_parser.py new file mode 100644 index 000000000..5fba261a5 --- /dev/null +++ b/taric_parsers/tests/quota_parser/test_quota_unblocking_event_parser.py @@ -0,0 +1,105 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from quotas.models import QuotaEvent +from taric_parsers.parsers.measure_parser import * +from taric_parsers.parsers.quota_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestQuotaEventParserV2: + """ + Could be one of any of the below: + + .. code:: XML + + + + + + + + + + + """ + + target_parser_class = QuotaUnblockingEventParserV2 + + def test_it_handles_population_from_expected_data_structure_quota_balance_event( + self, + ): + expected_data_example = { + "quota_definition_sid": "123", + "occurrence_timestamp": "2020-09-25T14:58:58", + "unblocking_date": "2021-07-04", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.occurrence_timestamp == datetime(2020, 9, 25, 14, 58, 58) + assert target.quota_definition__sid == 123 + assert target.unblocking_date == "2021-07-04" + + assert target.data == '{"unblocking.date": "2021-07-04"}' + + def test_import(self, superuser): + importer = preload_import( + "quota_unblocking_event_CREATE.xml", + __file__, + ) + + assert len(importer.parsed_transactions) == 6 + assert len(importer.parsed_transactions[5].parsed_messages) == 1 + + target_message = importer.parsed_transactions[5].parsed_messages[0] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.quota_definition__sid == 100 + assert target.occurrence_timestamp == datetime(2020, 9, 25, 14, 58, 58) + assert target.unblocking_date == "2021-01-01" + + assert target.data == '{"unblocking.date": "2021-01-01"}' + + assert len(importer.issues()) == 0 + + assert ( + QuotaEvent.objects.all() + .filter(subrecord_code=self.target_parser_class.subrecord_code) + .count() + == 1 + ) + + def test_import_update(self, superuser): + preload_import("quota_unblocking_event_CREATE.xml", __file__, True) + importer = preload_import("quota_unblocking_event_UPDATE.xml", __file__) + + assert len(importer.issues()) == 1 + + assert "Taric objects of type QuotaEvent can't be updated" in str( + importer.issues()[0], + ) + + def test_import_delete(self, superuser): + preload_import("quota_unblocking_event_CREATE.xml", __file__, True) + importer = preload_import("quota_unblocking_event_DELETE.xml", __file__) + + assert len(importer.issues()) == 0 + assert QuotaEvent.objects.all().count() == 2 diff --git a/taric_parsers/tests/quota_parser/test_quota_unsuspension_event_parser.py b/taric_parsers/tests/quota_parser/test_quota_unsuspension_event_parser.py new file mode 100644 index 000000000..7d29f33b5 --- /dev/null +++ b/taric_parsers/tests/quota_parser/test_quota_unsuspension_event_parser.py @@ -0,0 +1,106 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from quotas.models import QuotaEvent +from taric_parsers.parsers.measure_parser import * +from taric_parsers.parsers.quota_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestQuotaEventParserV2: + """ + Could be one of any of the below: + + .. code:: XML + + + + + + + + + + + """ + + target_parser_class = QuotaUnsuspensionEventParserV2 + + def test_it_handles_population_from_expected_data_structure_quota_balance_event( + self, + ): + expected_data_example = { + "quota_definition_sid": "123", + "occurrence_timestamp": "2020-09-25T14:58:58", + "unsuspension_date": "2021-07-04", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.occurrence_timestamp == datetime(2020, 9, 25, 14, 58, 58) + assert target.quota_definition__sid == 123 + assert target.unsuspension_date == "2021-07-04" + + assert target.data == '{"unsuspension.date": "2021-07-04"}' + + def test_import(self, superuser): + importer = preload_import( + "quota_unsuspension_event_CREATE.xml", + __file__, + ) + + assert len(importer.parsed_transactions) == 6 + assert len(importer.parsed_transactions[5].parsed_messages) == 1 + + target_message = importer.parsed_transactions[5].parsed_messages[0] + + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + assert target.quota_definition__sid == 100 + assert target.occurrence_timestamp == datetime(2020, 9, 25, 14, 58, 58) + assert target.unsuspension_date == "2021-01-01" + + assert target.data == '{"unsuspension.date": "2021-01-01"}' + + assert len(importer.issues()) == 0 + + assert ( + QuotaEvent.objects.all() + .filter(subrecord_code=self.target_parser_class.subrecord_code) + .count() + == 1 + ) + + def test_import_update(self, superuser): + preload_import("quota_unsuspension_event_CREATE.xml", __file__, True) + importer = preload_import("quota_unsuspension_event_UPDATE.xml", __file__) + + assert len(importer.issues()) == 1 + + assert "Taric objects of type QuotaEvent can't be updated" in str( + importer.issues()[0], + ) + + def test_import_delete(self, superuser): + preload_import("quota_unsuspension_event_CREATE.xml", __file__, True) + importer = preload_import("quota_unsuspension_event_DELETE.xml", __file__) + + assert len(importer.issues()) == 0 + + assert QuotaEvent.objects.all().count() == 2 diff --git a/taric_parsers/tests/regulation_parsers/importer_examples/base_regulation_CREATE.xml b/taric_parsers/tests/regulation_parsers/importer_examples/base_regulation_CREATE.xml new file mode 100644 index 000000000..9a9b21ca5 --- /dev/null +++ b/taric_parsers/tests/regulation_parsers/importer_examples/base_regulation_CREATE.xml @@ -0,0 +1,69 @@ + + + + + + + 1 + 150 + 00 + 3 + 3 + + ABC + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 150 + 05 + 3 + 3 + + ABC + ZZ + Some Description x + + + + + + + + + + 1 + 285 + 00 + 3 + 3 + + 1 + Z0000001 + 2023-01-01 + ABCDE + 7 + 2023-01-01 + 1 + 7 + 0 + Some Info Text + 1 + ABC + 2021-01-01 + 2022-01-01 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/regulation_parsers/importer_examples/base_regulation_DELETE.xml b/taric_parsers/tests/regulation_parsers/importer_examples/base_regulation_DELETE.xml new file mode 100644 index 000000000..83a8f572d --- /dev/null +++ b/taric_parsers/tests/regulation_parsers/importer_examples/base_regulation_DELETE.xml @@ -0,0 +1,33 @@ + + + + + + + 1 + 285 + 00 + 3 + 2 + + 1 + Z0000001 + 2023-01-11 + ABCDE + 7 + 2023-01-11 + 1 + 7 + 0 + Some Info Text with changes + 1 + ABC + 2021-01-11 + 2022-01-11 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/regulation_parsers/importer_examples/base_regulation_UPDATE.xml b/taric_parsers/tests/regulation_parsers/importer_examples/base_regulation_UPDATE.xml new file mode 100644 index 000000000..c7933cd20 --- /dev/null +++ b/taric_parsers/tests/regulation_parsers/importer_examples/base_regulation_UPDATE.xml @@ -0,0 +1,33 @@ + + + + + + + 1 + 285 + 00 + 3 + 1 + + 1 + Z0000001 + 2023-01-11 + ABCDE + 7 + 2023-01-11 + 1 + 7 + 0 + Some Info Text with changes + 1 + ABC + 2021-01-11 + 2022-01-11 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/regulation_parsers/importer_examples/fts_regulation_action_CREATE.xml b/taric_parsers/tests/regulation_parsers/importer_examples/fts_regulation_action_CREATE.xml new file mode 100644 index 000000000..f25ca025b --- /dev/null +++ b/taric_parsers/tests/regulation_parsers/importer_examples/fts_regulation_action_CREATE.xml @@ -0,0 +1,117 @@ + + + + + + + 1 + 150 + 00 + 3 + 3 + + ABC + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 150 + 05 + 3 + 3 + + ABC + ZZ + Some Description x + + + + + + + + + + 1 + 285 + 00 + 3 + 3 + + 1 + AB123400 + 2023-01-01 + ABCDE + 7 + 2023-01-01 + 1 + 7 + 0 + Some Info Text + 1 + ABC + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 285 + 00 + 3 + 3 + + 3 + CD567800 + 2023-01-01 + ABCDE + 7 + 2023-01-01 + 1 + 7 + 0 + Some Info Text + 1 + ABC + 2021-01-01 + 2022-01-01 + + + + + + + + + + 1 + 305 + 00 + 3 + 3 + + 1 + AB123400 + 3 + CD567800 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/regulation_parsers/importer_examples/regulation_group_CREATE.xml b/taric_parsers/tests/regulation_parsers/importer_examples/regulation_group_CREATE.xml new file mode 100644 index 000000000..0b06b40a9 --- /dev/null +++ b/taric_parsers/tests/regulation_parsers/importer_examples/regulation_group_CREATE.xml @@ -0,0 +1,39 @@ + + + + + + + 1 + 150 + 00 + 3 + 3 + + ABC + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 150 + 05 + 3 + 3 + + ABC + ZZ + Some Description x + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/regulation_parsers/importer_examples/regulation_group_DELETE.xml b/taric_parsers/tests/regulation_parsers/importer_examples/regulation_group_DELETE.xml new file mode 100644 index 000000000..3caae8d5f --- /dev/null +++ b/taric_parsers/tests/regulation_parsers/importer_examples/regulation_group_DELETE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 150 + 00 + 3 + 2 + + ABC + 2021-01-11 + 2022-01-11 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/regulation_parsers/importer_examples/regulation_group_UPDATE.xml b/taric_parsers/tests/regulation_parsers/importer_examples/regulation_group_UPDATE.xml new file mode 100644 index 000000000..8c94db12d --- /dev/null +++ b/taric_parsers/tests/regulation_parsers/importer_examples/regulation_group_UPDATE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 150 + 00 + 3 + 1 + + ABC + 2021-01-11 + 2022-01-11 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/regulation_parsers/importer_examples/regulation_group_description_CREATE.xml b/taric_parsers/tests/regulation_parsers/importer_examples/regulation_group_description_CREATE.xml new file mode 100644 index 000000000..0b06b40a9 --- /dev/null +++ b/taric_parsers/tests/regulation_parsers/importer_examples/regulation_group_description_CREATE.xml @@ -0,0 +1,39 @@ + + + + + + + 1 + 150 + 00 + 3 + 3 + + ABC + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 150 + 05 + 3 + 3 + + ABC + ZZ + Some Description x + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/regulation_parsers/importer_examples/regulation_group_description_DELETE.xml b/taric_parsers/tests/regulation_parsers/importer_examples/regulation_group_description_DELETE.xml new file mode 100644 index 000000000..ae10a1a6c --- /dev/null +++ b/taric_parsers/tests/regulation_parsers/importer_examples/regulation_group_description_DELETE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 150 + 05 + 3 + 2 + + ABC + ZZ + Some Description with changes + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/regulation_parsers/importer_examples/regulation_group_description_UPDATE.xml b/taric_parsers/tests/regulation_parsers/importer_examples/regulation_group_description_UPDATE.xml new file mode 100644 index 000000000..1ec020a7e --- /dev/null +++ b/taric_parsers/tests/regulation_parsers/importer_examples/regulation_group_description_UPDATE.xml @@ -0,0 +1,22 @@ + + + + + + + 1 + 150 + 05 + 3 + 1 + + ABC + ZZ + Some Description with changes + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/regulation_parsers/importer_examples/regulation_replacement_CREATE.xml b/taric_parsers/tests/regulation_parsers/importer_examples/regulation_replacement_CREATE.xml new file mode 100644 index 000000000..79d7bec48 --- /dev/null +++ b/taric_parsers/tests/regulation_parsers/importer_examples/regulation_replacement_CREATE.xml @@ -0,0 +1,222 @@ + + + + + + + 1 + 150 + 00 + 3 + 3 + + ABC + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 150 + 05 + 3 + 3 + + ABC + ZZ + Some Description x + + + + + + + + + + 1 + 285 + 00 + 3 + 3 + + 1 + Z0000001 + 2023-01-01 + ABCDE + 7 + 2023-01-01 + 1 + 7 + 0 + Some Info Text + 1 + ABC + 2021-01-01 + 2022-01-01 + + + + + + + + 1 + 285 + 00 + 3 + 3 + + 3 + Z0000002 + 2023-01-01 + ABCDE + 7 + 2023-01-01 + 1 + 7 + 0 + Some Info Text + 1 + ABC + 2021-01-01 + 2022-01-01 + + + + + + + + + + 1 + 140 + 00 + 3 + 3 + + A + 2021-01-01 + 2022-01-01 + 6 + + + + + + + + 1 + 140 + 05 + 3 + 3 + + A + zz + Some Description + + + + + + + + + + 1 + 235 + 00 + 3 + 3 + + ZZZ + 2021-01-01 + 2022-01-01 + 1 + 2 + 3 + 4 + 5 + 6 + A + + + + + + + + 1 + 235 + 05 + 3 + 3 + + ZZZ + ZZ + Some Description x + + + + + + + + + + 1 + 250 + 00 + 3 + 3 + + 8 + AB01 + 2021-01-01 + 2022-01-01 + 1 + + + + + + + + + + 1 + 305 + 00 + 3 + 3 + + 1 + Z0000001 + 3 + Z0000002 + ZZZ + AB01 + 09 + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/regulation_parsers/test_base_regulation_parser.py b/taric_parsers/tests/regulation_parsers/test_base_regulation_parser.py new file mode 100644 index 000000000..2ac166e56 --- /dev/null +++ b/taric_parsers/tests/regulation_parsers/test_base_regulation_parser.py @@ -0,0 +1,182 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from regulations.models import Group +from regulations.models import Regulation +from taric_parsers.parsers.additional_code_parsers import * +from taric_parsers.parsers.geo_area_parser import * +from taric_parsers.parsers.regulation_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestBaseRegulationParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + + + + + + + + + + + + + + + + """ + + target_parser_class = BaseRegulationParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "base_regulation_role": "1", + "base_regulation_id": "Z0000001", + "published_date": "2023-01-01", + "officialjournal_number": "ABCDE", + "officialjournal_page": "7", + "validity_start_date": "2021-01-01", + "validity_end_date": "2022-01-01", + "effective_end_date": "2023-01-01", + "community_code": "1", + "regulation_group_id": "ABC", + "replacement_indicator": "7", + "stopped_flag": "0", + "information_text": "Some Info Text", + "approved_flag": "1", + "antidumping_regulation_role": "1", + "related_antidumping_regulation_id": "A00000001", + "complete_abrogation_regulation_role": "1", + "complete_abrogation_regulation_id": "B00000001", + "explicit_abrogation_regulation_role": "1", + "explicit_abrogation_regulation_id": "C00000001", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + assert target.role_type == 1 + assert target.regulation_id == "Z0000001" + assert target.published_at == date(2023, 1, 1) + assert target.official_journal_number == "ABCDE" + assert target.official_journal_page == 7 + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + assert target.effective_end_date == date(2023, 1, 1) + assert target.community_code == 1 + assert target.regulation_group__group_id == "ABC" + assert target.replacement_indicator == 7 + assert target.stopped is False + assert target.information_text == "Some Info Text" + assert target.approved is True + + def test_import(self, superuser): + importer = preload_import( + "base_regulation_CREATE.xml", + __file__, + ) + + assert len(importer.parsed_transactions) == 2 + + target_message = importer.parsed_transactions[1].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + # verify all properties + assert target.role_type == 1 + assert target.regulation_id == "Z0000001" + assert target.published_at == date(2023, 1, 1) + assert target.official_journal_number == "ABCDE" + assert target.official_journal_page == 7 + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + assert target.effective_end_date == date(2023, 1, 1) + assert target.community_code == 1 + assert target.regulation_group__group_id == "ABC" + assert target.replacement_indicator == 7 + assert target.stopped is False + assert target.information_text == "Some Info Text" + assert target.approved is True + + assert len(importer.issues()) == 0 + + assert Group.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import( + "base_regulation_CREATE.xml", + __file__, + True, + ) + importer = preload_import( + "base_regulation_UPDATE.xml", + __file__, + ) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + target = target_message.taric_object + + # verify all properties + assert target.role_type == 1 + assert target.regulation_id == "Z0000001" + assert target.published_at == date(2023, 1, 11) + assert target.official_journal_number == "ABCDE" + assert target.official_journal_page == 7 + assert target.valid_between_lower == date(2021, 1, 11) + assert target.valid_between_upper == date(2022, 1, 11) + assert target.effective_end_date == date(2023, 1, 11) + assert target.community_code == 1 + assert target.regulation_group__group_id == "ABC" + assert target.replacement_indicator == 7 + assert target.stopped is False + assert target.information_text == "Some Info Text with changes" + assert target.approved is True + + assert len(importer.issues()) == 0 + + assert Regulation.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import( + "base_regulation_CREATE.xml", + __file__, + True, + ) + importer = preload_import( + "base_regulation_DELETE.xml", + __file__, + ) + + assert len(importer.issues()) == 0 + + assert Regulation.objects.all().count() == 2 diff --git a/taric_parsers/tests/regulation_parsers/test_full_temporary_stop_action_parser.py b/taric_parsers/tests/regulation_parsers/test_full_temporary_stop_action_parser.py new file mode 100644 index 000000000..5a66cb778 --- /dev/null +++ b/taric_parsers/tests/regulation_parsers/test_full_temporary_stop_action_parser.py @@ -0,0 +1,76 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from regulations.models import Suspension +from taric_parsers.parsers.regulation_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestFullTemporaryStopActionParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + """ + + target_parser_class = FullTemporaryStopActionParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "fts_regulation_role": "1", + "fts_regulation_id": "AB123400", + "stopped_regulation_role": "3", + "stopped_regulation_id": "CD567800", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.enacting_regulation__role_type == "1" + assert target.enacting_regulation__regulation_id == "AB123400" + assert target.target_regulation__role_type == "3" + assert target.target_regulation__regulation_id == "CD567800" + + def test_import(self, superuser): + importer = preload_import("fts_regulation_action_CREATE.xml", __file__) + + assert len(importer.parsed_transactions) == 3 + + target_message = importer.parsed_transactions[2].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + # verify all properties + assert target.enacting_regulation__role_type == "1" + assert target.enacting_regulation__regulation_id == "AB123400" + assert target.target_regulation__role_type == "3" + assert target.target_regulation__regulation_id == "CD567800" + + assert len(importer.issues()) == 0 + + assert Suspension.objects.all().count() == 1 diff --git a/taric_parsers/tests/regulation_parsers/test_full_temporary_stop_regulation_parser.py b/taric_parsers/tests/regulation_parsers/test_full_temporary_stop_regulation_parser.py new file mode 100644 index 000000000..0b695f2d2 --- /dev/null +++ b/taric_parsers/tests/regulation_parsers/test_full_temporary_stop_regulation_parser.py @@ -0,0 +1,98 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from taric_parsers.parsers.additional_code_parsers import * +from taric_parsers.parsers.geo_area_parser import * +from taric_parsers.parsers.regulation_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestFullTemporaryStopRegulationParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + + + + + + + + + + + "full.temporary.stop.regulation.role": "", + "full.temporary.stop.regulation.id": "", + "published.date": "", + "officialjournal.number": "", + "officialjournal.page": "", + "validity.start.date": "", + "validity.end.date": "", + "effective.enddate": "", + "complete.abrogation.regulation.role": "", + "complete.abrogation.regulation.id": "", + "explicit.abrogation.regulation.role": "", + "explicit.abrogation.regulation.id": "", + "replacement.indicator": "", + "information.text": "", + "approved.flag": "", + """ + + target_parser_class = FullTemporaryStopRegulationParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "full_temporary_stop_regulation_role": "3", + "full_temporary_stop_regulation_id": "AB123400", + "published_date": "2020-01-01", + "officialjournal_number": "ZZ123", + "officialjournal_page": "123", + "validity_start_date": "2020-01-01", + "validity_end_date": "2021-01-01", + "effective_enddate": "2022-01-01", + "complete_abrogation_regulation_role": "3", + "complete_abrogation_regulation_id": "CD123123", + "explicit_abrogation_regulation_role": "3", + "explicit_abrogation_regulation_id": "EF123123", + "replacement_indicator": "0", + "information_text": "info text", + "approved_flag": "1", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.enacting_regulation__role_type == 3 + assert target.enacting_regulation__regulation_id == "AB123400" + assert target.enacting_regulation__published_at == date(2020, 1, 1) + assert target.enacting_regulation__official_journal_number == "ZZ123" + assert target.enacting_regulation__official_journal_page == 123 + assert target.enacting_regulation__valid_between_lower == date(2020, 1, 1) + assert target.enacting_regulation__valid_between_upper == date(2021, 1, 1) + assert target.effective_end_date == date(2022, 1, 1) + assert target.enacting_regulation__replacement_indicator == 0 + assert target.enacting_regulation__information_text == "info text" + assert target.enacting_regulation__approved is True diff --git a/taric_parsers/tests/regulation_parsers/test_modification_regulation_parser.py b/taric_parsers/tests/regulation_parsers/test_modification_regulation_parser.py new file mode 100644 index 000000000..ea6fc50eb --- /dev/null +++ b/taric_parsers/tests/regulation_parsers/test_modification_regulation_parser.py @@ -0,0 +1,111 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from taric_parsers.parsers.additional_code_parsers import * +from taric_parsers.parsers.geo_area_parser import * +from taric_parsers.parsers.regulation_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestModificationRegulationParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + + + + + + + + + + + + + + + "modification_regulation_role":"", + "modification_regulation_id":"", + "published_date" type="Date":"", + "officialjournal_number":"", + "officialjournal_page":"", + "validity_start_date":"", + "validity_end_date":"", + "effective_end_date":"", + "base_regulation_role":"", + "base_regulation_id":"", + "complete_abrogation_regulation_role":"", + "complete_abrogation_regulation_id":"", + "explicit_abrogation_regulation_role":"", + "explicit_abrogation_regulation_id":"", + "replacement_indicator":"", + "stopped_flag":"", + "information_text":"", + "approved_flag":"", + """ + + target_parser_class = ModificationRegulationParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "modification_regulation_role": "3", + "modification_regulation_id": "AB123123", + "published_date": "2022-01-01", + "officialjournal_number": "AB123", + "officialjournal_page": "77", + "validity_start_date": "2022-01-01", + "validity_end_date": "2023-01-01", + "effective_end_date": "2021-01-01", + "base_regulation_role": "1", + "base_regulation_id": "CD123123", + "complete_abrogation_regulation_role": "3", + "complete_abrogation_regulation_id": "EF123123", + "explicit_abrogation_regulation_role": "3", + "explicit_abrogation_regulation_id": "GH123123", + "replacement_indicator": "0", + "stopped_flag": "0", + "information_text": "info text", + "approved_flag": "1", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.enacting_regulation__role_type == 3 + assert target.enacting_regulation__regulation_id == "AB123123" + assert target.enacting_regulation__published_at == date(2022, 1, 1) + assert target.enacting_regulation__official_journal_number == "AB123" + assert target.enacting_regulation__official_journal_page == 77 + assert target.enacting_regulation__valid_between_lower == date(2022, 1, 1) + assert target.enacting_regulation__valid_between_upper == date(2023, 1, 1) + assert target.enacting_regulation__effective_end_date == date(2021, 1, 1) + assert target.target_regulation__role_type == 1 + assert target.target_regulation__regulation_id == "CD123123" + assert target.enacting_regulation__replacement_indicator == 0 + assert target.enacting_regulation__stopped is False + assert target.enacting_regulation__information_text == "info text" + assert target.enacting_regulation__approved is True diff --git a/taric_parsers/tests/regulation_parsers/test_regulation_group_description_parser.py b/taric_parsers/tests/regulation_parsers/test_regulation_group_description_parser.py new file mode 100644 index 000000000..747716347 --- /dev/null +++ b/taric_parsers/tests/regulation_parsers/test_regulation_group_description_parser.py @@ -0,0 +1,114 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from regulations.models import Group +from taric_parsers.parsers.regulation_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestRegulationGroupDescriptionParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + """ + + target_parser_class = RegulationGroupDescriptionParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "regulation_group_id": "ASD", + "language_id": "ZZ", + "description": "Some Description", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.group_id == "ASD" + assert target.description == "Some Description" + + def test_import(self, superuser): + importer = preload_import( + "regulation_group_description_CREATE.xml", + __file__, + ) + + assert len(importer.parsed_transactions) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[1] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + # verify all properties + assert target.group_id == "ABC" + assert target.description == "Some Description x" + + assert len(importer.issues()) == 0 + + assert Group.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import( + "regulation_group_description_CREATE.xml", + __file__, + True, + ) + importer = preload_import( + "regulation_group_description_UPDATE.xml", + __file__, + ) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target = target_message.taric_object + + # verify all properties + assert target.group_id == "ABC" + assert target.description == "Some Description with changes" + + assert len(importer.issues()) == 0 + + assert Group.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import( + "regulation_group_description_CREATE.xml", + __file__, + True, + ) + importer = preload_import( + "regulation_group_description_DELETE.xml", + __file__, + ) + + assert len(importer.issues()) == 1 + + assert ( + "Children of Taric objects of type Group can't be deleted directly" + in str(importer.issues()[0]) + ) diff --git a/taric_parsers/tests/regulation_parsers/test_regulation_group_parser.py b/taric_parsers/tests/regulation_parsers/test_regulation_group_parser.py new file mode 100644 index 000000000..7ca2851ee --- /dev/null +++ b/taric_parsers/tests/regulation_parsers/test_regulation_group_parser.py @@ -0,0 +1,118 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from regulations.models import Group +from taric_parsers.parsers.additional_code_parsers import * +from taric_parsers.parsers.geo_area_parser import * +from taric_parsers.parsers.regulation_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestRegulationGroupParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + """ + + target_parser_class = RegulationGroupParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "regulation_group_id": "ASD", + "validity_start_date": "2021-01-01", + "validity_end_date": "2022-01-01", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.group_id == "ASD" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + + def test_import(self, superuser): + importer = preload_import( + "regulation_group_CREATE.xml", + __file__, + ) + + assert len(importer.parsed_transactions) == 1 + + target_message = importer.parsed_transactions[0].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + # verify all properties + assert target.group_id == "ABC" + assert target.valid_between_lower == date(2021, 1, 1) + assert target.valid_between_upper == date(2022, 1, 1) + + assert len(importer.issues()) == 0 + + assert Group.objects.all().count() == 1 + + def test_import_update(self, superuser): + preload_import( + "regulation_group_CREATE.xml", + __file__, + True, + ) + + importer = preload_import( + "regulation_group_UPDATE.xml", + __file__, + ) + + target_message = importer.parsed_transactions[0].parsed_messages[0] + + target = target_message.taric_object + + # verify all properties + assert target.group_id == "ABC" + assert target.valid_between_lower == date(2021, 1, 11) + assert target.valid_between_upper == date(2022, 1, 11) + + assert len(importer.issues()) == 0 + + assert Group.objects.all().count() == 2 + + def test_import_delete(self, superuser): + preload_import( + "regulation_group_CREATE.xml", + __file__, + True, + ) + + importer = preload_import( + "regulation_group_DELETE.xml", + __file__, + ) + + assert len(importer.issues()) == 0 + + assert Group.objects.all().count() == 2 diff --git a/taric_parsers/tests/regulation_parsers/test_regulation_replacement_parser.py b/taric_parsers/tests/regulation_parsers/test_regulation_replacement_parser.py new file mode 100644 index 000000000..385797069 --- /dev/null +++ b/taric_parsers/tests/regulation_parsers/test_regulation_replacement_parser.py @@ -0,0 +1,94 @@ +import pytest + +# note : need to import these objects to make them available to the parser +from common.tests.util import preload_import +from regulations.models import Replacement +from taric_parsers.parsers.additional_code_parsers import * +from taric_parsers.parsers.geo_area_parser import * +from taric_parsers.parsers.measure_parser import * +from taric_parsers.parsers.regulation_parser import * + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestRegulationReplacementParserV2: + """ + Example XML: + + .. code-block:: XML + + + + + + + + + + + + + + + """ + + target_parser_class = RegulationReplacementParserV2 + + def test_it_handles_population_from_expected_data_structure(self): + expected_data_example = { + "replacing_regulation_role": "3", + "replacing_regulation_id": "AB123123", + "replaced_regulation_role": "1", + "replaced_regulation_id": "CD123123", + "measure_type_id": "ABC1", + "geographical_area_id": "AB12", + "chapter_heading": "09", + } + + target = self.target_parser_class() + + target.populate( + 1, # transaction id + target.record_code, + target.subrecord_code, + 1, # sequence number + expected_data_example, + ) + + # verify all properties + assert target.enacting_regulation__role_type == 3 + assert target.enacting_regulation__regulation_id == "AB123123" + assert target.target_regulation__role_type == 1 + assert target.target_regulation__regulation_id == "CD123123" + assert target.measure_type_id == "ABC1" + assert target.geographical_area_id == "AB12" + assert target.chapter_heading == "09" + + def test_import(self, superuser): + importer = preload_import( + "regulation_replacement_CREATE.xml", + __file__, + ) + + assert len(importer.parsed_transactions) == 6 + + target_message = importer.parsed_transactions[5].parsed_messages[0] + assert target_message.record_code == self.target_parser_class.record_code + assert target_message.subrecord_code == self.target_parser_class.subrecord_code + assert type(target_message.taric_object) == self.target_parser_class + + target = target_message.taric_object + + # verify all properties + assert target.enacting_regulation__role_type == 1 + assert target.enacting_regulation__regulation_id == "Z0000001" + assert target.target_regulation__role_type == 3 + assert target.target_regulation__regulation_id == "Z0000002" + assert target.measure_type_id == "ZZZ" + assert target.geographical_area_id == "AB01" + assert target.chapter_heading == "09" + + assert len(importer.issues()) == 0 + + assert Replacement.objects.all().count() == 1 diff --git a/taric_parsers/tests/support/additional_code_CREATE.xml b/taric_parsers/tests/support/additional_code_CREATE.xml new file mode 100644 index 000000000..d81d37516 --- /dev/null +++ b/taric_parsers/tests/support/additional_code_CREATE.xml @@ -0,0 +1,55 @@ + + + + + + + 1 + 120 + 00 + 1 + 3 + + 5 + 3 + 2021-01-01 + 2021-12-31 + + + + + + + + 1 + 245 + 00 + 2 + 3 + + 1 + 5 + 3 + 2021-01-01 + + + + + + + + 1 + 120 + 05 + 3 + 3 + + 5 + zz + some description + + + + + + \ No newline at end of file diff --git a/taric_parsers/tests/support/broken.xml b/taric_parsers/tests/support/broken.xml new file mode 100644 index 000000000..804a92202 --- /dev/null +++ b/taric_parsers/tests/support/broken.xml @@ -0,0 +1,18 @@ + + + + + + + 392602 + 400 + 00 + 1 + 1 + + 100460 + 0709999030 + 80 + 2016-09-15 + 2021-12-31 + 0 diff --git a/taric_parsers/tests/support/dtd.xml b/taric_parsers/tests/support/dtd.xml new file mode 100644 index 000000000..511c6fe00 --- /dev/null +++ b/taric_parsers/tests/support/dtd.xml @@ -0,0 +1,7 @@ + + + + + text + diff --git a/taric_parsers/tests/support/invalid.xml b/taric_parsers/tests/support/invalid.xml new file mode 100644 index 000000000..8c0c12cf9 --- /dev/null +++ b/taric_parsers/tests/support/invalid.xml @@ -0,0 +1,24 @@ + + + + + + + 392602 + 400 + 00 + 1 + 1 + + 100460 + 0709999030 + 80 + 2016-09-15 + 2021-12-31 + 0 + + + + + + diff --git a/taric_parsers/tests/support/invalid_type.txt b/taric_parsers/tests/support/invalid_type.txt new file mode 100644 index 000000000..cd0875583 --- /dev/null +++ b/taric_parsers/tests/support/invalid_type.txt @@ -0,0 +1 @@ +Hello world! diff --git a/taric_parsers/tests/support/valid.xml b/taric_parsers/tests/support/valid.xml new file mode 100644 index 000000000..8d4393761 --- /dev/null +++ b/taric_parsers/tests/support/valid.xml @@ -0,0 +1,24 @@ + + + + + + + 392602 + 400 + 00 + 1 + 1 + + 100460 + 0709999030 + 80 + 2016-09-15 + 2021-12-31 + 0 + + + + + + diff --git a/taric_parsers/tests/test_chunker.py b/taric_parsers/tests/test_chunker.py new file mode 100644 index 000000000..43009dbb6 --- /dev/null +++ b/taric_parsers/tests/test_chunker.py @@ -0,0 +1,167 @@ +import xml.etree.ElementTree as ET +from io import BytesIO +from os import path +from typing import Sequence +from unittest import mock + +import pytest +from django.core.files.uploadedfile import SimpleUploadedFile + +from common.tests import factories +from importer.models import ImporterXMLChunk +from taric_parsers import chunker +from taric_parsers.chunker import chunk_taric +from taric_parsers.chunker import filter_transaction_records +from taric_parsers.chunker import find_or_create_chunk +from taric_parsers.namespaces import TTags + +from .test_namespaces import get_snippet_transaction + +TEST_FILES_PATH = path.join(path.dirname(__file__), "test_files") + +pytestmark = pytest.mark.django_db + + +def get_chunk_opener(id: str) -> bytes: + return ( + '' + '' + ).encode() + + +def get_basic_chunk_text(id: str) -> bytes: + return get_chunk_opener(id) + b"" + + +def filter_snippet_transaction( + xml: str, + Tags: TTags, + record_group: Sequence[str], +) -> ET.Element: + """Returns a filtered transaction with matching records from a record_group + only.""" + transaction = get_snippet_transaction(xml, Tags) + return filter_transaction_records(transaction, record_group) + + +@pytest.mark.importer_v2 +@mock.patch("taric_parsers.chunker.TemporaryFile") +def test_get_chunk(mock_temp_file: mock.MagicMock): + """Asserts that the correct chunk is found or created for writing to.""" + mock_temp_file.side_effect = BytesIO + chunks_in_progress = {} + + chunk1 = find_or_create_chunk(chunks_in_progress, "1") + chunk2 = find_or_create_chunk( + chunks_in_progress, + "2", + record_code="400", + chapter_heading="01", + ) + chunk1.seek(0) + chunk2.seek(0) + + assert chunks_in_progress.get(None) == chunk1 + assert chunks_in_progress.get(("400", "01")) == chunk2 + assert chunk1.read() == get_chunk_opener("1") + assert chunk2.read() == get_chunk_opener("2") + + +@pytest.mark.importer_v2 +def test_close_chunk(): + """Asserts that chunks are properly closed and added to the batch.""" + batch = factories.ImportBatchFactory.create() + chunk1 = BytesIO() + chunk2 = BytesIO() + chunk1.write(get_chunk_opener("1")) + chunk2.write(get_chunk_opener("2")) + + assert batch.chunks.count() == 0 + + chunker.close_chunk(chunk1, batch, None) + chunker.close_chunk(chunk2, batch, ("400", "01")) + + assert batch.chunks.count() == 2 + assert ( + batch.chunks.get(record_code=None).chunk_text + == get_basic_chunk_text("1").decode() + ) + assert ( + batch.chunks.get(record_code=400, chapter="01").chunk_text + == get_basic_chunk_text("2").decode() + ) + + +@pytest.mark.importer_v2 +def test_filter_transaction_records_positive( + taric_schema_tags, + record_group, + envelope_commodity, +): + """Asserts that matching records from the record_group are preserved in the + transaction.""" + + # filter_snippet_transaction calls get_snippet_transaction, + # which gets the first transaction from an xml envelope, + # and then calls filter_transaction_records, which checks whether this transaction contains + # a record identifier matching a value in TARIC_RECORD_CODES["commodities"] + transaction = filter_snippet_transaction( + envelope_commodity, + taric_schema_tags, + record_group, + ) + + assert transaction is not None + assert len(transaction) == 1 + + +@pytest.mark.importer_v2 +def test_filter_transaction_records_negative( + taric_schema_tags, + record_group, + envelope_measure, +): + """Asserts that non-matching records from the record_group are removed from + the transaction.""" + transaction = filter_snippet_transaction( + envelope_measure, + taric_schema_tags, + record_group, + ) + + assert transaction is None + + +@pytest.mark.importer_v2 +def test_chunk_taric(example_goods_taric_file_location): + """Tests that the chunker creates an ImporterXMLChunk object in the db from + the loaded XML file.""" + assert not ImporterXMLChunk.objects.count() + with open(f"{example_goods_taric_file_location}", "rb") as f: + content = f.read() + taric_file = SimpleUploadedFile("goods.xml", content, content_type="text/xml") + batch = factories.ImportBatchFactory.create() + chunk_taric(taric_file, batch) + assert ImporterXMLChunk.objects.count() + chunk = ImporterXMLChunk.objects.first() + assert chunk.chunk_text + + +@pytest.mark.importer_v2 +def test_chunk_taric_fails_with_split_job(example_goods_taric_file_location): + """Tests that the chunker creates an ImporterXMLChunk object in the db from + the loaded XML file.""" + assert not ImporterXMLChunk.objects.count() + with open(f"{example_goods_taric_file_location}", "rb") as f: + content = f.read() + taric_file = SimpleUploadedFile("goods.xml", content, content_type="text/xml") + batch = factories.ImportBatchFactory.create(split_job=True) + with pytest.raises(Exception) as e: + chunk_taric(taric_file, batch) + + assert ( + "Unexpected split job, split jobs are not compatible with importer v2." + in str(e) + ) + assert "Please split files up before importing." in str(e) diff --git a/taric_parsers/tests/test_for_generic_import_errors.py b/taric_parsers/tests/test_for_generic_import_errors.py new file mode 100644 index 000000000..bb0f72bb0 --- /dev/null +++ b/taric_parsers/tests/test_for_generic_import_errors.py @@ -0,0 +1,42 @@ +import pytest + +from common.tests.util import preload_import + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestForGenericImportErrors: + def test_correctly_raises_error_when_create_for_existing_sid(self): + preload_import("additional_code_CREATE.xml", __file__, True) + importer = preload_import("additional_code_second_CREATE.xml", __file__) + + assert len(importer.issues()) == 1 + + assert ( + "ERROR: Identity keys match existing non-deleted object in database (checking all published and unpublished data)" + in str(importer.issues()[0]) + ) + assert "additional.code >" in str(importer.issues()[0]) + assert ( + "{" + "'type': 'AdditionalcodeType 5: some description', " + "'update_type': 3, " + "'sid': 1, " + "'code': '3', " + "'valid_between': {'start': '2021-01-01', 'end': 'None'}}" + ) in str(importer.issues()[0]) + + def test_correctly_raises_error_when_update_for_non_existing_sid(self): + importer = preload_import("additional_code_UPDATE.xml", __file__) + + assert len(importer.issues()) == 1 + + # assert "Missing expected linked object NewAdditionalCodeTypeParser" in str(importer.issues()[0]) + # assert "link_data: {'sid': '5'}" in str(importer.issues()[0]) + + assert "Identity keys do not match an existing object" in str( + importer.issues()[0], + ) + assert "additional.code >" in str(importer.issues()[0]) + assert "link_data: {'sid': 1}" in str(importer.issues()[0]) diff --git a/taric_parsers/tests/test_forms.py b/taric_parsers/tests/test_forms.py new file mode 100644 index 000000000..21c8fd02e --- /dev/null +++ b/taric_parsers/tests/test_forms.py @@ -0,0 +1,60 @@ +from os import path +from unittest.mock import patch + +import pytest +from django.core.files.uploadedfile import SimpleUploadedFile + +from taric_parsers.forms import UploadTaricForm + +pytestmark = pytest.mark.django_db + +TEST_FILES_PATH = path.join(path.dirname(__file__), "support") + + +@pytest.mark.importer_v2 +def test_upload_taric_form_valid_envelope_id(): + with open(f"{TEST_FILES_PATH}/valid.xml", "rb") as upload_file: + data = { + "name": "test_upload", + } + file_data = { + "taric_file": SimpleUploadedFile( + upload_file.name, + upload_file.read(), + content_type="text/xml", + ), + } + form = UploadTaricForm(data, file_data) + + assert form.is_valid() + + +@pytest.mark.importer_v2 +@patch("taric_parsers.forms.chunk_taric") +@patch("taric_parsers.forms.run_batch") +def test_upload_taric_form_save(run_batch, chunk_taric, superuser): + with open(f"{TEST_FILES_PATH}/valid.xml", "rb") as upload_file: + chunk_taric.return_value = 1 + + data = { + "name": "test_upload", + } + file_data = { + "taric_file": SimpleUploadedFile( + upload_file.name, + upload_file.read(), + content_type="text/xml", + ), + } + form = UploadTaricForm(data, file_data) + form.is_valid() + batch = form.save(user=superuser) + + assert batch.name == "test_upload" + assert batch.goods_import is False + assert batch.split_job is False + # workbasket will not be created / associated until the background task has been run + assert batch.workbasket is None + + run_batch.assert_called_once() + chunk_taric.assert_called_once() diff --git a/taric_parsers/tests/test_namespaces.py b/taric_parsers/tests/test_namespaces.py new file mode 100644 index 000000000..acd87a0d8 --- /dev/null +++ b/taric_parsers/tests/test_namespaces.py @@ -0,0 +1,42 @@ +import xml.etree.ElementTree as ET + +import pytest + +from common.util import xml_fromstring +from importer.namespaces import Tag +from importer.namespaces import TTags + +quota_event = Tag("quota.event") +quota_balance_event = Tag("quota.balance.event") + +pytestmark = pytest.mark.django_db + + +def get_snippet_transaction( + xml: str, + Tags: TTags, +) -> ET.Element: + """Returns the first transaction element in a Taric envelope.""" + envelope = xml_fromstring(xml) + return Tags.ENV_TRANSACTION.first(envelope) + + +def test_tag_first(taric_schema_tags, envelope_commodity): + """Asserts that Tag.first locates the appropriate child in the parent + element.""" + transaction = get_snippet_transaction(envelope_commodity, taric_schema_tags) + assert transaction is not None + + +def test_tag_equals_with_name(tag_name): + """Asserts that Tag comparisons work when the tag name is not a patern.""" + assert tag_name.is_pattern is False + assert tag_name == quota_event + assert tag_name != quota_balance_event + + +def test_tag_equals_with_pattern(tag_regex): + """Asserts that Tag comparisons work when the tag name is a patern.""" + assert tag_regex.is_pattern is True + assert tag_regex != quota_event + assert tag_regex == quota_balance_event diff --git a/taric_parsers/tests/test_new_element_parser.py b/taric_parsers/tests/test_new_element_parser.py new file mode 100644 index 000000000..1c84fc2bd --- /dev/null +++ b/taric_parsers/tests/test_new_element_parser.py @@ -0,0 +1,797 @@ +from datetime import date +from typing import List + +import pytest + +from additional_codes.models import AdditionalCodeType +from common.tests import factories +from common.util import TaricDateRange +from taric_parsers.parser_model_link import ModelLink +from taric_parsers.parser_model_link import ModelLinkField +from taric_parsers.parsers.additional_code_parsers import ( # noqa + AdditionalCodeDescriptionParserV2, +) +from taric_parsers.parsers.additional_code_parsers import ( # noqa + AdditionalCodeDescriptionPeriodParserV2, +) +from taric_parsers.parsers.additional_code_parsers import AdditionalCodeParserV2 # noqa +from taric_parsers.parsers.additional_code_parsers import ( # noqa + AdditionalCodeTypeDescriptionParserV2, +) +from taric_parsers.parsers.additional_code_parsers import ( # noqa + AdditionalCodeTypeParserV2, +) +from taric_parsers.parsers.additional_code_parsers import ( # noqa + FootnoteAssociationAdditionalCodeParserV2, +) +from taric_parsers.parsers.certificate_parser import ( # noqa + CertificateDescriptionParserV2, +) +from taric_parsers.parsers.certificate_parser import ( # noqa + CertificateDescriptionPeriodParserV2, +) +from taric_parsers.parsers.certificate_parser import CertificateParserV2 # noqa +from taric_parsers.parsers.certificate_parser import ( # noqa + CertificateTypeDescriptionParserV2, +) +from taric_parsers.parsers.certificate_parser import CertificateTypeParserV2 # noqa +from taric_parsers.parsers.commodity_parser import ( # noqa + FootnoteAssociationGoodsNomenclatureParserV2, +) +from taric_parsers.parsers.commodity_parser import ( # noqa + GoodsNomenclatureDescriptionParserV2, +) +from taric_parsers.parsers.commodity_parser import ( # noqa + GoodsNomenclatureDescriptionPeriodParserV2, +) +from taric_parsers.parsers.commodity_parser import ( # noqa + GoodsNomenclatureIndentParserV2, +) +from taric_parsers.parsers.commodity_parser import ( # noqa + GoodsNomenclatureOriginParserV2, +) +from taric_parsers.parsers.commodity_parser import GoodsNomenclatureParserV2 # noqa +from taric_parsers.parsers.commodity_parser import ( # noqa + GoodsNomenclatureSuccessorParserV2, +) +from taric_parsers.parsers.footnote_parser import FootnoteDescriptionParserV2 # noqa +from taric_parsers.parsers.footnote_parser import ( # noqa + FootnoteDescriptionPeriodParserV2, +) +from taric_parsers.parsers.footnote_parser import FootnoteParserV2 # noqa +from taric_parsers.parsers.footnote_parser import ( # noqa + FootnoteTypeDescriptionParserV2, +) +from taric_parsers.parsers.footnote_parser import FootnoteTypeParserV2 # noqa +from taric_parsers.parsers.geo_area_parser import ( # noqa + GeographicalAreaDescriptionParserV2, +) +from taric_parsers.parsers.geo_area_parser import ( # noqa + GeographicalAreaDescriptionPeriodParserV2, +) +from taric_parsers.parsers.geo_area_parser import GeographicalAreaParserV2 # noqa +from taric_parsers.parsers.geo_area_parser import GeographicalMembershipParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + AdditionalCodeTypeMeasureTypeParserV2, +) +from taric_parsers.parsers.measure_parser import ( # noqa + DutyExpressionDescriptionParserV2, +) +from taric_parsers.parsers.measure_parser import DutyExpressionParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + FootnoteAssociationMeasureParserV2, +) +from taric_parsers.parsers.measure_parser import ( # noqa + MeasureActionDescriptionParserV2, +) +from taric_parsers.parsers.measure_parser import MeasureActionParserV2 # noqa +from taric_parsers.parsers.measure_parser import MeasureComponentParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + MeasureConditionCodeDescriptionParserV2, +) +from taric_parsers.parsers.measure_parser import MeasureConditionCodeParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + MeasureConditionComponentParserV2, +) +from taric_parsers.parsers.measure_parser import MeasureConditionParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + MeasureExcludedGeographicalAreaParserV2, +) +from taric_parsers.parsers.measure_parser import MeasurementParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + MeasurementUnitDescriptionParserV2, +) +from taric_parsers.parsers.measure_parser import MeasurementUnitParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + MeasurementUnitQualifierDescriptionParserV2, +) +from taric_parsers.parsers.measure_parser import ( # noqa + MeasurementUnitQualifierParserV2, +) +from taric_parsers.parsers.measure_parser import MeasureParserV2 # noqa +from taric_parsers.parsers.measure_parser import MeasureTypeDescriptionParserV2 # noqa +from taric_parsers.parsers.measure_parser import MeasureTypeParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + MeasureTypeSeriesDescriptionParserV2, +) +from taric_parsers.parsers.measure_parser import MeasureTypeSeriesParserV2 # noqa +from taric_parsers.parsers.measure_parser import MonetaryUnitDescriptionParserV2 # noqa +from taric_parsers.parsers.measure_parser import MonetaryUnitParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaAssociationParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaBalanceEventParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaBlockingParserV2 # noqa +from taric_parsers.parsers.quota_parser import ( # noqa + QuotaClosedAndTransferredEventParserV2, +) +from taric_parsers.parsers.quota_parser import QuotaCriticalEventParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaDefinitionParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaEventParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaExhaustionEventParserV2 # noqa +from taric_parsers.parsers.quota_parser import ( # noqa + QuotaOrderNumberOriginExclusionParserV2, +) +from taric_parsers.parsers.quota_parser import QuotaOrderNumberOriginParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaOrderNumberParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaReopeningEventParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaSuspensionParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaUnblockingEventParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaUnsuspensionEventParserV2 # noqa +from taric_parsers.parsers.regulation_parser import BaseRegulationParserV2 # noqa +from taric_parsers.parsers.regulation_parser import ( # noqa + FullTemporaryStopActionParserV2, +) +from taric_parsers.parsers.regulation_parser import ( # noqa + FullTemporaryStopRegulationParserV2, +) +from taric_parsers.parsers.regulation_parser import ( # noqa + ModificationRegulationParserV2, +) +from taric_parsers.parsers.regulation_parser import ( # noqa + RegulationGroupDescriptionParserV2, +) +from taric_parsers.parsers.regulation_parser import RegulationGroupParserV2 # noqa +from taric_parsers.parsers.regulation_parser import ( # noqa + RegulationReplacementParserV2, +) +from taric_parsers.parsers.taric_parser import BaseTaricParser +from taric_parsers.parsers.taric_parser import ParserHelper + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestNewElementParser: + def test_init_defaults(self): + target = BaseTaricParser + + assert target.model is None + assert target.issues == [] + assert target.transaction_id is None + assert target.record_code is None + assert target.subrecord_code is None + assert target.xml_object_tag is None + assert target.update_type is None + assert target.update_type_name is None + assert target.links_valid is None + assert target.value_mapping == {} + assert target.model_links == [] + assert target.parent_parser is None + assert target.parent_handler is None + assert target.xml_object_tag is None + + def test_links_raises_exception(self): + target = BaseTaricParser() + + with pytest.raises(Exception) as e: + target.links() + assert str(e) == "No parser defined for NewElementParser, is this correct?" + + def test_links_does_not_raises_exception_when_populated(self): + class TestElementParser(BaseTaricParser): + def __init__(self): + super().__init__() + self.model_links = [] + + target = TestElementParser() + + assert target.links() == [] + + def test_missing_child_attributes_returns_none_when_has_no_children(self): + target = BaseTaricParser() + + assert target.missing_child_attributes() is None + + def test_missing_child_attributes_returns_dict_when_has_children(self): + # setup parent + class MadeUpModel: + model_links = [] + + def __init__(self): + self.x = "z" + + class TestParentElementParser(BaseTaricParser): + model = MadeUpModel + xml_object_tag = "test.parent.element" + + def __init__(self): + super().__init__() + self.field_1 = None + + class TestChildElementParser(BaseTaricParser): + model = MadeUpModel + parent_parser = TestParentElementParser + model_links = [ + ModelLink( + MadeUpModel, + [ + ModelLinkField("parent_field_1", "field_1"), + ], + "test.parent.element", + ), + ] + + def __init__(self): + super().__init__() + self.parent_field_1 = "zzz" + + target = TestParentElementParser() + assert TestChildElementParser in ParserHelper.get_parser_classes() + assert len(ParserHelper.get_child_parsers(target)) == 1 + # + assert target.missing_child_attributes() == { + "TestChildElementParser": ["parent_field_1"], + } + + def test_missing_child_attributes_errors_when_attribute_not_present_on_parent(self): + # setup parent + class MadeUpModel: + def __init__(self): + self.x = "z" + + class TestParentElementParser(BaseTaricParser): + model = MadeUpModel + xml_object_tag = "test.parent.element" + + def __init__(self): + super().__init__() + + class TestChildElementParser(BaseTaricParser): + model = MadeUpModel + parent_parser = TestParentElementParser + model_links = [ + ModelLink( + MadeUpModel, + [ + ModelLinkField("parent_field_1", "field_1"), + ], + "test.parent.element", + ), + ] + + def __init__(self): + super().__init__() + self.parent_field_1 = "zzz" + + target = TestParentElementParser() + + e = None + with pytest.raises(Exception) as e: + target.missing_child_attributes() + + assert "Field referenced by child TestChildElementParser" in str(e) + assert "field_1 does not exist on parent TestParentElementParser" in str(e) + + def test_identity_fields_for_parent_raises_error_if_no_parent_parser(self): + class MadeUpModel: + def __init__(self): + self.x = "z" + + class TestParentElementParser(BaseTaricParser): + model = MadeUpModel + xml_object_tag = "test.parent.element" + + def __init__(self): + super().__init__() + + target = TestParentElementParser() + + e = None + with pytest.raises(Exception) as e: + target.identity_fields_for_parent() + + assert "Model TestParentElementParser has no parent parser" in str(e) + + def test_identity_fields_for_parent_raises_error_if_no_model_links(self): + class MadeUpModel: + def __init__(self): + self.x = "z" + + class TestParentElementParser(BaseTaricParser): + model = MadeUpModel + xml_object_tag = "test.parent.element" + + def __init__(self): + super().__init__() + + class TestChildElementParser(BaseTaricParser): + model = MadeUpModel + parent_parser = TestParentElementParser + model_links = [] + + def __init__(self): + super().__init__() + self.parent_field_1 = "zzz" + + target = TestChildElementParser() + + e = None + with pytest.raises(Exception) as e: + target.identity_fields_for_parent() + + assert ( + "Model TestChildElementParser appears to have a parent parser but no model links" + in str(e) + ) + + def test_identity_fields_for_parent_raises_error_if_no_model_link_to_parent(self): + class MadeUpModel: + def __init__(self): + self.x = "z" + + class AnotherMadeUpModel: + def __init__(self): + self.x = "z" + + class TestParentElementParser(BaseTaricParser): + model = MadeUpModel + xml_object_tag = "test.parent.element" + + def __init__(self): + super().__init__() + + class TestChildElementParser(BaseTaricParser): + model = MadeUpModel + parent_parser = TestParentElementParser + model_links = [ + ModelLink( + AnotherMadeUpModel, + [ + ModelLinkField("parent_field_1", "field_1"), + ], + "test.parent.element", + ), + ] + + def __init__(self): + super().__init__() + self.parent_field_1 = "zzz" + + target = TestChildElementParser() + + e = None + with pytest.raises(Exception) as e: + target.identity_fields_for_parent() + + assert ( + "Model TestChildElementParser appears to not have a model links to the parent model" + in str(e) + ) + + def test_populate_fails_of_record_code_differs(self): + class TestParentElementParser(BaseTaricParser): + xml_object_tag = "test.parent.element" + record_code = "000" + subrecord_code = "000" + + def __init__(self): + super().__init__() + + target = TestParentElementParser() + + e = None + with pytest.raises(Exception) as e: + target.populate(123, "900", "100", 1, {}) + + assert "Record code mismatch : expected : 000, got : 900" in str(e) + + def test_populate_fails_of_sub_record_code_differs(self): + class TestParentElementParser(BaseTaricParser): + xml_object_tag = "test.parent.element" + + record_code = "900" + subrecord_code = "100" + + def __init__(self): + super().__init__() + + target = TestParentElementParser() + + e = None + with pytest.raises(Exception) as e: + target.populate(123, "900", "200", 1, {}) + + assert "Sub-record code mismatch : expected : 100, got : 200" in str(e) + + def test_populate_succeeds_when_record_and_subrecord_match(self): + class TestParentElementParser(BaseTaricParser): + xml_object_tag = "test.parent.element" + + record_code = "900" + subrecord_code = "100" + + sid: int = None + validity_start: date = None + + def __init__(self): + super().__init__() + + target = TestParentElementParser() + + target.populate( + 123, + "900", + "100", + 1, + { + "validity_start": "2023-01-01", + "sid": "123", + }, + ) + + assert target.sid == 123 + assert target.validity_start == date(2023, 1, 1) + + def test_populate_fails_unexpected_fields_are_presented(self): + class TestParentElementParser(BaseTaricParser): + xml_object_tag = "test.parent.element" + + record_code = "900" + subrecord_code = "100" + + sid: int = None + validity_start: date = None + + def __init__(self): + super().__init__() + + target = TestParentElementParser() + + e = None + with pytest.raises(Exception) as e: + target.populate( + 123, + "900", + "100", + 1, + { + "validity_start": "2023-01-01", + "sid": "123", + "dfgdfg": "dsfgdfg", + }, + ) + + assert "test.parent.element TestParentElementParser " in str(e) + assert ( + "does not have a dfgdfg attribute, and can't assign value dsfgdfg" in str(e) + ) + + def test_populate_ignores_excluded_fields(self): + class TestParentElementParser(BaseTaricParser): + xml_object_tag = "test.parent.element" + + record_code = "900" + subrecord_code = "100" + + sid: int = None + validity_start: date = None + + def __init__(self): + super().__init__() + + target = TestParentElementParser() + + target.populate( + 123, + "900", + "100", + 1, + { + "validity_start": "2023-01-01", + "sid": "123", + "language_id": "fghfgh", + }, + ) + + assert target.sid == 123 + assert target.validity_start == date(2023, 1, 1) + + def test_populate_value_mapped_fields(self): + class TestParentElementParser(BaseTaricParser): + xml_object_tag = "test.parent.element" + + record_code = "900" + subrecord_code = "100" + + another_sid: int = None + validity_start: date = None + string_value: str = None + float_value: float = None + + value_mapping = { + "sid": "another_sid", + } + + def __init__(self): + super().__init__() + + target = TestParentElementParser() + + target.populate( + 123, + "900", + "100", + 1, + { + "validity_start": "2023-01-01", + "sid": "123", + "language_id": "fghfgh", + "update_type": 3, + "update_type_name": "CREATE", + "string_value": "happy", + "float_value": "12.897", + }, + ) + + assert target.another_sid == 123 + assert target.validity_start == date(2023, 1, 1) + + def test_populate_errors_for_invalid_data_type(self): + class TestParentElementParser(BaseTaricParser): + xml_object_tag = "test.parent.element" + + record_code = "900" + subrecord_code = "100" + + dict_value: dict = None + + def __init__(self): + super().__init__() + + target = TestParentElementParser() + + e = None + with pytest.raises(Exception) as e: + target.populate( + 123, + "900", + "100", + 1, + { + "dict_value": '{"fghfghfghfgh": "dfgdfgdfg"}', + }, + ) + + assert ( + "data type dict not handled, does the handler have the correct data type?" + in str(e) + ) + + def test_populate_correctly_makes_valid_between_with_upper_and_lower(self): + class TestParentElementParser(BaseTaricParser): + xml_object_tag = "test.parent.element" + + record_code = "900" + subrecord_code = "100" + valid_between = None + valid_between_lower: date = None + valid_between_upper: date = None + + dict_value: dict = None + + def __init__(self): + super().__init__() + + target = TestParentElementParser() + + target.populate( + 123, + "900", + "100", + 1, + { + "valid_between_lower": "2023-01-01", + "valid_between_upper": "2023-12-31", + }, + ) + + assert target.valid_between == TaricDateRange( + date(2023, 1, 1), + date(2023, 12, 31), + ) + assert target.valid_between_lower == date(2023, 1, 1) + assert target.valid_between_upper == date(2023, 12, 31) + + def test_populate_correctly_makes_valid_between_with_just_lower(self): + class TestParentElementParser(BaseTaricParser): + xml_object_tag = "test.parent.element" + + record_code = "900" + subrecord_code = "100" + valid_between = None + valid_between_lower: date = None + valid_between_upper: date = None + + dict_value: dict = None + + def __init__(self): + super().__init__() + + target = TestParentElementParser() + + target.populate( + 123, + "900", + "100", + 1, + { + "valid_between_lower": "2023-01-01", + }, + ) + + assert target.valid_between == TaricDateRange(date(2023, 1, 1)) + assert target.valid_between_lower == date(2023, 1, 1) + assert target.valid_between_upper is None + + def test_to_tap_model(self): + class MadeUpModel: + sequence_number: int = None + transaction_id: int = None + valid_between: TaricDateRange = None + + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + class TestParentElementParser(BaseTaricParser): + xml_object_tag = "test.parent.element" + + record_code = "900" + subrecord_code = "100" + valid_between = None + valid_between_lower: date = None + valid_between_upper: date = None + model = MadeUpModel + + model_links = [] + + def __init__(self): + super().__init__() + + parser = TestParentElementParser() + + parser.populate( + 123, + "900", + "100", + 1, + { + "valid_between_lower": "2023-01-01", + }, + ) + + transaction = factories.ApprovedTransactionFactory.create() + + target = parser.model_attributes(transaction) + + assert target["valid_between"] == TaricDateRange(date(2023, 1, 1)) + + def test_to_tap_model_invalid_peoperty(self): + class MadeUpModel: + sequence_number: int = None + transaction_id: int = None + valid_between: TaricDateRange = None + + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + class TestParentElementParser(BaseTaricParser): + xml_object_tag = "test.parent.element" + + record_code = "900" + subrecord_code = "100" + valid_between = None + valid_between_lower: date = None + valid_between_upper: date = None + invalid_property: str = None + model = MadeUpModel + + model_links = [] + + def __init__(self): + super().__init__() + + parser = TestParentElementParser() + + parser.populate( + 123, + "900", + "100", + 1, + { + "valid_between_lower": "2023-01-01", + "invalid_property": "dfgdfgdfg", + }, + ) + + transaction = factories.ApprovedTransactionFactory.create() + + e = None + with pytest.raises(Exception) as e: + parser.model_attributes(transaction) + + assert ( + "Error creating model MadeUpModel, model does not have an attribute invalid_property" + in str(e) + ) + + def test_model_attributes_returns_expected(self): + add_code_type = factories.AdditionalCodeTypeFactory.create(sid="A") + + class MadeUpModel: + sequence_number: int = None + transaction_id: int = None + valid_between: TaricDateRange = None + field_1: str = None + another_field: int = None + related_model: List[str] = None + additional_code_type = AdditionalCodeType + + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + class TestParentElementParser(BaseTaricParser): + xml_object_tag = "test.parent.element" + + record_code = "900" + subrecord_code = "100" + field_1: str = None + another_field: str = None + model = MadeUpModel + additional_code_type__sid: str = None + + model_links = [ + ModelLink( + AdditionalCodeType, + [ + ModelLinkField("additional_code_type__sid", "sid"), + ], + "additional.code.type", + ), + ] + + def __init__(self): + super().__init__() + + appr_transaction = factories.ApprovedTransactionFactory.create() + + parser = TestParentElementParser() + parser.populate( + 1, + "900", + "100", + 1, + { + "field_1": "aaa", + "another_field": "123", + "additional_code_type__sid": "A", + }, + ) + + # child_parser + target = parser.model_attributes(appr_transaction) + assert "field_1" in target.keys() + assert "another_field" in target.keys() + assert "additional_code_type" in target.keys() + assert len(target.keys()) == 3 diff --git a/taric_parsers/tests/test_new_parser_generics.py b/taric_parsers/tests/test_new_parser_generics.py new file mode 100644 index 000000000..59c185e1e --- /dev/null +++ b/taric_parsers/tests/test_new_parser_generics.py @@ -0,0 +1,854 @@ +import pytest + +from additional_codes.models import AdditionalCode +from additional_codes.models import AdditionalCodeDescription +from additional_codes.models import AdditionalCodeType +from additional_codes.models import FootnoteAssociationAdditionalCode +from certificates.models import Certificate +from certificates.models import CertificateDescription +from certificates.models import CertificateType +from commodities.models import FootnoteAssociationGoodsNomenclature +from commodities.models import GoodsNomenclature +from commodities.models import GoodsNomenclatureDescription +from commodities.models import GoodsNomenclatureIndent +from commodities.models import GoodsNomenclatureOrigin +from commodities.models import GoodsNomenclatureSuccessor +from footnotes.models import Footnote +from footnotes.models import FootnoteDescription +from footnotes.models import FootnoteType +from geo_areas.models import GeographicalArea +from geo_areas.models import GeographicalAreaDescription +from geo_areas.models import GeographicalMembership +from measures.models import AdditionalCodeTypeMeasureType +from measures.models import DutyExpression +from measures.models import FootnoteAssociationMeasure +from measures.models import Measure +from measures.models import MeasureAction +from measures.models import MeasureComponent +from measures.models import MeasureCondition +from measures.models import MeasureConditionCode +from measures.models import MeasureConditionComponent +from measures.models import MeasureExcludedGeographicalArea +from measures.models import Measurement +from measures.models import MeasurementUnit +from measures.models import MeasurementUnitQualifier +from measures.models import MeasureType +from measures.models import MeasureTypeSeries +from measures.models import MonetaryUnit +from quotas.models import QuotaAssociation +from quotas.models import QuotaBlocking +from quotas.models import QuotaDefinition +from quotas.models import QuotaEvent +from quotas.models import QuotaOrderNumber +from quotas.models import QuotaOrderNumberOrigin +from quotas.models import QuotaOrderNumberOriginExclusion +from quotas.models import QuotaSuspension +from regulations.models import Amendment +from regulations.models import Group +from regulations.models import Regulation +from regulations.models import Replacement +from regulations.models import Suspension +from taric_parsers.parsers.additional_code_parsers import ( # noqa + AdditionalCodeDescriptionParserV2, +) +from taric_parsers.parsers.additional_code_parsers import ( # noqa + AdditionalCodeDescriptionPeriodParserV2, +) +from taric_parsers.parsers.additional_code_parsers import AdditionalCodeParserV2 # noqa +from taric_parsers.parsers.additional_code_parsers import ( # noqa + AdditionalCodeTypeDescriptionParserV2, +) +from taric_parsers.parsers.additional_code_parsers import ( # noqa + AdditionalCodeTypeParserV2, +) +from taric_parsers.parsers.additional_code_parsers import ( # noqa + FootnoteAssociationAdditionalCodeParserV2, +) +from taric_parsers.parsers.certificate_parser import ( # noqa + CertificateDescriptionParserV2, +) +from taric_parsers.parsers.certificate_parser import ( # noqa + CertificateDescriptionPeriodParserV2, +) +from taric_parsers.parsers.certificate_parser import CertificateParserV2 # noqa +from taric_parsers.parsers.certificate_parser import ( # noqa + CertificateTypeDescriptionParserV2, +) +from taric_parsers.parsers.certificate_parser import CertificateTypeParserV2 # noqa +from taric_parsers.parsers.commodity_parser import ( # noqa + FootnoteAssociationGoodsNomenclatureParserV2, +) +from taric_parsers.parsers.commodity_parser import ( # noqa + GoodsNomenclatureDescriptionParserV2, +) +from taric_parsers.parsers.commodity_parser import ( # noqa + GoodsNomenclatureDescriptionPeriodParserV2, +) +from taric_parsers.parsers.commodity_parser import ( # noqa + GoodsNomenclatureIndentParserV2, +) +from taric_parsers.parsers.commodity_parser import ( # noqa + GoodsNomenclatureOriginParserV2, +) +from taric_parsers.parsers.commodity_parser import GoodsNomenclatureParserV2 # noqa +from taric_parsers.parsers.commodity_parser import ( # noqa + GoodsNomenclatureSuccessorParserV2, +) +from taric_parsers.parsers.footnote_parser import FootnoteDescriptionParserV2 # noqa +from taric_parsers.parsers.footnote_parser import ( # noqa + FootnoteDescriptionPeriodParserV2, +) +from taric_parsers.parsers.footnote_parser import FootnoteParserV2 # noqa +from taric_parsers.parsers.footnote_parser import ( # noqa + FootnoteTypeDescriptionParserV2, +) +from taric_parsers.parsers.footnote_parser import FootnoteTypeParserV2 # noqa +from taric_parsers.parsers.geo_area_parser import ( # noqa + GeographicalAreaDescriptionParserV2, +) +from taric_parsers.parsers.geo_area_parser import ( # noqa + GeographicalAreaDescriptionPeriodParserV2, +) +from taric_parsers.parsers.geo_area_parser import GeographicalAreaParserV2 # noqa +from taric_parsers.parsers.geo_area_parser import GeographicalMembershipParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + AdditionalCodeTypeMeasureTypeParserV2, +) +from taric_parsers.parsers.measure_parser import ( # noqa + DutyExpressionDescriptionParserV2, +) +from taric_parsers.parsers.measure_parser import DutyExpressionParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + FootnoteAssociationMeasureParserV2, +) +from taric_parsers.parsers.measure_parser import ( # noqa + MeasureActionDescriptionParserV2, +) +from taric_parsers.parsers.measure_parser import MeasureActionParserV2 # noqa +from taric_parsers.parsers.measure_parser import MeasureComponentParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + MeasureConditionCodeDescriptionParserV2, +) +from taric_parsers.parsers.measure_parser import MeasureConditionCodeParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + MeasureConditionComponentParserV2, +) +from taric_parsers.parsers.measure_parser import MeasureConditionParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + MeasureExcludedGeographicalAreaParserV2, +) +from taric_parsers.parsers.measure_parser import MeasurementParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + MeasurementUnitDescriptionParserV2, +) +from taric_parsers.parsers.measure_parser import MeasurementUnitParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + MeasurementUnitQualifierDescriptionParserV2, +) +from taric_parsers.parsers.measure_parser import ( # noqa + MeasurementUnitQualifierParserV2, +) +from taric_parsers.parsers.measure_parser import MeasureParserV2 # noqa +from taric_parsers.parsers.measure_parser import MeasureTypeDescriptionParserV2 # noqa +from taric_parsers.parsers.measure_parser import MeasureTypeParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + MeasureTypeSeriesDescriptionParserV2, +) +from taric_parsers.parsers.measure_parser import MeasureTypeSeriesParserV2 # noqa +from taric_parsers.parsers.measure_parser import MonetaryUnitDescriptionParserV2 # noqa +from taric_parsers.parsers.measure_parser import MonetaryUnitParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaAssociationParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaBalanceEventParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaBlockingParserV2 # noqa +from taric_parsers.parsers.quota_parser import ( # noqa + QuotaClosedAndTransferredEventParserV2, +) +from taric_parsers.parsers.quota_parser import QuotaCriticalEventParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaDefinitionParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaEventParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaExhaustionEventParserV2 # noqa +from taric_parsers.parsers.quota_parser import ( # noqa + QuotaOrderNumberOriginExclusionParserV2, +) +from taric_parsers.parsers.quota_parser import QuotaOrderNumberOriginParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaOrderNumberParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaReopeningEventParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaSuspensionParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaUnblockingEventParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaUnsuspensionEventParserV2 # noqa +from taric_parsers.parsers.regulation_parser import BaseRegulationParserV2 # noqa +from taric_parsers.parsers.regulation_parser import ( # noqa + FullTemporaryStopActionParserV2, +) +from taric_parsers.parsers.regulation_parser import ( # noqa + FullTemporaryStopRegulationParserV2, +) +from taric_parsers.parsers.regulation_parser import ( # noqa + ModificationRegulationParserV2, +) +from taric_parsers.parsers.regulation_parser import ( # noqa + RegulationGroupDescriptionParserV2, +) +from taric_parsers.parsers.regulation_parser import RegulationGroupParserV2 # noqa +from taric_parsers.parsers.regulation_parser import ( # noqa + RegulationReplacementParserV2, +) + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +@pytest.mark.parametrize( + ( + "parser_class", + "model_class", + "expected_xml_tag_name", + "links_to", + "child_to_other_parser", + ), + ( + # Additional Codes + ( + AdditionalCodeTypeParserV2, + AdditionalCodeType, + "additional.code.type", + [], + False, + ), + ( + AdditionalCodeTypeDescriptionParserV2, + AdditionalCodeType, + "additional.code.type.description", + [], + True, + ), + ( + AdditionalCodeParserV2, + AdditionalCode, + "additional.code", + [AdditionalCodeType], + False, + ), + ( + AdditionalCodeDescriptionPeriodParserV2, + AdditionalCodeDescription, + "additional.code.description.period", + [AdditionalCode], + True, + ), + ( + AdditionalCodeDescriptionParserV2, + AdditionalCodeDescription, + "additional.code.description", + [AdditionalCode], + False, + ), + ( + FootnoteAssociationAdditionalCodeParserV2, + FootnoteAssociationAdditionalCode, + "footnote.association.additional.code", + [Footnote, AdditionalCodeType, AdditionalCode], + False, + ), + # Certificates + ( + CertificateTypeParserV2, + CertificateType, + "certificate.type", + [], + False, + ), + ( + CertificateTypeDescriptionParserV2, + CertificateType, + "certificate.type.description", + [], + True, + ), + ( + CertificateParserV2, + Certificate, + "certificate", + [CertificateType], + False, + ), + ( + CertificateDescriptionParserV2, + CertificateDescription, + "certificate.description", + [CertificateType, Certificate], + False, + ), + ( + CertificateDescriptionPeriodParserV2, + CertificateDescription, + "certificate.description.period", + [CertificateType, Certificate], + True, + ), + # Commodities + ( + GoodsNomenclatureParserV2, + GoodsNomenclature, + "goods.nomenclature", + [], + False, + ), + ( + GoodsNomenclatureOriginParserV2, + GoodsNomenclatureOrigin, + "goods.nomenclature.origin", + [GoodsNomenclature], + False, + ), + ( + GoodsNomenclatureSuccessorParserV2, + GoodsNomenclatureSuccessor, + "goods.nomenclature.successor", + [GoodsNomenclature], + False, + ), + ( + GoodsNomenclatureDescriptionParserV2, + GoodsNomenclatureDescription, + "goods.nomenclature.description", + [GoodsNomenclature], + False, + ), + ( + GoodsNomenclatureDescriptionPeriodParserV2, + GoodsNomenclatureDescription, + "goods.nomenclature.description.period", + [GoodsNomenclature], + True, + ), + ( + GoodsNomenclatureIndentParserV2, + GoodsNomenclatureIndent, + "goods.nomenclature.indents", + [GoodsNomenclature], + False, + ), + ( + FootnoteAssociationGoodsNomenclatureParserV2, + FootnoteAssociationGoodsNomenclature, + "footnote.association.goods.nomenclature", + [GoodsNomenclature, Footnote], + False, + ), + # Footnotes + ( + FootnoteTypeParserV2, + FootnoteType, + "footnote.type", + [], + False, + ), + ( + FootnoteTypeDescriptionParserV2, + FootnoteType, + "footnote.type.description", + [FootnoteType], + True, + ), + ( + FootnoteParserV2, + Footnote, + "footnote", + [FootnoteType], + False, + ), + ( + FootnoteDescriptionParserV2, + FootnoteDescription, + "footnote.description", + [Footnote], + False, + ), + ( + FootnoteDescriptionPeriodParserV2, + FootnoteDescription, + "footnote.description.period", + [Footnote], + True, + ), + # Geo Areas + ( + GeographicalAreaParserV2, + GeographicalArea, + "geographical.area", + [GeographicalArea], + False, + ), + ( + GeographicalAreaDescriptionParserV2, + GeographicalAreaDescription, + "geographical.area.description", + [GeographicalArea], + False, + ), + ( + GeographicalAreaDescriptionPeriodParserV2, + GeographicalAreaDescription, + "geographical.area.description.period", + [GeographicalArea], + True, + ), + ( + GeographicalMembershipParserV2, + GeographicalMembership, + "geographical.membership", + [GeographicalArea], + False, + ), + # Measures + ( + MeasureTypeSeriesParserV2, + MeasureTypeSeries, + "measure.type.series", + [], + False, + ), + ( + MeasureTypeSeriesDescriptionParserV2, + MeasureTypeSeries, + "measure.type.series.description", + [], + True, + ), + ( + MeasurementUnitParserV2, + MeasurementUnit, + "measurement.unit", + [], + False, + ), + ( + MeasurementUnitDescriptionParserV2, + MeasurementUnit, + "measurement.unit.description", + [], + True, + ), + ( + MeasurementUnitQualifierParserV2, + MeasurementUnitQualifier, + "measurement.unit.qualifier", + [], + False, + ), + ( + MeasurementUnitQualifierDescriptionParserV2, + MeasurementUnitQualifier, + "measurement.unit.qualifier.description", + [], + True, + ), + ( + MeasurementParserV2, + Measurement, + "measurement", + [MeasurementUnit, MeasurementUnitQualifier], + False, + ), + ( + MonetaryUnitParserV2, + MonetaryUnit, + "monetary.unit", + [], + False, + ), + ( + MonetaryUnitDescriptionParserV2, + MonetaryUnit, + "monetary.unit.description", + [], + True, + ), + ( + DutyExpressionParserV2, + DutyExpression, + "duty.expression", + [], + False, + ), + ( + DutyExpressionDescriptionParserV2, + DutyExpression, + "duty.expression.description", + [], + True, + ), + ( + MeasureTypeParserV2, + MeasureType, + "measure.type", + [], + False, + ), + ( + MeasureTypeDescriptionParserV2, + MeasureType, + "measure.type.description", + [], + True, + ), + ( + AdditionalCodeTypeMeasureTypeParserV2, + AdditionalCodeTypeMeasureType, + "additional.code.type.measure.type", + [MeasureType, AdditionalCodeType], + False, + ), + ( + MeasureConditionCodeParserV2, + MeasureConditionCode, + "measure.condition.code", + [], + False, + ), + ( + MeasureConditionCodeDescriptionParserV2, + MeasureConditionCode, + "measure.condition.code.description", + [], + True, + ), + ( + MeasureActionParserV2, + MeasureAction, + "measure.action", + [], + False, + ), + ( + MeasureActionDescriptionParserV2, + MeasureAction, + "measure.action.description", + [], + True, + ), + ( + MeasureParserV2, + Measure, + "measure", + [ + MeasureType, + GeographicalArea, + GoodsNomenclature, + AdditionalCode, + QuotaOrderNumber, + Regulation, + ], + False, + ), + ( + MeasureComponentParserV2, + MeasureComponent, + "measure.component", + [ + Measure, + DutyExpression, + MonetaryUnit, + Measurement, + ], + False, + ), + ( + MeasureConditionParserV2, + MeasureCondition, + "measure.condition", + [ + Measure, + MeasureConditionCode, + MonetaryUnit, + Measurement, + MeasureAction, + Certificate, + ], + False, + ), + ( + MeasureConditionComponentParserV2, + MeasureConditionComponent, + "measure.condition.component", + [ + MeasureCondition, + DutyExpression, + MonetaryUnit, + Measurement, + ], + False, + ), + ( + MeasureExcludedGeographicalAreaParserV2, + MeasureExcludedGeographicalArea, + "measure.excluded.geographical.area", + [Measure, GeographicalArea], + False, + ), + ( + FootnoteAssociationMeasureParserV2, + FootnoteAssociationMeasure, + "footnote.association.measure", + [Measure, Footnote], + False, + ), + # Quotas + ( + QuotaOrderNumberParserV2, + QuotaOrderNumber, + "quota.order.number", + [], + False, + ), + ( + QuotaOrderNumberOriginParserV2, + QuotaOrderNumberOrigin, + "quota.order.number.origin", + [QuotaOrderNumber, GeographicalArea], + False, + ), + ( + QuotaOrderNumberOriginExclusionParserV2, + QuotaOrderNumberOriginExclusion, + "quota.order.number.origin.exclusions", + [QuotaOrderNumberOrigin, GeographicalArea], + False, + ), + ( + QuotaDefinitionParserV2, + QuotaDefinition, + "quota.definition", + [QuotaOrderNumber, MonetaryUnit, MeasurementUnit, MeasurementUnitQualifier], + False, + ), + ( + QuotaAssociationParserV2, + QuotaAssociation, + "quota.association", + [QuotaDefinition], + False, + ), + ( + QuotaSuspensionParserV2, + QuotaSuspension, + "quota.suspension.period", + [QuotaDefinition], + False, + ), + ( + QuotaBlockingParserV2, + QuotaBlocking, + "quota.blocking.period", + [QuotaDefinition], + False, + ), + ( + QuotaEventParserV2, + QuotaEvent, + "parent.quota.event", + [QuotaDefinition], + False, + ), + ( + QuotaBalanceEventParserV2, + QuotaEvent, + "quota.balance.event", + [QuotaDefinition], + False, + ), + ( + QuotaClosedAndTransferredEventParserV2, + QuotaEvent, + "quota.closed.and.transferred.event", + [QuotaDefinition], + False, + ), + ( + QuotaCriticalEventParserV2, + QuotaEvent, + "quota.critical.event", + [QuotaDefinition], + False, + ), + ( + QuotaExhaustionEventParserV2, + QuotaEvent, + "quota.exhaustion.event", + [QuotaDefinition], + False, + ), + ( + QuotaReopeningEventParserV2, + QuotaEvent, + "quota.reopening.event", + [QuotaDefinition], + False, + ), + ( + QuotaUnblockingEventParserV2, + QuotaEvent, + "quota.unblocking.event", + [QuotaDefinition], + False, + ), + ( + QuotaUnsuspensionEventParserV2, + QuotaEvent, + "quota.unsuspension.event", + [QuotaDefinition], + False, + ), + # Regulations + ( + RegulationGroupParserV2, + Group, + "regulation.group", + [], + False, + ), + ( + RegulationGroupDescriptionParserV2, + Group, + "regulation.group.description", + [], + True, + ), + ( + BaseRegulationParserV2, + Regulation, + "base.regulation", + [], + False, + ), + ( + ModificationRegulationParserV2, + Amendment, + "modification.regulation", + [], + False, + ), + ( + FullTemporaryStopRegulationParserV2, + Suspension, + "full.temporary.stop.regulation", + [], + False, + ), + ( + FullTemporaryStopActionParserV2, + Suspension, + "fts.regulation.action", + [], + False, + ), + ( + RegulationReplacementParserV2, + Replacement, + "regulation.replacement", + [], + False, + ), + ), +) +def test_importer_generics( + parser_class, + model_class, + expected_xml_tag_name, + links_to, + child_to_other_parser, +): + # verify xml tag name + assert parser_class.xml_object_tag == expected_xml_tag_name + + if parser_class.model != model_class: + print(f"for {parser_class} model {parser_class.model} is not {model_class}") + + # check we have the right model class for the parser + assert parser_class.model == model_class + + # check that we parent parsers for all parsers that should append to a parent parser + if child_to_other_parser: + assert parser_class.parent_parser is not None + else: + assert parser_class.parent_parser is None + + # check properties exist on target model + excluded_variable_names = [ + "__annotations__", + "__doc__", + "__module__", + "issues", + "model", + "model_links", + "parent_parser", + "value_mapping", + "xml_object_tag", + "valid_between_lower", + "valid_between_upper", + "record_code", + "identity_fields", + "non_taric_additional_fields", + "data_fields", + "last_published_description_with_period", + "allow_update_without_children", + "updates_allowed", + "deletes_allowed", + "skip_identity_check", + ] + + for variable_name in vars(parser_class).keys(): + if variable_name not in excluded_variable_names: + # Skip fields that are defined in data_fields - these are collected and appended to a data column + if variable_name in parser_class.data_fields: + continue + # where a variable name contains '__' it defines a related object and its properties, we only need to check + # that the part preceding the '__' exists + variable_first_part = variable_name.split("__")[0] + assert hasattr(parser_class.model, variable_first_part), ( + f"(Testing {parser_class.__name__})" + f"for {parser_class.model.__name__} no " + f"property named {variable_first_part} found." + ) + + # Check that each parser has a populated identity field, used to identify the record on updates + assert len(parser_class.identity_fields) > 0 + + # check that there is a direct link between parent model and child model when should_append_to_parent is True + # this link can be on the parent or the child + if child_to_other_parser: + child_to_parent_link = False + parent_to_child_link = False + + # check child + if parser_class.model_links: + for link in parser_class.model_links: + if link.model == parser_class.model: + child_to_parent_link = True + + # check parent + parent_parser = parser_class.parent_parser + + if parent_parser.model_links: + for link in parent_parser.model_links: + if link.model == parser_class.model: + parent_to_child_link = True + + # check that one exists + assert child_to_parent_link or parent_to_child_link + + # check both do not exist + assert child_to_parent_link is not parent_to_child_link + + # verify existence of link to other importer types + if len(links_to): + for klass in links_to: + link_exists = False + for link in parser_class.model_links: + if link.model == klass: + link_exists = True + + assert link_exists + else: + assert links_to == [] diff --git a/taric_parsers/tests/test_parser_helper.py b/taric_parsers/tests/test_parser_helper.py new file mode 100644 index 000000000..6495e7743 --- /dev/null +++ b/taric_parsers/tests/test_parser_helper.py @@ -0,0 +1,287 @@ +from datetime import date + +import pytest + +from common.util import TaricDateRange +from taric_parsers.parser_model_link import ModelLink +from taric_parsers.parser_model_link import ModelLinkField +from taric_parsers.parsers.additional_code_parsers import ( # noqa + AdditionalCodeDescriptionParserV2, +) +from taric_parsers.parsers.additional_code_parsers import ( # noqa + AdditionalCodeDescriptionPeriodParserV2, +) +from taric_parsers.parsers.additional_code_parsers import AdditionalCodeParserV2 # noqa +from taric_parsers.parsers.additional_code_parsers import ( # noqa + AdditionalCodeTypeDescriptionParserV2, +) +from taric_parsers.parsers.additional_code_parsers import ( # noqa + AdditionalCodeTypeParserV2, +) +from taric_parsers.parsers.additional_code_parsers import ( # noqa + FootnoteAssociationAdditionalCodeParserV2, +) +from taric_parsers.parsers.certificate_parser import ( # noqa + CertificateDescriptionParserV2, +) +from taric_parsers.parsers.certificate_parser import ( # noqa + CertificateDescriptionPeriodParserV2, +) +from taric_parsers.parsers.certificate_parser import CertificateParserV2 # noqa +from taric_parsers.parsers.certificate_parser import ( # noqa + CertificateTypeDescriptionParserV2, +) +from taric_parsers.parsers.certificate_parser import CertificateTypeParserV2 # noqa +from taric_parsers.parsers.commodity_parser import ( # noqa + FootnoteAssociationGoodsNomenclatureParserV2, +) +from taric_parsers.parsers.commodity_parser import ( # noqa + GoodsNomenclatureDescriptionParserV2, +) +from taric_parsers.parsers.commodity_parser import ( # noqa + GoodsNomenclatureDescriptionPeriodParserV2, +) +from taric_parsers.parsers.commodity_parser import ( # noqa + GoodsNomenclatureIndentParserV2, +) +from taric_parsers.parsers.commodity_parser import ( # noqa + GoodsNomenclatureOriginParserV2, +) +from taric_parsers.parsers.commodity_parser import GoodsNomenclatureParserV2 # noqa +from taric_parsers.parsers.commodity_parser import ( # noqa + GoodsNomenclatureSuccessorParserV2, +) +from taric_parsers.parsers.footnote_parser import FootnoteDescriptionParserV2 # noqa +from taric_parsers.parsers.footnote_parser import ( # noqa + FootnoteDescriptionPeriodParserV2, +) +from taric_parsers.parsers.footnote_parser import FootnoteParserV2 # noqa +from taric_parsers.parsers.footnote_parser import ( # noqa + FootnoteTypeDescriptionParserV2, +) +from taric_parsers.parsers.footnote_parser import FootnoteTypeParserV2 # noqa +from taric_parsers.parsers.geo_area_parser import ( # noqa + GeographicalAreaDescriptionParserV2, +) +from taric_parsers.parsers.geo_area_parser import ( # noqa + GeographicalAreaDescriptionPeriodParserV2, +) +from taric_parsers.parsers.geo_area_parser import GeographicalAreaParserV2 # noqa +from taric_parsers.parsers.geo_area_parser import GeographicalMembershipParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + AdditionalCodeTypeMeasureTypeParserV2, +) +from taric_parsers.parsers.measure_parser import ( # noqa + DutyExpressionDescriptionParserV2, +) +from taric_parsers.parsers.measure_parser import DutyExpressionParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + FootnoteAssociationMeasureParserV2, +) +from taric_parsers.parsers.measure_parser import ( # noqa + MeasureActionDescriptionParserV2, +) +from taric_parsers.parsers.measure_parser import MeasureActionParserV2 # noqa +from taric_parsers.parsers.measure_parser import MeasureComponentParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + MeasureConditionCodeDescriptionParserV2, +) +from taric_parsers.parsers.measure_parser import MeasureConditionCodeParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + MeasureConditionComponentParserV2, +) +from taric_parsers.parsers.measure_parser import MeasureConditionParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + MeasureExcludedGeographicalAreaParserV2, +) +from taric_parsers.parsers.measure_parser import MeasurementParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + MeasurementUnitDescriptionParserV2, +) +from taric_parsers.parsers.measure_parser import MeasurementUnitParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + MeasurementUnitQualifierDescriptionParserV2, +) +from taric_parsers.parsers.measure_parser import ( # noqa + MeasurementUnitQualifierParserV2, +) +from taric_parsers.parsers.measure_parser import MeasureParserV2 # noqa +from taric_parsers.parsers.measure_parser import MeasureTypeDescriptionParserV2 # noqa +from taric_parsers.parsers.measure_parser import MeasureTypeParserV2 # noqa +from taric_parsers.parsers.measure_parser import ( # noqa + MeasureTypeSeriesDescriptionParserV2, +) +from taric_parsers.parsers.measure_parser import MeasureTypeSeriesParserV2 # noqa +from taric_parsers.parsers.measure_parser import MonetaryUnitDescriptionParserV2 # noqa +from taric_parsers.parsers.measure_parser import MonetaryUnitParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaAssociationParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaBalanceEventParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaBlockingParserV2 # noqa +from taric_parsers.parsers.quota_parser import ( # noqa + QuotaClosedAndTransferredEventParserV2, +) +from taric_parsers.parsers.quota_parser import QuotaCriticalEventParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaDefinitionParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaEventParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaExhaustionEventParserV2 # noqa +from taric_parsers.parsers.quota_parser import ( # noqa + QuotaOrderNumberOriginExclusionParserV2, +) +from taric_parsers.parsers.quota_parser import QuotaOrderNumberOriginParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaOrderNumberParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaReopeningEventParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaSuspensionParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaUnblockingEventParserV2 # noqa +from taric_parsers.parsers.quota_parser import QuotaUnsuspensionEventParserV2 # noqa +from taric_parsers.parsers.regulation_parser import BaseRegulationParserV2 # noqa +from taric_parsers.parsers.regulation_parser import ( # noqa + FullTemporaryStopActionParserV2, +) +from taric_parsers.parsers.regulation_parser import ( # noqa + FullTemporaryStopRegulationParserV2, +) +from taric_parsers.parsers.regulation_parser import ( # noqa + ModificationRegulationParserV2, +) +from taric_parsers.parsers.regulation_parser import ( # noqa + RegulationGroupDescriptionParserV2, +) +from taric_parsers.parsers.regulation_parser import RegulationGroupParserV2 # noqa +from taric_parsers.parsers.regulation_parser import ( # noqa + RegulationReplacementParserV2, +) +from taric_parsers.parsers.taric_parser import BaseTaricParser +from taric_parsers.parsers.taric_parser import ParserHelper + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestParserHelper: + def test_get_parser_by_model(self): + class MadeUpModel: + sequence_number: int = None + transaction_id: int = None + valid_between: TaricDateRange = None + + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + class TestParentElementParser(BaseTaricParser): + xml_object_tag = "test.parent.element" + + record_code = "900" + subrecord_code = "100" + valid_between = None + valid_between_lower: date = None + valid_between_upper: date = None + invalid_property: str = None + model = MadeUpModel + + def __init__(self): + super().__init__() + + target = ParserHelper.get_parser_by_model + + result = target(MadeUpModel) + + assert result == TestParentElementParser + + def test_get_parser_by_model_errors_with_no_match(self): + class MadeUpModel: + sequence_number: int = None + transaction_id: int = None + valid_between: TaricDateRange = None + + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + e = None + with pytest.raises(Exception) as e: + ParserHelper.get_parser_by_model(MadeUpModel) + + assert "No parser class found for parsing MadeUpModel" in str(e) + + def test_get_child_parsers(self): + class MadeUpModel: + def __init__(self): + self.x = "z" + + class TestParentElementParser(BaseTaricParser): + model = MadeUpModel + xml_object_tag = "test.parent.element" + + def __init__(self): + super().__init__() + + class TestChildElementParser(BaseTaricParser): + model = MadeUpModel + parent_parser = TestParentElementParser + model_links = [ + ModelLink( + MadeUpModel, + [ + ModelLinkField("parent_field_1", "field_1"), + ], + "test.parent.element", + ), + ] + + def __init__(self): + super().__init__() + + result = ParserHelper.get_child_parsers(TestParentElementParser()) + + assert result[0] == TestChildElementParser + + def test_get_child_parsers_empty_list_when_no_children(self): + class MadeUpModel: + def __init__(self): + self.x = "z" + + class TestParentElementParser(BaseTaricParser): + model = MadeUpModel + xml_object_tag = "test.parent.element" + + def __init__(self): + super().__init__() + + result = ParserHelper.get_child_parsers(TestParentElementParser()) + + assert len(result) == 0 + + def test_get_parser_by_tag(self): + class MadeUpModel: + def __init__(self): + self.x = "z" + + class TestParentElementParser(BaseTaricParser): + model = MadeUpModel + xml_object_tag = "test.parent.element.zzz" + + def __init__(self): + super().__init__() + + matched_parser = ParserHelper.get_parser_by_tag( + TestParentElementParser.xml_object_tag, + ) + + assert matched_parser == TestParentElementParser + + def test_get_parser_by_tag_raises_exception_if_no_match(self): + with pytest.raises(Exception) as e: + ParserHelper.get_parser_by_tag("some.nonexistant.tag") + + assert "No parser class matching some.nonexistant.tag" in str(e) + + def test_get_parser_classes(self): + target = ParserHelper.get_parser_classes() + + assert len(target) > 65 + + def test_subclasses_for(self): + target = ParserHelper.subclasses_for(BaseTaricParser) + + assert len(target) > 65 diff --git a/taric_parsers/tests/test_taric_xml_sources.py b/taric_parsers/tests/test_taric_xml_sources.py new file mode 100644 index 000000000..24854c7e9 --- /dev/null +++ b/taric_parsers/tests/test_taric_xml_sources.py @@ -0,0 +1,55 @@ +import os + +import pytest + +from taric_parsers.taric_xml_source import TaricXMLFileSource +from taric_parsers.taric_xml_source import TaricXMLSourceBase +from taric_parsers.taric_xml_source import TaricXMLStringSource + +pytestmark = pytest.mark.django_db + + +@pytest.mark.importer_v2 +class TestTaricXMLSourceBase: + def test_init(self): + assert isinstance(TaricXMLSourceBase(), TaricXMLSourceBase) + + def test_get_xml_string(self): + target_inst = TaricXMLSourceBase() + with pytest.raises(Exception) as e: + target_inst.get_xml_string() + + assert "Implement on child class" in str(e) + + +@pytest.mark.importer_v2 +class TestTaricXMLStringSource: + def test_init(self): + assert isinstance(TaricXMLStringSource("xml"), TaricXMLStringSource) + + def test_get_xml_string(self): + target_inst = TaricXMLStringSource("xml") + assert target_inst.get_xml_string() == "xml" + + +@pytest.mark.importer_v2 +class TestTaricXMLFileSource: + def get_xml_file_path(self): + path_to_current_file = os.path.realpath(__file__) + current_directory = os.path.split(path_to_current_file)[0] + file_path_to_read = os.path.join( + current_directory, + "support", + "additional_code_CREATE.xml", + ) + return file_path_to_read + + def test_init(self): + assert isinstance( + TaricXMLFileSource(self.get_xml_file_path()), + TaricXMLFileSource, + ) + + def test_get_xml_string(self): + target_inst = TaricXMLFileSource(self.get_xml_file_path()) + assert "some description" in target_inst.get_xml_string() diff --git a/taric_parsers/tests/test_transaction_parser.py b/taric_parsers/tests/test_transaction_parser.py new file mode 100644 index 000000000..fe2374e39 --- /dev/null +++ b/taric_parsers/tests/test_transaction_parser.py @@ -0,0 +1,44 @@ +import os + +import pytest +from bs4 import BeautifulSoup + +from taric_parsers.parsers.additional_code_parsers import ( # noqa + AdditionalCodeDescriptionParserV2, +) +from taric_parsers.parsers.additional_code_parsers import AdditionalCodeParserV2 # noqa +from taric_parsers.parsers.additional_code_parsers import ( # noqa + AdditionalCodeTypeParserV2, +) +from taric_parsers.parsers.additional_code_parsers import ( # noqa + FootnoteAssociationAdditionalCodeParserV2, +) +from taric_parsers.parsers.taric_parser import TransactionParser + +pytestmark = pytest.mark.django_db + + +def get_test_xml_file(file_name): + path_to_current_file = os.path.realpath(__file__) + current_directory = os.path.split(path_to_current_file)[0] + return os.path.join(current_directory, "support", file_name) + + +@pytest.mark.importer_v2 +class TestTransactionParser: + def test_init(self): + taric3_file = get_test_xml_file("additional_code_CREATE.xml") + raw_xml = "" + + # load bs4 file + with open(taric3_file, "r") as file: + raw_xml = file.read() + + bs_taric3_file = BeautifulSoup(raw_xml, "xml") + + transactions = bs_taric3_file.find_all("env:transaction") + + parsed_transaction = TransactionParser(transactions[0], 1) + + assert len(transactions) == 1 + assert len(parsed_transaction.parsed_messages) == 3 diff --git a/taric_parsers/tests/test_views.py b/taric_parsers/tests/test_views.py new file mode 100644 index 000000000..8397ed238 --- /dev/null +++ b/taric_parsers/tests/test_views.py @@ -0,0 +1,104 @@ +from os import path +from unittest.mock import patch + +import pytest +from bs4 import BeautifulSoup +from django.contrib.auth.models import User +from django.core.files.uploadedfile import SimpleUploadedFile +from django.test import Client +from django.urls import reverse + +from common.tests import factories +from common.tests.factories import ImportBatchFactory + +pytestmark = pytest.mark.django_db + +TEST_FILES_PATH = path.join(path.dirname(__file__), "support") + + +@pytest.mark.importer_v2 +@pytest.mark.parametrize( + "url_name", + [ + "taric_parser_import_ui_details", + "taric_parser_import_ui_list", + "taric_parser_import_ui_create", + ], +) +def test_import_urls_requires_superuser( + valid_user: User, + admin_user: User, + client: Client, + url_name: str, +): + """Ensure only superusers can access the TARIC Parser views.""" + + if url_name == "taric_parser_import_ui_details": + # Seed data with import batch + ib = ImportBatchFactory.create() + url = reverse(url_name, args=[ib.pk]) + else: + url = reverse(url_name) + + bad_response = client.get(url) + assert bad_response.status_code == 302 + assert bad_response.url != url + + client.force_login(valid_user) + bad_response = client.get(url) + assert bad_response.status_code == 403 + + client.force_login(admin_user) + good_response = client.get(url) + assert good_response.status_code == 200 + assert good_response.request["PATH_INFO"] == url + + +@pytest.mark.importer_v2 +@patch("taric_parsers.forms.UploadTaricForm.save") +def test_import_success_redirect(mock_save, superuser_client): + mock_save.return_value = factories.ImportBatchFactory.create() + url = reverse("taric_parser_import_ui_create") + redirect_url = reverse("taric_parser_import_ui_list") + with open(f"{TEST_FILES_PATH}/valid.xml", "rb") as f: + content = f.read() + taric_file = SimpleUploadedFile("taric_file.xml", content, content_type="text/xml") + response = superuser_client.post( + url, + {"taric_file": taric_file, "name": "test file", "commodities_only": "on"}, + ) + assert response.status_code == 302 + assert response.url == redirect_url + + +@pytest.mark.importer_v2 +@pytest.mark.parametrize( + "file_name,error_msg", + [ + ( + "invalid.xml", + "The selected file could not be processed, please check the file and try again.", + ), + ( + "broken.xml", + "The selected file could not be processed, please check the file and try again.", + ), + # ("dtd.xml", "The selected file could not be processed, please check the file and try again."), + ( + "invalid_type.txt", + "The selected file could not be processed, please check the file and try again.", + ), + ], +) +def test_import_failure(file_name, error_msg, superuser_client): + url = reverse("taric_parser_import_ui_create") + with open(f"{TEST_FILES_PATH}/{file_name}", "rb") as f: + content = f.read() + taric_file = SimpleUploadedFile("taric_file.xml", content, content_type="text/xml") + response = superuser_client.post( + url, + {"taric_file": taric_file, "name": "test file"}, + ) + assert response.status_code == 200 + soup = BeautifulSoup(str(response.content), "html.parser") + assert error_msg in soup.select(".govuk-error-message")[0].text diff --git a/taric_parsers/urls.py b/taric_parsers/urls.py new file mode 100644 index 000000000..32b8290de --- /dev/null +++ b/taric_parsers/urls.py @@ -0,0 +1,23 @@ +from django.urls import path + +from taric_parsers import views + +importer_urlpatterns = [ + path( + "taric_parser//details/", + views.TaricImportDetails.as_view(), + name="taric_parser_import_ui_details", + ), + path( + "taric_parser/", + views.TaricImportList.as_view(), + name="taric_parser_import_ui_list", + ), + path( + "taric_parser/create/", + views.TaricImportUpload.as_view(), + name="taric_parser_import_ui_create", + ), +] + +urlpatterns = importer_urlpatterns diff --git a/taric_parsers/utils.py b/taric_parsers/utils.py new file mode 100644 index 000000000..9221f13c0 --- /dev/null +++ b/taric_parsers/utils.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +from hashlib import sha256 +from typing import Any +from typing import Dict +from typing import Iterable +from typing import List +from typing import Optional +from typing import Set +from typing import TypedDict + +from django.db.models.query_utils import DeferredAttribute + +from common.models import TrackedModel +from common.models.tracked_utils import get_relations + + +def col(label: str) -> int: + """Return the correct index given an Excel column letter.""" + if not label.isalpha(): + raise ValueError(f"'{label}' is not a valid column label.") + return ( + sum( + (ord(char) - ord("A") + 1) * (26**position) + for position, char in enumerate(reversed(label.upper())) + ) + - 1 + ) + + +class LinksType(TypedDict): + """ + A type defining the interface between a handler and its links. + + This constitutes: + + - identifying_fields - for the fields which are specific identifiers for the link. + - model - for the model being linked to. + - name - for the name of the link on the object being made. + - optional - whether this link is nullable. + """ + + identifying_fields: Optional[Iterable[str]] + model: TrackedModel + name: str + optional: bool + + +class DispatchedObjectType(TypedDict): + """ + A type defining the interface expected for any object passed to the + TariffObjectNursery. + + This constitutes: + - data - a dictionary with objects data + - tag - the type of record it reflects + - transaction_id - the transaction to attach the object to. + """ + + data: dict + tag: str + transaction_id: int + + +def generate_key( + tag: str, + identifying_fields: Iterable[str], + data: Dict[str, Any], +) -> str: + """ + Generate a unique hash key for each object. + + The identifying fields are the primary input for the hash however, + as these are likely to be duplicated across all the interdependent + objects - as well as a possibility , the serializer class and "name" have been + """ + hash_input = tag + for key in identifying_fields: + hash_input += str(data[key]) + + return sha256(hash_input.encode()).hexdigest() + + +def get_record_codes( + record: TrackedModel, + use_subrecord_codes: bool = False, +) -> List[str]: + key = record.record_code + + if use_subrecord_codes is False: + return [key] + + subrecord_code = record.subrecord_code + + if isinstance(subrecord_code, str): + return [f"{key}{subrecord_code}"] + + if isinstance(subrecord_code, DeferredAttribute): + return [f"{key}{code}" for code, _ in subrecord_code.field.choices] + + return [] + + +def build_dependency_tree(use_subrecord_codes: bool = False) -> Dict[str, Set[str]]: + """ + Build a dependency tree of all the TrackedModel subclasses mapped by record + code. + + The return value is a dictionary, mapped by record code, where the mapped values + are sets listing all the other record codes the mapped record code depends on. + + A dependency is defined as any foreign key relationship to another record code. + An example output is given below. + + .. code:: python + + { + "220": {"215", "210"}, + } + """ + + dependency_map = {} + + record_codes = { + code + for subclass in TrackedModel.__subclasses__() + for code in get_record_codes(subclass) + } + + for subclass in TrackedModel.__subclasses__(): + for record_code in get_record_codes(subclass): + if record_code not in dependency_map: + dependency_map[record_code] = set() + + for relation in get_relations(subclass).values(): + relation_codes = get_record_codes(relation) + + for relation_code in relation_codes: + if relation_code != record_code and relation_code in record_codes: + dependency_map[record_code].add(relation_code) + + return dependency_map + + +dependency_tree = build_dependency_tree() diff --git a/taric_parsers/validators.py b/taric_parsers/validators.py new file mode 100644 index 000000000..4855ff6b2 --- /dev/null +++ b/taric_parsers/validators.py @@ -0,0 +1,15 @@ +from django.db import models + + +class ImportStatus(models.TextChoices): + FAILED = "FAILED", "Failed" + """An issue was detected that prevented the import from proceeding.""" + + COMPLETED_WITH_WARNINGS = "COMPLETED_WITH_WARNINGS", "Completed with warnings" + """Non failure issues were detected, but the import completed.""" + + COMPLETED = "COMPLETED", "Completed" + """Import completed without any issues.""" + + EMPTY = "EMPTY", "Empty" + """No data to import.""" diff --git a/taric_parsers/views.py b/taric_parsers/views.py new file mode 100644 index 000000000..b29ad8bad --- /dev/null +++ b/taric_parsers/views.py @@ -0,0 +1,77 @@ +from xml.etree.ElementTree import ParseError + +from django.db.models import Count +from django.db.models import Q +from django.urls import reverse_lazy +from django.views.generic import DetailView +from django.views.generic import FormView + +from common.views import RequiresSuperuserMixin +from common.views import WithPaginationListView +from importer.filters import ImportBatchFilter +from importer.models import ImportBatch +from importer.models import ImporterChunkStatus +from importer.models import ImportIssueType +from taric_parsers import forms + + +class TaricImportList(RequiresSuperuserMixin, WithPaginationListView): + """UI endpoint for viewing and filtering TARIC parser imports.""" + + queryset = ( + ImportBatch.objects.all() + .order_by("-created_at") + .annotate( + import_issues_error_count=Count( + "issues", + filter=Q(issues__issue_type=ImportIssueType.ERROR), + distinct=True, + ), + import_issues_warning_count=Count( + "issues", + filter=Q(issues__issue_type=ImportIssueType.WARNING), + distinct=True, + ), + completed_chunks=Count( + "chunks", + filter=Q(chunks__status=ImporterChunkStatus.DONE), + distinct=True, + ), + ) + ) + + template_name = "taric_parser/list.jinja" + filterset_class = ImportBatchFilter + + +class TaricImportUpload(RequiresSuperuserMixin, FormView): + form_class = forms.UploadTaricForm + fields = ["name"] + success_url = reverse_lazy("taric_parser_import_ui_list") + template_name = "taric_parser/create.jinja" + + def form_valid(self, form): + try: + form.save(user=self.request.user) + except ParseError: + form.add_error( + "taric_file", + "The selected file could not be processed, please check the file and try again.", + ) + return super().form_invalid(form) + + return super().form_valid(form) + + +class TaricImportDetails(RequiresSuperuserMixin, DetailView): + """UI endpoint for viewing details of a TARIC parser import, and view + failures and errors.""" + + model = ImportBatch + queryset = ImportBatch.objects.all() + template_name = "taric_parser/details.jinja" + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context["issues"] = context["object"].issues.all() + return context diff --git a/urls.py b/urls.py index 7e6de7866..8a7ef47c0 100644 --- a/urls.py +++ b/urls.py @@ -1,5 +1,5 @@ """ -tamato URL Configuration. +Tamato URL Configuration. The `urlpatterns` list routes URLs to views. For more information please see: https://docs.djangoproject.com/en/3.0/topics/http/urls/ @@ -35,6 +35,7 @@ path("", include("quotas.urls")), path("", include("regulations.urls")), path("", include("reports.urls")), + path("", include("taric_parsers.urls")), path("", include("workbaskets.urls", namespace="workbaskets")), path("admin/", admin.site.urls), ]