From 1223d63c5d72cc637b4d71abd095998ac8c55e53 Mon Sep 17 00:00:00 2001 From: Aidan McMahon-Smith Date: Wed, 25 Sep 2024 17:17:00 +0200 Subject: [PATCH] Populate empty fields in comps.xml unit When pushing comps.xml files, if a unit already exists in Pulp, fields that have been removed from the comps.xml did not get overwritten. This resulted in unexpected older data being published from Pulp. This change extends the comps.xml parsing by giving an option to fill in all fields in comp unit classes from pulp-rpm. When the data is uploaded into pulp this will overwrite old unwanted data. --- pubtools/pulplib/_impl/comps.py | 57 +++++++++++++++- tests/comps/test_comps_parse.py | 112 +++++++++++++++++++++++++++++++- tests/data/sparse-comps.xml | 102 +++++++++++++++++++++++++++++ 3 files changed, 268 insertions(+), 3 deletions(-) create mode 100644 tests/data/sparse-comps.xml diff --git a/pubtools/pulplib/_impl/comps.py b/pubtools/pulplib/_impl/comps.py index fff9d9aa..3946eef8 100644 --- a/pubtools/pulplib/_impl/comps.py +++ b/pubtools/pulplib/_impl/comps.py @@ -5,6 +5,35 @@ from io import StringIO +# Fields that pulp expects for comps units +COMMON_FIELDS = [ + "description", + "translated_description", + "display_order", + "translated_name", + "name", +] +# Default values for comps units +UNIT_FIELD_DEFAULTS = { + "package_group": ( + {"default": False, "user_visible": False}, + [ + "default_package_names", + "optional_package_names", + "mandatory_package_names", + "default", + "user_visible", + "langonly", + "conditional_package_names", + ] + + COMMON_FIELDS, + ), + "package_category": ({}, ["packagegroupids"] + COMMON_FIELDS), + "package_environment": ({}, ["group_ids", "options"] + COMMON_FIELDS), + "package_langpacks": ({}, ["matches"]), +} + + class BooleanStringIO(StringIO): """A StringIO which coerces the output value into a boolean.""" @@ -333,7 +362,7 @@ def EndElementHandler(self, _name): self.current_buf = StringIO() -def units_for_xml(io): +def units_for_xml(io, fill_fields=False): """Parse comps.xml from a file-like object and return corresponding Pulp units. Arguments: @@ -343,6 +372,10 @@ def units_for_xml(io): Should be opened in binary mode, and should point at bytes making up a valid comps.xml document encoded with UTF-8, UTF-16, ISO-8859-1 (Latin1) or ASCII. + fill_fields (bool) + Whether we want the MongoEngine Document fields of the comps units + to be filled with default values. + Returns: list[dict] @@ -356,4 +389,24 @@ def units_for_xml(io): modules. """ parser = CompsParser() - return parser.parse(io) + content = parser.parse(io) + if fill_fields: + content = fill_unit_field_defaults(content) + return content + + +def fill_unit_field_defaults(units): + """ + Fill in missing comp unit fields with default values. + + Arguments: + units (dict) + Parsed in comps.xml file. + """ + + for unit in units: + defaults, fields = UNIT_FIELD_DEFAULTS[unit["_content_type_id"]] + for field in fields: + unit[field] = unit.get(field, defaults.get(field)) + + return units diff --git a/tests/comps/test_comps_parse.py b/tests/comps/test_comps_parse.py index ff5b107c..c4993381 100644 --- a/tests/comps/test_comps_parse.py +++ b/tests/comps/test_comps_parse.py @@ -2,7 +2,7 @@ import os -from pubtools.pulplib._impl.comps import units_for_xml +from pubtools.pulplib._impl.comps import units_for_xml, fill_unit_field_defaults # Don't use autoformatting in this file because we use u'' string literals # at least until py2 support is dropped, and black wants to remove them... @@ -128,3 +128,113 @@ def test_can_parse_units(data_path): {"install": "tkgate-%s", "name": "tkgate"}, ], } + + +def test_parse_populates_default_fields(data_path): + """units_for_xml parses comps.xml data and can populate fields with + default values.""" + + xml_path = os.path.join(data_path, "sparse-comps.xml") + + with open(xml_path, "rb") as f: + units = units_for_xml(f, fill_fields=True) + + assert len(units) == 8 + + assert units[0] == {"_content_type_id": "package_group", "id": "3d-printing", + "name": "3D Printing", + "translated_name": {"af": "3D-drukwerk", "bg": "3D Печатане"}, + "user_visible": True, "default_package_names": None, + "optional_package_names": None, "mandatory_package_names": None, + "default": False, "langonly": None, + "conditional_package_names": None, "description": None, + "translated_description": None, "display_order": None} + assert units[1] == {"_content_type_id": "package_group", "id": "admin-tools", + "name": "Administration Tools", + "translated_name": {"af": "Administrasienutsgoed", + "am": "የአስተዳደሩ መሣሪያዎች"}, + "description": "This group is a collection of graphical administration tools for the system, such as for managing user accounts and configuring system hardware.", + "translated_description": { + "sr": "Ова група је скуп графичких системских административних алатки, нпр. за управљање корисничким налозима и подешавање хардвера у систему.", + "sr@Latn": "Ova grupa je skup grafičkih sistemskih administrativnih alatki, npr. za upravljanje korisničkim nalozima i podešavanje hardvera u sistemu."}, + "mandatory_package_names": ["abrt-desktop", "gnome-disk-utility"], + "default_package_names": None, "optional_package_names": None, + "default": False, "user_visible": False, "langonly": None, + "conditional_package_names": None, "display_order": None} + assert units[2] == {"_content_type_id": "package_category", + "id": "kde-desktop-environment", "name": "KDE Desktop", + "translated_name": {"af": "KDE-werkskerm", "as": "KDE ডেস্কটপ"}, + "packagegroupids": ["kde-office", "kde-telepathy"], + "description": None, "translated_description": None, + "display_order": None} + assert units[3] == { + "_content_type_id": "package_category", + "id": "xfce-desktop-environment", + "name": "Xfce Desktop", + "translated_name": {"uk": u"Графічне середовище Xfce", "zh_CN": u"Xfce 桌面环境"}, + "description": "A lightweight desktop environment that works well on low end machines.", + "translated_description": { + "as": u"এটা লঘুভাৰৰ ডেষ্কট'প পৰিবেশ যি নিম্ন বিন্যাসৰ যন্ত্ৰত ভালকৈ কাম কৰি ।", + "ast": u"Un entornu d'escritoriu llixeru que furrula bien en máquines pequeñes.", + }, + "display_order": 15, + "packagegroupids": ["xfce-apps", "xfce-desktop"], + } + assert units[4] == {"_content_type_id": "package_environment", + "id": "basic-desktop-environment", "name": "Basic Desktop", + "description": "X Window System with a choice of window manager.", + "translated_description": { + "af": "X Window-stelsel met ’n keuse van vensterbestuurder.", + "bg": "X Window система с избор на мениджър на прозорци."}, + "display_order": None, + "group_ids": ["networkmanager-submodules", "standard"], + "options": None, "translated_name": None} + + assert units[5] == {"_content_type_id": "package_environment", + "id": "cinnamon-desktop-environment", "name": "Cinnamon Desktop", + "translated_name": {"en_GB": "Cinnamon Desktop", + "fr": "Bureau Cinnamon"}, + "translated_description": { + "ca": "Cinnamon proporciona un escriptori amb un disseny tradicional, funcionalitats avançades, facilitat d'ús, potent i flexible.", + "es": "Cinnamon proporciona un entorno de escritorio tradicional, con características avanzadas, fácil de usar, potente y flexible."}, + "display_order": 22, "group_ids": ["input-methods", "multimedia"], + "options": [{"group": "libreoffice", "default": False}], + "description": None} + + assert units[6] == {"_content_type_id": "package_langpacks", + "matches": [{"install": "stardict-dic-%s", "name": "stardict"}, + {"install": "tagainijisho-dic-%s", + "name": "tagainijisho-common"}, + {"install": "tesseract-langpack-%s", + "name": "tesseract"}, + {"install": "tkgate-%s", "name": "tkgate"}]} + + assert units[7] == {"_content_type_id": "package_langpacks", "matches": None} + + + +def test_fill_unit_field_defaults(): + """Missing fields are filled in with expected values.""" + units = [{"_content_type_id": "package_group"}, + {"_content_type_id": "package_category"}, + {"_content_type_id": "package_environment"}, + {"_content_type_id": "package_langpacks"}] + expected = [{"_content_type_id": "package_group", "default_package_names": None, + "optional_package_names": None, "mandatory_package_names": None, + "default": False, "user_visible": False, "langonly": None, + "conditional_package_names": None, "description": None, + "translated_description": None, "display_order": None, + "translated_name": None, "name": None}, + {"_content_type_id": "package_category", "packagegroupids": None, + "description": None, "translated_description": None, + "display_order": None, + "translated_name": None, "name": None}, + {"_content_type_id": "package_environment", "group_ids": None, + "options": None, "description": None, + "translated_description": None, + "display_order": None, "translated_name": None, "name": None}, + {"_content_type_id": "package_langpacks", "matches": None}] + + units = fill_unit_field_defaults(units) + + assert units == expected diff --git a/tests/data/sparse-comps.xml b/tests/data/sparse-comps.xml new file mode 100644 index 00000000..da5c1c1a --- /dev/null +++ b/tests/data/sparse-comps.xml @@ -0,0 +1,102 @@ + + + + + + + + + + 3d-printing + 3D Printing + 3D-drukwerk + 3D Печатане + true + + + admin-tools + Administration Tools + Administrasienutsgoed + የአስተዳደሩ መሣሪያዎች + This group is a collection of graphical administration tools for the system, such as for managing user accounts and configuring system hardware. + Ова група је скуп графичких системских административних алатки, нпр. за управљање корисничким налозима и подешавање хардвера у систему. + Ova grupa je skup grafičkih sistemskih administrativnih alatki, npr. za upravljanje korisničkim nalozima i podešavanje hardvera u sistemu. + + abrt-desktop + gnome-disk-utility + + + + + + kde-desktop-environment + KDE Desktop + KDE-werkskerm + KDE ডেস্কটপ + + kde-office + kde-telepathy + + + + xfce-desktop-environment + Xfce Desktop + Графічне середовище Xfce + Xfce 桌面环境 + A lightweight desktop environment that works well on low end machines. + এটা লঘুভাৰৰ ডেষ্কট'প পৰিবেশ যি নিম্ন বিন্যাসৰ যন্ত্ৰত ভালকৈ কাম কৰি । + Un entornu d'escritoriu llixeru que furrula bien en máquines pequeñes. + 15 + + xfce-apps + xfce-desktop + + + + + + basic-desktop-environment + Basic Desktop + X Window System with a choice of window manager. + X Window-stelsel met ’n keuse van vensterbestuurder. + X Window система с избор на мениджър на прозорци. + + + networkmanager-submodules + standard + + + + cinnamon-desktop-environment + Cinnamon Desktop + Cinnamon Desktop + Bureau Cinnamon + Cinnamon proporciona un escriptori amb un disseny tradicional, funcionalitats avançades, facilitat d'ús, potent i flexible. + Cinnamon proporciona un entorno de escritorio tradicional, con características avanzadas, fácil de usar, potente y flexible. + 22 + + input-methods + multimedia + + + libreoffice + + + + + + + + + + + + +