From 580367493e0bacc6d05ba68b1fd5d8df61b3e903 Mon Sep 17 00:00:00 2001 From: Aidan McMahon-Smith Date: Wed, 25 Sep 2024 17:17:00 +0200 Subject: [PATCH] Populate optional comps.xml unit fields [RHELDST-25995] When pushing comps.xml files, if a unit already exists in Pulp, fields that have been removed from the comps.xml did not get overwritten. This resulted in unexpected older data being published from Pulp. This change extends the comps.xml parsing by giving an option to fill in all fields in comp unit classes from pulp-rpm. When the data is uploaded into pulp this will overwrite old unwanted data. --- src/pubtools/pulplib/_impl/comps.py | 47 +++++++++- tests/comps/test_comps_parse.py | 122 +++++++++++++++++++++++++- tests/data/sparse-comps.xml | 102 +++++++++++++++++++++ tests/repository/test_upload_comps.py | 33 ++++--- 4 files changed, 292 insertions(+), 12 deletions(-) create mode 100644 tests/data/sparse-comps.xml diff --git a/src/pubtools/pulplib/_impl/comps.py b/src/pubtools/pulplib/_impl/comps.py index fff9d9aa..e2c50ca1 100644 --- a/src/pubtools/pulplib/_impl/comps.py +++ b/src/pubtools/pulplib/_impl/comps.py @@ -5,6 +5,34 @@ from io import StringIO +# All optional fields in the pulp-rpm comps.xml units +COMMON_FIELDS = [ + "description", + "translated_description", + "display_order", + "translated_name", + "name", +] +UNIT_FIELD_DEFAULTS = { + "package_group": ( + {"default": False, "user_visible": False}, + [ + "default_package_names", + "optional_package_names", + "mandatory_package_names", + "default", + "user_visible", + "langonly", + "conditional_package_names", + ] + + COMMON_FIELDS, + ), + "package_category": ({}, ["packagegroupids"] + COMMON_FIELDS), + "package_environment": ({}, ["group_ids", "options"] + COMMON_FIELDS), + "package_langpacks": ({}, ["matches"]), +} + + class BooleanStringIO(StringIO): """A StringIO which coerces the output value into a boolean.""" @@ -356,4 +384,21 @@ def units_for_xml(io): modules. """ parser = CompsParser() - return parser.parse(io) + return fill_unit_field_defaults(parser.parse(io)) + + +def fill_unit_field_defaults(units): + """ + Fill in missing comp unit fields with default values. + + Arguments: + units (dict) + Parsed comps.xml file. + """ + + for unit in units: + defaults, fields = UNIT_FIELD_DEFAULTS[unit["_content_type_id"]] + for field in fields: + unit[field] = unit.get(field, defaults.get(field)) + + return units diff --git a/tests/comps/test_comps_parse.py b/tests/comps/test_comps_parse.py index ff5b107c..add55f53 100644 --- a/tests/comps/test_comps_parse.py +++ b/tests/comps/test_comps_parse.py @@ -2,7 +2,7 @@ import os -from pubtools.pulplib._impl.comps import units_for_xml +from pubtools.pulplib._impl.comps import units_for_xml, fill_unit_field_defaults # Don't use autoformatting in this file because we use u'' string literals # at least until py2 support is dropped, and black wants to remove them... @@ -38,6 +38,10 @@ def test_can_parse_units(data_path): }, "translated_name": {"af": "3D-drukwerk", "bg": u"3D Печатане"}, "user_visible": True, + "optional_package_names": None, + "langonly": None, + "mandatory_package_names": None, + "display_order": None, } assert units[1] == { @@ -53,6 +57,11 @@ def test_can_parse_units(data_path): "default": False, "user_visible": True, "mandatory_package_names": ["abrt-desktop", "gnome-disk-utility"], + "conditional_package_names": None, + "default_package_names": None, + "display_order": None, + "langonly": None, + "optional_package_names": None, } assert units[2] == { @@ -115,6 +124,7 @@ def test_can_parse_units(data_path): "es": u"Cinnamon proporciona un entorno de escritorio tradicional, con características avanzadas, fácil de usar, potente y flexible.", }, "display_order": 22, + "description": None, "group_ids": ["input-methods", "multimedia"], "options": [{"default": False, "group": "libreoffice"}], } @@ -128,3 +138,113 @@ def test_can_parse_units(data_path): {"install": "tkgate-%s", "name": "tkgate"}, ], } + + +def test_parse_populates_default_fields(data_path): + """units_for_xml parses comps.xml data and can populate fields with + default values.""" + + xml_path = os.path.join(data_path, "sparse-comps.xml") + + with open(xml_path, "rb") as f: + units = units_for_xml(f) + + assert len(units) == 8 + + assert units[0] == {"_content_type_id": "package_group", "id": "3d-printing", + "name": "3D Printing", + "translated_name": {"af": "3D-drukwerk", "bg": "3D Печатане"}, + "user_visible": True, "default_package_names": None, + "optional_package_names": None, "mandatory_package_names": None, + "default": False, "langonly": None, + "conditional_package_names": None, "description": None, + "translated_description": None, "display_order": None} + assert units[1] == {"_content_type_id": "package_group", "id": "admin-tools", + "name": "Administration Tools", + "translated_name": {"af": "Administrasienutsgoed", + "am": "የአስተዳደሩ መሣሪያዎች"}, + "description": "This group is a collection of graphical administration tools for the system, such as for managing user accounts and configuring system hardware.", + "translated_description": { + "sr": "Ова група је скуп графичких системских административних алатки, нпр. за управљање корисничким налозима и подешавање хардвера у систему.", + "sr@Latn": "Ova grupa je skup grafičkih sistemskih administrativnih alatki, npr. za upravljanje korisničkim nalozima i podešavanje hardvera u sistemu."}, + "mandatory_package_names": ["abrt-desktop", "gnome-disk-utility"], + "default_package_names": None, "optional_package_names": None, + "default": False, "user_visible": False, "langonly": None, + "conditional_package_names": None, "display_order": None} + assert units[2] == {"_content_type_id": "package_category", + "id": "kde-desktop-environment", "name": "KDE Desktop", + "translated_name": {"af": "KDE-werkskerm", "as": "KDE ডেস্কটপ"}, + "packagegroupids": ["kde-office", "kde-telepathy"], + "description": None, "translated_description": None, + "display_order": None} + assert units[3] == { + "_content_type_id": "package_category", + "id": "xfce-desktop-environment", + "name": "Xfce Desktop", + "translated_name": {"uk": u"Графічне середовище Xfce", "zh_CN": u"Xfce 桌面环境"}, + "description": "A lightweight desktop environment that works well on low end machines.", + "translated_description": { + "as": u"এটা লঘুভাৰৰ ডেষ্কট'প পৰিবেশ যি নিম্ন বিন্যাসৰ যন্ত্ৰত ভালকৈ কাম কৰি ।", + "ast": u"Un entornu d'escritoriu llixeru que furrula bien en máquines pequeñes.", + }, + "display_order": 15, + "packagegroupids": ["xfce-apps", "xfce-desktop"], + } + assert units[4] == {"_content_type_id": "package_environment", + "id": "basic-desktop-environment", "name": "Basic Desktop", + "description": "X Window System with a choice of window manager.", + "translated_description": { + "af": "X Window-stelsel met ’n keuse van vensterbestuurder.", + "bg": "X Window система с избор на мениджър на прозорци."}, + "display_order": None, + "group_ids": ["networkmanager-submodules", "standard"], + "options": None, "translated_name": None} + + assert units[5] == {"_content_type_id": "package_environment", + "id": "cinnamon-desktop-environment", "name": "Cinnamon Desktop", + "translated_name": {"en_GB": "Cinnamon Desktop", + "fr": "Bureau Cinnamon"}, + "translated_description": { + "ca": "Cinnamon proporciona un escriptori amb un disseny tradicional, funcionalitats avançades, facilitat d'ús, potent i flexible.", + "es": "Cinnamon proporciona un entorno de escritorio tradicional, con características avanzadas, fácil de usar, potente y flexible."}, + "display_order": 22, "group_ids": ["input-methods", "multimedia"], + "options": [{"group": "libreoffice", "default": False}], + "description": None} + + assert units[6] == {"_content_type_id": "package_langpacks", + "matches": [{"install": "stardict-dic-%s", "name": "stardict"}, + {"install": "tagainijisho-dic-%s", + "name": "tagainijisho-common"}, + {"install": "tesseract-langpack-%s", + "name": "tesseract"}, + {"install": "tkgate-%s", "name": "tkgate"}]} + + assert units[7] == {"_content_type_id": "package_langpacks", "matches": None} + + + +def test_fill_unit_field_defaults(): + """Missing fields are filled in with expected values.""" + units = [{"_content_type_id": "package_group"}, + {"_content_type_id": "package_category"}, + {"_content_type_id": "package_environment"}, + {"_content_type_id": "package_langpacks"}] + expected = [{"_content_type_id": "package_group", "default_package_names": None, + "optional_package_names": None, "mandatory_package_names": None, + "default": False, "user_visible": False, "langonly": None, + "conditional_package_names": None, "description": None, + "translated_description": None, "display_order": None, + "translated_name": None, "name": None}, + {"_content_type_id": "package_category", "packagegroupids": None, + "description": None, "translated_description": None, + "display_order": None, + "translated_name": None, "name": None}, + {"_content_type_id": "package_environment", "group_ids": None, + "options": None, "description": None, + "translated_description": None, + "display_order": None, "translated_name": None, "name": None}, + {"_content_type_id": "package_langpacks", "matches": None}] + + units = fill_unit_field_defaults(units) + + assert units == expected diff --git a/tests/data/sparse-comps.xml b/tests/data/sparse-comps.xml new file mode 100644 index 00000000..da5c1c1a --- /dev/null +++ b/tests/data/sparse-comps.xml @@ -0,0 +1,102 @@ + + + + + + + + + + 3d-printing + 3D Printing + 3D-drukwerk + 3D Печатане + true + + + admin-tools + Administration Tools + Administrasienutsgoed + የአስተዳደሩ መሣሪያዎች + This group is a collection of graphical administration tools for the system, such as for managing user accounts and configuring system hardware. + Ова група је скуп графичких системских административних алатки, нпр. за управљање корисничким налозима и подешавање хардвера у систему. + Ova grupa je skup grafičkih sistemskih administrativnih alatki, npr. za upravljanje korisničkim nalozima i podešavanje hardvera u sistemu. + + abrt-desktop + gnome-disk-utility + + + + + + kde-desktop-environment + KDE Desktop + KDE-werkskerm + KDE ডেস্কটপ + + kde-office + kde-telepathy + + + + xfce-desktop-environment + Xfce Desktop + Графічне середовище Xfce + Xfce 桌面环境 + A lightweight desktop environment that works well on low end machines. + এটা লঘুভাৰৰ ডেষ্কট'প পৰিবেশ যি নিম্ন বিন্যাসৰ যন্ত্ৰত ভালকৈ কাম কৰি । + Un entornu d'escritoriu llixeru que furrula bien en máquines pequeñes. + 15 + + xfce-apps + xfce-desktop + + + + + + basic-desktop-environment + Basic Desktop + X Window System with a choice of window manager. + X Window-stelsel met ’n keuse van vensterbestuurder. + X Window система с избор на мениджър на прозорци. + + + networkmanager-submodules + standard + + + + cinnamon-desktop-environment + Cinnamon Desktop + Cinnamon Desktop + Bureau Cinnamon + Cinnamon proporciona un escriptori amb un disseny tradicional, funcionalitats avançades, facilitat d'ús, potent i flexible. + Cinnamon proporciona un entorno de escritorio tradicional, con características avanzadas, fácil de usar, potente y flexible. + 22 + + input-methods + multimedia + + + libreoffice + + + + + + + + + + + + + diff --git a/tests/repository/test_upload_comps.py b/tests/repository/test_upload_comps.py index 3296e772..d592538b 100644 --- a/tests/repository/test_upload_comps.py +++ b/tests/repository/test_upload_comps.py @@ -136,37 +136,50 @@ def test_upload_comps_xml(client, requests_mocker): imported_units.sort(key=lambda u: u["_content_type_id"]) # Did we import exactly the expected units? + assert imported_units == [ { "_content_type_id": "package_category", + "description": None, + "display_order": 10, "id": "kde-desktop-environment", "name": "KDE Desktop", - "display_order": 10, "packagegroupids": ["kde-office", "kde-telepathy"], "repo_id": "repo1", + "translated_description": None, + "translated_name": None, }, { "_content_type_id": "package_environment", - "id": "basic-desktop-environment", - "name": "Basic Desktop", - "translated_name": {"af": "Basiese werkskerm"}, "description": "X Window System with a choice of window manager.", + "display_order": None, "group_ids": ["networkmanager-submodules", "standard"], + "id": "basic-desktop-environment", + "name": "Basic Desktop", "options": [ - {"group": "xmonad", "default": True}, - {"group": "xmonad-mate", "default": False}, + {"default": True, "group": "xmonad"}, + {"default": False, "group": "xmonad-mate"}, ], "repo_id": "repo1", + "translated_description": None, + "translated_name": {"af": "Basiese werkskerm"}, }, { "_content_type_id": "package_group", - "id": "3d-printing", - "name": "3D Printing", - "translated_name": {"af": "3D-drukwerk"}, - "description": "3D printing software", + "conditional_package_names": None, + "default": False, "default_package_names": ["admesh"], + "description": "3D printing software", + "display_order": None, + "id": "3d-printing", + "langonly": None, "mandatory_package_names": ["blender"], + "name": "3D Printing", + "optional_package_names": None, "repo_id": "repo1", + "translated_description": None, + "translated_name": {"af": "3D-drukwerk"}, + "user_visible": False, }, { "_content_type_id": "package_langpacks",