Populate empty fields in comps.xml unit

When pushing comps.xml files, if a unit already exists in Pulp, fields that have been removed from the comps.xml did not get overwritten. This resulted in unexpected older data being published from Pulp. This change extends the comps.xml parsing by giving an option to fill in all fields in comp unit classes from pulp-rpm. When the data is uploaded into pulp this will overwrite old unwanted data.
amcmahon-rh · Sep 26, 2024 · 1223d63 · 1223d63
1 parent 6f26dd3
commit 1223d63
Show file tree

Hide file tree

Showing 3 changed files with 268 additions and 3 deletions.
diff --git a/pubtools/pulplib/_impl/comps.py b/pubtools/pulplib/_impl/comps.py
@@ -5,6 +5,35 @@
 from io import StringIO
 
 
+# Fields that pulp expects for comps units
+COMMON_FIELDS = [
+    "description",
+    "translated_description",
+    "display_order",
+    "translated_name",
+    "name",
+]
+# Default values for comps units
+UNIT_FIELD_DEFAULTS = {
+    "package_group": (
+        {"default": False, "user_visible": False},
+        [
+            "default_package_names",
+            "optional_package_names",
+            "mandatory_package_names",
+            "default",
+            "user_visible",
+            "langonly",
+            "conditional_package_names",
+        ]
+        + COMMON_FIELDS,
+    ),
+    "package_category": ({}, ["packagegroupids"] + COMMON_FIELDS),
+    "package_environment": ({}, ["group_ids", "options"] + COMMON_FIELDS),
+    "package_langpacks": ({}, ["matches"]),
+}
+
+
 class BooleanStringIO(StringIO):
     """A StringIO which coerces the output value into a boolean."""
 
@@ -333,7 +362,7 @@ def EndElementHandler(self, _name):
         self.current_buf = StringIO()
 
 
-def units_for_xml(io):
+def units_for_xml(io, fill_fields=False):
     """Parse comps.xml from a file-like object and return corresponding Pulp units.
 
     Arguments:
@@ -343,6 +372,10 @@ def units_for_xml(io):
             Should be opened in binary mode, and should point at bytes making up
             a valid comps.xml document encoded with UTF-8, UTF-16, ISO-8859-1 (Latin1)
             or ASCII.
+        fill_fields (bool)
+            Whether we want the MongoEngine Document fields of the comps units
+            to be filled with default values.
+
 
     Returns:
         list[dict]
@@ -356,4 +389,24 @@ def units_for_xml(io):
     modules.
     """
     parser = CompsParser()
-    return parser.parse(io)
+    content = parser.parse(io)
+    if fill_fields:
+        content = fill_unit_field_defaults(content)
+    return content
+
+
+def fill_unit_field_defaults(units):
+    """
+    Fill in missing comp unit fields with default values.
+
+    Arguments:
+    units (dict)
+        Parsed in comps.xml file.
+    """
+
+    for unit in units:
+        defaults, fields = UNIT_FIELD_DEFAULTS[unit["_content_type_id"]]
+        for field in fields:
+            unit[field] = unit.get(field, defaults.get(field))
+
+    return units
diff --git a/tests/comps/test_comps_parse.py b/tests/comps/test_comps_parse.py
@@ -2,7 +2,7 @@
 
 import os
 
-from pubtools.pulplib._impl.comps import units_for_xml
+from pubtools.pulplib._impl.comps import units_for_xml, fill_unit_field_defaults
 
 # Don't use autoformatting in this file because we use u'' string literals
 # at least until py2 support is dropped, and black wants to remove them...
@@ -128,3 +128,113 @@ def test_can_parse_units(data_path):
             {"install": "tkgate-%s", "name": "tkgate"},
         ],
     }
+
+
+def test_parse_populates_default_fields(data_path):
+    """units_for_xml parses comps.xml data and can populate fields with
+       default values."""
+
+    xml_path = os.path.join(data_path, "sparse-comps.xml")
+
+    with open(xml_path, "rb") as f:
+        units = units_for_xml(f, fill_fields=True)
+
+    assert len(units) == 8
+
+    assert units[0] == {"_content_type_id": "package_group", "id": "3d-printing",
+            "name": "3D Printing",
+            "translated_name": {"af": "3D-drukwerk", "bg": "3D Печатане"},
+            "user_visible": True, "default_package_names": None,
+            "optional_package_names": None, "mandatory_package_names": None,
+            "default": False, "langonly": None,
+            "conditional_package_names": None, "description": None,
+            "translated_description": None, "display_order": None}
+    assert units[1] == {"_content_type_id": "package_group", "id": "admin-tools",
+            "name": "Administration Tools",
+            "translated_name": {"af": "Administrasienutsgoed",
+                                "am": "የአስተዳደሩ መሣሪያዎች"},
+            "description": "This group is a collection of graphical administration tools for the system, such as for managing user accounts and configuring system hardware.",
+            "translated_description": {
+                "sr": "Ова група је скуп графичких системских административних алатки, нпр. за управљање корисничким налозима и подешавање хардвера у систему.",
+                "sr@Latn": "Ova grupa je skup grafičkih sistemskih administrativnih alatki, npr. za upravljanje korisničkim nalozima i podešavanje hardvera u sistemu."},
+            "mandatory_package_names": ["abrt-desktop", "gnome-disk-utility"],
+            "default_package_names": None, "optional_package_names": None,
+            "default": False, "user_visible": False, "langonly": None,
+            "conditional_package_names": None, "display_order": None}
+    assert units[2] == {"_content_type_id": "package_category",
+            "id": "kde-desktop-environment", "name": "KDE Desktop",
+            "translated_name": {"af": "KDE-werkskerm", "as": "KDE ডেস্কটপ"},
+            "packagegroupids": ["kde-office", "kde-telepathy"],
+            "description": None, "translated_description": None,
+            "display_order": None}
+    assert units[3] == {
+        "_content_type_id": "package_category",
+        "id": "xfce-desktop-environment",
+        "name": "Xfce Desktop",
+        "translated_name": {"uk": u"Графічне середовище Xfce", "zh_CN": u"Xfce 桌面环境"},
+        "description": "A lightweight desktop environment that works well on low end machines.",
+        "translated_description": {
+            "as": u"এটা লঘুভাৰৰ ডেষ্কট'প পৰিবেশ যি নিম্ন বিন্যাসৰ যন্ত্ৰত ভালকৈ কাম কৰি ।",
+            "ast": u"Un entornu d'escritoriu llixeru que furrula bien en máquines pequeñes.",
+        },
+        "display_order": 15,
+        "packagegroupids": ["xfce-apps", "xfce-desktop"],
+    }
+    assert units[4] == {"_content_type_id": "package_environment",
+            "id": "basic-desktop-environment", "name": "Basic Desktop",
+            "description": "X Window System with a choice of window manager.",
+            "translated_description": {
+                "af": "X Window-stelsel met ’n keuse van vensterbestuurder.",
+                "bg": "X Window система с избор на мениджър на прозорци."},
+            "display_order": None,
+            "group_ids": ["networkmanager-submodules", "standard"],
+            "options": None, "translated_name": None}
+
+    assert units[5] == {"_content_type_id": "package_environment",
+            "id": "cinnamon-desktop-environment", "name": "Cinnamon Desktop",
+            "translated_name": {"en_GB": "Cinnamon Desktop",
+                                "fr": "Bureau Cinnamon"},
+            "translated_description": {
+                "ca": "Cinnamon proporciona un escriptori amb un disseny tradicional, funcionalitats avançades, facilitat d'ús, potent i flexible.",
+                "es": "Cinnamon proporciona un entorno de escritorio tradicional, con características avanzadas, fácil de usar, potente y flexible."},
+            "display_order": 22, "group_ids": ["input-methods", "multimedia"],
+            "options": [{"group": "libreoffice", "default": False}],
+            "description": None}
+
+    assert units[6] == {"_content_type_id": "package_langpacks",
+            "matches": [{"install": "stardict-dic-%s", "name": "stardict"},
+                        {"install": "tagainijisho-dic-%s",
+                         "name": "tagainijisho-common"},
+                        {"install": "tesseract-langpack-%s",
+                         "name": "tesseract"},
+                        {"install": "tkgate-%s", "name": "tkgate"}]}
+
+    assert units[7] ==  {"_content_type_id": "package_langpacks", "matches": None}
+
+
+
+def test_fill_unit_field_defaults():
+    """Missing fields are filled in with expected values."""
+    units = [{"_content_type_id": "package_group"},
+             {"_content_type_id": "package_category"},
+             {"_content_type_id": "package_environment"},
+             {"_content_type_id": "package_langpacks"}]
+    expected = [{"_content_type_id": "package_group", "default_package_names": None,
+            "optional_package_names": None, "mandatory_package_names": None,
+            "default": False, "user_visible": False, "langonly": None,
+            "conditional_package_names": None, "description": None,
+            "translated_description": None, "display_order": None,
+            "translated_name": None, "name": None},
+           {"_content_type_id": "package_category", "packagegroupids": None,
+            "description": None, "translated_description": None,
+            "display_order": None,
+            "translated_name": None, "name": None},
+           {"_content_type_id": "package_environment", "group_ids": None,
+            "options": None, "description": None,
+            "translated_description": None,
+            "display_order": None, "translated_name": None, "name": None},
+           {"_content_type_id": "package_langpacks", "matches": None}]
+
+    units = fill_unit_field_defaults(units)
+
+    assert units == expected
diff --git a/tests/data/sparse-comps.xml b/tests/data/sparse-comps.xml
@@ -0,0 +1,102 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE comps
+  PUBLIC '-//Red Hat, Inc.//DTD Comps info//EN'
+  'comps.dtd'>
+
+<!--
+  A comps file for testing.
+
+  This was originally copied from sample-comps.xml, but trimmed down to
+  test field population
+-->
+
+<comps>
+
+
+  <group>
+    <id>3d-printing</id>
+    <name>3D Printing</name>
+    <name xml:lang="af">3D-drukwerk</name>
+    <name xml:lang="bg">3D Печатане</name>
+    <uservisible>true</uservisible>
+  </group>
+  <group>
+    <id>admin-tools</id>
+    <name>Administration Tools</name>
+    <name xml:lang="af">Administrasienutsgoed</name>
+    <name xml:lang="am">የአስተዳደሩ መሣሪያዎች</name>
+    <description>This group is a collection of graphical administration tools for the system, such as for managing user accounts and configuring system hardware.</description>
+    <description xml:lang="sr">Ова група је скуп графичких системских административних алатки, нпр. за управљање корисничким налозима и подешавање хардвера у систему.</description>
+    <description xml:lang="sr@Latn">Ova grupa je skup grafičkih sistemskih administrativnih alatki, npr. za upravljanje korisničkim nalozima i podešavanje hardvera u sistemu.</description>
+    <packagelist>
+      <packagereq type="mandatory">abrt-desktop</packagereq>
+      <packagereq type="mandatory">gnome-disk-utility</packagereq>
+    </packagelist>
+  </group>
+
+
+  <category>
+    <id>kde-desktop-environment</id>
+    <name>KDE Desktop</name>
+    <name xml:lang="af">KDE-werkskerm</name>
+    <name xml:lang="as">KDE ডেস্কটপ</name>
+    <grouplist>
+      <groupid>kde-office</groupid>
+      <groupid>kde-telepathy</groupid>
+    </grouplist>
+  </category>
+  <category>
+    <id>xfce-desktop-environment</id>
+    <name>Xfce Desktop</name>
+    <name xml:lang="uk">Графічне середовище Xfce</name>
+    <name xml:lang="zh_CN">Xfce 桌面环境</name>
+    <description>A lightweight desktop environment that works well on low end machines.</description>
+    <description xml:lang="as">এটা লঘুভাৰৰ ডেষ্কট'প পৰিবেশ যি নিম্ন বিন্যাসৰ যন্ত্ৰত ভালকৈ কাম কৰি ।</description>
+    <description xml:lang="ast">Un entornu d'escritoriu llixeru que furrula bien en máquines pequeñes.</description>
+    <display_order>15</display_order>
+    <grouplist>
+      <groupid>xfce-apps</groupid>
+      <groupid>xfce-desktop</groupid>
+    </grouplist>
+  </category>
+
+
+  <environment>
+    <id>basic-desktop-environment</id>
+    <name>Basic Desktop</name>
+    <description>X Window System with a choice of window manager.</description>
+    <description xml:lang="af">X Window-stelsel met ’n keuse van vensterbestuurder.</description>
+    <description xml:lang="bg">X Window система с избор на мениджър на прозорци.</description>
+    <display_order><!-- tweaked: try empty display order --></display_order>
+    <grouplist>
+      <groupid>networkmanager-submodules</groupid>
+      <groupid>standard</groupid>
+    </grouplist>
+  </environment>
+  <environment>
+    <id>cinnamon-desktop-environment</id>
+    <name>Cinnamon Desktop</name>
+    <name xml:lang="en_GB">Cinnamon Desktop</name>
+    <name xml:lang="fr">Bureau Cinnamon</name>
+    <description xml:lang="ca">Cinnamon proporciona un escriptori amb un disseny tradicional, funcionalitats avançades, facilitat d'ús, potent i flexible.</description>
+    <description xml:lang="es">Cinnamon proporciona un entorno de escritorio tradicional, con características avanzadas, fácil de usar, potente y flexible.</description>
+    <display_order>22</display_order>
+    <grouplist>
+      <groupid>input-methods</groupid>
+      <groupid>multimedia</groupid>
+    </grouplist>
+    <optionlist>
+      <groupid>libreoffice</groupid>
+    </optionlist>
+  </environment>
+
+  <langpacks>
+    <match install="stardict-dic-%s" name="stardict"/>
+    <match install="tagainijisho-dic-%s" name="tagainijisho-common"/>
+    <match install="tesseract-langpack-%s" name="tesseract"/>
+    <match install="tkgate-%s" name="tkgate"/>
+  </langpacks>
+  <langpacks>
+    <!-- An empty langpack doesn't really make much sense hey-->
+  </langpacks>
+</comps>