Build steps no longer write duplicate entries to def.json where app…

…ropriate Ticket: ENT-10830 Signed-off-by: jakub-nt <[email protected]>
cfengine · Jul 31, 2024 · 88ba4aa · 88ba4aa
1 parent 9dab8ac
commit 88ba4aa
Show file tree

Hide file tree

Showing 3 changed files with 74 additions and 3 deletions.
diff --git a/cfbs/build.py b/cfbs/build.py
@@ -3,6 +3,7 @@
 from cfbs.utils import (
     canonify,
     cp,
+    deduplicate_def_json,
     find,
     merge_json,
     mkdir,
@@ -115,6 +116,8 @@ def _perform_build_step(module, step, max_length):
             print("Warning: '%s' looks empty, adding nothing" % os.path.basename(src))
         if original:
             merged = merge_json(original, extras)
+            if os.path.basename(dst) == "def.json":
+                merged = deduplicate_def_json(merged)
         else:
             merged = extras
         write_json(dst, merged)
@@ -145,6 +148,7 @@ def _perform_build_step(module, step, max_length):
                     extra = read_json(os.path.join(root, f))
                     if extra:
                         merged = merge_json(merged, extra)
+                        merged = deduplicate_def_json(merged)
                 else:
                     s = os.path.join(root, f)
                     d = os.path.join(destination, dstarg, root[len(src) :], f)
@@ -182,6 +186,7 @@ def _perform_build_step(module, step, max_length):
         if original:
             log.debug("Original def.json: %s", pretty(original))
             merged = merge_json(original, extras)
+            merged = deduplicate_def_json(merged)
         else:
             merged = extras
         log.debug("Merged def.json: %s", pretty(merged))
@@ -208,7 +213,11 @@ def _perform_build_step(module, step, max_length):
         path = os.path.join(destination, "def.json")
         original = read_json(path)
         log.debug("Original def.json: %s" % pretty(original))
-        merged = merge_json(original, augment) if original else augment
+        if original:
+            merged = merge_json(original, augment)
+            merged = deduplicate_def_json(merged)
+        else:
+            merged = augment
         log.debug("Merged def.json: %s", pretty(merged))
         write_json(path, merged)
     elif operation == "bundles":
@@ -219,7 +228,11 @@ def _perform_build_step(module, step, max_length):
         path = os.path.join(destination, "def.json")
         original = read_json(path)
         log.debug("Original def.json: %s" % pretty(original))
-        merged = merge_json(original, augment) if original else augment
+        if original:
+            merged = merge_json(original, augment)
+            merged = deduplicate_def_json(merged)
+        else:
+            merged = augment
         log.debug("Merged def.json: %s", pretty(merged))
         write_json(path, merged)
     else:

diff --git a/cfbs/utils.py b/cfbs/utils.py
@@ -180,6 +180,25 @@ def merge_json(a, b, overwrite_callback=None, stack=None):
     return a
 
 
+def deduplicate_def_json(d):
+    if "inputs" in d:
+        d["inputs"] = deduplicate_list(d["inputs"])
+    if "augments" in d:
+        d["augments"] = deduplicate_list(d["augments"])
+
+    for variable in d.get("variables", {}).values():
+        if type(variable) is not dict:
+            continue
+        if "tags" in variable:
+            variable["tags"] = deduplicate_list(variable["tags"])
+
+    return d
+
+
+def deduplicate_list(l):
+    return list(OrderedDict.fromkeys(l))
+
+
 def cfbs_filename() -> str:
     return "cfbs.json"
 

diff --git a/tests/test_utils.py b/tests/test_utils.py
@@ -1,4 +1,4 @@
-from cfbs.utils import canonify, merge_json, loads_bundlenames
+from cfbs.utils import canonify, deduplicate_def_json, merge_json, loads_bundlenames
 
 
 def test_canonify():
@@ -55,6 +55,45 @@ def test_merge_json():
     assert merged == expected
 
 
+def test_deduplicate_def_json():
+    case = {
+        "inputs": [
+            "services/cfbs/inventory/company.cf",
+            "services/cfbs/inventory/company.cf",
+            "services/cfbs/inventory/company.cf",
+        ]
+    }
+    expected = {"inputs": ["services/cfbs/inventory/company.cf"]}
+
+    deduplicated = deduplicate_def_json(case)
+    assert deduplicated == expected
+
+    case = {
+        "augments": [
+            "/var/cfengine/augments/company.json",
+            "/var/cfengine/augments/company.json",
+        ],
+        "variables": {
+            "MyNamespace:my_bundle.Variable": {
+                "value": {"tags": ["dont-dedupe", "dont-dedupe"]},
+                "tags": ["inventory", "attribute_name=My Inventory", "inventory"],
+            }
+        },
+    }
+    expected = {
+        "augments": ["/var/cfengine/augments/company.json"],
+        "variables": {
+            "MyNamespace:my_bundle.Variable": {
+                "value": {"tags": ["dont-dedupe", "dont-dedupe"]},
+                "tags": ["inventory", "attribute_name=My Inventory"],
+            }
+        },
+    }
+
+    deduplicated = deduplicate_def_json(case)
+    assert deduplicated == expected
+
+
 def test_loads_bundlenames_single_bundle():
     policy = """bundle agent bogus
 {