From 88ba4aa7cc67b51d0097c4a299f3745ff08f9842 Mon Sep 17 00:00:00 2001
From: jakub-nt <175944085+jakub-nt@users.noreply.github.com>
Date: Wed, 31 Jul 2024 17:35:39 +0200
Subject: [PATCH] Build steps no longer write duplicate entries to `def.json`
 where appropriate

Ticket: ENT-10830
Signed-off-by: jakub-nt <175944085+jakub-nt@users.noreply.github.com>
---
 cfbs/build.py       | 17 +++++++++++++++--
 cfbs/utils.py       | 19 +++++++++++++++++++
 tests/test_utils.py | 41 ++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 74 insertions(+), 3 deletions(-)

diff --git a/cfbs/build.py b/cfbs/build.py
index 37ee32f8..37cc4277 100644
--- a/cfbs/build.py
+++ b/cfbs/build.py
@@ -3,6 +3,7 @@
 from cfbs.utils import (
     canonify,
     cp,
+    deduplicate_def_json,
     find,
     merge_json,
     mkdir,
@@ -115,6 +116,8 @@ def _perform_build_step(module, step, max_length):
             print("Warning: '%s' looks empty, adding nothing" % os.path.basename(src))
         if original:
             merged = merge_json(original, extras)
+            if os.path.basename(dst) == "def.json":
+                merged = deduplicate_def_json(merged)
         else:
             merged = extras
         write_json(dst, merged)
@@ -145,6 +148,7 @@ def _perform_build_step(module, step, max_length):
                     extra = read_json(os.path.join(root, f))
                     if extra:
                         merged = merge_json(merged, extra)
+                        merged = deduplicate_def_json(merged)
                 else:
                     s = os.path.join(root, f)
                     d = os.path.join(destination, dstarg, root[len(src) :], f)
@@ -182,6 +186,7 @@ def _perform_build_step(module, step, max_length):
         if original:
             log.debug("Original def.json: %s", pretty(original))
             merged = merge_json(original, extras)
+            merged = deduplicate_def_json(merged)
         else:
             merged = extras
         log.debug("Merged def.json: %s", pretty(merged))
@@ -208,7 +213,11 @@ def _perform_build_step(module, step, max_length):
         path = os.path.join(destination, "def.json")
         original = read_json(path)
         log.debug("Original def.json: %s" % pretty(original))
-        merged = merge_json(original, augment) if original else augment
+        if original:
+            merged = merge_json(original, augment)
+            merged = deduplicate_def_json(merged)
+        else:
+            merged = augment
         log.debug("Merged def.json: %s", pretty(merged))
         write_json(path, merged)
     elif operation == "bundles":
@@ -219,7 +228,11 @@ def _perform_build_step(module, step, max_length):
         path = os.path.join(destination, "def.json")
         original = read_json(path)
         log.debug("Original def.json: %s" % pretty(original))
-        merged = merge_json(original, augment) if original else augment
+        if original:
+            merged = merge_json(original, augment)
+            merged = deduplicate_def_json(merged)
+        else:
+            merged = augment
         log.debug("Merged def.json: %s", pretty(merged))
         write_json(path, merged)
     else:
diff --git a/cfbs/utils.py b/cfbs/utils.py
index 43afb4f3..18404e6e 100644
--- a/cfbs/utils.py
+++ b/cfbs/utils.py
@@ -180,6 +180,25 @@ def merge_json(a, b, overwrite_callback=None, stack=None):
     return a
 
 
+def deduplicate_def_json(d):
+    if "inputs" in d:
+        d["inputs"] = deduplicate_list(d["inputs"])
+    if "augments" in d:
+        d["augments"] = deduplicate_list(d["augments"])
+
+    for variable in d.get("variables", {}).values():
+        if type(variable) is not dict:
+            continue
+        if "tags" in variable:
+            variable["tags"] = deduplicate_list(variable["tags"])
+
+    return d
+
+
+def deduplicate_list(l):
+    return list(OrderedDict.fromkeys(l))
+
+
 def cfbs_filename() -> str:
     return "cfbs.json"
 
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 7a016bf1..78a1e54f 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -1,4 +1,4 @@
-from cfbs.utils import canonify, merge_json, loads_bundlenames
+from cfbs.utils import canonify, deduplicate_def_json, merge_json, loads_bundlenames
 
 
 def test_canonify():
@@ -55,6 +55,45 @@ def test_merge_json():
     assert merged == expected
 
 
+def test_deduplicate_def_json():
+    case = {
+        "inputs": [
+            "services/cfbs/inventory/company.cf",
+            "services/cfbs/inventory/company.cf",
+            "services/cfbs/inventory/company.cf",
+        ]
+    }
+    expected = {"inputs": ["services/cfbs/inventory/company.cf"]}
+
+    deduplicated = deduplicate_def_json(case)
+    assert deduplicated == expected
+
+    case = {
+        "augments": [
+            "/var/cfengine/augments/company.json",
+            "/var/cfengine/augments/company.json",
+        ],
+        "variables": {
+            "MyNamespace:my_bundle.Variable": {
+                "value": {"tags": ["dont-dedupe", "dont-dedupe"]},
+                "tags": ["inventory", "attribute_name=My Inventory", "inventory"],
+            }
+        },
+    }
+    expected = {
+        "augments": ["/var/cfengine/augments/company.json"],
+        "variables": {
+            "MyNamespace:my_bundle.Variable": {
+                "value": {"tags": ["dont-dedupe", "dont-dedupe"]},
+                "tags": ["inventory", "attribute_name=My Inventory"],
+            }
+        },
+    }
+
+    deduplicated = deduplicate_def_json(case)
+    assert deduplicated == expected
+
+
 def test_loads_bundlenames_single_bundle():
     policy = """bundle agent bogus
 {