D assistant take 2

contentctl/actions/build.py

@@ -1,19 +1,17 @@
+import datetime
+import json
+import pathlib
 import shutil
-
+import uuid
 from dataclasses import dataclass
 
 from contentctl.input.director import DirectorOutputDto
+from contentctl.objects.config import build
+from contentctl.objects.lookup import CSVLookup, Lookup_Type
+from contentctl.output.api_json_output import ApiJsonOutput
 from contentctl.output.conf_output import ConfOutput
 from contentctl.output.conf_writer import ConfWriter
-from contentctl.output.api_json_output import ApiJsonOutput
 from contentctl.output.data_source_writer import DataSourceWriter
-from contentctl.objects.lookup import CSVLookup, Lookup_Type
-import pathlib
-import json
-import datetime
-import uuid
-
-from contentctl.objects.config import build
 
 
 @dataclass(frozen=True)
@@ -46,9 +44,27 @@
                 / "data_sources.csv"
             )
 
+            if input_dto.director_output_dto.deprecation_documentation is not None:
+                input_dto.director_output_dto.deprecation_documentation.writeDeprecationCSV(
+                    input_dto.config.app,
+                    input_dto.config.getBuildDir() / "deprecation_info.csv",
+                )
+
+            deprecation_lookup_fake_yml_path = (
+                input_dto.config.getPackageDirectoryPath()
+                / "lookups"
+                / "deprecated_content.yml"
+            )
+
             DataSourceWriter.writeDataSourceCsv(
                 input_dto.director_output_dto.data_sources, data_sources_lookup_csv_path
             )
+            deprecation_lookup_csv_path = (
+                input_dto.config.getPackageDirectoryPath()
+                / "lookups"
+                / "deprecated_content.csv"
+            )
+
             input_dto.director_output_dto.addContentToDictMappings(
                 CSVLookup.model_construct(
                     name="data_sources",

contentctl/actions/validate.py

@@ -1,14 +1,14 @@
 import pathlib
 
-from contentctl.input.director import Director, DirectorOutputDto
-from contentctl.objects.config import validate
 from contentctl.enrichments.attack_enrichment import AttackEnrichment
 from contentctl.enrichments.cve_enrichment import CveEnrichment
-from contentctl.objects.atomic import AtomicEnrichment
-from contentctl.objects.lookup import FileBackedLookup
+from contentctl.helper.splunk_app import SplunkApp
 from contentctl.helper.utils import Utils
+from contentctl.input.director import Director, DirectorOutputDto
+from contentctl.objects.atomic import AtomicEnrichment
+from contentctl.objects.config import validate
 from contentctl.objects.data_source import DataSource
-from contentctl.helper.splunk_app import SplunkApp
+from contentctl.objects.lookup import FileBackedLookup
 
 
 class Validate:
@@ -27,6 +27,7 @@ def execute(self, input_dto: validate) -> DirectorOutputDto:
             [],
             [],
             [],
+            [],
         )
 
         director = Director(director_output_dto)

contentctl/helper/utils.py

@@ -1,40 +1,34 @@
-import os
-import git
-import shutil
-import requests
+import pathlib
 import random
+import shutil
 import string
+from math import ceil
 from timeit import default_timer
-import pathlib
+from typing import TYPE_CHECKING, Tuple, Union
 
-from typing import Union, Tuple
+import git
+import requests
 import tqdm
-from math import ceil
-
-from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from contentctl.objects.security_content_object import SecurityContentObject
-from contentctl.objects.security_content_object import SecurityContentObject
 
+from contentctl.objects.security_content_object import SecurityContentObject
 
 TOTAL_BYTES = 0
 ALWAYS_PULL = True
 
 
 class Utils:
     @staticmethod
-    def get_all_yml_files_from_directory(path: str) -> list[pathlib.Path]:
-        listOfFiles: list[pathlib.Path] = []
-        base_path = pathlib.Path(path)
-        if not base_path.exists():
-            return listOfFiles
-        for dirpath, dirnames, filenames in os.walk(path):
-            for file in filenames:
-                if file.endswith(".yml"):
-                    listOfFiles.append(pathlib.Path(os.path.join(dirpath, file)))
+    def get_all_yml_files_from_directory(path: pathlib.Path) -> list[pathlib.Path]:
+        if not path.exists():
+            raise FileNotFoundError(
+                f"Trying to find content files in the directory {path}, but it does not exist. "
+                "It is not mandatory to have content in this directory, but it must exist"
+            )
 
-        return sorted(listOfFiles)
+        return sorted(pathlib.Path(yml_path) for yml_path in path.glob("**/*.yml"))
 
     @staticmethod
     def get_security_content_files_from_directory(
@@ -490,3 +484,4 @@ def getPercent(numerator: float, denominator: float, decimal_places: int) -> str
         ratio = numerator / denominator
         percent = ratio * 100
         return Utils.getFixedWidth(percent, decimal_places) + "%"
+        return Utils.getFixedWidth(percent, decimal_places) + "%"