Merge pull request SEKOIA-IO#1085 from SEKOIA-IO/fix/update_utilities

Fix: Add useful utilities
HarishHary · May 30, 2024 · f8ff22a · f8ff22a
2 parents 9cec528 + 221d03f
commit f8ff22a
Show file tree

Hide file tree

Showing 20 changed files with 1,366 additions and 496 deletions.
diff --git a/utils/README.md b/utils/README.md
@@ -6,6 +6,8 @@
 	* [How to ?](#how-to-)
 		* [New module](#new-module)
 		* [New intake](#new-intake)
+        * [Publish intake-format](#publish-intake-format-to-sekoiaio)
+        * [Send events to platform](#send-events-to-sekoiaio)
 * [Parsers](#parsers)
 * [Events](#events)
 * [Manifests](#manifests)
@@ -620,4 +622,65 @@ data_sources:
 * Windows Error Reporting
 * Windows event logs
 * Windows Registry
-* WMI Objects
+* WMI Objects
+
+## Send events to SEKOIA.IO
+
+### Usage
+
+```console
+Usage: send_events.py [OPTIONS] COMMAND [ARGS]...
+
+Options:
+  --install-completion  Install completion for the current shell.
+  --show-completion     Show completion for the current shell, to copy it or
+                        customize the installation.
+  --help                Show this message and exit.
+
+Commands:
+  from-cli
+  from-intake-formats
+  from-text-file
+```
+
+### Example
+
+### From intake-formats input files
+
+Each test file `["input"]["message"]` will be sent as an event.
+
+```console
+poetry run python send_events.py from-intake-formats "<intake-key>" ./../module/format/
+```
+
+### From text file
+
+Each line will be sent as an event
+
+```console
+poetry run python3 send_events.py from-text-file "<intake-key>" ../Downloads/example.txt --url https://app.eur1.sekoia.io/api/v1/intake-http/batch 
+```
+
+### From the terminal
+
+Send one line from the terminal
+
+```console
+poetry run python send_events.py from-cli "<intake-key>" "<event>"
+```
+
+## Publish intake-format to SEKOIA.IO
+
+### Usage
+
+```console
+usage: poetry run publish_format.py [-h] path apikey url
+publish_format.py: error: the following arguments are required: path, apikey url
+```
+
+### Example
+
+### Publish
+```
+poetry run python publish_format.py ./../module/format/ '<API KEY>' '<URL>'
+```
diff --git a/utils/checks/__main__.py b/utils/checks/__main__.py
@@ -20,10 +20,7 @@ def find_modules(root_path: Path) -> list[Path]:
     filtered_elements = {"doc", "utils"}
 
     for element_path in root_path.iterdir():
-        if (
-            element_path.name not in filtered_elements
-            and not element_path.name.startswith(".")
-        ):
+        if element_path.name not in filtered_elements and not element_path.name.startswith("."):
             if element_path.is_dir():
                 module_paths.append(element_path)
 
@@ -50,9 +47,7 @@ def find_formats(root_path: Path, formats: list[Path] | None = None) -> list[Pat
     return sorted(format_paths)
 
 
-def check_format(
-    path: Path, module_result: CheckResult, args: argparse.Namespace
-) -> CheckResult:
+def check_format(path: Path, module_result: CheckResult, args: argparse.Namespace) -> CheckResult:
     valid = FormatValidator(path=path, module_result=module_result, args=args)
     valid.validate()
 
@@ -65,8 +60,7 @@ def check_module_formats(
     module_formats = find_formats(module_result.options["path"], formats=formats)
 
     result = [
-        check_format(path=module_format, module_result=module_result, args=args)
-        for module_format in module_formats
+        check_format(path=module_format, module_result=module_result, args=args) for module_format in module_formats
     ]
 
     return result
@@ -86,9 +80,9 @@ def check_module_uuids_and_slugs(check_module_results: list[CheckResult]):
                 f"than module {module_uuids[module_result.options['manifest_uuid']]}"
             )
 
-            module_uuids[
-                module_result.options["manifest_uuid"]
-            ] = module_result.options.get("manifest_uuid", "unknown")
+            module_uuids[module_result.options["manifest_uuid"]] = module_result.options.get(
+                "manifest_uuid", "unknown"
+            )
 
     # module slugs are unique
     module_slugs: dict[str, str] = dict()
@@ -103,9 +97,9 @@ def check_module_uuids_and_slugs(check_module_results: list[CheckResult]):
                 f"than module {module_slugs[module_result.options['manifest_slug']]}"
             )
 
-            module_slugs[
-                module_result.options["manifest_slug"]
-            ] = module_result.options.get("manifest_slug", "unknown")
+            module_slugs[module_result.options["manifest_slug"]] = module_result.options.get(
+                "manifest_slug", "unknown"
+            )
 
 
 def check_format_uuids_and_slugs(check_format_results: list[CheckResult]):
@@ -122,9 +116,9 @@ def check_format_uuids_and_slugs(check_format_results: list[CheckResult]):
                 f"than format {format_uuids[format_result.options['manifest_uuid']]}"
             )
 
-            format_uuids[
-                format_result.options["manifest_uuid"]
-            ] = format_result.options.get("manifest_uuid", "unknown")
+            format_uuids[format_result.options["manifest_uuid"]] = format_result.options.get(
+                "manifest_uuid", "unknown"
+            )
 
     # format slugs are unique
     format_slugs: dict[str, str] = dict()
@@ -139,9 +133,9 @@ def check_format_uuids_and_slugs(check_format_results: list[CheckResult]):
                 f"than format {format_slugs[format_result.options['manifest_slug']]}"
             )
 
-            format_slugs[
-                format_result.options["manifest_slug"]
-            ] = format_result.options.get("manifest_slug", "unknown")
+            format_slugs[format_result.options["manifest_slug"]] = format_result.options.get(
+                "manifest_slug", "unknown"
+            )
 
 
 def check_module(path: Path, args: argparse.Namespace) -> CheckResult:
@@ -153,9 +147,7 @@ def check_module(path: Path, args: argparse.Namespace) -> CheckResult:
 
 def main():
     parser = argparse.ArgumentParser(description="Check formats")
-    parser.add_argument(
-        "--changes", action="store_true", help="Only check modified formats and modules"
-    )
+    parser.add_argument("--changes", action="store_true", help="Only check modified formats and modules")
     parser.add_argument(
         "--ignore_missing_parsers",
         action="store_true",
@@ -182,9 +174,7 @@ def main():
     intake_formats = None
 
     if args.changes:
-        result = subprocess.run(
-            ["git", "diff", "--name-only", "origin/main"], capture_output=True
-        )
+        result = subprocess.run(["git", "diff", "--name-only", "origin/main"], capture_output=True)
         changed_modules = set()
         changed_formats = set()
         for changed_file in result.stdout.splitlines():
@@ -209,9 +199,7 @@ def main():
 
     in_error = False
 
-    check_module_results = [
-        check_module(INTAKES_PATH / module, args) for module in modules
-    ]
+    check_module_results = [check_module(INTAKES_PATH / module, args) for module in modules]
     check_module_uuids_and_slugs(check_module_results)
 
     print(
@@ -227,9 +215,7 @@ def main():
 
     check_format_results = []
     for check_module_result in check_module_results:
-        check_format_results.extend(
-            check_module_formats(check_module_result, intake_formats, args)
-        )
+        check_format_results.extend(check_module_formats(check_module_result, intake_formats, args))
     check_format_uuids_and_slugs(check_format_results)
 
     print(

diff --git a/utils/checks/validators/__init__.py b/utils/checks/validators/__init__.py
@@ -1,5 +1,3 @@
-from pathlib import Path
-
 from .base import INTAKES_PATH, Validator
 from .changelog import ChangelogValidator
 from .data_sources import ManifestDataSourcesValidator

diff --git a/utils/checks/validators/changelog.py b/utils/checks/validators/changelog.py
@@ -1,5 +1,4 @@
 import argparse
-import os
 from pathlib import Path
 
 from . import Validator

diff --git a/utils/checks/validators/data_sources.py b/utils/checks/validators/data_sources.py
@@ -23,13 +23,10 @@ def validate(cls, result: CheckResult, args: argparse.Namespace) -> None:
             return
 
         unsupported_data_sources = (
-            set(item.lower() for item in format_data_sources.keys())
-            - get_allowed_data_sources()
+            set(item.lower() for item in format_data_sources.keys()) - get_allowed_data_sources()
         )
         unsupported_data_sources_labels = [
-            item
-            for item in format_data_sources
-            if item.lower() in unsupported_data_sources
+            item for item in format_data_sources if item.lower() in unsupported_data_sources
         ]
         if len(unsupported_data_sources_labels) > 0:
             for data_source in unsupported_data_sources_labels:
@@ -39,9 +36,7 @@ def validate(cls, result: CheckResult, args: argparse.Namespace) -> None:
 
 @functools.cache
 def get_allowed_data_sources() -> set[str]:
-    with open(
-        INTAKES_PATH / "utils/checks/validators/data/data_sources.txt", "rt"
-    ) as f:
+    with open(INTAKES_PATH / "utils/checks/validators/data/data_sources.txt", "rt") as f:
         data_sources = set(item.lower() for item in f.read().split("\n"))
 
     return data_sources
diff --git a/utils/checks/validators/format.py b/utils/checks/validators/format.py
@@ -1,5 +1,4 @@
 import argparse
-import os
 from pathlib import Path
 
 from . import (
@@ -16,9 +15,7 @@
 
 
 class FormatValidator:
-    def __init__(
-        self, path: Path, module_result: CheckResult, args: argparse.Namespace
-    ) -> None:
+    def __init__(self, path: Path, module_result: CheckResult, args: argparse.Namespace) -> None:
         self.path = path
 
         format_name = path.name

diff --git a/utils/checks/validators/logo.py b/utils/checks/validators/logo.py
@@ -39,9 +39,7 @@ def has_transparency(img: Image):
         return False
 
     if not image_path.is_file():
-        result.errors.append(
-            f"Logo (`{image_path.relative_to(INTAKES_PATH)}`) is missing"
-        )
+        result.errors.append(f"Logo (`{image_path.relative_to(INTAKES_PATH)}`) is missing")
         return
 
     image = Image.open(image_path)

diff --git a/utils/checks/validators/manifest.py b/utils/checks/validators/manifest.py
@@ -1,5 +1,4 @@
 import argparse
-import os
 import re
 from pathlib import Path
 
@@ -22,18 +21,12 @@ def validate(cls, result: CheckResult, args: argparse.Namespace) -> None:
         module_meta_dir: Path = result.options["meta_dir"]
         module_manifest_file = module_meta_dir / "manifest.yml"
 
-        check_manifest(
-            manifest_file_path=module_manifest_file, result=result, args=args
-        )
+        check_manifest(manifest_file_path=module_manifest_file, result=result, args=args)
 
 
-def check_manifest(
-    manifest_file_path: Path, result: CheckResult, args: argparse.Namespace
-) -> None:
+def check_manifest(manifest_file_path: Path, result: CheckResult, args: argparse.Namespace) -> None:
     if not manifest_file_path.is_file():
-        result.errors.append(
-            f"manifest file (`{manifest_file_path.relative_to(INTAKES_PATH)}`) is missing"
-        )
+        result.errors.append(f"manifest file (`{manifest_file_path.relative_to(INTAKES_PATH)}`) is missing")
         return
 
     # check the format/module has a valid manifest
@@ -78,8 +71,5 @@ def check_manifest(
     if "description" not in manifest_content:
         result.errors.append("no description found in the manifest file")
 
-    elif (
-        not args.ignore_empty_descriptions
-        and len(manifest_content.get("description")) == 0
-    ):
+    elif not args.ignore_empty_descriptions and len(manifest_content.get("description")) == 0:
         result.errors.append("description is found, but empty")
diff --git a/utils/checks/validators/meta.py b/utils/checks/validators/meta.py
@@ -1,5 +1,4 @@
 import argparse
-import os
 from pathlib import Path
 
 from . import INTAKES_PATH, Validator
@@ -17,9 +16,7 @@ def validate(cls, result: CheckResult, args: argparse.Namespace) -> None:
 
         module_meta_dir = path / "_meta"
         if not module_meta_dir.is_dir():
-            result.errors.append(
-                f"Meta directory(`{module_meta_dir.relative_to(INTAKES_PATH)}`) is missing"
-            )
+            result.errors.append(f"Meta directory(`{module_meta_dir.relative_to(INTAKES_PATH)}`) is missing")
             return
 
         result.options["meta_dir"] = module_meta_dir