Merge branch 'refs/heads/dev'

CHANGELOG/3.6.0/238.md

@@ -0,0 +1 @@
+[FIX] Fixed connection issues. #238

CHANGELOG/3.6.0/242.md

@@ -0,0 +1 @@
+[MOD] Introduced an option to retrieve all completed scans from Tenable SC. #242

CHANGELOG/3.6.0/243.md

@@ -0,0 +1 @@
+[FIX] Added more options to the Cisco Cyber Vision executor. #243

CHANGELOG/3.6.0/date.md

@@ -0,0 +1 @@
+Jan 6th, 2025

RELEASE.md

@@ -1,3 +1,9 @@
+3.6.0 [Jan 6th, 2025]:
+---
+ * [MOD] Introduced an option to retrieve all completed scans from Tenable SC. #242
+ * [FIX] Fixed connection issues. #238
+ * [FIX] Added more options to the Cisco Cyber Vision executor. #243
+
 3.5.3 [Oct 24th, 2024]:
 ---
  * [MOD] Changed type of TARGETS and PORT_LIST to list in Nmap agent. #231

faraday_agent_dispatcher/__init__.py

@@ -20,4 +20,4 @@
 
 __author__ = """Faraday Development Team"""
 __email__ = "[email protected]"
-__version__ = "3.5.3"
+__version__ = "3.6.0"

faraday_agent_dispatcher/dispatcher.py

@@ -565,7 +565,7 @@ async def check_connection(self):
         server_url = api_url(
             self.host,
             self.api_port,
-            postfix="/_api/v3/info",
+            postfix="/_api/config",
             secure=self.api_ssl_enabled,
         )
         logger.debug(f"Validating server connection with {server_url}")

faraday_agent_dispatcher/dispatcher_io.py

@@ -568,7 +568,7 @@ async def check_connection(self):
         server_url = api_url(
             self.host,
             self.api_port,
-            postfix="/_api/v3/info",
+            postfix="/_api/config",
             secure=self.api_ssl_enabled,
         )
         logger.debug(f"Validating server connection with {server_url}")

faraday_agent_dispatcher/static/executors/official/cisco_cybervision.py

@@ -2,17 +2,17 @@
 import os
 import sys
 import json
-import time
 import datetime
-import requests
+
 from urllib3.exceptions import InsecureRequestWarning
-from faraday_agent_dispatcher.utils.severity_utils import severity_from_score
 
-API_BASE = "/api/3.0"
+import requests
 
+from faraday_agent_dispatcher.utils.severity_utils import severity_from_score
+from faraday_agent_dispatcher.utils.logging import log
 
-def log(msg, end="\n"):
-    print(msg, file=sys.stderr, flush=True, end=end)
+API_BASE = "/api/3.0"
+MY_PRESET_LABEL = "My preset"
 
 
 def parse_date(date_str):
@@ -22,73 +22,109 @@ def parse_date(date_str):
         return ""
 
 
-def cybervision_report_composer(url, token, preset_list, asset_tags, vuln_tags):
+def cybervision_report_composer(
+    url,
+    token,
+    preset_list,
+    asset_tags,
+    vuln_tags,
+    presets_containing=None,
+    only_my_presets=True,
+    only_preset_refresh=False,
+):
     req_headers = {"accept": "application/json", "x-token-id": token}
     requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
     presets_queue = []
     presets_id = {}
 
+    log(f"My Presets: {only_my_presets}")
+    log(f"Presets list: {preset_list}")
+    log(f"Refresh Presets: {only_preset_refresh}")
+    log(f"Presets Containing: {presets_containing}")
+
     # STAGE 1 - get preset list
     req_url = f"{url}{API_BASE}/presets"
     try:
+        log("Getting presets...")
         resp = requests.get(req_url, headers=req_headers, timeout=20, verify=False).json()
     except TimeoutError:
         log("Can't reach Cyber Vision: connection timed out")
         sys.exit(1)
     if "error" in resp:
         log(f"API Error: {resp['error']}")
         sys.exit(1)
-    for req_preset in preset_list:
+    if not preset_list:
+        log("No specific presets selected.")
         for preset in resp:
-            if preset["label"] == req_preset:
-                presets_id[preset["label"]] = preset["id"]
-                presets_queue.append(preset["id"])
+            preset_label = preset.get("label")
+            preset_id = preset.get("id")
+            if only_my_presets:
+                category = preset.get("category")
+                if not category:
+                    log(f"No category found for preset {preset_label}")
+                    continue
+                category_label = category.get("label")
+                if not category_label:
+                    log(f"No category found for preset {preset_label}")
+                    continue
+                if category_label == MY_PRESET_LABEL:
+                    if presets_containing:
+                        if presets_containing in preset_label:
+                            log(f"Preset {preset_label} contains {presets_containing}")
+                            presets_id[preset_label] = preset_id
+                            presets_queue.append(preset_id)
+                    else:
+                        presets_id[preset_label] = preset_id
+                        presets_queue.append(preset_id)
+            else:
+                if presets_containing:
+                    if presets_containing in preset_label:
+                        log(f"Preset {preset_label} contains {presets_containing}")
+                        presets_id[preset_label] = preset_id
+                        presets_queue.append(preset_id)
+                else:
+                    presets_id[preset_label] = preset_id
+                    presets_queue.append(preset_id)
+    else:
+        log("Presets list selected")
+        for req_preset in preset_list:
+            for preset in resp:
+                preset_label = preset.get("label")
+                preset_id = preset.get("id")
+                if preset_label == req_preset:
+                    presets_id[preset_label] = preset_id
+                    presets_queue.append(preset_id)
+
+    log(f"Added {len(presets_queue)} presets to process.")
     presets_id_inv = {v: k for k, v in presets_id.items()}
 
-    # STAGE 2 - get all vulns per preset
-    presets_vuln_collection = {}
-    step_c = 1
-    step_s = 100
-
-    for _id in presets_queue:  # post to update to latest computed data
-        req_refresh_url = f"{url}{API_BASE}/presets/{_id}/refreshData"
-        try:
-            resp = requests.post(req_refresh_url, headers=req_headers, timeout=20, verify=False)
-        except TimeoutError:
-            log("Can't reach Cyber Vision: connection timed out")
-            sys.exit(1)
-
-    for _id in presets_queue:
-        serv_c = 0
-        while True:  # wait until data is ready
-            req_test_url = f"{url}{API_BASE}/presets/{_id}/visualisations/vulnerability-list?page=1&size=5"
+    # Refresh STAGE. If set it will exit after finishing the refresh.
+    if only_preset_refresh:
+        log("Refreshing presets...")
+        for _id in presets_queue:  # post to update to latest computed data
+            req_refresh_url = f"{url}{API_BASE}/presets/{_id}/refreshData"
             try:
-                resp = requests.get(req_test_url, headers=req_headers, timeout=20, verify=False)
-                if "Service unavailable" in resp.content.decode("UTF-8"):
-                    if serv_c == 0:
-                        log(f"Preset {presets_id_inv[_id]} data is not ready, waiting...")
-                    serv_c += 1
-                else:
-                    log(f"Preset {presets_id_inv[_id]} data is ready!")
-                    serv_c = 0
-                    break
-                if serv_c >= 60:
-                    break
+                log(f"Refreshing preset {_id}")
+                resp = requests.post(req_refresh_url, headers=req_headers, timeout=20, verify=False)
             except TimeoutError:
                 log("Can't reach Cyber Vision: connection timed out")
                 sys.exit(1)
-            time.sleep(1)
-
-        if serv_c >= 60:
-            log(f"Error: Preset {presets_id_inv[_id]} took many time to refresh data")
-            continue
+        sys.exit(1)
 
+    # STAGE 2 - get all vulns per preset
+    presets_vuln_collection = {}
+    for _id in presets_queue:
+        log(f"Trying to get data for preset {presets_id_inv[_id]}...")
         step_c = 1
+        step_s = 100
         presets_vuln_collection[_id] = []
         while True:  # paged data fetch
             req_url = f"{url}{API_BASE}/presets/{_id}/visualisations/vulnerability-list?page={step_c}&size={step_s}"
             try:
                 resp = requests.get(req_url, headers=req_headers, timeout=20, verify=False)
+                if "Service unavailable" in resp.content.decode("UTF-8"):
+                    log(f"Preset {presets_id_inv[_id]} data is not ready, skipping...")
+                    break
                 resp = resp.json()
             except TimeoutError:
                 log("Can't reach Cyber Vision: connection timed out")
@@ -112,48 +148,63 @@ def cybervision_report_composer(url, token, preset_list, asset_tags, vuln_tags):
             continue
         for vuln_pack in presets_vuln_collection[pres_data[1]]:
             for vuln in vuln_pack:
-                if not vuln["device"]["label"] in hosts:
-                    hosts[vuln["device"]["label"]] = {
-                        "ip": vuln["device"]["label"],
-                        "description": f"Device ID {vuln['device']['id']}",
-                        "mac": "" if not vuln["device"]["mac"] else vuln["device"]["mac"],
+                log(vuln)
+                title = vuln.get("title", "")  # TODO: Que pasa si es vacío el titulo?
+                device = vuln.get("device")
+                if not device:
+                    log(f"Error: No device for {vuln} ({title})")
+                    continue
+                device_label = device.get("label")
+                device_id = device.get("id")
+                if device_label not in hosts:
+                    hosts[device_label] = {
+                        "ip": device_label,
+                        "description": f"Device ID {device_id}",
+                        "mac": device.get("mac", ""),
                         "vulnerabilities": [],
                         "tags": [pres_data[0]] + asset_tags,
                     }
-                if not vuln["title"] in [x["name"] for x in hosts[vuln["device"]["label"]]["vulnerabilities"]]:
+                if title not in [x["name"] for x in hosts[device_label]["vulnerabilities"]]:
+
                     try:
-                        vuln["cvss"] = float(vuln["cvss"])
+                        cvss_base_score = float(vuln.get("CVSS", -1.0))
                     except ValueError:
-                        vuln["cvss"] = -1.0
-                    hosts[vuln["device"]["label"]]["vulnerabilities"].append(
+                        cvss_base_score = -1.0
+
+                    hosts[device_label]["vulnerabilities"].append(
                         {
-                            "name": vuln["title"],
-                            "desc": vuln["summary"],
-                            "severity": severity_from_score(vuln["cvss"], 10.0),
-                            "refs": [{"name": ref["link"], "type": "other"} for ref in vuln["links"]],
-                            "external_id": vuln["id"],
+                            "name": title,
+                            "desc": vuln.get("summary"),
+                            "severity": severity_from_score(cvss_base_score, 10.0),
+                            "refs": [{"name": ref.get("link", ""), "type": "other"} for ref in vuln.get("links", [])],
+                            "external_id": "",
                             "type": "Vulnerability",
-                            "resolution": vuln["solution"],
-                            "data": vuln["fullDescription"],
+                            "resolution": vuln.get("solution"),
+                            "data": vuln.get("fullDescription"),
                             "status": "open",
-                            "cve": [x["cve"] for i, x in enumerate(vuln_pack) if x["title"] == vuln["title"]],
-                            "run_date": parse_date(vuln["publishTime"]),
+                            "cve": [x.get("cve") for i, x in enumerate(vuln_pack) if x.get("title") == title],
+                            "run_date": parse_date(vuln.get("publishTime")),
                             "tags": vuln_tags,
                             "cwe": [],
                         }
                     )
+
     data = {"hosts": [x[1] for x in hosts.items()]}
     print(json.dumps(data))
 
 
 def main():
     params_cybervision_token = os.getenv("CYBERVISION_TOKEN")
     params_cybervision_url = os.getenv("CYBERVISION_HTTPS_URL")
-    params_cybervision_presets_env = os.getenv("EXECUTOR_CONFIG_CYBERVISION_PRESETS")
 
-    params_cybervision_presets = []
-    if params_cybervision_presets_env:
-        params_cybervision_presets = json.loads(params_cybervision_presets_env)
+    fetch_specific_presets = os.getenv("EXECUTOR_CONFIG_SPECIFIC_PRESETS")
+    fetch_presets_containing = os.getenv("EXECUTOR_CONFIG_PRESETS_CONTAINING", None)
+    fetch_my_presets = bool(os.getenv("EXECUTOR_CONFIG_MY_PRESETS", False))
+    only_refresh_presets = bool(os.getenv("EXECUTOR_CONFIG_REFRESH_PRESETS", False))
+
+    params_fetch_specific_presets_list = []
+    if fetch_specific_presets:
+        params_fetch_specific_presets_list = json.loads(fetch_specific_presets)
 
     if not params_cybervision_url.startswith("https://"):
         log("Cyber Vision URL must be HTTPS")
@@ -169,9 +220,12 @@ def main():
     cybervision_report_composer(
         params_cybervision_url,
         params_cybervision_token,
-        params_cybervision_presets,
+        params_fetch_specific_presets_list,
         params_asset_tags,
         params_vulnerability_tags,
+        only_my_presets=fetch_my_presets,
+        presets_containing=fetch_presets_containing,
+        only_preset_refresh=only_refresh_presets,
     )
 
 

faraday_agent_dispatcher/static/executors/official/tenablesc.py

@@ -12,18 +12,20 @@ def log(msg):
     print(msg, file=sys.stderr)
 
 
-def get_only_usable_ids(tsc, scan_ids):
+def get_only_usable_ids(tsc, scan_ids, fetch_all_scans=False):
     tenable_scans = tsc.scan_instances.list()
     usable_tenable_scans = [str(scan["id"]) for scan in tenable_scans["usable"] if scan["status"] == "Completed"]
     log("*" * 10)
     log("Listing available scans ...")
     log(usable_tenable_scans)
     log("*" * 10)
+    if fetch_all_scans:
+        return usable_tenable_scans
     return [_id for _id in scan_ids if str(_id) in usable_tenable_scans]
 
 
 def process_scan(
-    tsc, scan_id, ignore_info=False, hostname_resolution=False, host_tag=False, service_tag=False, vuln_tag=False
+    tsc, scan_id, ignore_info=False, hostname_resolution=False, host_tag=None, service_tag=None, vuln_tag=None
 ):
     log(f"Processing scan id {scan_id}")
     try:
@@ -42,7 +44,6 @@ def process_scan(
             )
             plugin.parseOutputString(file.read())
             return plugin.get_json()
-    return {}
 
 
 def main():
@@ -58,7 +59,8 @@ def main():
     if host_tag:
         host_tag = host_tag.split(",")
 
-    tenable_scan_ids = os.getenv("EXECUTOR_CONFIG_TENABLE_SCAN_ID")
+    tenable_scan_ids = os.getenv("EXECUTOR_CONFIG_TENABLE_SCAN_ID", "[]")
+    tenable_fetch_all_completed_scans = bool(os.getenv("EXECUTOR_CONFIG_COMPLETED_SCANS", False))
     TENABLE_ACCESS_KEY = os.getenv("TENABLE_ACCESS_KEY")
     TENABLE_SECRET_KEY = os.getenv("TENABLE_SECRET_KEY")
     TENABLE_HOST = os.getenv("TENABLE_HOST")
@@ -71,19 +73,20 @@ def main():
         log("TenableSC Host not provided")
         exit(1)
 
-    if not tenable_scan_ids:
+    if not tenable_fetch_all_completed_scans and not tenable_scan_ids:
         log("TenableSC Scan ID not provided")
         exit(1)
 
-    # it should be a list but it is save as a str in the environment
     try:
         tenable_scan_ids_list = json.loads(tenable_scan_ids)
     except Exception as e:
         log(f"TenableSC Scan IDs could not be parsed {e}")
         exit(1)
 
     tsc = TenableSC(host=TENABLE_HOST, access_key=TENABLE_ACCESS_KEY, secret_key=TENABLE_SECRET_KEY)
-    usable_scan_ids = get_only_usable_ids(tsc, tenable_scan_ids_list)
+    usable_scan_ids = get_only_usable_ids(
+        tsc, tenable_scan_ids_list, fetch_all_scans=tenable_fetch_all_completed_scans
+    )
 
     if not usable_scan_ids:
         log("*" * 10)

faraday_agent_dispatcher/utils/logging.py

@@ -0,0 +1,5 @@
+import sys
+
+
+def log(msg, end="\n"):
+    print(msg, file=sys.stderr, flush=True, end=end)

setup.py

@@ -38,10 +38,10 @@
     "syslog_rfc5424_formatter",
     "requests",
     "itsdangerous",
-    "faraday-plugins>=1.19.0",
+    "faraday-plugins>=1.21.0",
     "python-owasp-zap-v2.4",
     "python-gvm",
-    "faraday_agent_parameters_types>=1.7.0",
+    "faraday_agent_parameters_types>=1.7.3",
     "pyyaml",
     "psutil",
     "pytenable",