diff --git a/VERSION b/VERSION index 8df6b88a..f6ed4357 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.5.14 +1.5.15 diff --git a/engines/owl_dns/Dockerfile b/engines/owl_dns/Dockerfile index af936540..60f0450a 100644 --- a/engines/owl_dns/Dockerfile +++ b/engines/owl_dns/Dockerfile @@ -1,5 +1,5 @@ FROM alpine:3.16.3 -LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.5.4" +LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.5.5" # Install dependencies RUN apk add --update --no-cache \ diff --git a/engines/owl_dns/VERSION b/engines/owl_dns/VERSION index 94fe62c2..9075be49 100644 --- a/engines/owl_dns/VERSION +++ b/engines/owl_dns/VERSION @@ -1 +1 @@ -1.5.4 +1.5.5 diff --git a/engines/owl_dns/__init__.py b/engines/owl_dns/__init__.py index 5b295cc2..da4cdc82 100644 --- a/engines/owl_dns/__init__.py +++ b/engines/owl_dns/__init__.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- __title__ = 'patrowl_engine_owl_dns' -__version__ = '1.4.32' +__version__ = '1.5.5' __author__ = 'Nicolas MATTIOCCO' __license__ = 'AGPLv3' -__copyright__ = 'Copyright (C) 2018-2022 Nicolas Mattiocco - @MaKyOtOx' +__copyright__ = 'Copyright (C) 2018-2023 Nicolas Mattiocco - @MaKyOtOx' diff --git a/engines/owl_dns/engine-owl_dns.py b/engines/owl_dns/engine-owl_dns.py index ae348f3a..8232fee2 100644 --- a/engines/owl_dns/engine-owl_dns.py +++ b/engines/owl_dns/engine-owl_dns.py @@ -65,6 +65,14 @@ def _loadconfig(): app.logger.error(f"Error: config file '{conf_file}' not found") return {"status": "error", "reason": "config file not found"} + if not os.path.isfile(this.scanner["seg_path"]): + this.scanner["status"] = "ERROR" + app.logger.error("Error: path to Secure Email Gateway providers not found") + return { + "status": "ERROR", + "reason": "path to Secure Email Gateway providers not found.", + } + if not os.path.isfile(this.scanner["external_ip_ranges_path"]): this.scanner["status"] = "ERROR" app.logger.error( @@ -210,6 +218,12 @@ def start_scan(): th = this.pool.submit(_dns_resolve, scan_id, asset["value"], False) this.scans[scan_id]["futures"].append(th) + if "do_seg_check" in scan["options"].keys() and data["options"]["do_seg_check"]: + for asset in data["assets"]: + if asset["datatype"] in ["domain", "fqdn"]: + th = this.pool.submit(_do_seg_check, scan_id, asset["value"]) + this.scans[scan_id]["futures"].append(th) + if "do_spf_check" in scan["options"].keys() and data["options"]["do_spf_check"]: for asset in data["assets"]: if asset["datatype"] == "domain": @@ -473,7 +487,7 @@ def _reverse_whois(scan_id, asset, datatype): wf_types = ["company", "owner"] elif datatype in ["keyword", "email"]: - print(asset) + # print(asset) if validators.email(asset): wf_types = ["email"] else: @@ -645,6 +659,50 @@ def _saas_check(scan_id: str, asset: str, datatype: str) -> dict: return res +def _do_seg_check(scan_id, asset_value): + seg_dict = [] + dns_records = __dns_resolve_asset(asset_value, "MX") + has_seg = False + + if len(dns_records) == 0: + # seg_dict = {"status": "failed", "reason": f"no MX records found for asset '{asset_value}'"} + return + + with open(this.scanner["seg_path"]) as seg_providers_file: + seg_providers = json.loads(seg_providers_file.read())["seg"] + + for dns_record in dns_records: + for dns_value in dns_record["values"]: + for seg_provider in seg_providers.keys(): + for mx_record in seg_providers[seg_provider]["mx_records"]: + if dns_value.endswith(mx_record): + seg_dict.append({seg_provider: seg_providers[seg_provider]}) + has_seg = True + break + + scan_lock = threading.RLock() + with scan_lock: + if "seg_dict" not in this.scans[scan_id]["findings"].keys(): + this.scans[scan_id]["findings"]["seg_dict"] = {} + this.scans[scan_id]["findings"]["seg_dict_dns_records"] = {} + + if asset_value not in this.scans[scan_id]["findings"].keys(): + this.scans[scan_id]["findings"]["seg_dict"][asset_value] = {} + this.scans[scan_id]["findings"]["seg_dict_dns_records"][asset_value] = {} + + if has_seg is True: + this.scans[scan_id]["findings"]["seg_dict"][asset_value] = copy.deepcopy( + seg_dict + ) + this.scans[scan_id]["findings"]["seg_dict_dns_records"][ + asset_value + ] = copy.deepcopy(dns_records) + else: + this.scans[scan_id]["findings"]["no_seg"] = { + asset_value: "MX records found but no Secure Email Gateway set" + } + + def _recursive_spf_lookups(spf_line): spf_lookups = 0 for word in spf_line.split(" "): @@ -814,7 +872,7 @@ def _get_whois(scan_id, asset): is_domain = __is_domain(asset) is_ip = __is_ip_addr(asset) - print(asset, is_domain, is_ip) + # print(asset, is_domain, is_ip) # Check the asset is a valid domain name or IP Address if not is_domain and not is_ip: @@ -1246,7 +1304,8 @@ def _parse_results(scan_id): issues = [] summary = {} - scan = this.scans[scan_id] + # scan = this.scans[scan_id] + scan = copy.deepcopy(this.scans[scan_id]) nb_vulns = { "info": 0, "low": 0, @@ -1315,6 +1374,52 @@ def _parse_results(scan_id): } ) + if "seg_dict" in scan["findings"].keys(): + for asset in scan["findings"]["seg_dict"].keys(): + seg_check = scan["findings"]["seg_dict"][asset] + if len(seg_check) == 0: + continue + seg_check_dns_records = scan["findings"]["seg_dict_dns_records"][asset] + for seg in seg_check: + seg_provider = list(seg.keys())[0] + seg_title = ( + f"{seg[seg_provider]['provider']}/{seg[seg_provider]['product']}" + ) + issues.append( + { + "issue_id": len(issues) + 1, + "severity": "info", + "confidence": "certain", + "target": {"addr": [asset], "protocol": "domain"}, + "title": f"Secure Email Gateway found: {seg_title}", + "description": f"{seg}\n", + "solution": "n/a", + "metadata": {"tags": ["domains", "seg"]}, + "type": "seg_check", + "raw": {"provider": seg, "mx_records": seg_check_dns_records}, + "timestamp": ts, + } + ) + + if "no_seg" in scan["findings"].keys(): + for asset in scan["findings"]["no_seg"].keys(): + seg_check_failed = scan["findings"]["no_seg"][asset] + issues.append( + { + "issue_id": len(issues) + 1, + "severity": "info", + "confidence": "certain", + "target": {"addr": [asset], "protocol": "domain"}, + "title": "No Secure Email Gateway found", + "description": f"{seg_check_failed}\n", + "solution": "n/a", + "metadata": {"tags": ["domains", "no_seg"]}, + "type": "seg_check", + "raw": seg_check_failed, + "timestamp": ts, + } + ) + if "spf_dict" in scan["findings"].keys(): for asset in scan["findings"]["spf_dict"].keys(): spf_check = scan["findings"]["spf_dict"][asset] @@ -1324,6 +1429,7 @@ def _parse_results(scan_id): ).hexdigest()[:6] spf_check.pop("spf_lookups") title_prefix = spf_check.pop("title_prefix") + for c in spf_check: h = str(c) + str(spf_check_dns_records) spf_hash = hashlib.sha1(h.encode("utf-8")).hexdigest()[:6] diff --git a/engines/owl_dns/etc/seg_list.json b/engines/owl_dns/etc/seg_list.json new file mode 100644 index 00000000..5eef5c8e --- /dev/null +++ b/engines/owl_dns/etc/seg_list.json @@ -0,0 +1,94 @@ +{ + "seg": { + "proofpoint": { + "provider": "ProofPoint", + "product": "Aegis Threat Protection Platform", + "mx_records": [ + ".gslb.pphosted.com.", + "mx1-us1.ppe-hosted.com", + "mx2-us1.ppe-hosted.com", + "mx1-eu1.ppe-hosted.com", + "mx2-eu1.ppe-hosted.com" + ], + "links": [ + "https://help.proofpoint.com/Proofpoint_Essentials/Email_Security/Administrator_Topics/000_gettingstarted/020_connectiondetails" + ] + }, + "microsoft": { + "provider": "Microsoft", + "product": "Microsoft 365 Defender Overview", + "mx_records": [ + ".mail.protection.outlook.com." + ], + "links": [] + }, + "cisco": { + "provider": "Cisco", + "product": "Secure Email", + "mx_records": [ + ".iphmx.com." + ], + "links": [ + "https://docs.ces.cisco.com/docs/hostnames" + ] + }, + "titanhq": { + "provider": "TitanHQ", + "product": "SpamTitan", + "mx_records": [ + ".spamtitan.titanhq.com.", + ".us1-smtp-mx1.titanhq.com." + ], + "links": [] + }, + "forcepoint": { + "provider": "Forcepoint", + "product": "Cloud Email Security", + "mx_records": [ + ".in.mailcontol.com.", + ".out.mailcontol.com ." + ], + "links": [ + "https://support.forcepoint.com/s/article/Where-do-I-need-to-point-my-mx-records-for-Cloud-Email-Security" + ] + }, + "sophos": { + "provider": "Sophos", + "product": "Email Security", + "mx_records": [ + "mx-01-us-west-2.prod.hydra.sophos.com", + "mx-02-us-west-2.prod.hydra.sophos.com", + "mx-01-us-east-2.prod.hydra.sophos.com", + "mx-02-us-east-2.prod.hydra.sophos.com", + "mx-01-eu-central-1.prod.hydra.sophos.com", + "mx-02-eu-central-1.prod.hydra.sophos.com", + "mx-01-eu-west-1.prod.hydra.sophos.com", + "mx-02-eu-west-1.prod.hydra.sophos.com", + "mx-01.eml100yul.ctr.sophos.com", + "mx-02.eml100yul.ctr.sophos.com", + "mx-01.eml100syd.ctr.sophos.com", + "mx-02.eml100syd.ctr.sophos.com", + "mx-01.eml100hnd.ctr.sophos.com", + "mx-02.eml100hnd.ctr.sophos.com", + "mx-01.eml100bom.ctr.sophos.com", + "mx-02.eml100bom.ctr.sophos.com", + "mx-01.eml100gru.ctr.sophos.com", + "mx-02.eml100gru.ctr.sophos.com" + ], + "links": [ + "https://support.forcepoint.com/s/article/Where-do-I-need-to-point-my-mx-records-for-Cloud-Email-Security" + ] + }, + "trendmicro": { + "provider": "TrendMicro", + "product": "Hosted Email Security (HES)", + "mx_records": [ + "in.hes.trendmicro.eu", + "in.hes.trendmicro.com" + ], + "links": [ + "https://success.trendmicro.com/dcx/s/solution/1055888-redirecting-mail-exchange-mx-records-to-hosted-email-security-hes?language=en_US&sfdcIFrameOrigin=null" + ] + } + } +} \ No newline at end of file diff --git a/engines/owl_dns/owl_dns.json.sample b/engines/owl_dns/owl_dns.json.sample index 5298a90b..f2eebd8e 100644 --- a/engines/owl_dns/owl_dns.json.sample +++ b/engines/owl_dns/owl_dns.json.sample @@ -7,6 +7,7 @@ "dnstwist_bin_path": "/opt/patrowl-engines/owl_dns/external-libs/dnstwist", "dnstwist_common_tlds": "/opt/patrowl-engines/owl_dns/external-libs/dnstwist/dictionaries/common_tlds.dict", "external_ip_ranges_path": "/opt/patrowl-engines/owl_dns/etc/ip-ranges.json", + "seg_path": "/opt/patrowl-engines/owl_dns/etc/seg_list.json", "whoisfreak_api_tokens": ["xx", "yy"], "names_path": "/opt/patrowl-engines/owl_dns/etc/names.txt", "options": {