From 936f08f77fe2fde6f2166cf429dd3c50c2a5f045 Mon Sep 17 00:00:00 2001 From: Jeremy Chapeau Date: Tue, 13 Jun 2023 18:34:34 +0200 Subject: [PATCH 1/8] add gandi intel --- cartography/cli.py | 15 +++ cartography/config.py | 4 + cartography/intel/gandi/__init__.py | 47 +++++++++ cartography/intel/gandi/organization.py | 42 ++++++++ cartography/intel/gandi/utils.py | 47 +++++++++ cartography/intel/gandi/zones.py | 117 +++++++++++++++++++++++ cartography/models/gandi/__init__.py | 0 cartography/models/gandi/dnsrecord.py | 72 ++++++++++++++ cartography/models/gandi/dnszone.py | 42 ++++++++ cartography/models/gandi/ip.py | 18 ++++ cartography/models/gandi/organization.py | 27 ++++++ cartography/sync.py | 2 + tests/data/gandi/__init__.py | 0 tests/data/gandi/organization.py | 15 +++ tests/data/gandi/zones.py | 91 ++++++++++++++++++ 15 files changed, 539 insertions(+) create mode 100644 cartography/intel/gandi/__init__.py create mode 100644 cartography/intel/gandi/organization.py create mode 100644 cartography/intel/gandi/utils.py create mode 100644 cartography/intel/gandi/zones.py create mode 100644 cartography/models/gandi/__init__.py create mode 100644 cartography/models/gandi/dnsrecord.py create mode 100644 cartography/models/gandi/dnszone.py create mode 100644 cartography/models/gandi/ip.py create mode 100644 cartography/models/gandi/organization.py create mode 100644 tests/data/gandi/__init__.py create mode 100644 tests/data/gandi/organization.py create mode 100644 tests/data/gandi/zones.py diff --git a/cartography/cli.py b/cartography/cli.py index 4ae159e40d..54525bc770 100644 --- a/cartography/cli.py +++ b/cartography/cli.py @@ -500,6 +500,14 @@ def _build_parser(self): 'The Duo api hostname' ), ) + parser.add_argument( + '--gandi-apikey-env-var', + type=str, + default=None, + help=( + 'The name of environment variable containing the gandi API key for authentication.' + ), + ) return parser def main(self, argv: str) -> int: @@ -666,6 +674,13 @@ def main(self, argv: str) -> int: config.duo_api_key = os.environ.get(config.duo_api_key_env_var) config.duo_api_secret = os.environ.get(config.duo_api_secret_env_var) + # Gandi config + if config.gandi_apikey_env_var: + logger.debug(f"Reading API key for Gandi from environment variable {config.gandi_apikey_env_var}") + config.gandi_apikey = os.environ.get(config.gandi_apikey_env_var) + else: + config.gandi_apikey = None + # Run cartography try: return cartography.sync.run_with_config(self.sync, config) diff --git a/cartography/config.py b/cartography/config.py index 7485b16058..54d53389b3 100644 --- a/cartography/config.py +++ b/cartography/config.py @@ -103,6 +103,8 @@ class Config: :param duo_api_key: The Duo api secret. Optional. :type duo_api_hostname: str :param duo_api_hostname: The Duo api hostname, e.g. "api-abc123.duosecurity.com". Optional. + :type: gandi_apikey: str + :param gandi_apikey: API authentication key for Gandi. Optional. """ def __init__( @@ -157,6 +159,7 @@ def __init__( duo_api_key=None, duo_api_secret=None, duo_api_hostname=None, + gandi_apikey=None, ): self.neo4j_uri = neo4j_uri self.neo4j_user = neo4j_user @@ -208,3 +211,4 @@ def __init__( self.duo_api_key = duo_api_key self.duo_api_secret = duo_api_secret self.duo_api_hostname = duo_api_hostname + self.gandi_apikey = gandi_apikey diff --git a/cartography/intel/gandi/__init__.py b/cartography/intel/gandi/__init__.py new file mode 100644 index 0000000000..c6eb37b654 --- /dev/null +++ b/cartography/intel/gandi/__init__.py @@ -0,0 +1,47 @@ +import logging + +import neo4j + +import cartography.intel.gandi.organization +import cartography.intel.gandi.zones +from cartography.config import Config +from cartography.intel.gandi.utils import GandiAPI +from cartography.util import timeit + +logger = logging.getLogger(__name__) + + +@timeit +def start_gandi_ingestion(neo4j_session: neo4j.Session, config: Config) -> None: + """ + If this module is configured, perform ingestion of Gandi data. Otherwise warn and exit + :param neo4j_session: Neo4J session for database interface + :param config: A cartography.config object + :return: None + """ + + if not config.gandi_apikey: + logger.info( + 'Gandi import is not configured - skipping this module. ' + 'See docs to configure.', + ) + return + + common_job_parameters = { + "UPDATE_TAG": config.update_tag, + } + + api_session = GandiAPI(config.gandi_apikey) + + cartography.intel.gandi.organization.sync( + neo4j_session, + api_session, + config.update_tag, + common_job_parameters, + ) + cartography.intel.gandi.zones.sync( + neo4j_session, + api_session, + config.update_tag, + common_job_parameters, + ) diff --git a/cartography/intel/gandi/organization.py b/cartography/intel/gandi/organization.py new file mode 100644 index 0000000000..bf5706c7ce --- /dev/null +++ b/cartography/intel/gandi/organization.py @@ -0,0 +1,42 @@ +import logging +from typing import Any +from typing import Dict +from typing import List + +import neo4j + +from cartography.client.core.tx import load +from cartography.intel.gandi.utils import GandiAPI +from cartography.models.gandi.organization import GandiOrganizationSchema +from cartography.util import timeit + +logger = logging.getLogger(__name__) + + +@timeit +def sync( + neo4j_session: neo4j.Session, + gandi_session: GandiAPI, + update_tag: int, + common_job_parameters: Dict[str, Any], +) -> None: + orgs = get(gandi_session) + load_organizations(neo4j_session, orgs, update_tag) + + +@timeit +def get(gandi_session: GandiAPI) -> List[Dict[str, Any]]: + return gandi_session.get_organizations() + + +def load_organizations( + neo4j_session: neo4j.Session, + data: List[Dict[str, Any]], + update_tag: int, +) -> None: + load( + neo4j_session, + GandiOrganizationSchema(), + data, + lastupdated=update_tag, + ) diff --git a/cartography/intel/gandi/utils.py b/cartography/intel/gandi/utils.py new file mode 100644 index 0000000000..9914df8f56 --- /dev/null +++ b/cartography/intel/gandi/utils.py @@ -0,0 +1,47 @@ +from typing import Any +from typing import Dict +from typing import List + +import requests + + +class GandiAPI: + TIMEOUT = (600, 600) + + def __init__(self, apikey: str) -> None: + self._base_url: str = 'https://api.gandi.net/v5/' + self._session = requests.Session() + self._session.headers.update( + { + 'Authorization': f'Apikey {apikey}', + 'User-Agent': 'Cartography/0.1', + }, + ) + + def get_organizations(self) -> List[Dict[str, Any]]: + result = [] + req = self._session.get(f'{self._base_url}/organization/organizations', timeout=self.TIMEOUT) + req.raise_for_status() + for org in req.json(): + if org['type'] == 'individual': + continue + result.append(org) + return result + + def get_domains(self) -> List[Dict[str, Any]]: + result = [] + # List domains + list_req = self._session.get(f'{self._base_url}/domain/domains', timeout=self.TIMEOUT) + list_req.raise_for_status() + for dom in list_req.json(): + # Get domains informations + req = self._session.get(dom['href']) + req.raise_for_status() + domain = req.json().copy() + # Get records + if 'dnssec' in domain['services']: + req = self._session.get(f'{self._base_url}/livedns/domains/{dom["fqdn"]}/records', timeout=self.TIMEOUT) + req.raise_for_status() + domain['records'] = req.json() + result.append(domain) + return result diff --git a/cartography/intel/gandi/zones.py b/cartography/intel/gandi/zones.py new file mode 100644 index 0000000000..e1cec52b53 --- /dev/null +++ b/cartography/intel/gandi/zones.py @@ -0,0 +1,117 @@ +import logging +from typing import Any +from typing import Dict +from typing import List +from typing import Tuple + +import neo4j +from dateutil import parser as dt_parse + +from cartography.client.core.tx import load +from cartography.intel.gandi.utils import GandiAPI +from cartography.models.gandi.dnsrecord import GandiDNSRecordSchema +from cartography.models.gandi.dnszone import GandiDNSZoneSchema +from cartography.models.gandi.ip import IPSchema +from cartography.util import timeit + +logger = logging.getLogger(__name__) + + +@timeit +def sync( + neo4j_session: neo4j.Session, + gandi_session: GandiAPI, + update_tag: int, + common_job_parameters: Dict[str, Any], +) -> None: + domains = get(gandi_session) + zones, records, ips = transform(domains) + load_zones(neo4j_session, zones, records, ips, update_tag) + # TODO: Cleanup zones & domains (when linked to organization) + + +@timeit +def get(gandi_session: GandiAPI) -> List[Dict[str, Any]]: + return gandi_session.get_domains() + + +@timeit +def transform(domains: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], List[Dict[str, Any]]]: + zones: List[Dict[str, Any]] = [] + records: List[Dict[str, Any]] = [] + ips = set() + + for dom in domains: + # Transform dates + for k in list(dom['dates'].keys()): + dom['dates'][k] = int(dt_parse.parse(dom['dates'][k]).timestamp() * 1000) if dom['dates'][k] else None + # Create record from nameservers + for ns in dom['nameservers']: + records.append({ + "id": f"{ns}+NS", + "rsset_name": "@", + "rsset_type": "NS", + "rsset_value": ns, + "registered_domain": dom['fqdn'], + }) + # Extract records + for rec in dom['records']: + rec['rrset_value'] = ','.join(rec['rrset_values']) + if rec['rrset_name'] == '@': + rec['id'] = rec['rrset_values'][0] + else: + rec['id'] = f"{rec['rrset_name']}.{dom['fqdn']}+{rec['rrset_type']}" + rec['registered_domain'] = dom['fqdn'] + # Split on IPs + if rec['rrset_type'] in ['A', 'AAAA']: + if len(rec['rrset_values']) >= 1: + for ip in rec['rrset_values']: + splited_record = rec.copy() + splited_record['resolved_ip'] = ip + ips.add(ip) + records.append(splited_record) + else: + records.append(rec) + else: + records.append(rec) + zones.append(dom) + # Format IPs + formated_ips: List[Dict[str, Any]] = [] + for ip in ips: + formated_ips.append({ + "id": ip, + "ip": ip, + }) + return zones, records, formated_ips + + +def load_zones( + neo4j_session: neo4j.Session, + zones: List[Dict[str, Any]], + records: List[Dict[str, Any]], + ips: List[Dict[str, Any]], + update_tag: int, +) -> None: + # Save zones + load( + neo4j_session, + GandiDNSZoneSchema(), + zones, + lastupdated=update_tag, + ) + + # Save Ips + load( + neo4j_session, + IPSchema(), + ips, + lastupdated=update_tag, + ) + + # Save records + load( + neo4j_session, + GandiDNSRecordSchema(), + records, + lastupdated=update_tag, + ) diff --git a/cartography/models/gandi/__init__.py b/cartography/models/gandi/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cartography/models/gandi/dnsrecord.py b/cartography/models/gandi/dnsrecord.py new file mode 100644 index 0000000000..c2c351b45c --- /dev/null +++ b/cartography/models/gandi/dnsrecord.py @@ -0,0 +1,72 @@ +from dataclasses import dataclass +from typing import Optional + +from cartography.models.core.common import PropertyRef +from cartography.models.core.nodes import CartographyNodeProperties +from cartography.models.core.nodes import CartographyNodeSchema +from cartography.models.core.nodes import ExtraNodeLabels +from cartography.models.core.relationships import CartographyRelProperties +from cartography.models.core.relationships import CartographyRelSchema +from cartography.models.core.relationships import LinkDirection +from cartography.models.core.relationships import make_target_node_matcher +from cartography.models.core.relationships import OtherRelationships +from cartography.models.core.relationships import TargetNodeMatcher + + +@dataclass(frozen=True) +class GandiDNSRecordProperties(CartographyNodeProperties): + id: PropertyRef = PropertyRef('id') + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + name: PropertyRef = PropertyRef('rrset_name', extra_index=True) + type: PropertyRef = PropertyRef('rrset_type') + ttl: PropertyRef = PropertyRef('rrset_ttl') + value: PropertyRef = PropertyRef('rrset_value') + registered_domain: PropertyRef = PropertyRef('registered_domain') + + +@dataclass(frozen=True) +class ZoneToRecordRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +# (:GandiDNSZone)<-[:MEMBER_OF_DNS_ZONE]-(:GandiDNSRecord) +class ZoneToRecordRel(CartographyRelSchema): + target_node_label: str = 'GandiDNSZone' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'name': PropertyRef('registered_domain')}, + ) + direction: LinkDirection = LinkDirection.OUTWARD + rel_label: str = "MEMBER_OF_DNS_ZONE" + properties: ZoneToRecordRelProperties = ZoneToRecordRelProperties() + + +@dataclass(frozen=True) +class RecordToIpRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +# (:Ip)<-[:DNS_POINTS_TO]-(:GandiDNSRecord) +class RecordToIpRel(CartographyRelSchema): + target_node_label: str = 'Ip' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('resolved_ip')}, + ) + direction: LinkDirection = LinkDirection.OUTWARD + rel_label: str = "DNS_POINTS_TO" + properties: RecordToIpRelProperties = RecordToIpRelProperties() + + +@dataclass(frozen=True) +class GandiDNSRecordSchema(CartographyNodeSchema): + label: str = 'GandiDNSRecord' + extra_node_labels: Optional[ExtraNodeLabels] = ExtraNodeLabels(['DNSRecord']) + properties: GandiDNSRecordProperties = GandiDNSRecordProperties() + other_relationships: OtherRelationships = OtherRelationships( + [ + ZoneToRecordRel(), + RecordToIpRel(), + ], + ) + # TODO: Link to organization: sub_resource_relationship: XXXRel = XXXRel() diff --git a/cartography/models/gandi/dnszone.py b/cartography/models/gandi/dnszone.py new file mode 100644 index 0000000000..706ea9c6de --- /dev/null +++ b/cartography/models/gandi/dnszone.py @@ -0,0 +1,42 @@ +from dataclasses import dataclass +from typing import Optional + +from cartography.models.core.common import PropertyRef +from cartography.models.core.nodes import CartographyNodeProperties +from cartography.models.core.nodes import CartographyNodeSchema +from cartography.models.core.nodes import ExtraNodeLabels + + +@dataclass(frozen=True) +class GandiDNSZoneNodeProperties(CartographyNodeProperties): + id: PropertyRef = PropertyRef('id') + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + name: PropertyRef = PropertyRef('fqdn', extra_index=True) + tld: PropertyRef = PropertyRef('tld') + # Dates + created_at: PropertyRef = PropertyRef('dates.created_at') + deletes_at: PropertyRef = PropertyRef('dates.deletes_at') + hold_begins_at: PropertyRef = PropertyRef('dates.hold_begins_at') + hold_ends_at: PropertyRef = PropertyRef('dates.hold_ends_at') + pending_delete_ends_at: PropertyRef = PropertyRef('dates.pending_delete_ends_at') + registry_created_at: PropertyRef = PropertyRef('dates.registry_created_at') + registry_ends_at: PropertyRef = PropertyRef('dates.registry_ends_at') + renew_begins_at: PropertyRef = PropertyRef('dates.renew_begins_at') + restore_ends_at: PropertyRef = PropertyRef('dates.restore_ends_at') + updated_at: PropertyRef = PropertyRef('dates.updated_at') + authinfo_expires_at: PropertyRef = PropertyRef('dates.authinfo_expires_at') + # Informations + status: PropertyRef = PropertyRef('status') + services: PropertyRef = PropertyRef('services') + # Autorenew + autorenew_duration: PropertyRef = PropertyRef('autorenew.duration') + autorenew_enabled: PropertyRef = PropertyRef('autorenew.enabled') + + +@dataclass(frozen=True) +class GandiDNSZoneSchema(CartographyNodeSchema): + label: str = 'GandiDNSZone' + extra_node_labels: Optional[ExtraNodeLabels] = ExtraNodeLabels(['DNSZone']) + properties: GandiDNSZoneNodeProperties = GandiDNSZoneNodeProperties() + # TODO: Link to organization: sub_resource_relationship: XXXRel = XXXRel() + # Use sharing_space.id or sharing_space.name to link to organization diff --git a/cartography/models/gandi/ip.py b/cartography/models/gandi/ip.py new file mode 100644 index 0000000000..60a1910cee --- /dev/null +++ b/cartography/models/gandi/ip.py @@ -0,0 +1,18 @@ +from dataclasses import dataclass + +from cartography.models.core.common import PropertyRef +from cartography.models.core.nodes import CartographyNodeProperties +from cartography.models.core.nodes import CartographyNodeSchema + + +@dataclass(frozen=True) +class IPNodeProperties(CartographyNodeProperties): + id: PropertyRef = PropertyRef('id') + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + ip: PropertyRef = PropertyRef('ip', extra_index=True) + + +@dataclass(frozen=True) +class IPSchema(CartographyNodeSchema): + label: str = 'Ip' + properties: IPNodeProperties = IPNodeProperties() diff --git a/cartography/models/gandi/organization.py b/cartography/models/gandi/organization.py new file mode 100644 index 0000000000..4c1adb0f22 --- /dev/null +++ b/cartography/models/gandi/organization.py @@ -0,0 +1,27 @@ +from dataclasses import dataclass + +from cartography.models.core.common import PropertyRef +from cartography.models.core.nodes import CartographyNodeProperties +from cartography.models.core.nodes import CartographyNodeSchema + + +@dataclass(frozen=True) +class GandiOrganizationNodeProperties(CartographyNodeProperties): + id: PropertyRef = PropertyRef('id') + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + name: PropertyRef = PropertyRef('name') + orgname: PropertyRef = PropertyRef('orgname') + firstname: PropertyRef = PropertyRef('firstname') + lastname: PropertyRef = PropertyRef('lastname') + type: PropertyRef = PropertyRef('type') + email: PropertyRef = PropertyRef('email') + reseller: PropertyRef = PropertyRef('reseller') + corporate: PropertyRef = PropertyRef('corporate') + siren: PropertyRef = PropertyRef('siren') + vat_number: PropertyRef = PropertyRef('vat_number') + + +@dataclass(frozen=True) +class GandiOrganizationSchema(CartographyNodeSchema): + label: str = 'GandiOrganization' + properties: GandiOrganizationNodeProperties = GandiOrganizationNodeProperties() diff --git a/cartography/sync.py b/cartography/sync.py index 08ee40e01a..9c086430dd 100644 --- a/cartography/sync.py +++ b/cartography/sync.py @@ -21,6 +21,7 @@ import cartography.intel.cve import cartography.intel.digitalocean import cartography.intel.duo +import cartography.intel.gandi import cartography.intel.gcp import cartography.intel.github import cartography.intel.gsuite @@ -53,6 +54,7 @@ 'lastpass': cartography.intel.lastpass.start_lastpass_ingestion, 'bigfix': cartography.intel.bigfix.start_bigfix_ingestion, 'duo': cartography.intel.duo.start_duo_ingestion, + 'gandi': cartography.intel.gandi.start_gandi_ingestion, 'analysis': cartography.intel.analysis.run, }) diff --git a/tests/data/gandi/__init__.py b/tests/data/gandi/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/data/gandi/organization.py b/tests/data/gandi/organization.py new file mode 100644 index 0000000000..8ec3fb4565 --- /dev/null +++ b/tests/data/gandi/organization.py @@ -0,0 +1,15 @@ +GANDI_ORGANIZATIONS = [ + { + 'id': '1aeab22b-1c3e-4829-a64f-a51d52014073', + 'name': 'LyftOSS', + 'orgname': 'LYFT', + 'firstname': 'John', + 'lastname': 'Doe', + 'type': 'company', + 'email': 'john.doe@lyft.com', + 'reseller': False, + 'corporate': False, + 'siren': '123456789', + 'vat_number': 'FR00123456789', + }, +] diff --git a/tests/data/gandi/zones.py b/tests/data/gandi/zones.py new file mode 100644 index 0000000000..8589415a48 --- /dev/null +++ b/tests/data/gandi/zones.py @@ -0,0 +1,91 @@ +GANDI_DNS_ZONES = [ + { + "fqdn": "evilcorp.com", + "tld": "com", + "status": [ + "clientTransferProhibited", + ], + "dates": { + "created_at": "2020-12-29T18:09:16Z", + "deletes_at": "2024-02-12T07:09:16Z", + "hold_begins_at": "2023-12-29T17:09:16Z", + "hold_ends_at": "2024-02-12T17:09:16Z", + "pending_delete_ends_at": "2024-03-18T17:09:16Z", + "registry_created_at": "2020-12-29T17:09:16Z", + "registry_ends_at": "2023-12-29T17:09:16Z", + "renew_begins_at": "2012-01-01T00:00:00Z", + "restore_ends_at": "2024-03-13T17:09:16Z", + "updated_at": "2023-05-19T13:26:38Z", + "authinfo_expires_at": "2024-05-18T13:26:38Z", + }, + "can_tld_lock": True, + "authinfo": "********", + "nameservers": [ + "ns-41-a.gandi.net", + "ns-135-b.gandi.net", + "ns-138-c.gandi.net", + ], + "tags": [], + "services": [ + "dnssec", + "gandilivedns", + "mailboxv2", + ], + "autorenew": { + "duration": 1, + "dates": [ + "2023-11-28T16:09:16Z", + "2023-12-14T17:09:16Z", + "2023-12-28T17:09:16Z", + ], + "org_id": "1aeab22b-1c3e-4829-a64f-a51d52014073", + "href": "https://api.gandi.net/v5/domain/domains/evilcorp.com/autorenew", + "enabled": True, + }, + "contacts": { + "owner": {}, + "admin": {}, + "tech": {}, + "bill": {}, + }, + "id": "3c31bfaa-3ba9-4f10-b468-404762ffa6a0", + "sharing_space": { + "id": "1aeab22b-1c3e-4829-a64f-a51d52014073", + "name": "LyftOSS", + "type": "organization", + "reseller": False, + }, + "href": "https://api.gandi.net/v5/domain/domains/evilcorp.com", + "reachability": "done", + "fqdn_unicode": "evilcorp.com", + "records": [ + { + "rrset_name": "@", + "rrset_type": "A", + "rrset_ttl": 10800, + "rrset_values": [ + "1.1.1.1", + ], + "rrset_href": "https://api.gandi.net/v5/livedns/domains/evilcorp.com/records/%40/A", + }, + { + "rrset_name": "@", + "rrset_type": "TXT", + "rrset_ttl": 10800, + "rrset_values": [ + "\"v=spf1 include:spf.mailjet.com ?all\"", + ], + "rrset_href": "https://api.gandi.net/v5/livedns/domains/evilcorp.com/records/%40/TXT", + }, + { + "rrset_name": "_DMARC", + "rrset_type": "CNAME", + "rrset_ttl": 10800, + "rrset_values": [ + "v=DMARC1;p=reject;pct=100;rua=mailto:postmaster@evilcorp.com", + ], + "rrset_href": "https://api.gandi.net/v5/livedns/domains/evilcorp.com/records/_DMARC/CNAME", + }, + ], + }, +] From cec0b4c611d2251c8e9c732174f90b4b417fc9e9 Mon Sep 17 00:00:00 2001 From: Jeremy Chapeau Date: Wed, 14 Jun 2023 10:38:46 +0200 Subject: [PATCH 2/8] fix typo --- cartography/intel/gandi/zones.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cartography/intel/gandi/zones.py b/cartography/intel/gandi/zones.py index e1cec52b53..c8d5dcf531 100644 --- a/cartography/intel/gandi/zones.py +++ b/cartography/intel/gandi/zones.py @@ -50,7 +50,7 @@ def transform(domains: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], List records.append({ "id": f"{ns}+NS", "rsset_name": "@", - "rsset_type": "NS", + "rrset_type": "NS", "rsset_value": ns, "registered_domain": dom['fqdn'], }) From 141d6da2fb55016b852921f83b3b21a81f9362d7 Mon Sep 17 00:00:00 2001 From: Jeremy Chapeau Date: Wed, 14 Jun 2023 15:55:00 +0200 Subject: [PATCH 3/8] fix records ingestion on non livedns --- cartography/intel/gandi/zones.py | 37 ++++++++++++++++---------------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/cartography/intel/gandi/zones.py b/cartography/intel/gandi/zones.py index c8d5dcf531..2571a786e0 100644 --- a/cartography/intel/gandi/zones.py +++ b/cartography/intel/gandi/zones.py @@ -54,26 +54,27 @@ def transform(domains: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], List "rsset_value": ns, "registered_domain": dom['fqdn'], }) - # Extract records - for rec in dom['records']: - rec['rrset_value'] = ','.join(rec['rrset_values']) - if rec['rrset_name'] == '@': - rec['id'] = rec['rrset_values'][0] - else: - rec['id'] = f"{rec['rrset_name']}.{dom['fqdn']}+{rec['rrset_type']}" - rec['registered_domain'] = dom['fqdn'] - # Split on IPs - if rec['rrset_type'] in ['A', 'AAAA']: - if len(rec['rrset_values']) >= 1: - for ip in rec['rrset_values']: - splited_record = rec.copy() - splited_record['resolved_ip'] = ip - ips.add(ip) - records.append(splited_record) + if "gandilivedns" in dom['services']: + # Extract records + for rec in dom['records']: + rec['rrset_value'] = ','.join(rec['rrset_values']) + if rec['rrset_name'] == '@': + rec['id'] = rec['rrset_values'][0] + else: + rec['id'] = f"{rec['rrset_name']}.{dom['fqdn']}+{rec['rrset_type']}" + rec['registered_domain'] = dom['fqdn'] + # Split on IPs + if rec['rrset_type'] in ['A', 'AAAA']: + if len(rec['rrset_values']) >= 1: + for ip in rec['rrset_values']: + splited_record = rec.copy() + splited_record['resolved_ip'] = ip + ips.add(ip) + records.append(splited_record) + else: + records.append(rec) else: records.append(rec) - else: - records.append(rec) zones.append(dom) # Format IPs formated_ips: List[Dict[str, Any]] = [] From e902b76c5ed74c8ff1d0a13a850fde097c227f16 Mon Sep 17 00:00:00 2001 From: Jeremy Chapeau Date: Wed, 14 Jun 2023 16:04:17 +0200 Subject: [PATCH 4/8] fix split multi value rrset --- cartography/intel/gandi/zones.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/cartography/intel/gandi/zones.py b/cartography/intel/gandi/zones.py index 2571a786e0..245c67bbbb 100644 --- a/cartography/intel/gandi/zones.py +++ b/cartography/intel/gandi/zones.py @@ -57,24 +57,27 @@ def transform(domains: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], List if "gandilivedns" in dom['services']: # Extract records for rec in dom['records']: - rec['rrset_value'] = ','.join(rec['rrset_values']) - if rec['rrset_name'] == '@': - rec['id'] = rec['rrset_values'][0] - else: - rec['id'] = f"{rec['rrset_name']}.{dom['fqdn']}+{rec['rrset_type']}" - rec['registered_domain'] = dom['fqdn'] - # Split on IPs - if rec['rrset_type'] in ['A', 'AAAA']: - if len(rec['rrset_values']) >= 1: + # No value + if len(rec['rrset_values']) == 0: + records.append(rec) + continue + # 1 or more values + for value in rec['rrset_values']: + rec_single = rec.copy() + if rec_single['rrset_name'] == '@': + rec['id'] = value + else: + rec_single['id'] = f"{rec['rrset_name']}.{dom['fqdn']}+{rec['rrset_type']}" + rec_single['registered_domain'] = dom['fqdn'] + # Split on IPs + if rec['rrset_type'] in ['A', 'AAAA']: for ip in rec['rrset_values']: - splited_record = rec.copy() + splited_record = rec_single.copy() splited_record['resolved_ip'] = ip ips.add(ip) records.append(splited_record) else: - records.append(rec) - else: - records.append(rec) + records.append(rec_single) zones.append(dom) # Format IPs formated_ips: List[Dict[str, Any]] = [] From 2473c1011e0959701baed56e64335dbccfc87e28 Mon Sep 17 00:00:00 2001 From: Jeremy Chapeau Date: Thu, 15 Jun 2023 11:09:52 +0200 Subject: [PATCH 5/8] fix missing id --- cartography/intel/gandi/zones.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cartography/intel/gandi/zones.py b/cartography/intel/gandi/zones.py index 245c67bbbb..ebd8ecb411 100644 --- a/cartography/intel/gandi/zones.py +++ b/cartography/intel/gandi/zones.py @@ -65,7 +65,7 @@ def transform(domains: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], List for value in rec['rrset_values']: rec_single = rec.copy() if rec_single['rrset_name'] == '@': - rec['id'] = value + rec_single['id'] = value else: rec_single['id'] = f"{rec['rrset_name']}.{dom['fqdn']}+{rec['rrset_type']}" rec_single['registered_domain'] = dom['fqdn'] From 8802382a9a501ce784e1aeabbeb65db776568d14 Mon Sep 17 00:00:00 2001 From: Jeremy Chapeau Date: Mon, 24 Jul 2023 13:27:51 +0200 Subject: [PATCH 6/8] fix: empty records --- cartography/intel/gandi/zones.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cartography/intel/gandi/zones.py b/cartography/intel/gandi/zones.py index ebd8ecb411..f5cc763d5d 100644 --- a/cartography/intel/gandi/zones.py +++ b/cartography/intel/gandi/zones.py @@ -56,7 +56,7 @@ def transform(domains: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], List }) if "gandilivedns" in dom['services']: # Extract records - for rec in dom['records']: + for rec in dom.get('records', []): # No value if len(rec['rrset_values']) == 0: records.append(rec) From 9f673ccd3f9711b33e2052333518bab87e8ec60a Mon Sep 17 00:00:00 2001 From: Jeremy Chapeau Date: Fri, 25 Aug 2023 16:50:05 +0200 Subject: [PATCH 7/8] add integration tests --- cartography/intel/gandi/zones.py | 15 +-- cartography/models/gandi/dnsrecord.py | 10 +- cartography/models/gandi/dnszone.py | 26 ++++- tests/data/gandi/zones.py | 16 +-- .../cartography/intel/gandi/__init__.py | 0 .../cartography/intel/gandi/test_dns.py | 99 +++++++++++++++++++ .../intel/gandi/test_organization.py | 31 ++++++ 7 files changed, 175 insertions(+), 22 deletions(-) create mode 100644 tests/integration/cartography/intel/gandi/__init__.py create mode 100644 tests/integration/cartography/intel/gandi/test_dns.py create mode 100644 tests/integration/cartography/intel/gandi/test_organization.py diff --git a/cartography/intel/gandi/zones.py b/cartography/intel/gandi/zones.py index f5cc763d5d..a15953fcb7 100644 --- a/cartography/intel/gandi/zones.py +++ b/cartography/intel/gandi/zones.py @@ -1,4 +1,5 @@ import logging +from hashlib import md5 from typing import Any from typing import Dict from typing import List @@ -48,7 +49,7 @@ def transform(domains: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], List # Create record from nameservers for ns in dom['nameservers']: records.append({ - "id": f"{ns}+NS", + "id": md5(f"{ns}+NS".encode()).hexdigest(), "rsset_name": "@", "rrset_type": "NS", "rsset_value": ns, @@ -57,17 +58,17 @@ def transform(domains: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], List if "gandilivedns" in dom['services']: # Extract records for rec in dom.get('records', []): - # No value + record_id = md5(f"{rec['rrset_name']}.{dom['fqdn']}+{rec['rrset_type']}".encode()) + # No value if len(rec['rrset_values']) == 0: + rec['id'] = record_id.hexdigest() records.append(rec) continue - # 1 or more values + # 1 or more values for value in rec['rrset_values']: + record_id.update(value.encode()) rec_single = rec.copy() - if rec_single['rrset_name'] == '@': - rec_single['id'] = value - else: - rec_single['id'] = f"{rec['rrset_name']}.{dom['fqdn']}+{rec['rrset_type']}" + rec_single['id'] = record_id.hexdigest() rec_single['registered_domain'] = dom['fqdn'] # Split on IPs if rec['rrset_type'] in ['A', 'AAAA']: diff --git a/cartography/models/gandi/dnsrecord.py b/cartography/models/gandi/dnsrecord.py index c2c351b45c..011fbf4a04 100644 --- a/cartography/models/gandi/dnsrecord.py +++ b/cartography/models/gandi/dnsrecord.py @@ -20,7 +20,7 @@ class GandiDNSRecordProperties(CartographyNodeProperties): name: PropertyRef = PropertyRef('rrset_name', extra_index=True) type: PropertyRef = PropertyRef('rrset_type') ttl: PropertyRef = PropertyRef('rrset_ttl') - value: PropertyRef = PropertyRef('rrset_value') + values: PropertyRef = PropertyRef('rrset_values') registered_domain: PropertyRef = PropertyRef('registered_domain') @@ -30,14 +30,14 @@ class ZoneToRecordRelProperties(CartographyRelProperties): @dataclass(frozen=True) -# (:GandiDNSZone)<-[:MEMBER_OF_DNS_ZONE]-(:GandiDNSRecord) +# (:GandiDNSZone)-[:RESOURCE]->(:GandiDNSRecord) class ZoneToRecordRel(CartographyRelSchema): target_node_label: str = 'GandiDNSZone' target_node_matcher: TargetNodeMatcher = make_target_node_matcher( {'name': PropertyRef('registered_domain')}, ) - direction: LinkDirection = LinkDirection.OUTWARD - rel_label: str = "MEMBER_OF_DNS_ZONE" + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "RESOURCE" properties: ZoneToRecordRelProperties = ZoneToRecordRelProperties() @@ -69,4 +69,4 @@ class GandiDNSRecordSchema(CartographyNodeSchema): RecordToIpRel(), ], ) - # TODO: Link to organization: sub_resource_relationship: XXXRel = XXXRel() + sub_resource_relationship: ZoneToRecordRel = ZoneToRecordRel() diff --git a/cartography/models/gandi/dnszone.py b/cartography/models/gandi/dnszone.py index 706ea9c6de..497a5a2539 100644 --- a/cartography/models/gandi/dnszone.py +++ b/cartography/models/gandi/dnszone.py @@ -5,6 +5,11 @@ from cartography.models.core.nodes import CartographyNodeProperties from cartography.models.core.nodes import CartographyNodeSchema from cartography.models.core.nodes import ExtraNodeLabels +from cartography.models.core.relationships import CartographyRelProperties +from cartography.models.core.relationships import CartographyRelSchema +from cartography.models.core.relationships import LinkDirection +from cartography.models.core.relationships import make_target_node_matcher +from cartography.models.core.relationships import TargetNodeMatcher @dataclass(frozen=True) @@ -13,6 +18,7 @@ class GandiDNSZoneNodeProperties(CartographyNodeProperties): lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) name: PropertyRef = PropertyRef('fqdn', extra_index=True) tld: PropertyRef = PropertyRef('tld') + organization_id: PropertyRef = PropertyRef('sharing_space.id') # Dates created_at: PropertyRef = PropertyRef('dates.created_at') deletes_at: PropertyRef = PropertyRef('dates.deletes_at') @@ -33,10 +39,26 @@ class GandiDNSZoneNodeProperties(CartographyNodeProperties): autorenew_enabled: PropertyRef = PropertyRef('autorenew.enabled') +@dataclass(frozen=True) +class OrganizationToDNSZoneRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +# (:GandiOrganization)-[:RESOURCE]->(:GandiDNSZone) +class OrganizationToDNSZoneRel(CartographyRelSchema): + target_node_label: str = 'GandiOrganization' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('sharing_space.id')}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "RESOURCE" + properties: OrganizationToDNSZoneRelProperties = OrganizationToDNSZoneRelProperties() + + @dataclass(frozen=True) class GandiDNSZoneSchema(CartographyNodeSchema): label: str = 'GandiDNSZone' extra_node_labels: Optional[ExtraNodeLabels] = ExtraNodeLabels(['DNSZone']) properties: GandiDNSZoneNodeProperties = GandiDNSZoneNodeProperties() - # TODO: Link to organization: sub_resource_relationship: XXXRel = XXXRel() - # Use sharing_space.id or sharing_space.name to link to organization + sub_resource_relationship: OrganizationToDNSZoneRel = OrganizationToDNSZoneRel() diff --git a/tests/data/gandi/zones.py b/tests/data/gandi/zones.py index 8589415a48..4d68664c0e 100644 --- a/tests/data/gandi/zones.py +++ b/tests/data/gandi/zones.py @@ -1,6 +1,6 @@ GANDI_DNS_ZONES = [ { - "fqdn": "evilcorp.com", + "fqdn": "lyft.com", "tld": "com", "status": [ "clientTransferProhibited", @@ -39,7 +39,7 @@ "2023-12-28T17:09:16Z", ], "org_id": "1aeab22b-1c3e-4829-a64f-a51d52014073", - "href": "https://api.gandi.net/v5/domain/domains/evilcorp.com/autorenew", + "href": "https://api.gandi.net/v5/domain/domains/lyft.com/autorenew", "enabled": True, }, "contacts": { @@ -55,9 +55,9 @@ "type": "organization", "reseller": False, }, - "href": "https://api.gandi.net/v5/domain/domains/evilcorp.com", + "href": "https://api.gandi.net/v5/domain/domains/lyft.com", "reachability": "done", - "fqdn_unicode": "evilcorp.com", + "fqdn_unicode": "lyft.com", "records": [ { "rrset_name": "@", @@ -66,7 +66,7 @@ "rrset_values": [ "1.1.1.1", ], - "rrset_href": "https://api.gandi.net/v5/livedns/domains/evilcorp.com/records/%40/A", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/lyft.com/records/%40/A", }, { "rrset_name": "@", @@ -75,16 +75,16 @@ "rrset_values": [ "\"v=spf1 include:spf.mailjet.com ?all\"", ], - "rrset_href": "https://api.gandi.net/v5/livedns/domains/evilcorp.com/records/%40/TXT", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/lyft.com/records/%40/TXT", }, { "rrset_name": "_DMARC", "rrset_type": "CNAME", "rrset_ttl": 10800, "rrset_values": [ - "v=DMARC1;p=reject;pct=100;rua=mailto:postmaster@evilcorp.com", + "v=DMARC1;p=reject;pct=100;rua=mailto:postmaster@lyft.com", ], - "rrset_href": "https://api.gandi.net/v5/livedns/domains/evilcorp.com/records/_DMARC/CNAME", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/lyft.com/records/_DMARC/CNAME", }, ], }, diff --git a/tests/integration/cartography/intel/gandi/__init__.py b/tests/integration/cartography/intel/gandi/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/cartography/intel/gandi/test_dns.py b/tests/integration/cartography/intel/gandi/test_dns.py new file mode 100644 index 0000000000..c732b6f384 --- /dev/null +++ b/tests/integration/cartography/intel/gandi/test_dns.py @@ -0,0 +1,99 @@ +from unittest.mock import Mock + +import cartography.intel.gandi.organization +import cartography.intel.gandi.zones +import tests.data.gandi.organization +import tests.data.gandi.zones +from tests.integration.util import check_nodes +from tests.integration.util import check_rels + +TEST_UPDATE_TAG = 123456789 +COMMON_JOB_PARAMETERS = {"UPDATE_TAG": TEST_UPDATE_TAG} + + +def test_load_gandi_zones(neo4j_session): + """ + Ensure that zones actually get loaded + """ + gandi_api = Mock( + get_organizations=Mock(return_value=tests.data.gandi.organization.GANDI_ORGANIZATIONS), + get_domains=Mock(return_value=tests.data.gandi.zones.GANDI_DNS_ZONES), + ) + + # Act + cartography.intel.gandi.organization.sync( + neo4j_session, + gandi_api, + TEST_UPDATE_TAG, + COMMON_JOB_PARAMETERS, + ) + cartography.intel.gandi.zones.sync( + neo4j_session, + gandi_api, + TEST_UPDATE_TAG, + COMMON_JOB_PARAMETERS, + ) + + # Assert zones exists + expected_nodes = { + ('3c31bfaa-3ba9-4f10-b468-404762ffa6a0', 'lyft.com', 'com', '1aeab22b-1c3e-4829-a64f-a51d52014073'), + } + assert check_nodes(neo4j_session, 'GandiDNSZone', ['id', 'name', 'tld', 'organization_id']) == expected_nodes + + # Asserts records exists + expected_nodes = { + ('e1c458121cc75d19801a5fc261267446', None, 'NS', 'lyft.com'), + ('09ca0af5ae3a005d87fd617c734006cc', None, 'NS', 'lyft.com'), + ('71b0716bcebab7d820f34417a7c8e017', '_DMARC', 'CNAME', 'lyft.com'), + ('32b136cf5f96c5240362292782772fa1', '@', 'A', 'lyft.com'), + ('31c058a2af95ce4a763113adc7af8a65', '@', 'TXT', 'lyft.com'), + ('aadda9b146e8ac211703fe758eea7579', None, 'NS', 'lyft.com'), + } + assert check_nodes(neo4j_session, 'GandiDNSRecord', ['id', 'name', 'type', 'registered_domain']) == expected_nodes + + # Asserts ips exists + expected_nodes = { + ('1.1.1.1',), + } + assert check_nodes(neo4j_session, 'Ip', ['ip']) == expected_nodes + + # Asserts zones are linked to organizations + expected_rels = { + ('1aeab22b-1c3e-4829-a64f-a51d52014073', '3c31bfaa-3ba9-4f10-b468-404762ffa6a0'), + } + assert check_rels( + neo4j_session, + 'GandiOrganization', 'id', + 'GandiDNSZone', 'id', + 'RESOURCE', + rel_direction_right=True, + ) == expected_rels + + # Asserts records are linked to zones + expected_rels = { + ('3c31bfaa-3ba9-4f10-b468-404762ffa6a0', 'e1c458121cc75d19801a5fc261267446'), + ('3c31bfaa-3ba9-4f10-b468-404762ffa6a0', '09ca0af5ae3a005d87fd617c734006cc'), + ('3c31bfaa-3ba9-4f10-b468-404762ffa6a0', 'aadda9b146e8ac211703fe758eea7579'), + ('3c31bfaa-3ba9-4f10-b468-404762ffa6a0', '71b0716bcebab7d820f34417a7c8e017'), + ('3c31bfaa-3ba9-4f10-b468-404762ffa6a0', '31c058a2af95ce4a763113adc7af8a65'), + ('3c31bfaa-3ba9-4f10-b468-404762ffa6a0', '32b136cf5f96c5240362292782772fa1'), + } + assert check_rels( + neo4j_session, + 'GandiDNSZone', 'id', + 'GandiDNSRecord', 'id', + 'RESOURCE', + rel_direction_right=True, + ) == expected_rels + + # Asserts records are linked to ips + excepted_rels = { + ('32b136cf5f96c5240362292782772fa1', '1.1.1.1'), + } + assert check_rels( + neo4j_session, + 'GandiDNSRecord', 'id', + 'Ip', 'ip', + 'DNS_POINTS_TO', + rel_direction_right=True, + ) == excepted_rels diff --git a/tests/integration/cartography/intel/gandi/test_organization.py b/tests/integration/cartography/intel/gandi/test_organization.py new file mode 100644 index 0000000000..40dd2844d2 --- /dev/null +++ b/tests/integration/cartography/intel/gandi/test_organization.py @@ -0,0 +1,31 @@ +from unittest.mock import Mock + +import cartography.intel.gandi.organization +import tests.data.gandi.organization +from tests.integration.util import check_nodes + +TEST_UPDATE_TAG = 123456789 +COMMON_JOB_PARAMETERS = {"UPDATE_TAG": TEST_UPDATE_TAG} + + +def test_load_gandi_organization(neo4j_session): + """ + Ensure that organization actually get loaded + """ + gandi_api = Mock( + get_organizations=Mock(return_value=tests.data.gandi.organization.GANDI_ORGANIZATIONS), + ) + + # Act + cartography.intel.gandi.organization.sync( + neo4j_session, + gandi_api, + TEST_UPDATE_TAG, + COMMON_JOB_PARAMETERS, + ) + + # Assert organization exists + expected_nodes = { + ('1aeab22b-1c3e-4829-a64f-a51d52014073', 'LyftOSS'), + } + assert check_nodes(neo4j_session, 'GandiOrganization', ['id', 'name']) == expected_nodes From 3f10d77a5bebbab912a167c27370ecaf0990b13a Mon Sep 17 00:00:00 2001 From: Jeremy Chapeau Date: Fri, 25 Aug 2023 16:55:19 +0200 Subject: [PATCH 8/8] bug: cleanup bug to match argument condition --- cartography/intel/gandi/zones.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cartography/intel/gandi/zones.py b/cartography/intel/gandi/zones.py index a15953fcb7..00c7bc684c 100644 --- a/cartography/intel/gandi/zones.py +++ b/cartography/intel/gandi/zones.py @@ -9,6 +9,7 @@ from dateutil import parser as dt_parse from cartography.client.core.tx import load +from cartography.graph.job import GraphJob from cartography.intel.gandi.utils import GandiAPI from cartography.models.gandi.dnsrecord import GandiDNSRecordSchema from cartography.models.gandi.dnszone import GandiDNSZoneSchema @@ -28,7 +29,7 @@ def sync( domains = get(gandi_session) zones, records, ips = transform(domains) load_zones(neo4j_session, zones, records, ips, update_tag) - # TODO: Cleanup zones & domains (when linked to organization) + cleanup(neo4j_session, common_job_parameters) @timeit @@ -120,3 +121,9 @@ def load_zones( records, lastupdated=update_tag, ) + + +def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None: + #BUG: GraphJob.from_node_schema(GandiDNSZoneSchema(), common_job_parameters).run(neo4j_session) + #BUG: GraphJob.from_node_schema(GandiDNSRecordSchema(), common_job_parameters).run(neo4j_session) + pass