New Intel: Gandi

cartography/cli.py

@@ -509,6 +509,14 @@ def _build_parser(self):
                 'Required if you are using the Semgrep intel module. Ignored otherwise.'
             ),
         )
+        parser.add_argument(
+            '--gandi-apikey-env-var',
+            type=str,
+            default=None,
+            help=(
+                'The name of environment variable containing the gandi API key for authentication.'
+            ),
+        )
         return parser
 
     def main(self, argv: str) -> int:
@@ -685,6 +693,13 @@ def main(self, argv: str) -> int:
         else:
             config.semgrep_app_token = None
 
+        # Gandi config
+        if config.gandi_apikey_env_var:
+            logger.debug(f"Reading API key for Gandi from environment variable {config.gandi_apikey_env_var}")
+            config.gandi_apikey = os.environ.get(config.gandi_apikey_env_var)
+        else:
+            config.gandi_apikey = None
+
         # Run cartography
         try:
             return cartography.sync.run_with_config(self.sync, config)

cartography/config.py

@@ -105,6 +105,8 @@ class Config:
     :param duo_api_hostname: The Duo api hostname, e.g. "api-abc123.duosecurity.com". Optional.
     :param semgrep_app_token: The Semgrep api token. Optional.
     :type semgrep_app_token: str
+    :type: gandi_apikey: str
+    :param gandi_apikey: API authentication key for Gandi. Optional.
     """
 
     def __init__(
@@ -160,6 +162,7 @@ def __init__(
         duo_api_secret=None,
         duo_api_hostname=None,
         semgrep_app_token=None,
+        gandi_apikey=None,
     ):
         self.neo4j_uri = neo4j_uri
         self.neo4j_user = neo4j_user
@@ -212,3 +215,4 @@ def __init__(
         self.duo_api_secret = duo_api_secret
         self.duo_api_hostname = duo_api_hostname
         self.semgrep_app_token = semgrep_app_token
+        self.gandi_apikey = gandi_apikey

cartography/intel/gandi/__init__.py

@@ -0,0 +1,47 @@
+import logging
+
+import neo4j
+
+import cartography.intel.gandi.organization
+import cartography.intel.gandi.zones
+from cartography.config import Config
+from cartography.intel.gandi.utils import GandiAPI
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def start_gandi_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    """
+    If this module is configured, perform ingestion of Gandi data. Otherwise warn and exit
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+
+    if not config.gandi_apikey:
+        logger.info(
+            'Gandi import is not configured - skipping this module. '
+            'See docs to configure.',
+        )
+        return
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+    }
+
+    api_session = GandiAPI(config.gandi_apikey)
+
+    cartography.intel.gandi.organization.sync(
+        neo4j_session,
+        api_session,
+        config.update_tag,
+        common_job_parameters,
+    )
+    cartography.intel.gandi.zones.sync(
+        neo4j_session,
+        api_session,
+        config.update_tag,
+        common_job_parameters,
+    )

cartography/intel/gandi/organization.py

@@ -0,0 +1,42 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+
+from cartography.client.core.tx import load
+from cartography.intel.gandi.utils import GandiAPI
+from cartography.models.gandi.organization import GandiOrganizationSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    gandi_session: GandiAPI,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    orgs = get(gandi_session)
+    load_organizations(neo4j_session, orgs, update_tag)
+
+
+@timeit
+def get(gandi_session: GandiAPI) -> List[Dict[str, Any]]:
+    return gandi_session.get_organizations()
+
+
+def load_organizations(
+    neo4j_session: neo4j.Session,
+    data: List[Dict[str, Any]],
+    update_tag: int,
+) -> None:
+    load(
+        neo4j_session,
+        GandiOrganizationSchema(),
+        data,
+        lastupdated=update_tag,
+    )

cartography/intel/gandi/utils.py

@@ -0,0 +1,47 @@
+from typing import Any
+from typing import Dict
+from typing import List
+
+import requests
+
+
+class GandiAPI:
+    TIMEOUT = (600, 600)
+
+    def __init__(self, apikey: str) -> None:
+        self._base_url: str = 'https://api.gandi.net/v5/'
+        self._session = requests.Session()
+        self._session.headers.update(
+            {
+                'Authorization': f'Apikey {apikey}',
+                'User-Agent': 'Cartography/0.1',
+            },
+        )
+
+    def get_organizations(self) -> List[Dict[str, Any]]:
+        result = []
+        req = self._session.get(f'{self._base_url}/organization/organizations', timeout=self.TIMEOUT)
+        req.raise_for_status()
+        for org in req.json():
+            if org['type'] == 'individual':
+                continue
+            result.append(org)
+        return result
+
+    def get_domains(self) -> List[Dict[str, Any]]:
+        result = []
+        # List domains
+        list_req = self._session.get(f'{self._base_url}/domain/domains', timeout=self.TIMEOUT)
+        list_req.raise_for_status()
+        for dom in list_req.json():
+            # Get domains informations
+            req = self._session.get(dom['href'])
+            req.raise_for_status()
+            domain = req.json().copy()
+            # Get records
+            if 'dnssec' in domain['services']:
+                req = self._session.get(f'{self._base_url}/livedns/domains/{dom["fqdn"]}/records', timeout=self.TIMEOUT)
+                req.raise_for_status()
+                domain['records'] = req.json()
+            result.append(domain)
+        return result

cartography/intel/gandi/zones.py

@@ -0,0 +1,129 @@
+import logging
+from hashlib import md5
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import Tuple
+
+import neo4j
+from dateutil import parser as dt_parse
+
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.intel.gandi.utils import GandiAPI
+from cartography.models.gandi.dnsrecord import GandiDNSRecordSchema
+from cartography.models.gandi.dnszone import GandiDNSZoneSchema
+from cartography.models.gandi.ip import IPSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    gandi_session: GandiAPI,
+    update_tag: int,
+    common_job_parameters: Dict[str, Any],
+) -> None:
+    domains = get(gandi_session)
+    zones, records, ips = transform(domains)
+    load_zones(neo4j_session, zones, records, ips, update_tag)
+    cleanup(neo4j_session, common_job_parameters)
+
+
+@timeit
+def get(gandi_session: GandiAPI) -> List[Dict[str, Any]]:
+    return gandi_session.get_domains()
+
+
+@timeit
+def transform(domains: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], List[Dict[str, Any]]]:
+    zones: List[Dict[str, Any]] = []
+    records: List[Dict[str, Any]] = []
+    ips = set()
+
+    for dom in domains:
+        # Transform dates
+        for k in list(dom['dates'].keys()):
+            dom['dates'][k] = int(dt_parse.parse(dom['dates'][k]).timestamp() * 1000) if dom['dates'][k] else None
+        # Create record from nameservers
+        for ns in dom['nameservers']:
+            records.append({
+                "id": md5(f"{ns}+NS".encode()).hexdigest(),
+                "rsset_name": "@",
+                "rrset_type": "NS",
+                "rsset_value": ns,
+                "registered_domain": dom['fqdn'],
+            })
+        if "gandilivedns" in dom['services']:
+            # Extract records
+            for rec in dom.get('records', []):
+                record_id = md5(f"{rec['rrset_name']}.{dom['fqdn']}+{rec['rrset_type']}".encode())
+                # No value
+                if len(rec['rrset_values']) == 0:
+                    rec['id'] = record_id.hexdigest()
+                    records.append(rec)
+                    continue
+                # 1 or more values
+                for value in rec['rrset_values']:
+                    record_id.update(value.encode())
+                    rec_single = rec.copy()
+                    rec_single['id'] = record_id.hexdigest()
+                    rec_single['registered_domain'] = dom['fqdn']
+                    # Split on IPs
+                    if rec['rrset_type'] in ['A', 'AAAA']:
+                        for ip in rec['rrset_values']:
+                            splited_record = rec_single.copy()
+                            splited_record['resolved_ip'] = ip
+                            ips.add(ip)
+                            records.append(splited_record)
+                    else:
+                        records.append(rec_single)
+        zones.append(dom)
+    # Format IPs
+    formated_ips: List[Dict[str, Any]] = []
+    for ip in ips:
+        formated_ips.append({
+            "id": ip,
+            "ip": ip,
+        })
+    return zones, records, formated_ips
+
+
+def load_zones(
+    neo4j_session: neo4j.Session,
+    zones: List[Dict[str, Any]],
+    records: List[Dict[str, Any]],
+    ips: List[Dict[str, Any]],
+    update_tag: int,
+) -> None:
+    # Save zones
+    load(
+        neo4j_session,
+        GandiDNSZoneSchema(),
+        zones,
+        lastupdated=update_tag,
+    )
+
+    # Save Ips
+    load(
+        neo4j_session,
+        IPSchema(),
+        ips,
+        lastupdated=update_tag,
+    )
+
+    # Save records
+    load(
+        neo4j_session,
+        GandiDNSRecordSchema(),
+        records,
+        lastupdated=update_tag,
+    )
+
+
+def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) -> None:
+    #BUG: GraphJob.from_node_schema(GandiDNSZoneSchema(), common_job_parameters).run(neo4j_session)
+    #BUG: GraphJob.from_node_schema(GandiDNSRecordSchema(), common_job_parameters).run(neo4j_session)
+    pass

cartography/models/gandi/dnsrecord.py

@@ -0,0 +1,72 @@
+from dataclasses import dataclass
+from typing import Optional
+
+from cartography.models.core.common import PropertyRef
+from cartography.models.core.nodes import CartographyNodeProperties
+from cartography.models.core.nodes import CartographyNodeSchema
+from cartography.models.core.nodes import ExtraNodeLabels
+from cartography.models.core.relationships import CartographyRelProperties
+from cartography.models.core.relationships import CartographyRelSchema
+from cartography.models.core.relationships import LinkDirection
+from cartography.models.core.relationships import make_target_node_matcher
+from cartography.models.core.relationships import OtherRelationships
+from cartography.models.core.relationships import TargetNodeMatcher
+
+
+@dataclass(frozen=True)
+class GandiDNSRecordProperties(CartographyNodeProperties):
+    id: PropertyRef = PropertyRef('id')
+    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+    name: PropertyRef = PropertyRef('rrset_name', extra_index=True)
+    type: PropertyRef = PropertyRef('rrset_type')
+    ttl: PropertyRef = PropertyRef('rrset_ttl')
+    values: PropertyRef = PropertyRef('rrset_values')
+    registered_domain: PropertyRef = PropertyRef('registered_domain')
+
+
+@dataclass(frozen=True)
+class ZoneToRecordRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+# (:GandiDNSZone)-[:RESOURCE]->(:GandiDNSRecord)
+class ZoneToRecordRel(CartographyRelSchema):
+    target_node_label: str = 'GandiDNSZone'
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {'name': PropertyRef('registered_domain')},
+    )
+    direction: LinkDirection = LinkDirection.INWARD
+    rel_label: str = "RESOURCE"
+    properties: ZoneToRecordRelProperties = ZoneToRecordRelProperties()
+
+
+@dataclass(frozen=True)
+class RecordToIpRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+# (:Ip)<-[:DNS_POINTS_TO]-(:GandiDNSRecord)
+class RecordToIpRel(CartographyRelSchema):
+    target_node_label: str = 'Ip'
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {'id': PropertyRef('resolved_ip')},
+    )
+    direction: LinkDirection = LinkDirection.OUTWARD
+    rel_label: str = "DNS_POINTS_TO"
+    properties: RecordToIpRelProperties = RecordToIpRelProperties()
+
+
+@dataclass(frozen=True)
+class GandiDNSRecordSchema(CartographyNodeSchema):
+    label: str = 'GandiDNSRecord'
+    extra_node_labels: Optional[ExtraNodeLabels] = ExtraNodeLabels(['DNSRecord'])
+    properties: GandiDNSRecordProperties = GandiDNSRecordProperties()
+    other_relationships: OtherRelationships = OtherRelationships(
+        [
+            ZoneToRecordRel(),
+            RecordToIpRel(),
+        ],
+    )
+    sub_resource_relationship: ZoneToRecordRel = ZoneToRecordRel()