Merge branch 'master' into gandi_intel

README.md

@@ -39,16 +39,13 @@ Start [here](https://lyft.github.io/cartography/install.html).
 ## Usage
 Start with our [tutorial](https://lyft.github.io/cartography/usage/tutorial.html). Our [data schema](https://lyft.github.io/cartography/usage/schema.html) is a helpful reference when you get stuck.
 
-## Contact
+## Community
 
 - Join us on `#cartography` on the [Lyft OSS Slack](https://join.slack.com/t/lyftoss/shared_invite/enQtOTYzODg5OTQwNDE2LTFiYjgwZWM3NTNhMTFkZjc4Y2IxOTI4NTdiNTdhNjQ4M2Q5NTIzMjVjOWI4NmVlNjRiZmU2YzA5NTc3MmFjYTQ).
-
-## Community Meeting
-
-Talk to us and see what we're working on at our [monthly community meeting](https://calendar.google.com/calendar/embed?src=lyft.com_p10o6ceuiieq9sqcn1ef61v1io%40group.calendar.google.com&ctz=America%2FLos_Angeles).
-- Meeting minutes are [here](https://docs.google.com/document/d/1VyRKmB0dpX185I15BmNJZpfAJ_Ooobwz0U1WIhjDxvw).
-- Recorded videos are posted [here](https://www.youtube.com/playlist?list=PLMga2YJvAGzidUWJB_fnG7EHI4wsDDsE1).
-- Our current project road map is [here](https://docs.google.com/document/d/18MOsGI-isFvag1fGk718Aht7wQPueWd4SqOI9KapBa8/edit#heading=h.15nsmgmjaaml).
+- Talk to us and see what we're working on at our [monthly community meeting](https://calendar.google.com/calendar/embed?src=lyft.com_p10o6ceuiieq9sqcn1ef61v1io%40group.calendar.google.com&ctz=America%2FLos_Angeles).
+  - Meeting minutes are [here](https://docs.google.com/document/d/1VyRKmB0dpX185I15BmNJZpfAJ_Ooobwz0U1WIhjDxvw).
+  - Recorded videos are posted [here](https://www.youtube.com/playlist?list=PLMga2YJvAGzidUWJB_fnG7EHI4wsDDsE1).
+- Our current project roadmap is [here](https://github.com/orgs/lyft/projects/26/views/1).
 
 ## Contributing
 Thank you for considering contributing to Cartography!
@@ -57,9 +54,12 @@ Thank you for considering contributing to Cartography!
 Legal stuff: This project is governed by [Lyft's code of conduct](https://github.com/lyft/code-of-conduct).
 All contributors and participants agree to abide by its terms.
 
+### Bug reports and feature requests and discussions
+Submit a GitHub issue to report a bug or request a new feature. If we decide that the issue needs more discussion - usually because the scope is too large or we need to make careful decision - we will convert the issue to a [GitHub Discussion](https://github.com/lyft/cartography/discussions).
+
 ### Developing Cartography
 
-Get started with our [developer documentation](https://lyft.github.io/cartography/dev/developer-guide.html).
+Get started with our [developer documentation](https://lyft.github.io/cartography/dev/developer-guide.html). Please feel free to submit your own PRs to update documentation if you've found a better way to explain something.
 
 #### Sign the Contributor License Agreement (CLA)
 
@@ -73,6 +73,7 @@ and follow the instructions to sign the CLA.
 1. [Thought Machine](https://thoughtmachine.net/)
 1. [MessageBird](https://messagebird.com)
 1. [Cloudanix](https://www.cloudanix.com/)
+1. [ZeusCloud](https://www.zeuscloud.io/)
 1. {Your company here} :-)
 
 If your organization uses Cartography, please file a PR and update this list. Say hi on Slack too!

cartography/cli.py

@@ -500,6 +500,15 @@ def _build_parser(self):
                 'The Duo api hostname'
             ),
         )
+        parser.add_argument(
+            '--semgrep-app-token-env-var',
+            type=str,
+            default=None,
+            help=(
+                'The name of environment variable containing the Semgrep app token key. '
+                'Required if you are using the Semgrep intel module. Ignored otherwise.'
+            ),
+        )
         parser.add_argument(
             '--gandi-apikey-env-var',
             type=str,
@@ -646,7 +655,7 @@ def main(self, argv: str) -> int:
             logger.debug(f"Reading config string for GSuite from environment variable {config.gsuite_tokens_env_var}")
             config.gsuite_config = os.environ.get(config.gsuite_tokens_env_var)
         else:
-            config.github_config = None
+            config.gsuite_tokens_env_var = None
 
         # Lastpass config
         if config.lastpass_cid_env_var:
@@ -673,6 +682,16 @@ def main(self, argv: str) -> int:
             )
             config.duo_api_key = os.environ.get(config.duo_api_key_env_var)
             config.duo_api_secret = os.environ.get(config.duo_api_secret_env_var)
+        else:
+            config.duo_api_key = None
+            config.duo_api_secret = None
+
+        # Semgrep config
+        if config.semgrep_app_token_env_var:
+            logger.debug(f"Reading Semgrep App Token from environment variable {config.semgrep_app_token_env_var}")
+            config.semgrep_app_token = os.environ.get(config.semgrep_app_token_env_var)
+        else:
+            config.semgrep_app_token = None
 
         # Gandi config
         if config.gandi_apikey_env_var:

cartography/config.py

@@ -103,6 +103,8 @@ class Config:
     :param duo_api_key: The Duo api secret. Optional.
     :type duo_api_hostname: str
     :param duo_api_hostname: The Duo api hostname, e.g. "api-abc123.duosecurity.com". Optional.
+    :param semgrep_app_token: The Semgrep api token. Optional.
+    :type semgrep_app_token: str
     :type: gandi_apikey: str
     :param gandi_apikey: API authentication key for Gandi. Optional.
     """
@@ -159,6 +161,7 @@ def __init__(
         duo_api_key=None,
         duo_api_secret=None,
         duo_api_hostname=None,
+        semgrep_app_token=None,
         gandi_apikey=None,
     ):
         self.neo4j_uri = neo4j_uri
@@ -211,4 +214,5 @@ def __init__(
         self.duo_api_key = duo_api_key
         self.duo_api_secret = duo_api_secret
         self.duo_api_hostname = duo_api_hostname
+        self.semgrep_app_token = semgrep_app_token
         self.gandi_apikey = gandi_apikey

cartography/data/indexes.cypher

@@ -321,10 +321,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:SpotlightVulnerability) ON (n.host_info_local_
 CREATE INDEX IF NOT EXISTS FOR (n:SpotlightVulnerability) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:SQSQueue) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:SQSQueue) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:SSMInstanceInformation) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:SSMInstanceInformation) ON (n.lastupdated);
-CREATE INDEX IF NOT EXISTS FOR (n:SSMInstancePatch) ON (n.id);
-CREATE INDEX IF NOT EXISTS FOR (n:SSMInstancePatch) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:User) ON (n.arn);
 CREATE INDEX IF NOT EXISTS FOR (n:User) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:AzureTenant) ON (n.id);

cartography/data/jobs/cleanup/github_repos_cleanup.json

@@ -38,6 +38,31 @@
     "query": "MATCH (:GitHubRepository)-[r:REQUIRES]->(:PythonLibrary) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
     "iterative": true,
     "iterationsize": 100
+  },
+  {
+    "query": "MATCH (:GitHubUser)-[r:OUTSIDE_COLLAB_ADMIN]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
+    "iterative": true,
+    "iterationsize": 100
+  },
+  {
+    "query": "MATCH (:GitHubUser)-[r:OUTSIDE_COLLAB_MAINTAIN]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
+    "iterative": true,
+    "iterationsize": 100
+  },
+  {
+    "query": "MATCH (:GitHubUser)-[r:OUTSIDE_COLLAB_READ]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
+    "iterative": true,
+    "iterationsize": 100
+  },
+  {
+    "query": "MATCH (:GitHubUser)-[r:OUTSIDE_COLLAB_TRIAGE]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
+    "iterative": true,
+    "iterationsize": 100
+  },
+  {
+    "query": "MATCH (:GitHubUser)-[r:OUTSIDE_COLLAB_WRITE]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
+    "iterative": true,
+    "iterationsize": 100
   }],
   "name": "cleanup GitHub repos data"
 }

cartography/data/jobs/cleanup/github_users_cleanup.json

@@ -14,31 +14,6 @@
     "iterative": true,
     "iterationsize": 100
   },
-  {
-    "query": "MATCH (:GitHubUser)-[r:OUTSIDE_COLLAB_ADMIN]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
-    "iterative": true,
-    "iterationsize": 100
-  },
-  {
-    "query": "MATCH (:GitHubUser)-[r:OUTSIDE_COLLAB_MAINTAIN]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
-    "iterative": true,
-    "iterationsize": 100
-  },
-  {
-    "query": "MATCH (:GitHubUser)-[r:OUTSIDE_COLLAB_READ]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
-    "iterative": true,
-    "iterationsize": 100
-  },
-  {
-    "query": "MATCH (:GitHubUser)-[r:OUTSIDE_COLLAB_TRIAGE]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
-    "iterative": true,
-    "iterationsize": 100
-  },
-  {
-    "query": "MATCH (:GitHubUser)-[r:OUTSIDE_COLLAB_WRITE]->(:GitHubRepository) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
-    "iterative": true,
-    "iterationsize": 100
-  },
   {
     "query": "MATCH (:GitHubUser)-[r:MEMBER_OF]->(:GitHubOrganization) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
     "iterative": true,

cartography/driftdetect/config.py

@@ -1,3 +1,6 @@
+from typing import Optional
+
+
 class UpdateConfig:
     """
     A common interface for the drift-detection update configuration.
@@ -18,10 +21,10 @@ class UpdateConfig:
 
     def __init__(
         self,
-        drift_detection_directory,
-        neo4j_uri,
-        neo4j_user=None,
-        neo4j_password=None,
+        drift_detection_directory: str,
+        neo4j_uri: str,
+        neo4j_user: Optional[str] = None,
+        neo4j_password: Optional[str] = None,
     ):
         self.neo4j_uri = neo4j_uri
         self.neo4j_user = neo4j_user
@@ -46,13 +49,13 @@ class GetDriftConfig:
 
     def __init__(
         self,
-        query_directory,
-        start_state,
-        end_state,
+        query_directory: str,
+        start_state: str,
+        end_state: str,
     ):
-        self.query_directory = query_directory
-        self.start_state = start_state
-        self.end_state = end_state
+        self.query_directory: str = query_directory
+        self.start_state: str = start_state
+        self.end_state: str = end_state
 
 
 class AddShortcutConfig:
@@ -72,10 +75,10 @@ class AddShortcutConfig:
 
     def __init__(
         self,
-        query_directory,
-        shortcut,
-        filename,
+        query_directory: str,
+        shortcut: str,
+        filename: str,
     ):
-        self.query_directory = query_directory
-        self.shortcut = shortcut
-        self.filename = filename
+        self.query_directory: str = query_directory
+        self.shortcut: str = shortcut
+        self.filename: str = filename

cartography/driftdetect/detect_deviations.py

@@ -1,8 +1,12 @@
 import logging
 import os
+from typing import List
+from typing import Union
 
 from marshmallow import ValidationError
 
+from cartography.driftdetect.config import GetDriftConfig
+from cartography.driftdetect.model import State
 from cartography.driftdetect.reporter import report_drift
 from cartography.driftdetect.serializers import ShortcutSchema
 from cartography.driftdetect.serializers import StateSchema
@@ -12,7 +16,7 @@
 logger = logging.getLogger(__name__)
 
 
-def run_drift_detection(config):
+def run_drift_detection(config: GetDriftConfig) -> None:
     try:
         if not valid_directory(config.query_directory):
             logger.error("Invalid Drift Detection Directory")
@@ -59,7 +63,7 @@ def run_drift_detection(config):
         logger.exception(msg)
 
 
-def perform_drift_detection(start_state, end_state):
+def perform_drift_detection(start_state: State, end_state: State):
     """
     Returns differences (additions and missing results) between two States.
 
@@ -81,7 +85,7 @@ def perform_drift_detection(start_state, end_state):
     return new_results, missing_results
 
 
-def compare_states(start_state, end_state):
+def compare_states(start_state: State, end_state: State):
     """
     Helper function for comparing differences between two States.
 
@@ -92,10 +96,12 @@ def compare_states(start_state, end_state):
     :return: list of tuples of differences between states in the form (dictionary, State object)
     """
     differences = []
+    # Use set for faster membership check
+    start_state_results = {tuple(res) for res in start_state.results}
     for result in end_state.results:
-        if result in start_state.results:
+        if tuple(result) in start_state_results:
             continue
-        drift = []
+        drift: List[Union[str, List[str]]] = []
         for field in result:
             value = field.split("|")
             if len(value) > 1:

cartography/driftdetect/get_states.py

@@ -1,12 +1,18 @@
 import logging
 import os.path
 import time
+from typing import Any
+from typing import Dict
+from typing import List
 
 import neo4j.exceptions
 from marshmallow import ValidationError
 from neo4j import GraphDatabase
 
+from cartography.client.core.tx import read_list_of_dicts_tx
 from cartography.driftdetect.add_shortcut import add_shortcut
+from cartography.driftdetect.config import UpdateConfig
+from cartography.driftdetect.model import State
 from cartography.driftdetect.serializers import ShortcutSchema
 from cartography.driftdetect.serializers import StateSchema
 from cartography.driftdetect.storage import FileSystem
@@ -15,7 +21,7 @@
 logger = logging.getLogger(__name__)
 
 
-def run_get_states(config):
+def run_get_states(config: UpdateConfig) -> None:
     """
     Handles neo4j errors and then updates detectors.
 
@@ -90,7 +96,13 @@ def run_get_states(config):
                 logger.exception(err)
 
 
-def get_query_state(session, query_directory, state_serializer, storage, filename):
+def get_query_state(
+        session: neo4j.Session,
+        query_directory: str,
+        state_serializer: StateSchema,
+        storage,
+        filename: str,
+) -> State:
     """
     Gets the most recent state of a query.
 
@@ -115,7 +127,7 @@ def get_query_state(session, query_directory, state_serializer, storage, filenam
     return state
 
 
-def get_state(session, state):
+def get_state(session: neo4j.Session, state: State) -> None:
     """
     Connects to a neo4j session, runs the validation query, then saves the results to a state.
 
@@ -126,12 +138,16 @@ def get_state(session, state):
     :return:
     """
 
-    new_results = session.run(state.validation_query)
+    new_results: List[Dict[str, Any]] = session.read_transaction(
+        read_list_of_dicts_tx,
+        state.validation_query,
+    )
     logger.debug(f"Updating results for {state.name}")
 
-    state.properties = new_results.keys()
-    results = []
+    # The keys will be the same across all items in the returned list
+    state.properties = list(new_results[0].keys()) if len(new_results) > 0 else []
 
+    results = []
     for record in new_results:
         values = []
         for field in record.values():