Merge branch 'master' into topic/chandan/logging_config

.cache/.gitignore

@@ -0,0 +1,5 @@
+# This .gitignore is a placeholder so that we can store .cache/ in github.
+# We are including a pre-created .cache in Github so that when
+# we run docker-compose in WSL2 during dev linting, WSL2 does not attempt
+# to create .cache/ with root as the owner. The contents of .cache
+# should never be pushed back upstream to the main cartography repo.

.dockerignore

@@ -1,4 +1,6 @@
 build/
 *.egg-info/
 venv/
-.git/
+.compose
+.cache
+.local

.env

@@ -0,0 +1,4 @@
+# This file is for docker-compose dev use so that mounting
+# Neo4j volumes doesn't fail with permissions errors.
+GID=1000
+UID=1000

.github/pull_request_template.md

@@ -20,4 +20,4 @@ If you are changing a node or relationship:
 - [ ] Update the [schema](https://github.com/lyft/cartography/tree/master/docs/root/modules) and [readme](https://github.com/lyft/cartography/blob/master/docs/schema/README.md).
 
 If you are implementing a new intel module:
-- [ ] Use the NodeSchema [data model](https://lyft.github.io/cartography/dev/writing-intel-modules.html#defining-a-node).
+- [ ] Use the NodeSchema [data model](https://cartography-cncf.github.io/cartography/dev/writing-intel-modules.html#defining-a-node).

.github/workflows/publish-to-ghcr.yml

@@ -44,7 +44,7 @@ jobs:
       - name: Build and push
         uses: docker/build-push-action@v3
         with:
-          file: dist.Dockerfile
+          file: Dockerfile
           context: .
           push: true # push the image to ghcr
           tags: ${{ steps.meta.outputs.tags }}

.github/workflows/test_suite.yml

@@ -90,7 +90,7 @@ jobs:
       - name: make test_integration
         run: make test_integration
 
-  build-dist-docker-image:
+  build-docker-image:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -106,7 +106,7 @@ jobs:
       - name: Build
         uses: docker/build-push-action@v3
         with:
-          file: dist.Dockerfile
+          file: Dockerfile
           push: false # only build the image, don't push it anywhere
           context: .
           tags: ${{ steps.meta.outputs.tags }}

.gitignore

@@ -16,3 +16,4 @@ build/
 .compose
 generated
 dist/
+.local

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ default_language_version:
     python: python3
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
     -   id: check-docstring-first
     -   id: check-executables-have-shebangs
@@ -17,14 +17,14 @@ repos:
     rev: 7.1.1
     hooks:
     -   id: flake8
--   repo: https://github.com/pre-commit/mirrors-autopep8
-    rev: v2.0.4
+-   repo: https://github.com/hhatto/autopep8
+    rev: v2.3.1
     hooks:
     -   id: autopep8
         # disable a few rewrites which will cause autopep8 to reflow
         args: [--in-place, '--ignore=E265,E501,W504']
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v3.17.0
+    rev: v3.19.0
     hooks:
     -   id: pyupgrade
         args: [--py36-plus]
@@ -34,12 +34,12 @@ repos:
     -   id: add-trailing-comma
         args: [--py36-plus]
 -   repo: https://github.com/asottile/reorder_python_imports
-    rev: v3.13.0
+    rev: v3.14.0
     hooks:
     -   id: reorder-python-imports
         args: [--py3-plus]
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.11.2
+    rev: v1.13.0
     hooks:
     -   id: mypy
         additional_dependencies:

Dockerfile

@@ -1,33 +1,21 @@
-FROM ubuntu:focal
+# This is a thin distribution of the cartography software.
+# It is published at ghcr.io.
+FROM python:3.10-slim
 
-WORKDIR /srv/cartography
+# the UID and GID to run cartography as
+# (https://github.com/hexops/dockerfile#do-not-use-a-uid-below-10000).
+ARG uid=10001
+ARG gid=10001
 
-ENV PATH=/venv/bin:$PATH
-RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends python3.10-dev python3-pip python3-setuptools openssl libssl-dev gcc pkg-config libffi-dev libxml2-dev libxmlsec1-dev curl make git && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
+WORKDIR /var/cartography
+ENV HOME=/var/cartography
 
-# Installs pip supported by python3.10
-RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3.10 get-pip.py
+RUN pip install cartography
 
-# Create cartography user so that we can give it ownership of the directory later for unit&integ tests
-RUN groupadd cartography && \
-    useradd -s /bin/bash -d /home/cartography -m -g cartography cartography
+USER ${uid}:${gid}
 
-# Installs python dependencies
-COPY setup.py test-requirements.txt ./
-RUN pip install -e . && \
-    pip install -r test-requirements.txt && \
-    # Grant write access to the directory for unit and integration test coverage files
-    chmod -R a+w /srv/cartography
+# verify that the binary at least runs
+RUN cartography -h
 
-# Install cartography, setting the owner so that tests work
-COPY --chown=cartography:cartography . /srv/cartography
-
-USER cartography
-
-# Sets the directory as safe due to a mismatch in the user that cloned the repo
-# and the user that is going to run the unit&integ tests.
-RUN git config --global --add safe.directory /srv/cartography
-RUN /usr/bin/git config --local user.name "cartography"
+ENTRYPOINT ["cartography"]
+CMD ["-h"]

GOVERNANCE.md

@@ -0,0 +1,16 @@
+# Process for becoming a maintainer
+Generally, express interest. This usually includes things like:
+- Contributing 2+ PRs to Cartography.
+- Regularly participating in discussion on GitHub, on Slack, or in our live video meetings.
+
+# Maintainer responsibilities
+- Hang out on Slack; delayed response is perfectly acceptable
+- Triage GitHub issues and perform pull request reviews for other maintainers and the community.
+- Deeply understand the project's best practices and development patterns.
+
+# When does a maintainer lose maintainer status
+- If a maintainer is no longer interested or cannot perform the maintainer duties listed above, they should volunteer to be moved to emeritus status.
+- If we do not hear from you in over a year, we may remove you from maintainer status without notice.
+
+# Conflict resolution
+We handle conflicts on a case-by-case basis. In general, let's talk it out (Slack, video call, etc). For larger issues we may need to involve more people to come to a decision.

LICENSE

@@ -187,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2019 Lyft, Inc.
+   Copyright 2024 The Linux Foundation
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README.md

@@ -5,77 +5,117 @@ Cartography is a Python tool that consolidates infrastructure assets and the rel
 ![Visualization of RDS nodes and AWS nodes](docs/root/images/accountsandrds.png)
 
 ## Why Cartography?
-Cartography aims to enable a broad set of exploration and automation scenarios.  It is particularly good at exposing otherwise hidden dependency relationships between your service's assets so that you may validate assumptions about security risks.
+Cartography aims to enable a broad set of exploration and automation scenarios. It is particularly good at exposing otherwise hidden dependency relationships between your service's assets so that you may validate assumptions about security risks.
 
-Service owners can generate asset reports, Red Teamers can discover attack paths, and Blue Teamers can identify areas for security improvement.   All can benefit from using the graph for manual exploration through a web frontend interface, or in an automated fashion by calling the APIs.
+Service owners can generate asset reports, Red Teamers can discover attack paths, and Blue Teamers can identify areas for security improvement. All can benefit from using the graph for manual exploration through a web frontend interface, or in an automated fashion by calling the APIs.
 
-Cartography is not the only [security](https://github.com/dowjones/hammer) [graph](https://github.com/BloodHoundAD/BloodHound) [tool](https://github.com/Netflix/security_monkey) [out](https://github.com/vysecurity/ANGRYPUPPY) [there](https://github.com/duo-labs/cloudmapper), but it differentiates itself by being fully-featured yet generic and [extensible](https://lyft.github.io/cartography/dev/writing-analysis-jobs.html) enough to help make anyone better understand their risk exposure, regardless of what platforms they use.  Rather than being focused on one core scenario or attack vector like the other linked tools, Cartography focuses on flexibility and exploration.
+Cartography is not the only [security](https://github.com/dowjones/hammer) [graph](https://github.com/BloodHoundAD/BloodHound) [tool](https://github.com/Netflix/security_monkey) [out](https://github.com/vysecurity/ANGRYPUPPY) [there](https://github.com/duo-labs/cloudmapper), but it differentiates itself by being fully-featured yet generic and [extensible](https://cartography-cncf.github.io/cartography/dev/writing-analysis-jobs.html) enough to help make anyone better understand their risk exposure, regardless of what platforms they use. Rather than being focused on one core scenario or attack vector like the other linked tools, Cartography focuses on flexibility and exploration.
 
 You can learn more about the story behind Cartography in our [presentation at BSidesSF 2019](https://www.youtube.com/watch?v=ZukUmZSKSek).
 
-## Install and configure
-Start [here](https://lyft.github.io/cartography/install.html).
 
 ## Supported platforms
 
-- [Amazon Web Services](https://lyft.github.io/cartography/modules/aws/index.html) - API Gateway, Config, EC2, ECS, ECR, Elasticsearch, Elastic Kubernetes Service (EKS), DynamoDB, IAM, Inspector, KMS, Lambda, RDS, Redshift, Route53, S3, Secrets Manager, Security Hub, SQS, SSM, STS, Tags
-- [Google Cloud Platform](https://lyft.github.io/cartography/modules/gcp/index.html) - Cloud Resource Manager, Compute, DNS, Storage, Google Kubernetes Engine
-- [Google GSuite](https://lyft.github.io/cartography/modules/gsuite/index.html) - users, groups
-- [Duo CRXcavator](https://lyft.github.io/cartography/modules/crxcavator/index.html) - Chrome extensions, GSuite users
+- [Amazon Web Services](https://cartography-cncf.github.io/cartography/modules/aws/index.html) - API Gateway, Config, EC2, ECS, ECR, Elasticsearch, Elastic Kubernetes Service (EKS), DynamoDB, IAM, Inspector, KMS, Lambda, RDS, Redshift, Route53, S3, Secrets Manager, Security Hub, SQS, SSM, STS, Tags
+- [Google Cloud Platform](https://cartography-cncf.github.io/cartography/modules/gcp/index.html) - Cloud Resource Manager, Compute, DNS, Storage, Google Kubernetes Engine
+- [Google GSuite](https://cartography-cncf.github.io/cartography/modules/gsuite/index.html) - users, groups
 - [Oracle Cloud Infrastructure](docs/setup/config/oci.md) - IAM
-- [Okta](https://lyft.github.io/cartography/modules/okta/index.html) - users, groups, organizations, roles, applications, factors, trusted origins, reply URIs
-- [Github](https://lyft.github.io/cartography/modules/github/index.html) - repos, branches, users, teams
-- [DigitalOcean](https://lyft.github.io/cartography/modules/digitalocean/index.html)
-- [Microsoft Azure](https://lyft.github.io/cartography/modules/azure/index.html) -  CosmosDB, SQL, Storage, Virtual Machine
-- [Kubernetes](https://lyft.github.io/cartography/modules/kubernetes/index.html) - Cluster, Namespace, Service, Pod, Container
-- [PagerDuty](https://lyft.github.io/cartography/modules/pagerduty/index.html) - Users, teams, services, schedules, escalation policies, integrations, vendors
-- [Crowdstrike Falcon](https://lyft.github.io/cartography/modules/crowdstrike/index.html) - Hosts, Spotlight vulnerabilites, CVEs
-- [NIST CVE](https://lyft.github.io/cartography/modules/cve/index.html) - Common Vulnerabilities and Exposures (CVE) data from NIST database
-- [Lastpass](https://lyft.github.io/cartography/modules/lastpass/index.html) - users
-- [BigFix](https://lyft.github.io/cartography/modules/bigfix/index.html) - Computers
-- [Duo](https://lyft.github.io/cartography/modules/duo/index.html) - Users, Groups, Endpoints
-- [Kandji](https://lyft.github.io/cartography/modules/kandji/index.html) - Devices
-- [SnipeIT](https://lyft.github.io/cartography/modules/snipeit/index.html) - Users, Assets
+- [Okta](https://cartography-cncf.github.io/cartography/modules/okta/index.html) - users, groups, organizations, roles, applications, factors, trusted origins, reply URIs
+- [GitHub](https://cartography-cncf.github.io/cartography/modules/github/index.html) - repos, branches, users, teams
+- [DigitalOcean](https://cartography-cncf.github.io/cartography/modules/digitalocean/index.html)
+- [Microsoft Azure](https://cartography-cncf.github.io/cartography/modules/azure/index.html) -  CosmosDB, SQL, Storage, Virtual Machine
+- [Kubernetes](https://cartography-cncf.github.io/cartography/modules/kubernetes/index.html) - Cluster, Namespace, Service, Pod, Container
+- [PagerDuty](https://cartography-cncf.github.io/cartography/modules/pagerduty/index.html) - Users, teams, services, schedules, escalation policies, integrations, vendors
+- [Crowdstrike Falcon](https://cartography-cncf.github.io/cartography/modules/crowdstrike/index.html) - Hosts, Spotlight vulnerabilities, CVEs
+- [NIST CVE](https://cartography-cncf.github.io/cartography/modules/cve/index.html) - Common Vulnerabilities and Exposures (CVE) data from NIST database
+- [Lastpass](https://cartography-cncf.github.io/cartography/modules/lastpass/index.html) - users
+- [BigFix](https://cartography-cncf.github.io/cartography/modules/bigfix/index.html) - Computers
+- [Duo](https://cartography-cncf.github.io/cartography/modules/duo/index.html) - Users, Groups, Endpoints
+- [Kandji](https://cartography-cncf.github.io/cartography/modules/kandji/index.html) - Devices
+- [SnipeIT](https://cartography-cncf.github.io/cartography/modules/snipeit/index.html) - Users, Assets
+
+
+## Philosophy
+Here are some points that can help you decide if adopting Cartography is a good fit for your problem.
+
+### What Cartography is
+- A simple Python script that pulls data from multiple providers and writes it to a Neo4j graph database in batches.
+- A powerful analysis tool that captures the current snapshot of the environment, building a uniquely useful inventory where you can ask complex questions such as:
+  - Which identities have access to which datastores?
+  - What are the cross-tenant permission relationships in the environment?
+  - What are the network paths in and out of the environment?
+  - What are the backup policies for my datastores?
+- Battle-tested in production by [many companies](#who-uses-cartography).
+- Straightforward to extend with your own custom plugins.
+- Provides a useful data-plane that you can build automation and CSPM (Cloud Security Posture Management) applications on top of.
+
+### What Cartography is not
+- A near-real time capability.
+  - Cartography is not designed for very fast updates. Cartography writes to the database in a batches (not streamed).
+  - Cartography is also limited by how most upstream sources only provide APIs to retrieve assets in a batched manner.
+- By itself, Cartography does not capture data changes over time.
+  - Although we do include a [drift detection](https://cartography-cncf.github.io/cartography/usage/drift-detect.html) feature.
+  - It's also possible to implement other processes in your Cartography installation to make this happen.
+
+
+## Install and configure
+
+### Trying out Cartography on a test machine
+Start [here](https://cartography-cncf.github.io/cartography/install.html) to set up a test graph and get data into it.
+
+### Setting up Cartography in production
+When you are ready to try it in production, read [here](https://cartography-cncf.github.io/cartography/ops.html) for recommendations on getting cartography spun up in your environment.
 
 ## Usage
-Start with our [tutorial](https://lyft.github.io/cartography/usage/tutorial.html). Our [data schema](https://lyft.github.io/cartography/usage/schema.html) is a helpful reference when you get stuck.
+
+### Querying the database directly
+
+![poweruser.png](docs/root/images/poweruser.png)
+
+Now that data is in the graph, you can quickly start with our [querying tutorial](https://cartography-cncf.github.io/cartography/usage/tutorial.html). Our [data schema](https://cartography-cncf.github.io/cartography/usage/schema.html) is a helpful reference when you get stuck.
+
+### Building applications around Cartography
+Directly querying Neo4j is already very useful as a sort of "swiss army knife" for security data problems, but you can also build applications and data pipelines around Cartography. View this doc on [applications](https://cartography-cncf.github.io/cartography/usage/applications.html).
+
 
 ## Community
 
-- Join us on `#cartography` on the [Lyft OSS Slack](https://join.slack.com/t/lyftoss/shared_invite/enQtOTYzODg5OTQwNDE2LTFiYjgwZWM3NTNhMTFkZjc4Y2IxOTI4NTdiNTdhNjQ4M2Q5NTIzMjVjOWI4NmVlNjRiZmU2YzA5NTc3MmFjYTQ).
+- Hang out with us on Slack: Join the CNCF Slack workspace [here](https://communityinviter.com/apps/cloud-native/cncf), and then join the `#cartography` channel.
 - Talk to us and see what we're working on at our [monthly community meeting](https://calendar.google.com/calendar/embed?src=lyft.com_p10o6ceuiieq9sqcn1ef61v1io%40group.calendar.google.com&ctz=America%2FLos_Angeles).
   - Meeting minutes are [here](https://docs.google.com/document/d/1VyRKmB0dpX185I15BmNJZpfAJ_Ooobwz0U1WIhjDxvw).
   - Recorded videos are posted [here](https://www.youtube.com/playlist?list=PLMga2YJvAGzidUWJB_fnG7EHI4wsDDsE1).
-- Our current project roadmap is [here](https://github.com/orgs/lyft/projects/26/views/1).
+
+## License
+
+This project is licensed under the [Apache 2.0 License](LICENSE).
 
 ## Contributing
 Thank you for considering contributing to Cartography!
 
 ### Code of conduct
-Legal stuff: This project is governed by [Lyft's code of conduct](https://github.com/lyft/code-of-conduct).
-All contributors and participants agree to abide by its terms.
+All contributors and participants of this project must follow the  [CNCF code of conduct](https://github.com/cncf/foundation/blob/main/code-of-conduct.md).
 
 ### Bug reports and feature requests and discussions
 Submit a GitHub issue to report a bug or request a new feature. If we decide that the issue needs more discussion - usually because the scope is too large or we need to make careful decision - we will convert the issue to a [GitHub Discussion](https://github.com/lyft/cartography/discussions).
 
 ### Developing Cartography
 
-Get started with our [developer documentation](https://lyft.github.io/cartography/dev/developer-guide.html). Please feel free to submit your own PRs to update documentation if you've found a better way to explain something.
-
-#### Sign the Contributor License Agreement (CLA)
-
-We require a CLA for code contributions, so before we can accept a pull request
-we need to have a signed CLA. Please [visit our CLA service](https://oss.lyft.com/cla)
-and follow the instructions to sign the CLA.
+Get started with our [developer documentation](https://cartography-cncf.github.io/cartography/dev/developer-guide.html). Please feel free to submit your own PRs to update documentation if you've found a better way to explain something.
 
 ## Who uses Cartography?
 
 1. [Lyft](https://www.lyft.com)
 1. [Thought Machine](https://thoughtmachine.net/)
 1. [MessageBird](https://messagebird.com)
 1. [Cloudanix](https://www.cloudanix.com/)
-1. [ZeusCloud](https://www.zeuscloud.io/)
 1. [Corelight](https://www.corelight.com/)
 1. {Your company here} :-)
 
 If your organization uses Cartography, please file a PR and update this list. Say hi on Slack too!
+
+---
+
+Cartography is a [Cloud Native Computing Foundation](https://www.cncf.io/) sandbox project.<br>
+<div style="background-color: white; display: inline-block; padding: 10px;">
+  <img src="docs/root/images/cncf-color.png" alt="CNCF Logo" width="200">
+</div>