Update DEV setup for CloudQuery

.devcontainer/devcontainer.json

@@ -19,7 +19,8 @@
       "version": "1.1.9",
       "tflint": "latest",
       "terragrunt": "0.36.7"
-    }
+    },
+    "ghcr.io/devcontainers/features/python:1": {}
   },
   "customizations": {
     "vscode": {

.gitignore

@@ -12,4 +12,18 @@ __pycache__
 
 connector.zip
 
-.cq
+.cq
+
+/tools/logs-analyzer/connection_logs.txt
+/tools/logs-analyzer/cloudquery.log
+/tools/logs-analyzer/cloudquery/*
+
+# Python
+*.pyc
+*.pyo
+*.pyd
+__pycache__/
+*.py[cod]
+*$py.class
+
+**/env/*

Makefile

@@ -1,4 +1,4 @@
-.PHONY: fmt fmt-ci install install-dev lint lint-ci test build-cq run-cloud-query
+.PHONY: fmt fmt-ci install install-dev lint lint-ci test build up shell down
 
 fmt:
 	black .
@@ -20,14 +20,36 @@ lint-ci: lint
 test:
 	pytest -s -vv .
 
-build-cq: 
-	docker build --build-arg CONFIG_FILE=config.yml -f images/cloud_asset_inventory/cloudquery/Dockerfile -t cq images/cloud_asset_inventory/cloudquery
+build:
+	docker-compose -f images/cloud_asset_inventory/dev/docker-compose.yml build
 
-build-dev-cq:
-	docker build --build-arg CONFIG_FILE=config.yml -f images/cloud_asset_inventory/cloudquery/dev/Dockerfile -t cq-dev images/cloud_asset_inventory/cloudquery/dev
+build-prod: # Local build of prod image
+	docker build \
+	-t cloud_asset_inventory:latest \
+	images/cloud_asset_inventory/cloudquery
 
-run-dev-cloud-query:
-	docker-compose -f images/cloud_asset_inventory/cloudquery/dev/docker-compose.yml up -d app
+start:
+	export AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID); \
+	export AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY); \
+	export AWS_SESSION_TOKEN=$(AWS_SESSION_TOKEN); \
+	docker-compose -f images/cloud_asset_inventory/dev/docker-compose.yml up -d
 
-stop-dev-cloud-query:
-	docker-compose -f images/cloud_asset_inventory/cloudquery/dev/docker-compose.yml down
+start-logging:
+	docker-compose -f images/cloud_asset_inventory/dev/docker-compose.yml exec -d app /usr/local/bin/log_connections.sh
+
+attach:
+	docker-compose -f images/cloud_asset_inventory/dev/docker-compose.yml exec app bash
+
+stop:
+	docker-compose -f images/cloud_asset_inventory/dev/docker-compose.yml down
+
+copy-logs:
+	-docker cp $(shell docker-compose -f images/cloud_asset_inventory/dev/docker-compose.yml ps -q app):/var/log/connection_logs.txt ./tools/logs-analyzer/connection_logs.txt
+	-docker cp $(shell docker-compose -f images/cloud_asset_inventory/dev/docker-compose.yml ps -q app):/var/log/cloudquery.log ./tools/logs-analyzer/cloudquery.log
+
+copy-data:
+	mkdir -p ./tools/logs-analyzer/cloudquery/data
+	docker cp $(shell docker-compose -f images/cloud_asset_inventory/dev/docker-compose.yml ps -q app):/cloudquery/data/. ./tools/logs-analyzer/cloudquery/data/
+
+delete-logs:
+	docker-compose -f images/cloud_asset_inventory/dev/docker-compose.yml exec app rm /var/log/connection_logs.txt

README.md

@@ -9,20 +9,25 @@ This repository will contain various tools used by CDS to ensure the confidentia
 
 - Cloud Asset Inventory: AWS, ECS, [CloudQuery](https://www.cloudquery.io/docs)
 - Content Security Policy (CSP) violation reporting: AWS, Lambda
-   - Onboard by adding the `report-uri https://csp-report-to.security.cdssandbox.xyz/report;` directive to your apps existing CSP
+  - Onboard by adding the `report-uri https://csp-report-to.security.cdssandbox.xyz/report;` directive to your apps existing CSP
 
 ## License
 
 This code is released under the MIT License. See [LICENSE](LICENSE).
 
 ## Maintenance
 
-### Upgrading CloudQuery
+_Note: If you are using Codespaces, all the environment setup is done for you. You can start working on the Cloud Asset Inventory right away._
 
-1. Update the `cloudquery` image tag in `Dockerfile` to the latest version (path: /workspace/images/cloud_asset_inventory/cloudquery/Dockerfile)
-2. In VS Code, run the devcontainer to build the new image and start the container
-3. Assume the PlatformSecurity role in the AWS account you want to scan and export the credentials to your environment
-4. Export the CQ_S3_BUCKET variable to your environment with the name cloudquery-794722365809-test as the value (ex: `export CQ_S3_BUCKET=cloudquery-794722365809-test`)
-4. Run `make build-cq` to build the CloudQuery binary
-5. Run `make run-cloud-query` to run the CloudQuery container
+See the Makefile for the available commands.
 
+Main commands:
+
+- **build**: Build the Docker image for the Cloud Asset Inventory in Codespaces using docker-compose (sets up the environment for network analysis and other resources)
+- **start**: Start the Docker container for the Cloud Asset Inventory in Codespaces
+- **start-logging**: Start the Docker container for the Cloud Asset Inventory in Codespaces with network logging enabled (using tcpdump)
+- **attach**: Attach a shell to the running Docker container for the Cloud Asset Inventory in Codespaces
+- **stop**: Stop the Docker container for the Cloud Asset Inventory in Codespaces
+- **copy-logs**: Copy the network logs and the Cloud Asset Inventory logs from the Docker container to the host machine
+- **copy-data**: Copy the data generated by the Cloud Asset Inventory from the Docker container to the host machine
+- **delete-logs**: Delete the network logs from the Docker container

images/cloud_asset_inventory/cloudquery/dependencies.txt

@@ -0,0 +1 @@
+curl

images/cloud_asset_inventory/cloudquery/get_cloudquery.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -euo pipefail
+
+TOOL=$1
+DISTRIBUTION=$2
+
+# Download the tool
+curl -L https://github.com/cloudquery/cloudquery/releases/download/$TOOL/$DISTRIBUTION -o cloudquery
+
+# Download checksums
+curl -L https://github.com/cloudquery/cloudquery/releases/download/$TOOL/checksums.txt -o checksums.txt
+
+# Extract the checksum for the package
+while read -r checksum file; do
+  if [[ $file == $DISTRIBUTION ]]; then
+    EXPECTED_CHECKSUM=$checksum
+    break
+  fi
+done < checksums.txt
+
+# Print the expected checksum
+echo "Expected checksum: $EXPECTED_CHECKSUM"
+
+# Calculate the actual checksum of the cloudquery file
+ACTUAL_CHECKSUM=$(sha256sum cloudquery | awk '{print $1}')
+
+# Print the actual checksum
+echo "Actual checksum: $ACTUAL_CHECKSUM"
+
+# Check if the checksums match
+if [ "$EXPECTED_CHECKSUM" != "$ACTUAL_CHECKSUM" ]; then
+  echo "Checksums do not match!"
+  exit 1
+else
+  echo "Checksums match!"
+fi
+
+rm checksums.txt

images/cloud_asset_inventory/dev/Dockerfile

@@ -0,0 +1,38 @@
+FROM debian:bullseye@sha256:ad47d883924a75f704037e49c71c20ef209fb3ba05be7070322cb45d2e089261
+
+# Set the working directory
+WORKDIR /app
+
+# Copy dependencies file
+COPY ./dev/dependencies.txt .
+
+# Install dependencies
+RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
+  xargs -a ./dependencies.txt apt-get install -y && \
+  rm -rf /var/lib/apt/lists/* && \
+  apt-get clean && \
+  rm dependencies.txt
+
+# Set the version
+ARG TOOL=cli-v5.13.1
+ARG DISTRIBUTION=cloudquery_linux_amd64
+
+# Download the CloudQuery CLI
+COPY ./cloudquery/get_cloudquery.sh .
+RUN ./get_cloudquery.sh ${TOOL} ${DISTRIBUTION} && \
+  mv cloudquery /usr/local/bin/cloudquery && \
+  chmod +x /usr/local/bin/cloudquery && \
+  rm get_cloudquery.sh
+
+# Accept the API key as a build argument
+ARG CLOUDQUERY_API_KEY
+ENV CLOUDQUERY_API_KEY=${CLOUDQUERY_API_KEY}
+
+# Copy the config file and install the plugins (requires the API key)
+ARG CONFIG_FILE=./dev/config.yml
+COPY ${CONFIG_FILE} ./config.yml
+RUN cloudquery plugin install config.yml
+
+# Copy the log_connections.sh file and make it executable
+COPY ./dev/log_connections.sh /usr/local/bin/log_connections.sh
+RUN chmod +x /usr/local/bin/log_connections.sh

images/cloud_asset_inventory/dev/config.yml

@@ -0,0 +1,49 @@
+kind: source
+spec:
+  name: aws
+  path: cloudquery/aws
+  version: "v25.5.0"
+  tables: ["*"]
+  skip_tables:
+    - aws_ec2_vpc_endpoint_services # this resource includes services that are available from AWS as well as other AWS Accounts
+    - aws_cloudtrail_events
+    - aws_docdb_cluster_parameter_groups
+    - aws_docdb_engine_versions
+    - aws_ec2_instance_types
+    - aws_elasticache_engine_versions
+    - aws_elasticache_parameter_groups
+    - aws_elasticache_reserved_cache_nodes_offerings
+    - aws_elasticache_service_updates
+    - aws_iam_group_last_accessed_details
+    - aws_iam_policy_last_accessed_details
+    - aws_iam_role_last_accessed_details
+    - aws_iam_user_last_accessed_details
+    - aws_lambda_functions
+    - aws_neptune_cluster_parameter_groups
+    - aws_neptune_db_parameter_groups
+    - aws_rds_cluster_parameter_groups
+    - aws_rds_db_parameter_groups
+    - aws_rds_engine_versions
+    - aws_servicequotas_services
+  destinations:
+    - local
+  spec:
+    regions:
+      - ca-central-1
+      - us-east-1
+      - us-west-2
+    org:
+      member_role_name: secopsAssetInventorySecurityAuditRole
+      admin_account:
+        role_arn: "arn:aws:iam::794722365809:role/secopsAssetInventoryCloudqueryRole"
+---
+kind: destination
+spec:
+  name: "local"
+  path: "cloudquery/file"
+  registry: "cloudquery"
+  version: "v4.0.6"
+  write_mode: "append"
+  spec:
+    path: "cloudquery/data/{{TABLE}}/{{UUID}}.{{FORMAT}}"
+    format: "parquet"

images/cloud_asset_inventory/dev/dependencies.txt

@@ -0,0 +1,2 @@
+tcpdump
+curl

images/cloud_asset_inventory/dev/docker-compose.yml

@@ -0,0 +1,19 @@
+services:
+  app:
+    build:
+      context: ../
+      dockerfile: dev/Dockerfile
+      args:
+        CLOUDQUERY_API_KEY: ${CLOUDQUERY_API_KEY}
+    volumes:
+      - ../..:/workspaces:cached
+      - logs:/var/log
+    command: sleep infinity
+    environment:
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}
+    cap_add:
+      - NET_ADMIN
+volumes:
+  logs:

images/cloud_asset_inventory/dev/log_connections.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+tcpdump -nn -i any dst port not 22 and dst net not 127.0.0.0/8 >> /var/log/connection_logs.txt 2>&1 

tools/itsg33-issue-generator/script.py

@@ -23,6 +23,7 @@
 CSV_FILE = "controls.csv"
 LOG_LEVEL = os.getenv("LOG_LEVEL", logging.INFO)
 
+
 class Header(Enum):
     FAMILY = 0
     CONTROL_ID = 1

tools/itsg33-issue-generator/tests/test_script.py

@@ -211,7 +211,7 @@ def test_get_labels(mock_get_suggested_assignment):
         "Priority: P1",
         "Class: Operational",
         "Suggested Assignment: IT Security",
-        "ITSG-33"
+        "ITSG-33",
     ]
 
 
@@ -327,7 +327,11 @@ def test_get_title_with_enhancement():
         "",
         "",
     ]
-    assert script.get_title(row) == "CM-11-2: User-installed Software | Prohibit Installation Without Privileged Status"
+    assert (
+        script.get_title(row)
+        == "CM-11-2: User-installed Software | Prohibit Installation Without Privileged Status"
+    )
+
 
 def test_get_title_with_enhancement_100():
     row = [