Merge pull request #153 from NASA-PDS/software-issues-repo#55

Secrets detection

.github/workflows/secrets-detection.yaml

@@ -0,0 +1,76 @@
+name: Secret Detection Workflow
+on:
+    push:
+        branches:
+            - main
+    pull_request:
+        branches:
+            - main
+
+jobs:
+    secret-detection:
+        runs-on: ubuntu-latest
+        steps:
+            -
+                name: Checkout code
+                uses: actions/checkout@v4
+            -
+                name: Install necessary packages
+                run: |
+                    pip install git+https://github.com/NASA-AMMOS/slim-detect-secrets.git@exp
+                    pip install jq
+
+            -
+                name: Create an initial .secrets.baseline if .secrets.baseline does not exist
+                run: |
+                    if [ ! -f .secrets.baseline ]; then
+                        # This generated baseline file will only be temporarily available on the GitHub side and will not appear in the user's local files.
+                        # Scanning an empty folder to generate an initial .secrets.baseline without secrets in the results.
+                        echo "⚠️ No existing .secrets.baseline file detected. Creating a new blank baseline file."
+                        mkdir empty-dir
+                        detect-secrets scan empty-dir > .secrets.baseline
+                        echo "✅ Blank .secrets.baseline file created successfully."
+                        rm -r empty-dir
+                    else
+                        echo "✅ Existing .secrets.baseline file detected. No new baseline file will be created."
+                    fi
+
+            -
+                name: Scan repository for secrets
+                run: |
+                    # scripts to scan repository for new secrets
+
+                    # backup the list of known secrets
+                    cp .secrets.baseline .secrets.new
+
+                    # find the secrets in the repository
+                    detect-secrets scan --disable-plugin AbsolutePathDetectorExperimental --baseline .secrets.new \
+                        --exclude-files '\.secrets..*' \
+                        --exclude-files '\.git.*' \
+                        --exclude-files '\.mypy_cache' \
+                        --exclude-files '\.pytest_cache' \
+                        --exclude-files '\.tox' \
+                        --exclude-files '\.venv' \
+                        --exclude-files 'venv' \
+                        --exclude-files 'dist' \
+                        --exclude-files 'build' \
+                        --exclude-files '.*\.egg-info'
+
+                    # if there is any difference between the known and newly detected secrets, break the build
+                    # Function to compare secrets without listing them
+                    compare_secrets() { diff <(jq -r '.results | keys[] as $key | "\($key),\(.[$key] | .[] | .hashed_secret)"' "$1" | sort) <(jq -r '.results | keys[] as $key | "\($key),\(.[$key] | .[] | .hashed_secret)"' "$2" | sort) >/dev/null; }
+
+                    # Check if there's any difference between the known and newly detected secrets
+                    if ! compare_secrets .secrets.baseline .secrets.new; then
+                        echo "⚠️ Attention Required! ⚠️" >&2
+                        echo "New secrets have been detected in your recent commit. Due to security concerns, we cannot display detailed information here and we cannot proceed until this issue is resolved." >&2
+                        echo "" >&2
+                        echo "Please follow the steps below on your local machine to reveal and handle the secrets:" >&2
+                        echo "" >&2
+                        echo "1️⃣ Run the 'detect-secrets' tool on your local machine. This tool will identify and clean up the secrets. You can find detailed instructions at this link: https://nasa-ammos.github.io/slim/continuous-testing/starter-kits/#detect-secrets" >&2
+                        echo "" >&2
+                        echo "2️⃣ After cleaning up the secrets, commit your changes and re-push your update to the repository." >&2
+                        echo "" >&2
+                        echo "Your efforts to maintain the security of our codebase are greatly appreciated!" >&2
+                        exit 1
+                    fi

.pre-commit-config.yaml

@@ -49,3 +49,23 @@ repos:
         language: system
         stages: [push]
         pass_filenames: false
+
+-   repo: https://github.com/NASA-AMMOS/slim-detect-secrets
+    # using commit id for now, will change to tag when official version is released
+    rev: 91e097ad4559ae6ab785c883dc5ed989202c7fbe
+    hooks:
+      - id: detect-secrets
+        args:
+          - '--baseline'
+          - '.secrets.baseline'
+          - --exclude-files '\.secrets..*'
+          - --exclude-files '\.pre-commit-config.yaml'
+          - --exclude-files '\.git.*'
+          - --exclude-files '\.mypy_cache'
+          - --exclude-files '\.pytest_cache'
+          - --exclude-files '\.tox'
+          - --exclude-files '\.venv'
+          - --exclude-files 'venv'
+          - --exclude-files 'dist'
+          - --exclude-files 'build'
+          - --exclude-files '.*\.egg-info'

.secrets.baseline

@@ -0,0 +1,208 @@
+{
+  "version": "1.4.0",
+  "plugins_used": [
+    {
+      "name": "ArtifactoryDetector"
+    },
+    {
+      "name": "AWSKeyDetector"
+    },
+    {
+      "name": "AWSSensitiveInfoDetectorExperimental"
+    },
+    {
+      "name": "AzureStorageKeyDetector"
+    },
+    {
+      "name": "Base64HighEntropyString",
+      "limit": 4.5
+    },
+    {
+      "name": "BasicAuthDetector"
+    },
+    {
+      "name": "CloudantDetector"
+    },
+    {
+      "name": "DiscordBotTokenDetector"
+    },
+    {
+      "name": "EmailAddressDetector"
+    },
+    {
+      "name": "GitHubTokenDetector"
+    },
+    {
+      "name": "HexHighEntropyString",
+      "limit": 3.0
+    },
+    {
+      "name": "IbmCloudIamDetector"
+    },
+    {
+      "name": "IbmCosHmacDetector"
+    },
+    {
+      "name": "IPPublicDetector"
+    },
+    {
+      "name": "JwtTokenDetector"
+    },
+    {
+      "name": "KeywordDetector",
+      "keyword_exclude": ""
+    },
+    {
+      "name": "MailchimpDetector"
+    },
+    {
+      "name": "NpmDetector"
+    },
+    {
+      "name": "PrivateKeyDetector"
+    },
+    {
+      "name": "SendGridDetector"
+    },
+    {
+      "name": "SlackDetector"
+    },
+    {
+      "name": "SoftlayerDetector"
+    },
+    {
+      "name": "SquareOAuthDetector"
+    },
+    {
+      "name": "StripeDetector"
+    },
+    {
+      "name": "TwilioKeyDetector"
+    }
+  ],
+  "filters_used": [
+    {
+      "path": "detect_secrets.filters.allowlist.is_line_allowlisted"
+    },
+    {
+      "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
+      "min_level": 2
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_indirect_reference"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_likely_id_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_lock_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_potential_uuid"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_sequential_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_swagger_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_templated_secret"
+    },
+    {
+      "path": "detect_secrets.filters.regex.should_exclude_file",
+      "pattern": [
+        "\\.secrets..*",
+        "\\.pre-commit-config.yaml",
+        "\\.git.*",
+        "\\.mypy_cache",
+        "\\.pytest_cache",
+        "\\.tox",
+        "\\.venv",
+        "venv",
+        "dist",
+        "build",
+        ".*\\.egg-info"
+      ]
+    }
+  ],
+  "results": {
+    "setup.cfg": [
+      {
+        "type": "Email Address",
+        "filename": "setup.cfg",
+        "hashed_secret": "3a6d7aa49a8e4a2fe32a5cd0e53da9cb96bd8d29",
+        "is_verified": false,
+        "line_number": 33
+      }
+    ],
+    "src/pds2/aipgen/tests/test_utils.py": [
+      {
+        "type": "Hex High Entropy String",
+        "filename": "src/pds2/aipgen/tests/test_utils.py",
+        "hashed_secret": "10a34637ad661d98ba3344717656fcc76209c2f8",
+        "is_verified": false,
+        "line_number": 48
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "src/pds2/aipgen/tests/test_utils.py",
+        "hashed_secret": "67a74306b06d0c01624fe0d0249a570f4d093747",
+        "is_verified": false,
+        "line_number": 49
+      }
+    ],
+    "test/data/insight_documents/urn-nasa-pds-insight_documents/document_hp3rad/release_notes.txt": [
+      {
+        "type": "Email Address",
+        "filename": "test/data/insight_documents/urn-nasa-pds-insight_documents/document_hp3rad/release_notes.txt",
+        "hashed_secret": "af6e7d5c95484d3b7d88a581b4d19912149baff0",
+        "is_verified": false,
+        "line_number": 15
+      }
+    ],
+    "test/data/insight_documents/urn-nasa-pds-insight_documents/document_ida/ida_release_notes.txt": [
+      {
+        "type": "Email Address",
+        "filename": "test/data/insight_documents/urn-nasa-pds-insight_documents/document_ida/ida_release_notes.txt",
+        "hashed_secret": "af6e7d5c95484d3b7d88a581b4d19912149baff0",
+        "is_verified": false,
+        "line_number": 15
+      }
+    ],
+    "test/data/insight_documents/urn-nasa-pds-insight_documents/document_rise/release_notes.txt": [
+      {
+        "type": "Email Address",
+        "filename": "test/data/insight_documents/urn-nasa-pds-insight_documents/document_rise/release_notes.txt",
+        "hashed_secret": "af6e7d5c95484d3b7d88a581b4d19912149baff0",
+        "is_verified": false,
+        "line_number": 15
+      }
+    ],
+    "test/data/insight_documents/urn-nasa-pds-insight_documents/document_seis/seis_release_notes.txt": [
+      {
+        "type": "Email Address",
+        "filename": "test/data/insight_documents/urn-nasa-pds-insight_documents/document_seis/seis_release_notes.txt",
+        "hashed_secret": "af6e7d5c95484d3b7d88a581b4d19912149baff0",
+        "is_verified": false,
+        "line_number": 15
+      }
+    ],
+    "test/data/insight_documents/urn-nasa-pds-insight_documents/readme.txt": [
+      {
+        "type": "Email Address",
+        "filename": "test/data/insight_documents/urn-nasa-pds-insight_documents/readme.txt",
+        "hashed_secret": "af6e7d5c95484d3b7d88a581b4d19912149baff0",
+        "is_verified": false,
+        "line_number": 32
+      }
+    ]
+  },
+  "generated_at": "2023-11-16T17:14:34Z"
+}