Merge pull request #14 from NASA-AMMOS/develop

Release Update of Python Starter Kit with Scanning

.github/PULL_REQUEST_TEMPLATE.md

@@ -10,4 +10,4 @@
 ## Testing
 - Provide some proof you've tested your changes 
 - Example: test results available at ...
-- Example: tested on operating system ...
+- Example: tested on operating system ...

.github/workflows/codeql.yml

@@ -0,0 +1,100 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# For more information see: 
+#     https://nasa.github.io/scrub/
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [main, develop]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [develop]
+  schedule:
+    # default branch on sundays at 5a
+    - cron: '0 5 * * 0'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # CodeQL supports ['cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby']
+        # Learn more about CodeQL language support at https://git.io/codeql-language-support
+        language: ['python']
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        #config-file: ./.github/workflows/codeql/codeql-config.yml
+        languages: ${{ matrix.language }}
+        queries: security-and-quality, security-extended
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+
+    - name: Post-Process Output
+      run: |
+        python3 -m pip install nasa-scrub
+
+        results_dir=`realpath ${{ github.workspace }}/../results`
+        sarif_files=`find $results_dir -name '*.sarif'`
+
+        for sarif_file in $sarif_files
+        do
+          output_file="$results_dir/$(basename $sarif_file .sarif).scrub"
+
+          python3 -m scrub.tools.parsers.translate_results $sarif_file $output_file ${{ github.workspace }} scrub
+        done
+
+        python3 -m scrub.tools.parsers.csv_parser $results_dir
+
+        echo "RESULTS_DIR=$results_dir" >> $GITHUB_ENV
+        
+      
+    - name: Upload CodeQL Artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: codeql-artifacts
+        path: ${{ env.RESULTS_DIR }}
+        if-no-files-found: error
+        overwrite: true
+        retention-days: 15

.github/workflows/codeql/codeql-config.yml

@@ -0,0 +1,5 @@
+name: "CodeQL config"
+queries: 
+  - name: Run custom queries
+  - uses: security-extended
+  - uses: security-and-quality

.github/workflows/pylint.yml

@@ -0,0 +1,71 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to disable certain Pylint checks in the 
+# "Analyze" configuration block below. 
+#
+# For more information see: 
+#     https://nasa-ammos.github.io/slim/docs/guides/software-lifecycle/application-starter-kits/python-starter-kit/
+#
+# ******** NOTE ********
+# Pylint is a Python-based linter that works to evaluate Python code.
+#
+name: "Pylint"
+
+on:
+  push:
+    branches: [main, develop]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [develop]
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+      contents: read
+      security-events: write
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.10'
+    - name: Upgrade tooling
+      run: |
+        python3 -m pip install --upgrade pip
+        pip3 install --upgrade build importlib_metadata setuptools setuptools_scm wheel
+        pip3 install pylint
+    - name: Install dependencies
+      run: |
+        pip3 install -r requirements.txt
+        pip3 install -e .
+    - name: Prepare PYTHONPATH
+      run: |
+        src_paths=`find ${PWD} -type f -maxdepth 3 -mindepth 2 -name "*.py" -exec dirname {} + | uniq`
+        pythonpathplus=""
+        for p in $src_paths
+        do
+            pythonpathplus="${pythonpathplus:+:${pythonpathplus}}:$p"
+        done
+        echo "PYTHONPATH=${PYTHONPATH:+:${PYTHONPATH}}${pythonpathplus}:." >> $GITHUB_ENV
+    - name: Analyze
+      run: |
+        # disable docstring checks
+        # See https://pylint.readthedocs.io/en/latest/user_guide/messages/messages_overview.html
+        # pylint --disable=C0114,C0115,C0116 --recursive=y --output=pylint_report.txt --exit-zero . 
+        pylint --recursive=y --output=pylint_report.txt --exit-zero . 
+      continue-on-error: true
+    - name: Upload Artifact
+      uses: actions/upload-artifact@v4
+      with:
+        name: pylint_report
+        path: pylint_report.txt
+        if-no-files-found: error
+        overwrite: true
+        retention-days: 15

.github/workflows/python-publish.yml

@@ -1,26 +1,37 @@
-# This workflows will upload a Python Package using Twine when a release is created
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
 # For more information see: 
-#     https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
-
-name: Upload Python Package
+#     https://nasa-ammos.github.io/slim/docs/guides/software-lifecycle/application-starter-kits/python-starter-kit/
+#
+# ******** NOTE ********
+# This file publishes to TestPyPi. To enable public PyPi the repository flag 
+# must be removed from the Twine upload call in the "Publish package" block.
+#
+name: "Upload Python Package"
 
 on:
   release:
-    types: [published]
     branches: [main]
+    types: [published]
 
 jobs:
   deploy:
+    name: Deploy
     runs-on: ubuntu-latest
+    permissions:
+      actions: write
+      contents: read
+      security-events: write
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v3
+    - name: Checkout repository
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.9'
+        python-version: '3.10'
     - name: Upgrade tooling
       run: |
         python3 -m pip install --upgrade pip

.github/workflows/secrets-detection.yml

@@ -0,0 +1,97 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to exclude files from analyses. (See "Scan" 
+# configuration block below.)
+#
+# For more information, see:
+#     https://nasa-ammos.github.io/slim/docs/guides/software-lifecycle/security/secrets-detection
+#
+# ******** NOTE ********
+# Detect Secrets will compare known values from the ".secrets.baseline" file 
+# located in the root of the repository. Should any false detections occur, 
+# this file should be committed locally with an exception added to .gitignore 
+# to prevent inadvertent modification or overwrite. 
+#
+name: "Secret Detection"
+on:
+  push:
+    branches: [main, develop]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [develop]
+
+jobs:
+  secret-detection:
+    name: Secret-Detection
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+      contents: read
+      security-events: write
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Upgrade tooling
+      run: |
+        python3 -m pip install --upgrade pip
+        pip install --upgrade git+https://github.com/NASA-AMMOS/slim-detect-secrets.git@exp
+        pip install --upgrade jq
+    - name: Create baseline config
+      run: |
+        if [ ! -f .secrets.baseline ] ; 
+        then
+            # This generated baseline file will only be temporarily available on the GitHub side and will not appear in the user's local files.
+            # Scanning an empty folder to generate an initial .secrets.baseline without secrets in the results.
+            echo "⚠️ No existing .secrets.baseline file detected. Creating a new blank baseline file."
+            mkdir empty-dir
+            detect-secrets scan empty-dir > .secrets.baseline
+            echo "✅ Blank .secrets.baseline file created successfully."
+            rm -r empty-dir
+        else
+            echo "✅ Existing .secrets.baseline file detected. No new baseline file will be created."
+        fi
+    - name: Scan
+      run: |
+        # scripts scan repository for new secrets
+        # backup list of known secrets
+        cp -pr .secrets.baseline .secrets.new
+        # find secrets in the repository
+        detect-secrets scan --disable-plugin AbsolutePathDetectorExperimental --baseline .secrets.new \
+            --exclude-files '\.secrets..*' \
+            --exclude-files '\.git.*' \
+            --exclude-files '\.mypy_cache' \
+            --exclude-files '\.pytest_cache' \
+            --exclude-files '\.tox' \
+            --exclude-files '\.venv' \
+            --exclude-files 'venv' \
+            --exclude-files 'dist' \
+            --exclude-files 'build' \
+            --exclude-files '.*\.egg-info'
+        # break build when new secrets discovered
+        # function compares baseline/new secrets w/o listing results -- success(0) when new secret found
+        compare_secrets() { diff <(jq -r '.results | keys[] as $key | "\($key),\(.[$key] | .[] | .hashed_secret)"' "${1}" | sort) <(jq -r '.results | keys[] as $key | "\($key),\(.[$key] | .[] | .hashed_secret)"' "${2}" | sort) | grep -q '>' ; }
+        # test baseline versus new secret files
+        if compare_secrets .secrets.baseline .secrets.new; 
+        then
+            echo "⚠️ Attention Required! ⚠️" >&2
+            echo "New secrets have been detected in your recent commit. Due to security concerns, we cannot display detailed information here and we cannot proceed until this issue is resolved." >&2
+            echo "" >&2
+            echo "Please follow the steps below on your local machine to reveal and handle the secrets:" >&2
+            echo "" >&2
+            echo "1️⃣ Run the 'detect-secrets' tool on your local machine. This tool will identify and clean up the secrets. You can find detailed instructions at this link: https://nasa-ammos.github.io/slim/continuous-testing/starter-kits/#detect-secrets" >&2
+            echo "" >&2
+            echo "2️⃣ After cleaning up the secrets, commit your changes and re-push your update to the repository." >&2
+            echo "" >&2
+            echo "Your efforts to maintain the security of our codebase are greatly appreciated!" >&2
+            exit 1
+        else
+            echo "🟢 Secrets tests PASSED! 🟢" >&1
+            echo "No new secrets were detected in comparison to any baseline configurations."  >&1
+            exit 0
+        fi 

.gitignore

@@ -51,6 +51,7 @@ coverage.xml
 *.py,cover
 .hypothesis/
 .pytest_cache/
+.secrets*
 
 # Translations
 *.mo

.secrets.baseline

@@ -0,0 +1,15 @@
+{
+  "version": "1.4.0",
+  "results": {
+    "setup.cfg": [
+      {
+        "type": "Email Address",
+        "filename": "setup.cfg",
+        "hashed_secret": "3d5f1cb1412e27257b118b0fbf9dcccf390be6cb",
+        "is_verified": false,
+        "line_number": 31
+      }
+    ]
+  },
+  "generated_at": "2024-03-14T20:03:34Z"
+}