Merge branch 'master' into dependabot/npm_and_yarn/docs/sharp-0.32.6

.github/CODEOWNERS

@@ -0,0 +1,10 @@
+# Each line is a file pattern followed by one or more owners.
+
+# These owners will be the default owners for everything in the
+# repo. Unless a later match takes precedence, these owners will be
+# requested for review when someone opens a pull request.
+* @rapidray12 @schmelz21 @cduhn17 @aloftus23 @Matthew-Grayson @nickviola
+
+# These folks own any files in the .github directory at the root of
+# the repository and any of its subdirectories.
+/.github/ @rapidray12 @schmelz21 @cduhn17 @aloftus23 @Matthew-Grayson @nickviola

.github/workflows/backend.yml

@@ -110,7 +110,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python 3.10
-        uses: actions/setup-python@v4.2.0
+        uses: actions/setup-python@v5.0.0
         with:
           python-version: '3.10'
       - uses: actions/cache@v3

.github/workflows/codeql.yml

@@ -27,16 +27,16 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         with:
           languages: ${{ matrix.language }}
           config-file: ./.github/codeql.yml
           queries: +security-and-quality
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
+        uses: github/codeql-action/autobuild@v3
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3
         with:
           category: "/language:${{ matrix.language }}"

.github/workflows/docs.yml

@@ -42,13 +42,13 @@ jobs:
         run: npm run lint
       - name: Build
         run: npm run build
-      - name: Deploy to GitHub Pages
-        if: github.event_name == 'push' && github.ref == 'refs/heads/master'
-        uses: crazy-max/[email protected]
-        with:
-          keep_history: false
-          target_branch: gh-pages
-          build_dir: docs/public
-          fqdn: docs.crossfeed.cyber.dhs.gov
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      # - name: Deploy to GitHub Pages
+      #   if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+      #   uses: crazy-max/[email protected]
+      #   with:
+      #     keep_history: false
+      #     target_branch: gh-pages
+      #     build_dir: docs/public
+      #     fqdn: docs.crossfeed.cyber.dhs.gov
+      #   env:
+      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/frontend.yml

@@ -84,11 +84,11 @@ jobs:
         run: cp stage.env .env && npm run build
 
       - name: Deploy Staging
-        run: aws s3 sync build s3://staging.crossfeed.cyber.dhs.gov --delete
+        run: npx sls deploy --stage=staging
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          AWS_DEFAULT_REGION: us-east-1
+          SLS_DEBUG: '*'
 
   deploy_prod:
     needs: [lint, test]
@@ -115,8 +115,8 @@ jobs:
         run: cp prod.env .env && npm run build
 
       - name: Deploy Production
-        run: aws s3 sync build s3://crossfeed.cyber.dhs.gov --delete
+        run: npx sls deploy --stage=staging
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          AWS_DEFAULT_REGION: us-east-1
+          SLS_DEBUG: '*'

.github/workflows/security.yml

@@ -78,7 +78,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python 3.10
-        uses: actions/setup-python@v4.2.0
+        uses: actions/setup-python@v5.0.0
         with:
           python-version: '3.10'
       - uses: actions/cache@v3

backend/.snyk

@@ -5,9 +5,9 @@ version: v1.22.1
 # ignores vulnerabilities until expiry date; change duration by modifying expiry date
 
 ignore: 
-    # ignore scrapy 2.9.0 for 6 months.
+    # ignore scrapy 2.x.x for 6 months.
     SNYK-PYTHON-SCRAPY-40690: 
         - '*':
-            reason: No fix available for version 2.9.x
-            expires: 2023-11-01T00:00:00.000Z
+            reason: No fix available up to version 2.11.0
+            expires: 2024-06-01T00:00:00.000Z
 patch: {}

backend/Dockerfile.pe

@@ -9,6 +9,8 @@ COPY src ./src
 
 RUN apt update && apt install git zlib1g-dev
 
+RUN apt-get update && apt-get install -y jq
+
 RUN wget -c https://www.python.org/ftp/python/3.10.11/Python-3.10.11.tar.xz && tar -Jxvf Python-3.10.11.tar.xz
 RUN cd Python-3.10.11 && ./configure && make -j4 && make altinstall
 RUN update-alternatives --install /usr/bin/python python /usr/local/bin/python3.10 1
@@ -17,8 +19,25 @@ RUN pip3.10 install --upgrade pip
 
 RUN apt remove dav1d && apt autoclean && apt autoremove
 
+# Install AWS CLI
+RUN curl --insecure "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+RUN unzip awscliv2.zip
+RUN ./aws/install
+
 # Install pe-source module
 # Sync the latest from cf-staging branch
-RUN git clone -b cf-source-staging https://github.com/cisagov/pe-reports.git && cd pe-reports && git checkout c9cbbd73b22ef38cabe1da6ba50aeb2dc0be4f99 && pip install .
+RUN git clone -b AL-staging-SQS https://github.com/cisagov/pe-reports.git && \
+    cd pe-reports && \
+    git checkout 6dcd017551ba14022e110c073e1bdbc804c795f8 && \
+    pip install .
+
+RUN python -m spacy download en_core_web_lg
+
+# Create database.ini 
+RUN echo "[database]" > database.ini \ 
+    && echo "user=$(cat db_user.txt)" >> database.ini \ 
+    && echo "password=$(cat db_password.txt)" >> database.ini
+
+COPY worker worker
 
-CMD ["./worker/pe-worker-entry.sh"]
+CMD ["./worker/generate_config.sh", "./worker/pe-worker-entry.sh"]

backend/env.yml

@@ -46,6 +46,7 @@ staging:
   PE_CLUSTER_NAME: pe-staging-worker
   SHODAN_QUEUE_URL: ${ssm:/crossfeed/staging/SHODAN_QUEUE_URL}
   SHODAN_SERVICE_NAME: pe-staging-shodan
+  EMAIL_BUCKET_NAME: cisa-crossfeed-staging-html-email
 
 prod:
   DB_DIALECT: 'postgres'
@@ -86,6 +87,7 @@ prod:
   PE_CLUSTER_NAME: pe-prod-worker
   SHODAN_QUEUE_URL: ${ssm:/crossfeed/prod/SHODAN_QUEUE_URL}
   SHODAN_SERVICE_NAME: pe-prod-shodan
+  EMAIL_BUCKET_NAME: cisa-crossfeed-staging-html-email
 
 dev-vpc:
   securityGroupIds:

backend/jest.config.js

@@ -23,6 +23,9 @@ module.exports = {
     '/node_modules/',
     '.*report.*' // Remove this when we enable report / vulnerability functionality
   ],
+  moduleNameMapper: {
+    '^axios$': require.resolve('axios')
+  },
   coverageThreshold: {
     global: {
       branches: 50