First commit to integration

Includes:
- AL-python-serverless latest
-  .github updates from prod to integration
- terraform/serverless updates from prod to integration

.github/workflows/backend.yml

@@ -5,14 +5,14 @@ on:
   push:
     branches:
       - develop
-      - production
+      - integration
     paths:
       - backend/**
       - .github/workflows/backend.yml
   pull_request:
     branches:
       - develop
-      - production
+      - integration
     paths:
       - backend/**
       - .github/workflows/backend.yml
@@ -26,51 +26,63 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
+      - id: setup-python
+        uses: actions/setup-python@v4
         with:
-          node-version: '18'
-      - name: Restore npm cache
-        uses: actions/cache@v3
+          python-version: '3.11'
+
+      - uses: actions/cache@v3
         with:
-          path: ~/.npm
-          key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }}
-          restore-keys: ${{ runner.os }}-node-
-      - name: Install dependencies
-        run: npm ci
-      - name: Lint
-        run: npm run lint
-  test:
+          path: ~/.cache/pip
+          key: pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: pip-
+
+      - run: pip install -r worker/requirements.txt
+
+      - name: Lint with pylint
+        run: pylint backend/src
+
+  test_build:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
-        with:
-          node-version: '18'
-      - name: Restore npm cache
-        uses: actions/cache@v3
-        with:
-          path: ~/.npm
-          key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }}
-          restore-keys: ${{ runner.os }}-node-
-      - name: Install dependencies
-        run: npm ci
+
       - name: Run site locally
         run: |
           cp dev.env.example .env
           docker compose up -d db backend es
           npm install -g wait-port
           wait-port -t 3000 5432 9200 9300
         working-directory: ./
+
       - name: Sync database
-        run: npm run syncdb
-        working-directory: ./backend
-      - name: Test
-        run: npm run test -- --collectCoverage --silent
+        run: make syncdb
+
       - name: Package
         run: npx sls package
         env:
           SLS_DEBUG: '*'
+
+  test_python:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python 3.10
+        uses: actions/[email protected]
+        with:
+          python-version: '3.10'
+      - name: Copy .env file
+        run: cp ../dev.env.example .env
+      - uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: pip-
+      - run: pip install -r requirements.txt
+      - run: pytest xfd_django/xfd_api/tests/
+
   test_worker:
     runs-on: ubuntu-latest
     timeout-minutes: 20
@@ -102,24 +114,7 @@ jobs:
           CROSSFEED_COMMAND_OPTIONS: '{"scanName": "test"}'
           DB_USERNAME: crossfeed
           DB_PASSWORD: password
-  test_python:
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python 3.10
-        uses: actions/[email protected]
-        with:
-          python-version: '3.10'
-      - name: Copy .env file
-        run: cp ../dev.env.example .env
-      - uses: actions/cache@v3
-        with:
-          path: ~/.cache/pip
-          key: pip-${{ hashFiles('**/requirements.txt') }}
-          restore-keys: pip-
-      - run: pip install -r worker/requirements.txt
-      - run: pytest
+
   build_worker:
     runs-on: ubuntu-latest
     timeout-minutes: 90
@@ -139,8 +134,9 @@ jobs:
       - name: Build worker container
         run: npm run build-worker
         working-directory: ./backend
+
   deploy_staging:
-    needs: [build_worker, lint, test, test_worker, test_python]
+    needs: [build_worker, lint, test_build, test_worker, test_python]
     runs-on: ubuntu-latest
     environment: staging
     concurrency: '1'
@@ -198,12 +194,12 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
-  deploy_prod:
-    needs: [build_worker, lint, test, test_python]
+  deploy_integration:
+    needs: [build_worker, lint, test_build, test_worker, test_python]
     runs-on: ubuntu-latest
-    environment: production
+    environment: integration
     concurrency: '1'
-    if: github.event_name == 'push' && github.ref == 'refs/heads/production'
+    if: github.event_name == 'push' && github.ref == 'refs/heads/integration'
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
@@ -219,29 +215,29 @@ jobs:
         run: npm ci
 
       - name: Ensure domain exists
-        run: npx sls create_domain --stage=prod
+        run: npx sls create_domain --stage=integration
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           SLS_DEBUG: '*'
 
       - name: Deploy backend
-        run: npx sls deploy --stage=prod
+        run: npx sls deploy --stage=integration
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           SLS_DEBUG: '*'
 
       - name: Deploy worker
-        run: npm run deploy-worker-prod
+        run: npm run deploy-worker-integration
         working-directory: backend
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
       - name: Run syncdb
         run: |
-          aws lambda invoke --function-name crossfeed-prod-syncdb --region us-east-1 \
+          aws lambda invoke --function-name crossfeed-integration-syncdb --region us-east-1 \
           /dev/stdout
         working-directory: backend
         env:
@@ -250,7 +246,7 @@ jobs:
 
       - name: Run syncmdl
         run: |
-          aws lambda invoke --function-name crossfeed-prod-syncmdl --region us-east-1 \
+          aws lambda invoke --function-name crossfeed-integration-syncmdl --region us-east-1 \
           /dev/stdout
         working-directory: backend
         env:

.github/workflows/frontend.yml

@@ -5,14 +5,14 @@ on:
   push:
     branches:
       - develop
-      - production
+      - integration
     paths:
       - 'frontend/**'
       - '.github/workflows/frontend.yml'
   pull_request:
     branches:
       - develop
-      - production
+      - integration
     paths:
       - 'frontend/**'
       - '.github/workflows/frontend.yml'
@@ -88,12 +88,12 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           SLS_DEBUG: '*'
 
-  deploy_prod:
+  deploy_integration:
     needs: [lint, test]
     runs-on: ubuntu-latest
-    environment: production
+    environment: integration
     concurrency: '1'
-    if: github.event_name == 'push' && github.ref == 'refs/heads/production'
+    if: github.event_name == 'push' && github.ref == 'refs/heads/integration'
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
@@ -108,11 +108,11 @@ jobs:
       - name: Install dependencies
         run: npm ci
 
-      - name: Build Production
-        run: cp prod.env .env && npm run build
+      - name: Build Integration
+        run: cp integration.env .env && npm run build
 
-      - name: Deploy Production
-        run: npx sls deploy --stage=staging
+      - name: Deploy Integration
+        run: npx sls deploy --stage=integration
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.github/workflows/infrastructure.yml

@@ -8,14 +8,14 @@ on:
   push:
     branches:
       - develop
-      - production
+      - integration
     paths:
       - infrastructure/**
       - .github/workflows/infrastructure.yml
   pull_request:
     branches:
       - develop
-      - production
+      - integration
     paths:
       - infrastructure/**
       - .github/workflows/infrastructure.yml
@@ -77,10 +77,10 @@ jobs:
       - if: ${{ always() }}
         run: rm stage.plan || true
 
-  prod:
+  integration:
     timeout-minutes: 4320
     runs-on: ubuntu-latest
-    environment: production
+    environment: integration
     concurrency: '1'
     steps:
       - uses: actions/checkout@v3
@@ -92,7 +92,7 @@ jobs:
           sudo mv terraform /usr/local/bin
 
       - name: Terraform init
-        run: terraform init -backend-config=prod.config -input=false
+        run: terraform init -backend-config=integration.config -input=false
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -104,17 +104,17 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
       - name: Terraform plan
-        run: terraform plan -var-file=prod.tfvars -out prod.plan
+        run: terraform plan -var-file=integration.tfvars -out integration.plan
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
       - name: Terraform apply
-        if: github.ref == 'refs/heads/production'
-        run: terraform apply prod.plan
+        if: github.ref == 'refs/heads/integration'
+        run: terraform apply integration.plan
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
       - if: ${{ always() }}
-        run: rm prod.plan || true
+        run: rm integration.plan || true

Makefile

@@ -0,0 +1,5 @@
+.PHONY: syncdb
+
+# Synchronize and populate the database
+syncdb:
+	docker compose exec backend python manage.py syncdb

backend/Dockerfile.python

@@ -14,4 +14,4 @@ COPY src/xfd_django .
 ENV DJANGO_SETTINGS_MODULE=xfd_django.settings
 
 # Command to run the FastAPI application
-CMD ["uvicorn", "--workers", "4", "xfd_django.asgi:app", "--host", "0.0.0.0", "--port", "8000", "--reload"]
+CMD ["uvicorn", "--workers", "4", "xfd_django.asgi:app", "--host", "0.0.0.0", "--port", "3000", "--reload"]