Adds integration page + tests (#12)

* fixed Makefile targets:
* api key works; needs a refactor
* added integration
* added testing framework
* addest tests
* added ci/cd workflows
* api key now compulsory when creating a user

.github/pull_request_template.md

@@ -1,5 +1,4 @@
 Reviewer:
-
 Estimate:
 
 ---
@@ -8,21 +7,21 @@ Estimate:
 
 Fixes: JIRA_TICKET_LINK
 
+## Description
+
 ### Goal
 
 ### Changes
 
-## How has this been tested?
+### Future Tasks (optional)
 
-## To-do before merge
-
-(Optional -- remove this section if not needed)
+## How has this been tested?
 
-- [ ] Ensure PR #56 is merged
+## To-do before merge (optional)
 
 ## Checklist
 
-Fill with `x` for completed. Delete any lines that are not relevant
+Fill with `x` for completed.
 
 - [ ] My code follows the style guidelines of this project
 - [ ] I have reviewed my own code to ensure good quality
@@ -32,3 +31,6 @@ Fill with `x` for completed. Delete any lines that are not relevant
 - [ ] I have updated the requirements (if applicable)
 - [ ] I have updated the README file (if applicable)
 - [ ] I have updated affected documentation (if applicable)
+- [ ] I have added a blogpost in Latest Updates
+- [ ] I have updated the CI/CD scripts in `.github/workflows/`
+- [ ] I have updated the Terraform code

.github/workflows/linting.yaml

@@ -0,0 +1,42 @@
+name: Python Linting
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - "**"
+    paths:
+      - "**.py"
+
+jobs:
+  python-lint-check:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.12"
+          cache: "pip"
+          cache-dependency-path: |
+            backend/requirements.txt
+            requirements-dev.txt
+      - run: |
+          python -m pip install --upgrade pip
+          pip install -r backend/requirements.txt
+          pip install -r requirements-dev.txt
+
+      - name: Run MyPy
+        run: |
+          mypy backend/app
+
+      - name: Run ruff
+        run: |
+          ruff check --exclude "backend/migrations" .
+
+      - name: Check code formatting with Black
+        run: |
+          cd backend
+          black --check .

.github/workflows/unit_tests.yaml

@@ -0,0 +1,71 @@
+name: Unit Tests
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - "**"
+    paths:
+      - "**.py"
+      - ".github/workflows/unit_tests.yaml"
+env:
+  POSTGRES_PASSWORD: postgres-test-pw
+  POSTGRES_USER: postgres-test-user
+  POSTGRES_DB: postgres-test-db
+  REDIS_HOST: redis://redis:6379
+  ADMIN_USERNAME: [email protected]
+  ADMIN_PASSWORD: test123
+  ADMIN_API_KEY: testkey123
+jobs:
+  container-job:
+    runs-on: ubuntu-20.04
+    container: node:20.7-bullseye
+    services:
+      postgres:
+        image: postgres:16.4
+        env:
+          POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }}
+          POSTGRES_USER: ${{ env.POSTGRES_USER }}
+          POSTGRES_DB: ${{ env.POSTGRES_DB }}
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+      redis:
+        image: redis:6.0-alpine
+        options: >-
+          --health-cmd "redis-cli ping || exit 1"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+    steps:
+      - name: install dependencies
+        run: apt-get update && apt-get install -y lsb-release && apt-get clean all
+      - name: Check out repository code
+        uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.12"
+      - name: Install Python libraries
+        run: |
+          python -m pip install -r backend/requirements.txt
+          python -m pip install -r requirements-dev.txt
+      - name: Run migrations and Add users
+        env:
+          POSTGRES_HOST: postgres
+        run: |
+          cd backend
+          python -m alembic upgrade head
+          python add_users_to_db.py
+      - name: Run Unit Tests
+        env:
+          PROMETHEUS_MULTIPROC_DIR: /tmp
+          REDIS_HOST: ${{ env.REDIS_HOST }}
+          POSTGRES_HOST: postgres
+        run: |
+          cd backend
+          python -m pytest -m "not slow" tests

Makefile

@@ -30,9 +30,9 @@ setup-db: guard-POSTGRES_USER guard-POSTGRES_PASSWORD guard-POSTGRES_DB
 	@docker system prune -f
 	@sleep 2
 	@docker run --name pg-experiment-local \
-		-e POSTGRES_USER=postgres \
-		-e POSTGRES_PASSWORD=postgres \
-		-e POSTGRES_DB=postgres \
+		-e POSTGRES_USER=$(POSTGRES_USER) \
+		-e POSTGRES_PASSWORD=$(POSTGRES_PASSWORD) \
+		-e POSTGRES_DB=$(POSTGRES_DB) \
 		-p 5432:5432 \
 		-d postgres:16.4
 	@sleep 5
@@ -42,6 +42,7 @@ setup-db: guard-POSTGRES_USER guard-POSTGRES_PASSWORD guard-POSTGRES_DB
         set +a && \
 	cd backend && \
 	python -m alembic upgrade head
+	python backend/add_users_to_db.py
 
 teardown-db:
 	@docker stop pg-experiment-local
@@ -56,16 +57,15 @@ setup-redis:
      -p 6379:6379 \
      -d redis:6.0-alpine
 
-make teardown-redis:
+teardown-redis:
 	@docker stop redis-experiment-local
 	@docker rm redis-experiment-local
 
 
-make run-backend:
+run-backend:
 	$(CONDA_ACTIVATE) $(PROJECT_NAME); \
 	make setup-db && make setup-redis && \
-	python backend/add_users_to_db.py && \
 	python backend/main.py
 
-make run-frontend:
+run-frontend:
 	cd frontend && npm run dev

backend/Makefile

@@ -0,0 +1,56 @@
+#!make
+
+include ./tests/test.env
+export
+
+.PHONY : tests
+
+# Main test target
+tests: setup-test-containers run-tests teardown-test-containers
+
+# Test runner
+# NB: `pytest-alembic` requires the DB engine to point an empty database. Thus, alembic
+# tests should be run first.
+
+guard-%:
+	@if [ -z "$$${*}" ]; then echo "ERROR: environment variable $* not set" && exit 1; fi
+
+run-tests:
+	python -m pytest -rPQ -m "not slow" tests
+
+## Helper targets
+setup-test-containers: setup-redis-test setup-test-db
+teardown-test-containers: teardown-test-db teardown-redis-test
+
+setup-test-db: guard-POSTGRES_PASSWORD guard-POSTGRES_USER guard-POSTGRES_DB
+	-@docker stop testdb
+	-@docker rm testdb
+	@docker system prune -f
+	@sleep 2
+	@docker run --name testdb \
+		-p 5433:5432 \
+		-e POSTGRES_PASSWORD \
+		-e POSTGRES_USER \
+		-e POSTGRES_DB \
+		-d postgres:16.4
+	@sleep 2
+	python -m alembic upgrade head
+	python add_users_to_db.py
+
+# Use port 6381 since port 6379 is used for dev and 6380 for docker-compose
+setup-redis-test:
+	-@docker stop redis-test
+	-@docker rm redis-test
+	@docker system prune -f
+	@sleep 2
+	@docker run --name redis-test \
+		-p 6381:6379 \
+		-d redis:6.0-alpine
+
+teardown-redis-test:
+	@docker stop redis-test
+	@docker rm redis-test
+
+teardown-test-db:
+	@docker stop testdb
+	@docker rm testdb