Merge pull request #34 from jackleland/jl/tests-ci

Add CI tests + linting, and fix buggy group-profiles in the admin interface

.dockerignore

@@ -5,4 +5,5 @@ docker-compose.yml
 .env
 .dockerignore
 data/
-.devcontainer/
+.devcontainer/
+.github/

.env.sample

@@ -0,0 +1,46 @@
+# Django settings
+SECRET_KEY=
+DEBUG=True
+DJANGO_ALLOWED_HOSTS=localhost 127.0.0.1 [::1]
+
+# MySQL/MariaDB settings
+MYSQL_ROOT_USER=root
+MYSQL_ROOT_PASSWORD=root
+MYSQL_DATABASE=atlas
+MYSQL_USER=atlas
+MYSQL_PASSWORD=atlas
+MYSQL_PORT=3306
+MYSQL_TEST_DATABASE=atlas_test
+MYSQL_TEST_PORT=3306
+MYSQL_TEST_USER=root
+MYSQL_TEST_PASSWORD=root
+
+# DJANGO Settings for testing with SQLite
+DJANGO_DB_ENGINE=django.db.backends.sqlite3
+DJANGO_MYSQL_DBNAME=memory
+TESTING=True
+
+WSGI_PREFIX=/default
+WSGI_PORT=8087
+DJANGO_SECRET_KEY=test
+DJANGO_MYSQL_DBUSER=${MYSQL_TEST_USER}
+DJANGO_MYSQL_DBPASS=${MYSQL_TEST_PASSWORD}
+DJANGO_MYSQL_DBHOST=localhost
+DJANGO_MYSQL_DBPORT=${MYSQL_TEST_PORT}
+DJANGO_MYSQL_TEST_DBNAME=${MYSQL_TEST_DATABASE}
+DJANGO_MYSQL_TEST_DBPORT=${MYSQL_TEST_PORT}
+DJANGO_MYSQL_TEST_DBUSER=${MYSQL_TEST_USER}
+DJANGO_MYSQL_TEST_DBPASS=${MYSQL_TEST_PASSWORD}
+DJANGO_TNS_DAEMON_SERVER=
+DJANGO_TNS_DAEMON_PORT=1010
+DJANGO_MPC_DAEMON_SERVER=
+DJANGO_MPC_DAEMON_PORT=1011
+DJANGO_LASAIR_TOKEN=
+DJANGO_LOG_LEVEL=INFO
+DJANGO_NAMESERVER_TOKEN=
+DJANGO_NAMESERVER_API_URL=
+DJANGO_NAMESERVER_MULTIPLIER=
+DJANGO_PANSTARRS_TOKEN=
+DJANGO_PANSTARRS_BASE_URL=
+DJANGO_DUSTMAP_LOCATION=
+API_TOKEN_EXPIRY=10

.github/workflows/ci.yaml

@@ -0,0 +1,71 @@
+name: Continuous Integration
+on: [push, pull_request]
+jobs:
+  linting:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10"]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+  testing-docker:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build test containers
+      run: |
+        # Copy the sample environment file and build the containers
+        cp .env.sample .env
+        # In the future we can populate the .env file with secrets, none needed for now
+        docker compose build 
+    - name: Start db container
+      run: docker compose up db
+    - name: Start test container
+      run: |
+        docker compose up tests --exit-code-from tests
+    - name: Stop containers
+      if: always()
+      run: docker compose down
+  testing-pip:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps: 
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+    - name: Install system dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y apache2 apache2-dev
+    - name: Install python dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install .
+    - name: Set initial environment
+      run: |
+        # Copy the sample environment file
+        cp .env.sample .env
+    - name: Run tests
+      run: |
+        python psat_server_web/atlas/manage.py makemigrations --noinput
+        python psat_server_web/atlas/manage.py test --noinput

.gitignore

@@ -53,3 +53,6 @@ recurrence_plots
 reports
 admin/
 django_tables2/
+
+# Log files
+*.log

Dockerfile

@@ -14,6 +14,7 @@ WORKDIR /app
 RUN apt-get update && apt-get install -y \
     build-essential \
     pkg-config \
+    apache2 \
     apache2-dev \
     libhdf5-dev \
     default-mysql-client \
@@ -26,8 +27,8 @@ RUN apt-get update && apt-get install -y \
 COPY . .
 
 # Install python dependencies and the package itself
-RUN pip install --no-cache-dir -e . && \
-    pip install --no-cache-dir mod_wsgi-standalone
+RUN pip install --no-cache-dir -e .
+    # pip install --no-cache-dir mod_wsgi-express
 
 WORKDIR /app/psat_server_web/atlas
 

README.md

@@ -11,3 +11,91 @@ site-packages/psat\_server\_web/atlas/media/images
 site-packages/psat\_server\_web/ps1/media/images
 
 directories which point to the location of the image stamps.
+
+--- 
+
+# Local development version 
+
+A localised development instance can be run with docker compose. You will need a 
+`.env` file in your repository root to define some environment variables, 
+looking something like:
+
+``` .env
+# Django settings
+SECRET_KEY={YOUR_SECRET_KEY_HERE}
+DJANGO_ALLOWED_HOSTS=localhost 127.0.0.1 [::1]
+
+# MySQL/MariaDB settings
+MYSQL_ROOT_USER=root
+MYSQL_ROOT_PASSWORD=root
+MYSQL_DATABASE=atlas
+MYSQL_USER=atlas
+MYSQL_PASSWORD=atlas
+MYSQL_PORT=3306
+MYSQL_TEST_DATABASE=atlas_test
+MYSQL_TEST_PORT=3306
+MYSQL_TEST_USER=root
+MYSQL_TEST_PASSWORD=root
+
+DJANGO_MYSQL_DBUSER=atlas
+DJANGO_LASAIR_TOKEN=
+
+```
+
+Several of the values are omitted but aren't necessary to run a development 
+environment. Also ruch simple passwords should not be used in production, but this should get you something running locally. Hopefully it's then as simple as running the following:
+
+``` bash
+docker compose up
+```
+
+Which will start 4 services, comprised of:
+- `db` - A MariaDB instance. By default served at `localhost:3036` 
+- `atlas-web` - A mod-wsgi instance to serve the django front end. By default 
+served at `localhost:8086` and requires `db` to be running.
+- `adminer` - A web interface for interacting with the db. By default served 
+at `localhost:8080` and requires `db` to be running.
+- `tests` - A single-use run of the unit tests using django's `manage.py test`. 
+Requires `db` to be running.
+
+Each of these services can be run independently with 
+
+``` bash
+docker compose up {service_name}
+```
+
+So if, for example, you wanted to run the tests you could execute 
+`docker compose up tests` and it would – after starting `db` if it is not 
+already running – perform a single run of the unit tests. 
+
+## A dummy database
+
+There are a few things you will likely need to do to get a fully working version 
+of the web-server so as to make your local version useful, namely: 
+1. Get a dummy database dump .sql file from one of the developers and place it into `data/init.sql`
+2. [Optional] Get an image dump and place it into `data/db_data/`
+3. Fix a mild issue with the database model (see next section)
+
+If you are having problems with any of this, contact one of the developers.
+
+### Issue with `TcsGravityEventAnnotations.map_iteration`
+
+If you are getting an error along the lines of 
+
+```
+atlas-web-1  | ERRORS:
+atlas-web-1  | atlas.TcsGravityEventAnnotations.map_iteration: (fields.E311) 'TcsGravityAlerts.map_iteration' must be unique because it is referenced by a foreign key.
+atlas-web-1  | 	HINT: Add unique=True to this field or add a UniqueConstraint (without condition) in the model Meta.constraints.
+```
+
+Then you may need to make an adjustment to the models, specifically by 
+uncommenting the line declaring `TcsGravityAlerts.map_iteration` which adds a 
+unique `kwarg`, and commenting the line without the unique `kwarg`, should solve 
+the problem. Specifically, the lines should look like 
+```
+# map_iteration = models.CharField(max_length=100, blank=True, null=True)
+map_iteration = models.CharField(max_length=100, blank=True, null=True, unique=True)
+``` 
+This is within the `TcsGravityAlerts(models.Model)` class. Again, please speak 
+to one of the developers if this is unclear. 
+

docker-compose.yml

@@ -17,8 +17,8 @@ services:
       MYSQL_PASSWORD: ${MYSQL_PASSWORD}
       MARIADB_DATABASE: ${MYSQL_DATABASE}
     healthcheck:
-      test: ["CMD", "mysql", "-h", "localhost", "-u", "root", "-p${MYSQL_ROOT_PASSWORD}", "-e", "SELECT 1"]
-      timeout: 20s
+      test: healthcheck.sh --su-mysql --connect --innodb_initialized
+      timeout: 5s
       retries: 10
 
   adminer:
@@ -73,4 +73,60 @@ services:
       - DJANGO_NAMESERVER_API_URL=''
       - DJANGO_LASAIR_TOKEN=${DJANGO_LASAIR_TOKEN}
       - DJANGO_DUSTMAP_LOCATION=/tmp/dustmap
+      - API_TOKEN_EXPIRY=10
+      - DJANGO_LOG_LEVEL=DEBUG
+      - DJANGO_DEBUG=True
+      - DJANGO_PANSTARRS_TOKEN=${PANSTARRS_TOKEN}
+      - DJANGO_PANSTARRS_BASE_URL=${PANSTARRS_BASE_URL}
 
+  tests:
+    build: .
+    image: local/psat-server-web
+      # Run the tests, using the root user to avoid permission issues
+    command: >
+      bash -c "python manage.py makemigrations --noinput && 
+      python manage.py test 
+      || exit $?"
+    volumes:
+    # Mount the code directories into the image to allow for live code changes
+    # while we develop
+      - ./data/db_data:/images 
+      - ./psat_server_web/atlas/atlasapi:/app/psat_server_web/atlas/atlasapi
+      - ./psat_server_web/atlas/atlas:/app/psat_server_web/atlas/atlas
+      - ./psat_server_web/atlas/accounts:/app/psat_server_web/atlas/accounts
+      - ./psat_server_web/atlas/tests:/app/psat_server_web/atlas/tests
+    ports:
+      - 8087:8087
+    depends_on:
+      db:
+        condition: service_healthy
+        restart: true
+    environment:
+      - WSGI_PREFIX=/atlas
+      - WSGI_PORT=8087
+      - DJANGO_DB_ENGINE=django.db.backends.mysql
+      - DJANGO_MYSQL_DBNAME=${MYSQL_TEST_DATABASE}
+      - DJANGO_MYSQL_DBUSER=${MYSQL_TEST_USER}
+      - DJANGO_MYSQL_DBPASS=${MYSQL_TEST_PASSWORD}
+      - DJANGO_MYSQL_DBHOST=db
+      - DJANGO_MYSQL_DBPORT=${MYSQL_TEST_PORT}
+      - DJANGO_MYSQL_TEST_DBNAME=${MYSQL_TEST_DATABASE}
+      - DJANGO_MYSQL_TEST_DBPORT=${MYSQL_TEST_PORT}
+      - DJANGO_MYSQL_TEST_DBUSER=${MYSQL_TEST_USER}
+      - DJANGO_MYSQL_TEST_DBPASS=${MYSQL_TEST_PASSWORD}
+      - DJANGO_SECRET_KEY=secret
+      - DJANGO_TNS_DAEMON_SERVER=psat-server-web
+      - DJANGO_TNS_DAEMON_PORT=8001
+      - DJANGO_MPC_DAEMON_SERVER=psat-server-web
+      - DJANGO_MPC_DAEMON_PORT=8002
+      - DJANGO_NAME_DEAMON_SERVER=psat-server-web
+      - DJANGO_NAME_DEAMON_PORT=8003
+      - DJANGO_NAMESERVER_MULTIPLIER=10000000
+      - DJANGO_NAMESERVER_TOKEN=''
+      - DJANGO_NAMESERVER_API_URL=''
+      - DJANGO_LASAIR_TOKEN=${DJANGO_LASAIR_TOKEN}
+      - DJANGO_DUSTMAP_LOCATION=/tmp/dustmap
+      - DJANGO_LOG_LEVEL=DEBUG
+      - API_TOKEN_EXPIRY=10
+      - DJANGO_PANSTARRS_TOKEN=${PANSTARRS_TOKEN}
+      - DJANGO_PANSTARRS_BASE_URL=${PANSTARRS_BASE_URL}

psat_server_web/atlas/accounts/admin.py

@@ -2,14 +2,30 @@
 Customise admin interface for Group model to include GroupProfile, so default 
 expiry time for tokens can be set. 
 """
+import logging
 
 from django.contrib import admin
 from django.contrib.auth.models import Group
+from django.forms.models import ModelForm
 from .models import GroupProfile
 
+
+logger = logging.getLogger(__name__)
+
+class AlwaysChangedModelForm(ModelForm):
+    def has_changed(self):
+        """ Should return True if data differs from initial. 
+        By always returning true even unchanged inlines will get validated and saved.
+        We need this because the GroupProfile needs to be created even if the default 
+        values haven't been changed. 
+        """
+        return True
+
 class GroupProfileInline(admin.StackedInline):
     model = GroupProfile
     can_delete = False
+    extra = 1
+    form = AlwaysChangedModelForm
 
 class GroupAdmin(admin.ModelAdmin):
     inlines = (GroupProfileInline,)

psat_server_web/atlas/accounts/signals.py

@@ -1,15 +1,27 @@
-from django.db.models.signals import post_save
+import logging
+
+from django.db.models import signals 
 from django.dispatch import receiver
 from django.contrib.auth.models import Group
-from .models import GroupProfile
 
-@receiver(post_save, sender=Group)
+logger = logging.getLogger(__name__)
+
+@receiver(signals.post_save, sender=Group)
 def create_group_profile(sender, instance, created, **kwargs):
+    # NOTE (2024-11-12 JL): This function is no longer needed as we have instead 
+    # solved the problem of duplicate GroupProfile creation by using the 
+    # AlwaysChangedModelForm in the admin interface, but I'm leaving this here 
+    # as a reference for future use of signals and logging.
+    logger.debug('create_group_profile called')
+    logger.debug('instance: %s', instance)
+    logger.debug('created: %s', created)
+
+    # Disconnect the signal so we don't get into a loop
+    signals.post_save.disconnect(create_group_profile, sender=Group)
+
     if created:
-        new_profile, profile_created = GroupProfile.objects.get_or_create(
-            group=instance
-        )
-        if profile_created:
-            # If we created a new profile then set the group's profile to it
-            instance.profile = new_profile 
-            instance.save()
+        logger.debug('In created block')
+
+    # Reconnect the signal once we're done
+    signals.post_save.connect(create_group_profile, sender=Group)
+    logger.debug('create_group_profile finished')