updates CI

.github/file-filters.yml

@@ -0,0 +1,45 @@
+# This is used by the action https://github.com/dorny/paths-filter
+docker: &docker
+  - added|modified: './docker/**/*'
+  - added|modified: './docker/*'
+
+dependencies: &dependencies
+  - 'pdm.lock'
+  - 'pyproject.toml'
+
+actions: &actions
+  - added|modified: './.github/**/*'
+
+python: &python
+  - added|modified: 'src/**'
+  - added|modified: 'tests/**'
+  - 'manage.py'
+
+changelog:
+  - added|modified: 'changes/**'
+  - 'CHANGELOG.md'
+
+mypy:
+  - *python
+  - 'mypy.ini'
+
+run_tests:
+  - *actions
+  - *python
+  - *docker
+  - *dependencies
+  - 'pytest.ini'
+
+migrations:
+  - added|modified: 'src/**/migrations/*'
+
+lint:
+  - *python
+  - '.flake8'
+  - 'pyproject.toml'
+
+docs:
+  - added|modified: './docs/**/*'
+  - modified: './src/country_workspace/config/__init__.py'
+  - modified: './github/workflows/docs.yml'
+  - modified: './github/file-filters.yml'

.github/workflows/lint.yml

@@ -0,0 +1,91 @@
+name: Lint
+on:
+  push:
+    branches:
+      - develop
+      - master
+      - staging
+      - release/*
+      - feature/*
+      - bugfix/*
+      - hotfix/*
+#  pull_request:
+#    branches: [develop, master]
+#    types: [synchronize, opened, reopened, ready_for_review]
+
+defaults:
+  run:
+    shell: bash
+
+
+concurrency:
+  group: "${{ github.workflow }}-${{ github.ref }}"
+  cancel-in-progress: true
+
+
+permissions:
+  contents: read
+
+jobs:
+  changes:
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
+    name: check files
+    runs-on: ubuntu-latest
+    timeout-minutes: 3
+    outputs:
+      lint: ${{ steps.changes.outputs.lint }}
+      docker: ${{ steps.changes.outputs.docker_base }}
+    steps:
+      - run: git config --global --add safe.directory $(realpath .)
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - id: changes
+        name: Check for backend file changes
+        uses: dorny/paths-filter@0bc4621a3135347011ad047f9ecf449bf72ce2bd # v3.0.0
+        with:
+          base: ${{ github.ref }}
+          token: ${{ github.token }}
+          filters: .github/file-filters.yml
+
+  flake8:
+    needs: changes
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.draft == false && needs.changes.outputs.lint
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Install requirements
+        run: pip install flake8 pycodestyle
+      - name: Check syntax
+        # Stop the build if there are Python syntax errors or undefined names
+        run: flake8 src/ --count --statistics --max-line-length=127
+
+      - name: Warnings
+        run: flake8 src --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --extend-exclude=""
+  isort:
+    needs: changes
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.draft == false && needs.changes.outputs.lint
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Install requirements
+        run: pip install isort
+      - name: iSort
+        run: isort src/ --check-only
+  black:
+    needs: changes
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.draft == false && needs.changes.outputs.lint
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Install requirements
+        run: pip install black
+      - name: Black
+        run: black src/ --check

.github/workflows/test.yml

@@ -3,77 +3,183 @@ name: Test
 on:
   push:
     branches:
-      - master
       - develop
-  pull_request:
+      - master
+      - staging
+      - release/*
+      - feature/*
+      - bugfix/*
+      - hotfix/*
+#  pull_request:
+#    branches: [ develop, master ]
+#    types: [ synchronize, opened, reopened, ready_for_review ]
+
+concurrency:
+  group: "${{ github.workflow }}-${{ github.ref }}"
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  id-token: write
+  attestations: write
+
 
 jobs:
+#  changes:
+#    if: (github.event_name != 'pull_request'
+#            || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)
+#            || github.event_name == 'create'
+#    runs-on: ubuntu-latest
+#    timeout-minutes: 1
+#    defaults:
+#      run:
+#        shell: bash
+#    outputs:
+#      run_tests: ${{ steps.changes.outputs.run_tests }}
+#    steps:
+#      - name: Checkout code
+#        uses: actions/[email protected]
+#      - id: changes
+#        name: Check for file changes
+#        uses: dorny/paths-filter@0bc4621a3135347011ad047f9ecf449bf72ce2bd # v3.0.0
+#        with:
+#          base: ${{ github.ref }}
+#          token: ${{ github.token }}
+#          filters: .github/file-filters.yml
   test:
+#    needs: [ changes ]
+#    if: needs.changes.outputs.run_tests == 'true'
     runs-on: ubuntu-latest
+    outputs:
+      image: ${{ env.IMAGE }}
+      commit: ${{env.sha_short}}
+      build_date: ${{env.BUILD_DATE}}
+      branch: ${{env.BRANCH}}
     services:
-      postgres:
-        image: postgres:12
+      redis:
+        image: redis
+        ports:
+          - 16379:6379
+      db:
+        image: postgres:14
         env:
-          POSTGRES_USER: postgres
+          POSTGRES_DATABASE: aurora
           POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: aurora
+          POSTGRES_USERNAME: postgres
         ports:
-          - 5432:5432
-        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
-    strategy:
-      fail-fast: false
-      matrix:
-        django-version: [ "4.2", ]
-        python-version: [ "3.12", ]
-        experimental: [ false ]
-#        include:
-#          - django-version: "5.0"
-#            python-version: "3.11"
-#            experimental: true
-    continue-on-error: ${{ matrix.experimental }}
+          - 15432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     env:
-      DATABASE_URL: postgres://postgres:[email protected]:5432/aurora
-      STATIC_URL: /static/
-      STATIC_ROOT:
-      PY_VER: ${{ matrix.python-version}}
-      DJ_VER: ${{ matrix.django-version}}
-      SECRET_KEY: super-secret-key
+      DOCKER_DEFAULT_PLATFORM: linux/amd64
+      DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index
+      DATABASE_URL: postgres://postgres:postgres@localhost:15432/aurora
+      CELERY_BROKER_URL: redis://localhost:16379/0"
+      CACHE_URL: redis://localhost:16379/0
+      DOCKER_BUILDKIT: 1
     steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Hack container for local development
+        if: ${{ env.ACT }}
+        run: |
+          echo /home/runner/externals/node20/bin >> $GITHUB_PATH
+      - name: Hack container for local development
+        run: |
+          echo BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_ENV
+      - name: Docker meta
+        id: meta
+        uses: docker/[email protected]
         with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Install pdm
-        run: python -m pip install --upgrade pdm
-
-      - name: Install dependencies
+          images: "unicef/aurora"
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=ref,event=tag
+            type=semver,pattern={{version}}
+            type=semver,pattern={{raw}}
+      - name: DockerHub login
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - id: last_commit
+        uses: ./.github/actions/last_commit
+      - id: calc
+        shell: bash
         run: |
-          pdm venv create
-          pdm venv activate in-project
-          pdm sync
+          set -x
+          LOCK_SHA=$(sha1sum uv.lock docker/bin/* docker/conf/* docker/Dockerfile | sha1sum | awk '{print $1}' | cut -c 1-8)
+          IMAGE=$(echo '${{env.DOCKER_METADATA_OUTPUT_JSON}}' | jq '.tags[0]')
+          echo "checksum=$LOCK_SHA" >> "$GITHUB_ENV"
+          echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+          echo "BUILD_DATE=$(date +"%Y-%m-%d %H:%M" )" >> $GITHUB_ENV
+          echo "IMAGE=$IMAGE" >> $GITHUB_ENV
 
-      - name: Run tests
-        run: pdm run pytest tests/ --create-db --cov --cov-report xml --junit-xml junit.xml
+      - name: Build Test Image
+        run: |
+             docker build \
+                --target tests \
+                -t ${{env.IMAGE}} \
+                --cache-from "type=gha" \
+                --cache-to "type=gha,mode=max" \
+                -f docker/Dockerfile .
 
-      - name: Upload pytest test results
-        uses: actions/upload-artifact@v4
-        with:
-          name: pytest-results
-          path: junit.xml
-        if: ${{ always() }}
+      - name: Run Test suite
+        run: |
+            mkdir output
+            docker run --rm \
+              --network host \
+              -e PYTHONPATH=/app/src \
+              -e DATABASE_URL=${DATABASE_URL} \
+              -e CELERY_BROKER_URL=${CELERY_BROKER_URL} \
+              -e CACHE_URL=${CACHE_URL} \
+              -v "./output/:/app/output" \
+              -v "./src/:/app/src" \
+              -v "./tests:/app/tests" \
+              -v "./pytest.ini:/app/pytest.ini" \
+              -t ${{env.IMAGE}} \
+              pytest tests/ -n auto -v --maxfail=5 --migrations --cov-report xml:./output/coverage.xml
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
-        if: matrix.python-version == 3.12
         continue-on-error: true
         with:
           env_vars: OS,PYTHON
           fail_ci_if_error: true
-          flags: unittests
-          files: ./coverage.xml
-          verbose: false
+          files: /app/output/coverage.xml
           token: ${{ secrets.CODECOV_TOKEN }}
+          verbose: false
           name: codecov-${{env.GITHUB_REF_NAME}}
+
+  release:
+    needs: [ test ]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: DockerHub login
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build Distro
+        run: |
+            docker build \
+                --target dist \
+                --cache-from "type=gha" \
+                --cache-to "type=gha,mode=max" \
+                --build-arg "VERSION=${{needs.test.outputs.commit}}" \
+                --build-arg "BUILD_DATE=${{needs.test.outputs.build_date}}" \
+                --build-arg "BRANCH=${{needs.test.outputs.branch}}" \
+                -t ${{needs.test.outputs.image}} \
+                -f docker/Dockerfile .
+            docker push ${{needs.test.outputs.image}}
+            docker inspect ${{needs.test.outputs.image}} | jq -r '.[0].Config.Labels'
+            echo "::notice::✅ Image ${{needs.test.outputs.image}} built and pushed"