updates

unicef · Oct 20, 2024 · a76f49e · a76f49e
1 parent fe04a3a
commit a76f49e
Show file tree

Hide file tree

Showing 18 changed files with 269 additions and 679 deletions.
diff --git a/.github/file-filters.yml b/.github/file-filters.yml
@@ -7,6 +7,9 @@ dependencies: &dependencies
   - 'pdm.lock'
   - 'pyproject.toml'
 
+actions: &actions
+  - added|modified: './github/**/*'
+
 python: &python
   - added|modified: 'src/**'
   - added|modified: 'tests/**'
@@ -20,11 +23,8 @@ mypy:
   - *python
   - 'mypy.ini'
 
-docker_base:
-  - *docker
-  - *dependencies
-
 run_tests:
+  - *actions
   - *python
   - *docker
   - *dependencies
@@ -34,6 +34,12 @@ migrations:
   - added|modified: 'src/**/migrations/*'
 
 lint:
-  -  *python
+  - *python
   - '.flake8'
   - 'pyproject.toml'
+
+docs:
+  - added|modified: './docs/**/*'
+  - modified: './src/country_workspace/config/__init__.py'
+  - modified: './github/workflows/docs.yml'
+  - modified: './github/file-filters.yml'
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -18,29 +18,65 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
+  changes:
+    runs-on: ubuntu-latest
+    timeout-minutes: 1
+    defaults:
+      run:
+        shell: bash
+    outputs:
+      docs: ${{ steps.changed_files.outputs.docs }}
+    steps:
+      - name: Checkout code
+        uses: actions/[email protected]
+      - id: changed_files
+        name: Check for file changes
+        uses: dorny/paths-filter@0bc4621a3135347011ad047f9ecf449bf72ce2bd # v3.0.0
+        with:
+          base: ${{ github.ref }}
+          token: ${{ github.token }}
+          filters: .github/file-filters.yml
   generate:
     name: Generate
+    if: needs.changes.outputs.docs == 'true'
+    needs: changes
     runs-on: ubuntu-latest
+    env:
+      PYTHONPATH: src/
     steps:
       - uses: actions/checkout@v4
+      - uses: yezz123/setup-uv@v4
+
+      - uses: actions/cache/restore@v4
+        id: restore-cache
+        with:
+          path: .venv
+          key: ${{ runner.os }}-${{ hashFiles('**/uv.lock') }}
+
+      - name: Install dependencies
+        run: uv sync --extra docs
+
       - name: Build Doc
-        run: |
-            pip install "mkdocs>=1.6.1" "markdown-include" "mkdocs-material>=9.5.36" \
-                          "mkdocs-awesome-pages-plugin>=2.9.3" "mkdocstrings-python" "mdx-gh-links>=0.4"
-            mkdocs build -d ./docs-output
+        run: .venv/bin/mkdocs build -d ./docs-output
 
+      - uses: actions/cache/save@v4
+        id: cache
+        if: always() && steps.restore-cache.outputs.cache-hit != 'true'
+        with:
+          path: .venv
+          key: ${{ runner.os }}-${{ hashFiles('**/uv.lock') }}
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v3
         with:
           path: ./docs-output
 
   # Deployment job
   deploy:
+    needs: generate
     environment:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
     runs-on: ubuntu-latest
-    needs: generate
     steps:
       - name: Deploy to GitHub Pages
         id: deployment

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -49,25 +49,9 @@ jobs:
           base: ${{ github.ref }}
           token: ${{ github.token }}
           filters: .github/file-filters.yml
-      - name: info
-        shell: bash
-        run: |
-          force_build="${{ contains(github.event.head_commit.message, 'ci:build') || contains(github.event.head_commit.message, 'ci:release')}}"
-          force_scan="${{ contains(github.event.head_commit.message, 'ci:scan') }}"
-          force_test="${{ contains(github.event.head_commit.message, 'ci:test') }}"
-
-          if [[ $force_build == "true" ]]; then
-            echo "::notice:: Forced build docker due to commit message"
-          elif [[ $force_test == "true" ]]; then
-            echo "::notice:: Forced python tests due to commit message"
-          elif [[ $force_scan == "true" ]]; then
-            echo "::notice:: Forced trivy scan due to commit message"
-          fi
-          if [[ $force_build == "true" || "${{needs.changes.outputs.run_tests}}" == "true" ]]; then
-              echo "BUILD=true" >> $GITHUB_ENV
-          fi
   test:
     needs: [ changes ]
+    if: needs.changes.outputs.run_tests == 'true'
     runs-on: ubuntu-latest
     services:
       redis:
@@ -104,7 +88,6 @@ jobs:
       - name: Hack container for local development
         run: |
           echo BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_ENV
-#      - uses: yezz123/setup-uv@v4
       - name: Docker meta
         id: meta
         uses: docker/[email protected]
@@ -131,6 +114,8 @@ jobs:
           set -x
           LOCK_SHA=$(sha1sum uv.lock docker/bin/* docker/conf/* docker/Dockerfile | sha1sum | awk '{print $1}' | cut -c 1-8)
           echo "checksum=$LOCK_SHA" >> "$GITHUB_ENV"
+          echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+          echo "BUILD_DATE=$(date +"%Y-%m-%d %H:%M" )" >> $GITHUB_ENV
 
       - name: Build Test Image
         run: |
@@ -140,6 +125,7 @@ jobs:
                 --cache-from "type=gha" \
                 --cache-to "type=gha,mode=max" \
                 -f docker/Dockerfile .
+
       - name: Run Test suite
         run: |
             mkdir output
@@ -169,8 +155,12 @@ jobs:
         run: |
              docker build \
                 --target dist \
-                -t "unicef/hope-country-workspace:${{env.BRANCH}}" \
                 --cache-from "type=gha" \
                 --cache-to "type=gha,mode=max" \
+                --build-arg "VERSION=${{env.sha_short}}" \
+                --build-arg "BUILD_DATE=${{env.BUILD_DATE}}" \
+                --build-arg "BRANCH=${{env.BRANCH}}" \
+                -t "unicef/hope-country-workspace:${{env.BRANCH}}" \
                 -f docker/Dockerfile .
               docker push "unicef/hope-country-workspace:${{env.BRANCH}}"
+              docker inspect unicef/hope-country-workspace:${{env.BRANCH}} | jq -r '.[0].Config.Labels'
diff --git a/README.md b/README.md
@@ -5,113 +5,3 @@
 [![codecov](https://codecov.io/github/unicef/hope-country-workspace/graph/badge.svg?token=FBUB7HML5S)](https://codecov.io/github/unicef/hope-country-workspace)
 [![Documentation](https://github.com/unicef/hope-country-workspace/actions/workflows/docs.yml/badge.svg)](https://unicef.github.io/hope-country-workspace/)
 [![Docker Pulls](https://img.shields.io/docker/pulls/unicef/hope-country-workspace)](https://hub.docker.com/repository/docker/unicef/hope-country-workspace/tags)
-
-
-## Spreadsheet import
-
-```mermaid
----
-title: Spreadsheet import
----
-graph TD
-;
-    A[Upload file] --> B{All records are valid?}
-    B -- Yes --> E[End]
-    B -- No --> C[Generate error report]
-    C --> D[Clear imported records]
-    D --> A
-```
-
-Previous import error report could be displayed on spreadsheet entry page.
-
-**TBD**:
-
-- Do we need to save data from previous import attempt if we still have to parse
-  each row on every import attempt?
-- Should we add a flag to show data from spreadsheet is ready for export to HOPE?
-- Do we need to display errors from previous import attempt on spreadsheet entry
-  page?
-
-```mermaid
----
-title: Table structure
----
-erDiagram
-    Spreadsheet {
-        string title
-    }
-    
-    Error {
-        string sheet
-        int row
-        string column
-    }
-    
-    Program ||--o{ Spreadsheet: has
-    Program ||--o{ Individual: has
-    Program ||--o{ Household: has
-    Program ||--o{ IndividualRoleInHousehold: has
-
-    Spreadsheet ||--o{ IndividualRecord: has
-    IndividualRecord }o--|| Individual: is
-    Spreadsheet ||--o{ HouseholdRecord: has
-    HouseholdRecord }o--|| Household: is
-    Spreadsheet ||--o{ IndividualRoleRecord: has
-    IndividualRoleRecord ||--o{ IndividualRoleInHousehold: is
-    Spreadsheet ||--o{ Error: has
-```
-
-## Kobo Import
-
-```mermaid
----
-title: Dataset import
----
-graph TD
-;
-    A[Fetch dataset] --> B{All records are valid?}
-    B -- Yes --> E[End]
-    B -- No --> C[Generate error report]
-    C --> D[Clear imported records]
-    D --> A
-```
-
-```mermaid
----
-title: Table structure
----
-erDiagram
-    Dataset {
-        string title
-        string url
-    }
-    
-    Error {
-        string data
-    }
-    
-    Program ||--o{ Dataset: has
-    Program ||--o{ Individual: has
-    Program ||--o{ Household: has
-    Program ||--o{ IndividualRoleInHousehold: has
- 
-    Dataset ||--o{ IndividualRecord: has
-    IndividualRecord }o--|| Individual: is
-    Dataset ||--o{ HouseholdRecord: has
-    HouseholdRecord }o--|| Household: is
-    Dataset ||--o{ IndividualRoleRecord: has
-    IndividualRoleRecord ||--o{ IndividualRoleInHousehold: is
-    Dataset ||--o{ Error: has
-```
-
-**TBD:**
-- Dataset format
-- Do we need to display errors from previous import attempt on dataset entry page?
-
-## Aurora Import
-
-## HOPE export
-
-**TBD**:
-
-- When deduplication is run in Country Workspace?
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -23,6 +23,8 @@ RUN groupadd --gid 1024 app \
 
 # ------- builder -------
 FROM python_base AS builder
+ARG BUILD_DATE
+ARG VERSION
 
 RUN set -x \
     && buildDeps="build-essential \
@@ -44,17 +46,21 @@ RUN pip install uv
 COPY pyproject.toml uv.lock AUTHORS LICENSE README.md MANIFEST.in /app/
 
 WORKDIR /app
-RUN uv sync --no-dev --no-editable --frozen
+RUN uv sync --no-dev --no-editable --frozen --extra distribution
 
 
 # ------- tests -------
 FROM builder AS tests
+LABEL distro="tests"
+LABEL org.opencontainers.image.created="$BUILD_DATE"
+LABEL org.opencontainers.image.version="$VERSION"
+
 ENV PATH=/app/.venv/bin:/usr/local/bin/:/usr/bin:/bin \
     DJANGO_SETTINGS_MODULE=country_workspace.config.settings \
     PYTHONPATH=""\
     UWSGI_PROCESSES=4
 WORKDIR /app
-RUN uv sync --no-editable --frozen \
+RUN uv sync --no-editable --frozen --extra distribution \
     && rm -fr build hope_country_workspace.egg-info
 
 # ------- package -------
@@ -66,15 +72,24 @@ ENV PATH=/app/.venv/bin:/usr/local/bin/:/usr/bin:/bin \
 WORKDIR /app
 COPY src/ /app/src/
 COPY pyproject.toml uv.lock AUTHORS LICENSE README.md MANIFEST.in /app/
-RUN uv sync --no-dev --no-editable --frozen
+RUN uv sync --no-dev --no-editable --frozen --extra distribution
 
 # ------- dist -------
 FROM python_base AS dist
+ARG BUILD_DATE
+ARG VERSION
+ARG BRANCH
+
 ENV PATH=/app/.venv/bin:/usr/local/bin/:/usr/bin:/bin \
     DJANGO_SETTINGS_MODULE=country_workspace.config.settings \
     PYTHONPATH=""\
     UWSGI_PROCESSES=4
-
+RUN <<EOF cat> /RELEASE
+{"version": "$VERSION",
+ "commit": "$GITHUB_SHA",
+ "date": "$BUILD_DATE",
+}
+EOF
 WORKDIR /app
 COPY docker/conf /conf/
 COPY docker/bin/* /usr/local/bin/
@@ -84,17 +99,16 @@ EXPOSE 8000
 ENTRYPOINT exec docker-entrypoint.sh "$0" "$@"
 CMD ["run"]
 
-
-#LABEL distro="final"
-#LABEL maintainer="[email protected]"
-#LABEL org.opencontainers.image.authors="[email protected]"
-#LABEL org.opencontainers.image.created="$BUILD_DATE"
-#LABEL org.opencontainers.image.description="Hope Country Workspace"
-#LABEL org.opencontainers.image.documentation="https://github.com/unicef/hope-country-workspace/"
-#LABEL org.opencontainers.image.licenses="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/blob/${SOURCE_COMMIT:-master}/LICENSE"
-#LABEL org.opencontainers.image.revision=$SOURCE_COMMIT
-#LABEL org.opencontainers.image.source="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/tree/${SOURCE_COMMIT:-master}/"
-#LABEL org.opencontainers.image.title="Hope Country Workspace"
-#LABEL org.opencontainers.image.version="$VERSION"
-#LABEL org.opencontainers.image.url="https://app.io/"
-#LABEL org.opencontainers.image.vendor="App ltd"
+LABEL maintainer="[email protected]"
+LABEL org.opencontainers.image.authors="[email protected]"
+LABEL org.opencontainers.image.description="Hope Country Workspace"
+LABEL org.opencontainers.image.documentation="https://github.com/unicef/hope-country-workspace/"
+LABEL org.opencontainers.image.title="Hope Country Workspace"
+LABEL org.opencontainers.image.licenses="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/blob/${SOURCE_COMMIT:-master}/LICENSE"
+LABEL org.opencontainers.image.revision=$SOURCE_COMMIT
+LABEL org.opencontainers.image.source="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/tree/${SOURCE_COMMIT:-master}/"
+LABEL org.opencontainers.image.url="https://hub.docker.com/repository/docker/unicef/hope-country-workspace/tags/"
+LABEL org.opencontainers.image.vendor="UNICEF"
+LABEL org.opencontainers.image.created="$BUILD_DATE"
+LABEL org.opencontainers.image.version="$VERSION"
+LABEL org.opencontainers.image.branch="$BRANCH"
diff --git a/docs/_hooks/hooks.py b/docs/_hooks/hooks.py
@@ -0,0 +1,20 @@
+# mypy: disable-error-code="typeddict-unknown-key"
+from datetime import datetime
+
+from mkdocs.config.defaults import MkDocsConfig
+from mkdocs.structure.files import Files
+from mkdocs.structure.nav import Navigation
+from mkdocs.structure.pages import Page
+from mkdocs.utils.templates import TemplateContext
+
+
+def on_pre_build(config: MkDocsConfig) -> None:
+    pass
+
+
+def on_page_markdown(markdown: str, page: Page, config: MkDocsConfig, files: Files) -> None:
+    pass
+
+
+def on_page_context(context: TemplateContext, nav: Navigation, page: Page, config: MkDocsConfig) -> None:
+    context["build_date"] = datetime.now().strftime("%a, %d, %b %Y - %H:%M")