From d71921d055910b27475dc1a5ba1fc8ad4739d564 Mon Sep 17 00:00:00 2001
From: Vilem Ded <51602017+vildead@users.noreply.github.com>
Date: Wed, 29 May 2024 09:20:08 +0200
Subject: [PATCH 01/43] Update setup.py to contain current version

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 01d90695..de5652a9 100644
--- a/setup.py
+++ b/setup.py
@@ -61,7 +61,7 @@
 
 setup(
     name="elixir-daisy",
-    version="1.7.12",
+    version="1.8.1",
     description="Elixir-LU DAISY",
     author="Pinar Alper, Valentin Grouès, Yohan Jarosz, Jacek Lebioda, Kavita Rege, Vilem Ded",
     author_email="lcsb.sysadmins@uni.lu",

From 6ca6921e6edd852b89629081006bad1e9a4a6977 Mon Sep 17 00:00:00 2001
From: Vilem Ded <vilem.ded@uni.lu>
Date: Wed, 5 Jun 2024 17:16:56 +0200
Subject: [PATCH 02/43] bump version to 1.8.2

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index de5652a9..c56299f7 100644
--- a/setup.py
+++ b/setup.py
@@ -61,7 +61,7 @@
 
 setup(
     name="elixir-daisy",
-    version="1.8.1",
+    version="1.8.2",
     description="Elixir-LU DAISY",
     author="Pinar Alper, Valentin Grouès, Yohan Jarosz, Jacek Lebioda, Kavita Rege, Vilem Ded",
     author_email="lcsb.sysadmins@uni.lu",

From dbf5c1be938c44f1c72eee2670fa971ddbb77a96 Mon Sep 17 00:00:00 2001
From: Vilem Ded <vilem.ded@uni.lu>
Date: Mon, 26 Aug 2024 08:47:16 +0200
Subject: [PATCH 03/43] validate version in setup.py upon every release

---
 .github/workflows/release.yml | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 .github/workflows/release.yml

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
new file mode 100644
index 00000000..cecc2900
--- /dev/null
+++ b/.github/workflows/release.yml
@@ -0,0 +1,27 @@
+name: Validate Version bump upon release
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  validate-version:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+
+    - name: Validate version in setup.py
+      run: |
+        VERSION=$(python setup.py --version)
+        echo "Version in setup.py: $VERSION"
+        if [[ "$VERSION" != "$GITHUB_REF_NAME" ]]; then
+          echo "Version mismatch: setup.py ($VERSION) vs release tag ($GITHUB_REF_NAME)"
+          exit 1
+        fi
\ No newline at end of file

From 1cc604dd087d4e15b3f92ec150456cd213ddca06 Mon Sep 17 00:00:00 2001
From: Moustapha Cheikh <moustapha.cheikh@uni.lu>
Date: Tue, 17 Sep 2024 10:34:31 +0200
Subject: [PATCH 04/43] Add support for backup and restore using docker

---
 .env.template       |   9 +++
 backup.md           | 110 +++++++++++++++++++++++++++++++++
 db.sh               | 147 ++++++++++++++++++++++++++++++++++++++++++++
 docker-compose.yaml |  36 +++++++----
 legacy_restore.sh   |  78 +++++++++++++++++++++++
 setup_cron.sh       |  23 +++++++
 6 files changed, 392 insertions(+), 11 deletions(-)
 create mode 100644 .env.template
 create mode 100644 backup.md
 create mode 100644 db.sh
 create mode 100644 legacy_restore.sh
 create mode 100644 setup_cron.sh

diff --git a/.env.template b/.env.template
new file mode 100644
index 00000000..b8bfe9ee
--- /dev/null
+++ b/.env.template
@@ -0,0 +1,9 @@
+# Database configuration
+DB_NAME=daisy
+DB_USER=daisy
+DB_PASSWORD=daisy
+
+# Backup configuration
+BACKUP_VOLUME=../backups
+ENABLE_BACKUPS=true
+BACKUP_SCHEDULE="0 0 * * *"
diff --git a/backup.md b/backup.md
new file mode 100644
index 00000000..3740f1d8
--- /dev/null
+++ b/backup.md
@@ -0,0 +1,110 @@
+# Database Backup and Restore Script
+
+## Overview
+Manage PostgreSQL database backups and Django media files in a Docker environment using `tar.gz` archives.
+
+## Key Functions
+- **Backup**: Creates a timestamped `tar.gz` archive of the PostgreSQL database and Django media files.
+- **Restore**: Restores from a specified `tar.gz` backup archive.
+
+## Docker Compose Integration
+The `backup` service in `docker-compose.yaml` manages backup and restore using the `db.sh` script.
+
+### Configuration
+
+- **Environment Variables**:
+  - `DB_HOST` (default: `db`)
+  - `DB_PORT` (default: `5432`)
+  - `DB_NAME` (default: `daisy`)
+  - `DB_USER` (default: `daisy`)
+  - `DB_PASSWORD` (default: `daisy`)
+  - `BACKUP_DIR` (default: `../backups`)
+  - `ENABLE_BACKUPS` (default: `true`)
+  - `BACKUP_SCHEDULE` (default: `"0 0 * * *"`)
+
+- **Volumes**:
+  - `${BACKUP_VOLUME:-../backups}:/backups`
+  - `.:/code`
+
+### Operations
+
+#### Enable Automatic Backups
+To ensure automatic backups are enabled, set `ENABLE_BACKUPS=true` (enabled by default):
+
+To checkout if the cron is added 
+
+```bash
+docker compose exec backup crontab -l
+```
+
+```bash
+ENABLE_BACKUPS=true docker compose up -d backup
+```
+This will configure automatic backups based on the `BACKUP_SCHEDULE`.
+
+#### Automatic Backups
+- Enabled by default (`ENABLE_BACKUPS=true`).
+- Schedule defined by `BACKUP_SCHEDULE` (cron format).
+
+To disable automatic backups:
+```bash
+ENABLE_BACKUPS=false docker compose up -d backup
+```
+
+To checkout if the cron was removed 
+
+```bash
+docker compose exec backup crontab -l
+```
+
+
+#### Manual Backup
+Create a manual backup:
+```bash
+docker compose exec backup /code/db.sh backup
+```
+- **Output**: `backup_<timestamp>.tar.gz` in the `BACKUP_DIR` (`../backups` by default).
+
+#### Restore Backup
+Restore from a specific backup file:
+```bash
+docker compose exec backup /code/db.sh restore ../backups/backup_<timestamp>.tar.gz
+docker compose run web python manage.py rebuild_index --noinput
+```
+- Replace `../backups/backup_<timestamp>.tar.gz` with the actual file path.
+
+
+#### List Cron Jobs
+View the automatic backup schedule:
+```bash
+docker compose exec backup crontab -l
+```
+
+#### List Backup Contents
+View contents of a backup archive:
+```bash
+tar -ztvf ../backups/backup_<timestamp>.tar.gz
+```
+
+
+#### Restore Legacy Backup
+Execute the `legacy_restore.sh` script inside the running container
+
+```bash
+# Copy the legacy backup file to the backup container
+docker cp ../daisy.tar.gz $(docker compose ps -q backup):/code/daisy.tar.gz
+
+# Execute the legacy_restore.sh script inside the running container
+docker compose exec backup /bin/sh -c "/code/legacy_restore.sh /code/daisy.tar.gz && rm /code/daisy.tar.gz"
+docker compose run web python manage.py rebuild_index --noinput
+```
+Replace `../daisy.tar.gz` with the actual path to legacy backup file.
+
+
+#### Reindex Solr
+
+To rebuild the Solr search index after a restore operation:
+
+```bash
+docker compose run web python manage.py rebuild_index --noinput
+```
\ No newline at end of file
diff --git a/db.sh b/db.sh
new file mode 100644
index 00000000..26ee25ef
--- /dev/null
+++ b/db.sh
@@ -0,0 +1,147 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Configuration
+BACKUP_DIR="${BACKUP_DIR:-../backups}"     # Directory to store backups
+DB_HOST="${DB_HOST:-db}"                   # PostgreSQL host
+DB_PORT="${DB_PORT:-5432}"                 # PostgreSQL port
+DB_NAME="${DB_NAME:-daisy}"                # PostgreSQL database name
+DB_USER="${DB_USER:-daisy}"                # PostgreSQL user
+DB_PASSWORD="${DB_PASSWORD:-daisy}"        # PostgreSQL password
+MEDIA_DIR="${MEDIA_DIR:-/code/documents}"  # Django media directory
+
+if [ "${BACKUP_DIR}" == "../backups" ] && [ ! -d "${BACKUP_DIR}" ]; then
+    mkdir -p "${BACKUP_DIR}"
+fi
+
+# Function to perform backup
+backup() {
+    local timestamp
+    timestamp=$(date +%Y-%m-%d_%H-%M-%S)
+    local backup_file="${BACKUP_DIR}/backup_${timestamp}.tar.gz"
+    local temp_backup_dir="${BACKUP_DIR}/temp_${timestamp}"
+
+    echo "Starting backup..."
+
+    # Create temporary backup directory
+    mkdir -p "${temp_backup_dir}"
+
+    # Backup database as SQL dump
+    echo "Backing up PostgreSQL database..."
+    if ! PGPASSWORD="${DB_PASSWORD}" pg_dump -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -d "${DB_NAME}" -F p -f "${temp_backup_dir}/db_backup.sql"; then
+        echo "ERROR: Database backup failed" >&2
+        rm -rf "${temp_backup_dir}"
+        exit 1
+    fi
+
+    # Backup media files
+    echo "Backing up Django media files..."
+    if [ -d "${MEDIA_DIR}" ]; then
+        if ! cp -R "${MEDIA_DIR}" "${temp_backup_dir}/documents"; then
+            echo "ERROR: Media files backup failed" >&2
+            rm -rf "${temp_backup_dir}"
+            exit 1
+        fi
+    else
+        echo "WARNING: Media directory not found. Skipping media backup."
+    fi
+
+    # Create tar.gz archive of the backups
+    echo "Creating tar.gz archive..."
+    if ! tar -czf "${backup_file}" -C "${BACKUP_DIR}" "temp_${timestamp}"; then
+        echo "ERROR: Archive creation failed" >&2
+        rm -rf "${temp_backup_dir}"
+        exit 1
+    fi
+
+    # Remove temporary backup directory
+    rm -rf "${temp_backup_dir}"
+
+    echo "Backup completed successfully: ${backup_file}"
+}
+
+# Function to perform restore
+restore() {
+    if [ $# -ne 1 ]; then
+        echo "Usage: $0 restore <backup_file.tar.gz>" >&2
+        exit 1
+    fi
+
+    local backup_file="$1"
+    local tmp_restore_dir="${BACKUP_DIR}/restore_temp"
+
+    if [ ! -f "${backup_file}" ]; then
+        echo "ERROR: Backup file not found: ${backup_file}" >&2
+        exit 1
+    fi
+
+    echo "Starting restoration..."
+
+    # Drop the existing database
+    echo "Dropping existing database..."
+    if ! PGPASSWORD="${DB_PASSWORD}" psql -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -d postgres -c "DROP DATABASE IF EXISTS ${DB_NAME};"; then
+        echo "ERROR: Failed to drop existing database" >&2
+        exit 1
+    fi
+
+    # Create a new database
+    echo "Creating new database..."
+    if ! PGPASSWORD="${DB_PASSWORD}" psql -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -d postgres -c "CREATE DATABASE ${DB_NAME};"; then
+        echo "ERROR: Failed to create new database" >&2
+        exit 1
+    fi
+
+    # Extract the backup archive
+    mkdir -p "${tmp_restore_dir}"
+    if ! tar -xzf "${backup_file}" -C "${tmp_restore_dir}"; then
+        echo "ERROR: Failed to extract backup archive" >&2
+        rm -rf "${tmp_restore_dir}"
+        exit 1
+    fi
+
+    # Identify the extracted directory (e.g., temp_2023-10-05_12-00-00)
+    local extracted_dir
+    extracted_dir=$(ls "${tmp_restore_dir}")
+
+    # Restore PostgreSQL database from SQL dump
+    echo "Restoring PostgreSQL database..."
+    if ! PGPASSWORD="${DB_PASSWORD}" psql -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -d "${DB_NAME}" -f "${tmp_restore_dir}/${extracted_dir}/db_backup.sql"; then
+        echo "ERROR: Database restoration failed" >&2
+        rm -rf "${tmp_restore_dir}"
+        exit 1
+    fi
+
+    # Restore media files
+    echo "Restoring Django media files..."
+    if [ -d "${tmp_restore_dir}/${extracted_dir}/documents" ]; then
+        rm -rf "${MEDIA_DIR:?}/"*
+        if ! cp -R "${tmp_restore_dir}/${extracted_dir}/documents/"* "${MEDIA_DIR}/"; then
+            echo "ERROR: Media files restoration failed" >&2
+            rm -rf "${tmp_restore_dir}"
+            exit 1
+        fi
+    else
+        echo "WARNING: No media backup found in the archive. Skipping media restoration."
+    fi
+
+    # Remove temporary restoration directory
+    rm -rf "${tmp_restore_dir}"
+
+    echo "Restoration completed successfully."
+}
+
+# Main script logic
+case "$1" in
+    backup)
+        backup
+        ;;
+    restore)
+        shift
+        restore "$@"
+        ;;
+    *)
+        echo "Usage: $0 {backup|restore <backup_file.tar.gz>}" >&2
+        exit 1
+        ;;
+esac
\ No newline at end of file
diff --git a/docker-compose.yaml b/docker-compose.yaml
index 8fa5fb6d..76cab05a 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -1,5 +1,3 @@
-version: '3'
-
 services:
     # web app
     web:
@@ -22,14 +20,12 @@ services:
         image: postgres:10.1
         restart: on-failure
         environment:
-            POSTGRES_PASSWORD: daisy
-            POSTGRES_USER: daisy
-            POSTGRES_DB: daisy
-        ports:
-            - "5433:5432"
+            POSTGRES_PASSWORD: ${DB_PASSWORD:-daisy}
+            POSTGRES_USER: ${DB_USER:-daisy}
+            POSTGRES_DB: ${DB_NAME:-daisy}
         volumes:
             - pgdata:/var/lib/postgresql/data
-    # web werver frontend
+    # web server frontend
     nginx:
         build: ./docker/nginx
         restart: on-failure
@@ -47,14 +43,13 @@ services:
             - "8983:8983"
         volumes:
             - solrdata:/opt/solr/server/solr
-     # rabbit mq
+    # rabbit mq
     mq:
         image: rabbitmq:3.9-management-alpine
         restart: on-failure
         ports:
             - "15672:15672"
             - "5672:5672"
-
     # task monitoring
     flower:
         image: mher/flower:0.9.7
@@ -85,9 +80,28 @@ services:
             - db
             - mq
         command: "celery -A elixir_daisy beat --loglevel=DEBUG --pidfile= --scheduler django_celery_beat.schedulers:DatabaseScheduler"
-
+    # Backup service
+    backup:
+        image: alpine:3.20.3
+        environment:
+            - DB_HOST=db
+            - DB_PORT=5432
+            - DB_NAME=${DB_NAME:-daisy}
+            - DB_USER=${DB_USER:-daisy}
+            - DB_PASSWORD=${DB_PASSWORD:-daisy}
+            - BACKUP_DIR=/backups
+            - MEDIA_DIR=/code/documents
+            - ENABLE_BACKUPS=${ENABLE_BACKUPS:-true}
+            - BACKUP_SCHEDULE=${BACKUP_SCHEDULE:-"0 0 * * *"}
+        volumes:
+            - ${BACKUP_VOLUME:-../backups}:/backups
+            - .:/code
+        depends_on:
+            - db
+        entrypoint: /code/setup_cron.sh
 
 volumes:
     pgdata:
     statics:
     solrdata:
+    backups:
diff --git a/legacy_restore.sh b/legacy_restore.sh
new file mode 100644
index 00000000..1c754110
--- /dev/null
+++ b/legacy_restore.sh
@@ -0,0 +1,78 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Configuration
+DB_HOST="${DB_HOST:-db}"
+DB_PORT="${DB_PORT:-5432}"
+DB_NAME="${DB_NAME:-daisy}"
+DB_USER="${DB_USER:-daisy}"
+DB_PASSWORD="${DB_PASSWORD:-daisy}"
+MEDIA_DIR="${MEDIA_DIR:-/code/documents}"
+
+if [ $# -ne 1 ]; then
+    echo "Usage: $0 path_to_daisy_backup_tar.gz" >&2
+    exit 1
+fi
+
+TAR_FILE=$1
+TEMP_DIR=$(mktemp -d)
+
+echo "Starting restoration..."
+
+# Extract the backup archive
+echo "Extracting files..."
+if ! tar -xzvf "$TAR_FILE" -C "$TEMP_DIR" \
+    --strip-components=2 \
+    home/daisy/daisy_dump.sql \
+    home/daisy/daisy/documents; then
+    echo "ERROR: Failed to extract backup archive" >&2
+    rm -rf "$TEMP_DIR"
+    exit 1
+fi
+
+echo "Extraction successful."
+echo "SQL dump: $TEMP_DIR/daisy_dump.sql"
+echo "Documents: $TEMP_DIR/daisy/documents"
+
+# Drop the existing database
+echo "Dropping existing database..."
+if ! PGPASSWORD="${DB_PASSWORD}" psql -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -d postgres -c "DROP DATABASE IF EXISTS ${DB_NAME};"; then
+    echo "ERROR: Failed to drop existing database" >&2
+    rm -rf "$TEMP_DIR"
+    exit 1
+fi
+
+# Create a new database
+echo "Creating new database..."
+if ! PGPASSWORD="${DB_PASSWORD}" psql -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -d postgres -c "CREATE DATABASE ${DB_NAME};"; then
+    echo "ERROR: Failed to create new database" >&2
+    rm -rf "$TEMP_DIR"
+    exit 1
+fi
+
+# Restore PostgreSQL database from SQL dump
+echo "Restoring PostgreSQL database..."
+if ! PGPASSWORD="${DB_PASSWORD}" psql -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -d "${DB_NAME}" -f "${TEMP_DIR}/daisy_dump.sql"; then
+    echo "ERROR: Database restoration failed" >&2
+    rm -rf "$TEMP_DIR"
+    exit 1
+fi
+
+# Restore media files
+echo "Restoring Django media files..."
+if [ -d "${TEMP_DIR}/daisy/documents" ]; then
+    rm -rf "${MEDIA_DIR:?}/"*
+    if ! cp -R "${TEMP_DIR}/daisy/documents/"* "${MEDIA_DIR}/"; then
+        echo "ERROR: Media files restoration failed" >&2
+        rm -rf "$TEMP_DIR"
+        exit 1
+    fi
+else
+    echo "WARNING: No media backup found in the archive. Skipping media restoration."
+fi
+
+# Remove temporary directory
+rm -rf "$TEMP_DIR"
+
+echo "Restoration completed successfully."
diff --git a/setup_cron.sh b/setup_cron.sh
new file mode 100644
index 00000000..77b0085c
--- /dev/null
+++ b/setup_cron.sh
@@ -0,0 +1,23 @@
+#!/bin/sh
+
+# Ensure the backup script is executable
+chmod +x /code/db.sh
+
+# Check if backups are enabled
+if [ "$ENABLE_BACKUPS" = "true" ]; then
+  # Install necessary packages
+  apk add --no-cache bash curl tar gzip postgresql-client openrc cronie
+
+  # Set up the crontab entry
+  echo "$BACKUP_SCHEDULE /code/db.sh backup" | tr -d '"' > /etc/crontabs/root
+
+  # Print the crontab contents for debugging
+  echo "Crontab contents:"
+  cat /etc/crontabs/root
+
+  # Start the cron daemon
+  crond -f
+else
+  echo "Backups are disabled. Set ENABLE_BACKUPS=true to enable."
+  tail -f /dev/null
+fi
\ No newline at end of file

From d9ffb8d1e10bac5292b1d6c3cfa651e62e1a0324 Mon Sep 17 00:00:00 2001
From: Moustapha Cheikh <moustapha.cheikh@uni.lu>
Date: Tue, 17 Sep 2024 10:38:41 +0200
Subject: [PATCH 05/43] Update main.yml

---
 .github/workflows/main.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 3dbb49f8..fcdfd6b6 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -25,17 +25,17 @@ jobs:
       run: cp elixir_daisy/settings_compose_ci.py elixir_daisy/settings_compose.py
 
     - name: Build and start containers
-      run: docker-compose up -d --build
+      run: docker compose up -d --build
 
     - name: Check code formatting with Black
-      run: docker-compose exec -T web black --check --verbose .
+      run: docker compose exec -T web black --check --verbose .
 
     - name: Install test dependencies
-      run: docker-compose exec -T web pip install ".[test]"
+      run: docker compose exec -T web pip install ".[test]"
 
     - name: Execute the tests
-      run: docker-compose exec -T web pytest
+      run: docker compose exec -T web pytest
 
     - name: Stop containers
       if: always()
-      run: docker-compose down
+      run: docker compose down

From b617baa2b1d9aeb4a5a820f7ee95e3d5eaf0f7e8 Mon Sep 17 00:00:00 2001
From: Moustapha Cheikh <moustapha.cheikh@uni.lu>
Date: Tue, 17 Sep 2024 10:45:08 +0200
Subject: [PATCH 06/43] changes the permissions before executing setup_cron.sh

---
 docker-compose.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index 76cab05a..030908fe 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -91,14 +91,15 @@ services:
             - DB_PASSWORD=${DB_PASSWORD:-daisy}
             - BACKUP_DIR=/backups
             - MEDIA_DIR=/code/documents
-            - ENABLE_BACKUPS=${ENABLE_BACKUPS:-true}
+            - ENABLE_BACKUPS=${ENABLE_BACKUPS:-false}
             - BACKUP_SCHEDULE=${BACKUP_SCHEDULE:-"0 0 * * *"}
         volumes:
             - ${BACKUP_VOLUME:-../backups}:/backups
             - .:/code
         depends_on:
             - db
-        entrypoint: /code/setup_cron.sh
+        entrypoint: >
+            sh -c "chmod +x /code/setup_cron.sh && exec /code/setup_cron.sh"
 
 volumes:
     pgdata:

From 5c38053afd220b0d50d42c3d99d5264a9d3f619a Mon Sep 17 00:00:00 2001
From: Moustapha Cheikh <moustapha.cheikh@uni.lu>
Date: Thu, 19 Sep 2024 09:50:11 +0200
Subject: [PATCH 07/43] Stop services before backup and restore

---
 backup.md           |  6 +++---
 db.sh               | 21 ++++++++++++++++++++-
 docker-compose.yaml | 38 ++++++++++++++++++++++++++++++++++----
 3 files changed, 57 insertions(+), 8 deletions(-)

diff --git a/backup.md b/backup.md
index 3740f1d8..62eae9fc 100644
--- a/backup.md
+++ b/backup.md
@@ -61,14 +61,14 @@ docker compose exec backup crontab -l
 #### Manual Backup
 Create a manual backup:
 ```bash
-docker compose exec backup /code/db.sh backup
+docker compose exec backup sh /code/db.sh backup
 ```
 - **Output**: `backup_<timestamp>.tar.gz` in the `BACKUP_DIR` (`../backups` by default).
 
 #### Restore Backup
 Restore from a specific backup file:
 ```bash
-docker compose exec backup /code/db.sh restore ../backups/backup_<timestamp>.tar.gz
+docker compose exec backup sh /code/db.sh restore ../backups/backup_<timestamp>.tar.gz
 docker compose run web python manage.py rebuild_index --noinput
 ```
 - Replace `../backups/backup_<timestamp>.tar.gz` with the actual file path.
@@ -95,7 +95,7 @@ Execute the `legacy_restore.sh` script inside the running container
 docker cp ../daisy.tar.gz $(docker compose ps -q backup):/code/daisy.tar.gz
 
 # Execute the legacy_restore.sh script inside the running container
-docker compose exec backup /bin/sh -c "/code/legacy_restore.sh /code/daisy.tar.gz && rm /code/daisy.tar.gz"
+docker compose exec backup /bin/sh -c "sh /code/legacy_restore.sh /code/daisy.tar.gz && rm /code/daisy.tar.gz"
 docker compose run web python manage.py rebuild_index --noinput
 ```
 Replace `../daisy.tar.gz` with the actual path to legacy backup file.
diff --git a/db.sh b/db.sh
index 26ee25ef..40a0e77c 100644
--- a/db.sh
+++ b/db.sh
@@ -15,6 +15,21 @@ if [ "${BACKUP_DIR}" == "../backups" ] && [ ! -d "${BACKUP_DIR}" ]; then
     mkdir -p "${BACKUP_DIR}"
 fi
 
+stop_services() {
+    echo "Stopping: nginx, flower, worker, beat, web, mq, solr"
+    docker compose stop nginx flower worker beat web mq solr
+    echo "Services stopped."
+}
+
+start_services() {
+    echo "Starting: solr, mq, web, worker, beat, flower, nginx"
+    docker compose up -d solr mq web worker beat flower nginx || {
+        echo "ERROR: Service startup failed. Check Docker logs." >&2
+        exit 1
+    }
+    echo "Services started."
+}
+
 # Function to perform backup
 backup() {
     local timestamp
@@ -134,14 +149,18 @@ restore() {
 # Main script logic
 case "$1" in
     backup)
+        stop_services
         backup
+        start_services
         ;;
     restore)
         shift
+        stop_services
         restore "$@"
+        start_services
         ;;
     *)
         echo "Usage: $0 {backup|restore <backup_file.tar.gz>}" >&2
         exit 1
         ;;
-esac
\ No newline at end of file
+esac
diff --git a/docker-compose.yaml b/docker-compose.yaml
index 030908fe..a41242f9 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -10,6 +10,8 @@ services:
             - statics:/static
             - solrdata:/solr
             - .:/code
+        networks:
+            - daisy_network
         depends_on:
             - db
             - solr
@@ -18,35 +20,44 @@ services:
     # database
     db:
         image: postgres:10.1
-        restart: on-failure
+        restart: unless-stopped
         environment:
             POSTGRES_PASSWORD: ${DB_PASSWORD:-daisy}
             POSTGRES_USER: ${DB_USER:-daisy}
             POSTGRES_DB: ${DB_NAME:-daisy}
+        networks:
+            - daisy_network
         volumes:
             - pgdata:/var/lib/postgresql/data
     # web server frontend
     nginx:
         build: ./docker/nginx
-        restart: on-failure
+        restart: unless-stopped
         volumes:
             - statics:/public/static:ro
         ports:
             - '80:80'
             - '443:443'
+        networks:
+            - daisy_network
         depends_on:
             - web
     # FTS db
     solr:
         build: ./docker/solr
+        restart: unless-stopped
         ports:
             - "8983:8983"
+        networks:
+            - daisy_network
         volumes:
             - solrdata:/opt/solr/server/solr
     # rabbit mq
     mq:
         image: rabbitmq:3.9-management-alpine
-        restart: on-failure
+        restart: unless-stopped
+        networks:
+            - daisy_network
         ports:
             - "15672:15672"
             - "5672:5672"
@@ -56,6 +67,9 @@ services:
         command: --broker=amqp://guest:guest@mq:5672// --broker_api=http://guest:guest@mq:15672/api/
         ports:
             - "5555:5555"
+        restart: unless-stopped
+        networks:
+            - daisy_network
         depends_on:
             - mq
     # task runner
@@ -68,6 +82,9 @@ services:
         depends_on:
             - db
             - mq
+        restart: unless-stopped
+        networks:
+            - daisy_network
         command: "celery -A elixir_daisy.celery_app worker --loglevel=DEBUG"
     # celery beat
     beat:
@@ -79,6 +96,8 @@ services:
         depends_on:
             - db
             - mq
+        networks:
+            - daisy_network
         command: "celery -A elixir_daisy beat --loglevel=DEBUG --pidfile= --scheduler django_celery_beat.schedulers:DatabaseScheduler"
     # Backup service
     backup:
@@ -93,16 +112,27 @@ services:
             - MEDIA_DIR=/code/documents
             - ENABLE_BACKUPS=${ENABLE_BACKUPS:-false}
             - BACKUP_SCHEDULE=${BACKUP_SCHEDULE:-"0 0 * * *"}
+            - SOLR_PORT=8983
+            - COMPOSE_PROJECT_NAME=daisy
+            - RABBITMQ_MANAGEMENT_PORT=${RABBITMQ_MANAGEMENT_PORT:-15672}
         volumes:
             - ${BACKUP_VOLUME:-../backups}:/backups
             - .:/code
+            - /var/run/docker.sock:/var/run/docker.sock
+        working_dir: /code
         depends_on:
             - db
+        networks:
+            - daisy_network
         entrypoint: >
-            sh -c "chmod +x /code/setup_cron.sh && exec /code/setup_cron.sh"
+            sh -c "apk add --no-cache docker-cli docker-compose postgresql-client lsof && chmod +x /code/setup_cron.sh && chmod +x /code/db.sh && exec /code/setup_cron.sh"
 
 volumes:
     pgdata:
     statics:
     solrdata:
     backups:
+
+networks:
+  daisy_network:
+    driver: bridge

From 6fbf038c27cc8265c3f93dba6061198909e6c791 Mon Sep 17 00:00:00 2001
From: Moustapha Cheikh <moustapha.cheikh@uni.lu>
Date: Fri, 20 Sep 2024 10:57:46 +0200
Subject: [PATCH 08/43] Update legecy restore script

---
 backup.md         |  4 +--
 legacy_restore.sh | 76 +++++++++++++++++++++++++++++++----------------
 2 files changed, 53 insertions(+), 27 deletions(-)

diff --git a/backup.md b/backup.md
index 62eae9fc..1d2d6e7a 100644
--- a/backup.md
+++ b/backup.md
@@ -92,10 +92,10 @@ Execute the `legacy_restore.sh` script inside the running container
 
 ```bash
 # Copy the legacy backup file to the backup container
-docker cp ../daisy.tar.gz $(docker compose ps -q backup):/code/daisy.tar.gz
+docker cp ../daisy.tar.gz $(docker compose ps -q backup):/code/daisy_prod.tar.gz
 
 # Execute the legacy_restore.sh script inside the running container
-docker compose exec backup /bin/sh -c "sh /code/legacy_restore.sh /code/daisy.tar.gz && rm /code/daisy.tar.gz"
+docker compose exec backup /bin/sh -c "sh /code/legacy_restore.sh /code/daisy_prod.tar.gz && rm /code/daisy_prod.tar.gz"
 docker compose run web python manage.py rebuild_index --noinput
 ```
 Replace `../daisy.tar.gz` with the actual path to legacy backup file.
diff --git a/legacy_restore.sh b/legacy_restore.sh
index 1c754110..e2ff16a2 100644
--- a/legacy_restore.sh
+++ b/legacy_restore.sh
@@ -9,6 +9,7 @@ DB_NAME="${DB_NAME:-daisy}"
 DB_USER="${DB_USER:-daisy}"
 DB_PASSWORD="${DB_PASSWORD:-daisy}"
 MEDIA_DIR="${MEDIA_DIR:-/code/documents}"
+SHOW_DB_LOGS="${SHOW_DB_LOGS:-true}"
 
 if [ $# -ne 1 ]; then
     echo "Usage: $0 path_to_daisy_backup_tar.gz" >&2
@@ -18,52 +19,77 @@ fi
 TAR_FILE=$1
 TEMP_DIR=$(mktemp -d)
 
-echo "Starting restoration..."
+echo "Step 1: Starting restoration process..."
 
-# Extract the backup archive
-echo "Extracting files..."
-if ! tar -xzvf "$TAR_FILE" -C "$TEMP_DIR" \
-    --strip-components=2 \
-    home/daisy/daisy_dump.sql \
-    home/daisy/daisy/documents; then
+echo "Step 2: Extracting backup archive..."
+if ! tar -xzf "$TAR_FILE" -C "$TEMP_DIR" > /dev/null 2>&1; then
     echo "ERROR: Failed to extract backup archive" >&2
     rm -rf "$TEMP_DIR"
     exit 1
 fi
 
-echo "Extraction successful."
-echo "SQL dump: $TEMP_DIR/daisy_dump.sql"
-echo "Documents: $TEMP_DIR/daisy/documents"
+echo "Step 3: Locating SQL dump and documents directory..."
+SQL_DUMP=$(find "$TEMP_DIR" -name "daisy_dump.sql")
+DOCUMENTS_DIR=$(find "$TEMP_DIR" -type d -name "documents" ! -path "*/templates/*" | head -n 1)
 
-# Drop the existing database
-echo "Dropping existing database..."
-if ! PGPASSWORD="${DB_PASSWORD}" psql -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -d postgres -c "DROP DATABASE IF EXISTS ${DB_NAME};"; then
+echo "Step 4: Verifying extracted contents..."
+echo "  - SQL dump found: $([ -n "$SQL_DUMP" ] && echo "Yes" || echo "No")"
+echo "  - Documents directory found: $([ -n "$DOCUMENTS_DIR" ] && echo "Yes" || echo "No")"
+
+if [ -z "$SQL_DUMP" ]; then
+    echo "ERROR: Could not find daisy_dump.sql in the archive" >&2
+    rm -rf "$TEMP_DIR"
+    exit 1
+fi
+
+echo "Step 5: Extraction successful."
+echo "  - SQL dump location: $SQL_DUMP"
+echo "  - Documents location: $DOCUMENTS_DIR"
+
+echo "Step 6: Dropping existing database..."
+if [ "$SHOW_DB_LOGS" = "true" ]; then
+    PGPASSWORD="${DB_PASSWORD}" psql -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -d postgres -c "DROP DATABASE IF EXISTS ${DB_NAME};"
+else
+    PGPASSWORD="${DB_PASSWORD}" psql -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -d postgres -c "DROP DATABASE IF EXISTS ${DB_NAME};" > /dev/null 2>&1
+fi
+
+if [ $? -ne 0 ]; then
     echo "ERROR: Failed to drop existing database" >&2
     rm -rf "$TEMP_DIR"
     exit 1
 fi
 
-# Create a new database
-echo "Creating new database..."
-if ! PGPASSWORD="${DB_PASSWORD}" psql -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -d postgres -c "CREATE DATABASE ${DB_NAME};"; then
+echo "Step 7: Creating new database..."
+if [ "$SHOW_DB_LOGS" = "true" ]; then
+    PGPASSWORD="${DB_PASSWORD}" psql -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -d postgres -c "CREATE DATABASE ${DB_NAME};"
+else
+    PGPASSWORD="${DB_PASSWORD}" psql -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -d postgres -c "CREATE DATABASE ${DB_NAME};" > /dev/null 2>&1
+fi
+
+if [ $? -ne 0 ]; then
     echo "ERROR: Failed to create new database" >&2
     rm -rf "$TEMP_DIR"
     exit 1
 fi
 
-# Restore PostgreSQL database from SQL dump
-echo "Restoring PostgreSQL database..."
-if ! PGPASSWORD="${DB_PASSWORD}" psql -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -d "${DB_NAME}" -f "${TEMP_DIR}/daisy_dump.sql"; then
+echo "Step 8: Restoring PostgreSQL database from SQL dump..."
+if [ "$SHOW_DB_LOGS" = "true" ]; then
+    PGPASSWORD="${DB_PASSWORD}" psql -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -d "${DB_NAME}" -f "${SQL_DUMP}"
+else
+    PGPASSWORD="${DB_PASSWORD}" psql -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -d "${DB_NAME}" -f "${SQL_DUMP}" > /dev/null 2>&1
+fi
+
+if [ $? -ne 0 ]; then
     echo "ERROR: Database restoration failed" >&2
     rm -rf "$TEMP_DIR"
     exit 1
 fi
 
-# Restore media files
-echo "Restoring Django media files..."
-if [ -d "${TEMP_DIR}/daisy/documents" ]; then
+echo "Step 9: Restoring Django media files..."
+if [ -n "$DOCUMENTS_DIR" ] && [ -d "$DOCUMENTS_DIR" ]; then
+    echo "  - Copying files from $DOCUMENTS_DIR to $MEDIA_DIR"
     rm -rf "${MEDIA_DIR:?}/"*
-    if ! cp -R "${TEMP_DIR}/daisy/documents/"* "${MEDIA_DIR}/"; then
+    if ! cp -R "${DOCUMENTS_DIR}/." "${MEDIA_DIR}/"; then
         echo "ERROR: Media files restoration failed" >&2
         rm -rf "$TEMP_DIR"
         exit 1
@@ -72,7 +98,7 @@ else
     echo "WARNING: No media backup found in the archive. Skipping media restoration."
 fi
 
-# Remove temporary directory
+echo "Step 10: Cleaning up temporary files..."
 rm -rf "$TEMP_DIR"
 
-echo "Restoration completed successfully."
+echo "Step 11: Restoration completed successfully."

From 7e9662522ed4ae25ed8351a77a66d493f55c9c5f Mon Sep 17 00:00:00 2001
From: Vilem Ded <vilem.ded@uni.lu>
Date: Fri, 20 Sep 2024 14:22:45 +0200
Subject: [PATCH 09/43] add missing endline

---
 .github/workflows/release.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index cecc2900..44b824c1 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -24,4 +24,4 @@ jobs:
         if [[ "$VERSION" != "$GITHUB_REF_NAME" ]]; then
           echo "Version mismatch: setup.py ($VERSION) vs release tag ($GITHUB_REF_NAME)"
           exit 1
-        fi
\ No newline at end of file
+        fi

From d2552d8bea8a482bf2d519354fca36042608d17e Mon Sep 17 00:00:00 2001
From: Vilem Ded <vilem.ded@uni.lu>
Date: Fri, 20 Sep 2024 14:31:53 +0200
Subject: [PATCH 10/43] docker compose with space

---
 .github/workflows/main.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 3dbb49f8..fcdfd6b6 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -25,17 +25,17 @@ jobs:
       run: cp elixir_daisy/settings_compose_ci.py elixir_daisy/settings_compose.py
 
     - name: Build and start containers
-      run: docker-compose up -d --build
+      run: docker compose up -d --build
 
     - name: Check code formatting with Black
-      run: docker-compose exec -T web black --check --verbose .
+      run: docker compose exec -T web black --check --verbose .
 
     - name: Install test dependencies
-      run: docker-compose exec -T web pip install ".[test]"
+      run: docker compose exec -T web pip install ".[test]"
 
     - name: Execute the tests
-      run: docker-compose exec -T web pytest
+      run: docker compose exec -T web pytest
 
     - name: Stop containers
       if: always()
-      run: docker-compose down
+      run: docker compose down

From d018b817ecde53b079e573bb0c5893a3f8441b70 Mon Sep 17 00:00:00 2001
From: Moustapha Cheikh <moustapha.cheikh@uni.lu>
Date: Mon, 23 Sep 2024 10:50:30 +0200
Subject: [PATCH 11/43] Update docker compose setup

---
 .env.template                 |   2 -
 DAISY_Docker_Compose_Guide.md | 644 ++++++++++++++++++++++++++++++++++
 backup.md                     | 110 ------
 backup_script.sh              |  10 +
 db.sh                         |  19 -
 docker-compose.yaml           |   7 +-
 setup_cron.sh                 |  23 --
 7 files changed, 657 insertions(+), 158 deletions(-)
 create mode 100644 DAISY_Docker_Compose_Guide.md
 delete mode 100644 backup.md
 create mode 100755 backup_script.sh
 mode change 100644 => 100755 db.sh
 delete mode 100644 setup_cron.sh

diff --git a/.env.template b/.env.template
index b8bfe9ee..f51ba9e7 100644
--- a/.env.template
+++ b/.env.template
@@ -5,5 +5,3 @@ DB_PASSWORD=daisy
 
 # Backup configuration
 BACKUP_VOLUME=../backups
-ENABLE_BACKUPS=true
-BACKUP_SCHEDULE="0 0 * * *"
diff --git a/DAISY_Docker_Compose_Guide.md b/DAISY_Docker_Compose_Guide.md
new file mode 100644
index 00000000..02ed9da9
--- /dev/null
+++ b/DAISY_Docker_Compose_Guide.md
@@ -0,0 +1,644 @@
+# Daisy Project Setup Guide
+
+This guide provides concise instructions for setting up and running the Daisy project using Docker Compose. It includes commands for managing the Django application and other services within the project.
+
+---
+
+## Table of Contents
+
+- [Prerequisites](#prerequisites)
+- [Getting Started](#getting-started)
+  - [Clone the Repository](#clone-the-repository)
+  - [Environment Variables](#environment-variables)
+- [Running the Project](#running-the-project)
+  - [Build and Start Services](#build-and-start-services)
+  - [Initialize the Database](#initialize-the-database)
+  - [Build the Solr Schema](#build-the-solr-schema)
+  - [Compile and Deploy Static Files](#compile-and-deploy-static-files)
+  - [Load Initial Data into the Database](#load-initial-data-into-the-database)
+  - [Load Demo Data (Optional)](#load-demo-data-optional)
+  - [Build the Search Index](#build-the-search-index)
+  - [Access the Application](#access-the-application)
+- [Managing the Django Application](#managing-the-django-application)
+  - [Access the Web Service](#access-the-web-service)
+  - [Run Django Commands](#run-django-commands)
+- [Managing Other Services](#managing-other-services)
+  - [PostgreSQL Database (`db` Service)](#postgresql-database-db-service)
+  - [Solr (`solr` Service)](#solr-solr-service)
+  - [RabbitMQ (`mq` Service)](#rabbitmq-mq-service)
+  - [Celery Worker (`worker` Service)](#celery-worker-worker-service)
+  - [Celery Beat (`beat` Service)](#celery-beat-beat-service)
+  - [Flower Monitoring Tool (`flower` Service)](#flower-monitoring-tool-flower-service)
+- [Backup and Restore Operations](#backup-and-restore-operations)
+  - [Backup Service (`backup`)](#backup-service-backup)
+  - [Restore from Backup](#restore-from-backup)
+  - [Restore Legacy Backup](#restore-legacy-backup)
+- [Importing and Exporting Data](#importing-and-exporting-data)
+  - [Import Data](#import-data)
+  - [Export Data](#export-data)
+- [Updating the Project](#updating-the-project)
+  - [Pull Latest Changes](#pull-latest-changes)
+  - [Rebuild Services After Code Changes](#rebuild-services-after-code-changes)
+  - [Database Backup Before Upgrade](#database-backup-before-upgrade)
+  - [Upgrade Steps](#upgrade-steps)
+- [Administration](#administration)
+- [Settings Reference](#settings-reference)
+- [Additional Tips](#additional-tips)
+- [Docker Compose Services Overview](#docker-compose-services-overview)
+- [Logs and Monitoring](#logs-and-monitoring)
+- [Clean Up](#clean-up)
+
+---
+
+## Prerequisites
+
+- **Docker**
+- **Docker Compose**
+
+---
+
+## Getting Started
+
+### Clone the Repository
+
+```bash
+git clone https://github.com/elixir-luxembourg/daisy.git
+cd daisy
+```
+
+### Environment Variables
+
+Create a `.env` file in the project root to override default environment variables if necessary.
+
+Example `.env` file:
+
+```env
+DB_NAME=daisy
+DB_USER=daisy
+DB_PASSWORD=daisy
+BACKUP_VOLUME=../backups
+```
+
+---
+
+## Running the Project
+
+### Build and Start Services
+
+Build and start all services defined in `docker-compose.yaml`:
+
+```bash
+docker compose up -d --build
+```
+
+### Initialize the Database
+
+Run database migrations:
+
+```bash
+docker compose exec web python manage.py migrate
+```
+
+### Build the Solr Schema
+
+Build the Solr schema required for full-text search:
+
+```bash
+docker compose exec web python manage.py build_solr_schema -c /solr/daisy/conf -r daisy -u default
+```
+
+### Compile and Deploy Static Files
+
+The project uses frontend assets that need to be compiled (e.g., with npm), you need to build them and collect static files.
+
+#### Install npm Dependencies
+
+```bash
+cd web/static/vendor
+npm ci
+```
+
+#### Build Frontend Assets
+
+```bash
+npm run build
+```
+
+#### Collect Static Files
+
+From the project root:
+
+```bash
+docker compose exec web python manage.py collectstatic --noinput
+```
+
+### Load Initial Data into the Database
+
+Load initial data, such as controlled vocabularies and initial list of institutions and cohorts.
+
+```bash
+docker compose exec web bash -c "
+    cd core/fixtures/ && \
+    wget https://git-r3lab.uni.lu/pinar.alper/metadata-tools/raw/master/metadata_tools/resources/edda.json && \
+    wget https://git-r3lab.uni.lu/pinar.alper/metadata-tools/raw/master/metadata_tools/resources/hpo.json && \
+    wget https://git-r3lab.uni.lu/pinar.alper/metadata-tools/raw/master/metadata_tools/resources/hdo.json && \
+    wget https://git-r3lab.uni.lu/pinar.alper/metadata-tools/raw/master/metadata_tools/resources/hgnc.json
+"
+docker compose exec web python manage.py load_initial_data
+```
+
+**Note:** This step can take several minutes to complete.
+
+### Load Demo Data (Optional)
+
+To load demo data, including mock datasets, projects, and a demo admin account:
+
+```bash
+docker compose exec web python manage.py load_demo_data
+```
+
+### Build the Search Index
+
+After loading data, build the search index for Solr:
+
+```bash
+docker compose exec web python manage.py rebuild_index --noinput
+```
+
+### Access the Application
+
+The application should now be accessible at:
+
+- **HTTP:** `http://localhost/`
+- **HTTPS:** `https://localhost/`
+
+If you loaded the demo data, you can log in with the demo credentials provided during the demo data setup.
+
+---
+
+## Managing the Django Application
+
+### Access the Web Service
+
+Shell into the `web` container:
+
+```bash
+docker compose exec web /bin/bash
+```
+
+### Run Django Commands
+
+Run Django management commands using the `web` service.
+
+#### Make Migrations
+
+```bash
+docker compose exec web python manage.py makemigrations
+```
+
+#### Apply Migrations
+
+```bash
+docker compose exec web python manage.py migrate
+```
+
+#### Create Superuser
+
+```bash
+docker compose exec web python manage.py createsuperuser
+```
+
+#### Collect Static Files
+
+```bash
+docker compose exec web python manage.py collectstatic --noinput
+```
+
+#### Rebuild Solr Index
+
+```bash
+docker compose exec web python manage.py rebuild_index --noinput
+```
+
+---
+
+## Managing Other Services
+
+### PostgreSQL Database (`db` Service)
+
+#### Access the Database Shell
+
+```bash
+docker compose exec db psql -U daisy -d
+```
+
+#### Execute SQL Commands
+
+Run SQL commands directly:
+
+```bash
+docker compose exec db psql -U daisy -d daisy -c "SELECT * FROM user;"
+```
+
+### Solr (`solr` Service)
+
+#### Access Solr Admin Interface
+
+Solr runs on port `8983`. Access it via:
+
+```
+http://localhost:8983/solr/
+```
+
+### RabbitMQ (`mq` Service)
+
+#### Access RabbitMQ Management Interface
+
+RabbitMQ management runs on port `15672`. Access it via:
+
+```
+http://localhost:15672/
+```
+
+- **Username:** `guest`
+- **Password:** `guest`
+
+### Celery Worker (`worker` Service)
+
+Logs for the Celery worker can be viewed with:
+
+```bash
+docker compose logs -f worker
+```
+
+### Celery Beat (`beat` Service)
+
+Logs for Celery Beat can be viewed with:
+
+```bash
+docker compose logs -f beat
+```
+
+### Flower Monitoring Tool (`flower` Service)
+
+Access Flower for task monitoring on port `5555`:
+
+```
+http://localhost:5555/
+```
+
+---
+
+## Backup and Restore Operations
+
+### Backup Service (`backup`)
+
+
+#### Manual Backup
+
+Create a manual backup:
+
+
+
+```bash
+chmod +x ./backup_script.sh && ./backup_script.sh
+```
+or
+
+```bash
+docker compose stop nginx flower worker beat web mq solr
+docker compose exec backup sh /code/db.sh backup
+docker compose up -d solr mq web worker beat flower nginx
+```
+
+Backup files are stored in the `BACKUP_DIR` (default is `../backups`).
+
+### Restore from Backup
+
+Restore from a specific backup file:
+
+```bash
+docker compose stop nginx flower worker beat web mq solr
+docker compose exec backup sh /code/db.sh restore ../backups/backup_<timestamp>.tar.gz
+docker compose up -d solr mq web worker beat flower nginx
+```
+
+Replace `<timestamp>` with the actual timestamp in the backup filename.
+
+Rebuild the Solr index after restoration:
+
+```bash
+docker compose exec web python manage.py rebuild_index --noinput
+```
+
+#### Scheduled Backup with Cron
+
+To schedule the backup script to run automatically at a specific time using cron, add the following line to your crontab:
+
+1. Open the crontab editor:
+
+    ```bash
+    crontab -e
+    ```
+
+2. Add the cron job entry (for example, to run the backup at 1 AM daily):
+
+    ```bash
+    0 1 * * * /path/to/backup_script.sh
+    ```
+
+3. Check if the cron job is added:
+
+    ```bash
+    docker compose exec backup crontab -l
+    ```
+
+Replace `/path/to/backup_script.sh` with the actual path to backup_script.
+
+
+### Restore Legacy Backup
+
+To restore from a legacy backup file (e.g., `daisy_prod.tar.gz`):
+
+```bash
+docker compose stop nginx flower worker beat web mq solr
+
+# Copy the legacy backup file into the backup container
+docker cp ../daisy_prod.tar.gz $(docker compose ps -q backup):/code/daisy_prod.tar.gz
+
+# Execute the legacy restore script inside the backup container
+docker compose exec backup sh /code/legacy_restore.sh /code/daisy_prod.tar.gz
+
+# Remove the backup file from the container
+docker compose exec backup rm /code/daisy_prod.tar.gz
+
+docker compose up -d solr mq web worker beat flower nginx
+
+# Rebuild the Solr index
+docker compose exec web python manage.py rebuild_index --noinput
+```
+
+---
+
+## Importing and Exporting Data
+
+### Import Data
+
+You can import data from JSON files using Django management commands.
+
+#### Import Projects
+
+```bash
+docker compose exec web python manage.py import_projects -f /path/to/projects.json
+```
+
+#### Import Datasets
+
+```bash
+docker compose exec web python manage.py import_datasets -f /path/to/datasets.json
+```
+
+#### Import Partners
+
+```bash
+docker compose exec web python manage.py import_partners -f /path/to/partners.json
+```
+
+To import multiple JSON files from a directory:
+
+```bash
+docker compose exec web python manage.py import_projects -d /path/to/directory/
+```
+
+### Export Data
+
+Export data to JSON files.
+
+#### Export Projects
+
+```bash
+docker compose exec web python manage.py export_projects -f /path/to/output/projects.json
+```
+
+#### Export Datasets
+
+```bash
+docker compose exec web python manage.py export_datasets -f /path/to/output/datasets.json
+```
+
+#### Export Partners
+
+```bash
+docker compose exec web python manage.py export_partners -f /path/to/output/partners.json
+```
+
+---
+
+## Updating the Project
+
+### Pull Latest Changes
+
+```bash
+git pull origin main
+```
+
+### Rebuild Services After Code Changes
+
+If you make changes to the code or dependencies, rebuild the affected services:
+
+```bash
+docker compose build web worker beat
+docker compose up -d
+```
+
+### Database Backup Before Upgrade
+
+Create a database backup before upgrading:
+
+```bash
+docker compose exec backup sh /code/db.sh backup
+```
+
+### Upgrade Steps
+
+1. **Pull Latest Changes:**
+
+    ```bash
+    git pull origin main
+    ```
+
+2. **Rebuild Docker Images:**
+
+    ```bash
+    docker compose build
+    ```
+
+3. **Apply Database Migrations:**
+
+    ```bash
+    docker compose exec web python manage.py migrate
+    ```
+
+4. **Update Solr Schema:**
+
+    ```bash
+    docker compose exec web python manage.py build_solr_schema -c /solr/daisy/conf -r daisy -u default
+    ```
+
+5. **Collect Static Files:**
+
+    ```bash
+    docker compose exec web python manage.py collectstatic --noinput
+    ```
+
+6. **Rebuild Search Index:**
+
+    ```bash
+    docker compose exec web python manage.py rebuild_index --noinput
+    ```
+
+7. **Optional - Import Users:**
+
+    If you use LDAP or Active Directory:
+
+    ```bash
+    docker compose exec web python manage.py import_users
+    ```
+
+---
+
+## Administration
+
+To access the admin interface:
+
+1. **Create a Superuser Account:**
+
+    ```bash
+    docker compose exec web python manage.py createsuperuser
+    ```
+
+2. **Access the Admin Site:**
+
+    ```
+    http://localhost/admin/
+    ```
+
+    Log in with your superuser credentials.
+
+---
+
+## Settings Reference
+
+Adjust the `elixir_daisy/settings_compose.py` file to configure the application.
+
+### Display Settings
+
+- **`COMPANY`**: Name used in model verbose names.
+- **`DEMO_MODE`**: Set to `True` to display a demo banner.
+- **`INSTANCE_LABEL`**: Label displayed in the navbar to differentiate deployments.
+- **`INSTANCE_PRIMARY_COLOR`**: Primary color for the navbar.
+- **`LOGIN_USERNAME_PLACEHOLDER`**: Placeholder text for the login form username.
+- **`LOGIN_PASSWORD_PLACEHOLDER`**: Placeholder text for the login form password.
+
+### Integration with Other Tools
+
+#### ID Service
+
+- **`IDSERVICE_FUNCTION`**: Path to the function generating IDs.
+- **`IDSERVICE_ENDPOINT`**: Endpoint for the ID service.
+
+#### REMS Integration
+
+- **`REMS_INTEGRATION_ENABLED`**: Enable REMS integration.
+- **`REMS_SKIP_IP_CHECK`**: Skip IP address checks.
+- **`REMS_ALLOWED_IP_ADDRESSES`**: List of allowed IP addresses.
+
+#### Keycloak Integration
+
+- **`KEYCLOAK_INTEGRATION`**: Enable Keycloak integration.
+- **`KEYCLOAK_URL`**: URL of the Keycloak instance.
+- **`KEYCLOAK_REALM_LOGIN`**: Realm login name.
+- **`KEYCLOAK_REALM_ADMIN`**: Realm admin name.
+- **`KEYCLOAK_USER`**: Username for Keycloak access.
+- **`KEYCLOAK_PASS`**: Password for Keycloak access.
+
+### Other Settings
+
+- **`SERVER_SCHEME`**: Scheme (`http` or `https`) for accessing Daisy.
+- **`SERVER_URL`**: URL for accessing Daisy (without scheme).
+- **`GLOBAL_API_KEY`**: Global API key (disabled if `None`).
+
+---
+
+## Additional Tips
+
+- **Running Custom Commands:** Use `docker compose run` for one-off commands without starting the entire service.
+
+  Example:
+
+  ```bash
+  docker compose run --rm web python manage.py shell
+  ```
+
+- **Accessing Other Services:** You can shell into other services similarly:
+
+  ```bash
+  docker compose exec db /bin/bash
+  docker compose exec solr /bin/bash
+  ```
+
+- **Environment Variables:** Override environment variables when running commands:
+
+  ```bash
+  DB_NAME=custom_db docker compose up -d
+  ```
+
+---
+
+## Docker Compose Services Overview
+
+- **web:** Django application server using Gunicorn.
+- **db:** PostgreSQL database.
+- **nginx:** Reverse proxy and static file server.
+- **solr:** Apache Solr for full-text search.
+- **mq:** RabbitMQ message broker.
+- **flower:** Monitoring tool for Celery tasks.
+- **worker:** Celery worker for asynchronous tasks.
+- **beat:** Celery Beat scheduler.
+- **backup:** Manages database backups and restoration.
+
+---
+
+## Logs and Monitoring
+
+### View Service Logs
+
+View logs for a specific service:
+
+```bash
+docker compose logs -f <service_name>
+```
+
+Replace `<service_name>` with `web`, `db`, `worker`, etc.
+
+### Check Container Status
+
+```bash
+docker compose ps
+```
+
+---
+
+## Clean Up
+
+### Remove All Containers and Volumes
+
+Stop containers and remove containers, networks, volumes, and images:
+
+```bash
+docker system prune -a
+docker compose down -v --rmi all
+```
+
+
+
diff --git a/backup.md b/backup.md
deleted file mode 100644
index 1d2d6e7a..00000000
--- a/backup.md
+++ /dev/null
@@ -1,110 +0,0 @@
-# Database Backup and Restore Script
-
-## Overview
-Manage PostgreSQL database backups and Django media files in a Docker environment using `tar.gz` archives.
-
-## Key Functions
-- **Backup**: Creates a timestamped `tar.gz` archive of the PostgreSQL database and Django media files.
-- **Restore**: Restores from a specified `tar.gz` backup archive.
-
-## Docker Compose Integration
-The `backup` service in `docker-compose.yaml` manages backup and restore using the `db.sh` script.
-
-### Configuration
-
-- **Environment Variables**:
-  - `DB_HOST` (default: `db`)
-  - `DB_PORT` (default: `5432`)
-  - `DB_NAME` (default: `daisy`)
-  - `DB_USER` (default: `daisy`)
-  - `DB_PASSWORD` (default: `daisy`)
-  - `BACKUP_DIR` (default: `../backups`)
-  - `ENABLE_BACKUPS` (default: `true`)
-  - `BACKUP_SCHEDULE` (default: `"0 0 * * *"`)
-
-- **Volumes**:
-  - `${BACKUP_VOLUME:-../backups}:/backups`
-  - `.:/code`
-
-### Operations
-
-#### Enable Automatic Backups
-To ensure automatic backups are enabled, set `ENABLE_BACKUPS=true` (enabled by default):
-
-To checkout if the cron is added 
-
-```bash
-docker compose exec backup crontab -l
-```
-
-```bash
-ENABLE_BACKUPS=true docker compose up -d backup
-```
-This will configure automatic backups based on the `BACKUP_SCHEDULE`.
-
-#### Automatic Backups
-- Enabled by default (`ENABLE_BACKUPS=true`).
-- Schedule defined by `BACKUP_SCHEDULE` (cron format).
-
-To disable automatic backups:
-```bash
-ENABLE_BACKUPS=false docker compose up -d backup
-```
-
-To checkout if the cron was removed 
-
-```bash
-docker compose exec backup crontab -l
-```
-
-
-#### Manual Backup
-Create a manual backup:
-```bash
-docker compose exec backup sh /code/db.sh backup
-```
-- **Output**: `backup_<timestamp>.tar.gz` in the `BACKUP_DIR` (`../backups` by default).
-
-#### Restore Backup
-Restore from a specific backup file:
-```bash
-docker compose exec backup sh /code/db.sh restore ../backups/backup_<timestamp>.tar.gz
-docker compose run web python manage.py rebuild_index --noinput
-```
-- Replace `../backups/backup_<timestamp>.tar.gz` with the actual file path.
-
-
-#### List Cron Jobs
-View the automatic backup schedule:
-```bash
-docker compose exec backup crontab -l
-```
-
-#### List Backup Contents
-View contents of a backup archive:
-```bash
-tar -ztvf ../backups/backup_<timestamp>.tar.gz
-```
-
-
-#### Restore Legacy Backup
-Execute the `legacy_restore.sh` script inside the running container
-
-```bash
-# Copy the legacy backup file to the backup container
-docker cp ../daisy.tar.gz $(docker compose ps -q backup):/code/daisy_prod.tar.gz
-
-# Execute the legacy_restore.sh script inside the running container
-docker compose exec backup /bin/sh -c "sh /code/legacy_restore.sh /code/daisy_prod.tar.gz && rm /code/daisy_prod.tar.gz"
-docker compose run web python manage.py rebuild_index --noinput
-```
-Replace `../daisy.tar.gz` with the actual path to legacy backup file.
-
-
-#### Reindex Solr
-
-To rebuild the Solr search index after a restore operation:
-
-```bash
-docker compose run web python manage.py rebuild_index --noinput
-```
\ No newline at end of file
diff --git a/backup_script.sh b/backup_script.sh
new file mode 100755
index 00000000..bf358497
--- /dev/null
+++ b/backup_script.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Stop services that should not run during the backup
+docker compose stop nginx flower worker beat web mq solr
+
+# Run the backup script inside the backup container
+docker compose exec backup sh /code/db.sh backup
+
+# Start the services back up after the backup is complete
+docker compose up -d solr mq web worker beat flower nginx
diff --git a/db.sh b/db.sh
old mode 100644
new mode 100755
index 40a0e77c..4f0cea5b
--- a/db.sh
+++ b/db.sh
@@ -15,21 +15,6 @@ if [ "${BACKUP_DIR}" == "../backups" ] && [ ! -d "${BACKUP_DIR}" ]; then
     mkdir -p "${BACKUP_DIR}"
 fi
 
-stop_services() {
-    echo "Stopping: nginx, flower, worker, beat, web, mq, solr"
-    docker compose stop nginx flower worker beat web mq solr
-    echo "Services stopped."
-}
-
-start_services() {
-    echo "Starting: solr, mq, web, worker, beat, flower, nginx"
-    docker compose up -d solr mq web worker beat flower nginx || {
-        echo "ERROR: Service startup failed. Check Docker logs." >&2
-        exit 1
-    }
-    echo "Services started."
-}
-
 # Function to perform backup
 backup() {
     local timestamp
@@ -149,15 +134,11 @@ restore() {
 # Main script logic
 case "$1" in
     backup)
-        stop_services
         backup
-        start_services
         ;;
     restore)
         shift
-        stop_services
         restore "$@"
-        start_services
         ;;
     *)
         echo "Usage: $0 {backup|restore <backup_file.tar.gz>}" >&2
diff --git a/docker-compose.yaml b/docker-compose.yaml
index a41242f9..1cc2a766 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -110,10 +110,7 @@ services:
             - DB_PASSWORD=${DB_PASSWORD:-daisy}
             - BACKUP_DIR=/backups
             - MEDIA_DIR=/code/documents
-            - ENABLE_BACKUPS=${ENABLE_BACKUPS:-false}
-            - BACKUP_SCHEDULE=${BACKUP_SCHEDULE:-"0 0 * * *"}
             - SOLR_PORT=8983
-            - COMPOSE_PROJECT_NAME=daisy
             - RABBITMQ_MANAGEMENT_PORT=${RABBITMQ_MANAGEMENT_PORT:-15672}
         volumes:
             - ${BACKUP_VOLUME:-../backups}:/backups
@@ -125,7 +122,9 @@ services:
         networks:
             - daisy_network
         entrypoint: >
-            sh -c "apk add --no-cache docker-cli docker-compose postgresql-client lsof && chmod +x /code/setup_cron.sh && chmod +x /code/db.sh && exec /code/setup_cron.sh"
+            sh -c "apk add --no-cache docker-cli docker-compose postgresql-client lsof &&
+               chmod +x /code/db.sh &&
+               tail -f /dev/null"
 
 volumes:
     pgdata:
diff --git a/setup_cron.sh b/setup_cron.sh
deleted file mode 100644
index 77b0085c..00000000
--- a/setup_cron.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/sh
-
-# Ensure the backup script is executable
-chmod +x /code/db.sh
-
-# Check if backups are enabled
-if [ "$ENABLE_BACKUPS" = "true" ]; then
-  # Install necessary packages
-  apk add --no-cache bash curl tar gzip postgresql-client openrc cronie
-
-  # Set up the crontab entry
-  echo "$BACKUP_SCHEDULE /code/db.sh backup" | tr -d '"' > /etc/crontabs/root
-
-  # Print the crontab contents for debugging
-  echo "Crontab contents:"
-  cat /etc/crontabs/root
-
-  # Start the cron daemon
-  crond -f
-else
-  echo "Backups are disabled. Set ENABLE_BACKUPS=true to enable."
-  tail -f /dev/null
-fi
\ No newline at end of file

From b691325055b70870b77b76e32095f53965292bd9 Mon Sep 17 00:00:00 2001
From: Vilem Ded <vilem.ded@uni.lu>
Date: Tue, 24 Sep 2024 21:47:21 +0200
Subject: [PATCH 12/43] linting the markdown files

---
 README.md | 142 +++++++++++++++++++++++++++++++-----------------------
 backup.md | 125 +++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 207 insertions(+), 60 deletions(-)
 create mode 100644 backup.md

diff --git a/README.md b/README.md
index 6119dcd4..a20e9abb 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,5 @@
 # Elixir Daisy
+
 ![Build Status](https://github.com/elixir-luxembourg/daisy/actions/workflows/main.yml/badge.svg)
 [![Python 3.9](https://img.shields.io/badge/python-3.9-blue.svg)](https://www.python.org/downloads/release/python-396/)
 
@@ -9,41 +10,47 @@ For more information, please refer to the official [Daisy documentation](https:/
 DAISY was published as an article [DAISY: A Data Information System for accountability under the General Data Protection Regulation](https://doi.org/10.1093/gigascience/giz140) in GigaScience journal.
 
 ## Demo deployment
+
 You are encouraged to try Daisy for yourself using our [DEMO deployment](https://daisy-demo.elixir-luxembourg.org/).
 
 ## Deployment using Docker
 
 ### Requirements
 
-* docker: https://docs.docker.com/install/
+* docker: <https://docs.docker.com/install/>
 
 ### Installation
 
 1. Get the source code
-    
+
     ```bash
     git clone git@github.com:elixir-luxembourg/daisy.git
     cd daisy
     ```
+
 1. Create your settings file
-    
-	```bash
-	cp elixir_daisy/settings_local.template.py elixir_daisy/settings_local.py
-	```
+
+ ```bash
+ cp elixir_daisy/settings_local.template.py elixir_daisy/settings_local.py
+ ```
+
     Optional: edit the file elixir_daisy/settings_local.py to adapt to your environment.
 
 1. Build daisy docker image  
+
     ```bash
     docker-compose up --build
     ```
+
     Wait for the build to finish and keep the process running
 1. Open a new shell and go to daisy folder
 
 1. Build the database
-    
+
     ```bash
     docker-compose exec web python manage.py migrate
     ```
+
 1. Build the solr schema
 
     ```bash
@@ -51,27 +58,30 @@ You are encouraged to try Daisy for yourself using our [DEMO deployment](https:/
     ```
 
 1. Compile and deploy static files
-    
+
     ```bash
     cd web/static/vendor
     npm run build
     cd ../../../
     docker-compose exec web python manage.py collectstatic
     ```
+
 1. Create initial data in the database
-    
+
     ```bash
     docker-compose exec web bash -c "cd core/fixtures/ && wget https://git-r3lab.uni.lu/pinar.alper/metadata-tools/raw/master/metadata_tools/resources/edda.json && wget https://git-r3lab.uni.lu/pinar.alper/metadata-tools/raw/master/metadata_tools/resources/hpo.json && wget https://git-r3lab.uni.lu/pinar.alper/metadata-tools/raw/master/metadata_tools/resources/hdo.json && wget https://git-r3lab.uni.lu/pinar.alper/metadata-tools/raw/master/metadata_tools/resources/hgnc.json"
     docker-compose exec web python manage.py load_initial_data
     ```
+
    Initial data includes, for instance, controlled vocabularies terms and initial list of institutions and cohorts.  
    **This step can take several minutes to complete**
-    
+
 1. Load demo data
-    
+
     ```bash
     docker-compose exec web python manage.py load_demo_data
     ```
+
     This will create mock datasets, projects and create a demo admin account.
 
 1. Optional - import users from an active directory instance
@@ -79,16 +89,16 @@ You are encouraged to try Daisy for yourself using our [DEMO deployment](https:/
     ```bash
     docker-compose exec web python manage.py import_users
     ```
-    
-1.  Build the search index
+
+1. Build the search index
 
     ```bash
     docker-compose exec web python manage.py rebuild_index -u default
-    ```    
+    ```
 
-1. Browse to https://localhost  
+1. Browse to <https://localhost>  
     a demo admin account is available:
-    
+
     ```
         username: admin
         password: demo
@@ -101,11 +111,9 @@ pre-commit install
 black --check .
 black .
 
-
 ### Operation manual
 
-
-#### Importing 
+#### Importing
 
 In addition to loading of initial data, DAISY database can be populated by importing Project, Dataset and Partners records from JSON files using commands `import_projects`, `import_datasets` and `import_partners` respectively.
  The commands for import are accepting one JSON file (flag `-f`): </br>
@@ -113,10 +121,12 @@ In addition to loading of initial data, DAISY database can be populated by impor
 ```bash
 docker-compose exec web python manage.py <COMMAND> -f ${PATH_TO_JSON_FILE}
 ```
+
 where ${PATH_TO_JSON_FILE} is the path to a json file containing the records definitions.
 See file daisy/data/demo/projects.json as an example.
- 
+
 Alternatively, you can specify directory containing multiple JSON files to be imported with `-d` flag:
+
 ```bash
 docker-compose exec web python manage.py <COMMAND> -d ${PATH_TO_DIR}
 ```
@@ -128,97 +138,101 @@ Information in the DAISY database can be exported to JSON files. The command for
 ```bash
 docker-compose exec web python manage.py export_partners -f ${JSON_FILE}
 ```
+
 where ${JSON_FILE} is the path to a json file that will be produced.  In addition to ````export_partners````, you can run ````export_projects```` and ````export_datasets```` in the same way.
 
 ### Upgrade to last Daisy version
 
 1. Create a database backup.
 
-	```bash
-	docker-compose exec db pg_dump daisy --port=5432 --username=daisy --no-password --clean > backup_`date +%y-%m-%d`.sql
-	```
-	
+ ```bash
+ docker-compose exec db pg_dump daisy --port=5432 --username=daisy --no-password --clean > backup_`date +%y-%m-%d`.sql
+ ```
+
 1. Make sure docker containers are stopped.
 
-	```bash
-	docker-compose stop
-	```
+ ```bash
+ docker-compose stop
+ ```
 
 3. Get last Daisy release.
 
-	```bash
-	git checkout master
-	git pull
-	```
+ ```bash
+ git checkout master
+ git pull
+ ```
 
 1. Rebuild and start the docker containers.
 
-	```bash
-	docker-compose up --build
-	```
-	Open a new terminal window to execute the following commands.
+ ```bash
+ docker-compose up --build
+ ```
+
+ Open a new terminal window to execute the following commands.
 
 1. Update the database schema.
 
-	```bash
-	docker-compose exec web python manage.py migrate
-	```
+ ```bash
+ docker-compose exec web python manage.py migrate
+ ```
 
 1. Update the solr schema.
 
-	```bash
-	docker-compose exec web python manage.py build_solr_schema -c /solr/daisy/conf -r daisy -u default
-	```
+ ```bash
+ docker-compose exec web python manage.py build_solr_schema -c /solr/daisy/conf -r daisy -u default
+ ```
 
 1. Collect static files.
- 
-	```bash
-	docker-compose exec web python manage.py collectstatic
-	```
 
-	
+ ```bash
+ docker-compose exec web python manage.py collectstatic
+ ```
+
 1. Rebuild the search index.
-    
+
     ```bash
     docker-compose exec web python manage.py rebuild_index -u default
     ```
+
 1. Reimport the users (optional).
-	    
+
     If LDAP was used during initial setup to import users, they have to be imported again:
-    
+
     ```bash
     docker-compose exec web python manage.py import_users
     ```
-    
-## Deployment without Docker - CentOS
 
+## Deployment without Docker - CentOS
 
 See [DEPLOYMENT](DEPLOYMENT.md).
 
-
 ## Development
 
 To be completed.
 
 ### Import users from active directory
+
 ```bash
 ./manage.py import_users
 ```
 
 ### Import projects, datasets or partners from external system
+
 Single file mode:
+
 ```bash
 ./manage.py import_projects -f path/to/json_file.json
 ```
 
 Batch mode:
+
 ```bash
 ./manage.py import_projects -d path/to/dir/with/json/files/
 ```
 
 Available commands: `import_projects`, `import_datasets`, `import_partners`.
 
-In case of problems, add `--verbose` flag to the command, and take a look inside `./log/daisy.log`. 
+In case of problems, add `--verbose` flag to the command, and take a look inside `./log/daisy.log`.
 
 ### Install js and css dependencies
 
@@ -228,6 +242,7 @@ npm ci
 ```
 
 ### Compile daisy.scss and React
+
 ```bash
 cd web/static/vendor
 npm run-script build
@@ -246,7 +261,9 @@ The following command will install the test dependencies and execute the tests:
 ```bash
 python setup.py pytest
 ```
+
 run test for a specific file:
+
 ```bash
 python setup.py pytest --addopts web/tests/test_dataset.py
 ```
@@ -262,27 +279,30 @@ pytest
 To get access to the admin page, you must log in with a superuser account.  
 On the `Users` section, you can give any user a `staff` status and he will be able to access any project/datasets.
 
-
 ## `settings.py` and `local_settings.py` reference
 
 ### Display
+
 | Key | Description | Expected values | Example value |
 |---|---|---|---|
 | `COMPANY` | A name that is used to generate verbose names of some models | str | `'LCSB'` |
-| `DEMO_MODE` | A flag which makes a simple banneer about demo mode appear in About page | bool | `False` | 
-| `INSTANCE_LABEL` | A name that is used in navbar header to help differentiate different deployments | str | `'Staging test VM'` | 
-| `INSTANCE_PRIMARY_COLOR` | A color that will be navbar header's background | str of a color | `'#076505'` | 
-| `LOGIN_USERNAME_PLACEHOLDER` | A helpful placeholder in login form for logins | str | `'@uni.lu'` | 
-| `LOGIN_PASSWORD_PLACEHOLDER` | A helpful placeholder in login form for passwords | str | `'Hint: use your AD password'` | 
+| `DEMO_MODE` | A flag which makes a simple banneer about demo mode appear in About page | bool | `False` |
+| `INSTANCE_LABEL` | A name that is used in navbar header to help differentiate different deployments | str | `'Staging test VM'` |
+| `INSTANCE_PRIMARY_COLOR` | A color that will be navbar header's background | str of a color | `'#076505'` |
+| `LOGIN_USERNAME_PLACEHOLDER` | A helpful placeholder in login form for logins | str | `'@uni.lu'` |
+| `LOGIN_PASSWORD_PLACEHOLDER` | A helpful placeholder in login form for passwords | str | `'Hint: use your AD password'` |
 
 ### Integration with other tools
+
 #### ID Service
+
 | Key | Description | Expected values | Example value |
 |---|---|---|---|
 | `IDSERVICE_FUNCTION` | Path to a function (`lambda: str`) that generates IDs for entities which are published | str | `'web.views.utils.generate_elu_accession'` |
 | `IDSERVICE_ENDPOINT` | In case LCSB's idservice function is being used, the setting contains the IDservice's URI | str | `'https://192.168.1.101/v1/api/` |
 
 #### REMS
+
 | Key | Description | Expected values | Example value |
 |---|---|---|---|
 | `REMS_INTEGRATION_ENABLED` | A feature flag for REMS integration. In practice, there's a dedicated endpoint which processes the information from REMS about dataset entitlements | str | `True` |
@@ -290,6 +310,7 @@ On the `Users` section, you can give any user a `staff` status and he will be ab
 | `REMS_ALLOWED_IP_ADDRESSES` | A list of IP addresses that should be considered trusted REMS instances. Beware of configuration difficulties when using reverse proxies. The check can be skipped with `REMS_SKIP_IP_CHECK` | dict[str] | `['127.0.0.1', '192.168.1.101']` |
 
 #### Keycloak
+
 | Key | Description | Expected values | Example value |
 |---|---|---|---|
 | `KEYCLOAK_INTEGRATION` | A feature flag for importing user information from Keycloak (OIDC IDs) | bool | `True` |
@@ -300,8 +321,9 @@ On the `Users` section, you can give any user a `staff` status and he will be ab
 | `KEYCLOAK_PASS` | Password to access Keycloak | str | `'secure123'` |
 
 ### Others
+
 | Key | Description | Expected values | Example value |
 |---|---|---|---|
 | `SERVER_SCHEME` | A URL's scheme to access your DAISY instance (http or https) | str | `'https'` |
 | `SERVER_URL` | A URL to access your DAISY instance (without the scheme) | str | `'example.com'` |
-| `GLOBAL_API_KEY` | An API key that is not connected with any user. Disabled if set to `None` | optional[str] | `'in-practice-you-dont-want-to-use-it-unless-debugging'` |
\ No newline at end of file
+| `GLOBAL_API_KEY` | An API key that is not connected with any user. Disabled if set to `None` | optional[str] | `'in-practice-you-dont-want-to-use-it-unless-debugging'` |
diff --git a/backup.md b/backup.md
new file mode 100644
index 00000000..300810d2
--- /dev/null
+++ b/backup.md
@@ -0,0 +1,125 @@
+# Database Backup and Restore Script
+
+## Overview
+
+Manage PostgreSQL database backups and Django media files in a Docker environment using `tar.gz` archives.
+
+## Key Functions
+
+- **Backup**: Creates a timestamped `tar.gz` archive of the PostgreSQL database and Django media files.
+- **Restore**: Restores from a specified `tar.gz` backup archive.
+
+## Docker Compose Integration
+
+The `backup` service in `docker-compose.yaml` manages backup and restore using the `db.sh` script.
+
+### Configuration
+
+- **Environment Variables**:
+  - `DB_HOST` (default: `db`)
+  - `DB_PORT` (default: `5432`)
+  - `DB_NAME` (default: `daisy`)
+  - `DB_USER` (default: `daisy`)
+  - `DB_PASSWORD` (default: `daisy`)
+  - `BACKUP_DIR` (default: `../backups`)
+  - `ENABLE_BACKUPS` (default: `true`)
+  - `BACKUP_SCHEDULE` (default: `"0 0 * * *"`)
+
+- **Volumes**:
+  - `${BACKUP_VOLUME:-../backups}:/backups`
+  - `.:/code`
+
+### Operations
+
+#### Enable Automatic Backups
+
+To ensure automatic backups are enabled, set `ENABLE_BACKUPS=true` (enabled by default):
+
+To checkout if the cron is added
+
+```bash
+docker compose exec backup crontab -l
+```
+
+```bash
+ENABLE_BACKUPS=true docker compose up -d backup
+```
+
+This will configure automatic backups based on the `BACKUP_SCHEDULE`.
+
+#### Automatic Backups
+
+- Enabled by default (`ENABLE_BACKUPS=true`).
+- Schedule defined by `BACKUP_SCHEDULE` (cron format).
+
+To disable automatic backups:
+
+```bash
+ENABLE_BACKUPS=false docker compose up -d backup
+```
+
+To checkout if the cron was removed
+
+```bash
+docker compose exec backup crontab -l
+```
+
+#### Manual Backup
+
+Create a manual backup:
+
+```bash
+docker compose exec backup sh /code/db.sh backup
+```
+
+- **Output**: `backup_<timestamp>.tar.gz` in the `BACKUP_DIR` (`../backups` by default).
+
+#### Restore Backup
+
+Restore from a specific backup file:
+
+```bash
+docker compose exec backup sh /code/db.sh restore ../backups/backup_<timestamp>.tar.gz
+docker compose run web python manage.py rebuild_index --noinput
+```
+
+- Replace `../backups/backup_<timestamp>.tar.gz` with the actual file path.
+
+#### List Cron Jobs
+
+View the automatic backup schedule:
+
+```bash
+docker compose exec backup crontab -l
+```
+
+#### List Backup Contents
+
+View contents of a backup archive:
+
+```bash
+tar -ztvf ../backups/backup_<timestamp>.tar.gz
+```
+
+#### Restore Legacy Backup
+
+Execute the `legacy_restore.sh` script inside the running container
+
+```bash
+# Copy the legacy backup file to the backup container
+docker cp ../daisy.tar.gz $(docker compose ps -q backup):/code/daisy_prod.tar.gz
+
+# Execute the legacy_restore.sh script inside the running container
+docker compose exec backup /bin/sh -c "sh /code/legacy_restore.sh /code/daisy_prod.tar.gz && rm /code/daisy_prod.tar.gz"
+docker compose run web python manage.py rebuild_index --noinput
+```
+
+Replace `../daisy.tar.gz` with the actual path to legacy backup file.
+
+#### Reindex Solr
+
+To rebuild the Solr search index after a restore operation:
+
+```bash
+docker compose run web python manage.py rebuild_index --noinput
+```

From cb85e6159d7ac953a30f180e1ec7f1e585935b13 Mon Sep 17 00:00:00 2001
From: Vilem Ded <vilem.ded@uni.lu>
Date: Tue, 24 Sep 2024 22:04:15 +0200
Subject: [PATCH 13/43] rephrase backup.md manual

---
 backup.md | 30 +++---------------------------
 1 file changed, 3 insertions(+), 27 deletions(-)

diff --git a/backup.md b/backup.md
index 300810d2..08aeecc6 100644
--- a/backup.md
+++ b/backup.md
@@ -2,7 +2,7 @@
 
 ## Overview
 
-Manage PostgreSQL database backups and Django media files in a Docker environment using `tar.gz` archives.
+This manual describes steps to perform for backup creation and its restoration.
 
 ## Key Functions
 
@@ -15,15 +15,7 @@ The `backup` service in `docker-compose.yaml` manages backup and restore using t
 
 ### Configuration
 
-- **Environment Variables**:
-  - `DB_HOST` (default: `db`)
-  - `DB_PORT` (default: `5432`)
-  - `DB_NAME` (default: `daisy`)
-  - `DB_USER` (default: `daisy`)
-  - `DB_PASSWORD` (default: `daisy`)
-  - `BACKUP_DIR` (default: `../backups`)
-  - `ENABLE_BACKUPS` (default: `true`)
-  - `BACKUP_SCHEDULE` (default: `"0 0 * * *"`)
+All variables can be set in the [environment file](.env.template). These include variables necessary for connection to the database, path to local folder where the backup is created and setup of cron tasks for backup.
 
 - **Volumes**:
   - `${BACKUP_VOLUME:-../backups}:/backups`
@@ -35,11 +27,6 @@ The `backup` service in `docker-compose.yaml` manages backup and restore using t
 
 To ensure automatic backups are enabled, set `ENABLE_BACKUPS=true` (enabled by default):
 
-To checkout if the cron is added
-
-```bash
-docker compose exec backup crontab -l
-```
 
 ```bash
 ENABLE_BACKUPS=true docker compose up -d backup
@@ -47,23 +34,12 @@ ENABLE_BACKUPS=true docker compose up -d backup
 
 This will configure automatic backups based on the `BACKUP_SCHEDULE`.
 
-#### Automatic Backups
-
-- Enabled by default (`ENABLE_BACKUPS=true`).
-- Schedule defined by `BACKUP_SCHEDULE` (cron format).
-
 To disable automatic backups:
 
 ```bash
 ENABLE_BACKUPS=false docker compose up -d backup
 ```
 
-To checkout if the cron was removed
-
-```bash
-docker compose exec backup crontab -l
-```
-
 #### Manual Backup
 
 Create a manual backup:
@@ -103,7 +79,7 @@ tar -ztvf ../backups/backup_<timestamp>.tar.gz
 
 #### Restore Legacy Backup
 
-Execute the `legacy_restore.sh` script inside the running container
+To restore backup created before version 1.8.1 on newer versions with docker deployment, execute the `legacy_restore.sh` script inside the running container
 
 ```bash
 # Copy the legacy backup file to the backup container

From 8e0cf186ed63d784332cd86ac4179e9262d551c8 Mon Sep 17 00:00:00 2001
From: Vilem Ded <vilem.ded@uni.lu>
Date: Tue, 24 Sep 2024 22:13:54 +0200
Subject: [PATCH 14/43] moving scripts to /scripts/ folder

---
 DAISY_Docker_Compose_Guide.md                  | 8 ++++----
 backup.md                                      | 8 ++++----
 docker-compose.yaml                            | 2 +-
 backup_script.sh => scripts/backup_script.sh   | 2 +-
 db.sh => scripts/db.sh                         | 0
 legacy_restore.sh => scripts/legacy_restore.sh | 0
 6 files changed, 10 insertions(+), 10 deletions(-)
 rename backup_script.sh => scripts/backup_script.sh (83%)
 rename db.sh => scripts/db.sh (100%)
 rename legacy_restore.sh => scripts/legacy_restore.sh (100%)

diff --git a/DAISY_Docker_Compose_Guide.md b/DAISY_Docker_Compose_Guide.md
index 02ed9da9..e178a83c 100644
--- a/DAISY_Docker_Compose_Guide.md
+++ b/DAISY_Docker_Compose_Guide.md
@@ -307,7 +307,7 @@ or
 
 ```bash
 docker compose stop nginx flower worker beat web mq solr
-docker compose exec backup sh /code/db.sh backup
+docker compose exec backup sh /code/scripts/db.sh backup
 docker compose up -d solr mq web worker beat flower nginx
 ```
 
@@ -319,7 +319,7 @@ Restore from a specific backup file:
 
 ```bash
 docker compose stop nginx flower worker beat web mq solr
-docker compose exec backup sh /code/db.sh restore ../backups/backup_<timestamp>.tar.gz
+docker compose exec backup sh /code/scripts/db.sh restore ../backups/backup_<timestamp>.tar.gz
 docker compose up -d solr mq web worker beat flower nginx
 ```
 
@@ -367,7 +367,7 @@ docker compose stop nginx flower worker beat web mq solr
 docker cp ../daisy_prod.tar.gz $(docker compose ps -q backup):/code/daisy_prod.tar.gz
 
 # Execute the legacy restore script inside the backup container
-docker compose exec backup sh /code/legacy_restore.sh /code/daisy_prod.tar.gz
+docker compose exec backup sh /code/scripts/legacy_restore.sh /code/daisy_prod.tar.gz
 
 # Remove the backup file from the container
 docker compose exec backup rm /code/daisy_prod.tar.gz
@@ -456,7 +456,7 @@ docker compose up -d
 Create a database backup before upgrading:
 
 ```bash
-docker compose exec backup sh /code/db.sh backup
+docker compose exec backup sh /code/scripts/db.sh backup
 ```
 
 ### Upgrade Steps
diff --git a/backup.md b/backup.md
index 08aeecc6..189014c4 100644
--- a/backup.md
+++ b/backup.md
@@ -11,7 +11,7 @@ This manual describes steps to perform for backup creation and its restoration.
 
 ## Docker Compose Integration
 
-The `backup` service in `docker-compose.yaml` manages backup and restore using the `db.sh` script.
+The `backup` service in `docker-compose.yaml` manages backup and restore using the `scripts/db.sh` script.
 
 ### Configuration
 
@@ -45,7 +45,7 @@ ENABLE_BACKUPS=false docker compose up -d backup
 Create a manual backup:
 
 ```bash
-docker compose exec backup sh /code/db.sh backup
+docker compose exec backup sh /code/scripts/db.sh backup
 ```
 
 - **Output**: `backup_<timestamp>.tar.gz` in the `BACKUP_DIR` (`../backups` by default).
@@ -55,7 +55,7 @@ docker compose exec backup sh /code/db.sh backup
 Restore from a specific backup file:
 
 ```bash
-docker compose exec backup sh /code/db.sh restore ../backups/backup_<timestamp>.tar.gz
+docker compose exec backup sh /code/scripts/db.sh restore ../backups/backup_<timestamp>.tar.gz
 docker compose run web python manage.py rebuild_index --noinput
 ```
 
@@ -86,7 +86,7 @@ To restore backup created before version 1.8.1 on newer versions with docker dep
 docker cp ../daisy.tar.gz $(docker compose ps -q backup):/code/daisy_prod.tar.gz
 
 # Execute the legacy_restore.sh script inside the running container
-docker compose exec backup /bin/sh -c "sh /code/legacy_restore.sh /code/daisy_prod.tar.gz && rm /code/daisy_prod.tar.gz"
+docker compose exec backup /bin/sh -c "sh /code/scripts/legacy_restore.sh /code/daisy_prod.tar.gz && rm /code/daisy_prod.tar.gz"
 docker compose run web python manage.py rebuild_index --noinput
 ```
 
diff --git a/docker-compose.yaml b/docker-compose.yaml
index 1cc2a766..f11d8694 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -123,7 +123,7 @@ services:
             - daisy_network
         entrypoint: >
             sh -c "apk add --no-cache docker-cli docker-compose postgresql-client lsof &&
-               chmod +x /code/db.sh &&
+               chmod +x /code/scripts/db.sh &&
                tail -f /dev/null"
 
 volumes:
diff --git a/backup_script.sh b/scripts/backup_script.sh
similarity index 83%
rename from backup_script.sh
rename to scripts/backup_script.sh
index bf358497..15cea18b 100755
--- a/backup_script.sh
+++ b/scripts/backup_script.sh
@@ -4,7 +4,7 @@
 docker compose stop nginx flower worker beat web mq solr
 
 # Run the backup script inside the backup container
-docker compose exec backup sh /code/db.sh backup
+docker compose exec backup sh /code/scripts/db.sh backup
 
 # Start the services back up after the backup is complete
 docker compose up -d solr mq web worker beat flower nginx
diff --git a/db.sh b/scripts/db.sh
similarity index 100%
rename from db.sh
rename to scripts/db.sh
diff --git a/legacy_restore.sh b/scripts/legacy_restore.sh
similarity index 100%
rename from legacy_restore.sh
rename to scripts/legacy_restore.sh

From de9a15117da761b5be19aa52ede0e07f39649c88 Mon Sep 17 00:00:00 2001
From: Vilem Ded <vilem.ded@uni.lu>
Date: Tue, 24 Sep 2024 22:15:44 +0200
Subject: [PATCH 15/43] moving documentation to doc/ folder

---
 backup.md => doc/backup.md                         | 0
 DAISY_Docker_Compose_Guide.md => doc/deployment.md | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename backup.md => doc/backup.md (100%)
 rename DAISY_Docker_Compose_Guide.md => doc/deployment.md (100%)

diff --git a/backup.md b/doc/backup.md
similarity index 100%
rename from backup.md
rename to doc/backup.md
diff --git a/DAISY_Docker_Compose_Guide.md b/doc/deployment.md
similarity index 100%
rename from DAISY_Docker_Compose_Guide.md
rename to doc/deployment.md

From f124b4007558cedebe244af77966f9c3f1e2bc97 Mon Sep 17 00:00:00 2001
From: Vilem Ded <vilem.ded@uni.lu>
Date: Tue, 24 Sep 2024 22:31:50 +0200
Subject: [PATCH 16/43] split documentation into several files

---
 README.md             |  24 ++-
 doc/administration.md | 246 +++++++++++++++++++++++++++++++
 doc/deployment.md     | 331 +-----------------------------------------
 doc/update.md         |  72 +++++++++
 4 files changed, 337 insertions(+), 336 deletions(-)
 create mode 100644 doc/administration.md
 create mode 100644 doc/update.md

diff --git a/README.md b/README.md
index a20e9abb..e4c22e42 100644
--- a/README.md
+++ b/README.md
@@ -13,7 +13,17 @@ DAISY was published as an article [DAISY: A Data Information System for accounta
 
 You are encouraged to try Daisy for yourself using our [DEMO deployment](https://daisy-demo.elixir-luxembourg.org/).
 
-## Deployment using Docker
+## Deployment
+
+DAISY can be fully deployed using Docker. For more instructions see the deployment [guide](doc/deployment.md).
+
+## Documentation
+
+See our
+
+- [Backup manual](doc/backup.md) for creating and restoring backups.
+- [Update manual](doc/update.md) for migration to newer version
+- [Management and administration manual](doc/administration.md) for regular maintenance tasks
 
 ### Requirements
 
@@ -104,12 +114,7 @@ You are encouraged to try Daisy for yourself using our [DEMO deployment](https:/
         password: demo
     ```
 
-### Linting
 
-pip install black==23.7.0
-pre-commit install
-black --check .
-black .
 
 ### Operation manual
 
@@ -208,7 +213,12 @@ See [DEPLOYMENT](DEPLOYMENT.md).
 
 ## Development
 
-To be completed.
+### Linting
+
+pip install black==23.7.0
+pre-commit install
+black --check .
+black .
 
 ### Import users from active directory
 
diff --git a/doc/administration.md b/doc/administration.md
new file mode 100644
index 00000000..5465e918
--- /dev/null
+++ b/doc/administration.md
@@ -0,0 +1,246 @@
+# Managing
+
+### Access the Web Service
+
+Shell into the `web` container:
+
+```bash
+docker compose exec web /bin/bash
+```
+
+### Run Django Commands
+
+Run Django management commands using the `web` service.
+
+#### Make Migrations
+
+```bash
+docker compose exec web python manage.py makemigrations
+```
+
+#### Apply Migrations
+
+```bash
+docker compose exec web python manage.py migrate
+```
+
+#### Create Superuser
+
+```bash
+docker compose exec web python manage.py createsuperuser
+```
+
+#### Collect Static Files
+
+```bash
+docker compose exec web python manage.py collectstatic --noinput
+```
+
+#### Rebuild Solr Index
+
+```bash
+docker compose exec web python manage.py rebuild_index --noinput
+```
+
+---
+
+## Managing Other Services
+
+### PostgreSQL Database (`db` Service)
+
+#### Access the Database Shell
+
+```bash
+docker compose exec db psql -U daisy -d
+```
+
+#### Execute SQL Commands
+
+Run SQL commands directly:
+
+```bash
+docker compose exec db psql -U daisy -d daisy -c "SELECT * FROM user;"
+```
+
+### Solr (`solr` Service)
+
+#### Access Solr Admin Interface
+
+Solr runs on port `8983`. Access it via:
+
+```
+http://localhost:8983/solr/
+```
+
+### RabbitMQ (`mq` Service)
+
+#### Access RabbitMQ Management Interface
+
+RabbitMQ management runs on port `15672`. Access it via:
+
+```
+http://localhost:15672/
+```
+
+- **Username:** `guest`
+- **Password:** `guest`
+
+### Celery Worker (`worker` Service)
+
+Logs for the Celery worker can be viewed with:
+
+```bash
+docker compose logs -f worker
+```
+
+### Celery Beat (`beat` Service)
+
+Logs for Celery Beat can be viewed with:
+
+```bash
+docker compose logs -f beat
+```
+
+### Flower Monitoring Tool (`flower` Service)
+
+Access Flower for task monitoring on port `5555`:
+
+```
+http://localhost:5555/
+```
+
+---
+
+
+## Administration
+
+To access the admin interface:
+
+1. **Create a Superuser Account:**
+
+    ```bash
+    docker compose exec web python manage.py createsuperuser
+    ```
+
+2. **Access the Admin Site:**
+
+    ```
+    http://localhost/admin/
+    ```
+
+    Log in with your superuser credentials.
+
+---
+
+## Settings Reference
+
+Adjust the `elixir_daisy/settings_compose.py` file to configure the application.
+
+### Display Settings
+
+- **`COMPANY`**: Name used in model verbose names.
+- **`DEMO_MODE`**: Set to `True` to display a demo banner.
+- **`INSTANCE_LABEL`**: Label displayed in the navbar to differentiate deployments.
+- **`INSTANCE_PRIMARY_COLOR`**: Primary color for the navbar.
+- **`LOGIN_USERNAME_PLACEHOLDER`**: Placeholder text for the login form username.
+- **`LOGIN_PASSWORD_PLACEHOLDER`**: Placeholder text for the login form password.
+
+### Integration with Other Tools
+
+#### ID Service
+
+- **`IDSERVICE_FUNCTION`**: Path to the function generating IDs.
+- **`IDSERVICE_ENDPOINT`**: Endpoint for the ID service.
+
+#### REMS Integration
+
+- **`REMS_INTEGRATION_ENABLED`**: Enable REMS integration.
+- **`REMS_SKIP_IP_CHECK`**: Skip IP address checks.
+- **`REMS_ALLOWED_IP_ADDRESSES`**: List of allowed IP addresses.
+
+#### Keycloak Integration
+
+- **`KEYCLOAK_INTEGRATION`**: Enable Keycloak integration.
+- **`KEYCLOAK_URL`**: URL of the Keycloak instance.
+- **`KEYCLOAK_REALM_LOGIN`**: Realm login name.
+- **`KEYCLOAK_REALM_ADMIN`**: Realm admin name.
+- **`KEYCLOAK_USER`**: Username for Keycloak access.
+- **`KEYCLOAK_PASS`**: Password for Keycloak access.
+
+### Other Settings
+
+- **`SERVER_SCHEME`**: Scheme (`http` or `https`) for accessing Daisy.
+- **`SERVER_URL`**: URL for accessing Daisy (without scheme).
+- **`GLOBAL_API_KEY`**: Global API key (disabled if `None`).
+
+---
+
+## Additional Tips
+
+- **Running Custom Commands:** Use `docker compose run` for one-off commands without starting the entire service.
+
+  Example:
+
+  ```bash
+  docker compose run --rm web python manage.py shell
+  ```
+
+- **Accessing Other Services:** You can shell into other services similarly:
+
+  ```bash
+  docker compose exec db /bin/bash
+  docker compose exec solr /bin/bash
+  ```
+
+- **Environment Variables:** Override environment variables when running commands:
+
+  ```bash
+  DB_NAME=custom_db docker compose up -d
+  ```
+
+---
+
+## Docker Compose Services Overview
+
+- **web:** Django application server using Gunicorn.
+- **db:** PostgreSQL database.
+- **nginx:** Reverse proxy and static file server.
+- **solr:** Apache Solr for full-text search.
+- **mq:** RabbitMQ message broker.
+- **flower:** Monitoring tool for Celery tasks.
+- **worker:** Celery worker for asynchronous tasks.
+- **beat:** Celery Beat scheduler.
+- **backup:** Manages database backups and restoration.
+
+---
+
+## Logs and Monitoring
+
+### View Service Logs
+
+View logs for a specific service:
+
+```bash
+docker compose logs -f <service_name>
+```
+
+Replace `<service_name>` with `web`, `db`, `worker`, etc.
+
+### Check Container Status
+
+```bash
+docker compose ps
+```
+
+---
+
+## Clean Up
+
+### Remove All Containers and Volumes
+
+Stop containers and remove containers, networks, volumes, and images:
+
+```bash
+docker system prune -a
+docker compose down -v --rmi all
+```
diff --git a/doc/deployment.md b/doc/deployment.md
index e178a83c..d57e461f 100644
--- a/doc/deployment.md
+++ b/doc/deployment.md
@@ -81,7 +81,7 @@ BACKUP_VOLUME=../backups
 
 ---
 
-## Running the Project
+## Installation
 
 ### Build and Start Services
 
@@ -176,133 +176,18 @@ If you loaded the demo data, you can log in with the demo credentials provided d
 
 ---
 
-## Managing the Django Application
-
-### Access the Web Service
-
-Shell into the `web` container:
-
-```bash
-docker compose exec web /bin/bash
-```
-
-### Run Django Commands
-
-Run Django management commands using the `web` service.
-
-#### Make Migrations
-
-```bash
-docker compose exec web python manage.py makemigrations
-```
-
-#### Apply Migrations
-
-```bash
-docker compose exec web python manage.py migrate
-```
-
-#### Create Superuser
-
-```bash
-docker compose exec web python manage.py createsuperuser
-```
-
-#### Collect Static Files
-
-```bash
-docker compose exec web python manage.py collectstatic --noinput
-```
-
-#### Rebuild Solr Index
-
-```bash
-docker compose exec web python manage.py rebuild_index --noinput
-```
-
----
-
-## Managing Other Services
-
-### PostgreSQL Database (`db` Service)
-
-#### Access the Database Shell
-
-```bash
-docker compose exec db psql -U daisy -d
-```
-
-#### Execute SQL Commands
-
-Run SQL commands directly:
-
-```bash
-docker compose exec db psql -U daisy -d daisy -c "SELECT * FROM user;"
-```
-
-### Solr (`solr` Service)
-
-#### Access Solr Admin Interface
-
-Solr runs on port `8983`. Access it via:
-
-```
-http://localhost:8983/solr/
-```
-
-### RabbitMQ (`mq` Service)
-
-#### Access RabbitMQ Management Interface
-
-RabbitMQ management runs on port `15672`. Access it via:
-
-```
-http://localhost:15672/
-```
-
-- **Username:** `guest`
-- **Password:** `guest`
-
-### Celery Worker (`worker` Service)
-
-Logs for the Celery worker can be viewed with:
-
-```bash
-docker compose logs -f worker
-```
-
-### Celery Beat (`beat` Service)
-
-Logs for Celery Beat can be viewed with:
-
-```bash
-docker compose logs -f beat
-```
-
-### Flower Monitoring Tool (`flower` Service)
-
-Access Flower for task monitoring on port `5555`:
-
-```
-http://localhost:5555/
-```
-
----
-
 ## Backup and Restore Operations
 
 ### Backup Service (`backup`)
 
-
 #### Manual Backup
 
 Create a manual backup:
 
-
-
 ```bash
 chmod +x ./backup_script.sh && ./backup_script.sh
 ```
+
 or
 
 ```bash
@@ -355,7 +240,6 @@ To schedule the backup script to run automatically at a specific time using cron
 
 Replace `/path/to/backup_script.sh` with the actual path to backup_script.
 
-
 ### Restore Legacy Backup
 
 To restore from a legacy backup file (e.g., `daisy_prod.tar.gz`):
@@ -431,214 +315,3 @@ docker compose exec web python manage.py export_datasets -f /path/to/output/data
 ```bash
 docker compose exec web python manage.py export_partners -f /path/to/output/partners.json
 ```
-
----
-
-## Updating the Project
-
-### Pull Latest Changes
-
-```bash
-git pull origin main
-```
-
-### Rebuild Services After Code Changes
-
-If you make changes to the code or dependencies, rebuild the affected services:
-
-```bash
-docker compose build web worker beat
-docker compose up -d
-```
-
-### Database Backup Before Upgrade
-
-Create a database backup before upgrading:
-
-```bash
-docker compose exec backup sh /code/scripts/db.sh backup
-```
-
-### Upgrade Steps
-
-1. **Pull Latest Changes:**
-
-    ```bash
-    git pull origin main
-    ```
-
-2. **Rebuild Docker Images:**
-
-    ```bash
-    docker compose build
-    ```
-
-3. **Apply Database Migrations:**
-
-    ```bash
-    docker compose exec web python manage.py migrate
-    ```
-
-4. **Update Solr Schema:**
-
-    ```bash
-    docker compose exec web python manage.py build_solr_schema -c /solr/daisy/conf -r daisy -u default
-    ```
-
-5. **Collect Static Files:**
-
-    ```bash
-    docker compose exec web python manage.py collectstatic --noinput
-    ```
-
-6. **Rebuild Search Index:**
-
-    ```bash
-    docker compose exec web python manage.py rebuild_index --noinput
-    ```
-
-7. **Optional - Import Users:**
-
-    If you use LDAP or Active Directory:
-
-    ```bash
-    docker compose exec web python manage.py import_users
-    ```
-
----
-
-## Administration
-
-To access the admin interface:
-
-1. **Create a Superuser Account:**
-
-    ```bash
-    docker compose exec web python manage.py createsuperuser
-    ```
-
-2. **Access the Admin Site:**
-
-    ```
-    http://localhost/admin/
-    ```
-
-    Log in with your superuser credentials.
-
----
-
-## Settings Reference
-
-Adjust the `elixir_daisy/settings_compose.py` file to configure the application.
-
-### Display Settings
-
-- **`COMPANY`**: Name used in model verbose names.
-- **`DEMO_MODE`**: Set to `True` to display a demo banner.
-- **`INSTANCE_LABEL`**: Label displayed in the navbar to differentiate deployments.
-- **`INSTANCE_PRIMARY_COLOR`**: Primary color for the navbar.
-- **`LOGIN_USERNAME_PLACEHOLDER`**: Placeholder text for the login form username.
-- **`LOGIN_PASSWORD_PLACEHOLDER`**: Placeholder text for the login form password.
-
-### Integration with Other Tools
-
-#### ID Service
-
-- **`IDSERVICE_FUNCTION`**: Path to the function generating IDs.
-- **`IDSERVICE_ENDPOINT`**: Endpoint for the ID service.
-
-#### REMS Integration
-
-- **`REMS_INTEGRATION_ENABLED`**: Enable REMS integration.
-- **`REMS_SKIP_IP_CHECK`**: Skip IP address checks.
-- **`REMS_ALLOWED_IP_ADDRESSES`**: List of allowed IP addresses.
-
-#### Keycloak Integration
-
-- **`KEYCLOAK_INTEGRATION`**: Enable Keycloak integration.
-- **`KEYCLOAK_URL`**: URL of the Keycloak instance.
-- **`KEYCLOAK_REALM_LOGIN`**: Realm login name.
-- **`KEYCLOAK_REALM_ADMIN`**: Realm admin name.
-- **`KEYCLOAK_USER`**: Username for Keycloak access.
-- **`KEYCLOAK_PASS`**: Password for Keycloak access.
-
-### Other Settings
-
-- **`SERVER_SCHEME`**: Scheme (`http` or `https`) for accessing Daisy.
-- **`SERVER_URL`**: URL for accessing Daisy (without scheme).
-- **`GLOBAL_API_KEY`**: Global API key (disabled if `None`).
-
----
-
-## Additional Tips
-
-- **Running Custom Commands:** Use `docker compose run` for one-off commands without starting the entire service.
-
-  Example:
-
-  ```bash
-  docker compose run --rm web python manage.py shell
-  ```
-
-- **Accessing Other Services:** You can shell into other services similarly:
-
-  ```bash
-  docker compose exec db /bin/bash
-  docker compose exec solr /bin/bash
-  ```
-
-- **Environment Variables:** Override environment variables when running commands:
-
-  ```bash
-  DB_NAME=custom_db docker compose up -d
-  ```
-
----
-
-## Docker Compose Services Overview
-
-- **web:** Django application server using Gunicorn.
-- **db:** PostgreSQL database.
-- **nginx:** Reverse proxy and static file server.
-- **solr:** Apache Solr for full-text search.
-- **mq:** RabbitMQ message broker.
-- **flower:** Monitoring tool for Celery tasks.
-- **worker:** Celery worker for asynchronous tasks.
-- **beat:** Celery Beat scheduler.
-- **backup:** Manages database backups and restoration.
-
----
-
-## Logs and Monitoring
-
-### View Service Logs
-
-View logs for a specific service:
-
-```bash
-docker compose logs -f <service_name>
-```
-
-Replace `<service_name>` with `web`, `db`, `worker`, etc.
-
-### Check Container Status
-
-```bash
-docker compose ps
-```
-
----
-
-## Clean Up
-
-### Remove All Containers and Volumes
-
-Stop containers and remove containers, networks, volumes, and images:
-
-```bash
-docker system prune -a
-docker compose down -v --rmi all
-```
-
-
-
diff --git a/doc/update.md b/doc/update.md
new file mode 100644
index 00000000..3cc26114
--- /dev/null
+++ b/doc/update.md
@@ -0,0 +1,72 @@
+# Updating the Project
+
+### Pull Latest Changes
+
+```bash
+git pull origin main
+```
+
+### Rebuild Services After Code Changes
+
+If you make changes to the code or dependencies, rebuild the affected services:
+
+```bash
+docker compose build web worker beat
+docker compose up -d
+```
+
+### Database Backup Before Upgrade
+
+Create a database backup before upgrading:
+
+```bash
+docker compose exec backup sh /code/scripts/db.sh backup
+```
+
+### Upgrade Steps
+
+1. **Pull Latest Changes:**
+
+    ```bash
+    git pull origin main
+    ```
+
+2. **Rebuild Docker Images:**
+
+    ```bash
+    docker compose build
+    ```
+
+3. **Apply Database Migrations:**
+
+    ```bash
+    docker compose exec web python manage.py migrate
+    ```
+
+4. **Update Solr Schema:**
+
+    ```bash
+    docker compose exec web python manage.py build_solr_schema -c /solr/daisy/conf -r daisy -u default
+    ```
+
+5. **Collect Static Files:**
+
+    ```bash
+    docker compose exec web python manage.py collectstatic --noinput
+    ```
+
+6. **Rebuild Search Index:**
+
+    ```bash
+    docker compose exec web python manage.py rebuild_index --noinput
+    ```
+
+7. **Optional - Import Users:**
+
+    If you use LDAP or Active Directory:
+
+    ```bash
+    docker compose exec web python manage.py import_users
+    ```
+
+---
\ No newline at end of file

From 0f0542d4324c8736ef021799bb0ce729cf438dd3 Mon Sep 17 00:00:00 2001
From: Vilem Ded <vilem.ded@uni.lu>
Date: Wed, 25 Sep 2024 09:28:03 +0200
Subject: [PATCH 17/43] restructure the documentation

- move old deployemnet and administration steps to legacy doc
- split tasks into several manuals (deployment, backup, development,
  etc.)
- update README file
---
 README.md                                 | 326 +---------------------
 doc/administration.md                     | 122 ++++++--
 doc/backup.md                             |  47 +++-
 doc/deployment.md                         | 163 +----------
 doc/development.md                        |  73 +++++
 DEPLOYMENT.md => doc/legacy-deployment.md |   5 +-
 doc/update.md                             |  10 +-
 7 files changed, 223 insertions(+), 523 deletions(-)
 create mode 100644 doc/development.md
 rename DEPLOYMENT.md => doc/legacy-deployment.md (99%)

diff --git a/README.md b/README.md
index e4c22e42..ed939f5b 100644
--- a/README.md
+++ b/README.md
@@ -13,327 +13,19 @@ DAISY was published as an article [DAISY: A Data Information System for accounta
 
 You are encouraged to try Daisy for yourself using our [DEMO deployment](https://daisy-demo.elixir-luxembourg.org/).
 
-## Deployment
-
-DAISY can be fully deployed using Docker. For more instructions see the deployment [guide](doc/deployment.md).
-
 ## Documentation
 
-See our
-
-- [Backup manual](doc/backup.md) for creating and restoring backups.
-- [Update manual](doc/update.md) for migration to newer version
-- [Management and administration manual](doc/administration.md) for regular maintenance tasks
-
-### Requirements
-
-* docker: <https://docs.docker.com/install/>
-
-### Installation
-
-1. Get the source code
-
-    ```bash
-    git clone git@github.com:elixir-luxembourg/daisy.git
-    cd daisy
-    ```
-
-1. Create your settings file
-
- ```bash
- cp elixir_daisy/settings_local.template.py elixir_daisy/settings_local.py
- ```
-
-    Optional: edit the file elixir_daisy/settings_local.py to adapt to your environment.
-
-1. Build daisy docker image  
-
-    ```bash
-    docker-compose up --build
-    ```
-
-    Wait for the build to finish and keep the process running
-1. Open a new shell and go to daisy folder
-
-1. Build the database
-
-    ```bash
-    docker-compose exec web python manage.py migrate
-    ```
-
-1. Build the solr schema
-
-    ```bash
-    docker-compose exec web python manage.py build_solr_schema -c /solr/daisy/conf -r daisy -u default
-    ```
-
-1. Compile and deploy static files
-
-    ```bash
-    cd web/static/vendor
-    npm run build
-    cd ../../../
-    docker-compose exec web python manage.py collectstatic
-    ```
-
-1. Create initial data in the database
-
-    ```bash
-    docker-compose exec web bash -c "cd core/fixtures/ && wget https://git-r3lab.uni.lu/pinar.alper/metadata-tools/raw/master/metadata_tools/resources/edda.json && wget https://git-r3lab.uni.lu/pinar.alper/metadata-tools/raw/master/metadata_tools/resources/hpo.json && wget https://git-r3lab.uni.lu/pinar.alper/metadata-tools/raw/master/metadata_tools/resources/hdo.json && wget https://git-r3lab.uni.lu/pinar.alper/metadata-tools/raw/master/metadata_tools/resources/hgnc.json"
-    docker-compose exec web python manage.py load_initial_data
-    ```
-
-   Initial data includes, for instance, controlled vocabularies terms and initial list of institutions and cohorts.  
-   **This step can take several minutes to complete**
-
-1. Load demo data
-
-    ```bash
-    docker-compose exec web python manage.py load_demo_data
-    ```
-
-    This will create mock datasets, projects and create a demo admin account.
-
-1. Optional - import users from an active directory instance
-
-    ```bash
-    docker-compose exec web python manage.py import_users
-    ```
-
-1. Build the search index
-
-    ```bash
-    docker-compose exec web python manage.py rebuild_index -u default
-    ```
-
-1. Browse to <https://localhost>  
-    a demo admin account is available:
-
-    ```
-        username: admin
-        password: demo
-    ```
-
-
-
-### Operation manual
-
-#### Importing
-
-In addition to loading of initial data, DAISY database can be populated by importing Project, Dataset and Partners records from JSON files using commands `import_projects`, `import_datasets` and `import_partners` respectively.
- The commands for import are accepting one JSON file (flag `-f`): </br>
-
-```bash
-docker-compose exec web python manage.py <COMMAND> -f ${PATH_TO_JSON_FILE}
-```
-
-where ${PATH_TO_JSON_FILE} is the path to a json file containing the records definitions.
-See file daisy/data/demo/projects.json as an example.
-
-Alternatively, you can specify directory containing multiple JSON files to be imported with `-d` flag:
-
-```bash
-docker-compose exec web python manage.py <COMMAND> -d ${PATH_TO_DIR}
-```
-
-#### Exporting  
-
-Information in the DAISY database can be exported to JSON files. The command for export are given below:</br>
-
-```bash
-docker-compose exec web python manage.py export_partners -f ${JSON_FILE}
-```
-
-where ${JSON_FILE} is the path to a json file that will be produced.  In addition to ````export_partners````, you can run ````export_projects```` and ````export_datasets```` in the same way.
-
-### Upgrade to last Daisy version
-
-1. Create a database backup.
-
- ```bash
- docker-compose exec db pg_dump daisy --port=5432 --username=daisy --no-password --clean > backup_`date +%y-%m-%d`.sql
- ```
-
-1. Make sure docker containers are stopped.
-
- ```bash
- docker-compose stop
- ```
-
-3. Get last Daisy release.
+DAISY comes with a **Docker deployment*. For more instructions see the deployment [guide](doc/deployment.md).
 
- ```bash
- git checkout master
- git pull
- ```
+See also our
 
-1. Rebuild and start the docker containers.
-
- ```bash
- docker-compose up --build
- ```
-
- Open a new terminal window to execute the following commands.
-
-1. Update the database schema.
-
- ```bash
- docker-compose exec web python manage.py migrate
- ```
-
-1. Update the solr schema.
-
- ```bash
- docker-compose exec web python manage.py build_solr_schema -c /solr/daisy/conf -r daisy -u default
- ```
-
-1. Collect static files.
-
- ```bash
- docker-compose exec web python manage.py collectstatic
- ```
-
-1. Rebuild the search index.
-
-    ```bash
-    docker-compose exec web python manage.py rebuild_index -u default
-    ```
-
-1. Reimport the users (optional).
-
-    If LDAP was used during initial setup to import users, they have to be imported again:
-
-    ```bash
-    docker-compose exec web python manage.py import_users
-    ```
-
-## Deployment without Docker - CentOS
-
-See [DEPLOYMENT](DEPLOYMENT.md).
-
-## Development
-
-### Linting
-
-pip install black==23.7.0
-pre-commit install
-black --check .
-black .
-
-### Import users from active directory
-
-```bash
-./manage.py import_users
-```
-
-### Import projects, datasets or partners from external system
-
-Single file mode:
-
-```bash
-./manage.py import_projects -f path/to/json_file.json
-```
-
-Batch mode:
-
-```bash
-./manage.py import_projects -d path/to/dir/with/json/files/
-```
-
-Available commands: `import_projects`, `import_datasets`, `import_partners`.
-
-In case of problems, add `--verbose` flag to the command, and take a look inside `./log/daisy.log`.
-
-### Install js and css dependencies
-
-```bash
-cd web/static/vendor/
-npm ci
-```
-
-### Compile daisy.scss and React
-
-```bash
-cd web/static/vendor
-npm run-script build
-```
-
-### Run the built-in web server (for development)
-
-```bash
-./manage.py runserver
-```
-
-### Run the tests
-
-The following command will install the test dependencies and execute the tests:
-
-```bash
-python setup.py pytest
-```
-
-run test for a specific file:
-
-```bash
-python setup.py pytest --addopts web/tests/test_dataset.py
-```
-
-If tests dependencies are already installed, one can also run the tests just by executing:
-
-```bash
-pytest
-```
-
-## Administration
-
-To get access to the admin page, you must log in with a superuser account.  
-On the `Users` section, you can give any user a `staff` status and he will be able to access any project/datasets.
-
-## `settings.py` and `local_settings.py` reference
-
-### Display
-
-| Key | Description | Expected values | Example value |
-|---|---|---|---|
-| `COMPANY` | A name that is used to generate verbose names of some models | str | `'LCSB'` |
-| `DEMO_MODE` | A flag which makes a simple banneer about demo mode appear in About page | bool | `False` |
-| `INSTANCE_LABEL` | A name that is used in navbar header to help differentiate different deployments | str | `'Staging test VM'` |
-| `INSTANCE_PRIMARY_COLOR` | A color that will be navbar header's background | str of a color | `'#076505'` |
-| `LOGIN_USERNAME_PLACEHOLDER` | A helpful placeholder in login form for logins | str | `'@uni.lu'` |
-| `LOGIN_PASSWORD_PLACEHOLDER` | A helpful placeholder in login form for passwords | str | `'Hint: use your AD password'` |
-
-### Integration with other tools
-
-#### ID Service
-
-| Key | Description | Expected values | Example value |
-|---|---|---|---|
-| `IDSERVICE_FUNCTION` | Path to a function (`lambda: str`) that generates IDs for entities which are published | str | `'web.views.utils.generate_elu_accession'` |
-| `IDSERVICE_ENDPOINT` | In case LCSB's idservice function is being used, the setting contains the IDservice's URI | str | `'https://192.168.1.101/v1/api/` |
-
-#### REMS
-
-| Key | Description | Expected values | Example value |
-|---|---|---|---|
-| `REMS_INTEGRATION_ENABLED` | A feature flag for REMS integration. In practice, there's a dedicated endpoint which processes the information from REMS about dataset entitlements | str | `True` |
-| `REMS_SKIP_IP_CHECK` | If set to `True`, there will be no IP checking if the request comes from trusted REMS instance. | bool | `False` |
-| `REMS_ALLOWED_IP_ADDRESSES` | A list of IP addresses that should be considered trusted REMS instances. Beware of configuration difficulties when using reverse proxies. The check can be skipped with `REMS_SKIP_IP_CHECK` | dict[str] | `['127.0.0.1', '192.168.1.101']` |
-
-#### Keycloak
+- [Backup manual](doc/backup.md) for creating and restoring backups.
+- [Update manual](doc/update.md) for migration to newer version.
+- [Management and administration manual](doc/administration.md) for regular maintenance tasks including starting/stopping the services, import/export of data, inspecting logs and clean up.
+- [Development manual](doc/development.md) for steps to setup the development environment and guidance on how to contribute.
 
-| Key | Description | Expected values | Example value |
-|---|---|---|---|
-| `KEYCLOAK_INTEGRATION` | A feature flag for importing user information from Keycloak (OIDC IDs) | bool | `True` |
-| `KEYCLOAK_URL` | URL to the Keycloak instance | str | `'https://keycloak.lcsb.uni.lu/auth/'` |
-| `KEYCLOAK_REALM_LOGIN` | Realm's login name in your Keycloak instance | str | `'master'` |
-| `KEYCLOAK_REALM_ADMIN` | Realm's admin name in your Keycloak instance | str | `'master'` |
-| `KEYCLOAK_USER` | Username to access Keycloak | str | `'username'` |
-| `KEYCLOAK_PASS` | Password to access Keycloak | str | `'secure123'` |
+For legacy deployment (<1.8.1), please refer to the [Legacy deployment and administration manual](doc/legacy-deployment.md).
 
-### Others
+### Acknowledgement
 
-| Key | Description | Expected values | Example value |
-|---|---|---|---|
-| `SERVER_SCHEME` | A URL's scheme to access your DAISY instance (http or https) | str | `'https'` |
-| `SERVER_URL` | A URL to access your DAISY instance (without the scheme) | str | `'example.com'` |
-| `GLOBAL_API_KEY` | An API key that is not connected with any user. Disabled if set to `None` | optional[str] | `'in-practice-you-dont-want-to-use-it-unless-debugging'` |
+This work was supported by [ELIXIR Luxembourg](https://elixir-luxembourg.org/).
\ No newline at end of file
diff --git a/doc/administration.md b/doc/administration.md
index 5465e918..cf2e3c2e 100644
--- a/doc/administration.md
+++ b/doc/administration.md
@@ -111,7 +111,6 @@ http://localhost:5555/
 
 ---
 
-
 ## Administration
 
 To access the admin interface:
@@ -134,44 +133,57 @@ To access the admin interface:
 
 ## Settings Reference
 
-Adjust the `elixir_daisy/settings_compose.py` file to configure the application.
+To get access to the admin page, you must log in with a superuser account.  
+On the `Users` section, you can give any user a `staff` status and he will be able to access any project/datasets.
+
+### `local_settings.py` reference
 
-### Display Settings
+#### Display
 
-- **`COMPANY`**: Name used in model verbose names.
-- **`DEMO_MODE`**: Set to `True` to display a demo banner.
-- **`INSTANCE_LABEL`**: Label displayed in the navbar to differentiate deployments.
-- **`INSTANCE_PRIMARY_COLOR`**: Primary color for the navbar.
-- **`LOGIN_USERNAME_PLACEHOLDER`**: Placeholder text for the login form username.
-- **`LOGIN_PASSWORD_PLACEHOLDER`**: Placeholder text for the login form password.
+| Key                          | Description                                                                      | Expected values | Example value                  |
+| ---------------------------- | -------------------------------------------------------------------------------- | --------------- | ------------------------------ |
+| `COMPANY`                    | A name that is used to generate verbose names of some models                     | str             | `'LCSB'`                       |
+| `DEMO_MODE`                  | A flag which makes a simple banneer about demo mode appear in About page         | bool            | `False`                        |
+| `INSTANCE_LABEL`             | A name that is used in navbar header to help differentiate different deployments | str             | `'Staging test VM'`            |
+| `INSTANCE_PRIMARY_COLOR`     | A color that will be navbar header's background                                  | str of a color  | `'#076505'`                    |
+| `LOGIN_USERNAME_PLACEHOLDER` | A helpful placeholder in login form for logins                                   | str             | `'@uni.lu'`                    |
+| `LOGIN_PASSWORD_PLACEHOLDER` | A helpful placeholder in login form for passwords                                | str             | `'Hint: use your AD password'` |
 
-### Integration with Other Tools
+#### Integration with other tools
 
-#### ID Service
+##### ID Service
 
-- **`IDSERVICE_FUNCTION`**: Path to the function generating IDs.
-- **`IDSERVICE_ENDPOINT`**: Endpoint for the ID service.
+| Key                  | Description                                                                               | Expected values | Example value                              |
+| -------------------- | ----------------------------------------------------------------------------------------- | --------------- | ------------------------------------------ |
+| `IDSERVICE_FUNCTION` | Path to a function (`lambda: str`) that generates IDs for entities which are published    | str             | `'web.views.utils.generate_elu_accession'` |
+| `IDSERVICE_ENDPOINT` | In case LCSB's idservice function is being used, the setting contains the IDservice's URI | str             | `'https://192.168.1.101/v1/api/`           |
 
-#### REMS Integration
+##### REMS
 
-- **`REMS_INTEGRATION_ENABLED`**: Enable REMS integration.
-- **`REMS_SKIP_IP_CHECK`**: Skip IP address checks.
-- **`REMS_ALLOWED_IP_ADDRESSES`**: List of allowed IP addresses.
+| Key                         | Description                                                                                                                                                                                  | Expected values | Example value                    |
+| --------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------- | -------------------------------- |
+| `REMS_INTEGRATION_ENABLED`  | A feature flag for REMS integration. In practice, there's a dedicated endpoint which processes the information from REMS about dataset entitlements                                          | str             | `True`                           |
+| `REMS_SKIP_IP_CHECK`        | If set to `True`, there will be no IP checking if the request comes from trusted REMS instance.                                                                                              | bool            | `False`                          |
+| `REMS_ALLOWED_IP_ADDRESSES` | A list of IP addresses that should be considered trusted REMS instances. Beware of configuration difficulties when using reverse proxies. The check can be skipped with `REMS_SKIP_IP_CHECK` | dict[str]       | `['127.0.0.1', '192.168.1.101']` |
 
-#### Keycloak Integration
+##### Keycloak
 
-- **`KEYCLOAK_INTEGRATION`**: Enable Keycloak integration.
-- **`KEYCLOAK_URL`**: URL of the Keycloak instance.
-- **`KEYCLOAK_REALM_LOGIN`**: Realm login name.
-- **`KEYCLOAK_REALM_ADMIN`**: Realm admin name.
-- **`KEYCLOAK_USER`**: Username for Keycloak access.
-- **`KEYCLOAK_PASS`**: Password for Keycloak access.
+| Key                    | Description                                                            | Expected values | Example value                          |
+| ---------------------- | ---------------------------------------------------------------------- | --------------- | -------------------------------------- |
+| `KEYCLOAK_INTEGRATION` | A feature flag for importing user information from Keycloak (OIDC IDs) | bool            | `True`                                 |
+| `KEYCLOAK_URL`         | URL to the Keycloak instance                                           | str             | `'https://keycloak.lcsb.uni.lu/auth/'` |
+| `KEYCLOAK_REALM_LOGIN` | Realm's login name in your Keycloak instance                           | str             | `'master'`                             |
+| `KEYCLOAK_REALM_ADMIN` | Realm's admin name in your Keycloak instance                           | str             | `'master'`                             |
+| `KEYCLOAK_USER`        | Username to access Keycloak                                            | str             | `'username'`                           |
+| `KEYCLOAK_PASS`        | Password to access Keycloak                                            | str             | `'secure123'`                          |
 
-### Other Settings
+#### Others
 
-- **`SERVER_SCHEME`**: Scheme (`http` or `https`) for accessing Daisy.
-- **`SERVER_URL`**: URL for accessing Daisy (without scheme).
-- **`GLOBAL_API_KEY`**: Global API key (disabled if `None`).
+| Key              | Description                                                               | Expected values | Example value                                            |
+| ---------------- | ------------------------------------------------------------------------- | --------------- | -------------------------------------------------------- |
+| `SERVER_SCHEME`  | A URL's scheme to access your DAISY instance (http or https)              | str             | `'https'`                                                |
+| `SERVER_URL`     | A URL to access your DAISY instance (without the scheme)                  | str             | `'example.com'`                                          |
+| `GLOBAL_API_KEY` | An API key that is not connected with any user. Disabled if set to `None` | optional[str]   | `'in-practice-you-dont-want-to-use-it-unless-debugging'` |
 
 ---
 
@@ -244,3 +256,57 @@ Stop containers and remove containers, networks, volumes, and images:
 docker system prune -a
 docker compose down -v --rmi all
 ```
+
+## Importing and Exporting Data
+
+In addition to loading of initial data, DAISY database can be populated by importing Project, Dataset and Partners records from JSON files using commands `import_projects`, `import_datasets` and `import_partners` respectively. JSON files are validated using the [Elixir-LU JSON schemas](https://github.com/elixir-luxembourg/json-schemas).
+
+### Import Data
+
+You can import data from JSON files using Django management commands.
+
+#### Import Projects
+
+```bash
+docker compose exec web python manage.py import_projects -f /path/to/projects.json
+```
+
+#### Import Datasets
+
+```bash
+docker compose exec web python manage.py import_datasets -f /path/to/datasets.json
+```
+
+#### Import Partners
+
+```bash
+docker compose exec web python manage.py import_partners -f /path/to/partners.json
+```
+
+To import multiple JSON files from a directory:
+
+```bash
+docker compose exec web python manage.py import_projects -d /path/to/directory/
+```
+
+### Export Data
+
+Information in the DAISY database can be exported to JSON files. The command for export are given below:</br>
+
+#### Export Projects
+
+```bash
+docker compose exec web python manage.py export_projects -f /path/to/output/projects.json
+```
+
+#### Export Datasets
+
+```bash
+docker compose exec web python manage.py export_datasets -f /path/to/output/datasets.json
+```
+
+#### Export Partners
+
+```bash
+docker compose exec web python manage.py export_partners -f /path/to/output/partners.json
+```
diff --git a/doc/backup.md b/doc/backup.md
index 189014c4..2ea5da89 100644
--- a/doc/backup.md
+++ b/doc/backup.md
@@ -27,7 +27,6 @@ All variables can be set in the [environment file](.env.template). These include
 
 To ensure automatic backups are enabled, set `ENABLE_BACKUPS=true` (enabled by default):
 
-
 ```bash
 ENABLE_BACKUPS=true docker compose up -d backup
 ```
@@ -50,7 +49,31 @@ docker compose exec backup sh /code/scripts/db.sh backup
 
 - **Output**: `backup_<timestamp>.tar.gz` in the `BACKUP_DIR` (`../backups` by default).
 
-#### Restore Backup
+## Scheduled Backup with Cron
+
+To schedule the backup script to run automatically at a specific time using cron, add the following line to your crontab:
+
+1. Open the crontab editor:
+
+    ```bash
+    crontab -e
+    ```
+
+2. Add the cron job entry (for example, to run the backup at 1 AM daily):
+
+    ```bash
+    0 1 * * * /path/to/backup_script.sh
+    ```
+
+3. Check if the cron job is added:
+
+    ```bash
+    docker compose exec backup crontab -l
+    ```
+
+Replace `/path/to/backup_script.sh` with the actual path to backup_script.
+
+## Restore Backup
 
 Restore from a specific backup file:
 
@@ -61,7 +84,13 @@ docker compose run web python manage.py rebuild_index --noinput
 
 - Replace `../backups/backup_<timestamp>.tar.gz` with the actual file path.
 
-#### List Cron Jobs
+Rebuild the Solr index after restoration:
+
+```bash
+docker compose exec web python manage.py rebuild_index --noinput
+```
+
+## List Cron Jobs
 
 View the automatic backup schedule:
 
@@ -69,7 +98,7 @@ View the automatic backup schedule:
 docker compose exec backup crontab -l
 ```
 
-#### List Backup Contents
+## List Backup Contents
 
 View contents of a backup archive:
 
@@ -77,7 +106,7 @@ View contents of a backup archive:
 tar -ztvf ../backups/backup_<timestamp>.tar.gz
 ```
 
-#### Restore Legacy Backup
+## Restore Legacy Backup
 
 To restore backup created before version 1.8.1 on newer versions with docker deployment, execute the `legacy_restore.sh` script inside the running container
 
@@ -91,11 +120,3 @@ docker compose run web python manage.py rebuild_index --noinput
 ```
 
 Replace `../daisy.tar.gz` with the actual path to legacy backup file.
-
-#### Reindex Solr
-
-To rebuild the Solr search index after a restore operation:
-
-```bash
-docker compose run web python manage.py rebuild_index --noinput
-```
diff --git a/doc/deployment.md b/doc/deployment.md
index d57e461f..f2997894 100644
--- a/doc/deployment.md
+++ b/doc/deployment.md
@@ -2,54 +2,6 @@
 
 This guide provides concise instructions for setting up and running the Daisy project using Docker Compose. It includes commands for managing the Django application and other services within the project.
 
----
-
-## Table of Contents
-
-- [Prerequisites](#prerequisites)
-- [Getting Started](#getting-started)
-  - [Clone the Repository](#clone-the-repository)
-  - [Environment Variables](#environment-variables)
-- [Running the Project](#running-the-project)
-  - [Build and Start Services](#build-and-start-services)
-  - [Initialize the Database](#initialize-the-database)
-  - [Build the Solr Schema](#build-the-solr-schema)
-  - [Compile and Deploy Static Files](#compile-and-deploy-static-files)
-  - [Load Initial Data into the Database](#load-initial-data-into-the-database)
-  - [Load Demo Data (Optional)](#load-demo-data-optional)
-  - [Build the Search Index](#build-the-search-index)
-  - [Access the Application](#access-the-application)
-- [Managing the Django Application](#managing-the-django-application)
-  - [Access the Web Service](#access-the-web-service)
-  - [Run Django Commands](#run-django-commands)
-- [Managing Other Services](#managing-other-services)
-  - [PostgreSQL Database (`db` Service)](#postgresql-database-db-service)
-  - [Solr (`solr` Service)](#solr-solr-service)
-  - [RabbitMQ (`mq` Service)](#rabbitmq-mq-service)
-  - [Celery Worker (`worker` Service)](#celery-worker-worker-service)
-  - [Celery Beat (`beat` Service)](#celery-beat-beat-service)
-  - [Flower Monitoring Tool (`flower` Service)](#flower-monitoring-tool-flower-service)
-- [Backup and Restore Operations](#backup-and-restore-operations)
-  - [Backup Service (`backup`)](#backup-service-backup)
-  - [Restore from Backup](#restore-from-backup)
-  - [Restore Legacy Backup](#restore-legacy-backup)
-- [Importing and Exporting Data](#importing-and-exporting-data)
-  - [Import Data](#import-data)
-  - [Export Data](#export-data)
-- [Updating the Project](#updating-the-project)
-  - [Pull Latest Changes](#pull-latest-changes)
-  - [Rebuild Services After Code Changes](#rebuild-services-after-code-changes)
-  - [Database Backup Before Upgrade](#database-backup-before-upgrade)
-  - [Upgrade Steps](#upgrade-steps)
-- [Administration](#administration)
-- [Settings Reference](#settings-reference)
-- [Additional Tips](#additional-tips)
-- [Docker Compose Services Overview](#docker-compose-services-overview)
-- [Logs and Monitoring](#logs-and-monitoring)
-- [Clean Up](#clean-up)
-
----
-
 ## Prerequisites
 
 - **Docker**
@@ -68,18 +20,7 @@ cd daisy
 
 ### Environment Variables
 
-Create a `.env` file in the project root to override default environment variables if necessary.
-
-Example `.env` file:
-
-```env
-DB_NAME=daisy
-DB_USER=daisy
-DB_PASSWORD=daisy
-BACKUP_VOLUME=../backups
-```
-
----
+Create a `.env` file in the project root to override default environment variables if necessary. See [.env.template](env.template) file for more detail.
 
 ## Installation
 
@@ -151,12 +92,14 @@ docker compose exec web python manage.py load_initial_data
 
 ### Load Demo Data (Optional)
 
-To load demo data, including mock datasets, projects, and a demo admin account:
+In case you are provisioning a demo instance, following command loads demo data, including mock datasets, projects, and a demo admin account:
 
 ```bash
 docker compose exec web python manage.py load_demo_data
 ```
 
+You can log in with the demo admin credentials provided during the demo data setup (username: `admin`, password:`admin` by default) or as one of the regular users (see the `About` page for more detail).
+
 ### Build the Search Index
 
 After loading data, build the search index for Solr:
@@ -172,51 +115,7 @@ The application should now be accessible at:
 - **HTTP:** `http://localhost/`
 - **HTTPS:** `https://localhost/`
 
-If you loaded the demo data, you can log in with the demo credentials provided during the demo data setup.
-
----
-
-## Backup and Restore Operations
-
-### Backup Service (`backup`)
-
-#### Manual Backup
-
-Create a manual backup:
-
-```bash
-chmod +x ./backup_script.sh && ./backup_script.sh
-```
-
-or
-
-```bash
-docker compose stop nginx flower worker beat web mq solr
-docker compose exec backup sh /code/scripts/db.sh backup
-docker compose up -d solr mq web worker beat flower nginx
-```
-
-Backup files are stored in the `BACKUP_DIR` (default is `../backups`).
-
-### Restore from Backup
-
-Restore from a specific backup file:
-
-```bash
-docker compose stop nginx flower worker beat web mq solr
-docker compose exec backup sh /code/scripts/db.sh restore ../backups/backup_<timestamp>.tar.gz
-docker compose up -d solr mq web worker beat flower nginx
-```
-
-Replace `<timestamp>` with the actual timestamp in the backup filename.
-
-Rebuild the Solr index after restoration:
-
-```bash
-docker compose exec web python manage.py rebuild_index --noinput
-```
-
-#### Scheduled Backup with Cron
+## Scheduled Backup with Cron
 
 To schedule the backup script to run automatically at a specific time using cron, add the following line to your crontab:
 
@@ -263,55 +162,3 @@ docker compose exec web python manage.py rebuild_index --noinput
 ```
 
 ---
-
-## Importing and Exporting Data
-
-### Import Data
-
-You can import data from JSON files using Django management commands.
-
-#### Import Projects
-
-```bash
-docker compose exec web python manage.py import_projects -f /path/to/projects.json
-```
-
-#### Import Datasets
-
-```bash
-docker compose exec web python manage.py import_datasets -f /path/to/datasets.json
-```
-
-#### Import Partners
-
-```bash
-docker compose exec web python manage.py import_partners -f /path/to/partners.json
-```
-
-To import multiple JSON files from a directory:
-
-```bash
-docker compose exec web python manage.py import_projects -d /path/to/directory/
-```
-
-### Export Data
-
-Export data to JSON files.
-
-#### Export Projects
-
-```bash
-docker compose exec web python manage.py export_projects -f /path/to/output/projects.json
-```
-
-#### Export Datasets
-
-```bash
-docker compose exec web python manage.py export_datasets -f /path/to/output/datasets.json
-```
-
-#### Export Partners
-
-```bash
-docker compose exec web python manage.py export_partners -f /path/to/output/partners.json
-```
diff --git a/doc/development.md b/doc/development.md
new file mode 100644
index 00000000..f791c558
--- /dev/null
+++ b/doc/development.md
@@ -0,0 +1,73 @@
+
+# Development
+
+## Linting
+
+pip install black==23.7.0
+pre-commit install
+black --check .
+black .
+
+## Import users from active directory
+
+```bash
+./manage.py import_users
+```
+
+## Import projects, datasets or partners from external system
+
+Single file mode:
+
+```bash
+./manage.py import_projects -f path/to/json_file.json
+```
+
+Batch mode:
+
+```bash
+./manage.py import_projects -d path/to/dir/with/json/files/
+```
+
+Available commands: `import_projects`, `import_datasets`, `import_partners`.
+
+In case of problems, add `--verbose` flag to the command, and take a look inside `./log/daisy.log`.
+
+## Install js and css dependencies
+
+```bash
+cd web/static/vendor/
+npm ci
+```
+
+## Compile daisy.scss and React
+
+```bash
+cd web/static/vendor
+npm run-script build
+```
+
+## Run the built-in web server (for development)
+
+```bash
+./manage.py runserver
+```
+
+## Run the tests
+
+The following command will install the test dependencies and execute the tests:
+
+```bash
+python setup.py pytest
+```
+
+run test for a specific file:
+
+```bash
+python setup.py pytest --addopts web/tests/test_dataset.py
+```
+
+If tests dependencies are already installed, one can also run the tests just by executing:
+
+```bash
+pytest
+```
diff --git a/DEPLOYMENT.md b/doc/legacy-deployment.md
similarity index 99%
rename from DEPLOYMENT.md
rename to doc/legacy-deployment.md
index e2515fe6..0bc5bfeb 100644
--- a/DEPLOYMENT.md
+++ b/doc/legacy-deployment.md
@@ -1,4 +1,7 @@
-# Installation
+
+**DISCLAIMER**: This is a set of instructions for legacy deployment and administration for version <1.8.1. It will be removed in future.
+
+# Legacy deployment and administration manual
 
 Following instructions are for CentOS Linux 7 (Core).
 
diff --git a/doc/update.md b/doc/update.md
index 3cc26114..8cdfa8db 100644
--- a/doc/update.md
+++ b/doc/update.md
@@ -1,12 +1,12 @@
 # Updating the Project
 
-### Pull Latest Changes
+## Pull Latest Changes
 
 ```bash
 git pull origin main
 ```
 
-### Rebuild Services After Code Changes
+## Rebuild Services After Code Changes
 
 If you make changes to the code or dependencies, rebuild the affected services:
 
@@ -15,7 +15,7 @@ docker compose build web worker beat
 docker compose up -d
 ```
 
-### Database Backup Before Upgrade
+## Database Backup Before Upgrade
 
 Create a database backup before upgrading:
 
@@ -23,7 +23,7 @@ Create a database backup before upgrading:
 docker compose exec backup sh /code/scripts/db.sh backup
 ```
 
-### Upgrade Steps
+## Upgrade Steps
 
 1. **Pull Latest Changes:**
 
@@ -68,5 +68,3 @@ docker compose exec backup sh /code/scripts/db.sh backup
     ```bash
     docker compose exec web python manage.py import_users
     ```
-
----
\ No newline at end of file

From 0f0ad34faaa0753e46b122aab8f4f1b9cf81935f Mon Sep 17 00:00:00 2001
From: Moustapha Cheikh <moustapha.cheikh@uni.lu>
Date: Thu, 26 Sep 2024 09:04:18 +0200
Subject: [PATCH 18/43] Support setting restore

---
 docker-compose.yaml       |  1 +
 env.template              |  7 +++++++
 scripts/backup_script.sh  |  0
 scripts/db.sh             |  3 ++-
 scripts/legacy_restore.sh | 30 +++++++++++++++++++++++++++---
 5 files changed, 37 insertions(+), 4 deletions(-)
 create mode 100644 env.template
 mode change 100755 => 100644 scripts/backup_script.sh
 mode change 100755 => 100644 scripts/db.sh

diff --git a/docker-compose.yaml b/docker-compose.yaml
index f11d8694..fa442086 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -112,6 +112,7 @@ services:
             - MEDIA_DIR=/code/documents
             - SOLR_PORT=8983
             - RABBITMQ_MANAGEMENT_PORT=${RABBITMQ_MANAGEMENT_PORT:-15672}
+            - BACKUP_VOLUME=${BACKUP_VOLUME:-../backups}
         volumes:
             - ${BACKUP_VOLUME:-../backups}:/backups
             - .:/code
diff --git a/env.template b/env.template
new file mode 100644
index 00000000..f51ba9e7
--- /dev/null
+++ b/env.template
@@ -0,0 +1,7 @@
+# Database configuration
+DB_NAME=daisy
+DB_USER=daisy
+DB_PASSWORD=daisy
+
+# Backup configuration
+BACKUP_VOLUME=../backups
diff --git a/scripts/backup_script.sh b/scripts/backup_script.sh
old mode 100755
new mode 100644
diff --git a/scripts/db.sh b/scripts/db.sh
old mode 100755
new mode 100644
index 4f0cea5b..f5c43e20
--- a/scripts/db.sh
+++ b/scripts/db.sh
@@ -10,6 +10,7 @@ DB_NAME="${DB_NAME:-daisy}"                # PostgreSQL database name
 DB_USER="${DB_USER:-daisy}"                # PostgreSQL user
 DB_PASSWORD="${DB_PASSWORD:-daisy}"        # PostgreSQL password
 MEDIA_DIR="${MEDIA_DIR:-/code/documents}"  # Django media directory
+BACKUP_VOLUME="${BACKUP_VOLUME:-../backups}"  # Backup volume
 
 if [ "${BACKUP_DIR}" == "../backups" ] && [ ! -d "${BACKUP_DIR}" ]; then
     mkdir -p "${BACKUP_DIR}"
@@ -58,7 +59,7 @@ backup() {
     # Remove temporary backup directory
     rm -rf "${temp_backup_dir}"
 
-    echo "Backup completed successfully: ${backup_file}"
+    echo "Backup completed successfully: ${BACKUP_VOLUME}/backup_${timestamp}.tar.gz"
 }
 
 # Function to perform restore
diff --git a/scripts/legacy_restore.sh b/scripts/legacy_restore.sh
index e2ff16a2..5e5c0895 100644
--- a/scripts/legacy_restore.sh
+++ b/scripts/legacy_restore.sh
@@ -28,13 +28,15 @@ if ! tar -xzf "$TAR_FILE" -C "$TEMP_DIR" > /dev/null 2>&1; then
     exit 1
 fi
 
-echo "Step 3: Locating SQL dump and documents directory..."
+echo "Step 3: Locating SQL dump, documents directory, and settings_local.py..."
 SQL_DUMP=$(find "$TEMP_DIR" -name "daisy_dump.sql")
 DOCUMENTS_DIR=$(find "$TEMP_DIR" -type d -name "documents" ! -path "*/templates/*" | head -n 1)
+SETTINGS_LOCAL=$(find "$TEMP_DIR" -type f -name "settings_local.py" | head -n 1)
 
 echo "Step 4: Verifying extracted contents..."
 echo "  - SQL dump found: $([ -n "$SQL_DUMP" ] && echo "Yes" || echo "No")"
 echo "  - Documents directory found: $([ -n "$DOCUMENTS_DIR" ] && echo "Yes" || echo "No")"
+echo "  - settings_local.py found: $([ -n "$SETTINGS_LOCAL" ] && echo "Yes" || echo "No")"
 
 if [ -z "$SQL_DUMP" ]; then
     echo "ERROR: Could not find daisy_dump.sql in the archive" >&2
@@ -45,6 +47,7 @@ fi
 echo "Step 5: Extraction successful."
 echo "  - SQL dump location: $SQL_DUMP"
 echo "  - Documents location: $DOCUMENTS_DIR"
+echo "  - settings_local.py location: $SETTINGS_LOCAL"
 
 echo "Step 6: Dropping existing database..."
 if [ "$SHOW_DB_LOGS" = "true" ]; then
@@ -98,7 +101,28 @@ else
     echo "WARNING: No media backup found in the archive. Skipping media restoration."
 fi
 
-echo "Step 10: Cleaning up temporary files..."
+echo "Step 10: Restoring settings_local.py..."
+if [ -n "$SETTINGS_LOCAL" ] && [ -f "$SETTINGS_LOCAL" ]; then
+    echo "  - Copying settings_local.py to /code/elixir_daisy/"
+    if [ -f "/code/elixir_daisy/settings_local.py" ]; then
+        echo "  - Existing settings_local.py found. Replacing it."
+        if ! cp -f "$SETTINGS_LOCAL" "/code/elixir_daisy/settings_local.py"; then
+            echo "ERROR: Failed to replace existing settings_local.py" >&2
+            rm -rf "$TEMP_DIR"
+            exit 1
+        fi
+    else
+        if ! cp "$SETTINGS_LOCAL" "/code/elixir_daisy/settings_local.py"; then
+            echo "ERROR: Failed to restore settings_local.py" >&2
+            rm -rf "$TEMP_DIR"
+            exit 1
+        fi
+    fi
+else
+    echo "WARNING: settings_local.py not found in the archive. Skipping restoration."
+fi
+
+echo "Step 11: Cleaning up temporary files..."
 rm -rf "$TEMP_DIR"
 
-echo "Step 11: Restoration completed successfully."
+echo "Step 12: Restoration completed successfully."

From 0ae9faee065c6fa7c685fb4abed59523b7ae012c Mon Sep 17 00:00:00 2001
From: Moustapha Cheikh <moustapha.cheikh@uni.lu>
Date: Tue, 24 Sep 2024 10:45:50 +0200
Subject: [PATCH 19/43] Fix metaclass conflict in NotifyMixin and refactor
 related models

---
 core/models/access.py    |  4 ++--
 core/models/dataset.py   |  4 ++--
 core/models/document.py  |  4 ++--
 core/models/project.py   |  4 ++--
 core/models/utils.py     |  6 ------
 notification/__init__.py | 12 ++++--------
 6 files changed, 12 insertions(+), 22 deletions(-)

diff --git a/core/models/access.py b/core/models/access.py
index 25c13a7a..6ca25c36 100644
--- a/core/models/access.py
+++ b/core/models/access.py
@@ -13,7 +13,7 @@
 
 from enumchoicefield import EnumChoiceField, ChoiceEnum
 
-from .utils import CoreModel, CoreNotifyMeta
+from .utils import CoreModel
 from notification import NotifyMixin
 from notification.models import NotificationVerb, Notification
 from core.utils import DaisyLogger
@@ -35,7 +35,7 @@ class StatusChoices(ChoiceEnum):
     terminated = "Terminated"
 
 
-class Access(CoreModel, NotifyMixin, metaclass=CoreNotifyMeta):
+class Access(CoreModel, NotifyMixin):
     """
     Represents the access given to an internal (LCSB) entity over data storage locations.
     """
diff --git a/core/models/dataset.py b/core/models/dataset.py
index 938ea2a2..b52d4860 100644
--- a/core/models/dataset.py
+++ b/core/models/dataset.py
@@ -18,7 +18,7 @@
 from core.permissions.mapping import PERMISSION_MAPPING
 from notification import NotifyMixin
 from notification.models import Notification, NotificationVerb
-from .utils import CoreTrackedModel, TextFieldWithInputWidget, CoreNotifyMeta
+from .utils import CoreTrackedModel, TextFieldWithInputWidget
 from .partner import HomeOrganisation
 
 if typing.TYPE_CHECKING:
@@ -28,7 +28,7 @@
 logger = DaisyLogger(__name__)
 
 
-class Dataset(CoreTrackedModel, NotifyMixin, metaclass=CoreNotifyMeta):
+class Dataset(CoreTrackedModel, NotifyMixin):
     class Meta:
         app_label = "core"
         get_latest_by = "added"
diff --git a/core/models/document.py b/core/models/document.py
index c1a791c3..7f1626a4 100644
--- a/core/models/document.py
+++ b/core/models/document.py
@@ -17,7 +17,7 @@
 from django.core.files.storage import default_storage
 from django.urls import reverse
 
-from .utils import CoreModel, CoreNotifyMeta
+from .utils import CoreModel
 from core.utils import DaisyLogger
 from notification.models import Notification, NotificationVerb
 from notification import NotifyMixin
@@ -39,7 +39,7 @@ def get_file_name(instance, filename):
     )
 
 
-class Document(CoreModel, NotifyMixin, metaclass=CoreNotifyMeta):
+class Document(CoreModel, NotifyMixin):
     """
     Represents a document
     """
diff --git a/core/models/project.py b/core/models/project.py
index 8baa7c87..8f55f5ab 100644
--- a/core/models/project.py
+++ b/core/models/project.py
@@ -19,7 +19,7 @@
 from notification.models import NotificationVerb, Notification
 from core.utils import DaisyLogger
 
-from .utils import CoreTrackedModel, COMPANY, CoreNotifyMeta
+from .utils import CoreTrackedModel, COMPANY
 from .partner import HomeOrganisation
 
 
@@ -30,7 +30,7 @@
 logger = DaisyLogger(__name__)
 
 
-class Project(CoreTrackedModel, NotifyMixin, metaclass=CoreNotifyMeta):
+class Project(CoreTrackedModel, NotifyMixin):
     class Meta:
         app_label = "core"
         get_latest_by = "added"
diff --git a/core/models/utils.py b/core/models/utils.py
index fbdb4515..6f2bd02c 100644
--- a/core/models/utils.py
+++ b/core/models/utils.py
@@ -10,8 +10,6 @@
 from django.utils.module_loading import import_string
 from django.contrib.auth.hashers import make_password
 
-from notification import NotifyMixin
-
 COMPANY = getattr(settings, "COMPANY", "Company")
 
 
@@ -48,10 +46,6 @@ class Meta:
         abstract = True
 
 
-class CoreNotifyMeta(type(CoreModel), type(NotifyMixin)):
-    pass
-
-
 class CoreTrackedModel(CoreModel):
     elu_accession = models.CharField(
         unique=True,
diff --git a/notification/__init__.py b/notification/__init__.py
index 7d95d411..c87378d9 100644
--- a/notification/__init__.py
+++ b/notification/__init__.py
@@ -1,4 +1,3 @@
-from abc import ABC, abstractmethod
 import typing
 from typing import List, Optional
 from datetime import timedelta
@@ -11,15 +10,14 @@
     User = settings.AUTH_USER_MODEL
 
 
-class NotifyMixin(ABC):
+class NotifyMixin:
     @staticmethod
-    @abstractmethod
     def get_notification_recipients() -> List["User"]:
         """
         Should query the users based on their notification settings
         and the entity.
         """
-        pass
+        raise NotImplementedError("Subclasses must implement this method")
 
     @classmethod
     def make_notifications(cls, exec_date: "date"):
@@ -41,7 +39,6 @@ def make_notifications(cls, exec_date: "date"):
             cls.make_notifications_for_user(day_offset, exec_date, user)
 
     @classmethod
-    @abstractmethod
     def make_notifications_for_user(
         cls, day_offset: "timedelta", exec_date: "date", user: "User"
     ):
@@ -53,15 +50,14 @@ def make_notifications_for_user(
             exec_date: The date of execution of the task.
             user: The user to create the notification for.
         """
-        pass
+        raise NotImplementedError("Subclasses must implement this method")
 
     @staticmethod
-    @abstractmethod
     def notify(user: "User", obj: object, verb: "NotificationVerb"):
         """
         Notify the user about the entity.
         """
-        pass
+        raise NotImplementedError("Subclasses must implement this method")
 
     @staticmethod
     def get_notification_setting(user: "User"):

From 9425dd56d45baa15e8024b4e9969896e6f1c69e5 Mon Sep 17 00:00:00 2001
From: Moustapha Cheikh <moustapha.cheikh@uni.lu>
Date: Thu, 26 Sep 2024 09:24:12 +0200
Subject: [PATCH 20/43] Update main.yml

---
 .github/workflows/main.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 3dbb49f8..fcdfd6b6 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -25,17 +25,17 @@ jobs:
       run: cp elixir_daisy/settings_compose_ci.py elixir_daisy/settings_compose.py
 
     - name: Build and start containers
-      run: docker-compose up -d --build
+      run: docker compose up -d --build
 
     - name: Check code formatting with Black
-      run: docker-compose exec -T web black --check --verbose .
+      run: docker compose exec -T web black --check --verbose .
 
     - name: Install test dependencies
-      run: docker-compose exec -T web pip install ".[test]"
+      run: docker compose exec -T web pip install ".[test]"
 
     - name: Execute the tests
-      run: docker-compose exec -T web pytest
+      run: docker compose exec -T web pytest
 
     - name: Stop containers
       if: always()
-      run: docker-compose down
+      run: docker compose down

From a66f1e132db4100fccb54357b29c8c2d3c49e29a Mon Sep 17 00:00:00 2001
From: Moustapha Cheikh <moustapha.cheikh@uni.lu>
Date: Fri, 27 Sep 2024 10:20:54 +0200
Subject: [PATCH 21/43] Update legacy backup script to support test/production
 backups

---
 doc/backup.md             |   4 +-
 scripts/legacy_restore.sh | 119 +++++++++++++++++++++++++++-----------
 2 files changed, 88 insertions(+), 35 deletions(-)

diff --git a/doc/backup.md b/doc/backup.md
index 2ea5da89..96418100 100644
--- a/doc/backup.md
+++ b/doc/backup.md
@@ -112,11 +112,11 @@ To restore backup created before version 1.8.1 on newer versions with docker dep
 
 ```bash
 # Copy the legacy backup file to the backup container
-docker cp ../daisy.tar.gz $(docker compose ps -q backup):/code/daisy_prod.tar.gz
+docker cp ../daisy_prod.tar.gz $(docker compose ps -q backup):/code/daisy_prod.tar.gz
 
 # Execute the legacy_restore.sh script inside the running container
 docker compose exec backup /bin/sh -c "sh /code/scripts/legacy_restore.sh /code/daisy_prod.tar.gz && rm /code/daisy_prod.tar.gz"
 docker compose run web python manage.py rebuild_index --noinput
 ```
 
-Replace `../daisy.tar.gz` with the actual path to legacy backup file.
+Replace `../daisy_prod.tar.gz` with the actual path to legacy backup file.
diff --git a/scripts/legacy_restore.sh b/scripts/legacy_restore.sh
index 5e5c0895..a44b2414 100644
--- a/scripts/legacy_restore.sh
+++ b/scripts/legacy_restore.sh
@@ -1,6 +1,7 @@
-#!/bin/bash
+#!/bin/sh
 
-set -euo pipefail
+# Exit immediately if a command exits with a non-zero status and treat unset variables as errors
+set -eu
 
 # Configuration
 DB_HOST="${DB_HOST:-db}"
@@ -11,45 +12,100 @@ DB_PASSWORD="${DB_PASSWORD:-daisy}"
 MEDIA_DIR="${MEDIA_DIR:-/code/documents}"
 SHOW_DB_LOGS="${SHOW_DB_LOGS:-true}"
 
+# Check if TAR_FILE argument is provided
 if [ $# -ne 1 ]; then
     echo "Usage: $0 path_to_daisy_backup_tar.gz" >&2
     exit 1
 fi
 
 TAR_FILE=$1
-TEMP_DIR=$(mktemp -d)
 
-echo "Step 1: Starting restoration process..."
+# Initialize variables before defining cleanup
+EXTRACT_LIST=""
 
-echo "Step 2: Extracting backup archive..."
-if ! tar -xzf "$TAR_FILE" -C "$TEMP_DIR" > /dev/null 2>&1; then
-    echo "ERROR: Failed to extract backup archive" >&2
-    rm -rf "$TEMP_DIR"
+# Create a temporary directory for extraction
+TEMP_DIR=$(mktemp -d /tmp/restore_temp.XXXXXX)
+
+# Function to clean up temporary directories on exit
+cleanup() {
+    rm -rf "$TEMP_DIR" "$EXTRACT_LIST"
+}
+trap cleanup EXIT
+
+# Ensure TEMP_DIR was created successfully
+if [ ! -d "$TEMP_DIR" ]; then
+    echo "ERROR: Failed to create temporary directory." >&2
     exit 1
 fi
 
-echo "Step 3: Locating SQL dump, documents directory, and settings_local.py..."
-SQL_DUMP=$(find "$TEMP_DIR" -name "daisy_dump.sql")
-DOCUMENTS_DIR=$(find "$TEMP_DIR" -type d -name "documents" ! -path "*/templates/*" | head -n 1)
-SETTINGS_LOCAL=$(find "$TEMP_DIR" -type f -name "settings_local.py" | head -n 1)
+echo "Step 1: Starting restoration process..."
 
-echo "Step 4: Verifying extracted contents..."
-echo "  - SQL dump found: $([ -n "$SQL_DUMP" ] && echo "Yes" || echo "No")"
-echo "  - Documents directory found: $([ -n "$DOCUMENTS_DIR" ] && echo "Yes" || echo "No")"
-echo "  - settings_local.py found: $([ -n "$SETTINGS_LOCAL" ] && echo "Yes" || echo "No")"
+echo "Step 2: Searching for daisy_dump.sql, documents directory, and settings_local.py in the archive..."
 
-if [ -z "$SQL_DUMP" ]; then
-    echo "ERROR: Could not find daisy_dump.sql in the archive" >&2
-    rm -rf "$TEMP_DIR"
+# Initialize search result variables
+SQL_DUMP_PATH=""
+DOCUMENTS_DIR_PATH=""
+SETTINGS_LOCAL_PATH=""
+
+# Search for daisy_dump.sql
+SQL_DUMP_PATH=$(tar -tzf "$TAR_FILE" | grep "/daisy_dump\.sql$" | head -n 1 || true)
+
+# Search for documents directory (any depth) excluding those under 'templates'
+DOCUMENTS_DIR_PATH=$(tar -tzf "$TAR_FILE" | grep "/documents/$" | grep -v "/templates/" | head -n 1 || true)
+
+# Search for settings_local.py (optional)
+SETTINGS_LOCAL_PATH=$(tar -tzf "$TAR_FILE" | grep "/settings_local\.py$" | head -n 1 || true)
+
+# Check if SQL dump is found
+if [ -z "$SQL_DUMP_PATH" ]; then
+    echo "ERROR: SQL dump 'daisy_dump.sql' not found in the archive." >&2
+    # Proceeding without exiting to allow extraction of other files
+fi
+
+# Check if documents directory is found
+if [ -z "$DOCUMENTS_DIR_PATH" ]; then
+    echo "WARNING: Documents directory not found in the archive or it is under 'templates/'." >&2
+fi
+
+# settings_local.py is optional
+if [ -z "$SETTINGS_LOCAL_PATH" ]; then
+    echo "INFO: settings_local.py not found in the archive. Continuing without it." >&2
+fi
+
+# Prepare list of paths to extract
+EXTRACT_LIST=$(mktemp /tmp/extract_list.XXXXXX)
+
+# Add found paths to the extract list if they exist
+if [ -n "$SQL_DUMP_PATH" ]; then
+    echo "$SQL_DUMP_PATH" >> "$EXTRACT_LIST"
+fi
+
+if [ -n "$DOCUMENTS_DIR_PATH" ]; then
+    echo "$DOCUMENTS_DIR_PATH" >> "$EXTRACT_LIST"
+fi
+
+if [ -n "$SETTINGS_LOCAL_PATH" ]; then
+    echo "$SETTINGS_LOCAL_PATH" >> "$EXTRACT_LIST"
+fi
+
+# Check if EXTRACT_LIST has entries
+if [ ! -s "$EXTRACT_LIST" ]; then
+    echo "ERROR: No files or directories to extract. Exiting." >&2
     exit 1
 fi
 
-echo "Step 5: Extraction successful."
-echo "  - SQL dump location: $SQL_DUMP"
-echo "  - Documents location: $DOCUMENTS_DIR"
-echo "  - settings_local.py location: $SETTINGS_LOCAL"
+echo "Step 3: Extracting the necessary files and directories..."
+if ! tar -xzf "$TAR_FILE" -C "$TEMP_DIR" -T "$EXTRACT_LIST" 2>&1; then
+    echo "ERROR: Failed to extract necessary files from the backup archive." >&2
+    exit 1
+fi
+
+echo "Step 4: Verifying extracted contents..."
+SQL_DUMP=$(find "$TEMP_DIR" -type f -name "daisy_dump.sql" | head -n 1 || true)
+DOCUMENTS_DIR=$(find "$TEMP_DIR" -type d -name "documents" | grep -v "/templates/" | head -n 1 || true)
+SETTINGS_LOCAL=$(find "$TEMP_DIR" -type f -name "settings_local.py" | head -n 1 || true)
 
-echo "Step 6: Dropping existing database..."
+echo "Step 5: Dropping existing database..."
 if [ "$SHOW_DB_LOGS" = "true" ]; then
     PGPASSWORD="${DB_PASSWORD}" psql -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -d postgres -c "DROP DATABASE IF EXISTS ${DB_NAME};"
 else
@@ -62,7 +118,7 @@ if [ $? -ne 0 ]; then
     exit 1
 fi
 
-echo "Step 7: Creating new database..."
+echo "Step 6: Creating new database..."
 if [ "$SHOW_DB_LOGS" = "true" ]; then
     PGPASSWORD="${DB_PASSWORD}" psql -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -d postgres -c "CREATE DATABASE ${DB_NAME};"
 else
@@ -75,7 +131,7 @@ if [ $? -ne 0 ]; then
     exit 1
 fi
 
-echo "Step 8: Restoring PostgreSQL database from SQL dump..."
+echo "Step 7: Restoring PostgreSQL database from SQL dump..."
 if [ "$SHOW_DB_LOGS" = "true" ]; then
     PGPASSWORD="${DB_PASSWORD}" psql -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -d "${DB_NAME}" -f "${SQL_DUMP}"
 else
@@ -88,9 +144,8 @@ if [ $? -ne 0 ]; then
     exit 1
 fi
 
-echo "Step 9: Restoring Django media files..."
+echo "Step 8: Restoring Django media files..."
 if [ -n "$DOCUMENTS_DIR" ] && [ -d "$DOCUMENTS_DIR" ]; then
-    echo "  - Copying files from $DOCUMENTS_DIR to $MEDIA_DIR"
     rm -rf "${MEDIA_DIR:?}/"*
     if ! cp -R "${DOCUMENTS_DIR}/." "${MEDIA_DIR}/"; then
         echo "ERROR: Media files restoration failed" >&2
@@ -101,11 +156,9 @@ else
     echo "WARNING: No media backup found in the archive. Skipping media restoration."
 fi
 
-echo "Step 10: Restoring settings_local.py..."
+echo "Step 9: Restoring settings_local.py..."
 if [ -n "$SETTINGS_LOCAL" ] && [ -f "$SETTINGS_LOCAL" ]; then
-    echo "  - Copying settings_local.py to /code/elixir_daisy/"
     if [ -f "/code/elixir_daisy/settings_local.py" ]; then
-        echo "  - Existing settings_local.py found. Replacing it."
         if ! cp -f "$SETTINGS_LOCAL" "/code/elixir_daisy/settings_local.py"; then
             echo "ERROR: Failed to replace existing settings_local.py" >&2
             rm -rf "$TEMP_DIR"
@@ -122,7 +175,7 @@ else
     echo "WARNING: settings_local.py not found in the archive. Skipping restoration."
 fi
 
-echo "Step 11: Cleaning up temporary files..."
+echo "Step 10: Cleaning up temporary files..."
 rm -rf "$TEMP_DIR"
 
-echo "Step 12: Restoration completed successfully."
+echo "Step 11: Restoration completed successfully."

From cc8298cd49f6daed8dbca83040b06ee38ed60749 Mon Sep 17 00:00:00 2001
From: Moustapha Cheikh <moustapha.cheikh@uni.lu>
Date: Fri, 27 Sep 2024 14:54:39 +0200
Subject: [PATCH 22/43] Update legacy backup script

---
 scripts/legacy_restore.sh | 49 ++++++++++++++-------------------------
 1 file changed, 17 insertions(+), 32 deletions(-)

diff --git a/scripts/legacy_restore.sh b/scripts/legacy_restore.sh
index a44b2414..3e447baa 100644
--- a/scripts/legacy_restore.sh
+++ b/scripts/legacy_restore.sh
@@ -40,6 +40,7 @@ fi
 
 echo "Step 1: Starting restoration process..."
 
+# Step 2: Search for necessary files and directories
 echo "Step 2: Searching for daisy_dump.sql, documents directory, and settings_local.py in the archive..."
 
 # Initialize search result variables
@@ -48,7 +49,7 @@ DOCUMENTS_DIR_PATH=""
 SETTINGS_LOCAL_PATH=""
 
 # Search for daisy_dump.sql
-SQL_DUMP_PATH=$(tar -tzf "$TAR_FILE" | grep "/daisy_dump\.sql$" | head -n 1 || true)
+SQL_DUMP_PATH=$(tar -tzf "$TAR_FILE" | grep "/daisy_dump\.sql$" | grep -v "/daisy/" | head -n 1 || true)
 
 # Search for documents directory (any depth) excluding those under 'templates'
 DOCUMENTS_DIR_PATH=$(tar -tzf "$TAR_FILE" | grep "/documents/$" | grep -v "/templates/" | head -n 1 || true)
@@ -88,50 +89,33 @@ if [ -n "$SETTINGS_LOCAL_PATH" ]; then
     echo "$SETTINGS_LOCAL_PATH" >> "$EXTRACT_LIST"
 fi
 
+# Debug: Show the list of paths to extract
+echo "DEBUG: Paths to extract:"
+cat "$EXTRACT_LIST"
+
 # Check if EXTRACT_LIST has entries
 if [ ! -s "$EXTRACT_LIST" ]; then
     echo "ERROR: No files or directories to extract. Exiting." >&2
     exit 1
 fi
 
+# Step 3: Extract only the necessary files and directories
 echo "Step 3: Extracting the necessary files and directories..."
 if ! tar -xzf "$TAR_FILE" -C "$TEMP_DIR" -T "$EXTRACT_LIST" 2>&1; then
     echo "ERROR: Failed to extract necessary files from the backup archive." >&2
     exit 1
 fi
 
+# Step 4: Verify extracted contents
 echo "Step 4: Verifying extracted contents..."
-SQL_DUMP=$(find "$TEMP_DIR" -type f -name "daisy_dump.sql" | head -n 1 || true)
+SQL_DUMP=$(find "$TEMP_DIR" -type f -name "daisy_dump.sql" | grep -v "/home/daisy/" | head -n 1 || true)
 DOCUMENTS_DIR=$(find "$TEMP_DIR" -type d -name "documents" | grep -v "/templates/" | head -n 1 || true)
 SETTINGS_LOCAL=$(find "$TEMP_DIR" -type f -name "settings_local.py" | head -n 1 || true)
 
-echo "Step 5: Dropping existing database..."
-if [ "$SHOW_DB_LOGS" = "true" ]; then
-    PGPASSWORD="${DB_PASSWORD}" psql -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -d postgres -c "DROP DATABASE IF EXISTS ${DB_NAME};"
-else
-    PGPASSWORD="${DB_PASSWORD}" psql -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -d postgres -c "DROP DATABASE IF EXISTS ${DB_NAME};" > /dev/null 2>&1
-fi
-
-if [ $? -ne 0 ]; then
-    echo "ERROR: Failed to drop existing database" >&2
-    rm -rf "$TEMP_DIR"
-    exit 1
-fi
-
-echo "Step 6: Creating new database..."
-if [ "$SHOW_DB_LOGS" = "true" ]; then
-    PGPASSWORD="${DB_PASSWORD}" psql -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -d postgres -c "CREATE DATABASE ${DB_NAME};"
-else
-    PGPASSWORD="${DB_PASSWORD}" psql -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -d postgres -c "CREATE DATABASE ${DB_NAME};" > /dev/null 2>&1
-fi
-
-if [ $? -ne 0 ]; then
-    echo "ERROR: Failed to create new database" >&2
-    rm -rf "$TEMP_DIR"
-    exit 1
-fi
-
-echo "Step 7: Restoring PostgreSQL database from SQL dump..."
+echo "  - SQL dump found: $([ -n "$SQL_DUMP" ] && echo 'Yes' || echo 'No')"
+echo "  - Documents directory found: $([ -n "$DOCUMENTS_DIR" ] && echo 'Yes' || echo 'No')"
+echo "  - settings_local.py found: $([ -n "$SETTINGS_LOCAL" ] && echo 'Yes' || echo 'No')"
+echo "Step 8: Restoring PostgreSQL database from SQL dump..."
 if [ "$SHOW_DB_LOGS" = "true" ]; then
     PGPASSWORD="${DB_PASSWORD}" psql -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -d "${DB_NAME}" -f "${SQL_DUMP}"
 else
@@ -144,8 +128,9 @@ if [ $? -ne 0 ]; then
     exit 1
 fi
 
-echo "Step 8: Restoring Django media files..."
+echo "Step 9: Restoring Django media files..."
 if [ -n "$DOCUMENTS_DIR" ] && [ -d "$DOCUMENTS_DIR" ]; then
+    echo "  - Copying files from $DOCUMENTS_DIR to $MEDIA_DIR"
     rm -rf "${MEDIA_DIR:?}/"*
     if ! cp -R "${DOCUMENTS_DIR}/." "${MEDIA_DIR}/"; then
         echo "ERROR: Media files restoration failed" >&2
@@ -175,7 +160,7 @@ else
     echo "WARNING: settings_local.py not found in the archive. Skipping restoration."
 fi
 
-echo "Step 10: Cleaning up temporary files..."
+echo "Step 11: Cleaning up temporary files..."
 rm -rf "$TEMP_DIR"
 
-echo "Step 11: Restoration completed successfully."
+echo "Step 12: Restoration completed successfully."
\ No newline at end of file

From d25f9b322389e786a56c463246fc47e8922fde21 Mon Sep 17 00:00:00 2001
From: Moustapha Cheikh <moustapha.cheikh@uni.lu>
Date: Fri, 27 Sep 2024 14:54:51 +0200
Subject: [PATCH 23/43] Add mising migration file

---
 core/migrations/0037_auto_20240924_1020.py | 60 ++++++++++++++++++++++
 1 file changed, 60 insertions(+)
 create mode 100644 core/migrations/0037_auto_20240924_1020.py

diff --git a/core/migrations/0037_auto_20240924_1020.py b/core/migrations/0037_auto_20240924_1020.py
new file mode 100644
index 00000000..dfe7ff49
--- /dev/null
+++ b/core/migrations/0037_auto_20240924_1020.py
@@ -0,0 +1,60 @@
+# Generated by Django 3.2.23 on 2024-09-24 08:20
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0036_use_restriction_codes"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="cohort",
+            name="elu_accession",
+            field=models.CharField(
+                blank=True,
+                help_text="Unique persistent identifier of the record.",
+                max_length=20,
+                null=True,
+                unique=True,
+                verbose_name="Accession number",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="dataset",
+            name="elu_accession",
+            field=models.CharField(
+                blank=True,
+                help_text="Unique persistent identifier of the record.",
+                max_length=20,
+                null=True,
+                unique=True,
+                verbose_name="Accession number",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="partner",
+            name="elu_accession",
+            field=models.CharField(
+                blank=True,
+                help_text="Unique persistent identifier of the record.",
+                max_length=20,
+                null=True,
+                unique=True,
+                verbose_name="Accession number",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="project",
+            name="elu_accession",
+            field=models.CharField(
+                blank=True,
+                help_text="Unique persistent identifier of the record.",
+                max_length=20,
+                null=True,
+                unique=True,
+                verbose_name="Accession number",
+            ),
+        ),
+    ]

From 2325528553c6a16d6fffc3fac78110e5d67bc6d5 Mon Sep 17 00:00:00 2001
From: Vilem Ded <vilem.ded@uni.lu>
Date: Fri, 27 Sep 2024 21:31:36 +0200
Subject: [PATCH 24/43] remove duplicated step

---
 doc/backup.md | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/doc/backup.md b/doc/backup.md
index 96418100..5228c501 100644
--- a/doc/backup.md
+++ b/doc/backup.md
@@ -84,12 +84,6 @@ docker compose run web python manage.py rebuild_index --noinput
 
 - Replace `../backups/backup_<timestamp>.tar.gz` with the actual file path.
 
-Rebuild the Solr index after restoration:
-
-```bash
-docker compose exec web python manage.py rebuild_index --noinput
-```
-
 ## List Cron Jobs
 
 View the automatic backup schedule:

From 32ec1f1548088873058bf7f698020501b87028d9 Mon Sep 17 00:00:00 2001
From: Vilem Ded <vilem.ded@uni.lu>
Date: Fri, 27 Sep 2024 21:34:51 +0200
Subject: [PATCH 25/43] removing duplicated steps from update.md

---
 doc/update.md | 17 +----------------
 1 file changed, 1 insertion(+), 16 deletions(-)

diff --git a/doc/update.md b/doc/update.md
index 8cdfa8db..2776d87d 100644
--- a/doc/update.md
+++ b/doc/update.md
@@ -1,20 +1,5 @@
 # Updating the Project
 
-## Pull Latest Changes
-
-```bash
-git pull origin main
-```
-
-## Rebuild Services After Code Changes
-
-If you make changes to the code or dependencies, rebuild the affected services:
-
-```bash
-docker compose build web worker beat
-docker compose up -d
-```
-
 ## Database Backup Before Upgrade
 
 Create a database backup before upgrading:
@@ -28,7 +13,7 @@ docker compose exec backup sh /code/scripts/db.sh backup
 1. **Pull Latest Changes:**
 
     ```bash
-    git pull origin main
+    git pull origin master
     ```
 
 2. **Rebuild Docker Images:**

From e2591eb32aac5b330a88418b86c60d2b3a573c5e Mon Sep 17 00:00:00 2001
From: Vilem Ded <vilem.ded@uni.lu>
Date: Fri, 27 Sep 2024 21:59:46 +0200
Subject: [PATCH 26/43] making section on django commands less verbose

---
 doc/administration.md | 21 +++++----------------
 1 file changed, 5 insertions(+), 16 deletions(-)

diff --git a/doc/administration.md b/doc/administration.md
index cf2e3c2e..dea66011 100644
--- a/doc/administration.md
+++ b/doc/administration.md
@@ -1,6 +1,6 @@
-# Managing
+# Management and administration
 
-### Access the Web Service
+## Access the Web Service
 
 Shell into the `web` container:
 
@@ -10,25 +10,14 @@ docker compose exec web /bin/bash
 
 ### Run Django Commands
 
-Run Django management commands using the `web` service.
+Run Django management commands, e.g. `makemigrations`, `migrate`, `createsuperuser`, etc., using the `web` service:
 
-#### Make Migrations
 
 ```bash
-docker compose exec web python manage.py makemigrations
+docker compose exec web python manage.py <django-command>
 ```
 
-#### Apply Migrations
-
-```bash
-docker compose exec web python manage.py migrate
-```
-
-#### Create Superuser
-
-```bash
-docker compose exec web python manage.py createsuperuser
-```
+See `docker compose exec web python manage.py --help` for all available commands.
 
 #### Collect Static Files
 

From 3d964c44ffc705525c6eca7e4d1cb072877e80e9 Mon Sep 17 00:00:00 2001
From: Vilem Ded <vilem.ded@uni.lu>
Date: Fri, 4 Oct 2024 15:53:22 +0200
Subject: [PATCH 27/43] fix deployement guide

---
 doc/deployment.md | 26 +++++++++++---------------
 1 file changed, 11 insertions(+), 15 deletions(-)

diff --git a/doc/deployment.md b/doc/deployment.md
index f2997894..f1fdbfd2 100644
--- a/doc/deployment.md
+++ b/doc/deployment.md
@@ -20,7 +20,7 @@ cd daisy
 
 ### Environment Variables
 
-Create a `.env` file in the project root to override default environment variables if necessary. See [.env.template](env.template) file for more detail.
+Create a `.env` file in the project root to override default environment variables if necessary. See [.env.template](env.template) file for more detail. Additionally, create `elixir_daisy/settings_local.py` file from `elixir_daisy/settings_local.template.py`.
 
 ## Installation
 
@@ -55,14 +55,13 @@ The project uses frontend assets that need to be compiled (e.g., with npm), you
 #### Install npm Dependencies
 
 ```bash
-cd web/static/vendor
-npm ci
+docker compose exec web npm --prefix web/static/vendor ci
 ```
 
 #### Build Frontend Assets
 
 ```bash
-npm run build
+docker compose exec web npm --prefix web/static/vendor run build
 ```
 
 #### Collect Static Files
@@ -110,35 +109,32 @@ docker compose exec web python manage.py rebuild_index --noinput
 
 ### Access the Application
 
-The application should now be accessible at:
-
-- **HTTP:** `http://localhost/`
-- **HTTPS:** `https://localhost/`
+The application should now be accessible on `https://localhost/`
 
 ## Scheduled Backup with Cron
 
 To schedule the backup script to run automatically at a specific time using cron, add the following line to your crontab:
 
-1. Open the crontab editor:
+1. Ensure the destination location for backups in `.env` file (`BACKUP_VOLUME` variable)
+
+2. Open the crontab editor:
 
     ```bash
     crontab -e
     ```
 
-2. Add the cron job entry (for example, to run the backup at 1 AM daily):
+3. Add the cron job entry (for example, to run the backup at 1 AM daily) with path to the backup script:
 
     ```bash
-    0 1 * * * /path/to/backup_script.sh
+    0 1 * * * <path-to-project-root>/scripts/backup_script.sh
     ```
 
-3. Check if the cron job is added:
+4. Check if the cron job is added:
 
     ```bash
-    docker compose exec backup crontab -l
+    crontab -l
     ```
 
-Replace `/path/to/backup_script.sh` with the actual path to backup_script.
-
 ### Restore Legacy Backup
 
 To restore from a legacy backup file (e.g., `daisy_prod.tar.gz`):

From d9a8b858eb5abe6f1da66dea9d4a182339ec8e85 Mon Sep 17 00:00:00 2001
From: Vilem Ded <vilem.ded@uni.lu>
Date: Fri, 4 Oct 2024 15:58:44 +0200
Subject: [PATCH 28/43] make scripts executable by default

---
 scripts/backup_script.sh  | 0
 scripts/db.sh             | 0
 scripts/legacy_restore.sh | 0
 3 files changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 scripts/backup_script.sh
 mode change 100644 => 100755 scripts/db.sh
 mode change 100644 => 100755 scripts/legacy_restore.sh

diff --git a/scripts/backup_script.sh b/scripts/backup_script.sh
old mode 100644
new mode 100755
diff --git a/scripts/db.sh b/scripts/db.sh
old mode 100644
new mode 100755
diff --git a/scripts/legacy_restore.sh b/scripts/legacy_restore.sh
old mode 100644
new mode 100755

From bbfba8cec46155c1336ae6ce36c52e06f81fe7fa Mon Sep 17 00:00:00 2001
From: Vilem Ded <vilem.ded@uni.lu>
Date: Fri, 4 Oct 2024 16:30:32 +0200
Subject: [PATCH 29/43] removing redundancy in backup steps

---
 doc/backup.md     | 14 +++++++-------
 doc/deployment.md | 20 +-------------------
 2 files changed, 8 insertions(+), 26 deletions(-)

diff --git a/doc/backup.md b/doc/backup.md
index 5228c501..2bf028a9 100644
--- a/doc/backup.md
+++ b/doc/backup.md
@@ -53,26 +53,26 @@ docker compose exec backup sh /code/scripts/db.sh backup
 
 To schedule the backup script to run automatically at a specific time using cron, add the following line to your crontab:
 
-1. Open the crontab editor:
+1. Ensure the destination location for backups in `.env` file (`BACKUP_VOLUME` variable)
+
+2. Open the crontab editor:
 
     ```bash
     crontab -e
     ```
 
-2. Add the cron job entry (for example, to run the backup at 1 AM daily):
+3. Add the cron job entry (for example, to run the backup at 1 AM daily) with path to the backup script:
 
     ```bash
-    0 1 * * * /path/to/backup_script.sh
+    0 1 * * * <path-to-project-root>/scripts/backup_script.sh
     ```
 
-3. Check if the cron job is added:
+4. Check if the cron job is added:
 
     ```bash
-    docker compose exec backup crontab -l
+    crontab -l
     ```
 
-Replace `/path/to/backup_script.sh` with the actual path to backup_script.
-
 ## Restore Backup
 
 Restore from a specific backup file:
diff --git a/doc/deployment.md b/doc/deployment.md
index f1fdbfd2..89d12334 100644
--- a/doc/deployment.md
+++ b/doc/deployment.md
@@ -113,27 +113,9 @@ The application should now be accessible on `https://localhost/`
 
 ## Scheduled Backup with Cron
 
-To schedule the backup script to run automatically at a specific time using cron, add the following line to your crontab:
+To ensure the backups are properly set up, please refer to the [Backup manual](backup.md#Scheduled-Backup-with-Cron)
 
-1. Ensure the destination location for backups in `.env` file (`BACKUP_VOLUME` variable)
 
-2. Open the crontab editor:
-
-    ```bash
-    crontab -e
-    ```
-
-3. Add the cron job entry (for example, to run the backup at 1 AM daily) with path to the backup script:
-
-    ```bash
-    0 1 * * * <path-to-project-root>/scripts/backup_script.sh
-    ```
-
-4. Check if the cron job is added:
-
-    ```bash
-    crontab -l
-    ```
 
 ### Restore Legacy Backup
 

From bbd19809080d92947383ad1d52cb62a3de2fc048 Mon Sep 17 00:00:00 2001
From: Vilem Ded <vilem.ded@uni.lu>
Date: Fri, 4 Oct 2024 16:49:55 +0200
Subject: [PATCH 30/43] add timezone

---
 .env.template       |  3 +++
 docker-compose.yaml | 13 +++++++++++++
 2 files changed, 16 insertions(+)

diff --git a/.env.template b/.env.template
index f51ba9e7..58523a62 100644
--- a/.env.template
+++ b/.env.template
@@ -5,3 +5,6 @@ DB_PASSWORD=daisy
 
 # Backup configuration
 BACKUP_VOLUME=../backups
+
+# Timezone
+TZ=Europe/Paris
\ No newline at end of file
diff --git a/docker-compose.yaml b/docker-compose.yaml
index fa442086..317efc55 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -6,6 +6,7 @@ services:
             - '5000'
         environment:
             - DJANGO_SETTINGS_MODULE=elixir_daisy.settings_compose
+            - TZ=${TZ}
         volumes:
             - statics:/static
             - solrdata:/solr
@@ -25,6 +26,7 @@ services:
             POSTGRES_PASSWORD: ${DB_PASSWORD:-daisy}
             POSTGRES_USER: ${DB_USER:-daisy}
             POSTGRES_DB: ${DB_NAME:-daisy}
+            TZ: ${TZ}
         networks:
             - daisy_network
         volumes:
@@ -33,6 +35,8 @@ services:
     nginx:
         build: ./docker/nginx
         restart: unless-stopped
+        environment:
+            - - TZ=${TZ}
         volumes:
             - statics:/public/static:ro
         ports:
@@ -46,6 +50,8 @@ services:
     solr:
         build: ./docker/solr
         restart: unless-stopped
+        environment:
+            - TZ=${TZ}
         ports:
             - "8983:8983"
         networks:
@@ -56,6 +62,8 @@ services:
     mq:
         image: rabbitmq:3.9-management-alpine
         restart: unless-stopped
+        environment:
+            - TZ=${TZ}
         networks:
             - daisy_network
         ports:
@@ -65,6 +73,8 @@ services:
     flower:
         image: mher/flower:0.9.7
         command: --broker=amqp://guest:guest@mq:5672// --broker_api=http://guest:guest@mq:15672/api/
+        environment:
+            - TZ=${TZ}
         ports:
             - "5555:5555"
         restart: unless-stopped
@@ -77,6 +87,7 @@ services:
         build: .
         environment:
             - DJANGO_SETTINGS_MODULE=elixir_daisy.settings_compose
+            - TZ=${TZ}
         volumes:
             - .:/code
         depends_on:
@@ -91,6 +102,7 @@ services:
         build: .
         environment:
             - DJANGO_SETTINGS_MODULE=elixir_daisy.settings_compose
+            - TZ=${TZ}
         volumes:
             - .:/code
         depends_on:
@@ -113,6 +125,7 @@ services:
             - SOLR_PORT=8983
             - RABBITMQ_MANAGEMENT_PORT=${RABBITMQ_MANAGEMENT_PORT:-15672}
             - BACKUP_VOLUME=${BACKUP_VOLUME:-../backups}
+            - TZ=${TZ}
         volumes:
             - ${BACKUP_VOLUME:-../backups}:/backups
             - .:/code

From e493663caba9ad060025cc275a64c10dba326951 Mon Sep 17 00:00:00 2001
From: Vilem Ded <vilem.ded@uni.lu>
Date: Fri, 4 Oct 2024 16:54:14 +0200
Subject: [PATCH 31/43] fix timezone - set default

---
 docker-compose.yaml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index 317efc55..c11fe6ba 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -6,7 +6,7 @@ services:
             - '5000'
         environment:
             - DJANGO_SETTINGS_MODULE=elixir_daisy.settings_compose
-            - TZ=${TZ}
+            - TZ=${TZ:-UTC}
         volumes:
             - statics:/static
             - solrdata:/solr
@@ -36,7 +36,7 @@ services:
         build: ./docker/nginx
         restart: unless-stopped
         environment:
-            - - TZ=${TZ}
+            - - TZ=${TZ:-UTC}
         volumes:
             - statics:/public/static:ro
         ports:
@@ -51,7 +51,7 @@ services:
         build: ./docker/solr
         restart: unless-stopped
         environment:
-            - TZ=${TZ}
+            - TZ=${TZ:-UTC}
         ports:
             - "8983:8983"
         networks:
@@ -63,7 +63,7 @@ services:
         image: rabbitmq:3.9-management-alpine
         restart: unless-stopped
         environment:
-            - TZ=${TZ}
+            - TZ=${TZ:-UTC}
         networks:
             - daisy_network
         ports:
@@ -74,7 +74,7 @@ services:
         image: mher/flower:0.9.7
         command: --broker=amqp://guest:guest@mq:5672// --broker_api=http://guest:guest@mq:15672/api/
         environment:
-            - TZ=${TZ}
+            - TZ=${TZ:-UTC}
         ports:
             - "5555:5555"
         restart: unless-stopped
@@ -87,7 +87,7 @@ services:
         build: .
         environment:
             - DJANGO_SETTINGS_MODULE=elixir_daisy.settings_compose
-            - TZ=${TZ}
+            - TZ=${TZ:-UTC}
         volumes:
             - .:/code
         depends_on:
@@ -102,7 +102,7 @@ services:
         build: .
         environment:
             - DJANGO_SETTINGS_MODULE=elixir_daisy.settings_compose
-            - TZ=${TZ}
+            - TZ=${TZ:-UTC}
         volumes:
             - .:/code
         depends_on:
@@ -125,7 +125,7 @@ services:
             - SOLR_PORT=8983
             - RABBITMQ_MANAGEMENT_PORT=${RABBITMQ_MANAGEMENT_PORT:-15672}
             - BACKUP_VOLUME=${BACKUP_VOLUME:-../backups}
-            - TZ=${TZ}
+            - TZ=${TZ:-UTC}
         volumes:
             - ${BACKUP_VOLUME:-../backups}:/backups
             - .:/code

From ed5c2e9cce8bf8571366afe6d22e387c1c50ffba Mon Sep 17 00:00:00 2001
From: Vilem Ded <vilem.ded@uni.lu>
Date: Fri, 4 Oct 2024 16:57:51 +0200
Subject: [PATCH 32/43] fix timezone - set default

---
 docker-compose.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index c11fe6ba..e52a31e6 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -26,7 +26,7 @@ services:
             POSTGRES_PASSWORD: ${DB_PASSWORD:-daisy}
             POSTGRES_USER: ${DB_USER:-daisy}
             POSTGRES_DB: ${DB_NAME:-daisy}
-            TZ: ${TZ}
+            TZ: ${TZ:-UTC}
         networks:
             - daisy_network
         volumes:

From 39b67a8b1b9c5734a041a0c0229277607c7379a2 Mon Sep 17 00:00:00 2001
From: Vilem Ded <vilem.ded@uni.lu>
Date: Fri, 4 Oct 2024 16:58:25 +0200
Subject: [PATCH 33/43] fix timezone - set default

---
 docker-compose.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index e52a31e6..8f3445c0 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -36,7 +36,7 @@ services:
         build: ./docker/nginx
         restart: unless-stopped
         environment:
-            - - TZ=${TZ:-UTC}
+            - TZ=${TZ:-UTC}
         volumes:
             - statics:/public/static:ro
         ports:

From 58f79959ddce5524ad45f3c939bdca368bf6684f Mon Sep 17 00:00:00 2001
From: Vilem Ded <vilem.ded@uni.lu>
Date: Mon, 7 Oct 2024 08:38:17 +0200
Subject: [PATCH 34/43] revert timzone fix

---
 .env.template       |  3 ---
 docker-compose.yaml | 13 -------------
 2 files changed, 16 deletions(-)

diff --git a/.env.template b/.env.template
index 58523a62..f51ba9e7 100644
--- a/.env.template
+++ b/.env.template
@@ -5,6 +5,3 @@ DB_PASSWORD=daisy
 
 # Backup configuration
 BACKUP_VOLUME=../backups
-
-# Timezone
-TZ=Europe/Paris
\ No newline at end of file
diff --git a/docker-compose.yaml b/docker-compose.yaml
index 8f3445c0..fa442086 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -6,7 +6,6 @@ services:
             - '5000'
         environment:
             - DJANGO_SETTINGS_MODULE=elixir_daisy.settings_compose
-            - TZ=${TZ:-UTC}
         volumes:
             - statics:/static
             - solrdata:/solr
@@ -26,7 +25,6 @@ services:
             POSTGRES_PASSWORD: ${DB_PASSWORD:-daisy}
             POSTGRES_USER: ${DB_USER:-daisy}
             POSTGRES_DB: ${DB_NAME:-daisy}
-            TZ: ${TZ:-UTC}
         networks:
             - daisy_network
         volumes:
@@ -35,8 +33,6 @@ services:
     nginx:
         build: ./docker/nginx
         restart: unless-stopped
-        environment:
-            - TZ=${TZ:-UTC}
         volumes:
             - statics:/public/static:ro
         ports:
@@ -50,8 +46,6 @@ services:
     solr:
         build: ./docker/solr
         restart: unless-stopped
-        environment:
-            - TZ=${TZ:-UTC}
         ports:
             - "8983:8983"
         networks:
@@ -62,8 +56,6 @@ services:
     mq:
         image: rabbitmq:3.9-management-alpine
         restart: unless-stopped
-        environment:
-            - TZ=${TZ:-UTC}
         networks:
             - daisy_network
         ports:
@@ -73,8 +65,6 @@ services:
     flower:
         image: mher/flower:0.9.7
         command: --broker=amqp://guest:guest@mq:5672// --broker_api=http://guest:guest@mq:15672/api/
-        environment:
-            - TZ=${TZ:-UTC}
         ports:
             - "5555:5555"
         restart: unless-stopped
@@ -87,7 +77,6 @@ services:
         build: .
         environment:
             - DJANGO_SETTINGS_MODULE=elixir_daisy.settings_compose
-            - TZ=${TZ:-UTC}
         volumes:
             - .:/code
         depends_on:
@@ -102,7 +91,6 @@ services:
         build: .
         environment:
             - DJANGO_SETTINGS_MODULE=elixir_daisy.settings_compose
-            - TZ=${TZ:-UTC}
         volumes:
             - .:/code
         depends_on:
@@ -125,7 +113,6 @@ services:
             - SOLR_PORT=8983
             - RABBITMQ_MANAGEMENT_PORT=${RABBITMQ_MANAGEMENT_PORT:-15672}
             - BACKUP_VOLUME=${BACKUP_VOLUME:-../backups}
-            - TZ=${TZ:-UTC}
         volumes:
             - ${BACKUP_VOLUME:-../backups}:/backups
             - .:/code

From ed7b33973bbe7ea109459479ffed1481e43ad81c Mon Sep 17 00:00:00 2001
From: Vilem Ded <vilemded@gmail.com>
Date: Tue, 15 Oct 2024 09:43:17 +0200
Subject: [PATCH 35/43] removing work-in-progress doc

---
 doc/backup.md | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/doc/backup.md b/doc/backup.md
index 2bf028a9..27b4e3fc 100644
--- a/doc/backup.md
+++ b/doc/backup.md
@@ -23,22 +23,6 @@ All variables can be set in the [environment file](.env.template). These include
 
 ### Operations
 
-#### Enable Automatic Backups
-
-To ensure automatic backups are enabled, set `ENABLE_BACKUPS=true` (enabled by default):
-
-```bash
-ENABLE_BACKUPS=true docker compose up -d backup
-```
-
-This will configure automatic backups based on the `BACKUP_SCHEDULE`.
-
-To disable automatic backups:
-
-```bash
-ENABLE_BACKUPS=false docker compose up -d backup
-```
-
 #### Manual Backup
 
 Create a manual backup:

From b8aa49541f7328ce13f7596632d7a6881265af4d Mon Sep 17 00:00:00 2001
From: Moustapha Cheikh <moustapha.cheikh@uni.lu>
Date: Wed, 16 Oct 2024 08:25:13 +0200
Subject: [PATCH 36/43] Upgrade PostgreSQL from 10.1 to 16.4

---
 docker-compose.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index fa442086..adcaf2b7 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -19,7 +19,7 @@ services:
         command: gunicorn --reload -w 2 -b :5000 --pid /run/gunicorn.pid elixir_daisy.wsgi
     # database
     db:
-        image: postgres:10.1
+        image: postgres:16.4-bullseye
         restart: unless-stopped
         environment:
             POSTGRES_PASSWORD: ${DB_PASSWORD:-daisy}

From 788d63bb5bb306ce7efb672e51e1c5430501993a Mon Sep 17 00:00:00 2001
From: Moustapha Cheikh <moustapha.cheikh@uni.lu>
Date: Wed, 16 Oct 2024 14:22:26 +0200
Subject: [PATCH 37/43] Update backup.md

---
 doc/backup.md | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/doc/backup.md b/doc/backup.md
index 27b4e3fc..ad55721d 100644
--- a/doc/backup.md
+++ b/doc/backup.md
@@ -98,3 +98,34 @@ docker compose run web python manage.py rebuild_index --noinput
 ```
 
 Replace `../daisy_prod.tar.gz` with the actual path to legacy backup file.
+
+### Updating Django Settings to support Docker Compose
+
+After the restore script. you need to change settings_local.py to work with Docker Compose, follow these key changes:
+
+1. **Database Settings:**
+   - Change `HOST` from `'localhost'` to the service name defined in Docker (`'db'`):
+   ```python
+   'HOST': 'db',
+   ```
+
+2. **Haystack Solr Configuration:**
+   - Change Solr URL and Admin URL to use the Docker service name (`'solr'`):
+   ```python
+   'URL': 'http://solr:8983/solr/daisy',
+   'ADMIN_URL': 'http://solr:8983/solr/admin/cores',
+   ```
+
+3. **Static Files:**
+   - Update `STATIC_ROOT` to a directory accessible by Docker containers:
+   ```python
+   STATIC_ROOT = "/static/"
+   ```
+
+4. **Allowed Hosts:**
+   - Ensure you update the `ALLOWED_HOSTS` to include the IP addresses or domains used by your Docker environment:
+   ```python
+   ALLOWED_HOSTS = ['IP addresses', 'daisy domain here']
+   ```
+
+By making these adjustments,  daisy should now work seamlessly with Docker Compose.

From 71ebe52ad4dbe155df914036e632a4cdcd7483e3 Mon Sep 17 00:00:00 2001
From: Moustapha Cheikh <moustapha.cheikh@uni.lu>
Date: Wed, 16 Oct 2024 14:26:18 +0200
Subject: [PATCH 38/43] Update backup.md

Add migrate and collectstatic to the restore doc
---
 doc/backup.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/backup.md b/doc/backup.md
index ad55721d..f104fdb3 100644
--- a/doc/backup.md
+++ b/doc/backup.md
@@ -95,6 +95,11 @@ docker cp ../daisy_prod.tar.gz $(docker compose ps -q backup):/code/daisy_prod.t
 # Execute the legacy_restore.sh script inside the running container
 docker compose exec backup /bin/sh -c "sh /code/scripts/legacy_restore.sh /code/daisy_prod.tar.gz && rm /code/daisy_prod.tar.gz"
 docker compose run web python manage.py rebuild_index --noinput
+
+# you may also need to run migrate if you have new migration after this backup was created
+docker compose exec web python manage.py migrate
+docker compose exec web python manage.py collectstatic
+
 ```
 
 Replace `../daisy_prod.tar.gz` with the actual path to legacy backup file.

From 8f99ff5639236e658e789eeec8a1c12abc7e450d Mon Sep 17 00:00:00 2001
From: Moustapha Cheikh <moustapha.cheikh@uni.lu>
Date: Wed, 16 Oct 2024 15:30:32 +0200
Subject: [PATCH 39/43] Create Contribution.md

---
 Contribution.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)
 create mode 100644 Contribution.md

diff --git a/Contribution.md b/Contribution.md
new file mode 100644
index 00000000..f8c141cb
--- /dev/null
+++ b/Contribution.md
@@ -0,0 +1,46 @@
+# Contribution
+
+This guide outlines the process for contributing to this repository.
+
+## GitHub Workflow
+
+We follow a standard GitHub workflow for contributions:
+
+1. Fork the repository
+2. Create a new branch
+3. Make your changes
+4. Submit a pull request
+
+## Pull Requests
+
+- All major contributions should be made through pull requests (PRs).
+- Branch names should start with the issue number (e.g., `124-fancy-new-feature`).
+- PRs can be opened as soon as the branch is created. Use draft PRs for work in progress.
+- PR titles should include the issue number (e.g., "#98: Improve error handling").
+- Include relevant implementation notes and any deviations from the original issue description in the PR description.
+- If the changes impact the user interface, include screenshots in the PR description.
+- Resolve any merge conflicts before requesting a review.
+
+## Definition of "Done" (DoD)
+
+Before marking a PR as ready for review, ensure:
+
+- All issue requirements are implemented
+- CI pipeline is passing
+- Test coverage is maintained or improved
+- Unit tests cover the new implementation (except for very minor changes)
+- Documentation (including README) is updated if necessary
+- New methods and classes have docstrings and type hints
+
+## Code Review Process
+
+- At least one approval from another team member is required before merging.
+- Reviewers should check out the branch and test the feature, not just review the code.
+- Use GitHub's review features for inline comments and general feedback.
+- The PR author should address all comments and request re-review when ready.
+- Reviewers should resolve comment threads once satisfied with the changes.
+- After all threads are resolved and the PR is approved, it can be merged.
+
+## After Merging
+
+- Close the related issue once the PR is merged and the implementation has been verified in the test environment.

From 6e46f8bfbb80de46f54c62f06c290c3f287ccff8 Mon Sep 17 00:00:00 2001
From: Moustapha Cheikh <moustapha.cheikh@uni.lu>
Date: Wed, 16 Oct 2024 15:34:14 +0200
Subject: [PATCH 40/43] Rename Contribution.md to CONTRIBUTING.md

---
 Contribution.md => CONTRIBUTING.md | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename Contribution.md => CONTRIBUTING.md (100%)

diff --git a/Contribution.md b/CONTRIBUTING.md
similarity index 100%
rename from Contribution.md
rename to CONTRIBUTING.md

From 0a63475b3a67c3053a34270102e65e77f420f46b Mon Sep 17 00:00:00 2001
From: Moustapha Cheikh <moustapha.cheikh@uni.lu>
Date: Thu, 17 Oct 2024 08:10:20 +0200
Subject: [PATCH 41/43] Update CONTRIBUTING.md

---
 CONTRIBUTING.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index f8c141cb..eb4e6228 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -6,7 +6,7 @@ This guide outlines the process for contributing to this repository.
 
 We follow a standard GitHub workflow for contributions:
 
-1. Fork the repository
+1. Fork the repository [if needed]
 2. Create a new branch
 3. Make your changes
 4. Submit a pull request

From 426c2a74c27e2d2bbe0e4497f79eb66698c952db Mon Sep 17 00:00:00 2001
From: Alexander Kozarev <al1102@yandex.ru>
Date: Mon, 21 Oct 2024 09:15:04 +0200
Subject: [PATCH 42/43] 391 Rems external id (#526)

* rems external id

* black

* black

* update settings

* update notes in task

* update migration

* test config

* test config

* tests

* access request in template

* update migration

---------

Co-authored-by: Aleksandr KOZAREV <aleksandr.kozarev@unilxw1vp42mx.uni.lux>
---
 core/lcsb/rems.py                          | 100 +++++++++++++++------
 core/migrations/0038_auto_20241018_1733.py |  33 +++++++
 core/models/access.py                      |  12 +++
 core/tasks.py                              |  12 ++-
 core/tests/test_lcsb.py                    |  99 +++++++++++++++++++-
 elixir_daisy/settings_ci.py                |   6 ++
 elixir_daisy/settings_local.template.py    |   5 ++
 elixir_daisy/settings_tests.py             |   1 +
 setup.py                                   |   1 +
 test/factories.py                          |   9 ++
 web/templates/datasets/dataset.html        |  10 ++-
 web/views/datasets.py                      |   1 +
 12 files changed, 257 insertions(+), 32 deletions(-)
 create mode 100644 core/migrations/0038_auto_20241018_1733.py

diff --git a/core/lcsb/rems.py b/core/lcsb/rems.py
index d7f21cc3..07d41562 100644
--- a/core/lcsb/rems.py
+++ b/core/lcsb/rems.py
@@ -1,12 +1,13 @@
 import json
 import urllib3
-
 from datetime import datetime, date, timedelta
 from dateutil.parser import isoparse
 from typing import Dict, Union, Tuple
 
 from django.conf import settings
 from django.http import HttpRequest
+from django.db.models import Q
+import requests
 
 from core.synchronizers import (
     DummyAccountSynchronizer,
@@ -24,7 +25,6 @@
 from core.models.dataset import Dataset
 from core.models.user import User
 from core.utils import DaisyLogger
-from django.db.models import Q
 
 logger = DaisyLogger(__name__)
 
@@ -107,7 +107,7 @@ def handle_rems_callback(request: HttpRequest) -> bool:
     return all(statuses)
 
 
-def extract_rems_data(data: Dict[str, str]) -> Tuple[str, str, str, str, date]:
+def extract_rems_data(data: Dict[str, str]) -> Tuple[int, str, str, str, date]:
     """
     Extracts the data from the REMS request
     """
@@ -172,7 +172,7 @@ def handle_rems_entitlement(data: Dict[str, str]) -> bool:
 
 
 def create_rems_entitlement(
-    obj: Union[Access, User], application: str, dataset_id: str, expiration_date: date
+    obj: Union[Access, User], application: int, dataset_id: str, expiration_date: date
 ) -> bool:
     """
     Tries to find a dataset with `elu_accession` equal to `dataset_id`.
@@ -180,31 +180,26 @@ def create_rems_entitlement(
     Assumes that the Dataset exists, otherwise will throw an exception.
     """
     dataset = Dataset.objects.get(elu_accession=dataset_id)
-    notes = build_access_notes_rems(application)
+    external_id = get_rems_external_id(application)
+    notes = build_access_notes_rems(application, external_id)
     system_rems_user = get_or_create_rems_user()
 
+    access_kwargs = {
+        "dataset": dataset,
+        "access_notes": notes,
+        "granted_on": datetime.now(),
+        "was_generated_automatically": True,
+        "created_by": system_rems_user,
+        "status": StatusChoices.active,
+        "grant_expires_on": expiration_date,
+        "application_id": application,
+        "application_external_id": external_id,
+    }
+
     if isinstance(obj, User):
-        new_logbook_entry = Access(
-            user=obj,
-            dataset=dataset,
-            access_notes=notes,
-            granted_on=datetime.now(),
-            was_generated_automatically=True,
-            created_by=system_rems_user,
-            status=StatusChoices.active,
-            grant_expires_on=expiration_date,
-        )
+        new_logbook_entry = Access(user=obj, **access_kwargs)
     elif isinstance(obj, Contact):
-        new_logbook_entry = Access(
-            contact=obj,
-            dataset=dataset,
-            access_notes=notes,
-            granted_on=datetime.now(),
-            was_generated_automatically=True,
-            created_by=system_rems_user,
-            status=StatusChoices.active,
-            grant_expires_on=expiration_date,
-        )
+        new_logbook_entry = Access(contact=obj, **access_kwargs)
     else:
         klass = obj.__class__.__name__
         raise TypeError(
@@ -215,11 +210,14 @@ def create_rems_entitlement(
     return True
 
 
-def build_access_notes_rems(application: str) -> str:
+def build_access_notes_rems(application: int, external_id: str) -> str:
     """
     Build and return note to be attached to the access created from a REMS application
     """
-    return f"Set automatically by REMS data access request #{application}"
+    result = f"Set automatically by REMS data access request #{application}"
+    if external_id:
+        result += f" ({external_id})"
+    return result
 
 
 def get_or_create_rems_user() -> User:
@@ -240,3 +238,51 @@ def get_or_create_rems_user() -> User:
     else:
         system_rems_user = system_rems_user.first()
     return system_rems_user
+
+
+def get_rems_application(application_id: int) -> dict:
+    headers = {
+        "x-rems-api-key": getattr(settings, "REMS_API_KEY"),
+        "x-rems-user-id": getattr(settings, "REMS_API_USER"),
+    }
+
+    request_url = getattr(settings, "REMS_URL") + "api/applications"
+    if application_id is not None:
+        request_url = request_url + "/" + str(application_id)
+
+    response = requests.get(
+        request_url, headers=headers, verify=getattr(settings, "REMS_VERIFY_SSL")
+    )
+    response.raise_for_status()
+    return json.loads(response.text)
+
+
+def get_rems_external_id(application_id: int) -> str:
+    attempt = 0
+    max_retries = getattr(settings, "REMS_RETRIES")
+
+    while attempt < max_retries:
+        try:
+            application_data = get_rems_application(application_id)
+            return application_data.get("application/external-id")
+        except Exception as e:
+            attempt += 1
+            logger.error(
+                f"REMS :: Exception on requesting external ID for application {application_id}",
+                exc_info=e,
+            )
+    return None
+
+
+def bulk_update_rems_external_ids():
+    accesses = Access.objects.filter(
+        Q(application_external_id__isnull=True) & Q(application_id__isnull=False)
+    )
+    for access in accesses:
+        external_id = get_rems_external_id(access.application_id)
+        access.application_external_id = external_id
+        access.access_notes = build_access_notes_rems(
+            access.application_id, external_id
+        )
+    Access.objects.bulk_update(accesses, ["application_external_id", "access_notes"])
+    logger.info(f"REMS :: Accesses updated: {len(accesses)}")
diff --git a/core/migrations/0038_auto_20241018_1733.py b/core/migrations/0038_auto_20241018_1733.py
new file mode 100644
index 00000000..9f7714a0
--- /dev/null
+++ b/core/migrations/0038_auto_20241018_1733.py
@@ -0,0 +1,33 @@
+# Generated by Django 3.2.23 on 2024-10-15 13:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0037_auto_20240924_1020"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="access",
+            name="application_external_id",
+            field=models.CharField(
+                blank=True,
+                help_text="REMS application external ID.",
+                max_length=200,
+                null=True,
+                verbose_name="Other access request ID",
+            ),
+        ),
+        migrations.AddField(
+            model_name="access",
+            name="application_id",
+            field=models.IntegerField(
+                blank=True,
+                help_text="REMS application ID.",
+                null=True,
+                verbose_name="Access request ID",
+            ),
+        ),
+    ]
diff --git a/core/models/access.py b/core/models/access.py
index 6ca25c36..80380edb 100644
--- a/core/models/access.py
+++ b/core/models/access.py
@@ -185,6 +185,18 @@ def clean(self):
         help_text="Was the entry generated automatically, e.g. by REMS?",
     )
 
+    application_id = models.IntegerField(
+        "Access request ID", blank=True, null=True, help_text="REMS application ID."
+    )
+
+    application_external_id = models.CharField(
+        "Other access request ID",
+        blank=True,
+        null=True,
+        max_length=200,
+        help_text="REMS application external ID.",
+    )
+
     def __str__(self):
         try:
             if self.contact:
diff --git a/core/tasks.py b/core/tasks.py
index feff973a..4bbda9df 100644
--- a/core/tasks.py
+++ b/core/tasks.py
@@ -1,7 +1,9 @@
 from datetime import date
+
 from celery import shared_task
+
 from core.models.access import Access
-from core.lcsb.rems import synchronizer
+from core.lcsb.rems import synchronizer, bulk_update_rems_external_ids
 
 
 @shared_task
@@ -19,3 +21,11 @@ def run_synchronizer():
     Task to synchronize users and contacts with the external system
     """
     synchronizer.synchronize_all()
+
+
+@shared_task
+def update_rems_access_external_id():
+    """
+    Task to update external id for REMS accesses
+    """
+    bulk_update_rems_external_ids()
diff --git a/core/tests/test_lcsb.py b/core/tests/test_lcsb.py
index 01b365b8..0e61a01c 100644
--- a/core/tests/test_lcsb.py
+++ b/core/tests/test_lcsb.py
@@ -1,7 +1,9 @@
 import datetime
 from typing import Dict, List
 
+from django.conf import settings
 import pytest
+import requests_mock
 
 from core.lcsb.oidc import KeycloakSynchronizationBackend
 from core.lcsb.rems import (
@@ -9,6 +11,8 @@
     extract_rems_data,
     build_default_expiration_date,
     check_existence_automatic,
+    get_rems_external_id,
+    bulk_update_rems_external_ids,
 )
 from core.models.access import Access
 from core.models.access import StatusChoices
@@ -162,8 +166,8 @@ def test_check_existence_automatic_positive():
         title="Test", local_custodians=[user], elu_accession=resource_id
     )
     dataset.save()
-    create_rems_entitlement(user, "Test Application", resource_id, expiration_date)
     application_id = 4056
+    create_rems_entitlement(user, application_id, resource_id, expiration_date)
     email = "john.doe@uni.lu"
     data = {
         "application": application_id,
@@ -174,6 +178,10 @@ def test_check_existence_automatic_positive():
     }
     assert check_existence_automatic(data)
 
+    access = Access.objects.get(application_id=application_id)
+    assert access
+    assert not access.application_external_id
+
 
 def test_check_existence_automatic_negative_mismatch():
     resource_id = "TEST-2-5591E3-1"
@@ -184,7 +192,7 @@ def test_check_existence_automatic_negative_mismatch():
         title="Test", local_custodians=[user], elu_accession=resource_id
     )
     dataset.save()
-    create_rems_entitlement(user, "Test Application", resource_id, expiration_date)
+    create_rems_entitlement(user, 1, resource_id, expiration_date)
     application_id = 4056
     email = "john.doe@uni.lu"
     # resource id is different
@@ -254,7 +262,7 @@ def test_add_rems_entitlements():
     )
     dataset.save()
 
-    create_rems_entitlement(user, "Test Application", elu_accession, expiration_date)
+    create_rems_entitlement(user, 1, elu_accession, expiration_date)
     user.delete()
 
 
@@ -312,3 +320,88 @@ def test_get_user_or_contact_by_oidc_id():
     user_found, contact_found, user, contact = get_user_or_contact_by_oidc_id("98765")
     assert not user_found
     assert contact_found
+
+
+def test_get_rems_external_id(requests_mock):
+    application_id = 1
+    external_id = "2024/1"
+    request_url = (
+        getattr(settings, "REMS_URL") + "api/applications" + "/" + str(application_id)
+    )
+    requests_mock.get(request_url, json={"application/external-id": external_id})
+
+    rems_external_id = get_rems_external_id(application_id)
+
+    assert rems_external_id == external_id
+
+
+def test_get_rems_external_id_(requests_mock):
+    application_id = 1
+    request_url = (
+        getattr(settings, "REMS_URL") + "api/applications" + "/" + str(application_id)
+    )
+    requests_mock.get(request_url, json={"error": "not found"}, status_code=404)
+
+    rems_external_id = get_rems_external_id(application_id)
+
+    assert not rems_external_id
+
+
+def test_create_access_with_external_id():
+    resource_id = "TEST-2-5591E3-1"
+    expiration_date = datetime.date.today() + datetime.timedelta(days=1)
+    application_id = 4056
+    external_id = "2024/1"
+    request_url = (
+        getattr(settings, "REMS_URL") + "api/applications" + "/" + str(application_id)
+    )
+
+    user = UserFactory(oidc_id="12345", email="example@example.org")
+    user.save()
+    dataset = DatasetFactory(
+        title="Test", local_custodians=[user], elu_accession=resource_id
+    )
+    dataset.save()
+
+    with requests_mock.Mocker() as m:
+        m.get(request_url, json={"application/external-id": external_id})
+        create_rems_entitlement(user, application_id, resource_id, expiration_date)
+
+    access = Access.objects.get(application_id=application_id)
+    assert access
+    assert access.application_external_id == "2024/1"
+
+
+def test_update_accesses_external_id():
+    access_0 = AccessFactory.create()
+    access_1 = AccessFactory.create(rems_id=True)
+    access_2 = AccessFactory.create(rems_id=True, rems_external_id=True)
+
+    rems_url = getattr(settings, "REMS_URL")
+    with requests_mock.Mocker() as m:
+        m.get(
+            rems_url + f"api/applications/{access_0.application_id}",
+            json={"application/external-id": "new_external_id-0"},
+        )
+        m.get(
+            rems_url + f"api/applications/{access_1.application_id}",
+            json={"application/external-id": "new_external_id-1"},
+        )
+        m.get(
+            rems_url + f"api/applications/{access_2.application_id}",
+            json={"application/external-id": "new_external_id-2"},
+        )
+        bulk_update_rems_external_ids()
+
+    updated_access_0 = Access.objects.get(id=access_0.id)
+    assert not updated_access_0.application_external_id
+
+    updated_access_1 = Access.objects.get(id=access_1.id)
+    assert updated_access_1.application_external_id == "new_external_id-1"
+    assert (
+        updated_access_1.access_notes
+        == f"Set automatically by REMS data access request #{access_1.id} (new_external_id-1)"
+    )
+
+    updated_access_2 = Access.objects.get(id=access_2.id)
+    assert updated_access_2.application_external_id == access_2.application_external_id
diff --git a/elixir_daisy/settings_ci.py b/elixir_daisy/settings_ci.py
index 9d3a4743..13475607 100644
--- a/elixir_daisy/settings_ci.py
+++ b/elixir_daisy/settings_ci.py
@@ -73,3 +73,9 @@
 
 REMS_INTEGRATION_ENABLED = True
 REMS_ALLOWED_IP_ADDRESSES = ["127.0.0.1"]
+REMS_URL = "http://localhost:3000/"
+REMS_RETRIES = 1
+REMS_API_KEY = "test"
+REMS_API_USER = "test"
+REMS_VERIFY_SSL = True
+REMS_SKIP_IP_CHECK = False
diff --git a/elixir_daisy/settings_local.template.py b/elixir_daisy/settings_local.template.py
index 9053c358..403003fa 100644
--- a/elixir_daisy/settings_local.template.py
+++ b/elixir_daisy/settings_local.template.py
@@ -70,6 +70,7 @@
 REMS_URL = ""
 REMS_API_USER = ""
 REMS_VERIFY_SSL = True
+REMS_RETRIES = 3
 
 # IDSERVICE_FUNCTION = 'core.lcsb.idservice.generate_identifier'
 IDSERVICE_ENDPOINT = "https://10.240.16.199:8080/v1/api/id"
@@ -105,4 +106,8 @@
         "task": "core.tasks.run_synchronizer",
         "schedule": crontab(minute=0, hour=2),  # Execute task at 2am
     },
+    "update-rems-application-external-id": {
+        "task": "core.tasks.update_rems_access_external_id",
+        "schedule": crontab(minute=0, hour=3),  # Execute task at 3am
+    },
 }
diff --git a/elixir_daisy/settings_tests.py b/elixir_daisy/settings_tests.py
index 0d6fec26..ba0b68dc 100644
--- a/elixir_daisy/settings_tests.py
+++ b/elixir_daisy/settings_tests.py
@@ -66,3 +66,4 @@
 
 REMS_INTEGRATION_ENABLED = True
 REMS_ALLOWED_IP_ADDRESSES = ["*"]
+REMS_URL = "http://localhost:3000/"
diff --git a/setup.py b/setup.py
index 01d90695..bbb6dba2 100644
--- a/setup.py
+++ b/setup.py
@@ -54,6 +54,7 @@
     "pytest-solr==1.0a1",
     "pytest-celery",
     "pytest-mock==3.12.0",
+    "requests-mock==1.12.1",
 ]
 
 dev_requirements = ["pre-commit==3.3.3"]
diff --git a/test/factories.py b/test/factories.py
index c602faa5..20a6b1ac 100644
--- a/test/factories.py
+++ b/test/factories.py
@@ -196,10 +196,19 @@ class Meta:
         model = "core.Access"
         django_get_or_create = ("user",)
 
+    id = factory.Sequence(lambda n: n)
     access_notes = factory.Faker("sentence")
     dataset = factory.SubFactory(DatasetFactory)
     user = factory.SubFactory(UserFactory)
 
+    class Params:
+        rems_id = factory.Trait(
+            application_id=factory.Sequence(lambda n: n),
+        )
+        rems_external_id = factory.Trait(
+            application_external_id=factory.Faker("sentence"),
+        )
+
 
 class StorageResourceFactory(factory.django.DjangoModelFactory):
     """
diff --git a/web/templates/datasets/dataset.html b/web/templates/datasets/dataset.html
index a2cf2435..aa24d68b 100644
--- a/web/templates/datasets/dataset.html
+++ b/web/templates/datasets/dataset.html
@@ -346,6 +346,7 @@ <h2 class="card-title">
                         <th>Remarks</th>
                         <th>Granted on</th>
                         <th>Expires on</th>
+                        <th>Access requests</th>
                         {% if can_edit %}
                             <th>To whom</th>
                             <th style="width:7em">Actions</th>
@@ -360,9 +361,16 @@ <h2 class="card-title">
                         <tr>
                             <td>{{ access.status }}</td>
                             <td><p>{{ access.display_locations |truncatechars:90 }}</p></td>
-                            <td>{{ access.access_notes |truncatechars:45  }}<br /><small>(Created {% if access.was_generated_automatically %}automatically{% endif %} by user: {{access.created_by}})</small></td>
+                            <td>{{ access.access_notes |truncatechars:75  }}<br /><small>(Created {% if access.was_generated_automatically %}automatically{% endif %} by user: {{access.created_by}})</small></td>
                             <td><p>{{ access.granted_on | default:"-"  }}</p></td>
                             <td><p>{{ access.grant_expires_on | default:"-"  }}</p></td>
+                            <td><p>
+                                {% if access.application_id %}
+                                    <a href="{{ rems_application_url }}{{ access.application_id }}">{{ access.application_external_id}}</a>
+                                {% else %}
+                                    -
+                                {% endif %}
+                            </p></td>
                             {% if can_edit %}
                                 <td>
                                     {% if access.user %}
diff --git a/web/views/datasets.py b/web/views/datasets.py
index 833760e0..65e44326 100644
--- a/web/views/datasets.py
+++ b/web/views/datasets.py
@@ -197,6 +197,7 @@ def get_context_data(self, **kwargs):
         )
         context["company_name"] = COMPANY
         context["exposure_list"] = Exposure.objects.filter(dataset=self.object)
+        context["rems_application_url"] = getattr(settings, "REMS_URL") + "application/"
         return context
 
 

From ef4aebe70caf79d30b9b3ca1838c42f56a44bec7 Mon Sep 17 00:00:00 2001
From: Vilem Ded <vilemded@gmail.com>
Date: Mon, 11 Nov 2024 11:04:02 +0100
Subject: [PATCH 43/43] matching of the version is now done properly (including
 v prefix)

---
 .github/workflows/release.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 44b824c1..22875693 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -21,7 +21,7 @@ jobs:
       run: |
         VERSION=$(python setup.py --version)
         echo "Version in setup.py: $VERSION"
-        if [[ "$VERSION" != "$GITHUB_REF_NAME" ]]; then
-          echo "Version mismatch: setup.py ($VERSION) vs release tag ($GITHUB_REF_NAME)"
+        if [[ "v$VERSION" != "$GITHUB_REF_NAME" ]]; then
+          echo "Version mismatch: setup.py (v$VERSION) vs release tag ($GITHUB_REF_NAME)"
           exit 1
         fi