feat: make nutripatrol production-ready (#29)

- fix container-deploy.yml for staging deployment
- add environment info to sentry (dev, staging, prod)
- add peewee_migrate to apply create/apply DB migrations.
- remove remaining search-a-licious specific config
- expose API on port 9010

.env

@@ -5,7 +5,7 @@ COMPOSE_PATH_SEPARATOR=;
 # dev is default target
 COMPOSE_FILE=docker-compose.yml;docker/dev.yml
 
-API_PORT=127.0.0.1:8000
+API_EXPOSE=127.0.0.1:8000
 
 # by default on dev desktop, no restart
 RESTART_POLICY=no
@@ -25,4 +25,7 @@ POSTGRES_PASSWORD=postgres
 
 # The top-level domain used for Open Food Facts,
 # it's either `net` (staging) or `org` (production) 
-OFF_TLD=net
+OFF_TLD=net
+
+# Environment name (mostly used for Sentry): dev, staging, prod
+ENVIRONMENT=dev

.github/workflows/container-deploy.yml

@@ -29,15 +29,19 @@ jobs:
     environment: ${{ matrix.env }}
     concurrency: ${{ matrix.env }}
     steps:
+    - name: Set common variables
+      run: |
+        echo "SSH_PROXY_HOST=ovh1.openfoodfacts.org" >> $GITHUB_ENV
+        echo "SSH_USERNAME=off" >> $GITHUB_ENV
     - name: Set various variable for staging (net) deployment
       if: matrix.env == 'nutripatrol-net'
       run: |
-        # direct container access
-        echo "OPENFOODFACTS_API_URL=https://off:[email protected]" >> $GITHUB_ENV
-        # deploy target
         echo "SSH_HOST=10.1.0.200" >> $GITHUB_ENV
-        echo "SSH_PROXY_HOST=ovh1.openfoodfacts.org" >> $GITHUB_ENV
-        echo "SSH_USERNAME=off" >> $GITHUB_ENV
+        echo "ENVIRONMENT=staging" >> $GITHUB_ENV
+        if: matrix.env == 'nutripatrol-org'
+        run: |
+          echo "SSH_HOST=10.1.0.201" >> $GITHUB_ENV
+          echo "ENVIRONMENT=prod" >> $GITHUB_ENV
     - name: Wait for docker image container build workflow
       uses: tomchv/[email protected]
       id: wait-build
@@ -99,32 +103,20 @@ jobs:
 
           mv .env .env-dev
           # init .env
-          echo "# Env file generated by container-deploy action"> .env
+          echo "# Env file generated by container-deploy action" > .env
           # Set Docker Compose variables
           echo "DOCKER_CLIENT_TIMEOUT=180" >> .env
           echo "COMPOSE_HTTP_TIMEOUT=180" >> .env
           echo "COMPOSE_PROJECT_NAME=nutripatrol" >> .env
           echo "COMPOSE_PATH_SEPARATOR=;" >> .env
           echo "COMPOSE_FILE=docker-compose.yml;docker/prod.yml" >> .env
-          # Copy variables that are same as dev
-          grep '\(STACK_VERSION\|ES_PORT\)' .env-dev >> .env
           # Set docker variables
           echo "TAG=sha-${{ github.sha }}" >> .env
           echo "RESTART_POLICY=always" >> .env
-          # Set App variables
-          echo "CLUSTER_NAME=${{ matrix.env }}-es-cluster" >> .env
-          echo "SEARCH_PORT=8180" >> .env
-          echo "ES_VUE_PORT=8181" >> .env
-          echo "REDIS_PORT=8182" >> .env
-          echo "MEM_LIMIT=4294967296" >> .env
-          # this is the network shared with productopener
-          echo "COMMON_NET_NAME=po_webnet">> .env
-          echo "OPENFOODFACTS_API_URL=${{ env.OPENFOODFACTS_API_URL }}" >> .env
-          # This secret is to be generated using htpasswd, see .env file
-          # use simple quotes to avoid interpolation of $apr1$ !
-          echo 'NGINX_BASIC_AUTH_USER_PASSWD=${{ secrets.NGINX_BASIC_AUTH_USER_PASSWD }}' >> .env
           echo "SENTRY_DNS=${{ secrets.SENTRY_DSN }}" >> .env
-          echo "CONFIG_PATH=data/config/openfoodfacts.yml" >> .env
+          echo "ENVIRONMENT=${{ env.ENVIRONMENT }}" >> .env
+          # Expose API on port 9010
+          echo "API_EXPOSE=0.0.0.0:9010" >> .env
 
     - name: Create Docker volumes
       uses: appleboy/ssh-action@master
@@ -152,8 +144,11 @@ jobs:
         script_stop: false
         script: |
           cd ${{ matrix.env }}
-          docker-compose down
-          docker-compose up -d --remove-orphans 2>&1
+          make pull
+          # Apply migrations
+          make migrate-db
+          # Launch new version
+          make up
 
     - name: Check services are up
       uses: appleboy/ssh-action@master
@@ -183,7 +178,7 @@ jobs:
         script_stop: false
         script: |
           cd ${{ matrix.env }}
-          docker system prune -af
+          make prune
 
     - uses: frankie567/[email protected]
       if: ${{ always() }}

Dockerfile

@@ -15,6 +15,7 @@ RUN groupadd -g $USER_GID off && \
     chown off:off -R /opt/nutripatrol /home/off
 WORKDIR /opt/nutripatrol
 COPY --chown=off:off requirements.txt requirements.txt
+COPY --chown=off:off migrations /opt/nutripatrol/migrations
 RUN  pip install --no-cache-dir --upgrade -r requirements.txt
 
 USER off:off

Makefile

@@ -41,6 +41,10 @@ build:
 	docker-compose build
 
 
+# pull images from image repository
+pull:
+	${DOCKER_COMPOSE} pull
+
 up:
 ifdef service
 	${DOCKER_COMPOSE} up -d ${service} 2>&1
@@ -52,3 +56,40 @@ endif
 down:
 	@echo "🥫 Bringing down containers …"
 	${DOCKER_COMPOSE} down
+
+
+#-----------#
+# Utilities #
+#-----------#
+
+guard-%: # guard clause for targets that require an environment variable (usually used as an argument)
+	@ if [ "${${*}}" = "" ]; then \
+   		echo "Environment variable '$*' is mandatory"; \
+   		echo use "make ${MAKECMDGOALS} $*=you-args"; \
+   		exit 1; \
+	fi;
+
+
+#------------#
+#  Database  #
+#------------#
+
+# apply DB migrations
+migrate-db:
+	${DOCKER_COMPOSE} run --rm --no-deps api python -m app migrate-db
+
+# add a new DB revision
+add-revision: guard-name
+	${DOCKER_COMPOSE} run --rm --no-deps api python -m app add-revision ${name}
+
+
+#---------#
+# Cleanup #
+#---------#
+prune:
+	@echo "🥫 Pruning unused Docker artifacts (save space) …"
+	docker system prune -af
+
+prune_cache:
+	@echo "🥫 Pruning Docker builder cache …"
+	docker builder prune -f

app/__main__.py

@@ -0,0 +1,4 @@
+from app.cli import main
+
+if __name__ == "__main__":
+    main()

app/cli.py

@@ -0,0 +1,35 @@
+import typer
+
+app = typer.Typer()
+
+
+@app.command()
+def migrate_db():
+    """Run unapplied DB migrations."""
+    from openfoodfacts.utils import get_logger
+
+    from app.models import db, run_migration
+
+    get_logger()
+
+    with db.connection_context():
+        run_migration()
+
+
+@app.command()
+def add_revision(
+    name: str = typer.Argument(..., help="name of the revision"),
+):
+    """Create a new migration file using peewee_migrate."""
+    from openfoodfacts.utils import get_logger
+
+    from app.models import add_revision, db
+
+    get_logger()
+
+    with db.connection_context():
+        add_revision(name)
+
+
+def main() -> None:
+    app()

app/config.py

@@ -1,8 +1,11 @@
 from enum import StrEnum
+from pathlib import Path
 
 from openfoodfacts import Environment
 from pydantic_settings import BaseSettings
 
+PROJECT_DIR = Path(__file__).parent.parent
+
 
 class LoggingLevel(StrEnum):
     NOTSET = "NOTSET"
@@ -36,6 +39,8 @@ class Settings(BaseSettings):
     postgres_password: str = "postgres"
     postgres_port: int = 5432
     off_tld: Environment = Environment.net
+    environment: str = "dev"
+    migration_dir: Path = PROJECT_DIR / "migrations"
 
 
 settings = Settings()

app/models.py

@@ -5,7 +5,9 @@
     ForeignKeyField,
     Model,
     PostgresqlDatabase,
+    TextField,
 )
+from peewee_migrate import Router
 
 from .config import settings
 
@@ -19,12 +21,13 @@
 
 
 class TicketModel(Model):
-    barcode = CharField(null=True)
-    type = CharField()
-    url = CharField()
-    status = CharField()
+    # barcode of the product, if any
+    barcode = TextField(null=True)
+    type = CharField(max_length=50)
+    url = TextField()
+    status = CharField(max_length=50)
     image_id = CharField(null=True)
-    flavor = CharField()
+    flavor = CharField(max_length=20)
     created_at = DateTimeField()
 
     class Meta:
@@ -33,8 +36,8 @@ class Meta:
 
 
 class ModeratorActionModel(Model):
-    action_type = CharField()
-    user_id = CharField()
+    action_type = CharField(max_length=20)
+    user_id = TextField()
     ticket = ForeignKeyField(TicketModel, backref="moderator_actions")
     created_at = DateTimeField()
 
@@ -45,19 +48,33 @@ class Meta:
 
 class FlagModel(Model):
     ticket = ForeignKeyField(TicketModel, backref="flags")
-    barcode = CharField(null=True)
-    type = CharField()
-    url = CharField()
-    user_id = CharField()
-    device_id = CharField()
+    barcode = TextField(null=True)
+    type = CharField(max_length=50)
+    url = TextField()
+    user_id = TextField()
+    device_id = TextField()
     source = CharField()
     confidence = FloatField(null=True)
     image_id = CharField(null=True)
-    flavor = CharField()
-    reason = CharField(null=True)
-    comment = CharField(max_length=500, null=True)
+    flavor = CharField(max_length=20)
+    reason = TextField(null=True)
+    comment = TextField(null=True)
     created_at = DateTimeField()
 
     class Meta:
         database = db
         table_name = "flags"
+
+
+def run_migration():
+    """Run all unapplied migrations."""
+    # embedding schema does not exist at DB initialization
+    router = Router(db, migrate_dir=settings.migration_dir)
+    # Run all unapplied migrations
+    router.run()
+
+
+def add_revision(name: str):
+    """Create a migration revision."""
+    router = Router(db, migrate_dir=settings.migration_dir)
+    router.create(name, auto=True)

app/utils.py

@@ -4,6 +4,8 @@
 from sentry_sdk.integrations import Integration
 from sentry_sdk.integrations.logging import LoggingIntegration
 
+from app.config import settings
+
 
 def init_sentry(sentry_dsn: str | None, integrations: list[Integration] | None = None):
     if sentry_dsn:
@@ -17,4 +19,5 @@ def init_sentry(sentry_dsn: str | None, integrations: list[Integration] | None =
         sentry_sdk.init(  # type:ignore  # mypy say it's abstract
             sentry_dsn,
             integrations=integrations,
+            environment=settings.environment,
         )

docker-compose.yml

@@ -10,15 +10,14 @@ x-api-common: &api-common
     - POSTGRES_PASSWORD
     - POSTGRES_DB
     - POSTGRES_HOST
-    - OFF_TLD
   networks:
     - default
 
 services:
   api:
     <<: *api-common
     ports:
-      - "${API_PORT}:8000"
+      - "${API_EXPOSE:-127.0.0.1:8000}:8000"
 
   postgres:
     restart: $RESTART_POLICY

docker/dev.yml

@@ -10,6 +10,8 @@ x-api-base: &api-base
   volumes:
     # mount code dynamically
     - "./app:/opt/nutripatrol/app"
+    # mount migrations dynamically
+    - "./migrations:/opt/nutripatrol/migrations"
 
 services:
   api: