diff --git a/.env b/.env index 6a9a3cc..db3cd67 100644 --- a/.env +++ b/.env @@ -5,7 +5,7 @@ COMPOSE_PATH_SEPARATOR=; # dev is default target COMPOSE_FILE=docker-compose.yml;docker/dev.yml -API_PORT=127.0.0.1:8000 +API_EXPOSE=127.0.0.1:8000 # by default on dev desktop, no restart RESTART_POLICY=no @@ -25,4 +25,7 @@ POSTGRES_PASSWORD=postgres # The top-level domain used for Open Food Facts, # it's either `net` (staging) or `org` (production) -OFF_TLD=net \ No newline at end of file +OFF_TLD=net + +# Environment name (mostly used for Sentry): dev, staging, prod +ENVIRONMENT=dev \ No newline at end of file diff --git a/.github/workflows/container-deploy.yml b/.github/workflows/container-deploy.yml index c335525..260d8a9 100644 --- a/.github/workflows/container-deploy.yml +++ b/.github/workflows/container-deploy.yml @@ -29,15 +29,19 @@ jobs: environment: ${{ matrix.env }} concurrency: ${{ matrix.env }} steps: + - name: Set common variables + run: | + echo "SSH_PROXY_HOST=ovh1.openfoodfacts.org" >> $GITHUB_ENV + echo "SSH_USERNAME=off" >> $GITHUB_ENV - name: Set various variable for staging (net) deployment if: matrix.env == 'nutripatrol-net' run: | - # direct container access - echo "OPENFOODFACTS_API_URL=https://off:off@world.openfoodfacts.net" >> $GITHUB_ENV - # deploy target echo "SSH_HOST=10.1.0.200" >> $GITHUB_ENV - echo "SSH_PROXY_HOST=ovh1.openfoodfacts.org" >> $GITHUB_ENV - echo "SSH_USERNAME=off" >> $GITHUB_ENV + echo "ENVIRONMENT=staging" >> $GITHUB_ENV + if: matrix.env == 'nutripatrol-org' + run: | + echo "SSH_HOST=10.1.0.201" >> $GITHUB_ENV + echo "ENVIRONMENT=prod" >> $GITHUB_ENV - name: Wait for docker image container build workflow uses: tomchv/wait-my-workflow@v1.1.0 id: wait-build @@ -99,32 +103,20 @@ jobs: mv .env .env-dev # init .env - echo "# Env file generated by container-deploy action"> .env + echo "# Env file generated by container-deploy action" > .env # Set Docker Compose variables echo "DOCKER_CLIENT_TIMEOUT=180" >> .env echo "COMPOSE_HTTP_TIMEOUT=180" >> .env echo "COMPOSE_PROJECT_NAME=nutripatrol" >> .env echo "COMPOSE_PATH_SEPARATOR=;" >> .env echo "COMPOSE_FILE=docker-compose.yml;docker/prod.yml" >> .env - # Copy variables that are same as dev - grep '\(STACK_VERSION\|ES_PORT\)' .env-dev >> .env # Set docker variables echo "TAG=sha-${{ github.sha }}" >> .env echo "RESTART_POLICY=always" >> .env - # Set App variables - echo "CLUSTER_NAME=${{ matrix.env }}-es-cluster" >> .env - echo "SEARCH_PORT=8180" >> .env - echo "ES_VUE_PORT=8181" >> .env - echo "REDIS_PORT=8182" >> .env - echo "MEM_LIMIT=4294967296" >> .env - # this is the network shared with productopener - echo "COMMON_NET_NAME=po_webnet">> .env - echo "OPENFOODFACTS_API_URL=${{ env.OPENFOODFACTS_API_URL }}" >> .env - # This secret is to be generated using htpasswd, see .env file - # use simple quotes to avoid interpolation of $apr1$ ! - echo 'NGINX_BASIC_AUTH_USER_PASSWD=${{ secrets.NGINX_BASIC_AUTH_USER_PASSWD }}' >> .env echo "SENTRY_DNS=${{ secrets.SENTRY_DSN }}" >> .env - echo "CONFIG_PATH=data/config/openfoodfacts.yml" >> .env + echo "ENVIRONMENT=${{ env.ENVIRONMENT }}" >> .env + # Expose API on port 9010 + echo "API_EXPOSE=0.0.0.0:9010" >> .env - name: Create Docker volumes uses: appleboy/ssh-action@master @@ -152,8 +144,11 @@ jobs: script_stop: false script: | cd ${{ matrix.env }} - docker-compose down - docker-compose up -d --remove-orphans 2>&1 + make pull + # Apply migrations + make migrate-db + # Launch new version + make up - name: Check services are up uses: appleboy/ssh-action@master @@ -183,7 +178,7 @@ jobs: script_stop: false script: | cd ${{ matrix.env }} - docker system prune -af + make prune - uses: frankie567/grafana-annotation-action@v1.0.3 if: ${{ always() }} diff --git a/Dockerfile b/Dockerfile index 0fe6597..97dba93 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,7 @@ RUN groupadd -g $USER_GID off && \ chown off:off -R /opt/nutripatrol /home/off WORKDIR /opt/nutripatrol COPY --chown=off:off requirements.txt requirements.txt +COPY --chown=off:off migrations /opt/nutripatrol/migrations RUN pip install --no-cache-dir --upgrade -r requirements.txt USER off:off diff --git a/Makefile b/Makefile index bfc8c8c..b70a9d5 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,10 @@ build: docker-compose build +# pull images from image repository +pull: + ${DOCKER_COMPOSE} pull + up: ifdef service ${DOCKER_COMPOSE} up -d ${service} 2>&1 @@ -52,3 +56,40 @@ endif down: @echo "🥫 Bringing down containers …" ${DOCKER_COMPOSE} down + + +#-----------# +# Utilities # +#-----------# + +guard-%: # guard clause for targets that require an environment variable (usually used as an argument) + @ if [ "${${*}}" = "" ]; then \ + echo "Environment variable '$*' is mandatory"; \ + echo use "make ${MAKECMDGOALS} $*=you-args"; \ + exit 1; \ + fi; + + +#------------# +# Database # +#------------# + +# apply DB migrations +migrate-db: + ${DOCKER_COMPOSE} run --rm --no-deps api python -m app migrate-db + +# add a new DB revision +add-revision: guard-name + ${DOCKER_COMPOSE} run --rm --no-deps api python -m app add-revision ${name} + + +#---------# +# Cleanup # +#---------# +prune: + @echo "🥫 Pruning unused Docker artifacts (save space) …" + docker system prune -af + +prune_cache: + @echo "🥫 Pruning Docker builder cache …" + docker builder prune -f diff --git a/app/__main__.py b/app/__main__.py new file mode 100644 index 0000000..c0bcdad --- /dev/null +++ b/app/__main__.py @@ -0,0 +1,4 @@ +from app.cli import main + +if __name__ == "__main__": + main() diff --git a/app/cli.py b/app/cli.py new file mode 100644 index 0000000..5958d30 --- /dev/null +++ b/app/cli.py @@ -0,0 +1,35 @@ +import typer + +app = typer.Typer() + + +@app.command() +def migrate_db(): + """Run unapplied DB migrations.""" + from openfoodfacts.utils import get_logger + + from app.models import db, run_migration + + get_logger() + + with db.connection_context(): + run_migration() + + +@app.command() +def add_revision( + name: str = typer.Argument(..., help="name of the revision"), +): + """Create a new migration file using peewee_migrate.""" + from openfoodfacts.utils import get_logger + + from app.models import add_revision, db + + get_logger() + + with db.connection_context(): + add_revision(name) + + +def main() -> None: + app() diff --git a/app/config.py b/app/config.py index 40a22ce..16b90c9 100644 --- a/app/config.py +++ b/app/config.py @@ -1,8 +1,11 @@ from enum import StrEnum +from pathlib import Path from openfoodfacts import Environment from pydantic_settings import BaseSettings +PROJECT_DIR = Path(__file__).parent.parent + class LoggingLevel(StrEnum): NOTSET = "NOTSET" @@ -36,6 +39,8 @@ class Settings(BaseSettings): postgres_password: str = "postgres" postgres_port: int = 5432 off_tld: Environment = Environment.net + environment: str = "dev" + migration_dir: Path = PROJECT_DIR / "migrations" settings = Settings() diff --git a/app/models.py b/app/models.py index e4426db..81c5f7f 100644 --- a/app/models.py +++ b/app/models.py @@ -5,7 +5,9 @@ ForeignKeyField, Model, PostgresqlDatabase, + TextField, ) +from peewee_migrate import Router from .config import settings @@ -19,12 +21,13 @@ class TicketModel(Model): - barcode = CharField(null=True) - type = CharField() - url = CharField() - status = CharField() + # barcode of the product, if any + barcode = TextField(null=True) + type = CharField(max_length=50) + url = TextField() + status = CharField(max_length=50) image_id = CharField(null=True) - flavor = CharField() + flavor = CharField(max_length=20) created_at = DateTimeField() class Meta: @@ -33,8 +36,8 @@ class Meta: class ModeratorActionModel(Model): - action_type = CharField() - user_id = CharField() + action_type = CharField(max_length=20) + user_id = TextField() ticket = ForeignKeyField(TicketModel, backref="moderator_actions") created_at = DateTimeField() @@ -45,19 +48,33 @@ class Meta: class FlagModel(Model): ticket = ForeignKeyField(TicketModel, backref="flags") - barcode = CharField(null=True) - type = CharField() - url = CharField() - user_id = CharField() - device_id = CharField() + barcode = TextField(null=True) + type = CharField(max_length=50) + url = TextField() + user_id = TextField() + device_id = TextField() source = CharField() confidence = FloatField(null=True) image_id = CharField(null=True) - flavor = CharField() - reason = CharField(null=True) - comment = CharField(max_length=500, null=True) + flavor = CharField(max_length=20) + reason = TextField(null=True) + comment = TextField(null=True) created_at = DateTimeField() class Meta: database = db table_name = "flags" + + +def run_migration(): + """Run all unapplied migrations.""" + # embedding schema does not exist at DB initialization + router = Router(db, migrate_dir=settings.migration_dir) + # Run all unapplied migrations + router.run() + + +def add_revision(name: str): + """Create a migration revision.""" + router = Router(db, migrate_dir=settings.migration_dir) + router.create(name, auto=True) diff --git a/app/utils.py b/app/utils.py index 3062a7a..f6870ca 100644 --- a/app/utils.py +++ b/app/utils.py @@ -4,6 +4,8 @@ from sentry_sdk.integrations import Integration from sentry_sdk.integrations.logging import LoggingIntegration +from app.config import settings + def init_sentry(sentry_dsn: str | None, integrations: list[Integration] | None = None): if sentry_dsn: @@ -17,4 +19,5 @@ def init_sentry(sentry_dsn: str | None, integrations: list[Integration] | None = sentry_sdk.init( # type:ignore # mypy say it's abstract sentry_dsn, integrations=integrations, + environment=settings.environment, ) diff --git a/docker-compose.yml b/docker-compose.yml index 8846ce9..5af2c7e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,6 @@ x-api-common: &api-common - POSTGRES_PASSWORD - POSTGRES_DB - POSTGRES_HOST - - OFF_TLD networks: - default @@ -18,7 +17,7 @@ services: api: <<: *api-common ports: - - "${API_PORT}:8000" + - "${API_EXPOSE:-127.0.0.1:8000}:8000" postgres: restart: $RESTART_POLICY diff --git a/docker/dev.yml b/docker/dev.yml index 2caec6f..95c266a 100644 --- a/docker/dev.yml +++ b/docker/dev.yml @@ -10,6 +10,8 @@ x-api-base: &api-base volumes: # mount code dynamically - "./app:/opt/nutripatrol/app" + # mount migrations dynamically + - "./migrations:/opt/nutripatrol/migrations" services: api: diff --git a/migrations/001_initial.py b/migrations/001_initial.py new file mode 100644 index 0000000..df0ea1e --- /dev/null +++ b/migrations/001_initial.py @@ -0,0 +1,67 @@ +"""Peewee migrations -- 001_initial.py.""" + +import peewee as pw +from peewee_migrate import Migrator + + +def migrate(migrator: Migrator, database: pw.Database, *, fake=False): + """Write your migrations here.""" + + @migrator.create_model + class TicketModel(pw.Model): + id = pw.AutoField() + barcode = pw.TextField(null=True) + type = pw.CharField(max_length=50) + url = pw.TextField() + status = pw.CharField(max_length=50) + image_id = pw.CharField(max_length=255, null=True) + flavor = pw.CharField(max_length=20) + created_at = pw.DateTimeField() + + class Meta: + table_name = "tickets" + + @migrator.create_model + class FlagModel(pw.Model): + id = pw.AutoField() + ticket = pw.ForeignKeyField( + column_name="ticket_id", field="id", model=migrator.orm["tickets"] + ) + barcode = pw.TextField(null=True) + type = pw.CharField(max_length=50) + url = pw.TextField() + user_id = pw.TextField() + device_id = pw.TextField() + source = pw.CharField(max_length=255) + confidence = pw.FloatField(null=True) + image_id = pw.CharField(max_length=255, null=True) + flavor = pw.CharField(max_length=20) + reason = pw.TextField(null=True) + comment = pw.TextField(null=True) + created_at = pw.DateTimeField() + + class Meta: + table_name = "flags" + + @migrator.create_model + class ModeratorActionModel(pw.Model): + id = pw.AutoField() + action_type = pw.CharField(max_length=20) + user_id = pw.TextField() + ticket = pw.ForeignKeyField( + column_name="ticket_id", field="id", model=migrator.orm["tickets"] + ) + created_at = pw.DateTimeField() + + class Meta: + table_name = "moderator_actions" + + +def rollback(migrator: Migrator, database: pw.Database, *, fake=False): + """Write your rollback migrations here.""" + + migrator.remove_model("moderator_actions") + + migrator.remove_model("flags") + + migrator.remove_model("tickets") diff --git a/requirements.txt b/requirements.txt index ac11390..e19123c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,6 @@ pydantic-settings==2.0.3 sentry-sdk[fastapi]==1.31.0 jinja2==3.1.3 peewee==3.17.0 -psycopg2-binary==2.9.9 \ No newline at end of file +peewee-migrate==1.12.2 +psycopg2-binary==2.9.9 +typer==0.9.0 \ No newline at end of file