Merge main into sweep/add-sweep-config

.env

@@ -0,0 +1,17 @@
+# set project name to have a short one
+COMPOSE_PROJECT_NAME=nutripatrol
+# unify separator with windows style
+COMPOSE_PATH_SEPARATOR=;
+# dev is default target
+COMPOSE_FILE=docker-compose.yml;docker/dev.yml
+
+API_PORT=127.0.0.1:8000
+
+# by default on dev desktop, no restart
+RESTART_POLICY=no
+
+# Sentry DNS for bug tracking, used only in staging and production
+SENTRY_DNS=
+
+# Log level to use, DEBUG by default in dev
+LOG_LEVEL=DEBUG

.flake8

@@ -0,0 +1,5 @@
+[flake8]
+ignore = E203, E501, W503
+max-line-length = 88
+exclude = .git,__pycache__,build,dist,*_pb2.py,.venv
+max-doc-length = 79

.github/dependabot.yml

@@ -0,0 +1,22 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: docker
+    directory: "/"
+    schedule:
+      interval: daily
+    open-pull-requests-limit: 30
+  - package-ecosystem: github-actions
+    directory: "/"
+    schedule:
+      interval: daily
+    open-pull-requests-limit: 30
+  - package-ecosystem: pip
+    directory: "/app"
+    schedule:
+      interval: daily
+    open-pull-requests-limit: 30

.github/workflows/auto-assign-pr.yml

@@ -0,0 +1,15 @@
+# .github/workflows/auto-author-assign.yml
+name: 'Auto Author Assign'
+
+on:
+  pull_request_target:
+    types: [opened, reopened]
+
+permissions:
+  pull-requests: write
+
+jobs:
+  assign-author:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: toshimaru/[email protected]

.github/workflows/codeql.yml

@@ -0,0 +1,74 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "main", deploy-* ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ "main" ]
+  schedule:
+    - cron: '25 5 * * 5'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines.
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #   echo "Run, Build Application using script"
+    #   ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/container-build.yml

@@ -0,0 +1,57 @@
+name: Container Image Build CI
+
+on:
+  push:
+    branches:
+      - main
+      - deploy-*
+    tags:
+      - v*.*.*
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        image_name:
+        - api
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 1
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+      with:
+        version: v0.6.0
+        buildkitd-flags: --debug
+
+    - name: Login to DockerHub
+      uses: docker/login-action@v2
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Docker meta
+      id: meta
+      uses: docker/metadata-action@v4
+      with:
+        images: |
+          ghcr.io/${{ github.repository }}/${{ matrix.image_name }}
+        tags: |
+          type=semver,pattern={{version}}
+          type=ref,event=pr
+          type=ref,event=branch
+          type=sha,format=long
+
+    - name: Build and push
+      uses: docker/build-push-action@v3
+      with:
+        context: .
+        push: true
+        file: Dockerfile
+        cache-from: type=registry,ref=ghcr.io/${{ github.repository }}/${{ matrix.image_name }}:buildcache
+        cache-to: type=registry,ref=ghcr.io/${{ github.repository }}/${{ matrix.image_name }}:buildcache,mode=max
+        tags: ${{ steps.meta.outputs.tags }}
+        labels: ${{ steps.meta.outputs.labels }}

.github/workflows/container-deploy.yml

@@ -0,0 +1,194 @@
+name: Container Image Deployment CI
+
+on:
+  push:
+    branches:
+      - main
+      - deploy-*
+# only staging for now, not prod
+#     tags:
+#       - v*.*.*
+
+
+# Note on secrets used for connection
+# They are configured as environment secrets
+# HOST is the internal ip of VM containing docker
+# PROXY_HOST is the host of VMs
+# USERNAME is the user used for operations
+# SSH_PRIVATE_KEY is the private key (shared between VM and host)
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        env:
+          # only stagging for now
+          # Note: env is also the name of the directory on the server
+          - nutripatrol-net
+    environment: ${{ matrix.env }}
+    concurrency: ${{ matrix.env }}
+    steps:
+    - name: Set various variable for staging (net) deployment
+      if: matrix.env == 'nutripatrol-net'
+      run: |
+        # direct container access
+        echo "OPENFOODFACTS_API_URL=https://off:[email protected]" >> $GITHUB_ENV
+        # deploy target
+        echo "SSH_HOST=10.1.0.200" >> $GITHUB_ENV
+        echo "SSH_PROXY_HOST=ovh1.openfoodfacts.org" >> $GITHUB_ENV
+        echo "SSH_USERNAME=off" >> $GITHUB_ENV
+    - name: Wait for docker image container build workflow
+      uses: tomchv/[email protected]
+      id: wait-build
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        checkName: build (api)
+        ref: ${{ github.event.pull_request.head.sha || github.sha }}
+        intervalSeconds: 10
+        timeoutSeconds: 600 # 10m
+
+    - name: Do something if build isn't launch
+      if: steps.wait-build.outputs.conclusion == 'does not exist'
+      run: echo job does not exist && true
+
+    - name: Do something if build fail
+      if: steps.wait-build.outputs.conclusion == 'failure'
+      run: echo fail && false # fail if build fail
+
+    - name: Do something if build timeout
+      if: steps.wait-build.outputs.conclusion == 'timed_out'
+      run: echo Timeout && false # fail if build time out
+
+    - name: Checkout git repository
+      uses: appleboy/ssh-action@master
+      with:
+        host: ${{ env.SSH_HOST }}
+        username: ${{ env.SSH_USERNAME }}
+        key: ${{ secrets.SSH_PRIVATE_KEY }}
+        proxy_host: ${{ env.SSH_PROXY_HOST }}
+        proxy_username: ${{ env.SSH_USERNAME }}
+        proxy_key: ${{ secrets.SSH_PRIVATE_KEY }}
+        script_stop: false
+        script: |
+          # Clone Git repository if not already there
+          [ ! -d '${{ matrix.env }}' ] && git clone --depth 1 https://github.com/${{ github.repository }} ${{ matrix.env }} --no-single-branch 2>&1
+
+          # Go to repository directory
+          cd ${{ matrix.env }}
+
+          # Fetch newest commits (in case it wasn't freshly cloned)
+          git fetch --depth 1
+
+          # Checkout current commit SHA
+          git checkout -qf ${{ github.sha }}
+
+    - name: Set environment variables
+      uses: appleboy/ssh-action@master
+      with:
+        host: ${{ env.SSH_HOST }}
+        username: ${{ env.SSH_USERNAME }}
+        key: ${{ secrets.SSH_PRIVATE_KEY }}
+        proxy_host: ${{ env.SSH_PROXY_HOST }}
+        proxy_username: ${{ env.SSH_USERNAME }}
+        proxy_key: ${{ secrets.SSH_PRIVATE_KEY }}
+        script_stop: false
+        script: |
+          # Go to repository directory
+          cd ${{ matrix.env }}
+
+          mv .env .env-dev
+          # init .env
+          echo "# Env file generated by container-deploy action"> .env
+          # Set Docker Compose variables
+          echo "DOCKER_CLIENT_TIMEOUT=180" >> .env
+          echo "COMPOSE_HTTP_TIMEOUT=180" >> .env
+          echo "COMPOSE_PROJECT_NAME=nutripatrol" >> .env
+          echo "COMPOSE_PATH_SEPARATOR=;" >> .env
+          echo "COMPOSE_FILE=docker-compose.yml;docker/prod.yml" >> .env
+          # Copy variables that are same as dev
+          grep '\(STACK_VERSION\|ES_PORT\)' .env-dev >> .env
+          # Set docker variables
+          echo "TAG=sha-${{ github.sha }}" >> .env
+          echo "RESTART_POLICY=always" >> .env
+          # Set App variables
+          echo "CLUSTER_NAME=${{ matrix.env }}-es-cluster" >> .env
+          echo "SEARCH_PORT=8180" >> .env
+          echo "ES_VUE_PORT=8181" >> .env
+          echo "REDIS_PORT=8182" >> .env
+          echo "MEM_LIMIT=4294967296" >> .env
+          # this is the network shared with productopener
+          echo "COMMON_NET_NAME=po_webnet">> .env
+          echo "OPENFOODFACTS_API_URL=${{ env.OPENFOODFACTS_API_URL }}" >> .env
+          # This secret is to be generated using htpasswd, see .env file
+          # use simple quotes to avoid interpolation of $apr1$ !
+          echo 'NGINX_BASIC_AUTH_USER_PASSWD=${{ secrets.NGINX_BASIC_AUTH_USER_PASSWD }}' >> .env
+          echo "SENTRY_DNS=${{ secrets.SENTRY_DSN }}" >> .env
+          echo "CONFIG_PATH=data/config/openfoodfacts.yml" >> .env
+
+    - name: Create Docker volumes
+      uses: appleboy/ssh-action@master
+      with:
+        host: ${{ env.SSH_HOST }}
+        username: ${{ env.SSH_USERNAME }}
+        key: ${{ secrets.SSH_PRIVATE_KEY }}
+        proxy_host: ${{ env.SSH_PROXY_HOST }}
+        proxy_username: ${{ env.SSH_USERNAME }}
+        proxy_key: ${{ secrets.SSH_PRIVATE_KEY }}
+        script_stop: false
+        script: |
+          cd ${{ matrix.env }}
+          make create_external_volumes
+
+    - name: Start services
+      uses: appleboy/ssh-action@master
+      with:
+        host: ${{ env.SSH_HOST }}
+        username: ${{ env.SSH_USERNAME }}
+        key: ${{ secrets.SSH_PRIVATE_KEY }}
+        proxy_host: ${{ env.SSH_PROXY_HOST }}
+        proxy_username: ${{ env.SSH_USERNAME }}
+        proxy_key: ${{ secrets.SSH_PRIVATE_KEY }}
+        script_stop: false
+        script: |
+          cd ${{ matrix.env }}
+          docker-compose down
+          docker-compose up -d --remove-orphans 2>&1
+
+    - name: Check services are up
+      uses: appleboy/ssh-action@master
+      if: ${{ always() }}
+      with:
+        host: ${{ env.SSH_HOST }}
+        username: ${{ env.SSH_USERNAME }}
+        key: ${{ secrets.SSH_PRIVATE_KEY }}
+        proxy_host: ${{ env.SSH_PROXY_HOST }}
+        proxy_username: ${{ env.SSH_USERNAME }}
+        proxy_key: ${{ secrets.SSH_PRIVATE_KEY }}
+        script_stop: false
+        script: |
+          cd ${{ matrix.env }}
+          make livecheck
+
+    - name: Cleanup obsolete Docker objects
+      uses: appleboy/ssh-action@master
+      if: ${{ always() }}
+      with:
+        host: ${{ env.SSH_HOST }}
+        username: ${{ env.SSH_USERNAME }}
+        key: ${{ secrets.SSH_PRIVATE_KEY }}
+        proxy_host: ${{ env.SSH_PROXY_HOST }}
+        proxy_username: ${{ env.SSH_USERNAME }}
+        proxy_key: ${{ secrets.SSH_PRIVATE_KEY }}
+        script_stop: false
+        script: |
+          cd ${{ matrix.env }}
+          docker system prune -af
+
+    - uses: frankie567/[email protected]
+      if: ${{ always() }}
+      with:
+        apiHost: https://grafana.openfoodfacts.org
+        apiToken: ${{ secrets.GRAFANA_API_TOKEN }}
+        text: <a href="https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}">Deployment ${{ steps.livecheck.outcome }} on ${{ matrix.env }}</a>
+        tags: type:deployment,origin:github,status:${{ steps.livecheck.outcome }},repo:${{ github.repository }},sha:${{ github.sha }},app:robotoff,env:${{ matrix.env }}