feat: implement real multi-platform support (OFF, OBF,...)

- delete server_domain in image, product_insight and prediction tables - add server_type field to image and prediction tables - use ProductIdentifier (barcode + server_type) instead of barcode in codebase See #894
openfoodfacts · Apr 13, 2023 · 9464f46 · 9464f46
1 parent 232e5c6
commit 9464f46
Show file tree

Hide file tree

Showing 57 changed files with 1,284 additions and 931 deletions.
diff --git a/.env b/.env
@@ -23,10 +23,10 @@ ROBOTOFF_INSTANCE=dev
 
 # Overwrites the Product Opener domain used. If empty, the domain will
 # be inferred from `ROBOTOFF_INSTANCE`
-ROBOTOFF_DOMAIN=openfoodfacts.net
+ROBOTOFF_TLD=net
 
 # if you want to connect to a Product Opener dev instance on localhost, use:
-# STATIC_OFF_DOMAIN=http://openfoodfacts.localhost
+# STATIC_DOMAIN=http://openfoodfacts.localhost
 # ROBOTOFF_SCHEME=http  # for dev scheme is http
 
 # for dev only on localhost

diff --git a/.github/workflows/container-deploy.yml b/.github/workflows/container-deploy.yml
@@ -25,7 +25,7 @@ jobs:
         echo "SSH_PROXY_HOST=ovh2.openfoodfacts.org" >> $GITHUB_ENV
         echo "SSH_USERNAME=off" >> $GITHUB_ENV
         echo "ROBOTOFF_INSTANCE=dev" >> $GITHUB_ENV
-        echo "ROBOTOFF_DOMAIN=openfoodfacts.net" >> $GITHUB_ENV
+        echo "ROBOTOFF_TLD=net" >> $GITHUB_ENV
         echo "MONGO_URI=mongodb://10.1.0.200:27017" >> $GITHUB_ENV
         echo "INFLUXDB_HOST=10.1.0.200" >> $GITHUB_ENV
     - name: Set various variable for production deployment
@@ -35,7 +35,7 @@ jobs:
         echo "SSH_PROXY_HOST=ovh2.openfoodfacts.org" >> $GITHUB_ENV
         echo "SSH_USERNAME=off" >> $GITHUB_ENV
         echo "ROBOTOFF_INSTANCE=prod" >> $GITHUB_ENV
-        echo "ROBOTOFF_DOMAIN=openfoodfacts.org" >> $GITHUB_ENV
+        echo "ROBOTOFF_TLD=org" >> $GITHUB_ENV
         echo "MONGO_URI=mongodb://213.36.253.195:27017" >> $GITHUB_ENV
         echo "INFLUXDB_HOST=10.1.0.201" >> $GITHUB_ENV
     - name: Wait for container build workflow
@@ -111,7 +111,7 @@ jobs:
 
           # Set app variables
           echo "ROBOTOFF_INSTANCE=${{ env.ROBOTOFF_INSTANCE }}" >> .env
-          echo "ROBOTOFF_DOMAIN=${{ env.ROBOTOFF_DOMAIN }}" >> .env
+          echo "ROBOTOFF_TLD=${{ env.ROBOTOFF_TLD }}" >> .env
           echo "REDIS_HOST=redis.robotoff_default" >> .env
           echo "POSTGRES_HOST=postgres.robotoff_default" >> .env
           echo "POSTGRES_DB=postgres" >> .env
@@ -132,7 +132,7 @@ jobs:
           echo "INFLUXDB_AUTH_TOKEN=${{ secrets.INFLUXDB_AUTH_TOKEN }}" >> .env
           echo "SLACK_TOKEN=${{ secrets.SLACK_TOKEN }}" >> .env
           echo "GUNICORN_NUM_WORKERS=8"
-          echo "EVENTS_API_URL=https://event.${{ env.ROBOTOFF_DOMAIN }}" >> .env
+          echo "EVENTS_API_URL=https://event.openfoodfacts.${{ env.ROBOTOFF_TLD }}" >> .env
           # TODO: remove this url when we have a proper server running for this purpose
           echo "IMAGE_MODERATION_SERVICE_URL=https://amathjourney.com/api/off-annotation/flag-image"
 

diff --git a/doc/explanations/interactions-product-opener.md b/doc/explanations/interactions-product-opener.md
@@ -14,7 +14,6 @@ Product Opener calls `POST /api/v1/webhook/product` whenever a product is update
 
 - `barcode`: the barcode of product
 - `action`:  either `updated` or `deleted`
-- `server_domain`: the server domain (ex: `api.openfoodfacts.org`)
 
 After receiving a `product_update` webhook call, Robotoff does the following [^product_update]:
 
@@ -29,7 +28,6 @@ Product Opener calls `POST /api/v1/images/import` whenever an new image is uploa
 - `barcode`: the barcode of product
 - `image_url`:  the URL of the image
 - `ocr_url`:  the URL of the OCR result (JSON file)
-- `server_domain`: the server domain (ex: `api.openfoodfacts.org`)
 
 After receiving a `import_image` webhook call, Robotoff does the following [^image_import]:
 

diff --git a/doc/how-to-guides/test-and-debug.md b/doc/how-to-guides/test-and-debug.md
@@ -66,11 +66,8 @@ Write test cases every time you write a new feature, to test a feature or to und
 
 There are even cases where automated tests are your only chance to test you code. For example: when you write code to post notifications on Slack channel you can only test them  by writing a unit test case. 
 
-There are instances when Robotoff tries to connect to MongoDB via Open Food Facts server. For local testing we do not yet provide a standarized approach to add a MongoDB Docker in the same network and configure Robotoff to use it.
-
-In such cases you will have to mock the function which calls MongoDB. Feel free to reuse the existing test cases.
-
-To identify parts of the code where Robotoff connects to MongoDB or to Open Food Facts server (the part you should mock), keep an eye for variables like `server_url`, `server_domain` or `settings.OFF_SERVER_DOMAIN`.
+There are instances when Robotoff tries to connect to MongoDB via Open Food Facts server. To disable this
+feature (this is disabled by default on local environments), set `DISABLE_PRODUCT_CHECK=1` in your `.env`.
 
 # Debugging guide
 

diff --git a/doc/references/api.yml b/doc/references/api.yml
@@ -5,8 +5,6 @@ info:
     Robotoff provides a simple API allowing consumers to fetch predictions and annotate them.
 
     All endpoints must be prefixed with `/api/v1`. The full URL is `https://robotoff.openfoodfacts.org/api/v1/{endpoint}`.
-
-    Robotoff can interact with all Openfoodfacts products: Openfoodfacts, Openbeautyfacts, etc. and all environments (production, development, pro). The `server_domain` field should be used to specify the product/environment: `api.openfoodfacts.org` for OFF-prod, `api.openfoodfacts.net` for OFF-dev, `api.openbeautyfacts.org` for OBF-prod,...
   contact: {}
   version: "1.0"
 servers:
@@ -30,7 +28,7 @@ paths:
             default: 1
             minimum: 1
         - $ref: "#/components/parameters/barcode_path"
-        - $ref: "#/components/parameters/server_domain"
+        - $ref: "#/components/parameters/server_type"
         - $ref: "#/components/parameters/lang"
       responses:
         "200":
@@ -57,7 +55,7 @@ paths:
       parameters:
         - $ref: "#/components/parameters/lang"
         - $ref: "#/components/parameters/count"
-        - $ref: "#/components/parameters/server_domain"
+        - $ref: "#/components/parameters/server_type"
         - $ref: "#/components/parameters/insight_types"
         - $ref: "#/components/parameters/country"
         - $ref: "#/components/parameters/brands"
@@ -110,7 +108,7 @@ paths:
       parameters:
         - $ref: "#/components/parameters/lang"
         - $ref: "#/components/parameters/count"
-        - $ref: "#/components/parameters/server_domain"
+        - $ref: "#/components/parameters/server_type"
         - $ref: "#/components/parameters/insight_types"
         - $ref: "#/components/parameters/country"
         - $ref: "#/components/parameters/brands"
@@ -151,7 +149,7 @@ paths:
       parameters:
         - $ref: "#/components/parameters/lang"
         - $ref: "#/components/parameters/count"
-        - $ref: "#/components/parameters/server_domain"
+        - $ref: "#/components/parameters/server_type"
         - $ref: "#/components/parameters/insight_types"
         - $ref: "#/components/parameters/country"
         - $ref: "#/components/parameters/brands"
@@ -180,7 +178,7 @@ paths:
             type: number
             default: 25
             minimum: 1
-        - $ref: "#/components/parameters/server_domain"
+        - $ref: "#/components/parameters/server_type"
         - $ref: "#/components/parameters/insight_type"
         - $ref: "#/components/parameters/country"
         - $ref: "#/components/parameters/page"
@@ -226,7 +224,7 @@ paths:
         - $ref: "#/components/parameters/insight_type"
         - $ref: "#/components/parameters/country"
         - $ref: "#/components/parameters/value_tag"
-        - $ref: "#/components/parameters/server_domain"
+        - $ref: "#/components/parameters/server_type"
         - $ref: "#/components/parameters/count"
         - $ref: "#/components/parameters/predictor"
       responses:
@@ -249,6 +247,7 @@ paths:
       summary: Get all insights for a specific product
       parameters:
         - $ref: "#/components/parameters/barcode_path"
+        - $ref: "#/components/parameters/server_type"
       responses:
         "200":
           description: ""
@@ -330,6 +329,7 @@ paths:
       tags:
         - Insights
       parameters:
+          - $ref: "#/components/parameters/server_type"
           - $ref: "#/components/parameters/value_tag"
           - $ref: "#/components/parameters/insight_types"
           - name: barcode
@@ -450,12 +450,7 @@ paths:
                   format: uri
                   description: URL of the OCR JSON file associated with the image
                 server_domain:
-                  type: string
-                  description: |
-                    The server domain associated with the image/product.
-                    If the server domain does not match with the server configuration,
-                    the import task will be rejected by Robotoff.
-                  example: "api.openfoodfacts.org"
+                  $ref: "#/components/schemas/ServerDomainParameter"
               required:
                 - "barcode"
                 - "image_url"
@@ -472,12 +467,16 @@ paths:
                   status:
                     type: string
                     description: |
-                      status of the import operation, either `scheduled` if it was
-                      successfully scheduled or rejected if the `server_domain` did
-                      not match Robotoff configured server domain.
+                      status of the import operation, always `scheduled`
                     enum:
-                      - "rejected"
                       - "scheduled"
+        "400":
+          description: "HTTP Bad Request error, if the `server_domain` parameter is invalid"
+          content:
+            application/json:
+              schema:
+                type: object
+
 
   /images/logos:
     get:
@@ -517,6 +516,7 @@ paths:
         Search for logos detected using the universal-logo-detector model that 
         meet some criteria (annotation status, annotated, type,...)
       parameters:
+        - $ref: "#/components/parameters/server_type"
         - name: count
           description: Number of results to return
           in: query
@@ -561,11 +561,6 @@ paths:
           schema:
             type: boolean
             default: false
-        - name: server_domain
-          in: query
-          description: The server domain
-          schema:
-            type: string
         - name: annotated
           description: The annotation status of the logo.
             If not provided, both annotated and non-annotated logos are returned
@@ -660,6 +655,17 @@ paths:
                       description: The barcode of the product to categorize
                       minLength: 1
                       example: 0748162621021
+                    server_type:
+                      type: string
+                      description: |
+                        The server type (=project) to use, such as 'off' (Open Food Facts), 'obf' (Open Beauty Facts),...
+                        Only 'off' is currently supported for category prediction
+                      default: 'off'
+                      enum:
+                        - 'off'
+                        - 'obf'
+                        - 'opff'
+                        - 'opf'
                     deepest_only:
                       type: boolean
                       description: |
@@ -875,6 +881,20 @@ components:
         id: 3cd5aecd-edcc-4237-87d0-6595fc4e53c9
         type: label
         barcode: 9782012805866
+    ServerDomainParameter:
+      description: |
+        The server domain associated with the image/product.
+
+        If the `server_domain` top level domain does not match the server configuration,
+        an HTTP 400 error will be raised
+      type: string
+      example: "api.openfoodfacts.org"
+      enum:
+        - "api.openfoodfacts.org"
+        - "api.openbeautyfacts.org"
+        - "api.openproductfacts.org"
+        - "api.openpetfoodfacts.org"
+        - "api.pro.openfoodfacts.org"
   parameters:
     lang:
       name: lang
@@ -891,13 +911,18 @@ components:
         type: integer
         default: 25
         minimum: 1
-    server_domain:
-      name: server_domain
+    server_type:
+      name: server_type
       in: query
-      description: The server domain
+      description: The server type (=project) to use, such as 'off' (Open Food Facts), 'obf' (Open Beauty Facts),... 
       schema:
         type: string
-        default: api.openfoodfacts.org
+        default: 'off'
+        enum:
+          - 'off'
+          - 'obf'
+          - 'opff'
+          - 'opf'
     insight_types:
       name: insight_types
       in: query

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -14,9 +14,9 @@ x-robotoff-base-env:
   &robotoff-base-env
   LOG_LEVEL:
   ROBOTOFF_INSTANCE:
-  ROBOTOFF_DOMAIN:
+  ROBOTOFF_TLD:
   ROBOTOFF_SCHEME:
-  STATIC_OFF_DOMAIN:
+  STATIC_DOMAIN:
   GUNICORN_NUM_WORKERS:
   ROBOTOFF_UPDATED_PRODUCT_WAIT:
   REDIS_HOST: