Merge branch 'main' into rgseaa

.deepsource.toml

@@ -0,0 +1,19 @@
+version = 1
+
+test_patterns = ["tests/**"]
+exclude_patterns = ["legacy/**"]
+
+[[analyzers]]
+name = "python"
+enabled = true
+
+[analyzers.meta]
+runtime_version = "3.x.x"
+
+[[transformers]]
+name = "black"
+enabled = false
+
+[[transformers]]
+name = "isort"
+enabled = false

.dockerignore

@@ -1,9 +1,12 @@
 es
 ml
 ocr.jsonl.gz
+data
 datasets
 local_models
 models
 tf_models
 .git
-.idea
+.idea
+weights
+debug

.env

@@ -0,0 +1,102 @@
+COMPOSE_PATH_SEPARATOR=;
+COMPOSE_FILE=docker-compose.yml;docker/dev.yml;docker/mongodb.yml;docker/ml.yml
+
+# Docker
+RESTART_POLICY=no
+# for dev only: connection to local product opener network
+PO_LOCAL_NET=po_default
+
+# User uid - in dev align this with your own user uid / gid
+# see https://gist.github.com/alexgarel/6e6158ee869d6db2192e0441fd58576e
+# OFF_UID=1000
+# OFF_GID=1000
+
+# Robotoff
+TAG=latest
+
+# Robotoff instance gives the environment, either `prod` or `dev`
+# (`dev` by default).
+# If `prod` is used, openfoodfacts.org domain will be used by default,
+# and openfoodfacts.net if `dev` is used.
+# Messages to Slack are only enabled if `ROBOTOFF_INSTANCE=prod`.
+ROBOTOFF_INSTANCE=dev
+
+# Overwrites the Product Opener domain used. If empty, the domain will
+# be inferred from `ROBOTOFF_INSTANCE`
+ROBOTOFF_TLD=net
+
+# if you want to connect to a Product Opener dev instance on localhost, use:
+# STATIC_DOMAIN=http://openfoodfacts.localhost
+# ROBOTOFF_SCHEME=http  # for dev scheme is http
+
+# for dev only on localhost
+ROBOTOFF_EXPOSE=127.0.0.1:5500
+EVENTS_API_URL=
+
+# ElasticSearch
+ELASTIC_HOST=elasticsearch
+ELASTIC_USER=elastic
+ELASTIC_PASSWORD=elastic
+
+# Postgres
+# use network qualified name for dev for we have two networks
+POSTGRES_HOST=postgres
+POSTGRES_DB=postgres
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres
+# Expose postgres on localhost for dev
+# POSTGRES_EXPOSE=127.0.0.1:5432
+
+# Triton ML inference server
+TRITON_HOST=triton
+
+# Fasttext (langid) inference server
+FASTTEXT_HOST=fasttext
+
+# InfluxDB
+INFLUXDB_HOST=
+INFLUXDB_PORT=8086
+INFLUXDB_BUCKET=off_metrics
+INFLUXDB_AUTH_TOKEN=
+# on linux, this will work if you have an influxdb listening on 0.0.0.0
+# INFLUXDB_HOST=host.docker.internal
+
+# MongoDB (dev settings, using robotoff-specific MongoDB instance by default)
+# To use Product Opener instance, use following commented line
+# MONGO_URI=mongodb://mongodb.po_default:27017
+MONGO_URI=mongodb://mongodb:27017
+
+# Redis
+REDIS_HOST=redis
+
+# OpenFoodFacts API
+OFF_PASSWORD=
+OFF_USER=
+
+# Utils
+SENTRY_DSN=
+
+# We need this envvar as the path is different in prod and local environments:
+# we launch directly docker/ml.yml in preprod/prod, while it's part of docker-compose.yml
+# in local environment
+TRITON_MODELS_DIR=./models/triton
+FASTTEXT_MODEL_DIR=./models
+
+# Enable/disable MongoDB access. All insights/predictions are checked
+# against MongoDB, we disable by default locally to be able to easily test
+# image import
+ENABLE_MONGODB_ACCESS=0
+
+# gunicorn --auto-reload is not compatible with preload_app
+# so it has to be disabled when developing, to allow hot reload
+GUNICORN_PRELOAD_APP=0
+
+# The log level for the robotoff service
+LOG_LEVEL=DEBUG
+
+# PostgreSQL configuration
+
+# Use 16G in production
+ROBOTOFF_POSTGRES_SHARED_BUFFERS=1GB
+# Use 1G in production
+ROBOTOFF_POSTGRES_WORK_MEM=64MB

.flake8

@@ -0,0 +1,7 @@
+[flake8]
+ignore = E203, E501, W503
+max-line-length = 88
+exclude = .git,__pycache__,build,dist,*_pb2.py,.venv
+per-file-ignores =
+    robotoff/cli/main.py:B008
+max-doc-length = 79

.gitattributes

@@ -0,0 +1,12 @@
+# Basic .gitattributes to work with docker, even on windows
+# prefer keeping line ending unix style
+# since most files are used in linux docker containers
+* text=auto eol=lf
+[core]
+    # avoid line ending conversion on windows
+    autocrlf=false
+    # try to respect symlinks even on windows
+    symlinks=true
+*.json.gz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text

.github/CODEOWNERS

@@ -0,0 +1,7 @@
+
+# These owners will be the default owners for everything in
+# the repo. Unless a later match takes precedence,
+# review when someone opens a pull request.
+# For more on how to customize the CODEOWNERS file - https://help.github.com/en/articles/about-code-owners
+
+*       @openfoodfacts/robotoff

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,37 +1,29 @@
 ---
 name: Bug report
-about: Create a report to help us improve
+about: Create a report to help us improve Robotoff
 title: ''
-labels: ''
+labels: 'bug'
 assignees: ''
 
 ---
 
-**Describe the bug**
-A clear and concise description of what the bug is.
+### What
+<!-- A clear and concise description of what the bug is.-->
 
-**To Reproduce**
-Steps to reproduce the behavior:
+### Steps to reproduce the behavior:
 1. Go to '...'
 2. Click on '....'
 3. See error
 
-**Expected behavior**
-A clear and concise description of what you expected to happen.
+### Expected behavior
+<!-- A clear and concise description of what you expected to happen. -->
 
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
+### Screenshots
+<!-- (Optional, delete if not needed) -->
 
-**Desktop (please complete the following information):**
- - OS: [e.g. iOS]
- - Browser [e.g. chrome, safari]
- - Version [e.g. 22]
+### Platform (Desktop, Mobile, Hunger Games)
+ - OS: <!-- [e.g. iOS, Android, Desktop] -->
+ - Platform <!-- [e.g. Desktop, Mobile, Hunger Games] -->
 
-**Smartphone (please complete the following information):**
- - Device: [e.g. iPhone6]
- - OS: [e.g. iOS8.1]
- - Browser [e.g. stock browser, safari]
- - Version [e.g. 22]
-
-**Additional context**
-Add any other context about the problem here.
+### Part of
+- <!-- Add the issue number preceded by # (that also allows searching by name for similar issues): eg #374 -->

.github/ISSUE_TEMPLATE/epic.md

@@ -0,0 +1,17 @@
+---
+name: Epic
+about: Template for epics. Epics group several user stories together into a main piece
+  of added value.
+title: ''
+labels: "♞ epic"
+assignees: ''
+
+---
+
+### Who for
+
+### What
+
+### Why
+
+### Part of

.github/ISSUE_TEMPLATE/feature_request.md

@@ -2,19 +2,21 @@
 name: Feature request
 about: Suggest an idea for this project
 title: ''
-labels: ''
+labels: 'enhancement'
 assignees: ''
 
 ---
 
-**Is your feature request related to a problem? Please describe.**
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+### Problem
+<!-- Please replace me by a clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->
 
-**Describe the solution you'd like**
-A clear and concise description of what you want to happen.
+### Proposed solution
+<!-- Please replace me by a clear and concise description of what you want to happen) -->
 
-**Describe alternatives you've considered**
-A clear and concise description of any alternative solutions or features you've considered.
+### Additional context
+<!-- Please replace me by any other context or screenshots about the feature request here.) -->
 
-**Additional context**
-Add any other context or screenshots about the feature request here.
+### Mockups
+
+### Part of
+- #374 <!-- Please remplace 374 by the most specific parent issue possible -->

.github/labeler.yml

@@ -0,0 +1,113 @@
+# Please see https://github.com/actions/labeler for reference
+
+github_actions:
+- changed-files:
+  - any-glob-to-any-file: '.github/**/*'
+
+crowdin:
+- changed-files:
+  - any-glob-to-any-file: 'crowdin.yml'
+
+documentation:
+- changed-files:
+  - any-glob-to-any-file: ['*.MD']
+
+tests:
+- changed-files:
+  - any-glob-to-any-file: 'tests/**/*'
+
+integration tests:
+- changed-files:
+  - any-glob-to-any-file: 'tests/integration/**/*'
+  - any-glob-to-any-file: 'tests/integration/insights/**/*'
+
+unit tests:
+- changed-files:
+  - any-glob-to-any-file: 'tests/unit/**/*'
+  - any-glob-to-any-file: 'tests/unit/insights/**/*'
+  - any-glob-to-any-file: 'tests/unit/cli/**/*'
+  - any-glob-to-any-file: 'tests/unit/utils/**/*'
+  - any-glob-to-any-file: 'tests/unit/prediction/**/*'
+  - any-glob-to-any-file: 'tests/unit/elasticsearch/**/*'
+  - any-glob-to-any-file: 'tests/unit/prediction/category/**/*'
+  - any-glob-to-any-file: 'tests/unit/prediction/category/neural/**/*'
+
+robotoff app:
+- changed-files:
+  - any-glob-to-any-file: 'robotoff/app/**/*'
+
+robotoff-cli:
+- changed-files:
+  - any-glob-to-any-file: 'robotoff/cli/**/*'
+  - any-glob-to-any-file: 'tests/unit/cli/**/*'
+
+utils:
+- changed-files:
+  - any-glob-to-any-file: 'robotoff/utils/**/*'
+  - any-glob-to-any-file: 'tests/unit/utils/**/*'
+
+workers:
+- changed-files:
+  - any-glob-to-any-file: 'robotoff/workers/**/*'
+  - any-glob-to-any-file: 'robotoff/workers/tasks/**/*'
+
+insights:
+- changed-files:
+  - any-glob-to-any-file: 'robotoff/insights/**/*'
+  - any-glob-to-any-file: 'tests/integration/insights/**/*'
+  - any-glob-to-any-file: 'tests/unit/insights/**/*'
+
+scheduler:
+- changed-files:
+  - any-glob-to-any-file: 'robotoff/scheduler/**/*'
+
+predictions:
+- changed-files:
+  - any-glob-to-any-file: 'robotoff/prediction/**/*'
+  - any-glob-to-any-file: 'tests/unit/prediction/**/*'
+
+ocr prediction:
+- changed-files:
+  - any-glob-to-any-file: 'robotoff/prediction/ocr/**/*'
+  - any-glob-to-any-file: 'tests/unit/prediction/ocr/**/*'
+  - any-glob-to-any-file: 'legacy/ocr_category/prediction_from_ocr/**/*'
+
+elasticsearch:
+- changed-files:
+  - any-glob-to-any-file: 'robotoff/elasticsearch/**/*'
+  - any-glob-to-any-file: 'tests/unit/elasticsearch/**/*'
+  - any-glob-to-any-file: 'robotoff/elasticsearch/product/**/*'
+  - any-glob-to-any-file: 'robotoff/elasticsearch/category/**/*'
+  - any-glob-to-any-file: 'tests/unit/elasticsearch/category/**/*'
+  - any-glob-to-any-file: 'tests/unit/elasticsearch/**/*'
+  - any-glob-to-any-file: 'robotoff/elasticsearch/category/**/*'
+  - any-glob-to-any-file: 'tests/unit/elasticsearch/category/**/*'
+  - any-glob-to-any-file: 'robotoff/elasticsearch/product/**/*'
+
+object detection:
+- changed-files:
+  - any-glob-to-any-file: 'robotoff/prediction/object_detection/utils/**/*'
+  - any-glob-to-any-file: 'robotoff/prediction/object_detection/**/*'
+
+category prediction:
+- changed-files:
+  - any-glob-to-any-file: 'robotoff/prediction/category/**/*'
+  - any-glob-to-any-file: 'tests/unit/prediction/category/**/*'
+  - any-glob-to-any-file: 'tests/unit/elasticsearch/category/**/*'
+  - any-glob-to-any-file: 'robotoff/elasticsearch/category/**/*'
+  - any-glob-to-any-file: 'tests/unit/elasticsearch/category/**/*'
+  - any-glob-to-any-file: 'tests/unit/prediction/category/**/*'
+  - any-glob-to-any-file: 'legacy/ocr_category/prediction_from_ocr/**/*'
+
+neural category prediction:
+- changed-files:
+  - any-glob-to-any-file: 'robotoff/prediction/category/neural/**/*'
+  - any-glob-to-any-file: 'tests/unit/prediction/category/neural/**/*'
+
+stores:
+- changed-files:
+  - any-glob-to-any-file: 'data/ocr/store_regex.txt'
+
+legacy:
+- legacy/**/*
+- legacy/ocr_category/prediction_from_ocr/**/*