feat: Use openfoodfacts-query service for facet queries instead of product_tags collection

.env

@@ -45,6 +45,9 @@ MONGODB_CACHE_SIZE=8 # GB
 MONGO_INITDB_ROOT_USERNAME=root
 MONGO_INITDB_ROOT_PASSWORD=test
 ROBOTOFF_URL=http://robotoff.openfoodfacts.localhost:5500 # connect to Robotoff running in separate docker-compose deployment
+# connect to openfoodfacts-query running in separate docker-compose deployment.
+# To test locally change to http://host.docker.internal:5510
+QUERY_URL=http://query:5510
 EVENTS_URL=
 FACETS_KP_URL = https://facets-kp.openfoodfacts.org/render-to-html
 # use this to push products to openfoodfacts-search

.github/labeler.yml

@@ -12,7 +12,6 @@ nginx:
 
 mongodb:
 - conf/mongodb/create_indexes.js
-- scripts/refresh_products_tags.js
 - .github/workflows/mongo-deploy.yml
 - scripts/checkmongodb.pl
 - scripts/update_all_products_from_dir_in_mongodb.pl

.github/workflows/daily.yml

@@ -84,7 +84,7 @@ jobs:
         echo "SSH_HOST=10.1.0.200" >> $GITHUB_ENV
         echo "SSH_PROXY_HOST=ovh1.openfoodfacts.org" >> $GITHUB_ENV
         echo "SSH_USERNAME=off" >> $GITHUB_ENV
-    - name: Refresh MongoDB products_tags collection
+    - name: Refresh Postgres cache
       uses: appleboy/ssh-action@master
       with:
         host: ${{ env.SSH_HOST }}

Makefile

@@ -192,10 +192,8 @@ create_mongodb_indexes:
 	${DOCKER_COMPOSE} exec -T mongodb //bin/sh -c "mongo off /data/db/create_indexes.js"
 
 refresh_product_tags:
-	@echo "🥫 Refreshing products tags (update MongoDB products_tags collection) …"
-# get id for mongodb container
-	docker cp scripts/refresh_products_tags.js $(shell docker-compose ps -q mongodb):/data/db
-	${DOCKER_COMPOSE} exec -T mongodb //bin/sh -c "mongo off /data/db/refresh_products_tags.js"
+	@echo "🥫 Refreshing product data cached in Postgres …"
+	${DOCKER_COMPOSE} run --rm backend perl /opt/product-opener/scripts/refresh_postgres.pl ${from}
 
 import_sample_data:
 	@echo "🥫 Importing sample data (~200 products) into MongoDB …"

conf/apache.conf

@@ -14,6 +14,7 @@ PerlPassEnv PRODUCT_OPENER_DOMAIN
 PerlPassEnv PRODUCT_OPENER_PORT
 PerlPassEnv PRODUCERS_PLATFORM
 PerlPassEnv ROBOTOFF_URL
+PerlPassEnv QUERY_URL
 PerlPassEnv EVENTS_URL
 PerlPassEnv FACETS_KP_URL
 PerlPassEnv EVENTS_USERNAME

docker-compose.yml

@@ -14,6 +14,7 @@ x-backend-conf: &backend-conf
     - POSTGRES_USER
     - POSTGRES_PASSWORD
     - ROBOTOFF_URL
+    - QUERY_URL
     - EVENTS_URL
     - FACETS_KP_URL
     - EVENTS_USERNAME

docs/dev/how-to-quick-start-guide.md

@@ -96,6 +96,7 @@ The `.env` file contains ProductOpener default settings:
 | `PRODUCT_OPENER_FLAVOR_SHORT`                                     | can be modified to run different flavors of OpenFoodFacts, amongst `off` (default), `obf`, `oppf`, `opf`.|
 | `PRODUCERS_PLATFORM`                                              | can be set to `1` to build / run the **producer platform**.|
 | `ROBOTOFF_URL`                                                    | can be set to **connect with a Robotoff instance**.|
+| `QUERY_URL`                                                       | can be set to **connect with a Query instance**.|
 | `REDIS_URL` | can be set to **connect with a Redis instance for populating the search index**.|
 | `GOOGLE_CLOUD_VISION_API_KEY`                                     | can be set to **enable OCR using Google Cloud Vision**.|
 | `CROWDIN_PROJECT_IDENTIFIER` and `CROWDIN_PROJECT_KEY`            | can be set to **run translations**.|

lib/ProductOpener/Config2_docker.pm

@@ -47,6 +47,7 @@ BEGIN {
 		$crowdin_project_identifier
 		$crowdin_project_key
 		$robotoff_url
+		$query_url
 		$events_url
 		$facets_kp_url
 		$events_username
@@ -105,6 +106,10 @@ $log_emails = $ENV{OFF_LOG_EMAILS} // 0;
 # enable an in-site robotoff-asker in the product page
 $robotoff_url = $ENV{ROBOTOFF_URL};
 
+# Set this to your instance of https://github.com/openfoodfacts/openfoodfacts-query/ to
+# enable product counts and aggregations / facets
+$query_url = $ENV{QUERY_URL};
+
 # Set this to your instance of https://github.com/openfoodfacts/openfoodfacts-events
 # enable creating events for some actions (e.g. when a product is edited)
 $events_url = $ENV{EVENTS_URL};

lib/ProductOpener/Config2_sample.pm

@@ -39,6 +39,7 @@ BEGIN {
 		$crowdin_project_identifier
 		$crowdin_project_key
 		$robotoff_url
+		$query_url
 		$events_url
 		$events_username
 		$events_password
@@ -75,6 +76,7 @@ $crowdin_project_key = '';
 # Set this to your instance of https://github.com/openfoodfacts/robotoff/ to
 # enable an in-site robotoff-asker in the product page
 $robotoff_url = '';
+$query_url = '';
 
 # Set this to your instance of https://github.com/openfoodfacts/openfoodfacts-events
 # enable creating events for some actions (e.g. when a product is edited)

lib/ProductOpener/Config_obf.pm

@@ -49,6 +49,7 @@ BEGIN {
 		$crowdin_project_key
 
 		$robotoff_url
+		$query_url
 		$events_url
 		$events_username
 		$events_password
@@ -206,6 +207,7 @@ $crowdin_project_key = $ProductOpener::Config2::crowdin_project_key;
 # Set this to your instance of https://github.com/openfoodfacts/robotoff/ to
 # enable an in-site robotoff-asker in the product page
 $robotoff_url = $ProductOpener::Config2::robotoff_url;
+$query_url = $ProductOpener::Config2::query_url;
 
 # Set this to your instance of https://github.com/openfoodfacts/openfoodfacts-events
 # enable creating events for some actions (e.g. when a product is edited)

lib/ProductOpener/Config_off.pm

@@ -49,6 +49,7 @@ BEGIN {
 
 		$log_emails
 		$robotoff_url
+		$query_url
 		$events_url
 		$events_username
 		$events_password
@@ -354,6 +355,7 @@ $crowdin_project_key = $ProductOpener::Config2::crowdin_project_key;
 # Set this to your instance of https://github.com/openfoodfacts/robotoff/ to
 # enable an in-site robotoff-asker in the product page
 $robotoff_url = $ProductOpener::Config2::robotoff_url;
+$query_url = $ProductOpener::Config2::query_url;
 
 # do we want to send emails
 $log_emails = $ProductOpener::Config2::log_emails;
@@ -461,7 +463,6 @@ my $manifest = {
 };
 $options{manifest} = $manifest;
 
-$options{mongodb_supports_sample} = 0;    # from MongoDB 3.2 onward
 $options{display_random_sample_of_products_after_edits} = 0;    # from MongoDB 3.2 onward
 
 $options{favicons} = <<HTML
@@ -702,6 +703,7 @@ $options{replace_existing_values_when_importing_those_tags_fields} = {
 );
 
 # fields for drilldown facet navigation
+# If adding to this list ensure that the tables are being replicated to Postgres in the openfoodfacts-query repo
 
 @drilldown_fields = qw(
 	nutrition_grades

lib/ProductOpener/Config_opf.pm

@@ -49,6 +49,7 @@ BEGIN {
 		$crowdin_project_key
 
 		$robotoff_url
+		$query_url
 		$events_url
 		$events_username
 		$events_password
@@ -204,6 +205,7 @@ $crowdin_project_key = $ProductOpener::Config2::crowdin_project_key;
 # Set this to your instance of https://github.com/openfoodfacts/robotoff/ to
 # enable an in-site robotoff-asker in the product page
 $robotoff_url = $ProductOpener::Config2::robotoff_url;
+$query_url = $ProductOpener::Config2::query_url;
 
 # Set this to your instance of https://github.com/openfoodfacts/openfoodfacts-events
 # enable creating events for some actions (e.g. when a product is edited)

lib/ProductOpener/Config_opff.pm

@@ -49,6 +49,7 @@ BEGIN {
 		$crowdin_project_key
 
 		$robotoff_url
+		$query_url
 		$events_url
 		$events_username
 		$events_password
@@ -203,6 +204,7 @@ $crowdin_project_key = $ProductOpener::Config2::crowdin_project_key;
 # Set this to your instance of https://github.com/openfoodfacts/robotoff/ to
 # enable an in-site robotoff-asker in the product page
 $robotoff_url = $ProductOpener::Config2::robotoff_url;
+$query_url = $ProductOpener::Config2::query_url;
 
 # Set this to your instance of https://github.com/openfoodfacts/openfoodfacts-events
 # enable creating events for some actions (e.g. when a product is edited)

lib/ProductOpener/Data.pm

@@ -27,19 +27,10 @@ ProductOpener::Data - methods to create or get the mongoDB client and fetch "dat
 The module implements the methods required to fetch certain collections from the MongoDB database.
 The functions used in this module are responsible for executing queries, to get connection to database and also to select the collection required.
 
-The module exposes 2 distinct kinds of collections, products and products_tags, returned by the Data::get_products_collections
-and the Data::get_products_tags_collections methods respectively.
-
 The products collection contains a complete document for each product in the OpenFoodFacts database which exposes all
 available information about the product.
 
-The products_tags collection contains a stripped down version of the data in the products collection, where each
-product entry has a select few fields, including fields used in tags. The main purpose of having this copy is to
-improve performance of aggregate queries for an improved user experience and more efficient resource usage. This
-collection was initially proposed in L<issue#1610|https://github.com/openfoodfacts/openfoodfacts-server/issues/1610> on
-GitHub, where some additional context is available.
-
-Obsolete products that have been withdrawn from the market have separate collections: products_obsolete and products_obsolete_tags
+Obsolete products that have been withdrawn from the market have a separate collection: products_obsolete
 
 =cut
 
@@ -52,6 +43,8 @@ BEGIN {
 	use vars qw(@ISA @EXPORT_OK %EXPORT_TAGS);
 	@EXPORT_OK = qw(
 		&execute_query
+		&execute_aggregate_tags_query
+		&execute_count_tags_query
 		&get_database
 		&get_collection
 		&get_products_collection
@@ -71,6 +64,7 @@ use ProductOpener::Config qw/:all/;
 
 use MongoDB;
 use Tie::IxHash;
+use JSON::PP;
 use Log::Any qw($log);
 
 use Action::CircuitBreaker;
@@ -116,6 +110,48 @@ sub execute_query ($sub) {
 	)->run();
 }
 
+sub execute_aggregate_tags_query ($aggregate_parameters) {
+	return execute_tags_query('aggregate', $aggregate_parameters);
+}
+
+sub execute_count_tags_query ($query_ref) {
+	return execute_tags_query('count', $query_ref);
+}
+
+sub execute_tags_query ($type, $parameters) {
+	if ((defined $query_url) and (length($query_url) > 0)) {
+		$query_url =~ s/^\s+|\s+$//g;
+		my $path = "$query_url/$type";
+		$log->debug('Executing PostgreSQL ' . $type . ' query on ' . $path, {query => $parameters})
+			if $log->is_debug();
+
+		my $ua = LWP::UserAgent->new();
+		my $resp = $ua->post(
+			$path,
+			Content => encode_json($parameters),
+			'Content-Type' => 'application/json; charset=utf-8'
+		);
+		if ($resp->is_success) {
+			return decode_json($resp->decoded_content);
+		}
+		else {
+			$log->warn(
+				"query response not ok",
+				{
+					code => $resp->code,
+					status_line => $resp->status_line,
+					response => $resp
+				}
+			) if $log->is_warn();
+			return;
+		}
+	}
+	else {
+		$log->debug('QUERY_URL not defined') if $log->is_debug();
+		return;
+	}
+}
+
 =head2 get_products_collection( $parameters_ref )
 
 C<get_products_collection()> establishes a connection to MongoDB and uses timeout as an argument. This then selects a collection
@@ -139,16 +175,6 @@ This is useful when moving products to another flavour
 If set to a true value, the function returns a collection that contains only obsolete products,
 otherwise it returns the collection with products that are not obsolete.
 
-=head4 tags
-
-If set to a true value, the function may return a smaller collection that contains only the *_tags fields,
-in order to speed aggregate queries. The smaller collection is created every night,
-and may therefore contain slightly stale data.
-
-As of 2023/03/13, we return the products_tags collection for non obsolete products.
-For obsolete products, we currently return the products_obsolete collection, but we might
-create a separate products_obsolete_tags collection in the future, if it becomes necessary to create one.
-
 =head3 Return values
 
 Returns a mongoDB collection object.
@@ -161,11 +187,6 @@ sub get_products_collection ($parameters_ref = {}) {
 	if ($parameters_ref->{obsolete}) {
 		$collection .= '_obsolete';
 	}
-	# We don't have a products_obsolete_tags collection at this point
-	# if it changes, the following elsif should be changed to a if
-	elsif ($parameters_ref->{tags}) {
-		$collection .= '_tags';
-	}
 	return get_collection($database, $collection, $parameters_ref->{timeout});
 }