feat: Use openfoodfacts-query service for facet queries instead of pr…

…oduct_tags collection (#8947)

* Call local postgres service instead of mongo

* Initial count support

* Removing unused code

* Separate queries so easier to mock results

* Make QUERY_URL configurable

* Removing references to products_tags

* Note and tweaks

* Perl tidy fixes

* Get from query first then fallback

* Fix merge issue

* Update port

* Tidy up comment

* Fix comment block

* Remove commented code that prevented caching

* Removed $@ reference as not being used

* Move estimate count into a separate function

* Check fixes

* extra_hosts for Linux

* Allow Produuct Opener to continue to use product_tags during transition

* Perltidy fixes

* Prevent errors when query_urll not set on refresh_product_tags

* Set query_url for staging

* Make sure product_tags is not used when no_cache=1

.env

@@ -45,6 +45,9 @@ MONGODB_CACHE_SIZE=8 # GB
 MONGO_INITDB_ROOT_USERNAME=root
 MONGO_INITDB_ROOT_PASSWORD=test
 ROBOTOFF_URL=http://robotoff.openfoodfacts.localhost:5500 # connect to Robotoff running in separate docker-compose deployment
+# connect to openfoodfacts-query running in separate docker-compose deployment.
+# To test locally change to http://host.docker.internal:5510
+QUERY_URL=http://query:5510
 EVENTS_URL=
 FACETS_KP_URL = https://facets-kp.openfoodfacts.org/render-to-html
 # use this to push products to openfoodfacts-search

.github/workflows/container-deploy.yml

@@ -29,6 +29,7 @@ jobs:
         echo "REDIS_URL=searchredis:6379" >> $GITHUB_ENV
         echo "MONGODB_HOST=10.1.0.200" >> $GITHUB_ENV
         echo "ROBOTOFF_URL=https://robotoff.openfoodfacts.net" >> $GITHUB_ENV
+        echo "QUERY_URL=http://10.1.0.200:5511" >> $GITHUB_ENV
         echo "PRODUCT_OPENER_DOMAIN=openfoodfacts.net" >> $GITHUB_ENV
         echo "PRODUCT_OPENER_FLAVOR=openfoodfacts" >> $GITHUB_ENV
         echo "PRODUCT_OPENER_FLAVOR_SHORT=off" >> $GITHUB_ENV
@@ -144,6 +145,7 @@ jobs:
           echo "POSTGRES_USER=${{ env.POSTGRES_USER }}" >> .env
           echo "POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}" >> .env
           echo "ROBOTOFF_URL=${{ env.ROBOTOFF_URL }}" >> .env
+          echo "QUERY_URL=${{ env.QUERY_URL }}" >> .env
           echo "REDIS_URL=${{ env.REDIS_URL }}" >> .env
           echo "FACETS_KP_URL=${{ env.FACETS_KP_URL }}" >> .env
           echo "GOOGLE_CLOUD_VISION_API_KEY=${{ secrets.GOOGLE_CLOUD_VISION_API_KEY }}" >> .env

Makefile

@@ -201,6 +201,8 @@ refresh_product_tags:
 # get id for mongodb container
 	docker cp scripts/refresh_products_tags.js $(shell docker-compose ps -q mongodb):/data/db
 	${DOCKER_COMPOSE} exec -T mongodb //bin/sh -c "mongo off /data/db/refresh_products_tags.js"
+	@echo "🥫 Refreshing product data cached in Postgres …"
+	${DOCKER_COMPOSE} run --rm backend perl /opt/product-opener/scripts/refresh_postgres.pl ${from}
 
 import_sample_data:
 	@echo "🥫 Importing sample data (~200 products) into MongoDB …"

conf/apache.conf

@@ -14,6 +14,7 @@ PerlPassEnv PRODUCT_OPENER_DOMAIN
 PerlPassEnv PRODUCT_OPENER_PORT
 PerlPassEnv PRODUCERS_PLATFORM
 PerlPassEnv ROBOTOFF_URL
+PerlPassEnv QUERY_URL
 PerlPassEnv EVENTS_URL
 PerlPassEnv FACETS_KP_URL
 PerlPassEnv EVENTS_USERNAME

docker-compose.yml

@@ -14,6 +14,7 @@ x-backend-conf: &backend-conf
     - POSTGRES_USER
     - POSTGRES_PASSWORD
     - ROBOTOFF_URL
+    - QUERY_URL
     - EVENTS_URL
     - FACETS_KP_URL
     - EVENTS_USERNAME

docker/dev.yml

@@ -20,6 +20,11 @@ x-backend-conf: &backend-conf
     - ./docker/docker-entrypoint.sh:/docker-entrypoint.sh
     # we use this for debugging, from times to times
     - ./debug:/mnt/podata/debug/
+  # Allow the container to connect to the host when using Linux.
+  # This is needed to access other services that are running outside of Docker
+  # e.g. when developing with Robotoff or openfoodfacts-query
+  extra_hosts:
+    - "host.docker.internal:host-gateway"
 
 x-minion-db-network: &minion-db-network
   networks:

docs/dev/how-to-quick-start-guide.md

@@ -96,6 +96,7 @@ The `.env` file contains ProductOpener default settings:
 | `PRODUCT_OPENER_FLAVOR_SHORT`                                     | can be modified to run different flavors of OpenFoodFacts, amongst `off` (default), `obf`, `oppf`, `opf`.|
 | `PRODUCERS_PLATFORM`                                              | can be set to `1` to build / run the **producer platform**.|
 | `ROBOTOFF_URL`                                                    | can be set to **connect with a Robotoff instance**.|
+| `QUERY_URL`                                                       | can be set to **connect with a Query instance**.|
 | `REDIS_URL` | can be set to **connect with a Redis instance for populating the search index**.|
 | `GOOGLE_CLOUD_VISION_API_KEY`                                     | can be set to **enable OCR using Google Cloud Vision**.|
 | `CROWDIN_PROJECT_IDENTIFIER` and `CROWDIN_PROJECT_KEY`            | can be set to **run translations**.|

lib/ProductOpener/Config2_docker.pm

@@ -47,6 +47,7 @@ BEGIN {
 		$crowdin_project_identifier
 		$crowdin_project_key
 		$robotoff_url
+		$query_url
 		$events_url
 		$facets_kp_url
 		$events_username
@@ -105,6 +106,10 @@ $log_emails = $ENV{OFF_LOG_EMAILS} // 0;
 # enable an in-site robotoff-asker in the product page
 $robotoff_url = $ENV{ROBOTOFF_URL};
 
+# Set this to your instance of https://github.com/openfoodfacts/openfoodfacts-query/ to
+# enable product counts and aggregations / facets
+$query_url = $ENV{QUERY_URL};
+
 # Set this to your instance of https://github.com/openfoodfacts/openfoodfacts-events
 # enable creating events for some actions (e.g. when a product is edited)
 $events_url = $ENV{EVENTS_URL};

lib/ProductOpener/Config2_sample.pm

@@ -39,6 +39,7 @@ BEGIN {
 		$crowdin_project_identifier
 		$crowdin_project_key
 		$robotoff_url
+		$query_url
 		$events_url
 		$events_username
 		$events_password
@@ -75,6 +76,7 @@ $crowdin_project_key = '';
 # Set this to your instance of https://github.com/openfoodfacts/robotoff/ to
 # enable an in-site robotoff-asker in the product page
 $robotoff_url = '';
+$query_url = '';
 
 # Set this to your instance of https://github.com/openfoodfacts/openfoodfacts-events
 # enable creating events for some actions (e.g. when a product is edited)

lib/ProductOpener/Config_obf.pm

@@ -49,6 +49,7 @@ BEGIN {
 		$crowdin_project_key
 
 		$robotoff_url
+		$query_url
 		$events_url
 		$events_username
 		$events_password
@@ -206,6 +207,7 @@ $crowdin_project_key = $ProductOpener::Config2::crowdin_project_key;
 # Set this to your instance of https://github.com/openfoodfacts/robotoff/ to
 # enable an in-site robotoff-asker in the product page
 $robotoff_url = $ProductOpener::Config2::robotoff_url;
+$query_url = $ProductOpener::Config2::query_url;
 
 # Set this to your instance of https://github.com/openfoodfacts/openfoodfacts-events
 # enable creating events for some actions (e.g. when a product is edited)

lib/ProductOpener/Config_off.pm

@@ -49,6 +49,7 @@ BEGIN {
 
 		$log_emails
 		$robotoff_url
+		$query_url
 		$events_url
 		$events_username
 		$events_password
@@ -354,6 +355,7 @@ $crowdin_project_key = $ProductOpener::Config2::crowdin_project_key;
 # Set this to your instance of https://github.com/openfoodfacts/robotoff/ to
 # enable an in-site robotoff-asker in the product page
 $robotoff_url = $ProductOpener::Config2::robotoff_url;
+$query_url = $ProductOpener::Config2::query_url;
 
 # do we want to send emails
 $log_emails = $ProductOpener::Config2::log_emails;
@@ -461,7 +463,6 @@ my $manifest = {
 };
 $options{manifest} = $manifest;
 
-$options{mongodb_supports_sample} = 0;    # from MongoDB 3.2 onward
 $options{display_random_sample_of_products_after_edits} = 0;    # from MongoDB 3.2 onward
 
 $options{favicons} = <<HTML
@@ -702,6 +703,7 @@ $options{replace_existing_values_when_importing_those_tags_fields} = {
 );
 
 # fields for drilldown facet navigation
+# If adding to this list ensure that the tables are being replicated to Postgres in the openfoodfacts-query repo
 
 @drilldown_fields = qw(
 	nutrition_grades

lib/ProductOpener/Config_opf.pm

@@ -49,6 +49,7 @@ BEGIN {
 		$crowdin_project_key
 
 		$robotoff_url
+		$query_url
 		$events_url
 		$events_username
 		$events_password
@@ -204,6 +205,7 @@ $crowdin_project_key = $ProductOpener::Config2::crowdin_project_key;
 # Set this to your instance of https://github.com/openfoodfacts/robotoff/ to
 # enable an in-site robotoff-asker in the product page
 $robotoff_url = $ProductOpener::Config2::robotoff_url;
+$query_url = $ProductOpener::Config2::query_url;
 
 # Set this to your instance of https://github.com/openfoodfacts/openfoodfacts-events
 # enable creating events for some actions (e.g. when a product is edited)

lib/ProductOpener/Config_opff.pm

@@ -49,6 +49,7 @@ BEGIN {
 		$crowdin_project_key
 
 		$robotoff_url
+		$query_url
 		$events_url
 		$events_username
 		$events_password
@@ -203,6 +204,7 @@ $crowdin_project_key = $ProductOpener::Config2::crowdin_project_key;
 # Set this to your instance of https://github.com/openfoodfacts/robotoff/ to
 # enable an in-site robotoff-asker in the product page
 $robotoff_url = $ProductOpener::Config2::robotoff_url;
+$query_url = $ProductOpener::Config2::query_url;
 
 # Set this to your instance of https://github.com/openfoodfacts/openfoodfacts-events
 # enable creating events for some actions (e.g. when a product is edited)

lib/ProductOpener/Data.pm

@@ -52,6 +52,8 @@ BEGIN {
 	use vars qw(@ISA @EXPORT_OK %EXPORT_TAGS);
 	@EXPORT_OK = qw(
 		&execute_query
+		&execute_aggregate_tags_query
+		&execute_count_tags_query
 		&get_database
 		&get_collection
 		&get_products_collection
@@ -71,6 +73,7 @@ use ProductOpener::Config qw/:all/;
 
 use MongoDB;
 use Tie::IxHash;
+use JSON::PP;
 use Log::Any qw($log);
 
 use Action::CircuitBreaker;
@@ -116,6 +119,48 @@ sub execute_query ($sub) {
 	)->run();
 }
 
+sub execute_aggregate_tags_query ($aggregate_parameters) {
+	return execute_tags_query('aggregate', $aggregate_parameters);
+}
+
+sub execute_count_tags_query ($query_ref) {
+	return execute_tags_query('count', $query_ref);
+}
+
+sub execute_tags_query ($type, $parameters) {
+	if ((defined $query_url) and (length($query_url) > 0)) {
+		$query_url =~ s/^\s+|\s+$//g;
+		my $path = "$query_url/$type";
+		$log->debug('Executing PostgreSQL ' . $type . ' query on ' . $path, {query => $parameters})
+			if $log->is_debug();
+
+		my $ua = LWP::UserAgent->new();
+		my $resp = $ua->post(
+			$path,
+			Content => encode_json($parameters),
+			'Content-Type' => 'application/json; charset=utf-8'
+		);
+		if ($resp->is_success) {
+			return decode_json($resp->decoded_content);
+		}
+		else {
+			$log->warn(
+				"query response not ok",
+				{
+					code => $resp->code,
+					status_line => $resp->status_line,
+					response => $resp
+				}
+			) if $log->is_warn();
+			return;
+		}
+	}
+	else {
+		$log->debug('QUERY_URL not defined') if $log->is_debug();
+		return;
+	}
+}
+
 =head2 get_products_collection( $parameters_ref )
 
 C<get_products_collection()> establishes a connection to MongoDB and uses timeout as an argument. This then selects a collection