feat: Synonyms in taxonomized suggestions

docs/api/ref/api-v3.yml

@@ -4,8 +4,8 @@ x-stoplight:
 info:
   title: Open Food Facts Open API V3 - under development
   description: |
-    As a developer, the Open Food Facts API allows you to get information 
-    and contribute to the products database. You can create great apps to 
+    As a developer, the Open Food Facts API allows you to get information
+    and contribute to the products database. You can create great apps to
     help people make better food choices and also provide data to enhance the database.
   termsOfService: 'https://openweathermap.org/terms'
   contact:
@@ -58,12 +58,12 @@ paths:
           in: query
           name: fields
           description: |-
-            Comma separated list of fields requested in the response. 
+            Comma separated list of fields requested in the response.
 
-            Special values: 
+            Special values:
             * "none": returns no fields
             * "raw": returns all fields as stored internally in the database
-            * "all": returns all fields except generated fields that need to be explicitly requested such as "knowledge_panels". 
+            * "all": returns all fields except generated fields that need to be explicitly requested such as "knowledge_panels".
 
             Defaults to "all" for READ requests. The "all" value can also be combined with fields like "attribute_groups" and "knowledge_panels".'
       responses:
@@ -247,6 +247,11 @@ paths:
                         description: Array of sorted strings suggestions in the language requested in the "lc" field.
                         items:
                           type: string
+                      matched_synonyms:
+                        type: array
+                        description: Array of strings, which are the synonyms the search best matched to, with indices corresponding with the canonical names in the suggestions field.
+                        items:
+                          type: string
       operationId: get-api-v3-taxonomy_suggestions-taxonomy
       description: |-
         Open Food Facts uses multilingual [taxonomies](https://wiki.openfoodfacts.org/Global_taxonomies) to normalize entries for categories, labels, ingredients, packaging shapes / materials / recycling instructions and many more fields.
@@ -299,6 +304,11 @@ paths:
           in: query
           name: limit
           description: 'Maximum number of suggestions. Default is 25, max is 400.'
+        - schema:
+            type: string
+          in: query
+          name: get_synonyms
+          description: 'Whether or not to include "matched_synonyms" in the response. Set to 1 to include.'
         - schema:
             type: string
           in: query

html/js/product-multilingual.js

@@ -542,14 +542,22 @@ function initializeTagifyInput(el) {
         autocomplete: true,
         whitelist: get_recents(el.id) || [],
         dropdown: {
-            enabled: 0
+            enabled: 0,
+            maxItems: 100
         }
     });
 
     let abortController;
     let debounceTimer;
     const timeoutWait = 300;
 
+    function updateSuggestions() {
+        const value = input.state.inputText;
+        const lc = (/^\w\w:/).exec(value);
+        const term = lc ? value.substring(lc[0].length) : value;
+        input.dropdown.show(term);
+    }
+
     input.on("input", function (event) {
         const value = event.detail.value;
         input.whitelist = null; // reset the whitelist
@@ -565,16 +573,48 @@ function initializeTagifyInput(el) {
 
                 abortController = new AbortController();
 
-                fetch(el.dataset.autocomplete + "&string=" + value, {
+                fetch(el.dataset.autocomplete + "&string=" + value + "&get_synonyms=1", {
                     signal: abortController.signal
                 }).
                     then((RES) => RES.json()).
                     then(function (json) {
-                        input.whitelist = json.suggestions;
-                        input.dropdown.show(value); // render the suggestions dropdown
+                        const lc = (/^\w\w:/).exec(value);
+                        let whitelist;
+                        if (lc) {
+                            whitelist = json.matched_synonyms.map(function (e) {
+                                return {"value": lc + e, "searchBy": e};
+                            });
+                        } else {
+                            whitelist = json.matched_synonyms;
+                        }
+                        const synonymMap = Object.create(null);
+                        for (let i = 0; i < json.suggestions.length; i++) {
+                            synonymMap[json.matched_synonyms[i]] = json.suggestions[i];
+                        }
+                        input.synonymMap = synonymMap;
+                        input.whitelist = whitelist;
+                        updateSuggestions(); // render the suggestions dropdown
                     });
             }, timeoutWait);
         }
+        updateSuggestions();
+    });
+
+    input.on("dropdown:show", function() {
+        if (!input.synonymMap) {
+            return;
+        }
+        $(input.DOM.dropdown).find("div.tagify__dropdown__item").each(function(_,e) {
+            let synonymName = e.getAttribute("value");
+            const lc = (/^\w\w:/).exec(synonymName);
+            if (lc) {
+                synonymName = synonymName.substring(3);
+            }
+            const canonicalName = input.synonymMap[synonymName];
+            if (canonicalName && canonicalName !== synonymName) {
+                e.innerHTML += " (&rarr; <i>" + canonicalName + "</i>)";
+            }
+        });
     });
 
     input.on("add", function (event) {

lib/ProductOpener/APITaxonomySuggestions.pm

@@ -91,7 +91,10 @@ sub taxonomy_suggestions_api ($request_ref) {
 	};
 
 	# Options define how many suggestions should be returned, in which format etc.
-	my $options_ref = {limit => request_param($request_ref, 'limit')};
+	my $options_ref = {
+		limit => request_param($request_ref, 'limit'),
+		get_synonyms => request_param($request_ref, 'get_synonyms')
+	};
 
 	# Validate input parameters
 
@@ -123,8 +126,11 @@ sub taxonomy_suggestions_api ($request_ref) {
 	# Generate suggestions
 	else {
 
-		$response_ref->{suggestions}
-			= [get_taxonomy_suggestions($tagtype, $search_lc, $string, $context_ref, $options_ref)];
+		my @suggestions = get_taxonomy_suggestions($tagtype, $search_lc, $string, $context_ref, $options_ref);
+		$log->debug("taxonomy_suggestions_api", @suggestions) if $log->is_debug();
+		$response_ref->{suggestions} = [map {$_->{tag}} @suggestions];
+		$response_ref->{matched_synonyms} = [map {ucfirst($_->{matched_synonym})} @suggestions]
+			if $options_ref->{get_synonyms};
 	}
 
 	$log->debug("taxonomy_suggestions_api - stop", {request => $request_ref}) if $log->is_debug();

lib/ProductOpener/TaxonomySuggestions.pm

@@ -121,14 +121,16 @@ sub get_taxonomy_suggestions ($tagtype, $search_lc, $string, $context_ref, $opti
 	) if $log->is_debug();
 
 	# Check if we have cached suggestions
+	my $options_relavant = {%$options_ref};
+	delete $options_relavant->{get_synonyms};
 	my $key = generate_cache_key(
 		"get_taxonomy_suggestions",
 		{
 			tagtype => $tagtype,
 			search_lc => $search_lc,
 			string => $string,
 			context_ref => $context_ref,
-			options_ref => $options_ref
+			options_ref => $options_relavant
 		}
 	);
 
@@ -312,23 +314,30 @@ sub match_stringids ($stringid, $fuzzystringid, $synonymid) {
 
 # best_match is used to see how well matches the best matching synonym
 
-sub best_match ($stringid, $fuzzystringid, $synonyms_ids_ref) {
+sub best_match ($search_lc, $stringid, $fuzzystringid, $synonyms_ref) {
 
-	my $best_match = "none";
+	my $best_type = "none";
+	my $best_match = 0;
 
-	foreach my $synonymid (@$synonyms_ids_ref) {
+	foreach my $synonym (@$synonyms_ref) {
+		my $synonymid = get_string_id_for_lang($search_lc, $synonym);
 		my $match = match_stringids($stringid, $fuzzystringid, $synonymid);
+		# Prefer to use the earlier ones from the list for when the canonical name has the same match type as a synonym
+		next if $match eq "none" or $match eq $best_type;
 		if ($match eq "start") {
 			# Best match, we can return without looking at the other synonyms
-			return "start";
+			$best_type = $match;
+			$best_match = $synonym;
+			last;
 		}
 		elsif (($match eq "inside")
-			or (($match eq "fuzzy") and ($best_match eq "none")))
+			or (($match eq "fuzzy") and ($best_type eq "none")))
 		{
-			$best_match = $match;
+			$best_type = $match;
+			$best_match = $synonym;
 		}
 	}
-	return $best_match;
+	return {type => $best_type, match => $best_match};
 }
 
 =head2 filter_suggestions_matching_string ($tags_ref, $tagtype, $search_lc, $string, $options_ref)
@@ -365,6 +374,8 @@ sub filter_suggestions_matching_string ($tags_ref, $tagtype, $search_lc, $string
 	my $limit = $options_ref->{limit} || 25;
 	# Set a hard limit of 400
 	$limit = min(int($limit), 400);
+	# Whether or not to get synonyms
+	my $get_synonyms = $options_ref->{get_synonyms} || 0;
 
 	$log->debug(
 		"filter_suggestions_matching_string",
@@ -374,7 +385,8 @@ sub filter_suggestions_matching_string ($tags_ref, $tagtype, $search_lc, $string
 			search_lc => $search_lc,
 			string => $string,
 			options_ref => $options_ref,
-			limit => $limit
+			limit => $limit,
+			get_synonyms => $get_synonyms
 		}
 	) if $log->is_debug();
 
@@ -424,39 +436,43 @@ sub filter_suggestions_matching_string ($tags_ref, $tagtype, $search_lc, $string
 			my $tag_xx = display_taxonomy_tag("xx", $tagtype, $canon_tagid);
 
 			# Build a list of normalized synonyms in the search language and the wildcard xx: language
-			my @synonyms_ids = map {get_string_id_for_lang($search_lc, $_)} (
+			my @synonyms = (
 				@{deep_get(\%synonyms_for, $tagtype, $search_lc, get_string_id_for_lang($search_lc, $tag)) || []},
 				@{deep_get(\%synonyms_for, $tagtype, "xx", get_string_id_for_lang("xx", $tag_xx)) || []}
 			);
 
 			# check how well the synonyms match the input string
-			my $best_match = best_match($stringid, $fuzzystringid, \@synonyms_ids);
+			my $best_match = best_match($search_lc, $stringid, $fuzzystringid, \@synonyms);
 
 			$log->debug(
-				"synonyms_ids for canon_tagid",
+				"synonyms for canon_tagid",
 				{
 					tagtype => $tagtype,
 					canon_tagid => $canon_tagid,
 					tag => $tag,
-					synonym_ids => \@synonyms_ids,
+					synonyms => \@synonyms,
 					best_match => $best_match
 				}
 			) if $log->is_debug();
 
+			my $to_add = {
+				tag => $tag,
+				matched_synonym => $best_match->{match}
+			};
 			# matching at start, best matches
-			if ($best_match eq "start") {
-				push @suggestions, $tag;
+			if ($best_match->{type} eq "start") {
+				push @suggestions, $to_add;
 				# count matches at start so that we can return only if we have enough matches
 				$suggestions_count++;
 				last if $suggestions_count >= $limit;
 			}
 			# matching inside
-			elsif ($best_match eq "inside") {
-				push @suggestions_c, $tag;
+			elsif ($best_match->{type} eq "inside") {
+				push @suggestions_c, $to_add;
 			}
 			# fuzzy match
-			elsif ($best_match eq "fuzzy") {
-				push @suggestions_f, $tag;
+			elsif ($best_match->{type} eq "fuzzy") {
+				push @suggestions_f, $to_add;
 			}
 		}
 	}

tests/integration/api_v3_taxonomy_suggestions.t

@@ -77,6 +77,12 @@ my $tests_ref = [
 		path => '/api/v3/taxonomy_suggestions?tagtype=categories&string=Café&lc=fr',
 		expected_status_code => 200,
 	},
+	{
+		test_case => 'allergens-string-fr-o-get-synonyms',
+		method => 'GET',
+		path => '/api/v3/taxonomy_suggestions?tagtype=allergens&string=o&lc=fr&get_synonyms=1',
+		expected_status_code => 200,
+	},
 	# Packaging suggestions return most popular suggestions first
 	{
 		test_case => 'packaging-shapes',