Merge pull request #2525 from openfoodfacts/data-quality

Data quality warning for products with main language set to unknown + 5 consonants
openfoodfacts · Nov 1, 2019 · cdae4cf · cdae4cf
2 parents a71eb57 + 42823c2
commit cdae4cf
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 2 deletions.
diff --git a/lib/ProductOpener/DataQualityCommon.pm b/lib/ProductOpener/DataQualityCommon.pm
@@ -65,10 +65,37 @@ sub check_bugs($) {
 
 	my $product_ref = shift;
 
+	check_bug_missing_or_unknown_main_language($product_ref);
+
 	check_bug_code_missing($product_ref);
 	check_bug_created_t_missing($product_ref);
 }
 
+=head2 check_bug_missing_or_unknown_main_language( PRODUCT_REF )
+
+Products that do not have the lc or lang field set, or a lang field set to "xx" (unknown)
+
+lc and lang fields should always be set, but there has been some bugs in the past
+that caused them not to be set in certain conditions.
+
+=cut
+
+sub check_bug_missing_or_unknown_main_language($) {
+
+	my $product_ref = shift;
+
+	if ((not (defined $product_ref->{lc}))) {
+		push @{$product_ref->{data_quality_bugs_tags}}, "en:main-language-code-missing";
+	}
+
+	if ((not (defined $product_ref->{lang}))) {
+		push @{$product_ref->{data_quality_bugs_tags}}, "en:main-language-missing";
+	}
+	elsif ($product_ref->{lang} eq 'xx') {
+		push @{$product_ref->{data_quality_bugs_warnings}}, "en:main-language-unknown";
+	}
+}
+
 sub check_bug_code_missing($) {
 
 	my $product_ref = shift;

diff --git a/lib/ProductOpener/DataQualityFood.pm b/lib/ProductOpener/DataQualityFood.pm
@@ -808,9 +808,9 @@ sub check_ingredients($) {
 
 				# Dutch and other languages can have 4 consecutive consonants
 				if ($display_lc !~ /de|nl/) {
-					if ($product_ref->{$ingredients_text_lc} =~ /[bcdfghjklmnpqrstvwxz]{4}/is) {
+					if ($product_ref->{$ingredients_text_lc} =~ /[bcdfghjklmnpqrstvwxz]{5}/is) {
 
-						push @{$product_ref->{data_quality_warnings_tags}}, "en:ingredients-" . $display_lc . "-4-consonants";
+						push @{$product_ref->{data_quality_warnings_tags}}, "en:ingredients-" . $display_lc . "-5-consonants";
 					}
 				}
 

diff --git a/scripts/update_all_products.pl b/scripts/update_all_products.pl
@@ -431,6 +431,15 @@
 			}
 		}
 
+		if (($fix_missing_lc) and (not defined $product_ref->{lang})) {
+			print STDERR "lc: " . $product_ref->{lc} . "\n";
+			if ((defined $product_ref->{lc}) and ($product_ref->{lc} =~ /^[a-z][a-z]$/)) {
+				print STDERR "fixing missing lang, using lc: " . $product_ref->{lc} . "\n";
+				$product_ref->{lang} = $product_ref->{lc};
+				$product_values_changed = 1;
+			}
+		}
+
 		# Fix ingredients_n that was set as string
 		if (defined $product_ref->{ingredients_n}) {
 			$product_ref->{ingredients_n} += 0;