fix: reduce number of products from 100 to 50 on lists of products (#…

…10374)
openfoodfacts · May 31, 2024 · cb09ab5 · cb09ab5
1 parent 9a19a18
commit cb09ab5
Show file tree

Hide file tree

Showing 43 changed files with 199 additions and 114 deletions.
diff --git a/cgi/recent_changes.pl b/cgi/recent_changes.pl
@@ -46,9 +46,9 @@
 
 my $query_ref = {};
 
-my $limit = 0 + (single_param('page_size') || $page_size);
-if (($limit < 2) or ($limit > 1000)) {
-	$limit = $page_size;
+my $limit = 0 + (single_param('page_size') || $options{default_recent_changes_page_size});
+if ($limit > $options{max_recent_changes_page_size}) {
+	$limit = $options{max_recent_changes_page_size};
 }
 
 my $page = 0 + (single_param('page') || 1);

diff --git a/cgi/search.pl b/cgi/search.pl
@@ -227,9 +227,9 @@
 	$sort_by = 'unique_scans_n';
 }
 
-my $limit = 0 + (single_param('page_size') || $page_size);
-if (($limit < 2) or ($limit > 1000)) {
-	$limit = $page_size;
+my $limit = 0 + (single_param('page_size') || $options{default_web_products_page_size});
+if ($limit > $options{max_products_page_size}) {
+	$limit = $options{max_products_page_size};
 }
 
 my $graph_ref = {graph_title => remove_tags_and_quote(decode utf8 => single_param("graph_title"))};

diff --git a/lib/ProductOpener/Config_obf.pm b/lib/ProductOpener/Config_obf.pm
@@ -78,8 +78,6 @@ BEGIN {
 		$display_size
 		$zoom_size
 
-		$page_size
-
 		%options
 		%server_options
 
@@ -186,6 +184,21 @@ $flavor = 'obf';
 	twitter_account => "OpenBeautyFacts",
 );
 
+$options{export_limit} = 10000;
+
+# Recent changes limits
+$options{default_recent_changes_page_size} = 20;
+$options{max_recent_changes_page_size} = 1000;
+
+# List of products limits
+$options{default_api_products_page_size} = 20;
+$options{default_web_products_page_size} = 50;
+$options{max_products_page_size} = 100;
+
+# List of tags limits
+$options{default_tags_page_size} = 100;
+$options{max_tags_page_size} = 1000;
+
 @edit_rules = ();
 
 # server constants
@@ -242,8 +255,6 @@ $small_size = 200;
 $display_size = 400;
 $zoom_size = 800;
 
-$page_size = 20;
-
 $google_analytics = <<HTML
 HTML
 	;

diff --git a/lib/ProductOpener/Config_off.pm b/lib/ProductOpener/Config_off.pm
@@ -78,8 +78,6 @@ BEGIN {
 		$display_size
 		$zoom_size
 
-		$page_size
-
 		%options
 		%server_options
 
@@ -207,6 +205,19 @@ $flavor = 'off';
 
 $options{export_limit} = 10000;
 
+# Recent changes limits
+$options{default_recent_changes_page_size} = 20;
+$options{max_recent_changes_page_size} = 1000;
+
+# List of products limits
+$options{default_api_products_page_size} = 20;
+$options{default_web_products_page_size} = 50;
+$options{max_products_page_size} = 100;
+
+# List of tags limits
+$options{default_tags_page_size} = 100;
+$options{max_tags_page_size} = 1000;
+
 $options{users_who_can_upload_small_images} = {
 	map {$_ => 1}
 		qw(
@@ -457,8 +468,6 @@ $small_size = 200;
 $display_size = 400;
 $zoom_size = 800;
 
-$page_size = 24;
-
 $google_analytics = <<HTML
 <!-- Matomo -->
 <script>

diff --git a/lib/ProductOpener/Config_opf.pm b/lib/ProductOpener/Config_opf.pm
@@ -77,8 +77,6 @@ BEGIN {
 		$display_size
 		$zoom_size
 
-		$page_size
-
 		%options
 		%server_options
 
@@ -174,7 +172,31 @@ $flavor = "opf";
 	teolemon
 );
 
-$options{product_type} = "product";
+%options = (
+	site_name => "Open Products Facts",
+	product_type => "products",
+	og_image_url => "https://world.openproductsfacts.org/images/misc/openproductsfacts-logo-en.png",
+	#android_apk_app_link => "https://world.openbeautyfacts.org/images/apps/obf.apk",
+	#android_app_link => "https://play.google.com/store/apps/details?id=org.openbeautyfacts.scanner",
+	#ios_app_link => "https://apps.apple.com/app/open-beauty-facts/id1122926380",
+	#facebook_page_url => "https://www.facebook.com/openbeautyfacts",
+	#twitter_account => "OpenBeautyFacts",
+);
+
+$options{export_limit} = 10000;
+
+# Recent changes limits
+$options{default_recent_changes_page_size} = 20;
+$options{max_recent_changes_page_size} = 1000;
+
+# List of products limits
+$options{default_api_products_page_size} = 20;
+$options{default_web_products_page_size} = 50;
+$options{max_products_page_size} = 100;
+
+# List of tags limits
+$options{default_tags_page_size} = 100;
+$options{max_tags_page_size} = 1000;
 
 @edit_rules = ();
 
@@ -232,8 +254,6 @@ $small_size = 200;
 $display_size = 400;
 $zoom_size = 800;
 
-$page_size = 20;
-
 $google_analytics = <<HTML
 HTML
 	;

diff --git a/lib/ProductOpener/Config_opff.pm b/lib/ProductOpener/Config_opff.pm
@@ -77,8 +77,6 @@ BEGIN {
 		$display_size
 		$zoom_size
 
-		$page_size
-
 		%options
 		%server_options
 
@@ -174,7 +172,31 @@ $flavor = "opff";
 	teolemon
 );
 
-$options{product_type} = "petfood";
+%options = (
+	site_name => "Open Pet Food Facts",
+	product_type => "petfood",
+	og_image_url => "https://world.openpetfoodfacts.org/images/misc/openpetfoodfacts-logo-en.png",
+	#android_apk_app_link => "https://world.openbeautyfacts.org/images/apps/obf.apk",
+	#android_app_link => "https://play.google.com/store/apps/details?id=org.openbeautyfacts.scanner",
+	#ios_app_link => "https://apps.apple.com/app/open-beauty-facts/id1122926380",
+	#facebook_page_url => "https://www.facebook.com/openbeautyfacts",
+	#twitter_account => "OpenBeautyFacts",
+);
+
+$options{export_limit} = 10000;
+
+# Recent changes limits
+$options{default_recent_changes_page_size} = 20;
+$options{max_recent_changes_page_size} = 1000;
+
+# List of products limits
+$options{default_api_products_page_size} = 20;
+$options{default_web_products_page_size} = 50;
+$options{max_products_page_size} = 100;
+
+# List of tags limits
+$options{default_tags_page_size} = 100;
+$options{max_tags_page_size} = 1000;
 
 @edit_rules = ();
 
@@ -232,8 +254,6 @@ $small_size = 200;
 $display_size = 400;
 $zoom_size = 800;
 
-$page_size = 20;
-
 $google_analytics = <<HTML
 HTML
 	;

diff --git a/lib/ProductOpener/Display.pm b/lib/ProductOpener/Display.pm
@@ -287,9 +287,6 @@ foreach my $file (sort keys %file_timestamps) {
 # On demand exports can be very big, limit the number of products
 my $export_limit = 10000;
 
-# TODO: explain why such a high number
-my $tags_page_size = 10000;
-
 if (defined $options{export_limit}) {
 	$export_limit = $options{export_limit};
 }
@@ -1674,13 +1671,18 @@ sub query_list_of_tags ($request_ref, $query_ref) {
 	{
 		$limit = 999999999999;
 	}
-	elsif (defined $request_ref->{tags_page_size}) {
-		$limit = $request_ref->{tags_page_size};
+	elsif (defined $request_ref->{page_size}) {
+		$limit = $request_ref->{page_size};
+		if ($limit > $options{max_tags_page_size}) {
+			$limit = $options{max_tags_page_size};
+		}
 	}
 	else {
-		$limit = $tags_page_size;
+		$limit = $options{default_tags_page_size};
 	}
 
+	$request_ref->{page_size} = $limit;
+
 	my $skip = 0;
 	if (defined $page) {
 		$skip = ($page - 1) * $limit;
@@ -2299,16 +2301,20 @@ sub display_list_of_tags ($request_ref, $query_ref) {
 		}
 
 		$html .= "</tbody></table></div>";
-		# if there are more than $tags_page_size lines, add pagination. Except for ?stats=1 and ?filter display
+		# if there are more lines than the limit, add pagination. Except for ?stats=1 and ?filter display
 		$log->info("PAGINATION: BEFORE\n");
-		if (    $request_ref->{structured_response}{count} >= $tags_page_size
+		if (    $request_ref->{structured_response}{count} >= $request_ref->{page_size}
 			and not(defined single_param("stats"))
 			and not(defined single_param("filter")))
 		{
 			$log->info("PAGINATION: CALLING\n");
 			$html .= "\n<hr>"
-				. display_pagination($request_ref, $request_ref->{structured_response}{count},
-				$tags_page_size, $request_ref->{page});
+				. display_pagination(
+				$request_ref,
+				$request_ref->{structured_response}{count},
+				$request_ref->{page_size},
+				$request_ref->{page}
+				);
 		}
 
 		if ((defined single_param("stats")) and (single_param("stats"))) {
@@ -4437,7 +4443,7 @@ sub display_search_results ($request_ref) {
 		my $search_api_url = $formatted_subdomain . "/api/v0" . $current_link;
 		$search_api_url =~ s/(\&|\?)(page|page_size|limit)=(\d+)//;
 		$search_api_url .= "&fields=code,product_display_name,url,image_front_small_url,attribute_groups";
-		$search_api_url .= "&page_size=100";
+		$search_api_url .= "&page_size=" . $options{default_web_products_page_size};
 		if ($search_api_url !~ /\?/) {
 			$search_api_url =~ s/\&/\?/;
 		}
@@ -4971,14 +4977,21 @@ sub search_and_display_products ($request_ref, $query_ref, $sort_by, $limit, $pa
 	elsif (defined $request_ref->{page_size}) {
 		$limit = $request_ref->{page_size};
 	}
-	# If user preferences are turned on, return 100 products per page
+	# If user preferences are turned on, return 50 products per page
 	elsif (not defined $request_ref->{api}) {
-		$limit = 100;
+		$limit = $options{default_web_products_page_size};
 	}
 	else {
-		$limit = $page_size;
+		$limit = $options{default_api_products_page_size};
 	}
 
+	# Make sure we are not over the limit
+	if ($limit > $options{max_products_page_size}) {
+		$limit = $options{max_products_page_size};
+	}
+
+	$request_ref->{page_size} = $limit;
+
 	my $skip = 0;
 	if (defined $page) {
 		$skip = ($page - 1) * $limit;
@@ -10726,9 +10739,15 @@ sub display_recent_changes ($request_ref, $query_ref, $limit, $page) {
 		$limit = $request_ref->{page_size};
 	}
 	else {
-		$limit = $page_size;
+		$limit = $options{default_recent_changes_page_size};
 	}
 
+	if ($limit > $options{max_recent_changes_page_size}) {
+		$limit = $options{max_recent_changes_page_size};
+	}
+
+	$request_ref->{page_size} = $limit;
+
 	my $skip = 0;
 	if (defined $page) {
 		$skip = ($page - 1) * $limit;

diff --git a/lib/ProductOpener/Routing.pm b/lib/ProductOpener/Routing.pm
@@ -644,16 +644,22 @@ sub set_rate_limit_attributes ($request_ref, $ip) {
 		my $block_message = "Rate-limiter blocking: the user has reached the rate-limit";
 		# Check if rate-limit blocking is enabled
 		if ($rate_limiter_blocking_enabled) {
-			# Check that the IP address is not in the allow list
-			if (not defined $options{rate_limit_allow_list}{$ip}) {
-				# The user has reached the rate-limit, we block the request
-				$request_ref->{rate_limiter_blocking} = 1;
+			# Check that the ip is not local (e.g. integration tests)
+			if ($ip eq "127.0.0.1") {
+				# The IP address is local, we don't block the request
+				$block_message
+					= "Rate-limiter blocking is disabled for local IP addresses, but the user has reached the rate-limit";
 			}
-			else {
+			# Check that the IP address is not in the allow list
+			elsif (defined $options{rate_limit_allow_list}{$ip}) {
 				# The IP address is in the allow list, we don't block the request
 				$block_message
 					= "Rate-limiter blocking is disabled for the user, but the user has reached the rate-limit";
 			}
+			else {
+				# The user has reached the rate-limit, we block the request
+				$request_ref->{rate_limiter_blocking} = 1;
+			}
 		}
 		else {
 			# Rate-limit blocking is disabled, we just log a warning

diff --git a/tests/integration/expected_test_results/api_v2_product_read/get-existing-product.json b/tests/integration/expected_test_results/api_v2_product_read/get-existing-product.json
@@ -102,7 +102,7 @@
             "origins_of_ingredients" : {
                "aggregated_origins" : [
                   {
-                     "epi_score" : "0",
+                     "epi_score" : 0,
                      "origin" : "en:unknown",
                      "percent" : 100,
                      "transportation_score" : null

diff --git a/...ntegration/expected_test_results/api_v2_product_read/get-fields-all-knowledge-panels.json b/...ntegration/expected_test_results/api_v2_product_read/get-fields-all-knowledge-panels.json
@@ -102,7 +102,7 @@
             "origins_of_ingredients" : {
                "aggregated_origins" : [
                   {
-                     "epi_score" : "0",
+                     "epi_score" : 0,
                      "origin" : "en:unknown",
                      "percent" : 100,
                      "transportation_score" : null

diff --git a/...ed_test_results/api_v2_product_read/get-fields-attribute-groups-all-knowledge-panels.json b/...ed_test_results/api_v2_product_read/get-fields-attribute-groups-all-knowledge-panels.json
@@ -750,7 +750,7 @@
             "origins_of_ingredients" : {
                "aggregated_origins" : [
                   {
-                     "epi_score" : "0",
+                     "epi_score" : 0,
                      "origin" : "en:unknown",
                      "percent" : 100,
                      "transportation_score" : null

diff --git a/...cted_test_results/api_v2_product_write/get-product-ingredients-text-without-language.json b/...cted_test_results/api_v2_product_write/get-product-ingredients-text-without-language.json
@@ -97,7 +97,7 @@
             "origins_of_ingredients" : {
                "aggregated_origins" : [
                   {
-                     "epi_score" : "0",
+                     "epi_score" : 0,
                      "origin" : "en:unknown",
                      "percent" : 100,
                      "transportation_score" : null

diff --git a/tests/integration/expected_test_results/api_v2_product_write/get-product.json b/tests/integration/expected_test_results/api_v2_product_write/get-product.json
@@ -109,7 +109,7 @@
             "origins_of_ingredients" : {
                "aggregated_origins" : [
                   {
-                     "epi_score" : "0",
+                     "epi_score" : 0,
                      "origin" : "en:unknown",
                      "percent" : 100,
                      "transportation_score" : null

diff --git a/...integration/expected_test_results/api_v3_product_read/get-existing-product-gs1-caret.json b/...integration/expected_test_results/api_v3_product_read/get-existing-product-gs1-caret.json
@@ -104,7 +104,7 @@
             "origins_of_ingredients" : {
                "aggregated_origins" : [
                   {
-                     "epi_score" : "0",
+                     "epi_score" : 0,
                      "origin" : "en:unknown",
                      "percent" : 100,
                      "transportation_score" : null