From 7bda9cc7993762f3458df9652fdcd9a3af4e7929 Mon Sep 17 00:00:00 2001 From: Deveshi Dwivedi <120312681+deveshidwivedi@users.noreply.github.com> Date: Tue, 2 Apr 2024 20:32:41 +0530 Subject: [PATCH 1/6] update to add obsolete field --- scripts/export_database.pl | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/scripts/export_database.pl b/scripts/export_database.pl index d6597448eee1c..2c4c6644c6212 100755 --- a/scripts/export_database.pl +++ b/scripts/export_database.pl @@ -265,6 +265,10 @@ sub sanitize_field_content { $csv .= "${nid}_100g" . "\t"; } + if (grep { $_ eq 'products_obsolete' } @collections) { + $csv .= "obsolete\t"; + } + $csv =~ s/\t$/\n/; print $OUT $csv; @@ -274,6 +278,7 @@ sub sanitize_field_content { get_products_collection({obsolete => 1, timeout => 3 * 60 * 60 * 1000}) ); + push @export_fields, "obsolete"; my $count = 0; my %ingredients = (); @@ -298,6 +303,12 @@ sub sanitize_field_content { my $url = "http://world-$lc.$server_domain" . product_url($product_ref); my $code = ($product_ref->{code} // ''); + if ($product_ref->{obsolete}) { + if ($exported_products{$code}) { + next; + } + } + $code eq '' and next; $code < 1 and next; @@ -339,6 +350,15 @@ sub sanitize_field_content { $field_value = sanitize_field_content($field_value, $BAD, "$code barcode -> field $field:"); } + #Obsolete field + if ($field eq "obsolete") { + $field_value = $product_ref->{obsolete} ? 1 : 0; + } + + else { + $field_value = ($product_ref->{$field} // ""); + } + # Add field value to CSV file $csv .= $field_value . "\t"; @@ -509,6 +529,7 @@ sub sanitize_field_content { print $OUT $csv; print $RDF $rdf; + $exported_products{$code} = 1; } } From c67f0b8b6826f388a1b2d762c69922466296ffc4 Mon Sep 17 00:00:00 2001 From: Deveshi Dwivedi <120312681+deveshidwivedi@users.noreply.github.com> Date: Fri, 5 Apr 2024 02:25:23 +0530 Subject: [PATCH 2/6] add obsolete to export_fields --- lib/ProductOpener/Config_opff.pm | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/ProductOpener/Config_opff.pm b/lib/ProductOpener/Config_opff.pm index 42a032988d50b..45091bfe4493d 100644 --- a/lib/ProductOpener/Config_opff.pm +++ b/lib/ProductOpener/Config_opff.pm @@ -355,6 +355,7 @@ XML pnns_groups_1 pnns_groups_2 states + obsolete ); # for ingredients OCR, we use tesseract-ocr From 4ec90d8abf128c23412b80b2338347854d17403c Mon Sep 17 00:00:00 2001 From: Deveshi Dwivedi <120312681+deveshidwivedi@users.noreply.github.com> Date: Fri, 5 Apr 2024 02:26:03 +0530 Subject: [PATCH 3/6] add obsolete to export_fields --- lib/ProductOpener/Config_obf.pm | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/ProductOpener/Config_obf.pm b/lib/ProductOpener/Config_obf.pm index 4d058f7a46777..0b07dd530658b 100644 --- a/lib/ProductOpener/Config_obf.pm +++ b/lib/ProductOpener/Config_obf.pm @@ -318,6 +318,7 @@ HTML ingredients_that_may_be_from_palm_oil_n ingredients_that_may_be_from_palm_oil states + obsolete ); # for ingredients OCR, we use tesseract-ocr From 941d3b6fe044a9de5b9eb95cf0c4225442ede7c4 Mon Sep 17 00:00:00 2001 From: Deveshi Dwivedi <120312681+deveshidwivedi@users.noreply.github.com> Date: Fri, 5 Apr 2024 02:26:44 +0530 Subject: [PATCH 4/6] add obsolete to export_fields --- lib/ProductOpener/Config_off.pm | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/ProductOpener/Config_off.pm b/lib/ProductOpener/Config_off.pm index 05a3fb2cc3863..60040a9c16c7e 100644 --- a/lib/ProductOpener/Config_off.pm +++ b/lib/ProductOpener/Config_off.pm @@ -855,6 +855,7 @@ $options{replace_existing_values_when_importing_those_tags_fields} = { popularity_tags completeness last_image_t + obsolete ); # List of fields that can be imported on the producers platform From fa8aeeba4c08cb52850d08f9c0969cadd0e3e1d9 Mon Sep 17 00:00:00 2001 From: Deveshi Dwivedi <120312681+deveshidwivedi@users.noreply.github.com> Date: Fri, 5 Apr 2024 02:27:07 +0530 Subject: [PATCH 5/6] add obsolete to export_fields --- lib/ProductOpener/Config_opf.pm | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/ProductOpener/Config_opf.pm b/lib/ProductOpener/Config_opf.pm index e2f76af9916e3..d648f46ea1aca 100644 --- a/lib/ProductOpener/Config_opf.pm +++ b/lib/ProductOpener/Config_opf.pm @@ -313,6 +313,7 @@ HTML additives_n additives states + obsolete ); # for ingredients OCR, we use tesseract-ocr From aa13461b946e194eb4080964a52e9b6f4358bd93 Mon Sep 17 00:00:00 2001 From: Deveshi Dwivedi <120312681+deveshidwivedi@users.noreply.github.com> Date: Fri, 5 Apr 2024 02:31:46 +0530 Subject: [PATCH 6/6] update to add obsolete field to CSV correctly --- scripts/export_database.pl | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/scripts/export_database.pl b/scripts/export_database.pl index 2c4c6644c6212..6398b42daacbd 100755 --- a/scripts/export_database.pl +++ b/scripts/export_database.pl @@ -265,10 +265,6 @@ sub sanitize_field_content { $csv .= "${nid}_100g" . "\t"; } - if (grep { $_ eq 'products_obsolete' } @collections) { - $csv .= "obsolete\t"; - } - $csv =~ s/\t$/\n/; print $OUT $csv; @@ -278,9 +274,9 @@ sub sanitize_field_content { get_products_collection({obsolete => 1, timeout => 3 * 60 * 60 * 1000}) ); - push @export_fields, "obsolete"; my $count = 0; my %ingredients = (); + my %exported_products = (); foreach my $collection (@collections) { @@ -355,9 +351,7 @@ sub sanitize_field_content { $field_value = $product_ref->{obsolete} ? 1 : 0; } - else { - $field_value = ($product_ref->{$field} // ""); - } + $field_value //= ""; # Add field value to CSV file $csv .= $field_value . "\t";