diff --git a/scraper/src/tests/typesense_helper/commit_tmp_test.py b/scraper/src/tests/typesense_helper/commit_tmp_test.py index cb53117..9061dbd 100644 --- a/scraper/src/tests/typesense_helper/commit_tmp_test.py +++ b/scraper/src/tests/typesense_helper/commit_tmp_test.py @@ -307,6 +307,15 @@ def test_transform_record(): 'lvl5': None, 'lvl6': None, }, + 'hierarchy_radio': { + 'lvl0': None, + 'lvl1': None, + 'lvl2': 'Electronics', + 'lvl3': None, + 'lvl4': None, + 'lvl5': None, + 'lvl6': None, + }, 'version': '1.0,2.0', 'extra_field': None, } @@ -317,6 +326,7 @@ def test_transform_record(): 'hierarchy.lvl0': 'Home', 'hierarchy.lvl1': 'Products', 'hierarchy.lvl2': 'Electronics', + 'hierarchy_radio.lvl2': 'Electronics', 'version': ['1.0', '2.0'], 'hierarchy': { 'lvl0': 'Home', @@ -327,6 +337,15 @@ def test_transform_record(): 'lvl5': None, 'lvl6': None, }, + 'hierarchy_radio': { + 'lvl0': None, + 'lvl1': None, + 'lvl2': 'Electronics', + 'lvl3': None, + 'lvl4': None, + 'lvl5': None, + 'lvl6': None, + }, } # Act @@ -354,6 +373,15 @@ def test_add_records(typesense_client): 'lvl5': None, 'lvl6': None, }, + 'hierarchy_radio': { + 'lvl0': None, + 'lvl1': None, + 'lvl2': 'Electronics', + 'lvl3': None, + 'lvl4': None, + 'lvl5': None, + 'lvl6': None, + }, 'version': '1.0,2.0', 'anchor': 'See More', 'content': 'Explore our range of electronics', @@ -374,6 +402,15 @@ def test_add_records(typesense_client): 'lvl5': None, 'lvl6': None, }, + 'hierarchy_radio': { + 'lvl0': 'About Us', + 'lvl1': None, + 'lvl2': None, + 'lvl3': None, + 'lvl4': None, + 'lvl5': None, + 'lvl6': None, + }, 'version': '1.0', 'anchor': 'Learn More', 'content': 'Discover more about our company values', @@ -406,7 +443,17 @@ def test_add_records(typesense_client): 'lvl5': None, 'lvl6': None, }, + 'hierarchy_radio': { + 'lvl0': 'About Us', + 'lvl1': None, + 'lvl2': None, + 'lvl3': None, + 'lvl4': None, + 'lvl5': None, + 'lvl6': None, + }, 'hierarchy.lvl0': 'About Us', + 'hierarchy_radio.lvl0': 'About Us', 'id': '1', 'item_priority': 2000001001, 'language': 'en', @@ -429,9 +476,19 @@ def test_add_records(typesense_client): 'lvl5': None, 'lvl6': None, }, + 'hierarchy_radio': { + 'lvl0': None, + 'lvl1': None, + 'lvl2': 'Electronics', + 'lvl3': None, + 'lvl4': None, + 'lvl5': None, + 'lvl6': None, + }, 'hierarchy.lvl0': 'Home', 'hierarchy.lvl1': 'Products', 'hierarchy.lvl2': 'Electronics', + 'hierarchy_radio.lvl2': 'Electronics', 'id': '0', 'item_priority': 1000002003, 'language': 'en', @@ -473,6 +530,15 @@ def test_commit_tmp_collection(typesense_client): 'lvl5': None, 'lvl6': None, }, + 'hierarchy_radio': { + 'lvl0': None, + 'lvl1': None, + 'lvl2': 'Electronics', + 'lvl3': None, + 'lvl4': None, + 'lvl5': None, + 'lvl6': None, + }, 'version': '1.0,2.0', 'anchor': 'See More', 'content': 'Explore our range of electronics', @@ -493,6 +559,15 @@ def test_commit_tmp_collection(typesense_client): 'lvl5': None, 'lvl6': None, }, + 'hierarchy_radio': { + 'lvl0': 'About Us', + 'lvl1': None, + 'lvl2': None, + 'lvl3': None, + 'lvl4': None, + 'lvl5': None, + 'lvl6': None, + }, 'version': '1.0', 'anchor': 'Learn More', 'content': 'Discover more about our company values', @@ -568,6 +643,15 @@ def test_commit_tmp_collection_with_curation_rules(typesense_client): 'lvl5': None, 'lvl6': None, }, + 'hierarchy_radio': { + 'lvl0': None, + 'lvl1': None, + 'lvl2': 'Electronics', + 'lvl3': None, + 'lvl4': None, + 'lvl5': None, + 'lvl6': None, + }, 'version': '1.0,2.0', 'anchor': 'See More', 'content': 'Explore our range of electronics', @@ -588,6 +672,15 @@ def test_commit_tmp_collection_with_curation_rules(typesense_client): 'lvl5': None, 'lvl6': None, }, + 'hierarchy_radio': { + 'lvl0': 'About Us', + 'lvl1': None, + 'lvl2': None, + 'lvl3': None, + 'lvl4': None, + 'lvl5': None, + 'lvl6': None, + }, 'version': '1.0', 'anchor': 'Learn More', 'content': 'Discover more about our company values', diff --git a/scraper/src/typesense_helper.py b/scraper/src/typesense_helper.py index 9a2d1e2..c29bf87 100644 --- a/scraper/src/typesense_helper.py +++ b/scraper/src/typesense_helper.py @@ -207,11 +207,12 @@ def transform_record(record): + transformed_record['weight']['position_descending'] ) - # Flatten nested hierarchy field + # Flatten nested hierarchy fields for x in range(0, 7): - if record['hierarchy'][f'lvl{x}'] is None: - continue - transformed_record[f'hierarchy.lvl{x}'] = record['hierarchy'][f'lvl{x}'] + if record['hierarchy'][f'lvl{x}'] is not None: + transformed_record[f'hierarchy.lvl{x}'] = record['hierarchy'][f'lvl{x}'] + if record['hierarchy_radio'][f'lvl{x}'] is not None: + transformed_record[f'hierarchy_radio.lvl{x}'] = record['hierarchy_radio'][f'lvl{x}'] # Convert version to array if 'version' in record and type(record['version']) == str: