Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added method to clean input csv of NBSP unicode chars V2 #17

Merged
merged 3 commits into from
Dec 29, 2023
Merged

Added method to clean input csv of NBSP unicode chars V2 #17

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
74cc476
handle nbsp
jamesgreen-moj Dec 29, 2023
2fe84e0
added tests
jamesgreen-moj Dec 29, 2023
8f7f8c0
Commit changes made by code formatters
github-actions[bot] Dec 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions src/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,15 @@ def convert_csv_to_json(file_path):
csv_rows = []
with open(file_path) as csvfile:
reader = csv.DictReader(csvfile, skipinitialspace=True, quotechar='"')
title = reader.fieldnames

# Here we clean any Non-breaking spaces and convert to normal spacing. \xa0 converts to ' '
for row in reader:
csv_rows.extend([{title[i]: row[title[i]]
for i in range(len(title))}])
temp_overwritable_dict = {}
for key, value in row.items():
cleaned_key = key.replace('\xa0', ' ')
cleaned_value = value.replace('\xa0', ' ')
temp_overwritable_dict.update({cleaned_key: cleaned_value})
csv_rows.append(temp_overwritable_dict)

if csv_rows == None or csv_rows == []:
raise ValueError('Failed to convert CSV file to JSON. Exiting script.')
Expand Down
59 changes: 50 additions & 9 deletions test/test_main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,18 @@ class TestCsvToJson(unittest.TestCase):
def setUp(self):
# Create a temporary CSV file for testing
self.csv_data = [
{'Site Name': 'Test location 1', 'Site Address': '40 Mayflower Dr, Plymouth PL2 3DG', 'Enable GovWifi': ' "TRUE"',
'Enable MoJWifi': ' "FALSE"', 'GovWifi Radius Key': '00000DD0000BC0EEE000', 'Wired NACS Radius Key': '00000DD0000BC0EEE000'},
{'Site Name': 'Test location 2', 'Site Address': '102 Petty France, London SW1H 9AJ', 'Enable GovWifi': ' "TRUE"',
'Enable MoJWifi': ' "FALSE"', 'GovWifi Radius Key': '0D0E0DDE000BC0EEE000', 'Wired NACS Radius Key': '00000DD0000BC0EEE000'},
{'Site Name': 'Test location 3', 'Site Address': 'Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB', 'Enable GovWifi': ' "TRUE"',
'Enable MoJWifi': ' "FALSE"', 'GovWifi Radius Key': '0D0E0DDE080BC0EEE000', 'Wired NACS Radius Key': '00000DD0000BC0EEE000'}
{'Site Name': 'Test location 1', 'Site Address': '40 Mayflower Dr, Plymouth PL2 3DG',
'Enable GovWifi': ' "TRUE"',
'Enable MoJWifi': ' "FALSE"', 'GovWifi Radius Key': '00000DD0000BC0EEE000',
'Wired NACS Radius Key': '00000DD0000BC0EEE000'},
{'Site Name': 'Test location 2', 'Site Address': '102 Petty France, London SW1H 9AJ',
'Enable GovWifi': ' "TRUE"',
'Enable MoJWifi': ' "FALSE"', 'GovWifi Radius Key': '0D0E0DDE000BC0EEE000',
'Wired NACS Radius Key': '00000DD0000BC0EEE000'},
{'Site Name': 'Test location 3', 'Site Address': 'Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB',
'Enable GovWifi': ' "TRUE"',
'Enable MoJWifi': ' "FALSE"', 'GovWifi Radius Key': '0D0E0DDE080BC0EEE000',
'Wired NACS Radius Key': '00000DD0000BC0EEE000'}
]
self.csv_file = tempfile.NamedTemporaryFile(
mode='w', delete=False, newline='', suffix='.csv')
Expand Down Expand Up @@ -51,6 +57,39 @@ def test_given_file_path_when_csv_file_not_found_then_raise_FileNotFoundError(se
with self.assertRaises(FileNotFoundError):
convert_csv_to_json(nonexistent_file_path)

def test_given_csv_while_when_csv_file_contains_nbsp_then_convert_to_normal_spacing(self):
csv_data = [
{'Site\xa0Name': 'Test location 1', 'Site Address': '40\xa0Mayflower\xa0Dr, Plymouth PL2 3DG',
'Enable GovWifi': ' "TRUE"',
'Enable\xa0MoJWifi': ' "FALSE"', 'GovWifi\xa0Radius Key': '00000DD0000BC0EEE000',
'Wired\xa0NACS Radius Key': '00000DD0000BC0EEE000'},
{'Site\xa0Name': 'Test location 2', 'Site Address': '102\xa0Petty\xa0France, London SW1H 9AJ',
'Enable GovWifi': ' "TRUE"',
'Enable\xa0MoJWifi': ' "FALSE"', 'GovWifi\xa0Radius Key': '0D0E0DDE000BC0EEE000',
'Wired\xa0NACS Radius Key': '00000DD0000BC0EEE000'},
{'Site\xa0Name': 'Test location 3', 'Site Address': 'Met\xa0Office, FitzRoy Road,\xa0Exeter, Devon, EX1 3PB',
'Enable GovWifi': ' "TRUE"',
'Enable\xa0MoJWifi': ' "FALSE"', 'GovWifi\xa0Radius Key': '0D0E0DDE080BC0EEE000',
'Wired\xa0NACS Radius Key': '00000DD0000BC0EEE000'}
]
csv_file = tempfile.NamedTemporaryFile(
mode='w', delete=False, newline='', suffix='.csv')
csv_writer = csv.DictWriter(csv_file, fieldnames=[
'Site\xa0Name',
'Site Address',
'Enable GovWifi',
'Enable\xa0MoJWifi',
'GovWifi\xa0Radius Key',
'Wired\xa0NACS Radius Key'
])
csv_writer.writeheader()
csv_writer.writerows(csv_data)
csv_file.close()

expected_json = self.csv_data
actual_json = convert_csv_to_json(csv_file.name)
self.assertEqual(actual_json, expected_json)


class TestAddGeocodingToJson(unittest.TestCase):

Expand All @@ -72,16 +111,18 @@ def test_given_site_name_and_site_address_in_json_format_when_function_called_th
{'Site Name': 'Site1', 'Site Address': '40 Mayflower Dr, Plymouth PL2 3DG'},
{'Site Name': 'Site2', 'Site Address': '102 Petty France, London SW1H 9AJ'},
{'Site Name': 'Site3',
'Site Address': 'Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB'}
'Site Address': 'Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB'}
]

expected_data = [
{'Site Name': 'Site1', 'Site Address': '40 Mayflower Dr, Plymouth PL2 3DG', 'gps': {
'latitude': 50.3868633, 'longitude': -4.1539256}, 'country_code': 'GB', 'time_zone': 'Europe/London'},
{'Site Name': 'Site2', 'Site Address': '102 Petty France, London SW1H 9AJ', 'gps': {
'latitude': 51.499929300000005, 'longitude': -0.13477761285315926}, 'country_code': 'GB', 'time_zone': 'Europe/London'},
'latitude': 51.499929300000005, 'longitude': -0.13477761285315926}, 'country_code': 'GB',
'time_zone': 'Europe/London'},
{'Site Name': 'Site3', 'Site Address': 'Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB', 'gps': {
'latitude': 50.727350349999995, 'longitude': -3.4744726127760086}, 'country_code': 'GB', 'time_zone': 'Europe/London'}
'latitude': 50.727350349999995, 'longitude': -3.4744726127760086}, 'country_code': 'GB',
'time_zone': 'Europe/London'}
]

actual_data = add_geocoding_to_json(data)
Expand Down