diff --git a/cloudnetpy/instruments/__init__.py b/cloudnetpy/instruments/__init__.py index 0bc871e1..5f8baf9b 100644 --- a/cloudnetpy/instruments/__init__.py +++ b/cloudnetpy/instruments/__init__.py @@ -9,5 +9,6 @@ from .mrr import mrr2nc from .pollyxt import pollyxt2nc from .radiometrics import radiometrics2nc +from .rain_e_h3 import rain_e_h32nc from .rpg import rpg2nc from .weather_station import ws2nc diff --git a/cloudnetpy/instruments/cloudnet_instrument.py b/cloudnetpy/instruments/cloudnet_instrument.py index 3e915bfe..17dba8b8 100644 --- a/cloudnetpy/instruments/cloudnet_instrument.py +++ b/cloudnetpy/instruments/cloudnet_instrument.py @@ -112,3 +112,34 @@ def _get_zenith_angle(self) -> float | None: if np.isnan(zenith_angle) or zenith_angle is ma.masked: return None return zenith_angle + + +class CSVFile(CloudnetInstrument): + def __init__(self, site_meta: dict): + super().__init__() + self.site_meta = site_meta + self._data: dict = {} + + def add_date(self) -> None: + dt = self._data["time"][0] + self.date = dt.strftime("%Y %m %d").split() + + def add_data(self) -> None: + for key, value in self._data.items(): + parsed = ( + utils.datetime2decimal_hours(value) + if key == "time" + else ma.array(value) + ) + self.data[key] = CloudnetArray(parsed, key) + + def normalize_rainfall_amount(self) -> None: + if "rainfall_amount" in self.data: + amount = self.data["rainfall_amount"][:] + offset = 0 + for i in range(1, len(amount)): + if amount[i] + offset < amount[i - 1]: + offset += amount[i - 1] + amount[i] += offset + amount -= amount[0] + self.data["rainfall_amount"].data = amount diff --git a/cloudnetpy/instruments/rain_e_h3.py b/cloudnetpy/instruments/rain_e_h3.py new file mode 100644 index 00000000..ac1b73b7 --- /dev/null +++ b/cloudnetpy/instruments/rain_e_h3.py @@ -0,0 +1,166 @@ +import csv +import datetime +from os import PathLike + +import numpy as np + +from cloudnetpy import output +from cloudnetpy.exceptions import ValidTimeStampError +from cloudnetpy.instruments import instruments +from cloudnetpy.instruments.cloudnet_instrument import CSVFile + + +def rain_e_h32nc( + input_file: str | PathLike, + output_file: str, + site_meta: dict, + uuid: str | None = None, + date: str | datetime.date | None = None, +): + """Converts rain_e_h3 rain-gauge into Cloudnet Level 1b netCDF file. + + Args: + input_file: Filename of rain_e_h3 CSV file. + output_file: Output filename. + site_meta: Dictionary containing information about the site. Required key + is `name`. + uuid: Set specific UUID for the file. + date: Expected date of the measurements as YYYY-MM-DD or datetime.date object. + + Returns: + UUID of the generated file. + + Raises: + WeatherStationDataError : Unable to read the file. + ValidTimeStampError: No valid timestamps found. + """ + rain = RainEH3(site_meta) + if isinstance(date, str): + date = datetime.date.fromisoformat(date) + rain.parse_input_file(input_file, date) + rain.add_data() + rain.add_date() + rain.convert_units() + rain.normalize_rainfall_amount() + rain.add_site_geolocation() + attributes = output.add_time_attribute({}, rain.date) + output.update_attributes(rain.data, attributes) + return output.save_level1b(rain, output_file, uuid) + + +class RainEH3(CSVFile): + def __init__(self, site_meta: dict): + super().__init__(site_meta) + self.instrument = instruments.RAIN_E_H3 + self._data = { + "time": [], + "rainfall_rate": [], + "rainfall_amount": [], + } + + def parse_input_file( + self, filepath: str | PathLike, date: datetime.date | None = None + ) -> None: + with open(filepath, encoding="latin1") as f: + data = list(csv.reader(f, delimiter=";")) + n_values = np.median([len(row) for row in data]).astype(int) + + if n_values == 22: + self._read_talker_protocol_22_columns(data, date) + elif n_values == 16: + self._read_talker_protocol_16_columns(data, date) + else: + msg = "Only talker protocol with 16 or 22 columns is supported." + raise NotImplementedError(msg) + + def _read_talker_protocol_16_columns( + self, data: list, date: datetime.date | None = None + ) -> None: + """Old Lindenberg data format. + + 0 date DD.MM.YYYY + 1 time + 2 precipitation intensity in mm/h + 3 precipitation accumulation in mm + 4 housing contact + 5 top temperature + 6 bottom temperature + 7 heater status + 8 error code + 9 system status + 10 talker interval in seconds + 11 operating hours + 12 device type + 13 user data storage 1 + 14 user data storage 2 + 15 user data storage 3 + + """ + for row in data: + if len(row) != 16: + continue + try: + dt = datetime.datetime.strptime( + f"{row[0]} {row[1]}", "%d.%m.%Y %H:%M:%S" + ) + except ValueError: + continue + if date and date != dt.date(): + continue + self._data["time"].append(dt) + self._data["rainfall_rate"].append(float(row[2])) + self._data["rainfall_amount"].append(float(row[3])) + if not self._data["time"]: + raise ValidTimeStampError + + def _read_talker_protocol_22_columns( + self, data: list, date: datetime.date | None = None + ) -> None: + """Columns according to header in Lindenberg data. + + 0 datetime utc + 1 date + 2 time + 3 precipitation intensity in mm/h + 4 precipitation accumulation in mm + 5 housing contact + 6 top temperature + 7 bottom temperature + 8 heater status + 9 error code + 10 system status + 11 talker interval in seconds + 12 operating hours + 13 device type + 14 user data storage 1 + 15 user data storage 2 + 16 user data storage 3 + 17 user data storage 4 + 18 serial number + 19 hardware version + 20 firmware version + 21 external temperature * checksum + + """ + for row in data: + if len(row) != 22: + continue + try: + dt = datetime.datetime.strptime(f"{row[0]}", "%Y-%m-%d %H:%M:%S") + except ValueError: + continue + if date and date != dt.date(): + continue + self._data["time"].append(dt) + self._data["rainfall_rate"].append(float(row[3])) + self._data["rainfall_amount"].append(float(row[4])) + self.serial_number = row[18] + if not self._data["time"]: + raise ValidTimeStampError + + def convert_units(self) -> None: + rainfall_rate = self.data["rainfall_rate"][:] + self.data["rainfall_rate"].data = rainfall_rate / 3600 / 1000 # mm/h -> m/s + self.data["rainfall_amount"].data = ( + self.data["rainfall_amount"][:] / 1000 + ) # mm -> m diff --git a/cloudnetpy/instruments/weather_station.py b/cloudnetpy/instruments/weather_station.py index a729297c..dabccea4 100644 --- a/cloudnetpy/instruments/weather_station.py +++ b/cloudnetpy/instruments/weather_station.py @@ -12,7 +12,7 @@ from cloudnetpy.constants import HPA_TO_PA, MM_H_TO_M_S, SEC_IN_HOUR from cloudnetpy.exceptions import ValidTimeStampError, WeatherStationDataError from cloudnetpy.instruments import instruments -from cloudnetpy.instruments.cloudnet_instrument import CloudnetInstrument +from cloudnetpy.instruments.cloudnet_instrument import CSVFile from cloudnetpy.instruments.toa5 import read_toa5 from cloudnetpy.utils import datetime2decimal_hours @@ -79,28 +79,13 @@ def ws2nc( return output.save_level1b(ws, output_file, uuid) -class WS(CloudnetInstrument): +class WS(CSVFile): def __init__(self, site_meta: dict): - super().__init__() - self._data: dict - self.site_meta = site_meta + super().__init__(site_meta) self.instrument = instruments.GENERIC_WEATHER_STATION date: list[str] - def add_date(self) -> None: - first_date = self._data["time"][0].date() - self.date = [ - str(first_date.year), - str(first_date.month).zfill(2), - str(first_date.day).zfill(2), - ] - - def add_data(self) -> None: - for key, value in self._data.items(): - parsed = datetime2decimal_hours(value) if key == "time" else ma.array(value) - self.data[key] = CloudnetArray(parsed, key) - def calculate_rainfall_amount(self) -> None: if "rainfall_amount" in self.data: return @@ -137,17 +122,6 @@ def convert_rainfall_rate(self) -> None: def convert_pressure(self) -> None: self.data["air_pressure"].data = self.data["air_pressure"][:] * HPA_TO_PA - def normalize_rainfall_amount(self) -> None: - if "rainfall_amount" in self.data: - amount = self.data["rainfall_amount"][:] - offset = 0 - for i in range(1, len(amount)): - if amount[i] + offset < amount[i - 1]: - offset += amount[i - 1] - amount[i] += offset - amount -= amount[0] - self.data["rainfall_amount"].data = amount - def convert_time(self) -> None: pass diff --git a/tests/unit/data/rain_e_h3/20241231_raine_lindenberg.csv b/tests/unit/data/rain_e_h3/20241231_raine_lindenberg.csv new file mode 100644 index 00000000..7bf65e9d --- /dev/null +++ b/tests/unit/data/rain_e_h3/20241231_raine_lindenberg.csv @@ -0,0 +1,23 @@ +datetime_utc;date;time;precipitation intensity [mm/h];precipitation accumulation [mm];housing contact;top temperature [°C];bottom temperature [°C];heater status;error code;system status;talker interval [s];operating hours;device type;user data1;user data2;user data3;user data4;serial number;hardware version;firmware version;external temperature [°C];checksum +2024-12-31 00:00:35;te:2024.12.31;00:01:03;0.000;514.761;0;5.00;5.00;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.50*9A +2024-12-31 00:01:35;te:2024.12.31;00:02:03;0.000;514.761;0;5.06;5.00;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.50*93 +2024-12-31 00:02:35;te:2024.12.31;00:03:03;0.000;514.761;0;5.06;5.00;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.44*8F +2024-12-31 00:03:35;te:2024.12.31;00:04:03;0.000;514.761;0;5.19;5.00;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.44*8A +2024-12-31 00:04:35;te:2024.12.31;00:05:03;0.000;514.761;0;5.19;5.00;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.44*89 +2024-12-31 00:05:35;te:2024.12.31;00:06:03;0.000;514.761;0;5.19;5.06;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.44*82 +2024-12-31 00:06:35;te:2024.12.31;00:07:03;0.000;514.761;0;5.12;5.06;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.44*88 +2024-12-31 00:07:35;te:2024.12.31;00:08:03;0.000;514.761;0;5.12;5.00;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.44*8D +2024-12-31 00:08:35;te:2024.12.31;00:09:03;0.000;514.761;0;5.19;5.06;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.44*7F +2024-12-31 00:09:35;te:2024.12.31;00:10:03;0.000;514.761;0;5.19;5.12;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.44*8A +2024-12-31 00:10:35;te:2024.12.31;00:11:03;0.000;514.761;0;5.12;5.19;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.44*89 +2024-12-31 00:11:35;te:2024.12.31;00:12:03;0.000;514.761;0;5.06;5.12;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.50*8F +2024-12-31 00:12:35;te:2024.12.31;00:13:03;0.000;514.761;0;5.06;5.12;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.44*8B +2024-12-31 00:13:35;te:2024.12.31;00:14:03;0.000;514.761;0;4.94;5.06;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.44*81 +2024-12-31 00:14:35;te:2024.12.31;00:15:03;0.000;514.761;0;4.94;5.00;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.44*86 +2024-12-31 00:15:35;te:2024.12.31;00:16:03;0.000;514.761;0;5.12;5.12;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.50*8E +2024-12-31 00:16:35;te:2024.12.31;00:17:03;0.000;514.761;0;5.00;5.12;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.50*90 +2024-12-31 00:17:35;te:2024.12.31;00:18:03;0.000;514.761;0;4.87;5.06;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.50*7E +2024-12-31 00:18:35;te:2024.12.31;00:19:03;0.000;514.761;0;4.81;4.94;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.50*7D +2024-12-31 00:19:35;te:2024.12.31;00:20:03;0.000;514.761;0;4.87;4.81;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.50*83 +2024-12-31 00:20:35;te:2024.12.31;00:21:03;0.000;514.761;0;5.00;4.81;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.50*90 +2024-12-31 00:21:35;te:2024.12.31;00:22:03;0.000;514.761;0;5.06;4.81;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.50*89 diff --git a/tests/unit/data/rain_e_h3/Lindenberg_RainE_20230514.txt b/tests/unit/data/rain_e_h3/Lindenberg_RainE_20230514.txt new file mode 100644 index 00000000..c97c70fa --- /dev/null +++ b/tests/unit/data/rain_e_h3/Lindenberg_RainE_20230514.txt @@ -0,0 +1,18 @@ +14.05.2023;00:00:39;0.000;0.406;0;9.56;11.50;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS +14.05.2023;00:01:39;0.000;0.406;0;9.50;11.44;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS +14.05.2023;00:02:39;0.000;0.406;0;9.44;11.44;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS +14.05.2023;00:03:39;0.000;0.406;0;9.37;11.37;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS +14.05.2023;00:04:39;0.000;0.406;0;9.31;11.31;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS +14.05.2023;00:05:39;0.000;0.406;0;9.25;11.31;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS +14.05.2023;00:06:39;0.000;0.406;0;9.19;11.25;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS +14.05.2023;00:07:39;0.000;0.406;0;9.12;11.19;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS +14.05.2023;00:08:39;0.000;0.406;0;9.12;11.19;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS +14.05.2023;00:09:39;0.000;0.406;0;9.06;11.12;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS +14.05.2023;00:10:39;0.000;0.406;0;9.00;11.06;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS +14.05.2023;00:11:39;0.000;0.406;0;9.06;11.06;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS +14.05.2023;00:12:39;0.000;0.406;0;9.12;11.12;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS +14.05.2023;00:13:39;0.000;0.406;0;9.06;11.12;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS +14.05.2023;00:14:39;0.000;0.406;0;9.06;11.06;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS +14.05.2023;00:15:39;0.000;0.406;0;9.06;11.06;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS +14.05.2023;00:16:39;0.000;0.406;0;9.00;11.00;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS +14.05.2023;00:17:39;0.000;0.406;0;8.94;10.94;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS diff --git a/tests/unit/test_rain_e_h3.py b/tests/unit/test_rain_e_h3.py new file mode 100644 index 00000000..9a2f2b19 --- /dev/null +++ b/tests/unit/test_rain_e_h3.py @@ -0,0 +1,80 @@ +import os +from tempfile import TemporaryDirectory + +from cloudnetpy.cloudnetarray import CloudnetArray +import pytest + +from cloudnetpy.exceptions import ValidTimeStampError +from cloudnetpy.instruments import rain_e_h32nc +from tests.unit.all_products_fun import Check +import numpy as np +from numpy import ma + +SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__)) + +SITE_META = { + "name": "Palaiseau", + "latitude": 50, + "longitude": 104.5, + "altitude": 50, +} + + +class RainEH3(Check): + temp_dir = TemporaryDirectory() + temp_path = temp_dir.name + "/test.nc" + + def test_rainfall_amount(self): + assert self.nc.variables["rainfall_amount"][0] == 0.0 + assert (np.diff(self.nc.variables["rainfall_amount"][:]) >= 0).all() + + def test_global_attributes(self): + assert self.nc.cloudnet_file_type == "rain-gauge" + assert self.nc.title == f"rain[e]H3 rain-gauge from {self.site_meta['name']}" + assert self.nc.source == "LAMBRECHT meteo GmbH rain[e]H3" + assert self.nc.year == self.date[:4] + assert self.nc.month == self.date[5:7] + assert self.nc.day == self.date[8:10] + assert self.nc.location == self.site_meta["name"] + + +class TestRainEH3(RainEH3): + date = "2024-12-31" + temp_dir = TemporaryDirectory() + temp_path = temp_dir.name + "/test.nc" + site_meta = SITE_META + filename = f"{SCRIPT_PATH}/data/rain_e_h3/20241231_raine_lindenberg.csv" + uuid = rain_e_h32nc(filename, temp_path, site_meta) + + def test_dimensions(self): + assert self.nc.dimensions["time"].size == 22 + + +class TestRainEH3File2(RainEH3): + date = "2023-05-14" + temp_dir = TemporaryDirectory() + temp_path = temp_dir.name + "/test.nc" + site_meta = SITE_META + filename = f"{SCRIPT_PATH}/data/rain_e_h3/Lindenberg_RainE_20230514.txt" + uuid = rain_e_h32nc(filename, temp_path, site_meta) + + def test_dimensions(self): + assert self.nc.dimensions["time"].size == 18 + + +class TestDateArgument(RainEH3): + date = "2024-12-31" + temp_dir = TemporaryDirectory() + temp_path = temp_dir.name + "/test.nc" + filename = f"{SCRIPT_PATH}/data/rain_e_h3/20241231_raine_lindenberg.csv" + site_meta = SITE_META + uuid = rain_e_h32nc(filename, temp_path, site_meta, date=date) + + def test_invalid_date(self): + with pytest.raises(ValidTimeStampError): + rain_e_h32nc( + self.filename, + self.temp_path, + SITE_META, + date="2022-01-05", + )