Skip to content

Commit

Permalink
Add rain[e]H3 rain gauge processing (#113)
Browse files Browse the repository at this point in the history
  • Loading branch information
@tukiains
tukiains authored Jan 17, 2025
1 parent 95cf37d commit ba13893
Show file tree
Hide file tree
Showing 7 changed files with 322 additions and 29 deletions.
1 change: 1 addition & 0 deletions cloudnetpy/instruments/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,6 @@
from .mrr import mrr2nc
from .pollyxt import pollyxt2nc
from .radiometrics import radiometrics2nc
from .rain_e_h3 import rain_e_h32nc
from .rpg import rpg2nc
from .weather_station import ws2nc
31 changes: 31 additions & 0 deletions cloudnetpy/instruments/cloudnet_instrument.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,3 +112,34 @@ def _get_zenith_angle(self) -> float | None:
if np.isnan(zenith_angle) or zenith_angle is ma.masked:
return None
return zenith_angle


class CSVFile(CloudnetInstrument):
def __init__(self, site_meta: dict):
super().__init__()
self.site_meta = site_meta
self._data: dict = {}

def add_date(self) -> None:
dt = self._data["time"][0]
self.date = dt.strftime("%Y %m %d").split()

def add_data(self) -> None:
for key, value in self._data.items():
parsed = (
utils.datetime2decimal_hours(value)
if key == "time"
else ma.array(value)
)
self.data[key] = CloudnetArray(parsed, key)

def normalize_rainfall_amount(self) -> None:
if "rainfall_amount" in self.data:
amount = self.data["rainfall_amount"][:]
offset = 0
for i in range(1, len(amount)):
if amount[i] + offset < amount[i - 1]:
offset += amount[i - 1]
amount[i] += offset
amount -= amount[0]
self.data["rainfall_amount"].data = amount
166 changes: 166 additions & 0 deletions cloudnetpy/instruments/rain_e_h3.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
import csv
import datetime
from os import PathLike

import numpy as np

from cloudnetpy import output
from cloudnetpy.exceptions import ValidTimeStampError
from cloudnetpy.instruments import instruments
from cloudnetpy.instruments.cloudnet_instrument import CSVFile


def rain_e_h32nc(
input_file: str | PathLike,
output_file: str,
site_meta: dict,
uuid: str | None = None,
date: str | datetime.date | None = None,
):
"""Converts rain_e_h3 rain-gauge into Cloudnet Level 1b netCDF file.
Args:
input_file: Filename of rain_e_h3 CSV file.
output_file: Output filename.
site_meta: Dictionary containing information about the site. Required key
is `name`.
uuid: Set specific UUID for the file.
date: Expected date of the measurements as YYYY-MM-DD or datetime.date object.
Returns:
UUID of the generated file.
Raises:
WeatherStationDataError : Unable to read the file.
ValidTimeStampError: No valid timestamps found.
"""
rain = RainEH3(site_meta)
if isinstance(date, str):
date = datetime.date.fromisoformat(date)
rain.parse_input_file(input_file, date)
rain.add_data()
rain.add_date()
rain.convert_units()
rain.normalize_rainfall_amount()
rain.add_site_geolocation()
attributes = output.add_time_attribute({}, rain.date)
output.update_attributes(rain.data, attributes)
return output.save_level1b(rain, output_file, uuid)


class RainEH3(CSVFile):
def __init__(self, site_meta: dict):
super().__init__(site_meta)
self.instrument = instruments.RAIN_E_H3
self._data = {
"time": [],
"rainfall_rate": [],
"rainfall_amount": [],
}

def parse_input_file(
self, filepath: str | PathLike, date: datetime.date | None = None
) -> None:
with open(filepath, encoding="latin1") as f:
data = list(csv.reader(f, delimiter=";"))
n_values = np.median([len(row) for row in data]).astype(int)

if n_values == 22:
self._read_talker_protocol_22_columns(data, date)
elif n_values == 16:
self._read_talker_protocol_16_columns(data, date)
else:
msg = "Only talker protocol with 16 or 22 columns is supported."
raise NotImplementedError(msg)

def _read_talker_protocol_16_columns(
self, data: list, date: datetime.date | None = None
) -> None:
"""Old Lindenberg data format.
0 date DD.MM.YYYY
1 time
2 precipitation intensity in mm/h
3 precipitation accumulation in mm
4 housing contact
5 top temperature
6 bottom temperature
7 heater status
8 error code
9 system status
10 talker interval in seconds
11 operating hours
12 device type
13 user data storage 1
14 user data storage 2
15 user data storage 3
"""
for row in data:
if len(row) != 16:
continue
try:
dt = datetime.datetime.strptime(
f"{row[0]} {row[1]}", "%d.%m.%Y %H:%M:%S"
)
except ValueError:
continue
if date and date != dt.date():
continue
self._data["time"].append(dt)
self._data["rainfall_rate"].append(float(row[2]))
self._data["rainfall_amount"].append(float(row[3]))
if not self._data["time"]:
raise ValidTimeStampError

def _read_talker_protocol_22_columns(
self, data: list, date: datetime.date | None = None
) -> None:
"""Columns according to header in Lindenberg data.
0 datetime utc
1 date
2 time
3 precipitation intensity in mm/h
4 precipitation accumulation in mm
5 housing contact
6 top temperature
7 bottom temperature
8 heater status
9 error code
10 system status
11 talker interval in seconds
12 operating hours
13 device type
14 user data storage 1
15 user data storage 2
16 user data storage 3
17 user data storage 4
18 serial number
19 hardware version
20 firmware version
21 external temperature * checksum
"""
for row in data:
if len(row) != 22:
continue
try:
dt = datetime.datetime.strptime(f"{row[0]}", "%Y-%m-%d %H:%M:%S")
except ValueError:
continue
if date and date != dt.date():
continue
self._data["time"].append(dt)
self._data["rainfall_rate"].append(float(row[3]))
self._data["rainfall_amount"].append(float(row[4]))
self.serial_number = row[18]
if not self._data["time"]:
raise ValidTimeStampError

def convert_units(self) -> None:
rainfall_rate = self.data["rainfall_rate"][:]
self.data["rainfall_rate"].data = rainfall_rate / 3600 / 1000 # mm/h -> m/s
self.data["rainfall_amount"].data = (
self.data["rainfall_amount"][:] / 1000
) # mm -> m
32 changes: 3 additions & 29 deletions cloudnetpy/instruments/weather_station.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from cloudnetpy.constants import HPA_TO_PA, MM_H_TO_M_S, SEC_IN_HOUR
from cloudnetpy.exceptions import ValidTimeStampError, WeatherStationDataError
from cloudnetpy.instruments import instruments
from cloudnetpy.instruments.cloudnet_instrument import CloudnetInstrument
from cloudnetpy.instruments.cloudnet_instrument import CSVFile
from cloudnetpy.instruments.toa5 import read_toa5
from cloudnetpy.utils import datetime2decimal_hours

Expand Down Expand Up @@ -79,28 +79,13 @@ def ws2nc(
return output.save_level1b(ws, output_file, uuid)


class WS(CloudnetInstrument):
class WS(CSVFile):
def __init__(self, site_meta: dict):
super().__init__()
self._data: dict
self.site_meta = site_meta
super().__init__(site_meta)
self.instrument = instruments.GENERIC_WEATHER_STATION

date: list[str]

def add_date(self) -> None:
first_date = self._data["time"][0].date()
self.date = [
str(first_date.year),
str(first_date.month).zfill(2),
str(first_date.day).zfill(2),
]

def add_data(self) -> None:
for key, value in self._data.items():
parsed = datetime2decimal_hours(value) if key == "time" else ma.array(value)
self.data[key] = CloudnetArray(parsed, key)

def calculate_rainfall_amount(self) -> None:
if "rainfall_amount" in self.data:
return
Expand Down Expand Up @@ -137,17 +122,6 @@ def convert_rainfall_rate(self) -> None:
def convert_pressure(self) -> None:
self.data["air_pressure"].data = self.data["air_pressure"][:] * HPA_TO_PA

def normalize_rainfall_amount(self) -> None:
if "rainfall_amount" in self.data:
amount = self.data["rainfall_amount"][:]
offset = 0
for i in range(1, len(amount)):
if amount[i] + offset < amount[i - 1]:
offset += amount[i - 1]
amount[i] += offset
amount -= amount[0]
self.data["rainfall_amount"].data = amount

def convert_time(self) -> None:
pass

Expand Down
23 changes: 23 additions & 0 deletions tests/unit/data/rain_e_h3/20241231_raine_lindenberg.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
datetime_utc;date;time;precipitation intensity [mm/h];precipitation accumulation [mm];housing contact;top temperature [�C];bottom temperature [�C];heater status;error code;system status;talker interval [s];operating hours;device type;user data1;user data2;user data3;user data4;serial number;hardware version;firmware version;external temperature [�C];checksum
2024-12-31 00:00:35;te:2024.12.31;00:01:03;0.000;514.761;0;5.00;5.00;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.50*9A
2024-12-31 00:01:35;te:2024.12.31;00:02:03;0.000;514.761;0;5.06;5.00;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.50*93
2024-12-31 00:02:35;te:2024.12.31;00:03:03;0.000;514.761;0;5.06;5.00;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.44*8F
2024-12-31 00:03:35;te:2024.12.31;00:04:03;0.000;514.761;0;5.19;5.00;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.44*8A
2024-12-31 00:04:35;te:2024.12.31;00:05:03;0.000;514.761;0;5.19;5.00;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.44*89
2024-12-31 00:05:35;te:2024.12.31;00:06:03;0.000;514.761;0;5.19;5.06;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.44*82
2024-12-31 00:06:35;te:2024.12.31;00:07:03;0.000;514.761;0;5.12;5.06;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.44*88
2024-12-31 00:07:35;te:2024.12.31;00:08:03;0.000;514.761;0;5.12;5.00;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.44*8D
2024-12-31 00:08:35;te:2024.12.31;00:09:03;0.000;514.761;0;5.19;5.06;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.44*7F
2024-12-31 00:09:35;te:2024.12.31;00:10:03;0.000;514.761;0;5.19;5.12;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.44*8A
2024-12-31 00:10:35;te:2024.12.31;00:11:03;0.000;514.761;0;5.12;5.19;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.44*89
2024-12-31 00:11:35;te:2024.12.31;00:12:03;0.000;514.761;0;5.06;5.12;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.50*8F
2024-12-31 00:12:35;te:2024.12.31;00:13:03;0.000;514.761;0;5.06;5.12;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.44*8B
2024-12-31 00:13:35;te:2024.12.31;00:14:03;0.000;514.761;0;4.94;5.06;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.44*81
2024-12-31 00:14:35;te:2024.12.31;00:15:03;0.000;514.761;0;4.94;5.00;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.44*86
2024-12-31 00:15:35;te:2024.12.31;00:16:03;0.000;514.761;0;5.12;5.12;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.50*8E
2024-12-31 00:16:35;te:2024.12.31;00:17:03;0.000;514.761;0;5.00;5.12;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.50*90
2024-12-31 00:17:35;te:2024.12.31;00:18:03;0.000;514.761;0;4.87;5.06;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.50*7E
2024-12-31 00:18:35;te:2024.12.31;00:19:03;0.000;514.761;0;4.81;4.94;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.50*7D
2024-12-31 00:19:35;te:2024.12.31;00:20:03;0.000;514.761;0;4.87;4.81;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.50*83
2024-12-31 00:20:35;te:2024.12.31;00:21:03;0.000;514.761;0;5.00;4.81;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.50*90
2024-12-31 00:21:35;te:2024.12.31;00:22:03;0.000;514.761;0;5.06;4.81;1;0;0;60;16740;rain[e]H3;Lindenberg ;MF ;ACTRIS ; ;850383.0067;1.20;1.55;-0.50*89
18 changes: 18 additions & 0 deletions tests/unit/data/rain_e_h3/Lindenberg_RainE_20230514.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
14.05.2023;00:00:39;0.000;0.406;0;9.56;11.50;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS
14.05.2023;00:01:39;0.000;0.406;0;9.50;11.44;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS
14.05.2023;00:02:39;0.000;0.406;0;9.44;11.44;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS
14.05.2023;00:03:39;0.000;0.406;0;9.37;11.37;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS
14.05.2023;00:04:39;0.000;0.406;0;9.31;11.31;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS
14.05.2023;00:05:39;0.000;0.406;0;9.25;11.31;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS
14.05.2023;00:06:39;0.000;0.406;0;9.19;11.25;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS
14.05.2023;00:07:39;0.000;0.406;0;9.12;11.19;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS
14.05.2023;00:08:39;0.000;0.406;0;9.12;11.19;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS
14.05.2023;00:09:39;0.000;0.406;0;9.06;11.12;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS
14.05.2023;00:10:39;0.000;0.406;0;9.00;11.06;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS
14.05.2023;00:11:39;0.000;0.406;0;9.06;11.06;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS
14.05.2023;00:12:39;0.000;0.406;0;9.12;11.12;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS
14.05.2023;00:13:39;0.000;0.406;0;9.06;11.12;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS
14.05.2023;00:14:39;0.000;0.406;0;9.06;11.06;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS
14.05.2023;00:15:39;0.000;0.406;0;9.06;11.06;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS
14.05.2023;00:16:39;0.000;0.406;0;9.00;11.00;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS
14.05.2023;00:17:39;0.000;0.406;0;8.94;10.94;0;0;0;60;2582;rain[e]H3;Lindenberg ;MF ;ACTRIS
80 changes: 80 additions & 0 deletions tests/unit/test_rain_e_h3.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
import os
from tempfile import TemporaryDirectory

from cloudnetpy.cloudnetarray import CloudnetArray
import pytest

from cloudnetpy.exceptions import ValidTimeStampError
from cloudnetpy.instruments import rain_e_h32nc
from tests.unit.all_products_fun import Check
import numpy as np
from numpy import ma

SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__))

SITE_META = {
"name": "Palaiseau",
"latitude": 50,
"longitude": 104.5,
"altitude": 50,
}


class RainEH3(Check):
temp_dir = TemporaryDirectory()
temp_path = temp_dir.name + "/test.nc"

def test_rainfall_amount(self):
assert self.nc.variables["rainfall_amount"][0] == 0.0
assert (np.diff(self.nc.variables["rainfall_amount"][:]) >= 0).all()

def test_global_attributes(self):
assert self.nc.cloudnet_file_type == "rain-gauge"
assert self.nc.title == f"rain[e]H3 rain-gauge from {self.site_meta['name']}"
assert self.nc.source == "LAMBRECHT meteo GmbH rain[e]H3"
assert self.nc.year == self.date[:4]
assert self.nc.month == self.date[5:7]
assert self.nc.day == self.date[8:10]
assert self.nc.location == self.site_meta["name"]


class TestRainEH3(RainEH3):
date = "2024-12-31"
temp_dir = TemporaryDirectory()
temp_path = temp_dir.name + "/test.nc"
site_meta = SITE_META
filename = f"{SCRIPT_PATH}/data/rain_e_h3/20241231_raine_lindenberg.csv"
uuid = rain_e_h32nc(filename, temp_path, site_meta)

def test_dimensions(self):
assert self.nc.dimensions["time"].size == 22


class TestRainEH3File2(RainEH3):
date = "2023-05-14"
temp_dir = TemporaryDirectory()
temp_path = temp_dir.name + "/test.nc"
site_meta = SITE_META
filename = f"{SCRIPT_PATH}/data/rain_e_h3/Lindenberg_RainE_20230514.txt"
uuid = rain_e_h32nc(filename, temp_path, site_meta)

def test_dimensions(self):
assert self.nc.dimensions["time"].size == 18


class TestDateArgument(RainEH3):
date = "2024-12-31"
temp_dir = TemporaryDirectory()
temp_path = temp_dir.name + "/test.nc"
filename = f"{SCRIPT_PATH}/data/rain_e_h3/20241231_raine_lindenberg.csv"
site_meta = SITE_META
uuid = rain_e_h32nc(filename, temp_path, site_meta, date=date)

def test_invalid_date(self):
with pytest.raises(ValidTimeStampError):
rain_e_h32nc(
self.filename,
self.temp_path,
SITE_META,
date="2022-01-05",
)

0 comments on commit ba13893

Please sign in to comment.