From e9983f4ec409e2e3a5016c22c8d96d5c68004cb9 Mon Sep 17 00:00:00 2001 From: Clay Harrison Date: Wed, 17 Jul 2024 11:53:30 +0200 Subject: [PATCH 1/6] Change to uint8_nan for swath_indicator --- src/ascat/read_native/eps_native.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ascat/read_native/eps_native.py b/src/ascat/read_native/eps_native.py index 8ec4b42..e690921 100644 --- a/src/ascat/read_native/eps_native.py +++ b/src/ascat/read_native/eps_native.py @@ -1321,7 +1321,7 @@ def read_szx_fmv_11(eps_file): data[f] = raw_data[f.upper()].flatten()[idx_nodes] fields = [("longitude", long_nan), ("latitude", long_nan), - ("swath_indicator", int8_nan)] + ("swath_indicator", uint8_nan)] for f, nan_val in fields: data[f] = raw_data[f.upper()].flatten() @@ -1622,7 +1622,7 @@ def read_smx_fmv_12(eps_file): fields = [("longitude", long_nan, long_nan), ("latitude", long_nan, long_nan), - ("swath_indicator", int8_nan, int8_nan), + ("swath_indicator", int8_nan, uint8_nan), ("soil_moisture", uint_nan, uint_nan), ("soil_moisture_error", uint_nan, uint_nan), ("sigma40", long_nan, long_nan), From 945fe83da2e8fda8b296e380c0a5c2aa65a74811 Mon Sep 17 00:00:00 2001 From: Clay Harrison Date: Wed, 17 Jul 2024 11:58:40 +0200 Subject: [PATCH 2/6] raw_data nan should also be uint8 --- src/ascat/read_native/eps_native.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ascat/read_native/eps_native.py b/src/ascat/read_native/eps_native.py index e690921..2ed8f64 100644 --- a/src/ascat/read_native/eps_native.py +++ b/src/ascat/read_native/eps_native.py @@ -1622,7 +1622,7 @@ def read_smx_fmv_12(eps_file): fields = [("longitude", long_nan, long_nan), ("latitude", long_nan, long_nan), - ("swath_indicator", int8_nan, uint8_nan), + ("swath_indicator", uint8_nan, uint8_nan), ("soil_moisture", uint_nan, uint_nan), ("soil_moisture_error", uint_nan, uint_nan), ("sigma40", long_nan, long_nan), From 7da13bb9b9711f7f2cc7cf7387dd69d6f5841e63 Mon Sep 17 00:00:00 2001 From: Clay Harrison Date: Thu, 18 Jul 2024 13:51:08 +0200 Subject: [PATCH 3/6] Move from np.core.defchararray to np.char --- src/ascat/read_native/hdf5.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ascat/read_native/hdf5.py b/src/ascat/read_native/hdf5.py index 7d443fb..9cadaf0 100644 --- a/src/ascat/read_native/hdf5.py +++ b/src/ascat/read_native/hdf5.py @@ -107,7 +107,7 @@ def read(self, generic=False, to_xarray=False): "FORMAT_MAJOR_VERSION", "FORMAT_MINOR_VERSION"] for f in fields: - pos = np.core.defchararray.startswith( + pos = np.char.startswith( mdr_metadata["MPHR/MPHR_TABLE"]["EntryName"], f.encode()) var = mdr_metadata["MPHR/MPHR_TABLE"]["EntryValue"][ pos][0].decode() From 5a862cbb5f6dfc5e612afb001b068bc8a7fd8198 Mon Sep 17 00:00:00 2001 From: Clay Harrison Date: Thu, 18 Jul 2024 15:19:34 +0200 Subject: [PATCH 4/6] Fix f_kp, f_usable, swath_indicator dtype casts --- src/ascat/read_native/eps_native.py | 31 +++++++++++++++++++---------- src/ascat/utils.py | 25 +++++++++++++++++++++++ 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/src/ascat/read_native/eps_native.py b/src/ascat/read_native/eps_native.py index 2ed8f64..45279af 100644 --- a/src/ascat/read_native/eps_native.py +++ b/src/ascat/read_native/eps_native.py @@ -43,6 +43,7 @@ from ascat.utils import get_toi_subset, get_roi_subset from ascat.utils import get_bit, set_bit +from ascat.utils import dtype_to_nan short_cds_time = np.dtype([("day", ">u2"), ("time", ">u4")]) long_cds_time = np.dtype([("day", ">u2"), ("ms", ">u4"), ("mms", ">u2")]) @@ -793,9 +794,11 @@ def conv_epsl1bszf_generic(data, metadata, gen_fields_lut, skip_fields): (data[new_name] > valid_range[1])) data[new_name].set_fill_value(nan_val) else: + invalid = data[var_name] == dtype_to_nan[np.dtype(data[var_name].dtype)] data[new_name] = np.ma.array(data.pop(var_name).astype(new_dtype)) data[new_name].mask = ((data[new_name] < valid_range[0]) | - (data[new_name] > valid_range[1])) + (data[new_name] > valid_range[1]) | + invalid) data[new_name].set_fill_value(nan_val) return data @@ -817,12 +820,15 @@ def conv_epsl1bszx_generic(data, metadata): data : dict of numpy.ndarray Converted dataset. """ + # template - "old_var_name": ("new_name", new dtype ) gen_fields_lut = { "inc_angle_trip": ("inc", np.float32, uint_nan), "azi_angle_trip": ("azi", np.float32, int_nan), "sigma0_trip": ("sig", np.float32, long_nan), "kp": ("kp", np.float32, uint_nan), - "f_kp": ("kp_quality", np.uint8, uint8_nan) + "f_kp": ("kp_quality", np.uint8, None), # "f_kp": ("kp_quality", np.int8, uint8_nan), + "f_usable": ("f_usable", np.int8, uint8_nan), + "swath_indicator": ("swath_indicator", np.int8, uint8_nan), } skip_fields = ["flagfield_rf1", "f_f", "f_v", "f_oa", "f_sa", "f_tel"] @@ -832,9 +838,11 @@ def conv_epsl1bszx_generic(data, metadata): data.pop(var_name) for var_name, (new_name, new_dtype, nan_val) in gen_fields_lut.items(): + invalid = data[var_name] == nan_val data[new_name] = data.pop(var_name).astype(new_dtype) if nan_val is not None: - data[new_name][data[new_name] == nan_val] = float32_nan + new_nan_val = dtype_to_nan[np.dtype(new_dtype)] + data[new_name][invalid] = new_nan_val data["sat_id"] = np.repeat(metadata["sat_id"], data["time"].size) @@ -879,7 +887,8 @@ def conv_epsl2szx_generic(data, metadata): "frozen_soil_probability": ("frozen_prob", np.uint8, None), "innudation_or_wetland": ("wetland", np.uint8, None), "topographical_complexity": ("topo", np.uint8, None), - "kp": ("kp", np.float32, uint_nan) + "kp": ("kp", np.float32, uint_nan), + "swath_indicator": ("swath_indicator", np.int8, uint8_nan) } skip_fields = ["flagfield_rf1", "f_f", "f_v", "f_oa", "f_sa", "f_tel"] @@ -889,9 +898,11 @@ def conv_epsl2szx_generic(data, metadata): data.pop(var_name) for var_name, (new_name, new_dtype, nan_val) in gen_fields_lut.items(): + invalid = data[var_name] == nan_val data[new_name] = data.pop(var_name).astype(new_dtype) if nan_val is not None: - data[new_name][data[new_name] == nan_val] = float32_nan + new_nan_val = dtype_to_nan[np.dtype(new_dtype)] + data[new_name][invalid] = new_nan_val data["sat_id"] = np.repeat(metadata["sat_id"], data["time"].size) @@ -1330,7 +1341,7 @@ def read_szx_fmv_11(eps_file): fields = [("sigma0_trip", long_nan), ("inc_angle_trip", uint_nan), ("azi_angle_trip", int_nan), ("kp", uint_nan), - ("f_kp", int8_nan), ("f_usable", int8_nan), ("f_f", uint_nan), + ("f_kp", uint8_nan), ("f_usable", uint8_nan), ("f_f", uint_nan), ("f_v", uint_nan), ("f_oa", uint_nan), ("f_sa", uint_nan), ("f_tel", uint_nan), ("f_land", uint_nan)] @@ -1406,7 +1417,7 @@ def read_szx_fmv_12(eps_file): data[f] = raw_data[f.upper()].flatten()[idx_nodes] fields = [("longitude", long_nan), ("latitude", long_nan), - ("swath indicator", int8_nan)] + ("swath indicator", uint8_nan)] for f, nan_val in fields: data[f] = raw_data[f.upper()].flatten() @@ -1415,8 +1426,8 @@ def read_szx_fmv_12(eps_file): fields = [("sigma0_trip", long_nan), ("inc_angle_trip", uint_nan), ("azi_angle_trip", int_nan), ("kp", uint_nan), - ("num_val_trip", ulong_nan), ("f_kp", int8_nan), - ("f_usable", int8_nan), ("f_f", uint_nan), ("f_v", uint_nan), + ("num_val_trip", ulong_nan), ("f_kp", uint8_nan), + ("f_usable", uint8_nan), ("f_f", uint_nan), ("f_v", uint_nan), ("f_oa", uint_nan), ("f_sa", uint_nan), ("f_tel", uint_nan), ("f_ref", uint_nan), ("f_land", uint_nan)] @@ -1861,7 +1872,7 @@ def read_szx_fmv_13(eps_file): fields = [("sigma0_trip", long_nan), ("inc_angle_trip", uint_nan), ("azi_angle_trip", int_nan), ("kp", uint_nan), - ("num_val_trip", ulong_nan), ("f_kp", int8_nan), + ("num_val_trip", ulong_nan), ("f_kp", uint8_nan), ("f_usable", int8_nan), ("land_frac", uint_nan)] for f, nan_val in fields: diff --git a/src/ascat/utils.py b/src/ascat/utils.py index b3392f1..0ea5fd9 100644 --- a/src/ascat/utils.py +++ b/src/ascat/utils.py @@ -32,6 +32,31 @@ import numpy as np import xarray as xr +int8_nan = np.iinfo(np.int8).min +uint8_nan = np.iinfo(np.uint8).max +int16_nan = np.iinfo(np.int16).min +uint16_nan = np.iinfo(np.uint16).max +int32_nan = np.iinfo(np.int32).min +uint32_nan = np.iinfo(np.uint32).max +int64_nan = np.iinfo(np.int64).min +uint64_nan = np.iinfo(np.uint64).max +float32_nan = -999999. +float64_nan = -999999. + +dtype_to_nan = { + np.dtype('int8'): int8_nan, + np.dtype('uint8'): uint8_nan, + np.dtype('int16'): int16_nan, + np.dtype('uint16'): uint16_nan, + np.dtype('int32'): int32_nan, + np.dtype('uint32'): uint32_nan, + np.dtype('int64'): int64_nan, + np.dtype('uint64'): uint64_nan, + np.dtype('float32'): float32_nan, + np.dtype('float64'): float64_nan, + np.dtype(' Date: Thu, 18 Jul 2024 15:20:09 +0200 Subject: [PATCH 5/6] Fix flagfield_gen2 nan assignment --- src/ascat/read_native/eps_native.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ascat/read_native/eps_native.py b/src/ascat/read_native/eps_native.py index 45279af..bd0e933 100644 --- a/src/ascat/read_native/eps_native.py +++ b/src/ascat/read_native/eps_native.py @@ -1557,7 +1557,7 @@ def read_szf_fmv_12(eps_file, ignore_noise_ool=False): fields = [("longitude_full", long_nan), ("latitude_full", long_nan), ("sigma0_full", long_nan), ("inc_angle_full", uint_nan), ("azi_angle_full", int_nan), ("land_frac", uint_nan), - ("flagfield_gen2", int8_nan)] + ("flagfield_gen2", uint8_nan)] for f, nan_val in fields: data[f] = eps_file.mdr[f.upper()].flatten() From 37f912c3cf295186e0465c2db98f7a038038ae96 Mon Sep 17 00:00:00 2001 From: Clay Harrison Date: Thu, 18 Jul 2024 16:07:01 +0200 Subject: [PATCH 6/6] Fix nans in generic l1b bufr conversion --- src/ascat/read_native/bufr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ascat/read_native/bufr.py b/src/ascat/read_native/bufr.py index 9ccf6cb..bbfbb1e 100644 --- a/src/ascat/read_native/bufr.py +++ b/src/ascat/read_native/bufr.py @@ -266,7 +266,7 @@ def conv_bufrl1b_generic(data, metadata): data.pop(f[2]))).T.astype(new_dtype) if nan_val is not None: valid = data[new_name] != nan_val - data[new_name][~valid] = float32_nan + data[new_name][~valid] = nan_val_dict[new_dtype] data[new_name][valid] *= s if data['node_num'].max() == 82: