From 89a0e82ed15255d047f3248aafe058a7282a4272 Mon Sep 17 00:00:00 2001 From: Simo Tukiainen Date: Tue, 5 Dec 2023 14:59:09 +0200 Subject: [PATCH] Use tempfiles --- mwrpy/level2/lev2_collocated.py | 361 +++++++++++++++++--------------- 1 file changed, 187 insertions(+), 174 deletions(-) diff --git a/mwrpy/level2/lev2_collocated.py b/mwrpy/level2/lev2_collocated.py index 539e8fe..6ebe8f2 100644 --- a/mwrpy/level2/lev2_collocated.py +++ b/mwrpy/level2/lev2_collocated.py @@ -1,4 +1,4 @@ -import os +from tempfile import NamedTemporaryFile import netCDF4 @@ -12,121 +12,134 @@ def generate_lev2_single( output_file: str, coeff_files: list[str] | None = None, ): - products = { - "2I01": "lwp", - "2I02": "iwv", - "2P01": "t_prof", - "2P03": "abs_hum", - "2P04": "rel_hum", - "2P07": "t_pot", - "2P08": "eq_temp", - } - - for prod, file in products.items(): - lev2_to_nc( - prod, - mwr_l1c_file, - output_file=file, - site=site, - temp_file=products["2P01"] if prod in ("2P04", "2P07", "2P08") else None, - hum_file=products["2P03"] if prod in ("2P04", "2P07", "2P08") else None, - coeff_files=coeff_files, - ) - with ( - netCDF4.Dataset(output_file, "w", format="NETCDF4_CLASSIC") as nc_output, - netCDF4.Dataset(products["2I01"], "r") as nc_lwp, - netCDF4.Dataset(products["2I02"], "r") as nc_iwv, - netCDF4.Dataset(products["2P01"], "r") as nc_t_prof, - netCDF4.Dataset(products["2P03"], "r") as nc_abs_hum, - netCDF4.Dataset(products["2P04"], "r") as nc_rel_hum, - netCDF4.Dataset(products["2P07"], "r") as nc_t_pot, - netCDF4.Dataset(products["2P08"], "r") as nc_eq_temp, + NamedTemporaryFile() as lwp_file, + NamedTemporaryFile() as iwv_file, + NamedTemporaryFile() as t_prof_file, + NamedTemporaryFile() as abs_hum_file, + NamedTemporaryFile() as rel_hum_file, + NamedTemporaryFile() as t_pot_file, + NamedTemporaryFile() as eq_temp_file, ): - nc_output.createDimension("height", len(nc_t_prof.variables["height"][:])) - nc_output.createDimension("time", len(nc_lwp.variables["time"][:])) - nc_output.createDimension("bnds", 2) - - for source, variables in ( + for prod, file in zip( + ("2I01", "2I02", "2P01", "2P03", "2P04", "2P07", "2P08"), ( - nc_iwv, + lwp_file.name, + iwv_file.name, + t_prof_file.name, + abs_hum_file.name, + rel_hum_file.name, + t_pot_file.name, + eq_temp_file.name, + ), + strict=True, + ): + lev2_to_nc( + prod, + mwr_l1c_file, + output_file=file, + site=site, + temp_file=t_prof_file.name + if prod in ("2P04", "2P07", "2P08") + else None, + hum_file=abs_hum_file.name + if prod in ("2P04", "2P07", "2P08") + else None, + coeff_files=coeff_files, + ) + + with ( + netCDF4.Dataset(output_file, "w", format="NETCDF4_CLASSIC") as nc_output, + netCDF4.Dataset(lwp_file.name, "r") as nc_lwp, + netCDF4.Dataset(iwv_file.name, "r") as nc_iwv, + netCDF4.Dataset(t_prof_file.name, "r") as nc_t_prof, + netCDF4.Dataset(abs_hum_file.name, "r") as nc_abs_hum, + netCDF4.Dataset(rel_hum_file.name, "r") as nc_rel_hum, + netCDF4.Dataset(t_pot_file.name, "r") as nc_t_pot, + netCDF4.Dataset(eq_temp_file.name, "r") as nc_eq_temp, + ): + nc_output.createDimension("height", len(nc_t_prof.variables["height"][:])) + nc_output.createDimension("time", len(nc_lwp.variables["time"][:])) + nc_output.createDimension("bnds", 2) + + for source, variables in ( ( - "iwv", - "iwv_random_error", - "iwv_systematic_error", - "iwv_quality_flag", - "iwv_quality_flag_status", + nc_iwv, + ( + "iwv", + "iwv_random_error", + "iwv_systematic_error", + "iwv_quality_flag", + "iwv_quality_flag_status", + ), ), - ), - ( - nc_abs_hum, ( - "absolute_humidity", - "absolute_humidity_random_error", - "absolute_humidity_systematic_error", - "absolute_humidity_quality_flag", - "absolute_humidity_quality_flag_status", + nc_abs_hum, + ( + "absolute_humidity", + "absolute_humidity_random_error", + "absolute_humidity_systematic_error", + "absolute_humidity_quality_flag", + "absolute_humidity_quality_flag_status", + ), ), - ), - ( - nc_t_prof, ( - "temperature", - "temperature_random_error", - "temperature_systematic_error", - "height", - "temperature_quality_flag", - "temperature_quality_flag_status", + nc_t_prof, + ( + "temperature", + "temperature_random_error", + "temperature_systematic_error", + "height", + "temperature_quality_flag", + "temperature_quality_flag_status", + ), ), - ), - ( - nc_lwp, ( - "time", - "time_bnds", - "latitude", - "longitude", - "altitude", - "lwp", - "lwp_offset", - "lwp_random_error", - "lwp_systematic_error", - "elevation_angle", - "azimuth_angle", - "lwp_quality_flag", - "lwp_quality_flag_status", + nc_lwp, + ( + "time", + "time_bnds", + "latitude", + "longitude", + "altitude", + "lwp", + "lwp_offset", + "lwp_random_error", + "lwp_systematic_error", + "elevation_angle", + "azimuth_angle", + "lwp_quality_flag", + "lwp_quality_flag_status", + ), ), - ), - ( - nc_rel_hum, ( - "relative_humidity", - "relative_humidity_random_error", - "relative_humidity_systematic_error", + nc_rel_hum, + ( + "relative_humidity", + "relative_humidity_random_error", + "relative_humidity_systematic_error", + ), ), - ), - ( - nc_t_pot, ( - "potential_temperature", - "potential_temperature_random_error", - "potential_temperature_systematic_error", + nc_t_pot, + ( + "potential_temperature", + "potential_temperature_random_error", + "potential_temperature_systematic_error", + ), ), - ), - ( - nc_eq_temp, ( - "equivalent_potential_temperature", - "equivalent_potential_temperature_random_error", - "equivalent_potential_temperature_systematic_error", + nc_eq_temp, + ( + "equivalent_potential_temperature", + "equivalent_potential_temperature_random_error", + "equivalent_potential_temperature_systematic_error", + ), ), - ), - ): - copy_variables(source, nc_output, variables) - - copy_global(nc_lwp, nc_output, nc_lwp.ncattrs()) + ): + copy_variables(source, nc_output, variables) - _delete_temp_files(products) + copy_global(nc_lwp, nc_output, nc_lwp.ncattrs()) return nc_output @@ -137,93 +150,93 @@ def generate_lev2_multi( output_file: str, coeff_files: list[str] | None = None, ): - products = { - "2P02": "temp", - "2P03": "abs_hum", - "2P04": "rel_hum", - "2P07": "t_pot", - "2P08": "eq_temp", - } - - for prod, file in products.items(): - lev2_to_nc( - prod, - mwr_l1c_file, - output_file=file, - site=site, - temp_file=products["2P02"] if prod not in ("2P02", "2P03") else None, - hum_file=products["2P03"] if prod not in ("2P02", "2P03") else None, - coeff_files=coeff_files, - ) - with ( - netCDF4.Dataset(output_file, "w", format="NETCDF4_CLASSIC") as nc_output, - netCDF4.Dataset(products["2P02"], "r") as nc_temp, - netCDF4.Dataset(products["2P04"], "r") as nc_rel_hum, - netCDF4.Dataset(products["2P07"], "r") as nc_t_pot, - netCDF4.Dataset(products["2P08"], "r") as nc_eq_temp, + NamedTemporaryFile() as temperature_file, + NamedTemporaryFile() as abs_hum_file, + NamedTemporaryFile() as rel_hum_file, + NamedTemporaryFile() as t_pot_file, + NamedTemporaryFile() as eq_temp_file, ): - nc_output.createDimension("time", len(nc_temp.variables["time"][:])) - nc_output.createDimension("height", len(nc_temp.variables["height"][:])) - nc_output.createDimension("bnds", 2) - - for source, variables in ( + for prod, file in zip( + ("2P02", "2P03", "2P04", "2P07", "2P08"), ( - nc_temp, + temperature_file.name, + abs_hum_file.name, + rel_hum_file.name, + t_pot_file.name, + eq_temp_file.name, + ), + strict=True, + ): + lev2_to_nc( + prod, + mwr_l1c_file, + output_file=file, + site=site, + temp_file=temperature_file.name + if prod not in ("2P02", "2P03") + else None, + hum_file=abs_hum_file.name if prod not in ("2P02", "2P03") else None, + coeff_files=coeff_files, + ) + + with ( + netCDF4.Dataset(output_file, "w", format="NETCDF4_CLASSIC") as nc_output, + netCDF4.Dataset(temperature_file.name, "r") as nc_temp, + netCDF4.Dataset(rel_hum_file.name, "r") as nc_rel_hum, + netCDF4.Dataset(t_pot_file.name, "r") as nc_t_pot, + netCDF4.Dataset(eq_temp_file.name, "r") as nc_eq_temp, + ): + nc_output.createDimension("time", len(nc_temp.variables["time"][:])) + nc_output.createDimension("height", len(nc_temp.variables["height"][:])) + nc_output.createDimension("bnds", 2) + + for source, variables in ( ( - "time", - "time_bnds", - "height", - "latitude", - "longitude", - "altitude", - "elevation_angle", - "azimuth_angle", - "temperature", - "temperature_random_error", - "temperature_systematic_error", - "temperature_quality_flag", - "temperature_quality_flag_status", + nc_temp, + ( + "time", + "time_bnds", + "height", + "latitude", + "longitude", + "altitude", + "elevation_angle", + "azimuth_angle", + "temperature", + "temperature_random_error", + "temperature_systematic_error", + "temperature_quality_flag", + "temperature_quality_flag_status", + ), ), - ), - ( - nc_rel_hum, ( - "relative_humidity", - "relative_humidity_random_error", - "relative_humidity_systematic_error", + nc_rel_hum, + ( + "relative_humidity", + "relative_humidity_random_error", + "relative_humidity_systematic_error", + ), ), - ), - ( - nc_t_pot, ( - "potential_temperature", - "potential_temperature_random_error", - "potential_temperature_systematic_error", + nc_t_pot, + ( + "potential_temperature", + "potential_temperature_random_error", + "potential_temperature_systematic_error", + ), ), - ), - ( - nc_eq_temp, ( - "equivalent_potential_temperature", - "equivalent_potential_temperature_random_error", - "equivalent_potential_temperature_systematic_error", + nc_eq_temp, + ( + "equivalent_potential_temperature", + "equivalent_potential_temperature_random_error", + "equivalent_potential_temperature_systematic_error", + ), ), - ), - ): - copy_variables(source, nc_output, variables) - - copy_global(nc_temp, nc_output, nc_temp.ncattrs()) + ): + copy_variables(source, nc_output, variables) - _delete_temp_files(products) + copy_global(nc_temp, nc_output, nc_temp.ncattrs()) return nc_output - - -def _delete_temp_files(products: dict): - for _, file in products.items(): - if os.path.exists(file): - try: - os.remove(file) - except PermissionError: - pass