From e0c106d5e86d1efec5f8dfd397c707c36f4aca17 Mon Sep 17 00:00:00 2001 From: dabhicusp Date: Fri, 15 Sep 2023 08:13:37 +0000 Subject: [PATCH 1/5] Ar variable updated. --- src/arco_era5/source_data.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/arco_era5/source_data.py b/src/arco_era5/source_data.py index e8ed566..33e0456 100644 --- a/src/arco_era5/source_data.py +++ b/src/arco_era5/source_data.py @@ -58,6 +58,37 @@ "2m_temperature", "10m_v_component_of_wind", "10m_u_component_of_wind", + "volumetric_soil_water_layer_1", + "volumetric_soil_water_layer_2", + "volumetric_soil_water_layer_3", + "volumetric_soil_water_layer_4", + "mean_top_net_long_wave_radiation_flux", + "mean_top_net_short_wave_radiation_flux", + "high_vegetation_cover", + "ice_temperature_layer_1", + "ice_temperature_layer_2", + "ice_temperature_layer_3", + "ice_temperature_layer_4", + "lake_cover", + "lake_depth", + "lake_ice_depth", + "lake_ice_temperature", + "lake_mix_layer_depth", + "lake_mix_layer_temperature", + "lake_total_layer_temperature", + "land_sea_mask", + "leaf_area_index_high_vegetation", + "leaf_area_index_low_vegetation", + "low_vegetation_cover", + "skin_temperature", + "snow_depth", + "soil_temperature_level_1", + "soil_temperature_level_2", + "soil_temperature_level_3", + "soil_temperature_level_4", + "soil_type", + "type_of_high_vegetation", + "type_of_low_vegetation" ) MULTILEVEL_VARIABLES = ( @@ -67,6 +98,11 @@ "u_component_of_wind", "v_component_of_wind", "vertical_velocity", + "potential_vorticity", + "specific_cloud_ice_water_content", + "specific_cloud_liquid_water_content", + "fraction_of_cloud_cover", + "ozone_mass_mixing_ratio" ) # Variables that correspond to an integration over a `TIME_RESOLUTION_HOURS` From 07b6d297fe9dff0536ac97e3dd1353b55c0a5840 Mon Sep 17 00:00:00 2001 From: dabhicusp Date: Sat, 16 Sep 2023 05:42:14 +0000 Subject: [PATCH 2/5] Other variabls are added. --- raw/era5_sl_hourly.cfg | 220 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 220 insertions(+) diff --git a/raw/era5_sl_hourly.cfg b/raw/era5_sl_hourly.cfg index 48c43e1..a5587d9 100644 --- a/raw/era5_sl_hourly.cfg +++ b/raw/era5_sl_hourly.cfg @@ -85,3 +85,223 @@ variable= soil_type type_of_high_vegetation type_of_low_vegetation + 100m_u_component_of_wind + 100m_v_component_of_wind + 10m_u_component_of_neutral_wind + 10m_v_component_of_neutral_wind + 10m_wind_gust_since_previous_post_processing + 2m_dewpoint_temperature + air_density_over_the_oceans + angle_of_sub_gridscale_orography + anisotropy_of_sub_gridscale_orography + benjamin_feir_index + boundary_layer_dissipation + boundary_layer_height + charnock + clear_sky_direct_solar_radiation_at_surface + cloud_base_height + coefficient_of_drag_with_waves + convective_available_potential_energy + convective_inhibition + convective_precipitation + convective_rain_rate + convective_snowfall + convective_snowfall_rate_water_equivalent + downward_uv_radiation_at_the_surface + duct_base_height + eastward_gravity_wave_surface_stress + eastward_turbulent_surface_stress + evaporation + forecast_albedo + forecast_logarithm_of_surface_roughness_for_heat + forecast_surface_roughness + free_convective_velocity_over_the_oceans + friction_velocity + geopotential + gravity_wave_dissipation + high_cloud_cover + instantaneous_10m_wind_gust + instantaneous_eastward_turbulent_surface_stress + instantaneous_large_scale_surface_precipitation_fraction + instantaneous_moisture_flux + instantaneous_northward_turbulent_surface_stress + instantaneous_surface_sensible_heat_flux + k_index + lake_bottom_temperature + lake_shape_factor + large_scale_precipitation + large_scale_precipitation_fraction + large_scale_rain_rate + large_scale_snowfall + large_scale_snowfall_rate_water_equivalent + low_cloud_cover + maximum_2m_temperature_since_previous_post_processing + maximum_individual_wave_height + maximum_total_precipitation_rate_since_previous_post_processing + mean_boundary_layer_dissipation + mean_convective_precipitation_rate + mean_convective_snowfall_rate + mean_direction_of_total_swell + mean_direction_of_wind_waves + mean_eastward_gravity_wave_surface_stress + mean_eastward_turbulent_surface_stress + mean_evaporation_rate + mean_gravity_wave_dissipation + mean_large_scale_precipitation_fraction + mean_large_scale_precipitation_rate + mean_large_scale_snowfall_rate + mean_northward_gravity_wave_surface_stress + mean_northward_turbulent_surface_stress + mean_period_of_total_swell + mean_period_of_wind_waves + mean_potential_evaporation_rate + mean_runoff_rate + mean_snow_evaporation_rate + mean_snowfall_rate + mean_snowmelt_rate + mean_square_slope_of_waves + mean_sub_surface_runoff_rate + mean_surface_direct_short_wave_radiation_flux + mean_surface_direct_short_wave_radiation_flux_clear_sky + mean_surface_downward_long_wave_radiation_flux + mean_surface_downward_long_wave_radiation_flux_clear_sky + mean_surface_downward_short_wave_radiation_flux + mean_surface_downward_short_wave_radiation_flux_clear_sky + mean_surface_downward_uv_radiation_flux + mean_surface_latent_heat_flux + mean_surface_net_long_wave_radiation_flux + mean_surface_net_long_wave_radiation_flux_clear_sky + mean_surface_net_short_wave_radiation_flux + mean_surface_net_short_wave_radiation_flux_clear_sky + mean_surface_runoff_rate + mean_surface_sensible_heat_flux + mean_top_downward_short_wave_radiation_flux + mean_top_net_long_wave_radiation_flux_clear_sky + mean_top_net_short_wave_radiation_flux_clear_sky + mean_total_precipitation_rate + mean_vertical_gradient_of_refractivity_inside_trapping_layer + mean_vertically_integrated_moisture_divergence + mean_wave_direction + mean_wave_direction_of_first_swell_partition + mean_wave_direction_of_second_swell_partition + mean_wave_direction_of_third_swell_partition + mean_wave_period + mean_wave_period_based_on_first_moment + mean_wave_period_based_on_first_moment_for_swell + mean_wave_period_based_on_first_moment_for_wind_waves + mean_wave_period_based_on_second_moment_for_swell + mean_wave_period_based_on_second_moment_for_wind_waves + mean_wave_period_of_first_swell_partition + mean_wave_period_of_second_swell_partition + mean_wave_period_of_third_swell_partition + mean_zero_crossing_wave_period + medium_cloud_cover + minimum_2m_temperature_since_previous_post_processing + minimum_total_precipitation_rate_since_previous_post_processing + minimum_vertical_gradient_of_refractivity_inside_trapping_layer + model_bathymetry + near_ir_albedo_for_diffuse_radiation + near_ir_albedo_for_direct_radiation + normalized_energy_flux_into_ocean + normalized_energy_flux_into_waves + normalized_stress_into_ocean + northward_gravity_wave_surface_stress + northward_turbulent_surface_stress + ocean_surface_stress_equivalent_10m_neutral_wind_direction + ocean_surface_stress_equivalent_10m_neutral_wind_speed + peak_wave_period + period_corresponding_to_maximum_individual_wave_height + potential_evaporation + precipitation_type + runoff + significant_height_of_combined_wind_waves_and_swell + significant_height_of_total_swell + significant_height_of_wind_waves + significant_wave_height_of_first_swell_partition + significant_wave_height_of_second_swell_partition + significant_wave_height_of_third_swell_partition + skin_reservoir_content + slope_of_sub_gridscale_orography + snow_albedo + snow_density + snow_evaporation + snowfall + snowmelt + standard_deviation_of_filtered_subgrid_orography + standard_deviation_of_orography + sub_surface_runoff + surface_latent_heat_flux + surface_net_solar_radiation + surface_net_solar_radiation_clear_sky + surface_net_thermal_radiation + surface_net_thermal_radiation_clear_sky + surface_runoff + surface_sensible_heat_flux + surface_solar_radiation_downward_clear_sky + surface_solar_radiation_downwards + surface_thermal_radiation_downward_clear_sky + surface_thermal_radiation_downwards + temperature_of_snow_layer + top_net_solar_radiation + top_net_solar_radiation_clear_sky + top_net_thermal_radiation + top_net_thermal_radiation_clear_sky + total_column_cloud_ice_water + total_column_cloud_liquid_water + total_column_ozone + total_column_rain_water + total_column_snow_water + total_column_supercooled_liquid_water + total_column_water + total_sky_direct_solar_radiation_at_surface + total_totals_index + trapping_layer_base_height + trapping_layer_top_height + u_component_stokes_drift + uv_visible_albedo_for_diffuse_radiation + uv_visible_albedo_for_direct_radiation + v_component_stokes_drift + vertical_integral_of_divergence_of_cloud_frozen_water_flux + vertical_integral_of_divergence_of_cloud_liquid_water_flux + vertical_integral_of_divergence_of_geopotential_flux + vertical_integral_of_divergence_of_kinetic_energy_flux + vertical_integral_of_divergence_of_mass_flux + vertical_integral_of_divergence_of_moisture_flux + vertical_integral_of_divergence_of_ozone_flux + vertical_integral_of_divergence_of_thermal_energy_flux + vertical_integral_of_divergence_of_total_energy_flux + vertical_integral_of_eastward_cloud_frozen_water_flux + vertical_integral_of_eastward_cloud_liquid_water_flux + vertical_integral_of_eastward_geopotential_flux + vertical_integral_of_eastward_heat_flux + vertical_integral_of_eastward_kinetic_energy_flux + vertical_integral_of_eastward_mass_flux + vertical_integral_of_eastward_ozone_flux + vertical_integral_of_eastward_total_energy_flux + vertical_integral_of_eastward_water_vapour_flux + vertical_integral_of_energy_conversion + vertical_integral_of_kinetic_energy + vertical_integral_of_mass_of_atmosphere + vertical_integral_of_mass_tendency + vertical_integral_of_northward_cloud_frozen_water_flux + vertical_integral_of_northward_cloud_liquid_water_flux + vertical_integral_of_northward_geopotential_flux + vertical_integral_of_northward_heat_flux + vertical_integral_of_northward_kinetic_energy_flux + vertical_integral_of_northward_mass_flux + vertical_integral_of_northward_ozone_flux + vertical_integral_of_northward_total_energy_flux + vertical_integral_of_northward_water_vapour_flux + vertical_integral_of_potential_and_internal_energy + vertical_integral_of_potential_internal_and_latent_energy + vertical_integral_of_temperature + vertical_integral_of_thermal_energy + vertical_integral_of_total_energy + vertically_integrated_moisture_divergence + wave_spectral_directional_width + wave_spectral_directional_width_for_swell + wave_spectral_directional_width_for_wind_waves + wave_spectral_kurtosis + wave_spectral_peakedness + wave_spectral_skewness + zero_degree_level From a6da24536b791dc5bce4c2747401a5bf36d50465 Mon Sep 17 00:00:00 2001 From: dabhicusp Date: Sat, 16 Sep 2023 06:33:43 +0000 Subject: [PATCH 3/5] Source_data updated. --- src/arco_era5/source_data.py | 238 ++++++++++++++++++++++++++++++++--- 1 file changed, 222 insertions(+), 16 deletions(-) diff --git a/src/arco_era5/source_data.py b/src/arco_era5/source_data.py index 33e0456..a432a7c 100644 --- a/src/arco_era5/source_data.py +++ b/src/arco_era5/source_data.py @@ -29,23 +29,9 @@ "{day:02d}/{variable}/{pressure_level}.nc") STATIC_VARIABLES = ( - "type_of_low_vegetation", - "type_of_high_vegetation", - "standard_deviation_of_orography", - "standard_deviation_of_filtered_subgrid_orography", - "soil_type", - "slope_of_sub_gridscale_orography", - "low_vegetation_cover", - "land_sea_mask", - "lake_depth", - "lake_cover", - "high_vegetation_cover", - "geopotential_at_surface", - "anisotropy_of_sub_gridscale_orography", - "angle_of_sub_gridscale_orography", ) -# TODO(alvarosg): Add more variables. + SINGLE_LEVEL_VARIABLES = ( "total_precipitation", "total_column_water_vapour", @@ -88,7 +74,227 @@ "soil_temperature_level_4", "soil_type", "type_of_high_vegetation", - "type_of_low_vegetation" + "type_of_low_vegetation", + "100m_u_component_of_wind", + "100m_v_component_of_wind", + "10m_u_component_of_neutral_wind", + "10m_v_component_of_neutral_wind", + "10m_wind_gust_since_previous_post_processing", + "2m_dewpoint_temperature", + "air_density_over_the_oceans", + "angle_of_sub_gridscale_orography", + "anisotropy_of_sub_gridscale_orography", + "benjamin_feir_index", + "boundary_layer_dissipation", + "boundary_layer_height", + "charnock", + "clear_sky_direct_solar_radiation_at_surface", + "cloud_base_height", + "coefficient_of_drag_with_waves", + "convective_available_potential_energy", + "convective_inhibition", + "convective_precipitation", + "convective_rain_rate", + "convective_snowfall", + "convective_snowfall_rate_water_equivalent", + "downward_uv_radiation_at_the_surface", + "duct_base_height", + "eastward_gravity_wave_surface_stress", + "eastward_turbulent_surface_stress", + "evaporation", + "forecast_albedo", + "forecast_logarithm_of_surface_roughness_for_heat", + "forecast_surface_roughness", + "free_convective_velocity_over_the_oceans", + "friction_velocity", + "geopotential", + "gravity_wave_dissipation", + "high_cloud_cover", + "instantaneous_10m_wind_gust", + "instantaneous_eastward_turbulent_surface_stress", + "instantaneous_large_scale_surface_precipitation_fraction", + "instantaneous_moisture_flux", + "instantaneous_northward_turbulent_surface_stress", + "instantaneous_surface_sensible_heat_flux", + "k_index", + "lake_bottom_temperature", + "lake_shape_factor", + "large_scale_precipitation", + "large_scale_precipitation_fraction", + "large_scale_rain_rate", + "large_scale_snowfall", + "large_scale_snowfall_rate_water_equivalent", + "low_cloud_cover", + "maximum_2m_temperature_since_previous_post_processing", + "maximum_individual_wave_height", + "maximum_total_precipitation_rate_since_previous_post_processing", + "mean_boundary_layer_dissipation", + "mean_convective_precipitation_rate", + "mean_convective_snowfall_rate", + "mean_direction_of_total_swell", + "mean_direction_of_wind_waves", + "mean_eastward_gravity_wave_surface_stress", + "mean_eastward_turbulent_surface_stress", + "mean_evaporation_rate", + "mean_gravity_wave_dissipation", + "mean_large_scale_precipitation_fraction", + "mean_large_scale_precipitation_rate", + "mean_large_scale_snowfall_rate", + "mean_northward_gravity_wave_surface_stress", + "mean_northward_turbulent_surface_stress", + "mean_period_of_total_swell", + "mean_period_of_wind_waves", + "mean_potential_evaporation_rate", + "mean_runoff_rate", + "mean_snow_evaporation_rate", + "mean_snowfall_rate", + "mean_snowmelt_rate", + "mean_square_slope_of_waves", + "mean_sub_surface_runoff_rate", + "mean_surface_direct_short_wave_radiation_flux", + "mean_surface_direct_short_wave_radiation_flux_clear_sky", + "mean_surface_downward_long_wave_radiation_flux", + "mean_surface_downward_long_wave_radiation_flux_clear_sky", + "mean_surface_downward_short_wave_radiation_flux", + "mean_surface_downward_short_wave_radiation_flux_clear_sky", + "mean_surface_downward_uv_radiation_flux", + "mean_surface_latent_heat_flux", + "mean_surface_net_long_wave_radiation_flux", + "mean_surface_net_long_wave_radiation_flux_clear_sky", + "mean_surface_net_short_wave_radiation_flux", + "mean_surface_net_short_wave_radiation_flux_clear_sky", + "mean_surface_runoff_rate", + "mean_surface_sensible_heat_flux", + "mean_top_downward_short_wave_radiation_flux", + "mean_top_net_long_wave_radiation_flux_clear_sky", + "mean_top_net_short_wave_radiation_flux_clear_sky", + "mean_total_precipitation_rate", + "mean_vertical_gradient_of_refractivity_inside_trapping_layer", + "mean_vertically_integrated_moisture_divergence", + "mean_wave_direction", + "mean_wave_direction_of_first_swell_partition", + "mean_wave_direction_of_second_swell_partition", + "mean_wave_direction_of_third_swell_partition", + "mean_wave_period", + "mean_wave_period_based_on_first_moment", + "mean_wave_period_based_on_first_moment_for_swell", + "mean_wave_period_based_on_first_moment_for_wind_waves", + "mean_wave_period_based_on_second_moment_for_swell", + "mean_wave_period_based_on_second_moment_for_wind_waves", + "mean_wave_period_of_first_swell_partition", + "mean_wave_period_of_second_swell_partition", + "mean_wave_period_of_third_swell_partition", + "mean_zero_crossing_wave_period", + "medium_cloud_cover", + "minimum_2m_temperature_since_previous_post_processing", + "minimum_total_precipitation_rate_since_previous_post_processing", + "minimum_vertical_gradient_of_refractivity_inside_trapping_layer", + "model_bathymetry", + "near_ir_albedo_for_diffuse_radiation", + "near_ir_albedo_for_direct_radiation", + "normalized_energy_flux_into_ocean", + "normalized_energy_flux_into_waves", + "normalized_stress_into_ocean", + "northward_gravity_wave_surface_stress", + "northward_turbulent_surface_stress", + "ocean_surface_stress_equivalent_10m_neutral_wind_direction", + "ocean_surface_stress_equivalent_10m_neutral_wind_speed", + "peak_wave_period", + "period_corresponding_to_maximum_individual_wave_height", + "potential_evaporation", + "precipitation_type", + "runoff", + "significant_height_of_combined_wind_waves_and_swell", + "significant_height_of_total_swell", + "significant_height_of_wind_waves", + "significant_wave_height_of_first_swell_partition", + "significant_wave_height_of_second_swell_partition", + "significant_wave_height_of_third_swell_partition", + "skin_reservoir_content", + "slope_of_sub_gridscale_orography", + "snow_albedo", + "snow_density", + "snow_evaporation", + "snowfall", + "snowmelt", + "standard_deviation_of_filtered_subgrid_orography", + "standard_deviation_of_orography", + "sub_surface_runoff", + "surface_latent_heat_flux", + "surface_net_solar_radiation", + "surface_net_solar_radiation_clear_sky", + "surface_net_thermal_radiation", + "surface_net_thermal_radiation_clear_sky", + "surface_runoff", + "surface_sensible_heat_flux", + "surface_solar_radiation_downward_clear_sky", + "surface_solar_radiation_downwards", + "surface_thermal_radiation_downward_clear_sky", + "surface_thermal_radiation_downwards", + "temperature_of_snow_layer", + "top_net_solar_radiation", + "top_net_solar_radiation_clear_sky", + "top_net_thermal_radiation", + "top_net_thermal_radiation_clear_sky", + "total_column_cloud_ice_water", + "total_column_cloud_liquid_water", + "total_column_ozone", + "total_column_rain_water", + "total_column_snow_water", + "total_column_supercooled_liquid_water", + "total_column_water", + "total_sky_direct_solar_radiation_at_surface", + "total_totals_index", + "trapping_layer_base_height", + "trapping_layer_top_height", + "u_component_stokes_drift", + "uv_visible_albedo_for_diffuse_radiation", + "uv_visible_albedo_for_direct_radiation", + "v_component_stokes_drift", + "vertical_integral_of_divergence_of_cloud_frozen_water_flux", + "vertical_integral_of_divergence_of_cloud_liquid_water_flux", + "vertical_integral_of_divergence_of_geopotential_flux", + "vertical_integral_of_divergence_of_kinetic_energy_flux", + "vertical_integral_of_divergence_of_mass_flux", + "vertical_integral_of_divergence_of_moisture_flux", + "vertical_integral_of_divergence_of_ozone_flux", + "vertical_integral_of_divergence_of_thermal_energy_flux", + "vertical_integral_of_divergence_of_total_energy_flux", + "vertical_integral_of_eastward_cloud_frozen_water_flux", + "vertical_integral_of_eastward_cloud_liquid_water_flux", + "vertical_integral_of_eastward_geopotential_flux", + "vertical_integral_of_eastward_heat_flux", + "vertical_integral_of_eastward_kinetic_energy_flux", + "vertical_integral_of_eastward_mass_flux", + "vertical_integral_of_eastward_ozone_flux", + "vertical_integral_of_eastward_total_energy_flux", + "vertical_integral_of_eastward_water_vapour_flux", + "vertical_integral_of_energy_conversion", + "vertical_integral_of_kinetic_energy", + "vertical_integral_of_mass_of_atmosphere", + "vertical_integral_of_mass_tendency", + "vertical_integral_of_northward_cloud_frozen_water_flux", + "vertical_integral_of_northward_cloud_liquid_water_flux", + "vertical_integral_of_northward_geopotential_flux", + "vertical_integral_of_northward_heat_flux", + "vertical_integral_of_northward_kinetic_energy_flux", + "vertical_integral_of_northward_mass_flux", + "vertical_integral_of_northward_ozone_flux", + "vertical_integral_of_northward_total_energy_flux", + "vertical_integral_of_northward_water_vapour_flux", + "vertical_integral_of_potential_and_internal_energy", + "vertical_integral_of_potential_internal_and_latent_energy", + "vertical_integral_of_temperature", + "vertical_integral_of_thermal_energy", + "vertical_integral_of_total_energy", + "vertically_integrated_moisture_divergence", + "wave_spectral_directional_width", + "wave_spectral_directional_width_for_swell", + "wave_spectral_directional_width_for_wind_waves", + "wave_spectral_kurtosis", + "wave_spectral_peakedness", + "wave_spectral_skewness", + "zero_degree_level" ) MULTILEVEL_VARIABLES = ( From c9fbd4c184bd7331b2c6a21d5fa74da7869c603e Mon Sep 17 00:00:00 2001 From: dabhicusp Date: Sat, 16 Sep 2023 13:00:59 +0000 Subject: [PATCH 4/5] Geo-potential is added. --- src/arco_era5/source_data.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/arco_era5/source_data.py b/src/arco_era5/source_data.py index a432a7c..24b36b2 100644 --- a/src/arco_era5/source_data.py +++ b/src/arco_era5/source_data.py @@ -107,7 +107,7 @@ "forecast_surface_roughness", "free_convective_velocity_over_the_oceans", "friction_velocity", - "geopotential", + "geopotential_at_surface", "gravity_wave_dissipation", "high_cloud_cover", "instantaneous_10m_wind_gust", @@ -396,8 +396,12 @@ def read_single_level_vars(year, month, day, variables=SINGLE_LEVEL_VARIABLES, root_path = pathlib.Path(root_path) output = {} for variable in variables: + if variable in _VARIABLE_TO_ERA5_FILE_NAME: + era5_variable = _VARIABLE_TO_ERA5_FILE_NAME[variable] + else: + era5_variable = variable relative_path = SINGLE_LEVEL_SUBDIR_TEMPLATE.format( - year=year, month=month, day=day, variable=variable) + year=year, month=month, day=day, variable=era5_variable) output[variable] = _read_nc_dataset(root_path / relative_path) return xarray.Dataset(output) @@ -441,7 +445,7 @@ def get_var_attrs_dict(root_path=GCP_DIRECTORY): var_attrs_dict = {} for variables, template, rename_dict in [ (STATIC_VARIABLES, STATIC_SUBDIR_TEMPLATE, _VARIABLE_TO_ERA5_FILE_NAME), - (SINGLE_LEVEL_VARIABLES, SINGLE_LEVEL_SUBDIR_TEMPLATE, {}), + (SINGLE_LEVEL_VARIABLES, SINGLE_LEVEL_SUBDIR_TEMPLATE, _VARIABLE_TO_ERA5_FILE_NAME), (MULTILEVEL_VARIABLES, MULTILEVEL_SUBDIR_TEMPLATE, {}), ]: for variable in variables: From 9bea4348046bec64f804a37b0579c102cc67aec0 Mon Sep 17 00:00:00 2001 From: dabhicusp Date: Mon, 18 Sep 2023 11:11:38 +0000 Subject: [PATCH 5/5] Static variables were removed from pipeline. --- src/arco_era5/__init__.py | 4 ++-- src/arco_era5/source_data.py | 21 ------------------ src/netcdf_to_zarr.py | 43 +----------------------------------- 3 files changed, 3 insertions(+), 65 deletions(-) diff --git a/src/arco_era5/__init__.py b/src/arco_era5/__init__.py index 3c457f5..95a3810 100644 --- a/src/arco_era5/__init__.py +++ b/src/arco_era5/__init__.py @@ -11,6 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from .source_data import GCP_DIRECTORY,STATIC_VARIABLES,SINGLE_LEVEL_VARIABLES,MULTILEVEL_VARIABLES,PRESSURE_LEVELS_GROUPS, TIME_RESOLUTION_HOURS -from .source_data import get_var_attrs_dict, read_multilevel_vars, read_static_vars, read_single_level_vars, daily_date_iterator, align_coordinates, parse_arguments +from .source_data import GCP_DIRECTORY,SINGLE_LEVEL_VARIABLES,MULTILEVEL_VARIABLES,PRESSURE_LEVELS_GROUPS, TIME_RESOLUTION_HOURS +from .source_data import get_var_attrs_dict, read_multilevel_vars, read_single_level_vars, daily_date_iterator, align_coordinates, parse_arguments from .pangeo import run, parse_args diff --git a/src/arco_era5/source_data.py b/src/arco_era5/source_data.py index 24b36b2..b3f02ea 100644 --- a/src/arco_era5/source_data.py +++ b/src/arco_era5/source_data.py @@ -18,8 +18,6 @@ GCP_DIRECTORY = "gs://gcp-public-data-arco-era5/raw" -STATIC_SUBDIR_TEMPLATE = "date-variable-static/2021/12/31/{variable}/static.nc" - SINGLE_LEVEL_SUBDIR_TEMPLATE = ( "date-variable-single_level/{year}/{month:02d}/" "{day:02d}/{variable}/surface.nc") @@ -28,10 +26,6 @@ "date-variable-pressure_level/{year}/{month:02d}/" "{day:02d}/{variable}/{pressure_level}.nc") -STATIC_VARIABLES = ( -) - - SINGLE_LEVEL_VARIABLES = ( "total_precipitation", "total_column_water_vapour", @@ -376,20 +370,6 @@ def _read_nc_dataset(gpath_file): return dataarray -def read_static_vars(variables=STATIC_VARIABLES, root_path=GCP_DIRECTORY): - """xarray.Dataset with static variables for single level data from nc files.""" - root_path = pathlib.Path(root_path) - output = {} - for variable in variables: - if variable in _VARIABLE_TO_ERA5_FILE_NAME: - era5_variable = _VARIABLE_TO_ERA5_FILE_NAME[variable] - else: - era5_variable = variable - relative_path = STATIC_SUBDIR_TEMPLATE.format(variable=era5_variable) - output[variable] = _read_nc_dataset(root_path / relative_path) - return xarray.Dataset(output) - - def read_single_level_vars(year, month, day, variables=SINGLE_LEVEL_VARIABLES, root_path=GCP_DIRECTORY): """xarray.Dataset with variables for singel level data from nc files.""" @@ -444,7 +424,6 @@ def get_var_attrs_dict(root_path=GCP_DIRECTORY): # have to download the data files to get this into. var_attrs_dict = {} for variables, template, rename_dict in [ - (STATIC_VARIABLES, STATIC_SUBDIR_TEMPLATE, _VARIABLE_TO_ERA5_FILE_NAME), (SINGLE_LEVEL_VARIABLES, SINGLE_LEVEL_SUBDIR_TEMPLATE, _VARIABLE_TO_ERA5_FILE_NAME), (MULTILEVEL_VARIABLES, MULTILEVEL_SUBDIR_TEMPLATE, {}), ]: diff --git a/src/netcdf_to_zarr.py b/src/netcdf_to_zarr.py index 6cf0293..987a1d0 100644 --- a/src/netcdf_to_zarr.py +++ b/src/netcdf_to_zarr.py @@ -52,7 +52,6 @@ from arco_era5 import ( GCP_DIRECTORY, - STATIC_VARIABLES, SINGLE_LEVEL_VARIABLES, MULTILEVEL_VARIABLES, PRESSURE_LEVELS_GROUPS, @@ -60,7 +59,6 @@ get_var_attrs_dict, read_multilevel_vars, read_single_level_vars, - read_static_vars, daily_date_iterator, align_coordinates, parse_arguments @@ -115,14 +113,6 @@ def make_template(data_path: str, start_date: str, end_date: str, time_chunk_siz time_size = len(coords["time"]) template_dataset = {} - for name in STATIC_VARIABLES: - template_dataset[name] = xa.Variable( - dims=("latitude", "longitude"), - data=dask.array.zeros( - shape=(lat_size, lon_size), - dtype=np.float32), - attrs=var_attrs_dict[name]) - for name in SINGLE_LEVEL_VARIABLES: template_dataset[name] = xa.Variable( dims=("time", "latitude", "longitude"), @@ -201,28 +191,6 @@ def offset_along_time_axis(start_date: str, year: int, month: int, day: int) -> return time_delta.days * _HOURS_PER_DAY // TIME_RESOLUTION_HOURS -def load_static_data(args, data_path: str) -> t.Tuple[xb.Key, xa.Dataset]: - """Loads all static data, with an xarray_beam key for it..""" - - logging.info("Loading static data") - - dataset = read_static_vars( - variables=STATIC_VARIABLES, root_path=data_path) - - logging.info("static data loaded.") - # It is crucial to actually "load" as otherwise we get a pickle error. - dataset = dataset.load() - - # Technically the static data has a time coordinate, but we don't need it. - dataset = dataset.squeeze("time").drop("time") - dataset = align_coordinates(dataset) - - offsets = {"latitude": 0, "longitude": 0} - key = xb.Key(offsets, vars=set(dataset.data_vars.keys())) - logging.info("Finished loading static data") - return key, dataset - - def define_pipeline( root: beam.Pipeline, input_path: str, @@ -272,17 +240,8 @@ def define_pipeline( | "ChunksToZarrTemporal" >> chunks_to_zarr ) - logging.info("Setting up static variables.") - static_variables_chunks = ( - root - # This is a single element with no parameters. - | "DummySingleElement" >> beam.Create(range(1)) - | "StaticVariableFromNetCDFFile" >> beam.Map(load_static_data, - data_path=input_path) - | "ChunksToZarrStatic" >> chunks_to_zarr - ) logging.info("Finished defining pipeline.") - return (temporal_variables_chunks, static_variables_chunks) + return temporal_variables_chunks def main():