diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index cc2c87dfeb..9ab5068274 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -615,12 +615,12 @@ Key Description Default value if not ``modeling_realm`` Realm attribute include `atm`, `ice` No default (needs to be and `oce` specified in extra facets or recipe if default DRS is used) -```special_attr`` A special attribute in the filename No default +``frequency_attribute`` A special attribute in the filename No default `ACCESS-ESM` raw data, it's related to frquency of raw data ``sub_dataset`` Part of the ACCESS-ESM raw dataset No default root, need to specify if you want to - use the cmoriser + use the cmoriser ==================== ====================================== ================================= .. _data-retrieval: diff --git a/esmvalcore/cmor/_fixes/access/_base_fix.py b/esmvalcore/cmor/_fixes/access/_base_fix.py index 659fffd364..c376801c20 100644 --- a/esmvalcore/cmor/_fixes/access/_base_fix.py +++ b/esmvalcore/cmor/_fixes/access/_base_fix.py @@ -2,6 +2,8 @@ import logging +import numpy as np + from iris.cube import CubeList from esmvalcore.cmor._fixes.native_datasets import NativeDatasetFix @@ -27,3 +29,37 @@ def get_cubes_from_multivar(self, cubes): for name in name_list: data_list.append(self.get_cube(cubes, name)) return CubeList(data_list) + + def fix_ocean_dim_coords(self, cube): + """Fix dim coords of ocean variables""" + cube.dim_coords[-2].points = np.array([int(i) for i in range(300)]) + cube.dim_coords[-2].standard_name = None + cube.dim_coords[-2].var_name = 'j' + cube.dim_coords[-2].long_name = 'cell index along second dimension' + cube.dim_coords[-2].attributes = None + + cube.dim_coords[-1].points = np.array([int(i) for i in range(360)]) + cube.dim_coords[-1].standard_name = None + cube.dim_coords[-1].var_name = 'i' + cube.dim_coords[-1].long_name = 'cell index along first dimension' + cube.dim_coords[-1].attributes = None + + def fix_ocean_aux_coords(self, cube): + """Fix aux coords of ocean variables""" + temp_points=[] + for i in cube.aux_coords[-1].points: + temp_points.append([j + 360 for j in i if j < 0]+[j for j in i if j >= 0]) + cube.aux_coords[-1].points = np.array(temp_points) + cube.aux_coords[-1].standard_name = 'longitude' + cube.aux_coords[-1].long_name = 'longitude' + cube.aux_coords[-1].var_name = 'longitude' + cube.aux_coords[-1].attributes = None + + temp_points=[] + for i in cube.aux_coords[-2].points: + temp_points.append([j.astype(np.float64) for j in i]) + cube.aux_coords[-2].points = np.array(temp_points) + cube.aux_coords[-2].standard_name = 'latitude' + cube.aux_coords[-2].long_name = 'latitude' + cube.aux_coords[-2].var_name = 'latitude' + cube.aux_coords[-2].attributes = None diff --git a/esmvalcore/cmor/_fixes/access/access_esm1_5.py b/esmvalcore/cmor/_fixes/access/access_esm1_5.py index 6b8bc662e7..b4d24568d9 100644 --- a/esmvalcore/cmor/_fixes/access/access_esm1_5.py +++ b/esmvalcore/cmor/_fixes/access/access_esm1_5.py @@ -5,6 +5,8 @@ from ._base_fix import AccessFix +from cf_units import Unit + logger = logging.getLogger(__name__) @@ -125,3 +127,65 @@ def fix_height_value(self, cube): """Fix height value to make it comparable to other dataset.""" if cube.coord('height').points[0] != 2: cube.coord('height').points = [2] + +class Tos(AccessFix): + """Fixes for Tos""" + + def fix_metadata(self, cubes): + """Fix metadata. + + Parameters + ---------- + cubes : iris.cube.CubeList + Input cubes. + + Returns + ------- + iris.cube.CubeList + """ + + cube = self.get_cube(cubes) + + self.fix_ocean_dim_coords(cube) + self.fix_ocean_aux_coords(cube) + + return CubeList([cube]) + + +class So(AccessFix): + """FIxes for So""" + + def fix_metadata(self, cubes): + """Fix metadata. + + Parameters + ---------- + cubes : iris.cube.CubeList + Input cubes. + + Returns + ------- + iris.cube.CubeList + """ + + cube = self.get_cube(cubes) + + self.fix_ocean_dim_coords(cube) + self.fix_ocean_aux_coords(cube) + self.fix_depth_metadata(cube) + self.fix_so_units(cube) + + return CubeList([cube]) + + def fix_depth_metadata(self, cube): + """fix depth metadata""" + cube.dim_coords[1].standard_name = 'depth' + cube.dim_coords[1].long_name = 'ocean depth coordinate' + cube.dim_coords[1].var_name = 'lev' + cube.dim_coords[1].attributes = {'positive':'down'} + + def fix_so_units(self, cube): + """fix units of so""" + cube.attributes.pop('invalid_units') + cube.units=Unit(0.001) + \ No newline at end of file diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml index c81324142a..bbd751911c 100644 --- a/esmvalcore/config-developer.yml +++ b/esmvalcore/config-developer.yml @@ -200,8 +200,11 @@ ACCESS: input_dir: default: - '{dataset}/{sub_dataset}/{exp}/{modeling_realm}/netCDF' + - '{dataset}/{sub_dataset}/{exp}/{modeling_realm}/' input_file: - default: '{sub_dataset}.{special_attr}-*.nc' + default: + - '{sub_dataset}.{frequency_attribute}-2000*.nc' + - 'ocean_{frequency_attribute}.nc-2000*' output_file: '{project}_{dataset}_{mip}_{exp}_{institute}_{sub_dataset}_{special_attr}_{short_name}' cmor_type: 'CMIP6' cmor_default_table_prefix: 'CMIP6_' diff --git a/esmvalcore/config/extra_facets/access-mappings.yml b/esmvalcore/config/extra_facets/access-mappings.yml index 9d4eb0621b..884c0b3596 100644 --- a/esmvalcore/config/extra_facets/access-mappings.yml +++ b/esmvalcore/config/extra_facets/access-mappings.yml @@ -7,6 +7,7 @@ ACCESS-ESM1-5: '*': + # atm tas: raw_name: fld_s03i236 @@ -66,3 +67,121 @@ ACCESS-ESM1-5: - fld_s01i201 modeling_realm: atm + clivi: + raw_name: fld_s30i406 + modeling_realm: atm + + evspsbl: + raw_name: fld_s03i223 + modeling_realm: atm + + hfls: + raw_name: fld_s03i234 + modeling_realm: atm + + hfss: + raw_name: fld_s03i217 + modeling_realm: atm + + hur: + raw_name: fld_s30i206 + modeling_realm: atm + + hurs: + raw_name: fld_s03i245 + modeling_realm: atm + + huss: + raw_name: fld_s03i237 + modeling_realm: atm + + prsn: + raw_name: fld_s05i215 + modeling_realm: atm + + rldscs: + raw_name: fld_s02i208 + modeling_realm: atm + + rsdt: + raw_name: fld_s01i207 + modeling_realm: atm + + rsuscs: + raw_name: fld_s01i211 + modeling_realm: atm + + rsut: + raw_name: fld_s01i208 + modeling_realm: atm + + rsutcs: + raw_name: fld_s01i209 + modeling_realm: atm + + sci: + raw_name: fld_s05i270 + modeling_realm: atm + + sfcWind: + raw_name: fld_s03i230 + modeling_realm: atm + + sfcWindmax: + raw_name: fld_s03i227_max + modeling_realm: atm + + tauu: + raw_name: fld_s03i460 + modeling_realm: atm + + tauv: + raw_name: fld_s03i461 + modeling_realm: atm + + ts: + raw_name: fld_s00i024 + modeling_realm: atm + + ua: + raw_name: fld_s30i201 + modeling_realm: atm + + uas: + raw_name: fld_s03i209 + modeling_realm: atm + + va: + raw_name: fld_s30i202 + modeling_realm: atm + + vas: + raw_name: fld_s03i210 + modeling_realm: atm + + wap: + raw_name: fld_s30i208 + modeling_realm: atm + + zg: + raw_name: fld_s30i207 + modeling_realm: atm + + rls: + raw_name: fld_s02i201 + modeling_realm: atm + + rss: + raw_name: fld_s01i201 + modeling_realm: atm + + # ocean + + tos: + raw_name: sst + modeling_realm: ocn + + so: + raw_name: salt + modeling_realm: ocn + diff --git a/test_recipe/recipe_access_noncmip_test.yml b/test_recipe/recipe_access_noncmip_test.yml new file mode 100644 index 0000000000..70f8ec8322 --- /dev/null +++ b/test_recipe/recipe_access_noncmip_test.yml @@ -0,0 +1,163 @@ +# ESMValTool +# recipe_access_noncmip.yml + +documentation: + title: ACCESS-NONCMIP-TEST + + description: ACCESS-NONCMIP-TEST + + authors: + - unmaintained + + maintainer: + - unmaintained + + +datasets: + - {project: ACCESS, dataset: ACCESS-ESM1-5, mip: Amon, sub_dataset: HI-CN-05, exp: history, modeling_realm: ocn, frequency_filename: month, start_year: 2000, end_year: 2000} + + # - {dataset: ACCESS-ESM1-5, activity: CMIP ,project: CMIP6, grid: gn, + # exp: historical, ensemble: r1i1p1f1, start_year: 1986, end_year: 1986} + # - {dataset: UKESM1-0-LL, activity: CMIP ,project: CMIP6, grid: gn, # + # exp: historical, ensemble: r1i1p1f2, start_year: 1986, end_year: 1986} + # - {dataset: CESM2, activity: CMIP ,project: CMIP6, grid: gn, # institution NCAR + # exp: historical, ensemble: r1i1p1f1, start_year: 1986, end_year: 1986} + +preprocessors: + statistics_test: + multi_model_statistics: + ignore_scalar_coords: true + span: overlap + statistics: [mean, gmean, hmean, max, mean, median, min, peak, {operrator: percentile, percent: 50}, rms, std_dev, sum, variance] + exclude: [reference_dataset] + annual_statistics: + operator: mean + area_statistics: + operator: median + axis_statistics: + axis: y + operator: mean + normalize: divide + climate_statistics: + operator: max + seasons: DJF + daily_statistics: + operator: peak + decadal_statistics: + operator: min + ensemble_statistics: + ignore_scalar_coords: true + span: overlap + statistics: [mean, gmean, hmean, max, mean, median, min, peak, {operrator:percentile, percent:50}, rms, std_dev, sum, variance] + hourly_statistics: + hours: 12 + meridional_statistics: + operator: rms + normalize: subtract + monthly_statistics: + operator: mean + rolling_window_statistics: + coordinate: latitude + operator: std_dev + seasonal_statistics: + operator: variance + seasons: MAM + volume_statistics: + operator: sum + normalize: divide + zonal_statistics: + operator: variance + normalize: divide + + other_test: + accumulate_coordinate: + coordinate: latitude + add_supplementary_variables: + supplementary_cubes: [tas] + amplitude: + coords: ['day_of_year', 'year'] + anomalies: + period: mon + standardize: true + seasons: JJA + bias: + bias_type: absolute + clip: + minimum: 0 + clip_timerange: + timerange: 1985-01-16 00:00:00.000/1985-02-16 00:00:00.000 + concatenate: + check_level: strict + detrend: + dimention: y + method: constant + distance_metric: + metric: rmse + coords: [x,y] + extract_levels: + levels: [0., 10., 100., 1000.] + scheme: linear_extrapolate + extract_region: + start_longitude: -80. + end_longitude: 30. + start_latitude: -80. + end_latitude: 80. + regrid: + target_grid: 2x2 + scheme: linear + mask_fillvalues: + threshold_fraction: 0.95 + + + +diagnostics: + access_dataload_test: + variables: + + tos_test: + short_name: tos + mip: Omon + preprocessors: [statistics_test, other_test] + so_test: + short_name: so + mip: Omon + preprocessors: [statistics_test, other_test] + + # preprocessor: annual_mean_global # can remove this + # pr: + # psl: + # additional_datasets: + # - {project: ACCESS, institute: ACCESS-ESM1-5, mip: Amon, dataset: HI-CN-05, exp: history, modeling_realm: atm, special_attr: pa} + scripts: null + +# diagnostics: + + # ********************************************************************** + # Flato et al. (2013) - IPCC AR5, chap. 9 + # similar to fig. 9.4 + # ********************************************************************** + # Multi model mean, multi model mean bias, mean absolute error, and + # mean relative error (geographical ditributions) + # ********************************************************************** + + # clouds_bias_pr: + # title: Precipitation climatology (MMM) + # description: IPCC AR5 Ch. 9, Fig. 9.4 (precipitation) + # themes: + # - clouds + # realms: + # - atmos + # variables: + # pr: + # preprocessor: [statistics_test, other_test] + # reference_dataset: GPCP-V2.2 + # mip: Amon + # additional_datasets: + # - {dataset: GPCP-V2.2, project: obs4MIPs, level: L3, start_year: 1986, end_year: 1986, tier: 1} + # scripts: + # clim: + # script: clouds/clouds_bias.ncl + # projection: Robinson + # timemean: annualclim + # plot_abs_diff: true + # plot_rel_diff: true \ No newline at end of file diff --git a/tests/integration/cmor/_fixes/access/test_access_esm1_5.py b/tests/integration/cmor/_fixes/access/test_access_esm1_5.py index 01fb0e0529..4ea7071a2e 100644 --- a/tests/integration/cmor/_fixes/access/test_access_esm1_5.py +++ b/tests/integration/cmor/_fixes/access/test_access_esm1_5.py @@ -5,10 +5,10 @@ import numpy as np import pytest from cf_units import Unit -from iris.coords import DimCoord +from iris.coords import DimCoord, AuxCoord from iris.cube import Cube, CubeList -import esmvalcore.cmor._fixes.access.access_esm1_5 +import esmvalcore.cmor._fixes.access.access_esm1_5 from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix from esmvalcore.cmor.table import CoordinateInfo, get_var_info @@ -25,6 +25,74 @@ 'time_origin': 'will_be_removed' }, ) + +time_ocn_coord = DimCoord( + [int(i) for i in range(1,13)], + standard_name = 'time', + var_name = 'time', + long_name = 'time', + units = Unit('days since 0000-01-01', calendar='noleap'), + attributes = { + 'calendar_type': 'GREGORIAN', + 'cartesian_axis': 'T' + }, +) + +lat_ocn_coord = DimCoord( + np.linspace(-90,210,300), + standard_name ='latitude', + long_name ='tcell latitude', + var_name ='yt_ocean', + units = 'degrees_N', + attributes = { + 'cartesian_axis' :'Y', + } +) + +lon_ocn_coord = DimCoord( + np.linspace(-90,270,360), + standard_name ='longitude', + long_name ='tcell longitude', + var_name ='xt_ocean', + units = 'degrees_E', + attributes = { + 'cartesian_axis' :'X', + } +) + +depth_ocn_coord = DimCoord( + [0, 1], + long_name = 'tcell zstar depth', + var_name = 'st_ocean', + units = 'meter', + attributes = { + 'cartesian_axis' :'Z', + 'edges' : 'st_edges_ocean', + 'positive' : 'down', + }, +) + +lat_ocn_aux_coord = AuxCoord( + da.arange(300 * 360, dtype=np.float32).reshape(300, 360), + standard_name ='latitude', + long_name ='tracer latitude', + var_name ='geolat_t', + attributes = { + 'valid_range' :'[-91. 91]', + } +) + +lon_ocn_aux_coord = AuxCoord( + da.arange(300 * 360, dtype=np.float32).reshape(300, 360), + standard_name ='longitude', + long_name ='tracer longitude', + var_name ='geolon_t', + attributes = { + 'valid_range' :'[-281. 361]', + } +) + + lat_coord = DimCoord( [0, 10], standard_name='latitude', @@ -97,6 +165,25 @@ def check_tas_metadata(cubes): assert 'positive' not in cube.attributes return cube +def check_tos_metadata(cubes): + """Check tas metadata.""" + assert len(cubes) == 1 + cube= cubes[0] + assert cube.var_name == 'tos' + assert cube.standard_name == 'sea_surface_temperature' + assert cube.long_name == 'Sea Surface Temperature' + assert cube.units == 'degC' + return cube + +def check_so_metadata(cubes): + """Check tas metadata.""" + assert len(cubes) == 1 + cube= cubes[0] + assert cube.var_name == 'so' + assert cube.standard_name == 'sea_water_salinity' + assert cube.long_name == 'Sea Water Salinity' + assert cube.units == Unit(0.001) + return cube def check_pr_metadata(cubes): """Check pr metadata.""" @@ -129,6 +216,9 @@ def check_lat(cube): assert lat.units == 'degrees_north' assert lat.attributes == {} +def check_ocn_lat(cube): + """Check latitude coordinate of ocean variable cube.""" + def check_lon(cube): """Check longitude coordinate of cube.""" @@ -151,6 +241,37 @@ def check_heightxm(cube, height_value): np.testing.assert_allclose(height.points, [height_value]) assert height.bounds is None +def check_ocean_dim_coords(cube): + """Check dim_coords of ocean variables""" + assert cube.dim_coords[-2].points == np.array([int(i) for i in range(300)]) + assert cube.dim_coords[-2].standard_name == None + assert cube.dim_coords[-2].var_name == 'j' + assert cube.dim_coords[-2].long_name == 'cell index along second dimension' + assert cube.dim_coords[-2].attributes == None + + assert cube.dim_coords[-1].points == np.array([int(i) for i in range(360)]) + assert cube.dim_coords[-1].standard_name == None + assert cube.dim_coords[-1].var_name == 'i' + assert cube.dim_coords[-1].long_name == 'cell index along first dimension' + assert cube.dim_coords[-1].attributes == None + +def check_ocean_aux_coords(cube): + """Check aux_coords of ocean variables""" + assert cube.aux_coords[-2].shape ==(300,360) + assert cube.aux_coords[-2].dtype == np.dtype('float64') + assert cube.aux_coords[-2].standard_name == 'latitude' + assert cube.aux_coords[-2].long_name == 'latitude' + assert cube.aux_coords[-2].var_name == 'latitude' + assert cube.aux_coords[-2].attributes == None + + assert cube.aux_coords[-1].shape ==(300,360) + assert max(cube.aux_coords[-1].point) < 360 + assert min(cube.aux_coords[-1].point) > 0 + assert cube.aux_coords[-1].standard_name == 'longitude' + assert cube.aux_coords[-1].long_name == 'longitude' + assert cube.aux_coords[-1].var_name == 'longitude' + assert cube.aux_coords[-1].attributes == None + def assert_plev_metadata(cube): """Assert plev metadata is correct.""" @@ -410,3 +531,78 @@ def test_rlus_fix(): fix = get_fix('Amon', 'mon', 'rlus') fixed_cubes = fix.fix_metadata(cubes_3d) np.testing.assert_allclose(fixed_cubes[0].data, cube_result.data) + +def test_tos_fix(): + """Test fix 'tos'""" + coord_dim = [ + (time_ocn_coord, 0), + (lat_ocn_coord, 1), + (lon_ocn_coord, 2), + ] + + coord_aux = [ + (lat_ocn_aux_coord, (1, 2)), + (lon_ocn_aux_coord, (1, 2)), + ] + + cube_tos= Cube( + da.arange(12 * 300 * 360, dtype=np.float32).reshape(12, 300, 360), + var_name='sst', + units=Unit('degrees K'), + dim_coords_and_dims=coord_dim, + aux_coords_and_dims=coord_aux, + attributes={}, + ) + + cubes_tos=CubeList([cube_tos]) + fix_tos = get_fix('Omon', 'mon', 'tos') + fix_allvar = get_fix_allvar('Omon', 'mon', 'tos') + fixed_cubes = fix_tos.fix_metadata(cubes_tos) + print(fixed_cubes) + fixed_cubes = fix_allvar.fix_metadata(fixed_cubes) + fixed_cube=check_tos_metadata(fixed_cubes) + + check_ocean_dim_coords(fixed_cube) + check_ocean_aux_coords(fixed_cube) + assert fixed_cube.shape == (12, 300, 360) + + +def test_so_fix(): + """Test fix 'so'""" + coord_dim = [ + (time_ocn_coord, 0), + (depth_ocn_coord, 1), + (lat_ocn_coord, 2), + (lon_ocn_coord, 3), + ] + + coord_aux = [ + (lat_ocn_aux_coord, (2, 3)), + (lon_ocn_aux_coord, (2, 3)), + ] + + cube_so= Cube( + da.arange(12 * 2 * 300 * 360, dtype=np.float32).reshape(12, 2, 300, 360), + var_name='salt', + units='unknown', + dim_coords_and_dims=coord_dim, + aux_coords_and_dims=coord_aux, + attributes={ + 'invalid_units': 'psu', + }, + ) + + cubes_so=CubeList([cube_so]) + fix_so = get_fix('Omon', 'mon', 'so') + fix_allvar = get_fix_allvar('Omon', 'mon', 'so') + fixed_cubes = fix_so.fix_metadata(cubes_so) + fixed_cubes = fix_allvar.fix_metadata(fixed_cubes) + fixed_cube=check_so_metadata(fixed_cubes) + + check_ocean_dim_coords(fixed_cube) + check_ocean_aux_coords(fixed_cube) + assert fixed_cube.shape == (12, 2, 300, 360) + + + +