From 8b4a63afdf11a153c23f51bbe0c3a4fc856b0a71 Mon Sep 17 00:00:00 2001 From: Karen Garcia Perdomo <85649962+Karen-A-Garcia@users.noreply.github.com> Date: Fri, 29 Nov 2024 08:06:57 -0800 Subject: [PATCH] Missing 2m height coordinate and monotonicity for tasmin in CESM2 and CESM2-WACCM (#2574) Co-authored-by: Karen Garcia Perdomo Co-authored-by: Valeriu Predoi --- esmvalcore/cmor/_fixes/cmip6/cesm2.py | 187 +++++++++++++++- esmvalcore/cmor/_fixes/cmip6/cesm2_fv2.py | 4 + esmvalcore/cmor/_fixes/cmip6/cesm2_waccm.py | 12 + .../cmor/_fixes/cmip6/cesm2_waccm_fv2.py | 4 + .../cmor/_fixes/cmip6/test_cesm2.py | 207 +++++++++++++++++- .../cmor/_fixes/cmip6/test_cesm2_fv2.py | 15 ++ .../cmor/_fixes/cmip6/test_cesm2_waccm.py | 41 ++++ .../cmor/_fixes/cmip6/test_cesm2_waccm_fv2.py | 15 ++ 8 files changed, 476 insertions(+), 9 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cmip6/cesm2.py b/esmvalcore/cmor/_fixes/cmip6/cesm2.py index 0c5c0eed94..9b190adc63 100644 --- a/esmvalcore/cmor/_fixes/cmip6/cesm2.py +++ b/esmvalcore/cmor/_fixes/cmip6/cesm2.py @@ -2,6 +2,7 @@ from shutil import copyfile +import iris import numpy as np from netCDF4 import Dataset @@ -160,7 +161,7 @@ class Tas(Prw): def fix_metadata(self, cubes): """ - Add height (2m) coordinate. + Add height (2m) coordinate and time coordinate. Fix also done for prw. Fix latitude_bounds and longitude_bounds data type and round to 4 d.p. @@ -172,15 +173,72 @@ def fix_metadata(self, cubes): Returns ------- - iris.cube.CubeList + iris.cube.CubeList, iris.cube.CubeList """ super().fix_metadata(cubes) # Specific code for tas cube = self.get_cube_from_list(cubes) add_scalar_height_coord(cube) - - return cubes + new_list = iris.cube.CubeList() + for cube in cubes: + try: + old_time = cube.coord("time") + except iris.exceptions.CoordinateNotFoundError: + new_list.append(cube) + else: + if old_time.is_monotonic(): + new_list.append(cube) + else: + time_units = old_time.units + time_data = old_time.points + + # erase erroneously copy-pasted points + time_diff = np.diff(time_data) + idx_neg = np.where(time_diff <= 0.0)[0] + while len(idx_neg) > 0: + time_data = np.delete(time_data, idx_neg[0] + 1) + time_diff = np.diff(time_data) + idx_neg = np.where(time_diff <= 0.0)[0] + + # create the new time coord + new_time = iris.coords.DimCoord( + time_data, + standard_name="time", + var_name="time", + units=time_units, + ) + + # create a new cube with the right shape + dims = ( + time_data.shape[0], + cube.coord("latitude").shape[0], + cube.coord("longitude").shape[0], + ) + data = cube.data + new_data = np.ma.append( + data[: dims[0] - 1, :, :], data[-1, :, :] + ) + new_data = new_data.reshape(dims) + + tmp_cube = iris.cube.Cube( + new_data, + standard_name=cube.standard_name, + long_name=cube.long_name, + var_name=cube.var_name, + units=cube.units, + attributes=cube.attributes, + cell_methods=cube.cell_methods, + dim_coords_and_dims=[ + (new_time, 0), + (cube.coord("latitude"), 1), + (cube.coord("longitude"), 2), + ], + ) + + new_list.append(tmp_cube) + + return new_list class Sftlf(Fix): @@ -286,3 +344,124 @@ def fix_metadata(self, cubes): if z_coord.standard_name is None: fix_ocean_depth_coord(cube) return cubes + + +class Pr(Fix): + """Fixes for pr.""" + + def fix_metadata(self, cubes): + """Fix time coordinates. + + Parameters + ---------- + cubes : iris.cube.CubeList + Cubes to fix + + Returns + ------- + iris.cube.CubeList + """ + new_list = iris.cube.CubeList() + for cube in cubes: + try: + old_time = cube.coord("time") + except iris.exceptions.CoordinateNotFoundError: + new_list.append(cube) + else: + if old_time.is_monotonic(): + new_list.append(cube) + else: + time_units = old_time.units + time_data = old_time.points + + # erase erroneously copy-pasted points + time_diff = np.diff(time_data) + idx_neg = np.where(time_diff <= 0.0)[0] + while len(idx_neg) > 0: + time_data = np.delete(time_data, idx_neg[0] + 1) + time_diff = np.diff(time_data) + idx_neg = np.where(time_diff <= 0.0)[0] + + # create the new time coord + new_time = iris.coords.DimCoord( + time_data, + standard_name="time", + var_name="time", + units=time_units, + ) + + # create a new cube with the right shape + dims = ( + time_data.shape[0], + cube.coord("latitude").shape[0], + cube.coord("longitude").shape[0], + ) + data = cube.data + new_data = np.ma.append( + data[: dims[0] - 1, :, :], data[-1, :, :] + ) + new_data = new_data.reshape(dims) + + tmp_cube = iris.cube.Cube( + new_data, + standard_name=cube.standard_name, + long_name=cube.long_name, + var_name=cube.var_name, + units=cube.units, + attributes=cube.attributes, + cell_methods=cube.cell_methods, + dim_coords_and_dims=[ + (new_time, 0), + (cube.coord("latitude"), 1), + (cube.coord("longitude"), 2), + ], + ) + + new_list.append(tmp_cube) + return new_list + + +class Tasmin(Pr): + """Fixes for tasmin.""" + + def fix_metadata(self, cubes): + """Fix time and height 2m coordinates. + + Fix for time coming from Pr. + + Parameters + ---------- + cubes : iris.cube.CubeList + Cubes to fix + + Returns + ------- + iris.cube.CubeList + + """ + for cube in cubes: + add_scalar_height_coord(cube, height=2.0) + return cubes + + +class Tasmax(Pr): + """Fixes for tasmax.""" + + def fix_metadata(self, cubes): + """Fix time and height 2m coordinates. + + Fix for time coming from Pr. + + Parameters + ---------- + cubes : iris.cube.CubeList + Cubes to fix + + Returns + ------- + iris.cube.CubeList + + """ + for cube in cubes: + add_scalar_height_coord(cube, height=2.0) + return cubes diff --git a/esmvalcore/cmor/_fixes/cmip6/cesm2_fv2.py b/esmvalcore/cmor/_fixes/cmip6/cesm2_fv2.py index 4c55a20f03..2ce4b1073f 100644 --- a/esmvalcore/cmor/_fixes/cmip6/cesm2_fv2.py +++ b/esmvalcore/cmor/_fixes/cmip6/cesm2_fv2.py @@ -4,6 +4,7 @@ from .cesm2 import Cl as BaseCl from .cesm2 import Fgco2 as BaseFgco2 from .cesm2 import Omon as BaseOmon +from .cesm2 import Pr as BasePr from .cesm2 import Tas as BaseTas Cl = BaseCl @@ -25,3 +26,6 @@ Tas = BaseTas + + +Pr = BasePr diff --git a/esmvalcore/cmor/_fixes/cmip6/cesm2_waccm.py b/esmvalcore/cmor/_fixes/cmip6/cesm2_waccm.py index 156a656b5f..d3bbc4dafe 100644 --- a/esmvalcore/cmor/_fixes/cmip6/cesm2_waccm.py +++ b/esmvalcore/cmor/_fixes/cmip6/cesm2_waccm.py @@ -6,7 +6,10 @@ from .cesm2 import Cl as BaseCl from .cesm2 import Fgco2 as BaseFgco2 from .cesm2 import Omon as BaseOmon +from .cesm2 import Pr as BasePr from .cesm2 import Tas as BaseTas +from .cesm2 import Tasmax as BaseTasmax +from .cesm2 import Tasmin as BaseTasmin class Cl(BaseCl): @@ -64,4 +67,13 @@ def fix_file(self, filepath, output_dir, add_unique_suffix=False): Siconc = SiconcFixScalarCoord +Pr = BasePr + + Tas = BaseTas + + +Tasmin = BaseTasmin + + +Tasmax = BaseTasmax diff --git a/esmvalcore/cmor/_fixes/cmip6/cesm2_waccm_fv2.py b/esmvalcore/cmor/_fixes/cmip6/cesm2_waccm_fv2.py index 23f77fbd07..f888bb6244 100644 --- a/esmvalcore/cmor/_fixes/cmip6/cesm2_waccm_fv2.py +++ b/esmvalcore/cmor/_fixes/cmip6/cesm2_waccm_fv2.py @@ -3,6 +3,7 @@ from ..common import SiconcFixScalarCoord from .cesm2 import Fgco2 as BaseFgco2 from .cesm2 import Omon as BaseOmon +from .cesm2 import Pr as BasePr from .cesm2 import Tas as BaseTas from .cesm2_waccm import Cl as BaseCl from .cesm2_waccm import Cli as BaseCli @@ -27,3 +28,6 @@ Tas = BaseTas + + +Pr = BasePr diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2.py index 5d504f6084..24df5db059 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cesm2.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2.py @@ -4,7 +4,9 @@ import unittest.mock import iris +import iris.cube import numpy as np +import pandas as pd import pytest from cf_units import Unit @@ -14,8 +16,11 @@ Clw, Fgco2, Omon, + Pr, Siconc, Tas, + Tasmax, + Tasmin, Tos, ) from esmvalcore.cmor._fixes.common import SiconcFixScalarCoord @@ -202,7 +207,7 @@ def test_clw_fix(): def tas_cubes(): """Cubes to test fixes for ``tas``.""" time_coord = iris.coords.DimCoord( - [0.0, 1.0], + [0.0, 1.0, 2.0], var_name="time", standard_name="time", units="days since 1850-01-01 00:00:00", @@ -219,12 +224,12 @@ def tas_cubes(): (lon_coord, 2), ] ta_cube = iris.cube.Cube( - np.ones((2, 2, 2)), + np.ones((3, 2, 2)), var_name="ta", dim_coords_and_dims=coord_specs, ) tas_cube = iris.cube.Cube( - np.ones((2, 2, 2)), + np.ones((3, 2, 2)), var_name="tas", dim_coords_and_dims=coord_specs, ) @@ -336,17 +341,19 @@ def test_tas_fix_metadata(tas_cubes): with pytest.raises(iris.exceptions.CoordinateNotFoundError): cube.coord("height") height_coord = iris.coords.AuxCoord( - 2.0, + [2.0], var_name="height", standard_name="height", long_name="height", units=Unit("m"), attributes={"positive": "up"}, ) + vardef = get_var_info("CMIP6", "Amon", "tas") fix = Tas(vardef) out_cubes = fix.fix_metadata(tas_cubes) - assert out_cubes is tas_cubes + assert out_cubes[0] is tas_cubes[0] + assert out_cubes[1] is tas_cubes[1] for cube in out_cubes: assert cube.coord("longitude").has_bounds() assert cube.coord("latitude").has_bounds() @@ -357,6 +364,20 @@ def test_tas_fix_metadata(tas_cubes): with pytest.raises(iris.exceptions.CoordinateNotFoundError): cube.coord("height") + # de-monotonize time points + for cube in tas_cubes: + time = cube.coord("time") + points = np.array(time.points) + points[-1] = points[0] + dims = cube.coord_dims(time) + cube.remove_coord(time) + time = iris.coords.AuxCoord.from_coord(time) + cube.add_aux_coord(time.copy(points), dims) + + out_cubes = fix.fix_metadata(tas_cubes) + for cube in out_cubes: + assert cube.coord("time").is_monotonic() + def test_tos_fix_metadata(tos_cubes): """Test ``fix_metadata`` for ``tos``.""" @@ -420,3 +441,179 @@ def test_fgco2_fix_metadata(): def test_siconc_fix(): """Test fix for ``siconc``.""" assert Siconc is SiconcFixScalarCoord + + +@pytest.fixture +def pr_cubes(): + correct_time_coord = iris.coords.DimCoord( + points=[1.0, 2.0, 3.0, 4.0, 5.0], + var_name="time", + standard_name="time", + units="days since 1850-01-01", + ) + + lat_coord = iris.coords.DimCoord( + [0.0], var_name="lat", standard_name="latitude" + ) + + lon_coord = iris.coords.DimCoord( + [0.0], var_name="lon", standard_name="longitude" + ) + + correct_coord_specs = [ + (correct_time_coord, 0), + (lat_coord, 1), + (lon_coord, 2), + ] + + correct_pr_cube = iris.cube.Cube( + np.ones((5, 1, 1)), + var_name="pr", + units="kg m-2 s-1", + dim_coords_and_dims=correct_coord_specs, + ) + + scalar_cube = iris.cube.Cube(0.0, var_name="ps") + + return iris.cube.CubeList([correct_pr_cube, scalar_cube]) + + +def test_get_pr_fix(): + """Test pr fix.""" + fix = Fix.get_fixes("CMIP6", "CESM2", "day", "pr") + assert fix == [Pr(None), GenericFix(None)] + + +def test_pr_fix_metadata(pr_cubes): + """Test metadata fix.""" + vardef = get_var_info("CMIP6", "day", "pr") + fix = Pr(vardef) + + out_cubes = fix.fix_metadata(pr_cubes) + assert out_cubes[0].var_name == "pr" + coord = out_cubes[0].coord("time") + assert pd.Series(coord.points).is_monotonic_increasing + + # de-monotonize time points + for cube in pr_cubes: + if cube.var_name == "pr": + time = cube.coord("time") + points = np.array(time.points) + points[-1] = points[0] + dims = cube.coord_dims(time) + cube.remove_coord(time) + time = iris.coords.AuxCoord.from_coord(time) + cube.add_aux_coord(time.copy(points), dims) + + out_cubes = fix.fix_metadata(pr_cubes) + for cube in out_cubes: + if cube.var_name == "tas": + assert cube.coord("time").is_monotonic() + + +@pytest.fixture +def tasmin_cubes(): + correct_lat_coord = iris.coords.DimCoord( + [0.0], var_name="lat", standard_name="latitude" + ) + wrong_lat_coord = iris.coords.DimCoord( + [0.0], var_name="latitudeCoord", standard_name="latitude" + ) + correct_lon_coord = iris.coords.DimCoord( + [0.0], var_name="lon", standard_name="longitude" + ) + wrong_lon_coord = iris.coords.DimCoord( + [0.0], var_name="longitudeCoord", standard_name="longitude" + ) + correct_cube = iris.cube.Cube( + [[2.0]], + var_name="tasmin", + dim_coords_and_dims=[(correct_lat_coord, 0), (correct_lon_coord, 1)], + ) + wrong_cube = iris.cube.Cube( + [[2.0]], + var_name="ta", + dim_coords_and_dims=[(wrong_lat_coord, 0), (wrong_lon_coord, 1)], + ) + scalar_cube = iris.cube.Cube(0.0, var_name="ps") + return iris.cube.CubeList([correct_cube, wrong_cube, scalar_cube]) + + +@pytest.fixture +def tasmax_cubes(): + correct_lat_coord = iris.coords.DimCoord( + [0.0], var_name="lat", standard_name="latitude" + ) + wrong_lat_coord = iris.coords.DimCoord( + [0.0], var_name="latitudeCoord", standard_name="latitude" + ) + correct_lon_coord = iris.coords.DimCoord( + [0.0], var_name="lon", standard_name="longitude" + ) + wrong_lon_coord = iris.coords.DimCoord( + [0.0], var_name="longitudeCoord", standard_name="longitude" + ) + correct_cube = iris.cube.Cube( + [[2.0]], + var_name="tasmax", + dim_coords_and_dims=[(correct_lat_coord, 0), (correct_lon_coord, 1)], + ) + wrong_cube = iris.cube.Cube( + [[2.0]], + var_name="ta", + dim_coords_and_dims=[(wrong_lat_coord, 0), (wrong_lon_coord, 1)], + ) + scalar_cube = iris.cube.Cube(0.0, var_name="ps") + return iris.cube.CubeList([correct_cube, wrong_cube, scalar_cube]) + + +def test_get_tasmin_fix(): + fix = Fix.get_fixes("CMIP6", "CESM2", "day", "tasmin") + assert fix == [Tasmin(None), GenericFix(None)] + + +def test_tasmin_fix_metadata(tasmin_cubes): + for cube in tasmin_cubes: + with pytest.raises(iris.exceptions.CoordinateNotFoundError): + cube.coord("height") + height_coord = iris.coords.AuxCoord( + 2.0, + var_name="height", + standard_name="height", + long_name="height", + units=Unit("m"), + attributes={"positive": "up"}, + ) + vardef = get_var_info("CMIP6", "day", "tasmin") + fix = Tasmin(vardef) + + out_cubes = fix.fix_metadata(tasmin_cubes) + assert out_cubes[0].var_name == "tasmin" + coord = out_cubes[0].coord("height") + assert coord == height_coord + + +def test_get_tasmax_fix(): + fix = Fix.get_fixes("CMIP6", "CESM2", "day", "tasmax") + assert fix == [Tasmax(None), GenericFix(None)] + + +def test_tasmax_fix_metadata(tasmax_cubes): + for cube in tasmax_cubes: + with pytest.raises(iris.exceptions.CoordinateNotFoundError): + cube.coord("height") + height_coord = iris.coords.AuxCoord( + 2.0, + var_name="height", + standard_name="height", + long_name="height", + units=Unit("m"), + attributes={"positive": "up"}, + ) + vardef = get_var_info("CMIP6", "day", "tasmax") + fix = Tasmax(vardef) + + out_cubes = fix.fix_metadata(tasmax_cubes) + assert out_cubes[0].var_name == "tasmax" + coord = out_cubes[0].coord("height") + assert coord == height_coord diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2_fv2.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2_fv2.py index 50e67e5d0f..89bc345bbb 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cesm2_fv2.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2_fv2.py @@ -2,6 +2,7 @@ from esmvalcore.cmor._fixes.cmip6.cesm2 import Cl as BaseCl from esmvalcore.cmor._fixes.cmip6.cesm2 import Fgco2 as BaseFgco2 +from esmvalcore.cmor._fixes.cmip6.cesm2 import Pr as BasePr from esmvalcore.cmor._fixes.cmip6.cesm2 import Tas as BaseTas from esmvalcore.cmor._fixes.cmip6.cesm2_fv2 import ( Cl, @@ -9,6 +10,7 @@ Clw, Fgco2, Omon, + Pr, Siconc, Tas, ) @@ -76,8 +78,21 @@ def test_get_tas_fix(): """Test getting of fix.""" fix = Fix.get_fixes("CMIP6", "CESM2-FV2", "Amon", "tas") assert fix == [Tas(None), GenericFix(None)] + fix = Fix.get_fixes("CMIP6", "CESM2-FV2", "day", "tas") + assert fix == [Tas(None), GenericFix(None)] def test_tas_fix(): """Test fix for ``tas``.""" assert Tas is BaseTas + + +def test_get_pr_fix(): + """Test getting of fix.""" + fix = Fix.get_fixes("CMIP6", "CESM2-FV2", "day", "pr") + assert fix == [Pr(None), GenericFix(None)] + + +def test_pr_fix(): + """Test fix for ``Pr``.""" + assert Pr is BasePr diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py index 363bf0d80c..8e4409542f 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py @@ -9,15 +9,21 @@ from esmvalcore.cmor._fixes.cmip6.cesm2 import Cl as BaseCl from esmvalcore.cmor._fixes.cmip6.cesm2 import Fgco2 as BaseFgco2 +from esmvalcore.cmor._fixes.cmip6.cesm2 import Pr as BasePr from esmvalcore.cmor._fixes.cmip6.cesm2 import Tas as BaseTas +from esmvalcore.cmor._fixes.cmip6.cesm2 import Tasmax as BaseTasmax +from esmvalcore.cmor._fixes.cmip6.cesm2 import Tasmin as BaseTasmin from esmvalcore.cmor._fixes.cmip6.cesm2_waccm import ( Cl, Cli, Clw, Fgco2, Omon, + Pr, Siconc, Tas, + Tasmax, + Tasmin, ) from esmvalcore.cmor._fixes.common import SiconcFixScalarCoord from esmvalcore.cmor._fixes.fix import GenericFix @@ -119,8 +125,43 @@ def test_get_tas_fix(): """Test getting of fix.""" fix = Fix.get_fixes("CMIP6", "CESM2-WACCM", "Amon", "tas") assert fix == [Tas(None), GenericFix(None)] + fix = Fix.get_fixes("CMIP6", "CESM2-WACCM", "day", "tas") + assert fix == [Tas(None), GenericFix(None)] def test_tas_fix(): """Test fix for ``tas``.""" assert Tas is BaseTas + + +def test_get_pr_fix(): + """Test getting of fix.""" + fix = Fix.get_fixes("CMIP6", "CESM2-WACCM", "day", "pr") + assert fix == [Pr(None), GenericFix(None)] + + +def test_pr_fix(): + """Test fix for ``Pr``.""" + assert Pr is BasePr + + +def test_get_tasmin_fix(): + """Test getting of fix.""" + fix = Fix.get_fixes("CMIP6", "CESM2-WACCM", "day", "tasmin") + assert fix == [Tasmin(None), GenericFix(None)] + + +def test_tasmin_fix(): + """Test fix for ``Tasmin``.""" + assert Tasmin is BaseTasmin + + +def test_get_tasmax_fix(): + """Test getting of fix.""" + fix = Fix.get_fixes("CMIP6", "CESM2-WACCM", "day", "tasmax") + assert fix == [Tasmax(None), GenericFix(None)] + + +def test_tasmax_fix(): + """Test fix for ``Tasmax``.""" + assert Tasmax is BaseTasmax diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm_fv2.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm_fv2.py index e61fec5745..6d115234f2 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm_fv2.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm_fv2.py @@ -1,6 +1,7 @@ """Tests for the fixes of CESM2-WACCM-FV2.""" from esmvalcore.cmor._fixes.cmip6.cesm2 import Fgco2 as BaseFgco2 +from esmvalcore.cmor._fixes.cmip6.cesm2 import Pr as BasePr from esmvalcore.cmor._fixes.cmip6.cesm2 import Tas as BaseTas from esmvalcore.cmor._fixes.cmip6.cesm2_waccm import Cl as BaseCl from esmvalcore.cmor._fixes.cmip6.cesm2_waccm_fv2 import ( @@ -9,6 +10,7 @@ Clw, Fgco2, Omon, + Pr, Siconc, Tas, ) @@ -76,8 +78,21 @@ def test_get_tas_fix(): """Test getting of fix.""" fix = Fix.get_fixes("CMIP6", "CESM2-WACCM-FV2", "Amon", "tas") assert fix == [Tas(None), GenericFix(None)] + fix = Fix.get_fixes("CMIP6", "CESM2-WACCM-FV2", "day", "tas") + assert fix == [Tas(None), GenericFix(None)] def test_tas_fix(): """Test fix for ``tas``.""" assert Tas is BaseTas + + +def test_get_pr_fix(): + """Test getting of fix.""" + fix = Fix.get_fixes("CMIP6", "CESM2-WACCM_FV2", "day", "pr") + assert fix == [Pr(None), GenericFix(None)] + + +def test_pr_fix(): + """Test fix for ``Pr``.""" + assert Pr is BasePr