From 58d2971902a008938edb9230650f5318fdd023b6 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Fri, 25 Jun 2021 12:54:34 +0200 Subject: [PATCH 1/4] Implemented fully lazy climate_statistics --- esmvalcore/preprocessor/_time.py | 19 +++-- tests/unit/preprocessor/_time/test_time.py | 94 +++++++++++++++++----- 2 files changed, 84 insertions(+), 29 deletions(-) diff --git a/esmvalcore/preprocessor/_time.py b/esmvalcore/preprocessor/_time.py index 3cf1d5f4d0..55a95eda5a 100644 --- a/esmvalcore/preprocessor/_time.py +++ b/esmvalcore/preprocessor/_time.py @@ -233,13 +233,18 @@ def get_time_weights(cube): Array of time weights for averaging. """ time = cube.coord('time') - time_weights = time.bounds[..., 1] - time.bounds[..., 0] - time_weights = time_weights.squeeze() - if time_weights.shape == (): - time_weights = da.broadcast_to(time_weights, cube.shape) - else: - time_weights = iris.util.broadcast_to_shape(time_weights, cube.shape, - cube.coord_dims('time')) + coord_dims = cube.coord_dims('time') + + # Multidimensional time coordinates are not supported: In this case, + # weights cannot be simply calculated as difference between the bounds + if len(coord_dims) > 1: + raise ValueError( + f"Weighted statistical operations are not supported for " + f"{len(coord_dims):d}D time coordinates, expected " + f"0D or 1D") + + # Extract 1D time weights (= lengths of time intervals) + time_weights = time.core_bounds()[:, 1] - time.core_bounds()[:, 0] return time_weights diff --git a/tests/unit/preprocessor/_time/test_time.py b/tests/unit/preprocessor/_time/test_time.py index 1677c05fcd..860fc62c34 100644 --- a/tests/unit/preprocessor/_time/test_time.py +++ b/tests/unit/preprocessor/_time/test_time.py @@ -42,6 +42,7 @@ def _create_sample_cube(calendar='gregorian'): + """Create sample cube.""" cube = Cube(np.arange(1, 25), var_name='co2', units='J') cube.add_dim_coord( iris.coords.DimCoord( @@ -63,6 +64,7 @@ def add_auxiliary_coordinate(cubelist): class TestExtractMonth(tests.Test): """Tests for extract_month.""" + def setUp(self): """Prepare tests.""" self.cube = _create_sample_cube() @@ -96,6 +98,7 @@ def test_bad_month_raises(self): class TestTimeSlice(tests.Test): """Tests for extract_time.""" + def setUp(self): """Prepare tests.""" self.cube = _create_sample_cube() @@ -167,6 +170,7 @@ def test_extract_time_no_time(self): class TestClipStartEndYear(tests.Test): """Tests for clip_start_end_year.""" + def setUp(self): """Prepare tests.""" self.cube = _create_sample_cube() @@ -210,6 +214,7 @@ def test_clip_start_end_year_no_time(self): class TestExtractSeason(tests.Test): """Tests for extract_season.""" + def setUp(self): """Prepare tests.""" self.cube = _create_sample_cube() @@ -293,7 +298,8 @@ def test_get_jf(self): class TestClimatology(tests.Test): - """Test class for :func:`esmvalcore.preprocessor._time.climatology`""" + """Test class for :func:`esmvalcore.preprocessor._time.climatology`.""" + @staticmethod def _create_cube(data, times, bounds): time = iris.coords.DimCoord(times, @@ -494,7 +500,8 @@ def test_time_rms(self): class TestSeasonalStatistics(tests.Test): - """Test :func:`esmvalcore.preprocessor._time.seasonal_statistics`""" + """Test :func:`esmvalcore.preprocessor._time.seasonal_statistics`.""" + @staticmethod def _create_cube(data, times): time = iris.coords.DimCoord(times, @@ -581,7 +588,8 @@ def test_season_custom_spans_full_season(self): class TestMonthlyStatistics(tests.Test): - """Test :func:`esmvalcore.preprocessor._time.monthly_statistics`""" + """Test :func:`esmvalcore.preprocessor._time.monthly_statistics`.""" + @staticmethod def _create_cube(data, times): time = iris.coords.DimCoord(times, @@ -648,7 +656,8 @@ def test_sum(self): class TestHourlyStatistics(tests.Test): - """Test :func:`esmvalcore.preprocessor._time.hourly_statistics`""" + """Test :func:`esmvalcore.preprocessor._time.hourly_statistics`.""" + @staticmethod def _create_cube(data, times): time = iris.coords.DimCoord(times, @@ -711,7 +720,8 @@ def test_sum(self): class TestDailyStatistics(tests.Test): - """Test :func:`esmvalcore.preprocessor._time.monthly_statistics`""" + """Test :func:`esmvalcore.preprocessor._time.monthly_statistics`.""" + @staticmethod def _create_cube(data, times): time = iris.coords.DimCoord(times, @@ -775,6 +785,7 @@ def test_sum(self): class TestRegridTimeYearly(tests.Test): """Tests for regrid_time with monthly frequency.""" + def setUp(self): """Prepare tests.""" self.cube_1 = _create_sample_cube() @@ -832,6 +843,7 @@ def test_regrid_time_year(self): class TestRegridTimeMonthly(tests.Test): """Tests for regrid_time with monthly frequency.""" + def setUp(self): """Prepare tests.""" self.cube_1 = _create_sample_cube() @@ -899,6 +911,7 @@ def test_regrid_time_different_calendar_bounds(self): class TestRegridTimeDaily(tests.Test): """Tests for regrid_time with daily frequency.""" + def setUp(self): """Prepare tests.""" self.cube_1 = _create_sample_cube() @@ -952,6 +965,7 @@ def test_regrid_time_day(self): class TestRegridTime6Hourly(tests.Test): """Tests for regrid_time with 6-hourly frequency.""" + def setUp(self): """Prepare tests.""" self.cube_1 = _create_sample_cube() @@ -1005,6 +1019,7 @@ def test_regrid_time_6hour(self): class TestRegridTime3Hourly(tests.Test): """Tests for regrid_time with 3-hourly frequency.""" + def setUp(self): """Prepare tests.""" self.cube_1 = _create_sample_cube() @@ -1058,6 +1073,7 @@ def test_regrid_time_3hour(self): class TestRegridTime1Hourly(tests.Test): """Tests for regrid_time with hourly frequency.""" + def setUp(self): """Prepare tests.""" self.cube_1 = _create_sample_cube() @@ -1111,6 +1127,7 @@ def test_regrid_time_hour(self): class TestTimeseriesFilter(tests.Test): """Tests for regrid_time with hourly frequency.""" + def setUp(self): """Prepare tests.""" self.cube = _create_sample_cube() @@ -1133,7 +1150,6 @@ def test_timeseries_filter_simple(self): def test_timeseries_filter_timecoord(self): """Test missing time axis.""" - import iris.exceptions new_cube = self.cube.copy() new_cube.remove_coord(new_cube.coord('time')) with self.assertRaises(iris.exceptions.CoordinateNotFoundError): @@ -1204,7 +1220,7 @@ def test_decadal_average(existing_coord): if existing_coord: def get_decade(coord, value): - """Callback function to get decades from cube.""" + """Get decades from cube.""" date = coord.units.num2date(value) return date.year - date.year % 10 @@ -1225,7 +1241,7 @@ def test_decadal_sum(existing_coord): if existing_coord: def get_decade(coord, value): - """Callback function to get decades from cube.""" + """Get decades from cube.""" date = coord.units.num2date(value) return date.year - date.year % 10 @@ -1289,6 +1305,7 @@ def make_map_data(number_years=2): @pytest.mark.parametrize('period', ['full']) def test_standardized_anomalies(period, standardize=True): + """Test standardized ``anomalies``.""" cube = make_map_data(number_years=2) result = anomalies(cube, period, standardize=standardize) if period == 'full': @@ -1309,6 +1326,7 @@ def test_standardized_anomalies(period, standardize=True): @pytest.mark.parametrize('period, reference', PARAMETERS) def test_anomalies_preserve_metadata(period, reference, standardize=False): + """Test that ``anomalies`` preserves metadata.""" cube = make_map_data(number_years=2) cube.var_name = "si" cube.units = "m" @@ -1323,6 +1341,7 @@ def test_anomalies_preserve_metadata(period, reference, standardize=False): @pytest.mark.parametrize('period, reference', PARAMETERS) def test_anomalies(period, reference, standardize=False): + """Test ``anomalies``.""" cube = make_map_data(number_years=2) result = anomalies(cube, period, reference, standardize=standardize) if reference is None: @@ -1374,6 +1393,7 @@ def test_anomalies(period, reference, standardize=False): def test_anomalies_custom_season(): + """Test ``anomalies`` with custom season.""" cube = make_map_data(number_years=2) result = anomalies(cube, 'season', seasons=('jfmamj', 'jasond')) anom = np.concatenate(( @@ -1406,6 +1426,16 @@ def get_1d_time(): return time +def get_2d_time(): + """Get 2D time coordinate.""" + time = iris.coords.AuxCoord([[20., 45.]], + standard_name='time', + bounds=[[[15., 30.], [30., 60.]]], + units=Unit('days since 1950-01-01', + calendar='gregorian')) + return time + + def get_lon_coord(): """Get longitude coordinate.""" lons = iris.coords.DimCoord([1.5, 2.5, 3.5], @@ -1444,9 +1474,8 @@ def test_get_time_weights(): """Test ``get_time_weights`` for complex cube.""" cube = _make_cube() weights = get_time_weights(cube) - assert weights.shape == cube.shape - np.testing.assert_allclose( - weights, [[[[15.0, 15.0, 15.0]]], [[[30.0, 30.0, 30.0]]]]) + assert weights.shape == (2, ) + np.testing.assert_allclose(weights, [15.0, 30.0]) def test_get_time_weights_0d_time(): @@ -1457,8 +1486,8 @@ def test_get_time_weights_0d_time(): units='K', aux_coords_and_dims=[(time, ())]) weights = get_time_weights(cube) - assert weights.shape == cube.shape - np.testing.assert_allclose(weights, 30.0) + assert weights.shape == (1, ) + np.testing.assert_allclose(weights, [30.0]) def test_get_time_weights_0d_time_1d_lon(): @@ -1471,8 +1500,8 @@ def test_get_time_weights_0d_time_1d_lon(): aux_coords_and_dims=[(time, ())], dim_coords_and_dims=[(lons, 0)]) weights = get_time_weights(cube) - assert weights.shape == cube.shape - np.testing.assert_allclose(weights, [30.0, 30.0, 30.0]) + assert weights.shape == (1, ) + np.testing.assert_allclose(weights, [30.0]) def test_get_time_weights_1d_time(): @@ -1483,7 +1512,7 @@ def test_get_time_weights_1d_time(): units='K', dim_coords_and_dims=[(time, 0)]) weights = get_time_weights(cube) - assert weights.shape == cube.shape + assert weights.shape == (2, ) np.testing.assert_allclose(weights, [15.0, 30.0]) @@ -1496,9 +1525,19 @@ def test_get_time_weights_1d_time_1d_lon(): units='K', dim_coords_and_dims=[(time, 0), (lons, 1)]) weights = get_time_weights(cube) - assert weights.shape == cube.shape - np.testing.assert_allclose(weights, - [[15.0, 15.0, 15.0], [30.0, 30.0, 30.0]]) + assert weights.shape == (2, ) + np.testing.assert_allclose(weights, [15.0, 30.0]) + + +def test_get_time_weights_2d_time(): + """Test ``get_time_weights`` for 1D time coordinate.""" + time = get_2d_time() + cube = iris.cube.Cube([[0.0, 1.0]], + var_name='x', + units='K', + aux_coords_and_dims=[(time, (0, 1))]) + with pytest.raises(ValueError): + get_time_weights(cube) def test_climate_statistics_0d_time_1d_lon(): @@ -1519,7 +1558,7 @@ def test_climate_statistics_0d_time_1d_lon(): np.testing.assert_allclose(new_cube.data, [1.0, -1.0, 42.0]) -def test_climate_statistics_complex_cube(): +def test_climate_statistics_complex_cube_sum(): """Test climate statistics.""" cube = _make_cube() new_cube = climate_statistics(cube, operator='sum', period='full') @@ -1528,8 +1567,18 @@ def test_climate_statistics_complex_cube(): np.testing.assert_allclose(new_cube.data, [[[45.0, 45.0, 45.0]]]) +def test_climate_statistics_complex_cube_mean(): + """Test climate statistics.""" + cube = _make_cube() + new_cube = climate_statistics(cube, operator='mean', period='full') + assert cube.shape == (2, 1, 1, 3) + assert new_cube.shape == (1, 1, 3) + np.testing.assert_allclose(new_cube.data, [[[1.0, 1.0, 1.0]]]) + + class TestResampleHours(tests.Test): - """Test :func:`esmvalcore.preprocessor._time.resample_hours`""" + """Test :func:`esmvalcore.preprocessor._time.resample_hours`.""" + @staticmethod def _create_cube(data, times): time = iris.coords.DimCoord(times, @@ -1618,7 +1667,8 @@ def test_resample_same_interval(self): class TestResampleTime(tests.Test): - """Test :func:`esmvalcore.preprocessor._time.resample_hours`""" + """Test :func:`esmvalcore.preprocessor._time.resample_hours`.""" + @staticmethod def _create_cube(data, times): time = iris.coords.DimCoord(times, From 955fc59f05c1eb72ea173f10f8696d1d8c7f3d07 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Sat, 26 Jun 2021 11:23:01 +0200 Subject: [PATCH 2/4] Create codecov.yml --- codecov.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 codecov.yml diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000000..53316e3208 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,7 @@ +coverage: + status: + project: + informational: true + patch: + default: + target: 100% From fbff57703dd04a81a0d427b95051fed6703d97f0 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Sat, 26 Jun 2021 11:25:59 +0200 Subject: [PATCH 3/4] Fix project --- codecov.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/codecov.yml b/codecov.yml index 53316e3208..87ce359403 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,7 +1,8 @@ coverage: status: project: - informational: true + default: + informational: true patch: default: target: 100% From 91a93360b745f5e6699f20ae007deeb689dec7a5 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Sat, 26 Jun 2021 11:38:20 +0200 Subject: [PATCH 4/4] Disable project status --- codecov.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/codecov.yml b/codecov.yml index 87ce359403..66c9e1c864 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,8 +1,6 @@ coverage: status: - project: - default: - informational: true + project: off patch: default: target: 100%