From 35b015ce307f1eb12000981ec62ead8009b21cbe Mon Sep 17 00:00:00 2001 From: ValentinaHutter Date: Wed, 18 Sep 2024 11:35:09 +0200 Subject: [PATCH 1/6] update array_find --- .../process_implementations/arrays.py | 6 ++++++ tests/test_arrays.py | 10 +++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/openeo_processes_dask/process_implementations/arrays.py b/openeo_processes_dask/process_implementations/arrays.py index c66b5546..729a44f6 100644 --- a/openeo_processes_dask/process_implementations/arrays.py +++ b/openeo_processes_dask/process_implementations/arrays.py @@ -190,6 +190,12 @@ def array_find( mask = ~np.array((data == value).any(axis=axis)) if np.isnan(value): mask = True + if reverse: + if axis is None: + size = data.size + else: + size = data.shape[axis] + idxs = size - 1 - idxs logger.warning( "array_find: numpy has no sentinel value for missing data in integer arrays, therefore np.masked_array is used to return the indices of found elements. Further operations might fail if not defined for masked arrays." diff --git a/tests/test_arrays.py b/tests/test_arrays.py index 0152ac09..6f9fc857 100644 --- a/tests/test_arrays.py +++ b/tests/test_arrays.py @@ -222,14 +222,14 @@ def test_array_contains_object_dtype(): [ ([1, 0, 3, 2], 3, 2, None, False), ([1, 0, 3, 2, np.nan, 3], np.nan, 999999, None, False), - ([1, 0, 3, 2], 3, 2, None, False), + ([1, 0, 3, 0, 2], 0, 1, None, False), ([[1, 0, 3, 2], [5, 3, 6, 8]], 3, [999999, 1, 0, 999999], 0, False), ([[1, 0, 3, 2], [5, 3, 6, 8]], 3, [2, 1], 1, False), - ([1, 0, 3, 2], 3, 1, None, True), + ([1, 0, 3, 2], 3, 2, None, True), ([1, 0, 3, 2, np.nan, 3], np.nan, 999999, None, True), - ([1, 0, 3, 2], 3, 1, None, True), - ([[1, 0, 3, 2], [5, 3, 6, 8]], 3, [999999, 0, 1, 999999], 0, True), - ([[1, 0, 3, 2], [5, 3, 6, 8]], 3, [1, 2], 1, True), + ([1, 0, 3, 0, 2], 0, 3, None, True), + ([[1, 0, 3, 2], [5, 3, 6, 8]], 3, [999999, 1, 0, 999999], 0, True), + ([[1, 0, 3, 2], [5, 3, 6, 8]], 3, [2, 1], 1, True), ], ) def test_array_find(data, value, expected, axis, reverse): From e5f8089ccf420916c3c244e93a161ee6b00e8b03 Mon Sep 17 00:00:00 2001 From: ValentinaHutter Date: Wed, 18 Sep 2024 14:20:01 +0200 Subject: [PATCH 2/6] update array contains --- openeo_processes_dask/process_implementations/arrays.py | 6 ++---- tests/test_arrays.py | 3 ++- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/openeo_processes_dask/process_implementations/arrays.py b/openeo_processes_dask/process_implementations/arrays.py index 729a44f6..14decd5a 100644 --- a/openeo_processes_dask/process_implementations/arrays.py +++ b/openeo_processes_dask/process_implementations/arrays.py @@ -165,10 +165,8 @@ def array_contains(data: ArrayLike, value: Any, axis=None) -> bool: value_is_valid = True if len(np.shape(data)) != 1 and axis is None: return False - if not value_is_valid: + if not value_is_valid or pd.isnull(value): return False - if pd.isnull(value): - return np.isnan(data).any(axis=axis) else: return np.isin(data, value).any(axis=axis) @@ -188,7 +186,7 @@ def array_find( idxs = (data == value).argmax(axis=axis) mask = ~np.array((data == value).any(axis=axis)) - if np.isnan(value): + if not isinstance(value, str) and np.isnan(value): mask = True if reverse: if axis is None: diff --git a/tests/test_arrays.py b/tests/test_arrays.py index 6f9fc857..6f66c990 100644 --- a/tests/test_arrays.py +++ b/tests/test_arrays.py @@ -180,7 +180,7 @@ def test_array_append(data, value, expected): ([1, 2, 3], 2, True), (["A", "B", "C"], "b", False), ([1, 2, 3], "2", False), - ([1, 2, np.nan], np.nan, True), + ([1, 2, np.nan], np.nan, False), ([[2, 1], [3, 4]], [1, 2], False), ([[2, 1], [3, 4]], 2, False), ([1, 2, 3], np.int64(2), True), @@ -230,6 +230,7 @@ def test_array_contains_object_dtype(): ([1, 0, 3, 0, 2], 0, 3, None, True), ([[1, 0, 3, 2], [5, 3, 6, 8]], 3, [999999, 1, 0, 999999], 0, True), ([[1, 0, 3, 2], [5, 3, 6, 8]], 3, [2, 1], 1, True), + (["A", "B", "C"], "b", 99999, None, False), ], ) def test_array_find(data, value, expected, axis, reverse): From e7fc372d13e3baae87d4eeafad0ca9082329c06f Mon Sep 17 00:00:00 2001 From: ValentinaHutter Date: Wed, 18 Sep 2024 16:14:47 +0200 Subject: [PATCH 3/6] add array_interpolate --- .../process_implementations/arrays.py | 12 ++++++++++++ tests/test_arrays.py | 15 +++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/openeo_processes_dask/process_implementations/arrays.py b/openeo_processes_dask/process_implementations/arrays.py index 14decd5a..3d27f8c1 100644 --- a/openeo_processes_dask/process_implementations/arrays.py +++ b/openeo_processes_dask/process_implementations/arrays.py @@ -33,6 +33,7 @@ "array_find", "array_labels", "array_apply", + "array_interpolate_linear", "first", "last", "order", @@ -234,6 +235,17 @@ def array_apply( ) +def array_interpolate_linear(data: ArrayLike): + x = np.arange(len(data)) + valid = np.isfinite(data) + if len(x[valid]) < 2: + return data + data[~valid] = np.interp( + x[~valid], x[valid], data[valid], left=np.nan, right=np.nan + ) + return data + + def first( data: ArrayLike, ignore_nodata: Optional[bool] = True, diff --git a/tests/test_arrays.py b/tests/test_arrays.py index 6f66c990..9af5f901 100644 --- a/tests/test_arrays.py +++ b/tests/test_arrays.py @@ -264,6 +264,21 @@ def test_array_apply(process_registry): assert (output_cube == [2, 3, 4, 5, 6, 7]).all() +@pytest.mark.parametrize( + "data, expected", + [ + ([np.nan, 1, np.nan, 6, np.nan, -8], [np.nan, 1, 3.5, 6, -1, -8]), + ([np.nan, 1, np.nan, np.nan], [np.nan, 1, np.nan, np.nan]), + ], +) +def test_array_interpolate_linear(data, expected): + assert np.array_equal( + array_interpolate_linear(data), + expected, + equal_nan=True, + ) + + def test_first(): assert first(np.array([1, 0, 3, 2])) == 1 assert pd.isnull(first(np.array([np.nan, 2, 3]), ignore_nodata=False)) From 4de6d18bde2c19b6cfcb5ffee0c1b503826e1201 Mon Sep 17 00:00:00 2001 From: ValentinaHutter Date: Wed, 18 Sep 2024 16:22:53 +0200 Subject: [PATCH 4/6] add array_interpolate --- .../process_implementations/arrays.py | 2 ++ tests/test_arrays.py | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/openeo_processes_dask/process_implementations/arrays.py b/openeo_processes_dask/process_implementations/arrays.py index 3d27f8c1..9e89acae 100644 --- a/openeo_processes_dask/process_implementations/arrays.py +++ b/openeo_processes_dask/process_implementations/arrays.py @@ -236,6 +236,8 @@ def array_apply( def array_interpolate_linear(data: ArrayLike): + if isinstance(data, list): + data = np.array(data) x = np.arange(len(data)) valid = np.isfinite(data) if len(x[valid]) < 2: diff --git a/tests/test_arrays.py b/tests/test_arrays.py index 9af5f901..f6c45653 100644 --- a/tests/test_arrays.py +++ b/tests/test_arrays.py @@ -277,6 +277,16 @@ def test_array_interpolate_linear(data, expected): expected, equal_nan=True, ) + assert np.array_equal( + array_interpolate_linear(np.array(data)), + expected, + equal_nan=True, + ) + assert np.array_equal( + array_interpolate_linear(da.from_array(np.array(data))), + expected, + equal_nan=True, + ) def test_first(): From fb379aced0d7757fc4813fe90335009a3db202c7 Mon Sep 17 00:00:00 2001 From: ValentinaHutter Date: Wed, 18 Sep 2024 16:42:57 +0200 Subject: [PATCH 5/6] add array_interpolate --- tests/test_arrays.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_arrays.py b/tests/test_arrays.py index f6c45653..09e4a49a 100644 --- a/tests/test_arrays.py +++ b/tests/test_arrays.py @@ -283,7 +283,7 @@ def test_array_interpolate_linear(data, expected): equal_nan=True, ) assert np.array_equal( - array_interpolate_linear(da.from_array(np.array(data))), + array_interpolate_linear(da.from_array(np.array(data))).compute(), expected, equal_nan=True, ) From 71505f38ac01375abd8ac2be106c61752b52ebed Mon Sep 17 00:00:00 2001 From: ValentinaHutter Date: Thu, 19 Sep 2024 08:16:01 +0200 Subject: [PATCH 6/6] update tests! --- tests/test_arrays.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_arrays.py b/tests/test_arrays.py index 09e4a49a..631d9d01 100644 --- a/tests/test_arrays.py +++ b/tests/test_arrays.py @@ -277,13 +277,15 @@ def test_array_interpolate_linear(data, expected): expected, equal_nan=True, ) + data_np = np.array(data) assert np.array_equal( - array_interpolate_linear(np.array(data)), + array_interpolate_linear(data_np), expected, equal_nan=True, ) + data_da = da.from_array(data_np) assert np.array_equal( - array_interpolate_linear(da.from_array(np.array(data))).compute(), + array_interpolate_linear(data_da), expected, equal_nan=True, )