Modify chunkung a little bit

openclimatefix · Jul 21, 2023 · fddfdbe · fddfdbe
1 parent 59162e2
commit fddfdbe
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 5 deletions.
diff --git a/src/nwp_consumer/internal/inputs/ceda/client.py b/src/nwp_consumer/internal/inputs/ceda/client.py
@@ -157,7 +157,8 @@ def mapTemp(self, *, p: pathlib.Path) -> xr.Dataset:
         try:
             datasets: list[xr.Dataset] = cfgrib.open_datasets(
                 path=p.as_posix(),
-                backend_kwargs={"indexpath": ""}
+                backend_kwargs={"indexpath": ""},
+
             )
         except Exception as e:
             log.warn(
@@ -240,7 +241,7 @@ def mapTemp(self, *, p: pathlib.Path) -> xr.Dataset:
             .sortby("variable") \
             .chunk({
                 "init_time": 1,
-                "step": 1,
+                "step": -1,
                 "variable": -1,
                 "y": len(wholesaleDataset.y) // 2,
                 "x": len(wholesaleDataset.x) // 2,

diff --git a/src/nwp_consumer/internal/inputs/ceda/test_client.py b/src/nwp_consumer/internal/inputs/ceda/test_client.py
@@ -57,7 +57,6 @@ def test_convertsWholesale2FileCorrectly(self):
         # Ensure the correct variables are in the variable dimension
         self.assertListEqual(['dlwrf', 'dswrf', 'hcc', 'lcc', 'mcc', 'sde'], sorted(list(out.coords["variable"].values)))
 
-
 # --------- Static methods --------- #
 
 class TestIsWantedFile(unittest.TestCase):

diff --git a/src/nwp_consumer/internal/inputs/metoffice/client.py b/src/nwp_consumer/internal/inputs/metoffice/client.py
@@ -153,7 +153,8 @@ def mapTemp(self, *, p: pathlib.Path) -> xr.Dataset:
             parameterDataset: xr.Dataset = xr.open_dataset(
                 p.as_posix(),
                 engine='cfgrib',
-                backend_kwargs={'read_keys': ['name', 'parameterNumber'], 'indexpath': ''}
+                backend_kwargs={'read_keys': ['name', 'parameterNumber'], 'indexpath': ''},
+                chunks={},
             )
         except Exception as e:
             log.warn(
@@ -212,7 +213,7 @@ def mapTemp(self, *, p: pathlib.Path) -> xr.Dataset:
             .sortby("variable") \
             .chunk({
                 "init_time": 1,
-                "step": 1,
+                "step": -1,
                 "variable": -1,
                 "y": len(parameterDataset.y) // 2,
                 "x": len(parameterDataset.x) // 2,