resolve merge conflicts from rebase

.github/workflows/flink.yaml

@@ -32,7 +32,6 @@ jobs:
             "pangeo-forge-recipes==0.10.0",
             # save some cycles and infer it might
             # work on all versions between lowest and highest
-            # "pangeo-forge-recipes==0.10.0",
             # "pangeo-forge-recipes==0.10.3",
             "pangeo-forge-recipes==0.10.4",
         ]

.github/workflows/unit-test.yml

@@ -18,7 +18,6 @@ jobs:
           "pangeo-forge-recipes==0.10.0",
           # save some cycles and infer it might
           # work on all versions between lowest and highest
-          # "pangeo-forge-recipes==0.10.0",
           # "pangeo-forge-recipes==0.10.3",
           "pangeo-forge-recipes==0.10.4",
         ]

MANIFEST.in

@@ -1,2 +1 @@
 include LICENSE
-include pangeo_forge_runner/commands/pangeo-forge-recipes-0.9-requirements.txt

docs/reference/storage.md

@@ -14,9 +14,3 @@ where `pangeo-forge-runner` puts intermediate & final data products.
 ```{eval-rst}
 .. autoconfigurable::  pangeo_forge_runner.storage.InputCacheStorage
 ```
-
-## MetadataCacheStorage
-
-```{eval-rst}
-.. autoconfigurable::  pangeo_forge_runner.storage.MetadataCacheStorage
-```

pangeo_forge_runner/commands/bake.py

@@ -18,13 +18,13 @@
 from ..bakery.flink import FlinkOperatorBakery
 from ..bakery.local import LocalDirectBakery
 from ..plugin import get_injections, get_injectionspecs_from_entrypoints
+<<<<<<< HEAD
+=======
+from ..storage import InputCacheStorage, TargetStorage
+>>>>>>> main
 from ..stream_capture import redirect_stderr, redirect_stdout
 from .base import BaseCommand, common_aliases, common_flags
 
-PFR_0_9_REQUIREMENTS_FILE_PATH = (
-    Path(__file__).parent / "pangeo-forge-recipes-0.9-requirements.txt"
-)
-
 
 class Bake(BaseCommand):
     """
@@ -197,15 +197,13 @@ def start(self):
 
                 from ..storage import (
                     InputCacheStorage,
-                    MetadataCacheStorage,
                     TargetStorage,
                 )
 
                 # Create our storage configurations. Traitlets will do its magic, populate these
                 # with appropriate config from config file / commandline / defaults.
                 target_storage = TargetStorage(parent=self)
                 input_cache_storage = InputCacheStorage(parent=self)
-                metadata_cache_storage = MetadataCacheStorage(parent=self)
 
                 self.log.info(
                     f"Target Storage is {target_storage}\n", extra={"status": "setup"}
@@ -214,10 +212,6 @@ def start(self):
                     f"Input Cache Storage is {input_cache_storage}\n",
                     extra={"status": "setup"},
                 )
-                self.log.info(
-                    f"Metadata Cache Storage is {metadata_cache_storage}\n",
-                    extra={"status": "setup"},
-                )
 
                 injection_specs = get_injectionspecs_from_entrypoints()
 

pangeo_forge_runner/storage.py

@@ -105,11 +105,3 @@ class InputCacheStorage(StorageTargetConfig):
     """
 
     pangeo_forge_target_class = "CacheFSSpecTarget"
-
-
-class MetadataCacheStorage(StorageTargetConfig):
-    """
-    Storage configuration for caching metadata during recipe baking
-    """
-
-    pangeo_forge_target_class = "MetadataTarget"

tests/integration/flink/test_flink_integration.py

@@ -5,7 +5,6 @@
 from importlib.metadata import version
 from pathlib import Path
 
-import pytest
 import xarray as xr
 from packaging.version import parse as parse_version
 
@@ -22,12 +21,6 @@ def test_flink_bake(minio_service, flinkversion, pythonversion, beamversion):
     pfr_version = parse_version(version("pangeo-forge-recipes"))
     if pfr_version >= parse_version("0.10"):
         recipe_version_ref = "0.10.x"
-    else:
-        recipe_version_ref = "0.9.x"
-        pytest.xfail(
-            f"{pfr_version = }, which is < 0.10. "
-            "Flink tests timeout with this recipes version, so we xfail this test."
-        )
 
     bucket = "s3://gpcp-out"
     config = {
@@ -50,11 +43,6 @@ def test_flink_bake(minio_service, flinkversion, pythonversion, beamversion):
             "fsspec_args": fsspec_args,
             "root_path": bucket + "/input-cache/{job_name}",
         },
-        "MetadataCacheStorage": {
-            "fsspec_class": "s3fs.S3FileSystem",
-            "fsspec_args": fsspec_args,
-            "root_path": bucket + "/metadata-cache/{job_name}",
-        },
         "FlinkOperatorBakery": {
             "flink_version": flinkversion,
             "job_manager_resources": {"memory": "1024m", "cpu": 0.30},

tests/integration/test_dataflow_integration.py

@@ -21,7 +21,9 @@ def test_dataflow_integration():
     if pfr_version >= parse_version("0.10"):
         recipe_version_ref = "0.10.x"
     else:
-        recipe_version_ref = "0.9.x"
+        raise ValueError(
+            f"Unsupported pfr_version: {pfr_version}. Please upgrade to 0.10 or newer."
+        )
     bucket = "gs://pangeo-forge-runner-ci-testing"
     config = {
         "Bake": {
@@ -38,10 +40,6 @@ def test_dataflow_integration():
             "fsspec_class": "gcsfs.GCSFileSystem",
             "root_path": bucket + "/input-cache/{job_name}",
         },
-        "MetadataCacheStorage": {
-            "fsspec_class": "gcsfs.GCSFileSystem",
-            "root_path": bucket + "/metadata-cache/{job_name}",
-        },
     }
 
     with tempfile.NamedTemporaryFile("w", suffix=".json") as f:

tests/unit/test_bake.py

@@ -113,7 +113,9 @@ def recipes_version_ref(request):
     if pfr_version >= parse_version("0.10"):
         recipes_version_ref = "0.10.x"
     else:
-        recipes_version_ref = "0.9.x"
+        raise ValueError(
+            f"Unsupported pfr_version: {pfr_version}. Please upgrade to 0.10 or newer."
+        )
     return (
         recipes_version_ref
         if not request.param == "dict_object"
@@ -173,11 +175,6 @@ def test_gpcp_bake(
             "fsspec_args": fsspec_args,
             "root_path": "s3://gpcp/input-cache/",
         },
-        "MetadataCacheStorage": {
-            "fsspec_class": "s3fs.S3FileSystem",
-            "fsspec_args": fsspec_args,
-            "root_path": "s3://gpcp/metadata-cache/",
-        },
     }
 
     if no_input_cache:
@@ -211,9 +208,6 @@ def test_gpcp_bake(
         if expected_error:
             assert proc.returncode == 1
             stdout[-1] == expected_error
-        elif no_input_cache and recipes_version_ref == "0.9.x":
-            # no_input_cache is only supported in 0.10.x and above
-            assert proc.returncode == 1
         else:
             assert proc.returncode == 0