Merge remote-tracking branch 'upstream/main' into injections

.github/workflows/dataflow.yaml

@@ -14,12 +14,14 @@ jobs:
     # run on:
     #  - all pushes to main
     #  - schedule defined above
-    #  - a PR was just labeled 'test-dataflow'
-    #  - a PR with 'test-dataflow' label was opened, reopened, or synchronized
+    #  - a PR was just labeled 'test-dataflow' or 'test-all'
+    #  - a PR with 'test-dataflow' or 'test-all' label was opened, reopened, or synchronized
     if: |
       github.event_name == 'push' ||
       github.event_name == 'schedule' ||
+      github.event.label.name == 'test-all' ||
       github.event.label.name == 'test-dataflow' ||
+      contains( github.event.pull_request.labels.*.name, 'test-all') ||
       contains( github.event.pull_request.labels.*.name, 'test-dataflow')
     runs-on: ubuntu-latest
     strategy:

.github/workflows/flink.yaml

@@ -5,9 +5,24 @@ on:
     branches: [ "main" ]
   pull_request:
     branches: [ "main" ]
+    types: [ opened, reopened, synchronize, labeled ]
+  schedule:
+    - cron: '0 4 * * *' # run once a day at 4 AM
 
 jobs:
   build:
+    # run on:
+    #  - all pushes to main
+    #  - schedule defined above
+    #  - a PR was just labeled 'test-flink' or 'test-all'
+    #  - a PR with 'test-flink' or 'test-all' label was opened, reopened, or synchronized
+    if: |
+      github.event_name == 'push' ||
+      github.event_name == 'schedule' ||
+      github.event.label.name == 'test-all' ||
+      github.event.label.name == 'test-flink' ||
+      contains( github.event.pull_request.labels.*.name, 'test-all') ||
+      contains( github.event.pull_request.labels.*.name, 'test-flink')
 
     runs-on: ubuntu-latest
 
@@ -39,7 +54,7 @@ jobs:
 
     - name: Setup FlinkOperator
       run: |
-        FLINK_OPERATOR_VERSION=1.3.0
+        FLINK_OPERATOR_VERSION=1.5.0
         helm repo add flink-operator-repo https://downloads.apache.org/flink/flink-kubernetes-operator-${FLINK_OPERATOR_VERSION}
         helm install flink-kubernetes-operator flink-operator-repo/flink-kubernetes-operator --wait
 

.github/workflows/release.yaml

@@ -0,0 +1,33 @@
+name: Release
+
+on:
+  push:
+    tags: ["**"]
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    environment:
+      name: release
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade setuptools setuptools-scm wheel twine toml
+      - name: Build
+        run: |
+          python setup.py sdist bdist_wheel
+          python setup.py --version
+          twine check dist/*
+          ls -l dist
+      - name: Publish
+        uses: pypa/gh-action-pypi-publish@release/v1
+        if: startsWith(github.ref, 'refs/tags/')
+

.gitignore

@@ -26,6 +26,7 @@ share/python-wheels/
 .installed.cfg
 *.egg
 MANIFEST
+_version.py
 
 # PyInstaller
 #  Usually these files are written by a python script from a template

docs/development/release.md

@@ -0,0 +1,17 @@
+# Release
+
+Releases are automated by the `release.yaml` GitHub Workflow,
+which is triggered by tag events.
+
+To cut a new release, those with push permissions to the repo, may run:
+
+```console
+git tag $VERSION
+git push origin --tags
+```
+
+Where `$VERSION` is a three-element, dot-delimited semantic version of the form
+`v{MAJOR}.{MINOR}.{PATCH}`, which is appropriately incremented from the prior tag.
+
+And `origin` is assumed to be the remote corresponding to
+`pangeo-forge/pangeo-forge-runner`.

docs/index.md

@@ -24,3 +24,11 @@ tutorial/flink
 
 reference/index
 ```
+
+## Development
+
+```{toctree}
+:maxdepth: 2
+
+development/release
+```

docs/requirements.txt

@@ -2,5 +2,5 @@ autodoc-traits
 myst-parser
 pre-commit
 pydata-sphinx-theme
-sphinx>=1.7
+sphinx>=1.7,<7.2  # https://github.com/sphinx-doc/sphinx/issues/11631
 sphinx-copybutton

pangeo_forge_runner/commands/base.py

@@ -7,7 +7,7 @@
 
 from pythonjsonlogger import jsonlogger
 from repo2docker import contentproviders
-from traitlets import Bool, Instance, List, Unicode
+from traitlets import Bool, Dict, Instance, List, Unicode
 from traitlets.config import Application
 
 # Common aliases we want to support in *all* commands
@@ -41,6 +41,21 @@ class BaseCommand(Application):
 
     log_level = logging.INFO
 
+    logging_config = Dict(
+        {},
+        config=True,
+        help="""
+        Logging configuration for this python application.
+
+        When set, this value is passed to logging.config.dictConfig,
+        and can be used to configure how logs *throughout the application*
+        are handled, not just for logs from this application alone.
+
+        See https://docs.python.org/3/library/logging.config.html#logging.config.dictConfig
+        for more details.
+        """,
+    )
+
     repo = Unicode(
         "",
         config=True,
@@ -196,8 +211,26 @@ def json_excepthook(self, etype, evalue, traceback):
 
     def initialize(self, argv=None):
         super().initialize(argv)
-        # Load traitlets config from a config file if present
-        self.load_config_file(self.config_file)
+        # Load traitlets config from a config file if passed
+        if self.config_file:
+            self.load_config_file(self.config_file)
+            if (
+                not os.path.exists(self.config_file)
+                and self.config_file != "pangeo_forge_runner_config.py"
+            ):
+                # Throw an explicit error and exit if config file isn't present
+                print(
+                    f"Could not read config from file {self.config_file}. Make sure it exists and is readable",
+                    file=sys.stderr,
+                )
+                sys.exit(1)
+
+        # Allow arbitrary logging config if set
+        # We do this first up so any custom logging we set up ourselves
+        # is not affected, as by default dictConfig will replace all
+        # existing config.
+        if self.logging_config:
+            logging.config.dictConfig(self.logging_config)
 
         # The application communicates with the outside world via
         # stdout, and we structure this communication via logging.

pyproject.toml

@@ -1,3 +1,14 @@
+[build-system]
+requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2"]
+
+[project]
+name = "pangeo-forge-runner"
+dynamic = ["version"]
+
+[tool.setuptools_scm]
+write_to = "pangeo_forge_runner/_version.py"
+write_to_template = "__version__ = '{version}'"
+
 [tool.isort]
 # Prevent isort & black from fighting each otherd
 profile = "black"

setup.py

@@ -4,13 +4,11 @@
     readme = f.read()
 
 setup(
-    name="pangeo-forge-runner",
     description="Commandline tool to manage pangeo-forge feedstocks",
     long_description=readme,
     long_description_content_type="text/markdown",
     author="Yuvi Panda",
     author_email="[email protected]",
-    version="0.7.2",
     packages=find_packages(),
     python_requires=">=3.9",
     install_requires=[

tests/integration/test_flink.py

@@ -42,13 +42,13 @@ def test_flink_bake(minio):
             "pangeo-forge-runner",
             "bake",
             "--repo",
-            "https://github.com/pangeo-forge/gpcp-feedstock.git",
+            "https://github.com/pforgetest/gpcp-from-gcs-feedstock.git",
             "--ref",
-            "2cde04745189665a1f5a05c9eae2a98578de8b7f",
+            "beam-refactor",
             "-f",
             f.name,
         ]
-        proc = subprocess.run(cmd)
+        proc = subprocess.run(cmd, capture_output=True)
 
         assert proc.returncode == 0
 

tests/unit/test_expand_meta.py

@@ -1,4 +1,6 @@
 import json
+import os
+import secrets
 import subprocess
 
 invocations = [
@@ -75,3 +77,12 @@ def test_expand_meta_no_json():
         out = subprocess.check_output(cmd, encoding="utf-8")
         last_line = out.splitlines()[-1]
         assert json.loads(last_line) == invocation["meta"]
+
+
+def test_missing_config_file():
+    non_existent_path = secrets.token_hex() + ".py"
+    assert not os.path.exists(non_existent_path)
+    cmd = ["pangeo-forge-runner", "expand-meta", "--config", non_existent_path]
+    proc = subprocess.run(cmd, encoding="utf-8", capture_output=True, text=True)
+    assert proc.returncode == 1
+    assert "Could not read config from file" in proc.stderr