Tweak staging invocation (#131)

* Separate staging test run from prod test run This was originally combined into a single invocation for convenience but I believe this is more work than gain: * xfails alone became painful if staging expected failures are not the same as prod (which seems like a normal thing at times). * The test results are much clearer ("Staging conformance failure" is not as critical as "production conformance failure") This means the caller does need to run the action twice to get both tests Signed-off-by: Jussi Kukkonen <[email protected]> * cli protocol: Change --staging argument order If we promise the order is static, let's make it something sensible (don't have --staging after the artifact input). Signed-off-by: Jussi Kukkonen <[email protected]> * cli protocol: Don't promise option presence We already have --trusted-root that also is not always present. Signed-off-by: Jussi Kukkonen <[email protected]> * linter: Fix warnings from modern ruff * Update config to current style * use "ruff check" instead of "ruff" Signed-off-by: Jussi Kukkonen <[email protected]> * self-test: Let's run this daily Results should not change... except if infrastructure changes. Signed-off-by: Jussi Kukkonen <[email protected]> --------- Signed-off-by: Jussi Kukkonen <[email protected]>
sigstore · Mar 14, 2024 · 45d9f6d · 45d9f6d
1 parent 36c89ee
commit 45d9f6d
Show file tree

Hide file tree

Showing 8 changed files with 100 additions and 75 deletions.
diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml
@@ -5,6 +5,9 @@ on:
     branches:
       - main
   pull_request:
+  workflow_dispatch:
+  schedule:
+    - cron: '10 10 * * *'
 
 jobs:
   selftest:
@@ -22,8 +25,13 @@ jobs:
 
       - name: conformance test sigstore-python
         uses: ./
-        id: sigstore-conformance
         with:
           entrypoint: ${{ github.workspace }}/sigstore-python-conformance
-          enable-staging: true
+          xfail: "test_verify_with_trust_root test_verify_dsse_bundle_with_trust_root"
+
+      - name: Staging conformance test sigstore-python
+        uses: ./
+        with:
+          entrypoint: ${{ github.workspace }}/sigstore-python-conformance
+          environment: staging
           xfail: "test_verify_with_trust_root test_verify_dsse_bundle_with_trust_root"
diff --git a/Makefile b/Makefile
@@ -18,5 +18,5 @@ dev: env/pyvenv.cfg
 .PHONY: lint
 lint: env/pyvenv.cfg $(ALL_PY_SRCS)
 	./env/bin/python -m black $(ALL_PY_SRCS)
-	./env/bin/python -m ruff --fix $(ALL_PY_SRCS)
+	./env/bin/python -m ruff check --fix $(ALL_PY_SRCS)
 	./env/bin/python -m mypy action.py test/
diff --git a/README.md b/README.md
@@ -43,10 +43,19 @@ client-under-test [CLI protocol](docs/cli_protocol.md).
         runs-on: ubuntu-latest
         steps:
           - uses: actions/checkout@v4
+
           # insert your client installation steps here
+
+          # Run tests against production Sigstore environment
+          - uses: sigstore/[email protected]
+            with:
+              entrypoint: my-conformance-client
+
+          # Run tests against staging Sigstore environment
           - uses: sigstore/[email protected]
             with:
               entrypoint: my-conformance-client
+              environment: staging
     ```
 
 See [sigstore-python conformance test](https://github.com/sigstore/sigstore-python/blob/main/.github/workflows/conformance.yml)
@@ -57,8 +66,8 @@ for a complete example.
 The important action inputs are
 * `entrypoint`: required string. A command that implements the client-under-test
   [CLI protocol](docs/cli_protocol.md)
-* `enable-staging`: optional boolean. When true, the test suite will run tests against
-  staging infrastructure in addition to running them against production infrastructure
+* `environment`: 'production' (default) or 'staging'. This selects the Sigstore environment to
+  run against
 * `xfail`: optional string. Whitespace separated test names that are expected to fail.
 
 See [action.yml](action.yml) for full list of inputs.
@@ -77,8 +86,7 @@ The test suite can be configured with
   [CLI specification](https://github.com/sigstore/sigstore-conformance/blob/main/docs/cli_protocol.md)
 * `--identity-token=$GITHUB_TOKEN` where GITHUB_TOKEN is a GitHub token with actions:read
   access for public repositories (--identity-token is only required for signing tests)
-* optional (and currently experimental) `--staging`: This instructs the test suite to run
-  against Sigstore staging infrastructure
+* optional `--staging`: This instructs the test suite to run against Sigstore staging infrastructure
 * The environment variable `GHA_SIGSTORE_CONFORMANCE_XFAIL` can be used to
   set expected results
 

diff --git a/action.py b/action.py
@@ -14,7 +14,6 @@
 _RENDER_SUMMARY = os.getenv("GHA_SIGSTORE_CONFORMANCE_SUMMARY", "true") == "true"
 _DEBUG = os.getenv("GHA_SIGSTORE_CONFORMANCE_INTERNAL_BE_CAREFUL_DEBUG", "false") != "false"
 _ACTION_PATH = Path(os.getenv("GITHUB_ACTION_PATH"))  # type: ignore
-_ENABLE_STAGING = os.getenv("GHA_SIGSTORE_CONFORMANCE_ENABLE_STAGING", "false").lower() == "true"
 
 
 def _summary(msg):
@@ -27,7 +26,7 @@ def _debug(msg):
         print(f"\033[93mDEBUG: {msg}\033[0m", file=sys.stderr)
 
 
-def _sigstore_conformance(staging: bool) -> int:
+def _sigstore_conformance(environment: str) -> int:
     args = []
 
     if _DEBUG:
@@ -37,8 +36,10 @@ def _sigstore_conformance(staging: bool) -> int:
     if entrypoint:
         args.extend(["--entrypoint", entrypoint])
 
-    if staging:
+    if environment == "staging":
         args.append("--staging")
+    elif environment != "production":
+        raise ValueError(f"Unknown environment '{environment}'")
 
     skip_signing = os.getenv("GHA_SIGSTORE_CONFORMANCE_SKIP_SIGNING", "false").lower() == "true"
     if skip_signing:
@@ -48,17 +49,15 @@ def _sigstore_conformance(staging: bool) -> int:
     if gh_token:
         args.extend(["--github-token", gh_token])
 
-    infra = "staging" if staging else "production"
-    print(f"running sigstore-conformance against Sigstore {infra} infrastructure")
+    print(f"running sigstore-conformance against Sigstore {environment} infrastructure")
     _debug(f"running: sigstore-conformance {[str(a) for a in args]}")
 
     return pytest.main([str(_ACTION_PATH / "test"), *args])
 
 
-# Run against production, then optionally against staging
-status = _sigstore_conformance(staging=False)
-if _ENABLE_STAGING:
-    status += _sigstore_conformance(staging=True)
+# Run against chosen environment
+environment = os.getenv("GHA_SIGSTORE_CONFORMANCE_ENVIRONMENT", "production")
+status = _sigstore_conformance(environment)
 
 if status == 0:
     _summary("🎉 sigstore-conformance exited successfully")

diff --git a/action.yml b/action.yml
@@ -15,10 +15,10 @@ inputs:
     description: "skip tests that involve signing (default false)"
     required: false
     default: "false"
-  enable-staging:
-    description: "Test against staging infrastructure as well as production (default false)"
+  environment:
+    description: "'production' (default) or 'staging'"
     required: false
-    default: "false"
+    default: "production"
   xfail:
     description: "one or more tests that are expected to fail, whitespace-separated"
     required: false
@@ -38,7 +38,7 @@ runs:
       run: |
         ${{ github.action_path }}/action.py
       env:
-        GHA_SIGSTORE_CONFORMANCE_ENABLE_STAGING: "${{ inputs.enable-staging }}"
+        GHA_SIGSTORE_CONFORMANCE_ENVIRONMENT: "${{ inputs.environment }}"
         GHA_SIGSTORE_CONFORMANCE_ENTRYPOINT: "${{ inputs.entrypoint }}"
         GHA_SIGSTORE_CONFORMANCE_INTERNAL_BE_CAREFUL_DEBUG: "${{ inputs.internal-be-careful-debug }}"
         GHA_SIGSTORE_CONFORMANCE_SKIP_SIGNING: "${{ inputs.skip-signing }}"

diff --git a/docs/cli_protocol.md b/docs/cli_protocol.md
@@ -24,26 +24,20 @@ client's native CLI accepts.
 This is the set of subcommands that the test CLI must support. Each subcommand
 has a provided syntax and list of descriptions for each argument.
 
-To simplify argument parsing, all arguments are required, except `--staging`, and will **always** be
-supplied by the conformance suite in the order that they are specified in the
-templates below.
-
-All commands below are allowed to run against staging by appending the `--staging` in the command, for example:
-
-```console
-${ENTRYPOINT} sign --identity-token TOKEN --signature FILE --certificate FILE FILE --staging
-```
+To simplify argument parsing, arguments will **always** be supplied by the
+conformance suite in the order that they are specified in the templates below.
 
 ### Sign
 
 #### Signature and certificate flow
 
 ```console
-${ENTRYPOINT} sign --identity-token TOKEN --signature FILE --certificate FILE FILE
+${ENTRYPOINT} sign [--staging] --identity-token TOKEN --signature FILE --certificate FILE FILE
 ```
 
 | Option | Description |
 | --- | --- |
+| `--staging`        | Presence indicates client should use Sigstore staging infrastructure |
 | `--identity-token` | The OIDC identity token to use |
 | `--signature FILE` | The path to write the signature to |
 | `--certificate FILE` | The path to write the signing certificate to |
@@ -52,11 +46,12 @@ ${ENTRYPOINT} sign --identity-token TOKEN --signature FILE --certificate FILE FI
 #### Bundle flow
 
 ```console
-${ENTRYPOINT} sign-bundle --identity-token TOKEN --bundle FILE FILE
+${ENTRYPOINT} sign-bundle [--staging] --identity-token TOKEN --bundle FILE FILE
 ```
 
 | Option | Description |
 | --- | --- |
+| `--staging`        | Presence indicates client should use Sigstore staging infrastructure |
 | `--identity-token` | The OIDC identity token to use |
 | `--bundle FILE` | The path to write the bundle to |
 | `FILE` | The artifact to sign |
@@ -66,11 +61,12 @@ ${ENTRYPOINT} sign-bundle --identity-token TOKEN --bundle FILE FILE
 #### Signature and certificate flow
 
 ```console
-${ENTRYPOINT} verify --signature FILE --certificate FILE --certificate-identity IDENTITY --certificate-oidc-issuer URL [--trusted-root FILE] FILE
+${ENTRYPOINT} verify [--staging] --signature FILE --certificate FILE --certificate-identity IDENTITY --certificate-oidc-issuer URL [--trusted-root FILE] FILE
 ```
 
 | Option | Description |
 | --- | --- |
+| `--staging`        | Presence indicates client should use Sigstore staging infrastructure |
 | `--signature FILE` | The path to the signature to verify |
 | `--certificate FILE` | The path to the signing certificate to verify |
 | `--certificate-identity IDENTITY` | The expected identity in the signing certificate's SAN extension |
@@ -81,11 +77,12 @@ ${ENTRYPOINT} verify --signature FILE --certificate FILE --certificate-identity
 #### Bundle flow
 
 ```console
-${ENTRYPOINT} verify-bundle --bundle FILE --certificate-identity IDENTITY --certificate-oidc-issuer URL [--trusted-root FILE] FILE
+${ENTRYPOINT} verify-bundle [--staging] --bundle FILE --certificate-identity IDENTITY --certificate-oidc-issuer URL [--trusted-root FILE] FILE
 ```
 
 | Option | Description |
 | --- | --- |
+| `--staging`        | Presence indicates client should use Sigstore staging infrastructure |
 | `--bundle FILE` | The path to the Sigstore bundle to verify |
 | `--certificate-identity IDENTITY` | The expected identity in the signing certificate's SAN extension |
 | `--certificate-oidc-issuer URL` | The expected OIDC issuer for the signing certificate |

diff --git a/pyproject.toml b/pyproject.toml
@@ -3,4 +3,4 @@ line-length = 100
 
 [tool.ruff]
 line-length = 100
-select = ["E", "F", "I", "W", "UP"]
+lint.select = ["E", "F", "I", "W", "UP"]
diff --git a/test/client.py b/test/client.py
@@ -128,8 +128,6 @@ def run(self, *args) -> None:
         """
         self.completed_process = None
         full_command = [self.entrypoint, *args]
-        if self.staging:
-            full_command.append("--staging")
 
         try:
             self.completed_process = subprocess.run(
@@ -187,16 +185,20 @@ def _sign_for_sigcrt(
         This is an overload of `sign` for the signature/certificate flow and should not
         be called directly.
         """
-        args = [
-            "sign",
-            "--identity-token",
-            self.identity_token,
-            "--signature",
-            materials.signature,
-            "--certificate",
-            materials.certificate,
-            artifact,
-        ]
+        args: list[str | os.PathLike] = ["sign"]
+        if self.staging:
+            args.append("--staging")
+        args.extend(
+            [
+                "--identity-token",
+                self.identity_token,
+                "--signature",
+                materials.signature,
+                "--certificate",
+                materials.certificate,
+                artifact,
+            ]
+        )
 
         self.run(*args)
 
@@ -208,14 +210,18 @@ def _sign_for_bundle(self, materials: BundleMaterials, artifact: os.PathLike) ->
         This is an overload of `sign` for the bundle flow and should not be called directly.
         """
 
-        args = [
-            "sign-bundle",
-            "--identity-token",
-            self.identity_token,
-            "--bundle",
-            materials.bundle,
-            artifact,
-        ]
+        args: list[str | os.PathLike] = ["sign-bundle"]
+        if self.staging:
+            args.append("--staging")
+        args.extend(
+            [
+                "--identity-token",
+                self.identity_token,
+                "--bundle",
+                materials.bundle,
+                artifact,
+            ]
+        )
 
         self.run(*args)
 
@@ -243,17 +249,21 @@ def _verify_for_sigcrt(
         not be called directly.
         """
 
-        args = [
-            "verify",
-            "--signature",
-            materials.signature,
-            "--certificate",
-            materials.certificate,
-            "--certificate-identity",
-            CERTIFICATE_IDENTITY,
-            "--certificate-oidc-issuer",
-            CERTIFICATE_OIDC_ISSUER,
-        ]
+        args: list[str | os.PathLike] = ["verify"]
+        if self.staging:
+            args.append("--staging")
+        args.extend(
+            [
+                "--signature",
+                materials.signature,
+                "--certificate",
+                materials.certificate,
+                "--certificate-identity",
+                CERTIFICATE_IDENTITY,
+                "--certificate-oidc-issuer",
+                CERTIFICATE_OIDC_ISSUER,
+            ]
+        )
 
         if getattr(materials, "trusted_root", None) is not None:
             args.extend(["--trusted-root", materials.trusted_root])
@@ -270,16 +280,19 @@ def _verify_for_bundle(self, materials: BundleMaterials, artifact: os.PathLike)
         This is an overload of `verify` for the bundle flow and should not be called
         directly.
         """
-
-        args = [
-            "verify-bundle",
-            "--bundle",
-            materials.bundle,
-            "--certificate-identity",
-            CERTIFICATE_IDENTITY,
-            "--certificate-oidc-issuer",
-            CERTIFICATE_OIDC_ISSUER,
-        ]
+        args: list[str | os.PathLike] = ["verify-bundle"]
+        if self.staging:
+            args.append("--staging")
+        args.extend(
+            [
+                "--bundle",
+                materials.bundle,
+                "--certificate-identity",
+                CERTIFICATE_IDENTITY,
+                "--certificate-oidc-issuer",
+                CERTIFICATE_OIDC_ISSUER,
+            ]
+        )
 
         if getattr(materials, "trusted_root", None) is not None:
             args.extend(["--trusted-root", materials.trusted_root])