From 8e29756084ece8df484f5994daa8e68631d0d58f Mon Sep 17 00:00:00 2001
From: Robin Krahl <robin@nitrokey.com>
Date: Thu, 13 Apr 2023 12:32:17 +0200
Subject: [PATCH] nk3 test: Add --deny-skipped option

This patch adds an option to the nk3 test subcommand that causes one or
more skipped tests to fail the test run.  This can be used in production
to make sure that all selected tests are actually executed.
---
 pynitrokey/cli/nk3/__init__.py | 12 ++++++++++--
 pynitrokey/cli/nk3/test.py     |  9 +++++++--
 2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/pynitrokey/cli/nk3/__init__.py b/pynitrokey/cli/nk3/__init__.py
index 0c733467..637e8d2d 100644
--- a/pynitrokey/cli/nk3/__init__.py
+++ b/pynitrokey/cli/nk3/__init__.py
@@ -245,6 +245,13 @@ def rng(ctx: Context, length: int) -> None:
     default=False,
     help="List the selected tests instead of running them",
 )
+@click.option(
+    "--deny-skipped",
+    "deny_skipped",
+    is_flag=True,
+    default=False,
+    help="Return an error if one or more tests have been skipped",
+)
 @click.pass_obj
 def test(
     ctx: Context,
@@ -254,6 +261,7 @@ def test(
     include: Optional[str],
     exclude: Optional[str],
     list_: bool,
+    deny_skipped: bool,
 ) -> None:
     """Run some tests on all connected Nitrokey 3 devices."""
     from .test import (
@@ -296,7 +304,7 @@ def test(
     results = []
     test_ctx = TestContext(pin=pin)
     for device in devices:
-        results.append(run_tests(test_ctx, device, test_selector))
+        results.append(run_tests(test_ctx, device, test_selector, deny_skipped))
 
     n = len(devices)
     success = sum(results)
@@ -551,7 +559,7 @@ def provision_fido2(ctx: Context, key_file: BinaryIO, cert_file: BinaryIO) -> No
             raise CliException(f"Missing subject {name} in certificate")
     if subject_attrs["OU"] != "Authenticator Attestation":
         raise CliException(
-            f"Unexpected certificate subject OU {subject_attrs['OU']} (expected "
+            f"Unexpected certificate subject OU {subject_attrs['OU']!r} (expected "
             "Authenticator Attestation)"
         )
 
diff --git a/pynitrokey/cli/nk3/test.py b/pynitrokey/cli/nk3/test.py
index f6b0bad3..4a0efc33 100644
--- a/pynitrokey/cli/nk3/test.py
+++ b/pynitrokey/cli/nk3/test.py
@@ -342,7 +342,9 @@ def list_tests(selector: TestSelector) -> None:
         print(f"- {test_case.name}: {test_case.description}")
 
 
-def run_tests(ctx: TestContext, device: Nitrokey3Base, selector: TestSelector) -> bool:
+def run_tests(
+    ctx: TestContext, device: Nitrokey3Base, selector: TestSelector, deny_skipped: bool
+) -> bool:
     test_cases = selector.select()
     if not test_cases:
         raise CliException("No test cases selected", support_hint=False)
@@ -388,4 +390,7 @@ def run_tests(ctx: TestContext, device: Nitrokey3Base, selector: TestSelector) -
     local_print("")
     local_print(f"{n} tests, {success} successful, {skipped} skipped, {failed} failed")
 
-    return all([result.status != TestStatus.FAILURE for result in results])
+    if deny_skipped:
+        return all([result.status == TestStatus.SUCCESS for result in results])
+    else:
+        return all([result.status != TestStatus.FAILURE for result in results])