diff --git a/files/validate_config.py b/files/validate_config.py
index dfcb7845..d0776806 100755
--- a/files/validate_config.py
+++ b/files/validate_config.py
@@ -103,14 +103,14 @@ def confirm_submission_privkey_file(self):
             gpg_env = {"GNUPGHOME": d}
             # Call out to gpg to confirm it's a valid keyfile
             try:
-                subprocess.check_call(
-                    gpg_cmd, env=gpg_env, stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL
+                subprocess.check_output(
+                    gpg_cmd, env=gpg_env, stderr=subprocess.STDOUT
                 )
                 result = True
-            except subprocess.CalledProcessError:
-                # suppress error since "result" is checked next
-                pass
-
+            except subprocess.CalledProcessError as err:
+                if err.output and "No pinentry" in err.output.decode():
+                    raise ValidationError(f"PGP key is passphrase-protected.")
+                # Otherwise, continue; "result" is checked next
         if not result:
             raise ValidationError(f"PGP secret key is not valid: {self.secret_key_filepath}")