From 8df1757688e7e9864ae19bfee37df97df2e42e36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 18 Dec 2024 04:30:14 +0100 Subject: [PATCH 01/10] tests: assertEquals -> assertEqual --- splitgpg2tests/tests.py | 68 ++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/splitgpg2tests/tests.py b/splitgpg2tests/tests.py index 6b51224..700bfcf 100644 --- a/splitgpg2tests/tests.py +++ b/splitgpg2tests/tests.py @@ -80,25 +80,25 @@ def setUp(self): cmd = 'gpg2 -a --export user@localhost' p = self.backend.run(cmd, passio_popen=True, passio_stderr=True) (pubkey, stderr) = p.communicate() - self.assertEquals(p.returncode, 0, + self.assertEqual(p.returncode, 0, '{} failed: {}'.format(cmd, stderr.decode())) cmd = 'gpg2 --import' p = self.frontend.run(cmd, passio_popen=True, passio_stderr=True) (stdout, stderr) = p.communicate(pubkey) - self.assertEquals(p.returncode, 0, + self.assertEqual(p.returncode, 0, '{} failed: {}{}'.format(cmd, stdout.decode(), stderr.decode())) # and set as trusted cmd = 'gpg2 --with-colons --list-key user@localhost' p = self.frontend.run(cmd, passio_popen=True, passio_stderr=True) (stdout, stderr) = p.communicate() - self.assertEquals(p.returncode, 0, + self.assertEqual(p.returncode, 0, '{} failed: {}{}'.format(cmd, stdout.decode(), stderr.decode())) fpr = [l for l in stdout.splitlines() if l.startswith(b'fpr:')][0] cmd = 'gpg2 --with-colons --import-ownertrust' p = self.frontend.run(cmd, passio_popen=True, passio_stderr=True) (stdout, stderr) = p.communicate( fpr.replace(b'fpr:::::::::', b'') + b'6:\n') - self.assertEquals(p.returncode, 0, + self.assertEqual(p.returncode, 0, '{} failed: {}{}'.format(cmd, stdout.decode(), stderr.decode())) @@ -106,19 +106,19 @@ class TC_00_Direct(SplitGPGBase): def test_000_version(self): cmd = 'gpg2 --version' p = self.frontend.run(cmd, wait=True) - self.assertEquals(p, 0, '{} failed'.format(cmd)) + self.assertEqual(p, 0, '{} failed'.format(cmd)) def test_010_list_keys(self): cmd = 'gpg2 --list-keys' p = self.frontend.run(cmd, passio_popen=True, passio_stderr=True) (keys, stderr) = p.communicate() - self.assertEquals(p.returncode, 0, + self.assertEqual(p.returncode, 0, '{} failed: {}'.format(cmd, stderr.decode())) self.assertIn("Qubes test", keys.decode()) cmd = 'gpg2 --list-secret-keys' p = self.frontend.run(cmd, passio_popen=True, passio_stderr=True) (keys, stderr) = p.communicate() - self.assertEquals(p.returncode, 0, + self.assertEqual(p.returncode, 0, '{} failed: {}'.format(cmd, stderr.decode())) self.assertIn("Qubes test", keys.decode()) @@ -128,25 +128,25 @@ def test_020_export_secret_key_deny(self): cmd = 'gpg2 -a --export-secret-keys user@localhost' p = self.frontend.run(cmd, passio_popen=True, passio_stderr=True) keys, stderr = p.communicate() - self.assertNotEquals(p.returncode, 0, + self.assertNotEqual(p.returncode, 0, '{} succeeded unexpectedly: {}'.format(cmd, stderr.decode())) - self.assertEquals(keys.decode(), '') + self.assertEqual(keys.decode(), '') def test_030_sign_verify(self): msg = "Test message" cmd = 'gpg2 -a --sign -u user@localhost' p = self.frontend.run(cmd, passio_popen=True, passio_stderr=True) (signature, stderr) = p.communicate(msg.encode()) - self.assertEquals(p.returncode, 0, + self.assertEqual(p.returncode, 0, '{} failed: {}'.format(cmd, stderr.decode())) - self.assertNotEquals('', signature.decode()) + self.assertNotEqual('', signature.decode()) cmd = "gpg2" p = self.frontend.run(cmd, passio_popen=True, passio_stderr=True) decoded_msg, verification_result = p.communicate(signature) - self.assertEquals(p.returncode, 0, + self.assertEqual(p.returncode, 0, '{} failed: {}'.format(cmd, verification_result.decode())) - self.assertEquals(decoded_msg.decode(), msg) + self.assertEqual(decoded_msg.decode(), msg) self.assertIn('\ngpg: Good signature from', verification_result.decode()) def test_031_sign_verify_detached(self): @@ -155,15 +155,15 @@ def test_031_sign_verify_detached(self): cmd = 'gpg2 --output=signature.asc -a -b --sign -u user@localhost message' p = self.frontend.run(cmd, passio_popen=True, passio_stderr=True) stdout, stderr = p.communicate() - self.assertEquals(p.returncode, 0, + self.assertEqual(p.returncode, 0, '{} failed: {}'.format(cmd, stderr.decode())) cmd = 'gpg2 --verify signature.asc message' p = self.frontend.run(cmd, passio_popen=True, passio_stderr=True) decoded_msg, verification_result = p.communicate() - self.assertEquals(p.returncode, 0, + self.assertEqual(p.returncode, 0, '{} failed: {}'.format(cmd, verification_result.decode())) - self.assertEquals(decoded_msg.decode(), '') + self.assertEqual(decoded_msg.decode(), '') self.assertIn('\ngpg: Good signature from', verification_result.decode()) # break the message and check again @@ -171,9 +171,9 @@ def test_031_sign_verify_detached(self): cmd = 'gpg2 --verify signature.asc message' p = self.frontend.run(cmd, passio_popen=True, passio_stderr=True) decoded_msg, verification_result = p.communicate() - self.assertNotEquals(p.returncode, 0, + self.assertNotEqual(p.returncode, 0, '{} unexpecedly succeeded: {}'.format(cmd, verification_result.decode())) - self.assertEquals(decoded_msg.decode(), '') + self.assertEqual(decoded_msg.decode(), '') self.assertIn('\ngpg: BAD signature from', verification_result.decode()) def test_040_encrypt_decrypt(self): @@ -181,32 +181,32 @@ def test_040_encrypt_decrypt(self): cmd = 'gpg2 --trust-model tofu -a --encrypt -r user@localhost' p = self.frontend.run(cmd, passio_popen=True, passio_stderr=True) (encrypted, stderr) = p.communicate(msg.encode()) - self.assertEquals(p.returncode, 0, + self.assertEqual(p.returncode, 0, '{} failed: {}'.format(cmd, stderr.decode())) - self.assertNotEquals('', encrypted.decode()) + self.assertNotEqual('', encrypted.decode()) cmd = "gpg2 --decrypt" p = self.frontend.run(cmd, passio_popen=True, passio_stderr=True) decoded_msg, stderr = p.communicate(encrypted) - self.assertEquals(p.returncode, 0, + self.assertEqual(p.returncode, 0, '{} failed: {}'.format(cmd, stderr.decode())) - self.assertEquals(decoded_msg.decode(), msg) + self.assertEqual(decoded_msg.decode(), msg) def test_041_sign_encrypt_decrypt(self): msg = "Test message" cmd = 'gpg2 --trust-model tofu -a --sign --encrypt -u user@localhost -r user@localhost' p = self.frontend.run(cmd, passio_popen=True, passio_stderr=True) (encrypted, stderr) = p.communicate(msg.encode()) - self.assertEquals(p.returncode, 0, + self.assertEqual(p.returncode, 0, '{} failed: {}'.format(cmd, stderr.decode())) - self.assertNotEquals('', encrypted.decode()) + self.assertNotEqual('', encrypted.decode()) cmd = "gpg2 --decrypt" p = self.frontend.run(cmd, passio_popen=True, passio_stderr=True) decoded_msg, verification_result = p.communicate(encrypted) - self.assertEquals(p.returncode, 0, + self.assertEqual(p.returncode, 0, '{} failed: {}'.format(cmd, verification_result.decode())) - self.assertEquals(decoded_msg.decode(), msg) + self.assertEqual(decoded_msg.decode(), msg) self.assertIn('\ngpg: Good signature from', verification_result.decode()) def test_050_generate(self): @@ -287,7 +287,7 @@ def test_060_import_secret(self): passio_popen=True) p.communicate(stdout) # secret key import should be refused - self.assertNotEquals(p.returncode, 0) + self.assertNotEqual(p.returncode, 0) p = self.frontend.run('gpg2 --list-keys', passio_popen=True) @@ -381,7 +381,7 @@ def get_key_fpr(self): cmd = 'gpg2 -K --with-colons' p = self.frontend.run(cmd, passio_popen=True) (stdout, _) = p.communicate() - self.assertEquals(p.returncode, 0, 'Failed to determin key id') + self.assertEqual(p.returncode, 0, 'Failed to determin key id') keyid = stdout.decode('utf-8').split('\n')[1] keyid = keyid.split(':')[9] keyid = keyid[-16:] @@ -456,7 +456,7 @@ def test_000_send_receive_default(self): self.scriptpath, self.tb_name, self.profile_dir, self.imap_pw), passio_popen=True) (stdout, _) = p.communicate() - self.assertEquals(p.returncode, 0, + self.assertEqual(p.returncode, 0, 'Thunderbird send/receive failed: {}'.format( stdout.decode('ascii', 'ignore'))) @@ -468,7 +468,7 @@ def test_010_send_receive_inline_signed_only(self): self.scriptpath, self.tb_name, self.profile_dir, self.imap_pw), passio_popen=True) (stdout, _) = p.communicate() - self.assertEquals(p.returncode, 0, + self.assertEqual(p.returncode, 0, 'Thunderbird send/receive failed: {}'.format( stdout.decode('ascii', 'ignore'))) @@ -480,7 +480,7 @@ def test_020_send_receive_inline_with_attachment(self): self.scriptpath, self.tb_name, self.profile_dir, self.imap_pw), passio_popen=True) (stdout, _) = p.communicate() - self.assertEquals(p.returncode, 0, + self.assertEqual(p.returncode, 0, 'Thunderbird send/receive failed: {}'.format( stdout.decode('ascii', 'ignore'))) @@ -550,7 +550,7 @@ def test_000_send_receive_signed_encrypted(self): self.scriptpath), passio_popen=True) (stdout, _) = p.communicate() - self.assertEquals(p.returncode, 0, + self.assertEqual(p.returncode, 0, 'Evolution send/receive failed: {}'.format( stdout.decode('ascii', 'ignore'))) @@ -561,7 +561,7 @@ def test_010_send_receive_signed_only(self): self.scriptpath), passio_popen=True) (stdout, _) = p.communicate() - self.assertEquals(p.returncode, 0, + self.assertEqual(p.returncode, 0, 'Evolution send/receive failed: {}'.format( stdout.decode('ascii', 'ignore'))) @@ -573,7 +573,7 @@ def test_020_send_receive_with_attachment(self): self.scriptpath), passio_popen=True) (stdout, _) = p.communicate() - self.assertEquals(p.returncode, 0, + self.assertEqual(p.returncode, 0, 'Evolution send/receive failed: {}'.format( stdout.decode('ascii', 'ignore'))) From 373ca733bf4a47ed481f3d748fb5799c8e8729cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 18 Dec 2024 04:30:32 +0100 Subject: [PATCH 02/10] tests: fix creating config dir --- splitgpg2tests/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/splitgpg2tests/tests.py b/splitgpg2tests/tests.py index 700bfcf..415a414 100644 --- a/splitgpg2tests/tests.py +++ b/splitgpg2tests/tests.py @@ -63,7 +63,7 @@ def setUp(self): if 'whonix' in self.template: self.backend.run("date -s +10min", user="root", wait=True) - p = self.backend.run('mkdir .config; cat > .config/qubes-split-gpg2/qubes-split-gpg2.conf', passio_popen=True) + p = self.backend.run('mkdir -p .config/qubes-split-gpg2; cat > .config/qubes-split-gpg2/qubes-split-gpg2.conf', passio_popen=True) p.communicate( b'[DEFAULT]\n' b'autoaccept = yes\n' From b3330f2cb94274795fb0cdaf2ad52b400098be48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 18 Dec 2024 04:42:04 +0100 Subject: [PATCH 03/10] Factor in FlowControlMixin It's not recommended to use it directly, but copy into own project, so do just that. See https://github.com/python/cpython/issues/79174 --- splitgpg2/stdiostream.py | 73 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 splitgpg2/stdiostream.py diff --git a/splitgpg2/stdiostream.py b/splitgpg2/stdiostream.py new file mode 100644 index 0000000..cb58ac3 --- /dev/null +++ b/splitgpg2/stdiostream.py @@ -0,0 +1,73 @@ +# +# based on asyncio library: +# Copyright (C) 2001 Python Software Foundation +# +# + +import collections +from asyncio import protocols, events + + +class FlowControlMixin(protocols.Protocol): + """Reusable flow control logic for StreamWriter.drain(). + + This implements the protocol methods pause_writing(), + resume_writing() and connection_lost(). If the subclass overrides + these it must call the super methods. + + StreamWriter.drain() must wait for _drain_helper() coroutine. + """ + + def __init__(self, loop=None): + if loop is None: + self._loop = events.get_event_loop() + else: + self._loop = loop + self._paused = False + self._drain_waiters = collections.deque() + self._connection_lost = False + + def pause_writing(self): + assert not self._paused + self._paused = True + if self._loop.get_debug(): + logger.debug("%r pauses writing", self) + + def resume_writing(self): + assert self._paused + self._paused = False + if self._loop.get_debug(): + logger.debug("%r resumes writing", self) + + for waiter in self._drain_waiters: + if not waiter.done(): + waiter.set_result(None) + + def connection_lost(self, exc): + self._connection_lost = True + # Wake up the writer(s) if currently paused. + if not self._paused: + return + + for waiter in self._drain_waiters: + if not waiter.done(): + if exc is None: + waiter.set_result(None) + else: + waiter.set_exception(exc) + + async def _drain_helper(self): + if self._connection_lost: + raise ConnectionResetError('Connection lost') + if not self._paused: + return + waiter = self._loop.create_future() + self._drain_waiters.append(waiter) + try: + await waiter + finally: + self._drain_waiters.remove(waiter) + + def _get_close_waiter(self, stream): + raise NotImplementedError + From 2eb10acb15ecd8c05b301cbf4bdac6ba972a63e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 18 Dec 2024 04:51:41 +0100 Subject: [PATCH 04/10] Convert FlowControlMixin to StdoutWriterProtocol Add support for waiting for stream close. And actually use the new class. --- splitgpg2/__init__.py | 4 +++- splitgpg2/stdiostream.py | 21 +++++++++++---------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/splitgpg2/__init__.py b/splitgpg2/__init__.py index 7fc50a7..fe1c562 100755 --- a/splitgpg2/__init__.py +++ b/splitgpg2/__init__.py @@ -46,6 +46,8 @@ from typing import Optional, Dict, Callable, Awaitable, Tuple, Pattern, List, \ Union, Any, TypeVar, Set, TYPE_CHECKING, Coroutine, Sequence, cast +from .stdiostream import StdoutWriterProtocol + if TYPE_CHECKING: from typing_extensions import Protocol from typing import TypeAlias @@ -1405,7 +1407,7 @@ def open_stdinout_connection(*, write_transport, write_protocol = loop.run_until_complete( loop.connect_write_pipe( - lambda: asyncio.streams.FlowControlMixin(loop), + lambda: StdoutWriterProtocol(loop), sys.stdout.buffer)) writer = asyncio.StreamWriter(write_transport, write_protocol, None, loop) diff --git a/splitgpg2/stdiostream.py b/splitgpg2/stdiostream.py index cb58ac3..06ec3b8 100644 --- a/splitgpg2/stdiostream.py +++ b/splitgpg2/stdiostream.py @@ -2,19 +2,18 @@ # based on asyncio library: # Copyright (C) 2001 Python Software Foundation # +# Copyright (C) 2024 Marek Marczykowski-Górecki +# # import collections from asyncio import protocols, events - -class FlowControlMixin(protocols.Protocol): +class StdoutWriterProtocol(protocols.Protocol): """Reusable flow control logic for StreamWriter.drain(). - This implements the protocol methods pause_writing(), resume_writing() and connection_lost(). If the subclass overrides these it must call the super methods. - StreamWriter.drain() must wait for _drain_helper() coroutine. """ @@ -26,18 +25,15 @@ def __init__(self, loop=None): self._paused = False self._drain_waiters = collections.deque() self._connection_lost = False + self._closed = self._loop.create_future() def pause_writing(self): assert not self._paused self._paused = True - if self._loop.get_debug(): - logger.debug("%r pauses writing", self) def resume_writing(self): assert self._paused self._paused = False - if self._loop.get_debug(): - logger.debug("%r resumes writing", self) for waiter in self._drain_waiters: if not waiter.done(): @@ -55,6 +51,11 @@ def connection_lost(self, exc): waiter.set_result(None) else: waiter.set_exception(exc) + if not self._closed.done(): + if exc is None: + self._closed.set_result(None) + else: + self._closed.set_exception(exc) async def _drain_helper(self): if self._connection_lost: @@ -68,6 +69,6 @@ async def _drain_helper(self): finally: self._drain_waiters.remove(waiter) + # pylint: disable=unused-argument def _get_close_waiter(self, stream): - raise NotImplementedError - + return self._closed From 4ee0498bf65fc49ac32eeb35f7a7979252626fe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 18 Dec 2024 04:54:41 +0100 Subject: [PATCH 05/10] ci: drop R4.1 add R4.3 --- .gitlab-ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d24fc49..1b8a36c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,16 +1,16 @@ include: - - project: 'QubesOS/qubes-continuous-integration' - file: '/r4.1/gitlab-base.yml' - - project: 'QubesOS/qubes-continuous-integration' - file: '/r4.1/gitlab-dom0.yml' - - project: 'QubesOS/qubes-continuous-integration' - file: '/r4.1/gitlab-vm.yml' - project: 'QubesOS/qubes-continuous-integration' file: '/r4.2/gitlab-base.yml' - project: 'QubesOS/qubes-continuous-integration' file: '/r4.2/gitlab-host.yml' - project: 'QubesOS/qubes-continuous-integration' file: '/r4.2/gitlab-vm.yml' + - project: 'QubesOS/qubes-continuous-integration' + file: '/r4.3/gitlab-base.yml' + - project: 'QubesOS/qubes-continuous-integration' + file: '/r4.3/gitlab-host.yml' + - project: 'QubesOS/qubes-continuous-integration' + file: '/r4.3/gitlab-vm.yml' checks:tests: stage: checks From 7cc989fa45c018e58781a02cbe5aff7dc415bbf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Wed, 18 Dec 2024 12:14:48 +0100 Subject: [PATCH 06/10] Add type annotations to StdoutWriterProtocol --- splitgpg2/stdiostream.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/splitgpg2/stdiostream.py b/splitgpg2/stdiostream.py index 06ec3b8..7638b3c 100644 --- a/splitgpg2/stdiostream.py +++ b/splitgpg2/stdiostream.py @@ -7,7 +7,8 @@ # import collections -from asyncio import protocols, events +from asyncio import protocols, events, Future +from typing import Optional, Any class StdoutWriterProtocol(protocols.Protocol): """Reusable flow control logic for StreamWriter.drain(). @@ -17,21 +18,22 @@ class StdoutWriterProtocol(protocols.Protocol): StreamWriter.drain() must wait for _drain_helper() coroutine. """ - def __init__(self, loop=None): + def __init__(self, loop: Optional[events.AbstractEventLoop] = None) -> None: if loop is None: self._loop = events.get_event_loop() else: self._loop = loop self._paused = False - self._drain_waiters = collections.deque() + self._drain_waiters: collections.deque[Future[None]] = \ + collections.deque() self._connection_lost = False self._closed = self._loop.create_future() - def pause_writing(self): + def pause_writing(self) -> None: assert not self._paused self._paused = True - def resume_writing(self): + def resume_writing(self) -> None: assert self._paused self._paused = False @@ -39,7 +41,7 @@ def resume_writing(self): if not waiter.done(): waiter.set_result(None) - def connection_lost(self, exc): + def connection_lost(self, exc: Optional[BaseException]) -> None: self._connection_lost = True # Wake up the writer(s) if currently paused. if not self._paused: @@ -57,7 +59,7 @@ def connection_lost(self, exc): else: self._closed.set_exception(exc) - async def _drain_helper(self): + async def _drain_helper(self) -> None: if self._connection_lost: raise ConnectionResetError('Connection lost') if not self._paused: @@ -70,5 +72,5 @@ async def _drain_helper(self): self._drain_waiters.remove(waiter) # pylint: disable=unused-argument - def _get_close_waiter(self, stream): + def _get_close_waiter(self, stream: Any) -> Future[None]: return self._closed From e6fa1f8cafeaeb30bae1d641f603f77130661cdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 19 Dec 2024 11:57:02 +0100 Subject: [PATCH 07/10] Do not output keyring import messages to the client stderr Those messages are not relevant for the client, and may leak some info about the configuration. --- splitgpg2/__init__.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/splitgpg2/__init__.py b/splitgpg2/__init__.py index fe1c562..e238e40 100755 --- a/splitgpg2/__init__.py +++ b/splitgpg2/__init__.py @@ -328,15 +328,21 @@ def setup_subkey_keyring(self) -> None: self.gnupghome, self.source_keyring_dir) with subprocess.Popen(export_cmd, stdout=subprocess.PIPE, - stdin=subprocess.DEVNULL) as exporter, ( + stdin=subprocess.DEVNULL, + stderr=subprocess.PIPE) as exporter, ( subprocess.Popen(import_cmd, - stdin=exporter.stdout)) as importer: + stdin=exporter.stdout, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE)) as importer: pass if exporter.returncode or importer.returncode: self.log.warning('Unable to export keys. If your key has a ' 'passphrase, you might want to save it to a ' 'file and use passphrase-file and ' - 'pinentry-mode loopback in gpg.conf') + 'pinentry-mode loopback in gpg.conf.') + self.log.warning("Exporter output: %s", exporter.stderr) + self.log.warning("Importer output: %s %s", + importer.stdout, importer.stderr) self.log.info('Subkey-only keyring %r created', self.gnupghome) From 49385e9965e0ace0789eb8d8ed3bfa639dab3024 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 19 Dec 2024 11:59:15 +0100 Subject: [PATCH 08/10] tests: log stdout/stderr on key generation failure --- splitgpg2tests/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/splitgpg2tests/tests.py b/splitgpg2tests/tests.py index 415a414..38c9ce2 100644 --- a/splitgpg2tests/tests.py +++ b/splitgpg2tests/tests.py @@ -43,7 +43,7 @@ def setUp(self): p = self.backend.run('mkdir -p -m 0700 .gnupg; gpg2 --gen-key --batch', passio_popen=True, passio_stderr=True) - p.communicate(''' + stdout, stderr = p.communicate(''' Key-Type: RSA Key-Length: 1024 Key-Usage: sign @@ -59,7 +59,7 @@ def setUp(self): if p.returncode == 127: self.skipTest('gpg2 not installed') elif p.returncode != 0: - self.fail('key generation failed') + self.fail('key generation failed: {}{}'.format(stdout, stderr)) if 'whonix' in self.template: self.backend.run("date -s +10min", user="root", wait=True) From bd30173c1996dba720a150076a49e988d823fec0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 19 Dec 2024 11:59:37 +0100 Subject: [PATCH 09/10] tests: generate signing subkey Since the subkeys-only keyring feature is enabled by default (source_keyring_dir != "no"), signing with the GnuPG's defaults fails. Generate a signing subkey using gpg2 --quick-add-key. --- splitgpg2tests/tests.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/splitgpg2tests/tests.py b/splitgpg2tests/tests.py index 38c9ce2..01b9719 100644 --- a/splitgpg2tests/tests.py +++ b/splitgpg2tests/tests.py @@ -60,6 +60,24 @@ def setUp(self): self.skipTest('gpg2 not installed') elif p.returncode != 0: self.fail('key generation failed: {}{}'.format(stdout, stderr)) + + cmd = 'gpg2 --with-colons --list-key user@localhost' + p = self.backend.run(cmd, passio_popen=True, passio_stderr=True) + (stdout, stderr) = p.communicate() + self.assertEqual(p.returncode, 0, + '{} failed: {}{}'.format(cmd, stdout.decode(), stderr.decode())) + fpr = [l for l in stdout.splitlines() if l.startswith(b'fpr:')][0] + fpr = fpr.decode().split(":")[9] + # add signing subkey + p = self.backend.run( + f"gpg2 --batch --passphrase "" --quick-add-key " + f"{fpr} rsa sign never", + passio_popen=True, + passio_stderr=True) + stdout, stderr = p.communicate() + if p.returncode != 0: + self.fail('subkey generation failed: {}{}'.format( + stdout, stderr)) if 'whonix' in self.template: self.backend.run("date -s +10min", user="root", wait=True) From b2fb67d19a75171f07dbd986e8567d134a59723a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Thu, 19 Dec 2024 12:04:37 +0100 Subject: [PATCH 10/10] tests: update Thunderbird/Evolution interactions Sync with app-linux-split-gpg, especially the following commits: a652de5 test: avoid false negative from sending status dialog 4bcba03 tests: update for Thunderbird 102 15fda8c tests: disable end-of-year message, and similar popups b9c13d0 tests: fix clicking top buttons in evolution d65f098 tests: use distribution's dogtail package 00394da tests: try harder to avoid donation prompt during tests 3bf5616 tests: update for Thunderbird 115 7477c2d tests: switch from smtpd to aiosmtpd 17f96e0 tests: handle both Save and Save All dialogs e46af5f tests: adjust for Thunderbird 128 --- splitgpg2tests/tests.py | 16 ++-- tests/test_evolution.py | 32 +++++--- tests/test_thunderbird.py | 160 ++++++++++++++++++++++++++++---------- 3 files changed, 152 insertions(+), 56 deletions(-) diff --git a/splitgpg2tests/tests.py b/splitgpg2tests/tests.py index 01b9719..62b6735 100644 --- a/splitgpg2tests/tests.py +++ b/splitgpg2tests/tests.py @@ -360,18 +360,18 @@ def setUp(self): # run as root to not deal with /var/mail permission issues self.frontend.run( - 'touch /var/mail/user; chown user:user /var/mail/user', user='root', + 'mkdir -p Mail/new Mail/cur Mail/tmp', wait=True) # SMTP configuration self.smtp_server = self.frontend.run( - 'python3 /usr/share/split-gpg2-tests/test_smtpd.py', - user='root', passio_popen=True) + 'aiosmtpd -n -c aiosmtpd.handlers.Mailbox /home/user/Mail', + passio_popen=True) # IMAP configuration self.imap_pw = "pass" self.frontend.run( - 'echo "mail_location=mbox:~/Mail:INBOX=/var/mail/%u" |\ + 'echo "mail_location=maildir:~/Mail" |\ sudo tee /etc/dovecot/conf.d/100-mail.conf', wait=True) self.frontend.run('sudo systemctl restart dovecot', wait=True) self.frontend.run( # set a user password because IMAP needs one for auth @@ -422,6 +422,8 @@ def setup_tb_profile(self, setup_openpgp): user_pref("mail.identity.id1.useremail", "user@localhost"); user_pref("mail.identity.id1.smtpServer", "smtp1"); user_pref("mail.identity.id1.compose_html", false); +user_pref("datareporting.policy.dataSubmissionEnabled", false); // avoid message popups +user_pref("app.donation.eoy.version.viewed", 100); // avoid message popups """ imap_server = """ user_pref("mail.server.server1.userName", "user"); @@ -542,11 +544,11 @@ def setUp(self): # run as root to not deal with /var/mail permission issues self.frontend.run( - 'touch /var/mail/user; chown user /var/mail/user', user='root', + 'mkdir -p Mail/new Mail/cur Mail/tmp', wait=True) self.smtp_server = self.frontend.run( - 'python3 /usr/share/split-gpg2-tests/test_smtpd.py', - user='root', passio_popen=True) + 'aiosmtpd -n -c aiosmtpd.handlers.Mailbox /home/user/Mail', + passio_popen=True) p = self.frontend.run( 'PYTHONPATH=$HOME/dogtail python3 {} setup 2>&1'.format( diff --git a/tests/test_evolution.py b/tests/test_evolution.py index 8f69b0d..d6fd3a8 100644 --- a/tests/test_evolution.py +++ b/tests/test_evolution.py @@ -58,6 +58,16 @@ def open_accounts(app): def get_sibling_offset(node, offset): return node.parent.children[node.indexInParent+offset] +def get_sibling_button_maybe(button): + try: + # if there is a sibling button (without the name) that's the one that works + button_sibling = button.parent.children[button.indexInParent + 1] + if button_sibling.roleName == "push button": + return button_sibling + except KeyError: + pass + return button + def add_local_account(app): accounts_tab = None settings = None @@ -78,16 +88,16 @@ def add_local_account(app): wizard.button('Next').doActionNamed('click') # Receiving Email tab time.sleep(2) - wizard.menuItem('Local delivery').doActionNamed('click') - wizard.childLabelled('Local Delivery File:').parent.button('(None)').\ + wizard.menuItem('Maildir-format mail directories').doActionNamed('click') + wizard.childLabelled('Mail Directory:', showingOnly=True).parent.menuItem('Other…').\ doActionNamed('click') - file_chooser = app.child('Choose a local delivery file', + file_chooser = app.child('Choose a Maildir mail directory', roleName='file chooser') file_chooser.child('File System Root').doActionNamed('click') - file_chooser.child('var').doActionNamed('activate') - file_chooser.child('spool').doActionNamed('activate') - file_chooser.child('mail').doActionNamed('activate') + file_chooser.child('home').doActionNamed('activate') file_chooser.child('user').doActionNamed('activate') + file_chooser.child('Mail').doActionNamed('activate') + file_chooser.button('Open').doActionNamed('click') time.sleep(1) wizard.button('Next').doActionNamed('click') # Receiving Options tab @@ -138,7 +148,9 @@ def attach(app, compose_window, path): file_chooser.button('Attach').doActionNamed('click') def send_email(app, sign=False, encrypt=False, inline=False, attachment=None): - app.button('New').doActionNamed('click') + new_button = app.button('New') + new_button = get_sibling_button_maybe(new_button) + new_button.doActionNamed('click') new_message = app.child('Compose Message', roleName='frame') new_message.textentry('To:').text = 'user@localhost,' new_message.childLabelled('Subject:').text = subject @@ -160,8 +172,10 @@ def send_email(app, sign=False, encrypt=False, inline=False, attachment=None): new_message.button('Send').doActionNamed('click') def receive_message(app, signed=False, encrypted=False, attachment=None): - app.button('Send / Receive').doActionNamed('click') - app.child(name='Inbox.*', roleName='table cell').doActionNamed('edit') + send_receive = app.button('Send / Receive') + send_receive = get_sibling_button_maybe(send_receive) + send_receive.doActionNamed('click') + app.child(name='Inbox .*', roleName='table cell').doActionNamed('edit') messages = app.child('Messages', roleName='panel') messages.child(subject).grabFocus() message = app.child('Evolution Mail Display', roleName='document web') diff --git a/tests/test_thunderbird.py b/tests/test_thunderbird.py index 77081b7..eb8d4ba 100644 --- a/tests/test_thunderbird.py +++ b/tests/test_thunderbird.py @@ -119,7 +119,7 @@ def wrapper(*args, **kwargs): func(*args, **kwargs) break # if successful except Exception as e: - if retry == max_tries: + if retry == max_tries-1: raise e else: print("failed during setup in {}.\n Retrying".format( @@ -151,20 +151,27 @@ def export_pub_key(): @retry_if_failed(max_tries=3) def enter_imap_passwd(tb): - # check new mail so client can realize IMAP requires entering a password - get_messages(tb) + try: + pass_prompt = tb.app.findChild(orPredicate( + GenericPredicate(name='Enter your password for user', roleName='frame'), + GenericPredicate(name='Enter your password for user', roleName='dialog') + )) + except tree.SearchError: + # check new mail so client can realize IMAP requires entering a password + get_messages(tb) # password entry pass_prompt = tb.app.findChild(orPredicate( GenericPredicate(name='Enter your password for user', roleName='frame'), GenericPredicate(name='Enter your password for user', roleName='dialog') )) pass_textbox = pass_prompt.findChild(GenericPredicate(roleName='password text')) - pass_textbox.text = tb.imap_pw + pass_textbox.typeText(tb.imap_pw) pass_prompt.childNamed("Use Password Manager to remember this password.")\ .doActionNamed('check') pass_prompt.findChild(orPredicate( GenericPredicate(name='OK', roleName='push button'), # tb < 91 - GenericPredicate(name='Sign in', roleName='push button')) # tb >= 91 + GenericPredicate(name='Sign in', roleName='push button'), # tb >= 91, tb < 128 + GenericPredicate(name='OK', roleName='button')) # tb >= 128 ).doActionNamed('press') def accept_qubes_attachments(tb): @@ -237,8 +244,14 @@ def configure_openpgp_account(tb): accept_dialog = tb.app.findChild(orPredicate( GenericPredicate(name='.*(%s).*' % keyid), GenericPredicate(name='.[0-9A-F]*%s' % keyid), - )).parent - accept_dialog.childNamed('OK').doActionNamed('press') + GenericPredicate(name='ID: 0x%s' % keyid), + )).parent.parent + try: + accept_dialog.childNamed("Accepted.*").doActionNamed("select") + except tree.SearchError: + # old TB + pass + accept_dialog.childNamed('OK|Import').doActionNamed('press') tb.app.childNamed('Success! Keys imported.*').childNamed('OK').doActionNamed( 'press') doubleClick(*key_manager.findChild( @@ -256,16 +269,37 @@ def configure_openpgp_account(tb): def get_messages(tb): - tb.app.child(name='user@localhost', - roleName='table row').doActionNamed('activate') - tb.app.button('Get Messages').doActionNamed('press') - tb.app.menuItem('Get All New Messages').doActionNamed('click') - tb.app.child(name='Inbox.*', roleName='table row').doActionNamed( - 'activate') + try: + # TB >= 115 + try: + # TB >= 128 + tb.app.child('Get Messages', roleName='button').doActionNamed('press') + except tree.SearchError: + # TB < 128 + tb.app.button('Get Messages').doActionNamed('press') + tb.app.child(name='Inbox.*', roleName='tree item').doActionNamed( + 'activate') + except tree.SearchError: + # TB < 115 + tb.app.child(name='user@localhost', + roleName='table row').doActionNamed('activate') + tb.app.button('Get Messages').doActionNamed('press') + tb.app.menuItem('Get All New Messages').doActionNamed('click') + tb.app.child(name='Inbox.*', roleName='table row').doActionNamed( + 'activate') + def attach(tb, compose_window, path): - compose_window.button('Attach').button('Attach').doActionNamed('press') - compose_window.button('Attach').menuItem('File.*').doActionNamed('click') + try: + # TB >= 128 + compose_window.child('Attach', roleName='button').\ + doActionNamed('press') + compose_window.child('Attach', roleName='button').\ + menuItem('File.*').doActionNamed('click') + except tree.SearchError: + # TB < 128 + compose_window.button('Attach').button('Attach').doActionNamed('press') + compose_window.button('Attach').menuItem('File.*').doActionNamed('click') # for some reason on some thunderbird versions do not expose 'Attach File' # dialog through accessibility API, use xdotool instead subprocess.check_call( @@ -293,18 +327,25 @@ def attach(tb, compose_window, path): def send_email(tb, sign=False, encrypt=False, inline=False, attachment=None): config.searchCutoffCount = 20 - write = tb.app.button('Write') + try: + # TB >= 128 + write = tb.app.child(name='New Message', roleName='button') + except tree.SearchError: + try: + write = tb.app.button('New Message') + except tree.SearchError: + write = tb.app.button('Write') config.searchCutoffCount = defaultCutoffCount write.doActionNamed('press') compose = tb.app.child(name='Write: .*', roleName='frame') to_entry = compose.findChild(TBEntry(name='To')) - to_entry.text = 'user@localhost' + to_entry.typeText('user@localhost') # lets thunderbird settle down on default values (after filling recipients) time.sleep(1) subject_entry = compose.findChild( orPredicate(GenericPredicate(name='Subject:', roleName='entry'), TBEntry(name='Subject'))) - subject_entry.text = subject + subject_entry.typeText(subject) try: compose_document = compose.child(roleName='document web') try: @@ -315,18 +356,29 @@ def send_email(tb, sign=False, encrypt=False, inline=False, attachment=None): except tree.SearchError: compose.child( roleName='document frame').text = 'This is test message' - security = compose.findChild( - GenericPredicate(name='Security', roleName='push button')) + try: + # TB >= 128 + security = compose.findChild( + GenericPredicate(name='Security|OpenPGP', roleName='button')) + except tree.SearchError: + # TB < 128 + security = compose.findChild( + GenericPredicate(name='Security|OpenPGP', roleName='push button')) security.doActionNamed('press') - sign_button = security.childNamed('Digitally Sign This Message') - encrypt_button = security.childNamed('Require Encryption') + sign_button = security.childNamed('Digitally Sign.*') + encrypt_button = security.childNamed('Require Encryption|Encrypt') if sign_button.checked != sign: sign_button.doActionNamed('click') if encrypt_button.checked != encrypt: encrypt_button.doActionNamed('click') if attachment: attach(tb, compose, attachment) - compose.button('Send').doActionNamed('press') + try: + # TB >= 128 + compose.child('Send', roleName='button').doActionNamed('press') + except tree.SearchError: + # TB < 128 + compose.button('Send').doActionNamed('press') config.searchCutoffCount = 5 try: if encrypt: @@ -363,16 +415,36 @@ def send_email(tb, sign=False, encrypt=False, inline=False, attachment=None): def receive_message(tb, signed=False, encrypted=False, attachment=None): get_messages(tb) - config.searchCutoffCount = 5 + if encrypted: + config.searchCutoffCount = 5 + try: + # TB >= 128 + tb.app.child(name='user[^,]*, .*, \.\.\..*', + roleName='table row').doActionNamed('clickAncestor') + except tree.SearchError: + try: + # TB >= 115 + tb.app.child(name='user[^,]*, .*, \.\.\..*', + roleName='tree item').doActionNamed('activate') + except tree.SearchError: + # TB < 115 + tb.app.child(name='Encrypted Message .*|.*\.\.\. .*', + roleName='table row').doActionNamed('activate') + finally: + config.searchCutoffCount = defaultCutoffCount try: - tb.app.child(name='Encrypted Message .*|.*\.\.\. .*', - roleName='table row').doActionNamed('activate') + # TB >= 128 + tb.app.child(name='.*{}.*'.format(subject), + roleName='table row').doActionNamed('clickAncestor') except tree.SearchError: - pass - finally: - config.searchCutoffCount = defaultCutoffCount - tb.app.child(name='.*{}.*'.format(subject), - roleName='table row').doActionNamed('activate') + try: + # TB >= 115 + tb.app.child(name='.*{}.*'.format(subject), + roleName='tree item').doActionNamed('activate') + except tree.SearchError: + # TB < 115 + tb.app.child(name='.*{}.*'.format(subject), + roleName='table row').doActionNamed('activate') # wait a little to TB decrypt/check the message time.sleep(2) # dogtail always add '$' at the end of regexp; and also "Escape all @@ -399,11 +471,10 @@ def receive_message(tb, signed=False, encrypted=False, attachment=None): # msg_body = msg.text config.searchCutoffCount = 5 try: - if signed or encrypted: - tb.app.button('OpenPGP.*').doActionNamed('press') - # 'Message Security - OpenPGP' is an internal label, - # nested 2 levels into the popup - message_security = tb.app.child('Message Security - OpenPGP') + tb.app.button('OpenPGP.*').doActionNamed('press') + # 'Message Security - OpenPGP' is an internal label, + # nested 2 levels into the popup + message_security = tb.app.child('Message Security - OpenPGP') except tree.SearchError: # alternative way of opening 'message security' keyCombo('s') @@ -430,8 +501,14 @@ def receive_message(tb, signed=False, encrypted=False, attachment=None): attachment_size = attachment_label.parent.children[ attachment_label.indexInParent + 1 + offset] assert attachment_size.text[0] != '0' - attachment_save = attachment_label.parent.children[ - attachment_label.indexInParent + 2 + offset].button('Save.*') + attachment_save_parent = attachment_label.parent.children[ + attachment_label.indexInParent + 2 + offset] + try: + # TB >= 128 + attachment_save = attachment_save_parent.child('Save.*', roleName='button') + except tree.SearchError: + # TB < 128 + attachment_save = attachment_save_parent.button('Save.*') try: # try child button first attachment_save.children[1].doActionNamed('press') @@ -444,11 +521,14 @@ def receive_message(tb, signed=False, encrypted=False, attachment=None): # for some reasons some Thunderbird versions do not expose 'Attach File' # dialog through accessibility API, use xdotool instead save_as = tb.app.findChild( - GenericPredicate(name='Save All Attachments', + GenericPredicate(name='Save All Attachments|Save Attachment', roleName='file chooser')) click(*save_as.childNamed('Home').position) click(*save_as.childNamed('Desktop').position) - save_as.childNamed('Open').doActionNamed('click') + if save_as.name == 'Save Attachment': + save_as.childNamed('Save').doActionNamed('click') + else: + save_as.childNamed('Open').doActionNamed('click') # save_as = tb.app.dialog('Save .*Attachment.*') # places = save_as.child(roleName='table', # name='Places')