From 426edd69835f7e4b386e2b769ae97a140a4b91f1 Mon Sep 17 00:00:00 2001 From: heevasti Date: Thu, 22 Aug 2024 10:14:24 +0200 Subject: [PATCH 1/5] [QMI-090] Added RPCable `__enter__` and `__exit__` methods to `QMI_Instrument` and `QMI_TaskRunner` classes, which inherit from `QMI_RpcProxy`. The latter has also now the same methods implemented but not as RPC methods. Added warnings of obsoletion into `open_close` and `start_stop_join` context managers in `qmi.utils.context_managers`. Added unit-tests, and added and improved docstrings. --- CHANGELOG.md | 5 + qmi/core/instrument.py | 15 +- qmi/core/rpc.py | 54 +++++- qmi/core/task.py | 15 +- qmi/utils/context_managers.py | 13 ++ tests/core/test_rpc.py | 97 +++++++++-- tests/core/test_tasks.py | 320 +++++++++++++++++++++++++++++++++- 7 files changed, 496 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fb8f8f6..bbb01ffc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## \[x.y.z] - Unreleased +### Added +- The `QMI_Instrument` and `QMI_TaskRunner` (which inherit from `QMI_RpcObject`) are now equipped with specific `__enter__` and `__exit__` methods, which in the case of `QMI_Instrument` + also open and close the instrument when run with a `with` context manager protocol. Meanwhile `QMI_TaskRunner` starts and stops then joins a QMI task thread. In practise, these context managers + can be used instead of the to-be-obsoleted `open_close` and `start_stop_join` context managers. The context manager protocol cannot be used for `QMI_RpcObject` directly. + ### Changed - The CI pipelines are now using reusable workflows, placed in reusable-ci-workflows.yml. - The file names for the different pipeline actions were also changed to be more descriptive. diff --git a/qmi/core/instrument.py b/qmi/core/instrument.py index bdf37961..be369905 100644 --- a/qmi/core/instrument.py +++ b/qmi/core/instrument.py @@ -47,7 +47,7 @@ class QMI_Instrument(QMI_RpcObject): (measurements, getting and setting of parameters). Driver should implement a method `reset()` when applicable. - This methods returns the instrument to its default settings. + This method returns the instrument to its default settings. Drivers should implement a method `get_idn()` when applicable. This method returns an instance of `QMI_InstrumentIdentification`. @@ -66,6 +66,19 @@ def __init__(self, context: 'qmi.core.context.QMI_Context', name: str) -> None: super().__init__(context, name) self._is_open = False + @rpc_method + def __enter__(self) -> "QMI_Instrument": + """The `__enter__` methods is decorated as `rpc_method` so that `QMI_RpcProxy` can call it when using the + proxy with a `with` context manager. This method also opens the instrument.""" + self.open() + return self + + @rpc_method + def __exit__(self, *args, **kwargs) -> None: + """The `__exit__` methods is decorated as `rpc_method` so that `QMI_RpcProxy` can call it when using the + proxy with a `with` context manager. This method also closes the instrument.""" + self.close() + def release_rpc_object(self) -> None: """Give a warning if the instrument is removed while still open.""" if self._is_open: diff --git a/qmi/core/rpc.py b/qmi/core/rpc.py index eca4b725..9200cd80 100644 --- a/qmi/core/rpc.py +++ b/qmi/core/rpc.py @@ -83,7 +83,7 @@ def square(self, x): Since QMI V0.29.1 it is also possible to lock and unlock with a custom token from other proxies as well, as long as the contexts for the other proxies have the same name. -Example 1: +Example 1:: # Two proxies in same context. proxy1 = context.make_rpc_object("my_object", MyRpcTestClass) proxy2 = context.get_rpc_object_by_name("my_context.my_object") @@ -94,7 +94,7 @@ def square(self, x): proxy2.unlock(lock_token=custom_token) proxy2.is_locked() # Returns False -Example 2: +Example 2:: # Three proxies in different contexts. The first one serves as an "object provider". c1 = QMI_Context("c1", config) c1.start() @@ -557,7 +557,20 @@ def blocking_rpc_method_call(context: "qmi.core.context.QMI_Context", class QMI_RpcNonBlockingProxy: - """Proxy class for RPC objects that performs non-blocking calls.""" + """Proxy class for RPC objects that performs non-blocking calls. Direct instantiation is not recommended. + + This is always also instantiated in `QMI_RpcProxy` as `self.rpc_nonblocking` attribute. Typically, if user wants + to use a non-blocking call with an RPC object `rpc_proxy`, they would do: + ```python + future = rpc_proxy.rpc_nonblocking.some_rpc_command(args) + # Other stuff can be done here in the meanwhile, and the `proxy` is not blocked while waiting to return + retval = future.wait() + ``` + Instead of the usual + ```python + retval = rpc_proxy.some_rpc_commands(args) + ``` + """ def __init__(self, context: "qmi.core.context.QMI_Context", descriptor: RpcObjectDescriptor) -> None: @@ -602,7 +615,11 @@ def __repr__(self) -> str: class QMI_RpcProxy: - """Proxy class for RPC objects that performs blocking calls.""" + """Proxy class for RPC objects that performs blocking calls. All RPC objects created return this proxy class to + enable RPC communication between objects. + + Direct instantiation of this class is not meant to be done by users; internal use only! + """ def __init__(self, context: "qmi.core.context.QMI_Context", descriptor: RpcObjectDescriptor) -> None: @@ -659,6 +676,26 @@ def make_rpc_forward_function(method_name: str): # Add non-blocking proxy. self.rpc_nonblocking = QMI_RpcNonBlockingProxy(context, descriptor) + def __enter__(self) -> "QMI_RpcProxy": + """The context manager definition is needed for the proxy as it will always be returned from QMI contexts, + instead of the actual RPC object instance. Trying to use context management directly on this class will + cause recursion error, but for actual RPC objects not. + + The with keyword checks if the `QMI_RpcProxy` class (not instance) implements the context manager protocol, + i.e. `__enter__` and `__exit__`, so they exist here as stubs. These stubs make an RPC to the actual `__enter__` + and `__exit__` methods on the relevant RPC object. If in those classes `__enter__` and `__exit__` are + decorated as `rpc_methods`, we do not see a recursion error. + + `rpc_method` decorated `__enter__` and `__exit__` methods are currently implemented in `QMI_Instrument` and + `QMI_TaskRunner` classes. + """ + self.__enter__() + return self + + def __exit__(self, *args, **kwargs): + self.__exit__() + return + def __repr__(self) -> str: return "".format(self._rpc_object_address, self._rpc_class_fqn) @@ -891,9 +928,13 @@ def __init__(self, def __repr__(self) -> str: return "{}({!r})".format(type(self).__name__, self._name) + @rpc_method + def __enter__(self): + raise NotImplementedError(f"{type(self)} is not meant to be used with context manager.") + def lock(self, timeout: float = 0.0, lock_token: Optional[str] = None) -> bool: - """Lock the remote object. If timeout is given, try every 0.1s within the given timeout value. The remote object - can be locked with an optional custom lock token by giving a string into `lock_token` keyword argument. + """Lock the remote object. If timeout is given, try every 0.1s within the given timeout value. The remote + object can be locked with an optional custom lock token by giving a string into `lock_token` keyword argument. If successful, this proxy is the only proxy that can invoke RPC methods on the remote object; other proxies will receive an "object is locked" response. The return value indicates if the lock was granted; a denied lock @@ -1354,7 +1395,6 @@ def make_proxy(self) -> QMI_RpcProxy: # This may raise an exception! rpc_object = self._rpc_thread.rpc_object() - return QMI_RpcProxy(self._context, rpc_object.rpc_object_descriptor) def handle_message(self, message: QMI_Message) -> None: diff --git a/qmi/core/task.py b/qmi/core/task.py index 0a11c98b..6e6924b3 100644 --- a/qmi/core/task.py +++ b/qmi/core/task.py @@ -262,7 +262,7 @@ def update_settings(self) -> bool: to this task, this method copies the new settings to `self.settings` and returns True. - Otherwise the settings remain the same and this method returns False. + Otherwise, the settings remain the same and this method returns False. Returns: True if there are new settings, False if the settings are unchanged. @@ -602,6 +602,19 @@ def __init__(self, assert state == _TaskThread.State.READY_TO_RUN assert self._thread.task is not None + @rpc_method + def __enter__(self): + """The `__enter__` methods is decorated as `rpc_method` so that `QMI_RpcProxy` can call it when using the + proxy with a `with` context manager. This method also calls to start the task thread.""" + return self.start() + + @rpc_method + def __exit__(self, *args, **kwargs): + """The `__exit__` methods is decorated as `rpc_method` so that `QMI_RpcProxy` can call it when using the + proxy with a `with` context manager. This method also calls to stop and join the task thread.""" + self.stop() + self.join() + def release_rpc_object(self) -> None: """Ensure the task is joined before it is removed from the context.""" if not self._joined: diff --git a/qmi/utils/context_managers.py b/qmi/utils/context_managers.py index 207ea7c8..ebb99261 100644 --- a/qmi/utils/context_managers.py +++ b/qmi/utils/context_managers.py @@ -1,6 +1,7 @@ """Context managers for QMI RPC protocol contexts.""" from contextlib import contextmanager from typing import Iterator, Optional, Protocol, TypeVar +import warnings from qmi.core.pubsub import QMI_SignalReceiver, QMI_SignalSubscriber @@ -47,6 +48,12 @@ def start_stop(thing: _SS, *args, **kwargs) -> Iterator[_SS]: @contextmanager def start_stop_join(thing: _SSJ, *args, **kwargs) -> Iterator[_SSJ]: + warnings.warn( + "This context manager is obsoleted. The tasks can now by managed directly by their own context manager " + "by calling `with qmi.make_task('task_name', TaskClass, args, kwargs) as task: ...`.", + DeprecationWarning, + stacklevel=3 + ) thing.start(*args, **kwargs) try: yield thing @@ -57,6 +64,12 @@ def start_stop_join(thing: _SSJ, *args, **kwargs) -> Iterator[_SSJ]: @contextmanager def open_close(thing: _OC) -> Iterator[_OC]: + warnings.warn( + "This context manager is obsoleted. The instruments can now by managed directly by their own context manager " + "by calling `with qmi.make_instrument('instr_name', InstrClass, args, kwargs) as instr: ...`.", + DeprecationWarning, + stacklevel=3 + ) thing.open() try: yield thing diff --git a/tests/core/test_rpc.py b/tests/core/test_rpc.py index 2cdfd843..c2ef3f76 100644 --- a/tests/core/test_rpc.py +++ b/tests/core/test_rpc.py @@ -3,20 +3,23 @@ import logging import math import time +from typing import NamedTuple import unittest -from unittest.mock import Mock +from unittest.mock import Mock, MagicMock from qmi.core.config_defs import CfgQmi, CfgContext from qmi.core.context import QMI_Context -from qmi.core.exceptions import QMI_MessageDeliveryException, QMI_UsageException, QMI_InvalidOperationException,\ - QMI_DuplicateNameException -from qmi.core.rpc import QMI_RpcObject, rpc_method, QMI_RpcTimeoutException, QMI_RpcFuture - +from qmi.core.exceptions import ( + QMI_MessageDeliveryException, QMI_UsageException, QMI_InvalidOperationException, QMI_DuplicateNameException +) +from qmi.core.rpc import ( + QMI_RpcObject, rpc_method, QMI_RpcTimeoutException, QMI_RpcFuture, QMI_RpcProxy, QMI_RpcNonBlockingProxy +) from threading import Timer class MyRpcTestClass(QMI_RpcObject): - + """An RPC test class""" _rpc_constants = ["CONSTANT_NUMBER"] CONSTANT_NUMBER = 42 @@ -44,7 +47,7 @@ def my_lock_method(self, lock_token): class MyRpcSubClass(MyRpcTestClass): - + """An RPC sub class""" _rpc_constants = ["CONSTANT_STRING"] CONSTANT_STRING = "testing" @@ -54,6 +57,67 @@ def remote_log(self, x): return math.log(x) +class ProxyInterface(NamedTuple): + """For testing proxy descriptor.""" + rpc_class_module: str = "SomeClass" + rpc_class_name: str = "ClassyName" + rpc_class_docstring: str = """This is Some Classy docstring.""" + constants: list = [] + methods: list = [] + signals: list = [] + + +class ProxyDescriptor(NamedTuple): + """For testing proxy descriptor.""" + address: MagicMock = MagicMock() + category: MagicMock = MagicMock() + interface: ProxyInterface = ProxyInterface() + + +class TestRpcProxy(unittest.TestCase): + """Test instantiating QMI_RpcProxy and QMI_RpcNonBlockingProxy classes.""" + + def test_create_instance(self): + """Test creating an instance of QMI_RpcProxy.""" + # Arrange + proxy_interface = ProxyInterface() + expected_class_fqn = ".".join([proxy_interface.rpc_class_module, proxy_interface.rpc_class_name]) + expected_docstring = proxy_interface.rpc_class_docstring + # Act + proxy = QMI_RpcProxy(QMI_Context("test_rpcproxy"), ProxyDescriptor()) + # Assert + self.assertIsInstance(proxy, QMI_RpcProxy) + self.assertEqual(expected_class_fqn, proxy._rpc_class_fqn) + self.assertEqual(expected_docstring, proxy.__doc__) + + def test_context_manager_excepts(self): + """Test creating an instance of QMI_RpcProxy with context manager excepts with RecursionError.""" + # Act + with self.assertRaises(RecursionError): + with QMI_RpcProxy(QMI_Context("test_rpcproxy"), ProxyDescriptor()): + pass + + def test_create_nonblocking_instance(self): + """Test creating an instance of QMI_RpcNonBlockingProxy.""" + # Arrange + proxy_interface = ProxyInterface() + expected_class_fqn = ".".join([proxy_interface.rpc_class_module, proxy_interface.rpc_class_name]) + expected_docstring = proxy_interface.rpc_class_docstring + # Act + proxy = QMI_RpcNonBlockingProxy(QMI_Context("test_rpcnonblockingproxy"), ProxyDescriptor()) + # Assert + self.assertIsInstance(proxy, QMI_RpcNonBlockingProxy) + self.assertEqual(expected_class_fqn, proxy._rpc_class_fqn) + self.assertEqual(expected_docstring, proxy.__doc__) + + def test_nonblocking_context_manager_excepts(self): + """Test creating an instance of QMI_RpcProxy with context manager excepts with TypeError.""" + # Act + with self.assertRaises((AttributeError, TypeError)): # TypeError as no __enter__ and __exit__ present + with QMI_RpcNonBlockingProxy(QMI_Context("test_rpcproxy"), ProxyDescriptor()): + pass + + class TestRPC(unittest.TestCase): def setUp(self): @@ -63,7 +127,6 @@ def setUp(self): logging.getLogger("qmi.core.messaging").setLevel(logging.ERROR) # Start two contexts. - config = CfgQmi( contexts={ "c1": CfgContext(tcp_server_port=0), @@ -97,11 +160,10 @@ def tearDown(self): logging.getLogger("qmi.core.messaging").setLevel(logging.NOTSET) def test_blocking_rpc(self): - + """Test for blocking RPC calls.""" # Instantiate the class, as a thing to be serviced from context c1. # This gives us a proxy to the instance. proxy1 = self.c1.make_rpc_object("tc1", MyRpcTestClass) - # Check nominal behavior. result = proxy1.remote_sqrt(256.0) self.assertEqual(result, 16.0) @@ -116,6 +178,9 @@ def test_blocking_rpc(self): # Check nominal behavior. result = proxy2.remote_sqrt(256.0) self.assertEqual(result, 16.0) + # Assert also that docstring gets passed to proxy. + self.assertEqual(MyRpcTestClass.__doc__, proxy1.__doc__) + self.assertEqual(proxy1.__doc__, proxy2.__doc__) # Check exception behavior. with self.assertRaises(ValueError): @@ -163,6 +228,9 @@ def test_nonblocking_rpc(self): assert isinstance(future, QMI_RpcFuture) result = future.wait() self.assertEqual(result, 16.0) + # Assert also that docstring gets passed to proxy. + self.assertEqual(MyRpcTestClass.__doc__, proxy1.__doc__) + self.assertEqual(proxy1.__doc__, proxy2.__doc__) # Check exception behavior. with self.assertRaises(ValueError): @@ -255,6 +323,9 @@ def test_subclass(self): # Make an RPC call to a method defined by the base class. result = proxy2.remote_sqrt(100.0) self.assertAlmostEqual(result, 10.0) + # Assert also that docstring gets passed to proxy. + self.assertEqual(MyRpcSubClass.__doc__, proxy1.__doc__) + self.assertEqual(proxy1.__doc__, proxy2.__doc__) def test_constants(self): @@ -397,6 +468,12 @@ def test_get_address_of_RpcObject(self): self.assertEqual(expected, address1) self.assertEqual(expected, address2) + def test_no_context_manager_allowed(self): + """Making an RPC object with context manager is not allowed.""" + with self.assertRaises(NotImplementedError): + with self.c1.make_rpc_object("tc1", MyRpcSubClass): + pass + class TestRpcMethodDecorator(unittest.TestCase): class ObjectWithGoodMethodName(QMI_RpcObject): diff --git a/tests/core/test_tasks.py b/tests/core/test_tasks.py index 1e2e6e71..68c9e6ac 100644 --- a/tests/core/test_tasks.py +++ b/tests/core/test_tasks.py @@ -12,7 +12,7 @@ import qmi import qmi.core.exceptions -from qmi.utils.context_managers import start_stop +from qmi.utils.context_managers import start_stop, start_stop_join from qmi.core.pubsub import QMI_Signal, QMI_SignalReceiver from qmi.core.rpc import QMI_RpcObject, rpc_method from qmi.core.task import ( @@ -251,8 +251,23 @@ def publish_signals(self): class TestQMITaskContextManager(unittest.TestCase): + """Test the various context managers.""" + def test_with_context_manager(self): + """Test the 'with' context manager run as QMI_TaskRunner.""" + logging.getLogger("qmi.core.task").setLevel(logging.ERROR) + logging.getLogger("qmi.core.rpc").setLevel(logging.ERROR) + qmi.start("test-taskrunner") + with qmi.make_task( + "taskrunner", SimpleTestTask, False, False, 1.0, 2.0 + ) as task: + task.get_status() + + qmi.stop() + logging.getLogger("qmi.core.task").setLevel(logging.NOTSET) + logging.getLogger("qmi.core.rpc").setLevel(logging.NOTSET) + def test_start_stop_context_manager(self): - # Test the context manager run as QMI_TaskRunner + """Test the 'start_stop' context manager run as QMI_TaskRunner.""" logging.getLogger("qmi.core.task").setLevel(logging.ERROR) logging.getLogger("qmi.core.rpc").setLevel(logging.ERROR) qmi.start("test-taskrunner") @@ -267,6 +282,21 @@ def test_start_stop_context_manager(self): logging.getLogger("qmi.core.task").setLevel(logging.NOTSET) logging.getLogger("qmi.core.rpc").setLevel(logging.NOTSET) + def test_start_stop_join_context_manager(self): + """Test the 'start_stop_join' context manager run as QMI_TaskRunner.""" + logging.getLogger("qmi.core.task").setLevel(logging.ERROR) + logging.getLogger("qmi.core.rpc").setLevel(logging.ERROR) + qmi.start("test-taskrunner2") + task: QMI_TaskRunner = qmi.make_task( + "taskrunner2", SimpleTestTask, False, False, 1.0, 2.0 + ) + with start_stop_join(task): + task.get_status() + + qmi.stop() + logging.getLogger("qmi.core.task").setLevel(logging.NOTSET) + logging.getLogger("qmi.core.rpc").setLevel(logging.NOTSET) + class TestQMITasks(unittest.TestCase): def setUp(self): @@ -281,7 +311,72 @@ def tearDown(self): logging.getLogger("qmi.core.task").setLevel(logging.NOTSET) logging.getLogger("qmi.core.rpc").setLevel(logging.NOTSET) + def test_context_manager(self): + """Test the 'with' context manager does the same as start_stop_join context manager.""" + with qmi.make_task( + "simple_task_init", + SimpleTestTask, + raise_exception_in_init=False, + raise_exception_in_run=False, + duration=1.0, + value=5, + ) as simple_task: + self.assertTrue(simple_task.is_running()) + + self.assertFalse(simple_task.is_running()) + + def test_exception_double_start(self): + """Test QMI_UsageException is raised if the task is started twice.""" + with self.assertRaises(qmi.core.exceptions.QMI_UsageException): + task_proxy = qmi.make_task( + "simple_task_init", + SimpleTestTask, + raise_exception_in_init=False, + raise_exception_in_run=False, + duration=1.0, + value=5, + ) + task_proxy.start() + task_proxy.start() + + # Remove task proxy and re-run within context manager. + qmi.context().remove_rpc_object(task_proxy) + + with self.assertRaises(qmi.core.exceptions.QMI_UsageException): + with qmi.make_task( + "simple_task_init", + SimpleTestTask, + raise_exception_in_init=False, + raise_exception_in_run=False, + duration=1.0, + value=5, + ) as task_proxy: + task_proxy.start() + + def test_exception_duplicate_task(self): + """Test QMI_UsageException is raised if the task is created twice.""" + task_proxy = qmi.make_task( + "simple_task_init", + SimpleTestTask, + raise_exception_in_init=False, + raise_exception_in_run=False, + duration=1.0, + value=5, + ) + + with self.assertRaises(qmi.core.exceptions.QMI_DuplicateNameException): + with qmi.make_task( + "simple_task_init", + SimpleTestTask, + raise_exception_in_init=False, + raise_exception_in_run=False, + duration=1.0, + value=5, + ) as task_proxy: + pass + def test_exception_during_init(self): + """Test QMI_TaskInitException is raised.""" with self.assertRaises(qmi.core.exceptions.QMI_TaskInitException): qmi.make_task( "simple_task_init", @@ -293,6 +388,7 @@ def test_exception_during_init(self): ) def test_exception_during_run(self): + """Test QMI_TaskRunException is raised.""" task_proxy = qmi.make_task( "simple_task_run", SimpleTestTask, @@ -306,7 +402,23 @@ def test_exception_during_run(self): with self.assertRaises(qmi.core.exceptions.QMI_TaskRunException): task_proxy.join() + def test_exception_during_context_run(self): + """Test QMI_TaskRunException is raised within context.""" + with self.assertRaises(qmi.core.exceptions.QMI_TaskRunException): + with qmi.make_task( + "simple_task_run", + SimpleTestTask, + raise_exception_in_init=False, + raise_exception_in_run=True, + duration=1.0, + value=6, + ) as task_proxy: + time.sleep(2.0) + + self.assertFalse(task_proxy.is_running()) + def test_run_to_completion(self): + """Test task stops running after completion of the task.""" task_proxy = qmi.make_task( "simple_task_complete", SimpleTestTask, @@ -328,6 +440,7 @@ def test_run_to_completion(self): self.assertEqual(status, 7) def test_run_to_stopped(self): + """Test task stops running if it is called to stop.""" task_proxy = qmi.make_task( "simple_task_stopped", SimpleTestTask, @@ -352,7 +465,53 @@ def test_run_to_stopped(self): status = task_proxy.get_status() self.assertIsNone(status) + def test_context_run_to_completion(self): + """Test task stops running after completion of the task within context manager.""" + with qmi.make_task( + "simple_task_complete", + SimpleTestTask, + raise_exception_in_init=False, + raise_exception_in_run=False, + duration=1.0, + value=7, + ) as task_proxy: + self.assertTrue(task_proxy.is_running()) + time.sleep(2.0) + self.assertFalse(task_proxy.is_running()) + t1 = time.monotonic() + + t2 = time.monotonic() + self.assertLess(t2 - t1, 0.1) + status = task_proxy.get_status() + self.assertEqual(status, 7) + + def test_context_run_to_stopped(self): + """Test task stops running within context manager if it is called to stop.""" + with qmi.make_task( + "simple_task_stopped", + SimpleTestTask, + raise_exception_in_init=False, + raise_exception_in_run=False, + duration=2.0, + value=8, + ) as task_proxy: + time.sleep(0.5) + self.assertTrue(task_proxy.is_running()) + t1 = time.monotonic() + task_proxy.stop() + t2 = time.monotonic() + self.assertLess(t2 - t1, 0.1) + time.sleep(0.1) + self.assertFalse(task_proxy.is_running()) + t1 = time.monotonic() + + t2 = time.monotonic() + self.assertLess(t2 - t1, 0.1) + status = task_proxy.get_status() + self.assertIsNone(status) + def test_update_settings(self): + """Test updating task settings.""" task_proxy = qmi.make_task( "simple_task_update", SimpleTestTask, @@ -379,7 +538,36 @@ def test_update_settings(self): self.assertEqual(status, 101) self.assertEqual(settings_received, new_settings) + def test_update_settings_with_context(self): + """Test updating task settings within a context manager.""" + recv = QMI_SignalReceiver() + with qmi.make_task( + "simple_task_update", + SimpleTestTask, + raise_exception_in_init=False, + raise_exception_in_run=False, + duration=2.0, + value=9, + ) as task_proxy: + task_proxy.sig_settings_updated.subscribe(recv) + time.sleep(0.5) + old_settings = task_proxy.get_settings() + new_settings = SimpleTestTask.Settings("hello", 101) + task_proxy.set_settings(new_settings) + time.sleep(0.5) + active_settings = task_proxy.get_settings() + task_proxy.join() + status = task_proxy.get_status() + settings_received = recv.get_next_signal().args[0] + + self.assertNotEqual(old_settings, active_settings) + self.assertEqual(active_settings, new_settings) + self.assertEqual(status, 101) + self.assertEqual(settings_received, new_settings) + def test_get_pending_settings(self): + """A more complex test to obtain pending settings for slow cycle tasks, where the new settings are not updated + frequently.""" event = threading.Event() task_proxy = qmi.make_task( "slow_task", SlowTestTask, duration=2.0, value=9, event=event @@ -407,7 +595,36 @@ def test_get_pending_settings(self): self.assertEqual(pending_settings, new_settings) self.assertIsNone(post_pending_settings) + def test_get_pending_settings_with_context(self): + """A more complex test to obtain pending settings for slow cycle tasks, where the new settings are not updated + frequently, within a context manager.""" + event = threading.Event() + recv = QMI_SignalReceiver() + with qmi.make_task( + "slow_task", SlowTestTask, duration=2.0, value=9, event=event + ) as task_proxy: + task_proxy.sig_settings_updated.subscribe(recv) + pre_pending_settings = task_proxy.get_pending_settings() + new_settings = SimpleTestTask.Settings("hello", 101) + # When setting new settings with QMI_TaskRunner.set_settings the new settings are put into a FiFo queue. + task_proxy.set_settings(new_settings) + # The settings won't be updated in the proxy before update_settings() has been called in a LoopTask + # instance. This triggers obtaining the next item in the FiFo queue. + # In the run() of SlowTestTask class, update_settings() called only after the event is set. + pending_settings = task_proxy.get_pending_settings() + event.set() + time.sleep(0.4) + self.assertIsNone(task_proxy.get_pending_settings()) + event.set() + time.sleep(0.4) + post_pending_settings = task_proxy.get_pending_settings() + + self.assertIsNone(pre_pending_settings) + self.assertEqual(pending_settings, new_settings) + self.assertIsNone(post_pending_settings) + def test_signals(self): + """Test the published signal in task gets added to the queue and received correctly.""" task_proxy = qmi.make_task( "simple_task_signals", SimpleTestTask, @@ -430,6 +647,7 @@ def test_signals(self): self.assertEqual(sig.args, (10,)) def test_stop_before_start(self): + """Test that just making a task doesn't start it, but also does not fail if `stop()` or `join()` is called.""" task_proxy = qmi.make_task( "simple_task_stop", SimpleTestTask, @@ -450,6 +668,7 @@ def test_stop_before_start(self): self.assertIsNone(status) def test_receive_signals(self): + """Test that signals are received one cycle (0.1s) after sending, and that the task stops at -1.""" publisher_proxy = qmi.context().make_rpc_object("test_publisher", TestPublisher) task_proxy = qmi.make_task( "receiving_task", @@ -474,7 +693,33 @@ def test_receive_signals(self): self.assertFalse(task_proxy.is_running()) task_proxy.join() + def test_receive_signals_in_context(self): + """Test that signals are received one cycle (0.1s) after sending within a context manager.""" + publisher_proxy = qmi.context().make_rpc_object("test_publisher", TestPublisher) + with qmi.make_task( + "receiving_task", + SignalWaitingTask, + wait_timeout=None, + context_id=self._ctx_qmi_id, + ) as task_proxy: + self.assertTrue(task_proxy.is_running()) + time.sleep(0.1) + status = task_proxy.get_status() + self.assertIsNone(status) + publisher_proxy.send_signal(8) + time.sleep(0.1) + status = task_proxy.get_status() + self.assertEqual(status, 8) + publisher_proxy.send_signal(9) + time.sleep(0.1) + status = task_proxy.get_status() + self.assertEqual(status, 9) + publisher_proxy.send_signal(-1) + time.sleep(0.1) + self.assertFalse(task_proxy.is_running()) + def test_timeout_while_waiting_for_signal(self): + """Test when a wait times out, it raises an exception.""" qmi.context().make_rpc_object("test_publisher", TestPublisher) task_proxy = qmi.make_task( "receiving_task", @@ -493,6 +738,24 @@ def test_timeout_while_waiting_for_signal(self): with self.assertRaises(qmi.core.exceptions.QMI_TaskRunException): task_proxy.join() + def test_timeout_while_waiting_for_signal_in_context(self): + """Test when a wait times out, it raises an exception within a context manager.""" + qmi.context().make_rpc_object("test_publisher", TestPublisher) + # task will have raised a QMI_TimeoutException while waiting for a signal + with self.assertRaises(qmi.core.exceptions.QMI_TaskRunException): + with qmi.make_task( + "receiving_task", + SignalWaitingTask, + wait_timeout=1.0, + context_id=self._ctx_qmi_id, + ) as task_proxy: + time.sleep(0.1) + self.assertTrue(task_proxy.is_running()) + time.sleep(1.4) + self.assertFalse(task_proxy.is_running()) + status = task_proxy.get_status() + self.assertEqual(status, -101) # indicates task got QMI_TimeoutException + def test_stop_while_waiting_for_signal(self): qmi.context().make_rpc_object("test_publisher", TestPublisher) task_proxy = qmi.make_task( @@ -514,7 +777,6 @@ def test_stop_while_waiting_for_signal(self): def test_run_to_loop_finish(self): """Test and assert that loop runs normally from start to finish if there are no glitches.""" # Arrange - status_signals_received = [] settings_signals_received = [] nr_of_loops = 3 @@ -561,7 +823,57 @@ def test_run_to_loop_finish(self): time.sleep(0.1 * loop_period) status_signals_received.append(task_proxy.get_status().value) - time.sleep(2 * loop_period) # Give time to finalize + task_proxy.join() # Give time to finalize + # Assert + self.assertListEqual(status_signals_expected, status_signals_received) + self.assertListEqual(settings_signals_expected, settings_signals_received) + self.assertFalse(task_proxy.is_running()) + + def test_run_to_loop_finish_in_context(self): + """Test and assert that loop runs normally within context manager if there are no glitches.""" + # Arrange + status_signals_received = [] + settings_signals_received = [] + nr_of_loops = 3 + increase_loop = False + initial_status_value = -1 + loop_period = 0.1 + policy = QMI_LoopTaskMissedLoopPolicy.IMMEDIATE + status_signals_expected = list(range(initial_status_value, nr_of_loops, 1)) + [1] + settings_signals_expected = list(range(1, 4)) + receiver = QMI_SignalReceiver() + + with qmi.make_task( + "loop_task_finish2", + LoopTestTask, + increase_loop=increase_loop, + nr_of_loops=nr_of_loops, + status_value=initial_status_value, + loop_period=loop_period, + policy=policy, + ) as task_proxy: + # Act + publisher_proxy = qmi.get_task(f"{self._ctx_qmi_id}.loop_task_finish2") + publisher_proxy.sig_settings_updated.subscribe(receiver) + # Test that prepare has done its job + status_signals_received.append(task_proxy.get_status().value) + # LoopTestTask does 3x 1 second loops --> should be finished after 3 seconds + for n in range(nr_of_loops): + # Test that the status changes at the end of each loop, after receiver signal increments + while task_proxy.get_status().value == status_signals_received[-1]: + time.sleep(0.1 * loop_period) + + settings_signals_received.append( + receiver.get_next_signal(timeout=2 * loop_period).args[-1] + ) + status_signals_received.append(task_proxy.get_status().value) + + # Test that finalize_loop sets status back to 1 + while task_proxy.get_status().value == status_signals_received[-1]: + time.sleep(0.1 * loop_period) + + status_signals_received.append(task_proxy.get_status().value) + # Assert self.assertListEqual(status_signals_expected, status_signals_received) self.assertListEqual(settings_signals_expected, settings_signals_received) From 8f27fcddcd45f811105f2172ed649b75e0c03293 Mon Sep 17 00:00:00 2001 From: heevasti Date: Thu, 22 Aug 2024 10:35:45 +0200 Subject: [PATCH 2/5] [QMI-090] Fix error in scheduled full-ci by installing dependencies for unit-tests. --- .github/workflows/scheduled-full-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/scheduled-full-ci.yml b/.github/workflows/scheduled-full-ci.yml index f9e561c0..d542c951 100644 --- a/.github/workflows/scheduled-full-ci.yml +++ b/.github/workflows/scheduled-full-ci.yml @@ -29,6 +29,7 @@ jobs: - name: Run unit tests and generate report if: always() run: | + pip install -e . pip install unittest-xml-reporting python -m xmlrunner --output-file testresults.xml discover --start-directory=tests --pattern="test_*.py" From b577fc3e1f61ac3de042d8a4c62d2bbec375269b Mon Sep 17 00:00:00 2001 From: heevasti Date: Fri, 23 Aug 2024 10:56:46 +0200 Subject: [PATCH 3/5] [QMI-090] Forgot to update the documentation as well. --- documentation/sphinx/source/design.rst | 72 ++++++++++++++++++++++++ documentation/sphinx/source/tutorial.rst | 42 ++++---------- 2 files changed, 84 insertions(+), 30 deletions(-) diff --git a/documentation/sphinx/source/design.rst b/documentation/sphinx/source/design.rst index 6d864391..9b66baaf 100644 --- a/documentation/sphinx/source/design.rst +++ b/documentation/sphinx/source/design.rst @@ -244,6 +244,78 @@ could lead into unexpected hardware responses and/or other kinds of issues, and Further, the proxies have the possibility of *locking* their objects to be controlled by a specific context only. The use of the ``lock()``, ``unlock()``, ``force_unlock()`` and ``is_locked()`` methods are illustrated in the Tutorial. +**Context management** +====================== + +QMI offers a few context managers to facilitate better control of the QMI contexts, instruments, tasks and signals. + +QMI contexts can be started and stopped with a `start_stop` context manager, available in ``qmi.utils.context_managers`` module. +The following code based on the ``with`` statement:: + + with start_stop(qmi, "name"): + custom_code_here ... + +has the same effect as:: + + qmi.start("name") + try: + custom_code_here ... + finally: + qmi.stop() + +both ensuring that ``qmi.stop()`` will be called even when an error occurs in the custom code. + +We can make instruments and tasks in the QMI context. For automatic opening and closing of an instrument driver instance +based on `QMI_Instrument`, we can do:: + + with qmi.make_instrument("instrument_name", InstrumentClass, ...) as instr: + custom_code_here... + +which has the same effect as:: + + instr = qmi.make_instrument("instrument_name", InstrumentClass) + instr.open() + try: + custom_code_here... + finally: + instr.close() + +Alternatively, the `open_close` context manager,from ``qmi.utils.context_managers`` can be used, but this context manager +will be obsoleted. That option requires making the instrument instance first and then giving it as an input to the context manager. + +For tasks we can use the context management protocol to automatically start the task thread when entering a task's `QMI_LoopTask` context, +and stopping and joining to it at exit. Similar to the instrument, we can do:: + + with qmi.make_task("task_name", TaskClass, ...) as task: + task_code_here... + +And the task should be stopped and joined after the task is finished. In the ``qmi.utils.context_managers`` is also context manager +`start_stop_join` to do this, but it will be obsoleted. + +Further context managers in ``qmi.utils.context_managers`` are `lock_unlock` and `subscribe_unsubscribe` context managers. +The `lock_unlock` manager is meant for RPC objects that the user wants to lock while they are used by some script or task. +Typical use:: + + some_instr = qmi.get_instrument(...) + with lock_unlock(some_instr): + priviledged_code_here... + +The `lock_unlock` context manager accepts also extra input arguments, so that `timeout` and `lock_token` arguments can +also be given for the context manager. + +And the final `subscribe_unsubscribe` context manager is meant to be used with signals. For example, a task has signal +named `sig_send_data` in the task's class. And we want to subscribe to it to receive data updates. If a task is f.ex. +obtained from another context, and we want to receive in `data_receiver`:: + + signal_task = qmi.get_task(...) + data_receiver = QMI_SignalReceiver() + with subscribe_unsubscribe(signal_task.sig_send_data, data_receiver): + data = data_receiver.get_next_signal() + +If the task is 'running' and publishing data, the receiver should receive the data from it and then unsubscribe from +the signal again. Forgetting to unsubscribe from the signal could possibly lead to memory issues if the receiver is +still present, because then the published data could keep accumulating into the receiver queue. + **Messaging** ============= diff --git a/documentation/sphinx/source/tutorial.rst b/documentation/sphinx/source/tutorial.rst index 4927897c..bd1b70be 100644 --- a/documentation/sphinx/source/tutorial.rst +++ b/documentation/sphinx/source/tutorial.rst @@ -183,7 +183,7 @@ However, if you find yourself in a situation in which the locking proxy was lost >>> nsg2.is_locked() False -It is also possible to unlock from another context proxy by providing the context name as well. See the example below how to connect from another context to an instrument. +It is also possible to unlock from another instrument proxy by providing the context name as well. See the example below, how to get from another context the instrument proxy. >>> import qmi >>> qmi.start("client") @@ -355,7 +355,7 @@ To set up a simple measurement script, create a file ``measure_demo.py`` with th #!/usr/bin/env python3 import qmi - from qmi.utils.context_managers import start_stop, open_close + from qmi.utils.context_managers import start_stop from qmi.instruments.dummy.noisy_sine_generator import NoisySineGenerator def measure_data(nsg): @@ -368,8 +368,7 @@ To set up a simple measurement script, create a file ``measure_demo.py`` with th def main(): with start_stop(qmi, "measure_demo"): - nsg = qmi.make_instrument("nsg", NoisySineGenerator) - with open_close(nsg): + with qmi.make_instrument("nsg", NoisySineGenerator) as nsg: measure_data(nsg) if __name__ == "__main__": @@ -383,21 +382,8 @@ Note that the script uses :py:class:`qmi.utils.context_managers.start_stop` to start and stop the QMI framework. This is just a convenient way to make sure that ``qmi.start()`` and ``qmi.stop()`` will always be called. -The following code based on the ``with`` statement:: - - with start_stop(qmi, "name"): - custom_code_here ... - -has the same effect as:: - - qmi.start("name") - custom_code_here ... - qmi.stop() - -with the difference that the ``with`` mechanism ensures that ``qmi.stop()`` -will be called even when an error occurs in the custom code. -Similarly, the script uses :py:class:`qmi.utils.context_managers.open_close` -to open and close to represent the calls to ``nsg.open()`` and ``nsg.close()``. +Similarly, the `QMI_Instrument` objects are equipped with context managers that open and close the +the instrument, calling ``nsg.open()`` and ``nsg.close()`` at the creation and destruction of the instance. .. note:: Some users prefer to invoke scripts from an interactive Python session, @@ -464,14 +450,13 @@ the task and continues to perform other activities:: import time import qmi - from qmi.utils.context_managers import start_stop, open_close + from qmi.utils.context_managers import start_stop from qmi.instruments.dummy.noisy_sine_generator import NoisySineGenerator from demo_task import DemoTask def main(): with start_stop(qmi, "task_demo"): - nsg = qmi.make_instrument("nsg", NoisySineGenerator) - with open_close(nsg): + with qmi.make_instrument("nsg", NoisySineGenerator) as nsg: task = qmi.make_task("task", DemoTask) task.start() print("the task has been started") @@ -585,11 +570,10 @@ Try making the following class:: raise qmi.core.exceptions.QMI_TaskRunException("No such attribute in task: 'amplitude_factor'") And give it as the ``task_runner`` input when making the task, it is possible to change the value from outside the task. -We now also switch to use the ``start_stop_join`` context manager for tasks:: +We now also switch to use the internal context manager for tasks:: ... - task = qmi.make_task("task", DemoTask, task_runner=CustomTaskRunner) - with start_stop_join(task): + with qmi.make_task("task", DemoTask, task_runner=CustomTaskRunner) as task: print("the task has been started") time.sleep(1) task.set_amplitude_factor(1.0) @@ -631,7 +615,7 @@ of the task in the script instead of inside the task. We now rewrite the script import time import qmi - from qmi.utils.context_managers import start_stop, open_close + from qmi.utils.context_managers import start_stop from qmi.instruments.dummy.noisy_sine_generator import NoisySineGenerator from demo_task import DemoTask @@ -644,10 +628,8 @@ of the task in the script instead of inside the task. We now rewrite the script def main_2(): with start_stop(qmi, "task_demo"): - nsg = qmi.make_instrument("nsg", NoisySineGenerator) - with open_close(nsg): - task = qmi.make_task("task", DemoRpcControlTask, task_runner=CustomRpcControlTaskRunner) - with start_stop_join(task): + with qmi.make_instrument("nsg", NoisySineGenerator) as nsg: + with qmi.make_task("task", DemoRpcControlTask, task_runner=CustomRpcControlTaskRunner) as task: print("the task has been started") for i in range(5): sample = nsg.get_sample() From 5e58cd856d76731e4f88deae3671481bf79db760 Mon Sep 17 00:00:00 2001 From: heevasti Date: Fri, 23 Aug 2024 11:26:38 +0200 Subject: [PATCH 4/5] [QMI-090] Not installing the qmi package in editable mode for the pipelines. --- .github/workflows/reusable-ci-workflows.yml | 2 +- .github/workflows/scheduled-full-ci.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable-ci-workflows.yml b/.github/workflows/reusable-ci-workflows.yml index f025a93d..2a637ed7 100644 --- a/.github/workflows/reusable-ci-workflows.yml +++ b/.github/workflows/reusable-ci-workflows.yml @@ -37,7 +37,7 @@ jobs: sudo apt-get update -qy sudo apt-get install -y bc pip install --upgrade pip - pip install -e '.[dev]' + pip install '.[dev]' - name: Run pylint run: | diff --git a/.github/workflows/scheduled-full-ci.yml b/.github/workflows/scheduled-full-ci.yml index d542c951..ef9333e3 100644 --- a/.github/workflows/scheduled-full-ci.yml +++ b/.github/workflows/scheduled-full-ci.yml @@ -29,7 +29,7 @@ jobs: - name: Run unit tests and generate report if: always() run: | - pip install -e . + pip install . pip install unittest-xml-reporting python -m xmlrunner --output-file testresults.xml discover --start-directory=tests --pattern="test_*.py" From c921bac43743fdfbb9149b02de66622058b2b93c Mon Sep 17 00:00:00 2001 From: heevasti Date: Fri, 23 Aug 2024 11:51:35 +0200 Subject: [PATCH 5/5] [QMI-090] Adding link to python docs. --- qmi/core/rpc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qmi/core/rpc.py b/qmi/core/rpc.py index 9200cd80..ac8daf8f 100644 --- a/qmi/core/rpc.py +++ b/qmi/core/rpc.py @@ -684,7 +684,8 @@ def __enter__(self) -> "QMI_RpcProxy": The with keyword checks if the `QMI_RpcProxy` class (not instance) implements the context manager protocol, i.e. `__enter__` and `__exit__`, so they exist here as stubs. These stubs make an RPC to the actual `__enter__` and `__exit__` methods on the relevant RPC object. If in those classes `__enter__` and `__exit__` are - decorated as `rpc_methods`, we do not see a recursion error. + decorated as `rpc_methods`, we do not see a recursion error. See for further details in: + https://docs.python.org/3/reference/datamodel.html#special-method-lookup `rpc_method` decorated `__enter__` and `__exit__` methods are currently implemented in `QMI_Instrument` and `QMI_TaskRunner` classes.