From d4eb5cda08a2a8531a1e1f5f2ef2e2ee5d0fcf9a Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Tue, 13 Feb 2024 15:33:54 -0500 Subject: [PATCH 01/56] Add dependencies and update pre-commit config --- .pre-commit-config.yaml | 6 +++--- pyproject.toml | 11 +++++++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ac556dd..5f75507 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,14 +33,14 @@ repos: - id: rst-inline-touching-normal - repo: https://github.com/pre-commit/mirrors-prettier - rev: "v3.1.0" + rev: "v4.0.0-alpha.8" hooks: - id: prettier types_or: [yaml, markdown, html, css, scss, javascript, json] args: [--prose-wrap=always] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.1.14" + rev: "v0.2.1" hooks: - id: ruff args: ["--fix", "--show-fixes"] @@ -80,7 +80,7 @@ repos: additional_dependencies: ["validate-pyproject-schema-store[all]"] - repo: https://github.com/python-jsonschema/check-jsonschema - rev: "0.27.3" + rev: "0.28.0" hooks: - id: check-dependabot - id: check-github-workflows diff --git a/pyproject.toml b/pyproject.toml index e02b752..a2d73db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,11 @@ classifiers = [ "Typing :: Typed", ] dynamic = ["version"] -dependencies = [] +dependencies = [ + "caproto", + "h5py", + "numpy", +] [project.optional-dependencies] test = [ @@ -38,6 +42,9 @@ test = [ "pytest-cov >=3", ] dev = [ + "ipython", + "pre-commit", + "pylint", "pytest >=6", "pytest-cov >=3", ] @@ -121,7 +128,7 @@ extend-select = [ "RET", # flake8-return "RUF", # Ruff-specific "SIM", # flake8-simplify - "T20", # flake8-print + # "T20", # flake8-print "UP", # pyupgrade "YTT", # flake8-2020 "EXE", # flake8-executable From 517213ceaa9002ad2670e2dce548039e5c552d28 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Tue, 13 Feb 2024 15:42:53 -0500 Subject: [PATCH 02/56] Create base classes and utility functions --- .pre-commit-config.yaml | 16 +-- pyproject.toml | 2 +- src/srx_caproto_iocs/base.py | 152 ++++++++++++++++++++ src/srx_caproto_iocs/sis_scaler/__init__.py | 0 src/srx_caproto_iocs/utils.py | 11 ++ src/srx_caproto_iocs/zebra/__init__.py | 0 src/srx_caproto_iocs/zebra/caproto_ioc.py | 27 ++++ 7 files changed, 199 insertions(+), 9 deletions(-) create mode 100644 src/srx_caproto_iocs/base.py create mode 100644 src/srx_caproto_iocs/sis_scaler/__init__.py create mode 100644 src/srx_caproto_iocs/utils.py create mode 100644 src/srx_caproto_iocs/zebra/__init__.py create mode 100644 src/srx_caproto_iocs/zebra/caproto_ioc.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5f75507..199f43e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,14 +46,14 @@ repos: args: ["--fix", "--show-fixes"] - id: ruff-format - - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.8.0" - hooks: - - id: mypy - files: src|tests - args: [] - additional_dependencies: - - pytest + # - repo: https://github.com/pre-commit/mirrors-mypy + # rev: "v1.8.0" + # hooks: + # - id: mypy + # files: src|tests + # args: [] + # additional_dependencies: + # - pytest - repo: https://github.com/codespell-project/codespell rev: "v2.2.6" diff --git a/pyproject.toml b/pyproject.toml index a2d73db..c9ada46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -115,7 +115,7 @@ src = ["src"] extend-select = [ "B", # flake8-bugbear "I", # isort - "ARG", # flake8-unused-arguments + # "ARG", # flake8-unused-arguments "C4", # flake8-comprehensions "EM", # flake8-errmsg "ICN", # flake8-import-conventions diff --git a/src/srx_caproto_iocs/base.py b/src/srx_caproto_iocs/base.py new file mode 100644 index 0000000..f837f19 --- /dev/null +++ b/src/srx_caproto_iocs/base.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +import threading +import uuid +from enum import Enum +from pathlib import Path + +from caproto import ChannelType +from caproto.server import PVGroup, pvproperty + +from .utils import now + + +class AcqStatuses(Enum): + """Enum class for acquisition statuses.""" + + IDLE = "Done" + ACQUIRING = "Count" + + +class StageStates(Enum): + """Enum class for stage states.""" + + UNSTAGED = "unstaged" + STAGED = "staged" + + +class GenericSaveIOC(PVGroup): + """Generic Caproto Save IOC""" + + write_dir = pvproperty( + value="/tmp", + doc="The directory to write data to. It support datetime formatting, e.g. '/tmp/det/%Y/%m/%d/'", + string_encoding="utf-8", + report_as_string=True, + max_length=255, + ) + file_name = pvproperty( + value="test.h5", + doc="The file name of the file to write to. It support .format() based formatting, e.g. 'scan_{num:06d}.h5'", + string_encoding="utf-8", + report_as_string=True, + max_length=255, + ) + full_file_path = pvproperty( + value="", + doc="Full path to the data file", + dtype=str, + read_only=True, + max_length=255, + ) + + # TODO: check non-negative value in @frame_num.putter. + frame_num = pvproperty(value=0, doc="Frame counter", dtype=int) + + stage = pvproperty( + value=StageStates.UNSTAGED.value, + enum_strings=[x.value for x in StageStates], + dtype=ChannelType.ENUM, + doc="Stage/unstage the detector", + ) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._request_queue = None + self._response_queue = None + + _queue = pvproperty(value=0, doc="A PV to facilitate threading-based queue") + + @_queue.startup + async def _queue(self, instance, async_lib): + """The startup behavior of the count property to set up threading queues.""" + # pylint: disable=unused-argument + self._request_queue = async_lib.ThreadsafeQueue() + self._response_queue = async_lib.ThreadsafeQueue() + + # Start a separate thread that consumes requests and sends responses. + thread = threading.Thread( + target=self.saver, + daemon=True, + kwargs={ + "request_queue": self._request_queue, + "response_queue": self._response_queue, + }, + ) + thread.start() + + @stage.putter + async def stage(self, instance, value): + """The stage method to perform preparation of a dataset to save the data.""" + if ( + instance.value in [True, StageStates.STAGED.value] + and value == StageStates.STAGED.value + ): + msg = "The device is already staged. Unstage it first." + print(msg) + return False + + if value == StageStates.STAGED.value: + # Steps: + # 1. Render 'write_dir' with datetime lib and replace any blank spaces with underscores. + # 2. Render 'file_name' with .format(). + # 3. Replace blank spaces with underscores. + + date = now(as_object=True) + write_dir = Path(date.strftime(self.write_dir.value).replace(" ", "_")) + if not write_dir.exists(): + msg = f"Path '{write_dir}' does not exist." + print(msg) + return False + + file_name = self.file_name.value + uid = "" if "{uid" not in file_name else str(uuid.uuid4()) + full_file_path = write_dir / file_name.format( + num=self.frame_num.value, uid=uid + ) + full_file_path = str(full_file_path) + full_file_path.replace(" ", "_") + + print(f"{now()}: {full_file_path = }") + + await self.full_file_path.write(full_file_path) + + return True + + return False + + @staticmethod + def saver(request_queue, response_queue): + """The saver callback for threading-based queueing.""" + while True: + received = request_queue.get() + filename = received["filename"] + data = received["data"] + try: + # save_hdf5(fname=filename, data=data) + print(f"{now()}: saved {data.shape} data into:\n {filename}") + + success = True + error_message = "" + except Exception as exc: # pylint: disable=broad-exception-caught + # The GeRM detector happens to response twice for a single + # ".CNT" put, so capture an attempt to save the file with the + # same name here and do nothing. + success = False + error_message = exc + print( + f"Cannot save file {filename!r} due to the following exception:\n{exc}" + ) + + response = {"success": success, "error_message": error_message} + response_queue.put(response) diff --git a/src/srx_caproto_iocs/sis_scaler/__init__.py b/src/srx_caproto_iocs/sis_scaler/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/srx_caproto_iocs/utils.py b/src/srx_caproto_iocs/utils.py new file mode 100644 index 0000000..b34f530 --- /dev/null +++ b/src/srx_caproto_iocs/utils.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +import datetime + + +def now(as_object=False): + """A helper function to return ISO 8601 formatted datetime string.""" + _now = datetime.datetime.now() + if as_object: + return _now + return _now.isoformat() diff --git a/src/srx_caproto_iocs/zebra/__init__.py b/src/srx_caproto_iocs/zebra/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/srx_caproto_iocs/zebra/caproto_ioc.py b/src/srx_caproto_iocs/zebra/caproto_ioc.py new file mode 100644 index 0000000..91ed729 --- /dev/null +++ b/src/srx_caproto_iocs/zebra/caproto_ioc.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +import textwrap + +from caproto.server import run, template_arg_parser + +from ..base import GenericSaveIOC + + +class ZebraSaveIOC(GenericSaveIOC): + """Zebra Caproto Save IOC""" + + +if __name__ == "__main__": + parser, split_args = template_arg_parser( + default_prefix="", desc=textwrap.dedent(ZebraSaveIOC.__doc__) + ) + + parsed_args = parser.parse_args() + prefix = parsed_args.prefix + if not prefix: + parser.error("The 'prefix' argument must be specified.") + + ioc_options, run_options = split_args(parsed_args) + + ioc = ZebraSaveIOC(**ioc_options) # TODO: pass libca IOC PVs of interest + run(ioc.pvdb, **run_options) From bd46a1a284f17e7588e48e8e2baaf488141d070b Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Tue, 13 Feb 2024 15:46:14 -0500 Subject: [PATCH 03/56] CI: run ci workflow on all pushes --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index efcc345..aeb7d0d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,8 +4,6 @@ on: workflow_dispatch: pull_request: push: - branches: - - main concurrency: group: ${{ github.workflow }}-${{ github.ref }} From f393c63b13628d5e12eac0bda191715b44c62b67 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Tue, 13 Feb 2024 15:58:19 -0500 Subject: [PATCH 04/56] CI: Exclude pypy tests --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aeb7d0d..2d2ab4a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,9 +41,9 @@ jobs: python-version: ["3.8", "3.12"] runs-on: [ubuntu-latest, macos-latest, windows-latest] - include: - - python-version: pypy-3.10 - runs-on: ubuntu-latest + # include: + # - python-version: pypy-3.10 + # runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 From cc80d49956e07af50b67daa32f1a190144da028a Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Tue, 13 Feb 2024 16:44:45 -0500 Subject: [PATCH 05/56] TST: add ophyd-based test for Zebra Caproto IOC --- pyproject.toml | 1 + src/srx_caproto_iocs/zebra/ophyd.py | 12 ++++++++++ tests/conftest.py | 10 ++++++++ tests/test_zebra_ophyd.py | 36 +++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+) create mode 100644 src/srx_caproto_iocs/zebra/ophyd.py create mode 100644 tests/conftest.py create mode 100644 tests/test_zebra_ophyd.py diff --git a/pyproject.toml b/pyproject.toml index c9ada46..7c62cf6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ dependencies = [ "caproto", "h5py", "numpy", + "ophyd", ] [project.optional-dependencies] diff --git a/src/srx_caproto_iocs/zebra/ophyd.py b/src/srx_caproto_iocs/zebra/ophyd.py new file mode 100644 index 0000000..cd58a5e --- /dev/null +++ b/src/srx_caproto_iocs/zebra/ophyd.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +from ophyd import Component as Cpt +from ophyd import Device, EpicsSignal, EpicsSignalRO + + +class ZebraWithCaprotoIOC(Device): + write_dir = Cpt(EpicsSignal, "write_dir") + file_name = Cpt(EpicsSignal, "file_name") + full_file_path = Cpt(EpicsSignalRO, "full_file_path") + frame_num = Cpt(EpicsSignal, "frame_num") + ioc_stage = Cpt(EpicsSignal, "ioc_stage") diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..2a993a1 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,10 @@ +from __future__ import annotations + +import pytest + +from srx_caproto_iocs.zebra.ophyd import ZebraWithCaprotoIOC + + +@pytest.fixture() +def zebra_ophyd_caproto(): + return ZebraWithCaprotoIOC("XF:05IDD-ES:1{Dev:Zebra2}:") diff --git a/tests/test_zebra_ophyd.py b/tests/test_zebra_ophyd.py new file mode 100644 index 0000000..4e81397 --- /dev/null +++ b/tests/test_zebra_ophyd.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from ophyd.status import SubscriptionStatus + +from srx_caproto_iocs.utils import now + + +# TODO: use pytest.mark.parametrize to implement many use cases. +def test_zebra_ophyd_caproto(tmp_path, zebra_ophyd_caproto): + date = now(as_object=True) + write_dir_root = tmp_path / "zebra/" + dir_template = f"{write_dir_root}/%Y/%m/" + write_dir = date.strftime(dir_template) + write_dir.mkdir() + + print(f"{write_dir = }\n{type(write_dir) = }") + + file_template = "scan_{num:06d}_{uid}.hdf5" + + dev = zebra_ophyd_caproto + dev.write_dir.put(dir_template) + dev.file_name.put(file_template) + + def cb(value, old_value, **kwargs): + if value == "staged" and old_value == "unstaged": + return True + return False + + st = SubscriptionStatus(dev.ioc_stage, callback=cb, run=False) + dev.ioc_stage.set("stage").wait() + st.wait() + + full_file_path = dev.full_file_path.get() + print(f"{full_file_path = }") + + assert full_file_path From e320f286b6b87e0039dc77b1249169e61d94368b Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Tue, 13 Feb 2024 18:00:08 -0500 Subject: [PATCH 06/56] Working ophyd tests --- pyproject.toml | 1 + src/srx_caproto_iocs/zebra/caproto_ioc.py | 2 +- src/srx_caproto_iocs/zebra/ophyd.py | 10 ++-- tests/conftest.py | 4 +- tests/test_zebra_ophyd.py | 60 ++++++++++++----------- 5 files changed, 43 insertions(+), 34 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7c62cf6..6905d19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ dependencies = [ "h5py", "numpy", "ophyd", + "pyepics", # does not work with 'setuptools' version higher than v66.1.1 ] [project.optional-dependencies] diff --git a/src/srx_caproto_iocs/zebra/caproto_ioc.py b/src/srx_caproto_iocs/zebra/caproto_ioc.py index 91ed729..d4cb4a3 100644 --- a/src/srx_caproto_iocs/zebra/caproto_ioc.py +++ b/src/srx_caproto_iocs/zebra/caproto_ioc.py @@ -8,7 +8,7 @@ class ZebraSaveIOC(GenericSaveIOC): - """Zebra Caproto Save IOC""" + """Zebra caproto save IOC.""" if __name__ == "__main__": diff --git a/src/srx_caproto_iocs/zebra/ophyd.py b/src/srx_caproto_iocs/zebra/ophyd.py index cd58a5e..67d74ac 100644 --- a/src/srx_caproto_iocs/zebra/ophyd.py +++ b/src/srx_caproto_iocs/zebra/ophyd.py @@ -5,8 +5,10 @@ class ZebraWithCaprotoIOC(Device): - write_dir = Cpt(EpicsSignal, "write_dir") - file_name = Cpt(EpicsSignal, "file_name") - full_file_path = Cpt(EpicsSignalRO, "full_file_path") + """An ophyd Device which works with the Zebra caproto extension IOC.""" + + write_dir = Cpt(EpicsSignal, "write_dir", string=True) + file_name = Cpt(EpicsSignal, "file_name", string=True) + full_file_path = Cpt(EpicsSignalRO, "full_file_path", string=True) frame_num = Cpt(EpicsSignal, "frame_num") - ioc_stage = Cpt(EpicsSignal, "ioc_stage") + ioc_stage = Cpt(EpicsSignal, "stage", string=True) diff --git a/tests/conftest.py b/tests/conftest.py index 2a993a1..bbf2d7b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,4 +7,6 @@ @pytest.fixture() def zebra_ophyd_caproto(): - return ZebraWithCaprotoIOC("XF:05IDD-ES:1{Dev:Zebra2}:") + dev = ZebraWithCaprotoIOC("XF:05IDD-ES:1{Dev:Zebra2}:", name="zebra_ophyd_caproto") + yield dev + dev.ioc_stage.put("unstaged") diff --git a/tests/test_zebra_ophyd.py b/tests/test_zebra_ophyd.py index 4e81397..db6e0fe 100644 --- a/tests/test_zebra_ophyd.py +++ b/tests/test_zebra_ophyd.py @@ -1,36 +1,40 @@ from __future__ import annotations +import tempfile +from pathlib import Path + +import pytest from ophyd.status import SubscriptionStatus from srx_caproto_iocs.utils import now # TODO: use pytest.mark.parametrize to implement many use cases. -def test_zebra_ophyd_caproto(tmp_path, zebra_ophyd_caproto): - date = now(as_object=True) - write_dir_root = tmp_path / "zebra/" - dir_template = f"{write_dir_root}/%Y/%m/" - write_dir = date.strftime(dir_template) - write_dir.mkdir() - - print(f"{write_dir = }\n{type(write_dir) = }") - - file_template = "scan_{num:06d}_{uid}.hdf5" - - dev = zebra_ophyd_caproto - dev.write_dir.put(dir_template) - dev.file_name.put(file_template) - - def cb(value, old_value, **kwargs): - if value == "staged" and old_value == "unstaged": - return True - return False - - st = SubscriptionStatus(dev.ioc_stage, callback=cb, run=False) - dev.ioc_stage.set("stage").wait() - st.wait() - - full_file_path = dev.full_file_path.get() - print(f"{full_file_path = }") - - assert full_file_path +@pytest.mark.parametrize("date_template", ["%Y/%m/", "%Y/%m/%d", "mydir/%Y/%m/%d"]) +def test_zebra_ophyd_caproto(zebra_ophyd_caproto, date_template): + with tempfile.TemporaryDirectory() as tmpdirname: + date = now(as_object=True) + write_dir_root = Path(tmpdirname) + dir_template = f"{write_dir_root}/{date_template}" + write_dir = Path(date.strftime(dir_template)) + write_dir.mkdir(parents=True, exist_ok=True) + + print(f"{write_dir = }") + + file_template = "scan_{num:06d}_{uid}.hdf5" + + dev = zebra_ophyd_caproto + dev.write_dir.put(dir_template) + dev.file_name.put(file_template) + + def cb(value, old_value, **kwargs): + if value == "staged" and old_value == "unstaged": + return True + return False + + st = SubscriptionStatus(dev.ioc_stage, callback=cb, run=False) + dev.ioc_stage.put("staged") + st.wait() + + full_file_path = dev.full_file_path.get() + print(f"{full_file_path = }") From a61223fc503da3aeb00437b699f857a0d08f5a40 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Tue, 13 Feb 2024 18:15:39 -0500 Subject: [PATCH 07/56] More fixes for the tests --- .github/workflows/ci.yml | 2 +- pyproject.toml | 5 ++++- tests/conftest.py | 23 +++++++++++++++++++++++ tests/test_zebra_ophyd.py | 3 ++- 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d2ab4a..02225c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,7 +61,7 @@ jobs: - name: Test package run: >- python -m pytest -ra --cov --cov-report=xml --cov-report=term - --durations=20 + --durations=20 -m "(not hardware) and (not tiled)" - name: Upload coverage report uses: codecov/codecov-action@v4.0.1 diff --git a/pyproject.toml b/pyproject.toml index 6905d19..6e2ec7d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,7 +85,10 @@ log_cli_level = "INFO" testpaths = [ "tests", ] - +markers = [ + "hardware: marks tests as requiring the hardware IOC to be available/running (deselect with '-m \"not hardware\"')", + "tiled: marks tests as requiring tiled", +] [tool.coverage] run.source = ["srx_caproto_iocs"] diff --git a/tests/conftest.py b/tests/conftest.py index bbf2d7b..ad1f2e5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,8 @@ from __future__ import annotations +import subprocess +import time as ttime + import pytest from srx_caproto_iocs.zebra.ophyd import ZebraWithCaprotoIOC @@ -10,3 +13,23 @@ def zebra_ophyd_caproto(): dev = ZebraWithCaprotoIOC("XF:05IDD-ES:1{Dev:Zebra2}:", name="zebra_ophyd_caproto") yield dev dev.ioc_stage.put("unstaged") + + +@pytest.fixture(scope="session") +def _caproto_ioc(): + command = 'EPICS_CAS_BEACON_ADDR_LIST=127.0.0.1 EPICS_CAS_AUTO_BEACON_ADDR_LIST=no python -m srx_caproto_iocs.zebra.caproto_ioc --prefix="XF:05IDD-ES:1{{Dev:Zebra2}}:" --list-pvs' + p = subprocess.Popen( + command.split(), + start_new_session=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + shell=True, + ) + ttime.sleep(2.0) + + yield + + std_out, std_err = p.communicate() + std_out = std_out.decode() + print(std_out) + p.terminate() diff --git a/tests/test_zebra_ophyd.py b/tests/test_zebra_ophyd.py index db6e0fe..10d6527 100644 --- a/tests/test_zebra_ophyd.py +++ b/tests/test_zebra_ophyd.py @@ -9,7 +9,8 @@ from srx_caproto_iocs.utils import now -# TODO: use pytest.mark.parametrize to implement many use cases. +@pytest.mark.hardware() +@pytest.mark.usefixtures("_caproto_ioc") @pytest.mark.parametrize("date_template", ["%Y/%m/", "%Y/%m/%d", "mydir/%Y/%m/%d"]) def test_zebra_ophyd_caproto(zebra_ophyd_caproto, date_template): with tempfile.TemporaryDirectory() as tmpdirname: From ca99a7efca6648786d2cb640bef5c8d96f33d7b4 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Wed, 14 Feb 2024 12:06:21 -0500 Subject: [PATCH 08/56] TST: add a caproto IOC fixture to be able to start and test against it in CI --- pyproject.toml | 1 + tests/conftest.py | 26 ++++++++++++++++++++------ tests/test_zebra_ophyd.py | 9 ++++----- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6e2ec7d..7cb474a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,6 +88,7 @@ testpaths = [ markers = [ "hardware: marks tests as requiring the hardware IOC to be available/running (deselect with '-m \"not hardware\"')", "tiled: marks tests as requiring tiled", + "cloud_friendly: marks tests to be able to execute in the CI in the cloud", ] [tool.coverage] diff --git a/tests/conftest.py b/tests/conftest.py index ad1f2e5..9fd0122 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ from __future__ import annotations import subprocess +import sys import time as ttime import pytest @@ -16,20 +17,33 @@ def zebra_ophyd_caproto(): @pytest.fixture(scope="session") -def _caproto_ioc(): - command = 'EPICS_CAS_BEACON_ADDR_LIST=127.0.0.1 EPICS_CAS_AUTO_BEACON_ADDR_LIST=no python -m srx_caproto_iocs.zebra.caproto_ioc --prefix="XF:05IDD-ES:1{{Dev:Zebra2}}:" --list-pvs' +def caproto_ioc(wait=3): + env = { + "EPICS_CAS_BEACON_ADDR_LIST": "127.0.0.1", + "EPICS_CAS_AUTO_BEACON_ADDR_LIST": "no", + } + command = ( + sys.executable + + " -m srx_caproto_iocs.zebra.caproto_ioc --prefix=XF:05IDD-ES:1{{Dev:Zebra2}}: --list-pvs" + ) + print( + f"Starting caproto IOC in via a fixture using the following command:\n\n {command}\n" + ) p = subprocess.Popen( command.split(), start_new_session=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - shell=True, + shell=False, + env=env, ) - ttime.sleep(2.0) + print(f"Wait for {wait} seconds...") + ttime.sleep(wait) - yield + yield p + + p.terminate() std_out, std_err = p.communicate() std_out = std_out.decode() print(std_out) - p.terminate() diff --git a/tests/test_zebra_ophyd.py b/tests/test_zebra_ophyd.py index 10d6527..7b4f81f 100644 --- a/tests/test_zebra_ophyd.py +++ b/tests/test_zebra_ophyd.py @@ -9,10 +9,9 @@ from srx_caproto_iocs.utils import now -@pytest.mark.hardware() -@pytest.mark.usefixtures("_caproto_ioc") +@pytest.mark.cloud_friendly() @pytest.mark.parametrize("date_template", ["%Y/%m/", "%Y/%m/%d", "mydir/%Y/%m/%d"]) -def test_zebra_ophyd_caproto(zebra_ophyd_caproto, date_template): +def test_zebra_ophyd_caproto(caproto_ioc, zebra_ophyd_caproto, date_template): with tempfile.TemporaryDirectory() as tmpdirname: date = now(as_object=True) write_dir_root = Path(tmpdirname) @@ -20,8 +19,6 @@ def test_zebra_ophyd_caproto(zebra_ophyd_caproto, date_template): write_dir = Path(date.strftime(dir_template)) write_dir.mkdir(parents=True, exist_ok=True) - print(f"{write_dir = }") - file_template = "scan_{num:06d}_{uid}.hdf5" dev = zebra_ophyd_caproto @@ -39,3 +36,5 @@ def cb(value, old_value, **kwargs): full_file_path = dev.full_file_path.get() print(f"{full_file_path = }") + + assert full_file_path, "The returned 'full_file_path' did not change." From d2d25e2184cc8f7eb27954dddfe31c469b522ef2 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Wed, 14 Feb 2024 12:16:41 -0500 Subject: [PATCH 09/56] CI: disable Win/OSX builds; more python versions to test with; pin setuptools to an older version; more verbose test output --- .github/workflows/ci.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02225c6..45f3bcf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,8 +38,9 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.12"] - runs-on: [ubuntu-latest, macos-latest, windows-latest] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + # runs-on: [ubuntu-latest, macos-latest, windows-latest] + runs-on: [ubuntu-latest] # include: # - python-version: pypy-3.10 @@ -56,12 +57,15 @@ jobs: allow-prereleases: true - name: Install package - run: python -m pip install .[test] + run: | + set -vxeuo pipefail + python -m pip install .[test] + python -m pip install setuptools==66.1.1 - name: Test package run: >- python -m pytest -ra --cov --cov-report=xml --cov-report=term - --durations=20 -m "(not hardware) and (not tiled)" + --durations=20 -m "(not hardware) and (not tiled)" -s -vv - name: Upload coverage report uses: codecov/codecov-action@v4.0.1 From e1510894eb725bb19871fc1862f95d63387ab358 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Wed, 14 Feb 2024 12:46:20 -0500 Subject: [PATCH 10/56] TST: add broadcast address a smarter way --- pyproject.toml | 2 +- tests/conftest.py | 28 +++++++++++++++++++++++++--- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7cb474a..2ce4bc1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ classifiers = [ ] dynamic = ["version"] dependencies = [ - "caproto", + "caproto[complete]", "h5py", "numpy", "ophyd", diff --git a/tests/conftest.py b/tests/conftest.py index 9fd0122..a824cce 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,9 +1,13 @@ from __future__ import annotations +import os +import socket import subprocess import sys import time as ttime +from pprint import pformat +import netifaces import pytest from srx_caproto_iocs.zebra.ophyd import ZebraWithCaprotoIOC @@ -18,16 +22,34 @@ def zebra_ophyd_caproto(): @pytest.fixture(scope="session") def caproto_ioc(wait=3): + first_three = ".".join(socket.gethostbyname(socket.gethostname()).split(".")[:3]) + broadcast = f"{first_three}.255" + + print(f"{broadcast = }") + env = { - "EPICS_CAS_BEACON_ADDR_LIST": "127.0.0.1", + "EPICS_CAS_BEACON_ADDR_LIST": os.getenv("EPICS_CA_ADDR_LIST", broadcast), "EPICS_CAS_AUTO_BEACON_ADDR_LIST": "no", } + + print(f"Updating env with:\n\n{pformat(env)}\n") + os.environ.update(env) + + interfaces = netifaces.interfaces() + print(f"{interfaces = }") + for interface in interfaces: + addrs = netifaces.ifaddresses(interface) + try: + print(f"{interface = }: {pformat(addrs[netifaces.AF_INET])}") + except Exception as e: + print(f"{interface = }: exception:\n {e}") + command = ( sys.executable + " -m srx_caproto_iocs.zebra.caproto_ioc --prefix=XF:05IDD-ES:1{{Dev:Zebra2}}: --list-pvs" ) print( - f"Starting caproto IOC in via a fixture using the following command:\n\n {command}\n" + f"\nStarting caproto IOC in via a fixture using the following command:\n\n {command}\n" ) p = subprocess.Popen( command.split(), @@ -35,7 +57,7 @@ def caproto_ioc(wait=3): stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False, - env=env, + env=os.environ, ) print(f"Wait for {wait} seconds...") ttime.sleep(wait) From 7a9944a21be995dfe359684cfa9f37247548bc53 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Wed, 14 Feb 2024 12:49:24 -0500 Subject: [PATCH 11/56] CI: Do not run tests twice on a PR from the same repo's branch --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45f3bcf..68bfe14 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,6 +32,11 @@ jobs: pipx run nox -s pylint checks: + # pull requests are a duplicate of a branch push if within the same repo. + if: + github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name != github.repository + name: Check Python ${{ matrix.python-version }} on ${{ matrix.runs-on }} runs-on: ${{ matrix.runs-on }} needs: [pre-commit] From d7ec78c21c67ab28bf2beec08a54cfee44dedb5b Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Wed, 14 Feb 2024 14:23:05 -0500 Subject: [PATCH 12/56] TST: print caproto stdout and stderr --- tests/conftest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index a824cce..2843eb5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -68,4 +68,6 @@ def caproto_ioc(wait=3): std_out, std_err = p.communicate() std_out = std_out.decode() - print(std_out) + sep = "=" * 80 + print(f"STDOUT:\n{sep}\n{std_out}") + print(f"STDERR:\n{sep}\n{std_err}") From 09a08ca9fcef628398952f2e037c9a0e36cb93ef Mon Sep 17 00:00:00 2001 From: Maksim Rakitin Date: Wed, 14 Feb 2024 17:28:43 -0500 Subject: [PATCH 13/56] Use base IOC/ophyd in CI tests --- src/srx_caproto_iocs/base.py | 37 ++++++++++++++++++- src/srx_caproto_iocs/sis_scaler/__init__.py | 1 + src/srx_caproto_iocs/zebra/__init__.py | 1 + src/srx_caproto_iocs/zebra/caproto_ioc.py | 13 ++----- src/srx_caproto_iocs/zebra/ophyd.py | 11 +----- tests/conftest.py | 18 +++++---- ...test_zebra_ophyd.py => test_base_ophyd.py} | 4 +- 7 files changed, 54 insertions(+), 31 deletions(-) rename tests/{test_zebra_ophyd.py => test_base_ophyd.py} (91%) diff --git a/src/srx_caproto_iocs/base.py b/src/srx_caproto_iocs/base.py index f837f19..4445c2c 100644 --- a/src/srx_caproto_iocs/base.py +++ b/src/srx_caproto_iocs/base.py @@ -1,12 +1,15 @@ from __future__ import annotations +import textwrap import threading import uuid from enum import Enum from pathlib import Path from caproto import ChannelType -from caproto.server import PVGroup, pvproperty +from caproto.server import PVGroup, pvproperty, run, template_arg_parser +from ophyd import Component as Cpt +from ophyd import Device, EpicsSignal, EpicsSignalRO from .utils import now @@ -25,7 +28,7 @@ class StageStates(Enum): STAGED = "staged" -class GenericSaveIOC(PVGroup): +class CaprotoSaveIOC(PVGroup): """Generic Caproto Save IOC""" write_dir = pvproperty( @@ -150,3 +153,33 @@ def saver(request_queue, response_queue): response = {"success": success, "error_message": error_message} response_queue.put(response) + + +class OphydDeviceWithCaprotoIOC(Device): + """An ophyd Device which works with the base caproto extension IOC.""" + + write_dir = Cpt(EpicsSignal, "write_dir", string=True) + file_name = Cpt(EpicsSignal, "file_name", string=True) + full_file_path = Cpt(EpicsSignalRO, "full_file_path", string=True) + frame_num = Cpt(EpicsSignal, "frame_num") + ioc_stage = Cpt(EpicsSignal, "stage", string=True) + + +def check_args(parser_, split_args_): + """Helper function to process caproto CLI args.""" + parsed_args = parser_.parse_args() + prefix = parsed_args.prefix + if not prefix: + parser_.error("The 'prefix' argument must be specified.") + + ioc_opts, run_opts = split_args_(parsed_args) + return ioc_opts, run_opts + + +if __name__ == "__main__": + parser, split_args = template_arg_parser( + default_prefix="", desc=textwrap.dedent(CaprotoSaveIOC.__doc__) + ) + ioc_options, run_options = check_args(parser, split_args) + ioc = CaprotoSaveIOC(**ioc_options) + run(ioc.pvdb, **run_options) diff --git a/src/srx_caproto_iocs/sis_scaler/__init__.py b/src/srx_caproto_iocs/sis_scaler/__init__.py index e69de29..479fe07 100644 --- a/src/srx_caproto_iocs/sis_scaler/__init__.py +++ b/src/srx_caproto_iocs/sis_scaler/__init__.py @@ -0,0 +1 @@ +""""SIS scaler Caproto IOC code.""" diff --git a/src/srx_caproto_iocs/zebra/__init__.py b/src/srx_caproto_iocs/zebra/__init__.py index e69de29..19c633b 100644 --- a/src/srx_caproto_iocs/zebra/__init__.py +++ b/src/srx_caproto_iocs/zebra/__init__.py @@ -0,0 +1 @@ +""""Zebra Caproto IOC code.""" diff --git a/src/srx_caproto_iocs/zebra/caproto_ioc.py b/src/srx_caproto_iocs/zebra/caproto_ioc.py index d4cb4a3..f2ae288 100644 --- a/src/srx_caproto_iocs/zebra/caproto_ioc.py +++ b/src/srx_caproto_iocs/zebra/caproto_ioc.py @@ -4,10 +4,10 @@ from caproto.server import run, template_arg_parser -from ..base import GenericSaveIOC +from ..base import CaprotoSaveIOC, check_args -class ZebraSaveIOC(GenericSaveIOC): +class ZebraSaveIOC(CaprotoSaveIOC): """Zebra caproto save IOC.""" @@ -15,13 +15,6 @@ class ZebraSaveIOC(GenericSaveIOC): parser, split_args = template_arg_parser( default_prefix="", desc=textwrap.dedent(ZebraSaveIOC.__doc__) ) - - parsed_args = parser.parse_args() - prefix = parsed_args.prefix - if not prefix: - parser.error("The 'prefix' argument must be specified.") - - ioc_options, run_options = split_args(parsed_args) - + ioc_options, run_options = check_args(parser, split_args) ioc = ZebraSaveIOC(**ioc_options) # TODO: pass libca IOC PVs of interest run(ioc.pvdb, **run_options) diff --git a/src/srx_caproto_iocs/zebra/ophyd.py b/src/srx_caproto_iocs/zebra/ophyd.py index 67d74ac..2c17aa0 100644 --- a/src/srx_caproto_iocs/zebra/ophyd.py +++ b/src/srx_caproto_iocs/zebra/ophyd.py @@ -1,14 +1,7 @@ from __future__ import annotations -from ophyd import Component as Cpt -from ophyd import Device, EpicsSignal, EpicsSignalRO +from ..base import OphydDeviceWithCaprotoIOC -class ZebraWithCaprotoIOC(Device): +class ZebraWithCaprotoIOC(OphydDeviceWithCaprotoIOC): """An ophyd Device which works with the Zebra caproto extension IOC.""" - - write_dir = Cpt(EpicsSignal, "write_dir", string=True) - file_name = Cpt(EpicsSignal, "file_name", string=True) - full_file_path = Cpt(EpicsSignalRO, "full_file_path", string=True) - frame_num = Cpt(EpicsSignal, "frame_num") - ioc_stage = Cpt(EpicsSignal, "stage", string=True) diff --git a/tests/conftest.py b/tests/conftest.py index 2843eb5..77e76f4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,18 +10,23 @@ import netifaces import pytest -from srx_caproto_iocs.zebra.ophyd import ZebraWithCaprotoIOC +from srx_caproto_iocs.base import OphydDeviceWithCaprotoIOC + +CAPROTO_PV_PREFIX = "BASE:{{Dev:Save1}}:" +OPHYD_PV_PREFIX = CAPROTO_PV_PREFIX.replace("{{", "{").replace("}}", "}") @pytest.fixture() -def zebra_ophyd_caproto(): - dev = ZebraWithCaprotoIOC("XF:05IDD-ES:1{Dev:Zebra2}:", name="zebra_ophyd_caproto") +def base_ophyd_device(): + dev = OphydDeviceWithCaprotoIOC( + OPHYD_PV_PREFIX, name="ophyd_device_with_caproto_ioc" + ) yield dev dev.ioc_stage.put("unstaged") @pytest.fixture(scope="session") -def caproto_ioc(wait=3): +def base_caproto_ioc(wait=3): first_three = ".".join(socket.gethostbyname(socket.gethostname()).split(".")[:3]) broadcast = f"{first_three}.255" @@ -44,10 +49,7 @@ def caproto_ioc(wait=3): except Exception as e: print(f"{interface = }: exception:\n {e}") - command = ( - sys.executable - + " -m srx_caproto_iocs.zebra.caproto_ioc --prefix=XF:05IDD-ES:1{{Dev:Zebra2}}: --list-pvs" - ) + command = f"{sys.executable} -m srx_caproto_iocs.base --prefix={CAPROTO_PV_PREFIX} --list-pvs" print( f"\nStarting caproto IOC in via a fixture using the following command:\n\n {command}\n" ) diff --git a/tests/test_zebra_ophyd.py b/tests/test_base_ophyd.py similarity index 91% rename from tests/test_zebra_ophyd.py rename to tests/test_base_ophyd.py index 7b4f81f..eb04ce3 100644 --- a/tests/test_zebra_ophyd.py +++ b/tests/test_base_ophyd.py @@ -11,7 +11,7 @@ @pytest.mark.cloud_friendly() @pytest.mark.parametrize("date_template", ["%Y/%m/", "%Y/%m/%d", "mydir/%Y/%m/%d"]) -def test_zebra_ophyd_caproto(caproto_ioc, zebra_ophyd_caproto, date_template): +def test_base_ophyd_templates(base_caproto_ioc, base_ophyd_device, date_template): with tempfile.TemporaryDirectory() as tmpdirname: date = now(as_object=True) write_dir_root = Path(tmpdirname) @@ -21,7 +21,7 @@ def test_zebra_ophyd_caproto(caproto_ioc, zebra_ophyd_caproto, date_template): file_template = "scan_{num:06d}_{uid}.hdf5" - dev = zebra_ophyd_caproto + dev = base_ophyd_device dev.write_dir.put(dir_template) dev.file_name.put(file_template) From eb361155da892eb0312469593b08e14c77b0bb12 Mon Sep 17 00:00:00 2001 From: Maksim Rakitin Date: Wed, 14 Feb 2024 17:52:50 -0500 Subject: [PATCH 14/56] Add saving capabilities to the base IOC --- src/srx_caproto_iocs/base.py | 64 +++++++++++++++++++++++++++++++---- src/srx_caproto_iocs/utils.py | 36 ++++++++++++++++++++ 2 files changed, 93 insertions(+), 7 deletions(-) diff --git a/src/srx_caproto_iocs/base.py b/src/srx_caproto_iocs/base.py index 4445c2c..72d394a 100644 --- a/src/srx_caproto_iocs/base.py +++ b/src/srx_caproto_iocs/base.py @@ -2,23 +2,26 @@ import textwrap import threading +import time as ttime import uuid -from enum import Enum +from enum import Enum, auto from pathlib import Path +import numpy as np from caproto import ChannelType +from caproto.ioc_examples.mini_beamline import no_reentry from caproto.server import PVGroup, pvproperty, run, template_arg_parser from ophyd import Component as Cpt from ophyd import Device, EpicsSignal, EpicsSignalRO -from .utils import now +from .utils import now, save_hdf5 class AcqStatuses(Enum): """Enum class for acquisition statuses.""" - IDLE = "Done" - ACQUIRING = "Count" + IDLE = auto() + ACQUIRING = auto() class StageStates(Enum): @@ -60,11 +63,22 @@ class CaprotoSaveIOC(PVGroup): value=StageStates.UNSTAGED.value, enum_strings=[x.value for x in StageStates], dtype=ChannelType.ENUM, - doc="Stage/unstage the detector", + doc="Stage/unstage the device", ) - def __init__(self, *args, **kwargs): + acquire = pvproperty( + value=AcqStatuses.IDLE.value, + enum_strings=[x.value for x in AcqStatuses], + dtype=ChannelType.ENUM, + doc="Acquire signal to save a dataset.", + ) + + def __init__(self, *args, update_rate=10.0, **kwargs): super().__init__(*args, **kwargs) + + self._update_rate = update_rate + self._update_period = 1.0 / update_rate + self._request_queue = None self._response_queue = None @@ -128,6 +142,42 @@ async def stage(self, instance, value): return False + def _get_current_dataset(self): + return np.random.random((10, 20)) + + @acquire.putter + @no_reentry + async def acquire(self, instance, value): + """The acquire method to perform an individual acquisition of a data point.""" + if value != AcqStatuses.ACQUIRING.value: + return False + + if ( + instance.value in [True, AcqStatuses.ACQUIRING.value] + and value == AcqStatuses.ACQUIRING.value + ): + print( + f"The device is already acquiring. Please wait until the '{AcqStatuses.IDLE.value}' status." + ) + return True + + # Delegate saving the resulting data to a blocking callback in a thread. + payload = { + "filename": self.full_file_path.value, + "data": self._get_current_dataset(), + "uid": str(uuid.uuid4()), + "timestamp": ttime.time(), + } + + await self._request_queue.async_put(payload) + response = await self._response_queue.async_get() + + if response["success"]: + # Increment the counter only on a successful saving of the file. + await self.frame_num.write(self.frame_num.value + 1) + + return False + @staticmethod def saver(request_queue, response_queue): """The saver callback for threading-based queueing.""" @@ -136,7 +186,7 @@ def saver(request_queue, response_queue): filename = received["filename"] data = received["data"] try: - # save_hdf5(fname=filename, data=data) + save_hdf5(fname=filename, data=data) print(f"{now()}: saved {data.shape} data into:\n {filename}") success = True diff --git a/src/srx_caproto_iocs/utils.py b/src/srx_caproto_iocs/utils.py index b34f530..8348255 100644 --- a/src/srx_caproto_iocs/utils.py +++ b/src/srx_caproto_iocs/utils.py @@ -2,6 +2,9 @@ import datetime +import h5py +import numpy as np + def now(as_object=False): """A helper function to return ISO 8601 formatted datetime string.""" @@ -9,3 +12,36 @@ def now(as_object=False): if as_object: return _now return _now.isoformat() + + +def save_hdf5( + fname, + data, + group_name="/entry", + group_path="data/data", + dtype="float32", + mode="x", + update_existing=False, +): + """The function to export the data to an HDF5 file.""" + h5file_desc = h5py.File(fname, mode, libver="latest") + frame_shape = data.shape + if not update_existing: + group = h5file_desc.create_group(group_name) + dataset = group.create_dataset( + "data/data", + data=np.full(fill_value=np.nan, shape=(1, *frame_shape)), + maxshape=(None, *frame_shape), + chunks=(1, *frame_shape), + dtype=dtype, + ) + frame_num = 0 + else: + dataset = h5file_desc[f"{group_name}/{group_path}"] + frame_num = dataset.shape[0] + + h5file_desc.swmr_mode = True + + dataset.resize((frame_num + 1, *frame_shape)) + dataset[frame_num, :, :] = data + dataset.flush() From d29d4426b7c40ce85aef49a3edaaefc0ede1a6d3 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Wed, 14 Feb 2024 18:11:45 -0500 Subject: [PATCH 15/56] Fixed saving capabilities --- pyproject.toml | 1 + src/srx_caproto_iocs/base.py | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2ce4bc1..e204d7c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ test = [ ] dev = [ "ipython", + "nexpy", "pre-commit", "pylint", "pytest >=6", diff --git a/src/srx_caproto_iocs/base.py b/src/srx_caproto_iocs/base.py index 72d394a..ceb456b 100644 --- a/src/srx_caproto_iocs/base.py +++ b/src/srx_caproto_iocs/base.py @@ -4,7 +4,7 @@ import threading import time as ttime import uuid -from enum import Enum, auto +from enum import Enum from pathlib import Path import numpy as np @@ -20,8 +20,8 @@ class AcqStatuses(Enum): """Enum class for acquisition statuses.""" - IDLE = auto() - ACQUIRING = auto() + IDLE = "idle" + ACQUIRING = "acquiring" class StageStates(Enum): @@ -82,10 +82,10 @@ def __init__(self, *args, update_rate=10.0, **kwargs): self._request_queue = None self._response_queue = None - _queue = pvproperty(value=0, doc="A PV to facilitate threading-based queue") + queue = pvproperty(value=0, doc="A PV to facilitate threading-based queue") - @_queue.startup - async def _queue(self, instance, async_lib): + @queue.startup + async def queue(self, instance, async_lib): """The startup behavior of the count property to set up threading queues.""" # pylint: disable=unused-argument self._request_queue = async_lib.ThreadsafeQueue() From 42f073bb0376bd0fbdba241d2a2f9db3457f637f Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Thu, 15 Feb 2024 01:24:36 -0500 Subject: [PATCH 16/56] Make it work on MacOS --- tests/test_base_ophyd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_base_ophyd.py b/tests/test_base_ophyd.py index eb04ce3..9d3fae4 100644 --- a/tests/test_base_ophyd.py +++ b/tests/test_base_ophyd.py @@ -12,7 +12,7 @@ @pytest.mark.cloud_friendly() @pytest.mark.parametrize("date_template", ["%Y/%m/", "%Y/%m/%d", "mydir/%Y/%m/%d"]) def test_base_ophyd_templates(base_caproto_ioc, base_ophyd_device, date_template): - with tempfile.TemporaryDirectory() as tmpdirname: + with tempfile.TemporaryDirectory(prefix="/tmp/") as tmpdirname: date = now(as_object=True) write_dir_root = Path(tmpdirname) dir_template = f"{write_dir_root}/{date_template}" From d78df6feaf549b4e895e2ad735719f9e53cefb6b Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Thu, 15 Feb 2024 01:25:02 -0500 Subject: [PATCH 17/56] Make it work with `act` act -W .github/workflows/ci.yml -j checks --matrix python-version:3.11 NOTE: Python 3.12 build fails on a datetime/TZ issue. | OpenBLAS WARNING - could not determine the L2 cache size on this system, assuming 256k | ImportError while loading conftest '/Users/mrakitin/src/NSLS-II/SRX/srx-caproto-iocs/tests/conftest.py'. | tests/conftest.py:13: in | from srx_caproto_iocs.base import OphydDeviceWithCaprotoIOC | /opt/hostedtoolcache/Python/3.12.2/x64/lib/python3.12/site-packages/srx_caproto_iocs/base.py:11: in | from caproto import ChannelType | /opt/hostedtoolcache/Python/3.12.2/x64/lib/python3.12/site-packages/caproto/__init__.py:2: in | from ._utils import * | /opt/hostedtoolcache/Python/3.12.2/x64/lib/python3.12/site-packages/caproto/_utils.py:29: in | from ._dbr import SubscriptionType | /opt/hostedtoolcache/Python/3.12.2/x64/lib/python3.12/site-packages/caproto/_dbr.py:17: in | from ._constants import (EPICS2UNIX_EPOCH, EPICS_EPOCH, MAX_ENUM_STATES, | /opt/hostedtoolcache/Python/3.12.2/x64/lib/python3.12/site-packages/caproto/_constants.py:11: in | EPICS_EPOCH = datetime.datetime.utcfromtimestamp(EPICS2UNIX_EPOCH) | E DeprecationWarning: datetime.datetime.utcfromtimestamp() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.fromtimestamp(timestamp, datetime.UTC). --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e204d7c..0e41821 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ classifiers = [ ] dynamic = ["version"] dependencies = [ - "caproto[complete]", + "caproto", "h5py", "numpy", "ophyd", @@ -40,6 +40,7 @@ dependencies = [ [project.optional-dependencies] test = [ + "netifaces", "pytest >=6", "pytest-cov >=3", ] From f18d187d163125eb8e920e88f6f1fdef23a50c8f Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Thu, 15 Feb 2024 10:32:57 -0500 Subject: [PATCH 18/56] Working saving example of multiple frames with ophyd --- src/srx_caproto_iocs/base.py | 66 +++++++++++++++++++++++++++++++---- src/srx_caproto_iocs/utils.py | 54 ++++++++++++++++------------ tests/conftest.py | 2 +- tests/test_base_ophyd.py | 33 ++++++++++++------ 4 files changed, 114 insertions(+), 41 deletions(-) diff --git a/src/srx_caproto_iocs/base.py b/src/srx_caproto_iocs/base.py index ceb456b..cd9beab 100644 --- a/src/srx_caproto_iocs/base.py +++ b/src/srx_caproto_iocs/base.py @@ -7,12 +7,13 @@ from enum import Enum from pathlib import Path -import numpy as np +import skimage.data from caproto import ChannelType from caproto.ioc_examples.mini_beamline import no_reentry from caproto.server import PVGroup, pvproperty, run, template_arg_parser from ophyd import Component as Cpt from ophyd import Device, EpicsSignal, EpicsSignalRO +from ophyd.status import SubscriptionStatus from .utils import now, save_hdf5 @@ -142,14 +143,18 @@ async def stage(self, instance, value): return False - def _get_current_dataset(self): - return np.random.random((10, 20)) + def _get_current_dataset(self, frame): + dataset = skimage.data.cells3d().sum(axis=1) + return dataset[frame, ...] @acquire.putter @no_reentry async def acquire(self, instance, value): """The acquire method to perform an individual acquisition of a data point.""" - if value != AcqStatuses.ACQUIRING.value: + if ( + value != AcqStatuses.ACQUIRING.value + # or self.stage.value not in [True, StageStates.STAGED.value] + ): return False if ( @@ -161,12 +166,16 @@ async def acquire(self, instance, value): ) return True + await self.acquire.write(AcqStatuses.ACQUIRING.value) + # Delegate saving the resulting data to a blocking callback in a thread. payload = { "filename": self.full_file_path.value, - "data": self._get_current_dataset(), + "data": self._get_current_dataset(frame=self.frame_num.value), "uid": str(uuid.uuid4()), "timestamp": ttime.time(), + "frame_number": self.frame_num.value, + "update_existing": self.frame_num.value > 0, } await self._request_queue.async_put(payload) @@ -176,6 +185,8 @@ async def acquire(self, instance, value): # Increment the counter only on a successful saving of the file. await self.frame_num.write(self.frame_num.value + 1) + # await self.acquire.write(AcqStatuses.IDLE.value) + return False @staticmethod @@ -185,9 +196,15 @@ def saver(request_queue, response_queue): received = request_queue.get() filename = received["filename"] data = received["data"] + frame_number = received["frame_number"] + update_existing = received["update_existing"] try: - save_hdf5(fname=filename, data=data) - print(f"{now()}: saved {data.shape} data into:\n {filename}") + save_hdf5( + fname=filename, data=data, mode="a", update_existing=update_existing + ) + print( + f"{now()}: saved {frame_number=} {data.shape} data into:\n {filename}" + ) success = True error_message = "" @@ -213,6 +230,41 @@ class OphydDeviceWithCaprotoIOC(Device): full_file_path = Cpt(EpicsSignalRO, "full_file_path", string=True) frame_num = Cpt(EpicsSignal, "frame_num") ioc_stage = Cpt(EpicsSignal, "stage", string=True) + acquire = Cpt(EpicsSignal, "acquire", string=True) + + def set(self, command): + """The set method with values for staging and acquiring.""" + + print(f"{now()}: {command = }") + if command in [StageStates.STAGED.value, "stage"]: + expected_old_value = StageStates.UNSTAGED.value + expected_new_value = StageStates.STAGED.value + obj = self.ioc_stage + cmd = StageStates.STAGED.value + + if command in [StageStates.UNSTAGED.value, "unstage"]: + expected_old_value = StageStates.STAGED.value + expected_new_value = StageStates.UNSTAGED.value + obj = self.ioc_stage + cmd = StageStates.UNSTAGED.value + + if command in [AcqStatuses.ACQUIRING.value, "acquire"]: + expected_old_value = AcqStatuses.ACQUIRING.value + expected_new_value = AcqStatuses.IDLE.value + obj = self.acquire + cmd = AcqStatuses.ACQUIRING.value + + def cb(value, old_value, **kwargs): + # pylint: disable=unused-argument + print(f"{now()}: {old_value} -> {value}") + if value == expected_new_value and old_value == expected_old_value: + return True + return False + + st = SubscriptionStatus(obj, callback=cb, run=False) + print(f"{now()}: {cmd = }") + obj.put(cmd) + return st def check_args(parser_, split_args_): diff --git a/src/srx_caproto_iocs/utils.py b/src/srx_caproto_iocs/utils.py index 8348255..07e99cb 100644 --- a/src/srx_caproto_iocs/utils.py +++ b/src/srx_caproto_iocs/utils.py @@ -23,25 +23,35 @@ def save_hdf5( mode="x", update_existing=False, ): - """The function to export the data to an HDF5 file.""" - h5file_desc = h5py.File(fname, mode, libver="latest") - frame_shape = data.shape - if not update_existing: - group = h5file_desc.create_group(group_name) - dataset = group.create_dataset( - "data/data", - data=np.full(fill_value=np.nan, shape=(1, *frame_shape)), - maxshape=(None, *frame_shape), - chunks=(1, *frame_shape), - dtype=dtype, - ) - frame_num = 0 - else: - dataset = h5file_desc[f"{group_name}/{group_path}"] - frame_num = dataset.shape[0] - - h5file_desc.swmr_mode = True - - dataset.resize((frame_num + 1, *frame_shape)) - dataset[frame_num, :, :] = data - dataset.flush() + """The function to export the data to an HDF5 file. + + Check https://docs.h5py.org/en/stable/high/file.html#opening-creating-files for modes: + + r Readonly, file must exist (default) + r+ Read/write, file must exist + w Create file, truncate if exists + w- or x Create file, fail if exists + a Read/write if exists, create otherwise + """ + with h5py.File(fname, mode, libver="latest") as h5file_desc: + frame_shape = data.shape + if not update_existing: + group = h5file_desc.create_group(group_name) + dataset = group.create_dataset( + group_path, + data=np.full(fill_value=np.nan, shape=(1, *frame_shape)), + maxshape=(None, *frame_shape), + chunks=(1, *frame_shape), + dtype=dtype, + ) + frame_num = 0 + else: + dataset = h5file_desc[f"{group_name}/{group_path}"] + frame_num = dataset.shape[0] + + # https://docs.h5py.org/en/stable/swmr.html + h5file_desc.swmr_mode = True + + dataset.resize((frame_num + 1, *frame_shape)) + dataset[frame_num, :, :] = data + dataset.flush() diff --git a/tests/conftest.py b/tests/conftest.py index 77e76f4..5137c09 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,7 +26,7 @@ def base_ophyd_device(): @pytest.fixture(scope="session") -def base_caproto_ioc(wait=3): +def base_caproto_ioc(wait=5): first_three = ".".join(socket.gethostbyname(socket.gethostname()).split(".")[:3]) broadcast = f"{first_three}.255" diff --git a/tests/test_base_ophyd.py b/tests/test_base_ophyd.py index 9d3fae4..2ed76f7 100644 --- a/tests/test_base_ophyd.py +++ b/tests/test_base_ophyd.py @@ -1,17 +1,21 @@ from __future__ import annotations import tempfile +import time as ttime from pathlib import Path +import h5py import pytest -from ophyd.status import SubscriptionStatus from srx_caproto_iocs.utils import now @pytest.mark.cloud_friendly() -@pytest.mark.parametrize("date_template", ["%Y/%m/", "%Y/%m/%d", "mydir/%Y/%m/%d"]) -def test_base_ophyd_templates(base_caproto_ioc, base_ophyd_device, date_template): +# @pytest.mark.parametrize("date_template", ["%Y/%m/", "%Y/%m/%d", "mydir/%Y/%m/%d"]) +@pytest.mark.parametrize("date_template", ["%Y/%m/"]) +def test_base_ophyd_templates( + base_caproto_ioc, base_ophyd_device, date_template, num_frames=50 +): with tempfile.TemporaryDirectory(prefix="/tmp/") as tmpdirname: date = now(as_object=True) write_dir_root = Path(tmpdirname) @@ -25,16 +29,23 @@ def test_base_ophyd_templates(base_caproto_ioc, base_ophyd_device, date_template dev.write_dir.put(dir_template) dev.file_name.put(file_template) - def cb(value, old_value, **kwargs): - if value == "staged" and old_value == "unstaged": - return True - return False - - st = SubscriptionStatus(dev.ioc_stage, callback=cb, run=False) - dev.ioc_stage.put("staged") - st.wait() + dev.set("stage").wait() full_file_path = dev.full_file_path.get() print(f"{full_file_path = }") + for i in range(num_frames): + print(f"Collecting frame {i}...") + dev.set("acquire").wait() + ttime.sleep(0.1) + + dev.set("unstage").wait() + assert full_file_path, "The returned 'full_file_path' did not change." + assert Path(full_file_path).is_file(), f"No such file '{full_file_path}'" + + with h5py.File(full_file_path, "r", swmr=True) as f: + dataset = f["/entry/data/data"] + assert dataset.shape == (num_frames, 256, 256) + + ttime.sleep(1.0) From fbf17aaaffdbc18a826878c44c734227c2fe529e Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Thu, 15 Feb 2024 10:40:56 -0500 Subject: [PATCH 19/56] Add `scikit-image` to the dependencies --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 0e41821..bc7bf93 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ dependencies = [ "numpy", "ophyd", "pyepics", # does not work with 'setuptools' version higher than v66.1.1 + "scikit-image", ] [project.optional-dependencies] From 384c00119fef898f47efc0460a3519df9435e450 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Thu, 15 Feb 2024 14:43:30 -0500 Subject: [PATCH 20/56] Add missing skimage dependency to pull the dataset --- pyproject.toml | 1 + scripts/run-caproto-ioc.sh | 9 +++++++++ scripts/test-file-saving.sh | 21 +++++++++++++++++++++ 3 files changed, 31 insertions(+) create mode 100644 scripts/run-caproto-ioc.sh create mode 100644 scripts/test-file-saving.sh diff --git a/pyproject.toml b/pyproject.toml index bc7bf93..a7ee4d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ dependencies = [ "ophyd", "pyepics", # does not work with 'setuptools' version higher than v66.1.1 "scikit-image", + "pooch", # datasets for skimage ] [project.optional-dependencies] diff --git a/scripts/run-caproto-ioc.sh b/scripts/run-caproto-ioc.sh new file mode 100644 index 0000000..50549ca --- /dev/null +++ b/scripts/run-caproto-ioc.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# shellcheck source=/dev/null +. /etc/profile.d/epics.sh + +export EPICS_CAS_AUTO_BEACON_ADDR_LIST="no" +export EPICS_CAS_BEACON_ADDR_LIST="${EPICS_CA_ADDR_LIST}" + +python -m srx_caproto_iocs.base --prefix="BASE:{{Dev:Save1}}:" --list-pvs diff --git a/scripts/test-file-saving.sh b/scripts/test-file-saving.sh new file mode 100644 index 0000000..bc4e0ae --- /dev/null +++ b/scripts/test-file-saving.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# shellcheck source=/dev/null +. /etc/profile.d/epics.sh + +data_dir="/tmp/test/$(date +%Y/%m/%d)" + +if [ ! -d "${data_dir}" ]; then + mkdir -v -p "${data_dir}" +fi + +caput "BASE:{Dev:Save1}:write_dir" "${data_dir}" +caput "BASE:{Dev:Save1}:file_name" "saveme_{num:06d}_{uid}.h5" +caput "BASE:{Dev:Save1}:stage" 1 +for i in $(seq 50); do + echo "$i" + sleep 0.1 + caput "BASE:{Dev:Save1}:acquire" 1 +done + +caput "BASE:{Dev:Save1}:stage" 0 From 7dbbaca2b1f360e70d141f38fe25b57920c30ab2 Mon Sep 17 00:00:00 2001 From: Maksim Rakitin Date: Thu, 15 Feb 2024 15:13:38 -0500 Subject: [PATCH 21/56] PKG: apply setuptools limitation better --- .github/workflows/ci.yml | 3 +-- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 68bfe14..19d52df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8", "3.9", "3.10", "3.11"] # runs-on: [ubuntu-latest, macos-latest, windows-latest] runs-on: [ubuntu-latest] @@ -65,7 +65,6 @@ jobs: run: | set -vxeuo pipefail python -m pip install .[test] - python -m pip install setuptools==66.1.1 - name: Test package run: >- diff --git a/pyproject.toml b/pyproject.toml index a7ee4d0..e31a6d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["hatchling", "hatch-vcs"] +requires = ["hatchling", "hatch-vcs", "setuptools>=61,<67"] build-backend = "hatchling.build" From 8248509dbc22a468dd2d4f72156afdec69480000 Mon Sep 17 00:00:00 2001 From: Maksim Rakitin Date: Thu, 15 Feb 2024 15:14:10 -0500 Subject: [PATCH 22/56] CI: configure PyPI uploading --- .github/workflows/cd.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 6ea4db9..0c7874a 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -47,6 +47,4 @@ jobs: - uses: pypa/gh-action-pypi-publish@release/v1 if: github.event_name == 'release' && github.event.action == 'published' with: - # Remember to tell (test-)pypi about this repo before publishing - # Remove this line to publish to PyPI - repository-url: https://test.pypi.org/legacy/ + packages-dir: ./dist/ From 69d43060d7d2901311e737a916ba8e7d9baf7271 Mon Sep 17 00:00:00 2001 From: Maksim Rakitin Date: Thu, 15 Feb 2024 15:25:26 -0500 Subject: [PATCH 23/56] Add convenience script to run `act` locally --- scripts/run-act.sh | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 scripts/run-act.sh diff --git a/scripts/run-act.sh b/scripts/run-act.sh new file mode 100644 index 0000000..212dd68 --- /dev/null +++ b/scripts/run-act.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +act -W .github/workflows/ci.yml -j checks --matrix python-version:3.11 From 15bb697a7ddf718d6ab091455573c23f60ef2821 Mon Sep 17 00:00:00 2001 From: Hiran Wijesinghe Date: Fri, 16 Feb 2024 17:33:08 -0500 Subject: [PATCH 24/56] add `suid` to filename formatter --- src/srx_caproto_iocs/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/srx_caproto_iocs/base.py b/src/srx_caproto_iocs/base.py index cd9beab..7a463f3 100644 --- a/src/srx_caproto_iocs/base.py +++ b/src/srx_caproto_iocs/base.py @@ -130,7 +130,7 @@ async def stage(self, instance, value): file_name = self.file_name.value uid = "" if "{uid" not in file_name else str(uuid.uuid4()) full_file_path = write_dir / file_name.format( - num=self.frame_num.value, uid=uid + num=self.frame_num.value, uid=uid, suid=uid[:8] ) full_file_path = str(full_file_path) full_file_path.replace(" ", "_") From a4e70d43d329c4eab60f1bd8695823468cddbfcc Mon Sep 17 00:00:00 2001 From: Hiran Wijesinghe Date: Fri, 16 Feb 2024 17:46:34 -0500 Subject: [PATCH 25/56] fix bug where `uid` not generated if only `suid` is used in `file_name` --- src/srx_caproto_iocs/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/srx_caproto_iocs/base.py b/src/srx_caproto_iocs/base.py index 7a463f3..56d85ce 100644 --- a/src/srx_caproto_iocs/base.py +++ b/src/srx_caproto_iocs/base.py @@ -128,7 +128,10 @@ async def stage(self, instance, value): return False file_name = self.file_name.value - uid = "" if "{uid" not in file_name else str(uuid.uuid4()) + uid = "" if not any( + s in file_name for s in ("{uid", "{suid") + ) else str(uuid.uuid4()) + uid = "" if "{uid" not in file_name or "{suid" not in file_name else str(uuid.uuid4()) full_file_path = write_dir / file_name.format( num=self.frame_num.value, uid=uid, suid=uid[:8] ) From 1d3aae333ca9c5810efd06f4ce6f1f11723ceaa6 Mon Sep 17 00:00:00 2001 From: Hiran Wijesinghe Date: Fri, 16 Feb 2024 19:52:06 -0500 Subject: [PATCH 26/56] fix typo, make more readable --- src/srx_caproto_iocs/base.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/srx_caproto_iocs/base.py b/src/srx_caproto_iocs/base.py index 56d85ce..3404e3f 100644 --- a/src/srx_caproto_iocs/base.py +++ b/src/srx_caproto_iocs/base.py @@ -128,10 +128,11 @@ async def stage(self, instance, value): return False file_name = self.file_name.value - uid = "" if not any( - s in file_name for s in ("{uid", "{suid") - ) else str(uuid.uuid4()) - uid = "" if "{uid" not in file_name or "{suid" not in file_name else str(uuid.uuid4()) + + uid = ( + str(uuid.uuid4()) if "{uid" in file_name or "{suid" in file_name else "" + ) + full_file_path = write_dir / file_name.format( num=self.frame_num.value, uid=uid, suid=uid[:8] ) From 618cbb384dca8cc30b95294601c5562da1aac78c Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Sat, 17 Feb 2024 20:24:54 -0500 Subject: [PATCH 27/56] CNL: remove irrelevant comment and add a docstring to the `_get_current_dataset` method --- src/srx_caproto_iocs/base.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/srx_caproto_iocs/base.py b/src/srx_caproto_iocs/base.py index 3404e3f..03bbd08 100644 --- a/src/srx_caproto_iocs/base.py +++ b/src/srx_caproto_iocs/base.py @@ -148,6 +148,11 @@ async def stage(self, instance, value): return False def _get_current_dataset(self, frame): + """The method to return a desired dataset. + + See https://scikit-image.org/docs/stable/auto_examples/data/plot_3d.html + for details about the dataset returned by the base class' method. + """ dataset = skimage.data.cells3d().sum(axis=1) return dataset[frame, ...] @@ -213,9 +218,6 @@ def saver(request_queue, response_queue): success = True error_message = "" except Exception as exc: # pylint: disable=broad-exception-caught - # The GeRM detector happens to response twice for a single - # ".CNT" put, so capture an attempt to save the file with the - # same name here and do nothing. success = False error_message = exc print( From 1ac8b6fbf89b5f6c1d3d644973ee693aca4c2a54 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Sat, 17 Feb 2024 23:29:03 -0500 Subject: [PATCH 28/56] TST: add input arg for a number of frames to the testing script --- scripts/test-file-saving.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/test-file-saving.sh b/scripts/test-file-saving.sh index bc4e0ae..6631d09 100644 --- a/scripts/test-file-saving.sh +++ b/scripts/test-file-saving.sh @@ -3,6 +3,8 @@ # shellcheck source=/dev/null . /etc/profile.d/epics.sh +num="${1:-50}" + data_dir="/tmp/test/$(date +%Y/%m/%d)" if [ ! -d "${data_dir}" ]; then @@ -12,7 +14,7 @@ fi caput "BASE:{Dev:Save1}:write_dir" "${data_dir}" caput "BASE:{Dev:Save1}:file_name" "saveme_{num:06d}_{uid}.h5" caput "BASE:{Dev:Save1}:stage" 1 -for i in $(seq 50); do +for i in $(seq "$num"); do echo "$i" sleep 0.1 caput "BASE:{Dev:Save1}:acquire" 1 From 34e291cf29aa1f02347339d8235b12308699ea83 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Sat, 17 Feb 2024 23:53:23 -0500 Subject: [PATCH 29/56] Make multiple files saving working correctly --- src/srx_caproto_iocs/base.py | 17 +++++++---------- src/srx_caproto_iocs/utils.py | 3 ++- tests/test_base_ophyd.py | 3 +-- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src/srx_caproto_iocs/base.py b/src/srx_caproto_iocs/base.py index 03bbd08..20bdb76 100644 --- a/src/srx_caproto_iocs/base.py +++ b/src/srx_caproto_iocs/base.py @@ -89,8 +89,8 @@ def __init__(self, *args, update_rate=10.0, **kwargs): async def queue(self, instance, async_lib): """The startup behavior of the count property to set up threading queues.""" # pylint: disable=unused-argument - self._request_queue = async_lib.ThreadsafeQueue() - self._response_queue = async_lib.ThreadsafeQueue() + self._request_queue = async_lib.ThreadsafeQueue(maxsize=1) + self._response_queue = async_lib.ThreadsafeQueue(maxsize=1) # Start a separate thread that consumes requests and sends responses. thread = threading.Thread( @@ -154,7 +154,8 @@ def _get_current_dataset(self, frame): for details about the dataset returned by the base class' method. """ dataset = skimage.data.cells3d().sum(axis=1) - return dataset[frame, ...] + # This particular example dataset has 60 frames available, so we will cycle the slices for frame>=60. + return dataset[frame % dataset.shape[0], ...] @acquire.putter @no_reentry @@ -184,7 +185,6 @@ async def acquire(self, instance, value): "uid": str(uuid.uuid4()), "timestamp": ttime.time(), "frame_number": self.frame_num.value, - "update_existing": self.frame_num.value > 0, } await self._request_queue.async_put(payload) @@ -206,11 +206,8 @@ def saver(request_queue, response_queue): filename = received["filename"] data = received["data"] frame_number = received["frame_number"] - update_existing = received["update_existing"] try: - save_hdf5( - fname=filename, data=data, mode="a", update_existing=update_existing - ) + save_hdf5(fname=filename, data=data, mode="a") print( f"{now()}: saved {frame_number=} {data.shape} data into:\n {filename}" ) @@ -262,13 +259,13 @@ def set(self, command): def cb(value, old_value, **kwargs): # pylint: disable=unused-argument - print(f"{now()}: {old_value} -> {value}") + # print(f"{now()}: {old_value} -> {value}") if value == expected_new_value and old_value == expected_old_value: return True return False st = SubscriptionStatus(obj, callback=cb, run=False) - print(f"{now()}: {cmd = }") + # print(f"{now()}: {cmd = }") obj.put(cmd) return st diff --git a/src/srx_caproto_iocs/utils.py b/src/srx_caproto_iocs/utils.py index 07e99cb..750b480 100644 --- a/src/srx_caproto_iocs/utils.py +++ b/src/srx_caproto_iocs/utils.py @@ -1,6 +1,7 @@ from __future__ import annotations import datetime +from pathlib import Path import h5py import numpy as np @@ -21,7 +22,6 @@ def save_hdf5( group_path="data/data", dtype="float32", mode="x", - update_existing=False, ): """The function to export the data to an HDF5 file. @@ -33,6 +33,7 @@ def save_hdf5( w- or x Create file, fail if exists a Read/write if exists, create otherwise """ + update_existing = Path(fname).is_file() with h5py.File(fname, mode, libver="latest") as h5file_desc: frame_shape = data.shape if not update_existing: diff --git a/tests/test_base_ophyd.py b/tests/test_base_ophyd.py index 2ed76f7..b157dfd 100644 --- a/tests/test_base_ophyd.py +++ b/tests/test_base_ophyd.py @@ -11,8 +11,7 @@ @pytest.mark.cloud_friendly() -# @pytest.mark.parametrize("date_template", ["%Y/%m/", "%Y/%m/%d", "mydir/%Y/%m/%d"]) -@pytest.mark.parametrize("date_template", ["%Y/%m/"]) +@pytest.mark.parametrize("date_template", ["%Y/%m/", "%Y/%m/%d", "mydir/%Y/%m/%d"]) def test_base_ophyd_templates( base_caproto_ioc, base_ophyd_device, date_template, num_frames=50 ): From a179b09643003511e4bfdd1993e0a3443be13138 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Sat, 17 Feb 2024 23:54:12 -0500 Subject: [PATCH 30/56] Update the script for manual testing to report the resulting `:full_file_path` --- scripts/test-file-saving.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/test-file-saving.sh b/scripts/test-file-saving.sh index 6631d09..6591dbd 100644 --- a/scripts/test-file-saving.sh +++ b/scripts/test-file-saving.sh @@ -1,7 +1,11 @@ #!/bin/bash +set -vxeuo pipefail + # shellcheck source=/dev/null -. /etc/profile.d/epics.sh +if [ -f "/etc/profile.d/epics.sh" ]; then + . /etc/profile.d/epics.sh +fi num="${1:-50}" @@ -14,6 +18,7 @@ fi caput "BASE:{Dev:Save1}:write_dir" "${data_dir}" caput "BASE:{Dev:Save1}:file_name" "saveme_{num:06d}_{uid}.h5" caput "BASE:{Dev:Save1}:stage" 1 +caget -S "BASE:{Dev:Save1}:full_file_path" for i in $(seq "$num"); do echo "$i" sleep 0.1 @@ -21,3 +26,7 @@ for i in $(seq "$num"); do done caput "BASE:{Dev:Save1}:stage" 0 + +caget -S "BASE:{Dev:Save1}:full_file_path" + +exit 0 From 66df054d71f1052cb741821f09d40ff4a8bc0347 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Sun, 18 Feb 2024 00:04:50 -0500 Subject: [PATCH 31/56] Update caproto startup and testing scripts --- scripts/run-caproto-ioc.sh | 8 ++++++-- scripts/test-file-saving.sh | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/run-caproto-ioc.sh b/scripts/run-caproto-ioc.sh index 50549ca..76c8257 100644 --- a/scripts/run-caproto-ioc.sh +++ b/scripts/run-caproto-ioc.sh @@ -1,9 +1,13 @@ #!/bin/bash +set -vxeuo pipefail + # shellcheck source=/dev/null -. /etc/profile.d/epics.sh +if [ -f "/etc/profile.d/epics.sh" ]; then + . /etc/profile.d/epics.sh +fi export EPICS_CAS_AUTO_BEACON_ADDR_LIST="no" -export EPICS_CAS_BEACON_ADDR_LIST="${EPICS_CA_ADDR_LIST}" +export EPICS_CAS_BEACON_ADDR_LIST="${EPICS_CA_ADDR_LIST:-127.0.0.255}" python -m srx_caproto_iocs.base --prefix="BASE:{{Dev:Save1}}:" --list-pvs diff --git a/scripts/test-file-saving.sh b/scripts/test-file-saving.sh index 6591dbd..423d569 100644 --- a/scripts/test-file-saving.sh +++ b/scripts/test-file-saving.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -vxeuo pipefail +set -euo pipefail # shellcheck source=/dev/null if [ -f "/etc/profile.d/epics.sh" ]; then From 08bcbf225a7348883e666b8594830f3817b57f89 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Sun, 18 Feb 2024 01:30:04 -0500 Subject: [PATCH 32/56] PKG: install dependencies if skimage via extra --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e31a6d2..d81d23d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,8 +36,7 @@ dependencies = [ "numpy", "ophyd", "pyepics", # does not work with 'setuptools' version higher than v66.1.1 - "scikit-image", - "pooch", # datasets for skimage + "scikit-image[data]", ] [project.optional-dependencies] From 1c6d5be730ef8143b9bed7c0de630eecf9e647c9 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Sun, 18 Feb 2024 01:30:39 -0500 Subject: [PATCH 33/56] CI: upload testing results as artifacts --- .github/workflows/ci.yml | 14 ++++++++++ src/srx_caproto_iocs/base.py | 2 +- tests/test_base_ophyd.py | 54 +++++++++++++++++++----------------- 3 files changed, 44 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 19d52df..7fbd578 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,6 +50,8 @@ jobs: # include: # - python-version: pypy-3.10 # runs-on: ubuntu-latest + env: + TZ: America/New_York steps: - uses: actions/checkout@v4 @@ -61,6 +63,11 @@ jobs: python-version: ${{ matrix.python-version }} allow-prereleases: true + - name: Set env vars + run: | + export DATETIME_STRING=$(date +%Y%m%d%H%M%S) + echo "DATETIME_STRING=${DATETIME_STRING}" >> $GITHUB_ENV + - name: Install package run: | set -vxeuo pipefail @@ -71,5 +78,12 @@ jobs: python -m pytest -ra --cov --cov-report=xml --cov-report=term --durations=20 -m "(not hardware) and (not tiled)" -s -vv + - name: Upload test artifacts + uses: actions/upload-artifact@v4 + with: + name: test-artifacts-{{ env.DATETIME_STRING }} + path: /tmp/srx-caproto-iocs/ + retention-days: 14 + - name: Upload coverage report uses: codecov/codecov-action@v4.0.1 diff --git a/src/srx_caproto_iocs/base.py b/src/srx_caproto_iocs/base.py index 20bdb76..2ad314f 100644 --- a/src/srx_caproto_iocs/base.py +++ b/src/srx_caproto_iocs/base.py @@ -238,7 +238,7 @@ class OphydDeviceWithCaprotoIOC(Device): def set(self, command): """The set method with values for staging and acquiring.""" - print(f"{now()}: {command = }") + # print(f"{now()}: {command = }") if command in [StageStates.STAGED.value, "stage"]: expected_old_value = StageStates.UNSTAGED.value expected_new_value = StageStates.STAGED.value diff --git a/tests/test_base_ophyd.py b/tests/test_base_ophyd.py index b157dfd..ad00dab 100644 --- a/tests/test_base_ophyd.py +++ b/tests/test_base_ophyd.py @@ -1,7 +1,8 @@ from __future__ import annotations -import tempfile +import shutil import time as ttime +import uuid from pathlib import Path import h5py @@ -13,38 +14,41 @@ @pytest.mark.cloud_friendly() @pytest.mark.parametrize("date_template", ["%Y/%m/", "%Y/%m/%d", "mydir/%Y/%m/%d"]) def test_base_ophyd_templates( - base_caproto_ioc, base_ophyd_device, date_template, num_frames=50 + base_caproto_ioc, base_ophyd_device, date_template, num_frames=50, remove=False ): - with tempfile.TemporaryDirectory(prefix="/tmp/") as tmpdirname: - date = now(as_object=True) - write_dir_root = Path(tmpdirname) - dir_template = f"{write_dir_root}/{date_template}" - write_dir = Path(date.strftime(dir_template)) - write_dir.mkdir(parents=True, exist_ok=True) + tmpdirname = f"/tmp/srx-caproto-iocs/{str(uuid.uuid4())[:2]}" + date = now(as_object=True) + write_dir_root = Path(tmpdirname) + dir_template = f"{write_dir_root}/{date_template}" + write_dir = Path(date.strftime(dir_template)) + write_dir.mkdir(parents=True, exist_ok=True) - file_template = "scan_{num:06d}_{uid}.hdf5" + file_template = "scan_{num:06d}_{uid}.hdf5" - dev = base_ophyd_device - dev.write_dir.put(dir_template) - dev.file_name.put(file_template) + dev = base_ophyd_device + dev.write_dir.put(dir_template) + dev.file_name.put(file_template) - dev.set("stage").wait() + dev.set("stage").wait() - full_file_path = dev.full_file_path.get() - print(f"{full_file_path = }") + full_file_path = dev.full_file_path.get() + print(f"{full_file_path = }") - for i in range(num_frames): - print(f"Collecting frame {i}...") - dev.set("acquire").wait() - ttime.sleep(0.1) + for i in range(num_frames): + print(f"Collecting frame {i}...") + dev.set("acquire").wait() + ttime.sleep(0.1) - dev.set("unstage").wait() + dev.set("unstage").wait() - assert full_file_path, "The returned 'full_file_path' did not change." - assert Path(full_file_path).is_file(), f"No such file '{full_file_path}'" + assert full_file_path, "The returned 'full_file_path' did not change." + assert Path(full_file_path).is_file(), f"No such file '{full_file_path}'" - with h5py.File(full_file_path, "r", swmr=True) as f: - dataset = f["/entry/data/data"] - assert dataset.shape == (num_frames, 256, 256) + with h5py.File(full_file_path, "r", swmr=True) as f: + dataset = f["/entry/data/data"] + assert dataset.shape == (num_frames, 256, 256) ttime.sleep(1.0) + + if remove: + shutil.rmtree(tmpdirname) From 5be9909f48f7565d3ba3f1115a15ce0cf1a1a989 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Sun, 18 Feb 2024 01:53:39 -0500 Subject: [PATCH 34/56] CI: fix uploading of artifacts --- .github/workflows/ci.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7fbd578..47db85d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,6 +65,13 @@ jobs: - name: Set env vars run: | + + export REPOSITORY_NAME=${GITHUB_REPOSITORY#*/} # just the repo, as opposed to org/repo + echo "REPOSITORY_NAME=${REPOSITORY_NAME}" >> $GITHUB_ENV + + export PYTHONVER=$(echo ${{ matrix.python-version }} | sed 's/\.//g') + echo "PYTHONVER=${PYTHONVER}" >> $GITHUB_ENV + export DATETIME_STRING=$(date +%Y%m%d%H%M%S) echo "DATETIME_STRING=${DATETIME_STRING}" >> $GITHUB_ENV @@ -81,7 +88,9 @@ jobs: - name: Upload test artifacts uses: actions/upload-artifact@v4 with: - name: test-artifacts-{{ env.DATETIME_STRING }} + name: + ${{ env.REPOSITORY_NAME }}-${{ env.DATETIME_STRING }}-py${{ + env.PYTHONVER }} path: /tmp/srx-caproto-iocs/ retention-days: 14 From 41fca537ad13ecd2765afe9a195a847e8cae6b00 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Sun, 18 Feb 2024 01:59:11 -0500 Subject: [PATCH 35/56] CI: update artifact names --- .github/workflows/ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47db85d..35705f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,9 +88,7 @@ jobs: - name: Upload test artifacts uses: actions/upload-artifact@v4 with: - name: - ${{ env.REPOSITORY_NAME }}-${{ env.DATETIME_STRING }}-py${{ - env.PYTHONVER }} + name: ${{env.REPOSITORY_NAME}}-py${{env.PYTHONVER}}-${{env.DATETIME_STRING}} path: /tmp/srx-caproto-iocs/ retention-days: 14 From e75be6c5b1118990eeac0acf3c7d0fed10e2b5f3 Mon Sep 17 00:00:00 2001 From: Hiran Wijesinghe Date: Wed, 21 Feb 2024 12:20:59 -0500 Subject: [PATCH 36/56] add test parameters to test hidden spaces in write dir --- tests/test_base_ophyd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_base_ophyd.py b/tests/test_base_ophyd.py index ad00dab..1538f36 100644 --- a/tests/test_base_ophyd.py +++ b/tests/test_base_ophyd.py @@ -12,7 +12,7 @@ @pytest.mark.cloud_friendly() -@pytest.mark.parametrize("date_template", ["%Y/%m/", "%Y/%m/%d", "mydir/%Y/%m/%d"]) +@pytest.mark.parametrize("date_template", ["%Y/%m/", "%Y/%m/%d", "mydir/%Y/%m/%d", "disguised_spaces_%c"]) def test_base_ophyd_templates( base_caproto_ioc, base_ophyd_device, date_template, num_frames=50, remove=False ): From a33c7357c32c9cf26bd6509d26d6732a3e263923 Mon Sep 17 00:00:00 2001 From: Hiran Wijesinghe Date: Wed, 21 Feb 2024 15:19:44 -0500 Subject: [PATCH 37/56] fix lint errors from `e75be6c` --- tests/test_base_ophyd.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_base_ophyd.py b/tests/test_base_ophyd.py index 1538f36..f5ecade 100644 --- a/tests/test_base_ophyd.py +++ b/tests/test_base_ophyd.py @@ -12,7 +12,9 @@ @pytest.mark.cloud_friendly() -@pytest.mark.parametrize("date_template", ["%Y/%m/", "%Y/%m/%d", "mydir/%Y/%m/%d", "disguised_spaces_%c"]) +@pytest.mark.parametrize( + "date_template", ["%Y/%m/", "%Y/%m/%d", "mydir/%Y/%m/%d", "disguised_spaces_%c"] +) def test_base_ophyd_templates( base_caproto_ioc, base_ophyd_device, date_template, num_frames=50, remove=False ): From 9724d199768df26fcba439b4009df686342885f5 Mon Sep 17 00:00:00 2001 From: Maksim Rakitin Date: Wed, 21 Feb 2024 17:32:40 -0500 Subject: [PATCH 38/56] WIP: test with string IOC --- pyproject.toml | 3 +- src/srx_caproto_iocs/example/__init__.py | 1 + src/srx_caproto_iocs/example/caproto_ioc.py | 60 ++++++++++++++ src/srx_caproto_iocs/example/ophyd.py | 15 ++++ tests/conftest.py | 88 ++++++++++++++------- tests/test_string_ioc.py | 52 ++++++++++++ 6 files changed, 191 insertions(+), 28 deletions(-) create mode 100644 src/srx_caproto_iocs/example/__init__.py create mode 100644 src/srx_caproto_iocs/example/caproto_ioc.py create mode 100644 src/srx_caproto_iocs/example/ophyd.py create mode 100644 tests/test_string_ioc.py diff --git a/pyproject.toml b/pyproject.toml index d81d23d..ac54ad8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,6 +92,7 @@ markers = [ "hardware: marks tests as requiring the hardware IOC to be available/running (deselect with '-m \"not hardware\"')", "tiled: marks tests as requiring tiled", "cloud_friendly: marks tests to be able to execute in the CI in the cloud", + "needs_epics_core: marks tests as requiring epics-core executables such as caget, caput, etc." ] [tool.coverage] @@ -135,7 +136,7 @@ extend-select = [ "PT", # flake8-pytest-style "PTH", # flake8-use-pathlib "RET", # flake8-return - "RUF", # Ruff-specific + # "RUF", # Ruff-specific "SIM", # flake8-simplify # "T20", # flake8-print "UP", # pyupgrade diff --git a/src/srx_caproto_iocs/example/__init__.py b/src/srx_caproto_iocs/example/__init__.py new file mode 100644 index 0000000..5baeb5f --- /dev/null +++ b/src/srx_caproto_iocs/example/__init__.py @@ -0,0 +1 @@ +""""Example Caproto IOC code.""" diff --git a/src/srx_caproto_iocs/example/caproto_ioc.py b/src/srx_caproto_iocs/example/caproto_ioc.py new file mode 100644 index 0000000..d4b38a9 --- /dev/null +++ b/src/srx_caproto_iocs/example/caproto_ioc.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import textwrap + +from caproto import ChannelType +from caproto.server import PVGroup, pvproperty, run, template_arg_parser + +from ..base import check_args + + +class CaprotoStringIOC(PVGroup): + """Test channel types for strings.""" + + common_kwargs = {"max_length": 255, "string_encoding": "utf-8"} + + bare_string = pvproperty( + value="bare_string", doc="A test for a bare string", **common_kwargs + ) + implicit_string_type = pvproperty( + value="implicit_string_type", + doc="A test for an implicit string type", + report_as_string=True, + **common_kwargs, + ) + string_type = pvproperty( + value="string_type", + doc="A test for a string type", + dtype=str, + report_as_string=True, + **common_kwargs, + ) + string_type_enum = pvproperty( + value="string_type_enum", + doc="A test for a string type", + dtype=ChannelType.STRING, + **common_kwargs, + ) + char_type_as_string = pvproperty( + value="char_type_as_string", + doc="A test for a char type reported as string", + report_as_string=True, + dtype=ChannelType.CHAR, + **common_kwargs, + ) + char_type = pvproperty( + value="char_type", + doc="A test for a char type not reported as string", + dtype=ChannelType.CHAR, + **common_kwargs, + ) + + +if __name__ == "__main__": + parser, split_args = template_arg_parser( + default_prefix="", desc=textwrap.dedent(CaprotoStringIOC.__doc__) + ) + ioc_options, run_options = check_args(parser, split_args) + ioc = CaprotoStringIOC(**ioc_options) + + run(ioc.pvdb, **run_options) diff --git a/src/srx_caproto_iocs/example/ophyd.py b/src/srx_caproto_iocs/example/ophyd.py new file mode 100644 index 0000000..68c20c5 --- /dev/null +++ b/src/srx_caproto_iocs/example/ophyd.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from ophyd import Component as Cpt +from ophyd import Device, EpicsSignal + + +class OphydChannelTypes(Device): + """An ophyd Device which works with the CaprotoIOCChannelTypes caproto IOC.""" + + bare_string = Cpt(EpicsSignal, "bare_string", string=True) + implicit_string_type = Cpt(EpicsSignal, "implicit_string_type", string=True) + string_type = Cpt(EpicsSignal, "string_type", string=True) + string_type_enum = Cpt(EpicsSignal, "string_type_enum", string=True) + char_type_as_string = Cpt(EpicsSignal, "char_type_as_string", string=True) + char_type = Cpt(EpicsSignal, "char_type", string=True) diff --git a/tests/conftest.py b/tests/conftest.py index 5137c09..0512b94 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,62 +5,91 @@ import subprocess import sys import time as ttime -from pprint import pformat -import netifaces import pytest from srx_caproto_iocs.base import OphydDeviceWithCaprotoIOC +from srx_caproto_iocs.example.ophyd import OphydChannelTypes CAPROTO_PV_PREFIX = "BASE:{{Dev:Save1}}:" OPHYD_PV_PREFIX = CAPROTO_PV_PREFIX.replace("{{", "{").replace("}}", "}") -@pytest.fixture() -def base_ophyd_device(): - dev = OphydDeviceWithCaprotoIOC( - OPHYD_PV_PREFIX, name="ophyd_device_with_caproto_ioc" - ) - yield dev - dev.ioc_stage.put("unstaged") - - -@pytest.fixture(scope="session") -def base_caproto_ioc(wait=5): +def get_epics_env(): first_three = ".".join(socket.gethostbyname(socket.gethostname()).split(".")[:3]) broadcast = f"{first_three}.255" print(f"{broadcast = }") - env = { + # from pprint import pformat + # import netifaces + # interfaces = netifaces.interfaces() + # print(f"{interfaces = }") + # for interface in interfaces: + # addrs = netifaces.ifaddresses(interface) + # try: + # print(f"{interface = }: {pformat(addrs[netifaces.AF_INET])}") + # except Exception as e: + # print(f"{interface = }: exception:\n {e}") + + return { "EPICS_CAS_BEACON_ADDR_LIST": os.getenv("EPICS_CA_ADDR_LIST", broadcast), "EPICS_CAS_AUTO_BEACON_ADDR_LIST": "no", } - print(f"Updating env with:\n\n{pformat(env)}\n") - os.environ.update(env) - interfaces = netifaces.interfaces() - print(f"{interfaces = }") - for interface in interfaces: - addrs = netifaces.ifaddresses(interface) - try: - print(f"{interface = }: {pformat(addrs[netifaces.AF_INET])}") - except Exception as e: - print(f"{interface = }: exception:\n {e}") +def start_ioc_subprocess(ioc_name="srx_caproto_iocs.base", pv_prefix=CAPROTO_PV_PREFIX): + env = get_epics_env() - command = f"{sys.executable} -m srx_caproto_iocs.base --prefix={CAPROTO_PV_PREFIX} --list-pvs" + command = f"{sys.executable} -m {ioc_name} --prefix={pv_prefix} --list-pvs" print( f"\nStarting caproto IOC in via a fixture using the following command:\n\n {command}\n" ) - p = subprocess.Popen( + return subprocess.Popen( command.split(), start_new_session=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False, - env=os.environ, + env=env, + ) + + +@pytest.fixture(scope="session") +def base_caproto_ioc(wait=5): + p = start_ioc_subprocess( + ioc_name="srx_caproto_iocs.base", pv_prefix=CAPROTO_PV_PREFIX + ) + + print(f"Wait for {wait} seconds...") + ttime.sleep(wait) + + yield p + + p.terminate() + + std_out, std_err = p.communicate() + std_out = std_out.decode() + sep = "=" * 80 + print(f"STDOUT:\n{sep}\n{std_out}") + print(f"STDERR:\n{sep}\n{std_err}") + + +@pytest.fixture() +def base_ophyd_device(): + dev = OphydDeviceWithCaprotoIOC( + OPHYD_PV_PREFIX, name="ophyd_device_with_caproto_ioc" ) + yield dev + dev.ioc_stage.put("unstaged") + + +@pytest.fixture(scope="session") +def caproto_ioc_channel_types(wait=5): + p = start_ioc_subprocess( + ioc_name="srx_caproto_iocs.example.caproto_ioc", pv_prefix=CAPROTO_PV_PREFIX + ) + print(f"Wait for {wait} seconds...") ttime.sleep(wait) @@ -73,3 +102,8 @@ def base_caproto_ioc(wait=5): sep = "=" * 80 print(f"STDOUT:\n{sep}\n{std_out}") print(f"STDERR:\n{sep}\n{std_err}") + + +@pytest.fixture() +def ophyd_channel_types(): + return OphydChannelTypes(OPHYD_PV_PREFIX, name="ophyd_channel_type") diff --git a/tests/test_string_ioc.py b/tests/test_string_ioc.py new file mode 100644 index 0000000..d91edec --- /dev/null +++ b/tests/test_string_ioc.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import string +import subprocess + +import pytest + +LIMIT = 40 - 1 +STRING_39 = string.ascii_letters[:LIMIT] +STRING_LONGER = string.ascii_letters + + +@pytest.mark.cloud_friendly() +@pytest.mark.parametrize("value", [STRING_39, STRING_LONGER]) +def test_strings( + # caproto_ioc_channel_types, + ophyd_channel_types, + value, +): + ophyd_channel_types.bare_string.put(value) + + if len(value) <= LIMIT: + ophyd_channel_types.implicit_string_type.put(value) + else: + with pytest.raises(ValueError): + ophyd_channel_types.implicit_string_type.put(value) + + if len(value) <= LIMIT: + ophyd_channel_types.string_type.put(value) + else: + with pytest.raises(ValueError): + ophyd_channel_types.string_type.put(value) + + if len(value) <= LIMIT: + ophyd_channel_types.char_type_as_string.put(value) + else: + with pytest.raises(ValueError): + ophyd_channel_types.char_type_as_string.put(value) + + ophyd_channel_types.char_type.put(value) + + +@pytest.mark.cloud_friendly() +@pytest.mark.needs_epics_core() +@pytest.mark.parametrize("value", [STRING_39, STRING_LONGER]) +def test_with_epics_core(ophyd_channel_types, value): + for cpt in ophyd_channel_types.component_names: + ret = subprocess.run( + ["caput", "-S", getattr(ophyd_channel_types, cpt).pvname, value], + check=False, + ) + print(f"{cpt=}: {ret.returncode=}\n") From dc98043b2ca396d718cd5d82c685ff7112a5984b Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Wed, 21 Feb 2024 19:38:47 -0500 Subject: [PATCH 39/56] STY: fix linting errors --- tests/test_string_ioc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_string_ioc.py b/tests/test_string_ioc.py index d91edec..31af7ac 100644 --- a/tests/test_string_ioc.py +++ b/tests/test_string_ioc.py @@ -5,7 +5,7 @@ import pytest -LIMIT = 40 - 1 +LIMIT = 39 STRING_39 = string.ascii_letters[:LIMIT] STRING_LONGER = string.ascii_letters @@ -22,19 +22,19 @@ def test_strings( if len(value) <= LIMIT: ophyd_channel_types.implicit_string_type.put(value) else: - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="byte string too long"): ophyd_channel_types.implicit_string_type.put(value) if len(value) <= LIMIT: ophyd_channel_types.string_type.put(value) else: - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="byte string too long"): ophyd_channel_types.string_type.put(value) if len(value) <= LIMIT: ophyd_channel_types.char_type_as_string.put(value) else: - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="byte string too long"): ophyd_channel_types.char_type_as_string.put(value) ophyd_channel_types.char_type.put(value) From cfe47e2db45b619cd2849c98e1d04a72151155dc Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Wed, 21 Feb 2024 19:39:14 -0500 Subject: [PATCH 40/56] Parametrize the script to start different caproto IOCs --- scripts/run-caproto-ioc.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/run-caproto-ioc.sh b/scripts/run-caproto-ioc.sh index 76c8257..9cfaefe 100644 --- a/scripts/run-caproto-ioc.sh +++ b/scripts/run-caproto-ioc.sh @@ -2,6 +2,8 @@ set -vxeuo pipefail +CAPROTO_IOC="${1:-srx_caproto_iocs.base}" + # shellcheck source=/dev/null if [ -f "/etc/profile.d/epics.sh" ]; then . /etc/profile.d/epics.sh @@ -10,4 +12,4 @@ fi export EPICS_CAS_AUTO_BEACON_ADDR_LIST="no" export EPICS_CAS_BEACON_ADDR_LIST="${EPICS_CA_ADDR_LIST:-127.0.0.255}" -python -m srx_caproto_iocs.base --prefix="BASE:{{Dev:Save1}}:" --list-pvs +python -m "${CAPROTO_IOC}" --prefix="BASE:{{Dev:Save1}}:" --list-pvs From 9f1f5c68ec99c7e7f1f364c8ddb03b0857e07c81 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Wed, 21 Feb 2024 22:57:40 -0500 Subject: [PATCH 41/56] TST: `cainfo` and `caput` tests for string caproto IOC --- tests/conftest.py | 7 ++++- tests/test_string_ioc.py | 68 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 69 insertions(+), 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0512b94..f4e30ea 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,7 @@ import os import socket +import string import subprocess import sys import time as ttime @@ -106,4 +107,8 @@ def caproto_ioc_channel_types(wait=5): @pytest.fixture() def ophyd_channel_types(): - return OphydChannelTypes(OPHYD_PV_PREFIX, name="ophyd_channel_type") + dev = OphydChannelTypes(OPHYD_PV_PREFIX, name="ophyd_channel_type") + letters = iter(list(string.ascii_letters)) + for cpt in sorted(dev.component_names): + getattr(dev, cpt).put(next(letters)) + return dev diff --git a/tests/test_string_ioc.py b/tests/test_string_ioc.py index 31af7ac..af2e46c 100644 --- a/tests/test_string_ioc.py +++ b/tests/test_string_ioc.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re import string import subprocess @@ -13,7 +14,7 @@ @pytest.mark.cloud_friendly() @pytest.mark.parametrize("value", [STRING_39, STRING_LONGER]) def test_strings( - # caproto_ioc_channel_types, + caproto_ioc_channel_types, ophyd_channel_types, value, ): @@ -31,6 +32,12 @@ def test_strings( with pytest.raises(ValueError, match="byte string too long"): ophyd_channel_types.string_type.put(value) + if len(value) <= LIMIT: + ophyd_channel_types.string_type_enum.put(value) + else: + with pytest.raises(ValueError, match="byte string too long"): + ophyd_channel_types.string_type_enum.put(value) + if len(value) <= LIMIT: ophyd_channel_types.char_type_as_string.put(value) else: @@ -40,13 +47,64 @@ def test_strings( ophyd_channel_types.char_type.put(value) +@pytest.mark.cloud_friendly() +@pytest.mark.needs_epics_core() +def test_cainfo(caproto_ioc_channel_types, ophyd_channel_types): + for cpt in sorted(ophyd_channel_types.component_names): + command = ["cainfo", getattr(ophyd_channel_types, cpt).pvname] + command_str = " ".join(command) + ret = subprocess.run( + command, + check=False, + capture_output=True, + ) + stdout = ret.stdout.decode() + print( + f"command: {command_str}\n {ret.returncode=}\n STDOUT:\n{ret.stdout.decode()}\n STDERR:\n{ret.stderr.decode()}\n" + ) + assert ret.returncode == 0 + if cpt in [ + "char_type_as_string", + "implicit_string_type", + "string_type", + "string_type_enum", + ]: + assert "Native data type: DBF_STRING" in stdout + else: + assert "Native data type: DBF_CHAR" in stdout + + @pytest.mark.cloud_friendly() @pytest.mark.needs_epics_core() @pytest.mark.parametrize("value", [STRING_39, STRING_LONGER]) -def test_with_epics_core(ophyd_channel_types, value): - for cpt in ophyd_channel_types.component_names: +def test_caput(caproto_ioc_channel_types, ophyd_channel_types, value): + option = "" + for cpt in sorted(ophyd_channel_types.component_names): + if cpt in [ + "char_type_as_string", + "implicit_string_type", + "string_type", + "string_type_enum", + ]: + option = "-s" + would_trim = True + else: + option = "-S" + would_trim = False + command = ["caput", option, getattr(ophyd_channel_types, cpt).pvname, value] + command_str = " ".join(command) ret = subprocess.run( - ["caput", "-S", getattr(ophyd_channel_types, cpt).pvname, value], + command, check=False, + capture_output=True, + ) + stdout = ret.stdout.decode() + print( + f"command: {command_str}\n {ret.returncode=}\n STDOUT:\n{stdout}\n STDERR:\n{ret.stderr.decode()}\n" ) - print(f"{cpt=}: {ret.returncode=}\n") + assert ret.returncode == 0 + actual = re.search("New : (.*)", stdout).group(1).split()[-1].rstrip() + if not would_trim or len(value) == LIMIT: + assert actual == value + else: + assert len(actual) < len(value) From 2d05e7646e4215ec2a7d9aecf0f4fd68a9104afa Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Thu, 22 Feb 2024 00:33:17 -0500 Subject: [PATCH 42/56] TST: fix the blocking issue in the tests --- tests/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index f4e30ea..9cf01ec 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -46,13 +46,14 @@ def start_ioc_subprocess(ioc_name="srx_caproto_iocs.base", pv_prefix=CAPROTO_PV_ print( f"\nStarting caproto IOC in via a fixture using the following command:\n\n {command}\n" ) + os.environ.update(env) return subprocess.Popen( command.split(), start_new_session=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False, - env=env, + env=os.environ, ) From 9c9f53988c06fe338fa16c26ff54f45136d03e4f Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Thu, 22 Feb 2024 02:30:41 -0500 Subject: [PATCH 43/56] CI: install `epics-base` package from conda-forge (for `caget`, `caput`, `cainfo`, etc.) --- .github/workflows/ci.yml | 33 ++++++++++++++++++++++++++++----- pyproject.toml | 1 - 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 35705f2..1e227f6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,16 +53,15 @@ jobs: env: TZ: America/New_York + defaults: + run: + shell: bash -l {0} + steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - allow-prereleases: true - - name: Set env vars run: | @@ -75,9 +74,33 @@ jobs: export DATETIME_STRING=$(date +%Y%m%d%H%M%S) echo "DATETIME_STRING=${DATETIME_STRING}" >> $GITHUB_ENV + # - uses: actions/setup-python@v5 + # with: + # python-version: ${{ matrix.python-version }} + # allow-prereleases: true + + - name: Set up Python ${{ matrix.python-version }} with conda + uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: + ${{ env.REPOSITORY_NAME }}-py${{ matrix.python-version }} + python-version: ${{ matrix.python-version }} + mamba-version: "*" + miniforge-version: latest + channels: conda-forge + use-mamba: true + + - name: Install epics-base packages from CF + run: | + set -vxeuo pipefail + mamba install -c conda-forge -y epics-base "setuptools<67" + conda env list + conda list + - name: Install package run: | set -vxeuo pipefail + which caput python -m pip install .[test] - name: Test package diff --git a/pyproject.toml b/pyproject.toml index ac54ad8..580497f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,6 @@ dependencies = [ [project.optional-dependencies] test = [ - "netifaces", "pytest >=6", "pytest-cov >=3", ] From 590e5543de21db6ff86a8370b3627f1ee402f8d8 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Thu, 22 Feb 2024 02:52:49 -0500 Subject: [PATCH 44/56] Minor tweaks and addressing review comments from PR#2 --- .github/workflows/ci.yml | 2 +- scripts/run-act.sh | 4 +++- scripts/test-file-saving.sh | 5 +---- src/srx_caproto_iocs/example/ophyd.py | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e227f6..3088332 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,7 +64,7 @@ jobs: - name: Set env vars run: | - + set -x export REPOSITORY_NAME=${GITHUB_REPOSITORY#*/} # just the repo, as opposed to org/repo echo "REPOSITORY_NAME=${REPOSITORY_NAME}" >> $GITHUB_ENV diff --git a/scripts/run-act.sh b/scripts/run-act.sh index 212dd68..768ad28 100644 --- a/scripts/run-act.sh +++ b/scripts/run-act.sh @@ -1,3 +1,5 @@ #!/bin/bash -act -W .github/workflows/ci.yml -j checks --matrix python-version:3.11 +PYTHON_VERSION="${1:-3.11}" + +act -W .github/workflows/ci.yml -j checks --matrix python-version:"${PYTHON_VERSION}" diff --git a/scripts/test-file-saving.sh b/scripts/test-file-saving.sh index 423d569..2159796 100644 --- a/scripts/test-file-saving.sh +++ b/scripts/test-file-saving.sh @@ -10,10 +10,7 @@ fi num="${1:-50}" data_dir="/tmp/test/$(date +%Y/%m/%d)" - -if [ ! -d "${data_dir}" ]; then - mkdir -v -p "${data_dir}" -fi +mkdir -v -p "${data_dir}" caput "BASE:{Dev:Save1}:write_dir" "${data_dir}" caput "BASE:{Dev:Save1}:file_name" "saveme_{num:06d}_{uid}.h5" diff --git a/src/srx_caproto_iocs/example/ophyd.py b/src/srx_caproto_iocs/example/ophyd.py index 68c20c5..a44ac08 100644 --- a/src/srx_caproto_iocs/example/ophyd.py +++ b/src/srx_caproto_iocs/example/ophyd.py @@ -5,7 +5,7 @@ class OphydChannelTypes(Device): - """An ophyd Device which works with the CaprotoIOCChannelTypes caproto IOC.""" + """An ophyd Device which works with the CaprotoStringIOC caproto IOC.""" bare_string = Cpt(EpicsSignal, "bare_string", string=True) implicit_string_type = Cpt(EpicsSignal, "implicit_string_type", string=True) From ab77f7ff51676409de017f407eaf2027bd64acb2 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Thu, 22 Feb 2024 03:27:32 -0500 Subject: [PATCH 45/56] CI: use micromamba for the python env --- .github/workflows/ci.yml | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3088332..6bdf544 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,22 +80,12 @@ jobs: # allow-prereleases: true - name: Set up Python ${{ matrix.python-version }} with conda - uses: conda-incubator/setup-miniconda@v3 + uses: mamba-org/setup-micromamba@v1 with: - activate-environment: - ${{ env.REPOSITORY_NAME }}-py${{ matrix.python-version }} - python-version: ${{ matrix.python-version }} - mamba-version: "*" - miniforge-version: latest - channels: conda-forge - use-mamba: true - - - name: Install epics-base packages from CF - run: | - set -vxeuo pipefail - mamba install -c conda-forge -y epics-base "setuptools<67" - conda env list - conda list + init-shell: bash + environment-name: ${{env.REPOSITORY_NAME}}-py${{matrix.python-version}} + create-args: >- + python=${{ matrix.python-version }} epics-base setuptools<67 - name: Install package run: | From def5ded510af630193b8cbc18901a8686d608aa7 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Thu, 22 Feb 2024 03:56:00 -0500 Subject: [PATCH 46/56] Change `write_dir` and `file_name` PV types to CHAR and fix the relevant test --- src/srx_caproto_iocs/base.py | 4 ++-- tests/test_base_ophyd.py | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/srx_caproto_iocs/base.py b/src/srx_caproto_iocs/base.py index 2ad314f..9fb1faf 100644 --- a/src/srx_caproto_iocs/base.py +++ b/src/srx_caproto_iocs/base.py @@ -39,14 +39,14 @@ class CaprotoSaveIOC(PVGroup): value="/tmp", doc="The directory to write data to. It support datetime formatting, e.g. '/tmp/det/%Y/%m/%d/'", string_encoding="utf-8", - report_as_string=True, + dtype=ChannelType.CHAR, max_length=255, ) file_name = pvproperty( value="test.h5", doc="The file name of the file to write to. It support .format() based formatting, e.g. 'scan_{num:06d}.h5'", string_encoding="utf-8", - report_as_string=True, + dtype=ChannelType.CHAR, max_length=255, ) full_file_path = pvproperty( diff --git a/tests/test_base_ophyd.py b/tests/test_base_ophyd.py index f5ecade..d052bb1 100644 --- a/tests/test_base_ophyd.py +++ b/tests/test_base_ophyd.py @@ -22,7 +22,12 @@ def test_base_ophyd_templates( date = now(as_object=True) write_dir_root = Path(tmpdirname) dir_template = f"{write_dir_root}/{date_template}" - write_dir = Path(date.strftime(dir_template)) + + # We pre-create the test directory in advance as the IOC is not supposed to create one. + # The assumption for the IOC is that the directory will exist before saving a file to that. + # We need to replace the blank spaces below for it to work, as the IOC will replace + # any blank spaces in `full_file_path` before returning the value. + write_dir = Path(date.strftime(dir_template).replace(" ", "_")) write_dir.mkdir(parents=True, exist_ok=True) file_template = "scan_{num:06d}_{uid}.hdf5" From ccca4d3c63669aeffc3e4c1868df6d564bfd1989 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Thu, 22 Feb 2024 03:59:53 -0500 Subject: [PATCH 47/56] Minor improvements --- scripts/run-act.sh | 2 ++ tests/conftest.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/run-act.sh b/scripts/run-act.sh index 768ad28..074a8c6 100644 --- a/scripts/run-act.sh +++ b/scripts/run-act.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -vxeuo pipefail + PYTHON_VERSION="${1:-3.11}" act -W .github/workflows/ci.yml -j checks --matrix python-version:"${PYTHON_VERSION}" diff --git a/tests/conftest.py b/tests/conftest.py index 9cf01ec..a605f72 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -109,7 +109,7 @@ def caproto_ioc_channel_types(wait=5): @pytest.fixture() def ophyd_channel_types(): dev = OphydChannelTypes(OPHYD_PV_PREFIX, name="ophyd_channel_type") - letters = iter(list(string.ascii_letters)) + letters = iter(string.ascii_letters) for cpt in sorted(dev.component_names): getattr(dev, cpt).put(next(letters)) return dev From 7d0e104cb63c737509efe214a3d7e486f429712c Mon Sep 17 00:00:00 2001 From: Hiran Wijesinghe Date: Fri, 23 Feb 2024 04:06:24 -0500 Subject: [PATCH 48/56] sanitize file path better * replaces unsupport characters with underscores _ * github artifact upload does not support the special chars: - Double quote " - Colon : - Less than < - Greater than > - Vertical bar | - Asterisk * - Question mark ? - Carriage return \r - Line feed \n * note that \r, \n are included in whitespace regex symbol \s --- src/srx_caproto_iocs/base.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/srx_caproto_iocs/base.py b/src/srx_caproto_iocs/base.py index 9fb1faf..570794d 100644 --- a/src/srx_caproto_iocs/base.py +++ b/src/srx_caproto_iocs/base.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re import textwrap import threading import time as ttime @@ -116,12 +117,15 @@ async def stage(self, instance, value): if value == StageStates.STAGED.value: # Steps: - # 1. Render 'write_dir' with datetime lib and replace any blank spaces with underscores. - # 2. Render 'file_name' with .format(). - # 3. Replace blank spaces with underscores. + # 1. Render 'write_dir' with datetime lib + # 2. Replace unsupported characters with underscores (sanitize). + # 3. Check if sanitized 'write_dir' exists + # 4. Render 'file_name' with .format(). + # 5. Replace unsupported characters with underscores. + sanitizer = re.compile(pattern=r"[\":<>|\*\?\s]") date = now(as_object=True) - write_dir = Path(date.strftime(self.write_dir.value).replace(" ", "_")) + write_dir = Path(sanitizer.sub("_", date.strftime(self.write_dir.value))) if not write_dir.exists(): msg = f"Path '{write_dir}' does not exist." print(msg) @@ -136,8 +140,7 @@ async def stage(self, instance, value): full_file_path = write_dir / file_name.format( num=self.frame_num.value, uid=uid, suid=uid[:8] ) - full_file_path = str(full_file_path) - full_file_path.replace(" ", "_") + full_file_path = sanitizer.sub("_", str(full_file_path)) print(f"{now()}: {full_file_path = }") From 571539ed508b78a71afd8fb122d96cc6d7027877 Mon Sep 17 00:00:00 2001 From: Hiran Wijesinghe Date: Fri, 23 Feb 2024 09:18:51 -0500 Subject: [PATCH 49/56] update `test_base_ophyd.py` to use the correct path sanitizer --- tests/test_base_ophyd.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_base_ophyd.py b/tests/test_base_ophyd.py index d052bb1..7e514bd 100644 --- a/tests/test_base_ophyd.py +++ b/tests/test_base_ophyd.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re import shutil import time as ttime import uuid @@ -25,9 +26,10 @@ def test_base_ophyd_templates( # We pre-create the test directory in advance as the IOC is not supposed to create one. # The assumption for the IOC is that the directory will exist before saving a file to that. - # We need to replace the blank spaces below for it to work, as the IOC will replace - # any blank spaces in `full_file_path` before returning the value. - write_dir = Path(date.strftime(dir_template).replace(" ", "_")) + # We need to substitute the unsupported characters below for it to work, as the IOC will do + # the same in `full_file_path` before returning the value. + sanitizer = re.compile(pattern=r"[\":<>|\*\?\s]") + write_dir = Path(sanitizer.sub("_", date.strftime(dir_template))) write_dir.mkdir(parents=True, exist_ok=True) file_template = "scan_{num:06d}_{uid}.hdf5" From af0f9861c060681dada00ffc66b0b7e36cbb8702 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Tue, 14 May 2024 18:46:21 -0400 Subject: [PATCH 50/56] Dev updates for the Zebra IOC --- scripts/run-caproto-ioc.sh | 5 +- scripts/run-caproto-zebra-ioc.sh | 7 ++ src/srx_caproto_iocs/base.py | 7 +- src/srx_caproto_iocs/zebra/caproto_ioc.py | 119 +++++++++++++++++++++- 4 files changed, 133 insertions(+), 5 deletions(-) create mode 100644 scripts/run-caproto-zebra-ioc.sh diff --git a/scripts/run-caproto-ioc.sh b/scripts/run-caproto-ioc.sh index 9cfaefe..808b8dd 100644 --- a/scripts/run-caproto-ioc.sh +++ b/scripts/run-caproto-ioc.sh @@ -3,7 +3,8 @@ set -vxeuo pipefail CAPROTO_IOC="${1:-srx_caproto_iocs.base}" - +DEFAULT_PREFIX="BASE:{{Dev:Save1}}:" +CAPROTO_IOC_PREFIX="${2:-${DEFAULT_PREFIX}}" # shellcheck source=/dev/null if [ -f "/etc/profile.d/epics.sh" ]; then . /etc/profile.d/epics.sh @@ -12,4 +13,4 @@ fi export EPICS_CAS_AUTO_BEACON_ADDR_LIST="no" export EPICS_CAS_BEACON_ADDR_LIST="${EPICS_CA_ADDR_LIST:-127.0.0.255}" -python -m "${CAPROTO_IOC}" --prefix="BASE:{{Dev:Save1}}:" --list-pvs +python -m "${CAPROTO_IOC}" --prefix="${CAPROTO_IOC_PREFIX}" --list-pvs diff --git a/scripts/run-caproto-zebra-ioc.sh b/scripts/run-caproto-zebra-ioc.sh new file mode 100644 index 0000000..1d4118c --- /dev/null +++ b/scripts/run-caproto-zebra-ioc.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -vxeuo pipefail + +SCRIPT_DIR="$(dirname "$0")" + +bash "${SCRIPT_DIR}/run-caproto-ioc.sh" srx_caproto_iocs.zebra.caproto_ioc "XF:05IDD-ES:1{{Dev:Zebra2}}:" diff --git a/src/srx_caproto_iocs/base.py b/src/srx_caproto_iocs/base.py index 570794d..6c3152b 100644 --- a/src/srx_caproto_iocs/base.py +++ b/src/srx_caproto_iocs/base.py @@ -104,8 +104,7 @@ async def queue(self, instance, async_lib): ) thread.start() - @stage.putter - async def stage(self, instance, value): + async def _stage(self, instance, value): """The stage method to perform preparation of a dataset to save the data.""" if ( instance.value in [True, StageStates.STAGED.value] @@ -150,6 +149,10 @@ async def stage(self, instance, value): return False + @stage.putter + async def stage(self, *args, **kwargs): + return await self._stage(*args, **kwargs) + def _get_current_dataset(self, frame): """The method to return a desired dataset. diff --git a/src/srx_caproto_iocs/zebra/caproto_ioc.py b/src/srx_caproto_iocs/zebra/caproto_ioc.py index f2ae288..5970327 100644 --- a/src/srx_caproto_iocs/zebra/caproto_ioc.py +++ b/src/srx_caproto_iocs/zebra/caproto_ioc.py @@ -1,20 +1,137 @@ from __future__ import annotations import textwrap +from pprint import pformat +from caproto.asyncio.client import Context from caproto.server import run, template_arg_parser from ..base import CaprotoSaveIOC, check_args +from ..utils import now + +# def export_nano_zebra_data(zebra, filepath, fastaxis): +# j = 0 +# while zebra.pc.data_in_progress.get() == 1: +# print("Waiting for zebra...") +# ttime.sleep(0.1) +# j += 1 +# if j > 10: +# print("THE ZEBRA IS BEHAVING BADLY CARRYING ON") +# break + +# time_d = zebra.pc.data.time.get() +# enc1_d = zebra.pc.data.enc1.get() +# enc2_d = zebra.pc.data.enc2.get() +# enc3_d = zebra.pc.data.enc3.get() + +# px = zebra.pc.pulse_step.get() +# if fastaxis == 'NANOHOR': +# # Add half pixelsize to correct encoder +# enc1_d = enc1_d + (px / 2) +# elif fastaxis == 'NANOVER': +# # Add half pixelsize to correct encoder +# enc2_d = enc2_d + (px / 2) +# elif fastaxis == 'NANOZ': +# # Add half pixelsize to correct encoder +# enc3_d = enc3_d + (px / 2) + +# size = (len(time_d),) +# with h5py.File(filepath, "w") as f: +# dset0 = f.create_dataset("zebra_time", size, dtype="f") +# dset0[...] = np.array(time_d) +# dset1 = f.create_dataset("enc1", size, dtype="f") +# dset1[...] = np.array(enc1_d) +# dset2 = f.create_dataset("enc2", size, dtype="f") +# dset2[...] = np.array(enc2_d) +# dset3 = f.create_dataset("enc3", size, dtype="f") +# dset3[...] = np.array(enc3_d) + + +# class ZebraPositionCaptureData(Device): +# """ +# Data arrays for the Zebra position capture function and their metadata. +# """ +# # Data arrays +# ... +# enc1 = Cpt(EpicsSignal, "PC_ENC1") # XF:05IDD-ES:1{Dev:Zebra2}:PC_ENC1 +# enc2 = Cpt(EpicsSignal, "PC_ENC2") # XF:05IDD-ES:1{Dev:Zebra2}:PC_ENC2 +# enc3 = Cpt(EpicsSignal, "PC_ENC3") # XF:05IDD-ES:1{Dev:Zebra2}:PC_ENC3 +# time = Cpt(EpicsSignal, "PC_TIME") # XF:05IDD-ES:1{Dev:Zebra2}:PC_TIME +# ... + +# class ZebraPositionCapture(Device): +# """ +# Signals for the position capture function of the Zebra +# """ + +# # Configuration settings and status PVs +# ... +# pulse_step = Cpt(EpicsSignalWithRBV, "PC_PULSE_STEP") # XF:05IDD-ES:1{Dev:Zebra2}:PC_PULSE_STEP +# ... +# data_in_progress = Cpt(EpicsSignalRO, "ARRAY_ACQ") # XF:05IDD-ES:1{Dev:Zebra2}:ARRAY_ACQ +# ... +# data = Cpt(ZebraPositionCaptureData, "") + +# nanoZebra = SRXZebra( +# "XF:05IDD-ES:1{Dev:Zebra2}:", name="nanoZebra", +# read_attrs=["pc.data.enc1", "pc.data.enc2", "pc.data.enc3", "pc.data.time"], +# ) class ZebraSaveIOC(CaprotoSaveIOC): """Zebra caproto save IOC.""" + # enc1 = Cpt(EpicsSignal, "PC_ENC1") + # enc2 = Cpt(EpicsSignal, "PC_ENC2") + # enc3 = Cpt(EpicsSignal, "PC_ENC3") + # enc4 = Cpt(EpicsSignal, "PC_ENC4") + + # data_in_progress = pvproperty() # + + # time_d = pvproperty() + # enc1_d = pvproperty() + # enc2_d = pvproperty() + # enc3_d = pvproperty() + # pulse_step = pvproperty() + + def __init__(self, *args, external_pvs=None, **kwargs): + """Init method. + + external_pvs : dict + a dictionary of external PVs with keys as human-readable names. + """ + super().__init__(*args, **kwargs) + self._external_pvs = external_pvs + + async def _stage(self, *args, **kwargs): + await super()._stage(*args, **kwargs) + client_context = Context() + res = {} + if self._external_pvs is not None: + pvobjects = await client_context.get_pvs(*self._external_pvs.values()) + for i, (name, pv) in enumerate(self._external_pvs.items()): # noqa: B007 + pvobject = pvobjects[i] + # print(f"{now()}: {pvobject = }") + ret = await pvobject.read() + # print(f"{now()}: {val.data}") + res[name] = {"data": ret.data, "shape": ret.data.shape} + print(f"{now()}:\n{pformat(res)}") + return True + if __name__ == "__main__": parser, split_args = template_arg_parser( default_prefix="", desc=textwrap.dedent(ZebraSaveIOC.__doc__) ) ioc_options, run_options = check_args(parser, split_args) - ioc = ZebraSaveIOC(**ioc_options) # TODO: pass libca IOC PVs of interest + + external_pvs = { + "pulse_step": "XF:05IDD-ES:1{Dev:Zebra2}:PC_PULSE_STEP", + "data_in_progress": "XF:05IDD-ES:1{Dev:Zebra2}:ARRAY_ACQ", + "enc1": "XF:05IDD-ES:1{Dev:Zebra2}:PC_ENC1", + "enc2": "XF:05IDD-ES:1{Dev:Zebra2}:PC_ENC2", + "enc3": "XF:05IDD-ES:1{Dev:Zebra2}:PC_ENC3", + "time": "XF:05IDD-ES:1{Dev:Zebra2}:PC_TIME", + } + + ioc = ZebraSaveIOC(external_pvs=external_pvs, **ioc_options) run(ioc.pvdb, **run_options) From 0afd370e6316d2858706f5f2ffaf3ad98128262e Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Tue, 14 May 2024 19:47:50 -0400 Subject: [PATCH 51/56] WIP: can save `enc1` 1-D data --- src/srx_caproto_iocs/base.py | 8 +++--- src/srx_caproto_iocs/utils.py | 32 ++++++++++++++++++++--- src/srx_caproto_iocs/zebra/caproto_ioc.py | 29 ++++++++++---------- 3 files changed, 48 insertions(+), 21 deletions(-) diff --git a/src/srx_caproto_iocs/base.py b/src/srx_caproto_iocs/base.py index 6c3152b..725f4ab 100644 --- a/src/srx_caproto_iocs/base.py +++ b/src/srx_caproto_iocs/base.py @@ -16,7 +16,7 @@ from ophyd import Device, EpicsSignal, EpicsSignalRO from ophyd.status import SubscriptionStatus -from .utils import now, save_hdf5 +from .utils import now, save_hdf5_1d class AcqStatuses(Enum): @@ -153,7 +153,7 @@ async def _stage(self, instance, value): async def stage(self, *args, **kwargs): return await self._stage(*args, **kwargs) - def _get_current_dataset(self, frame): + async def _get_current_dataset(self, frame): """The method to return a desired dataset. See https://scikit-image.org/docs/stable/auto_examples/data/plot_3d.html @@ -187,7 +187,7 @@ async def acquire(self, instance, value): # Delegate saving the resulting data to a blocking callback in a thread. payload = { "filename": self.full_file_path.value, - "data": self._get_current_dataset(frame=self.frame_num.value), + "data": await self._get_current_dataset(frame=self.frame_num.value), "uid": str(uuid.uuid4()), "timestamp": ttime.time(), "frame_number": self.frame_num.value, @@ -213,7 +213,7 @@ def saver(request_queue, response_queue): data = received["data"] frame_number = received["frame_number"] try: - save_hdf5(fname=filename, data=data, mode="a") + save_hdf5_1d(fname=filename, data=data, mode="x", group_path="enc1") print( f"{now()}: saved {frame_number=} {data.shape} data into:\n {filename}" ) diff --git a/src/srx_caproto_iocs/utils.py b/src/srx_caproto_iocs/utils.py index 750b480..fd17f4c 100644 --- a/src/srx_caproto_iocs/utils.py +++ b/src/srx_caproto_iocs/utils.py @@ -15,7 +15,33 @@ def now(as_object=False): return _now.isoformat() -def save_hdf5( +def save_hdf5_1d( + fname, + data, + group_path="data", + dtype="float32", + mode="x", +): + """The function to export the 1-D data to an HDF5 file. + + Check https://docs.h5py.org/en/stable/high/file.html#opening-creating-files for modes: + + r Readonly, file must exist (default) + r+ Read/write, file must exist + w Create file, truncate if exists + w- or x Create file, fail if exists + a Read/write if exists, create otherwise + """ + with h5py.File(fname, mode, libver="latest") as h5file_desc: + dataset = h5file_desc.create_dataset( + group_path, + data=data, + dtype=dtype, + ) + dataset.flush() + + +def save_hdf5_nd( fname, data, group_name="/entry", @@ -23,7 +49,7 @@ def save_hdf5( dtype="float32", mode="x", ): - """The function to export the data to an HDF5 file. + """The function to export the N-D data to an HDF5 file (N>1). Check https://docs.h5py.org/en/stable/high/file.html#opening-creating-files for modes: @@ -54,5 +80,5 @@ def save_hdf5( h5file_desc.swmr_mode = True dataset.resize((frame_num + 1, *frame_shape)) - dataset[frame_num, :, :] = data + dataset[frame_num, ...] = data dataset.flush() diff --git a/src/srx_caproto_iocs/zebra/caproto_ioc.py b/src/srx_caproto_iocs/zebra/caproto_ioc.py index 5970327..69b0403 100644 --- a/src/srx_caproto_iocs/zebra/caproto_ioc.py +++ b/src/srx_caproto_iocs/zebra/caproto_ioc.py @@ -1,7 +1,6 @@ from __future__ import annotations import textwrap -from pprint import pformat from caproto.asyncio.client import Context from caproto.server import run, template_arg_parser @@ -102,20 +101,22 @@ def __init__(self, *args, external_pvs=None, **kwargs): super().__init__(*args, **kwargs) self._external_pvs = external_pvs - async def _stage(self, *args, **kwargs): - await super()._stage(*args, **kwargs) + # async def _stage(self, *args, **kwargs): + # ret = await super()._stage(*args, **kwargs) + # return ret + + async def _get_current_dataset(self, frame, external_pv="enc1"): client_context = Context() - res = {} - if self._external_pvs is not None: - pvobjects = await client_context.get_pvs(*self._external_pvs.values()) - for i, (name, pv) in enumerate(self._external_pvs.items()): # noqa: B007 - pvobject = pvobjects[i] - # print(f"{now()}: {pvobject = }") - ret = await pvobject.read() - # print(f"{now()}: {val.data}") - res[name] = {"data": ret.data, "shape": ret.data.shape} - print(f"{now()}:\n{pformat(res)}") - return True + (pvobject,) = await client_context.get_pvs(self._external_pvs[external_pv]) + print(f"{pvobject = }") + # pvobject = pvobjects[0] + ret = await pvobject.read() + + dataset = ret.data + + print(f"{now()}:\n{dataset} {dataset.shape}") + + return dataset if __name__ == "__main__": From 6524861cb14e98cac913af1baa817a081d260749 Mon Sep 17 00:00:00 2001 From: Maksim Rakitin Date: Thu, 16 May 2024 13:29:49 -0400 Subject: [PATCH 52/56] Vendor acquire/saver methods from the base IOC in the Zebra one --- src/srx_caproto_iocs/base.py | 2 +- src/srx_caproto_iocs/zebra/caproto_ioc.py | 82 +++++++++++++++++++++-- 2 files changed, 77 insertions(+), 7 deletions(-) diff --git a/src/srx_caproto_iocs/base.py b/src/srx_caproto_iocs/base.py index 725f4ab..1bf4658 100644 --- a/src/srx_caproto_iocs/base.py +++ b/src/srx_caproto_iocs/base.py @@ -213,7 +213,7 @@ def saver(request_queue, response_queue): data = received["data"] frame_number = received["frame_number"] try: - save_hdf5_1d(fname=filename, data=data, mode="x", group_path="enc1") + save_hdf5_nd(fname=filename, data=data, mode="x", group_path="enc1") print( f"{now()}: saved {frame_number=} {data.shape} data into:\n {filename}" ) diff --git a/src/srx_caproto_iocs/zebra/caproto_ioc.py b/src/srx_caproto_iocs/zebra/caproto_ioc.py index 69b0403..153a9b8 100644 --- a/src/srx_caproto_iocs/zebra/caproto_ioc.py +++ b/src/srx_caproto_iocs/zebra/caproto_ioc.py @@ -118,6 +118,73 @@ async def _get_current_dataset(self, frame, external_pv="enc1"): return dataset + @acquire.putter + @no_reentry + async def acquire(self, instance, value): + """The acquire method to perform an individual acquisition of a data point.""" + if ( + value != AcqStatuses.ACQUIRING.value + # or self.stage.value not in [True, StageStates.STAGED.value] + ): + return False + + if ( + instance.value in [True, AcqStatuses.ACQUIRING.value] + and value == AcqStatuses.ACQUIRING.value + ): + print( + f"The device is already acquiring. Please wait until the '{AcqStatuses.IDLE.value}' status." + ) + return True + + await self.acquire.write(AcqStatuses.ACQUIRING.value) + + # Delegate saving the resulting data to a blocking callback in a thread. + payload = { + "filename": self.full_file_path.value, + "data": await self._get_current_dataset(frame=self.frame_num.value), + "uid": str(uuid.uuid4()), + "timestamp": ttime.time(), + "frame_number": self.frame_num.value, + } + + await self._request_queue.async_put(payload) + response = await self._response_queue.async_get() + + if response["success"]: + # Increment the counter only on a successful saving of the file. + await self.frame_num.write(self.frame_num.value + 1) + + # await self.acquire.write(AcqStatuses.IDLE.value) + + return False + + @staticmethod + def saver(request_queue, response_queue): + """The saver callback for threading-based queueing.""" + while True: + received = request_queue.get() + filename = received["filename"] + data = received["data"] + frame_number = received["frame_number"] + try: + save_hdf5_1d(fname=filename, data=data, mode="x", group_path="enc1") + print( + f"{now()}: saved {frame_number=} {data.shape} data into:\n {filename}" + ) + + success = True + error_message = "" + except Exception as exc: # pylint: disable=broad-exception-caught + success = False + error_message = exc + print( + f"Cannot save file {filename!r} due to the following exception:\n{exc}" + ) + + response = {"success": success, "error_message": error_message} + response_queue.put(response) + if __name__ == "__main__": parser, split_args = template_arg_parser( @@ -125,13 +192,16 @@ async def _get_current_dataset(self, frame, external_pv="enc1"): ) ioc_options, run_options = check_args(parser, split_args) + external_pv_prefix = ioc_options["prefix"].replace("{{", "{").replace("}}", "}") # "XF:05IDD-ES:1{Dev:Zebra2}:" + external_pvs = { - "pulse_step": "XF:05IDD-ES:1{Dev:Zebra2}:PC_PULSE_STEP", - "data_in_progress": "XF:05IDD-ES:1{Dev:Zebra2}:ARRAY_ACQ", - "enc1": "XF:05IDD-ES:1{Dev:Zebra2}:PC_ENC1", - "enc2": "XF:05IDD-ES:1{Dev:Zebra2}:PC_ENC2", - "enc3": "XF:05IDD-ES:1{Dev:Zebra2}:PC_ENC3", - "time": "XF:05IDD-ES:1{Dev:Zebra2}:PC_TIME", + "pulse_step": external_pv_prefix + "PC_PULSE_STEP", + "data_in_progress": external_pv_prefix + "ARRAY_ACQ", + "enc1": external_pv_prefix + "PC_ENC1", + "enc2": external_pv_prefix + "PC_ENC2", + "enc3": external_pv_prefix + "PC_ENC3", + "enc4": external_pv_prefix + "PC_ENC4", + "time": external_pv_prefix + "PC_TIME", } ioc = ZebraSaveIOC(external_pvs=external_pvs, **ioc_options) From adb96a3ef5fb2ef89a69a8adb8a8b4a99953ea41 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Fri, 17 May 2024 14:44:12 -0400 Subject: [PATCH 53/56] Prototyped IOC to accept the a dict of arrays to save - the ophyd/bluesky side should do the logic and call the saver when done --- src/srx_caproto_iocs/base.py | 2 +- src/srx_caproto_iocs/utils.py | 16 ++-- src/srx_caproto_iocs/zebra/caproto_ioc.py | 110 ++++++++++------------ 3 files changed, 58 insertions(+), 70 deletions(-) diff --git a/src/srx_caproto_iocs/base.py b/src/srx_caproto_iocs/base.py index 1bf4658..412d33c 100644 --- a/src/srx_caproto_iocs/base.py +++ b/src/srx_caproto_iocs/base.py @@ -16,7 +16,7 @@ from ophyd import Device, EpicsSignal, EpicsSignalRO from ophyd.status import SubscriptionStatus -from .utils import now, save_hdf5_1d +from .utils import now, save_hdf5_nd class AcqStatuses(Enum): diff --git a/src/srx_caproto_iocs/utils.py b/src/srx_caproto_iocs/utils.py index fd17f4c..737fe3d 100644 --- a/src/srx_caproto_iocs/utils.py +++ b/src/srx_caproto_iocs/utils.py @@ -15,10 +15,9 @@ def now(as_object=False): return _now.isoformat() -def save_hdf5_1d( +def save_hdf5_zebra( fname, data, - group_path="data", dtype="float32", mode="x", ): @@ -33,12 +32,13 @@ def save_hdf5_1d( a Read/write if exists, create otherwise """ with h5py.File(fname, mode, libver="latest") as h5file_desc: - dataset = h5file_desc.create_dataset( - group_path, - data=data, - dtype=dtype, - ) - dataset.flush() + for pvname, value in data.items(): + dataset = h5file_desc.create_dataset( + pvname, + data=value, + dtype=dtype, + ) + dataset.flush() def save_hdf5_nd( diff --git a/src/srx_caproto_iocs/zebra/caproto_ioc.py b/src/srx_caproto_iocs/zebra/caproto_ioc.py index 153a9b8..405fd8f 100644 --- a/src/srx_caproto_iocs/zebra/caproto_ioc.py +++ b/src/srx_caproto_iocs/zebra/caproto_ioc.py @@ -2,11 +2,10 @@ import textwrap -from caproto.asyncio.client import Context -from caproto.server import run, template_arg_parser +from caproto.server import pvproperty, run, template_arg_parser from ..base import CaprotoSaveIOC, check_args -from ..utils import now +from ..utils import now, save_hdf5_zebra # def export_nano_zebra_data(zebra, filepath, fastaxis): # j = 0 @@ -76,10 +75,36 @@ # read_attrs=["pc.data.enc1", "pc.data.enc2", "pc.data.enc3", "pc.data.time"], # ) +DEFAULT_MAX_LENGTH = 100_000 + class ZebraSaveIOC(CaprotoSaveIOC): """Zebra caproto save IOC.""" + enc1 = pvproperty( + value=0, + doc="enc1 data", + max_length=DEFAULT_MAX_LENGTH, + ) + + enc2 = pvproperty( + value=0, + doc="enc2 data", + max_length=DEFAULT_MAX_LENGTH, + ) + + enc3 = pvproperty( + value=0, + doc="enc3 data", + max_length=DEFAULT_MAX_LENGTH, + ) + + zebra_time = pvproperty( + value=0, + doc="zebra time", + max_length=DEFAULT_MAX_LENGTH, + ) + # enc1 = Cpt(EpicsSignal, "PC_ENC1") # enc2 = Cpt(EpicsSignal, "PC_ENC2") # enc3 = Cpt(EpicsSignal, "PC_ENC3") @@ -87,9 +112,8 @@ class ZebraSaveIOC(CaprotoSaveIOC): # data_in_progress = pvproperty() # + # time_d = pvproperty() - # enc1_d = pvproperty() - # enc2_d = pvproperty() - # enc3_d = pvproperty() + # enc1_d = pvproperty()TypeError: ZebraSaveIOC._get_current_dataset() got an unexpected keyword argument 'frame' + # pulse_step = pvproperty() def __init__(self, *args, external_pvs=None, **kwargs): @@ -105,60 +129,23 @@ def __init__(self, *args, external_pvs=None, **kwargs): # ret = await super()._stage(*args, **kwargs) # return ret - async def _get_current_dataset(self, frame, external_pv="enc1"): - client_context = Context() - (pvobject,) = await client_context.get_pvs(self._external_pvs[external_pv]) - print(f"{pvobject = }") - # pvobject = pvobjects[0] - ret = await pvobject.read() + async def _get_current_dataset( + self, *args, **kwargs + ): # , frame, external_pv="enc1"): + # client_context = Context() + # (pvobject,) = await client_context.get_pvs(self._external_pvs[external_pv]) + # print(f"{pvobject = }") + # # pvobject = pvobjects[0] + # ret = await pvobject.read() - dataset = ret.data + dataset = {} + for pvname in ["enc1", "enc2", "enc3", "zebra_time"]: + dataset[pvname] = getattr(self, pvname).value - print(f"{now()}:\n{dataset} {dataset.shape}") + print(f"{now()}:\n{dataset}") return dataset - @acquire.putter - @no_reentry - async def acquire(self, instance, value): - """The acquire method to perform an individual acquisition of a data point.""" - if ( - value != AcqStatuses.ACQUIRING.value - # or self.stage.value not in [True, StageStates.STAGED.value] - ): - return False - - if ( - instance.value in [True, AcqStatuses.ACQUIRING.value] - and value == AcqStatuses.ACQUIRING.value - ): - print( - f"The device is already acquiring. Please wait until the '{AcqStatuses.IDLE.value}' status." - ) - return True - - await self.acquire.write(AcqStatuses.ACQUIRING.value) - - # Delegate saving the resulting data to a blocking callback in a thread. - payload = { - "filename": self.full_file_path.value, - "data": await self._get_current_dataset(frame=self.frame_num.value), - "uid": str(uuid.uuid4()), - "timestamp": ttime.time(), - "frame_number": self.frame_num.value, - } - - await self._request_queue.async_put(payload) - response = await self._response_queue.async_get() - - if response["success"]: - # Increment the counter only on a successful saving of the file. - await self.frame_num.write(self.frame_num.value + 1) - - # await self.acquire.write(AcqStatuses.IDLE.value) - - return False - @staticmethod def saver(request_queue, response_queue): """The saver callback for threading-based queueing.""" @@ -166,12 +153,11 @@ def saver(request_queue, response_queue): received = request_queue.get() filename = received["filename"] data = received["data"] - frame_number = received["frame_number"] + # 'frame_number' is not used for this exporter. + frame_number = received["frame_number"] # noqa: F841 try: - save_hdf5_1d(fname=filename, data=data, mode="x", group_path="enc1") - print( - f"{now()}: saved {frame_number=} {data.shape} data into:\n {filename}" - ) + save_hdf5_zebra(fname=filename, data=data, mode="x") + print(f"{now()}: saved data into:\n {filename}") success = True error_message = "" @@ -192,7 +178,9 @@ def saver(request_queue, response_queue): ) ioc_options, run_options = check_args(parser, split_args) - external_pv_prefix = ioc_options["prefix"].replace("{{", "{").replace("}}", "}") # "XF:05IDD-ES:1{Dev:Zebra2}:" + external_pv_prefix = ( + ioc_options["prefix"].replace("{{", "{").replace("}}", "}") + ) # "XF:05IDD-ES:1{Dev:Zebra2}:" external_pvs = { "pulse_step": external_pv_prefix + "PC_PULSE_STEP", From d7ea796793c47deaafd90e68c3152977a3d7444a Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Fri, 17 May 2024 15:01:36 -0400 Subject: [PATCH 54/56] Add support for SIS scaler exporting --- src/srx_caproto_iocs/zebra/caproto_ioc.py | 47 ++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/src/srx_caproto_iocs/zebra/caproto_ioc.py b/src/srx_caproto_iocs/zebra/caproto_ioc.py index 405fd8f..152abc2 100644 --- a/src/srx_caproto_iocs/zebra/caproto_ioc.py +++ b/src/srx_caproto_iocs/zebra/caproto_ioc.py @@ -1,7 +1,9 @@ from __future__ import annotations import textwrap +from enum import Enum +from caproto import ChannelType from caproto.server import pvproperty, run, template_arg_parser from ..base import CaprotoSaveIOC, check_args @@ -78,9 +80,23 @@ DEFAULT_MAX_LENGTH = 100_000 +class DevTypes(Enum): + """Enum class for devices.""" + + ZEBRA = "zebra" + SCALER = "scaler" + + class ZebraSaveIOC(CaprotoSaveIOC): """Zebra caproto save IOC.""" + dev_type = pvproperty( + value=DevTypes.ZEBRA, + enum_strings=[x.value for x in DevTypes], + dtype=ChannelType.ENUM, + doc="Pick device type", + ) + enc1 = pvproperty( value=0, doc="enc1 data", @@ -105,6 +121,30 @@ class ZebraSaveIOC(CaprotoSaveIOC): max_length=DEFAULT_MAX_LENGTH, ) + i0 = pvproperty( + value=0, + doc="i0 data", + max_length=DEFAULT_MAX_LENGTH, + ) + + im = pvproperty( + value=0, + doc="im data", + max_length=DEFAULT_MAX_LENGTH, + ) + + it = pvproperty( + value=0, + doc="it data", + max_length=DEFAULT_MAX_LENGTH, + ) + + sis_time = pvproperty( + value=0, + doc="sis time", + max_length=DEFAULT_MAX_LENGTH, + ) + # enc1 = Cpt(EpicsSignal, "PC_ENC1") # enc2 = Cpt(EpicsSignal, "PC_ENC2") # enc3 = Cpt(EpicsSignal, "PC_ENC3") @@ -138,8 +178,13 @@ async def _get_current_dataset( # # pvobject = pvobjects[0] # ret = await pvobject.read() + if self.dev_type == DevTypes.ZEBRA: + pvnames = ["enc1", "enc2", "enc3", "zebra_time"] + else: + pvnames = ["i0", "im", "it", "sis_time"] + dataset = {} - for pvname in ["enc1", "enc2", "enc3", "zebra_time"]: + for pvname in pvnames: dataset[pvname] = getattr(self, pvname).value print(f"{now()}:\n{dataset}") From afdb4888d88e773636ee9c499747ecd4eda2dca4 Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Fri, 17 May 2024 15:09:48 -0400 Subject: [PATCH 55/56] Fix enum default value for the dev_type --- src/srx_caproto_iocs/zebra/caproto_ioc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/srx_caproto_iocs/zebra/caproto_ioc.py b/src/srx_caproto_iocs/zebra/caproto_ioc.py index 152abc2..04cb2db 100644 --- a/src/srx_caproto_iocs/zebra/caproto_ioc.py +++ b/src/srx_caproto_iocs/zebra/caproto_ioc.py @@ -91,7 +91,7 @@ class ZebraSaveIOC(CaprotoSaveIOC): """Zebra caproto save IOC.""" dev_type = pvproperty( - value=DevTypes.ZEBRA, + value=DevTypes.ZEBRA.value, enum_strings=[x.value for x in DevTypes], dtype=ChannelType.ENUM, doc="Pick device type", From 246d8857aefd1e70009f9f772f31d99db6101a9b Mon Sep 17 00:00:00 2001 From: Max Rakitin Date: Fri, 17 May 2024 15:14:27 -0400 Subject: [PATCH 56/56] Clean up irrelevant code --- src/srx_caproto_iocs/zebra/caproto_ioc.py | 60 +++++++++-------------- 1 file changed, 23 insertions(+), 37 deletions(-) diff --git a/src/srx_caproto_iocs/zebra/caproto_ioc.py b/src/srx_caproto_iocs/zebra/caproto_ioc.py index 04cb2db..1f39217 100644 --- a/src/srx_caproto_iocs/zebra/caproto_ioc.py +++ b/src/srx_caproto_iocs/zebra/caproto_ioc.py @@ -145,29 +145,14 @@ class ZebraSaveIOC(CaprotoSaveIOC): max_length=DEFAULT_MAX_LENGTH, ) - # enc1 = Cpt(EpicsSignal, "PC_ENC1") - # enc2 = Cpt(EpicsSignal, "PC_ENC2") - # enc3 = Cpt(EpicsSignal, "PC_ENC3") - # enc4 = Cpt(EpicsSignal, "PC_ENC4") + # def __init__(self, *args, external_pvs=None, **kwargs): + # """Init method. - # data_in_progress = pvproperty() # + - # time_d = pvproperty() - # enc1_d = pvproperty()TypeError: ZebraSaveIOC._get_current_dataset() got an unexpected keyword argument 'frame' - - # pulse_step = pvproperty() - - def __init__(self, *args, external_pvs=None, **kwargs): - """Init method. - - external_pvs : dict - a dictionary of external PVs with keys as human-readable names. - """ - super().__init__(*args, **kwargs) - self._external_pvs = external_pvs - - # async def _stage(self, *args, **kwargs): - # ret = await super()._stage(*args, **kwargs) - # return ret + # external_pvs : dict + # a dictionary of external PVs with keys as human-readable names. + # """ + # super().__init__(*args, **kwargs) + # self._external_pvs = external_pvs async def _get_current_dataset( self, *args, **kwargs @@ -223,19 +208,20 @@ def saver(request_queue, response_queue): ) ioc_options, run_options = check_args(parser, split_args) - external_pv_prefix = ( - ioc_options["prefix"].replace("{{", "{").replace("}}", "}") - ) # "XF:05IDD-ES:1{Dev:Zebra2}:" - - external_pvs = { - "pulse_step": external_pv_prefix + "PC_PULSE_STEP", - "data_in_progress": external_pv_prefix + "ARRAY_ACQ", - "enc1": external_pv_prefix + "PC_ENC1", - "enc2": external_pv_prefix + "PC_ENC2", - "enc3": external_pv_prefix + "PC_ENC3", - "enc4": external_pv_prefix + "PC_ENC4", - "time": external_pv_prefix + "PC_TIME", - } - - ioc = ZebraSaveIOC(external_pvs=external_pvs, **ioc_options) + # external_pv_prefix = ( + # ioc_options["prefix"].replace("{{", "{").replace("}}", "}") + # ) # "XF:05IDD-ES:1{Dev:Zebra2}:" + + # external_pvs = { + # "pulse_step": external_pv_prefix + "PC_PULSE_STEP", + # "data_in_progress": external_pv_prefix + "ARRAY_ACQ", + # "enc1": external_pv_prefix + "PC_ENC1", + # "enc2": external_pv_prefix + "PC_ENC2", + # "enc3": external_pv_prefix + "PC_ENC3", + # "enc4": external_pv_prefix + "PC_ENC4", + # "time": external_pv_prefix + "PC_TIME", + # } + + # ioc = ZebraSaveIOC(external_pvs=external_pvs, **ioc_options) + ioc = ZebraSaveIOC(**ioc_options) run(ioc.pvdb, **run_options)