diff --git a/docs/tutorials/commandline-hdf.md b/docs/tutorials/commandline-hdf.md index 7c835620c..33ca58210 100644 --- a/docs/tutorials/commandline-hdf.md +++ b/docs/tutorials/commandline-hdf.md @@ -86,6 +86,41 @@ $ h5diff /tmp/panda-capture-1.h5 /tmp/panda-capture-2.h5 $ h5diff /tmp/panda-capture-1.h5 /tmp/panda-capture-3.h5 ``` +## Absolute timestamps + +Starting with v3.0, PandABox firmware supports absolute timestamping of +the collected data. PandA still collects relative timestamps for +individual data points that are saved as arrays to HDF5 file. In addition, +the absolute timestamp for the start of the measurement is saved to HDF5 +file and can be used to convert relative timestamps to absolute timestamps + +The absolute timestamp is saved as a set of attributes of the root group of +the HDF5 file. The attributes are optional and set only if the respective +parameters were captured by PandABox and received by the IOC. The following +attributes are used: + +- ``arm_time`` - the time when the Panda (PCAP block) was armed, saved as + a string in the ISO 8601 UTC format. This parameter is mostly used for + debugging. + +- ``start_time`` - the start time (PCAP block is armed and enabled) of + the measurement in seconds since the epoch, saved as a string in the + ISO 8601 UTC format. Uses hardware provided timestamp (e.g. PTP or MRF) if available, + falling back to the system timestamp. + +- ``hw_time_offset_ns`` - the offset in nanoseconds (*int64*) that should be added to + to ``start_time`` to get back to the system timestamp. The attribute is + present only if Panda is configured to use hardware-based absolute timestamps + (PTP or MRF). + +The following code may be used to read the absolute timestamp from the HDF5 file. +Use ``pandas.Timestamp`` object if nanosecond accuracy is required (standard +``dataframe`` object is limited to microsecond precision). + +```{literalinclude} ../../examples/load_abs_timestamps.py +``` + + ## Collecting more data faster The test data is produced by a SEQ Block, configured to produce a high level diff --git a/examples/load_abs_timestamps.py b/examples/load_abs_timestamps.py new file mode 100644 index 000000000..4b2dc4061 --- /dev/null +++ b/examples/load_abs_timestamps.py @@ -0,0 +1,29 @@ +import sys + +import h5py +import pandas as pd + +if __name__ == "__main__": + with h5py.File(sys.argv[1], "r") as f: + arm_time = f.attrs.get("arm_time", None) + start_time = f.attrs.get("start_time", None) + hw_time_offset_ns = f.attrs.get("hw_time_offset_ns", None) + + print(f"Arm time: {arm_time!r}") + print(f"Start time: {start_time!r}") + print(f"Hardware time offset: {hw_time_offset_ns!r} ns") + + if start_time: + # Compute and print the start time that includes the offset + ts_start = pd.Timestamp(start_time) + if hw_time_offset_ns: + ts_start += pd.Timedelta(nanoseconds=hw_time_offset_ns) + print(f"Start time (system clock instead of hardware clock): {ts_start}") + + +# Expected output: +# +# Arm time: '2024-03-05T20:27:12.607841574Z' +# Start time: '2024-03-05T20:27:12.605729480Z' +# Hardware time offset: 2155797 ns +# Start TS including the offset: 2024-03-05T20:27:08.605729480Z diff --git a/src/pandablocks/connections.py b/src/pandablocks/connections.py index 0e2cc8294..2f6440ec2 100644 --- a/src/pandablocks/connections.py +++ b/src/pandablocks/connections.py @@ -328,12 +328,21 @@ def _handle_header_body(self): self._frame_dtype = np.dtype( [(f"{f.name}.{f.capture}", f.type) for f in fields] ) + + try: + hw_time_offset_ns = np.int64(data.get("hw_time_offset_ns", "")) + except ValueError: + hw_time_offset_ns = None + yield StartData( fields=fields, missed=int(data.get("missed")), process=str(data.get("process")), format=str(data.get("format")), sample_bytes=sample_bytes, + arm_time=data.get("arm_time", None), + start_time=data.get("start_time", None), + hw_time_offset_ns=hw_time_offset_ns, ) self._next_handler = self._handle_header_end diff --git a/src/pandablocks/hdf.py b/src/pandablocks/hdf.py index 9a6a0fe87..00d47b73e 100644 --- a/src/pandablocks/hdf.py +++ b/src/pandablocks/hdf.py @@ -119,6 +119,15 @@ def open_file(self, data: StartData): raw = data.process == "Raw" self.datasets = [self.create_dataset(field, raw) for field in data.fields] self.hdf_file.swmr_mode = True + + # Save parameters + if data.arm_time is not None: + self.hdf_file.attrs["arm_time"] = data.arm_time + if data.start_time is not None: + self.hdf_file.attrs["start_time"] = data.start_time + if data.hw_time_offset_ns is not None: + self.hdf_file.attrs["hw_time_offset_ns"] = data.hw_time_offset_ns + logging.info( f"Opened '{self.file_path}' with {data.sample_bytes} byte samples " f"stored in {len(self.datasets)} datasets" diff --git a/src/pandablocks/responses.py b/src/pandablocks/responses.py index 39b9defe6..a68bb2984 100644 --- a/src/pandablocks/responses.py +++ b/src/pandablocks/responses.py @@ -266,6 +266,9 @@ class StartData(Data): process: str format: str sample_bytes: int + arm_time: Optional[str] + start_time: Optional[str] + hw_time_offset_ns: Optional[int] @dataclass diff --git a/tests/conftest.py b/tests/conftest.py index 1c77d6ad8..1ed7d365b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -42,6 +42,26 @@ def fast_dump(): yield chunked_read(f, 500) +@pytest_asyncio.fixture +def fast_dump_with_extra_header_params(): + """ + Add header parameters to `fast_dump.bin` binary stream. The fixture + is used for testing absolute timing parameters passed in the header. + """ + + def fast_dump(extra_header_params): + param_to_replace = 'sample_bytes="52"' + params = " ".join(f'{k}="{v}"' for k, v in extra_header_params.items()) + params = " ".join([param_to_replace, params]) if params else param_to_replace + with open(Path(__file__).parent / "data_dumps/fast_dump.bin", "rb") as f: + # Simulate larger chunked read + data = chunked_read(f, 500) + for buffer in data: + yield buffer.replace(param_to_replace.encode(), params.encode()) + + return fast_dump + + @pytest_asyncio.fixture def raw_dump(): with open(Path(__file__).parent / "data_dumps/raw_dump.bin", "rb") as f: @@ -150,7 +170,7 @@ def __eq__(self, o): def slow_dump_expected(): yield [ ReadyData(), - StartData(DUMP_FIELDS, 0, "Scaled", "Framed", 52), + StartData(DUMP_FIELDS, 0, "Scaled", "Framed", 52, None, None, None), FrameData(Rows([0, 1, 1, 3, 5.6e-08, 1, 2])), FrameData(Rows([8, 2, 2, 6, 1.000000056, 2, 4])), FrameData(Rows([0, 3, 3, 9, 2.000000056, 3, 6])), @@ -164,7 +184,7 @@ def slow_dump_expected(): def fast_dump_expected(): yield [ ReadyData(), - StartData(DUMP_FIELDS, 0, "Scaled", "Framed", 52), + StartData(DUMP_FIELDS, 0, "Scaled", "Framed", 52, None, None, None), FrameData( Rows( [0, 1, 1, 3, 5.6e-08, 1, 2], diff --git a/tests/test_asyncio.py b/tests/test_asyncio.py index fd1881eda..dc54e1bbf 100644 --- a/tests/test_asyncio.py +++ b/tests/test_asyncio.py @@ -1,4 +1,5 @@ import asyncio +import copy import pytest @@ -50,6 +51,51 @@ async def test_asyncio_data( assert fast_dump_expected == events +@pytest.mark.parametrize( + "timing_params", + [ + {}, + { + "arm_time": "2024-03-05T20:27:12.607841574Z", + "start_time": "2024-03-05T20:27:12.608875498Z", + }, + { + "arm_time": "2024-03-05T20:27:12.607841574Z", + "start_time": "2024-03-05T20:27:12.608875498Z", + "hw_time_offset_ns": 100555, + }, + ], +) +async def test_asyncio_data_with_abs_timing( + dummy_server_async, + fast_dump_with_extra_header_params, + fast_dump_expected, + timing_params, +): + """ + The test for handling of `arm_time`, `start_time` and `hw_time_offset_ns` + parameters passed in the header. The test is reusing the existing `fast_dump` + and `fast_dump_expected` by adding timing parameters to the header in + the binary stream and replacing the expected `StartData` attributes with + the expected values. + """ + dummy_server_async.data = fast_dump_with_extra_header_params(timing_params) + events = [] + async with AsyncioClient("localhost") as client: + async for data in client.data(frame_timeout=1): + events.append(data) + if len(events) == len(fast_dump_expected): + break + fast_dump_expected = list(fast_dump_expected) + + # Replace attributes in `StartData` with the expected values + fast_dump_expected[1] = copy.deepcopy(fast_dump_expected[1]) + for attr_name in timing_params: + setattr(fast_dump_expected[1], attr_name, timing_params[attr_name]) + + assert fast_dump_expected == events + + async def test_asyncio_data_timeout(dummy_server_async, fast_dump): dummy_server_async.data = fast_dump async with AsyncioClient("localhost") as client: diff --git a/tests/test_hdf.py b/tests/test_hdf.py index d0981c71a..362e8da48 100644 --- a/tests/test_hdf.py +++ b/tests/test_hdf.py @@ -3,11 +3,7 @@ import numpy as np -from pandablocks.hdf import ( - Pipeline, - create_default_pipeline, - stop_pipeline, -) +from pandablocks.hdf import Pipeline, create_default_pipeline, stop_pipeline from pandablocks.responses import EndData, EndReason, FieldCapture, FrameData, StartData @@ -44,6 +40,9 @@ def __init__(self): "Scaled", "Framed", 52, + "2024-03-05T20:27:12.607841574Z", + "2024-03-05T20:27:12.608875498Z", + 100555, ), ) pipeline[0].queue.put_nowait(