87 make hdf datasets always be float64 unless they are pcap bits

src/pandablocks/connections.py

@@ -303,15 +303,22 @@ def _handle_header_body(self):
         if line == b"</header>":
             fields = []
             root = ET.fromstring(self._header)
+
             for field in root.find("fields"):
                 fields.append(
                     FieldCapture(
                         name=str(field.get("name")),
                         type=np.dtype(field.get("type")),
                         capture=str(field.get("capture")),
-                        scale=float(field.get("scale", 1)),
-                        offset=float(field.get("offset", 0)),
-                        units=str(field.get("units", "")),
+                        scale=float(scale)
+                        if (scale := field.get("scale")) is not None
+                        else None,
+                        offset=float(offset)
+                        if (offset := field.get("offset")) is not None
+                        else None,
+                        units=str(units)
+                        if (units := field.get("units")) is not None
+                        else None,
                     )
                 )
             data = root.find("data")
@@ -323,7 +330,7 @@ def _handle_header_body(self):
                 name, capture = SAMPLES_FIELD.rsplit(".", maxsplit=1)
                 fields.insert(
                     0,
-                    FieldCapture(name, np.dtype("uint32"), capture),
+                    FieldCapture(name, np.dtype("uint32"), capture, 1.0, 0.0, ""),
                 )
             self._frame_dtype = np.dtype(
                 [(f"{f.name}.{f.capture}", f.type) for f in fields]

src/pandablocks/hdf.py

@@ -111,16 +111,13 @@ def __init__(
     def create_dataset(self, field: FieldCapture, raw: bool):
         # Data written in a big stack, growing in that dimension
         assert self.hdf_file, "File not open yet"
-        if raw and (field.capture == "Mean" or field.scale != 1 or field.offset != 0):
-            # Processor outputs a float
-            dtype = np.dtype("float64")
-        else:
-            # No processor, datatype passed through
-            dtype = field.type
+
         dataset_name = self.capture_record_hdf_names.get(field.name, {}).get(
             field.capture, f"{field.name}.{field.capture}"
         )
 
+        dtype = field.raw_mode_dataset_dtype if raw else field.type
+
         return self.hdf_file.create_dataset(
             f"/{dataset_name}",
             dtype=dtype,
@@ -201,7 +198,7 @@ def mean_callable(data):
                 return (data[column_name] * field.scale / gate_duration) + field.offset
 
             return mean_callable
-        elif raw and (field.scale != 1 or field.offset != 0):
+        elif raw and not field.is_pcap_bits and (field.scale != 1 or field.offset != 0):
             return lambda data: data[column_name] * field.scale + field.offset
         else:
             return lambda data: data[column_name]

src/pandablocks/responses.py

@@ -223,21 +223,43 @@ class EndReason(Enum):
 class FieldCapture:
     """Information about a field that is being captured
 
+    If scale, offset, and units are all `None`, then the field is a
+    ``PCAP.BITS``.
+
     Attributes:
         name: Name of captured field
         type: Numpy data type of the field as transmitted
         capture: Value of CAPTURE field used to enable this field
-        scale: Scaling factor, default 1.0
-        offset: Offset, default 0.0
-        units: Units string, default ""
+        scale: Scaling factor
+        offset: Offset
+        units: Units string
     """
 
     name: str
     type: np.dtype
     capture: str
-    scale: float = 1.0
-    offset: float = 0.0
-    units: str = ""
+    scale: Optional[float]
+    offset: Optional[float]
+    units: Optional[str]
+
+    @property
+    def raw_mode_dataset_dtype(self) -> np.dtype:
+        """We use double for all dtypes, unless the field is a PCAP.BITS."""
+
+        if self.is_pcap_bits:
+            return self.type
+
+        if None in (self.scale, self.offset, self.units):
+            raise ValueError(
+                "If any of `scale`, `offset`, or `units` is set, all must be set"
+            )
+
+        return np.dtype("float64")
+
+    @property
+    def is_pcap_bits(self) -> bool:
+        """Return True if this field is a PCAP.BITS field"""
+        return self.scale is None and self.offset is None and self.units is None
 
 
 class Data:
@@ -289,8 +311,8 @@ class FrameData(Data):
         ...       (2, 12)],
         ...      dtype=[('COUNTER1.OUT.Value', '<f8'), ('COUNTER2.OUT.Value', '<f8')])
         >>> fdata = FrameData(data)
-        >>> fdata.data[0] # Row view
-        (0., 10.)
+        >>> (fdata.data[0]['COUNTER1.OUT.Value'], fdata.data[0]['COUNTER2.OUT.Value'])
+        (np.float64(0.0), np.float64(10.0))
         >>> fdata.column_names # Column names
         ('COUNTER1.OUT.Value', 'COUNTER2.OUT.Value')
         >>> fdata.data['COUNTER1.OUT.Value'] # Column view

src/pandablocks/utils.py

@@ -62,7 +62,9 @@ def words_to_table(
 
         if field_info.subtype == "int":
             # First convert from 2's complement to offset, then add in offset.
-            temp = (value ^ (1 << (bit_length - 1))) + (-1 << (bit_length - 1))
+            temp = (value.astype(np.int64) ^ (1 << (bit_length - 1))) + (
+                -1 << (bit_length - 1)
+            )
             packing_value = temp.astype(np.int32)
         elif field_info.subtype == "enum" and convert_enum_indices:
             assert field_info.labels, f"Enum field {field_name} has no labels"

tests/conftest.py

@@ -88,9 +88,9 @@ def overrun_dump():
         name="PCAP.BITS2",
         type=np.dtype("uint32"),
         capture="Value",
-        scale=1,
-        offset=0,
-        units="",
+        scale=None,
+        offset=None,
+        units=None,
     ),
     FieldCapture(
         name="COUNTER1.OUT",

tests/test_hdf.py

@@ -61,6 +61,48 @@ def __init__(self):
         stop_pipeline(pipeline)
 
 
+def test_field_capture_pcap_bits():
+    pcap_bits_frame_data = FieldCapture(
+        name="PCAP.BITS",
+        type=np.dtype("uint32"),
+        capture="Value",
+        scale=None,
+        offset=None,
+        units=None,
+    )
+
+    assert pcap_bits_frame_data.is_pcap_bits
+    assert pcap_bits_frame_data.raw_mode_dataset_dtype is np.dtype("uint32")
+
+    some_other_frame_data = FieldCapture(
+        name="some_other_frame_data",
+        type=np.dtype("uint32"),
+        capture="Value",
+        scale=1.0,
+        offset=0.0,
+        units="",
+    )
+
+    assert not some_other_frame_data.is_pcap_bits
+    assert some_other_frame_data.raw_mode_dataset_dtype is np.dtype("float64")
+
+    malformed_frame_data = FieldCapture(
+        name="malformed_frame_data",
+        type=np.dtype("uint32"),
+        capture="Value",
+        scale=None,
+        offset=0.0,
+        units="",
+    )
+
+    assert not some_other_frame_data.is_pcap_bits
+    with pytest.raises(
+        ValueError,
+        match="If any of `scale`, `offset`, or `units` is set, all must be set",
+    ):
+        assert malformed_frame_data.raw_mode_dataset_dtype is np.dtype("float64")
+
+
 @pytest.mark.parametrize(
     "capture_record_hdf_names,expected_names",
     [