From c509dfdd68c3c790898a54adeb3e54bdfa934b61 Mon Sep 17 00:00:00 2001 From: Lin Zhu Date: Tue, 23 May 2023 12:58:07 +0200 Subject: [PATCH 1/6] Add seq_table2cmd func - convert one seq talbe line into a command to send --- pandablocks/utils.py | 65 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 pandablocks/utils.py diff --git a/pandablocks/utils.py b/pandablocks/utils.py new file mode 100644 index 000000000..bdc6e7da4 --- /dev/null +++ b/pandablocks/utils.py @@ -0,0 +1,65 @@ + + +def seq_table2cmd(repeats: int, trigger: int, position: int, time1: int, phase1: dict, time2: int, phase2: dict): + """ + Function to encode (repeats, trigger, phase1, phase2) into 32bit integer (code). + + Return a list of str [code, position, time1, time2] to send to pandabox sequencer (SEQ) + table. + + Example: + ```python + keys = ["a", "b", "c", "d", "e", "f"] + repeats = 1 + time1 = 0 + position = 0 + trigger = "Immediate" + phase1 = {key: False for key in keys} + time2 = 0 + phase2 = {key: False if key != "a" else True for key in keys} + pandabox.send( + Put( + "SEQ1.TABLE",seq_table( + repeats, trigger, position, time1, phase2, time2, phase2 + ), + ) + ) + ``` + """ + trigger_options = { + "Immediate": 0, + "bita=0": 1, + "bita=1": 2, + "bitb=0": 3, + "bitb=1": 4, + "bitc=0": 5, + "bitc=1": 6, + "posa>=position": 7, + "posa<=position": 8, + "posb>=position": 9, + "posb<=position": 10, + "posc>=position": 11, + "posc<=position": 12, + } + + # _b binary code + repeats_b = "{0:016b}".format(repeats) # 16 bits + trigger_b = "{0:04b}".format(trigger_options[trigger]) # 4 bits (17-20) + phase1_b = "" + for key, value in sorted(phase1.items()): # 6 bits (a-f) + phase1_b = "1" + phase1_b if value else "0" + phase1_b + phase2_b = "" + for key, value in sorted(phase2.items()): # 6 bits (a-f) + phase2_b = "1" + phase2_b if value else "0" + phase2_b + code_b = phase2_b + phase1_b + trigger_b + repeats_b # 32 bits code + code = int(code_b, 2) + + # a table line = [code position time1 time2] + pos_cmd = [ + f"{code:d}", + f"{int(position):d}", + f"{int(time1):d}", + f"{int(time2):d}", + ] + + return pos_cmd From 39af91891912018634d448d1fd1a3ce5d3f6548d Mon Sep 17 00:00:00 2001 From: Eva Lott Date: Tue, 13 Jun 2023 09:40:04 +0100 Subject: [PATCH 2/6] Added functions to convert to and from column indexed tables --- pandablocks/utils.py | 198 ++++++++++++++++++++++++++++++------------- pyproject.toml | 2 +- tests/test_utils.py | 176 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 314 insertions(+), 62 deletions(-) create mode 100644 tests/test_utils.py diff --git a/pandablocks/utils.py b/pandablocks/utils.py index bdc6e7da4..14ab33708 100644 --- a/pandablocks/utils.py +++ b/pandablocks/utils.py @@ -1,65 +1,141 @@ +from typing import Dict, List, Sequence, Union +import numpy as np +import numpy.typing as npt -def seq_table2cmd(repeats: int, trigger: int, position: int, time1: int, phase1: dict, time2: int, phase2: dict): +from pandablocks.responses import TableFieldInfo + +UnpackedArray = Union[ + npt.NDArray[np.int32], npt.NDArray[np.uint8], npt.NDArray[np.uint16] +] + + +def words_to_table( + words: Sequence[str], table_field_info: TableFieldInfo +) -> Dict[str, List]: + """Unpacks the given `packed` data based on the fields provided. + Returns the unpacked data in {column_name: column_data} column-indexed format + + Args: + words: The list of data for this table, from PandA. Each item is + expected to be the string representation of a uint32. + table_fields_info: The info for tables, containing the number of words per row, + and the bit information for fields. + Returns: + unpacked: A dict of lists, one item per field. """ - Function to encode (repeats, trigger, phase1, phase2) into 32bit integer (code). - - Return a list of str [code, position, time1, time2] to send to pandabox sequencer (SEQ) - table. - - Example: - ```python - keys = ["a", "b", "c", "d", "e", "f"] - repeats = 1 - time1 = 0 - position = 0 - trigger = "Immediate" - phase1 = {key: False for key in keys} - time2 = 0 - phase2 = {key: False if key != "a" else True for key in keys} - pandabox.send( - Put( - "SEQ1.TABLE",seq_table( - repeats, trigger, position, time1, phase2, time2, phase2 - ), - ) - ) - ``` + + row_words = table_field_info.row_words + data = np.array(words, dtype=np.uint32) + # Convert 1-D array into 2-D, one row element per row in the PandA table + data = data.reshape(len(data) // row_words, row_words) + packed = data.T + + # Ensure fields are in bit-order + table_fields = dict( + sorted( + table_field_info.fields.items(), + key=lambda item: item[1].bit_low, + ) + ) + + unpacked: Dict[str, List] = {} + + for field_name, field_info in table_fields.items(): + offset = field_info.bit_low + bit_length = field_info.bit_high - field_info.bit_low + 1 + + # The word offset indicates which column this field is in + # (column is exactly one 32-bit word) + word_offset = offset // 32 + + # bit offset is location of field inside the word + bit_offset = offset & 0x1F + + # Mask to remove every bit that isn't in the range we want + mask = (1 << bit_length) - 1 + + value: UnpackedArray = (packed[word_offset] >> bit_offset) & mask + + if field_info.subtype == "int": + # First convert from 2's complement to offset, then add in offset. + value = (value ^ (1 << (bit_length - 1))) + (-1 << (bit_length - 1)) + value = value.astype(np.int32) + else: + # Use shorter types, as these are used in waveform creation + if bit_length <= 8: + value = value.astype(np.uint8) + elif bit_length <= 16: + value = value.astype(np.uint16) + + value_list = value.tolist() + # Convert back to label from integer + if field_info.labels: + value_list = [field_info.labels[x] for x in value_list] + + unpacked.update({field_name: value_list}) + + return unpacked + + +def table_to_words( + table: Dict[str, Sequence], table_field_info: TableFieldInfo +) -> List[str]: + """Pack the records based on the field definitions into the format PandA expects + for table writes. + + Args: + row_words: The number of 32-bit words per row + table_fields_info: The info for tables, containing the number of words per row, + and the bit information for fields. + Returns: + List[str]: The list of data ready to be sent to PandA """ - trigger_options = { - "Immediate": 0, - "bita=0": 1, - "bita=1": 2, - "bitb=0": 3, - "bitb=1": 4, - "bitc=0": 5, - "bitc=1": 6, - "posa>=position": 7, - "posa<=position": 8, - "posb>=position": 9, - "posb<=position": 10, - "posc>=position": 11, - "posc<=position": 12, - } - - # _b binary code - repeats_b = "{0:016b}".format(repeats) # 16 bits - trigger_b = "{0:04b}".format(trigger_options[trigger]) # 4 bits (17-20) - phase1_b = "" - for key, value in sorted(phase1.items()): # 6 bits (a-f) - phase1_b = "1" + phase1_b if value else "0" + phase1_b - phase2_b = "" - for key, value in sorted(phase2.items()): # 6 bits (a-f) - phase2_b = "1" + phase2_b if value else "0" + phase2_b - code_b = phase2_b + phase1_b + trigger_b + repeats_b # 32 bits code - code = int(code_b, 2) - - # a table line = [code position time1 time2] - pos_cmd = [ - f"{code:d}", - f"{int(position):d}", - f"{int(time1):d}", - f"{int(time2):d}", - ] - - return pos_cmd + row_words = table_field_info.row_words + + # Ensure fields are in bit-order + table_fields = dict( + sorted( + table_field_info.fields.items(), + key=lambda item: item[1].bit_low, + ) + ) + + # Iterate over the zipped fields and their associated records to construct the + # packed array. + packed = None + + for column_name, column in table.items(): + field_details = table_fields[column_name] + if field_details.labels: + # Must convert the list of ints into strings + column = [field_details.labels.index(x) for x in column] + + # PandA always handles tables in uint32 format + column_value = np.uint32(np.array(column)) + + if packed is None: + # Create 1-D array sufficiently long to exactly hold the entire table + packed = np.zeros((len(column), row_words), dtype=np.uint32) + else: + assert len(packed) == len(column), ( + f"Table record {column_name} has mismatched length " + "compared to other records, cannot pack data" + ) + + offset = field_details.bit_low + + # The word offset indicates which column this field is in + # (each column is one 32-bit word) + word_offset = offset // 32 + # bit offset is location of field inside the word + bit_offset = offset & 0x1F + + # Slice to get the column to apply the values to. + # bit shift the value to the relevant bits of the word + packed[:, word_offset] |= column_value << bit_offset + + assert isinstance(packed, np.ndarray) # Squash mypy warning + + # 2-D array -> 1-D array -> list[int] -> list[str] + return [str(x) for x in packed.flatten().tolist()] diff --git a/pyproject.toml b/pyproject.toml index e6e5a15b8..64253b60b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,4 @@ [build-system] # Pin versions compatible with dls-python3 for reproducible wheels requires = ["setuptools==44.1.1", "wheel==0.33.1"] -build-backend = "setuptools.build_meta" \ No newline at end of file +build-backend = "setuptools.build_meta" diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 000000000..9357f2165 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,176 @@ +from typing import Dict + +import pytest + +from pandablocks.responses import TableFieldDetails, TableFieldInfo +from pandablocks.utils import table_to_words, words_to_table + + +@pytest.fixture +def table_fields() -> Dict[str, TableFieldDetails]: + """Table field definitions, taken from a SEQ.TABLE instance. + Associated with table_data and table_field_info fixtures""" + return { + "REPEATS": TableFieldDetails( + subtype="uint", + bit_low=0, + bit_high=15, + description="Number of times the line will repeat", + labels=None, + ), + "TRIGGER": TableFieldDetails( + subtype="enum", + bit_low=16, + bit_high=19, + description="The trigger condition to start the phases", + labels=[ + "Immediate", + "BITA=0", + "BITA=1", + "BITB=0", + "BITB=1", + "BITC=0", + "BITC=1", + "POSA>=POSITION", + "POSA<=POSITION", + "POSB>=POSITION", + "POSB<=POSITION", + "POSC>=POSITION", + "POSC<=POSITION", + ], + ), + "POSITION": TableFieldDetails( + subtype="int", + bit_low=32, + bit_high=63, + description="The position that can be used in trigger condition", + labels=None, + ), + "TIME1": TableFieldDetails( + subtype="uint", + bit_low=64, + bit_high=95, + description="The time the optional phase 1 should take", + labels=None, + ), + "OUTA1": TableFieldDetails( + subtype="uint", + bit_low=20, + bit_high=20, + description="Output A value during phase 1", + labels=None, + ), + "OUTB1": TableFieldDetails( + subtype="uint", + bit_low=21, + bit_high=21, + description="Output B value during phase 1", + labels=None, + ), + "OUTC1": TableFieldDetails( + subtype="uint", + bit_low=22, + bit_high=22, + description="Output C value during phase 1", + labels=None, + ), + "OUTD1": TableFieldDetails( + subtype="uint", + bit_low=23, + bit_high=23, + description="Output D value during phase 1", + labels=None, + ), + "OUTE1": TableFieldDetails( + subtype="uint", + bit_low=24, + bit_high=24, + description="Output E value during phase 1", + labels=None, + ), + "OUTF1": TableFieldDetails( + subtype="uint", + bit_low=25, + bit_high=25, + description="Output F value during phase 1", + labels=None, + ), + "TIME2": TableFieldDetails( + subtype="uint", + bit_low=96, + bit_high=127, + description="The time the mandatory phase 2 should take", + labels=None, + ), + "OUTA2": TableFieldDetails( + subtype="uint", + bit_low=26, + bit_high=26, + description="Output A value during phase 2", + labels=None, + ), + "OUTB2": TableFieldDetails( + subtype="uint", + bit_low=27, + bit_high=27, + description="Output B value during phase 2", + labels=None, + ), + "OUTC2": TableFieldDetails( + subtype="uint", + bit_low=28, + bit_high=28, + description="Output C value during phase 2", + labels=None, + ), + "OUTD2": TableFieldDetails( + subtype="uint", + bit_low=29, + bit_high=29, + description="Output D value during phase 2", + labels=None, + ), + "OUTE2": TableFieldDetails( + subtype="uint", + bit_low=30, + bit_high=30, + description="Output E value during phase 2", + labels=None, + ), + "OUTF2": TableFieldDetails( + subtype="uint", + bit_low=31, + bit_high=31, + description="Output F value during phase 2", + labels=None, + ), + } + + +@pytest.fixture +def table_field_info(table_fields) -> TableFieldInfo: + """Table data associated with table_fields and table_data fixtures""" + return TableFieldInfo( + "table", None, "Sequencer table of lines", 16384, table_fields, 4 + ) + + +def test_table_to_words_and_words_to_table(table_field_info): + table = dict( + REPEATS=[1, 0], + TRIGGER=["Immediate", "Immediate"], + POSITION=[0, 1], + TIME1=[0, 1], + TIME2=[0, 1], + ) + + table["OUTA1"] = [False, True] + table["OUTA2"] = [True, False] + for key in "BCDEF": + table[f"OUT{key}1"] = table[f"OUT{key}2"] = [False, False] + + words = table_to_words(table, table_field_info) + assert words == ["67108865", "0", "0", "0", "1048576", "1", "1", "1"] + + # Verify the methods are inverse + assert words_to_table(words, table_field_info) == table From 9b2354705508dc83cc5732835b2ad27f990f86ac Mon Sep 17 00:00:00 2001 From: Eva Lott Date: Wed, 14 Jun 2023 14:09:08 +0100 Subject: [PATCH 3/6] Made PR corrections and added large number tests --- pandablocks/utils.py | 66 +++++++++++++++++--------------------------- tests/test_utils.py | 35 +++++++++++++++++------ 2 files changed, 52 insertions(+), 49 deletions(-) diff --git a/pandablocks/utils.py b/pandablocks/utils.py index 14ab33708..e7eec9c49 100644 --- a/pandablocks/utils.py +++ b/pandablocks/utils.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Sequence, Union +from typing import Dict, List, Sequence, Union, Iterable, cast import numpy as np import numpy.typing as npt @@ -6,13 +6,17 @@ from pandablocks.responses import TableFieldInfo UnpackedArray = Union[ - npt.NDArray[np.int32], npt.NDArray[np.uint8], npt.NDArray[np.uint16] + npt.NDArray[np.int32], + npt.NDArray[np.uint8], + npt.NDArray[np.uint16], + npt.NDArray[np.bool_], + Sequence[str], ] def words_to_table( - words: Sequence[str], table_field_info: TableFieldInfo -) -> Dict[str, List]: + words: Iterable[str], table_field_info: TableFieldInfo +) -> Dict[str, UnpackedArray]: """Unpacks the given `packed` data based on the fields provided. Returns the unpacked data in {column_name: column_data} column-indexed format @@ -31,17 +35,9 @@ def words_to_table( data = data.reshape(len(data) // row_words, row_words) packed = data.T - # Ensure fields are in bit-order - table_fields = dict( - sorted( - table_field_info.fields.items(), - key=lambda item: item[1].bit_low, - ) - ) + unpacked: Dict[str, UnpackedArray] = {} - unpacked: Dict[str, List] = {} - - for field_name, field_info in table_fields.items(): + for field_name, field_info in table_field_info.fields.items(): offset = field_info.bit_low bit_length = field_info.bit_high - field_info.bit_low + 1 @@ -55,31 +51,26 @@ def words_to_table( # Mask to remove every bit that isn't in the range we want mask = (1 << bit_length) - 1 - value: UnpackedArray = (packed[word_offset] >> bit_offset) & mask + value = (packed[word_offset] >> bit_offset) & mask if field_info.subtype == "int": # First convert from 2's complement to offset, then add in offset. - value = (value ^ (1 << (bit_length - 1))) + (-1 << (bit_length - 1)) - value = value.astype(np.int32) + packing_value = (value ^ (1 << (bit_length - 1))) + (-1 << (bit_length - 1)) + packing_value = value.astype(np.int32) + elif field_info.labels: + packing_value = [field_info.labels[x] for x in value] + elif bit_length == 1: + packing_value = value.astype(np.bool_) else: - # Use shorter types, as these are used in waveform creation - if bit_length <= 8: - value = value.astype(np.uint8) - elif bit_length <= 16: - value = value.astype(np.uint16) - - value_list = value.tolist() - # Convert back to label from integer - if field_info.labels: - value_list = [field_info.labels[x] for x in value_list] + packing_value = value - unpacked.update({field_name: value_list}) + unpacked.update({field_name: packing_value}) return unpacked def table_to_words( - table: Dict[str, Sequence], table_field_info: TableFieldInfo + table: Dict[str, Iterable], table_field_info: TableFieldInfo ) -> List[str]: """Pack the records based on the field definitions into the format PandA expects for table writes. @@ -93,20 +84,12 @@ def table_to_words( """ row_words = table_field_info.row_words - # Ensure fields are in bit-order - table_fields = dict( - sorted( - table_field_info.fields.items(), - key=lambda item: item[1].bit_low, - ) - ) - # Iterate over the zipped fields and their associated records to construct the # packed array. packed = None for column_name, column in table.items(): - field_details = table_fields[column_name] + field_details = table_field_info.fields[column_name] if field_details.labels: # Must convert the list of ints into strings column = [field_details.labels.index(x) for x in column] @@ -115,7 +98,10 @@ def table_to_words( column_value = np.uint32(np.array(column)) if packed is None: - # Create 1-D array sufficiently long to exactly hold the entire table + # Create 1-D array sufficiently long to exactly hold the entire table, cast + # to prevent type error, this will still work if column is another iterable + # e.g numpy array + column = cast(List, column) packed = np.zeros((len(column), row_words), dtype=np.uint32) else: assert len(packed) == len(column), ( @@ -135,7 +121,7 @@ def table_to_words( # bit shift the value to the relevant bits of the word packed[:, word_offset] |= column_value << bit_offset - assert isinstance(packed, np.ndarray) # Squash mypy warning + assert isinstance(packed, np.ndarray), "Table has no columns" # Squash mypy warning # 2-D array -> 1-D array -> list[int] -> list[str] return [str(x) for x in packed.flatten().tolist()] diff --git a/tests/test_utils.py b/tests/test_utils.py index 9357f2165..dcd1b6dbe 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,4 +1,4 @@ -from typing import Dict +from typing import Dict, List, Iterable, OrderedDict import pytest @@ -155,13 +155,22 @@ def table_field_info(table_fields) -> TableFieldInfo: ) -def test_table_to_words_and_words_to_table(table_field_info): - table = dict( +def ensure_matching_order(list1: List, list2: List): + old_index = 0 + for list1_element in list1: + new_index = list2.index(list1_element) + if new_index < old_index: + return False + old_index = new_index + + +def test_table_to_words_and_words_to_table(table_field_info: TableFieldInfo): + table: Dict[str, Iterable] = dict( REPEATS=[1, 0], TRIGGER=["Immediate", "Immediate"], - POSITION=[0, 1], - TIME1=[0, 1], - TIME2=[0, 1], + POSITION=[-20, 2**31 - 1], + TIME1=[12, 2**32 - 1], + TIME2=[32, 1], ) table["OUTA1"] = [False, True] @@ -170,7 +179,15 @@ def test_table_to_words_and_words_to_table(table_field_info): table[f"OUT{key}1"] = table[f"OUT{key}2"] = [False, False] words = table_to_words(table, table_field_info) - assert words == ["67108865", "0", "0", "0", "1048576", "1", "1", "1"] + # assert words == ["67108865", "0", "0", "0", "1048576", "1", "1", "1"] + output = words_to_table(words, table_field_info) + + # Test the correct keys are outputted + assert output.keys() == table.keys() + + # Check the items have been inserted in panda order + sorted_table = OrderedDict({key: table[key] for key in output.keys()}) + assert sorted_table != OrderedDict(table) - # Verify the methods are inverse - assert words_to_table(words, table_field_info) == table + # Check the values are the same + assert [(x, list(y)) for x, y in output.items()] == list(sorted_table.items()) From 458b977e5ccc7ca9a166f34db4d047412f136728 Mon Sep 17 00:00:00 2001 From: Eva Lott Date: Wed, 14 Jun 2023 14:37:56 +0100 Subject: [PATCH 4/6] Updated docstrings and added a reference to utils to the api.rst --- docs/reference/api.rst | 8 ++++++++ pandablocks/utils.py | 16 +++++++++------- tests/test_utils.py | 13 +++++++++++-- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/docs/reference/api.rst b/docs/reference/api.rst index d804c0b56..b9071f801 100644 --- a/docs/reference/api.rst +++ b/docs/reference/api.rst @@ -12,6 +12,7 @@ from code: - `pandablocks.asyncio`: An asyncio client that uses the control and data connections - `pandablocks.blocking`: A blocking client that uses the control and data connections - `pandablocks.hdf`: Some helpers for efficiently writing data responses to disk +- `pandablocks.utils`: General utility methods for use with pandablocks .. automodule:: pandablocks.commands @@ -82,3 +83,10 @@ from code: gives multi-CPU benefits without hitting the limit of the GIL. .. seealso:: `library-hdf`, `performance` + +.. automodule:: pandablocks.utils + + Utilities + --------- + + This package contains general methods for working with pandablocks. diff --git a/pandablocks/utils.py b/pandablocks/utils.py index e7eec9c49..38b5b6f6d 100644 --- a/pandablocks/utils.py +++ b/pandablocks/utils.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Sequence, Union, Iterable, cast +from typing import Dict, Iterable, List, Sequence, Union, cast import numpy as np import numpy.typing as npt @@ -21,12 +21,13 @@ def words_to_table( Returns the unpacked data in {column_name: column_data} column-indexed format Args: - words: The list of data for this table, from PandA. Each item is + words: An iterable of data for this table, from PandA. Each item is expected to be the string representation of a uint32. table_fields_info: The info for tables, containing the number of words per row, and the bit information for fields. Returns: - unpacked: A dict of lists, one item per field. + unpacked: A dict containing record information, where keys are field names + and values are numpy arrays of record values in that column. """ row_words = table_field_info.row_words @@ -72,13 +73,14 @@ def words_to_table( def table_to_words( table: Dict[str, Iterable], table_field_info: TableFieldInfo ) -> List[str]: - """Pack the records based on the field definitions into the format PandA expects + """Convert records based on the field definitions into the format PandA expects for table writes. Args: - row_words: The number of 32-bit words per row - table_fields_info: The info for tables, containing the number of words per row, - and the bit information for fields. + table: A dict containing record information, where keys are field names + and values are iterables of record values in that column. + table_field_info: The info for tables, containing the dict `fields` for + information on each field, and the number of words per row. Returns: List[str]: The list of data ready to be sent to PandA """ diff --git a/tests/test_utils.py b/tests/test_utils.py index dcd1b6dbe..b028df118 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Iterable, OrderedDict +from typing import Dict, Iterable, List, OrderedDict import pytest @@ -179,7 +179,16 @@ def test_table_to_words_and_words_to_table(table_field_info: TableFieldInfo): table[f"OUT{key}1"] = table[f"OUT{key}2"] = [False, False] words = table_to_words(table, table_field_info) - # assert words == ["67108865", "0", "0", "0", "1048576", "1", "1", "1"] + assert words == [ + "67108865", + "4294967276", + "12", + "32", + "1048576", + "2147483647", + "4294967295", + "1", + ] output = words_to_table(words, table_field_info) # Test the correct keys are outputted From 02265cd68b2cf607ea17b14e578fddcb348528e0 Mon Sep 17 00:00:00 2001 From: Eva Lott Date: Tue, 27 Jun 2023 11:34:57 +0100 Subject: [PATCH 5/6] Added tests from pandablocks-ioc --- pandablocks/utils.py | 7 +- tests/test_utils.py | 151 ++++++++++++++++++++++++++++++++++++++----- 2 files changed, 138 insertions(+), 20 deletions(-) diff --git a/pandablocks/utils.py b/pandablocks/utils.py index 38b5b6f6d..fa39c3a04 100644 --- a/pandablocks/utils.py +++ b/pandablocks/utils.py @@ -9,6 +9,7 @@ npt.NDArray[np.int32], npt.NDArray[np.uint8], npt.NDArray[np.uint16], + npt.NDArray[np.uint32], npt.NDArray[np.bool_], Sequence[str], ] @@ -27,7 +28,8 @@ def words_to_table( and the bit information for fields. Returns: unpacked: A dict containing record information, where keys are field names - and values are numpy arrays of record values in that column. + and values are numpy arrays or a sequence of strings of record values + in that column. """ row_words = table_field_info.row_words @@ -57,11 +59,8 @@ def words_to_table( if field_info.subtype == "int": # First convert from 2's complement to offset, then add in offset. packing_value = (value ^ (1 << (bit_length - 1))) + (-1 << (bit_length - 1)) - packing_value = value.astype(np.int32) elif field_info.labels: packing_value = [field_info.labels[x] for x in value] - elif bit_length == 1: - packing_value = value.astype(np.bool_) else: packing_value = value diff --git a/tests/test_utils.py b/tests/test_utils.py index b028df118..442279ba9 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,9 +1,11 @@ +from itertools import chain from typing import Dict, Iterable, List, OrderedDict +import numpy as np import pytest from pandablocks.responses import TableFieldDetails, TableFieldInfo -from pandablocks.utils import table_to_words, words_to_table +from pandablocks.utils import UnpackedArray, table_to_words, words_to_table @pytest.fixture @@ -155,16 +157,69 @@ def table_field_info(table_fields) -> TableFieldInfo: ) -def ensure_matching_order(list1: List, list2: List): - old_index = 0 - for list1_element in list1: - new_index = list2.index(list1_element) - if new_index < old_index: - return False - old_index = new_index +@pytest.fixture +def table_1(table_fields: Dict[str, TableFieldDetails]) -> OrderedDict[str, np.ndarray]: + """The unpacked equivalent of table_data""" + array_values: List[np.ndarray] = [ + np.array([5, 0, 50000], dtype=np.uint16), + # Below labels correspond to numeric values [0, 6, 0] + np.array(["Immediate", "BITC=1", "Immediate"], dtype=" OrderedDict[str, np.ndarray]: + # To ensure the order is different: we want to check packed values + # come out in panda order + to_insert = list(table_1.items()) + data: OrderedDict[str, np.ndarray] = OrderedDict() + for insert_iteration in chain(range(5), [7, 6, 5], range(8, len(to_insert))): + field_name, data_array = to_insert[insert_iteration] + data[str(field_name)] = data_array + + return data + +@pytest.fixture +def table_data_1() -> List[str]: + """Table data associated with table_fields and table_field_info fixtures. + See table_unpacked_data for the unpacked equivalent""" + return [ + "2457862149", + "4294967291", + "100", + "0", + "269877248", + "678", + "0", + "55", + "4293968720", + "0", + "9", + "9999", + ] -def test_table_to_words_and_words_to_table(table_field_info: TableFieldInfo): + +@pytest.fixture +def table_2() -> Dict[str, Iterable]: table: Dict[str, Iterable] = dict( REPEATS=[1, 0], TRIGGER=["Immediate", "Immediate"], @@ -178,8 +233,12 @@ def test_table_to_words_and_words_to_table(table_field_info: TableFieldInfo): for key in "BCDEF": table[f"OUT{key}1"] = table[f"OUT{key}2"] = [False, False] - words = table_to_words(table, table_field_info) - assert words == [ + return table + + +@pytest.fixture +def table_data_2() -> List[str]: + return [ "67108865", "4294967276", "12", @@ -189,14 +248,74 @@ def test_table_to_words_and_words_to_table(table_field_info: TableFieldInfo): "4294967295", "1", ] - output = words_to_table(words, table_field_info) + + +def test_table_packing_pack_length_mismatched( + table_1: OrderedDict[str, Iterable], + table_field_info: TableFieldInfo, +): + assert table_field_info.row_words + + # Adjust one of the record lengths so it mismatches + field_info = table_field_info.fields[("OUTC1")] + assert field_info + table_1["OUTC1"] = np.array([1, 2, 3, 4, 5, 6, 7, 8]) + + with pytest.raises(AssertionError): + table_to_words(table_1, table_field_info) + + +@pytest.mark.parametrize( + "table_fixture_name,table_data_fixture_name", + [("table_1_not_in_panda_order", "table_data_1"), ("table_2", "table_data_2")], +) +def test_table_to_words_and_words_to_table( + table_fixture_name: str, + table_data_fixture_name: str, + table_field_info: TableFieldInfo, + request, +): + table: Dict[str, Iterable] = request.getfixturevalue(table_fixture_name) + table_data: List[str] = request.getfixturevalue(table_data_fixture_name) + + output_data = table_to_words(table, table_field_info) + assert output_data == table_data + output_table = words_to_table(output_data, table_field_info) # Test the correct keys are outputted - assert output.keys() == table.keys() + assert output_table.keys() == table.keys() # Check the items have been inserted in panda order - sorted_table = OrderedDict({key: table[key] for key in output.keys()}) - assert sorted_table != OrderedDict(table) + sorted_output_table = OrderedDict({key: table[key] for key in output_table.keys()}) + assert sorted_output_table != OrderedDict(table) # Check the values are the same - assert [(x, list(y)) for x, y in output.items()] == list(sorted_table.items()) + for output_key in output_table.keys(): + np.testing.assert_equal(output_table[output_key], table[output_key]) + + +def test_table_packing_unpack( + table_1: OrderedDict[str, np.ndarray], + table_field_info: TableFieldInfo, + table_data_1: List[str], +): + assert table_field_info.row_words + output_table = words_to_table(table_data_1, table_field_info) + + actual: UnpackedArray + for field_name, actual in output_table.items(): + expected = table_1[str(field_name)] + np.testing.assert_array_equal(actual, expected) + + +def test_table_packing_pack( + table_1: Dict[str, Iterable], + table_field_info: TableFieldInfo, + table_data_1: List[str], +): + """Test table unpacking works as expected""" + assert table_field_info.row_words + unpacked = table_to_words(table_1, table_field_info) + + for actual, expected in zip(unpacked, table_data_1): + assert actual == expected From 6a06054f27db7fd3a864a7c7a9d222bb90667dad Mon Sep 17 00:00:00 2001 From: Eva Lott Date: Wed, 28 Jun 2023 11:36:47 +0100 Subject: [PATCH 6/6] Cleaned tests and restricted numpy arrays in UnpackedArray to np.int32 or np.uint32 --- pandablocks/utils.py | 5 +- tests/test_utils.py | 118 ++++++++++++++++++++++++++++--------------- 2 files changed, 78 insertions(+), 45 deletions(-) diff --git a/pandablocks/utils.py b/pandablocks/utils.py index fa39c3a04..7970731c9 100644 --- a/pandablocks/utils.py +++ b/pandablocks/utils.py @@ -7,10 +7,7 @@ UnpackedArray = Union[ npt.NDArray[np.int32], - npt.NDArray[np.uint8], - npt.NDArray[np.uint16], npt.NDArray[np.uint32], - npt.NDArray[np.bool_], Sequence[str], ] @@ -96,7 +93,7 @@ def table_to_words( column = [field_details.labels.index(x) for x in column] # PandA always handles tables in uint32 format - column_value = np.uint32(np.array(column)) + column_value = np.array(column, dtype=np.uint32) if packed is None: # Create 1-D array sufficiently long to exactly hold the entire table, cast diff --git a/tests/test_utils.py b/tests/test_utils.py index 442279ba9..6057dd635 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,4 +1,3 @@ -from itertools import chain from typing import Dict, Iterable, List, OrderedDict import numpy as np @@ -158,50 +157,84 @@ def table_field_info(table_fields) -> TableFieldInfo: @pytest.fixture -def table_1(table_fields: Dict[str, TableFieldDetails]) -> OrderedDict[str, np.ndarray]: - """The unpacked equivalent of table_data""" - array_values: List[np.ndarray] = [ - np.array([5, 0, 50000], dtype=np.uint16), - # Below labels correspond to numeric values [0, 6, 0] - np.array(["Immediate", "BITC=1", "Immediate"], dtype=" OrderedDict[str, Iterable]: + return OrderedDict( + { + "REPEATS": [5, 0, 50000], + "TRIGGER": ["Immediate", "BITC=1", "Immediate"], + "POSITION": [-5, 678, 0], + "TIME1": [100, 0, 9], + "OUTA1": [0, 1, 1], + "OUTB1": [0, 0, 1], + "OUTC1": [0, 0, 1], + "OUTD1": [1, 0, 1], + "OUTE1": [0, 0, 1], + "OUTF1": [1, 0, 1], + "TIME2": [0, 55, 9999], + "OUTA2": [0, 0, 1], + "OUTB2": [0, 0, 1], + "OUTC2": [1, 1, 1], + "OUTD2": [0, 0, 1], + "OUTE2": [0, 0, 1], + "OUTF2": [1, 0, 1], + } + ) @pytest.fixture -def table_1_not_in_panda_order( - table_1: OrderedDict[str, np.ndarray] -) -> OrderedDict[str, np.ndarray]: - # To ensure the order is different: we want to check packed values - # come out in panda order - to_insert = list(table_1.items()) - data: OrderedDict[str, np.ndarray] = OrderedDict() - for insert_iteration in chain(range(5), [7, 6, 5], range(8, len(to_insert))): - field_name, data_array = to_insert[insert_iteration] - data[str(field_name)] = data_array +def table_1_np_arrays() -> OrderedDict[str, Iterable]: + # Intentionally not in panda order. Whatever types the np arrays are, + # the outputs from words_to_table will be uint32 or int32. + return OrderedDict( + { + "POSITION": np.array([-5, 678, 0], dtype=np.int32), + "TIME1": np.array([100, 0, 9], dtype=np.uint32), + "OUTA1": np.array([0, 1, 1], dtype=np.uint8), + "OUTB1": np.array([0, 0, 1], dtype=np.uint8), + "OUTD1": np.array([1, 0, 1], dtype=np.uint8), + "OUTE1": np.array([0, 0, 1], dtype=np.uint8), + "OUTC1": np.array([0, 0, 1], dtype=np.uint8), + "OUTF1": np.array([1, 0, 1], dtype=np.uint8), + "TIME2": np.array([0, 55, 9999], dtype=np.uint32), + "OUTA2": np.array([0, 0, 1], dtype=np.uint8), + "OUTB2": np.array([0, 0, 1], dtype=np.uint8), + "REPEATS": np.array([5, 0, 50000], dtype=np.uint32), + "OUTC2": np.array([1, 1, 1], dtype=np.uint8), + "OUTD2": np.array([0, 0, 1], dtype=np.uint8), + "OUTE2": np.array([0, 0, 1], dtype=np.uint8), + "OUTF2": np.array([1, 0, 1], dtype=np.uint8), + "TRIGGER": np.array(["Immediate", "BITC=1", "Immediate"], dtype=" OrderedDict[str, Iterable]: + return OrderedDict( + { + "REPEATS": [5, 0, 50000], + "TRIGGER": ["Immediate", "BITC=1", "Immediate"], + "POSITION": [-5, 678, 0], + "TIME1": [100, 0, 9], + "OUTA1": [0, 1, 1], + "OUTB1": [0, 0, 1], + "OUTC1": [0, 0, 1], + "OUTD1": [1, 0, 1], + "OUTF1": [1, 0, 1], + "OUTE1": [0, 0, 1], + "TIME2": [0, 55, 9999], + "OUTA2": [0, 0, 1], + "OUTC2": [1, 1, 1], + "OUTB2": [0, 0, 1], + "OUTD2": [0, 0, 1], + "OUTE2": [0, 0, 1], + "OUTF2": [1, 0, 1], + } + ) @pytest.fixture def table_data_1() -> List[str]: - """Table data associated with table_fields and table_field_info fixtures. - See table_unpacked_data for the unpacked equivalent""" return [ "2457862149", "4294967291", @@ -267,7 +300,11 @@ def test_table_packing_pack_length_mismatched( @pytest.mark.parametrize( "table_fixture_name,table_data_fixture_name", - [("table_1_not_in_panda_order", "table_data_1"), ("table_2", "table_data_2")], + [ + ("table_1_not_in_panda_order", "table_data_1"), + ("table_2", "table_data_2"), + ("table_1_np_arrays", "table_data_1"), + ], ) def test_table_to_words_and_words_to_table( table_fixture_name: str, @@ -295,7 +332,7 @@ def test_table_to_words_and_words_to_table( def test_table_packing_unpack( - table_1: OrderedDict[str, np.ndarray], + table_1_np_arrays: OrderedDict[str, np.ndarray], table_field_info: TableFieldInfo, table_data_1: List[str], ): @@ -304,7 +341,7 @@ def test_table_packing_unpack( actual: UnpackedArray for field_name, actual in output_table.items(): - expected = table_1[str(field_name)] + expected = table_1_np_arrays[str(field_name)] np.testing.assert_array_equal(actual, expected) @@ -313,7 +350,6 @@ def test_table_packing_pack( table_field_info: TableFieldInfo, table_data_1: List[str], ): - """Test table unpacking works as expected""" assert table_field_info.row_words unpacked = table_to_words(table_1, table_field_info)