diff --git a/pyulog/core.py b/pyulog/core.py index 6069325..a8f2f69 100644 --- a/pyulog/core.py +++ b/pyulog/core.py @@ -92,7 +92,7 @@ def parse_string(cstr): return ret def __init__(self, log_file, message_name_filter_list=None, disable_str_exceptions=True, - parse_header_only=False): + parse_header_only=False, sort_data=False): """ Initialize the object & load the file. @@ -100,6 +100,8 @@ def __init__(self, log_file, message_name_filter_list=None, disable_str_exceptio :param message_name_filter_list: list of strings, to only load messages with the given names. If None, load everything. :param disable_str_parser_exceptions: If True, ignore string parsing errors + :param sort_data: If True, sort the data by timestamp so comparison doesn't fail when + the data was streamed in a different order """ self._debug = False @@ -134,7 +136,7 @@ def __init__(self, log_file, message_name_filter_list=None, disable_str_exceptio ULog._disable_str_exceptions = disable_str_exceptions if log_file is not None: - self._load_file(log_file, message_name_filter_list, parse_header_only) + self._load_file(log_file, message_name_filter_list, parse_header_only, sort_data) ## parsed data @@ -823,7 +825,8 @@ def _add_message_info_multiple(self, msg_info): self._msg_info_multiple_dict[msg_info.key] = [[msg_info.value]] self._msg_info_multiple_dict_types[msg_info.key] = msg_info.type - def _load_file(self, log_file, message_name_filter_list, parse_header_only=False): + def _load_file(self, log_file, message_name_filter_list, parse_header_only=False, + sort_data=False): """ load and parse an ULog file into memory """ if isinstance(log_file, str): self._file_handle = open(log_file, "rb") #pylint: disable=consider-using-with @@ -853,6 +856,18 @@ def _load_file(self, log_file, message_name_filter_list, parse_header_only=False # read the whole file, or the rest if data appended self._read_file_data(message_name_filter_list) + if sort_data: + for dataset in self._data_list: + for field_name in dataset.data.keys(): + # we must lexsort because unordered data often causes + # overlapping data points, hence we sort by the overlapping + # value too + sort_idx = np.lexsort(( + dataset.data['timestamp'], + dataset.data[field_name], + )) + dataset.data[field_name] = dataset.data[field_name][sort_idx] + self._file_handle.close() del self._file_handle diff --git a/test/sample_unordered.ulg b/test/sample_unordered.ulg new file mode 100644 index 0000000..fa8acbd Binary files /dev/null and b/test/sample_unordered.ulg differ diff --git a/test/test_ulog.py b/test/test_ulog.py index 85f95c3..e706dc0 100644 --- a/test/test_ulog.py +++ b/test/test_ulog.py @@ -23,6 +23,7 @@ class TestULog(unittest.TestCase): @data('sample', 'sample_appended', 'sample_appended_multiple', + 'sample_unordered', 'sample_logging_tagged_and_default_params') def test_write_ulog(self, base_name): ''' @@ -32,9 +33,9 @@ def test_write_ulog(self, base_name): ulog_file_name = os.path.join(TEST_PATH, base_name + '.ulg') written_ulog_file_name = os.path.join(tmpdirname, base_name + '_copy.ulg') - original = pyulog.ULog(ulog_file_name) + original = pyulog.ULog(ulog_file_name, sort_data=True) original.write_ulog(written_ulog_file_name) - copied = pyulog.ULog(written_ulog_file_name) + copied = pyulog.ULog(written_ulog_file_name, sort_data=True) for original_key, original_value in original.__dict__.items(): copied_value = getattr(copied, original_key)