Test unordered logs

pyulog/core.py

@@ -92,14 +92,16 @@ def parse_string(cstr):
         return ret
 
     def __init__(self, log_file, message_name_filter_list=None, disable_str_exceptions=True,
-                 parse_header_only=False):
+                 parse_header_only=False, sort_data=False):
         """
         Initialize the object & load the file.
 
         :param log_file: a file name (str) or a readable file object
         :param message_name_filter_list: list of strings, to only load messages
                with the given names. If None, load everything.
         :param disable_str_parser_exceptions: If True, ignore string parsing errors
+        :param sort_data: If True, sort the data by timestamp so comparison doesn't fail when
+               the data was streamed in a different order
         """
 
         self._debug = False
@@ -134,7 +136,7 @@ def __init__(self, log_file, message_name_filter_list=None, disable_str_exceptio
         ULog._disable_str_exceptions = disable_str_exceptions
 
         if log_file is not None:
-            self._load_file(log_file, message_name_filter_list, parse_header_only)
+            self._load_file(log_file, message_name_filter_list, parse_header_only, sort_data)
 
     ## parsed data
 
@@ -823,7 +825,8 @@ def _add_message_info_multiple(self, msg_info):
             self._msg_info_multiple_dict[msg_info.key] = [[msg_info.value]]
             self._msg_info_multiple_dict_types[msg_info.key] = msg_info.type
 
-    def _load_file(self, log_file, message_name_filter_list, parse_header_only=False):
+    def _load_file(self, log_file, message_name_filter_list, parse_header_only=False,
+                   sort_data=False):
         """ load and parse an ULog file into memory """
         if isinstance(log_file, str):
             self._file_handle = open(log_file, "rb") #pylint: disable=consider-using-with
@@ -853,6 +856,18 @@ def _load_file(self, log_file, message_name_filter_list, parse_header_only=False
         # read the whole file, or the rest if data appended
         self._read_file_data(message_name_filter_list)
 
+        if sort_data:
+            for dataset in self._data_list:
+                for field_name in dataset.data.keys():
+                    # we must lexsort because unordered data often causes
+                    # overlapping data points, hence we sort by the overlapping
+                    # value too
+                    sort_idx = np.lexsort((
+                        dataset.data['timestamp'],
+                        dataset.data[field_name],
+                    ))
+                    dataset.data[field_name] = dataset.data[field_name][sort_idx]
+
         self._file_handle.close()
         del self._file_handle
 

test/test_ulog.py

@@ -23,6 +23,7 @@ class TestULog(unittest.TestCase):
     @data('sample',
           'sample_appended',
           'sample_appended_multiple',
+          'sample_unordered',
           'sample_logging_tagged_and_default_params')
     def test_write_ulog(self, base_name):
         '''
@@ -32,9 +33,9 @@ def test_write_ulog(self, base_name):
             ulog_file_name = os.path.join(TEST_PATH, base_name + '.ulg')
             written_ulog_file_name = os.path.join(tmpdirname, base_name + '_copy.ulg')
 
-            original = pyulog.ULog(ulog_file_name)
+            original = pyulog.ULog(ulog_file_name, sort_data=True)
             original.write_ulog(written_ulog_file_name)
-            copied = pyulog.ULog(written_ulog_file_name)
+            copied = pyulog.ULog(written_ulog_file_name, sort_data=True)
 
         for original_key, original_value in original.__dict__.items():
             copied_value = getattr(copied, original_key)