Visibilities now stored as HDF5 files. Closes Issue #40

software/python_modules/tart/tart/imaging/test/test_visibility.py

@@ -2,6 +2,7 @@
 import datetime
 import h5py
 import numpy as np
+import time
 
 from tart.util import skyloc
 from tart.util import angle
@@ -27,7 +28,7 @@ def dummy_vis():
 
 def dummy_vis_list():
     ret = []
-    for i in range(10):
+    for i in range(2):
         ret.append(dummy_vis())
     return ret 
 
@@ -41,6 +42,7 @@ def check_vis(self, dut, dut2):
         delta = np.sum(np.abs(np.array(dut.v) - np.array(dut2.v)))
         self.assertTrue( delta <= 1.0e10 )
         self.assertTrue((np.array(dut.baselines) == np.array(dut2.baselines)).all())
+        self.assertEqual(dut.timestamp, dut2.timestamp)
 
         keys = dut.config.Dict.keys()
         for k in keys:
@@ -55,24 +57,24 @@ def test_load_save(self):
     def test_list_load_save_hdf(self):
         dut_list = dummy_vis_list()
         fname = 'test_vis_list_io.hdf'
-        visibility.list_save(self.v_array, fname)
+        visibility.list_save(dut_list, fname)
         dut2_list = visibility.list_load(fname)
-        for dut, dut2 in zip(dut_list, dut2_list):
-            self.check_vis(dut, dut2)
+        for x, y in zip(dut_list, dut2_list):
+            self.check_vis(x, y)
 
     def test_list_load_save_pkl(self):
         dut_list = dummy_vis_list()
         fname = 'test_vis_list_io.pkl'
-        visibility.list_save(self.v_array, fname)
+        visibility.list_save(dut_list, fname)
         dut2_list = visibility.list_load(fname)
         for dut, dut2 in zip(dut_list, dut2_list):
             self.check_vis(dut, dut2)
 
     def test_hdf5(self):
         dut = dummy_vis()
-        visibility.to_hdf5(dut, 'test_vis.hdf')
+        visibility.to_hdf5([dut], 'test_vis.hdf')
         dut2 = visibility.from_hdf5('test_vis.hdf')
-        self.check_vis(dut, dut2)
+        self.check_vis(dut, dut2[0])
 
     def test_pkl(self):
         dut = dummy_vis()

software/python_modules/tart/tart/imaging/visibility.py

@@ -108,6 +108,9 @@ def toString(self):
             ret += " V(%s)=%g, I%g" % (str(b), np.abs(self.v[i]), np.angle(self.v[i]))
         return ret
 
+    def __repr__(self):
+        return "vis(ts={})".format(self.timestamp)
+
 
 
 def Visibility_Lsq(vis1, vis2):
@@ -155,7 +158,7 @@ def list_save(vis_list, filename):
     elif ('.vis' == file_extension):
         to_pkl(vis_list, filename)
     elif ('.hdf' == file_extension):
-        to_pkl(vis_list, filename)
+        to_hdf5(vis_list, filename)
     else:
         raise RuntimeError("Unknown visibility file extension {}".format(file_extension))
 
@@ -178,7 +181,7 @@ def list_load(filename):
         vis_list = from_pkl(filename)
         return vis_list
     elif ('.hdf' == file_extension):
-        vis_list = from_pkl(filename)
+        vis_list = from_hdf5(filename)
         return vis_list
     else:
         raise RuntimeError("Unknown file extension {}".format(file_extension))
@@ -204,40 +207,60 @@ def from_pkl(filename):
     Deal with HDF5 Files
 '''
 
-def to_hdf5(vis, filename):
+def to_hdf5(vis_list, filename):
     ''' Save the Observation object,
         to a portable HDF5 format
     '''
+    if not isinstance(vis_list, list):
+        raise RuntimeError("vis_list must be a list of visibility objects")
+
+    vis0 = vis_list[0]
+
+    vis_data = [vis.v for vis in vis_list]
+    vis_ts = [vis.timestamp.isoformat() for vis in vis_list]
+
+
     with h5py.File(filename, "w") as h5f:
-        dt = h5py.special_dtype(vlen=bytes)
+        dt = h5py.special_dtype(vlen=str)
+        conftype = h5py.special_dtype(vlen=bytes)
 
-        conf_dset = h5f.create_dataset('config', (1,), dtype=dt)
-        conf_dset[0] = vis.config.to_json()
-
-        ts_dset = h5f.create_dataset('timestamp', (1,), dtype=dt)
-        ts_dset[0] = vis.timestamp.isoformat()
-
-        h5f.create_dataset('baselines', data=vis.baselines)
-        h5f.create_dataset('vis', data=vis.v)
+        conf_dset = h5f.create_dataset('config', (1,), dtype=conftype)
+        conf_dset[0] = vis0.config.to_json()
+        h5f.create_dataset('phase_elaz', data=[vis0.phase_el.to_degrees(), vis0.phase_az.to_degrees()])
+        h5f.create_dataset('baselines', data=vis0.baselines)
+
+        h5f.create_dataset('vis', data=np.array(vis_data))
+        h5f.create_dataset("timestamp", data=np.array(vis_ts, dtype=object), dtype=dt)
+
+        #ts_dset = h5f.create_dataset('timestamp', (len(vis_ts),), dtype=dt)
+        #for i,ts in enumerate(vis_ts):
+            #print(ts)
+            #ts_dset[i] = ts
 
-        h5f.create_dataset('phase_elaz', data=[vis.phase_el.to_degrees(), vis.phase_az.to_degrees()])
 
 def from_hdf5(filename):
     ''' Load the Visibility object,
         from a portable HDF5 format
     '''
+
+    ret = []
+
     with h5py.File(filename, "r") as h5f:
         config_json = np.string_(h5f['config'][0])
         config = settings.from_json(config_json)
-        timestamp = dateutil.parser.parse(h5f['timestamp'][0])
-
-        hdf_vis = h5f['vis'][:]
         hdf_baselines = h5f['baselines'][:]
         hdf_phase_elaz = h5f['phase_elaz'][:]
 
-    ret = Visibility(config=config, timestamp=timestamp)
-    ret.set_visibilities(v=hdf_vis, b=hdf_baselines)
-    ret.phase_el = hdf_phase_elaz[0]
-    ret.phase_az = hdf_phase_elaz[1]
-
+        hdf_timestamps = h5f['timestamp']
+        timestamps = [dateutil.parser.parse(x) for x in hdf_timestamps]
+
+        hdf_vis = h5f['vis'][:]
+
+        for ts, v in zip(timestamps, hdf_vis):
+            vis = Visibility(config=config, timestamp=ts)
+            vis.set_visibilities(v=v, b=hdf_baselines)
+            vis.phase_el = hdf_phase_elaz[0]
+            vis.phase_az = hdf_phase_elaz[1]
+            ret.append(vis)
+
     return ret

software/python_modules/tart_web_api/README.md

@@ -42,4 +42,5 @@ in which case changes to the source-code will be immediately available to projec
 
 ## NEWS
 
+* Version 0.2b4. Default to 'vis' mode. Export vis files as hdf
 * Version 0.1.8 Python3 compatability

software/python_modules/tart_web_api/setup.py

@@ -8,7 +8,7 @@
     description='Transient Array Radio Telescope High-level HTTP Interface',
     long_description=readme,
     long_description_content_type="text/markdown",
-    version='0.2.0b3',
+    version='0.2.0b4',
     packages=['tart_web_api'],
     include_package_data=True,
     install_requires=[

software/python_modules/tart_web_api/tart_web_api/config.py

@@ -14,7 +14,7 @@ def init_config(manager):
     runtime_config['verbose'] = False
     runtime_config['centre'] = True
     runtime_config['modes_available'] = ['off', 'diag', 'raw', 'vis', 'vis_save', 'cal', 'rt_syn_img']
-    runtime_config['mode'] = 'off'
+    runtime_config['mode'] = 'vis'
     runtime_config['loop_mode'] = 'loop'
     runtime_config['loop_mode_available'] = ['loop', 'single', 'loop_n']
     runtime_config['loop_n'] = 5

software/python_modules/tart_web_api/tart_web_api/service.py

@@ -297,9 +297,11 @@ def vis_stream_acquire(self):
                 if len(self.vislist) >= chunksize:
                     logging.info('reached chunksize of {}'.format(chunksize))
                     if self.config['vis']['save'] == 1:
-                        fname = "{}/vis_{}.vis".format(self.config['vis']['base_path'], 
+                        fname = "{}/vis_{}.hdf".format(self.config['vis']['base_path'], 
                                                    vis.timestamp.strftime('%Y-%m-%d_%H_%M_%S.%f'))
-                        visibility.Visibility_Save(self.vislist, fname)
+                        # visibility.Visibility_Save(self.vislist, fname)
+                        visibility.list_save(self.vislist, fname)
+
                         logging.info("saved to {}".format(vis, fname))
                         ret['filename'] = fname
                         ret['sha256'] = sha256_checksum(fname)