diff --git a/software/python_modules/tart/tart/imaging/test/test_visibility.py b/software/python_modules/tart/tart/imaging/test/test_visibility.py index e1cc750b..1119cef7 100644 --- a/software/python_modules/tart/tart/imaging/test/test_visibility.py +++ b/software/python_modules/tart/tart/imaging/test/test_visibility.py @@ -2,6 +2,7 @@ import datetime import h5py import numpy as np +import time from tart.util import skyloc from tart.util import angle @@ -27,7 +28,7 @@ def dummy_vis(): def dummy_vis_list(): ret = [] - for i in range(10): + for i in range(2): ret.append(dummy_vis()) return ret @@ -41,6 +42,7 @@ def check_vis(self, dut, dut2): delta = np.sum(np.abs(np.array(dut.v) - np.array(dut2.v))) self.assertTrue( delta <= 1.0e10 ) self.assertTrue((np.array(dut.baselines) == np.array(dut2.baselines)).all()) + self.assertEqual(dut.timestamp, dut2.timestamp) keys = dut.config.Dict.keys() for k in keys: @@ -55,24 +57,24 @@ def test_load_save(self): def test_list_load_save_hdf(self): dut_list = dummy_vis_list() fname = 'test_vis_list_io.hdf' - visibility.list_save(self.v_array, fname) + visibility.list_save(dut_list, fname) dut2_list = visibility.list_load(fname) - for dut, dut2 in zip(dut_list, dut2_list): - self.check_vis(dut, dut2) + for x, y in zip(dut_list, dut2_list): + self.check_vis(x, y) def test_list_load_save_pkl(self): dut_list = dummy_vis_list() fname = 'test_vis_list_io.pkl' - visibility.list_save(self.v_array, fname) + visibility.list_save(dut_list, fname) dut2_list = visibility.list_load(fname) for dut, dut2 in zip(dut_list, dut2_list): self.check_vis(dut, dut2) def test_hdf5(self): dut = dummy_vis() - visibility.to_hdf5(dut, 'test_vis.hdf') + visibility.to_hdf5([dut], 'test_vis.hdf') dut2 = visibility.from_hdf5('test_vis.hdf') - self.check_vis(dut, dut2) + self.check_vis(dut, dut2[0]) def test_pkl(self): dut = dummy_vis() diff --git a/software/python_modules/tart/tart/imaging/visibility.py b/software/python_modules/tart/tart/imaging/visibility.py index 22074987..19268b65 100644 --- a/software/python_modules/tart/tart/imaging/visibility.py +++ b/software/python_modules/tart/tart/imaging/visibility.py @@ -108,6 +108,9 @@ def toString(self): ret += " V(%s)=%g, I%g" % (str(b), np.abs(self.v[i]), np.angle(self.v[i])) return ret + def __repr__(self): + return "vis(ts={})".format(self.timestamp) + def Visibility_Lsq(vis1, vis2): @@ -155,7 +158,7 @@ def list_save(vis_list, filename): elif ('.vis' == file_extension): to_pkl(vis_list, filename) elif ('.hdf' == file_extension): - to_pkl(vis_list, filename) + to_hdf5(vis_list, filename) else: raise RuntimeError("Unknown visibility file extension {}".format(file_extension)) @@ -178,7 +181,7 @@ def list_load(filename): vis_list = from_pkl(filename) return vis_list elif ('.hdf' == file_extension): - vis_list = from_pkl(filename) + vis_list = from_hdf5(filename) return vis_list else: raise RuntimeError("Unknown file extension {}".format(file_extension)) @@ -204,40 +207,60 @@ def from_pkl(filename): Deal with HDF5 Files ''' -def to_hdf5(vis, filename): +def to_hdf5(vis_list, filename): ''' Save the Observation object, to a portable HDF5 format ''' + if not isinstance(vis_list, list): + raise RuntimeError("vis_list must be a list of visibility objects") + + vis0 = vis_list[0] + + vis_data = [vis.v for vis in vis_list] + vis_ts = [vis.timestamp.isoformat() for vis in vis_list] + + with h5py.File(filename, "w") as h5f: - dt = h5py.special_dtype(vlen=bytes) + dt = h5py.special_dtype(vlen=str) + conftype = h5py.special_dtype(vlen=bytes) - conf_dset = h5f.create_dataset('config', (1,), dtype=dt) - conf_dset[0] = vis.config.to_json() - - ts_dset = h5f.create_dataset('timestamp', (1,), dtype=dt) - ts_dset[0] = vis.timestamp.isoformat() - - h5f.create_dataset('baselines', data=vis.baselines) - h5f.create_dataset('vis', data=vis.v) + conf_dset = h5f.create_dataset('config', (1,), dtype=conftype) + conf_dset[0] = vis0.config.to_json() + h5f.create_dataset('phase_elaz', data=[vis0.phase_el.to_degrees(), vis0.phase_az.to_degrees()]) + h5f.create_dataset('baselines', data=vis0.baselines) + + h5f.create_dataset('vis', data=np.array(vis_data)) + h5f.create_dataset("timestamp", data=np.array(vis_ts, dtype=object), dtype=dt) + + #ts_dset = h5f.create_dataset('timestamp', (len(vis_ts),), dtype=dt) + #for i,ts in enumerate(vis_ts): + #print(ts) + #ts_dset[i] = ts - h5f.create_dataset('phase_elaz', data=[vis.phase_el.to_degrees(), vis.phase_az.to_degrees()]) def from_hdf5(filename): ''' Load the Visibility object, from a portable HDF5 format ''' + + ret = [] + with h5py.File(filename, "r") as h5f: config_json = np.string_(h5f['config'][0]) config = settings.from_json(config_json) - timestamp = dateutil.parser.parse(h5f['timestamp'][0]) - - hdf_vis = h5f['vis'][:] hdf_baselines = h5f['baselines'][:] hdf_phase_elaz = h5f['phase_elaz'][:] - ret = Visibility(config=config, timestamp=timestamp) - ret.set_visibilities(v=hdf_vis, b=hdf_baselines) - ret.phase_el = hdf_phase_elaz[0] - ret.phase_az = hdf_phase_elaz[1] - + hdf_timestamps = h5f['timestamp'] + timestamps = [dateutil.parser.parse(x) for x in hdf_timestamps] + + hdf_vis = h5f['vis'][:] + + for ts, v in zip(timestamps, hdf_vis): + vis = Visibility(config=config, timestamp=ts) + vis.set_visibilities(v=v, b=hdf_baselines) + vis.phase_el = hdf_phase_elaz[0] + vis.phase_az = hdf_phase_elaz[1] + ret.append(vis) + return ret diff --git a/software/python_modules/tart_web_api/README.md b/software/python_modules/tart_web_api/README.md index ab3fbff2..dd349aaf 100644 --- a/software/python_modules/tart_web_api/README.md +++ b/software/python_modules/tart_web_api/README.md @@ -42,4 +42,5 @@ in which case changes to the source-code will be immediately available to projec ## NEWS +* Version 0.2b4. Default to 'vis' mode. Export vis files as hdf * Version 0.1.8 Python3 compatability diff --git a/software/python_modules/tart_web_api/setup.py b/software/python_modules/tart_web_api/setup.py index 851626b4..919f8a4b 100644 --- a/software/python_modules/tart_web_api/setup.py +++ b/software/python_modules/tart_web_api/setup.py @@ -8,7 +8,7 @@ description='Transient Array Radio Telescope High-level HTTP Interface', long_description=readme, long_description_content_type="text/markdown", - version='0.2.0b3', + version='0.2.0b4', packages=['tart_web_api'], include_package_data=True, install_requires=[ diff --git a/software/python_modules/tart_web_api/tart_web_api/config.py b/software/python_modules/tart_web_api/tart_web_api/config.py index f2305040..38813d2c 100644 --- a/software/python_modules/tart_web_api/tart_web_api/config.py +++ b/software/python_modules/tart_web_api/tart_web_api/config.py @@ -14,7 +14,7 @@ def init_config(manager): runtime_config['verbose'] = False runtime_config['centre'] = True runtime_config['modes_available'] = ['off', 'diag', 'raw', 'vis', 'vis_save', 'cal', 'rt_syn_img'] - runtime_config['mode'] = 'off' + runtime_config['mode'] = 'vis' runtime_config['loop_mode'] = 'loop' runtime_config['loop_mode_available'] = ['loop', 'single', 'loop_n'] runtime_config['loop_n'] = 5 diff --git a/software/python_modules/tart_web_api/tart_web_api/service.py b/software/python_modules/tart_web_api/tart_web_api/service.py index cbce53ac..0bf1b5c1 100644 --- a/software/python_modules/tart_web_api/tart_web_api/service.py +++ b/software/python_modules/tart_web_api/tart_web_api/service.py @@ -297,9 +297,11 @@ def vis_stream_acquire(self): if len(self.vislist) >= chunksize: logging.info('reached chunksize of {}'.format(chunksize)) if self.config['vis']['save'] == 1: - fname = "{}/vis_{}.vis".format(self.config['vis']['base_path'], + fname = "{}/vis_{}.hdf".format(self.config['vis']['base_path'], vis.timestamp.strftime('%Y-%m-%d_%H_%M_%S.%f')) - visibility.Visibility_Save(self.vislist, fname) + # visibility.Visibility_Save(self.vislist, fname) + visibility.list_save(self.vislist, fname) + logging.info("saved to {}".format(vis, fname)) ret['filename'] = fname ret['sha256'] = sha256_checksum(fname)