extra dump before decode (#287)

* add test for RMF

* Inverted tests and added comments

* change path for test data in test_rmf

* extra dump before decode

* narrow down test case

* patch

* Update oda_api/data_products.py

* Update oda_api/data_products.py

---------

Co-authored-by: Carlo Ferrigno <[email protected]>
Co-authored-by: Volodymyr Savchenko <[email protected]>

oda_api/data_products.py

@@ -18,6 +18,8 @@
 
 import typing
 
+import traceback
+
 from json_tricks import numpy_encode,dumps,loads,numeric_types_hook,hashodict,json_numpy_obj_hook
 from astropy.io import fits as pf
 from astropy.io import ascii as astropy_io_ascii
@@ -295,9 +297,12 @@ def from_fits_hdu(cls,hdu,name=None):
         if name is None or name == '':
             name=hdu.name
 
-        return cls(data=hdu.data,
+        r = cls(data=hdu.data,
                    data_header={k:v for k, v in hdu.header.items()},
                    hdu_type=cls._map_hdu_type(hdu),name=name)
+        # this is needed to re-read the file due to variable length file
+        r.to_fits_hdu()
+        return r
 
 
     def to_fits_hdu(self):
@@ -325,6 +330,9 @@ def to_fits_hdu(self):
                                     header=pf.header.Header(self.header),
                                     hdu_type=self.hdu_type,units_dict=self.units_dict)
         except Exception as e:
+            error_message = 'an exception occurred in oda_api when binary products are formatted to fits header: ' + repr(e)
+            logger.error(error_message)
+            logger.error('traceback: %s', traceback.format_exc())
             raise Exception("an exception occurred in oda_api when binary products are formatted to fits header: " + repr(e))
 
 
@@ -638,6 +646,7 @@ def from_fits_file(cls,filename,ext=None,hdu_name=None,meta_data={},name=''):
             hdul=_hdul
 
         return cls(data_unit=[NumpyDataUnit.from_fits_hdu(h) for h in  hdul],meta_data=meta_data,name=name)
+
 
     @classmethod
     def decode(cls, encoded_obj: typing.Union[str, dict], from_json=False):

tests/test_data_products.py

@@ -21,6 +21,9 @@
 from astropy.table import Table
 from matplotlib import pyplot as plt
 
+import base64
+import pickle
+
 secret_key = 'secretkey_test'
 default_exp_time = int(time.time()) + 5000
 default_token_payload = dict(
@@ -33,6 +36,52 @@
     mssub=True
 )
 
+
+def test_variable_length_rmf():
+    # "compressed" (by removing small matrix values, special RMF spec) RMF is stored in variable length table
+    # which is not well supported by astropy
+
+    isgri_rmf_dp_post_scan = NumpyDataProduct.from_fits_file("tests/test_data/isgri_rmf_Crab.fits")
+    isgri_rmf_dp_post_scan.data_unit[2].to_fits_hdu()
+    encoded_numpy_data_prod_postscan = isgri_rmf_dp_post_scan.encode()
+    decoded_numpy_data_prod_postscan = NumpyDataProduct.decode(encoded_numpy_data_prod_postscan)                
+    decoded_numpy_data_prod_postscan.data_unit[2].to_fits_hdu()
+
+    isgri_rmf_dp = NumpyDataProduct.from_fits_file("tests/test_data/isgri_rmf_Crab.fits")
+    encoded_numpy_data_prod = isgri_rmf_dp.encode()
+    decoded_numpy_data_prod = NumpyDataProduct.decode(encoded_numpy_data_prod)                    
+    decoded_numpy_data_prod.data_unit[2].to_fits_hdu()
+
+
+def test_rmf():
+    isgri_rmf_dp = NumpyDataProduct.from_fits_file("tests/test_data/isgri_rmf_Crab.fits")
+
+    for ID, _d in enumerate(isgri_rmf_dp.data_unit):
+        print(ID, _d.header['EXTNAME'], _d.to_fits_hdu())
+
+    encoded_numpy_data_prod = isgri_rmf_dp.encode()
+    decoded_numpy_data_prod = NumpyDataProduct.decode(encoded_numpy_data_prod)
+
+    # this is the higher-level call causing the above error in nb2workflow
+    _hdul = fits.HDUList()
+    for ID, _d in enumerate(decoded_numpy_data_prod.data_unit):
+        print(ID, _d.header['EXTNAME'])
+        try:
+            _hdul.append(_d.to_fits_hdu())
+        except Exception as ee:
+            # print(ee)
+            raise Exception(ee)
+
+    # this reproduces the commands done inside the above call (?)
+    binarys = base64.b64decode(encoded_numpy_data_prod['data_unit_list'][2]['binarys'])
+    try:
+        pickle.loads(binarys, encoding='bytes')
+        print('pickle test successful')
+    except Exception as ee:
+        raise Exception(ee)
+
+
+
 # TODO: adapt to new product types and implement corresponding tests
 def encode_decode(ndp: typing.Union[NumpyDataProduct, 
                                     ODAAstropyTable, 
@@ -188,4 +237,4 @@ def test_new_binary_product():
     decoded.write_file('decbinprd.foo')
     assert filecmp.cmp(infile, 'decbinprd.foo')
     os.remove('decbinprd.foo') 
-
+