Merge pull request #92 from LSSTDESC/90-fix-hdf5-chunk-write

90 fix hdf5 chunk write

qp/ensemble.py

@@ -599,16 +599,24 @@ def plot_native(self, key=0, **kwargs):
         return self._gen_class.plot_native(self[key], **kwargs)
 
     def _get_allocation_kwds(self, npdf):
-        return self._gen_class.get_allocation_kwds(npdf,
-                                                   **{**self.metadata(),
-                                                      **self.objdata()})
+        tables = self.build_tables()
+        groups = tables.items()
+        keywords = {}
+        for group, tab in tables.items():
+            if group != 'meta':
+                keywords[group]={}
+                for key, array in tab.items():
+                    shape = list(array.shape)
+                    shape[0] = npdf
+                    keywords[group][key] = (shape, array.dtype)
+        return keywords
 
     def initializeHdf5Write(self, filename, npdf):
         """set up the output write for an ensemble, but set size to npdf rather than
         the size of the ensemble, as the "initial chunk" will not contain the full data
         """
         kwds = self._get_allocation_kwds(npdf)
-        group, fout = io.initializeHdf5Write(filename, 'data', **kwds)
+        group, fout = io.initializeHdf5Write(filename, **kwds)
         return group, fout
 
     def writeHdf5Chunk(self, fname, start, end):
@@ -619,7 +627,9 @@ def writeHdf5Chunk(self, fname, start, end):
         start : `int` starting index of h5py file
         end : `int` ending index in h5py file
         """
-        io.writeDictToHdf5Chunk(fname, self.objdata(), start, end)
+        odict = self.build_tables().copy()
+        odict.pop('meta')
+        io.writeDictToHdf5Chunk(fname, odict, start, end)
 
     def finalizeHdf5Write(self, filename):
         """write ensemble metadata to the output file

setup.py

@@ -24,5 +24,5 @@
                       "numpy",
                       "scipy>=1.7.0",
                       "scikit-learn",
-                      "tables_io"],
+                      "tables_io>=0.7.4"],
 )