Use int32 for compression-sake

astronomy-commons · May 6, 2024 · 3745a22 · 3745a22
1 parent cfaccef
commit 3745a22
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 8 deletions.
diff --git a/src/hipscat_import/catalog/map_reduce.py b/src/hipscat_import/catalog/map_reduce.py
@@ -95,7 +95,7 @@ def map_to_pixels(
         FileNotFoundError: if the file does not exist, or is a directory
     """
     sparse_map = healsparse.HealSparseMap.make_empty(
-        hp.order2nside(highest_order), hp.order2nside(highest_order), np.int64, sentinel=0
+        hp.order2nside(highest_order), hp.order2nside(highest_order), np.int32, sentinel=0
     )
     if use_hipscat_index:
         read_columns = [HIPSCAT_ID_COLUMN]
@@ -106,8 +106,8 @@ def map_to_pixels(
         input_file, file_reader, highest_order, ra_column, dec_column, use_hipscat_index, read_columns
     ):
         mapped_pixel, count_at_pixel = np.unique(mapped_pixels, return_counts=True)
-        mapped_pixel = mapped_pixel.astype(np.int64)
-        sparse_map[mapped_pixel] += count_at_pixel.astype(np.int64)
+        mapped_pixel = mapped_pixel.astype(np.int32)
+        sparse_map[mapped_pixel] += count_at_pixel.astype(np.int32)
     ResumePlan.write_partial_healsparse_map(
         tmp_path=resume_path,
         mapping_key=mapping_key,

diff --git a/src/hipscat_import/catalog/resume_plan.py b/src/hipscat_import/catalog/resume_plan.py
@@ -112,7 +112,7 @@ def read_histogram(self, healpix_order):
         file_name = file_io.append_paths_to_pointer(self.tmp_path, self.HISTOGRAM_BINARY_FILE)
         if not file_io.does_file_or_directory_exist(file_name):
             # Combining coverage maps from partials
-            full_map = healsparse.HealSparseMap.make_empty(hp_nside, hp_nside, np.int64, sentinel=0)
+            full_map = healsparse.HealSparseMap.make_empty(hp_nside, hp_nside, np.int32, sentinel=0)
             histogram_files = self._get_partial_filenames()
 
             for file in histogram_files:
@@ -132,7 +132,7 @@ def read_histogram(self, healpix_order):
                 + "from scratch with the current order set `resume` to False."
             )
 
-        return hp_map.generate_healpix_map(dtype=np.int64, sentinel=0)
+        return hp_map.generate_healpix_map(dtype=np.int32, sentinel=0)
 
     def _get_partial_filenames(self):
         remaining_map_files = self.get_remaining_map_keys()

diff --git a/tests/hipscat_import/conftest.py b/tests/hipscat_import/conftest.py
@@ -170,19 +170,19 @@ def resume_dir(test_data_dir):
 
 @pytest.fixture
 def small_sky_healsparse_map():
-    sub_map = healsparse.HealSparseMap.make_empty(hp.order2nside(0), hp.order2nside(0), np.int64, sentinel=0)
+    sub_map = healsparse.HealSparseMap.make_empty(hp.order2nside(0), hp.order2nside(0), np.int32, sentinel=0)
     sub_map[11] = 131
     return sub_map
 
 
 @pytest.fixture
 def empty_healsparse_order0():
-    return healsparse.HealSparseMap.make_empty(hp.order2nside(0), hp.order2nside(0), np.int64, sentinel=0)
+    return healsparse.HealSparseMap.make_empty(hp.order2nside(0), hp.order2nside(0), np.int32, sentinel=0)
 
 
 @pytest.fixture
 def empty_healsparse_order1():
-    return healsparse.HealSparseMap.make_empty(hp.order2nside(1), hp.order2nside(1), np.int64, sentinel=0)
+    return healsparse.HealSparseMap.make_empty(hp.order2nside(1), hp.order2nside(1), np.int32, sentinel=0)
 
 
 @pytest.fixture