Use valid HI source catalogs, improve catalog access methods (#527)

* Remove SOUTH_CATALOG_FACTOR, make tests and examples comply with new code

* Replace HI flux catalog with correct file

* Begin replacing BATTYE sky with correct catalog

* Replace BATTYE catalogs with corrected HI catalog

* Resolve field count in SkyModel for OSKAR model

* Use corrected goldenfiles for line emission test

* Use correct golden files for mosaic test

* Reorder skymodel fields

* Use correct indices in SkyModel

* Correctly handle sources with and without ID column data

* Remove __del__ method since it's unpredictable

* Resolve notebooks

karabo/data/external_data.py

@@ -115,6 +115,14 @@ def __init__(self) -> None:
         )
 
 
+class HISourcesSmallCatalogDownloadObject(SingleFileDownloadObject):
+    def __init__(self) -> None:
+        super().__init__(
+            remote_file_path="HI_sources_small_catalog.h5",
+            remote_base_url=cscs_karabo_public_base_url,
+        )
+
+
 class BATTYESurveyDownloadObject(SingleFileDownloadObject):
     def __init__(self) -> None:
         raise NotImplementedError(
@@ -124,9 +132,9 @@ def __init__(self) -> None:
 
 class DilutedBATTYESurveyDownloadObject(SingleFileDownloadObject):
     def __init__(self) -> None:
-        super().__init__(
-            remote_file_path="point_sources_OSKAR1_diluted5000.h5",
-            remote_base_url=cscs_karabo_public_base_url,
+        raise NotImplementedError(
+            """This catalog has incorrect flux data.
+            Use HISourcesSmallCatalogDownloadObject instead."""
         )
 
 

karabo/examples/source_detection_big_files.ipynb

@@ -120,9 +120,7 @@
    "source": [
     "# Get GLEAM Survey Sky\n",
     "phase_center = [21.44213503, -30.70729488]\n",
-    "sky = measure_memory(\n",
-    "    SkyModel.get_BATTYE_sky, which=\"diluted\"\n",
-    ")  # Replace with 'full' for full sky"
+    "sky = measure_memory(SkyModel.get_sample_simulated_catalog)"
    ]
   },
   {

karabo/performance_test/time_karabo_reconstruction.py

@@ -1,9 +1,12 @@
+import os
+from pathlib import Path
+
 from karabo.simulation.line_emission import karabo_reconstruction
 from karabo.simulation.sky_model import SkyModel
 
 if __name__ == "__main__":
     print("Loading sky model")
-    sky = SkyModel.get_BATTYE_sky(which="diluted")
+    sky = SkyModel.get_sample_simulated_catalog()
     phase_center = [21.44213503, -30.70729488]
     print("Filtering sky model")
     sky = sky.filter_by_radius_euclidean_flat_approximation(
@@ -13,8 +16,13 @@
         phase_center[1],
     )
     print("Reconstructing sky model")
+    OUTPUT_PATH = "test_recon/"
+    SCRATCH_PATH = os.getenv("SCRATCH")
+    if SCRATCH_PATH is not None:
+        OUTPUT_PATH = str(Path(SCRATCH_PATH) / OUTPUT_PATH)
+
     karabo_reconstruction(
-        "/scratch/snx3000/vtimmel/karabo_folder/test_recon/",
+        OUTPUT_PATH,
         sky=sky,
         pdf_plot=True,
         cut=3,