HS82 running on test degree to merge

Perhaps it will run on Vulcain without changes now. 30Gb decals will reduce to 5 gb after processing and not as many surveys as xmm-lss. Yannick noticed major error in DECals - needs aperture correction and band names changing inline with Legacy survey on #2, #3, #4, #5, #7, #8, #15 and #16
H-E-L-P · Nov 22, 2017 · 7f78616 · 7f78616
1 parent 842f20b
commit 7f78616
Show file tree

Hide file tree

Showing 14 changed files with 947 additions and 1,207 deletions.
diff --git a/dmu1/dmu1_ml_Bootes/1_Reformatting.ipynb b/dmu1/dmu1_ml_Bootes/1_Reformatting.ipynb
@@ -69,8 +69,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "OUT_DIR =  os.environ.get('TMP_DIR', \"./data_tmp\")\n",
-    "SUFFIX = os.environ.get('SUFFIX', time.strftime(\"_%Y%m%d\"))\n",
+    "TMP_DIR = os.environ.get('TMP_DIR', \"./data_tmp\")\n",
+    "OUT_DIR = os.environ.get('OUT_DIR', \"./data\")\n",
     "try:\n",
     "    os.makedirs(OUT_DIR)\n",
     "except FileExistsError:\n",
@@ -95,12 +95,12 @@
    },
    "outputs": [],
    "source": [
-    "imported_columns = OrderedDict({\n",
-    "        \"objID\": \"ps1_id\",\n",
-    "        \"raMean\": \"ps1_ra\",\n",
-    "        \"decMean\": \"ps1_dec\",\n",
-    "\n",
-    "    })\n",
+    "#imported_columns = OrderedDict({\n",
+    "#        \"objID\": \"ps1_id\",\n",
+    "#        \"raMean\": \"ps1_ra\",\n",
+    "#        \"decMean\": \"ps1_dec\",\n",
+    "#\n",
+    "#    })\n",
     "\n",
     "\n",
     "i_catalogue = Table.read(\"../../dmu0/dmu0_Bootes_Brown/data/Bootes_merged_Icorr_2014a_all_ap2_07112017.fits\") #[list(imported_columns)]\n",

diff --git a/dmu1/dmu1_ml_Bootes/2_Checks_and_diagnostics.ipynb b/dmu1/dmu1_ml_Bootes/2_Checks_and_diagnostics.ipynb
@@ -12,21 +12,37 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "This notebook was run with herschelhelp_internal version: \n",
+      "04829ed (Thu Nov 2 16:57:19 2017 +0000) [with local modifications]\n"
+     ]
+    }
+   ],
    "source": [
     "from herschelhelp_internal import git_version\n",
     "print(\"This notebook was run with herschelhelp_internal version: \\n{}\".format(git_version()))"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "collapsed": true
-   },
-   "outputs": [],
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/rs548/anaconda/envs/herschelhelp_internal/lib/python3.6/site-packages/seaborn/apionly.py:6: UserWarning: As seaborn no longer sets a default style on import, the seaborn.apionly module is deprecated. It will be removed in a future version.\n",
+      "  warnings.warn(msg, UserWarning)\n"
+     ]
+    }
+   ],
    "source": [
     "%matplotlib inline\n",
     "#%config InlineBackend.figure_format = 'svg'\n",
@@ -54,9 +70,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Diagnostics done using: master_catalogue_bootes_20171121.fits\n"
+     ]
+    }
+   ],
    "source": [
     "OUT_DIR = os.environ.get('OUT_DIR', \"./data\")\n",
     "SUFFIX = find_last_ml_suffix()\n",

diff --git a/dmu1/dmu1_ml_Herschel-Stripe-82/1.10_SDSS-S82.ipynb b/dmu1/dmu1_ml_Herschel-Stripe-82/1.10_SDSS-S82.ipynb
diff --git a/dmu1/dmu1_ml_Herschel-Stripe-82/1.1_HSC-SSP.ipynb b/dmu1/dmu1_ml_Herschel-Stripe-82/1.1_HSC-SSP.ipynb
diff --git a/dmu1/dmu1_ml_Herschel-Stripe-82/1.2_VISTA-VHS.ipynb b/dmu1/dmu1_ml_Herschel-Stripe-82/1.2_VISTA-VHS.ipynb
diff --git a/dmu1/dmu1_ml_Herschel-Stripe-82/1.3_VICS82.ipynb b/dmu1/dmu1_ml_Herschel-Stripe-82/1.3_VICS82.ipynb
diff --git a/dmu1/dmu1_ml_Herschel-Stripe-82/1.4_UKIDSS-LAS.ipynb b/dmu1/dmu1_ml_Herschel-Stripe-82/1.4_UKIDSS-LAS.ipynb
diff --git a/dmu1/dmu1_ml_Herschel-Stripe-82/1.5_PanSTARRS-3SS.ipynb b/dmu1/dmu1_ml_Herschel-Stripe-82/1.5_PanSTARRS-3SS.ipynb
diff --git a/dmu1/dmu1_ml_Herschel-Stripe-82/1.6_SHELA.ipynb b/dmu1/dmu1_ml_Herschel-Stripe-82/1.6_SHELA.ipynb
diff --git a/dmu1/dmu1_ml_Herschel-Stripe-82/1.7_SpIES.ipynb b/dmu1/dmu1_ml_Herschel-Stripe-82/1.7_SpIES.ipynb
diff --git a/dmu1/dmu1_ml_Herschel-Stripe-82/1.8_DECaLS.ipynb b/dmu1/dmu1_ml_Herschel-Stripe-82/1.8_DECaLS.ipynb
diff --git a/dmu1/dmu1_ml_Herschel-Stripe-82/1.9_RCSLenS.ipynb b/dmu1/dmu1_ml_Herschel-Stripe-82/1.9_RCSLenS.ipynb
diff --git a/dmu1/dmu1_ml_Herschel-Stripe-82/2_Merging.ipynb b/dmu1/dmu1_ml_Herschel-Stripe-82/2_Merging.ipynb
diff --git a/dmu1/dmu1_ml_Herschel-Stripe-82/3_Checks_and_diagnostics.ipynb b/dmu1/dmu1_ml_Herschel-Stripe-82/3_Checks_and_diagnostics.ipynb