DECam adding to checks across southern fields

#17 #11, #18, #20'
H-E-L-P · Jan 18, 2018 · 203d709 · 203d709
1 parent d3ed06a
commit 203d709
Show file tree

Hide file tree

Showing 6 changed files with 240 additions and 1,156 deletions.
diff --git a/dmu1/dmu1_ml_AKARI-SEP/3_Checks_and_diagnostics.ipynb b/dmu1/dmu1_ml_AKARI-SEP/3_Checks_and_diagnostics.ipynb
diff --git a/dmu1/dmu1_ml_ELAIS-N1/3_Checks_and_diagnostics.ipynb b/dmu1/dmu1_ml_ELAIS-N1/3_Checks_and_diagnostics.ipynb
@@ -2106,9 +2106,9 @@
     "                        labels=(\"SDSS {} (fiberMag)\".format(band[-1]), \"{} (aperture)\".format(band)))\n",
     "    \n",
     "        sdss_mag_tot = sdss[\"petroMag_{}\".format(band[-1])]\n",
-    "        master_cat_mag_tot = master_catalogue[\"m_ap_{}\".format(band.replace(\" \", \"_\").lower())][ml_sdss_idx]\n",
+    "        master_cat_mag_tot = master_catalogue[\"m_{}\".format(band.replace(\" \", \"_\").lower())][ml_sdss_idx]\n",
     "        \n",
-    "        nb_compare_mags(sdss_mag_ap, master_cat_mag_ap,\n",
+    "        nb_compare_mags(sdss_mag_tot, master_cat_mag_tot,\n",
     "                        labels=(\"SDSS {} (petroMag)\".format(band[-1]), \"{} (total)\".format(band)))"
    ]
   },
@@ -2519,7 +2519,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.0"
+   "version": "3.6.1"
   }
  },
  "nbformat": 4,

diff --git a/dmu1/dmu1_ml_ELAIS-S1/3_Checks_and_diagnostics.ipynb b/dmu1/dmu1_ml_ELAIS-S1/3_Checks_and_diagnostics.ipynb
diff --git a/dmu1/dmu1_ml_SGP/3_Checks_and_diagnostics.ipynb b/dmu1/dmu1_ml_SGP/3_Checks_and_diagnostics.ipynb
diff --git a/dmu1/dmu1_ml_SSDF/2_Merging.ipynb b/dmu1/dmu1_ml_SSDF/2_Merging.ipynb
@@ -12,7 +12,9 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "collapsed": true
+   },
    "outputs": [],
    "source": [
     "from herschelhelp_internal import git_version\n",
@@ -123,7 +125,9 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "collapsed": true
+   },
    "outputs": [],
    "source": [
     "nb_merge_dist_plot(\n",
@@ -209,7 +213,9 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "collapsed": true
+   },
    "outputs": [],
    "source": [
     "master_catalogue[:10].show_in_notebook()"
@@ -278,7 +284,9 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "collapsed": true
+   },
    "outputs": [],
    "source": [
     "stellarity_columns = [column for column in master_catalogue.colnames\n",
@@ -346,7 +354,9 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "collapsed": true
+   },
    "outputs": [],
    "source": [
     "# Check that the HELP Ids are unique\n",
@@ -389,7 +399,9 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "collapsed": true
+   },
    "outputs": [],
    "source": [
     "nb_merge_dist_plot(\n",
@@ -523,18 +535,16 @@
     "    1 * ~np.isnan(master_catalogue['f_decam_y']) \n",
     ")\n",
     "\n",
-    "# VISTA-VHS is a catalogue of sources detected in any of the J, H, and K band (**TODO**: to be confirmed by\n",
-    "# Manda). We then can't say in how many bands a source was detected. Sources detected in mIR are sources\n",
-    "# having an associated vhs_id.\n",
     "nb_nir_flux = (\n",
-    "    1 * (master_catalogue['vhs_id'] != -1)\n",
+    "    1 * ~np.isnan(master_catalogue['f_vista_j']) +\n",
+    "    1 * ~np.isnan(master_catalogue['f_vista_h']) +    \n",
+    "    1 * ~np.isnan(master_catalogue['f_vista_k'])\n",
     ")\n",
     "\n",
-    "# For SSDF, we use the IRAC1 selected sources. For some sources, the catalogue does not contain IRAC1 flux (whereas the sources\n",
-    "# were detected there) but contains the error, which may indicate upper limits. To find the sources that were detected in\n",
-    "# IRAC we select the source which have an associated ssdf_id.\n",
+    "\n",
     "nb_mir_flux = (\n",
-    "    1 * (master_catalogue['ssdf_id'] != -1)\n",
+    "    1 * ~np.isnan(master_catalogue['f_irac_i1']) +\n",
+    "    1 * ~np.isnan(master_catalogue['f_irac_i2'])  \n",
     ")"
    ]
   },
@@ -572,7 +582,9 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "collapsed": true
+   },
    "outputs": [],
    "source": [
     "id_names = []\n",
@@ -654,7 +666,9 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "collapsed": true
+   },
    "outputs": [],
    "source": [
     "# We check for columns in the master catalogue that we will not save to disk.\n",

diff --git a/dmu1/dmu1_ml_SSDF/3_Checks_and_diagnostics.ipynb b/dmu1/dmu1_ml_SSDF/3_Checks_and_diagnostics.ipynb
@@ -140,13 +140,17 @@
     }
    ],
    "source": [
-    "venn2(\n",
+    "venn3(\n",
     "    [\n",
-    "        np.sum(flag_obs == 2),\n",
     "        np.sum(flag_obs == 4),\n",
-    "        np.sum(flag_obs == 6)\n",
+    "        np.sum(flag_obs == 2),\n",
+    "        np.sum(flag_obs == 6),\n",
+    "        np.sum(flag_obs == 1),\n",
+    "        np.sum(flag_obs == 5),\n",
+    "        np.sum(flag_obs == 3),\n",
+    "        np.sum(flag_obs == 7)\n",
     "    ],\n",
-    "    set_labels=('near-IR', 'mid-IR'),\n",
+    "    set_labels=('Optical', 'near-IR', 'mid-IR'),\n",
     "    subset_label_formatter=lambda x: \"{}%\".format(int(100*x/len(flag_obs)))\n",
     ")\n",
     "plt.title(\"Wavelength domain observations\");"
@@ -160,22 +164,23 @@
    },
    "outputs": [],
    "source": [
-    "# We have only one nIR and one mIR catalogue so the detection flag which uses a minimum of two detections\n",
-    "# will be 0 everywhere.\n",
-    "\n",
-    "#venn2(\n",
-    "#    [\n",
-    "#       np.sum(flag_det[flag_obs == 6] == 2),\n",
-    " #      np.sum(flag_det[flag_obs == 6] == 4),\n",
-    "#        np.sum(flag_det[flag_obs == 6] == 6)\n",
-    "#    ],\n",
-    "#    set_labels=('near-IR', 'mid-IR'),\n",
-    "#    subset_label_formatter=lambda x: \"{}%\".format(int(100*x/np.sum(flag_det != 0)))\n",
-    "#)\n",
-    "#plt.title(\"Detection of the {} sources detected\\n in any wavelength domains \"\n",
-    "#          \"(among {} sources)\".format(\n",
-    "#              locale.format('%d', np.sum(flag_det != 0), grouping=True),\n",
-    "#             locale.format('%d', len(flag_det), grouping=True)));"
+    "venn3(\n",
+    "    [\n",
+    "        np.sum(flag_det[flag_obs == 7] == 4),\n",
+    "        np.sum(flag_det[flag_obs == 7] == 2),\n",
+    "        np.sum(flag_det[flag_obs == 7] == 6),\n",
+    "        np.sum(flag_det[flag_obs == 7] == 1),\n",
+    "        np.sum(flag_det[flag_obs == 7] == 5),\n",
+    "        np.sum(flag_det[flag_obs == 7] == 3),\n",
+    "        np.sum(flag_det[flag_obs == 7] == 7)\n",
+    "    ],\n",
+    "    set_labels=('Optical', 'near-IR', 'mid-IR'),\n",
+    "    subset_label_formatter=lambda x: \"{}%\".format(int(100*x/np.sum(flag_det != 0)))\n",
+    ")\n",
+    "plt.title(\"Detection of the {} sources detected\\n in any wavelength domains \"\n",
+    "          \"(among {} sources)\".format(\n",
+    "              locale.format('%d', np.sum(flag_det != 0), grouping=True),\n",
+    "              locale.format('%d', len(flag_det), grouping=True)));"
    ]
   },
   {
@@ -195,10 +200,15 @@
    },
    "outputs": [],
    "source": [
-    "j_bands = [ \"VISTA j\"]\n",
-    "h_bands = [ \"VISTA h\"]\n",
-    "k_bands = [ \"VISTA k\"]\n",
-    "irac_bands = [\"IRAC I1\", \"IRAC I2\"]"
+    "g_bands = [\"DECam g\"]\n",
+    "r_bands = [\"DECam r\"]\n",
+    "i_bands = [\"DECam i\"]\n",
+    "z_bands = [\"DECam z\"]\n",
+    "y_bands = [\"DECam y\"]\n",
+    "j_bands =           [ \"VISTA j\"]\n",
+    "h_bands =           [ \"VISTA h\"]\n",
+    "k_bands =           [ \"VISTA k\"]\n",
+    "irac_bands =                    [\"IRAC I1\", \"IRAC I2\"]"
    ]
   },
   {
@@ -239,7 +249,7 @@
     }
    ],
    "source": [
-    "for bands in [j_bands + h_bands + k_bands, irac_bands]:\n",
+    "for bands in [g_bands + r_bands + i_bands + z_bands + y_bands, j_bands + h_bands + k_bands, irac_bands]:\n",
     "    colnames = [\"m_{}\".format(band.replace(\" \", \"_\").lower()) for band in bands]\n",
     "    nb_histograms(master_catalogue, colnames, bands)"
    ]