diff --git a/dmu1/dmu1_ml_Herschel-Stripe-82/3_Checks_and_diagnostics_subset.ipynb b/dmu1/dmu1_ml_Herschel-Stripe-82/3_Checks_and_diagnostics_subset.ipynb deleted file mode 100644 index f686af07..00000000 --- a/dmu1/dmu1_ml_Herschel-Stripe-82/3_Checks_and_diagnostics_subset.ipynb +++ /dev/null @@ -1,477 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "collapsed": true - }, - "source": [ - "# Herschel Stripe 82 master catalogue\n", - "## Checks and diagnostics" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from herschelhelp_internal import git_version\n", - "print(\"This notebook was run with herschelhelp_internal version: \\n{}\".format(git_version()))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%matplotlib inline\n", - "#%config InlineBackend.figure_format = 'svg'\n", - "\n", - "import matplotlib.pyplot as plt\n", - "plt.rc('figure', figsize=(10, 6))\n", - "plt.style.use('ggplot')\n", - "\n", - "import locale\n", - "locale.setlocale(locale.LC_ALL, 'en_GB')\n", - "\n", - "import os\n", - "import time\n", - "import itertools\n", - "\n", - "\n", - "from astropy.coordinates import SkyCoord\n", - "from astropy.table import Table\n", - "from astropy import units as u\n", - "import numpy as np\n", - "from matplotlib_venn import venn3\n", - "\n", - "from herschelhelp_internal.masterlist import nb_compare_mags, nb_ccplots, find_last_ml_suffix, quick_checks" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "OUT_DIR = os.environ.get('OUT_DIR', \"./data\")\n", - "SUFFIX = find_last_ml_suffix()\n", - "#SUFFIX = \"20170627\"\n", - "\n", - "master_catalogue_filename = \"master_catalogue_herschel-stripe-82_{}_{}.fits\".format('RANDOM10PCSAMPLE', SUFFIX)\n", - "master_catalogue = Table.read(\"{}/{}\".format(OUT_DIR, master_catalogue_filename))\n", - "\n", - "print(\"Diagnostics done using: {}\".format(master_catalogue_filename))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 0 - Quick checks" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "quick_checks(master_catalogue).show_in_notebook()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## I - Summary of wavelength domains" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "flag_obs = master_catalogue['flag_optnir_obs']\n", - "flag_det = master_catalogue['flag_optnir_det']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "venn3(\n", - " [\n", - " np.sum(flag_obs == 4),\n", - " np.sum(flag_obs == 2),\n", - " np.sum(flag_obs == 6),\n", - " np.sum(flag_obs == 1),\n", - " np.sum(flag_obs == 5),\n", - " np.sum(flag_obs == 3),\n", - " np.sum(flag_obs == 7)\n", - " ],\n", - " set_labels=('mid-IR', 'near-IR', 'Optical'),\n", - " subset_label_formatter=lambda x: \"{}%\".format(int(100*x/len(flag_obs)))\n", - ")\n", - "plt.title(\"Wavelength domain observations\");" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "venn3(\n", - " [\n", - " np.sum(flag_det[flag_obs == 7] == 4),\n", - " np.sum(flag_det[flag_obs == 7] == 2),\n", - " np.sum(flag_det[flag_obs == 7] == 6),\n", - " np.sum(flag_det[flag_obs == 7] == 1),\n", - " np.sum(flag_det[flag_obs == 7] == 5),\n", - " np.sum(flag_det[flag_obs == 7] == 3),\n", - " np.sum(flag_det[flag_obs == 7] == 7)\n", - " ],\n", - " set_labels=('mid-IR', 'near-IR', 'Optical'),\n", - " subset_label_formatter=lambda x: \"{}%\".format(int(100*x/np.sum(flag_det != 0)))\n", - ")\n", - "plt.title(\"Detection of the {} sources detected\\n in any wavelength domains \"\n", - " \"(among {} sources)\".format(\n", - " locale.format('%d', np.sum(flag_det != 0), grouping=True),\n", - " locale.format('%d', len(flag_det), grouping=True)));" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## II - Comparing magnitudes in similar filters\n", - "\n", - "The master list if composed of several catalogues containing magnitudes in similar filters on different instruments. We are comparing the magnitudes in these corresponding filters." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "u_bands = [ \"SDSS u\"]\n", - "g_bands = [ \"SDSS g\", \"DECam g\" ,\"GPC1 g\", \"Suprime g\", \"RCS g\"]\n", - "r_bands = [ \"SDSS r\", \"DECam r\" ,\"GPC1 r\", \"Suprime r\", \"RCS r\"]\n", - "i_bands = [ \"SDSS i\" ,\"GPC1 i\", \"Suprime i\", \"RCS i\"]\n", - "z_bands = [ \"SDSS z\", \"DECam z\" ,\"GPC1 z\", \"Suprime z\", \"RCS z\"]\n", - "y_bands = [ \"GPC1 y\", \"Suprime y\", \"RCS y\", \"VHS Y\", \"UKIDSS Y\"] #This will compare lower case y to upper case Y\n", - "J_bands = [ \"VHS J\", \"UKIDSS J\", \"VICS82 J\"]\n", - "H_bands = [ \"VHS H\", \"UKIDSS H\"]\n", - "K_bands = [ \"VHS K\", \"UKIDSS K\", \"VICS82 K\"]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": false - }, - "outputs": [], - "source": [ - "for band_of_a_kind in [u_bands, g_bands, r_bands, i_bands, z_bands, y_bands, J_bands, H_bands, K_bands]:\n", - " for band1, band2 in itertools.combinations(band_of_a_kind, 2):\n", - " \n", - " basecol1, basecol2 = band1.replace(\" \", \"_\").lower(), band2.replace(\" \", \"_\").lower()\n", - " \n", - " col1, col2 = \"m_ap_{}\".format(basecol1), \"m_ap_{}\".format(basecol2)\n", - " nb_compare_mags(master_catalogue[col1], master_catalogue[col2], \n", - " labels=(\"{} (aperture)\".format(band1), \"{} (aperture)\".format(band2)))\n", - " \n", - " col1, col2 = \"m_{}\".format(basecol1), \"m_{}\".format(basecol2)\n", - " nb_compare_mags(master_catalogue[col1], master_catalogue[col2], \n", - " labels=(\"{} (total)\".format(band1), \"{} (total)\".format(band2)))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## III - Comparing magnitudes to reference bands\n", - "\n", - "Cross-match the master list to SDSS and 2MASS to compare its magnitudes to SDSS and 2MASS ones." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "master_catalogue_coords = SkyCoord(master_catalogue['ra'], master_catalogue['dec'])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### III.a - Comparing u, g, r, i, and z bands to SDSS\n", - "\n", - "The catalogue is cross-matched to SDSS-DR13 withing 0.2 arcsecond. \n", - "\n", - "We compare the u, g, r, i, and z magnitudes to those from SDSS using `fiberMag` for the aperture magnitude and `petroMag` for the total magnitude." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "sdss = Table.read(\"../../dmu0/dmu0_SDSS-DR13/data/SDSS-DR13_Herschel-Stripe-82.fits\")\n", - "sdss_coords = SkyCoord(sdss['ra'] * u.deg, sdss['dec'] * u.deg)\n", - "\n", - "idx, d2d, _ = sdss_coords.match_to_catalog_sky(master_catalogue_coords)\n", - "mask = (d2d < 0.2 * u.arcsec)\n", - "\n", - "sdss = sdss[mask]\n", - "ml_sdss_idx = idx[mask]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": false - }, - "outputs": [], - "source": [ - "\n", - "for band_of_a_kind in [u_bands, g_bands, r_bands, i_bands, z_bands]:\n", - " for band in band_of_a_kind:\n", - " \n", - " sdss_mag_ap_colname = \"fiberMag_{}\".format(band[-1])\n", - " sdss_mag_tot_colname = \"petroMag_{}\".format(band[-1])\n", - " master_cat_mag_ap_colname = \"m_ap_{}\".format(band.replace(\" \", \"_\").lower())\n", - " master_cat_mag_tot_colname = \"m_{}\".format(band.replace(\" \", \"_\").lower())\n", - " \n", - " # Set SDSS magnitudes to NaN when the magnitude is negative as SDSS uses large\n", - " # negative numbers for missing magnitudes\n", - " sdss[sdss_mag_ap_colname][sdss[sdss_mag_ap_colname] < 0.] = np.nan\n", - " sdss[sdss_mag_tot_colname][sdss[sdss_mag_tot_colname] < 0.] = np.nan\n", - " \n", - "\n", - " sdss_mag_ap = sdss[sdss_mag_ap_colname]\n", - " master_cat_mag_ap = master_catalogue[master_cat_mag_ap_colname][ml_sdss_idx]\n", - " \n", - " nb_compare_mags(sdss_mag_ap, master_cat_mag_ap,\n", - " labels=(\"SDSS (Main survey DR13) {} (fiberMag)\".format(band[-1]), \"{} (aperture)\".format(band)))\n", - " \n", - " sdss_mag_tot = sdss[sdss_mag_tot_colname]\n", - " master_cat_mag_tot = master_catalogue[master_cat_mag_tot_colname][ml_sdss_idx]\n", - " \n", - " nb_compare_mags(sdss_mag_tot, master_cat_mag_tot,\n", - " labels=(\"SDSS (Main survey DR13) {} (petroMag)\".format(band[-1]), \"{} (total)\".format(band)))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### III.b - Comparing J and K bands to 2MASS\n", - "\n", - "The catalogue is cross-matched to 2MASS-PSC withing 0.2 arcsecond. \n", - "\n", - "We compare the UKIDSS total J and K magnitudes to those from 2MASS." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "twomass = Table.read(\"../../dmu0/dmu0_2MASS-point-sources/data/2MASS-PSC_Herschel-Stripe-82.fits\")\n", - "twomass_coords = SkyCoord(twomass['raj2000'], twomass['dej2000'])\n", - "\n", - "idx, d2d, _ = twomass_coords.match_to_catalog_sky(master_catalogue_coords)\n", - "mask = (d2d < 0.2 * u.arcsec)\n", - "\n", - "twomass = twomass[mask]\n", - "ml_twomass_idx = idx[mask]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "nb_compare_mags(twomass['jmag'], master_catalogue['m_ukidss_j'][ml_twomass_idx],\n", - " labels=(\"2MASS J\", \"UKIDSS LAS J (total)\"))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "nb_compare_mags(twomass['kmag'], master_catalogue['m_ukidss_k'][ml_twomass_idx],\n", - " labels=(\"2MASS K\", \"UKIDSS LAS K (total)\"))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "collapsed": true - }, - "source": [ - "## IV - Comparing aperture magnitudes to total ones." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "nb_ccplots(\n", - " master_catalogue['m_rcs_r'],\n", - " master_catalogue['m_ap_rcs_r'] - master_catalogue['m_rcs_r'],\n", - " \"r total magnitude (CFHT)\", \"r aperture mag - total mag (CFHT)\",\n", - " master_catalogue[\"stellarity\"],\n", - " invert_x=True\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "collapsed": true - }, - "source": [ - "## V - Color-color and magnitude-color plots" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": false - }, - "outputs": [], - "source": [ - "nb_ccplots(\n", - " master_catalogue['m_gpc1_g'] - master_catalogue['m_gpc1_i'],\n", - " master_catalogue['m_ukidss_j'] - master_catalogue['m_ukidss_k'],\n", - " \"g - i (PanSTARRS)\", \"J - K (UKIDSS LAS)\",\n", - " master_catalogue[\"stellarity\"]\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "nb_ccplots(\n", - " master_catalogue['m_gpc1_i'] - master_catalogue['m_irac_i1'],\n", - " master_catalogue['m_gpc1_g'] - master_catalogue['m_gpc1_i'],\n", - " \"PanSTARRS i - IRAC1\", \"g - i (PanSTARRS)\",\n", - " master_catalogue[\"stellarity\"]\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "nb_ccplots(\n", - " master_catalogue['m_rcs_g'] - master_catalogue['m_rcs_i'],\n", - " master_catalogue['m_rcs_r'] - master_catalogue['m_rcs_y'],\n", - " \"g - i (CFHT)\", \"r - y (CFHT)\",\n", - " master_catalogue[\"stellarity\"]\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "nb_ccplots(\n", - " master_catalogue['m_ukidss_j'] - master_catalogue['m_ukidss_k'],\n", - " master_catalogue['m_gpc1_g'] - master_catalogue['m_ukidss_j'],\n", - " \"J - K (UKIDSS LAS)\", \"g - J (PanSTARRS, UKIDSS)\",\n", - " master_catalogue[\"stellarity\"]\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "nb_ccplots(\n", - " master_catalogue['m_gpc1_i'] - master_catalogue['m_gpc1_z'],\n", - " master_catalogue['m_gpc1_z'] - master_catalogue['m_ukidss_j'],\n", - " \"i - z (PanSTARRS)\", \"z - J (PanSTARRS, UKIDSS LAS)\",\n", - " master_catalogue[\"stellarity\"]\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "nb_ccplots(\n", - " master_catalogue['m_gpc1_i'] - master_catalogue['m_gpc1_z'],\n", - " master_catalogue['m_irac_i1'] - master_catalogue['m_irac_i2'],\n", - " \"i - z (PanSTARRS)\", \"IRAC1 - IRAC2\",\n", - " master_catalogue[\"stellarity\"]\n", - ")" - ] - } - ], - "metadata": { - "anaconda-cloud": {}, - "kernelspec": { - "display_name": "Python (herschelhelp_internal)", - "language": "python", - "name": "helpint" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.1" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -}