Starting join based merge on XMM-LSS

#4 as well as some changes to starmask code
H-E-L-P · Dec 14, 2017 · 7ee5a53 · 7ee5a53
1 parent c89f2c0
commit 7ee5a53
Show file tree

Hide file tree

Showing 10 changed files with 15,838 additions and 42 deletions.
diff --git a/dmu1/dmu1_ml_Herschel-Stripe-82/1.10.1_SDSS-S82-IAC.ipynb b/dmu1/dmu1_ml_Herschel-Stripe-82/1.10.1_SDSS-S82-IAC.ipynb
diff --git a/dmu1/dmu1_ml_Herschel-Stripe-82/2.3_Make_full_catalogue.sh b/dmu1/dmu1_ml_Herschel-Stripe-82/2.3_Make_full_catalogue.sh
@@ -11,6 +11,6 @@
 #$ -m eas
 #$ -M [email protected]
 
-ls ./data/sub_catalogue_herschel-stripe-82_20171206_*.fits > fits_list.lis
+ls ./data/sub_catalogue_herschel-stripe-82_20171213_*.fits > fits_list.lis
 
-stilts tcat in=@fits_list.lis out=./data/master_catalogue_herschel-stripe-82_20171206.fits
+stilts tcat in=@fits_list.lis out=./data/master_catalogue_herschel-stripe-82_20171213.fits
diff --git a/dmu1/dmu1_ml_XMM-LSS/2.1_Megacam_merge.ipynb b/dmu1/dmu1_ml_XMM-LSS/2.1_Megacam_merge.ipynb
diff --git a/dmu1/dmu1_ml_XMM-LSS/2.2_WFCAM_merge.ipynb b/dmu1/dmu1_ml_XMM-LSS/2.2_WFCAM_merge.ipynb
diff --git a/dmu1/dmu1_ml_XMM-LSS/2.3_HSC_merge.ipynb b/dmu1/dmu1_ml_XMM-LSS/2.3_HSC_merge.ipynb
diff --git a/dmu1/dmu1_ml_XMM-LSS/2.4_VIRCAM_merge.ipynb b/dmu1/dmu1_ml_XMM-LSS/2.4_VIRCAM_merge.ipynb
diff --git a/dmu1/dmu1_ml_XMM-LSS/2.5_IRAC_merge.ipynb b/dmu1/dmu1_ml_XMM-LSS/2.5_IRAC_merge.ipynb
diff --git a/...mu1_ml_XMM-LSS/2_Merging_low_memory.ipynb → ...1_ml_XMM-LSS/2.6_Merging_low_memory.ipynb b/...mu1_ml_XMM-LSS/2_Merging_low_memory.ipynb → ...1_ml_XMM-LSS/2.6_Merging_low_memory.ipynb
diff --git a/dmu1/dmu1_ml_XMM-LSS/2_Merging.ipynb b/dmu1/dmu1_ml_XMM-LSS/2_Merging.ipynb
@@ -3137,7 +3137,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.0"
+   "version": "3.6.1"
   }
  },
  "nbformat": 4,

diff --git a/dmu4/dmu4_sm_Lockman-SWIRE/generate_holes.ipynb b/dmu4/dmu4_sm_Lockman-SWIRE/generate_holes.ipynb
@@ -55,8 +55,8 @@
    "outputs": [],
    "source": [
     "OUT_DIR = os.environ.get('OUT_DIR', \"./data\")\n",
-    "SUFFIX = os.environ.get('SUFFIX', time.strftime(\"_%Y%m%d\"))\n",
-    "SUFFIX = SUFFIX + '_v2'\n",
+    "SUFFIX = os.environ.get('SUFFIX', time.strftime(\"%Y%m%d\"))\n",
+    "SUFFIX = SUFFIX #+ '_v2'\n",
     "\n",
     "try:\n",
     "    os.makedirs(OUT_DIR)\n",
@@ -119,7 +119,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "metadata": {
     "scrolled": false
    },
@@ -128,7 +128,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "There are 3444 GAIA stars in Lockman-SWIRE     \n",
+      "There are 58813 GAIA stars in Lockman-SWIRE\n",
       "Starmask written to data/10_arcsec_holes_Lockman-SWIRE.reg\n"
      ]
     }
@@ -149,7 +149,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
@@ -222,7 +222,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
    "metadata": {
     "collapsed": true
    },
@@ -259,14 +259,31 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 8,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "There are 58813 GAIA stars in Lockman-SWIRE\n",
+      "Starmask written to data/holes_Lockman-SWIRE_megacam_r_20171214.reg\n",
+      "There are 58813 GAIA stars in Lockman-SWIRE\n",
+      "Starmask written to data/holes_Lockman-SWIRE_gpc1_r_20171214.reg\n",
+      "There are 58813 GAIA stars in Lockman-SWIRE\n",
+      "Starmask written to data/holes_Lockman-SWIRE_irac1_20171214.reg\n",
+      "There are 58813 GAIA stars in Lockman-SWIRE\n",
+      "Starmask written to data/holes_Lockman-SWIRE_ukidss_k_20171214.reg\n",
+      "There are 58813 GAIA stars in Lockman-SWIRE\n",
+      "Starmask written to data/holes_Lockman-SWIRE_wfc_r_20171214.reg\n"
+     ]
+    }
+   ],
    "source": [
     "for pristine_cat in per_catalogue_params:\n",
     "    starmask.create_holes('../../dmu0/dmu0_GAIA/data/GAIA_{}.fits'.format(pristine_cat[0]),    #GAIA catalogue\n",
-    "                          'data/holes_{}_{}{}.reg'.format(field,pristine_cat[1],SUFFIX), #output file\n",
-    "                          moc_files[pristine_cat[1]],                                       #moc of survey region\n",
+    "                          'data/holes_{}_{}_{}.reg'.format(field,pristine_cat[1],SUFFIX), #output file\n",
+    "                          '../../dmu2/dmu2_field_coverages/{}_MOC.fits'.format(field),                                       #moc of survey region\n",
     "                          AB = [pristine_cat[2],pristine_cat[3]],                           #radius as f of mag params\n",
     "                          mag_lim = pristine_cat[4])                                        #upper magnitude limit"
    ]
@@ -280,15 +297,65 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 9,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
+      "\n",
+      "WCSAXES =                    2 / Number of coordinate axes\n",
+      "CRPIX1  =            [astropy.io.fits.card]\n",
+      "WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
+      "     0.5 / Pixel coordinate of reference point\n",
+      "CRPIX2  =                0.5 / Pi [astropy.io.fits.card]\n",
+      "WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
+      "xel coordinate of reference point\n",
+      "CDELT1  =                0.675 / [deg] Coordin [astropy.io.fits.card]\n",
+      "WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
+      "ate increment at reference point\n",
+      "CDELT2  =                0.675 / [deg] Coordina [astropy.io.fits.card]\n",
+      "WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
+      "te increment at reference point\n",
+      "RADECSYSa= 'ICRS    '           / International  [astropy.io.fits.card]\n",
+      "WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
+      "Celestial Ref. System\n",
+      "CUNIT1  = 'deg     '                / Units of coordinate  [astropy.io.fits.card]\n",
+      "WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
+      "increment and value\n",
+      "CUNIT2  = 'deg     '                / Units of coordinate in [astropy.io.fits.card]\n",
+      "WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
+      "crement and value\n",
+      "CTYPE1  = 'RA---TAN'           / \n",
+      "CTYPE2  = 'DEC--TAN'         [astropy.io.fits.card]\n",
+      "WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
+      "   / \n",
+      "CRVAL1  =                  161.942787703 / [deg] Coordinate value at refer [astropy.io.fits.card]\n",
+      "WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
+      "ence point\n",
+      "CRVAL2  =                  59.0563805825 / [deg] Coordinate value at  [astropy.io.fits.card]\n",
+      "WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
+      "reference point\n",
+      "LONPOLE =                  0.0 / [deg] Native longitude of celes [astropy.io.fits.card]\n",
+      "WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
+      "tial pole\n",
+      "LATPOLE = 90.0 / [deg] Native latitude of celestial pole\n",
+      "              [astropy.io.fits.card]\n",
+      "WARNING: FITSFixedWarning:  0.5 / Pixel coordinate of reference point\n",
+      "a floating-point value was expected. [astropy.wcs.wcs]\n",
+      "WARNING: FITSFixedWarning:  / \n",
+      "the RADECSYS keyword is deprecated, use RADESYSa. [astropy.wcs.wcs]\n"
+     ]
+    }
+   ],
    "source": [
     "ORDER=16\n",
     "for pristine_cat in per_catalogue_params:\n",
-    "    starmask.reg2moc('data/holes_{}_{}{}.reg'.format(field,pristine_cat[1],SUFFIX),\n",
-    "            '../../dmu2/dmu2_field_coverages/' + field + '_MOC.fits',\n",
-    "            'data/holes_{}_{}_O{}_MOC.fits'.format(field,pristine_cat[1],str(ORDER)),\n",
+    "    starmask.reg2moc('data/holes_{}_{}_{}.reg'.format(field,pristine_cat[1],SUFFIX),\n",
+    "            '../../dmu2/dmu2_field_coverages/{}_MOC.fits'.format(field),\n",
+    "            'data/holes_{}_{}_{}_O{}_MOC.fits'.format(field,pristine_cat[1],SUFFIX,str(ORDER)),\n",
     "            ra_typ=fields[field][0],\n",
     "            dec_typ=fields[field][1],\n",
     "            order=ORDER)"