Skip to content

Commit 3203241

Browse files
raphaelshirleyraphaelshirley
committed
Changes to XMM-LSS
nearly ready to run on Vulcain. TODO: remove id columns and set correct number of objects per subcat and check output fits cat from test run. Also maybe correct masked values on sub cats. #4
1 parent b7687d7 commit 3203241

6 files changed

+884
-651
lines changed

‎dmu1/dmu1_ml_XMM-LSS/2.1_Megacam_merge.ipynb

+37-21
Large diffs are not rendered by default.

‎dmu1/dmu1_ml_XMM-LSS/2.2_UKIDSS_merge.ipynb

+31-73
Original file line numberDiff line numberDiff line change
@@ -65,12 +65,14 @@
6565
{
6666
"cell_type": "code",
6767
"execution_count": 3,
68-
"metadata": {},
68+
"metadata": {
69+
"collapsed": true
70+
},
6971
"outputs": [],
7072
"source": [
7173
"TMP_DIR = os.environ.get('TMP_DIR', \"./data_tmp\")\n",
7274
"OUT_DIR = os.environ.get('OUT_DIR', \"./data\")\n",
73-
"SUFFIX = '' # os.environ.get('SUFFIX', time.strftime(\"_%Y%m%d\"))\n",
75+
"SUFFIX = os.environ.get('SUFFIX', time.strftime(\"_%Y%m%d\"))\n",
7476
"\n",
7577
"try:\n",
7678
" os.makedirs(OUT_DIR)\n",
@@ -225,14 +227,28 @@
225227
{
226228
"cell_type": "code",
227229
"execution_count": 9,
230+
"metadata": {
231+
"collapsed": true
232+
},
233+
"outputs": [],
234+
"source": [
235+
"#Since this is not the final merged catalogue. We rename column names to make them unique\n",
236+
"master_catalogue['ra'].name = 'ukidss_ra'\n",
237+
"master_catalogue['dec'].name = 'ukidss_dec'\n",
238+
"master_catalogue['flag_merged'].name = 'ukidss_flag_merged'"
239+
]
240+
},
241+
{
242+
"cell_type": "code",
243+
"execution_count": 10,
228244
"metadata": {},
229245
"outputs": [
230246
{
231247
"data": {
232248
"text/html": [
233249
"<Table length=10>\n",
234-
"<table id=\"table4768201360-154881\" class=\"table-striped table-bordered table-condensed\">\n",
235-
"<thead><tr><th>idx</th><th>dxs_id</th><th>ra</th><th>dec</th><th>m_ap_ukidss_j</th><th>merr_ap_ukidss_j</th><th>m_ukidss_j</th><th>merr_ukidss_j</th><th>m_ap_ukidss_k</th><th>merr_ap_ukidss_k</th><th>m_ukidss_k</th><th>merr_ukidss_k</th><th>dxs_stellarity</th><th>f_ap_ukidss_j</th><th>ferr_ap_ukidss_j</th><th>f_ukidss_j</th><th>ferr_ukidss_j</th><th>flag_ukidss_j</th><th>f_ap_ukidss_k</th><th>ferr_ap_ukidss_k</th><th>f_ukidss_k</th><th>ferr_ukidss_k</th><th>flag_ukidss_k</th><th>dxs_flag_cleaned</th><th>dxs_flag_gaia</th><th>flag_merged</th><th>uds_id</th><th>m_ap_uds_j</th><th>merr_ap_uds_j</th><th>m_uds_j</th><th>merr_uds_j</th><th>m_ap_uds_h</th><th>merr_ap_uds_h</th><th>m_uds_h</th><th>merr_uds_h</th><th>m_ap_uds_k</th><th>merr_ap_uds_k</th><th>m_uds_k</th><th>merr_uds_k</th><th>uds_stellarity</th><th>f_ap_uds_j</th><th>ferr_ap_uds_j</th><th>f_uds_j</th><th>ferr_uds_j</th><th>flag_uds_j</th><th>f_ap_uds_h</th><th>ferr_ap_uds_h</th><th>f_uds_h</th><th>ferr_uds_h</th><th>flag_uds_h</th><th>f_ap_uds_k</th><th>ferr_ap_uds_k</th><th>f_uds_k</th><th>ferr_uds_k</th><th>flag_uds_k</th><th>uds_flag_cleaned</th><th>uds_flag_gaia</th></tr></thead>\n",
250+
"<table id=\"table4524300496-65840\" class=\"table-striped table-bordered table-condensed\">\n",
251+
"<thead><tr><th>idx</th><th>dxs_id</th><th>ukidss_ra</th><th>ukidss_dec</th><th>m_ap_ukidss_j</th><th>merr_ap_ukidss_j</th><th>m_ukidss_j</th><th>merr_ukidss_j</th><th>m_ap_ukidss_k</th><th>merr_ap_ukidss_k</th><th>m_ukidss_k</th><th>merr_ukidss_k</th><th>dxs_stellarity</th><th>f_ap_ukidss_j</th><th>ferr_ap_ukidss_j</th><th>f_ukidss_j</th><th>ferr_ukidss_j</th><th>flag_ukidss_j</th><th>f_ap_ukidss_k</th><th>ferr_ap_ukidss_k</th><th>f_ukidss_k</th><th>ferr_ukidss_k</th><th>flag_ukidss_k</th><th>dxs_flag_cleaned</th><th>dxs_flag_gaia</th><th>ukidss_flag_merged</th><th>uds_id</th><th>m_ap_uds_j</th><th>merr_ap_uds_j</th><th>m_uds_j</th><th>merr_uds_j</th><th>m_ap_uds_h</th><th>merr_ap_uds_h</th><th>m_uds_h</th><th>merr_uds_h</th><th>m_ap_uds_k</th><th>merr_ap_uds_k</th><th>m_uds_k</th><th>merr_uds_k</th><th>uds_stellarity</th><th>f_ap_uds_j</th><th>ferr_ap_uds_j</th><th>f_uds_j</th><th>ferr_uds_j</th><th>flag_uds_j</th><th>f_ap_uds_h</th><th>ferr_ap_uds_h</th><th>f_uds_h</th><th>ferr_uds_h</th><th>flag_uds_h</th><th>f_ap_uds_k</th><th>ferr_ap_uds_k</th><th>f_uds_k</th><th>ferr_uds_k</th><th>flag_uds_k</th><th>uds_flag_cleaned</th><th>uds_flag_gaia</th></tr></thead>\n",
236252
"<thead><tr><th></th><th></th><th>deg</th><th>deg</th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th><th></th></tr></thead>\n",
237253
"<tr><td>0</td><td>446679621523</td><td>35.5523293864</td><td>-3.89660860231</td><td>11.3550081253</td><td>0.000288982642815</td><td>9.84205055237</td><td>0.000124145561131</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>0.899999976158</td><td>104230.921875</td><td>27.742401123</td><td>419932.625</td><td>48.0160560608</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>False</td><td>3</td><td>False</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>False</td><td>0</td></tr>\n",
238254
"<tr><td>1</td><td>446679652426</td><td>35.5348871949</td><td>-3.71725661147</td><td>11.4726924896</td><td>0.000298160797684</td><td>11.3987607956</td><td>0.000252233905485</td><td>12.2191505432</td><td>0.000333435629727</td><td>12.4362030029</td><td>0.000336990662618</td><td>0.993865013123</td><td>93523.953125</td><td>25.6831951141</td><td>100114.164062</td><td>23.2581233978</td><td>False</td><td>47026.1757812</td><td>14.4420013428</td><td>38505.1289062</td><td>11.9512176514</td><td>False</td><td>False</td><td>3</td><td>False</td><td>-1</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>False</td><td>False</td><td>0</td></tr>\n",
@@ -267,14 +283,14 @@
267283
" datatables: 'https://cdn.datatables.net/1.10.12/js/jquery.dataTables.min'\n",
268284
"}});\n",
269285
"require([\"datatables\"], function(){\n",
270-
" console.log(\"$('#table4768201360-154881').dataTable()\");\n",
286+
" console.log(\"$('#table4524300496-65840').dataTable()\");\n",
271287
" \n",
272288
"jQuery.extend( jQuery.fn.dataTableExt.oSort, {\n",
273289
" \"optionalnum-asc\": astropy_sort_num,\n",
274290
" \"optionalnum-desc\": function (a,b) { return -astropy_sort_num(a, b); }\n",
275291
"});\n",
276292
"\n",
277-
" $('#table4768201360-154881').dataTable({\n",
293+
" $('#table4524300496-65840').dataTable({\n",
278294
" order: [],\n",
279295
" pageLength: 50,\n",
280296
" lengthMenu: [[10, 25, 50, 100, 500, 1000, -1], [10, 25, 50, 100, 500, 1000, 'All']],\n",
@@ -288,7 +304,7 @@
288304
"<IPython.core.display.HTML object>"
289305
]
290306
},
291-
"execution_count": 9,
307+
"execution_count": 10,
292308
"metadata": {},
293309
"output_type": "execute_result"
294310
}
@@ -306,7 +322,7 @@
306322
},
307323
{
308324
"cell_type": "code",
309-
"execution_count": 10,
325+
"execution_count": 11,
310326
"metadata": {
311327
"collapsed": true
312328
},
@@ -320,7 +336,7 @@
320336
},
321337
{
322338
"cell_type": "code",
323-
"execution_count": 11,
339+
"execution_count": 12,
324340
"metadata": {},
325341
"outputs": [
326342
{
@@ -361,7 +377,7 @@
361377
},
362378
{
363379
"cell_type": "code",
364-
"execution_count": 12,
380+
"execution_count": 13,
365381
"metadata": {
366382
"collapsed": true
367383
},
@@ -432,64 +448,6 @@
432448
"ukidss_origin.write(\"{}/xmm-lss_ukidss_fluxes_origins{}.fits\".format(OUT_DIR, SUFFIX), overwrite=True)"
433449
]
434450
},
435-
{
436-
"cell_type": "code",
437-
"execution_count": 13,
438-
"metadata": {},
439-
"outputs": [
440-
{
441-
"data": {
442-
"text/plain": [
443-
"['dxs_id',\n",
444-
" 'ra',\n",
445-
" 'dec',\n",
446-
" 'm_ap_ukidss_j',\n",
447-
" 'merr_ap_ukidss_j',\n",
448-
" 'm_ukidss_j',\n",
449-
" 'merr_ukidss_j',\n",
450-
" 'm_ap_ukidss_k',\n",
451-
" 'merr_ap_ukidss_k',\n",
452-
" 'm_ukidss_k',\n",
453-
" 'merr_ukidss_k',\n",
454-
" 'dxs_stellarity',\n",
455-
" 'f_ap_ukidss_j',\n",
456-
" 'ferr_ap_ukidss_j',\n",
457-
" 'f_ukidss_j',\n",
458-
" 'ferr_ukidss_j',\n",
459-
" 'flag_ukidss_j',\n",
460-
" 'f_ap_ukidss_k',\n",
461-
" 'ferr_ap_ukidss_k',\n",
462-
" 'f_ukidss_k',\n",
463-
" 'ferr_ukidss_k',\n",
464-
" 'flag_ukidss_k',\n",
465-
" 'dxs_flag_cleaned',\n",
466-
" 'dxs_flag_gaia',\n",
467-
" 'flag_merged',\n",
468-
" 'uds_id',\n",
469-
" 'm_ap_ukidss_h',\n",
470-
" 'merr_ap_ukidss_h',\n",
471-
" 'm_ukidss_h',\n",
472-
" 'merr_ukidss_h',\n",
473-
" 'uds_stellarity',\n",
474-
" 'f_ap_ukidss_h',\n",
475-
" 'ferr_ap_ukidss_h',\n",
476-
" 'f_ukidss_h',\n",
477-
" 'ferr_ukidss_h',\n",
478-
" 'flag_ukidss_h',\n",
479-
" 'uds_flag_cleaned',\n",
480-
" 'uds_flag_gaia',\n",
481-
" 'ukidss_intid']"
482-
]
483-
},
484-
"execution_count": 13,
485-
"metadata": {},
486-
"output_type": "execute_result"
487-
}
488-
],
489-
"source": [
490-
"master_catalogue.colnames"
491-
]
492-
},
493451
{
494452
"cell_type": "markdown",
495453
"metadata": {},
@@ -510,7 +468,7 @@
510468
},
511469
{
512470
"cell_type": "code",
513-
"execution_count": 14,
471+
"execution_count": 15,
514472
"metadata": {
515473
"collapsed": true
516474
},
@@ -532,14 +490,14 @@
532490
},
533491
{
534492
"cell_type": "code",
535-
"execution_count": 15,
493+
"execution_count": 16,
536494
"metadata": {},
537495
"outputs": [
538496
{
539497
"name": "stdout",
540498
"output_type": "stream",
541499
"text": [
542-
"Missing columns: {'dxs_stellarity', 'uds_stellarity', 'uds_flag_gaia', 'dxs_id', 'ukidss_intid', 'uds_flag_cleaned', 'uds_id', 'dxs_flag_gaia', 'dxs_flag_cleaned'}\n"
500+
"Missing columns: {'uds_flag_cleaned', 'ukidss_intid', 'uds_stellarity', 'ukidss_dec', 'dxs_stellarity', 'dxs_flag_gaia', 'dxs_id', 'uds_flag_gaia', 'uds_id', 'dxs_flag_cleaned', 'ukidss_flag_merged', 'ukidss_ra'}\n"
543501
]
544502
}
545503
],
@@ -550,13 +508,13 @@
550508
},
551509
{
552510
"cell_type": "code",
553-
"execution_count": 16,
511+
"execution_count": 18,
554512
"metadata": {
555513
"collapsed": true
556514
},
557515
"outputs": [],
558516
"source": [
559-
"master_catalogue.write(\"{}/ukidss_merged_catalogue_xmm-lss{}.fits\".format(TMP_DIR, SUFFIX), overwrite = True)"
517+
"master_catalogue.write(\"{}/ukidss_merged_catalogue_xmm-lss.fits\".format(TMP_DIR), overwrite = True)"
560518
]
561519
}
562520
],

0 commit comments

Comments
 (0)