Skip to content

Commit

Permalink
Updated starhole code to be run
Browse files Browse the repository at this point in the history 
Also some changes on HS82
  • Loading branch information
@raphaelshirley
raphaelshirley committed Dec 13, 2017
1 parent 042ca78 commit c89f2c0
Show file tree
Hide file tree
Showing 26 changed files with 1,151 additions and 3,832 deletions.
98 changes: 49 additions & 49 deletions dmu1/dmu1_ml_Bootes/1.2_DataFusion-Spitzer.ipynb
View file

Large diffs are not rendered by default.

235 changes: 20 additions & 215 deletions dmu1/dmu1_ml_Bootes/2_HELP_Merging.ipynb
View file

Large diffs are not rendered by default.

10 changes: 6 additions & 4 deletions dmu1/dmu1_ml_Herschel-Stripe-82/1.10.1_SDSS-S82-IAC.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,15 @@
"metadata": {},
"source": [
"# Herschel Stripe 82 master catalogue\n",
"## Preparation of SDSS Stripe 82 - IAC Legac data\n",
"## Preparation of SDSS Stripe 82 - IAC Legacy Survey data\n",
"\n",
"This catalogue comes from `dmu0_SDSS-S82`.\n",
"This catalogue comes from `dmu0_IAC_Stripe82_Legacy_Project`.\n",
"\n",
"One must choose between this catalogue and the official SDSS catalogue in 'dmu0_SDSS-S82'. Currently we choose this one.\n",
"\n",
"In the catalogue, we keep:\n",
"\n",
"- The `object_id` as unique object identifier;\n",
"- We generate a unique object identifier;\n",
"- The position;\n",
"- The u, g, r, i, z, aperture magnitude (for now in 3”);\n",
"- The u, g, r, i, z, auto fluxes and magnitudes.\n",
Expand Down Expand Up @@ -138,7 +140,7 @@
"for column in imported_columns:\n",
" catalogue[column].name = imported_columns[column]\n",
"\n",
"catalogue.add_column(Column(data=np.arange(len(catalogue)), name='sdss_id', dtype=float))\n",
"catalogue.add_column(Column(data=np.arange(len(catalogue)), name='sdss_id', dtype=int))\n",
" \n",
"epoch = 2013\n",
"\n",
Expand Down
113 changes: 60 additions & 53 deletions dmu1/dmu1_ml_Herschel-Stripe-82/2.2_Merging_low_memory.ipynb
View file

Large diffs are not rendered by default.

216 changes: 46 additions & 170 deletions dmu4/dmu4_sm_AKARI-NEP/generate_holes.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"# Generate hole region files\n",
"# AKARI-NEP Generate hole region files\n",
"\n",
"## I. Produce simple 10 arcsec holes. \n",
"First we produce the simplest possible hole regions. For every star brighter than 16 Mag it puts a 10 arcsec circle. \n",
Expand All @@ -14,33 +14,45 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"This notebook was run with herschelhelp_internal version: \n",
"91cdb7b (Fri Oct 20 11:48:11 2017 +0100) [with local modifications]\n"
]
}
],
"outputs": [],
"source": [
"from herschelhelp_internal import git_version\n",
"print(\"This notebook was run with herschelhelp_internal version: \\n{}\".format(git_version()))"
]
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"from herschelhelp_internal import starmask\n",
"from pymoc import MOC"
"from pymoc import MOC\n",
"\n",
"import time\n",
"import os"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"OUT_DIR = os.environ.get('OUT_DIR', \"./data\")\n",
"SUFFIX = os.environ.get('SUFFIX', time.strftime(\"_%Y%m%d\"))\n",
"SUFFIX = SUFFIX + '_WARNING-MADE-WITH-Lockman-SWIRE-PARAMS'\n",
"\n",
"try:\n",
" os.makedirs(OUT_DIR)\n",
"except FileExistsError:\n",
" pass\n",
"\n",
"field = 'AKARI-NEP'"
]
},
{
Expand All @@ -52,12 +64,14 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"\n",
"\n",
"#Field names plus ra dec of first circle\n",
"fields= {\n",
"# Field RA DEC\n",
Expand All @@ -84,8 +98,7 @@
"'xFLS': [261.387059958, 58.0184602211 ],\n",
"'XMM-13hr': [203.318355937, 37.4745777866 ],\n",
"'XMM-LSS': [32.9413834032, -6.02293494708 ]}\n",
"\n",
"field = 'Lockman-SWIRE'"
"\n"
]
},
{
Expand All @@ -97,20 +110,11 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": null,
"metadata": {
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"There are 3444 GAIA stars in Lockman-SWIRE \n",
"Starmask written to data/10_arcsec_holes_Lockman-SWIRE.reg\n"
]
}
],
"outputs": [],
"source": [
"starmask.create_holes('../../dmu0/dmu0_GAIA/data/GAIA_{}.fits'.format(field),\n",
" 'data/10_arcsec_holes_{}.reg'.format(field),\n",
Expand All @@ -127,59 +131,9 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
"\n",
"WCSAXES = 2 / Number of coordinate axes\n",
"CRPIX1 = [astropy.io.fits.card]\n",
"WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
" 0.5 / Pixel coordinate of reference point\n",
"CRPIX2 = 0.5 / Pi [astropy.io.fits.card]\n",
"WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
"xel coordinate of reference point\n",
"CDELT1 = 0.675 / [deg] Coordin [astropy.io.fits.card]\n",
"WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
"ate increment at reference point\n",
"CDELT2 = 0.675 / [deg] Coordina [astropy.io.fits.card]\n",
"WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
"te increment at reference point\n",
"RADECSYSa= 'ICRS ' / International [astropy.io.fits.card]\n",
"WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
"Celestial Ref. System\n",
"CUNIT1 = 'deg ' / Units of coordinate [astropy.io.fits.card]\n",
"WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
"increment and value\n",
"CUNIT2 = 'deg ' / Units of coordinate in [astropy.io.fits.card]\n",
"WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
"crement and value\n",
"CTYPE1 = 'RA---TAN' / \n",
"CTYPE2 = 'DEC--TAN' [astropy.io.fits.card]\n",
"WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
" / \n",
"CRVAL1 = 161.942787703 / [deg] Coordinate value at refer [astropy.io.fits.card]\n",
"WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
"ence point\n",
"CRVAL2 = 59.0563805825 / [deg] Coordinate value at [astropy.io.fits.card]\n",
"WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
"reference point\n",
"LONPOLE = 0.0 / [deg] Native longitude of celes [astropy.io.fits.card]\n",
"WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
"tial pole\n",
"LATPOLE = 90.0 / [deg] Native latitude of celestial pole\n",
" [astropy.io.fits.card]\n",
"WARNING: FITSFixedWarning: 0.5 / Pixel coordinate of reference point\n",
"a floating-point value was expected. [astropy.wcs.wcs]\n",
"WARNING: FITSFixedWarning: / \n",
"the RADECSYS keyword is deprecated, use RADESYSa. [astropy.wcs.wcs]\n"
]
}
],
"outputs": [],
"source": [
"ORDER=16\n",
"starmask.reg2moc('data/10_arcsec_holes_{}.reg'.format(field),\n",
Expand All @@ -200,13 +154,13 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# Run Lockman-SWIRE with parameters from IDL code. The IDL output files are in ./data/\n",
"# Run with parameters from IDL code. The IDL output files are in ./data/\n",
"\n",
"per_catalogue_params = [\n",
"# Field Band A B magnitude limit\n",
Expand All @@ -217,48 +171,20 @@
" [field, 'wfc_r', 3.43951, -0.173789, 18] # Using 18 as faint limit.\n",
"] \n",
"\n",
"#Combine SERVS and SWIRE mocs because irac1 fluxes can be from either\n",
"swire_moc = MOC(filename = '../../dmu0/dmu0_DataFusion-Spitzer/data/DF-SWIRE_{}_MOC.fits'.format(field))\n",
"servs_moc = MOC(filename = '../../dmu0/dmu0_DataFusion-Spitzer/data/DF-SERVS_{}_MOC.fits'.format(field))\n",
"irac1_moc = swire_moc + servs_moc\n",
"irac1_moc.write('data/SERVS_SWIRE_COVERAGE_irac1_{}_MOC.fits'.format(field), filetype='fits' ,overwrite=True)\n",
"\n",
"moc_files = {\n",
" 'megacam_r' :'data/SERVS_SWIRE_COVERAGE_irac1_{}_MOC.fits'.format(field),\n",
" 'gpc1_r' :'../../dmu0/dmu0_PanSTARRS1-3SS/data/PanSTARRS1-3SS_{}_MOC.fits'.format(field),\n",
" 'irac1' :'data/SERVS_SWIRE_COVERAGE_irac1_{}_MOC.fits'.format(field),\n",
" 'ukidss_k':'../../dmu0/dmu0_UKIDSS-DXS/data/UKIDSS-DXS_{}_MOC.fits'.format(field),\n",
" 'wfc_r':'../../dmu0/dmu0_INTWFC/data/lh_intwfc_v2.1_HELP_coverage_MOC.fits'\n",
"}"
"#We previously took a moc of the survey to only make holes on that area, which \n",
"#was a mistake as it led to missing holes for the XID+ run\n"
]
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"There are 3444 GAIA stars in Lockman-SWIRE \n",
"Starmask written to data/holes_Lockman-SWIRE_megacam_r.reg\n",
"There are 3444 GAIA stars in Lockman-SWIRE \n",
"Starmask written to data/holes_Lockman-SWIRE_gpc1_r.reg\n",
"There are 3444 GAIA stars in Lockman-SWIRE \n",
"Starmask written to data/holes_Lockman-SWIRE_irac1.reg\n",
"There are 3444 GAIA stars in Lockman-SWIRE \n",
"Starmask written to data/holes_Lockman-SWIRE_ukidss_k.reg\n",
"There are 3444 GAIA stars in Lockman-SWIRE \n",
"Starmask written to data/holes_Lockman-SWIRE_wfc_r.reg\n"
]
}
],
"outputs": [],
"source": [
"for pristine_cat in per_catalogue_params:\n",
" starmask.create_holes('../../dmu0/dmu0_GAIA/data/GAIA_' + pristine_cat[0] + '.fits', #GAIA catalogue\n",
" 'data/holes_' + pristine_cat[0] + '_' + pristine_cat[1] + '.reg', #output file\n",
" moc_files[pristine_cat[1]], #moc of survey region\n",
" 'data/holes_' + pristine_cat[0] + '_' + pristine_cat[1] + SUFFIX + '.reg', #output file\n",
" '../../dmu2/dmu2_field_coverages/' + field + '_MOC.fits', #moc of region in which to produce holes\n",
" AB = [pristine_cat[2],pristine_cat[3]], #radius as f of mag params\n",
" mag_lim = pristine_cat[4]) #upper magnitude limit"
]
Expand All @@ -272,65 +198,15 @@
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
"\n",
"WCSAXES = 2 / Number of coordinate axes\n",
"CRPIX1 = [astropy.io.fits.card]\n",
"WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
" 0.5 / Pixel coordinate of reference point\n",
"CRPIX2 = 0.5 / Pi [astropy.io.fits.card]\n",
"WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
"xel coordinate of reference point\n",
"CDELT1 = 0.675 / [deg] Coordin [astropy.io.fits.card]\n",
"WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
"ate increment at reference point\n",
"CDELT2 = 0.675 / [deg] Coordina [astropy.io.fits.card]\n",
"WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
"te increment at reference point\n",
"RADECSYSa= 'ICRS ' / International [astropy.io.fits.card]\n",
"WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
"Celestial Ref. System\n",
"CUNIT1 = 'deg ' / Units of coordinate [astropy.io.fits.card]\n",
"WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
"increment and value\n",
"CUNIT2 = 'deg ' / Units of coordinate in [astropy.io.fits.card]\n",
"WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
"crement and value\n",
"CTYPE1 = 'RA---TAN' / \n",
"CTYPE2 = 'DEC--TAN' [astropy.io.fits.card]\n",
"WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
" / \n",
"CRVAL1 = 161.942787703 / [deg] Coordinate value at refer [astropy.io.fits.card]\n",
"WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
"ence point\n",
"CRVAL2 = 59.0563805825 / [deg] Coordinate value at [astropy.io.fits.card]\n",
"WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
"reference point\n",
"LONPOLE = 0.0 / [deg] Native longitude of celes [astropy.io.fits.card]\n",
"WARNING: The following header keyword is invalid or follows an unrecognized non-standard convention:\n",
"tial pole\n",
"LATPOLE = 90.0 / [deg] Native latitude of celestial pole\n",
" [astropy.io.fits.card]\n",
"WARNING: FITSFixedWarning: 0.5 / Pixel coordinate of reference point\n",
"a floating-point value was expected. [astropy.wcs.wcs]\n",
"WARNING: FITSFixedWarning: / \n",
"the RADECSYS keyword is deprecated, use RADESYSa. [astropy.wcs.wcs]\n"
]
}
],
"outputs": [],
"source": [
"ORDER=16\n",
"for pristine_cat in per_catalogue_params:\n",
" starmask.reg2moc('data/holes_' + field + '_' + pristine_cat[1] + '.reg',\n",
" starmask.reg2moc('data/holes_' + field + '_' + pristine_cat[1] + SUFFIX + '.reg',\n",
" '../../dmu2/dmu2_field_coverages/' + field + '_MOC.fits',\n",
" 'data/holes_' + field + '_' + pristine_cat[1] + '_O' + str(ORDER) + '_MOC.fits',\n",
" 'data/holes_' + field + '_' + pristine_cat[1] + '_O' + str(ORDER) + SUFFIX + '.fits',\n",
" ra_typ=fields[field][0],\n",
" dec_typ=fields[field][1],\n",
" order=ORDER)"
Expand Down
Loading

0 comments on commit c89f2c0

Please sign in to comment.