Feature to ingest template images from surveys for O4 galaxies

supernova.sql

@@ -494,6 +494,35 @@ CREATE TABLE `notes` (
 
 
 
+# Dump of table o4_galaxies
+# ------------------------------------------------------------
+
+DROP TABLE IF EXISTS `o4_galaxies`;
+
+CREATE TABLE `o4_galaxies` (
+  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
+  `targetid` int(11) NOT NULL,
+  `event_id` varchar(255) NOT NULL,
+  `ra0` double NOT NULL,
+  `dec0` double NOT NULL,
+  `SDSS_g` varchar(255) DEFAULT NULL, 
+  `SDSS_r` varchar(255) DEFAULT NULL,
+  `SDSS_i` varchar(255) DEFAULT NULL,
+  `PS1_g` varchar(255) DEFAULT NULL, 
+  `PS1_r` varchar(255) DEFAULT NULL,
+  `PS1_i` varchar(255) DEFAULT NULL,
+  `DECam_g` varchar(255) DEFAULT NULL, 
+  `DECam_r` varchar(255) DEFAULT NULL,
+  `DECam_i` varchar(255) DEFAULT NULL,
+  `LCO_g` varchar(255) DEFAULT NULL, 
+  `LCO_r` varchar(255) DEFAULT NULL,
+  `LCO_i` varchar(255) DEFAULT NULL,
+  `ingested` tinyint(1) NOT NULL DEFAULT 0,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+
+
 # Dump of table obslog
 # ------------------------------------------------------------
 

trunk/bin/ingesttemplates.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+
+from runlsc import run_cmd
+from argparse import ArgumentParser
+
+
+if __name__ == "__main__":
+    basecmd = 'lscloop.py'
+    logfile = open('ingesttemplates.log', 'w+')
+
+    parser = ArgumentParser()
+    parser.add_argument("-f", "--filter", default='')
+    parser.add_argument("--targetid", default=None, type=int)
+    parser.add_argument("-t", "--timeout", type=float, default=86400.)
+    args = parser.parse_args()
+
+    if args.filter:
+        basecmd += ' -f ' + args.filter
+    if args.targetid is not None:
+        basecmd += ' --targetid ' + str(args.targetid)
+
+    run_cmd(basecmd + ' -s ingestsloan', logfile, args.timeout)
+    #run_cmd(basecmd + ' -s ingestps1', logfile, args.timeout)
+    #run_cmd(basecmd + ' -s ingestdecam', logfile, args.timeout)
+
+    ### Generate PSFs and run cosmic ray removal
+
+    run_cmd(basecmd + ' -e 19990101-20191231 -s psf --filetype 4 --use-sextractor --fwhm 5 --max_apercorr 2', logfile, args.timeout)
+    run_cmd(basecmd + ' -e 19990101-20191231 -s cosmic --filetype 4', logfile, args.timeout)
+

trunk/bin/lscingestsloan.py

@@ -10,19 +10,28 @@
     from LCOGTingest import db_ingest
 
     parser = argparse.ArgumentParser(description='Download and ingest SDSS or PS1 templates')
-    parser.add_argument('images', nargs='+', help='LCOGT images to use for field of view and filter')
+    parser.add_argument('images', nargs='*', help='LCOGT images to use for field of view and filter')
     parser.add_argument('--type', default='sloan', help='Survey to download from', choices=['sloan', 'ps1'])
     parser.add_argument('--ps1frames', default='')
     parser.add_argument('--show', action='store_true', help='show the assembled, swarped SDSS template in DS9')
     parser.add_argument('-F', '--force', action='store_true', help='redownload SDSS images if already present in CWD')
+    parser.add_argument('--targetid', default=None, type=int)
+    parser.add_argument('-T', '--tel', default='')
+    parser.add_argument('-f', '--filter', default='')
     args = parser.parse_args()
     imglist = args.images
     imgtype = args.type
     ps1frames = args.ps1frames
 
     if imgtype =='sloan':
-        for img in imglist:
-            image0, varimg = lsc.sloanimage(img,'sloan','', args.show, args.force)
+        if not imglist and args.targetid and args.tel and args.filter:
+            image0, varimg = lsc.sloanimage('', 'sloan', '', args.show, args.force, args.targetid, args.tel, args.filter)
+        elif not imglist:
+            image0=''
+            print 'no data selected'
+        else:
+            for img in imglist:
+                image0, varimg = lsc.sloanimage(img,'sloan','', args.show, args.force)
     elif imgtype =='ps1':
         print "WARNING: PS1 ingestion works at the moment with single object and filter\n "
         print "please, do not provide multiple objects and filter in the same query"

trunk/bin/lscloop.py

@@ -260,9 +260,7 @@ def run_absphot(img):
                         fake_temptel = 'sbig'
                     elif args.telescope == 'fs':
                         fake_temptel = 'spectral'
-                    elif args.telescope == 'fl':
-                        fake_temptel = 'sinistro'
-                    elif args.telescope == 'fa':
+                    elif args.telescope in ['fl', 'fa', '1m0']:
                         fake_temptel = 'sinistro'
                     elif args.telescope == 'ep':
                         fake_temptel = 'muscat'
@@ -330,5 +328,9 @@ def run_absphot(img):
                            str(ll0['psfmag'][i]), str(ll0['zcat'][i]), str(ll0['mag'][i]), str(ll0['abscat'][i]))
                 print '\n###  total number = ' + str(len(ll0['filename']))
                 lsc.myloopdef.run_merge(listfile[ll['dayobs'] == epo], args.force) # merge images using lacos and swarp
+    elif args.stage == 'ingestsloan':
+        lsc.myloopdef.run_ingestsloan('', 'sloan', show=args.show, force=args.force, targetid=args.targetid, tel='fa', filt=args.filter)
+    elif args.stage == 'ingestps1':
+        lsc.myloopdef.run_ingestsloan('', 'ps1', args.ps1frames, show=args.show, force=args.force, targetid=args.targetid, tel='fa', filt=args.filter)
     else:
         print '\n### no data selected'

trunk/bin/rungwgalaxydiff.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+
+from runlsc import run_cmd
+from lsc.mysqldef import query
+from lsc import conn
+
+if __name__ == "__main__":
+    logfile = open('gwgalaxydiff.log', 'w+')
+    galaxies_to_subtract = query(['select targetid, SDSS_g, SDSS_r, SDSS_i, PS1_g, PS1_r, PS1_i, DECam_g, DECam_r, DECam_i, LCO_g, LCO_r, LCO_i from o4_galaxies'], conn)
+    if galaxies_to_subtract:
+        for galaxy in galaxies_to_subtract:
+            targetid = int(galaxy['targetid'])
+            images_to_subtract = query(['select distinct filter from photlco where targetid={} and filetype=1'.format(targetid)], conn)
+            for filt_dict in images_to_subtract:
+                filt = filt_dict['filter'][0]
+                run_cmd('lscloop.py --targetid {} -f {} -e 20200101-20300101 --filetype 1 -s cosmic'.format(targetid, filt), logfile, 86400.0)
+                if galaxy['SDSS_'+filt]:
+                    run_cmd('lscloop.py --targetid {} -f {} -e 20200101-20300101 -s diff --temptel SDSS --tempdate 19990101-20191231 --normalize t -T 1m0'.format(targetid, filt), logfile, 86400.0)
+                elif galaxy['PS1_'+filt]:
+                    run_cmd('lscloop.py --targetid {} -f {} -e 20200101-20300101 -s diff --temptel PS1 --tempdate 19990101-20191231 --normalize t -T 1m0'.format(targetid, filt), logfile, 86400.0)
+                elif galaxy['DECam_'+filt]:
+                    run_cmd('lscloop.py --targetid {} -f {} -e 20200101-20300101 -s diff --temptel decam --tempdate 19990101-20191231 --normalize t -T 1m0'.format(targetid, filt), logfile, 86400.0)
+                elif galaxy['LCO_'+filt]:
+                    run_cmd('lscloop.py --targetid {} -f {} -e 20200101-20300101 -s diff --tempdate 19990101-20191231 --normalize t -T 1m0'.format(targetid, filt), logfile, 86400.0)
+
+
+

trunk/bin/runo4templateingest.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+
+from lsc.mysqldef import query
+from lsc import conn
+from runlsc import run_cmd
+
+
+if __name__ == "__main__":
+    ### Get list of galaxies with no templates ingested
+    logfile = open('runo4templateingest.log', 'w+')
+    undownloaded_galaxies = query(['select id, targetid from o4_galaxies where ingested = 0'], conn)
+    if undownloaded_galaxies:
+        for galaxy in undownloaded_galaxies:
+            ### Run the template ingestion
+            for filt in ['g', 'r', 'i']:
+                run_cmd('ingesttemplates.py -f {} --targetid {}'.format(filt, galaxy['targetid']), logfile)
+
+                ### Look for templates for this filter in each survey
+                template_query = query(["select filepath, filename from photlco where targetid={} and filetype=4 and filter='{}p'".format(galaxy['targetid'], filt)], conn)
+                if template_query:
+                    ### Update the filepath and ingested columns in the o4_galaxies table
+                    for temp in template_query:
+                        if 'SDSS' in temp['filename']:
+                            update_row = query(["update o4_galaxies set SDSS_{} = '{}', ingested = 1 where id = {}".format(filt, temp['filepath']+temp['filename'], int(galaxy['id']))], conn)
+                        elif 'PS' in temp['filename']:
+                            update_row = query(["update o4_galaxies set PS1_{} = '{}', ingested = 1 where id = {}".format(filt, temp['filepath']+temp['filename'], int(galaxy['id']))], conn)
+                        elif 'dec' in temp['filename'].lower():
+                            update_row = query(["update o4_galaxies set DECam_{} = '{}', ingested = 1 where id = {}".format(filt, temp['filepath']+temp['filename'], int(galaxy['id']))], conn)
+                        else:
+                            update_row = query(["update o4_galaxies set LCO_{} = '{}', ingested = 1 where id = {}".format(filt, temp['filepath']+temp['filename'], int(galaxy['id']))], conn)
+
+

trunk/src/lsc/externaldata.py

@@ -482,17 +482,27 @@ def northupeastleft(filename='', data=None, header=None):
         return data, header
 
 ##################################################################################
-def sloanimage(img,survey='sloan',frames=[], show=False, force=False):
+def sloanimage(img,survey='sloan',frames=[], show=False, force=False, targetid=None, tel='', filt=''):
    import sys
-   from lsc import readhdr, readkey3,deg2HMS,display_image
-   if show:
-      display_image(img,1,True,'','')
-   hdr = readhdr(img)
-   _ra = readkey3(hdr,'RA')
-   _dec = readkey3(hdr,'DEC')
-   _object = readkey3(hdr,'object')
-   _instrume = readkey3(hdr,'instrume')
-   _filter = readkey3(hdr,'filter')
+   from lsc import readhdr, readkey3,deg2HMS,display_image, conn
+   from lsc.mysqldef import query
+   if img:
+      if show:
+         display_image(img,1,True,'','')
+      hdr = readhdr(img)
+      _ra = readkey3(hdr,'RA')
+      _dec = readkey3(hdr,'DEC')
+      _object = readkey3(hdr,'object')
+      _instrume = readkey3(hdr,'instrume')
+      _filter = readkey3(hdr,'filter')
+   elif targetid:
+      targetquery = query(['select ra0, dec0 from targets where id={}'.format(targetid)], conn)
+      namequery = query(['select name from targetnames where targetid={}'.format(targetid)], conn)
+      _ra = float(targetquery[0]['ra0'])
+      _dec = float(targetquery[0]['dec0'])
+      _object = namequery[0]['name']
+      _instrume = tel
+      _filter = filt
    _radius = 1100
    filt={'up':'u','gp':'g','rp':'r','ip':'i','zs':'z'}
 

trunk/src/lsc/myloopdef.py

@@ -1964,8 +1964,10 @@ def run_merge(imglist, _redu=False):
 
 ########################################################################################
 
-def run_ingestsloan(imglist,imgtype = 'sloan', ps1frames='', show=False, force=False):
-    command = 'lscingestsloan.py ' + ' '.join(imglist)
+def run_ingestsloan(imglist,imgtype = 'sloan', ps1frames='', show=False, force=False, targetid=None, tel='fa', filt=''):
+    command = 'lscingestsloan.py'
+    if imglist:
+        command += ' '.join(imglist)
     if imgtype != 'sloan':
         command += ' --type ' + imgtype
     if ps1frames:
@@ -1974,6 +1976,8 @@ def run_ingestsloan(imglist,imgtype = 'sloan', ps1frames='', show=False, force=F
         command += ' --show'
     if force:
         command += ' -F'
+    if targetid and filt:
+        command += ' --targetid ' + str(targetid) + ' --tel ' + tel + ' --filter ' + filt[0]
     print command
     os.system(command)