-
Notifications
You must be signed in to change notification settings - Fork 1
/
iterateList_runZonalStats__ATL08.py
115 lines (88 loc) · 4.68 KB
/
iterateList_runZonalStats__ATL08.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
"""
3/21/2019: copied from iterateList_runZonalStats
used temporarily for ATL08 until ZS rewrite is done
# CURRENT (3/25):
"""
import os, sys
overwrite = False
lineS = sys.argv[1] # line start
lineE = sys.argv[2] # enter like 1 20 21 30, etc
# CHANGE:
name = 'LVIS' # for naming output dir and db/csv
outdir_base = '/att/gpfsfs/briskfs01/ppl/mwooten3/3DSI/oldCode_zonalStats/{}'.format(name)
inList = os.path.join(outdir_base, '_inputList.txt') # get list of stacks to run based on input
#inList = '/att/gpfsfs/briskfs01/ppl/pmontesa/userfs02/projects/3dsi/stacks/Out_SGM/completed_stack_files.txt'
# set up some parameters
#runScript = '/att/home/mwooten3/code/HRSI/run_GLAS_zonal_database.py'
runScript = '/att/home/mwooten3/code/HRSI/GLAS_zonalStats_to_database.py'
#outDir = os.path.join(outdir_base, 'outputs')
#shpDir = os.path.join(outdir_base, 'shp')
logDir = os.path.join(outdir_base, '_logs')
#for d in [outDir, shpDir, logDir]: # for extra measure
os.system('mkdir -p {}'.format(logDir))
# where all outputs for the run will go
mainDb = os.path.join(outdir_base, 'ATL08-{}__zonalStats_15m.csv'.format(name))
ATLgdb = '/att/gpfsfs/briskfs01/ppl/mwooten3/3DSI/ATL08/ATL08.gdb'
with open (inList, 'r') as il:
inStacks_all = [x.strip('\r\n') for x in il.readlines()] # _all stacks in list
inStacks = inStacks_all[int(lineS)-1:int(lineE)] # subset the scenes we are interested in based on inputs
print "\nProcessing {} stacks...".format(len(inStacks))
# now iterate through stacks and run
c = 0
for stack in inStacks:
c+=1
bname = os.path.basename(stack).strip(os.path.splitext(stack)[1]).strip('_stack')
logFile = os.path.join(logDir, 'ATL08-{}_{}__Log.txt'.format(name, bname))
outDir = os.path.join(outdir_base, bname)
os.system('mkdir -p {}'.format(outDir))
# skip files whose .shx file (last file to be created) already exists
if not overwrite:
if os.path.isfile(os.path.join(outDir, '{}__stats.shx'.format(bname))):
print "\nOutputs for {} already exist\n".format(bname)
continue
# call ex. for ATL08:
#python GLAS_zonalStats_to_database.py <vrt> <gdb> '' <outDir> '' <logfile> <db>
comm = "python {} {} {} ' ' {} ' ' {}".format(runScript, stack, ATLgdb, outDir, logFile, mainDb)
if mainDb:
comm += ' -mainDatabasePrefix {}'.format(mainDb)
print "\n{}/{}:".format(c, len(inStacks))
print comm
os.system(comm)
## # new 3/28/2019: make a copy of the output csv to go back into input dir
## bname = os.path.basename(os.path.splitext(stack)[0]).replace('_stack', '__stats.csv')
## outCsv = os.path.join(outDir, bname)
## toDir = os.path.dirname(stack)
##
## if os.path.isfile(outCsv):
## os.system('cp {} {}'.format(outCsv, toDir))
## else: print '{} DNE'.format(outCsv)
"""
KEEP THIS FOR NOW IN CASE WE HAVE TO COPY STUFF ABOUT SKIPPING STACKS THAT ARE FINISHED
for area in areas:
runScript = '/att/home/mwooten3/code/HRSI/run_GLAS_zonal_database.py'
indir = '/att/gpfsfs/briskfs01/ppl/wcwagne1/3DSI/hrsi_chms/{}/'.format(area) #*
# indir is set up like this: indir/<pairname>/stack.tif'
area = area.strip('Stacks_') # 11/27 just in case it already has Stacks_ in the name, remove it
globDir = glob.glob(os.path.join(indir, '*', '*stack.vrt')) # some stack dirs still use tif's not vrt
if len(globDir) == 0: globDir = glob.glob(os.path.join(indir, '*', '*stack.tif'))
outdir_base = '/att/gpfsfs/briskfs01/ppl/mwooten3/3DSI/GLAS_zonal/Stacks_{}'.format(area) #*
outDir = os.path.join(outdir_base, 'outputs')
shpDir = os.path.join(outdir_base, 'shp')
logDir = os.path.join(outdir_base, 'logs')
# set mainDb = '' if you want to use the default -- 11/27 use separate for now
mainDb = '/att/gpfsfs/briskfs01/ppl/mwooten3/3DSI/GLAS_zonal/Stacks_{0}/{0}_Stacks__zonalStats_15m.csv'.format(area)
for d in [outDir, shpDir, logDir]:
os.system('mkdir -p {}'.format(d))
# default buffer size (15m); default zstats
for pairStack in globDir:
## # temporarily skip list of pairs that were run - stopped after due to system err
## if os.path.basename(pairStack) in skipFiles:
## print "{} already in db. skipping".format(os.path.basename(pairStack))
## continue
comm = 'python {} {} -shpDir {} -outDir {} -logDir {}'.format(runScript, pairStack, shpDir, outDir, logDir)
## comm = 'python {} {} -shpDir {} -outDir {} -logDir {} -mainDatabasePrefix /att/gpfsfs/briskfs01/ppl/mwooten3/3DSI/3DSI_GLAS_stats_database_15m__p2.csv'.format(runScript, pairStack, shpDir, outDir, logDir) # TEMP
if mainDb:
comm += ' -mainDatabasePrefix {}'.format(mainDb)
print comm
os.system(comm)
"""