-
Notifications
You must be signed in to change notification settings - Fork 2
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Processing on OSG #67
base: master
Are you sure you want to change the base?
Changes from 1 commit
9da727a
b08ae8f
59e0457
c546e55
529a72e
8aa8fca
b3d4600
732517a
fbe8a9c
e49b71f
17ff365
85984da
b3479b9
d14a5d0
c7942c8
536de88
2bea618
755d882
5babaea
f7f86d9
47dfda8
71ee993
e261147
0fe90a6
c59c515
518f8cd
4724bde
9eb2b03
cbaa0da
2ea9df2
62017c1
d413397
759a6ae
90b4d63
45ae1bf
b285824
89eee80
fb7db43
798ba77
16c6971
d247b3c
56be9f4
77d8081
6d64324
bad0664
e921024
ea50970
1aa0af8
ab8c572
9cd2325
9a8ba4b
beb30ad
524917c
48bc695
8045b1a
2f80f25
13c0886
b3da3df
ac8000c
ef82706
fb3f681
29cb5ee
7673e68
755ff45
50f87a7
8f84294
99724dc
8206766
d955f5a
b28b706
b697d8b
1b34fd9
7d057d5
8b3ccb9
68154d5
2db8120
f1a7cc0
f8707bd
2c3de99
e7f218d
bc08822
e5a670b
2dbd1d0
d041356
6b18a4a
bdde49a
a8785f2
c1744d1
91b1b8b
c166b4b
0030597
948d14c
ea972a3
90046dd
f3878cc
d5fab9d
4dfd0c3
68b6a60
8cddb56
d092e2b
44d673e
221317b
0e697eb
889e932
9caed6d
305e4ec
b6000e3
de24493
58c948f
7ee5800
4e781c8
8d2a1e7
55fabbf
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
…ripts, etc. Includes dag_writer.py class that handles writing of dags and subdags.
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
[ | ||
{ | ||
"name": "login", | ||
"method": "scp", | ||
"hostname": "login.ci-connect.uchicago.edu", | ||
"dir_raw": "/xenon/xenon1t_test", | ||
"dir_processed": "/xenon/ershockley/processed", | ||
"upload_options": [], | ||
"username": "pdeperio", | ||
"download_options": [ | ||
"midway-login1"], | ||
"task_list": ["ProcessBatchQueue"], | ||
"dataset_list": ["161021_0841", "161021_0947", "161021_0952", "161021_1152", "161021_1253", "161021_1258", "161021_1358", "161021_1432"], | ||
"pax_processing_versions": ["v4.11.0", "v5.5.1"] | ||
} | ||
] |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
[ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. remove file |
||
{ | ||
"name": "login", | ||
"method": "scp", | ||
"hostname": "login.ci-connect.uchicago.edu", | ||
"dir_raw": "/xenon/xenon1t_test", | ||
"dir_processed": "/xenon/ershockley/processed", | ||
"upload_options": [], | ||
"username": "pdeperio", | ||
"download_options": ["midway-login1"], | ||
"task_list": ["ProcessBatchQueue"], | ||
"dataset_list": ["161021_0841", "161021_0947", "161021_0952", "161021_1152", "161021_1253", "161021_1258", "161021_1358", "161021_1432", "160812_1710", "160812_1810", "160812_1910", "160812_2010", "160812_2111", "160812_2211", "160812_2311", "160813_0011", "160813_0111", "160813_0212", "160813_0312", "160813_0412", "160813_0513", "160813_0613", "160813_0713", "160813_0813", "160813_0913", "160813_1014", "160813_1114", "160813_1214", "160813_1314", "160813_1414", "160813_1514", "160813_1615", "160813_1715", "160813_1815", "160813_1915", "160813_2016", "160813_2116", "160813_2216", "160813_2316", "160814_0016", "160814_0117", "160814_0217", "160814_0317", "160814_0417", "160814_0517", "160814_0618", "160814_0718", "160814_0818", "160814_0918", "160814_1018", "160814_1119", "160814_1219", "160814_1319", "160814_1419", "160814_1519", "160814_1619", "160814_1720", "160814_1820", "160814_1920", "160814_2020", "160814_2120", "160814_2221", "160814_2321", "160815_0021", "160815_0121", "160815_0221", "160815_0321", "160815_0422", "160815_0522", "160815_0622", "160815_0722", "160815_0823", "160815_0923", "160815_1023", "160815_1123", "160815_1223", "160815_1324", "160815_1914", "160815_2014", "160815_2114", "160815_2215", "160815_2315", "160816_0015", "160816_0115", "160816_0215", "160816_0316", "160816_0416", "160816_0516", "160816_0616", "160816_0716", "160816_0816", "160816_0917", "160816_1017", "160816_1117", "160816_1217", "160816_1317", "160816_1418", "160816_1518", "160816_1619", "160816_1719", "160816_1819", "160816_1920", "160816_2020", "160816_2120", "160816_2220", "160816_2320", "160817_0020", "160817_0121", "160817_0221", "160817_0321", "160817_0421", "160817_0521", "160817_0622", "160817_0722", "160817_0951", "160817_1540", "160817_1559", "160817_1659", "160817_1800", "160817_1900", "160817_2000", "160817_2100", "160817_2201", "160817_2301", "160818_1859", "160818_1959", "160818_2059", "160818_2159", "160818_2259", "160819_0000", "160819_0100", "160819_0200", "160819_0300", "160819_0401", "160819_0501", "160819_0601", "160819_0701", "160819_0801", "160819_0902", "160819_1002", "160819_1102", "160819_1202", "160819_1303", "160819_1403", "160819_1503", "160819_1603", "160819_1703", "160819_1804", "160819_1904", "160819_2004", "160819_2104", "160819_2205", "160819_2305", "160820_0005", "160820_0105", "160820_0206", "160820_0306", "160820_0406", "160820_0506", "160820_0606", "160820_0707", "160820_0807", "160820_0907", "160820_1702", "160820_1803", "160820_1903", "160820_2003", "160820_2103", "160820_2203", "160820_2304", "160821_0004", "160821_0104", "160821_0204", "160821_0305", "160821_0405", "160821_0505", "160821_0605", "160821_0706", "160821_0806", "160821_0906", "160821_1006", "160821_1107", "160821_1207", "160821_1611", "160821_1711", "160821_1811", "160821_1912", "160821_2012", "160821_2112", "160821_2212", "160821_2312", "160822_0013", "160822_0113", "160822_0213", "160822_0313", "160822_0414", "160823_0921", "160823_1022", "160823_1122", "160823_1222", "160823_1322", "160823_1423", "160823_1523", "160823_1623", "160823_1723", "160823_1823", "160823_1924", "160823_2024", "160823_2124", "160823_2224", "160823_2325", "160825_1539", "160825_1639", "160825_1739", "160825_1839", "160825_1940", "160825_2040", "160825_2140", "160825_2240", "160825_2341", "160826_0041", "160826_0141", "160826_1136", "160826_1237", "160826_1337", "160826_1437", "160826_1537", "160826_1638", "160826_1738", "160826_1838", "160826_1938", "160826_2039", "160826_2139", "160826_2239", "160826_2339", "160827_0040", "160827_0140", "160827_0240", "160827_0340", "160827_0440", "160827_0541", "160827_0641", "160827_0741", "160827_0743", "160827_0843", "160827_0943", "160827_1043", "160827_1144", "160827_1413", "160827_1414", "160827_1515", "160827_1615", "160827_1715", "160827_1815", "160827_1916", "160827_2016", "160827_2116", "160827_2216", "160827_2317", "160828_0017", "160828_0117", "160828_0217", "160828_0318", "160828_0418", "160828_0518", "160828_0618", "160828_0718", "160828_0819", "160828_0919", "160828_1019", "160828_1119", "160828_1220", "160828_1320", "160828_1755", "160828_1756", "160828_1856", "160828_1957", "160828_2057", "160828_2157", "160828_2257", "160828_2358", "160829_0058", "160829_0158", "160829_0258", "160829_0359", "160829_0459", "160829_0559", "160829_0708", "160829_0808", "160829_0908", "160829_1008", "160829_1109", "160829_1209", "160829_1309", "160829_1409", "160829_1510", "160829_1610", "160829_1710", "160829_1810", "160829_1911", "160829_2011", "160829_2111", "160829_2211", "160829_2312", "160830_0012", "160830_0112", "160830_0212", "160830_0313", "160830_0413", "160830_0513", "160830_0613", "160830_0713", "160830_0814", "160830_0914", "160830_1014", "160830_1114", "160830_1215", "160830_1415", "160830_1515", "160830_1616", "160830_1730", "160830_1731", "160830_1831", "160830_1931", "160830_2031", "160830_2132", "160830_2232", "160830_2332", "160831_0032", "160831_0133", "160831_0233", "160831_0333", "160831_0433", "160831_0538", "160831_0539", "160831_0640", "160831_0740", "160831_0840", "160831_0940", "160831_1041", "160831_1141", "160831_1241", "160831_1341", "160831_1441", "160831_1542", "160831_1642", "160831_1742", "160831_1842", "160831_1943", "160831_2043", "160831_2143", "160831_2243", "160831_2343", "160901_0044", "160901_0144", "160901_0244", "160901_0344", "160901_0445", "160901_0545", "160901_0645", "160901_0745", "160901_0846", "160901_0946", "160901_0947", "160901_1047", "160901_1147", "160901_1248", "160901_1348", "160901_1623", "160901_1723", "160901_1824", "160901_1924", "160901_2024", "160901_2124", "160901_2225", "160901_2325", "160902_0025", "160902_0125", "160902_0226", "160902_0326", "160902_0426", "160902_0527", "160902_0627", "160902_0727", "160902_0827", "160902_0927", "160902_1028", "160902_1848", "160902_1948", "160902_2049", "160902_2149", "160902_2249", "160902_2349", "160903_0050", "160903_0150", "160903_0250", "160903_0350", "160903_0450", "160903_0551", "160903_0651", "160903_0959", "160903_1100", "160903_1200", "160903_1300", "160903_1400", "160903_1501", "160903_1601", "160903_1701", "160903_1801", "160903_1902", "160903_2044", "160903_2144", "160903_2244", "160903_2345", "160904_0045", "160904_0145", "160904_0245", "160904_0346", "160904_0446", "160904_0546", "160904_0646", "160904_0930", "160904_0931", "160904_1031", "160904_1038", "160904_1058", "160904_1159", "160904_1259", "160904_1359", "160904_1459", "160904_1600", "160904_1700", "160904_1800", "160904_1900", "160904_2001", "160904_2101", "160904_2201", "160904_2301", "160905_0001", "160905_0102", "160905_0202", "160905_0302", "160905_0403", "160905_0503", "160905_0603", "160905_0703", "160905_1619", "160905_1719", "160905_1819", "160905_1920", "160905_2020", "160905_2120", "160905_2220", "160905_2321", "160906_0021", "160906_0121", "160906_0221", "160906_0222", "160906_0323", "160906_0423", "160906_0523", "160906_0623", "160906_0724", "160906_0824", "160906_1357", "160906_1457", "160906_1557", "160906_1658", "160906_1758", "160906_1858", "160906_1958", "160906_2059", "160906_2159"] | ||
} | ||
] |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
[ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. merge with main cax.json There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can't. Need separate config for a separate transfer-only process that happens in post-script of DAG. (cax_stash replication is also running a separate .json, but just not under version control.) |
||
{ | ||
"name": "login", | ||
"method": "scp", | ||
"hostname": "login.ci-connect.uchicago.edu", | ||
"dir_raw": "/xenon/xenon1t_test", | ||
"dir_processed": "/xenon/ershockley/processed", | ||
"upload_options": ["midway-login1"], | ||
"username": "pdeperio", | ||
"task_list": ["CopyPush"], | ||
"data_types" : ["processed"] | ||
}, | ||
|
||
{ | ||
"name": "midway-login1", | ||
"method": "scp", | ||
"hostname": "midway-login1.rcc.uchicago.edu", | ||
"dir_raw": "/project/lgrandi/xenon1t/raw", | ||
"dir_processed": "/project/lgrandi/xenon1t/processed", | ||
"dir_minitrees": "/project/lgrandi/xenon1t/minitrees", | ||
"upload_options": [], | ||
"username": "tunnell", | ||
"download_options": ["xe1t-datamanager"], | ||
"purge" : 25, | ||
"grid_cert": "/project/lgrandi/deployHQ/cax/user_cert" | ||
} | ||
] |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
remove