-
Notifications
You must be signed in to change notification settings - Fork 2
/
extractArticleInfo.py
98 lines (87 loc) · 3.65 KB
/
extractArticleInfo.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
__author__ = 'thodrek'
import cPickle as pickle
import argparse
from EventRegistry import *
import socket
import os
import sys
# Read input arguments
parser = argparse.ArgumentParser(description='Please use script as "python crawler.py -s <start_date> -e <end_date>. Data format should be "YYYY-MM-DD". Example: python crawler.py -l "United States" -c "Technology" -s 2014-08-16 -e 2014-09-27')
parser.add_argument('-s','--sdate',help="Specifies the start date for collecting events.",required=True)
parser.add_argument('-e','--edate',help="Specifies the end date for collecting events.",required=True)
args = vars(parser.parse_args())
print "Reading input parameters...",
if socket.gethostname() == 'balos.umiacs.umd.edu':
out_ev_prefix = '/scratch0/CIDRDemo/EventReg_Data/eventInfo_'+args['sdate']+'_'+args['edate']
out_art_prefix = '/scratch0/CIDRDemo/EventReg_Data/articleInfo_'+args['sdate']+'_'+args['edate']
else:
out_ev_prefix = '/tmp/eventInfo_'+args['sdate']+'_'+args['edate']
out_art_prefix = '/tmp/articleInfo_'+args['sdate']+'_'+args['edate']
start_date = datetime.datetime.strptime(args['sdate'], "%Y-%m-%d").date()
end_date = datetime.datetime.strptime(args['edate'], "%Y-%m-%d").date()
print "DONE."
# Initialize event registry connection
print "Initializing event registry connection...",
er = EventRegistry(host="http://eventregistry.org",logging=False)
print "DONE."
# Initialize query
print "Initializing query...",
q = QueryEvents()
q.setDateLimit(start_date, end_date)
q.addRequestedResult(RequestEventsUriList())
q.queryParams['lang'] = "eng"
q.queryParams['minArticlesInEvent'] = 50
print "DONE."
# Execute query
print "Issue query...",
res = er.execQuery(q)
if 'uriList' in res:
print "DONE. Total number of extracted event uris = ",len(res['uriList'])
else:
print "DONE. Problem with res:"
print res
sys.exit(-1)
# Iterate over events and extract article info
print "Retrieving articles..."
events_processed = 0.0
errors = 0.0
total_entries = len(res['uriList'])
eventInfo = {}
eventArticleInfo = {}
for eURI in res['uriList']:
# initialize query
qEv = QueryEvent(eURI)
qEv.addRequestedResult(RequestEventInfo())
qEv.addRequestedResult(RequestEventArticles(count = 50, lang=["eng"]))
# execute query
evInfoArticles = er.execQuery(qEv)
if evInfoArticles:
if 'articles' in evInfoArticles and 'info' in evInfoArticles:
# grab event info
eventInfo[eURI] = evInfoArticles['info']
# grab article information
eventArticleInfo[eURI] = evInfoArticles['articles']['results']
else:
print 'No articles or info in result!'
print evInfoArticles
errors += 1
else:
print 'NoneType returned!'
errors += 1
# update count and print progress
events_processed += 1.0
progress = events_processed*100.0/float(total_entries)
sys.stdout.write("Article information extraction progress: %10.2f%% (%d out of %d, errors %d) \r" % (progress,events_processed,total_entries, errors))
sys.stdout.flush()
print "DONE."
print "Storing output to pickle dictionaries..."
info_ev_filename = out_ev_prefix+".pkl"
info_art_filename = out_art_prefix+".pkl"
pickle.dump(eventInfo,open(info_ev_filename,"wb"))
pickle.dump(eventArticleInfo,open(info_art_filename,"wb"))
# if on remote server transfer code to balos
if socket.gethostname() != 'balos.umiacs.umd.edu':
os_command = "scp "+info_ev_filename+" [email protected]:/scratch0/CIDRDemo/EventReg_Data"
x = os.system(os_command)
os_command = "scp "+info_art_filename+" [email protected]:/scratch0/CIDRDemo/EventReg_Data"
x = os.system(os_command)