-
Notifications
You must be signed in to change notification settings - Fork 0
/
business_operations.py
146 lines (115 loc) · 5.4 KB
/
business_operations.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
import requests
from bs4 import BeautifulSoup
from google_sheets import harvest_sheet_tsv_http
import logging
import csv
from datetime import date
import pandas as pd
import os
from metrics import write_metric
def harvest_business_operations(path):
"""
Harvest Business Operations Spreadsheet
@param path: path where to write the TSV
@return: nothing
"""
logging.info("harvest_business_operations")
# harvest_social_media(path)
harvest_sheet_tsv_http(path=path,
name="business_operations",
url=os.getenv("SHEET_URL_BUSINESS_OPERATIONS"),
gid=0,
range_name='A:I',
columns=[])
def aggregate_bo(path, tsv):
"""
Aggregate data for business operations
@param path: Directory to write to
@param tsv: TSV with extracted data from the Google Spreadsheet
@return: nothing
"""
df = pd.read_csv(path + tsv + ".tsv", delimiter="\t")
df1 = df[df["Group"] == "Sponsored Research Administration"].iloc[:, 0:7]
df2 = df[df["Group"] != "Sponsored Research Administration"].iloc[:, 0:7]
update_date = df[df["Updated"].isnull() == False]["Updated"][0]
pd.DataFrame({"Department": ["Business Operations"], "Last Update": update_date}).to_csv(path + "update_dates.tsv",
sep='\t', index=False)
metric_df = df[df["Homepage"].isnull() == False].fillna("").values.tolist()[0]
df1.fillna("").to_csv(path + "sponsored_research_administration.tsv", sep='\t', index=False)
df2.fillna("").to_csv(path + "finance_and_administration.tsv", sep='\t', index=False)
write_metric(path=path, group=metric_df[0], metric=metric_df[1],
title=metric_df[1],
value=metric_df[6], unit=metric_df[5],
icon=metric_df[2], color=metric_df[4],
url=metric_df[3])
def harvest_social_media(path):
"""
Scrapes Twitter and Facebook for IQSS metrics
@param path: Directory to write to
@return: nothing
"""
# note: this function is not used, because it broke after changes on the website.
# should be replaced with API calls
logging.info("Harvesting Social Media")
collection_name = "social_media"
now = date.today().strftime('%Y-%m-%d')
# Write social_media_all once a day to prevent double records
with open(path + collection_name + "_all.tsv", 'r') as tsv_file:
reader = csv.reader(tsv_file, delimiter='\t')
for r in reader:
if r[0] == now:
logging.info("Social media already harvested")
return
# Twitter --------------
logging.info("Harvesting Twitter")
domain = 'https://twitter.com/IQSS'
data = requests.get(domain)
soup = BeautifulSoup(data.text, 'html.parser')
soup.find_all(class_='ProfileNav-item--followers')
followers = soup.find(class_='ProfileNav-item--followers').find_all('span', class_='ProfileNav-value')[
0].text.strip()
followers = int(followers.replace(",", ""))
tweets = soup.find(class_='ProfileNav-item--tweets').find('span',
class_='ProfileNav-value').text.strip()
tweets = int(tweets.replace(",", ""))
# TODO: check if records can be retrieved before writing to the database
posts = [['date', 'group', 'metric', 'title', 'value', 'unit', 'icon', 'color', 'url'],
[now, 'Social Media', 'Twitter Followers', 'Twitter', followers, 'Total Followers', 'fa fa-heart',
'light-blue',
"https://twitter.com/IQSS"],
[now, 'Social Media', 'Twitter Tweets', 'Twitter', tweets, 'Total Tweets', 'fab fa-twitter', 'light-blue',
"https://twitter.com/IQSS"]]
# Facebook --------------------
logging.info("Harvesting Facebook")
domain = 'https://www.facebook.com/iqssharvard'
data = requests.get(domain)
soup = BeautifulSoup(data.text, 'html.parser')
divs = soup.find_all('div')
fb_follow = ''
fb_like = ''
for d in divs:
if d.text.endswith('like this'):
if fb_like == '':
fb_like = d.text.split(' ')[0]
if d.text.endswith('follow this'):
if fb_follow == '':
fb_follow = d.text.split(' ')[0]
fb_like = int(fb_like.replace(",", ""))
fb_follow = int(fb_follow.replace(",", ""))
posts.append([now, 'Social Media', 'Facebook Likes', 'Facebook', fb_like, 'Total Likes', 'fa fa-thumbs-up', 'blue',
'https://www.facebook.com/iqssharvard'])
posts.append(
[now, 'Social Media', 'Facebook Followers', 'Facebook', fb_follow, 'Total Followers', 'fab fa-facebook-square',
'blue', 'https://www.facebook.com/iqssharvard'])
if len(posts) == 5: # 4 records including header
with open(path + collection_name + ".tsv", 'w') as tsv_file:
writer = csv.writer(tsv_file, delimiter='\t')
for row in posts:
writer.writerow(row)
with open(path + collection_name + "_all.tsv", 'a') as tsv_file:
writer = csv.writer(tsv_file, delimiter='\t')
for row in range(1, len(posts)):
writer.writerow(posts[row])
else:
logging.error('Could not load Facebook and Twitter stats (%s records)', str(len(posts) - 1))
logging.error('')