-
Notifications
You must be signed in to change notification settings - Fork 5
/
getCloneCount.py
139 lines (109 loc) · 3.88 KB
/
getCloneCount.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
import requests, json, os.path
# File Format where the first line is totals
# Counts: <#> Uniques: <#>
# <count> <timestamp> <uniques>
# <count> <timestamp> <uniques>
# BE CAREFUL; PLAINTEXT CREDENTIALS!
username = "username"
password = "password"
repos = ['VASim', 'ANMLZoo', 'Automata-to-Routing', 'GitHub-Clone-Scraper']
# Class defining the clone information returned by GitHub about each repo
class CloneRecord:
def __init__(self, count, timestamp, uniques):
self.count = count
self.timestamp = timestamp
self.uniques = uniques
# Parses a file line into a CloneRecord object
def parseFileRecord(record):
entries = record.split()
result = CloneRecord(entries[0], entries[1], entries[2])
return result
# Reads clone file into list of records in dictionary
def readCloneFile(fn, recordMap):
# open file, if it doesn't exist, create it
if os.path.isfile(fn):
f = open(fn, 'r')
else:
f = open(fn, 'w+')
writeTotalsLine(f,0,0)
f.seek(0)
# skip first line (contains totals)
parseTotals(f.readline())
# read entries
for line in f:
# parse line entry into object
record = parseFileRecord(line)
# map timestamp strings to record objects
recordMap[record.timestamp] = record
# Writes the first line of every file given a count and uniques total
def writeTotalsLine(f, count, uniques):
line = "Counts: " + str(count) + " Uniques: " + str(uniques) + "\n"
f.write(line)
# Writes the file given a record map. Automatically counts clone/unique totals
def writeCloneFile(fn, recordMap):
f = open(fn, 'w')
# get totals
countTotal = 0
uniquesTotal = 0
for timestamp, record in recordMap.items():
countTotal += int(record.count)
uniquesTotal += int(record.uniques)
# write totals
writeTotalsLine(f, countTotal, uniquesTotal)
# write entries
for timestamp, record in recordMap.items():
printRecord(record)
line = str(record.count) + " " + str(record.timestamp) + " " + str(record.uniques) + "\n"
f.write(line)
# Retrieves clone info from GitHub as a .json object
def getCloneCounts():
url = "https://api.github.com/repos/" + username + "/" + repo + "/traffic/clones"
response = requests.get(url, auth=(username,password))
return response.json()
# Pretty prints a record
def printRecord(record):
line = str(record.count) + " " + str(record.timestamp) + " " + str(record.uniques)
print line
# Parses first line in file. Returns tuple of totals
def parseTotals(line):
print line
entries = line.split()
return entries[1],entries[3]
#
def updateRepo(repo):
# Init recordMap
recordMap = dict()
# Choose filename
fn = repo + ".txt"
# Read configuration file for repo
print "Reading from old file..."
readCloneFile(fn, recordMap)
for timestamp, record in recordMap.items():
printRecord(record)
print "---------------------"
print "Records: " + str(len(recordMap))
# Get data from Github
print "Asking Github for data..."
counts = getCloneCounts()
print counts
print "---------------------"
# Build clone map
print "Building data structure from GitHub data..."
entries = counts["clones"]
for entry in entries:
ts = entry["timestamp"]
print entry["count"], ts, entry["uniques"]
recordMap[ts] = CloneRecord(entry["count"], ts, entry["uniques"])
print "---------------------"
# Write clone map to file
print "Writing data structure back to file..."
writeCloneFile(fn, recordMap)
print "---------------------"
print "DONE"
print ""
#####################################################
for repo in repos:
print "************************************************"
print " Updating data for " + repo
print "************************************************"
updateRepo(repo)