-
Notifications
You must be signed in to change notification settings - Fork 0
/
fetchpull-plot.gnuplot
152 lines (141 loc) · 9.26 KB
/
fetchpull-plot.gnuplot
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
set xdata time
set datafile separator ";"
set timefmt "%Y-%m-%d"
set format x "%Y-%m-%d"
# logarithmic seconds to show both realtime and bulk together
set logscale y
# styling
set xtics rotate by 45 right
set xlabel "retrieval date"
set ylabel "time to retrieve (seconds)"
set cbtics format ""
set cbtics add ("0" 0)
set cbtics add ("1" 1)
set cbtics add ("2" 2)
set cbtics add ("4" 3)
set cbtics add ("8" 4)
set cbtics add ("16" 5)
set cbtics add ("32" 6)
set cbtics add ("64" 7)
set cbtics add ("128" 8)
set cbtics add ("256" 9)
set cbtics add ("512" 10)
set cblabel "age / days since upload"
# add jitter for a bee swarm plot to get a better understanding of the data density without needing transparency
# overlap is vertical binning, spread is horizontal jitter, wrap limits horizontal spread
set jitter overlap 7 spread 0.02 wrap 1
# moving average, see https://gnuplot.sourceforge.net/demo/running_avg.html and https://stackoverflow.com/a/55935923/7666
# number of points in average
n = 64
array A[n]
samples(x) = $0 > (n - 1) ? n : int($0+1)
mod(x) = int(x) % n
avg_n(x) = (A[mod($0)+1]=x, (sum [i=1:samples($0)] A[i]) / samples($0))
# plot requests
set title "fetchpull: requests"
set term png size 800,600
set output "fetchpull-get-realtime.png"
plot "<(grep realtime\\;\\#t fetchpull-stats-get.csv)" using 1:3:(log((column(4)+1))/log(2)) palette lw 1 title "realtime succeeded", 3 title " 3 s" at end lw 2 lc "#cccccc", 10 title "10 s" at end lw 2 lc "#aaaaaa", 60 title "1 min" at end lw 2 lc "#aaaaaa", 300 title "5 min" at end lw 2 lc "#cccccc", \
"<(grep \\;1\\;realtime\\;\\#t fetchpull-stats-get.csv)" using 1:(avg_n($3)) title "running mean over previous 64 age 1 points" pt 7 ps 0.5 lw 2 lc rgb "dark-cyan"
replot
set output "fetchpull-get-small.png"
plot "<(grep small\\;\\#t fetchpull-stats-get.csv)" using 1:3:(log((column(4)+1))/log(2)) palette lw 1 title "small succeeded", 3 title " 3 s" at end lw 2 lc "#cccccc", 10 title "10 s" at end lw 2 lc "#aaaaaa", 60 title "1 min" at end lw 2 lc "#aaaaaa", 600 title "10 min" at end lw 2 lc "#cccccc", \
"<(grep \\;1\\;small\\;\\#t fetchpull-stats-get.csv)" using 1:(avg_n($3)) title "running mean over previous 64 age 1 points" pt 7 ps 0.5 lw 2 lc rgb "dark-cyan"
replot
set output "fetchpull-get-bulk.png"
plot "<(grep \\;bulk\\;\\#t fetchpull-stats-get.csv)" using 1:3:(log((column(4)+1))/log(2)) palette lw 1 title "bulk succeeded", 3 title " 3 s" at end lw 2 lc "#cccccc", 10 title "10 s" at end lw 2 lc "#aaaaaa", 60 title "1 min" at end lw 2 lc "#aaaaaa", 600 title "10 min" at end lw 2 lc "#cccccc", 3600 title "60 min" at end lw 2 lc "#dddddd", \
"<(grep \\;1\\;bulk\\;\\#t fetchpull-stats-get.csv)" using 1:(avg_n($3)) title "running mean over previous 64 age 1 points" pt 7 ps 0.5 lw 2 lc rgb "dark-cyan"
replot
# failed requests get less jitter
set jitter overlap 0.5 spread 0.1 wrap 1
set title "fetchpull: failed requests"
set term png size 800,600
set output "fetchpull-get-failed-realtime.png"
plot "<(grep realtime\\;\\#f fetchpull-stats-get.csv)" using 1:(column(3)<20000? column(3) : 1/0):(log((column(4)+1))/log(2)) palette lw 1 pt 4 title "realtime failed", 3 title " 3 s" at end lw 2 lc "#cccccc", 10 title "10 s" at end lw 2 lc "#aaaaaa", 60 title "1 min" at end lw 2 lc "#aaaaaa", 300 title "5 min" at end lw 2 lc "#cccccc", \
"<(grep \\;1\\;realtime\\;\\#f fetchpull-stats-get.csv)" using 1:(avg_n($3)) title "running mean over previous 64 age 1 points" pt 7 ps 0.5 lw 2 lc rgb "dark-cyan"
replot
set output "fetchpull-get-failed-small.png"
plot "<(grep small\\;\\#f fetchpull-stats-get.csv)" using 1:(column(3)<20000? column(3) : 1/0):(log((column(4)+1))/log(2)) palette lw 1 title "small failed", 3 title " 3 s" at end lw 2 lc "#cccccc", 10 title "10 s" at end lw 2 lc "#aaaaaa", 60 title "1 min" at end lw 2 lc "#aaaaaa", 600 title "10 min" at end lw 2 lc "#cccccc", \
"<(grep \\;1\\;small\\;\\#f fetchpull-stats-get.csv)" using 1:(avg_n($3)) title "running mean over previous 64 age 1 points" pt 7 ps 0.5 lw 2 lc rgb "dark-cyan"
replot
set output "fetchpull-get-failed-bulk.png"
plot "<(grep bulk\\;\\#f fetchpull-stats-get.csv)" using 1:(column(3)<20000? column(3) : 1/0):(log((column(4)+1))/log(2)) palette lw 1 title "bulk failed", 3 title " 3 s" at end lw 2 lc "#cccccc", 10 title "10 s" at end lw 2 lc "#aaaaaa", 60 title "1 min" at end lw 2 lc "#aaaaaa", 600 title "10 min" at end lw 2 lc "#cccccc", \
"<(grep \\;1\\;bulk\\;\\#f fetchpull-stats-get.csv)" using 1:(avg_n($3)) title "running mean over previous 64 age 1 points" pt 7 ps 0.5 lw 2 lc rgb "dark-cyan"
replot
# plot inserts
set output "fetchpull-put.png"
set title "fetchpull: inserts"
set ylabel "time to upload (seconds)"
set xlabel "upload date"
set cblabel "days until download"
plot "<(grep realtime\\;\\#t fetchpull-stats-put.csv)" using 1:3:(log((column(4)+1))/log(2)) palette lw 1 title "realtime succeeded", "<(grep small\\;\\#t fetchpull-stats-put.csv)" using 1:3:(log((column(4)+1))/log(2)) palette lw 1 title "small succeeded", "<(grep bulk\\;\\#t fetchpull-stats-put.csv)" using 1:3:(log((column(4)+1))/log(2)) palette lw 1 title "bulk succeeded", 3 title " 3 s" at end lw 2 lc "#cccccc", 10 title "10 s" at end lw 2 lc "#aaaaaa", 60 title "1 min" at end lw 2 lc "#aaaaaa", 600 title "10 min" at end lw 2 lc "#cccccc", 3600 title "60 min" at end lw 2 lc "#dddddd", \
"<(grep \\;1\\;realtime\\;\\#t fetchpull-stats-put.csv)" using 1:(avg_n($3)) title "running means, last 64, in 1 day" pt 7 ps 0.5 lw 2 lc rgb "dark-cyan", \
"<(grep \\;1\\;small\\;\\#t fetchpull-stats-put.csv)" using 1:(avg_n($3)) title "" pt 7 ps 0.5 lw 2 lc rgb "dark-cyan", \
"<(grep \\;1\\;bulk\\;\\#t fetchpull-stats-put.csv)" using 1:(avg_n($3)) title "" pt 7 ps 0.5 lw 2 lc rgb "dark-cyan"
replot
# plot inserts
set output "fetchpull-put-failed.png"
set title "fetchpull: failed inserts"
set ylabel "time to upload (seconds)"
set xlabel "upload date"
set cblabel "days until download"
plot "<(grep realtime\\;\\#f fetchpull-stats-put.csv)" using 1:(column(3)<20000? column(3) : 1/0):(log((column(4)+1))/log(2)) palette lw 1 pt 4 title "realtime failed", "<(grep small\\;\\#f fetchpull-stats-put.csv)" using 1:(column(3)<20000? column(3) : 1/0):(log((column(4)+1))/log(2)) palette lw 1 title "small failed", "<(grep bulk\\;\\#f fetchpull-stats-put.csv)" using 1:(column(3)<20000? column(3) : 1/0):(log((column(4)+1))/log(2)) palette lw 1 title "bulk failed", 3 title " 3 s" at end lw 2 lc "#cccccc", 10 title "10 s" at end lw 2 lc "#aaaaaa", 60 title "1 min" at end lw 2 lc "#aaaaaa", 600 title "10 min" at end lw 2 lc "#cccccc", 3600 title "60 min" at end lw 2 lc "#dddddd"
replot
# plot max age of requests
unset cbtics
set yrange [1:1000]
set logscale y
set logscale cb
set cbtics format ""
set cbtics add ("0" 0)
set cbtics add ("1" 1)
set cbtics add ("2" 2)
set cbtics add ("4" 4)
set cbtics add ("8" 8)
set cbtics add ("16" 16)
set cbtics add ("32" 32)
set cbtics add ("64" 64)
set cbtics add ("128" 128)
set cbtics add ("256" 256)
set cbtics add ("512" 512)
set cbtics add ("1024" 1024)
set cbtics add ("2048" 2048)
set cbtics add ("4096" 4096)
set ylabel "age / days since upload"
set cblabel "time to download (s)"
set title "fetchpull: lifetime download-time"
set term png size 800,600
set output "fetchpull-lifetime-realtime.png"
plot "<(grep realtime\\;\\#t fetchpull-stats-get.csv)" using 1:4:3 palette lw 1 title "realtime succeeded"
set output "fetchpull-lifetime-small.png"
plot "<(grep small\\;\\#t fetchpull-stats-get.csv)" using 1:4:3 palette lw 1 title "small succeeded"
set output "fetchpull-lifetime-bulk.png"
plot "<(grep bulk\\;\\#t fetchpull-stats-get.csv)" using 1:4:3 palette lw 1 title "bulk succeeded"
# download time heatmaps (do not work well yet)
# set view map
# set dgrid3d
# set pm3d interpolate 16,16
# set title "fetchpull: average download time"
#
# set output "fetchpull-lifetime-bulk-download-time.png"
# splot "<(grep 'bulk;#t' fetchpull-stats-get.csv)" using 1:4:3 palette with pm3d title "bulk succeeded"
# set output "fetchpull-lifetime-small-download-time.png"
# splot "<(grep 'small;#t' fetchpull-stats-get.csv)" using 1:4:3 palette with pm3d title "bulk succeeded"
# success count plots
set title "fetchpull: lifetime: monthly success-count"
unset datafile separator
unset logscale cb
unset cbtics
set cbtics format "%g "
set cblabel "successful downloads (count)"
# simple monthly binning
set timefmt "%Y-%m"
set format x "%Y-%m"
set output "fetchpull-lifetime-realtime-success-count.png"
plot "<(grep 'realtime;#t' fetchpull-stats-get.csv | sed 's/-..;/;/;s/;[^;]+//;s/;[^;]*//;s/;[^;]*//;s/;/ /g' | sort | uniq -c)" using 2:3:1 palette lw 4 title "realtime succeeded", 14 title "2 weeks" at begin lw 2 lc "#aaaaaa", 90 title "3 mo." at begin lw 2 lc "#cccccc"
set output "fetchpull-lifetime-small-success-count.png"
plot "<(grep 'small;#t' fetchpull-stats-get.csv | sed 's/-..;/;/;s/;[^;]+//;s/;[^;]*//;s/;[^;]*//;s/;/ /g' | sort | uniq -c)" using 2:3:1 palette lw 4 title "small succeeded", 14 title "2 weeks" at begin lw 2 lc "#aaaaaa", 90 title "3 mo." at begin lw 2 lc "#cccccc"
set output "fetchpull-lifetime-bulk-success-count.png"
plot "<(grep 'bulk;#t' fetchpull-stats-get.csv | sed 's/-..;/;/;s/;[^;]+//;s/;[^;]*//;s/;[^;]*//;s/;/ /g' | sort | uniq -c)" using 2:3:1 palette lw 4 title "bulk succeeded", 14 title "2 weeks" at begin lw 2 lc "#aaaaaa", 90 title "3 mo." at begin lw 2 lc "#cccccc"
replot
quit