-
Notifications
You must be signed in to change notification settings - Fork 0
/
tester.sh
executable file
·486 lines (368 loc) · 14.3 KB
/
tester.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
#!/bin/bash
# This is a test script for both `memory leaks` and correct outputs (corresponding to the output files that Yuval created),
# as well as an efficiency test, for those who don't care to optimize their lanky code.
# You're welcome.
# - Adam
#
# For help:
# bash tester.sh --help
# global variables
testers_path=$1
leaks=$2
output_file="./tmp/output.txt"
valgrind_file="./tmp/valgrind.txt"
# buffering between tests
function buffer() {
echo -e "\n"
echo "
*****
* *
* *
* *
*****"
echo -e "\n"
}
# =================
# REGULAR TEST
# =================
function regular_test() {
if [[ $interface == @(c|both) ]]; then
# testing the C interface
rm $results_dir/test_transcript_c.txt &> /dev/null
rm $results_dir/memory_transcript_c.txt &> /dev/null
touch $results_dir/test_transcript_c.txt
touch $results_dir/memory_transcript_c.txt
echo -e "\n\e[4;37mTesting correct outputs for the interface of \e[4;33m\e[1;33mC\e[0m:"
echo -e "\n\e[4;34m\e[1;34mRESULTS\e[0m"
test_interface c
buffer
fi
if [[ $interface == @(py|both) ]]; then
# if no regular tests were ordered, we shouldn't run the CPython interface test (happens when $regular == no, $leaks == yes)
if [[ $regular == "no" ]]; then
return
fi
# testing the CPython interface
rm $results_dir/test_transcript_py.txt &> /dev/null
touch $results_dir/test_transcript_py.txt
echo -e "\e[4;37mTesting correct outputs for the interface of \e[4;33m\e[1;33mPython\e[0m:"
echo -e "\n\e[4;34m\e[1;34mRESULTS\e[0m"
test_interface py
buffer
fi
}
# comprehensive interface test - specific interface only
function test_interface() {
# the first argument shall be the interface being tested
local_interface=$1
# else, run a test
test_goal $local_interface wam
test_goal $local_interface ddg
test_goal $local_interface lnorm
test_goal $local_interface jacobi
if [[ $local_interface == "py" ]]; then
test_goal $local_interface spk
fi
}
# comprehensive goal test - specific goal, specific interface only
function test_goal() {
# the first argument shall be the interface being tested c/py
# the second argument shall be the goal being tested
# running a test for the goal
if [[ "${2}" == "jacobi" ]]; then
for (( i = 0; i <= $jacobi; i++ )); do
echo -n "${1^^}: ${2^^}: ${testers_path}/jacobi_${i}.txt: "
individual_test $1 $2 jacobi_$i.txt
echo
done
else
for (( i = 0; i <= $spk; i++ )); do
echo -n "${1^^}: ${2^^}: ${testers_path}/spk_${i}.txt: "
individual_test $1 $2 spk_$i.txt
echo
done
fi
}
# invidividual test - specific input file, specific goal, specific interface only
function individual_test() {
# the first argument shall be the interface being tested: c/py
# the second argument shall be the goal being tested
# the third argument shall be the input file being used
# running the commands
if [[ "${1}" == "py" ]]; then # if we are testing the python interface
python3 spkmeans.py 0 $2 $testers_path/$3 &> $output_file
elif [[ "${1}" == "c" ]]; then # if we are testing the C interface
valgrind --leak-check=yes --log-file=$valgrind_file ./spkmeans $2 $testers_path/$3 &> $output_file
else
echo "Individual test function failed: Invalid interface"
return -1
fi
# if a regular test was asked
if [[ $regular == "yes" ]]; then
# calculating the difference between the desired output and the actual output
diff_result=$(diff $output_file $testers_path/outputs/$1/$2/$3 2>&1)
# verdicting if the test failed, then print an appropriate status
verdict_diff ${#diff_result}
# if the test failed, print a report of the 'diff' operation into the test transcript of the interface
if [[ ${#diff_result} -ne 0 ]]; then
echo -e "DIFF RESULT FOR: ${1}: ${2}: ${3}:\n${diff_result}\n\n" >> $results_dir/test_transcript_$1.txt
fi
fi
# if the interface is C, verdict if the test had a memory leak, and print an appropriate status accordingly
if [[ $leaks == "yes" ]]; then
if [[ $1 == "c" ]]; then
# buffering
if [[ $regular == "yes" ]]; then
echo -n ": "
fi
# running memory test
verdict_memory_loss
echo -e "MEMORY LEAK RESULT FOR: ${1}: ${2}: ${3}:\n\n" >> $results_dir/memory_transcript_c.txt
cat $valgrind_file >> $results_dir/memory_transcript_c.txt
echo -e "\n\n\n" >> $results_dir/memory_transcript_c.txt
fi
fi
}
function verdict_diff() {
# the first argument shall be the length (in characters) of the result of diff
if [[ $1 -eq 0 ]]; then
echo -ne '\033[1;32mSUCCESS\e[0m' # print out a 'success' message
else
echo -ne "\e[1;31mFAILED\e[0m" # print out a 'failed' message
fi
}
function verdict_memory_loss() {
# this function shall be used only upon the C interface comprehensive test
# this function assumes the $valgrind_file matches the very last running of a test using the C interface
bytes_lost=$(cat $valgrind_file | grep "HEAP SUMMARY" -A 1 | tail -n1 | awk '{print $6}')
if [[ ${bytes_lost//,} -eq 0 ]]; then
echo -ne '\033[1;32mNO MEMORY LEAK\e[0m' # print out a 'success' message
else
echo -ne "\e[1;31mMEMORY LEAK\e[0m" # print out a 'failed' message
fi
}
# =================
# EFFICIENCY TEST
# =================
function efficiency_test() {
# test message
echo -e "\e[4;37mTesting the \e[4;33m\e[1;33mefficiency of the algorithms\e[0m.\n\e[1;31mNOTICE: This test is going to take quite a while.\nMoreover, this test is subjective.\nAlbeit, don't be a bozo and do optimize your code - you might get a penalty for inefficient code.\e[0m"
# creating a maximized file
echo -e 'from sklearn import datasets
import numpy as np
def save_dataset(dataset, filename):
lines = map(stringify_row, dataset)
with open(filename + ".csv", "w") as f:
f.writelines(lines)
def stringify_row(row):
row = map(str, row)
return ",".join(row) + "\\n"
# datapoints
save_dataset(datasets.make_blobs(n_samples=1000, centers=3, cluster_std=5, n_features=10, shuffle=True, random_state=31)[0], "1000_blobs_10_feat")
# symmetric matrix
a = np.random.rand(1000, 1000)
m = np.tril(a) + np.tril(a, -1).T
save_dataset(m, "jacobi_input_10_6")' | python3
if [[ $interface == @(c|both) ]]; then
# testing the efficiency of the C interface
rm $results_dir/efficiency_transcript_c.txt &> /dev/null
touch $results_dir/efficiency_transcript_c.txt
echo -e "\n\e[4;37mTesting the efficiency for the interface: \e[4;33m\e[1;33mC\e[0m:"
echo -e "\n\e[4;34m\e[1;34mRESULTS\e[0m"
test_efficiency_interface c
buffer
fi
if [[ $interface == @(py|both) ]]; then
# testing the efficiency of the CPython interface
rm $results_dir/efficiency_transcript_py.txt &> /dev/null
touch $results_dir/efficiency_transcript_py.txt
echo -e "\e[4;37mTesting the efficiency for the interface: \e[4;33m\e[1;33mPython\e[0m:"
echo -e "\n\e[4;34m\e[1;34mRESULTS\e[0m"
test_efficiency_interface py
buffer
fi
rm jacobi_input_10_6.csv &> /dev/null
rm 1000_blobs_10_feat.csv &> /dev/null
}
function test_efficiency_interface() {
# the first argument shall be the interface being tested
local_interface=$1
test_efficiency_goal $local_interface wam
test_efficiency_goal $local_interface ddg
test_efficiency_goal $local_interface lnorm
test_efficiency_goal $local_interface jacobi
if [[ $local_interface == "py" ]]; then
test_efficiency_goal $local_interface spk
fi
}
function test_efficiency_goal() {
# the first argument is the interface that is currently being tested
# the second argument is the goal being tested
echo -n "${1^^}: ${2^^}: Checking efficiency: "
local time_result
if [[ $1 == "c" ]]; then
if [[ $2 == "jacobi" ]]; then
time_result=$(timeout 11.5 bash -c "time ./spkmeans jacobi jacobi_input_10_6.csv 1> /dev/null" 2>&1)
else
time_result=$(timeout 0.65 bash -c "time ./spkmeans ${2} 1000_blobs_10_feat.csv 1> /dev/null" 2>&1)
fi
else
if [[ $2 == "jacobi" ]]; then
time_result=$(timeout 8 bash -c "time python3 spkmeans.py 0 jacobi jacobi_input_10_6.csv 1> /dev/null" 2>&1)
elif [[ $2 == "spk" ]]; then
time_result=$(timeout 130 bash -c "time python3 spkmeans.py 0 spk 1000_blobs_10_feat.csv 1> /dev/null" 2>&1)
else
time_result=$(timeout 4.5 bash -c "time python3 spkmeans.py 0 ${2} 1000_blobs_10_feat.csv 1> /dev/null" 2>&1)
fi
fi
# verdicting result of test, and outputting a corresponding log message to the efficiency transcript
if [[ ${#time_result} -ne 0 ]]; then # test succeeded
echo -e '\033[1;32mSUCCESS\e[0m' # print out a 'success' message
else
echo -e "\e[1;31mFAILED\e[0m" # print out a 'failed' message
echo -e "REQUIRED EFFICIENCY FOR: ${1}: ${2}: is:\n" >> $results_dir/efficiency_transcript_$1.txt
if [[ $1 == "c" ]]; then
if [[ $2 == "jacobi" ]]; then
echo -ne "11.5" >> $results_dir/efficiency_transcript_$1.txt
else
echo -ne "0.65" >> $results_dir/efficiency_transcript_$1.txt
fi
else
if [[ $2 == "jacobi" ]]; then
echo -ne "8" >> $results_dir/efficiency_transcript_$1.txt
elif [[ $2 == "spk" ]]; then
echo -ne "130" >> $results_dir/efficiency_transcript_$1.txt
else
echo -ne "4.5" >> $results_dir/efficiency_transcript_$1.txt
fi
fi
echo -e " seconds.\nGood luck next time!\n\n\n" >> $results_dir/efficiency_transcript_$1.txt
fi
}
# =================
# Organizer
# =================
function comprehensive_test() {
mkdir ./tmp &> /dev/null
# Trying to build the necessary resources
if [[ $interface == @(c|both) ]]; then
comp_output=$(bash comp.sh 2>&1) # compiling
if [[ ${#comp_output} -ne 0 ]]; then
echo -e "\e[1;31mFailed to compile your C module with \`\033[4;31mcomp.sh\e[0m\e[1;31m\`!\n"
youre_a_bozo
fi
fi
if [[ $interface == @(py|both) ]]; then
build_output=$(python3 setup.py build_ext --inplace 2>&1 1>/dev/null)
if [[ ${#build_output} -ne 0 ]]; then
echo -e "\e[1;31mFailed to build the CPython extension with \`\033[4;31msetup.py\e[0m\e[1;31m\`!\n"
youre_a_bozo
fi
fi
# Running the necessary tests
if [[ $regular == "yes" || $leaks == "yes" ]]; then
regular_test
fi
if [[ $efficiency == "yes" ]]; then
efficiency_test
fi
# Summary
echo -e "\033[4;31m\e[1;31mDONE -> NOTICE:\e[0m\nDetailed regular tests' results are in: \e[1;34m${results_dir}/test_transcript_[c|py].txt\e[0m.\nDetailed memory leak tests' results have their memory reports at \e[1;34m${results_dir}/memory_transcript_c.txt\e[0m.\nDetailed efficiency tests' results have their efficiency reports at \e[1;34m${results_dir}/efficiency_transcript_[c|py].txt\e[0m.\n\e[4;37mOnly the results of failed tests will be viewed in the transcripts.\e[0m"
}
# =================
# PRELUDE
# =================
function instructions() {
echo -e "\e[4;37m\e[1;37mHelp\e[0m: bash tester.sh <testfiles> <interface> <regular> <leaks> <efficiency> [results_dir]\n
- testfiles <path>: The path of the directory containing the test files that were supplied with this test.
- interface [c|py|both]: Specifies the interface you want to test.
- regular [yes|no]: Specifies whether to run the regular tests or not.
- leaks [yes|no]: Specifies if you want memory leak tests. Applies to the \e[4;37m\e[1;37mC\e[0m interface only.
- efficiency [yes|no]: Specifies whether you want an efficiency test on the interfaces you chose.
- results_dir <path>: Creates a new directory to store the tests' results into (default: running directory).
\e[4;37m\e[1;37mNotes\e[0m:
(1) The efficiency tests are quite long, take that into account.
(2) Detailed regular tests' results are saved into \`\033[4;37mtest_transcript_<interface>.txt\e[0m\`.
(3) Detailed memory leak tests' results are saved into \`\033[4;37mmemory_transcript_c.txt\e[0m\`.
(4) Detailed Efficiency tests' results are saved into \`\033[4;37mefficiency_transcript_<interface>.txt\e[0m\`.
(5) Only failed tests' results are saved into their transcript.
(6) Efficiency tests become invalid when running on tau-related server (e.g., nova).
\e[4;37m\e[1;37mInstructions\e[0m:
(1) Avoid any build/dist/egg directories/files from the working directory. Could potentially lead to undefined behaviors of the test script.
(2) You shall run this shell script from within the directory that contains all of the files that you need to assign.
(3) You shall have the package \`\e[4;37mvalgrind\e[0m\` installed if you wish to run memory leak tests.
(4) You shall have the python packages \`\e[4;37mnumpy\e[0m\` and \`\e[4;37mscikit-learn\e[0m\` installed if you wish to run efficiency tests."
exit
}
function youre_a_bozo() {
echo -e "\e[0m
_ ____ _ _____ ___ ___ ____ ___ ________
| | _ | _ \ / \|_ _|_ _/ _ \ _ | __ ) / _ \__ / _ \
| | _| |_ | |_) | / _ \ | | | | | | | _| |_ | _ \| | | |/ / | | |
| |___ |_ _| | _ < / ___ \| | | | |_| | |_ _| | |_) | |_| / /| |_| |
|_____| |_| |_| \_\/_/ \_\_| |___\___/ |_| |____/ \___/____\___/
____ ___ ____ _____ ____ _ _____ _ _
_ / ___/ _ \| _ \| ____| _ / ___|| |/ /_ _| | | |
_| |_ | | | | | | |_) | _| _| |_ \___ \| ' / | || | | |
|_ _| | |__| |_| | __/| |___ |_ _| ___) | . \ | || |___| |___
|_| \____\___/|_| |_____| |_| |____/|_|\_\___|_____|_____|
___ ____ ____ _ _ _____
|_ _/ ___/ ___|| | | | ____|
| |\___ \___ \| | | | _|
| | ___) |__) | |_| | |___
|___|____/____/ \___/|_____|"
exit
}
# blah0
if [[ $1 == "--help" ]]; then # help
instructions
fi
# blah1
if [[ $# -le 3 ]]; then # args
instructions
youre_a_bozo
fi
# blah2
if [[ ! -d $1 ]]; then # testfiles
echo -e "\e[1;31mThe following directory is a non-existing directory \`\033[4;31m${1}\e[0m\e[1;31m\`!\n"
youre_a_bozo
else
testers_path=$1
jacobi=$(ls $testers_path | grep "jacobi" | cut -d "_" -f2 | cut -d "." -f1 | sort -n | tail -n1)
spk=$(ls $testers_path | grep "spk" | cut -d "_" -f2 | cut -d "." -f1 | sort -n | tail -n1)
fi
# blah3
if [[ $2 != @(c|py|both) ]]; then # interface
youre_a_bozo
else
interface=$2
fi
# blah4
if [[ $3 != @(yes|no) ]]; then # leaks
youre_a_bozo
else
regular=$3
fi
# blah5
if [[ $4 != @(yes|no) ]]; then # leaks
youre_a_bozo
else
leaks=$4
fi
# blah6
if [[ $5 != @(yes|no) ]]; then # efficiency
youre_a_bozo
else
efficiency=$5
fi
# blah7
if [[ $# -eq 6 ]]; then # results dir
if [[ ! -d $6 ]]; then
mkdir $6
fi
results_dir=$6
else
results_dir="."
fi
# run
comprehensive_test