-
Notifications
You must be signed in to change notification settings - Fork 2
/
run_tests
executable file
·170 lines (153 loc) · 6.82 KB
/
run_tests
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
#!/usr/bin/env python
import os
import sys
import glob
import re
import argparse
import subprocess
ap = argparse.ArgumentParser(description = 'Test a command for complicance with the vfmd spec.')
ap.add_argument('--dir', action='append',
help='Glob pattern for directory with testcases (default: \'tests/*/*\')')
ap.add_argument('--testcase', action='store', metavar='SLUG',
help='Glob pattern for testcase (specify filename without \'.md\')')
ap.add_argument('--actual-fails', action='store_true',
help='Print output of the given markdown command for failing tests')
ap.add_argument('--expected-fails', action='store_true',
help='Print expected output for failing tests')
ap.add_argument('markdown_command', action='store', metavar='markdown-command',
help='Command to test. The command should read Markdown ' +
'from stdin and output XHTML to stdout.')
args = ap.parse_args()
if not args.dir:
args.dir = ["tests/*/*"]
def read_file(filename):
with open(filename, 'r') as f:
return f.read()
def is_command_available(cmd, test_input):
xmllint_cmd = "xmllin --c14n --format -"
try:
process = subprocess.Popen(cmd.split(),
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
result = process.communicate(input=test_input)
if (result[1] or # something was written to stderr
process.returncode != 0): # non-zero exit code was returned
return False
except OSError:
return False
return True
def run_cmd(cmd, input_file):
input_file_contents = read_file(input_file)
process = subprocess.Popen(cmd.split(),
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
result = process.communicate(input=input_file_contents)
if (result[1] or # something was written to stderr
process.returncode != 0): # non-zero exit code was returned
print "Error: \"" + result[1] + "\" when running \"" + cmd + "\" on [" + input_file + "]"
return "";
return result[0] # return only stdout
def canonical_html(html):
xmllint_cmd = "xmllint --c14n --format -"
process = subprocess.Popen(xmllint_cmd.split(),
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
result = process.communicate(input="<html>" + html + "</html>")
if (result[1] or # something was written to stderr
process.returncode != 0): # non-zero exit code was returned
# If the html is not well-formed, we can't use xmllint.
# Instead, we use a crude hack with the assumption that all
# '<' and '>' characters in the html belong to html tags.
return re.sub(r'>(\s+)<', r'><', html)
return result[0] # return only stdout
# Main
# Check if we have xmllint available. We use that to canonicalize
# the actual and expected outputs.
xmllint_cmd = "xmllint --c14n --format -"
if (not is_command_available(xmllint_cmd, "<html></html>")):
print "Error: Unable to run '" + xmllint_cmd + "'"
sys.exit(1)
# Check if we are able to run the provided markdown command.
if (not is_command_available(args.markdown_command, "blah")):
print "Error: Unable to run '" + args.markdown_command + "'"
sys.exit(1)
pass_count = 0
fail_count = 0
error_count = 0
print ""
for tg in args.dir:
for t in glob.glob(tg):
if (not os.path.isdir(t)):
continue
t = t.strip('/')
if args.testcase:
testcases = glob.glob(t + "/" + args.testcase + ".md")
else:
testcases = glob.glob(t + "/*.md")
print t, "(" + str(len(testcases)) + " testcases" + "):"
has_normalize_script = (os.path.isfile(t + "/normalize.py"))
if has_normalize_script:
old_sys_path = sys.path
sys.path.insert(0, t)
from normalize import normalize
for md in testcases:
md_file = md[(len(t) + 1):]
slug = md_file[:-3]
md_input_file = t + "/" + md_file
actual_html_file = t + "/actual/" + slug + ".html"
expected_html_file = t + "/expected/" + slug + ".html"
if (os.path.isfile(md_input_file) and
os.path.isfile(expected_html_file)):
# Get the actual and expected outputs
actual_html = run_cmd(args.markdown_command, md_input_file).strip()
expected_html = read_file(expected_html_file).strip()
# Write the actual output to file
if not os.path.exists(t + "/actual"):
os.mkdir(t + "/actual")
with open(actual_html_file, "w") as f:
f.write(actual_html)
# If any of the outputs is null, it indicates an error in testing
if (actual_html == "" or expected_html == ""):
error_count += 1
continue
# Run the normalizer, if available
if has_normalize_script:
actual_html_n = normalize(slug, actual_html)
expected_html_n = normalize(slug, expected_html)
else:
actual_html_n = actual_html
expected_html_n = expected_html
# Canonicalize and compare the actual vs expected output
passed = (canonical_html(actual_html_n) == canonical_html(expected_html_n))
if passed:
print " PASS:", " ", slug
pass_count += 1
else:
print " FAIL:", "*", slug
fail_count += 1
# Print test output
if (not passed) and (args.actual_fails or args.expected_fails):
if args.actual_fails:
print "ACTUAL OUTPUT:"
print "--------------"
print actual_html
print "--------------"
if args.expected_fails:
print "EXPECTED OUTPUT:"
print "--------------"
print expected_html
print "--------------"
if has_normalize_script:
if 'normalize' in sys.modules:
del(sys.modules['normalize'])
sys.path = old_sys_path
print ""
total_count = pass_count + fail_count + error_count
if total_count == 0:
print "No testcases found"
else:
print "Passed: {:3} testcases".format(pass_count)
print "Failed: * {:3} testcases".format(fail_count)
print "Total: {:3} testcases".format(total_count)
if error_count:
print "{:3} testcases could not be run".format(error_count)
if (total_count != pass_count):
sys.exit(1)
sys.exit(0)