Skip to content

Commit

Permalink
SymQEMU tests: on test error, print differences of generated test ca…
Browse files Browse the repository at this point in the history
…se (#60)

* add colored hexdump diff of different test cases generated
* add new package dependencies to docker
  • Loading branch information
rmalmain authored May 15, 2024
1 parent 40f0b4c commit 6db2f6e
Show file tree
Hide file tree
Showing 2 changed files with 27 additions and 2 deletions.
5 changes: 4 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@ RUN apt update && apt install -y \
z3 \
libz3-dev \
libz3-dev \
libzstd-dev
libzstd-dev \
colordiff \
xxd \
wdiff

RUN pip install --user meson

Expand Down
24 changes: 23 additions & 1 deletion tests/symqemu/test.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
import pathlib
import shutil
import unittest
import tempfile
import subprocess

import util

Expand All @@ -16,13 +18,33 @@ def tearDown(self):
shutil.rmtree(self.SYMQEMU_OUTPUT_DIR)

def run_symqemu_and_assert_correct_result(self, binary_name):
symqemu_ref_output_dir = util.BINARIES_DIR / binary_name / 'expected_outputs'

util.run_symqemu_on_test_binary(binary_name=binary_name, generated_test_cases_output_dir=self.SYMQEMU_OUTPUT_DIR)

# `filecmp.dircmp` does a "shallow" comparison, but this is not a problem here because
# the timestamps should always be different, so the actual content of the files will be compared.
# See https://docs.python.org/3/library/filecmp.html#filecmp.dircmp
expected_vs_actual_output_comparison = filecmp.dircmp(self.SYMQEMU_OUTPUT_DIR, util.BINARIES_DIR / binary_name / 'expected_outputs')
expected_vs_actual_output_comparison = filecmp.dircmp(self.SYMQEMU_OUTPUT_DIR, symqemu_ref_output_dir)

for diff_file in expected_vs_actual_output_comparison.diff_files:
ref_file = symqemu_ref_output_dir / diff_file
gen_file = self.SYMQEMU_OUTPUT_DIR / diff_file

tmp_ref = tempfile.NamedTemporaryFile("w+")
subprocess.run(["xxd", f"{ref_file}"], stdout=tmp_ref, check=True)

tmp_gen = tempfile.NamedTemporaryFile("w+")
subprocess.run(["xxd", f"{gen_file}"], stdout=tmp_gen, check=True)

wdiff = subprocess.run(["wdiff", f"{tmp_ref.name}", f"{tmp_gen.name}"], capture_output=True)
colordiff = subprocess.run(["colordiff"], input=wdiff.stdout, capture_output=True)

print(f"===== Diff found in {diff_file} ======")
print(f"{colordiff.stdout.decode('utf-8').strip()}")
print(f"=================================")
print()

self.assertEqual(expected_vs_actual_output_comparison.diff_files, [])
self.assertEqual(expected_vs_actual_output_comparison.left_only, [])
self.assertEqual(expected_vs_actual_output_comparison.right_only, [])
Expand Down

0 comments on commit 6db2f6e

Please sign in to comment.