From 91a31ff03ccd07cbdb3d297098299c34cb0d51d2 Mon Sep 17 00:00:00 2001 From: Tobias Ribizel Date: Wed, 16 Aug 2023 13:33:51 +0200 Subject: [PATCH] test `-input_matrix` flag --- benchmark/test/conversion.py | 12 ++++ benchmark/test/matrix_statistics.py | 7 +++ benchmark/test/preconditioner.py | 7 +++ .../test/reference/conversion.matrix.stderr | 46 +++++++++++++++ .../test/reference/conversion.matrix.stdout | 19 +++++++ .../distributed_solver.matrix.stderr | 17 ++++++ .../distributed_solver.matrix.stdout | 57 +++++++++++++++++++ .../reference/matrix_statistics.matrix.stderr | 8 +++ .../reference/matrix_statistics.matrix.stdout | 38 +++++++++++++ .../reference/preconditioner.matrix.stderr | 42 ++++++++++++++ .../reference/preconditioner.matrix.stdout | 30 ++++++++++ benchmark/test/reference/solver.matrix.stderr | 17 ++++++ benchmark/test/reference/solver.matrix.stdout | 55 ++++++++++++++++++ .../test/reference/sparse_blas.matrix.stderr | 36 ++++++++++++ .../test/reference/sparse_blas.matrix.stdout | 25 ++++++++ benchmark/test/reference/spmv.matrix.stderr | 31 ++++++++++ benchmark/test/reference/spmv.matrix.stdout | 20 +++++++ benchmark/test/solver.py | 7 +++ benchmark/test/solver_distributed.py | 7 +++ benchmark/test/sparse_blas.py | 12 ++++ benchmark/test/spmv.py | 7 +++ benchmark/test/test_framework.py.in | 12 ++-- 22 files changed, 505 insertions(+), 7 deletions(-) create mode 100644 benchmark/test/reference/conversion.matrix.stderr create mode 100644 benchmark/test/reference/conversion.matrix.stdout create mode 100644 benchmark/test/reference/distributed_solver.matrix.stderr create mode 100644 benchmark/test/reference/distributed_solver.matrix.stdout create mode 100644 benchmark/test/reference/matrix_statistics.matrix.stderr create mode 100644 benchmark/test/reference/matrix_statistics.matrix.stdout create mode 100644 benchmark/test/reference/preconditioner.matrix.stderr create mode 100644 benchmark/test/reference/preconditioner.matrix.stdout create mode 100644 benchmark/test/reference/solver.matrix.stderr create mode 100644 benchmark/test/reference/solver.matrix.stdout create mode 100644 benchmark/test/reference/sparse_blas.matrix.stderr create mode 100644 benchmark/test/reference/sparse_blas.matrix.stdout create mode 100644 benchmark/test/reference/spmv.matrix.stderr create mode 100644 benchmark/test/reference/spmv.matrix.stdout diff --git a/benchmark/test/conversion.py b/benchmark/test/conversion.py index cf2e33983af..2eada100731 100755 --- a/benchmark/test/conversion.py +++ b/benchmark/test/conversion.py @@ -29,6 +29,18 @@ expected_stderr="conversion.simple.stderr", ) +# input matrixfile +test_framework.compare_output( + [ + "-input_matrix", + str(test_framework.matrixpath), + "-formats", + "coo,csr", + ], + expected_stdout="conversion.matrix.stdout", + expected_stderr="conversion.matrix.stderr", +) + # check that all conversions work test_framework.compare_output( [ diff --git a/benchmark/test/matrix_statistics.py b/benchmark/test/matrix_statistics.py index a29c80a0a7a..6e4d8b1d2f5 100755 --- a/benchmark/test/matrix_statistics.py +++ b/benchmark/test/matrix_statistics.py @@ -23,3 +23,10 @@ expected_stdout="matrix_statistics.simple.stdout", expected_stderr="matrix_statistics.simple.stderr", ) + +# input matrix file +test_framework.compare_output( + ["-input_matrix", str(test_framework.matrixpath)], + expected_stdout="matrix_statistics.matrix.stdout", + expected_stderr="matrix_statistics.matrix.stderr", +) diff --git a/benchmark/test/preconditioner.py b/benchmark/test/preconditioner.py index a5a8dd3f13f..e05e5b780ac 100755 --- a/benchmark/test/preconditioner.py +++ b/benchmark/test/preconditioner.py @@ -24,6 +24,13 @@ expected_stderr="preconditioner.simple.stderr", ) +# input matrix file +test_framework.compare_output( + ["-input_matrix", str(test_framework.matrixpath)], + expected_stdout="preconditioner.matrix.stdout", + expected_stderr="preconditioner.matrix.stderr", +) + # profiler annotations test_framework.compare_output( [ diff --git a/benchmark/test/reference/conversion.matrix.stderr b/benchmark/test/reference/conversion.matrix.stderr new file mode 100644 index 00000000000..813e04a2100 --- /dev/null +++ b/benchmark/test/reference/conversion.matrix.stderr @@ -0,0 +1,46 @@ +This is Ginkgo 1.7.0 (develop) + running with core module 1.7.0 (develop) +Running on reference(0) +Running with 2 warm iterations and 10 running iterations +The random seed for right hand sides is 42 +The formats are coo,csr +Benchmarking conversions. +Running test case +{ + "filename": "../../matrices/test/ani1.mtx", + "conversions": {} +} +Matrix is of size (36, 36) +Current state: +[ + { + "filename": "../../matrices/test/ani1.mtx", + "conversions": { + "coo-csr": { + "time": 1.0, + "repetitions": 10, + "completed": true + } + }, + "size": 36 + } +] +Current state: +[ + { + "filename": "../../matrices/test/ani1.mtx", + "conversions": { + "coo-csr": { + "time": 1.0, + "repetitions": 10, + "completed": true + }, + "csr-coo": { + "time": 1.0, + "repetitions": 10, + "completed": true + } + }, + "size": 36 + } +] diff --git a/benchmark/test/reference/conversion.matrix.stdout b/benchmark/test/reference/conversion.matrix.stdout new file mode 100644 index 00000000000..d3f62ec2b56 --- /dev/null +++ b/benchmark/test/reference/conversion.matrix.stdout @@ -0,0 +1,19 @@ + +[ + { + "filename": "../../matrices/test/ani1.mtx", + "conversions": { + "coo-csr": { + "time": 1.0, + "repetitions": 10, + "completed": true + }, + "csr-coo": { + "time": 1.0, + "repetitions": 10, + "completed": true + } + }, + "size": 36 + } +] diff --git a/benchmark/test/reference/distributed_solver.matrix.stderr b/benchmark/test/reference/distributed_solver.matrix.stderr new file mode 100644 index 00000000000..7eba2fc85dd --- /dev/null +++ b/benchmark/test/reference/distributed_solver.matrix.stderr @@ -0,0 +1,17 @@ +This is Ginkgo 1.7.0 (develop) + running with core module 1.7.0 (develop) +Running on reference(0) +Running with 2 warm iterations and 1 running iterations +The random seed for right hand sides is 42 +Running cg with 1000 iterations and residual goal of 1.000000e-06 +The number of right hand sides is 1 +Running test case +{ + "filename": "../../matrices/test/ani1.mtx", + "optimal": { + "spmv": "csr-csr" + }, + "solver": {} +} +Matrix is of size (36, 36) + Running solver: cg diff --git a/benchmark/test/reference/distributed_solver.matrix.stdout b/benchmark/test/reference/distributed_solver.matrix.stdout new file mode 100644 index 00000000000..157c40fdccf --- /dev/null +++ b/benchmark/test/reference/distributed_solver.matrix.stdout @@ -0,0 +1,57 @@ + +[ + { + "filename": "../../matrices/test/ani1.mtx", + "optimal": { + "spmv": "csr-csr" + }, + "solver": { + "cg": { + "recurrent_residuals": [], + "true_residuals": [], + "implicit_residuals": [], + "iteration_timestamps": [], + "rhs_norm": 1.0, + "generate": { + "components": { + "generate()": 1.0, + "free": 1.0, + "overhead": 1.0 + }, + "time": 1.0 + }, + "apply": { + "components": { + "apply()": 1.0, + "iteration": 1.0, + "allocate": 1.0, + "dense::fill": 1.0, + "cg::initialize": 1.0, + "advanced_apply()": 1.0, + "dense::row_gather": 1.0, + "csr::advanced_spmv": 1.0, + "dense::compute_squared_norm2": 1.0, + "dense::compute_sqrt": 1.0, + "copy()": 1.0, + "dense::copy": 1.0, + "dense::compute_conj_dot_dispatch": 1.0, + "check()": 1.0, + "residual_norm::residual_norm": 1.0, + "cg::step_1": 1.0, + "csr::spmv": 1.0, + "cg::step_2": 1.0, + "free": 1.0, + "overhead": 1.0 + }, + "iterations": 27, + "time": 1.0 + }, + "preconditioner": {}, + "residual_norm": 1.0, + "repetitions": 1, + "completed": true + } + }, + "size": 36 + } +] diff --git a/benchmark/test/reference/matrix_statistics.matrix.stderr b/benchmark/test/reference/matrix_statistics.matrix.stderr new file mode 100644 index 00000000000..0f7cc261c47 --- /dev/null +++ b/benchmark/test/reference/matrix_statistics.matrix.stderr @@ -0,0 +1,8 @@ +This is Ginkgo 1.7.0 (develop) + running with core module 1.7.0 (develop) +Running test case +{ + "filename": "../../matrices/test/ani1.mtx", + "problem": {} +} +Matrix is of size (36, 36) diff --git a/benchmark/test/reference/matrix_statistics.matrix.stdout b/benchmark/test/reference/matrix_statistics.matrix.stdout new file mode 100644 index 00000000000..20657f7abd5 --- /dev/null +++ b/benchmark/test/reference/matrix_statistics.matrix.stdout @@ -0,0 +1,38 @@ + +[ + { + "filename": "../../matrices/test/ani1.mtx", + "problem": { + "rows": 36, + "columns": 36, + "nonzeros": 208, + "row_distribution": { + "min": 4, + "q1": 4.5, + "median": 6.0, + "q3": 7.0, + "max": 9, + "mean": 5.777777777777778, + "variance": 2.061728395061728, + "skewness": 0.3366362745126052, + "kurtosis": 2.0507009932231366, + "hyperskewness": 1.9165991338199193, + "hyperflatness": 6.0545648993883665 + }, + "col_distribution": { + "min": 4, + "q1": 4.5, + "median": 6.0, + "q3": 7.0, + "max": 9, + "mean": 5.777777777777778, + "variance": 2.061728395061728, + "skewness": 0.3366362745126052, + "kurtosis": 2.0507009932231366, + "hyperskewness": 1.9165991338199193, + "hyperflatness": 6.0545648993883665 + } + }, + "size": 36 + } +] diff --git a/benchmark/test/reference/preconditioner.matrix.stderr b/benchmark/test/reference/preconditioner.matrix.stderr new file mode 100644 index 00000000000..d54c99ac971 --- /dev/null +++ b/benchmark/test/reference/preconditioner.matrix.stderr @@ -0,0 +1,42 @@ +This is Ginkgo 1.7.0 (develop) + running with core module 1.7.0 (develop) +Running on reference(0) +Running with 2 warm iterations and 10 running iterations +The random seed for right hand sides is 42 +Running with preconditioners: none +Running test case +{ + "filename": "../../matrices/test/ani1.mtx", + "preconditioner": {} +} +Matrix is of size (36, 36) +Current state: +[ + { + "filename": "../../matrices/test/ani1.mtx", + "preconditioner": { + "none": { + "generate": { + "components": { + "generate()": 1.0, + "overhead": 1.0 + }, + "time": 1.0, + "repetitions": 10 + }, + "apply": { + "components": { + "apply()": 1.0, + "copy()": 1.0, + "dense::copy": 1.0, + "overhead": 1.0 + }, + "time": 1.0, + "repetitions": 10 + }, + "completed": true + } + }, + "size": 36 + } +] diff --git a/benchmark/test/reference/preconditioner.matrix.stdout b/benchmark/test/reference/preconditioner.matrix.stdout new file mode 100644 index 00000000000..22c0c9cc1c1 --- /dev/null +++ b/benchmark/test/reference/preconditioner.matrix.stdout @@ -0,0 +1,30 @@ + +[ + { + "filename": "../../matrices/test/ani1.mtx", + "preconditioner": { + "none": { + "generate": { + "components": { + "generate()": 1.0, + "overhead": 1.0 + }, + "time": 1.0, + "repetitions": 10 + }, + "apply": { + "components": { + "apply()": 1.0, + "copy()": 1.0, + "dense::copy": 1.0, + "overhead": 1.0 + }, + "time": 1.0, + "repetitions": 10 + }, + "completed": true + } + }, + "size": 36 + } +] diff --git a/benchmark/test/reference/solver.matrix.stderr b/benchmark/test/reference/solver.matrix.stderr new file mode 100644 index 00000000000..78b75c301f7 --- /dev/null +++ b/benchmark/test/reference/solver.matrix.stderr @@ -0,0 +1,17 @@ +This is Ginkgo 1.7.0 (develop) + running with core module 1.7.0 (develop) +Running on reference(0) +Running with 2 warm iterations and 1 running iterations +The random seed for right hand sides is 42 +Running cg with 1000 iterations and residual goal of 1.000000e-06 +The number of right hand sides is 1 +Running test case +{ + "filename": "../../matrices/test/ani1.mtx", + "optimal": { + "spmv": "csr" + }, + "solver": {} +} +Matrix is of size (36, 36) + Running solver: cg diff --git a/benchmark/test/reference/solver.matrix.stdout b/benchmark/test/reference/solver.matrix.stdout new file mode 100644 index 00000000000..4a68d8a599b --- /dev/null +++ b/benchmark/test/reference/solver.matrix.stdout @@ -0,0 +1,55 @@ + +[ + { + "filename": "../../matrices/test/ani1.mtx", + "optimal": { + "spmv": "csr" + }, + "solver": { + "cg": { + "recurrent_residuals": [], + "true_residuals": [], + "implicit_residuals": [], + "iteration_timestamps": [], + "rhs_norm": 1.0, + "generate": { + "components": { + "generate()": 1.0, + "free": 1.0, + "overhead": 1.0 + }, + "time": 1.0 + }, + "apply": { + "components": { + "apply()": 1.0, + "iteration": 1.0, + "allocate": 1.0, + "dense::fill": 1.0, + "cg::initialize": 1.0, + "advanced_apply()": 1.0, + "csr::advanced_spmv": 1.0, + "dense::compute_norm2_dispatch": 1.0, + "copy()": 1.0, + "dense::copy": 1.0, + "dense::compute_conj_dot_dispatch": 1.0, + "check()": 1.0, + "residual_norm::residual_norm": 1.0, + "cg::step_1": 1.0, + "csr::spmv": 1.0, + "cg::step_2": 1.0, + "free": 1.0, + "overhead": 1.0 + }, + "iterations": 27, + "time": 1.0 + }, + "preconditioner": {}, + "residual_norm": 1.0, + "repetitions": 1, + "completed": true + } + }, + "size": 36 + } +] diff --git a/benchmark/test/reference/sparse_blas.matrix.stderr b/benchmark/test/reference/sparse_blas.matrix.stderr new file mode 100644 index 00000000000..404a761aec9 --- /dev/null +++ b/benchmark/test/reference/sparse_blas.matrix.stderr @@ -0,0 +1,36 @@ +This is Ginkgo 1.7.0 (develop) + running with core module 1.7.0 (develop) +Running on reference(0) +Running with 2 warm iterations and 10 running iterations +The random seed for right hand sides is 42 +The operations are transposeRunning test case +{ + "filename": "../../matrices/test/ani1.mtx", + "sparse_blas": {} +} +Matrix is of size (36, 36), 208 +Current state: +[ + { + "filename": "../../matrices/test/ani1.mtx", + "sparse_blas": { + "transpose": { + "time": 1.0, + "flops": 1.0, + "bandwidth": 1.0, + "repetitions": 10, + "components": { + "allocate": 1.0, + "components::fill_array": 1.0, + "csr::transpose": 1.0, + "free": 1.0, + "overhead": 1.0 + }, + "completed": true + } + }, + "rows": 36, + "cols": 36, + "nonzeros": 208 + } +] diff --git a/benchmark/test/reference/sparse_blas.matrix.stdout b/benchmark/test/reference/sparse_blas.matrix.stdout new file mode 100644 index 00000000000..ae983436081 --- /dev/null +++ b/benchmark/test/reference/sparse_blas.matrix.stdout @@ -0,0 +1,25 @@ + +[ + { + "filename": "../../matrices/test/ani1.mtx", + "sparse_blas": { + "transpose": { + "time": 1.0, + "flops": 1.0, + "bandwidth": 1.0, + "repetitions": 10, + "components": { + "allocate": 1.0, + "components::fill_array": 1.0, + "csr::transpose": 1.0, + "free": 1.0, + "overhead": 1.0 + }, + "completed": true + } + }, + "rows": 36, + "cols": 36, + "nonzeros": 208 + } +] diff --git a/benchmark/test/reference/spmv.matrix.stderr b/benchmark/test/reference/spmv.matrix.stderr new file mode 100644 index 00000000000..1096d64c74e --- /dev/null +++ b/benchmark/test/reference/spmv.matrix.stderr @@ -0,0 +1,31 @@ +This is Ginkgo 1.7.0 (develop) + running with core module 1.7.0 (develop) +Running on reference(0) +Running with 2 warm iterations and 10 running iterations +The random seed for right hand sides is 42 +The formats are coo +The number of right hand sides is 1 +Running test case +{ + "filename": "../../matrices/test/ani1.mtx", + "spmv": {} +} +Matrix is of size (36, 36) +Current state: +[ + { + "filename": "../../matrices/test/ani1.mtx", + "spmv": { + "coo": { + "storage": 3328, + "max_relative_norm2": 1.0, + "time": 1.0, + "repetitions": 10, + "completed": true + } + }, + "size": 36, + "nnz": 208, + "optimal": {} + } +] diff --git a/benchmark/test/reference/spmv.matrix.stdout b/benchmark/test/reference/spmv.matrix.stdout new file mode 100644 index 00000000000..b51f331f918 --- /dev/null +++ b/benchmark/test/reference/spmv.matrix.stdout @@ -0,0 +1,20 @@ + +[ + { + "filename": "../../matrices/test/ani1.mtx", + "spmv": { + "coo": { + "storage": 3328, + "max_relative_norm2": 1.0, + "time": 1.0, + "repetitions": 10, + "completed": true + } + }, + "size": 36, + "nnz": 208, + "optimal": { + "spmv": "coo" + } + } +] diff --git a/benchmark/test/solver.py b/benchmark/test/solver.py index e974f849276..025ee92707c 100755 --- a/benchmark/test/solver.py +++ b/benchmark/test/solver.py @@ -24,6 +24,13 @@ expected_stderr="solver.simple.stderr", ) +# input matrix file +test_framework.compare_output( + ["-input_matrix", str(test_framework.matrixpath)], + expected_stdout="solver.matrix.stdout", + expected_stderr="solver.matrix.stderr", +) + # profiler annotations test_framework.compare_output( [ diff --git a/benchmark/test/solver_distributed.py b/benchmark/test/solver_distributed.py index c6623723a43..54bbb030077 100644 --- a/benchmark/test/solver_distributed.py +++ b/benchmark/test/solver_distributed.py @@ -27,6 +27,13 @@ expected_stderr="distributed_solver.simple.stderr", ) +# input matrix file +test_framework.compare_output( + ["-input_matrix", str(test_framework.matrixpath)], + expected_stdout="distributed_solver.matrix.stdout", + expected_stderr="distributed_solver.matrix.stderr", +) + # profiler annotations test_framework.compare_output( [ diff --git a/benchmark/test/sparse_blas.py b/benchmark/test/sparse_blas.py index 7b0968a710c..724cdb866f0 100755 --- a/benchmark/test/sparse_blas.py +++ b/benchmark/test/sparse_blas.py @@ -29,6 +29,18 @@ expected_stderr="sparse_blas.simple.stderr", ) +# input matrix file +test_framework.compare_output( + [ + "-operations", + "transpose", + "-input_matrix", + str(test_framework.matrixpath), + ], + expected_stdout="sparse_blas.matrix.stdout", + expected_stderr="sparse_blas.matrix.stderr", +) + # profiler annotations (transpose has the smallest number of allocations) test_framework.compare_output( [ diff --git a/benchmark/test/spmv.py b/benchmark/test/spmv.py index 6e2d9f05d49..865f74bb6d0 100755 --- a/benchmark/test/spmv.py +++ b/benchmark/test/spmv.py @@ -24,6 +24,13 @@ expected_stderr="spmv.simple.stderr", ) +# input matrix file +test_framework.compare_output( + ["-input_matrix", str(test_framework.matrixpath)], + expected_stdout="spmv.matrix.stdout", + expected_stderr="spmv.matrix.stderr", +) + # profiler annotations test_framework.compare_output( [ diff --git a/benchmark/test/test_framework.py.in b/benchmark/test/test_framework.py.in index 6037f8c594e..1c762905c77 100644 --- a/benchmark/test/test_framework.py.in +++ b/benchmark/test/test_framework.py.in @@ -9,6 +9,7 @@ import sys sourcepath = pathlib.Path("@CMAKE_CURRENT_SOURCE_DIR@") binpath = pathlib.Path("@PROJECT_BINARY_DIR@") +matrixpath = pathlib.Path("../../matrices/test/ani1.mtx") generate = False if len(sys.argv) > 2 and sys.argv[2] == "--generate": generate = True @@ -99,8 +100,7 @@ def sanitize_json_in_text(lines: List[str]) -> List[str]: for begin, end, do_sanitize in combined_pairs ] reconstructed = [ - json.dumps(sanitize_json(json.loads(t)), - indent=4) if do_sanitize else t + json.dumps(sanitize_json(json.loads(t)), indent=4) if do_sanitize else t for t, do_sanitize in texts ] return "\n".join(reconstructed).split("\n") @@ -135,7 +135,7 @@ def determinize_text( break if keep: output_lines.append(line) - if output_lines[-1] != "": + if len(output_lines) == 0 or output_lines[-1] != "": output_lines.append("") try: return sanitize_json_in_text(output_lines) @@ -215,8 +215,7 @@ def compare_output_impl( print("FAIL: stdout differs") print( "\n".join( - difflib.unified_diff( - expected_stdout_processed, result_stdout_processed) + difflib.unified_diff(expected_stdout_processed, result_stdout_processed) ) ) failed = True @@ -224,8 +223,7 @@ def compare_output_impl( print("FAIL: stderr differs") print( "\n".join( - difflib.unified_diff( - expected_stderr_processed, result_stderr_processed) + difflib.unified_diff(expected_stderr_processed, result_stderr_processed) ) ) failed = True