Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add support for profiler and file input to benchmarks #1342

Merged
merged 8 commits into from
Jun 4, 2023
Merged

Add support for profiler and file input to benchmarks #1342

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
a0c60a9
fix GCR warning
upsj Apr 14, 2023
9fa95c8
add profiler support to benchmarks
upsj Apr 14, 2023
53dc5b6
make input stream configurable
upsj Apr 14, 2023
a86eef7
improve documentation
upsj Jun 1, 2023
9df2d42
remove unnecessary move-assignment operator
upsj Jun 2, 2023
f7ff448
improve flag documentation and disable detailed
upsj Jun 2, 2023
8109227
fix describe_config
upsj Jun 2, 2023
d94433b
extract common annotate functor
upsj Jun 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion benchmark/blas/blas.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ Parameters for a benchmark case are:
print_general_information(extra_information);
auto exec = executor_factory.at(FLAGS_executor)(FLAGS_gpu_timer);

rapidjson::IStreamWrapper jcin(std::cin);
rapidjson::IStreamWrapper jcin(get_input_stream());
rapidjson::Document test_cases;
test_cases.ParseStream(jcin);
if (!test_cases.IsArray()) {
Expand Down
40 changes: 37 additions & 3 deletions benchmark/blas/blas_common.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -434,6 +434,28 @@ dimensions parse_dims(rapidjson::Value& test_case)
}


std::string describe(rapidjson::Value& test_case)
{
std::stringstream ss;
auto optional_output = [&](const char* name) {
if (test_case.HasMember(name) && test_case[name].IsInt64()) {
ss << name << " = " << test_case[name].GetInt64() << " ";
}
};
optional_output("n");
optional_output("k");
optional_output("m");
optional_output("r");
optional_output("stride");
optional_output("stride_x");
optional_output("stride_y");
optional_output("stride_A");
optional_output("stride_B");
optional_output("stride_C");
return ss.str();
}


template <typename OpMap>
void apply_blas(const char* operation_name, std::shared_ptr<gko::Executor> exec,
std::shared_ptr<Timer> timer, const OpMap& operation_map,
Expand Down Expand Up @@ -501,6 +523,11 @@ void run_blas_benchmarks(std::shared_ptr<gko::Executor> exec,
{
auto operations = split(FLAGS_operations, ',');
auto& allocator = test_cases.GetAllocator();
auto profiler_hook = create_profiler_hook(exec);
if (profiler_hook) {
exec->add_logger(profiler_hook);
}
auto annotate = annotate_functor{profiler_hook};

for (auto& test_case : test_cases.GetArray()) {
try {
Expand All @@ -521,10 +548,14 @@ void run_blas_benchmarks(std::shared_ptr<gko::Executor> exec,
if (do_print) {
std::clog << "Running test case: " << test_case << std::endl;
}

// annotate the test case
auto test_case_range = annotate(describe(test_case));
for (const auto& operation_name : operations) {
apply_blas(operation_name.c_str(), exec, timer, operation_map,
test_case, allocator);
{
auto operation_range = annotate(operation_name.c_str());
apply_blas(operation_name.c_str(), exec, timer,
operation_map, test_case, allocator);
}

if (do_print) {
std::clog << "Current state:" << std::endl
Expand All @@ -538,4 +569,7 @@ void run_blas_benchmarks(std::shared_ptr<gko::Executor> exec,
<< std::endl;
}
}
if (profiler_hook) {
exec->remove_logger(profiler_hook);
}
}
2 changes: 1 addition & 1 deletion benchmark/blas/distributed/multi_vector.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ Parameters for a benchmark case are:

auto exec = executor_factory_mpi.at(FLAGS_executor)(comm.get());

std::string json_input = broadcast_json_input(std::cin, comm);
std::string json_input = broadcast_json_input(get_input_stream(), comm);
rapidjson::Document test_cases;
test_cases.Parse(json_input.c_str());
if (!test_cases.IsArray()) {
Expand Down
27 changes: 21 additions & 6 deletions benchmark/conversions/conversions.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,14 +123,21 @@ int main(int argc, char* argv[])
auto exec = executor_factory.at(FLAGS_executor)(FLAGS_gpu_timer);
auto formats = split(FLAGS_formats, ',');

rapidjson::IStreamWrapper jcin(std::cin);
rapidjson::IStreamWrapper jcin(get_input_stream());
rapidjson::Document test_cases;
test_cases.ParseStream(jcin);
if (!test_cases.IsArray()) {
print_config_error_and_exit();
}

auto& allocator = test_cases.GetAllocator();
auto profiler_hook = create_profiler_hook(exec);
if (profiler_hook) {
exec->add_logger(profiler_hook);
}
auto annotate = annotate_functor{profiler_hook};

DefaultSystemGenerator<> generator{};

for (auto& test_case : test_cases.GetArray()) {
std::clog << "Benchmarking conversions. " << std::endl;
Expand All @@ -146,7 +153,7 @@ int main(int argc, char* argv[])
std::clog << "Running test case: " << test_case << std::endl;
gko::matrix_data<etype, itype> data;
try {
data = DefaultSystemGenerator<>::generate_matrix_data(test_case);
data = generator.generate_matrix_data(test_case);
} catch (std::exception& e) {
std::cerr << "Error setting up matrix data, what(): " << e.what()
<< std::endl;
Expand All @@ -160,6 +167,8 @@ int main(int argc, char* argv[])
std::clog << "Matrix is of size (" << data.size[0] << ", "
<< data.size[1] << ")" << std::endl;
add_or_set_member(test_case, "size", data.size[0], allocator);
// annotate the test case
auto test_case_range = annotate(generator.describe_config(test_case));
for (const auto& format_from : formats) {
try {
auto matrix_from =
Expand All @@ -175,10 +184,13 @@ int main(int argc, char* argv[])
conversion_case.HasMember(conversion_name.c_str())) {
continue;
}

convert_matrix(matrix_from.get(), format_to.c_str(),
conversion_name.c_str(), exec, test_case,
allocator);
{
auto conversion_range =
annotate(conversion_name.c_str());
convert_matrix(matrix_from.get(), format_to.c_str(),
conversion_name.c_str(), exec, test_case,
allocator);
}
std::clog << "Current state:" << std::endl
<< test_cases << std::endl;
}
Expand All @@ -202,6 +214,9 @@ int main(int argc, char* argv[])
}
}
}
if (profiler_hook) {
exec->remove_logger(profiler_hook);
}

std::cout << test_cases << std::endl;
}
2 changes: 1 addition & 1 deletion benchmark/matrix_generator/matrix_generator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ int main(int argc, char* argv[])
std::clog << gko::version_info::get() << std::endl;

auto engine = get_engine();
rapidjson::IStreamWrapper jcin(std::cin);
rapidjson::IStreamWrapper jcin(get_input_stream());
rapidjson::Document configurations;
configurations.ParseStream(jcin);

Expand Down
2 changes: 1 addition & 1 deletion benchmark/matrix_statistics/matrix_statistics.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ int main(int argc, char* argv[])

std::clog << gko::version_info::get() << std::endl;

rapidjson::IStreamWrapper jcin(std::cin);
rapidjson::IStreamWrapper jcin(get_input_stream());
rapidjson::Document test_cases;
test_cases.ParseStream(jcin);
if (!test_cases.IsArray()) {
Expand Down
23 changes: 19 additions & 4 deletions benchmark/preconditioner/preconditioner.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -278,15 +278,19 @@ int main(int argc, char* argv[])
std::exit(1);
}

rapidjson::IStreamWrapper jcin(std::cin);
rapidjson::IStreamWrapper jcin(get_input_stream());
rapidjson::Document test_cases;
test_cases.ParseStream(jcin);
if (!test_cases.IsArray()) {
print_config_error_and_exit();
}

auto& allocator = test_cases.GetAllocator();

auto profiler_hook = create_profiler_hook(exec);
if (profiler_hook) {
exec->add_logger(profiler_hook);
}
auto annotate = annotate_functor{profiler_hook};
DefaultSystemGenerator<> generator{};

for (auto& test_case : test_cases.GetArray()) {
Expand All @@ -308,6 +312,10 @@ int main(int argc, char* argv[])
}
std::clog << "Running test case: " << test_case << std::endl;

// annotate the test case
auto test_case_range =
annotate(generator.describe_config(test_case));

auto data = generator.generate_matrix_data(test_case);

auto system_matrix =
Expand All @@ -322,8 +330,12 @@ int main(int argc, char* argv[])
<< std::endl;
add_or_set_member(test_case, "size", data.size[0], allocator);
for (const auto& precond_name : preconditioners) {
run_preconditioner(precond_name.c_str(), exec, system_matrix,
b.get(), x.get(), test_case, allocator);
{
auto precond_range = annotate(precond_name.c_str());
run_preconditioner(precond_name.c_str(), exec,
system_matrix, b.get(), x.get(),
test_case, allocator);
}
std::clog << "Current state:" << std::endl
<< test_cases << std::endl;
backup_results(test_cases);
Expand All @@ -333,6 +345,9 @@ int main(int argc, char* argv[])
<< std::endl;
}
}
if (profiler_hook) {
exec->remove_logger(profiler_hook);
}

std::cout << test_cases << std::endl;
}
6 changes: 3 additions & 3 deletions benchmark/solver/distributed/solver.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,12 +128,12 @@ int main(int argc, char* argv[])
}
}

std::string json_input = FLAGS_overhead
? R"(
std::string json_input =
FLAGS_overhead ? R"(
[{"filename": "overhead.mtx",
"optimal": {"spmv": "csr-csr"}]
)"
: broadcast_json_input(std::cin, comm);
: broadcast_json_input(get_input_stream(), comm);
rapidjson::Document test_cases;
test_cases.Parse(json_input.c_str());

Expand Down
2 changes: 1 addition & 1 deletion benchmark/solver/solver.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ int main(int argc, char* argv[])

rapidjson::Document test_cases;
if (!FLAGS_overhead) {
rapidjson::IStreamWrapper jcin(std::cin);
rapidjson::IStreamWrapper jcin(get_input_stream());
test_cases.ParseStream(jcin);
} else {
// Fake test case to run once
Expand Down
24 changes: 20 additions & 4 deletions benchmark/solver/solver_common.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -610,6 +610,11 @@ void run_solver_benchmarks(std::shared_ptr<gko::Executor> exec,
}

auto& allocator = test_cases.GetAllocator();
auto profiler_hook = create_profiler_hook(exec);
if (profiler_hook) {
exec->add_logger(profiler_hook);
}
auto annotate = annotate_functor{profiler_hook};

for (auto& test_case : test_cases.GetArray()) {
try {
Expand All @@ -628,6 +633,10 @@ void run_solver_benchmarks(std::shared_ptr<gko::Executor> exec,
})) {
continue;
}
// annotate the test case
auto test_case_range =
annotate(system_generator.describe_config(test_case));

if (do_print) {
std::clog << "Running test case: " << test_case << std::endl;
}
Expand Down Expand Up @@ -660,16 +669,20 @@ void run_solver_benchmarks(std::shared_ptr<gko::Executor> exec,
allocator);
auto precond_solver_name = begin(precond_solvers);
for (const auto& solver_name : solvers) {
auto solver_range = annotate(solver_name.c_str());
for (const auto& precond_name : preconds) {
if (do_print) {
std::clog
<< "\tRunning solver: " << *precond_solver_name
<< std::endl;
}
solve_system(solver_name, precond_name,
precond_solver_name->c_str(), exec, timer,
system_matrix, b.get(), x.get(), test_case,
allocator);
{
auto precond_range = annotate(precond_name.c_str());
solve_system(solver_name, precond_name,
precond_solver_name->c_str(), exec, timer,
system_matrix, b.get(), x.get(), test_case,
allocator);
}
if (do_print) {
backup_results(test_cases);
}
Expand All @@ -686,6 +699,9 @@ void run_solver_benchmarks(std::shared_ptr<gko::Executor> exec,
}
}
}
if (profiler_hook) {
exec->remove_logger(profiler_hook);
}
}


Expand Down
25 changes: 20 additions & 5 deletions benchmark/sparse_blas/sparse_blas.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ int main(int argc, char* argv[])

auto exec = executor_factory.at(FLAGS_executor)(FLAGS_gpu_timer);

rapidjson::IStreamWrapper jcin(std::cin);
rapidjson::IStreamWrapper jcin(get_input_stream());
rapidjson::Document test_cases;
test_cases.ParseStream(jcin);
if (!test_cases.IsArray()) {
Expand All @@ -166,9 +166,16 @@ int main(int argc, char* argv[])
print_general_information(extra_information);

auto& allocator = test_cases.GetAllocator();
auto profiler_hook = create_profiler_hook(exec);
if (profiler_hook) {
exec->add_logger(profiler_hook);
}
auto annotate = annotate_functor{profiler_hook};

auto operations = split(FLAGS_operations, ',');

DefaultSystemGenerator<> generator{};

for (auto& test_case : test_cases.GetArray()) {
try {
// set up benchmark
Expand All @@ -180,8 +187,7 @@ int main(int argc, char* argv[])
}
auto& sp_blas_case = test_case[benchmark_name];
std::clog << "Running test case: " << test_case << std::endl;
auto data =
DefaultSystemGenerator<>::generate_matrix_data(test_case);
auto data = generator.generate_matrix_data(test_case);
data.ensure_row_major_order();
std::clog << "Matrix is of size (" << data.size[0] << ", "
<< data.size[1] << "), " << data.nonzeros.size()
Expand All @@ -193,11 +199,17 @@ int main(int argc, char* argv[])

auto mtx = Mtx::create(exec, data.size, data.nonzeros.size());
mtx->read(data);
// annotate the test case
auto test_case_range =
annotate(generator.describe_config(test_case));
for (const auto& operation_name : operations) {
if (FLAGS_overwrite ||
!sp_blas_case.HasMember(operation_name.c_str())) {
apply_sparse_blas(operation_name.c_str(), exec, mtx.get(),
sp_blas_case, allocator);
{
auto operation_range = annotate(operation_name.c_str());
apply_sparse_blas(operation_name.c_str(), exec,
mtx.get(), sp_blas_case, allocator);
}
std::clog << "Current state:" << std::endl
<< test_cases << std::endl;
backup_results(test_cases);
Expand All @@ -215,6 +227,9 @@ int main(int argc, char* argv[])
}
}
}
if (profiler_hook) {
exec->remove_logger(profiler_hook);
}

std::cout << test_cases << std::endl;
}
2 changes: 1 addition & 1 deletion benchmark/spmv/distributed/spmv.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ int main(int argc, char* argv[])
}
}

std::string json_input = broadcast_json_input(std::cin, comm);
std::string json_input = broadcast_json_input(get_input_stream(), comm);
rapidjson::Document test_cases;
test_cases.Parse(json_input.c_str());
if (!test_cases.IsArray()) {
Expand Down
2 changes: 1 addition & 1 deletion benchmark/spmv/spmv.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ int main(int argc, char* argv[])
auto exec = executor_factory.at(FLAGS_executor)(FLAGS_gpu_timer);
auto formats = split(FLAGS_formats, ',');

rapidjson::IStreamWrapper jcin(std::cin);
rapidjson::IStreamWrapper jcin(get_input_stream());
rapidjson::Document test_cases;
test_cases.ParseStream(jcin);
if (!test_cases.IsArray()) {
Expand Down
Loading