diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cdb1cdf4..01475ffee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ Documentation for rocRAND is available at * Added host generator for MT19937 * Support for `rocrand_generate_poisson` in hipGraphs +* Added engine, distribution, mode, throughput_gigabytes_per_second, and lambda columns for csv format in + benchmark_rocrand_host_api and benchmark_rocrand_device_api. To see these new columns set --benchmark_format=csv + or --benchmark_out_format=csv --benchmark_out="outName.csv" ### Changes diff --git a/benchmark/benchmark_rocrand_device_api.cpp b/benchmark/benchmark_rocrand_device_api.cpp index 44ddec8fb..b38430efc 100644 --- a/benchmark/benchmark_rocrand_device_api.cpp +++ b/benchmark/benchmark_rocrand_device_api.cpp @@ -28,7 +28,9 @@ #include #include +#include "custom_csv_formater.hpp" #include +#include #include #include #include @@ -40,8 +42,10 @@ #endif template -__global__ __launch_bounds__(ROCRAND_DEFAULT_MAX_BLOCK_SIZE) void init_kernel( - EngineState* states, const unsigned long long seed, const unsigned long long offset) +__global__ __launch_bounds__(ROCRAND_DEFAULT_MAX_BLOCK_SIZE) +void init_kernel(EngineState* states, + const unsigned long long seed, + const unsigned long long offset) { const unsigned int state_id = blockIdx.x * blockDim.x + threadIdx.x; EngineState state; @@ -50,8 +54,9 @@ __global__ __launch_bounds__(ROCRAND_DEFAULT_MAX_BLOCK_SIZE) void init_kernel( } template -__global__ __launch_bounds__(ROCRAND_DEFAULT_MAX_BLOCK_SIZE) void generate_kernel( - EngineState* states, T* data, const size_t size, Generator generator) +__global__ +__launch_bounds__(ROCRAND_DEFAULT_MAX_BLOCK_SIZE) +void generate_kernel(EngineState* states, T* data, const size_t size, Generator generator) { const unsigned int state_id = blockIdx.x * blockDim.x + threadIdx.x; const unsigned int stride = gridDim.x * blockDim.x; @@ -119,12 +124,13 @@ struct runner }; template -__global__ __launch_bounds__(ROCRAND_DEFAULT_MAX_BLOCK_SIZE) void generate_kernel( - rocrand_state_mtgp32* states, T* data, const size_t size, Generator generator) +__global__ +__launch_bounds__(ROCRAND_DEFAULT_MAX_BLOCK_SIZE) +void generate_kernel(rocrand_state_mtgp32* states, T* data, const size_t size, Generator generator) { const unsigned int state_id = blockIdx.x; unsigned int index = blockIdx.x * blockDim.x + threadIdx.x; - unsigned int stride = gridDim.x * blockDim.x; + unsigned int stride = gridDim.x * blockDim.x; __shared__ rocrand_state_mtgp32 state; rocrand_mtgp32_block_copy(&states[state_id], &state); @@ -191,8 +197,8 @@ struct runner } }; -__global__ __launch_bounds__(ROCRAND_DEFAULT_MAX_BLOCK_SIZE) void init_kernel( - rocrand_state_lfsr113* states, const uint4 seed) +__global__ __launch_bounds__(ROCRAND_DEFAULT_MAX_BLOCK_SIZE) +void init_kernel(rocrand_state_lfsr113* states, const uint4 seed) { const unsigned int state_id = blockIdx.x * blockDim.x + threadIdx.x; rocrand_state_lfsr113 state; @@ -255,8 +261,9 @@ struct runner }; template -__global__ __launch_bounds__(ROCRAND_DEFAULT_MAX_BLOCK_SIZE) void init_sobol_kernel( - EngineState* states, SobolType* directions, SobolType offset) +__global__ +__launch_bounds__(ROCRAND_DEFAULT_MAX_BLOCK_SIZE) +void init_sobol_kernel(EngineState* states, SobolType* directions, SobolType offset) { const unsigned int dimension = blockIdx.y; const unsigned int state_id = blockIdx.x * blockDim.x + threadIdx.x; @@ -266,8 +273,12 @@ __global__ __launch_bounds__(ROCRAND_DEFAULT_MAX_BLOCK_SIZE) void init_sobol_ker } template -__global__ __launch_bounds__(ROCRAND_DEFAULT_MAX_BLOCK_SIZE) void init_scrambled_sobol_kernel( - EngineState* states, SobolType* directions, SobolType* scramble_constants, SobolType offset) +__global__ +__launch_bounds__(ROCRAND_DEFAULT_MAX_BLOCK_SIZE) +void init_scrambled_sobol_kernel(EngineState* states, + SobolType* directions, + SobolType* scramble_constants, + SobolType offset) { const unsigned int dimension = blockIdx.y; const unsigned int state_id = blockIdx.x * blockDim.x + threadIdx.x; @@ -281,8 +292,9 @@ __global__ __launch_bounds__(ROCRAND_DEFAULT_MAX_BLOCK_SIZE) void init_scrambled // generate_kernel for the normal and scrambled sobol generators template -__global__ __launch_bounds__(ROCRAND_DEFAULT_MAX_BLOCK_SIZE) void generate_sobol_kernel( - EngineState* states, T* data, const size_t size, Generator generator) +__global__ +__launch_bounds__(ROCRAND_DEFAULT_MAX_BLOCK_SIZE) +void generate_sobol_kernel(EngineState* states, T* data, const size_t size, Generator generator) { const unsigned int dimension = blockIdx.y; const unsigned int state_id = blockIdx.x * blockDim.x + threadIdx.x; @@ -614,7 +626,9 @@ struct generator_uint : public generator_type return "uniform-uint"; } - __device__ data_type operator()(Engine* state) const + __device__ + data_type + operator()(Engine* state) const { return rocrand(state); } @@ -630,7 +644,9 @@ struct generator_ullong : public generator_type return "uniform-ullong"; } - __device__ data_type operator()(Engine* state) const + __device__ + data_type + operator()(Engine* state) const { return rocrand(state); } @@ -646,7 +662,9 @@ struct generator_uniform : public generator_type return "uniform-float"; } - __device__ data_type operator()(Engine* state) const + __device__ + data_type + operator()(Engine* state) const { return rocrand_uniform(state); } @@ -662,7 +680,9 @@ struct generator_uniform_double : public generator_type return "uniform-double"; } - __device__ data_type operator()(Engine* state) const + __device__ + data_type + operator()(Engine* state) const { return rocrand_uniform_double(state); } @@ -678,7 +698,9 @@ struct generator_normal : public generator_type return "normal-float"; } - __device__ data_type operator()(Engine* state) const + __device__ + data_type + operator()(Engine* state) const { return rocrand_normal(state); } @@ -694,7 +716,9 @@ struct generator_normal_double : public generator_type return "normal-double"; } - __device__ data_type operator()(Engine* state) const + __device__ + data_type + operator()(Engine* state) const { return rocrand_normal_double(state); } @@ -710,7 +734,9 @@ struct generator_log_normal : public generator_type return "log-normal-float"; } - __device__ data_type operator()(Engine* state) const + __device__ + data_type + operator()(Engine* state) const { return rocrand_log_normal(state, 0.f, 1.f); } @@ -726,7 +752,9 @@ struct generator_log_normal_double : public generator_type return "log-normal-double"; } - __device__ data_type operator()(Engine* state) const + __device__ + data_type + operator()(Engine* state) const { return rocrand_log_normal_double(state, 0., 1.); } @@ -744,7 +772,9 @@ struct generator_poisson : public generator_type return "poisson(lambda=" + stream.str() + ")"; } - __device__ data_type operator()(Engine* state) + __device__ + data_type + operator()(Engine* state) { return rocrand_poisson(state, lambda); } @@ -774,7 +804,9 @@ struct generator_discrete_poisson : public generator_type ROCRAND_CHECK(rocrand_destroy_discrete_distribution(discrete_distribution)); } - __device__ data_type operator()(Engine* state) + __device__ + data_type + operator()(Engine* state) { return rocrand_discrete(state, discrete_distribution); } @@ -814,7 +846,9 @@ struct generator_discrete_custom : public generator_type ROCRAND_CHECK(rocrand_destroy_discrete_distribution(discrete_distribution)); } - __device__ data_type operator()(Engine* state) + __device__ + data_type + operator()(Engine* state) { return rocrand_discrete(state, discrete_distribution); } @@ -961,6 +995,14 @@ void add_benchmarks(const benchmark_context& ctx, int main(int argc, char* argv[]) { + // get paramaters before they are passed into + // benchmark::Initialize() + std::string outFormat = ""; + std::string filter = ""; + std::string consoleFormat = ""; + + getFormats(argc, argv, outFormat, filter, consoleFormat); + benchmark::Initialize(&argc, argv); cli::Parser parser(argc, argv); @@ -1045,8 +1087,16 @@ int main(int argc, char* argv[]) b->Unit(benchmark::kMillisecond); } + benchmark::BenchmarkReporter* console_reporter = getConsoleReporter(consoleFormat); + benchmark::BenchmarkReporter* out_file_reporter = getOutFileReporter(outFormat); + + std::string spec = (filter == "" || filter == "all") ? "." : filter; + // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); + if(outFormat == "") // default case + benchmark::RunSpecifiedBenchmarks(console_reporter, spec); + else + benchmark::RunSpecifiedBenchmarks(console_reporter, out_file_reporter, spec); HIP_CHECK(hipStreamDestroy(stream)); return 0; diff --git a/benchmark/benchmark_rocrand_generate.cpp b/benchmark/benchmark_rocrand_generate.cpp index a6bbabe12..c2fa286ab 100644 --- a/benchmark/benchmark_rocrand_generate.cpp +++ b/benchmark/benchmark_rocrand_generate.cpp @@ -18,36 +18,38 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -#include +#include #include -#include -#include +#include #include +#include #include -#include +#include #include "cmdparser.hpp" #include #include -#define HIP_CHECK(condition) \ - { \ - hipError_t error = condition; \ - if(error != hipSuccess){ \ - std::cout << "HIP error: " << error << " line: " << __LINE__ << std::endl; \ - exit(error); \ - } \ - } - -#define ROCRAND_CHECK(condition) \ - { \ - rocrand_status _status = condition; \ - if(_status != ROCRAND_STATUS_SUCCESS) { \ - std::cout << "ROCRAND error: " << _status << " line: " << __LINE__ << std::endl; \ - exit(_status); \ - } \ - } +#define HIP_CHECK(condition) \ + { \ + hipError_t error = condition; \ + if(error != hipSuccess) \ + { \ + std::cout << "HIP error: " << error << " line: " << __LINE__ << std::endl; \ + exit(error); \ + } \ + } + +#define ROCRAND_CHECK(condition) \ + { \ + rocrand_status _status = condition; \ + if(_status != ROCRAND_STATUS_SUCCESS) \ + { \ + std::cout << "ROCRAND error: " << _status << " line: " << __LINE__ << std::endl; \ + exit(_status); \ + } \ + } #ifndef DEFAULT_RAND_N const size_t DEFAULT_RAND_N = 1024 * 1024 * 128; @@ -56,32 +58,32 @@ const size_t DEFAULT_RAND_N = 1024 * 1024 * 128; typedef rocrand_rng_type rng_type_t; template -using generate_func_type = std::function; +using generate_func_type = std::function; template -void run_benchmark(const cli::Parser& parser, - const rng_type_t rng_type, - hipStream_t stream, +void run_benchmark(const cli::Parser& parser, + const rng_type_t rng_type, + hipStream_t stream, generate_func_type generate_func, - const std::string& distribution, - const std::string& engine, - const double lambda = 0.f) + const std::string& distribution, + const std::string& engine, + const double lambda = 0.f) { - const size_t size0 = parser.get("size"); - const size_t trials = parser.get("trials"); - const size_t dimensions = parser.get("dimensions"); - const size_t offset = parser.get("offset"); - const size_t size = (size0 / dimensions) * dimensions; - const std::string format = parser.get("format"); - - T * data; + const size_t size0 = parser.get("size"); + const size_t trials = parser.get("trials"); + const size_t dimensions = parser.get("dimensions"); + const size_t offset = parser.get("offset"); + const size_t size = (size0 / dimensions) * dimensions; + const std::string format = parser.get("format"); + + T* data; HIP_CHECK(hipMalloc(&data, size * sizeof(T))); rocrand_generator generator; ROCRAND_CHECK(rocrand_create_generator(&generator, rng_type)); rocrand_status status = rocrand_set_quasi_random_generator_dimensions(generator, dimensions); - if (status != ROCRAND_STATUS_TYPE_ERROR) // If the RNG is not quasi-random + if(status != ROCRAND_STATUS_TYPE_ERROR) // If the RNG is not quasi-random { ROCRAND_CHECK(status); } @@ -89,13 +91,13 @@ void run_benchmark(const cli::Parser& parser, ROCRAND_CHECK(rocrand_set_stream(generator, stream)); status = rocrand_set_offset(generator, offset); - if (status != ROCRAND_STATUS_TYPE_ERROR) // If the RNG is not pseudo-random + if(status != ROCRAND_STATUS_TYPE_ERROR) // If the RNG is not pseudo-random { ROCRAND_CHECK(status); } // Warm-up - for (size_t i = 0; i < 15; i++) + for(size_t i = 0; i < 15; i++) { ROCRAND_CHECK(generate_func(generator, data, size)); } @@ -106,7 +108,7 @@ void run_benchmark(const cli::Parser& parser, HIP_CHECK(hipEventCreate(&start)); HIP_CHECK(hipEventCreate(&stop)); HIP_CHECK(hipEventRecord(start, stream)); - for (size_t i = 0; i < trials; i++) + for(size_t i = 0; i < trials; i++) { ROCRAND_CHECK(generate_func(generator, data, size)); } @@ -117,43 +119,33 @@ void run_benchmark(const cli::Parser& parser, HIP_CHECK(hipEventDestroy(start)); HIP_CHECK(hipEventDestroy(stop)); - if (format.compare("csv") == 0) + if(format.compare("csv") == 0) { - std::cout << std::fixed << std::setprecision(3) - << engine << "," - << distribution << "," - << (trials * size * sizeof(T)) / - (elapsed / 1e3 * (1 << 30)) << "," - << (trials * size) / - (elapsed / 1e3 * (1 << 30)) << "," - << elapsed / trials << "," - << elapsed << "," - << size << ","; - if (distribution.compare("poisson") == 0 || distribution.compare("discrete-poisson") == 0) + std::cout << std::fixed << std::setprecision(3) << engine << "," << distribution << "," + << (trials * size * sizeof(T)) / (elapsed / 1e3 * (1 << 30)) << "," + << (trials * size) / (elapsed / 1e3 * (1 << 30)) << "," << elapsed / trials << "," + << elapsed << "," << size << ","; + if(distribution.compare("poisson") == 0 || distribution.compare("discrete-poisson") == 0) { - std::cout << lambda; + std::cout << lambda; } std::cout << std::endl; } else { - if (format.compare("console") != 0) + if(format.compare("console") != 0) { - std::cout << "Unknown format specified (must be either console or csv). Defaulting to console output." << std::endl; + std::cout << "Unknown format specified (must be either console or csv). Defaulting to " + "console output." + << std::endl; } - std::cout << std::fixed << std::setprecision(3) - << " " - << "Throughput = " - << std::setw(8) << (trials * size * sizeof(T)) / - (elapsed / 1e3 * (1 << 30)) - << " GB/s, Samples = " - << std::setw(8) << (trials * size) / - (elapsed / 1e3 * (1 << 30)) - << " GSample/s, AvgTime (1 trial) = " - << std::setw(8) << elapsed / trials - << " ms, Time (all) = " - << std::setw(8) << elapsed - << " ms, Size = " << size + std::cout << std::fixed << std::setprecision(3) << " " + << "Throughput = " << std::setw(8) + << (trials * size * sizeof(T)) / (elapsed / 1e3 * (1 << 30)) + << " GB/s, Samples = " << std::setw(8) + << (trials * size) / (elapsed / 1e3 * (1 << 30)) + << " GSample/s, AvgTime (1 trial) = " << std::setw(8) << elapsed / trials + << " ms, Time (all) = " << std::setw(8) << elapsed << " ms, Size = " << size << std::endl; } @@ -162,142 +154,164 @@ void run_benchmark(const cli::Parser& parser, } void run_benchmarks(const cli::Parser& parser, - const rng_type_t rng_type, + const rng_type_t rng_type, const std::string& distribution, const std::string& engine, - hipStream_t stream) + hipStream_t stream) { const std::string format = parser.get("format"); - if (distribution == "uniform-uint") - { - run_benchmark(parser, rng_type, stream, - [](rocrand_generator gen, unsigned int * data, size_t size) { - return rocrand_generate(gen, data, size); - }, - distribution, engine - ); - } - if (distribution == "uniform-uchar") + if(distribution == "uniform-uint") { - run_benchmark(parser, rng_type, stream, - [](rocrand_generator gen, unsigned char * data, size_t size) { - return rocrand_generate_char(gen, data, size); - }, - distribution, engine - ); + run_benchmark( + parser, + rng_type, + stream, + [](rocrand_generator gen, unsigned int* data, size_t size) + { return rocrand_generate(gen, data, size); }, + distribution, + engine); } - if (distribution == "uniform-ushort") + if(distribution == "uniform-uchar") { - run_benchmark(parser, rng_type, stream, - [](rocrand_generator gen, unsigned short * data, size_t size) { - return rocrand_generate_short(gen, data, size); - }, - distribution, engine - ); + run_benchmark( + parser, + rng_type, + stream, + [](rocrand_generator gen, unsigned char* data, size_t size) + { return rocrand_generate_char(gen, data, size); }, + distribution, + engine); } - if (distribution == "uniform-half") + if(distribution == "uniform-ushort") { - run_benchmark<__half>(parser, rng_type, stream, - [](rocrand_generator gen, __half * data, size_t size) { - return rocrand_generate_uniform_half(gen, data, size); - }, - distribution, engine - ); + run_benchmark( + parser, + rng_type, + stream, + [](rocrand_generator gen, unsigned short* data, size_t size) + { return rocrand_generate_short(gen, data, size); }, + distribution, + engine); } - if (distribution == "uniform-float") + if(distribution == "uniform-half") { - run_benchmark(parser, rng_type, stream, - [](rocrand_generator gen, float * data, size_t size) { - return rocrand_generate_uniform(gen, data, size); - }, - distribution, engine - ); + run_benchmark<__half>( + parser, + rng_type, + stream, + [](rocrand_generator gen, __half* data, size_t size) + { return rocrand_generate_uniform_half(gen, data, size); }, + distribution, + engine); } - if (distribution == "uniform-double") + if(distribution == "uniform-float") { - run_benchmark(parser, rng_type, stream, - [](rocrand_generator gen, double * data, size_t size) { - return rocrand_generate_uniform_double(gen, data, size); - }, - distribution, engine - ); + run_benchmark( + parser, + rng_type, + stream, + [](rocrand_generator gen, float* data, size_t size) + { return rocrand_generate_uniform(gen, data, size); }, + distribution, + engine); } - if (distribution == "normal-half") + if(distribution == "uniform-double") { - run_benchmark<__half>(parser, - rng_type, - stream, - [](rocrand_generator gen, __half* data, size_t size) { - return rocrand_generate_normal_half(gen, - data, - size, - __float2half(0.0f), - __float2half(1.0f)); - }, - distribution, - engine); + run_benchmark( + parser, + rng_type, + stream, + [](rocrand_generator gen, double* data, size_t size) + { return rocrand_generate_uniform_double(gen, data, size); }, + distribution, + engine); } - if (distribution == "normal-float") + if(distribution == "normal-half") { - run_benchmark(parser, rng_type, stream, - [](rocrand_generator gen, float * data, size_t size) { - return rocrand_generate_normal(gen, data, size, 0.0f, 1.0f); + run_benchmark<__half>( + parser, + rng_type, + stream, + [](rocrand_generator gen, __half* data, size_t size) { + return rocrand_generate_normal_half(gen, + data, + size, + __float2half(0.0f), + __float2half(1.0f)); }, - distribution, engine - ); + distribution, + engine); } - if (distribution == "normal-double") + if(distribution == "normal-float") { - run_benchmark(parser, rng_type, stream, - [](rocrand_generator gen, double * data, size_t size) { - return rocrand_generate_normal_double(gen, data, size, 0.0, 1.0); - }, - distribution, engine - ); + run_benchmark( + parser, + rng_type, + stream, + [](rocrand_generator gen, float* data, size_t size) + { return rocrand_generate_normal(gen, data, size, 0.0f, 1.0f); }, + distribution, + engine); } - if (distribution == "log-normal-half") + if(distribution == "normal-double") { - run_benchmark<__half>(parser, - rng_type, - stream, - [](rocrand_generator gen, __half* data, size_t size) - { - return rocrand_generate_log_normal_half(gen, - data, - size, - __float2half(0.0f), - __float2half(1.0f)); - }, - distribution, - engine); + run_benchmark( + parser, + rng_type, + stream, + [](rocrand_generator gen, double* data, size_t size) + { return rocrand_generate_normal_double(gen, data, size, 0.0, 1.0); }, + distribution, + engine); } - if (distribution == "log-normal-float") + if(distribution == "log-normal-half") { - run_benchmark(parser, rng_type, stream, - [](rocrand_generator gen, float * data, size_t size) { - return rocrand_generate_log_normal(gen, data, size, 0.0f, 1.0f); + run_benchmark<__half>( + parser, + rng_type, + stream, + [](rocrand_generator gen, __half* data, size_t size) + { + return rocrand_generate_log_normal_half(gen, + data, + size, + __float2half(0.0f), + __float2half(1.0f)); }, - distribution, engine - ); + distribution, + engine); } - if (distribution == "log-normal-double") + if(distribution == "log-normal-float") { - run_benchmark(parser, rng_type, stream, - [](rocrand_generator gen, double * data, size_t size) { - return rocrand_generate_log_normal_double(gen, data, size, 0.0, 1.0); - }, - distribution, engine - ); + run_benchmark( + parser, + rng_type, + stream, + [](rocrand_generator gen, float* data, size_t size) + { return rocrand_generate_log_normal(gen, data, size, 0.0f, 1.0f); }, + distribution, + engine); } - if (distribution == "poisson") + if(distribution == "log-normal-double") + { + run_benchmark( + parser, + rng_type, + stream, + [](rocrand_generator gen, double* data, size_t size) + { return rocrand_generate_log_normal_double(gen, data, size, 0.0, 1.0); }, + distribution, + engine); + } + if(distribution == "poisson") { const auto lambdas = parser.get>("lambda"); - for (double lambda : lambdas) + for(double lambda : lambdas) { - if (format.compare("console") == 0) + if(format.compare("console") == 0) { - std::cout << " " << "lambda " - << std::fixed << std::setprecision(1) << lambda << std::endl; + std::cout << " " + << "lambda " << std::fixed << std::setprecision(1) << lambda << std::endl; } run_benchmark( parser, @@ -330,66 +344,76 @@ const std::vector all_engines = { "scrambled_sobol64", }; -const std::vector all_distributions = { - "uniform-uint", - "uniform-uchar", - "uniform-ushort", - "uniform-half", - // "uniform-long-long", - "uniform-float", - "uniform-double", - "normal-half", - "normal-float", - "normal-double", - "log-normal-half", - "log-normal-float", - "log-normal-double", - "poisson" -}; - -int main(int argc, char *argv[]) +const std::vector all_distributions = {"uniform-uint", + "uniform-uchar", + "uniform-ushort", + "uniform-half", + // "uniform-long-long", + "uniform-float", + "uniform-double", + "normal-half", + "normal-float", + "normal-double", + "log-normal-half", + "log-normal-float", + "log-normal-double", + "poisson"}; + +int main(int argc, char* argv[]) { cli::Parser parser(argc, argv); - const std::string distribution_desc = - "space-separated list of distributions:" + - std::accumulate(all_distributions.begin(), all_distributions.end(), std::string(), - [](const std::string& a, const std::string& b) { - return a + "\n " + b; - } - ) + - "\n or all"; - const std::string engine_desc = - "space-separated list of random number engines:" + - std::accumulate(all_engines.begin(), all_engines.end(), std::string(), - [](const std::string& a, const std::string& b) { - return a + "\n " + b; - } - ) + - "\n or all"; + const std::string distribution_desc + = "space-separated list of distributions:" + + std::accumulate(all_distributions.begin(), + all_distributions.end(), + std::string(), + [](const std::string& a, const std::string& b) + { return a + "\n " + b; }) + + "\n or all"; + const std::string engine_desc = "space-separated list of random number engines:" + + std::accumulate(all_engines.begin(), + all_engines.end(), + std::string(), + [](const std::string& a, const std::string& b) + { return a + "\n " + b; }) + + "\n or all"; parser.set_optional("size", "size", DEFAULT_RAND_N, "number of values"); - parser.set_optional("dimensions", "dimensions", 1, "number of dimensions of quasi-random values"); + parser.set_optional("dimensions", + "dimensions", + 1, + "number of dimensions of quasi-random values"); parser.set_optional("offset", "offset", 0, "offset of generated pseudo-random values"); parser.set_optional("trials", "trials", 20, "number of trials"); - parser.set_optional>("dis", "dis", {"uniform-uint"}, distribution_desc); + parser.set_optional>("dis", + "dis", + {"uniform-uint"}, + distribution_desc); parser.set_optional>("engine", "engine", {"philox"}, engine_desc); - parser.set_optional>("lambda", "lambda", {10.0}, "space-separated list of lambdas of Poisson distribution"); - parser.set_optional("format", "format", {"console"}, "output format: console or csv"); + parser.set_optional>( + "lambda", + "lambda", + {10.0}, + "space-separated list of lambdas of Poisson distribution"); + parser.set_optional("format", + "format", + {"console"}, + "output format: console or csv"); parser.run_and_exit_if_error(); std::vector engines; { auto es = parser.get>("engine"); - if (std::find(es.begin(), es.end(), "all") != es.end()) + if(std::find(es.begin(), es.end(), "all") != es.end()) { engines = all_engines; } else { - for (auto e : all_engines) + for(auto e : all_engines) { - if (std::find(es.begin(), es.end(), e) != es.end()) + if(std::find(es.begin(), es.end(), e) != es.end()) engines.push_back(e); } } @@ -398,15 +422,15 @@ int main(int argc, char *argv[]) std::vector distributions; { auto ds = parser.get>("dis"); - if (std::find(ds.begin(), ds.end(), "all") != ds.end()) + if(std::find(ds.begin(), ds.end(), "all") != ds.end()) { distributions = all_distributions; } else { - for (auto d : all_distributions) + for(auto d : all_distributions) { - if (std::find(ds.begin(), ds.end(), d) != ds.end()) + if(std::find(ds.begin(), ds.end(), d) != ds.end()) distributions.push_back(d); } } @@ -430,27 +454,27 @@ int main(int argc, char *argv[]) hipStream_t stream; HIP_CHECK(hipStreamCreate(&stream)); - std::string format = parser.get("format"); - bool console_output = format.compare("console") == 0 ? true : false; + std::string format = parser.get("format"); + bool console_output = format.compare("console") == 0 ? true : false; - if (!console_output) + if(!console_output) { - std::cout << "Engine,Distribution,Throughput,Samples,AvgTime (1 Trial),Time(all),Size,Lambda" - << std::endl; - std::cout << ",,GB/s,GSample/s,ms),ms),values," - << std::endl; + std::cout + << "Engine,Distribution,Throughput,Samples,AvgTime (1 Trial),Time(all),Size,Lambda" + << std::endl; + std::cout << ",,GB/s,GSample/s,ms),ms),values," << std::endl; } - for (auto engine : engines) + for(auto engine : engines) { rng_type_t rng_type = ROCRAND_RNG_PSEUDO_XORWOW; - if (engine == "xorwow") + if(engine == "xorwow") rng_type = ROCRAND_RNG_PSEUDO_XORWOW; else if(engine == "mrg31k3p") rng_type = ROCRAND_RNG_PSEUDO_MRG31K3P; - else if (engine == "mrg32k3a") + else if(engine == "mrg32k3a") rng_type = ROCRAND_RNG_PSEUDO_MRG32K3A; - else if (engine == "philox") + else if(engine == "philox") rng_type = ROCRAND_RNG_PSEUDO_PHILOX4_32_10; else if(engine == "threefry2x32") rng_type = ROCRAND_RNG_PSEUDO_THREEFRY2_32_20; @@ -460,15 +484,15 @@ int main(int argc, char *argv[]) rng_type = ROCRAND_RNG_PSEUDO_THREEFRY4_32_20; else if(engine == "threefry4x64") rng_type = ROCRAND_RNG_PSEUDO_THREEFRY4_64_20; - else if (engine == "sobol32") + else if(engine == "sobol32") rng_type = ROCRAND_RNG_QUASI_SOBOL32; else if(engine == "scrambled_sobol32") rng_type = ROCRAND_RNG_QUASI_SCRAMBLED_SOBOL32; - else if (engine == "sobol64") + else if(engine == "sobol64") rng_type = ROCRAND_RNG_QUASI_SOBOL64; else if(engine == "scrambled_sobol64") rng_type = ROCRAND_RNG_QUASI_SCRAMBLED_SOBOL64; - else if (engine == "mtgp32") + else if(engine == "mtgp32") rng_type = ROCRAND_RNG_PSEUDO_MTGP32; else if(engine == "lfsr113") rng_type = ROCRAND_RNG_PSEUDO_LFSR113; @@ -480,11 +504,13 @@ int main(int argc, char *argv[]) exit(1); } - if (console_output) std::cout << engine << ":" << std::endl; + if(console_output) + std::cout << engine << ":" << std::endl; - for (auto distribution : distributions) + for(auto distribution : distributions) { - if (console_output) std::cout << " " << distribution << ":" << std::endl; + if(console_output) + std::cout << " " << distribution << ":" << std::endl; run_benchmarks(parser, rng_type, distribution, engine, stream); } std::cout << std::endl; diff --git a/benchmark/benchmark_rocrand_host_api.cpp b/benchmark/benchmark_rocrand_host_api.cpp index c29b21b91..b354ad6a3 100644 --- a/benchmark/benchmark_rocrand_host_api.cpp +++ b/benchmark/benchmark_rocrand_host_api.cpp @@ -23,10 +23,11 @@ #include +#include "custom_csv_formater.hpp" +#include #include -#include - #include +#include #include #include @@ -35,6 +36,7 @@ const size_t DEFAULT_RAND_N = 1024 * 1024 * 128; #endif typedef rocrand_rng_type rng_type_t; + template using generate_func_type = std::function; @@ -129,6 +131,14 @@ void run_benchmark(benchmark::State& state, int main(int argc, char* argv[]) { + // get paramaters before they are passed into + // benchmark::Initialize() + std::string outFormat = ""; + std::string filter = ""; + std::string consoleFormat = ""; + + getFormats(argc, argv, outFormat, filter, consoleFormat); + // Parse argv benchmark::Initialize(&argc, argv); @@ -455,14 +465,24 @@ int main(int argc, char* argv[]) } } } - // Use manual timing + for(auto& b : benchmarks) { b->UseManualTime(); b->Unit(benchmark::kMillisecond); } + + benchmark::BenchmarkReporter* console_reporter = getConsoleReporter(consoleFormat); + benchmark::BenchmarkReporter* out_file_reporter = getOutFileReporter(outFormat); + + std::string spec = (filter == "" || filter == "all") ? "." : filter; + // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); + if(outFormat == "") // default case + benchmark::RunSpecifiedBenchmarks(console_reporter, spec); + else + benchmark::RunSpecifiedBenchmarks(console_reporter, out_file_reporter, spec); + HIP_CHECK(hipStreamDestroy(stream)); return 0; diff --git a/benchmark/benchmark_utils.hpp b/benchmark/benchmark_utils.hpp index aa4007610..1ee5a4c2a 100644 --- a/benchmark/benchmark_utils.hpp +++ b/benchmark/benchmark_utils.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -27,6 +27,9 @@ #include #include +#include "custom_csv_formater.hpp" +#include + #define HIP_CHECK(condition) \ do \ { \ @@ -159,4 +162,87 @@ inline size_t next_power2(size_t x) return power; } +inline benchmark::BenchmarkReporter* getConsoleReporter(const std::string format) +{ + benchmark::BenchmarkReporter* reporter; + if(format == "csv") + { + static benchmark::customCSVReporter csv_reporter; + csv_reporter.SetErrorStream(&std::cout); + csv_reporter.SetOutputStream(&std::cout); + reporter = &csv_reporter; + } + else if(format == "json") + { + static benchmark::customCSVReporter json_reporter; + json_reporter.SetErrorStream(&std::cout); + json_reporter.SetOutputStream(&std::cout); + reporter = &json_reporter; + } + else + { + static benchmark::ConsoleReporter terminal_reporter; + terminal_reporter.SetErrorStream(&std::cout); + terminal_reporter.SetOutputStream(&std::cout); + reporter = &terminal_reporter; + } + + return reporter; +} + +inline benchmark::BenchmarkReporter* getOutFileReporter(const std::string format) +{ + benchmark::BenchmarkReporter* reporter = nullptr; + std::ofstream output_file; + if(format == "csv") + { + static benchmark::customCSVReporter csv_reporter; + csv_reporter.SetOutputStream(&output_file); + csv_reporter.SetErrorStream(&output_file); + reporter = &csv_reporter; + } + else if(format == "json") + { + static benchmark::JSONReporter json_reporter; + json_reporter.SetOutputStream(&output_file); + json_reporter.SetErrorStream(&output_file); + reporter = &json_reporter; + } + else if(format == "console") + { + static benchmark::ConsoleReporter console_reporter; + console_reporter.SetOutputStream(&output_file); + console_reporter.SetErrorStream(&output_file); + reporter = &console_reporter; + } + + return reporter; +} + +inline void getFormats(const int argc, + char* argv[], + std::string& outFormat, + std::string& filter, + std::string& consoleFormat) +{ + for(int i = 1; i < argc; i++) + { + std::string input(argv[i]); + int equalPos = input.find("="); + + if(equalPos < 0) + continue; + + std::string arg = std::string(input.begin() + 2, input.begin() + equalPos); + std::string argVal = std::string(input.begin() + 1 + equalPos, input.end()); + + if(arg == "benchmark_out_format") + outFormat = argVal; + else if(arg == "benchmark_filter") + filter = argVal; + else if(arg == "benchmark_format") + consoleFormat = argVal; + } +} + #endif // ROCRAND_BENCHMARK_UTILS_HPP_ diff --git a/benchmark/cmdparser.hpp b/benchmark/cmdparser.hpp index 1ae024715..27264f837 100644 --- a/benchmark/cmdparser.hpp +++ b/benchmark/cmdparser.hpp @@ -26,488 +26,622 @@ */ #pragma once +#include #include +#include #include #include #include -#include -#include -namespace cli { - struct CallbackArgs { - const std::vector& arguments; - std::ostream& output; - std::ostream& error; +namespace cli +{ +struct CallbackArgs +{ + const std::vector& arguments; + std::ostream& output; + std::ostream& error; +}; +class Parser +{ +private: + class CmdBase + { + public: + explicit CmdBase(const std::string& name, + const std::string& alternative, + const std::string& description, + bool required, + bool dominant, + bool variadic) + : name(name) + , command(name.size() > 0 ? "-" + name : "") + , alternative(alternative.size() > 0 ? "--" + alternative : "") + , description(description) + , required(required) + , handled(false) + , arguments({}) + , dominant(dominant) + , variadic(variadic) + {} + + virtual ~CmdBase() {} + + std::string name; + std::string command; + std::string alternative; + std::string description; + bool required; + bool handled; + std::vector arguments; + bool const dominant; + bool const variadic; + + virtual std::string print_value() const = 0; + virtual bool parse(std::ostream& output, std::ostream& error) = 0; + + bool is(const std::string& given) const + { + return given == command || given == alternative; + } }; - class Parser { - private: - class CmdBase { - public: - explicit CmdBase(const std::string& name, const std::string& alternative, const std::string& description, bool required, bool dominant, bool variadic) : - name(name), - command(name.size() > 0 ? "-" + name : ""), - alternative(alternative.size() > 0 ? "--" + alternative : ""), - description(description), - required(required), - handled(false), - arguments({}), - dominant(dominant), - variadic(variadic) { - } - virtual ~CmdBase() { - } - - std::string name; - std::string command; - std::string alternative; - std::string description; - bool required; - bool handled; - std::vector arguments; - bool const dominant; - bool const variadic; - - virtual std::string print_value() const = 0; - virtual bool parse(std::ostream& output, std::ostream& error) = 0; - - bool is(const std::string& given) const { - return given == command || given == alternative; - } - }; + template + struct ArgumentCountChecker + { + static constexpr bool Variadic = false; + }; - template - struct ArgumentCountChecker - { - static constexpr bool Variadic = false; - }; + template + struct ArgumentCountChecker> + { + static constexpr bool Variadic = true; + }; - template - struct ArgumentCountChecker> + template + class CmdFunction final : public CmdBase + { + public: + explicit CmdFunction(const std::string& name, + const std::string& alternative, + const std::string& description, + bool required, + bool dominant) + : CmdBase(name, + alternative, + description, + required, + dominant, + ArgumentCountChecker::Variadic) + {} + + bool parse(std::ostream& output, std::ostream& error) override { - static constexpr bool Variadic = true; - }; - - template - class CmdFunction final : public CmdBase { - public: - explicit CmdFunction(const std::string& name, const std::string& alternative, const std::string& description, bool required, bool dominant) : - CmdBase(name, alternative, description, required, dominant, ArgumentCountChecker::Variadic) { + try + { + CallbackArgs args{arguments, output, error}; + value = callback(args); + return true; } - - bool parse(std::ostream& output, std::ostream& error) override { - try { - CallbackArgs args { arguments, output, error }; - value = callback(args); - return true; - } catch (...) { - return false; - } - } - - std::string print_value() const override { - return ""; + catch(...) + { + return false; } + } - std::function callback; - T value; - }; + std::string print_value() const override + { + return ""; + } - template - class CmdArgument final : public CmdBase { - public: - explicit CmdArgument(const std::string& name, const std::string& alternative, const std::string& description, bool required, bool dominant) : - CmdBase(name, alternative, description, required, dominant, ArgumentCountChecker::Variadic) { - } + std::function callback; + T value; + }; - bool parse(std::ostream&, std::ostream&) override { - try { - value = Parser::parse(arguments, value); - return true; - } catch (...) { - return false; - } + template + class CmdArgument final : public CmdBase + { + public: + explicit CmdArgument(const std::string& name, + const std::string& alternative, + const std::string& description, + bool required, + bool dominant) + : CmdBase(name, + alternative, + description, + required, + dominant, + ArgumentCountChecker::Variadic) + {} + + bool parse(std::ostream&, std::ostream&) override + { + try + { + value = Parser::parse(arguments, value); + return true; } - - std::string print_value() const override { - return stringify(value); + catch(...) + { + return false; } + } - T value {}; - }; + std::string print_value() const override + { + return stringify(value); + } - static int parse(const std::vector& elements, const int&) { - if (elements.size() != 1) - throw std::bad_cast(); + T value{}; + }; - return std::stoi(elements[0]); - } + static int parse(const std::vector& elements, const int&) + { + if(elements.size() != 1) + throw std::bad_cast(); - static bool parse(const std::vector& elements, const bool& defval) { - if (elements.size() != 0) - throw std::runtime_error("A boolean command line parameter cannot have any arguments."); + return std::stoi(elements[0]); + } - return !defval; - } + static bool parse(const std::vector& elements, const bool& defval) + { + if(elements.size() != 0) + throw std::runtime_error("A boolean command line parameter cannot have any arguments."); - static double parse(const std::vector& elements, const double&) { - if (elements.size() != 1) - throw std::bad_cast(); + return !defval; + } - return std::stod(elements[0]); - } + static double parse(const std::vector& elements, const double&) + { + if(elements.size() != 1) + throw std::bad_cast(); - static float parse(const std::vector& elements, const float&) { - if (elements.size() != 1) - throw std::bad_cast(); + return std::stod(elements[0]); + } - return std::stof(elements[0]); - } + static float parse(const std::vector& elements, const float&) + { + if(elements.size() != 1) + throw std::bad_cast(); - static long double parse(const std::vector& elements, const long double&) { - if (elements.size() != 1) - throw std::bad_cast(); + return std::stof(elements[0]); + } - return std::stold(elements[0]); - } + static long double parse(const std::vector& elements, const long double&) + { + if(elements.size() != 1) + throw std::bad_cast(); - static unsigned int parse(const std::vector& elements, const unsigned int&) { - if (elements.size() != 1) - throw std::bad_cast(); + return std::stold(elements[0]); + } - return static_cast(std::stoul(elements[0])); - } + static unsigned int parse(const std::vector& elements, const unsigned int&) + { + if(elements.size() != 1) + throw std::bad_cast(); - static unsigned long parse(const std::vector& elements, const unsigned long&) { - if (elements.size() != 1) - throw std::bad_cast(); + return static_cast(std::stoul(elements[0])); + } - return std::stoul(elements[0]); - } - - static unsigned long long parse(const std::vector& elements, const unsigned long long&) { - if (elements.size() != 1) - throw std::bad_cast(); + static unsigned long parse(const std::vector& elements, const unsigned long&) + { + if(elements.size() != 1) + throw std::bad_cast(); - return std::stoull(elements[0]); - } + return std::stoul(elements[0]); + } - static long parse(const std::vector& elements, const long&) { - if (elements.size() != 1) - throw std::bad_cast(); + static unsigned long long parse(const std::vector& elements, + const unsigned long long&) + { + if(elements.size() != 1) + throw std::bad_cast(); - return std::stol(elements[0]); - } + return std::stoull(elements[0]); + } - static std::string parse(const std::vector& elements, const std::string&) { - if (elements.size() != 1) - throw std::bad_cast(); + static long parse(const std::vector& elements, const long&) + { + if(elements.size() != 1) + throw std::bad_cast(); - return elements[0]; - } + return std::stol(elements[0]); + } - template - static std::vector parse(const std::vector& elements, const std::vector&) { - const T defval = T(); - std::vector values { }; - std::vector buffer(1); + static std::string parse(const std::vector& elements, const std::string&) + { + if(elements.size() != 1) + throw std::bad_cast(); - for (const auto& element : elements) { - buffer[0] = element; - values.push_back(parse(buffer, defval)); - } + return elements[0]; + } - return values; - } + template + static std::vector parse(const std::vector& elements, const std::vector&) + { + const T defval = T(); + std::vector values{}; + std::vector buffer(1); - template - static std::string stringify(const T& value) { - return std::to_string(value); + for(const auto& element : elements) + { + buffer[0] = element; + values.push_back(parse(buffer, defval)); } - template - static std::string stringify(const std::vector& values) { - std::stringstream ss { }; - ss << "[ "; + return values; + } - for (const auto& value : values) { - ss << stringify(value) << " "; - } + template + static std::string stringify(const T& value) + { + return std::to_string(value); + } - ss << "]"; - return ss.str(); - } + template + static std::string stringify(const std::vector& values) + { + std::stringstream ss{}; + ss << "[ "; - static std::string stringify(const std::string& str) { - return str; + for(const auto& value : values) + { + ss << stringify(value) << " "; } - public: - explicit Parser(int argc, const char** argv) : - _appname(argv[0]) { - for (int i = 1; i < argc; ++i) { - _arguments.push_back(argv[i]); - } - enable_help(); - } + ss << "]"; + return ss.str(); + } - explicit Parser(int argc, char** argv) : - _appname(argv[0]) { - for (int i = 1; i < argc; ++i) { - _arguments.push_back(argv[i]); - } - enable_help(); - } + static std::string stringify(const std::string& str) + { + return str; + } - ~Parser() { - for (int i = 0, n = _commands.size(); i < n; ++i) { - delete _commands[i]; - } +public: + explicit Parser(int argc, const char** argv) : _appname(argv[0]) + { + for(int i = 1; i < argc; ++i) + { + _arguments.push_back(argv[i]); } + enable_help(); + } - bool has_help() const { - for (const auto command : _commands) { - if (command->name == "h" && command->alternative == "--help") { - return true; - } - } - - return false; + explicit Parser(int argc, char** argv) : _appname(argv[0]) + { + for(int i = 1; i < argc; ++i) + { + _arguments.push_back(argv[i]); } + enable_help(); + } - void enable_help() { - set_callback("h", "help", std::function([this](CallbackArgs& args){ - args.output << this->usage(); - exit(0); - return false; - }), "", true); + ~Parser() + { + for(int i = 0, n = _commands.size(); i < n; ++i) + { + delete _commands[i]; } + } - void disable_help() { - for (auto command = _commands.begin(); command != _commands.end(); ++command) { - if ((*command)->name == "h" && (*command)->alternative == "--help") { - _commands.erase(command); - break; - } + bool has_help() const + { + for(const auto command : _commands) + { + if(command->name == "h" && command->alternative == "--help") + { + return true; } } - template - void set_default(bool is_required, const std::string& description = "") { - auto command = new CmdArgument { "", "", description, is_required, false }; - _commands.push_back(command); - } - - template - void set_required(const std::string& name, const std::string& alternative, const std::string& description = "", bool dominant = false) { - auto command = new CmdArgument { name, alternative, description, true, dominant }; - _commands.push_back(command); + return false; + } + + void enable_help() + { + set_callback("h", + "help", + std::function( + [this](CallbackArgs& args) + { + args.output << this->usage(); + exit(0); + return false; + }), + "", + true); + } + + void disable_help() + { + for(auto command = _commands.begin(); command != _commands.end(); ++command) + { + if((*command)->name == "h" && (*command)->alternative == "--help") + { + _commands.erase(command); + break; + } } - - template - void set_optional(const std::string& name, const std::string& alternative, const T& defaultValue, const std::string& description = "", bool dominant = false) { - auto command = new CmdArgument { name, alternative, description, false, dominant }; - command->value = defaultValue; - _commands.push_back(command); + } + + template + void set_default(bool is_required, const std::string& description = "") + { + auto command = new CmdArgument{"", "", description, is_required, false}; + _commands.push_back(command); + } + + template + void set_required(const std::string& name, + const std::string& alternative, + const std::string& description = "", + bool dominant = false) + { + auto command = new CmdArgument{name, alternative, description, true, dominant}; + _commands.push_back(command); + } + + template + void set_optional(const std::string& name, + const std::string& alternative, + const T& defaultValue, + const std::string& description = "", + bool dominant = false) + { + auto command = new CmdArgument{name, alternative, description, false, dominant}; + command->value = defaultValue; + _commands.push_back(command); + } + + template + void set_callback(const std::string& name, + const std::string& alternative, + std::function callback, + const std::string& description = "", + bool dominant = false) + { + auto command = new CmdFunction{name, alternative, description, false, dominant}; + command->callback = callback; + _commands.push_back(command); + } + + inline void run_and_exit_if_error() + { + if(run() == false) + { + exit(1); } + } - template - void set_callback(const std::string& name, const std::string& alternative, std::function callback, const std::string& description = "", bool dominant = false) { - auto command = new CmdFunction { name, alternative, description, false, dominant }; - command->callback = callback; - _commands.push_back(command); - } + inline bool run() + { + return run(std::cout, std::cerr); + } - inline void run_and_exit_if_error() { - if (run() == false) { - exit(1); - } - } + inline bool run(std::ostream& output) + { + return run(output, std::cerr); + } - inline bool run() { - return run(std::cout, std::cerr); - } + bool run(std::ostream& output, std::ostream& error) + { + if(_arguments.size() > 0) + { + auto current = find_default(); - inline bool run(std::ostream& output) { - return run(output, std::cerr); - } + for(int i = 0, n = _arguments.size(); i < n; ++i) + { + auto isarg = _arguments[i].size() > 0 && _arguments[i][0] == '-'; + auto associated = isarg ? find(_arguments[i]) : nullptr; - bool run(std::ostream& output, std::ostream& error) { - if (_arguments.size() > 0) { - auto current = find_default(); - - for (int i = 0, n = _arguments.size(); i < n; ++i) { - auto isarg = _arguments[i].size() > 0 && _arguments[i][0] == '-'; - auto associated = isarg ? find(_arguments[i]) : nullptr; - - if (associated != nullptr) { - current = associated; - associated->handled = true; - } else if (current == nullptr) { - error << no_default(); - return false; - } else { - current->arguments.push_back(_arguments[i]); - current->handled = true; - if (!current->variadic) - { - // If the current command is not variadic, then no more arguments - // should be added to it. In this case, switch back to the default - // command. - current = find_default(); - } - } + if(associated != nullptr) + { + current = associated; + associated->handled = true; } - } - - // First, parse dominant arguments since they succeed even if required - // arguments are missing. - for (auto command : _commands) { - if (command->handled && command->dominant && !command->parse(output, error)) { - error << howto_use(command); + else if(current == nullptr) + { + error << no_default(); return false; } - } - - // Next, check for any missing arguments. - for (auto command : _commands) { - if (command->required && !command->handled) { - error << howto_required(command); - return false; + else + { + current->arguments.push_back(_arguments[i]); + current->handled = true; + if(!current->variadic) + { + // If the current command is not variadic, then no more arguments + // should be added to it. In this case, switch back to the default + // command. + current = find_default(); + } } } + } - // Finally, parse all remaining arguments. - for (auto command : _commands) { - if (command->handled && !command->dominant && !command->parse(output, error)) { - error << howto_use(command); - return false; - } + // First, parse dominant arguments since they succeed even if required + // arguments are missing. + for(auto command : _commands) + { + if(command->handled && command->dominant && !command->parse(output, error)) + { + error << howto_use(command); + return false; } - - return true; } - template - T get(const std::string& name) const { - for (const auto& command : _commands) { - if (command->name == name) { - auto cmd = dynamic_cast*>(command); - - if (cmd == nullptr) { - throw std::runtime_error("Invalid usage of the parameter " + name + " detected."); - } - - return cmd->value; - } + // Next, check for any missing arguments. + for(auto command : _commands) + { + if(command->required && !command->handled) + { + error << howto_required(command); + return false; } - - throw std::runtime_error("The parameter " + name + " could not be found."); } - template - T get_if(const std::string& name, std::function callback) const { - auto value = get(name); - return callback(value); + // Finally, parse all remaining arguments. + for(auto command : _commands) + { + if(command->handled && !command->dominant && !command->parse(output, error)) + { + error << howto_use(command); + return false; + } } - int requirements() const { - int count = 0; + return true; + } - for (const auto& command : _commands) { - if (command->required) { - ++count; + template + T get(const std::string& name) const + { + for(const auto& command : _commands) + { + if(command->name == name) + { + auto cmd = dynamic_cast*>(command); + + if(cmd == nullptr) + { + throw std::runtime_error("Invalid usage of the parameter " + name + + " detected."); } - } - return count; + return cmd->value; + } } - int commands() const { - return static_cast(_commands.size()); - } + throw std::runtime_error("The parameter " + name + " could not be found."); + } - inline const std::string& app_name() const { - return _appname; - } + template + T get_if(const std::string& name, std::function callback) const + { + auto value = get(name); + return callback(value); + } - protected: - CmdBase* find(const std::string& name) { - for (auto command : _commands) { - if (command->is(name)) { - return command; - } - } + int requirements() const + { + int count = 0; - return nullptr; + for(const auto& command : _commands) + { + if(command->required) + { + ++count; + } } - CmdBase* find_default() { - for (auto command : _commands) { - if (command->name == "") { - return command; - } - } + return count; + } + + int commands() const + { + return static_cast(_commands.size()); + } - return nullptr; + inline const std::string& app_name() const + { + return _appname; + } + +protected: + CmdBase* find(const std::string& name) + { + for(auto command : _commands) + { + if(command->is(name)) + { + return command; + } } - std::string usage() const { - std::stringstream ss { }; - ss << "Available parameters:\n\n"; + return nullptr; + } - for (const auto& command : _commands) { - ss << " " << command->command << "\t" << command->alternative; + CmdBase* find_default() + { + for(auto command : _commands) + { + if(command->name == "") + { + return command; + } + } - if (command->required == true) { - ss << "\t(required)"; - } + return nullptr; + } - ss << "\n " << command->description; + std::string usage() const + { + std::stringstream ss{}; + ss << "Available parameters:\n\n"; - if (command->required == false) { - ss << "\n " << "This parameter is optional. The default value is '" + command->print_value() << "'."; - } + for(const auto& command : _commands) + { + ss << " " << command->command << "\t" << command->alternative; - ss << "\n\n"; + if(command->required == true) + { + ss << "\t(required)"; } - return ss.str(); - } + ss << "\n " << command->description; - void print_help(std::stringstream& ss) const { - if (has_help()) { - ss << "For more help use --help or -h.\n"; + if(command->required == false) + { + ss << "\n " + << "This parameter is optional. The default value is '" + command->print_value() + << "'."; } - } - - std::string howto_required(CmdBase* command) const { - std::stringstream ss { }; - ss << "The parameter " << command->name << " is required.\n"; - ss << command->description << '\n'; - print_help(ss); - return ss.str(); - } - std::string howto_use(CmdBase* command) const { - std::stringstream ss { }; - ss << "The parameter " << command->name << " has invalid arguments.\n"; - ss << command->description << '\n'; - print_help(ss); - return ss.str(); + ss << "\n\n"; } - std::string no_default() const { - std::stringstream ss { }; - ss << "No default parameter has been specified.\n"; - ss << "The given argument must be used with a parameter.\n"; - print_help(ss); - return ss.str(); - } + return ss.str(); + } - private: - const std::string _appname; - std::vector _arguments; - std::vector _commands; - }; -} + void print_help(std::stringstream& ss) const + { + if(has_help()) + { + ss << "For more help use --help or -h.\n"; + } + } + + std::string howto_required(CmdBase* command) const + { + std::stringstream ss{}; + ss << "The parameter " << command->name << " is required.\n"; + ss << command->description << '\n'; + print_help(ss); + return ss.str(); + } + + std::string howto_use(CmdBase* command) const + { + std::stringstream ss{}; + ss << "The parameter " << command->name << " has invalid arguments.\n"; + ss << command->description << '\n'; + print_help(ss); + return ss.str(); + } + + std::string no_default() const + { + std::stringstream ss{}; + ss << "No default parameter has been specified.\n"; + ss << "The given argument must be used with a parameter.\n"; + print_help(ss); + return ss.str(); + } + +private: + const std::string _appname; + std::vector _arguments; + std::vector _commands; +}; +} // namespace cli diff --git a/benchmark/custom_csv_formater.hpp b/benchmark/custom_csv_formater.hpp new file mode 100644 index 000000000..5f89053c9 --- /dev/null +++ b/benchmark/custom_csv_formater.hpp @@ -0,0 +1,255 @@ +// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#pragma once + +#include +#include +namespace benchmark +{ + +class customCSVReporter : public BenchmarkReporter +{ +public: + customCSVReporter() : printed_header_(false) {} + bool ReportContext(const Context& context) override; + void ReportRuns(const std::vector& reports) override; + +private: + std::string CsvEscape(const std::string& s) + { + std::string tmp; + tmp.reserve(s.size() + 2); + for(char c : s) + { + switch(c) + { + case '"': tmp += "\"\""; break; + default: tmp += c; break; + } + } + return '"' + tmp + '"'; + } + + // Function to return an string for the calculated complexity + std::string GetBigOString(const BigO complexity) + { + switch(complexity) + { + case oN: return "N"; + case oNSquared: return "N^2"; + case oNCubed: return "N^3"; + case oLogN: return "lgN"; + case oNLogN: return "NlgN"; + case o1: return "(1)"; + default: return "f(N)"; + } + } + + void PrintRunData(const Run& report); + bool printed_header_; + std::set user_counter_names_; + + std::ostream* nullLog = nullptr; + + std::array elements = {"engine", + "distribution", + "mode", + "name", + "iterations", + "real_time", + "cpu_time", + "time_unit", + "bytes_per_second", + "throughput_gigabytes_per_second", + "lambda", + "items_per_second", + "label", + "error_occurred", + "error_message"}; +}; + +bool customCSVReporter::ReportContext(const Context& context) +{ + PrintBasicContext(&GetErrorStream(), context); + return true; +} + +void customCSVReporter::ReportRuns(const std::vector& reports) +{ + std::ostream& Out = GetOutputStream(); + + if(!printed_header_) + { + // save the names of all the user counters + for(const auto& run : reports) + { + for(const auto& cnt : run.counters) + { + if(cnt.first == "bytes_per_second" || cnt.first == "items_per_second") + continue; + user_counter_names_.insert(cnt.first); + } + } + + // print the header + for(auto B = elements.begin(); B != elements.end();) + { + Out << *B++; + if(B != elements.end()) + Out << ","; + } + for(auto B = user_counter_names_.begin(); B != user_counter_names_.end();) + { + Out << ",\"" << *B++ << "\""; + } + Out << "\n"; + + printed_header_ = true; + } + else + { + // check that all the current counters are saved in the name set + for(const auto& run : reports) + { + for(const auto& cnt : run.counters) + { + if(cnt.first == "bytes_per_second" || cnt.first == "items_per_second") + continue; + + // benchmark::internal::GetNullLogInstance() + *nullLog << "All counters must be present in each run. " + << "Counter named \"" << cnt.first + << "\" was not in a run after being added to the header"; + } + } + } + + // print results for each run + for(const auto& run : reports) + { + PrintRunData(run); + } +} + +void customCSVReporter::PrintRunData(const Run& run) +{ + std::ostream& Out = GetOutputStream(); + std::ostream& Err = GetErrorStream(); + + //get the name of the engine and distribution: + + std::string temp = run.benchmark_name(); + + std::string deviceName = std::string(temp.begin(), temp.begin() + temp.find("<")); + + temp.erase(0, temp.find("<") + 1); + + std::string engineName = std::string(temp.begin(), temp.begin() + temp.find(",")); + + temp.erase(0, engineName.size() + 1); + + std::string mode = "default"; + + if(deviceName != "device_kernel") + { + mode = std::string(temp.begin(), temp.begin() + temp.find(',')); + temp.erase(0, temp.find(",") + 1); + } + std::string disName = std::string(temp.begin(), temp.begin() + temp.find(">")); + + std::string lambda = ""; + + size_t ePos = disName.find("="); + if(ePos <= disName.size()) + { + lambda = std::string(disName.begin() + (ePos + 1), disName.end() - 1); + disName.erase(disName.begin() + disName.find("("), disName.end()); + } + + Out << engineName << "," << disName << "," << mode << ","; + Out << CsvEscape(run.benchmark_name()) << ","; + if(run.error_occurred) + { + Err << std::string(elements.size() - 3, ','); + Err << "true,"; + Err << CsvEscape(run.error_message) << "\n"; + return; + } + + // Do not print iteration on bigO and RMS report + if(!run.report_big_o && !run.report_rms) + { + Out << run.iterations; + } + Out << ","; + + Out << run.GetAdjustedRealTime() << ","; + Out << run.GetAdjustedCPUTime() << ","; + + // Do not print timeLabel on bigO and RMS report + if(run.report_big_o) + { + Out << GetBigOString(run.complexity); + } + else if(!run.report_rms) + { + Out << GetTimeUnitString(run.time_unit); + } + Out << ","; + + if(run.counters.find("bytes_per_second") != run.counters.end()) + { + Out << run.counters.at("bytes_per_second"); + } + Out << ","; + + double gbps = run.counters.at("bytes_per_second") / std::pow(1024, 3); + + Out << gbps << "," << lambda << ","; + + if(run.counters.find("items_per_second") != run.counters.end()) + { + Out << run.counters.at("items_per_second"); + } + Out << ","; + if(!run.report_label.empty()) + { + Out << CsvEscape(run.report_label); + } + Out << ",,"; // for error_occurred and error_message + + // Print user counters + for(const auto& ucn : user_counter_names_) + { + auto it = run.counters.find(ucn); + if(it == run.counters.end()) + { + Out << ","; + } + else + { + Out << "," << it->second; + } + } + Out << '\n'; +} + +} // namespace benchmark