diff --git a/examples/onnxruntime_demo/main/onnxruntime-src/include/onnxruntime_c_api.h b/examples/onnxruntime_demo/main/onnxruntime-src/include/onnxruntime_c_api.h deleted file mode 100644 index 24697f4e..00000000 --- a/examples/onnxruntime_demo/main/onnxruntime-src/include/onnxruntime_c_api.h +++ /dev/null @@ -1,3563 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -// See docs\c_cxx\README.md on generating the Doxygen documentation from this file - -/** \mainpage C & C++ APIs -* -*

C

-* -* ::OrtApi - Click here to go to the structure with all C API functions. -* -*

C++

-* -* ::Ort - Click here to go to the namespace holding all of the C++ wrapper classes -* -* It is a set of header only wrapper classes around the C API. The goal is to turn the C style return value error codes into C++ exceptions, and to -* automate memory management through standard C++ RAII principles. -* -* \addtogroup Global -* ONNX Runtime C API -* @{ -*/ - -#pragma once -#include -#include -#include - -/** \brief The API version defined in this header -* -* This value is used by some API functions to behave as this version of the header expects. -*/ -#define ORT_API_VERSION 12 - -#ifdef __cplusplus -extern "C" { -#endif - -//! @} -// SAL2 Definitions -#ifndef _WIN32 -#define _In_ -#define _In_z_ -#define _In_opt_ -#define _In_opt_z_ -#define _Out_ -#define _Outptr_ -#define _Out_opt_ -#define _Inout_ -#define _Inout_opt_ -#define _Frees_ptr_opt_ -#define _Ret_maybenull_ -#define _Ret_notnull_ -#define _Check_return_ -#define _Outptr_result_maybenull_ -#define _In_reads_(X) -#define _Inout_updates_all_(X) -#define _Out_writes_bytes_all_(X) -#define _Out_writes_all_(X) -#define _Success_(X) -#define _Outptr_result_buffer_maybenull_(X) -#define ORT_ALL_ARGS_NONNULL __attribute__((nonnull)) -#else -#include -#define ORT_ALL_ARGS_NONNULL -#endif - -#ifdef _WIN32 -// Define ORT_DLL_IMPORT if your program is dynamically linked to Ort. -// dllexport is not used, we use a .def file. -#ifdef ORT_DLL_IMPORT -#define ORT_EXPORT __declspec(dllimport) -#else -#define ORT_EXPORT -#endif -#define ORT_API_CALL _stdcall -#define ORT_MUST_USE_RESULT -#define ORTCHAR_T wchar_t -#else -// To make symbols visible on macOS/iOS -#ifdef __APPLE__ -#define ORT_EXPORT __attribute__((visibility("default"))) -#else -#define ORT_EXPORT -#endif -#define ORT_API_CALL -#define ORT_MUST_USE_RESULT __attribute__((warn_unused_result)) -#define ORTCHAR_T char -#endif - -#ifndef ORT_TSTR -#ifdef _WIN32 -#define ORT_TSTR(X) L##X -#else -#define ORT_TSTR(X) X -#endif -#endif - -// Any pointer marked with _In_ or _Out_, cannot be NULL. - -// Windows users should use unicode paths when possible to bypass the MAX_PATH limitation -// Every pointer marked with _In_ or _Out_, cannot be NULL. Caller should ensure that. -// for ReleaseXXX(...) functions, they can accept NULL pointer. - -#ifdef __cplusplus -// For any compiler with C++11 support, MSVC 2015 and greater, or Clang version supporting noexcept. -// Such complex condition is needed because compilers set __cplusplus value differently. -#ifndef __has_feature -#define __has_feature(x) 0 -#endif -#if ((__cplusplus >= 201103L) || (_MSC_VER >= 1900) || (defined(__has_feature) && __has_feature(cxx_noexcept))) -#define NO_EXCEPTION noexcept -#else -#define NO_EXCEPTION throw() -#endif -#else -#define NO_EXCEPTION -#endif - -// __VA_ARGS__ on Windows and Linux are different -#define ORT_API(RETURN_TYPE, NAME, ...) RETURN_TYPE ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION - -#define ORT_API_STATUS(NAME, ...) \ - _Success_(return == 0) _Check_return_ _Ret_maybenull_ OrtStatusPtr ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION ORT_MUST_USE_RESULT - -// XXX: Unfortunately, SAL annotations are known to not work with function pointers -#define ORT_API2_STATUS(NAME, ...) \ - _Check_return_ _Ret_maybenull_ OrtStatusPtr(ORT_API_CALL* NAME)(__VA_ARGS__) NO_EXCEPTION ORT_MUST_USE_RESULT - -// Used in *.cc files. Almost as same as ORT_API_STATUS, except without ORT_MUST_USE_RESULT and ORT_EXPORT -#define ORT_API_STATUS_IMPL(NAME, ...) \ - GSL_SUPPRESS(r .11) \ - _Success_(return == 0) _Check_return_ _Ret_maybenull_ OrtStatusPtr ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION - -#define ORT_CLASS_RELEASE(X) void(ORT_API_CALL * Release##X)(_Frees_ptr_opt_ Ort##X * input) - -#ifdef __DOXYGEN__ -#undef ORT_API_STATUS -#define ORT_API_STATUS(NAME, ...) OrtStatus* NAME(__VA_ARGS__) -#undef ORT_API2_STATUS -#define ORT_API2_STATUS(NAME, ...) OrtStatus* NAME(__VA_ARGS__) -#undef ORT_CLASS_RELEASE -#define ORT_CLASS_RELEASE(X) void Release##X(Ort##X* input) -#undef NO_EXCEPTION -#define NO_EXCEPTION -#endif -/** \addtogroup Global - * ONNX Runtime C API - * @{ - */ - -/** Copied from TensorProto::DataType -* Currently, Ort doesn't support complex64, complex128 -*/ -typedef enum ONNXTensorElementDataType { - ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED, - ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, // maps to c type float - ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8, // maps to c type uint8_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8, // maps to c type int8_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16, // maps to c type uint16_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16, // maps to c type int16_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32, // maps to c type int32_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64, // maps to c type int64_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING, // maps to c++ type std::string - ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL, - ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16, - ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE, // maps to c type double - ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32, // maps to c type uint32_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64, // maps to c type uint64_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64, // complex with float32 real and imaginary components - ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128, // complex with float64 real and imaginary components - ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16 // Non-IEEE floating-point format based on IEEE754 single-precision -} ONNXTensorElementDataType; - -// Synced with onnx TypeProto oneof -typedef enum ONNXType { - ONNX_TYPE_UNKNOWN, - ONNX_TYPE_TENSOR, - ONNX_TYPE_SEQUENCE, - ONNX_TYPE_MAP, - ONNX_TYPE_OPAQUE, - ONNX_TYPE_SPARSETENSOR, - ONNX_TYPE_OPTIONAL -} ONNXType; - -// These types are synced with internal -// SparseFormatFlags -typedef enum OrtSparseFormat { - ORT_SPARSE_UNDEFINED = 0, - ORT_SPARSE_COO = 0x1, - ORT_SPARSE_CSRC = 0x2, - ORT_SPARSE_BLOCK_SPARSE = 0x4 -} OrtSparseFormat; - -// Enum allows to query sparse tensor indices -enum OrtSparseIndicesFormat { - ORT_SPARSE_COO_INDICES, - ORT_SPARSE_CSR_INNER_INDICES, - ORT_SPARSE_CSR_OUTER_INDICES, - ORT_SPARSE_BLOCK_SPARSE_INDICES -}; - -/** \brief Logging severity levels - * - * In typical API usage, specifying a logging severity level specifies the minimum severity of log messages to show. - */ -typedef enum OrtLoggingLevel { - ORT_LOGGING_LEVEL_VERBOSE, ///< Verbose informational messages (least severe). - ORT_LOGGING_LEVEL_INFO, ///< Informational messages. - ORT_LOGGING_LEVEL_WARNING, ///< Warning messages. - ORT_LOGGING_LEVEL_ERROR, ///< Error messages. - ORT_LOGGING_LEVEL_FATAL, ///< Fatal error messages (most severe). -} OrtLoggingLevel; - -typedef enum OrtErrorCode { - ORT_OK, - ORT_FAIL, - ORT_INVALID_ARGUMENT, - ORT_NO_SUCHFILE, - ORT_NO_MODEL, - ORT_ENGINE_ERROR, - ORT_RUNTIME_EXCEPTION, - ORT_INVALID_PROTOBUF, - ORT_MODEL_LOADED, - ORT_NOT_IMPLEMENTED, - ORT_INVALID_GRAPH, - ORT_EP_FAIL, -} OrtErrorCode; - -typedef enum OrtOpAttrType { - ORT_OP_ATTR_UNDEFINED = 0, - ORT_OP_ATTR_INT, - ORT_OP_ATTR_INTS, - ORT_OP_ATTR_FLOAT, - ORT_OP_ATTR_FLOATS, - ORT_OP_ATTR_STRING, - ORT_OP_ATTR_STRINGS, -} OrtOpAttrType; - -//! @} -#define ORT_RUNTIME_CLASS(X) \ - struct Ort##X; \ - typedef struct Ort##X Ort##X; - -/** \addtogroup Global - * ONNX Runtime C API - * @{ - */ -// The actual types defined have an Ort prefix -ORT_RUNTIME_CLASS(Env); -ORT_RUNTIME_CLASS(Status); // nullptr for Status* indicates success -ORT_RUNTIME_CLASS(MemoryInfo); -ORT_RUNTIME_CLASS(IoBinding); -ORT_RUNTIME_CLASS(Session); //Don't call ReleaseSession from Dllmain (because session owns a thread pool) -ORT_RUNTIME_CLASS(Value); -ORT_RUNTIME_CLASS(RunOptions); -ORT_RUNTIME_CLASS(TypeInfo); -ORT_RUNTIME_CLASS(TensorTypeAndShapeInfo); -ORT_RUNTIME_CLASS(SessionOptions); -ORT_RUNTIME_CLASS(CustomOpDomain); -ORT_RUNTIME_CLASS(MapTypeInfo); -ORT_RUNTIME_CLASS(SequenceTypeInfo); -ORT_RUNTIME_CLASS(ModelMetadata); -ORT_RUNTIME_CLASS(ThreadPoolParams); -ORT_RUNTIME_CLASS(ThreadingOptions); -ORT_RUNTIME_CLASS(ArenaCfg); -ORT_RUNTIME_CLASS(PrepackedWeightsContainer); -ORT_RUNTIME_CLASS(TensorRTProviderOptionsV2); -ORT_RUNTIME_CLASS(CUDAProviderOptionsV2); -ORT_RUNTIME_CLASS(Op); -ORT_RUNTIME_CLASS(OpAttr); - -#ifdef _WIN32 -typedef _Return_type_success_(return == 0) OrtStatus* OrtStatusPtr; -#else -typedef OrtStatus* OrtStatusPtr; -#endif - -/** \brief Memory allocation interface -* -* Structure of function pointers that defines a memory allocator. This can be created and filled in by the user for custom allocators. -* -* When an allocator is passed to any function, be sure that the allocator object is not destroyed until the last allocated object using it is freed. -*/ -typedef struct OrtAllocator { - uint32_t version; ///< Must be initialized to ORT_API_VERSION - void*(ORT_API_CALL* Alloc)(struct OrtAllocator* this_, size_t size); ///< Returns a pointer to an allocated block of `size` bytes - void(ORT_API_CALL* Free)(struct OrtAllocator* this_, void* p); ///< Free a block of memory previously allocated with OrtAllocator::Alloc - const struct OrtMemoryInfo*(ORT_API_CALL* Info)(const struct OrtAllocator* this_); ///< Return a pointer to an ::OrtMemoryInfo that describes this allocator -} OrtAllocator; - -typedef void(ORT_API_CALL* OrtLoggingFunction)( - void* param, OrtLoggingLevel severity, const char* category, const char* logid, const char* code_location, - const char* message); - -/** \brief Graph optimization level -* -* Refer to https://www.onnxruntime.ai/docs/resources/graph-optimizations.html -* for an in-depth understanding of Graph Optimizations -*/ -typedef enum GraphOptimizationLevel { - ORT_DISABLE_ALL = 0, - ORT_ENABLE_BASIC = 1, - ORT_ENABLE_EXTENDED = 2, - ORT_ENABLE_ALL = 99 -} GraphOptimizationLevel; - -typedef enum ExecutionMode { - ORT_SEQUENTIAL = 0, - ORT_PARALLEL = 1, -} ExecutionMode; - -/** \brief Language projection identifiers -* /see OrtApi::SetLanguageProjection -*/ -typedef enum OrtLanguageProjection { - ORT_PROJECTION_C = 0, - ORT_PROJECTION_CPLUSPLUS = 1, - ORT_PROJECTION_CSHARP = 2, - ORT_PROJECTION_PYTHON = 3, - ORT_PROJECTION_JAVA = 4, - ORT_PROJECTION_WINML = 5, - ORT_PROJECTION_NODEJS = 6, -} OrtLanguageProjection; - -struct OrtKernelInfo; -typedef struct OrtKernelInfo OrtKernelInfo; -struct OrtKernelContext; -typedef struct OrtKernelContext OrtKernelContext; -struct OrtCustomOp; -typedef struct OrtCustomOp OrtCustomOp; - -typedef enum OrtAllocatorType { - OrtInvalidAllocator = -1, - OrtDeviceAllocator = 0, - OrtArenaAllocator = 1 -} OrtAllocatorType; - -/** \brief Memory types for allocated memory, execution provider specific types should be extended in each provider. -*/ -// Whenever this struct is updated, please also update the MakeKey function in onnxruntime / core / framework / execution_provider.cc -typedef enum OrtMemType { - OrtMemTypeCPUInput = -2, ///< Any CPU memory used by non-CPU execution provider - OrtMemTypeCPUOutput = -1, ///< CPU accessible memory outputted by non-CPU execution provider, i.e. CUDA_PINNED - OrtMemTypeCPU = OrtMemTypeCPUOutput, ///< Temporary CPU accessible memory allocated by non-CPU execution provider, i.e. CUDA_PINNED - OrtMemTypeDefault = 0, ///< The default allocator for execution provider -} OrtMemType; - -/** \brief Algorithm to use for cuDNN Convolution Op -*/ -typedef enum OrtCudnnConvAlgoSearch { - OrtCudnnConvAlgoSearchExhaustive, // expensive exhaustive benchmarking using cudnnFindConvolutionForwardAlgorithmEx - OrtCudnnConvAlgoSearchHeuristic, // lightweight heuristic based search using cudnnGetConvolutionForwardAlgorithm_v7 - OrtCudnnConvAlgoSearchDefault, // default algorithm using CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM -} OrtCudnnConvAlgoSearch; - -/** \brief CUDA Provider Options -* -* \see OrtApi::SessionOptionsAppendExecutionProvider_CUDA -*/ -typedef struct OrtCUDAProviderOptions { -#ifdef __cplusplus - OrtCUDAProviderOptions() : device_id{}, cudnn_conv_algo_search{OrtCudnnConvAlgoSearchExhaustive}, gpu_mem_limit{SIZE_MAX}, arena_extend_strategy{}, do_copy_in_default_stream{1}, has_user_compute_stream{}, user_compute_stream{}, default_memory_arena_cfg{} {} -#endif - - /** \brief CUDA device Id - * Defaults to 0. - */ - int device_id; - - /** \brief CUDA Convolution algorithm search configuration. - * See enum OrtCudnnConvAlgoSearch for more details. - * Defaults to OrtCudnnConvAlgoSearchExhaustive. - */ - OrtCudnnConvAlgoSearch cudnn_conv_algo_search; - - /** \brief CUDA memory limit (To use all possible memory pass in maximum size_t) - * Defaults to SIZE_MAX. - * \note If a ::OrtArenaCfg has been applied, it will override this field - */ - size_t gpu_mem_limit; - - /** \brief Strategy used to grow the memory arena - * 0 = kNextPowerOfTwo
- * 1 = kSameAsRequested
- * Defaults to 0. - * \note If a ::OrtArenaCfg has been applied, it will override this field - */ - int arena_extend_strategy; - - /** \brief Flag indicating if copying needs to take place on the same stream as the compute stream in the CUDA EP - * 0 = Use separate streams for copying and compute. - * 1 = Use the same stream for copying and compute. - * Defaults to 1. - * WARNING: Setting this to 0 may result in data races for some models. - * Please see issue #4829 for more details. - */ - int do_copy_in_default_stream; - - /** \brief Flag indicating if there is a user provided compute stream - * Defaults to 0. - */ - int has_user_compute_stream; - - /** \brief User provided compute stream. - * If provided, please set `has_user_compute_stream` to 1. - */ - void* user_compute_stream; - - /** \brief CUDA memory arena configuration parameters - */ - OrtArenaCfg* default_memory_arena_cfg; - -} OrtCUDAProviderOptions; - -/** \brief ROCM Provider Options -* -* \see OrtApi::SessionOptionsAppendExecutionProvider_ROCM -*/ -typedef struct OrtROCMProviderOptions { -#ifdef __cplusplus - OrtROCMProviderOptions() : device_id{}, miopen_conv_exhaustive_search{0}, gpu_mem_limit{SIZE_MAX}, arena_extend_strategy{}, do_copy_in_default_stream{1}, has_user_compute_stream{}, user_compute_stream{}, default_memory_arena_cfg{} {} -#endif - - /** \brief ROCM device Id - * Defaults to 0. - */ - int device_id; - - /** \brief ROCM MIOpen Convolution algorithm exaustive search option. - * Defaults to 0 (false). - */ - int miopen_conv_exhaustive_search; - - /** \brief ROCM memory limit (To use all possible memory pass in maximum size_t) - * Defaults to SIZE_MAX. - * \note If a ::OrtArenaCfg has been applied, it will override this field - */ - size_t gpu_mem_limit; - - /** \brief Strategy used to grow the memory arena - * 0 = kNextPowerOfTwo
- * 1 = kSameAsRequested
- * Defaults to 0. - * \note If a ::OrtArenaCfg has been applied, it will override this field - */ - int arena_extend_strategy; - - /** \brief Flag indicating if copying needs to take place on the same stream as the compute stream in the ROCM EP - * 0 = Use separate streams for copying and compute. - * 1 = Use the same stream for copying and compute. - * Defaults to 1. - * WARNING: Setting this to 0 may result in data races for some models. - * Please see issue #4829 for more details. - */ - int do_copy_in_default_stream; - - /** \brief Flag indicating if there is a user provided compute stream - * Defaults to 0. - */ - int has_user_compute_stream; - - /** \brief User provided compute stream. - * If provided, please set `has_user_compute_stream` to 1. - */ - void* user_compute_stream; - - /** \brief ROCM memory arena configuration parameters - */ - OrtArenaCfg* default_memory_arena_cfg; - -} OrtROCMProviderOptions; - -/** \brief TensorRT Provider Options -* -* \see OrtApi::SessionOptionsAppendExecutionProvider_TensorRT -*/ -typedef struct OrtTensorRTProviderOptions { - int device_id; ///< CUDA device id (0 = default device) - int has_user_compute_stream; // indicator of user specified CUDA compute stream. - void* user_compute_stream; // user specified CUDA compute stream. - int trt_max_partition_iterations; // maximum iterations for TensorRT parser to get capability - int trt_min_subgraph_size; // minimum size of TensorRT subgraphs - size_t trt_max_workspace_size; // maximum workspace size for TensorRT. - int trt_fp16_enable; // enable TensorRT FP16 precision. Default 0 = false, nonzero = true - int trt_int8_enable; // enable TensorRT INT8 precision. Default 0 = false, nonzero = true - const char* trt_int8_calibration_table_name; // TensorRT INT8 calibration table name. - int trt_int8_use_native_calibration_table; // use native TensorRT generated calibration table. Default 0 = false, nonzero = true - int trt_dla_enable; // enable DLA. Default 0 = false, nonzero = true - int trt_dla_core; // DLA core number. Default 0 - int trt_dump_subgraphs; // dump TRT subgraph. Default 0 = false, nonzero = true - int trt_engine_cache_enable; // enable engine caching. Default 0 = false, nonzero = true - const char* trt_engine_cache_path; // specify engine cache path - int trt_engine_decryption_enable; // enable engine decryption. Default 0 = false, nonzero = true - const char* trt_engine_decryption_lib_path; // specify engine decryption library path - int trt_force_sequential_engine_build; // force building TensorRT engine sequentially. Default 0 = false, nonzero = true - // This is the legacy struct and don't add new fields here. - // For new field that can be represented by string, please add it in include/onnxruntime/core/providers/tensorrt/tensorrt_provider_options.h - // For non-string field, need to create a new separate api to handle it. -} OrtTensorRTProviderOptions; - -/** \brief MIGraphX Provider Options -* -* \see OrtApi::SessionOptionsAppendExecutionProvider_MIGraphX -*/ -typedef struct OrtMIGraphXProviderOptions { - int device_id; // hip device id. - int migraphx_fp16_enable; // enable MIGraphX FP16 precision. Default 0 = false, nonzero = true - int migraphx_int8_enable; // enable MIGraphX INT8 precision. Default 0 = false, nonzero = true -} OrtMIGraphXProviderOptions; - -/** \brief OpenVINO Provider Options -* -* \see OrtApi::SessionOptionsAppendExecutionProvider_OpenVINO -*/ -typedef struct OrtOpenVINOProviderOptions { -#ifdef __cplusplus - OrtOpenVINOProviderOptions() : device_type{}, enable_vpu_fast_compile{}, device_id{}, - num_of_threads{}, use_compiled_network{}, blob_dump_path{}, - context{}, enable_opencl_throttling{}, enable_dynamic_shapes{} {} -#endif - /** \brief Device type string - * - * Valid settings are one of: "CPU_FP32", "GPU_FP32", "GPU_FP16", "MYRIAD_FP16", "VAD-M_FP16" or "VAD-F_FP32" - */ - const char* device_type; - unsigned char enable_vpu_fast_compile; ///< 0 = disabled, nonzero = enabled - const char* device_id; - size_t num_of_threads; ///< 0 = Use default number of threads - unsigned char use_compiled_network; ///< 0 = disabled, nonzero = enabled - const char* blob_dump_path; // path is set to empty by default - void* context; - unsigned char enable_opencl_throttling; ///< 0 = disabled, nonzero = enabled - unsigned char enable_dynamic_shapes; ///< 0 = disabled, nonzero = enabled -} OrtOpenVINOProviderOptions; - -struct OrtApi; -typedef struct OrtApi OrtApi; - -/** \brief The helper interface to get the right version of OrtApi -* -* Get a pointer to this structure through ::OrtGetApiBase -*/ -struct OrtApiBase { - /** \brief Get a pointer to the requested version of the ::OrtApi - * - * \param[in] version Must be ::ORT_API_VERSION - * \return The ::OrtApi for the version requested, nullptr will be returned if this version is unsupported, for example when using a runtime - * older than the version created with this header file. - */ - const OrtApi*(ORT_API_CALL* GetApi)(uint32_t version)NO_EXCEPTION; - const char*(ORT_API_CALL* GetVersionString)(void)NO_EXCEPTION; ///< Returns a null terminated string of the version of the Onnxruntime library (eg: "1.8.1") -}; -typedef struct OrtApiBase OrtApiBase; - -/** \brief The Onnxruntime library's entry point to access the C API -* -* Call this to get the a pointer to an ::OrtApiBase -*/ -ORT_EXPORT const OrtApiBase* ORT_API_CALL OrtGetApiBase(void) NO_EXCEPTION; - -/** \brief Thread work loop function -* -* Onnxruntime will provide the working loop on custom thread creation -* Argument is an onnxruntime built-in type which will be provided when thread pool calls OrtCustomCreateThreadFn -*/ -typedef void (*OrtThreadWorkerFn)(void* ort_worker_fn_param); - -typedef const struct OrtCustomHandleType { char __place_holder; } * OrtCustomThreadHandle; - -/** \brief Ort custom thread creation function -* -* The function should return a thread handle to be used in onnxruntime thread pools -* Onnxruntime will throw exception on return value of nullptr or 0, indicating that the function failed to create a thread -*/ -typedef OrtCustomThreadHandle (*OrtCustomCreateThreadFn)(void* ort_custom_thread_creation_options, OrtThreadWorkerFn ort_thread_worker_fn, void* ort_worker_fn_param); - -/** \brief Custom thread join function -* -* Onnxruntime thread pool destructor will call the function to join a custom thread. -* Argument ort_custom_thread_handle is the value returned by OrtCustomCreateThreadFn -*/ -typedef void (*OrtCustomJoinThreadFn)(OrtCustomThreadHandle ort_custom_thread_handle); - -/** \brief The C API -* -* All C API functions are defined inside this structure as pointers to functions. -* Call OrtApiBase::GetApi to get a pointer to it -* -* \nosubgrouping -*/ -struct OrtApi { - /// \name OrtStatus - /// @{ - - /** - * \brief Create an OrtStatus from a null terminated string - * - * \param[in] code - * \param[in] msg A null-terminated string. Its contents will be copied. - * \return A new OrtStatus object, must be destroyed with OrtApi::ReleaseStatus - */ - OrtStatus*(ORT_API_CALL* CreateStatus)(OrtErrorCode code, _In_ const char* msg)NO_EXCEPTION ORT_ALL_ARGS_NONNULL; - - /** \brief Get OrtErrorCode from OrtStatus - * - * \param[in] status - * \return OrtErrorCode that \p status was created with - */ - OrtErrorCode(ORT_API_CALL* GetErrorCode)(_In_ const OrtStatus* status) NO_EXCEPTION ORT_ALL_ARGS_NONNULL; - - /** \brief Get error string from OrtStatus - * - * \param[in] status - * \return The error message inside the `status`. Do not free the returned value. - */ - const char*(ORT_API_CALL* GetErrorMessage)(_In_ const OrtStatus* status)NO_EXCEPTION ORT_ALL_ARGS_NONNULL; - - /// @} - /// \name OrtEnv - /// @{ - - /** \brief Create an OrtEnv - * - * \param[in] log_severity_level The log severity level. - * \param[in] logid The log identifier. - * \param[out] out Returned newly created OrtEnv. Must be freed with OrtApi::ReleaseEnv - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateEnv, OrtLoggingLevel log_severity_level, _In_ const char* logid, _Outptr_ OrtEnv** out); - - /** \brief Create an OrtEnv - * - * \param[in] logging_function A pointer to a logging function. - * \param[in] logger_param A pointer to arbitrary data passed as the ::OrtLoggingFunction `param` parameter to - * `logging_function`. - * \param[in] log_severity_level The log severity level. - * \param[in] logid The log identifier. - * \param[out] out Returned newly created OrtEnv. Must be freed with OrtApi::ReleaseEnv - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateEnvWithCustomLogger, OrtLoggingFunction logging_function, _In_opt_ void* logger_param, - OrtLoggingLevel log_severity_level, _In_ const char* logid, _Outptr_ OrtEnv** out); - - /** \brief Enable Telemetry - * - * \note Telemetry events are on by default since they are lightweight - * \param[in] env - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(EnableTelemetryEvents, _In_ const OrtEnv* env); - /** \brief Disable Telemetry - * - * \see OrtApi::EnableTelemetryEvents - * \param[in] env - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(DisableTelemetryEvents, _In_ const OrtEnv* env); - - /// @} - /// \name OrtSession - /// @{ - - /** \brief Create an OrtSession from a model file - * - * \param[in] env - * \param[in] model_path - * \param[in] options - * \param[out] out Returned newly created OrtSession. Must be freed with OrtApi::ReleaseSession - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - // TODO: document the path separator convention? '/' vs '\' - // TODO: should specify the access characteristics of model_path. Is this read only during the - // execution of CreateSession, or does the OrtSession retain a handle to the file/directory - // and continue to access throughout the OrtSession lifetime? - // What sort of access is needed to model_path : read or read/write? - ORT_API2_STATUS(CreateSession, _In_ const OrtEnv* env, _In_ const ORTCHAR_T* model_path, - _In_ const OrtSessionOptions* options, _Outptr_ OrtSession** out); - - /** \brief Create an OrtSession from memory - * - * \param[in] env - * \param[in] model_data - * \param[in] model_data_length - * \param[in] options - * \param[out] out Returned newly created OrtSession. Must be freed with OrtApi::ReleaseSession - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateSessionFromArray, _In_ const OrtEnv* env, _In_ const void* model_data, size_t model_data_length, - _In_ const OrtSessionOptions* options, _Outptr_ OrtSession** out); - - /** \brief Run the model in an ::OrtSession - * - * Will not return until the model run has completed. Multiple threads might be used to run the model based on - * the options in the ::OrtSession and settings used when creating the ::OrtEnv - * - * \param[in] session - * \param[in] run_options If nullptr, will use a default ::OrtRunOptions - * \param[in] input_names Array of null terminated UTF8 encoded strings of the input names - * \param[in] inputs Array of ::OrtValue%s of the input values - * \param[in] input_len Number of elements in the input_names and inputs arrays - * \param[in] output_names Array of null terminated UTF8 encoded strings of the output names - * \param[in] output_names_len Number of elements in the output_names and outputs array - * \param[out] outputs Array of ::OrtValue%s that the outputs are stored in. This can also be - * an array of nullptr values, in this case ::OrtValue objects will be allocated and pointers - * to them will be set into the `outputs` array. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(Run, _Inout_ OrtSession* session, _In_opt_ const OrtRunOptions* run_options, - _In_reads_(input_len) const char* const* input_names, - _In_reads_(input_len) const OrtValue* const* inputs, size_t input_len, - _In_reads_(output_names_len) const char* const* output_names, size_t output_names_len, - _Inout_updates_all_(output_names_len) OrtValue** outputs); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Create an ::OrtSessionOptions object - * - * To use additional providers, you must build ORT with the extra providers enabled. Then call one of these - * functions to enable them in the session:
- * OrtSessionOptionsAppendExecutionProvider_CPU
- * OrtSessionOptionsAppendExecutionProvider_CUDA
- * OrtSessionOptionsAppendExecutionProvider_(remaining providers...)
- * The order they are called indicates the preference order as well. In other words call this method - * on your most preferred execution provider first followed by the less preferred ones. - * If none are called Ort will use its internal CPU execution provider. - * - * \param[out] options The newly created OrtSessionOptions. Must be freed with OrtApi::ReleaseSessionOptions - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateSessionOptions, _Outptr_ OrtSessionOptions** options); - - /** \brief Set filepath to save optimized model after graph level transformations - * - * \param[in] options - * \param[in] optimized_model_filepath - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetOptimizedModelFilePath, _Inout_ OrtSessionOptions* options, - _In_ const ORTCHAR_T* optimized_model_filepath); - - /** \brief Create a copy of an existing ::OrtSessionOptions - * - * \param[in] in_options OrtSessionOptions to copy - * \param[out] out_options Returned newly created ::OrtSessionOptions. Must be freed with OrtApi::ReleaseSessionOptions - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CloneSessionOptions, _In_ const OrtSessionOptions* in_options, - _Outptr_ OrtSessionOptions** out_options); - - /** \brief Set execution mode - * - * Controls whether you want to execute operators in your graph sequentially or in parallel. Usually when the model - * has many branches, setting this option to ExecutionMode.ORT_PARALLEL will give you better performance. - * See [docs/ONNX_Runtime_Perf_Tuning.md] for more details. - * - * \param[in] options - * \param[in] execution_mode - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetSessionExecutionMode, _Inout_ OrtSessionOptions* options, ExecutionMode execution_mode); - - /** \brief Enable profiling for a session - * - * \param[in] options - * \param[in] profile_file_prefix - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(EnableProfiling, _Inout_ OrtSessionOptions* options, _In_ const ORTCHAR_T* profile_file_prefix); - - /** \brief Disable profiling for a session - * - * \param[in] options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(DisableProfiling, _Inout_ OrtSessionOptions* options); - - /** \brief Enable the memory pattern optimization - * - * The idea is if the input shapes are the same, we could trace the internal memory allocation - * and generate a memory pattern for future request. So next time we could just do one allocation - * with a big chunk for all the internal memory allocation. - * \note Memory pattern optimization is only available when Sequential Execution mode is enabled (see OrtApi::SetSessionExecutionMode) - * - * \see OrtApi::DisableMemPattern - * - * \param[in] options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(EnableMemPattern, _Inout_ OrtSessionOptions* options); - - /** \brief Disable the memory pattern optimization - * - * \see OrtApi::EnableMemPattern - * - * \param[in] options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(DisableMemPattern, _Inout_ OrtSessionOptions* options); - - /** \brief Enable the memory arena on CPU - * - * Arena may pre-allocate memory for future usage. - * - * \param[in] options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(EnableCpuMemArena, _Inout_ OrtSessionOptions* options); - - /** \brief Disable the memory arena on CPU - * - * \param[in] options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(DisableCpuMemArena, _Inout_ OrtSessionOptions* options); - - /** \brief Set session log id - * - * \param[in] options - * \param[in] logid The log identifier. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetSessionLogId, _Inout_ OrtSessionOptions* options, const char* logid); - - /** \brief Set session log verbosity level - * - * Applies to session load, initialization, etc - * - * \param[in] options - * \param[in] session_log_verbosity_level \snippet{doc} snippets.dox Log Verbosity Level - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetSessionLogVerbosityLevel, _Inout_ OrtSessionOptions* options, int session_log_verbosity_level); - - /** \brief Set session log severity level - * - * \param[in] options - * \param[in] session_log_severity_level The log severity level (refer to ::OrtLoggingLevel for possible values). - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetSessionLogSeverityLevel, _Inout_ OrtSessionOptions* options, int session_log_severity_level); - - /** \brief Set the optimization level to apply when loading a graph - * - * Please see https://www.onnxruntime.ai/docs/resources/graph-optimizations.html for an in-depth explanation - * \param[in,out] options The session options object - * \param[in] graph_optimization_level The optimization level - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetSessionGraphOptimizationLevel, _Inout_ OrtSessionOptions* options, - GraphOptimizationLevel graph_optimization_level); - - /** \brief Sets the number of threads used to parallelize the execution within nodes - * - * When running a single node operation, ex. add, this sets the maximum number of threads to use. - * - * \note If built with OpenMP, this has no effect on the number of threads used. In this case - * use the OpenMP env variables to configure the number of intra op num threads. - * - * \param[in] options - * \param[in] intra_op_num_threads Number of threads to use
- * A value of 0 will use the default number of threads
- * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetIntraOpNumThreads, _Inout_ OrtSessionOptions* options, int intra_op_num_threads); - - /** \brief Sets the number of threads used to parallelize the execution of the graph - * - * If nodes can be run in parallel, this sets the maximum number of threads to use to run them in parallel. - * - * \note If sequential execution is enabled this value is ignored, it acts as if it was set to 1. - * - * \param[in] options - * \param[in] inter_op_num_threads Number of threads to use
- * A value of 0 will use the default number of threads
- * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetInterOpNumThreads, _Inout_ OrtSessionOptions* options, int inter_op_num_threads); - - /// @} - /// \name OrtCustomOpDomain - /// @{ - - /** \brief Create a custom op domain - * - * \param[in] domain - * \param[out] out Newly created domain. Must be freed with OrtApi::ReleaseCustomOpDomain - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateCustomOpDomain, _In_ const char* domain, _Outptr_ OrtCustomOpDomain** out); - - /** \brief Add a custom op to a custom op domain - * - * \note The OrtCustomOp* pointer must remain valid until the ::OrtCustomOpDomain using it is released - * - * \param[in] custom_op_domain - * \param[in] op - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CustomOpDomain_Add, _Inout_ OrtCustomOpDomain* custom_op_domain, _In_ const OrtCustomOp* op); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Add custom op domain to a session options - * - * \note The OrtCustomOpDomain* must not be deleted until all sessions using it are released - * - * \param[in] options - * \param[in] custom_op_domain - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(AddCustomOpDomain, _Inout_ OrtSessionOptions* options, _In_ OrtCustomOpDomain* custom_op_domain); - - /** \brief Register custom ops from a shared library - * - * Loads a shared library (dll on windows, so on linux, etc) named 'library_path' and looks for this entry point: - * OrtStatus* RegisterCustomOps(OrtSessionOptions * options, const OrtApiBase* api); - * It then passes in the provided session options to this function along with the api base. - * The handle to the loaded library is returned in library_handle. It can be freed by the caller after all sessions using the passed in - * session options are destroyed, or if an error occurs and it is non null. - * - * \param[in] options - * \param[in] library_path - * \param[out] library_handle OS specific handle to the loaded library (Use FreeLibrary on Windows, dlclose on Linux, etc.. to unload) - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(RegisterCustomOpsLibrary, _Inout_ OrtSessionOptions* options, _In_ const char* library_path, _Outptr_ void** library_handle); - - /// @} - /// \name OrtSession - /// @{ - - /** \brief Get input count for a session - * - * This number must also match the number of inputs passed to OrtApi::Run - * - * \see OrtApi::SessionGetInputTypeInfo, OrtApi::SessionGetInputName, OrtApi::Session - * - * \param[in] session - * \param[out] out Number of inputs - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetInputCount, _In_ const OrtSession* session, _Out_ size_t* out); - - /** \brief Get output count for a session - * - * This number must also match the number of outputs returned by OrtApi::Run - * - * \see OrtApi::SessionGetOutputTypeInfo, OrtApi::SessionGetOutputName, OrtApi::Session - * - * \param[in] session - * \param[out] out Number of outputs - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetOutputCount, _In_ const OrtSession* session, _Out_ size_t* out); - - /** \brief Get overridable initializer count - * - * \see OrtApi::SessionGetOverridableInitializerTypeInfo, OrtApi::SessionGetOverridableInitializerName - * - * \param[in] session - * \param[in] out - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetOverridableInitializerCount, _In_ const OrtSession* session, _Out_ size_t* out); - - /** \brief Get input type information - * - * \param[in] session - * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetInputCount returns (exclusive) - * \param[out] type_info Must be freed with OrtApi::ReleaseTypeInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetInputTypeInfo, _In_ const OrtSession* session, size_t index, _Outptr_ OrtTypeInfo** type_info); - - /** \brief Get output type information - * - * \param[in] session - * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetOutputCount returns (exclusive) - * \param[out] type_info Must be freed with OrtApi::ReleaseTypeInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetOutputTypeInfo, _In_ const OrtSession* session, size_t index, _Outptr_ OrtTypeInfo** type_info); - - /** \brief Get overridable initializer type information - * - * \param[in] session - * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetOverridableInitializerCount returns (exclusive) - * \param[out] type_info Must be freed with OrtApi::ReleaseTypeInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetOverridableInitializerTypeInfo, _In_ const OrtSession* session, size_t index, _Outptr_ OrtTypeInfo** type_info); - - /** \brief Get input name - * - * \param[in] session - * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetInputCount returns (exclusive) - * \param[in] allocator - * \param[out] value Set to a null terminated UTF-8 encoded string allocated using `allocator`. Must be freed using `allocator`. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetInputName, _In_ const OrtSession* session, size_t index, _Inout_ OrtAllocator* allocator, _Outptr_ char** value); - - /** \brief Get output name - * - * \param[in] session - * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetOutputCount returns (exclusive) - * \param[in] allocator - * \param[out] value Set to a null terminated UTF-8 encoded string allocated using `allocator`. Must be freed using `allocator`. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetOutputName, _In_ const OrtSession* session, size_t index, _Inout_ OrtAllocator* allocator, _Outptr_ char** value); - - /** \brief Get overridable initializer name - * - * \param[in] session - * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetOverridableInitializerCount returns (exclusive) - * \param[in] allocator - * \param[out] value Set to a null terminated UTF-8 encoded string allocated using `allocator`. Must be freed using `allocator`. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetOverridableInitializerName, _In_ const OrtSession* session, size_t index, - _Inout_ OrtAllocator* allocator, _Outptr_ char** value); - - /// @} - /// \name OrtRunOptions - /// @{ - - /** \brief Create an OrtRunOptions - * - * \param[out] out Returned newly created ::OrtRunOptions. Must be freed with OrtApi::ReleaseRunOptions - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateRunOptions, _Outptr_ OrtRunOptions** out); - - /** \brief Set per-run log verbosity level - * - * \see OrtApi::RunOptionsGetRunLogVerbosityLevel - * - * \param[in] options - * \param[in] log_verbosity_level \snippet{doc} snippets.dox Log Verbosity Level - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(RunOptionsSetRunLogVerbosityLevel, _Inout_ OrtRunOptions* options, int log_verbosity_level); - - /** \brief Set per-run log severity level - * - * \see OrtApi::RunOptionsGetRunLogSeverityLevel - * - * \param[in] options - * \param[in] log_severity_level The log severity level (refer to ::OrtLoggingLevel for possible values). - */ - ORT_API2_STATUS(RunOptionsSetRunLogSeverityLevel, _Inout_ OrtRunOptions* options, int log_severity_level); - - /** \brief Set per-run tag - * - * This is used in a per-run log identifier. - * - * \see OrtApi::RunOptionsGetRunTag - * - * \param[in] options - * \param[in] run_tag The run tag. - */ - ORT_API2_STATUS(RunOptionsSetRunTag, _Inout_ OrtRunOptions* options, _In_ const char* run_tag); - - /** \brief Get per-run log verbosity level - * - * \see OrtApi::RunOptionsSetRunLogVerbosityLevel - * - * \param[in] options - * \param[out] log_verbosity_level \snippet{doc} snippets.dox Log Verbosity Level - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(RunOptionsGetRunLogVerbosityLevel, _In_ const OrtRunOptions* options, - _Out_ int* log_verbosity_level); - - /** \brief Get per-run log severity level - * - * \see OrtApi::RunOptionsSetRunLogSeverityLevel - * - * \param[in] options - * \param[out] log_severity_level The log severity level (refer to ::OrtLoggingLevel for possible values). - */ - ORT_API2_STATUS(RunOptionsGetRunLogSeverityLevel, _In_ const OrtRunOptions* options, _Out_ int* log_severity_level); - - /** \brief Get per-run tag - * - * This is used in a per-run log identifier. - * - * \see OrtApi::RunOptionsSetRunTag - * - * \param[in] options - * \param[out] run_tag The run tag. - * Do not free this value, it is owned by `options`. It will be invalidated if the run tag - * changes (i.e., with OrtApi::RunOptionsSetRunTag) or `options` is freed. - */ - ORT_API2_STATUS(RunOptionsGetRunTag, _In_ const OrtRunOptions* options, _Out_ const char** run_tag); - - /** \brief Set terminate flag - * - * If a currently executing session needs to be force terminated, this can be called from another thread to force it to fail with an error. - * - * \param[in] options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(RunOptionsSetTerminate, _Inout_ OrtRunOptions* options); - - /** \brief Clears the terminate flag - * - * Used so the OrtRunOptions instance can be used in a new OrtApi::Run call without it instantly terminating - * - * \param[in] options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(RunOptionsUnsetTerminate, _Inout_ OrtRunOptions* options); - - /// @} - /// \name OrtValue - /// @{ - - /** \brief Create a tensor - * - * Create a tensor using a supplied ::OrtAllocator - * - * \param[in] allocator - * \param[in] shape Pointer to the tensor shape dimensions. - * \param[in] shape_len The number of tensor shape dimensions. - * \param[in] type - * \param[out] out Returns newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateTensorAsOrtValue, _Inout_ OrtAllocator* allocator, _In_ const int64_t* shape, size_t shape_len, - ONNXTensorElementDataType type, _Outptr_ OrtValue** out); - - /** \brief Create a tensor backed by a user supplied buffer - * - * Create a tensor with user's buffer. You can fill the buffer either before calling this function or after. - * p_data is owned by caller. ReleaseValue won't release p_data. - * - * \param[in] info Memory description of where the p_data buffer resides (CPU vs GPU etc). - * \param[in] p_data Pointer to the data buffer. - * \param[in] p_data_len The number of bytes in the data buffer. - * \param[in] shape Pointer to the tensor shape dimensions. - * \param[in] shape_len The number of tensor shape dimensions. - * \param[in] type The data type. - * \param[out] out Returns newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateTensorWithDataAsOrtValue, _In_ const OrtMemoryInfo* info, _Inout_ void* p_data, - size_t p_data_len, _In_ const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type, - _Outptr_ OrtValue** out); - - /** \brief Return if an ::OrtValue is a tensor type - * - * \param[in] value A tensor type (string tensors are not supported) - * \param[out] out Set to 1 iff ::OrtValue is a tensor, 0 otherwise - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(IsTensor, _In_ const OrtValue* value, _Out_ int* out); - - /** \brief Get a pointer to the raw data inside a tensor - * - * Used to read/write/modify the internal tensor data directly. - * \note The returned pointer is valid until the \p value is destroyed. - * - * \param[in] value A tensor type (string tensors are not supported) - * \param[out] out Filled in with a pointer to the internal storage - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetTensorMutableData, _In_ OrtValue* value, _Outptr_ void** out); - - /** \brief Set all strings at once in a string tensor - * - * \param[in,out] value A tensor of type ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING - * \param[in] s An array of strings. Each string in this array must be null terminated. - * \param[in] s_len Count of strings in s (Must match the size of \p value's tensor shape) - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(FillStringTensor, _Inout_ OrtValue* value, _In_ const char* const* s, size_t s_len); - - /** \brief Get total byte length for all strings in a string tensor - * - * Typically used with OrtApi::GetStringTensorContent - * - * \param[in] value A tensor of type ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING - * \param[out] len Total byte length of all strings (does not include trailing nulls) - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetStringTensorDataLength, _In_ const OrtValue* value, _Out_ size_t* len); - - /** \brief Get all strings from a string tensor - * - * An example of the results:
- * Given \p value is a string tensor with the strings { "This" "is" "a" "test" }
- * \p s must have a size of 11 bytes
- * \p offsets must have 4 elements
- * After the call, these values will be filled in:
- * \p s will contain "Thisisatest"
- * \p offsets will contain { 0, 4, 6, 7 }
- * The length of the last string is just s_len - offsets[last] - * - * \param[in] value A tensor of type ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING - * \param[in] s Buffer to sequentially write all tensor strings to. Each string is NOT null-terminated. - * \param[in] s_len Number of bytes of buffer pointed to by \p s (Get it from OrtApi::GetStringTensorDataLength) - * \param[out] offsets Array of start offsets into the strings written to \p s - * \param[in] offsets_len Number of elements in offsets - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetStringTensorContent, _In_ const OrtValue* value, _Out_writes_bytes_all_(s_len) void* s, - size_t s_len, _Out_writes_all_(offsets_len) size_t* offsets, size_t offsets_len); - - /// @} - /// \name OrtTypeInfo - /// @{ - - /** \brief Get ::OrtTensorTypeAndShapeInfo from an ::OrtTypeInfo - * - * \param[in] type_info - * \param[out] out Do not free this value, it will be valid until type_info is freed. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CastTypeInfoToTensorInfo, _In_ const OrtTypeInfo* type_info, - _Outptr_result_maybenull_ const OrtTensorTypeAndShapeInfo** out); - - /** \brief Get ::ONNXType from ::OrtTypeInfo - * - * \param[in] type_info - * \param[out] out - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetOnnxTypeFromTypeInfo, _In_ const OrtTypeInfo* type_info, _Out_ enum ONNXType* out); - - /// @} - /// \name OrtTensorTypeAndShapeInfo - /// @{ - - /** \brief Create an ::OrtTensorTypeAndShapeInfo object - * - * \param[out] out Returns newly created ::OrtTensorTypeAndShapeInfo. Must be freed with OrtApi::ReleaseTensorTypeAndShapeInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateTensorTypeAndShapeInfo, _Outptr_ OrtTensorTypeAndShapeInfo** out); - - /** \brief Set element type in ::OrtTensorTypeAndShapeInfo - * - * \param[in] info - * \param[in] type - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetTensorElementType, _Inout_ OrtTensorTypeAndShapeInfo* info, enum ONNXTensorElementDataType type); - - /** \brief Set shape information in ::OrtTensorTypeAndShapeInfo - * - * \param[in] info - * \param[in] dim_values Array with `dim_count` elements. Can contain negative values. - * \param[in] dim_count Number of elements in `dim_values` - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetDimensions, OrtTensorTypeAndShapeInfo* info, _In_ const int64_t* dim_values, size_t dim_count); - - /** \brief Get element type in ::OrtTensorTypeAndShapeInfo - * - * \see OrtApi::SetTensorElementType - * - * \param[in] info - * \param[out] out - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetTensorElementType, _In_ const OrtTensorTypeAndShapeInfo* info, - _Out_ enum ONNXTensorElementDataType* out); - - /** \brief Get dimension count in ::OrtTensorTypeAndShapeInfo - * - * \see OrtApi::GetDimensions - * - * \param[in] info - * \param[out] out - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetDimensionsCount, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ size_t* out); - - /** \brief Get dimensions in ::OrtTensorTypeAndShapeInfo - * - * \param[in] info - * \param[out] dim_values Array with `dim_values_length` elements. On return, filled with the dimensions stored in the ::OrtTensorTypeAndShapeInfo - * \param[in] dim_values_length Number of elements in `dim_values`. Use OrtApi::GetDimensionsCount to get this value - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetDimensions, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values, - size_t dim_values_length); - - /** \brief Get symbolic dimension names in ::OrtTensorTypeAndShapeInfo - * - * \param[in] info - * \param[in] dim_params Array with `dim_params_length` elements. On return filled with pointers to null terminated strings of the dimension names - * \param[in] dim_params_length Number of elements in `dim_params`. Use OrtApi::GetDimensionsCount to get this value - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetSymbolicDimensions, _In_ const OrtTensorTypeAndShapeInfo* info, - _Out_writes_all_(dim_params_length) const char* dim_params[], size_t dim_params_length); - - /** \brief Get total number of elements in a tensor shape from an ::OrtTensorTypeAndShapeInfo - * - * Return the number of elements specified by the tensor shape (all dimensions multiplied by each other). - * For 0 dimensions, 1 is returned. If any dimension is less than 0, the result is always -1. - * - * Examples:
- * [] = 1
- * [1,3,4] = 12
- * [2,0,4] = 0
- * [-1,3,4] = -1
- * - * \param[in] info - * \param[out] out Number of elements - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetTensorShapeElementCount, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ size_t* out); - - /// @} - /// \name OrtValue - /// @{ - - /** \brief Get type and shape information from a tensor ::OrtValue - * - * \param[in] value Must be a tensor (not a map/sequence/etc) or will return failure - * \param[out] out Newly created ::OrtTensorTypeAndShapeInfo. Must be freed with OrtApi::ReleaseTensorTypeAndShapeInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetTensorTypeAndShape, _In_ const OrtValue* value, _Outptr_ OrtTensorTypeAndShapeInfo** out); - - /** \brief Get type information of an OrtValue - * - * \param[in] value - * \param[out] out Newly created ::OrtTypeInfo. Must be freed with OrtApi::ReleaseTypeInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetTypeInfo, _In_ const OrtValue* value, _Outptr_result_maybenull_ OrtTypeInfo** out); - - /** \brief Get ONNXType of an ::OrtValue - * - * \param[in] value - * \param[out] out - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetValueType, _In_ const OrtValue* value, _Out_ enum ONNXType* out); - - /// @} - /// \name OrtMemoryInfo - /// @{ - - /** \brief Create an ::OrtMemoryInfo - * - * \param[in] name - * \param[in] type - * \param[in] id - * \param[in] mem_type - * \param[out] out Newly created ::OrtMemoryInfo. Must be freed with OrtAPi::ReleaseMemoryInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateMemoryInfo, _In_ const char* name, enum OrtAllocatorType type, int id, - enum OrtMemType mem_type, _Outptr_ OrtMemoryInfo** out); - - /** \brief Create an ::OrtMemoryInfo for CPU memory - * - * Special case version of OrtApi::CreateMemoryInfo for CPU based memory. Same as using OrtApi::CreateMemoryInfo with name = "Cpu" and id = 0. - * - * \param[in] type - * \param[in] mem_type - * \param[out] out - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateCpuMemoryInfo, enum OrtAllocatorType type, enum OrtMemType mem_type, - _Outptr_ OrtMemoryInfo** out); - - /** \brief Compare ::OrtMemoryInfo objects for equality - * - * Compares all settings of each ::OrtMemoryInfo for equality - * - * \param[in] info1 - * \param[in] info2 - * \param[out] out Set to 0 if equal, -1 if not equal - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CompareMemoryInfo, _In_ const OrtMemoryInfo* info1, _In_ const OrtMemoryInfo* info2, _Out_ int* out); - - /** \brief Get name from ::OrtMemoryInfo - * - * \param[in] ptr - * \param[out] out Writes null terminated string to this pointer. Do NOT free the returned pointer. It is valid for the lifetime of the ::OrtMemoryInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(MemoryInfoGetName, _In_ const OrtMemoryInfo* ptr, _Out_ const char** out); - - /** \brief Get the id from ::OrtMemoryInfo - */ - ORT_API2_STATUS(MemoryInfoGetId, _In_ const OrtMemoryInfo* ptr, _Out_ int* out); - - /** \brief Get the ::OrtMemType from ::OrtMemoryInfo - */ - ORT_API2_STATUS(MemoryInfoGetMemType, _In_ const OrtMemoryInfo* ptr, _Out_ OrtMemType* out); - - /** \brief Get the ::OrtAllocatorType from ::OrtMemoryInfo - */ - ORT_API2_STATUS(MemoryInfoGetType, _In_ const OrtMemoryInfo* ptr, _Out_ OrtAllocatorType* out); - - /// @} - /// \name OrtAllocator - /// @{ - - /// \brief Calls OrtAllocator::Alloc function - ORT_API2_STATUS(AllocatorAlloc, _Inout_ OrtAllocator* ort_allocator, size_t size, _Outptr_ void** out); - /// \brief Calls OrtAllocator::Free function - ORT_API2_STATUS(AllocatorFree, _Inout_ OrtAllocator* ort_allocator, void* p); - /// \brief Calls OrtAllocator::Info function - ORT_API2_STATUS(AllocatorGetInfo, _In_ const OrtAllocator* ort_allocator, _Outptr_ const struct OrtMemoryInfo** out); - - /** \brief Get the default allocator - * - * The default allocator is a CPU based, non-arena. Always returns the same pointer to the same default allocator. - * - * \param[out] out Returned value should NOT be freed - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetAllocatorWithDefaultOptions, _Outptr_ OrtAllocator** out); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Override session symbolic dimensions - * - * Override symbolic dimensions (by specific denotation strings) with actual values if known at session initialization time to enable - * optimizations that can take advantage of fixed values (such as memory planning, etc) - * - * \param[in] options - * \param[in] dim_denotation - * \param[in] dim_value - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(AddFreeDimensionOverride, _Inout_ OrtSessionOptions* options, _In_ const char* dim_denotation, - _In_ int64_t dim_value); - - /// @} - /// \name OrtValue - /// @{ - - /* Internal information (not seen in Doxygen) - * - * APIs to support non-tensor types - map and sequence. - * Currently only the following types are supported - * Note: the following types should be kept in sync with data_types.h - * Map types - * ========= - * std::map - * std::map - * std::map - * std::map - * std::map - * std::map - * std::map - * std::map - * - * Sequence types - * ============== - * std::vector - * std::vector - * std::vector - * std::vector - * std::vector> - * std::vector - */ - - /** \brief Get non tensor data from an ::OrtValue - * - * If `value` is of type ONNX_TYPE_MAP, you need to retrieve the keys and values - * separately. Use index=0 to retrieve keys and index=1 to retrieve values. - * If `value` is of type ONNX_TYPE_SEQUENCE, use index to retrieve the index'th element - * of the sequence. - * - * \param[in] value - * \param[in] index See above for usage based on `value` type - * \param[in] allocator Allocator used to allocate ::OrtValue - * \param[out] out Created ::OrtValue that holds the element requested. Must be freed with OrtApi::ReleaseValue - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetValue, _In_ const OrtValue* value, int index, _Inout_ OrtAllocator* allocator, - _Outptr_ OrtValue** out); - - /** \brief Get non tensor value count from an ::OrtValue - * - * If `value` is of type ONNX_TYPE_MAP 2 will always be returned. For ONNX_TYPE_SEQUENCE - * the number of elements in the sequence will be returned - * - * \param[in] value - * \param[out] out - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetValueCount, _In_ const OrtValue* value, _Out_ size_t* out); - - /** \brief Create a map or sequence ::OrtValue - * - * To construct a map (ONNX_TYPE_MAP), use num_values = 2 and `in` should be an array of 2 ::OrtValue%s - * representing keys and values.
- * - * To construct a sequence (ONNX_TYPE_SEQUENCE), use num_values = N where N is the number of the elements in the - * sequence. 'in' should be an array of N ::OrtValue%s. - * - * \param[in] in See above for details - * \param[in] num_values - * \param[in] value_type Must be either ONNX_TYPE_MAP or ONNX_TYPE_SEQUENCE - * \param[out] out Newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateValue, _In_reads_(num_values) const OrtValue* const* in, size_t num_values, - enum ONNXType value_type, _Outptr_ OrtValue** out); - - /** \brief Create an opaque (custom user defined type) ::OrtValue - * - * Constructs an ::OrtValue that contains a value of non-standard type created for - * experiments or while awaiting standardization. ::OrtValue in this case would contain - * an internal representation of the Opaque type. Opaque types are distinguished from - * each other by two strings 1) domain and 2) type name. The combination of the two - * must be unique, so the type representation is properly identified internally. The combination - * must be properly registered from within ORT at both compile/run time or by another API. - * - * To construct the ::OrtValue pass domain and type names, also a pointer to a data container - * the type of which must be known to both ORT and the client program. That data container may or may - * not match the internal representation of the Opaque type. The sizeof(data_container) is passed for - * verification purposes. - * - * \param[in] domain_name Null terminated string of the domain name - * \param[in] type_name Null terminated string of the type name - * \param[in] data_container User pointer Data to populate ::OrtValue - * \param[in] data_container_size Size in bytes of what `data_container` points to - * \param[out] out Newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateOpaqueValue, _In_z_ const char* domain_name, _In_z_ const char* type_name, - _In_ const void* data_container, size_t data_container_size, _Outptr_ OrtValue** out); - - /** \brief Get internal data from an opaque (custom user defined type) ::OrtValue - * - * Copies internal data from an opaque value into a user provided buffer - * - * \see OrtApi::CreateOpaqueValue - * - * \param[in] domain_name Null terminated string of the domain name - * \param[in] type_name Null terminated string of the type name - * \param[in] in The opaque ::OrtValue - * \param[out] data_container Buffer to copy data into - * \param[out] data_container_size Size in bytes of the buffer pointed to by data_container. Must match the size of the internal buffer. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetOpaqueValue, _In_ const char* domain_name, _In_ const char* type_name, _In_ const OrtValue* in, - _Out_ void* data_container, size_t data_container_size); - - /// @} - /// \name OrtKernelInfo - /// @{ - - /** \brief Get a float stored as an attribute in the graph node - * - * \param[in] info ::OrtKernelInfo instance - * \param[in] name Null terminated string of the name of the attribute - * \param[out] out Pointer to memory where the attribute will be stored - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(KernelInfoGetAttribute_float, _In_ const OrtKernelInfo* info, _In_ const char* name, - _Out_ float* out); - - /** \brief Fetch a 64-bit int stored as an attribute in the graph node - * - * \param[in] info ::OrtKernelInfo instance - * \param[in] name Null terminated string of the name of the attribute - * \param[out] out Pointer to memory where the attribute will be stored - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(KernelInfoGetAttribute_int64, _In_ const OrtKernelInfo* info, _In_ const char* name, - _Out_ int64_t* out); - - /** \brief Fetch a string stored as an attribute in the graph node - * - * If `out` is nullptr, the value of `size` is set to the true size of the string - * attribute, and a success status is returned. - * - * If the `size` parameter is greater than or equal to the actual string attribute's size, - * the value of `size` is set to the true size of the string attribute, the provided memory - * is filled with the attribute's contents, and a success status is returned. - * - * If the `size` parameter is less than the actual string attribute's size and `out` - * is not nullptr, the value of `size` is set to the true size of the string attribute - * and a failure status is returned.) - * - * \param[in] info ::OrtKernelInfo instance - * \param[in] name Null terminated string of the name of the attribute - * \param[out] out Pointer to memory where the attribute will be stored - * \param[in,out] size See above comments for details - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(KernelInfoGetAttribute_string, _In_ const OrtKernelInfo* info, _In_ const char* name, _Out_ char* out, - _Inout_ size_t* size); - - /// @} - /// \name OrtKernelContext - /// @{ - - /** \brief Used for custom operators, get the input count of a kernel - * - * \see ::OrtCustomOp - */ - ORT_API2_STATUS(KernelContext_GetInputCount, _In_ const OrtKernelContext* context, _Out_ size_t* out); - - /** \brief Used for custom operators, get the output count of a kernel - * - * \see ::OrtCustomOp - */ - ORT_API2_STATUS(KernelContext_GetOutputCount, _In_ const OrtKernelContext* context, _Out_ size_t* out); - - /** \brief Used for custom operators, get an input of a kernel - * - * \see ::OrtCustomOp - */ - ORT_API2_STATUS(KernelContext_GetInput, _In_ const OrtKernelContext* context, _In_ size_t index, - _Out_ const OrtValue** out); - - /** \brief Used for custom operators, get an output of a kernel - * - * \see ::OrtCustomOp - */ - ORT_API2_STATUS(KernelContext_GetOutput, _Inout_ OrtKernelContext* context, _In_ size_t index, - _In_ const int64_t* dim_values, size_t dim_count, _Outptr_ OrtValue** out); - - /// @} - /// \name OrtEnv - /// @{ - ORT_CLASS_RELEASE(Env); - /// @} - /// \name OrtStatus - /// @{ - ORT_CLASS_RELEASE(Status); - /// @} - /// \name OrtMemoryInfo - /// @{ - ORT_CLASS_RELEASE(MemoryInfo); - /// @} - /// \name OrtSession - /// @{ - ORT_CLASS_RELEASE(Session); //Don't call ReleaseSession from Dllmain (because session owns a thread pool) - /// @} - /// \name OrtValue - /// @{ - ORT_CLASS_RELEASE(Value); - /// @} - /// \name OrtRunOptions - /// @{ - ORT_CLASS_RELEASE(RunOptions); - /// @} - /// \name OrtTypeInfo - /// @{ - ORT_CLASS_RELEASE(TypeInfo); - /// @} - /// \name OrtTensorTypeAndShapeInfo - /// @{ - ORT_CLASS_RELEASE(TensorTypeAndShapeInfo); - /// @} - /// \name OrtSessionOptions - /// @{ - ORT_CLASS_RELEASE(SessionOptions); - /// @} - /// \name OrtCustomOpDomain - /// @{ - ORT_CLASS_RELEASE(CustomOpDomain); - - /// @} - /// \name OrtTypeInfo - /// @{ - - /** \brief Get denotation from type information - * - * Augments ::OrtTypeInfo to return denotations on the type. - * - * This is used by WinML to determine if an input/output is intended to be an Image or a Tensor. - * - * \param[in] type_info - * \param[out] denotation Pointer to the null terminated denotation string is written to this pointer. This pointer is valid until the object is destroyed or the name is changed, do not free. - * \param[out] len Length in bytes of the string returned in `denotation` - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetDenotationFromTypeInfo, _In_ const OrtTypeInfo* type_info, _Out_ const char** const denotation, - _Out_ size_t* len); - - /** \brief Get detailed map information from an ::OrtTypeInfo - * - * This augments ::OrtTypeInfo to return an ::OrtMapTypeInfo when the type is a map. - * The OrtMapTypeInfo has additional information about the map's key type and value type. - * - * This is used by WinML to support model reflection APIs. - * - * \param[out] type_info - * \param[out] out A pointer to the ::OrtMapTypeInfo. Do not free this value - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CastTypeInfoToMapTypeInfo, _In_ const OrtTypeInfo* type_info, - _Outptr_result_maybenull_ const OrtMapTypeInfo** out); - - /** \brief Cast ::OrtTypeInfo to an ::OrtSequenceTypeInfo - * - * This api augments ::OrtTypeInfo to return an ::OrtSequenceTypeInfo when the type is a sequence. - * The ::OrtSequenceTypeInfo has additional information about the sequence's element type. - * - * This is used by WinML to support model reflection APIs. - * - * \param[in] type_info - * \param[out] out A pointer to the OrtSequenceTypeInfo. Do not free this value - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CastTypeInfoToSequenceTypeInfo, _In_ const OrtTypeInfo* type_info, - _Outptr_result_maybenull_ const OrtSequenceTypeInfo** out); - - /// @} - /// \name OrtMapTypeInfo - /// @{ - - /** \brief Get key type from an ::OrtMapTypeInfo - * - * Key types are restricted to being scalar types. - * - * This is used by WinML to support model reflection APIs. - * - * \param[in] map_type_info - * \param[out] out - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetMapKeyType, _In_ const OrtMapTypeInfo* map_type_info, _Out_ enum ONNXTensorElementDataType* out); - - /** \brief Get the value type from an ::OrtMapTypeInfo - * - * \param[in] map_type_info - * \param[out] type_info - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetMapValueType, _In_ const OrtMapTypeInfo* map_type_info, _Outptr_ OrtTypeInfo** type_info); - - /// @} - /// \name OrtSequenceTypeInfo - /// @{ - - /** \brief Get element type from an ::OrtSequenceTypeInfo - * - * This is used by WinML to support model reflection APIs. - * - * \param[in] sequence_type_info - * \param[out] type_info - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetSequenceElementType, _In_ const OrtSequenceTypeInfo* sequence_type_info, - _Outptr_ OrtTypeInfo** type_info); - - /// @} - /// \name OrtMapTypeInfo - /// @{ - ORT_CLASS_RELEASE(MapTypeInfo); - /// @} - /// \name OrtSequenceTypeInfo - /// @{ - ORT_CLASS_RELEASE(SequenceTypeInfo); - - /// @} - /// \name OrtSession - /// @{ - - /** \brief End profiling and return filename of the profile data - * - * Profiling is turned on through OrtApi::EnableProfiling - * - * \param[in] session - * \param[in] allocator - * \param[out] out Null terminated string of the filename, allocated using `allocator`. Must be freed using `allocator` - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionEndProfiling, _In_ OrtSession* session, _Inout_ OrtAllocator* allocator, _Outptr_ char** out); - - /** \brief Get ::OrtModelMetadata from an ::OrtSession - * - * \param[in] session - * \param[out] out Newly created ::OrtModelMetadata. Must be freed using OrtApi::ReleaseModelMetadata - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetModelMetadata, _In_ const OrtSession* session, _Outptr_ OrtModelMetadata** out); - - /// @} - /// \name OrtModelMetadata - /// @{ - - /** \brief Get `producer name` from an ::OrtModelMetadata - * - * \param[in] model_metadata - * \param[in] allocator - * \param[out] value Set to a null terminated string allocated using `allocator`. Must be freed using `allocator` - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(ModelMetadataGetProducerName, _In_ const OrtModelMetadata* model_metadata, - _Inout_ OrtAllocator* allocator, _Outptr_ char** value); - - /** \brief Get `graph name` from an ::OrtModelMetadata - * - * \param[in] model_metadata - * \param[in] allocator - * \param[out] value Set to a null terminated string allocated using `allocator`. Must be freed using `allocator` - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(ModelMetadataGetGraphName, _In_ const OrtModelMetadata* model_metadata, - _Inout_ OrtAllocator* allocator, _Outptr_ char** value); - - /** \brief Get `domain` from an ::OrtModelMetadata - * - * \param[in] model_metadata - * \param[in] allocator - * \param[out] value Set to a null terminated string allocated using `allocator`. Must be freed using `allocator` - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(ModelMetadataGetDomain, _In_ const OrtModelMetadata* model_metadata, _Inout_ OrtAllocator* allocator, - _Outptr_ char** value); - - /** \brief Get `description` from an ::OrtModelMetadata - * - * \param[in] model_metadata - * \param[in] allocator - * \param[out] value Set to a null terminated string allocated using `allocator`. Must be freed using `allocator` - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(ModelMetadataGetDescription, _In_ const OrtModelMetadata* model_metadata, - _Inout_ OrtAllocator* allocator, _Outptr_ char** value); - - /** \brief Return data for a key in the custom metadata map in an ::OrtModelMetadata - * - * \param[in] model_metadata - * \param[in] allocator - * \param[in] key Null terminated string - * \param[out] value Set to a null terminated string allocated using `allocator`. Must be freed using `allocator` - * `value` will be set to nullptr if the given key is not found in the custom metadata map. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(ModelMetadataLookupCustomMetadataMap, _In_ const OrtModelMetadata* model_metadata, - _Inout_ OrtAllocator* allocator, _In_ const char* key, _Outptr_result_maybenull_ char** value); - - /** \brief Get version number from an ::OrtModelMetadata - * - * \param[in] model_metadata - * \param[out] value Set to the version number - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(ModelMetadataGetVersion, _In_ const OrtModelMetadata* model_metadata, _Out_ int64_t* value); - - ORT_CLASS_RELEASE(ModelMetadata); - - /// @} - /// \name OrtEnv - /// @{ - - /** \brief Create an OrtEnv - * - * Create an environment with global threadpools that will be shared across sessions. - * Use this in conjunction with OrtApi::DisablePerSessionThreads or else the session will use - * its own thread pools. - * - * \param[in] log_severity_level The log severity level. - * \param[in] logid The log identifier. - * \param[in] tp_options - * \param[out] out Returned newly created OrtEnv. Must be freed with OrtApi::ReleaseEnv - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateEnvWithGlobalThreadPools, OrtLoggingLevel log_severity_level, _In_ const char* logid, - _In_ const OrtThreadingOptions* tp_options, _Outptr_ OrtEnv** out); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Use global thread pool on a session - * - * Disable using per session thread pool and use the shared global threadpool. - * This should be used in conjunction with OrtApi::CreateEnvWithGlobalThreadPools. - * - * \param[in] options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(DisablePerSessionThreads, _Inout_ OrtSessionOptions* options); - - /// @} - /// \name OrtThreadingOptions - /// @{ - - /** \brief Create an ::OrtThreadingOptions - * - * \param[out] out Newly created ::OrtThreadingOptions. Must be freed with OrtApi::ReleaseThreadingOptions - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateThreadingOptions, _Outptr_ OrtThreadingOptions** out); - - ORT_CLASS_RELEASE(ThreadingOptions); - - /// @} - /// \name OrtModelMetadata - /// @{ - - /** - * - * \param[in] model_metadata - * \param[in] allocator - * \param[out] keys Array of null terminated strings (array count = num_keys) allocated using `allocator`. - * The strings and the pointer array must be freed using `allocator` - * `keys` will be set to nullptr if the custom metadata map is empty. - * \param[out] num_keys Set to the number of elements in the `keys` array - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(ModelMetadataGetCustomMetadataMapKeys, _In_ const OrtModelMetadata* model_metadata, - _Inout_ OrtAllocator* allocator, _Outptr_result_buffer_maybenull_(*num_keys) char*** keys, _Out_ int64_t* num_keys); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** - * - * Override symbolic dimensions (by specific name strings) with actual values - * if known at session initialization time to enable optimizations that can - * take advantage of fixed values (such as memory planning, etc) - * - */ - ORT_API2_STATUS(AddFreeDimensionOverrideByName, - _Inout_ OrtSessionOptions* options, _In_ const char* dim_name, - _In_ int64_t dim_value); - - /// @} - /// \name Misc - /// @{ - - /** \brief Get the names of all available providers - * - * \note The providers in the list are not guaranteed to be usable. They may fail to load due to missing system dependencies. - * For example, if the CUDA/cuDNN libraries are not installed, the CUDA provider will report an error when it is added to the session options. - * - * \param[out] out_ptr Set to a pointer to an array of null terminated strings of the available providers. The entries and the - * array itself must be freed using OrtApi::ReleaseAvailableProviders - * \param[out] provider_length Set to the number of entries in the `out_ptr` array - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetAvailableProviders, _Outptr_ char*** out_ptr, _Out_ int* provider_length); - - /** \brief Release data from OrtApi::GetAvailableProviders - * - * \param[in] ptr The `out_ptr` result from OrtApi::GetAvailableProviders. - * \param[in] providers_length The `provider_length` result from OrtApi::GetAvailableProviders - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(ReleaseAvailableProviders, _In_ char** ptr, - _In_ int providers_length); - - /// @} - /// \name OrtValue - /// @{ - - /** \brief Get the length of a single string in a string tensor - * - * \param[in] value A string tensor - * \param[in] index Index of the string in the tensor - * \param[out] out Set to number of bytes of the string element - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetStringTensorElementLength, _In_ const OrtValue* value, size_t index, _Out_ size_t* out); - - /** \brief Get a single string from a string tensor - * - * \param[in] value A string tensor - * \param[in] s_len Number of bytes in the `s` buffer. Must match the value returned by OrtApi::GetStringTensorElementLength. - * \param[in] index Index of the string in the tensor - * \param[out] s The string element contents in UTF-8 encoding. The string is NOT null-terminated. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetStringTensorElement, _In_ const OrtValue* value, size_t s_len, size_t index, _Out_writes_bytes_all_(s_len) void* s); - - /** \brief Set a single string in a string tensor - * - * \param[in] value A string tensor - * \param[in] s A null terminated UTF-8 encoded string - * \param[in] index Index of the string in the tensor to set - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(FillStringTensorElement, _Inout_ OrtValue* value, _In_ const char* s, size_t index); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Set a session configuration entry as a pair of strings - * - * If a configuration with same key exists, this will overwrite the configuration with the given config_value. - * - * The config_key and the format of config_value are defined in onnxruntime_session_options_config_keys.h - * - * \param[in] options - * \param[in] config_key A null terminated string representation of the config key - * \param[in] config_value A null terminated string representation of the config value - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(AddSessionConfigEntry, _Inout_ OrtSessionOptions* options, - _In_z_ const char* config_key, _In_z_ const char* config_value); - - /// @} - /// \name OrtAllocator - /// @{ - - /** \brief Create an allocator for an ::OrtSession following an ::OrtMemoryInfo - * - * \param[in] session - * \param[in] mem_info valid ::OrtMemoryInfo instance - * \param[out] out Newly created ::OrtAllocator. Must be freed with OrtApi::ReleaseAllocator - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateAllocator, _In_ const OrtSession* session, _In_ const OrtMemoryInfo* mem_info, - _Outptr_ OrtAllocator** out); - - /** \brief Release an ::OrtAllocator obtained from OrtApi::CreateAllocator - */ - ORT_CLASS_RELEASE(Allocator); - - /// @} - /// \name OrtSession - /// @{ - - /** \brief Run a model using Io Bindings for the inputs & outputs - * - * \see OrtApi::Run - * - * \param[in] session - * \param[in] run_options - * \param[in] binding_ptr - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(RunWithBinding, _Inout_ OrtSession* session, _In_ const OrtRunOptions* run_options, _In_ const OrtIoBinding* binding_ptr); - - /** \brief Create an ::OrtIoBinding instance - * - * An IoBinding object allows one to bind pre-allocated ::OrtValue%s to input names. - * Thus if you want to use a raw on device buffer as input or output you can avoid - * extra copy during runtime. - * - * \param[in] session - * \param[out] out Newly created ::OrtIoBinding. Must be freed with OrtApi::ReleaseIoBinding - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateIoBinding, _Inout_ OrtSession* session, _Outptr_ OrtIoBinding** out); - - /// @} - /// \name OrtIoBinding - /// @{ - - /** \brief Release an ::OrtIoBinding obtained from OrtApi::CreateIoBinding - */ - ORT_CLASS_RELEASE(IoBinding); - - /** \brief Bind an ::OrtValue to an ::OrtIoBinding input - * - * When using OrtApi::RunWithBinding this value is used for the named input - * - * \param[in] binding_ptr - * \param[in] name Name for the model input - * \param[in] val_ptr ::OrtValue of Tensor type. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(BindInput, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtValue* val_ptr); - - /** \brief Bind an ::OrtValue to an ::OrtIoBinding output - * - * When using OrtApi::RunWithBinding this value is used for the named output - * - * \param[in] binding_ptr - * \param[in] name Null terminated string of the model output name - * \param[in] val_ptr ::OrtValue of Tensor type. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(BindOutput, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtValue* val_ptr); - - /** \brief Bind an ::OrtIoBinding output to a device - * - * Binds the ::OrtValue to a device which is specified by ::OrtMemoryInfo. - * You can either create an instance of ::OrtMemoryInfo with a device id or obtain one from the allocator that you have created/are using - * This is useful when one or more outputs have dynamic shapes and, it is hard to pre-allocate and bind a chunk of - * memory within ::OrtValue ahead of time. - * - * \see OrtApi::RunWithBinding - * - * \param[in] binding_ptr - * \param[in] name Null terminated string of the device name - * \param[in] mem_info_ptr - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(BindOutputToDevice, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtMemoryInfo* mem_info_ptr); - - /** \brief Get the names of an ::OrtIoBinding's outputs - * - * Returns the names of the outputs in the order they were bound. This is useful after running the model - * with bound outputs because the returned names are in order in which output ::OrtValue are returned. This is useful if - * the order of outputs and their names is not known. - * - * \param[in] binding_ptr - * \param[in] allocator Allocator used to allocate continuous buffers for output strings and lengths. - * \param[out] buffer Returns an array of non-null terminated UTF-8 strings. The number of strings stored is returned in the count parameter. - * This buffer is allocated using `allocator` and must be freed using it. - * \param[out] lengths Returns an array of `count` lengths of the strings returned in `buffer` - * This buffer is allocated using `allocator` and must be freed using it. - * \param[out] count Number of strings returned. If `binding_ptr` has no bound outputs, zero is returned, - * no memory allocation is performed and buffer and lengths are set to nullptr. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetBoundOutputNames, _In_ const OrtIoBinding* binding_ptr, _In_ OrtAllocator* allocator, - _Out_ char** buffer, _Out_writes_all_(count) size_t** lengths, _Out_ size_t* count); - - /** \brief Get the output ::OrtValue objects from an ::OrtIoBinding - * - * Returns an array of pointers to individually allocated ::OrtValue%s that contain results of a model execution with OrtApi::RunWithBinding - * The array contains the same number of ::OrtValue%s and they are in the same order as they were bound with OrtApi::BindOutput - * or OrtApi::BindOutputToDevice. - * - * The returned ::OrtValue%s must be released using OrtApi::ReleaseValue after they are no longer needed. - * The array is allocated using the specified instance of the allocator and must be freed using the same allocator after - * all the ::OrtValue%s contained therein are individually released. - * - * \param[in] binding_ptr - * \param[in] allocator Allocator used to allocate output array - * \param[out] output Set to the allocated array of allocated ::OrtValue outputs. Set to nullptr if there are 0 outputs. - * \param[out] output_count Set to number of ::OrtValue%s returned - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetBoundOutputValues, _In_ const OrtIoBinding* binding_ptr, _In_ OrtAllocator* allocator, - _Out_writes_all_(output_count) OrtValue*** output, _Out_ size_t* output_count); - - /** \brief Clears any previously set Inputs for an ::OrtIoBinding - */ - void(ORT_API_CALL* ClearBoundInputs)(_Inout_ OrtIoBinding* binding_ptr) NO_EXCEPTION ORT_ALL_ARGS_NONNULL; - - /** \brief Clears any previously set Outputs for an ::OrtIoBinding - */ - void(ORT_API_CALL* ClearBoundOutputs)(_Inout_ OrtIoBinding* binding_ptr) NO_EXCEPTION ORT_ALL_ARGS_NONNULL; - - /// @} - /// \name OrtValue - /// @{ - - /** \brief Direct memory access to a specified tensor element - * - * For example, given a tensor with shape of [3,224,224], a pointer to the element at location [2,150,128] can be retrieved - * - * This function only works for numeric type tensors (No strings, etc). - * This is a no-copy method whose returned pointer is valid until the passed in ::OrtValue is free'd. - * - * \param[in] value - * \param[in] location_values Pointer to an array of index values that specify an element's location relative to its shape - * \param[in] location_values_count Number of elements in location_values. Must match the number of elements in the tensor's shape. - * \param[out] out Set to a pointer to the element specified - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(TensorAt, _Inout_ OrtValue* value, const int64_t* location_values, size_t location_values_count, _Outptr_ void** out); - - /// @} - /// \name OrtEnv - /// @{ - - /** \brief Create an allocator and register it with the ::OrtEnv - * - * Enables sharing the allocator between multiple sessions that use the same env instance. - * Lifetime of the created allocator will be valid for the duration of the environment. - * Returns an error if an allocator with the same ::OrtMemoryInfo is already registered. - * - * See https://onnxruntime.ai/docs/reference/api/c-api.html for details. - * - * \param[in] env ::OrtEnv instance - * \param[in] mem_info - * \param[in] arena_cfg Pass nullptr for defaults - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateAndRegisterAllocator, _Inout_ OrtEnv* env, _In_ const OrtMemoryInfo* mem_info, - _In_ const OrtArenaCfg* arena_cfg); - - /** \brief Set language projection - * - * Set the language projection for collecting telemetry data when Env is created. - * - * The default is ORT_PROJECTION_C, which means it will classify the language not in the list to C also. - * - * \param[in] ort_env - * \param[in] projection - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetLanguageProjection, _In_ const OrtEnv* ort_env, _In_ OrtLanguageProjection projection); - - /// @} - /// \name OrtSession - /// @{ - - /** \brief Return the time that profiling was started - * - * \note The timer precision varies per platform. On Windows and MacOS, the precision will be ~100ns - * - * \param[in] session - * \param[out] out nanoseconds of profiling's start time - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionGetProfilingStartTimeNs, _In_ const OrtSession* session, _Outptr_ uint64_t* out); - - /// @} - /// \name OrtThreadingOptions - /// @{ - - /** \brief Set global intra-op thread count - * - * This configures the global thread pool options to be used in the call to OrtApi::CreateEnvWithGlobalThreadPools - * - * \param[in] tp_options - * \param[in] intra_op_num_threads Number of threads, special values:
- * 0 = Use default thread count
- * 1 = The invoking thread will be used; no threads will be created in the thread pool. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetGlobalIntraOpNumThreads, _Inout_ OrtThreadingOptions* tp_options, int intra_op_num_threads); - - /** \brief Set global inter-op thread count - * - * This configures the global thread pool options to be used in the call to OrtApi::CreateEnvWithGlobalThreadPools - * - * \param[in] tp_options - * \param[in] inter_op_num_threads Number of threads, special values:
- * 0 = Use default thread count
- * 1 = The invoking thread will be used; no threads will be created in the thread pool. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetGlobalInterOpNumThreads, _Inout_ OrtThreadingOptions* tp_options, int inter_op_num_threads); - - /** \brief Set global spin control options - * - * This will configure the global thread pool options to be used in the call to OrtApi::CreateEnvWithGlobalThreadPools. - * Allow spinning of thread pools when their queues are empty. This will set the value for both - * inter_op and intra_op threadpools. - * - * \param[in] tp_options - * \param[in] allow_spinning Valid values are 0 or 1.
- * 0 = It won't spin (recommended if CPU usage is high)
- * 1 = Threadpool will spin to wait for queue to become non-empty - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetGlobalSpinControl, _Inout_ OrtThreadingOptions* tp_options, int allow_spinning); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Add a pre-allocated initializer to a session - * - * If a model contains an initializer with a name that is same as the name passed to this call, - * ORT will use this initializer instance instead of deserializing one from the model file. This - * is useful when you want to share the same initializer across sessions. - * - * \param[in] options - * \param[in] name Null terminated string of the initializer name - * \param[in] val ::OrtValue containing the initializer. Its lifetime and the underlying initializer buffer must be - * managed by the user (created using the OrtApi::CreateTensorWithDataAsOrtValue) and it must outlive the session object - * to which it is added. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(AddInitializer, _Inout_ OrtSessionOptions* options, _In_z_ const char* name, - _In_ const OrtValue* val); - - /// @} - /// \name OrtEnv - /// @{ - - /** - * Create a custom environment with global threadpools and logger that will be shared across sessions. - * Use this in conjunction with OrtApi::DisablePerSessionThreads or else the session will use - * its own thread pools. - * - * \param[in] logging_function A pointer to a logging function. - * \param[in] logger_param A pointer to arbitrary data passed as the ::OrtLoggingFunction `param` parameter to - * `logging_function`. - * \param[in] log_severity_level The log severity level. - * \param[in] logid The log identifier. - * \param[in] tp_options - * \param[out] out Newly created OrtEnv. Must be freed with OrtApi::ReleaseEnv - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateEnvWithCustomLoggerAndGlobalThreadPools, OrtLoggingFunction logging_function, _In_opt_ void* logger_param, OrtLoggingLevel log_severity_level, - _In_ const char* logid, _In_ const struct OrtThreadingOptions* tp_options, _Outptr_ OrtEnv** out); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Append CUDA provider to session options - * - * If CUDA is not available (due to a non CUDA enabled build, or if CUDA is not installed on the system), this function will return failure. - * - * \param[in] options - * \param[in] cuda_options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_CUDA, - _In_ OrtSessionOptions* options, _In_ const OrtCUDAProviderOptions* cuda_options); - - /** \brief Append ROCM execution provider to the session options - * - * If ROCM is not available (due to a non ROCM enabled build, or if ROCM is not installed on the system), this function will return failure. - * - * \param[in] options - * \param[in] rocm_options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_ROCM, - _In_ OrtSessionOptions* options, _In_ const OrtROCMProviderOptions* rocm_options); - - /** \brief Append OpenVINO execution provider to the session options - * - * If OpenVINO is not available (due to a non OpenVINO enabled build, or if OpenVINO is not installed on the system), this function will fail. - * - * \param[in] options - * \param[in] provider_options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_OpenVINO, - _In_ OrtSessionOptions* options, _In_ const OrtOpenVINOProviderOptions* provider_options); - - /// @} - /// \name OrtThreadingOptions - /// @{ - - /** \brief Set threading flush-to-zero and denormal-as-zero - * - * Sets global thread pool options to be used in the call to OrtApi::CreateEnvWithGlobalThreadPools. - * Flush-to-zero and denormal-as-zero are applied to threads in both intra and inter global thread pool. - * \note This option is not needed if the models used have no denormals. Having no denormals is recommended as this option may hurt model accuracy. - * - * \param[in] tp_options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetGlobalDenormalAsZero, _Inout_ OrtThreadingOptions* tp_options); - - /// @} - /// \name OrtArenaCfg - /// @{ - - /** \deprecated Use OrtApi::CreateArenaCfgV2 - * - * This will create the configuration of an arena that can eventually be used to define an arena based allocator's behavior - * - * \param[in] max_mem Use 0 to allow ORT to choose the default - * \param[in] arena_extend_strategy Use -1 to allow ORT to choose the default, 0 = kNextPowerOfTwo, 1 = kSameAsRequested - * \param[in] initial_chunk_size_bytes Use -1 to allow ORT to choose the default - * \param[in] max_dead_bytes_per_chunk Use -1 to allow ORT to choose the default - * \param[in] out A pointer to an OrtArenaCfg instance - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateArenaCfg, _In_ size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, - int max_dead_bytes_per_chunk, _Outptr_ OrtArenaCfg** out); - - ORT_CLASS_RELEASE(ArenaCfg); - - /// @} - /// \name OrtModelMetadata - /// @{ - - /** - * Use this to obtain the description of the graph present in the model - * (doc_string field of the GraphProto message within the ModelProto message). - * If it doesn't exist, an empty string will be returned. - * - * \param[in] model_metadata An instance of ::OrtModelMetadata - * \param[in] allocator Allocator used to allocate the string that will be returned back - * \param[out] value Set to a null terminated string allocated using `allocator`. The caller is responsible for freeing it using `allocator` - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(ModelMetadataGetGraphDescription, _In_ const OrtModelMetadata* model_metadata, - _Inout_ OrtAllocator* allocator, _Outptr_ char** value); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Append TensorRT provider to session options - * - * If TensorRT is not available (due to a non TensorRT enabled build, or if TensorRT is not installed on the system), this function will return failure. - * - * \param[in] options - * \param[in] tensorrt_options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_TensorRT, - _In_ OrtSessionOptions* options, _In_ const OrtTensorRTProviderOptions* tensorrt_options); - - /// @} - /// \name Misc - /// @{ - - /** \brief Set current GPU device ID - * - * Set the current device id of the GPU execution provider (CUDA/tensorrt/rocm). The device id should be less - * than the total number of devices available. This is only useful when multiple-GPUs are installed and it is - * required to restrict execution to a single GPU. - * - * \param[in] device_id - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetCurrentGpuDeviceId, _In_ int device_id); - - /** \brief Get current GPU device ID - * - * Get the current device id of the GPU execution provider (CUDA/tensorrt/rocm). - * - * \see OrtApi::SetCurrentGpuDeviceId - * - * \param[out] device_id - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetCurrentGpuDeviceId, _In_ int* device_id); - - /// @} - /// \name OrtKernelInfo - /// @{ - - /** \brief Fetch an array of int64_t values stored as an attribute in the graph node - * - * - * If `out` is nullptr, the value of `size` is set to the true size of the attribute - * array's size, and a success status is returned. - * - * If the `size` parameter is greater than or equal to the actual attribute array's size, - * the value of `size` is set to the true size of the attribute array's size, - * the provided memory is filled with the attribute's contents, - * and a success status is returned. - * - * If the `size` parameter is less than the actual attribute array's size and `out` - * is not nullptr, the value of `size` is set to the true size of the attribute array's size - * and a failure status is returned.) - * - * \param[in] info instance - * \param[in] name name of the attribute to be parsed - * \param[out] out pointer to memory where the attribute's contents are to be stored - * \param[in, out] size actual size of attribute array - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(KernelInfoGetAttributeArray_float, _In_ const OrtKernelInfo* info, _In_ const char* name, - _Out_ float* out, _Inout_ size_t* size); - - /** \brief Fetch an array of int64_t values stored as an attribute in the graph node - * - * If `out` is nullptr, the value of `size` is set to the true size of the attribute - * array's size, and a success status is returned. - * - * If the `size` parameter is greater than or equal to the actual attribute array's size, - * the value of `size` is set to the true size of the attribute array's size, - * the provided memory is filled with the attribute's contents, - * and a success status is returned. - * - * If the `size` parameter is less than the actual attribute array's size and `out` - * is not nullptr, the value of `size` is set to the true size of the attribute array's size - * and a failure status is returned.) - * - * \param[in] info instance - * \param[in] name name of the attribute to be parsed - * \param[out] out pointer to memory where the attribute's contents are to be stored - * \param[in, out] size actual size of attribute array - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(KernelInfoGetAttributeArray_int64, _In_ const OrtKernelInfo* info, _In_ const char* name, - _Out_ int64_t* out, _Inout_ size_t* size); - - /// @} - /// \name OrtArenaCfg - /// @{ - - /** \brief Create an ::OrtArenaCfg - * - * Create the configuration of an arena that can eventually be used to define an arena based allocator's behavior. - * - * Supported keys are (See https://onnxruntime.ai/docs/reference/api/c-api.html for details on what the - * following parameters mean and how to choose these values.): - * "max_mem": Maximum memory that can be allocated by the arena based allocator. - * Use 0 for ORT to pick the best value. Default is 0. - * "arena_extend_strategy": 0 = kNextPowerOfTwo, 1 = kSameAsRequested. - * Use -1 to allow ORT to choose the default. - * "initial_chunk_size_bytes": (Possible) Size of the first allocation in the arena. - * Only relevant if arena strategy is `kNextPowerOfTwo`. Use -1 to allow ORT to choose the default. - * Ultimately, the first allocation size is determined by the allocation memory request. - * "max_dead_bytes_per_chunk": Threshold of unused memory in an allocated chunk of arena memory after - * crossing which the current chunk is chunked into 2. - * "initial_growth_chunk_size_bytes": (Possible) Size of the second allocation in the arena. - * Only relevant if arena strategy is `kNextPowerOfTwo`. Use -1 to allow ORT to choose the default. - * Ultimately, the allocation size is determined by the allocation memory request. - * Further allocation sizes are governed by the arena extend strategy. - * - * \param[in] arena_config_keys Keys to configure the arena - * \param[in] arena_config_values Values to configure the arena - * \param[in] num_keys Number of keys in `arena_config_keys` and `arena_config_values` - * \param[out] out Newly created ::OrtArenaCfg. Must be freed with OrtApi::ReleaseArenaCfg - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateArenaCfgV2, _In_reads_(num_keys) const char* const* arena_config_keys, - _In_reads_(num_keys) const size_t* arena_config_values, _In_ size_t num_keys, - _Outptr_ OrtArenaCfg** out); - - /// @} - /// \name OrtRunOptions - /// @{ - - /** \brief Set a single run configuration entry as a pair of strings - * - * If a configuration with same key exists, this will overwrite the configuration with the given config_value - * - * The config_key and the format of config_value are defined in onnxruntime_run_options_config_keys.h - * - * \param[in] options - * \param[in] config_key A null terminated string representation of the config key - * \param[in] config_value A null terminated string representation of the config value - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(AddRunConfigEntry, _Inout_ OrtRunOptions* options, - _In_z_ const char* config_key, _In_z_ const char* config_value); - - /// @} - /// \name OrtPrepackedWeightsContainer - /// @{ - - /** \brief Create an ::OrtPrepackedWeightsContainer - * - * This container will hold pre-packed buffers of shared initializers for sharing between sessions - * (i.e.) if there are shared initializers that can be shared between sessions, the pre-packed buffers - * of these (if any) may possibly be shared to provide memory footprint savings. Pass this container - * to sessions that you would like to share pre-packed buffers of shared initializers at session - * creation time. - * - * \param[out] out Newly created ::OrtPrepackedWeightsContainer. Must be freed with OrtApi::ReleasePrepackedWeightsContainer - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreatePrepackedWeightsContainer, _Outptr_ OrtPrepackedWeightsContainer** out); - - /** \brief Release OrtPrepackedWeightsContainer instance - * - * \note instance must not be released until the sessions using it are released - */ - ORT_CLASS_RELEASE(PrepackedWeightsContainer); - - /// @} - /// \name OrtSession - /// @{ - - /** \brief Create session with prepacked weights container - * - * Same functionality offered by OrtApi::CreateSession except that a container that contains - * pre-packed weights' buffers is written into/read from by the created session. - * This is useful when used in conjunction with OrtApi::AddInitializer which injects - * shared initializer info into sessions. Wherever possible, the pre-packed versions of these - * shared initializers are cached in this container so that multiple sessions can just re-use - * these instead of duplicating these in memory. - * - * \param[in] env OrtEnv instance instance - * \param[in] model_path Null terminated string of the path (wchar on Windows, char otherwise) - * \param[in] options - * \param[in] prepacked_weights_container - * \param[out] out Newly created ::OrtSession. Must be freed with OrtApi::ReleaseSession - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateSessionWithPrepackedWeightsContainer, _In_ const OrtEnv* env, _In_ const ORTCHAR_T* model_path, - _In_ const OrtSessionOptions* options, _Inout_ OrtPrepackedWeightsContainer* prepacked_weights_container, - _Outptr_ OrtSession** out); - - /** \brief Create session from memory with prepacked weights container - * - * Same functionality offered by OrtApi::CreateSessionFromArray except that a container that contains - * pre-packed weights' buffers is written into/read from by the created session. - * This is useful when used in conjunction with OrtApi::AddInitializer which injects - * shared initializer info into sessions. Wherever possible, the pre-packed versions of these - * shared initializers are cached in this container so that multiple sessions can just re-use - * these instead of duplicating these in memory. - * - * \param[in] env - * \param[in] model_data Array of bytes holding the model - * \param[in] model_data_length Number of bytes in `model_data_model` - * \param[in] options - * \param[in] prepacked_weights_container - * \param[out] out Newly created ::OrtSession. Must be freed with OrtApi::ReleaseSession - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateSessionFromArrayWithPrepackedWeightsContainer, _In_ const OrtEnv* env, - _In_ const void* model_data, size_t model_data_length, - _In_ const OrtSessionOptions* options, _Inout_ OrtPrepackedWeightsContainer* prepacked_weights_container, - _Outptr_ OrtSession** out); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Append TensorRT execution provider to the session options - * - * If TensorRT is not available (due to a non TensorRT enabled build), this function will return failure. - * - * This is slightly different from OrtApi::SessionOptionsAppendExecutionProvider_TensorRT, it takes an - * ::OrtTensorRTProviderOptions which is publicly defined. This takes an opaque ::OrtTensorRTProviderOptionsV2 - * which must be created with OrtApi::CreateTensorRTProviderOptions. - * - * For OrtApi::SessionOptionsAppendExecutionProvider_TensorRT, the user needs to instantiate ::OrtTensorRTProviderOptions - * as well as allocate/release buffers for some members of ::OrtTensorRTProviderOptions. - * Here, OrtApi::CreateTensorRTProviderOptions and Ortapi::ReleaseTensorRTProviderOptions will do the memory management for you. - * - * \param[in] options - * \param[in] tensorrt_options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_TensorRT_V2, - _In_ OrtSessionOptions* options, _In_ const OrtTensorRTProviderOptionsV2* tensorrt_options); - - /// @} - /// \name OrtTensorRTProviderOptionsV2 - /// @{ - - /** \brief Create an OrtTensorRTProviderOptionsV2 - * - * \param[out] out Newly created ::OrtTensorRTProviderOptionsV2. Must be released with OrtApi::ReleaseTensorRTProviderOptions - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateTensorRTProviderOptions, _Outptr_ OrtTensorRTProviderOptionsV2** out); - - /** \brief Set options in a TensorRT Execution Provider. - * - * Please refer to https://www.onnxruntime.ai/docs/reference/execution-providers/TensorRT-ExecutionProvider.html#c-api-example - * to know the available keys and values. Key should be in null terminated string format of the member of ::OrtTensorRTProviderOptionsV2 - * and value should be its related range. - * - * For example, key="trt_max_workspace_size" and value="2147483648" - * - * \param[in] tensorrt_options - * \param[in] provider_options_keys Array of UTF-8 null-terminated string for provider options keys - * \param[in] provider_options_values Array of UTF-8 null-terminated string for provider options values - * \param[in] num_keys Number of elements in the `provider_option_keys` and `provider_options_values` arrays - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(UpdateTensorRTProviderOptions, _Inout_ OrtTensorRTProviderOptionsV2* tensorrt_options, - _In_reads_(num_keys) const char* const* provider_options_keys, - _In_reads_(num_keys) const char* const* provider_options_values, - _In_ size_t num_keys); - - /** \brief Get serialized TensorRT provider options string. - * - * For example, "trt_max_workspace_size=2147483648;trt_max_partition_iterations=10;trt_int8_enable=1;......" - * - * \param tensorrt_options - OrTensorRTProviderOptionsV2 instance - * \param allocator - a ptr to an instance of OrtAllocator obtained with OrtApi::CreateAllocator or OrtApi::GetAllocatorWithDefaultOptions - * the specified allocator will be used to allocate continuous buffers for output strings and lengths. - * \param ptr - is a UTF-8 null terminated string allocated using 'allocator'. The caller is responsible for using the same allocator to free it. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetTensorRTProviderOptionsAsString, _In_ const OrtTensorRTProviderOptionsV2* tensorrt_options, _Inout_ OrtAllocator* allocator, _Outptr_ char** ptr); - - /** \brief Release an ::OrtTensorRTProviderOptionsV2 - * - * \note This is an exception in the naming convention of other Release* functions, as the name of the method does not have the V2 suffix, but the type does - */ - void(ORT_API_CALL* ReleaseTensorRTProviderOptions)(_Frees_ptr_opt_ OrtTensorRTProviderOptionsV2* input); - - /// @} - /// \name OrtSessionOptions - /// @{ - - /** \brief Enable custom operators - * - * See onnxruntime-extensions: https://github.com/microsoft/onnxruntime-extensions.git - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(EnableOrtCustomOps, _Inout_ OrtSessionOptions* options); - - /// @} - /// \name OrtAllocator - /// @{ - - /** \brief Register a custom allocator - * - * Enables sharing between multiple sessions that use the same env instance. - * Returns an error if an allocator with the same ::OrtMemoryInfo is already registered. - * - * The behavior of this is exactly the same as OrtApi::CreateAndRegisterAllocator except - * instead of ORT creating an allocator based on provided info, in this case - * ORT uses the user-provided custom allocator. - * See https://onnxruntime.ai/docs/reference/api/c-api.html for details. - * - * \param[in] env - * \param[in] allocator User provided allocator - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(RegisterAllocator, _Inout_ OrtEnv* env, _In_ OrtAllocator* allocator); - - /** \brief Unregister a custom allocator - * - * It is an error if you provide an ::OrtMemoryInfo not corresponding to any - * registered allocators for sharing. - * - * \param[in] env - * \param[in] mem_info - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(UnregisterAllocator, _Inout_ OrtEnv* env, - _In_ const OrtMemoryInfo* mem_info); - - /// @} - /// \name OrtValue - /// @{ - - /** \brief Sets *out to 1 iff an ::OrtValue is a SparseTensor, and 0 otherwise - * - * \param[in] value existing ::OrtValue - * \param[out] out unless an error occurs, contains 1 iff the value contains an instance - * of sparse tensor or 0 otherwise. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(IsSparseTensor, _In_ const OrtValue* value, _Out_ int* out); - - /** \brief Create an ::OrtValue with a sparse tensor that is empty. - * - * Use FillSparseTensor() functions to populate sparse tensor with non-zero values and - * format specific indices data. - * Use ReleaseValue to destroy the sparse tensor, this will also release the buffer inside the output value - * if any was allocated. - * \param[in,out] allocator allocator to use when performing an allocation. Allocation will be performed - * by FillSparseTensor() APIs. The lifespan of the allocator instance must eclipse the lifespan - * this sparse tensor instance as the same allocator will be used to free memory. - * \param[in] dense_shape shape of the original dense tensor - * \param[in] dense_shape_len number of shape dimensions being passed - * \param[in] type must be one of TENSOR_ELEMENT_DATA_TYPE_xxxx - * \param[out] out Should be freed by calling ReleaseValue - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateSparseTensorAsOrtValue, _Inout_ OrtAllocator* allocator, _In_ const int64_t* dense_shape, - size_t dense_shape_len, ONNXTensorElementDataType type, _Outptr_ OrtValue** out); - - /** - * This fills populates an empty tensor that was created using OrtApi::CreateSparseTensorAsOrtValue. - * This will allocate required memory and copy the supplied NNZ values and COO indices into that memory allocation. - * Memory allocation is performed using the allocator that was specified with OrtApi::CreateSparseTensorAsOrtValue. - * - * \param[in,out] ort_value ::OrtValue to populate with data - * \param[in] data_mem_info serves to identify the location of the data to be copied. If the allocator specified - * at the creation time has memory info that is not the same as mem_info argument to this function a X-device copy will be performed. - * String data is assumed to be on CPU and will only be copied into a CPU allocated buffer. - * \param[in] values_shape pointer to values shape array - * \param[in] values_shape_len length of the values_shape - * \param[in] values pointer to an array of values. For strings, pass const char**. - * \param[in] indices_data pointer to a location of COO indices - * \param[in] indices_num number of COO indices - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(FillSparseTensorCoo, _Inout_ OrtValue* ort_value, _In_ const OrtMemoryInfo* data_mem_info, - _In_ const int64_t* values_shape, size_t values_shape_len, _In_ const void* values, - _In_ const int64_t* indices_data, size_t indices_num); - - /** - * This fills populates an empty tensor that was created using OrtApi::CreateSparseTensorAsOrtValue. - * This will allocate required memory and copy the supplied NNZ values and CSR indices into that memory allocation. - * Memory allocation is performed using the allocator that was specified with OrtApi::CreateSparseTensorAsOrtValue. - * - * \param[in,out] ort_value ::OrtValue to populate with data - * \param[in] data_mem_info serves to identify the location of the data to be copied. If the allocator specified - * at the creation time has memory info that is not the same as mem_info argument to this function a X-device copy will be performed. - * String data is assumed to be on CPU and will only be copied into a CPU allocated buffer. - * \param[in] values_shape pointer to values shape array - * \param[in] values_shape_len length of the values_shape - * \param[in] values - pointer to an array of values. For strings, pass const char**. - * \param[in] inner_indices_data pointer to a location of CSR inner indices - * \param[in] inner_indices_num number of CSR inner indices - * \param[in] outer_indices_data pointer to a location of CSR outer indices - * \param[in] outer_indices_num number of CSR outer indices - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(FillSparseTensorCsr, _Inout_ OrtValue* ort_value, _In_ const OrtMemoryInfo* data_mem_info, - _In_ const int64_t* values_shape, size_t values_shape_len, _In_ const void* values, - _In_ const int64_t* inner_indices_data, size_t inner_indices_num, - _In_ const int64_t* outer_indices_data, size_t outer_indices_num); - - /** - * This fills populates an empty tensor that was created using OrtApi::CreateSparseTensorAsOrtValue. - * This will allocate required memory and copy the supplied NNZ values and BlockSparse indices into that memory allocation. - * Memory allocation is performed using the allocator that was specified with OrtApi::CreateSparseTensorAsOrtValue. - * - * \param[in,out] ort_value ::OrtValue to populate with data - * \param[in] data_mem_info serves to identify the location of the data to be copied. If the allocator specified - * at the creation time has memory info that is not the same as mem_info argument to this function a X-device copy will be performed. - * String data is assumed to be on CPU and will only be copied into a CPU allocated buffer. - * \param[in] values_shape - * \param[in] values_shape_len - * \param[in] values structure with values information - * \param[in] indices_shape_data pointer to a location of indices shape - * \param[in] indices_shape_len length of the block sparse indices shape - * \param[in] indices_data pointer to a location of indices data. Shape will determine the length of the indices data. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(FillSparseTensorBlockSparse, _Inout_ OrtValue* ort_value, _In_ const OrtMemoryInfo* data_mem_info, - _In_ const int64_t* values_shape, size_t values_shape_len, _In_ const void* values, - _In_ const int64_t* indices_shape_data, size_t indices_shape_len, - _In_ const int32_t* indices_data); - - /** - * Create an ::OrtValue with a sparse tensor. This is the first step. - * Next, use UseIndices() functions to supply sparse tensor with - * format specific indices data and set its sparse format to a specific enum value. - * This will not perform memory allocations. It will - * use supplied user buffer which should outlive the created sparse tensor. - * Use OrtApi::ReleaseValue to destroy the sparse tensor. It would not release the supplied values buffer. - * This function can not be used to map strings from the user allocated memory. Strings must always be copied - * and have UTF-8 encoding. Therefore, use OrtApi::CreateSparseTensorAsOrtValue above and then fill it with data - * using appropriate Make*() function. - * - * \param[in] info memory info where sparse values reside. - * \param[in,out] p_data pointer to a user allocated buffer with values. To create a full sparse tensor with no non-zero - * values, pass nullptr - * \param[in] dense_shape shape of the original dense tensor - * \param[in] dense_shape_len number of shape dimensions being passed - * \param[in] values_shape shape of the values data. To create a fully sparse tensor with no non-zero values, - * pass {0} shape. - * \param[in] values_shape_len number of values shape dimensions - * \param[in] type must be one of TENSOR_ELEMENT_DATA_TYPE_xxxx - * \param[out] out Should be freed by calling ReleaseValue - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(CreateSparseTensorWithValuesAsOrtValue, _In_ const OrtMemoryInfo* info, _Inout_ void* p_data, - _In_ const int64_t* dense_shape, size_t dense_shape_len, - _In_ const int64_t* values_shape, size_t values_shape_len, - ONNXTensorElementDataType type, _Outptr_ OrtValue** out); - - /** - * This assigns Coo format indices to the SparseTensor that was created by - * OrtApi::CreateSparseTensorWithValuesAsOrtValue above. It also sets OrtSparseFormat to - * ORT_SPARSE_COO. This will not allocate any additional memory for data. The life span of - * indices_data buffer should eclipse the life span of this ::OrtValue. - * - * \param[in,out] ort_value ::OrtValue instance constructed with OrtApi::CreateSparseTensorWithValuesAsOrtValue - * \param[in,out] indices_data pointer to a user pre-allocated buffer or nullptr for fully sparse tensors. - * \param[in] indices_num number of COO indices. Should either be 0 for fully sparse tensors, be equal - * to the number of nnz values specified to OrtApi::CreateSparseTensorWithValuesAsOrtValue for 1-D {nnz} indices or - * be twice as number of nnz values for a 2-D indices {nnz, 2} - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(UseCooIndices, _Inout_ OrtValue* ort_value, _Inout_ int64_t* indices_data, size_t indices_num); - - /** - * The assigns CSR format indices to the SparseTensor that was created by - * OrtApi::CreateSparseTensorWithValuesAsOrtValue above. It also sets OrtSparseFormat to - * ORT_SPARSE_CSRC. This will not allocate any additional memory for data. The life spans of - * inner_data and outer_data buffers should eclipse the life span of this ::OrtValue. - * - * \param[in,out] ort_value ::OrtValue instance constructed with OrtApi::CreateSparseTensorWithValuesAsOrtValue - * \param[in,out] inner_data pointer to a user pre-allocated buffer or nullptr for fully sparse tensors. - * \param[in] inner_num number of inner CSR indices. Should either be 0 for fully sparse tensors or be equal - * to the number of nnz values specified to OrtApi::CreateSparseTensorWithValuesAsOrtValue. - * \param[in,out] outer_data pointer to user pre-allocated buffer or nullptr for fully sparse tensors. - * \param[in] outer_num number of CSR outer indices. Should either be 0 for fully sparse tensors or - * equal to rows + 1 of the dense shape. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(UseCsrIndices, _Inout_ OrtValue* ort_value, _Inout_ int64_t* inner_data, size_t inner_num, - _Inout_ int64_t* outer_data, size_t outer_num); - - /** - * The assigns BlockSparse format indices to the SparseTensor that was created by - * OrtApi::CreateSparseTensorWithValuesAsOrtValue above. It also sets OrtSparseFormat to - * ORT_SPARSE_BLOCK_SPARSE. This will not allocate any additional memory for data. The life span of - * indices_data buffer must eclipse the lifespan of this ::OrtValue. - * - * \param[in,out] ort_value OrtValue instance constructed with OrtApi::CreateSparseTensorWithValuesAsOrtValue - * \param[in] indices_shape pointer to indices shape. Use {0} for fully sparse tensors - * \param[in] indices_shape_len length of the indices shape - * \param[in,out] indices_data pointer to user pre-allocated buffer or nullptr for fully sparse tensors. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(UseBlockSparseIndices, _Inout_ OrtValue* ort_value, const int64_t* indices_shape, size_t indices_shape_len, _Inout_ int32_t* indices_data); - - /** \brief Returns sparse tensor format enum iff a given ort value contains an instance of sparse tensor. - * - * \param[in] ort_value ::OrtValue that contains an instance of sparse tensor - * \param[out] out pointer to out parameter - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetSparseTensorFormat, _In_ const OrtValue* ort_value, _Out_ enum OrtSparseFormat* out); - - /** \brief Returns data type and shape of sparse tensor values (nnz) iff ::OrtValue contains a SparseTensor. - * - * \param[in] ort_value An ::OrtValue that contains a fully constructed sparse tensor - * \param[out] out Must be freed by OrtApi::ReleaseTensorTypeAndShapeInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetSparseTensorValuesTypeAndShape, _In_ const OrtValue* ort_value, _Outptr_ OrtTensorTypeAndShapeInfo** out); - - /** \brief Returns numeric data for sparse tensor values (nnz). For string values use GetStringTensor*(). - * - * \param[in] ort_value an instance of ::OrtValue containing sparse tensor - * \param[out] out returns a pointer to values data. Do not attempt to free this ptr. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetSparseTensorValues, _In_ const OrtValue* ort_value, _Outptr_ const void** out); - - /** \brief Returns data type, shape for the type of indices specified by indices_format. - * - * \param[in] ort_value ::OrtValue containing sparse tensor. - * \param[in] indices_format One of the indices formats. It is an error to request a format that the sparse - * tensor does not contain. - * \param[out] out an instance of ::OrtTensorTypeAndShapeInfo. Must be freed by OrtApi::ReleaseTensorTypeAndShapeInfo - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetSparseTensorIndicesTypeShape, _In_ const OrtValue* ort_value, enum OrtSparseIndicesFormat indices_format, _Outptr_ OrtTensorTypeAndShapeInfo** out); - - /** \brief Returns indices data for the type of the indices specified by indices_format - * - * \param[in] ort_value ::OrtValue containing sparse tensor. - * \param[in] indices_format One of the indices formats. It is an error to request a format that the sparse tensor does not contain. - * \param[out] num_indices Pointer to where the number of indices entries is returned - * \param[out] indices Returned pointer to the indices data. Do not free the returned pointer as it refers to internal data owned by the ::OrtValue - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetSparseTensorIndices, _In_ const OrtValue* ort_value, enum OrtSparseIndicesFormat indices_format, _Out_ size_t* num_indices, _Outptr_ const void** indices); - /// @} - /// \name OrtSessionOptions - /// @{ - - /** - * \brief Sets out to 1 iff an optional type OrtValue has an element, 0 otherwise (OrtValue is None) - * Use this API to find if the optional type OrtValue is None or not. - * If the optional type OrtValue is not None, use the OrtValue just like any other OrtValue. - * For example, if you get an OrtValue that corresponds to Optional(tensor) and - * if HasValue() returns true, use it as tensor and so on. - - * \param[in] value Input OrtValue. - * \param[out] out indicating if the input OrtValue contains data (1) or if it is a None (0) - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(HasValue, _In_ const OrtValue* value, _Out_ int* out); - /// @} - /// \name OrtKernelContext - /// @{ - /** \brief Used for custom operators, gets the GPU compute stream to use to launch the custom a GPU kernel - * \see ::OrtCustomOp - * \param[in] context OrtKernelContext instance - * \param[out] out Returns pointer to a GPU compute stream that can be used to launch the custom GPU kernel. - * If retrieving the GPU compute stream is not relevant (GPU not enabled in the build, kernel partitioned to - * some other EP), then a nullptr is returned as the output param. - * Do not free or mutate the returned pointer as it refers to internal data owned by the underlying session. - * Only use it for custom kernel launching. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(KernelContext_GetGPUComputeStream, _In_ const OrtKernelContext* context, _Outptr_ void** out); - - /// @} - /// \name GetTensorMemoryInfo - /// @{ - /** \brief Returns a pointer to the ::OrtMemoryInfo of a Tensor - * \param[in] value ::OrtValue containing tensor. - * \param[out] mem_info ::OrtMemoryInfo of the tensor. Do NOT free the returned pointer. It is valid for the lifetime of the ::OrtValue - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetTensorMemoryInfo, _In_ const OrtValue* value, _Out_ const OrtMemoryInfo** mem_info); - - /// @} - /// \name GetExecutionProviderApi - /// @{ - /** \brief Get a pointer to the requested version of the Execution Provider specific - * API extensions to the OrtApi - * \param[in] provider_name The name of the execution provider name. Currently only the following - * values are supported: "DML". - * \param[in] version Must be ::ORT_API_VERSION. - * \param[out] provider_api A void pointer containing a reference to the execution provider versioned api structure. - * For example, the provider_api pointer can be cast to the OrtDmlApi* when the provider_name is "DML". - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(GetExecutionProviderApi, _In_ const char* provider_name, _In_ uint32_t version, _Outptr_ const void** provider_api); - - /// @} - - /// \name SessionOptions - /// @{ - /** \brief Set custom thread creation function - * - * \param[in] options Session options - * \param[in] ort_custom_create_thread_fn Custom thread creation function - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionOptionsSetCustomCreateThreadFn, _Inout_ OrtSessionOptions* options, _In_ OrtCustomCreateThreadFn ort_custom_create_thread_fn); - - /** \brief Set creation options for custom thread - * - * \param[in] options Session options - * \param[in] ort_custom_thread_creation_options Custom thread creation options (can be nullptr) - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionOptionsSetCustomThreadCreationOptions, _Inout_ OrtSessionOptions* options, _In_ void* ort_custom_thread_creation_options); - - /** \brief Set custom thread join function - * - * \param[in] options Session options - * \param[in] ort_custom_join_thread_fn Custom join thread function, must not be nullptr when ort_custom_create_thread_fn is set - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SessionOptionsSetCustomJoinThreadFn, _Inout_ OrtSessionOptions* options, _In_ OrtCustomJoinThreadFn ort_custom_join_thread_fn); - /// @} - - /// \name OrtThreadingOptions - /// @{ - /** \brief Set custom thread creation function for global thread pools - * - * \param[inout] tp_options - * \param[in] ort_custom_create_thread_fn Custom thread creation function - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetGlobalCustomCreateThreadFn, _Inout_ OrtThreadingOptions* tp_options, _In_ OrtCustomCreateThreadFn ort_custom_create_thread_fn); - - /** \brief Set custom thread creation options for global thread pools - * - * \param[inout] tp_options - * \param[in] ort_custom_thread_creation_options Custom thread creation options (can be nullptr) - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetGlobalCustomThreadCreationOptions, _Inout_ OrtThreadingOptions* tp_options, _In_ void* ort_custom_thread_creation_options); - - /** \brief Set custom thread join function for global thread pools - * - * \param[inout] tp_options - * \param[in] ort_custom_join_thread_fn Custom thread join function, must not be nullptr when global ort_custom_create_thread_fn is set - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SetGlobalCustomJoinThreadFn, _Inout_ OrtThreadingOptions* tp_options, _In_ OrtCustomJoinThreadFn ort_custom_join_thread_fn); - /// @} - - /** \brief Synchronize bound inputs. The call may be necessary for some providers, such as cuda, - * in case the system that allocated bound memory operated on a different stream. However, the - * operation is provider specific and could be a no-op. - * - * \param[inout] binding_ptr - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SynchronizeBoundInputs, _Inout_ OrtIoBinding* binding_ptr); - - /** \brief Synchronize bound outputs. The call may be necessary for some providers, such as cuda, - * in case the system that allocated bound memory operated on a different stream. However, the - * operation is provider specific and could be a no-op. - * - * \param[inout] binding_ptr - * - * \snippet{doc} snippets.dox OrtStatus Return Value - */ - ORT_API2_STATUS(SynchronizeBoundOutputs, _Inout_ OrtIoBinding* binding_ptr); - - /// \name OrtSessionOptions - /// @{ - - /** \brief Append CUDA execution provider to the session options - * - * If CUDA is not available (due to a non CUDA enabled build), this function will return failure. - * - * This is slightly different from OrtApi::SessionOptionsAppendExecutionProvider_CUDA, it takes an - * ::OrtCUDAProviderOptions which is publicly defined. This takes an opaque ::OrtCUDAProviderOptionsV2 - * which must be created with OrtApi::CreateCUDAProviderOptions. - * - * For OrtApi::SessionOptionsAppendExecutionProvider_CUDA, the user needs to instantiate ::OrtCUDAProviderOptions - * as well as allocate/release buffers for some members of ::OrtCUDAProviderOptions. - * Here, OrtApi::CreateCUDAProviderOptions and Ortapi::ReleaseCUDAProviderOptions will do the memory management for you. - * - * \param[in] options - * \param[in] cuda_options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.11. - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_CUDA_V2, - _In_ OrtSessionOptions* options, _In_ const OrtCUDAProviderOptionsV2* cuda_options); - - /// @} - /// \name OrtCUDAProviderOptionsV2 - /// @{ - - /** \brief Create an OrtCUDAProviderOptionsV2 - * - * \param[out] out Newly created ::OrtCUDAProviderOptionsV2. Must be released with OrtApi::ReleaseCudaProviderOptions - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.11. - */ - ORT_API2_STATUS(CreateCUDAProviderOptions, _Outptr_ OrtCUDAProviderOptionsV2** out); - - /** \brief Set options in a CUDA Execution Provider. - * - * Please refer to https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html#configuration-options - * to know the available keys and values. Key should be in null terminated string format of the member of ::OrtCUDAProviderOptionsV2 - * and value should be its related range. - * - * For example, key="device_id" and value="0" - * - * \param[in] cuda_options - * \param[in] provider_options_keys Array of UTF-8 null-terminated string for provider options keys - * \param[in] provider_options_values Array of UTF-8 null-terminated string for provider options values - * \param[in] num_keys Number of elements in the `provider_option_keys` and `provider_options_values` arrays - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.11. - */ - ORT_API2_STATUS(UpdateCUDAProviderOptions, _Inout_ OrtCUDAProviderOptionsV2* cuda_options, - _In_reads_(num_keys) const char* const* provider_options_keys, - _In_reads_(num_keys) const char* const* provider_options_values, - _In_ size_t num_keys); - - /** - * Get serialized CUDA provider options string. - * - * For example, "device_id=0;arena_extend_strategy=0;......" - * - * \param cuda_options - OrtCUDAProviderOptionsV2 instance - * \param allocator - a ptr to an instance of OrtAllocator obtained with CreateAllocator() or GetAllocatorWithDefaultOptions() - * the specified allocator will be used to allocate continuous buffers for output strings and lengths. - * \param ptr - is a UTF-8 null terminated string allocated using 'allocator'. The caller is responsible for using the same allocator to free it. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.11. - */ - ORT_API2_STATUS(GetCUDAProviderOptionsAsString, _In_ const OrtCUDAProviderOptionsV2* cuda_options, _Inout_ OrtAllocator* allocator, _Outptr_ char** ptr); - - /** \brief Release an ::OrtCUDAProviderOptionsV2 - * - * \note This is an exception in the naming convention of other Release* functions, as the name of the method does not have the V2 suffix, but the type does - * - * \since Version 1.11. - */ - void(ORT_API_CALL* ReleaseCUDAProviderOptions)(_Frees_ptr_opt_ OrtCUDAProviderOptionsV2* input); - - /// @} - - /** \brief Append MIGraphX provider to session options - * - * If MIGraphX is not available (due to a non MIGraphX enabled build, or if MIGraphX is not installed on the system), this function will return failure. - * - * \param[in] options - * \param[in] migraphx_options - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.11. - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_MIGraphX, - _In_ OrtSessionOptions* options, _In_ const OrtMIGraphXProviderOptions* migraphx_options); - - /** \brief Replace initialized Tensors with external data with the data provided in initializers. - * - * The function will find the initialized TensorProtos with external data in the graph with the provided names and - * replace them with the provided tensors. The API verifies that the TensorProto being replaced - * has an external data reference and has the same name, dimensions and data type as its replacement. The replacement - * will occur before any of the optimizations take place. The data will be copied into the graph - * since TensorProto can't refer to the user provided buffers. - * - * Once the model has been loaded, the OrtValue(s) added to SessionOptions instance will be removed - * from the internal SessionOptions copy to save memory, the user provided buffers can then be deallocated - * and the SessionOptions instance that refers to them can be destroyed. - * - * \param[in] options - * \param[in] initializer_names Array of null terminated UTF-8 encoded strings of the initializers names. - * \param[in] initializers Array of ::OrtValue type - * \param[in] initializers_num Number of elements in the initializer_names and initializers - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * - * \since Version 1.12. - */ - ORT_API2_STATUS(AddExternalInitializers, _In_ OrtSessionOptions* options, - _In_reads_(input_len) const char* const* initializer_names, - _In_reads_(input_len) const OrtValue* const* initializers, size_t initializers_num); - - /** \brief: Create attribute of onnxruntime operator - * - * \param[in] name Name of the attribute - * \param[in] data Data content of the attribute - * \param[in] len Number of bytes stored in data - * \param[in] type Data type - * \param[out] op_attr Attribute that has been created, which must be released by OrtApi::ReleaseOpAttr - * - * \since Version 1.12. - */ - ORT_API2_STATUS(CreateOpAttr, - _In_ const char* name, - _In_ const void* data, - _In_ int len, - _In_ OrtOpAttrType type, - _Outptr_ OrtOpAttr** op_attr); - - /* \brief: Release op attribute - * - * \param[in] opAttr Attribute created by OrtApi::CreateOpAttr - * - * \since Version 1.12. - */ - ORT_CLASS_RELEASE(OpAttr); - - /** \brief: Create onnxruntime native operator - * - * \param[in] info Kernel info - * \param[in] op_name Operator name - * \param[in] domain Operator domain - * \param[in] version Operator opset version - * \param[in] type_constraint_names Name of the type contraints, such as "T" or "T1" - * \param[in] type_constraint_values Type of each contraints - * \param[in] type_constraint_count Number of contraints - * \param[in] attr_values Attributes used to initialize the operator - * \param[in] attr_count Number of the attributes - * \param[in] input_count Number of inputs - * \param[in] output_count Number of outputs - * \param[out] ort_op Operator that has been created - * - * \since Version 1.12. - */ - ORT_API2_STATUS(CreateOp, - _In_ const OrtKernelInfo* info, - _In_ const char* op_name, - _In_ const char* domain, - _In_ int version, - _In_opt_ const char** type_constraint_names, - _In_opt_ const ONNXTensorElementDataType* type_constraint_values, - _In_opt_ int type_constraint_count, - _In_opt_ const OrtOpAttr* const* attr_values, - _In_opt_ int attr_count, - _In_ int input_count, - _In_ int output_count, - _Outptr_ OrtOp** ort_op); - - /** \brief: Invoke the operator created by OrtApi::CreateOp - * The inputs must follow the order as specified in onnx specification - * - * \param[in] context Kernel context - * \param[in] ort_op Operator that has been created - * \param[in] input_values Array of inputs - * \param[in] input_count Number of inputs - * \param[in] output_values Array of outputs - * \param[in] output_count Number of outputs - * - * \since Version 1.12. - */ - ORT_API2_STATUS(InvokeOp, - _In_ const OrtKernelContext* context, - _In_ const OrtOp* ort_op, - _In_ const OrtValue* const* input_values, - _In_ int input_count, - _Inout_ OrtValue* const* output_values, - _In_ int output_count); - - /* \brief: Release an onnxruntime operator - * - * \param[in] Op Operator created by OrtApi::CreateOp - * - * \since Version 1.12. - */ - ORT_CLASS_RELEASE(Op); - - /** \brief: Append execution provider to the session options. - * \param[in] provider_name - provider to add. - * \param[in] provider_options_keys - keys to configure the provider options - * \param[in] provider_options_values - values to configure the provider options - * \param[in] num_keys - number of keys passed in - * - * Currently supported providers: - * SNPE - * XNNPACK - * - * Note: If an execution provider has a dedicated SessionOptionsAppendExecutionProvider_ function - * that should be used to add it. - * - * SNPE supported keys: - * "runtime": SNPE runtime engine, options: "CPU", "CPU_FLOAT32", "GPU", "GPU_FLOAT32_16_HYBRID", "GPU_FLOAT16", - * "DSP", "DSP_FIXED8_TF", "AIP_FIXED_TF", "AIP_FIXED8_TF". - * Mapping to SNPE Runtime_t definition: CPU, CPU_FLOAT32 => zdl::DlSystem::Runtime_t::CPU; - * GPU, GPU_FLOAT32_16_HYBRID => zdl::DlSystem::Runtime_t::GPU; - * GPU_FLOAT16 => zdl::DlSystem::Runtime_t::GPU_FLOAT16; - * DSP, DSP_FIXED8_TF => zdl::DlSystem::Runtime_t::DSP. - * AIP_FIXED_TF, AIP_FIXED8_TF => zdl::DlSystem::Runtime_t::AIP_FIXED_TF. - * SNPE Runtime_t refers to https://developer.qualcomm.com/docs/snpe/group__c__plus__plus__apis.html - * "priority": execution priority, options: "low", "normal". - * "buffer_type": ITensor or user buffers, options: "ITENSOR", user buffer with different types - "TF8", "TF16", "UINT8", "FLOAT". - * "ITENSOR" -- default, ITensor which is float only. - * "TF8" -- quantized model required, "FLOAT" -- for both quantized or non-quantized model - * If SNPE is not available (due to a non Snpe enabled build or its dependencies not being installed), this function will fail. - * - * XNNPACK supported keys: - * None currently - * - * \since Version 1.12. - */ - ORT_API2_STATUS(SessionOptionsAppendExecutionProvider, _In_ OrtSessionOptions* options, - _In_ const char* provider_name, - _In_reads_(num_keys) const char* const* provider_options_keys, - _In_reads_(num_keys) const char* const* provider_options_values, - _In_ size_t num_keys); - - /* \brief: Get a copy of kernel info - * - * \param[in] info Kernel info - * \param[out] info_copy Copy of kernel info - * - * \since Version 1.12. - */ - ORT_API2_STATUS(CopyKernelInfo, - _In_ const OrtKernelInfo* info, - _Outptr_ OrtKernelInfo** info_copy); - - /* \brief: Release kernel info - * - * \param[in] KernelInfo A copy of kernel info returned by CopyKernelInfo - * - * \since Version 1.12. - */ - ORT_CLASS_RELEASE(KernelInfo); -}; - -/* - * Steps to use a custom op: - * 1 Create an OrtCustomOpDomain with the domain name used by the custom ops - * 2 Create an OrtCustomOp structure for each op and add them to the domain - * 3 Call OrtAddCustomOpDomain to add the custom domain of ops to the session options -*/ -#define OrtCustomOpApi OrtApi - -// Specifies some characteristics of inputs/outputs of custom ops: -// Specify if the inputs/outputs are one of: -// 1) Non-optional (input/output must be present in the node) -// 2) Optional (input/output may be absent in the node) -typedef enum OrtCustomOpInputOutputCharacteristic { - // TODO: Support 'Variadic' inputs/outputs - INPUT_OUTPUT_REQUIRED = 0, - INPUT_OUTPUT_OPTIONAL, -} OrtCustomOpInputOutputCharacteristic; - -/* - * The OrtCustomOp structure defines a custom op's schema and its kernel callbacks. The callbacks are filled in by - * the implementor of the custom op. -*/ -struct OrtCustomOp { - uint32_t version; // Must be initialized to ORT_API_VERSION - - // This callback creates the kernel, which is a user defined parameter that is passed to the Kernel* callbacks below. - void*(ORT_API_CALL* CreateKernel)(_In_ const struct OrtCustomOp* op, _In_ const OrtApi* api, - _In_ const OrtKernelInfo* info); - - // Returns the name of the op - const char*(ORT_API_CALL* GetName)(_In_ const struct OrtCustomOp* op); - - // Returns the type of the execution provider, return nullptr to use CPU execution provider - const char*(ORT_API_CALL* GetExecutionProviderType)(_In_ const struct OrtCustomOp* op); - - // Returns the count and types of the input & output tensors - ONNXTensorElementDataType(ORT_API_CALL* GetInputType)(_In_ const struct OrtCustomOp* op, _In_ size_t index); - size_t(ORT_API_CALL* GetInputTypeCount)(_In_ const struct OrtCustomOp* op); - ONNXTensorElementDataType(ORT_API_CALL* GetOutputType)(_In_ const struct OrtCustomOp* op, _In_ size_t index); - size_t(ORT_API_CALL* GetOutputTypeCount)(_In_ const struct OrtCustomOp* op); - - // Op kernel callbacks - void(ORT_API_CALL* KernelCompute)(_In_ void* op_kernel, _In_ OrtKernelContext* context); - void(ORT_API_CALL* KernelDestroy)(_In_ void* op_kernel); - - // Returns the characteristics of the input & output tensors - OrtCustomOpInputOutputCharacteristic(ORT_API_CALL* GetInputCharacteristic)(_In_ const struct OrtCustomOp* op, _In_ size_t index); - OrtCustomOpInputOutputCharacteristic(ORT_API_CALL* GetOutputCharacteristic)(_In_ const struct OrtCustomOp* op, _In_ size_t index); -}; - -/* - * This is the old way to add the CUDA provider to the session, please use SessionOptionsAppendExecutionProvider_CUDA above to access the latest functionality - * This function always exists, but will only succeed if Onnxruntime was built with CUDA support and the CUDA provider shared library exists - * - * \param device_id CUDA device id, starts from zero. -*/ -ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_CUDA, _In_ OrtSessionOptions* options, int device_id); - -/* - * This is the old way to add the MIGraphX provider to the session, please use - * SessionOptionsAppendExecutionProvider_MIGraphX above to access the latest functionality - * This function always exists, but will only succeed if Onnxruntime was built with - * HIP support and the MIGraphX provider shared library exists - * - * \param device_id HIP device id, starts from zero. -*/ -ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_MIGraphX, _In_ OrtSessionOptions* options, int device_id); - -#ifdef __cplusplus -} -#endif - -//! @} diff --git a/examples/onnxruntime_demo/main/onnxruntime-src/include/onnxruntime_cxx_api.h b/examples/onnxruntime_demo/main/onnxruntime-src/include/onnxruntime_cxx_api.h deleted file mode 100644 index 90659f06..00000000 --- a/examples/onnxruntime_demo/main/onnxruntime-src/include/onnxruntime_cxx_api.h +++ /dev/null @@ -1,1229 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -// Summary: The Ort C++ API is a header only wrapper around the Ort C API. -// -// The C++ API simplifies usage by returning values directly instead of error codes, throwing exceptions on errors -// and automatically releasing resources in the destructors. -// -// Each of the C++ wrapper classes holds only a pointer to the C internal object. Treat them like smart pointers. -// To create an empty object, pass 'nullptr' to the constructor (for example, Env e{nullptr};). -// -// Only move assignment between objects is allowed, there are no copy constructors. Some objects have explicit 'Clone' -// methods for this purpose. - -#pragma once -#include "onnxruntime_c_api.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef ORT_NO_EXCEPTIONS -#include -#endif - -/** \brief All C++ Onnxruntime APIs are defined inside this namespace -* -*/ -namespace Ort { - -/** \brief All C++ methods that can fail will throw an exception of this type -* -* If ORT_NO_EXCEPTIONS is defined, then any error will result in a call to abort() -*/ -struct Exception : std::exception { - Exception(std::string&& string, OrtErrorCode code) : message_{std::move(string)}, code_{code} {} - - OrtErrorCode GetOrtErrorCode() const { return code_; } - const char* what() const noexcept override { return message_.c_str(); } - - private: - std::string message_; - OrtErrorCode code_; -}; - -#ifdef ORT_NO_EXCEPTIONS -// The #ifndef is for the very special case where the user of this library wants to define their own way of handling errors. -// NOTE: This header expects control flow to not continue after calling ORT_CXX_API_THROW -#ifndef ORT_CXX_API_THROW -#define ORT_CXX_API_THROW(string, code) \ - do { \ - std::cerr << Ort::Exception(string, code) \ - .what() \ - << std::endl; \ - abort(); \ - } while (false) -#endif -#else -#define ORT_CXX_API_THROW(string, code) \ - throw Ort::Exception(string, code) -#endif - -// This is used internally by the C++ API. This class holds the global variable that points to the OrtApi, it's in a template so that we can define a global variable in a header and make -// it transparent to the users of the API. -template -struct Global { - static const OrtApi* api_; -}; - -// If macro ORT_API_MANUAL_INIT is defined, no static initialization will be performed. Instead, user must call InitApi() before using it. -template -#ifdef ORT_API_MANUAL_INIT -const OrtApi* Global::api_{}; -inline void InitApi() { Global::api_ = OrtGetApiBase()->GetApi(ORT_API_VERSION); } -#else -#if defined(_MSC_VER) && !defined(__clang__) -#pragma warning(push) -// "Global initializer calls a non-constexpr function." Therefore you can't use ORT APIs in the other global initializers. -// Please define ORT_API_MANUAL_INIT if it conerns you. -#pragma warning(disable : 26426) -#endif -const OrtApi* Global::api_ = OrtGetApiBase()->GetApi(ORT_API_VERSION); -#if defined(_MSC_VER) && !defined(__clang__) -#pragma warning(pop) -#endif -#endif - -/// This returns a reference to the OrtApi interface in use -inline const OrtApi& GetApi() { return *Global::api_; } - -/// This is a C++ wrapper for OrtApi::GetAvailableProviders() and returns a vector of strings representing the available execution providers. -std::vector GetAvailableProviders(); - -// This is used internally by the C++ API. This macro is to make it easy to generate overloaded methods for all of the various OrtRelease* functions for every Ort* type -// This can't be done in the C API since C doesn't have function overloading. -#define ORT_DEFINE_RELEASE(NAME) \ - inline void OrtRelease(Ort##NAME* ptr) { GetApi().Release##NAME(ptr); } - -ORT_DEFINE_RELEASE(Allocator); -ORT_DEFINE_RELEASE(MemoryInfo); -ORT_DEFINE_RELEASE(CustomOpDomain); -ORT_DEFINE_RELEASE(Env); -ORT_DEFINE_RELEASE(RunOptions); -ORT_DEFINE_RELEASE(Session); -ORT_DEFINE_RELEASE(SessionOptions); -ORT_DEFINE_RELEASE(TensorTypeAndShapeInfo); -ORT_DEFINE_RELEASE(SequenceTypeInfo); -ORT_DEFINE_RELEASE(MapTypeInfo); -ORT_DEFINE_RELEASE(TypeInfo); -ORT_DEFINE_RELEASE(Value); -ORT_DEFINE_RELEASE(ModelMetadata); -ORT_DEFINE_RELEASE(ThreadingOptions); -ORT_DEFINE_RELEASE(IoBinding); -ORT_DEFINE_RELEASE(ArenaCfg); - -#undef ORT_DEFINE_RELEASE - -/** \brief IEEE 754 half-precision floating point data type - * \details It is necessary for type dispatching to make use of C++ API - * The type is implicitly convertible to/from uint16_t. - * The size of the structure should align with uint16_t and one can freely cast - * uint16_t buffers to/from Ort::Float16_t to feed and retrieve data. - * - * Generally, you can feed any of your types as float16/blfoat16 data to create a tensor - * on top of it, providing it can form a continuous buffer with 16-bit elements with no padding. - * And you can also feed a array of uint16_t elements directly. For example, - * - * \code{.unparsed} - * uint16_t values[] = { 15360, 16384, 16896, 17408, 17664}; - * constexpr size_t values_length = sizeof(values) / sizeof(values[0]); - * std::vector dims = {values_length}; // one dimensional example - * Ort::MemoryInfo info("Cpu", OrtDeviceAllocator, 0, OrtMemTypeDefault); - * // Note we are passing bytes count in this api, not number of elements -> sizeof(values) - * auto float16_tensor = Ort::Value::CreateTensor(info, values, sizeof(values), - * dims.data(), dims.size(), ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16); - * \endcode - * - * Here is another example, a little bit more elaborate. Let's assume that you use your own float16 type and you want to use - * a templated version of the API above so the type is automatically set based on your type. You will need to supply an extra - * template specialization. - * - * \code{.unparsed} - * namespace yours { struct half {}; } // assume this is your type, define this: - * namespace Ort { - * template<> - * struct TypeToTensorType { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16; }; - * } //namespace Ort - * - * std::vector values; - * std::vector dims = {values.size()}; // one dimensional example - * Ort::MemoryInfo info("Cpu", OrtDeviceAllocator, 0, OrtMemTypeDefault); - * // Here we are passing element count -> values.size() - * auto float16_tensor = Ort::Value::CreateTensor(info, values.data(), values.size(), dims.data(), dims.size()); - * - * \endcode - */ -struct Float16_t { - uint16_t value; - constexpr Float16_t() noexcept : value(0) {} - constexpr Float16_t(uint16_t v) noexcept : value(v) {} - constexpr operator uint16_t() const noexcept { return value; } - constexpr bool operator==(const Float16_t& rhs) const noexcept { return value == rhs.value; }; - constexpr bool operator!=(const Float16_t& rhs) const noexcept { return value != rhs.value; }; -}; - -static_assert(sizeof(Float16_t) == sizeof(uint16_t), "Sizes must match"); - -/** \brief bfloat16 (Brain Floating Point) data type - * \details It is necessary for type dispatching to make use of C++ API - * The type is implicitly convertible to/from uint16_t. - * The size of the structure should align with uint16_t and one can freely cast - * uint16_t buffers to/from Ort::BFloat16_t to feed and retrieve data. - * - * See also code examples for Float16_t above. - */ -struct BFloat16_t { - uint16_t value; - constexpr BFloat16_t() noexcept : value(0) {} - constexpr BFloat16_t(uint16_t v) noexcept : value(v) {} - constexpr operator uint16_t() const noexcept { return value; } - constexpr bool operator==(const BFloat16_t& rhs) const noexcept { return value == rhs.value; }; - constexpr bool operator!=(const BFloat16_t& rhs) const noexcept { return value != rhs.value; }; -}; - -static_assert(sizeof(BFloat16_t) == sizeof(uint16_t), "Sizes must match"); - -/** \brief Used internally by the C++ API. C++ wrapper types inherit from this -* -* This is a zero cost abstraction to wrap the C API objects and delete them on destruction. -* There is a secondary class 'Unowned' that is used to prevent deletion on destruction (Used for return types that are -* not owned by the caller) -* -*/ -template -struct Base { - using contained_type = T; - - Base() = default; - Base(T* p) : p_{p} { - if (!p) - ORT_CXX_API_THROW("Allocation failure", ORT_FAIL); - } - ~Base() { OrtRelease(p_); } - - operator T*() { return p_; } - operator const T*() const { return p_; } - - /// \brief Releases ownership of the contained pointer - T* release() { - T* p = p_; - p_ = nullptr; - return p; - } - - protected: - Base(const Base&) = delete; - Base& operator=(const Base&) = delete; - Base(Base&& v) noexcept : p_{v.p_} { v.p_ = nullptr; } - void operator=(Base&& v) noexcept { - OrtRelease(p_); - p_ = v.release(); - } - - T* p_{}; - - template - friend struct Unowned; // This friend line is needed to keep the centos C++ compiler from giving an error -}; - -/** \brief Wraps an object that inherits from Ort::Base and stops it from deleting the contained pointer on destruction -* -* This has the effect of making it not own the memory held by Ort::Base. -*/ -template -struct Unowned : T { - Unowned(typename T::contained_type* p) : T{p} {} - Unowned(Unowned&& v) : T{v.p_} {} - ~Unowned() { this->release(); } -}; - -struct AllocatorWithDefaultOptions; -struct MemoryInfo; -struct Env; -struct TypeInfo; -struct Value; -struct ModelMetadata; - -namespace detail { -// Light functor to release memory with OrtAllocator -struct AllocatedFree { - OrtAllocator* allocator_; - explicit AllocatedFree(OrtAllocator* allocator) - : allocator_(allocator) {} - void operator()(void* ptr) const { if(ptr) allocator_->Free(allocator_, ptr); } -}; -} // namespace detail - -/** \brief unique_ptr typedef used to own strings allocated by OrtAllocators - * and release them at the end of the scope. The lifespan of the given allocator - * must eclipse the lifespan of AllocatedStringPtr instance - */ -using AllocatedStringPtr = std::unique_ptr; - -/** \brief The Env (Environment) -* -* The Env holds the logging state used by all other objects. -* Note: One Env must be created before using any other Onnxruntime functionality -*/ -struct Env : Base { - explicit Env(std::nullptr_t) {} ///< Create an empty Env object, must be assigned a valid one to be used - - /// \brief Wraps OrtApi::CreateEnv - Env(OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = ""); - - /// \brief Wraps OrtApi::CreateEnvWithCustomLogger - Env(OrtLoggingLevel logging_level, const char* logid, OrtLoggingFunction logging_function, void* logger_param); - - /// \brief Wraps OrtApi::CreateEnvWithGlobalThreadPools - Env(const OrtThreadingOptions* tp_options, OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = ""); - - /// \brief Wraps OrtApi::CreateEnvWithCustomLoggerAndGlobalThreadPools - Env(const OrtThreadingOptions* tp_options, OrtLoggingFunction logging_function, void* logger_param, - OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = ""); - - /// \brief C Interop Helper - explicit Env(OrtEnv* p) : Base{p} {} - - Env& EnableTelemetryEvents(); ///< Wraps OrtApi::EnableTelemetryEvents - Env& DisableTelemetryEvents(); ///< Wraps OrtApi::DisableTelemetryEvents - - Env& CreateAndRegisterAllocator(const OrtMemoryInfo* mem_info, const OrtArenaCfg* arena_cfg); ///< Wraps OrtApi::CreateAndRegisterAllocator -}; - -/** \brief Custom Op Domain -* -*/ -struct CustomOpDomain : Base { - explicit CustomOpDomain(std::nullptr_t) {} ///< Create an empty CustomOpDomain object, must be assigned a valid one to be used - - /// \brief Wraps OrtApi::CreateCustomOpDomain - explicit CustomOpDomain(const char* domain); - - void Add(OrtCustomOp* op); ///< Wraps CustomOpDomain_Add -}; - -struct RunOptions : Base { - explicit RunOptions(std::nullptr_t) {} ///< Create an empty RunOptions object, must be assigned a valid one to be used - RunOptions(); ///< Wraps OrtApi::CreateRunOptions - - RunOptions& SetRunLogVerbosityLevel(int); ///< Wraps OrtApi::RunOptionsSetRunLogVerbosityLevel - int GetRunLogVerbosityLevel() const; ///< Wraps OrtApi::RunOptionsGetRunLogVerbosityLevel - - RunOptions& SetRunLogSeverityLevel(int); ///< Wraps OrtApi::RunOptionsSetRunLogSeverityLevel - int GetRunLogSeverityLevel() const; ///< Wraps OrtApi::RunOptionsGetRunLogSeverityLevel - - RunOptions& SetRunTag(const char* run_tag); ///< wraps OrtApi::RunOptionsSetRunTag - const char* GetRunTag() const; ///< Wraps OrtApi::RunOptionsGetRunTag - - RunOptions& AddConfigEntry(const char* config_key, const char* config_value); ///< Wraps OrtApi::AddRunConfigEntry - - /** \brief Terminates all currently executing Session::Run calls that were made using this RunOptions instance - * - * If a currently executing session needs to be force terminated, this can be called from another thread to force it to fail with an error - * Wraps OrtApi::RunOptionsSetTerminate - */ - RunOptions& SetTerminate(); - - /** \brief Clears the terminate flag so this RunOptions instance can be used in a new Session::Run call without it instantly terminating - * - * Wraps OrtApi::RunOptionsUnsetTerminate - */ - RunOptions& UnsetTerminate(); -}; - -/** \brief Options object used when creating a new Session object -* -* Wraps ::OrtSessionOptions object and methods -*/ -struct SessionOptions : Base { - explicit SessionOptions(std::nullptr_t) {} ///< Create an empty SessionOptions object, must be assigned a valid one to be used - SessionOptions(); ///< Wraps OrtApi::CreateSessionOptions - explicit SessionOptions(OrtSessionOptions* p) : Base{p} {} ///< Used for interop with the C API - - SessionOptions Clone() const; ///< Creates and returns a copy of this SessionOptions object. Wraps OrtApi::CloneSessionOptions - - SessionOptions& SetIntraOpNumThreads(int intra_op_num_threads); ///< Wraps OrtApi::SetIntraOpNumThreads - SessionOptions& SetInterOpNumThreads(int inter_op_num_threads); ///< Wraps OrtApi::SetInterOpNumThreads - SessionOptions& SetGraphOptimizationLevel(GraphOptimizationLevel graph_optimization_level); ///< Wraps OrtApi::SetSessionGraphOptimizationLevel - - SessionOptions& EnableCpuMemArena(); ///< Wraps OrtApi::EnableCpuMemArena - SessionOptions& DisableCpuMemArena(); ///< Wraps OrtApi::DisableCpuMemArena - - SessionOptions& SetOptimizedModelFilePath(const ORTCHAR_T* optimized_model_file); ///< Wraps OrtApi::SetOptimizedModelFilePath - - SessionOptions& EnableProfiling(const ORTCHAR_T* profile_file_prefix); ///< Wraps OrtApi::EnableProfiling - SessionOptions& DisableProfiling(); ///< Wraps OrtApi::DisableProfiling - - SessionOptions& EnableOrtCustomOps(); ///< Wraps OrtApi::EnableOrtCustomOps - - SessionOptions& EnableMemPattern(); ///< Wraps OrtApi::EnableMemPattern - SessionOptions& DisableMemPattern(); ///< Wraps OrtApi::DisableMemPattern - - SessionOptions& SetExecutionMode(ExecutionMode execution_mode); ///< Wraps OrtApi::SetSessionExecutionMode - - SessionOptions& SetLogId(const char* logid); ///< Wraps OrtApi::SetSessionLogId - SessionOptions& SetLogSeverityLevel(int level); ///< Wraps OrtApi::SetSessionLogSeverityLevel - - SessionOptions& Add(OrtCustomOpDomain* custom_op_domain); ///< Wraps OrtApi::AddCustomOpDomain - - SessionOptions& DisablePerSessionThreads(); ///< Wraps OrtApi::DisablePerSessionThreads - - SessionOptions& AddConfigEntry(const char* config_key, const char* config_value); ///< Wraps OrtApi::AddSessionConfigEntry - SessionOptions& AddInitializer(const char* name, const OrtValue* ort_val); ///< Wraps OrtApi::AddInitializer - SessionOptions& AddExternalInitializers(const std::vector& names, const std::vector& ort_values); ///< Wraps OrtApi::AddExternalInitializers - - SessionOptions& AppendExecutionProvider_CUDA(const OrtCUDAProviderOptions& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_CUDA - SessionOptions& AppendExecutionProvider_CUDA_V2(const OrtCUDAProviderOptionsV2& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_CUDA_V2 - SessionOptions& AppendExecutionProvider_ROCM(const OrtROCMProviderOptions& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_ROCM - SessionOptions& AppendExecutionProvider_OpenVINO(const OrtOpenVINOProviderOptions& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_OpenVINO - SessionOptions& AppendExecutionProvider_TensorRT(const OrtTensorRTProviderOptions& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_TensorRT - SessionOptions& AppendExecutionProvider_TensorRT_V2(const OrtTensorRTProviderOptionsV2& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_TensorRT - SessionOptions& AppendExecutionProvider_MIGraphX(const OrtMIGraphXProviderOptions& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_MIGraphX - /// Wraps OrtApi::SessionOptionsAppendExecutionProvider. Currently supports SNPE and XNNPACK. - SessionOptions& AppendExecutionProvider(const std::string& provider_name, - const std::unordered_map& provider_options = {}); - - SessionOptions& SetCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn); ///< Wraps OrtApi::SessionOptionsSetCustomCreateThreadFn - SessionOptions& SetCustomThreadCreationOptions(void* ort_custom_thread_creation_options); ///< Wraps OrtApi::SessionOptionsSetCustomThreadCreationOptions - SessionOptions& SetCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn); ///< Wraps OrtApi::SessionOptionsSetCustomJoinThreadFn -}; - -/** \brief Wrapper around ::OrtModelMetadata -* -*/ -struct ModelMetadata : Base { - explicit ModelMetadata(std::nullptr_t) {} ///< Create an empty ModelMetadata object, must be assigned a valid one to be used - explicit ModelMetadata(OrtModelMetadata* p) : Base{p} {} ///< Used for interop with the C API - - /** \deprecated use GetProducerNameAllocated() - * [[deprecated]] - * This interface produces a pointer that must be released - * by the specified allocator and is often leaked. Not exception safe. - */ - char* GetProducerName(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetProducerName - - /** \brief Returns a copy of the producer name. - * - * \param allocator to allocate memory for the copy of the name returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr GetProducerNameAllocated(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetProducerName - - /** \deprecated use GetGraphNameAllocated() - * [[deprecated]] - * This interface produces a pointer that must be released - * by the specified allocator and is often leaked. Not exception safe. - */ - char* GetGraphName(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetGraphName - - /** \brief Returns a copy of the graph name. - * - * \param allocator to allocate memory for the copy of the name returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr GetGraphNameAllocated(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetGraphName - - /** \deprecated use GetDomainAllocated() - * [[deprecated]] - * This interface produces a pointer that must be released - * by the specified allocator and is often leaked. Not exception safe. - */ - char* GetDomain(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetDomain - - /** \brief Returns a copy of the domain name. - * - * \param allocator to allocate memory for the copy of the name returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr GetDomainAllocated(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetDomain - - /** \deprecated use GetDescriptionAllocated() - * [[deprecated]] - * This interface produces a pointer that must be released - * by the specified allocator and is often leaked. Not exception safe. - */ - char* GetDescription(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetDescription - - /** \brief Returns a copy of the description. - * - * \param allocator to allocate memory for the copy of the string returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr GetDescriptionAllocated(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetDescription - - /** \deprecated use GetGraphDescriptionAllocated() - * [[deprecated]] - * This interface produces a pointer that must be released - * by the specified allocator and is often leaked. Not exception safe. - */ - char* GetGraphDescription(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetGraphDescription - - /** \brief Returns a copy of the graph description. - * - * \param allocator to allocate memory for the copy of the string returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr GetGraphDescriptionAllocated(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetGraphDescription - - /** \deprecated use GetCustomMetadataMapKeysAllocated() - * [[deprecated]] - * This interface produces multiple pointers that must be released - * by the specified allocator and is often leaked. Not exception safe. - */ - char** GetCustomMetadataMapKeys(OrtAllocator* allocator, _Out_ int64_t& num_keys) const; ///< Wraps OrtApi::ModelMetadataGetCustomMetadataMapKeys - - std::vector GetCustomMetadataMapKeysAllocated(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetCustomMetadataMapKeys - - /** \deprecated use LookupCustomMetadataMapAllocated() - * [[deprecated]] - * This interface produces a pointer that must be released - * by the specified allocator and is often leaked. Not exception safe. - */ - char* LookupCustomMetadataMap(const char* key, OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataLookupCustomMetadataMap - - /** \brief Looks up a value by a key in the Custom Metadata map - * - * \param zero terminated string key to lookup - * \param allocator to allocate memory for the copy of the string returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * maybe nullptr if key is not found. - * - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr LookupCustomMetadataMapAllocated(const char* key, OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataLookupCustomMetadataMap - - int64_t GetVersion() const; ///< Wraps OrtApi::ModelMetadataGetVersion -}; - -/** \brief Wrapper around ::OrtSession -* -*/ -struct Session : Base { - explicit Session(std::nullptr_t) {} ///< Create an empty Session object, must be assigned a valid one to be used - Session(Env& env, const ORTCHAR_T* model_path, const SessionOptions& options); ///< Wraps OrtApi::CreateSession - Session(Env& env, const ORTCHAR_T* model_path, const SessionOptions& options, OrtPrepackedWeightsContainer* prepacked_weights_container); ///< Wraps OrtApi::CreateSessionWithPrepackedWeightsContainer - Session(Env& env, const void* model_data, size_t model_data_length, const SessionOptions& options); ///< Wraps OrtApi::CreateSessionFromArray - - /** \brief Run the model returning results in an Ort allocated vector. - * - * Wraps OrtApi::Run - * - * The caller provides a list of inputs and a list of the desired outputs to return. - * - * See the output logs for more information on warnings/errors that occur while processing the model. - * Common errors are.. (TODO) - * - * \param[in] run_options - * \param[in] input_names Array of null terminated strings of length input_count that is the list of input names - * \param[in] input_values Array of Value objects of length input_count that is the list of input values - * \param[in] input_count Number of inputs (the size of the input_names & input_values arrays) - * \param[in] output_names Array of C style strings of length output_count that is the list of output names - * \param[in] output_count Number of outputs (the size of the output_names array) - * \return A std::vector of Value objects that directly maps to the output_count (eg. output_name[0] is the first entry of the returned vector) - */ - std::vector Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count, - const char* const* output_names, size_t output_count); - - /** \brief Run the model returning results in user provided outputs - * Same as Run(const RunOptions&, const char* const*, const Value*, size_t,const char* const*, size_t) - */ - void Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count, - const char* const* output_names, Value* output_values, size_t output_count); - - void Run(const RunOptions& run_options, const struct IoBinding&); ///< Wraps OrtApi::RunWithBinding - - size_t GetInputCount() const; ///< Returns the number of model inputs - size_t GetOutputCount() const; ///< Returns the number of model outputs - size_t GetOverridableInitializerCount() const; ///< Returns the number of inputs that have defaults that can be overridden - - /** \deprecated use GetInputNameAllocated() - * [[deprecated]] - * This interface produces a pointer that must be released - * by the specified allocator and is often leaked. Not exception safe. - */ - char* GetInputName(size_t index, OrtAllocator* allocator) const; ///< Wraps OrtApi::SessionGetInputName - - /** \brief Returns a copy of input name at the specified index. - * - * \param index must less than the value returned by GetInputCount() - * \param allocator to allocate memory for the copy of the name returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr GetInputNameAllocated(size_t index, OrtAllocator* allocator) const; - - /** \deprecated use GetOutputNameAllocated() - * [[deprecated]] - * This interface produces a pointer that must be released - * by the specified allocator and is often leaked. Not exception safe. - */ - char* GetOutputName(size_t index, OrtAllocator* allocator) const; ///< Wraps OrtApi::SessionGetOutputName - - /** \brief Returns a copy of output name at then specified index. - * - * \param index must less than the value returned by GetOutputCount() - * \param allocator to allocate memory for the copy of the name returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr GetOutputNameAllocated(size_t index, OrtAllocator* allocator) const; - - /** \deprecated use GetOverridableInitializerNameAllocated() - * [[deprecated]] - * This interface produces a pointer that must be released - * by the specified allocator and is often leaked. Not exception safe. - */ - char* GetOverridableInitializerName(size_t index, OrtAllocator* allocator) const; ///< Wraps OrtApi::SessionGetOverridableInitializerName - - /** \brief Returns a copy of the overridable initializer name at then specified index. - * - * \param index must less than the value returned by GetOverridableInitializerCount() - * \param allocator to allocate memory for the copy of the name returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr GetOverridableInitializerNameAllocated(size_t index, OrtAllocator* allocator) const; ///< Wraps OrtApi::SessionGetOverridableInitializerName - - /** \deprecated use EndProfilingAllocated() - * [[deprecated]] - * This interface produces a pointer that must be released - * by the specified allocator and is often leaked. Not exception safe. - */ - char* EndProfiling(OrtAllocator* allocator) const; ///< Wraps OrtApi::SessionEndProfiling - - /** \brief Returns a copy of the profiling file name. - * - * \param allocator to allocate memory for the copy of the string returned - * \return a instance of smart pointer that would deallocate the buffer when out of scope. - * The OrtAllocator instances must be valid at the point of memory release. - */ - AllocatedStringPtr EndProfilingAllocated(OrtAllocator* allocator) const; ///< Wraps OrtApi::SessionEndProfiling - uint64_t GetProfilingStartTimeNs() const; ///< Wraps OrtApi::SessionGetProfilingStartTimeNs - ModelMetadata GetModelMetadata() const; ///< Wraps OrtApi::SessionGetModelMetadata - - TypeInfo GetInputTypeInfo(size_t index) const; ///< Wraps OrtApi::SessionGetInputTypeInfo - TypeInfo GetOutputTypeInfo(size_t index) const; ///< Wraps OrtApi::SessionGetOutputTypeInfo - TypeInfo GetOverridableInitializerTypeInfo(size_t index) const; ///< Wraps OrtApi::SessionGetOverridableInitializerTypeInfo -}; - -/** \brief Wrapper around ::OrtTensorTypeAndShapeInfo -* -*/ -struct TensorTypeAndShapeInfo : Base { - explicit TensorTypeAndShapeInfo(std::nullptr_t) {} ///< Create an empty TensorTypeAndShapeInfo object, must be assigned a valid one to be used - explicit TensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo* p) : Base{p} {} ///< Used for interop with the C API - - ONNXTensorElementDataType GetElementType() const; ///< Wraps OrtApi::GetTensorElementType - size_t GetElementCount() const; ///< Wraps OrtApi::GetTensorShapeElementCount - - size_t GetDimensionsCount() const; ///< Wraps OrtApi::GetDimensionsCount - void GetDimensions(int64_t* values, size_t values_count) const; ///< Wraps OrtApi::GetDimensions - void GetSymbolicDimensions(const char** values, size_t values_count) const; ///< Wraps OrtApi::GetSymbolicDimensions - - std::vector GetShape() const; ///< Uses GetDimensionsCount & GetDimensions to return a std::vector of the shape -}; - -/** \brief Wrapper around ::OrtSequenceTypeInfo -* -*/ -struct SequenceTypeInfo : Base { - explicit SequenceTypeInfo(std::nullptr_t) {} ///< Create an empty SequenceTypeInfo object, must be assigned a valid one to be used - explicit SequenceTypeInfo(OrtSequenceTypeInfo* p) : Base{p} {} ///< Used for interop with the C API - - TypeInfo GetSequenceElementType() const; ///< Wraps OrtApi::GetSequenceElementType -}; - -/** \brief Wrapper around ::OrtMapTypeInfo -* -*/ -struct MapTypeInfo : Base { - explicit MapTypeInfo(std::nullptr_t) {} ///< Create an empty MapTypeInfo object, must be assigned a valid one to be used - explicit MapTypeInfo(OrtMapTypeInfo* p) : Base{p} {} ///< Used for interop with the C API - - ONNXTensorElementDataType GetMapKeyType() const; ///< Wraps OrtApi::GetMapKeyType - TypeInfo GetMapValueType() const; ///< Wraps OrtApi::GetMapValueType -}; - -struct TypeInfo : Base { - explicit TypeInfo(std::nullptr_t) {} ///< Create an empty TypeInfo object, must be assigned a valid one to be used - explicit TypeInfo(OrtTypeInfo* p) : Base{p} {} ///< C API Interop - - Unowned GetTensorTypeAndShapeInfo() const; ///< Wraps OrtApi::CastTypeInfoToTensorInfo - Unowned GetSequenceTypeInfo() const; ///< Wraps OrtApi::CastTypeInfoToSequenceTypeInfo - Unowned GetMapTypeInfo() const; ///< Wraps OrtApi::CastTypeInfoToMapTypeInfo - - ONNXType GetONNXType() const; -}; - -struct Value : Base { - // This structure is used to feed sparse tensor values - // information for use with FillSparseTensor() API - // if the data type for the sparse tensor values is numeric - // use data.p_data, otherwise, use data.str pointer to feed - // values. data.str is an array of const char* that are zero terminated. - // number of strings in the array must match shape size. - // For fully sparse tensors use shape {0} and set p_data/str - // to nullptr. - struct OrtSparseValuesParam { - const int64_t* values_shape; - size_t values_shape_len; - union { - const void* p_data; - const char** str; - } data; - }; - - // Provides a way to pass shape in a single - // argument - struct Shape { - const int64_t* shape; - size_t shape_len; - }; - - /** \brief Creates a tensor with a user supplied buffer. Wraps OrtApi::CreateTensorWithDataAsOrtValue. - * \tparam T The numeric datatype. This API is not suitable for strings. - * \param info Memory description of where the p_data buffer resides (CPU vs GPU etc). - * \param p_data Pointer to the data buffer. - * \param p_data_element_count The number of elements in the data buffer. - * \param shape Pointer to the tensor shape dimensions. - * \param shape_len The number of tensor shape dimensions. - */ - template - static Value CreateTensor(const OrtMemoryInfo* info, T* p_data, size_t p_data_element_count, const int64_t* shape, size_t shape_len); - - /** \brief Creates a tensor with a user supplied buffer. Wraps OrtApi::CreateTensorWithDataAsOrtValue. - * \param info Memory description of where the p_data buffer resides (CPU vs GPU etc). - * \param p_data Pointer to the data buffer. - * \param p_data_byte_count The number of bytes in the data buffer. - * \param shape Pointer to the tensor shape dimensions. - * \param shape_len The number of tensor shape dimensions. - * \param type The data type. - */ - static Value CreateTensor(const OrtMemoryInfo* info, void* p_data, size_t p_data_byte_count, const int64_t* shape, size_t shape_len, - ONNXTensorElementDataType type); - -#if !defined(DISABLE_SPARSE_TENSORS) - /// - /// This is a simple forwarding method to the other overload that helps deducing - /// data type enum value from the type of the buffer. - /// - /// numeric datatype. This API is not suitable for strings. - /// Memory description where the user buffers reside (CPU vs GPU etc) - /// pointer to the user supplied buffer, use nullptr for fully sparse tensors - /// a would be dense shape of the tensor - /// non zero values shape. Use a single 0 shape for fully sparse tensors. - /// - template - static Value CreateSparseTensor(const OrtMemoryInfo* info, T* p_data, const Shape& dense_shape, - const Shape& values_shape); - - /// - /// Creates an OrtValue instance containing SparseTensor. This constructs - /// a sparse tensor that makes use of user allocated buffers. It does not make copies - /// of the user provided data and does not modify it. The lifespan of user provided buffers should - /// eclipse the life span of the resulting OrtValue. This call constructs an instance that only contain - /// a pointer to non-zero values. To fully populate the sparse tensor call UseIndices() API below - /// to supply a sparse format specific indices. - /// This API is not suitable for string data. Use CreateSparseTensor() with allocator specified so strings - /// can be properly copied into the allocated buffer. - /// - /// Memory description where the user buffers reside (CPU vs GPU etc) - /// pointer to the user supplied buffer, use nullptr for fully sparse tensors - /// a would be dense shape of the tensor - /// non zero values shape. Use a single 0 shape for fully sparse tensors. - /// data type - /// Ort::Value instance containing SparseTensor - static Value CreateSparseTensor(const OrtMemoryInfo* info, void* p_data, const Shape& dense_shape, - const Shape& values_shape, ONNXTensorElementDataType type); - - /// - /// Supplies COO format specific indices and marks the contained sparse tensor as being a COO format tensor. - /// Values are supplied with a CreateSparseTensor() API. The supplied indices are not copied and the user - /// allocated buffers lifespan must eclipse that of the OrtValue. - /// The location of the indices is assumed to be the same as specified by OrtMemoryInfo argument at the creation time. - /// - /// pointer to the user allocated buffer with indices. Use nullptr for fully sparse tensors. - /// number of indices entries. Use 0 for fully sparse tensors - void UseCooIndices(int64_t* indices_data, size_t indices_num); - - /// - /// Supplies CSR format specific indices and marks the contained sparse tensor as being a CSR format tensor. - /// Values are supplied with a CreateSparseTensor() API. The supplied indices are not copied and the user - /// allocated buffers lifespan must eclipse that of the OrtValue. - /// The location of the indices is assumed to be the same as specified by OrtMemoryInfo argument at the creation time. - /// - /// pointer to the user allocated buffer with inner indices or nullptr for fully sparse tensors - /// number of csr inner indices or 0 for fully sparse tensors - /// pointer to the user allocated buffer with outer indices or nullptr for fully sparse tensors - /// number of csr outer indices or 0 for fully sparse tensors - void UseCsrIndices(int64_t* inner_data, size_t inner_num, int64_t* outer_data, size_t outer_num); - - /// - /// Supplies BlockSparse format specific indices and marks the contained sparse tensor as being a BlockSparse format tensor. - /// Values are supplied with a CreateSparseTensor() API. The supplied indices are not copied and the user - /// allocated buffers lifespan must eclipse that of the OrtValue. - /// The location of the indices is assumed to be the same as specified by OrtMemoryInfo argument at the creation time. - /// - /// indices shape or a {0} for fully sparse - /// user allocated buffer with indices or nullptr for fully spare tensors - void UseBlockSparseIndices(const Shape& indices_shape, int32_t* indices_data); - -#endif // !defined(DISABLE_SPARSE_TENSORS) - - /** \brief Creates a tensor using a supplied OrtAllocator. Wraps OrtApi::CreateTensorAsOrtValue. - * \tparam T The numeric datatype. This API is not suitable for strings. - * \param allocator The allocator to use. - * \param shape Pointer to the tensor shape dimensions. - * \param shape_len The number of tensor shape dimensions. - */ - template - static Value CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len); - - /** \brief Creates a tensor using a supplied OrtAllocator. Wraps OrtApi::CreateTensorAsOrtValue. - * \param allocator The allocator to use. - * \param shape Pointer to the tensor shape dimensions. - * \param shape_len The number of tensor shape dimensions. - * \param type The data type. - */ - static Value CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type); - -#if !defined(DISABLE_SPARSE_TENSORS) - /// - /// This is a simple forwarding method the below CreateSparseTensor. - /// This helps to specify data type enum in terms of C++ data type. - /// Use CreateSparseTensor - /// - /// numeric data type only. String data enum must be specified explicitly. - /// allocator to use - /// a would be dense shape of the tensor - /// Ort::Value - template - static Value CreateSparseTensor(OrtAllocator* allocator, const Shape& dense_shape); - - /// - /// Creates an instance of OrtValue containing sparse tensor. The created instance has no data. - /// The data must be supplied by on of the FillSparseTensor() methods that take both non-zero values - /// and indices. The data will be copied into a buffer that would be allocated using the supplied allocator. - /// Use this API to create OrtValues that contain sparse tensors with all supported data types including - /// strings. - /// - /// allocator to use. The allocator lifespan must eclipse that of the resulting OrtValue - /// a would be dense shape of the tensor - /// data type - /// an instance of Ort::Value - static Value CreateSparseTensor(OrtAllocator* allocator, const Shape& dense_shape, ONNXTensorElementDataType type); - - /// - /// The API will allocate memory using the allocator instance supplied to the CreateSparseTensor() API - /// and copy the values and COO indices into it. If data_mem_info specifies that the data is located - /// at difference device than the allocator, a X-device copy will be performed if possible. - /// - /// specified buffer memory description - /// values buffer information. - /// coo indices buffer or nullptr for fully sparse data - /// number of COO indices or 0 for fully sparse data - void FillSparseTensorCoo(const OrtMemoryInfo* data_mem_info, const OrtSparseValuesParam& values_param, - const int64_t* indices_data, size_t indices_num); - - /// - /// The API will allocate memory using the allocator instance supplied to the CreateSparseTensor() API - /// and copy the values and CSR indices into it. If data_mem_info specifies that the data is located - /// at difference device than the allocator, a X-device copy will be performed if possible. - /// - /// specified buffer memory description - /// values buffer information - /// csr inner indices pointer or nullptr for fully sparse tensors - /// number of csr inner indices or 0 for fully sparse tensors - /// pointer to csr indices data or nullptr for fully sparse tensors - /// number of csr outer indices or 0 - void FillSparseTensorCsr(const OrtMemoryInfo* data_mem_info, - const OrtSparseValuesParam& values, - const int64_t* inner_indices_data, size_t inner_indices_num, - const int64_t* outer_indices_data, size_t outer_indices_num); - - /// - /// The API will allocate memory using the allocator instance supplied to the CreateSparseTensor() API - /// and copy the values and BlockSparse indices into it. If data_mem_info specifies that the data is located - /// at difference device than the allocator, a X-device copy will be performed if possible. - /// - /// specified buffer memory description - /// values buffer information - /// indices shape. use {0} for fully sparse tensors - /// pointer to indices data or nullptr for fully sparse tensors - void FillSparseTensorBlockSparse(const OrtMemoryInfo* data_mem_info, - const OrtSparseValuesParam& values, - const Shape& indices_shape, - const int32_t* indices_data); - - /// - /// The API returns the sparse data format this OrtValue holds in a sparse tensor. - /// If the sparse tensor was not fully constructed, i.e. Use*() or Fill*() API were not used - /// the value returned is ORT_SPARSE_UNDEFINED. - /// - /// Format enum - OrtSparseFormat GetSparseFormat() const; - - /// - /// The API returns type and shape information for stored non-zero values of the - /// sparse tensor. Use GetSparseTensorValues() to obtain values buffer pointer. - /// - /// TensorTypeAndShapeInfo values information - TensorTypeAndShapeInfo GetSparseTensorValuesTypeAndShapeInfo() const; - - /// - /// The API returns type and shape information for the specified indices. Each supported - /// indices have their own enum values even if a give format has more than one kind of indices. - /// Use GetSparseTensorIndicesData() to obtain pointer to indices buffer. - /// - /// enum requested - /// type and shape information - TensorTypeAndShapeInfo GetSparseTensorIndicesTypeShapeInfo(OrtSparseIndicesFormat format) const; - - /// - /// The API retrieves a pointer to the internal indices buffer. The API merely performs - /// a convenience data type casting on the return type pointer. Make sure you are requesting - /// the right type, use GetSparseTensorIndicesTypeShapeInfo(); - /// - /// type to cast to - /// requested indices kind - /// number of indices entries - /// Pinter to the internal sparse tensor buffer containing indices. Do not free this pointer. - template - const T* GetSparseTensorIndicesData(OrtSparseIndicesFormat indices_format, size_t& num_indices) const; - -#endif // !defined(DISABLE_SPARSE_TENSORS) - - static Value CreateMap(Value& keys, Value& values); ///< Wraps OrtApi::CreateValue - static Value CreateSequence(std::vector& values); ///< Wraps OrtApi::CreateValue - - template - static Value CreateOpaque(const char* domain, const char* type_name, const T&); ///< Wraps OrtApi::CreateOpaqueValue - - template - void GetOpaqueData(const char* domain, const char* type_name, T&) const; ///< Wraps OrtApi::GetOpaqueValue - - explicit Value(std::nullptr_t) {} ///< Create an empty Value object, must be assigned a valid one to be used - explicit Value(OrtValue* p) : Base{p} {} ///< Used for interop with the C API - Value(Value&&) = default; - Value& operator=(Value&&) = default; - - bool IsTensor() const; ///< Returns true if Value is a tensor, false for other types like map/sequence/etc - bool HasValue() const; /// < Return true if OrtValue contains data and returns false if the OrtValue is a None - -#if !defined(DISABLE_SPARSE_TENSORS) - /// - /// Returns true if the OrtValue contains a sparse tensor - /// - /// - bool IsSparseTensor() const; -#endif - - size_t GetCount() const; // If a non tensor, returns 2 for map and N for sequence, where N is the number of elements - Value GetValue(int index, OrtAllocator* allocator) const; - - /// - /// This API returns a full length of string data contained within either a tensor or a sparse Tensor. - /// For sparse tensor it returns a full length of stored non-empty strings (values). The API is useful - /// for allocating necessary memory and calling GetStringTensorContent(). - /// - /// total length of UTF-8 encoded bytes contained. No zero terminators counted. - size_t GetStringTensorDataLength() const; - - /// - /// The API copies all of the UTF-8 encoded string data contained within a tensor or a sparse tensor - /// into a supplied buffer. Use GetStringTensorDataLength() to find out the length of the buffer to allocate. - /// The user must also allocate offsets buffer with the number of entries equal to that of the contained - /// strings. - /// - /// Strings are always assumed to be on CPU, no X-device copy. - /// - /// user allocated buffer - /// length in bytes of the allocated buffer - /// a pointer to the offsets user allocated buffer - /// count of offsets, must be equal to the number of strings contained. - /// that can be obtained from the shape of the tensor or from GetSparseTensorValuesTypeAndShapeInfo() - /// for sparse tensors - void GetStringTensorContent(void* buffer, size_t buffer_length, size_t* offsets, size_t offsets_count) const; - - template - T* GetTensorMutableData(); ///< Wraps OrtApi::GetTensorMutableData - - template - const T* GetTensorData() const; ///< Wraps OrtApi::GetTensorMutableData - -#if !defined(DISABLE_SPARSE_TENSORS) - /// - /// The API returns a pointer to an internal buffer of the sparse tensor - /// containing non-zero values. The API merely does casting. Make sure you - /// are requesting the right data type by calling GetSparseTensorValuesTypeAndShapeInfo() - /// first. - /// - /// numeric data types only. Use GetStringTensor*() to retrieve strings. - /// a pointer to the internal values buffer. Do not free this pointer. - template - const T* GetSparseTensorValues() const; -#endif - - template - T& At(const std::vector& location); - - /// - /// The API returns type information for data contained in a tensor. For sparse - /// tensors it returns type information for contained non-zero values. - /// It returns dense shape for sparse tensors. - /// - /// TypeInfo - TypeInfo GetTypeInfo() const; - - /// - /// The API returns type information for data contained in a tensor. For sparse - /// tensors it returns type information for contained non-zero values. - /// It returns dense shape for sparse tensors. - /// - /// TensorTypeAndShapeInfo - TensorTypeAndShapeInfo GetTensorTypeAndShapeInfo() const; - - /// - /// The API returns a byte length of UTF-8 encoded string element - /// contained in either a tensor or a spare tensor values. - /// - /// - /// byte length for the specified string element - size_t GetStringTensorElementLength(size_t element_index) const; - - /// - /// The API copies UTF-8 encoded bytes for the requested string element - /// contained within a tensor or a sparse tensor into a provided buffer. - /// Use GetStringTensorElementLength() to obtain the length of the buffer to allocate. - /// - /// - /// - /// - void GetStringTensorElement(size_t buffer_length, size_t element_index, void* buffer) const; - - void FillStringTensor(const char* const* s, size_t s_len); - void FillStringTensorElement(const char* s, size_t index); -}; - -// Represents native memory allocation -struct MemoryAllocation { - MemoryAllocation(OrtAllocator* allocator, void* p, size_t size); - ~MemoryAllocation(); - MemoryAllocation(const MemoryAllocation&) = delete; - MemoryAllocation& operator=(const MemoryAllocation&) = delete; - MemoryAllocation(MemoryAllocation&&) noexcept; - MemoryAllocation& operator=(MemoryAllocation&&) noexcept; - - void* get() { return p_; } - size_t size() const { return size_; } - - private: - OrtAllocator* allocator_; - void* p_; - size_t size_; -}; - -struct AllocatorWithDefaultOptions { - AllocatorWithDefaultOptions(); - - operator OrtAllocator*() { return p_; } - operator const OrtAllocator*() const { return p_; } - - void* Alloc(size_t size); - // The return value will own the allocation - MemoryAllocation GetAllocation(size_t size); - void Free(void* p); - - const OrtMemoryInfo* GetInfo() const; - - private: - OrtAllocator* p_{}; -}; - -struct MemoryInfo : Base { - static MemoryInfo CreateCpu(OrtAllocatorType type, OrtMemType mem_type1); - - explicit MemoryInfo(std::nullptr_t) {} - explicit MemoryInfo(OrtMemoryInfo* p) : Base{p} {} ///< Used for interop with the C API - MemoryInfo(const char* name, OrtAllocatorType type, int id, OrtMemType mem_type); - - std::string GetAllocatorName() const; - OrtAllocatorType GetAllocatorType() const; - int GetDeviceId() const; - OrtMemType GetMemoryType() const; - - bool operator==(const MemoryInfo& o) const; -}; - -struct Allocator : public Base { - Allocator(const Session& session, const MemoryInfo&); - - void* Alloc(size_t size) const; - // The return value will own the allocation - MemoryAllocation GetAllocation(size_t size); - void Free(void* p) const; - Unowned GetInfo() const; -}; - -struct IoBinding : public Base { - explicit IoBinding(Session& session); - void BindInput(const char* name, const Value&); - void BindOutput(const char* name, const Value&); - void BindOutput(const char* name, const MemoryInfo&); - std::vector GetOutputNames() const; - std::vector GetOutputNames(Allocator&) const; - std::vector GetOutputValues() const; - std::vector GetOutputValues(Allocator&) const; - void ClearBoundInputs(); - void ClearBoundOutputs(); - void SynchronizeInputs(); - void SynchronizeOutputs(); - - private: - std::vector GetOutputNamesHelper(OrtAllocator*) const; - std::vector GetOutputValuesHelper(OrtAllocator*) const; -}; - -/*! \struct Ort::ArenaCfg - * \brief it is a structure that represents the configuration of an arena based allocator - * \details Please see docs/C_API.md for details - */ -struct ArenaCfg : Base { - explicit ArenaCfg(std::nullptr_t) {} ///< Create an empty ArenaCfg object, must be assigned a valid one to be used - /** - * Wraps OrtApi::CreateArenaCfg - * \param max_mem - use 0 to allow ORT to choose the default - * \param arena_extend_strategy - use -1 to allow ORT to choose the default, 0 = kNextPowerOfTwo, 1 = kSameAsRequested - * \param initial_chunk_size_bytes - use -1 to allow ORT to choose the default - * \param max_dead_bytes_per_chunk - use -1 to allow ORT to choose the default - * See docs/C_API.md for details on what the following parameters mean and how to choose these values - */ - ArenaCfg(size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, int max_dead_bytes_per_chunk); -}; - -// -// Custom OPs (only needed to implement custom OPs) -// - -struct CustomOpApi { - CustomOpApi(const OrtApi& api) : api_(api) {} - - template // T is only implemented for std::vector, std::vector, float, int64_t, and string - T KernelInfoGetAttribute(_In_ const OrtKernelInfo* info, _In_ const char* name); - - OrtTensorTypeAndShapeInfo* GetTensorTypeAndShape(_In_ const OrtValue* value); - size_t GetTensorShapeElementCount(_In_ const OrtTensorTypeAndShapeInfo* info); - ONNXTensorElementDataType GetTensorElementType(const OrtTensorTypeAndShapeInfo* info); - size_t GetDimensionsCount(_In_ const OrtTensorTypeAndShapeInfo* info); - void GetDimensions(_In_ const OrtTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values, size_t dim_values_length); - void SetDimensions(OrtTensorTypeAndShapeInfo* info, _In_ const int64_t* dim_values, size_t dim_count); - - template - T* GetTensorMutableData(_Inout_ OrtValue* value); - template - const T* GetTensorData(_Inout_ const OrtValue* value); - - const OrtMemoryInfo* GetTensorMemoryInfo(_In_ const OrtValue* value); - - std::vector GetTensorShape(const OrtTensorTypeAndShapeInfo* info); - void ReleaseTensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo* input); - size_t KernelContext_GetInputCount(const OrtKernelContext* context); - const OrtValue* KernelContext_GetInput(const OrtKernelContext* context, _In_ size_t index); - size_t KernelContext_GetOutputCount(const OrtKernelContext* context); - OrtValue* KernelContext_GetOutput(OrtKernelContext* context, _In_ size_t index, _In_ const int64_t* dim_values, size_t dim_count); - void* KernelContext_GetGPUComputeStream(const OrtKernelContext* context); - - void ThrowOnError(OrtStatus* result); - - OrtOpAttr* CreateOpAttr(_In_ const char* name, - _In_ const void* data, - _In_ int len, - _In_ OrtOpAttrType type); - - void ReleaseOpAttr(_Frees_ptr_opt_ OrtOpAttr* op_attr); - - OrtOp* CreateOp(_In_ const OrtKernelInfo* info, - _In_ const char* op_name, - _In_ const char* domain, - _In_ int version, - _In_opt_ const char** type_constraint_names, - _In_opt_ const ONNXTensorElementDataType* type_constraint_values, - _In_opt_ int type_constraint_count, - _In_opt_ const OrtOpAttr* const* attr_values, - _In_opt_ int attr_count, - _In_ int input_count, - _In_ int output_count); - - void InvokeOp(_In_ const OrtKernelContext* context, - _In_ const OrtOp* ort_op, - _In_ const OrtValue* const* input_values, - _In_ int input_count, - _Inout_ OrtValue* const* output_values, - _In_ int output_count); - - void ReleaseOp(_Frees_ptr_opt_ OrtOp* ort_op); - - OrtKernelInfo* CopyKernelInfo(_In_ const OrtKernelInfo* info); - - void ReleaseKernelInfo(_Frees_ptr_opt_ OrtKernelInfo* info_copy); - - private: - const OrtApi& api_; -}; - -template -struct CustomOpBase : OrtCustomOp { - CustomOpBase() { - OrtCustomOp::version = ORT_API_VERSION; - OrtCustomOp::CreateKernel = [](const OrtCustomOp* this_, const OrtApi* api, const OrtKernelInfo* info) { return static_cast(this_)->CreateKernel(*api, info); }; - OrtCustomOp::GetName = [](const OrtCustomOp* this_) { return static_cast(this_)->GetName(); }; - - OrtCustomOp::GetExecutionProviderType = [](const OrtCustomOp* this_) { return static_cast(this_)->GetExecutionProviderType(); }; - - OrtCustomOp::GetInputTypeCount = [](const OrtCustomOp* this_) { return static_cast(this_)->GetInputTypeCount(); }; - OrtCustomOp::GetInputType = [](const OrtCustomOp* this_, size_t index) { return static_cast(this_)->GetInputType(index); }; - - OrtCustomOp::GetOutputTypeCount = [](const OrtCustomOp* this_) { return static_cast(this_)->GetOutputTypeCount(); }; - OrtCustomOp::GetOutputType = [](const OrtCustomOp* this_, size_t index) { return static_cast(this_)->GetOutputType(index); }; - - OrtCustomOp::KernelCompute = [](void* op_kernel, OrtKernelContext* context) { static_cast(op_kernel)->Compute(context); }; -#if defined(_MSC_VER) && !defined(__clang__) -#pragma warning(push) -#pragma warning(disable : 26409) -#endif - OrtCustomOp::KernelDestroy = [](void* op_kernel) { delete static_cast(op_kernel); }; -#if defined(_MSC_VER) && !defined(__clang__) -#pragma warning(pop) -#endif - OrtCustomOp::GetInputCharacteristic = [](const OrtCustomOp* this_, size_t index) { return static_cast(this_)->GetInputCharacteristic(index); }; - OrtCustomOp::GetOutputCharacteristic = [](const OrtCustomOp* this_, size_t index) { return static_cast(this_)->GetOutputCharacteristic(index); }; - } - - // Default implementation of GetExecutionProviderType that returns nullptr to default to the CPU provider - const char* GetExecutionProviderType() const { return nullptr; } - - // Default implementations of GetInputCharacteristic() and GetOutputCharacteristic() below - // (inputs and outputs are required by default) - OrtCustomOpInputOutputCharacteristic GetInputCharacteristic(size_t /*index*/) const { - return OrtCustomOpInputOutputCharacteristic::INPUT_OUTPUT_REQUIRED; - } - - OrtCustomOpInputOutputCharacteristic GetOutputCharacteristic(size_t /*index*/) const { - return OrtCustomOpInputOutputCharacteristic::INPUT_OUTPUT_REQUIRED; - } -}; - -} // namespace Ort - -#include "onnxruntime_cxx_inline.h" diff --git a/examples/onnxruntime_demo/main/onnxruntime-src/include/onnxruntime_cxx_inline.h b/examples/onnxruntime_demo/main/onnxruntime-src/include/onnxruntime_cxx_inline.h deleted file mode 100644 index 9a572fa8..00000000 --- a/examples/onnxruntime_demo/main/onnxruntime-src/include/onnxruntime_cxx_inline.h +++ /dev/null @@ -1,1385 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -// Do not include this file directly. Please include "onnxruntime_cxx_api.h" instead. -// If interested in trying out features of the new experimental C++ API, include "experimental_onnxruntime_cxx_api.h" instead. -// -// These are the inline implementations of the C++ header APIs. They're in this separate file as to not clutter -// the main C++ file with implementation details. - -namespace Ort { - -inline void ThrowOnError(const OrtApi& ort, OrtStatus* status) { - if (status) { - std::string error_message = ort.GetErrorMessage(status); - OrtErrorCode error_code = ort.GetErrorCode(status); - ort.ReleaseStatus(status); - ORT_CXX_API_THROW(std::move(error_message), error_code); - } -} - -inline void ThrowOnError(OrtStatus* status) { - ThrowOnError(GetApi(), status); -} - -// This template converts a C++ type into it's ONNXTensorElementDataType -template -struct TypeToTensorType; -template <> -struct TypeToTensorType { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; }; -template <> -struct TypeToTensorType { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16; }; -template <> -struct TypeToTensorType { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16; }; -template <> -struct TypeToTensorType { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE; }; -template <> -struct TypeToTensorType { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8; }; -template <> -struct TypeToTensorType { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16; }; -template <> -struct TypeToTensorType { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32; }; -template <> -struct TypeToTensorType { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64; }; -template <> -struct TypeToTensorType { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8; }; -template <> -struct TypeToTensorType { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16; }; -template <> -struct TypeToTensorType { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32; }; -template <> -struct TypeToTensorType { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64; }; -template <> -struct TypeToTensorType { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL; }; - -inline MemoryAllocation::MemoryAllocation(OrtAllocator* allocator, void* p, size_t size) - : allocator_(allocator), p_(p), size_(size) { -} - -inline MemoryAllocation::~MemoryAllocation() { - if (p_ != nullptr) { - // We do not throw out of destructor - auto ret = GetApi().AllocatorFree(allocator_, p_); - static_cast(ret); - } -} - -inline MemoryAllocation::MemoryAllocation(MemoryAllocation&& o) noexcept : allocator_(nullptr), p_(nullptr), size_(0) { - *this = std::move(o); -} - -inline MemoryAllocation& MemoryAllocation::operator=(MemoryAllocation&& o) noexcept { - OrtAllocator* alloc = nullptr; - void* p = nullptr; - size_t sz = 0; - - // Swap out this - std::swap(alloc, allocator_); - std::swap(p, p_); - std::swap(sz, size_); - - // Swap with incoming - std::swap(allocator_, o.allocator_); - std::swap(p_, o.p_); - std::swap(size_, o.size_); - - // Destroy this instance if needed - MemoryAllocation this_alloc(alloc, p, sz); - return *this; -} - -inline AllocatorWithDefaultOptions::AllocatorWithDefaultOptions() { - ThrowOnError(GetApi().GetAllocatorWithDefaultOptions(&p_)); -} - -inline void* AllocatorWithDefaultOptions::Alloc(size_t size) { - void* out; - ThrowOnError(GetApi().AllocatorAlloc(p_, size, &out)); - return out; -} - -inline MemoryAllocation Ort::AllocatorWithDefaultOptions::GetAllocation(size_t size) { - void* out; - ThrowOnError(GetApi().AllocatorAlloc(p_, size, &out)); - MemoryAllocation result(p_, out, size); - return result; -} - -inline void AllocatorWithDefaultOptions::Free(void* p) { - ThrowOnError(GetApi().AllocatorFree(p_, p)); -} - -inline const OrtMemoryInfo* AllocatorWithDefaultOptions::GetInfo() const { - const OrtMemoryInfo* out; - ThrowOnError(GetApi().AllocatorGetInfo(p_, &out)); - return out; -} - -inline std::string MemoryInfo::GetAllocatorName() const { - const char* name = nullptr; - ThrowOnError(GetApi().MemoryInfoGetName(*this, &name)); - return std::string(name); -} - -inline OrtAllocatorType MemoryInfo::GetAllocatorType() const { - OrtAllocatorType type; - ThrowOnError(GetApi().MemoryInfoGetType(*this, &type)); - return type; -} - -inline int MemoryInfo::GetDeviceId() const { - int id = 0; - ThrowOnError(GetApi().MemoryInfoGetId(*this, &id)); - return id; -} - -inline OrtMemType MemoryInfo::GetMemoryType() const { - OrtMemType type; - ThrowOnError(GetApi().MemoryInfoGetMemType(*this, &type)); - return type; -} - -inline bool MemoryInfo::operator==(const MemoryInfo& o) const { - int comp_result = 0; - ThrowOnError(Ort::GetApi().CompareMemoryInfo(*this, o, &comp_result)); - return comp_result == 0; -} - -inline MemoryInfo MemoryInfo::CreateCpu(OrtAllocatorType type, OrtMemType mem_type) { - OrtMemoryInfo* p; - ThrowOnError(GetApi().CreateCpuMemoryInfo(type, mem_type, &p)); - return MemoryInfo(p); -} - -inline MemoryInfo::MemoryInfo(const char* name, OrtAllocatorType type, int id, OrtMemType mem_type) { - ThrowOnError(GetApi().CreateMemoryInfo(name, type, id, mem_type, &p_)); -} - -inline Allocator::Allocator(const Session& sess, const MemoryInfo& mem_info) { - ThrowOnError(GetApi().CreateAllocator(sess, mem_info, &p_)); -} - -inline void* Allocator::Alloc(size_t size) const { - void* out = nullptr; - ThrowOnError(GetApi().AllocatorAlloc(p_, size, &out)); - return out; -} - -inline MemoryAllocation Ort::Allocator::GetAllocation(size_t size) { - void* out = nullptr; - ThrowOnError(GetApi().AllocatorAlloc(p_, size, &out)); - MemoryAllocation result(p_, out, size); - return result; -} - -inline void Allocator::Free(void* p) const { - ThrowOnError(GetApi().AllocatorFree(p_, p)); -} - -inline Unowned Allocator::GetInfo() const { - const OrtMemoryInfo* out = nullptr; - ThrowOnError(GetApi().AllocatorGetInfo(p_, &out)); - return Unowned(const_cast(out)); -} - -inline IoBinding::IoBinding(Session& session) { - ThrowOnError(GetApi().CreateIoBinding(session, &p_)); -} - -inline void IoBinding::BindInput(const char* name, const Value& value) { - ThrowOnError(GetApi().BindInput(p_, name, value)); -} - -inline void IoBinding::BindOutput(const char* name, const Value& value) { - ThrowOnError(GetApi().BindOutput(p_, name, value)); -} - -inline void IoBinding::BindOutput(const char* name, const MemoryInfo& mem_info) { - ThrowOnError(GetApi().BindOutputToDevice(p_, name, mem_info)); -} - -inline std::vector IoBinding::GetOutputNamesHelper(OrtAllocator* allocator) const { - std::vector result; - auto free_fn = detail::AllocatedFree(allocator); - using Ptr = std::unique_ptr; - - char* buffer = nullptr; - size_t* lengths = nullptr; - size_t count = 0; - ThrowOnError(GetApi().GetBoundOutputNames(p_, allocator, &buffer, &lengths, &count)); - - if (count == 0) { - return result; - } - - Ptr buffer_g(buffer, free_fn); - Ptr lengths_g(lengths, free_fn); - - result.reserve(count); - for (size_t i = 0; i < count; ++i) { - auto sz = *lengths; - result.emplace_back(buffer, sz); - buffer += sz; - ++lengths; - } - return result; -} - -inline std::vector IoBinding::GetOutputNames() const { - AllocatorWithDefaultOptions allocator; - return GetOutputNamesHelper(allocator); -} - -inline std::vector IoBinding::GetOutputNames(Allocator& allocator) const { - return GetOutputNamesHelper(allocator); -} - -inline std::vector Ort::IoBinding::GetOutputValuesHelper(OrtAllocator* allocator) const { - std::vector result; - size_t owned = 0; - size_t output_count = 0; - // Lambda to release the buffer when no longer needed and - // make sure that we destroy all instances on exception - auto free_fn = [&owned, &output_count, allocator](OrtValue** buffer) { - if (buffer) { - while (owned < output_count) { - auto* p = buffer + owned++; - GetApi().ReleaseValue(*p); - } - allocator->Free(allocator, buffer); - } - }; - using Ptr = std::unique_ptr; - - OrtValue** output_buffer = nullptr; - ThrowOnError(GetApi().GetBoundOutputValues(p_, allocator, &output_buffer, &output_count)); - if (output_count == 0) { - return result; - } - - Ptr buffer_g(output_buffer, free_fn); - - result.reserve(output_count); - for (size_t i = 0; i < output_count; ++i) { - result.emplace_back(output_buffer[i]); - ++owned; - } - return result; -} - -inline std::vector Ort::IoBinding::GetOutputValues(Allocator& allocator) const { - return GetOutputValuesHelper(allocator); -} - -inline std::vector Ort::IoBinding::GetOutputValues() const { - AllocatorWithDefaultOptions allocator; - return GetOutputValuesHelper(allocator); -} - -inline void IoBinding::ClearBoundInputs() { - GetApi().ClearBoundInputs(p_); -} - -inline void IoBinding::ClearBoundOutputs() { - GetApi().ClearBoundOutputs(p_); -} - -inline void IoBinding::SynchronizeInputs() { - ThrowOnError(GetApi().SynchronizeBoundInputs(p_)); -} - -inline void IoBinding::SynchronizeOutputs() { - ThrowOnError(GetApi().SynchronizeBoundOutputs(p_)); -} - -inline ArenaCfg::ArenaCfg(size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, int max_dead_bytes_per_chunk) { - ThrowOnError(GetApi().CreateArenaCfg(max_mem, arena_extend_strategy, initial_chunk_size_bytes, max_dead_bytes_per_chunk, &p_)); -} - -inline Env::Env(OrtLoggingLevel logging_level, _In_ const char* logid) { - ThrowOnError(GetApi().CreateEnv(logging_level, logid, &p_)); - if (strcmp(logid, "onnxruntime-node") == 0) { - ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS)); - } else { - ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS)); - } -} - -inline Env::Env(OrtLoggingLevel logging_level, const char* logid, OrtLoggingFunction logging_function, void* logger_param) { - ThrowOnError(GetApi().CreateEnvWithCustomLogger(logging_function, logger_param, logging_level, logid, &p_)); - if (strcmp(logid, "onnxruntime-node") == 0) { - ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS)); - } else { - ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS)); - } -} - -inline Env::Env(const OrtThreadingOptions* tp_options, OrtLoggingLevel logging_level, _In_ const char* logid) { - ThrowOnError(GetApi().CreateEnvWithGlobalThreadPools(logging_level, logid, tp_options, &p_)); - if (strcmp(logid, "onnxruntime-node") == 0) { - ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS)); - } else { - ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS)); - } -} - -inline Env::Env(const OrtThreadingOptions* tp_options, OrtLoggingFunction logging_function, void* logger_param, - OrtLoggingLevel logging_level, _In_ const char* logid) { - ThrowOnError(GetApi().CreateEnvWithCustomLoggerAndGlobalThreadPools(logging_function, logger_param, logging_level, logid, tp_options, &p_)); - if (strcmp(logid, "onnxruntime-node") == 0) { - ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS)); - } else { - ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS)); - } -} - -inline Env& Env::EnableTelemetryEvents() { - ThrowOnError(GetApi().EnableTelemetryEvents(p_)); - return *this; -} - -inline Env& Env::DisableTelemetryEvents() { - ThrowOnError(GetApi().DisableTelemetryEvents(p_)); - return *this; -} - -inline Env& Env::CreateAndRegisterAllocator(const OrtMemoryInfo* mem_info, const OrtArenaCfg* arena_cfg) { - ThrowOnError(GetApi().CreateAndRegisterAllocator(p_, mem_info, arena_cfg)); - return *this; -} - -inline CustomOpDomain::CustomOpDomain(const char* domain) { - ThrowOnError(GetApi().CreateCustomOpDomain(domain, &p_)); -} - -inline void CustomOpDomain::Add(OrtCustomOp* op) { - ThrowOnError(GetApi().CustomOpDomain_Add(p_, op)); -} - -inline RunOptions::RunOptions() { - ThrowOnError(GetApi().CreateRunOptions(&p_)); -} - -inline RunOptions& RunOptions::SetRunLogVerbosityLevel(int level) { - ThrowOnError(GetApi().RunOptionsSetRunLogVerbosityLevel(p_, level)); - return *this; -} - -inline RunOptions& RunOptions::SetRunLogSeverityLevel(int level) { - ThrowOnError(GetApi().RunOptionsSetRunLogSeverityLevel(p_, level)); - return *this; -} - -inline int RunOptions::GetRunLogVerbosityLevel() const { - int out; - ThrowOnError(GetApi().RunOptionsGetRunLogVerbosityLevel(p_, &out)); - return out; -} - -inline int RunOptions::GetRunLogSeverityLevel() const { - int out; - ThrowOnError(GetApi().RunOptionsGetRunLogSeverityLevel(p_, &out)); - return out; -} - -inline RunOptions& RunOptions::SetRunTag(const char* run_tag) { - ThrowOnError(GetApi().RunOptionsSetRunTag(p_, run_tag)); - return *this; -} - -inline const char* RunOptions::GetRunTag() const { - const char* out; - ThrowOnError(GetApi().RunOptionsGetRunTag(p_, &out)); - return out; -} - -inline RunOptions& RunOptions::AddConfigEntry(const char* config_key, const char* config_value) { - ThrowOnError(GetApi().AddRunConfigEntry(p_, config_key, config_value)); - return *this; -} - -inline RunOptions& RunOptions::SetTerminate() { - ThrowOnError(GetApi().RunOptionsSetTerminate(p_)); - return *this; -} - -inline RunOptions& RunOptions::UnsetTerminate() { - ThrowOnError(GetApi().RunOptionsUnsetTerminate(p_)); - return *this; -} - -inline SessionOptions::SessionOptions() { - ThrowOnError(GetApi().CreateSessionOptions(&p_)); -} - -inline SessionOptions SessionOptions::Clone() const { - OrtSessionOptions* out; - ThrowOnError(GetApi().CloneSessionOptions(p_, &out)); - return SessionOptions{out}; -} - -inline SessionOptions& SessionOptions::SetIntraOpNumThreads(int intra_op_num_threads) { - ThrowOnError(GetApi().SetIntraOpNumThreads(p_, intra_op_num_threads)); - return *this; -} - -inline SessionOptions& SessionOptions::SetInterOpNumThreads(int inter_op_num_threads) { - ThrowOnError(GetApi().SetInterOpNumThreads(p_, inter_op_num_threads)); - return *this; -} - -inline SessionOptions& SessionOptions::SetGraphOptimizationLevel(GraphOptimizationLevel graph_optimization_level) { - ThrowOnError(GetApi().SetSessionGraphOptimizationLevel(p_, graph_optimization_level)); - return *this; -} - -inline SessionOptions& SessionOptions::SetOptimizedModelFilePath(const ORTCHAR_T* optimized_model_filepath) { - ThrowOnError(GetApi().SetOptimizedModelFilePath(p_, optimized_model_filepath)); - return *this; -} - -inline SessionOptions& SessionOptions::EnableProfiling(const ORTCHAR_T* profile_file_prefix) { - ThrowOnError(GetApi().EnableProfiling(p_, profile_file_prefix)); - return *this; -} - -inline SessionOptions& SessionOptions::DisableProfiling() { - ThrowOnError(GetApi().DisableProfiling(p_)); - return *this; -} - -inline SessionOptions& SessionOptions::EnableOrtCustomOps() { - ThrowOnError(GetApi().EnableOrtCustomOps(p_)); - return *this; -} - -inline SessionOptions& SessionOptions::EnableMemPattern() { - ThrowOnError(GetApi().EnableMemPattern(p_)); - return *this; -} - -inline SessionOptions& SessionOptions::DisableMemPattern() { - ThrowOnError(GetApi().DisableMemPattern(p_)); - return *this; -} - -inline SessionOptions& SessionOptions::EnableCpuMemArena() { - ThrowOnError(GetApi().EnableCpuMemArena(p_)); - return *this; -} - -inline SessionOptions& SessionOptions::DisableCpuMemArena() { - ThrowOnError(GetApi().DisableCpuMemArena(p_)); - return *this; -} - -inline SessionOptions& SessionOptions::SetExecutionMode(ExecutionMode execution_mode) { - ThrowOnError(GetApi().SetSessionExecutionMode(p_, execution_mode)); - return *this; -} - -inline SessionOptions& SessionOptions::SetLogId(const char* logid) { - ThrowOnError(GetApi().SetSessionLogId(p_, logid)); - return *this; -} - -inline SessionOptions& SessionOptions::SetLogSeverityLevel(int level) { - ThrowOnError(GetApi().SetSessionLogSeverityLevel(p_, level)); - return *this; -} - -inline SessionOptions& SessionOptions::Add(OrtCustomOpDomain* custom_op_domain) { - ThrowOnError(GetApi().AddCustomOpDomain(p_, custom_op_domain)); - return *this; -} - -inline SessionOptions& SessionOptions::AddConfigEntry(const char* config_key, const char* config_value) { - ThrowOnError(GetApi().AddSessionConfigEntry(p_, config_key, config_value)); - return *this; -} - -inline SessionOptions& SessionOptions::AddInitializer(const char* name, const OrtValue* ort_val) { - ThrowOnError(GetApi().AddInitializer(p_, name, ort_val)); - return *this; -} - -inline SessionOptions& SessionOptions::AddExternalInitializers(const std::vector& names, - const std::vector& ort_values) { - const size_t inputs_num = names.size(); - if (inputs_num != ort_values.size()) { - ORT_CXX_API_THROW("Expecting names and ort_values to have the same length", ORT_INVALID_ARGUMENT); - } - std::vector names_ptr; - std::vector ort_values_ptrs; - names_ptr.reserve(inputs_num); - ort_values_ptrs.reserve(inputs_num); - for (size_t i = 0; i < inputs_num; ++i) { - names_ptr.push_back(names[i].c_str()); - ort_values_ptrs.push_back(ort_values[i]); - } - ThrowOnError(GetApi().AddExternalInitializers(p_, names_ptr.data(), ort_values_ptrs.data(), inputs_num)); - return *this; -} - -inline SessionOptions& SessionOptions::AppendExecutionProvider_CUDA(const OrtCUDAProviderOptions& provider_options) { - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_CUDA(p_, &provider_options)); - return *this; -} - -inline SessionOptions& SessionOptions::AppendExecutionProvider_CUDA_V2(const OrtCUDAProviderOptionsV2& provider_options) { - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_CUDA_V2(p_, &provider_options)); - return *this; -} - -inline SessionOptions& SessionOptions::AppendExecutionProvider_ROCM(const OrtROCMProviderOptions& provider_options) { - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_ROCM(p_, &provider_options)); - return *this; -} - -inline SessionOptions& SessionOptions::AppendExecutionProvider_TensorRT(const OrtTensorRTProviderOptions& provider_options) { - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_TensorRT(p_, &provider_options)); - return *this; -} - -inline SessionOptions& SessionOptions::AppendExecutionProvider_TensorRT_V2(const OrtTensorRTProviderOptionsV2& provider_options) { - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_TensorRT_V2(p_, &provider_options)); - return *this; -} - -inline SessionOptions& SessionOptions::AppendExecutionProvider_MIGraphX(const OrtMIGraphXProviderOptions& provider_options) { - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_MIGraphX(p_, &provider_options)); - return *this; -} - -inline SessionOptions& SessionOptions::AppendExecutionProvider( - const std::string& provider_name, - const std::unordered_map& provider_options) { - auto num_entries = provider_options.size(); - std::vector keys, values; - if (num_entries > 0) { - keys.reserve(num_entries); - values.reserve(num_entries); - - for (const auto& entry : provider_options) { - keys.push_back(entry.first.c_str()); - values.push_back(entry.second.c_str()); - } - } - - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider(p_, provider_name.c_str(), - keys.data(), values.data(), num_entries)); - - return *this; -} - -inline SessionOptions& SessionOptions::SetCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn) { - ThrowOnError(GetApi().SessionOptionsSetCustomCreateThreadFn(p_, ort_custom_create_thread_fn)); - return *this; -} - -inline SessionOptions& SessionOptions::SetCustomThreadCreationOptions(void* ort_custom_thread_creation_options) { - ThrowOnError(GetApi().SessionOptionsSetCustomThreadCreationOptions(p_, ort_custom_thread_creation_options)); - return *this; -} - -inline SessionOptions& SessionOptions::SetCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn) { - ThrowOnError(GetApi().SessionOptionsSetCustomJoinThreadFn(p_, ort_custom_join_thread_fn)); - return *this; -} - -inline SessionOptions& SessionOptions::AppendExecutionProvider_OpenVINO(const OrtOpenVINOProviderOptions& provider_options) { - ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_OpenVINO(p_, &provider_options)); - return *this; -} - -inline Session::Session(Env& env, const ORTCHAR_T* model_path, const SessionOptions& options) { - ThrowOnError(GetApi().CreateSession(env, model_path, options, &p_)); -} - -inline Session::Session(Env& env, const ORTCHAR_T* model_path, const SessionOptions& options, - OrtPrepackedWeightsContainer* prepacked_weights_container) { - ThrowOnError(GetApi().CreateSessionWithPrepackedWeightsContainer(env, model_path, options, prepacked_weights_container, &p_)); -} - -inline Session::Session(Env& env, const void* model_data, size_t model_data_length, const SessionOptions& options) { - ThrowOnError(GetApi().CreateSessionFromArray(env, model_data, model_data_length, options, &p_)); -} - -inline std::vector Session::Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count, - const char* const* output_names, size_t output_names_count) { - std::vector output_values; - for (size_t i = 0; i < output_names_count; i++) - output_values.emplace_back(nullptr); - Run(run_options, input_names, input_values, input_count, output_names, output_values.data(), output_names_count); - return output_values; -} - -inline void Session::Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count, - const char* const* output_names, Value* output_values, size_t output_count) { - static_assert(sizeof(Value) == sizeof(OrtValue*), "Value is really just an array of OrtValue* in memory, so we can reinterpret_cast safely"); - auto ort_input_values = reinterpret_cast(const_cast(input_values)); - auto ort_output_values = reinterpret_cast(output_values); - ThrowOnError(GetApi().Run(p_, run_options, input_names, ort_input_values, input_count, output_names, output_count, ort_output_values)); -} - -inline void Session::Run(const RunOptions& run_options, const IoBinding& io_binding) { - ThrowOnError(GetApi().RunWithBinding(p_, run_options, io_binding)); -} - -inline size_t Session::GetInputCount() const { - size_t out; - ThrowOnError(GetApi().SessionGetInputCount(p_, &out)); - return out; -} - -inline size_t Session::GetOutputCount() const { - size_t out; - ThrowOnError(GetApi().SessionGetOutputCount(p_, &out)); - return out; -} - -inline size_t Session::GetOverridableInitializerCount() const { - size_t out; - ThrowOnError(GetApi().SessionGetOverridableInitializerCount(p_, &out)); - return out; -} - -inline char* Session::GetInputName(size_t index, OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().SessionGetInputName(p_, index, allocator, &out)); - return out; -} - -inline char* Session::GetOutputName(size_t index, OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().SessionGetOutputName(p_, index, allocator, &out)); - return out; -} - -inline AllocatedStringPtr Session::GetInputNameAllocated(size_t index, OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().SessionGetInputName(p_, index, allocator, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -inline AllocatedStringPtr Session::GetOutputNameAllocated(size_t index, OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().SessionGetOutputName(p_, index, allocator, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -inline char* Session::GetOverridableInitializerName(size_t index, OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().SessionGetOverridableInitializerName(p_, index, allocator, &out)); - return out; -} - -inline AllocatedStringPtr Session::GetOverridableInitializerNameAllocated(size_t index, OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().SessionGetOverridableInitializerName(p_, index, allocator, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -inline char* Session::EndProfiling(OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().SessionEndProfiling(p_, allocator, &out)); - return out; -} - -inline AllocatedStringPtr Session::EndProfilingAllocated(OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().SessionEndProfiling(p_, allocator, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -inline uint64_t Session::GetProfilingStartTimeNs() const { - uint64_t out; - ThrowOnError(GetApi().SessionGetProfilingStartTimeNs(p_, &out)); - return out; -} - -inline ModelMetadata Session::GetModelMetadata() const { - OrtModelMetadata* out; - ThrowOnError(GetApi().SessionGetModelMetadata(p_, &out)); - return ModelMetadata{out}; -} - -inline char* ModelMetadata::GetProducerName(OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().ModelMetadataGetProducerName(p_, allocator, &out)); - return out; -} - -inline AllocatedStringPtr ModelMetadata::GetProducerNameAllocated(OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().ModelMetadataGetProducerName(p_, allocator, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -inline char* ModelMetadata::GetGraphName(OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().ModelMetadataGetGraphName(p_, allocator, &out)); - return out; -} - -inline AllocatedStringPtr ModelMetadata::GetGraphNameAllocated(OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().ModelMetadataGetGraphName(p_, allocator, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -inline char* ModelMetadata::GetDomain(OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().ModelMetadataGetDomain(p_, allocator, &out)); - return out; -} - -inline AllocatedStringPtr ModelMetadata::GetDomainAllocated(OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().ModelMetadataGetDomain(p_, allocator, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -inline char* ModelMetadata::GetDescription(OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().ModelMetadataGetDescription(p_, allocator, &out)); - return out; -} - -inline AllocatedStringPtr Ort::ModelMetadata::GetDescriptionAllocated(OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().ModelMetadataGetDescription(p_, allocator, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -inline char* ModelMetadata::GetGraphDescription(OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().ModelMetadataGetGraphDescription(p_, allocator, &out)); - return out; -} - -inline AllocatedStringPtr ModelMetadata::GetGraphDescriptionAllocated(OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().ModelMetadataGetGraphDescription(p_, allocator, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -inline char* ModelMetadata::LookupCustomMetadataMap(const char* key, OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().ModelMetadataLookupCustomMetadataMap(p_, allocator, key, &out)); - return out; -} - -inline AllocatedStringPtr ModelMetadata::LookupCustomMetadataMapAllocated(const char* key, OrtAllocator* allocator) const { - char* out; - ThrowOnError(GetApi().ModelMetadataLookupCustomMetadataMap(p_, allocator, key, &out)); - return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); -} - -inline char** ModelMetadata::GetCustomMetadataMapKeys(OrtAllocator* allocator, _Out_ int64_t& num_keys) const { - char** out; - ThrowOnError(GetApi().ModelMetadataGetCustomMetadataMapKeys(p_, allocator, &out, &num_keys)); - return out; -} - -inline std::vector ModelMetadata::GetCustomMetadataMapKeysAllocated(OrtAllocator* allocator) const { - auto deletor = detail::AllocatedFree(allocator); - std::vector result; - - char** out = nullptr; - int64_t num_keys = 0; - ThrowOnError(GetApi().ModelMetadataGetCustomMetadataMapKeys(p_, allocator, &out, &num_keys)); - if (num_keys <= 0) { - return result; - } - - // array of pointers will be freed - std::unique_ptr array_guard(out, deletor); - // reserve may throw - auto strings_deletor = [&deletor, num_keys](char** out) { for(int64_t i = 0; i < num_keys; ++i) deletor(out[i]); }; - std::unique_ptr strings_guard(out, strings_deletor); - result.reserve(static_cast(num_keys)); - strings_guard.release(); - for (int64_t i = 0; i < num_keys; ++i) { - result.push_back(AllocatedStringPtr(out[i], deletor)); - } - - return result; -} - -inline int64_t ModelMetadata::GetVersion() const { - int64_t out; - ThrowOnError(GetApi().ModelMetadataGetVersion(p_, &out)); - return out; -} - -inline TypeInfo Session::GetInputTypeInfo(size_t index) const { - OrtTypeInfo* out; - ThrowOnError(GetApi().SessionGetInputTypeInfo(p_, index, &out)); - return TypeInfo{out}; -} - -inline TypeInfo Session::GetOutputTypeInfo(size_t index) const { - OrtTypeInfo* out; - ThrowOnError(GetApi().SessionGetOutputTypeInfo(p_, index, &out)); - return TypeInfo{out}; -} - -inline TypeInfo Session::GetOverridableInitializerTypeInfo(size_t index) const { - OrtTypeInfo* out; - ThrowOnError(GetApi().SessionGetOverridableInitializerTypeInfo(p_, index, &out)); - return TypeInfo{out}; -} - -inline ONNXTensorElementDataType TensorTypeAndShapeInfo::GetElementType() const { - ONNXTensorElementDataType out; - ThrowOnError(GetApi().GetTensorElementType(p_, &out)); - return out; -} - -inline size_t TensorTypeAndShapeInfo::GetElementCount() const { - size_t out; - ThrowOnError(GetApi().GetTensorShapeElementCount(p_, &out)); - return static_cast(out); -} - -inline size_t TensorTypeAndShapeInfo::GetDimensionsCount() const { - size_t out; - ThrowOnError(GetApi().GetDimensionsCount(p_, &out)); - return out; -} - -inline void TensorTypeAndShapeInfo::GetDimensions(int64_t* values, size_t values_count) const { - ThrowOnError(GetApi().GetDimensions(p_, values, values_count)); -} - -inline void TensorTypeAndShapeInfo::GetSymbolicDimensions(const char** values, size_t values_count) const { - ThrowOnError(GetApi().GetSymbolicDimensions(p_, values, values_count)); -} - -inline std::vector TensorTypeAndShapeInfo::GetShape() const { - std::vector out(GetDimensionsCount(), 0); - GetDimensions(out.data(), out.size()); - return out; -} - -inline Unowned TypeInfo::GetTensorTypeAndShapeInfo() const { - const OrtTensorTypeAndShapeInfo* out; - ThrowOnError(GetApi().CastTypeInfoToTensorInfo(p_, &out)); - return Unowned(const_cast(out)); -} - -inline Unowned TypeInfo::GetSequenceTypeInfo() const { - const OrtSequenceTypeInfo* out; - ThrowOnError(GetApi().CastTypeInfoToSequenceTypeInfo(p_, &out)); - return Unowned{const_cast(out)}; -} - -inline TypeInfo SequenceTypeInfo::GetSequenceElementType() const { - OrtTypeInfo* output; - ThrowOnError(GetApi().GetSequenceElementType(p_, &output)); - return TypeInfo{output}; -} - -inline Unowned TypeInfo::GetMapTypeInfo() const { - const OrtMapTypeInfo* out; - ThrowOnError(GetApi().CastTypeInfoToMapTypeInfo(p_, &out)); - return Unowned{const_cast(out)}; -} - -inline ONNXTensorElementDataType MapTypeInfo::GetMapKeyType() const { - ONNXTensorElementDataType out; - ThrowOnError(GetApi().GetMapKeyType(p_, &out)); - return out; -} - -inline TypeInfo MapTypeInfo::GetMapValueType() const { - OrtTypeInfo* output; - ThrowOnError(GetApi().GetMapValueType(p_, &output)); - return TypeInfo{output}; -} - -inline ONNXType TypeInfo::GetONNXType() const { - ONNXType out; - ThrowOnError(GetApi().GetOnnxTypeFromTypeInfo(p_, &out)); - return out; -} - -template -inline Value Value::CreateTensor(const OrtMemoryInfo* info, T* p_data, size_t p_data_element_count, const int64_t* shape, size_t shape_len) { - return CreateTensor(info, p_data, p_data_element_count * sizeof(T), shape, shape_len, TypeToTensorType::type); -} - -inline Value Value::CreateTensor(const OrtMemoryInfo* info, void* p_data, size_t p_data_byte_count, const int64_t* shape, size_t shape_len, - ONNXTensorElementDataType type) { - OrtValue* out; - ThrowOnError(GetApi().CreateTensorWithDataAsOrtValue(info, p_data, p_data_byte_count, shape, shape_len, type, &out)); - return Value{out}; -} - -#if !defined(DISABLE_SPARSE_TENSORS) -template -inline Value Value::CreateSparseTensor(const OrtMemoryInfo* info, T* p_data, const Shape& dense_shape, - const Shape& values_shape) { - return CreateSparseTensor(info, p_data, dense_shape, values_shape, TypeToTensorType::type); -} - -inline Value Value::CreateSparseTensor(const OrtMemoryInfo* info, void* p_data, const Shape& dense_shape, - const Shape& values_shape, ONNXTensorElementDataType type) { - OrtValue* out; - ThrowOnError(GetApi().CreateSparseTensorWithValuesAsOrtValue(info, p_data, dense_shape.shape, dense_shape.shape_len, - values_shape.shape, values_shape.shape_len, type, &out)); - return Value{out}; -} - -inline void Value::FillSparseTensorCoo(const OrtMemoryInfo* mem_info, const OrtSparseValuesParam& values_param, - const int64_t* indices_data, size_t indices_num) { - ThrowOnError(GetApi().FillSparseTensorCoo(p_, mem_info, values_param.values_shape, - values_param.values_shape_len, values_param.data.p_data, - indices_data, indices_num)); -} - -inline void Value::FillSparseTensorCsr(const OrtMemoryInfo* data_mem_info, - const OrtSparseValuesParam& values, - const int64_t* inner_indices_data, size_t inner_indices_num, - const int64_t* outer_indices_data, size_t outer_indices_num) { - ThrowOnError(GetApi().FillSparseTensorCsr(p_, data_mem_info, values.values_shape, values.values_shape_len, values.data.p_data, - inner_indices_data, inner_indices_num, - outer_indices_data, outer_indices_num)); -} - -inline void Value::FillSparseTensorBlockSparse(const OrtMemoryInfo* data_mem_info, - const OrtSparseValuesParam& values, - const Shape& indices_shape, - const int32_t* indices_data) { - ThrowOnError(GetApi().FillSparseTensorBlockSparse(p_, data_mem_info, values.values_shape, values.values_shape_len, values.data.p_data, - indices_shape.shape, indices_shape.shape_len, - indices_data)); -} - -inline void Value::UseCooIndices(int64_t* indices_data, size_t indices_num) { - ThrowOnError(GetApi().UseCooIndices(p_, indices_data, indices_num)); -} - -inline void Value::UseCsrIndices(int64_t* inner_data, size_t inner_num, int64_t* outer_data, size_t outer_num) { - ThrowOnError(GetApi().UseCsrIndices(p_, inner_data, inner_num, outer_data, outer_num)); -} - -inline void Value::UseBlockSparseIndices(const Shape& indices_shape, int32_t* indices_data) { - ThrowOnError(GetApi().UseBlockSparseIndices(p_, indices_shape.shape, indices_shape.shape_len, indices_data)); -} - -inline OrtSparseFormat Value::GetSparseFormat() const { - OrtSparseFormat format; - ThrowOnError(GetApi().GetSparseTensorFormat(p_, &format)); - return format; -} - -inline TensorTypeAndShapeInfo Value::GetSparseTensorValuesTypeAndShapeInfo() const { - OrtTensorTypeAndShapeInfo* output; - ThrowOnError(GetApi().GetSparseTensorValuesTypeAndShape(p_, &output)); - return TensorTypeAndShapeInfo{output}; -} - -inline TensorTypeAndShapeInfo Value::GetSparseTensorIndicesTypeShapeInfo(OrtSparseIndicesFormat indices_format) const { - OrtTensorTypeAndShapeInfo* output; - ThrowOnError(GetApi().GetSparseTensorIndicesTypeShape(p_, indices_format, &output)); - return TensorTypeAndShapeInfo{output}; -} - -template -inline const T* Value::GetSparseTensorIndicesData(OrtSparseIndicesFormat indices_format, size_t& num_indices) const { - const void* out; - ThrowOnError(GetApi().GetSparseTensorIndices(p_, indices_format, &num_indices, &out)); - return reinterpret_cast(out); -} -#endif // !defined(DISABLE_SPARSE_TENSORS) - -template -inline Value Value::CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len) { - return CreateTensor(allocator, shape, shape_len, TypeToTensorType::type); -} - -inline Value Value::CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type) { - OrtValue* out; - ThrowOnError(GetApi().CreateTensorAsOrtValue(allocator, shape, shape_len, type, &out)); - return Value{out}; -} - -#if !defined(DISABLE_SPARSE_TENSORS) -template -inline Value Value::CreateSparseTensor(OrtAllocator* allocator, const Shape& dense_shape) { - return CreateSparseTensor(allocator, dense_shape, TypeToTensorType::type); -} - -inline Value Value::CreateSparseTensor(OrtAllocator* allocator, const Shape& dense_shape, - ONNXTensorElementDataType type) { - OrtValue* out; - ThrowOnError(GetApi().CreateSparseTensorAsOrtValue(allocator, dense_shape.shape, dense_shape.shape_len, type, &out)); - return Value{out}; -} -#endif // !defined(DISABLE_SPARSE_TENSORS) - -inline Value Value::CreateMap(Value& keys, Value& values) { - OrtValue* out; - OrtValue* inputs[2] = {keys, values}; - ThrowOnError(GetApi().CreateValue(inputs, 2, ONNX_TYPE_MAP, &out)); - return Value{out}; -} - -inline Value Value::CreateSequence(std::vector& values) { - OrtValue* out; - std::vector values_ort{values.data(), values.data() + values.size()}; - ThrowOnError(GetApi().CreateValue(values_ort.data(), values_ort.size(), ONNX_TYPE_SEQUENCE, &out)); - return Value{out}; -} - -template -inline Value Value::CreateOpaque(const char* domain, const char* type_name, const T& data_container) { - OrtValue* out; - ThrowOnError(GetApi().CreateOpaqueValue(domain, type_name, &data_container, sizeof(T), &out)); - return Value{out}; -} - -template -inline void Value::GetOpaqueData(const char* domain, const char* type_name, T& out) const { - ThrowOnError(GetApi().GetOpaqueValue(domain, type_name, p_, &out, sizeof(T))); -} - -inline bool Value::IsTensor() const { - int out; - ThrowOnError(GetApi().IsTensor(p_, &out)); - return out != 0; -} - -inline bool Value::HasValue() const { - int out; - ThrowOnError(GetApi().HasValue(p_, &out)); - return out != 0; -} - -#if !defined(DISABLE_SPARSE_TENSORS) -inline bool Value::IsSparseTensor() const { - int out; - ThrowOnError(GetApi().IsSparseTensor(p_, &out)); - return out != 0; -} -#endif - -inline size_t Value::GetCount() const { - size_t out; - ThrowOnError(GetApi().GetValueCount(p_, &out)); - return out; -} - -inline Value Value::GetValue(int index, OrtAllocator* allocator) const { - OrtValue* out; - ThrowOnError(GetApi().GetValue(p_, index, allocator, &out)); - return Value{out}; -} - -inline size_t Value::GetStringTensorDataLength() const { - size_t out; - ThrowOnError(GetApi().GetStringTensorDataLength(p_, &out)); - return out; -} - -inline size_t Value::GetStringTensorElementLength(size_t element_index) const { - size_t out; - ThrowOnError(GetApi().GetStringTensorElementLength(p_, element_index, &out)); - return out; -} - -inline void Value::GetStringTensorContent(void* buffer, size_t buffer_length, size_t* offsets, size_t offsets_count) const { - ThrowOnError(GetApi().GetStringTensorContent(p_, buffer, buffer_length, offsets, offsets_count)); -} - -inline void Value::GetStringTensorElement(size_t buffer_length, size_t element_index, void* buffer) const { - ThrowOnError(GetApi().GetStringTensorElement(p_, buffer_length, element_index, buffer)); -} - -inline void Value::FillStringTensor(const char* const* s, size_t s_len) { - ThrowOnError(GetApi().FillStringTensor(p_, s, s_len)); -} - -inline void Value::FillStringTensorElement(const char* s, size_t index) { - ThrowOnError(GetApi().FillStringTensorElement(p_, s, index)); -} - -template -T* Value::GetTensorMutableData() { - T* out; - ThrowOnError(GetApi().GetTensorMutableData(p_, (void**)&out)); - return out; -} - -template -const T* Value::GetTensorData() const { - T* out; - ThrowOnError(GetApi().GetTensorMutableData(p_, (void**)&out)); - return out; -} - -#if !defined(DISABLE_SPARSE_TENSORS) -template -inline const T* Value::GetSparseTensorValues() const { - const void* out; - ThrowOnError(GetApi().GetSparseTensorValues(p_, &out)); - return reinterpret_cast(out); -} -#endif // !defined(DISABLE_SPARSE_TENSORS) - -template -inline T& Value::At(const std::vector& location) { - static_assert(!std::is_same::value, "this api does not support std::string"); - T* out; - ThrowOnError(GetApi().TensorAt(p_, location.data(), location.size(), (void**)&out)); - return *out; -} - -inline TypeInfo Value::GetTypeInfo() const { - OrtTypeInfo* output; - ThrowOnError(GetApi().GetTypeInfo(p_, &output)); - return TypeInfo{output}; -} - -inline TensorTypeAndShapeInfo Value::GetTensorTypeAndShapeInfo() const { - OrtTensorTypeAndShapeInfo* output; - ThrowOnError(GetApi().GetTensorTypeAndShape(p_, &output)); - return TensorTypeAndShapeInfo{output}; -} - -// -// Custom OP API Inlines -// -inline void CustomOpApi::ThrowOnError(OrtStatus* status) { - Ort::ThrowOnError(api_, status); -} - -template <> -inline float CustomOpApi::KernelInfoGetAttribute(_In_ const OrtKernelInfo* info, _In_ const char* name) { - float out; - ThrowOnError(api_.KernelInfoGetAttribute_float(info, name, &out)); - return out; -} - -template <> -inline int64_t CustomOpApi::KernelInfoGetAttribute(_In_ const OrtKernelInfo* info, _In_ const char* name) { - int64_t out; - ThrowOnError(api_.KernelInfoGetAttribute_int64(info, name, &out)); - return out; -} - -template <> -inline std::string CustomOpApi::KernelInfoGetAttribute(_In_ const OrtKernelInfo* info, _In_ const char* name) { - size_t size = 0; - std::string out; - - // Feed nullptr for the data buffer to query the true size of the string attribute - OrtStatus* status = api_.KernelInfoGetAttribute_string(info, name, nullptr, &size); - - if (status == nullptr) { - out.resize(size); - ThrowOnError(api_.KernelInfoGetAttribute_string(info, name, &out[0], &size)); - out.resize(size - 1); // remove the terminating character '\0' - } else { - ThrowOnError(status); - } - return out; -} - -template <> -inline std::vector CustomOpApi::KernelInfoGetAttribute(_In_ const OrtKernelInfo* info, _In_ const char* name) { - size_t size = 0; - std::vector out; - - // Feed nullptr for the data buffer to query the true size of the attribute - OrtStatus* status = api_.KernelInfoGetAttributeArray_float(info, name, nullptr, &size); - - if (status == nullptr) { - out.resize(size); - ThrowOnError(api_.KernelInfoGetAttributeArray_float(info, name, out.data(), &size)); - } else { - ThrowOnError(status); - } - return out; -} - -template <> -inline std::vector CustomOpApi::KernelInfoGetAttribute(_In_ const OrtKernelInfo* info, _In_ const char* name) { - size_t size = 0; - std::vector out; - - // Feed nullptr for the data buffer to query the true size of the attribute - OrtStatus* status = api_.KernelInfoGetAttributeArray_int64(info, name, nullptr, &size); - - if (status == nullptr) { - out.resize(size); - ThrowOnError(api_.KernelInfoGetAttributeArray_int64(info, name, out.data(), &size)); - } else { - ThrowOnError(status); - } - return out; -} -inline OrtTensorTypeAndShapeInfo* CustomOpApi::GetTensorTypeAndShape(_In_ const OrtValue* value) { - OrtTensorTypeAndShapeInfo* out; - ThrowOnError(api_.GetTensorTypeAndShape(value, &out)); - return out; -} - -inline size_t CustomOpApi::GetTensorShapeElementCount(_In_ const OrtTensorTypeAndShapeInfo* info) { - size_t out; - ThrowOnError(api_.GetTensorShapeElementCount(info, &out)); - return out; -} - -inline ONNXTensorElementDataType CustomOpApi::GetTensorElementType(const OrtTensorTypeAndShapeInfo* info) { - ONNXTensorElementDataType out; - ThrowOnError(api_.GetTensorElementType(info, &out)); - return out; -} - -inline size_t CustomOpApi::GetDimensionsCount(_In_ const OrtTensorTypeAndShapeInfo* info) { - size_t out; - ThrowOnError(api_.GetDimensionsCount(info, &out)); - return out; -} - -inline void CustomOpApi::GetDimensions(_In_ const OrtTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values, size_t dim_values_length) { - ThrowOnError(api_.GetDimensions(info, dim_values, dim_values_length)); -} - -inline void CustomOpApi::SetDimensions(OrtTensorTypeAndShapeInfo* info, _In_ const int64_t* dim_values, size_t dim_count) { - ThrowOnError(api_.SetDimensions(info, dim_values, dim_count)); -} - -template -inline T* CustomOpApi::GetTensorMutableData(_Inout_ OrtValue* value) { - T* data; - ThrowOnError(api_.GetTensorMutableData(value, reinterpret_cast(&data))); - return data; -} - -inline const OrtMemoryInfo* CustomOpApi::GetTensorMemoryInfo(_In_ const OrtValue* value) { - const OrtMemoryInfo* mem_info; - ThrowOnError(api_.GetTensorMemoryInfo(value, &mem_info)); - return mem_info; -} - -template -inline const T* CustomOpApi::GetTensorData(_Inout_ const OrtValue* value) { - return GetTensorMutableData(const_cast(value)); -} - -inline std::vector CustomOpApi::GetTensorShape(const OrtTensorTypeAndShapeInfo* info) { - std::vector output(GetDimensionsCount(info)); - GetDimensions(info, output.data(), output.size()); - return output; -} - -inline void CustomOpApi::ReleaseTensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo* input) { - api_.ReleaseTensorTypeAndShapeInfo(input); -} - -inline size_t CustomOpApi::KernelContext_GetInputCount(const OrtKernelContext* context) { - size_t out; - ThrowOnError(api_.KernelContext_GetInputCount(context, &out)); - return out; -} - -inline const OrtValue* CustomOpApi::KernelContext_GetInput(const OrtKernelContext* context, _In_ size_t index) { - const OrtValue* out; - ThrowOnError(api_.KernelContext_GetInput(context, index, &out)); - return out; -} - -inline size_t CustomOpApi::KernelContext_GetOutputCount(const OrtKernelContext* context) { - size_t out; - ThrowOnError(api_.KernelContext_GetOutputCount(context, &out)); - return out; -} - -inline OrtValue* CustomOpApi::KernelContext_GetOutput(OrtKernelContext* context, _In_ size_t index, - _In_ const int64_t* dim_values, size_t dim_count) { - OrtValue* out; - ThrowOnError(api_.KernelContext_GetOutput(context, index, dim_values, dim_count, &out)); - return out; -} - -inline void* CustomOpApi::KernelContext_GetGPUComputeStream(const OrtKernelContext* context) { - void* out; - ThrowOnError(api_.KernelContext_GetGPUComputeStream(context, &out)); - return out; -} - -inline OrtOpAttr* CustomOpApi::CreateOpAttr(_In_ const char* name, - _In_ const void* data, - _In_ int len, - _In_ OrtOpAttrType type) { - OrtOpAttr* op_attr{}; - ThrowOnError(api_.CreateOpAttr(name, data, len, type, &op_attr)); - return op_attr; -} - -inline void CustomOpApi::ReleaseOpAttr(_Frees_ptr_opt_ OrtOpAttr* op_attr) { - api_.ReleaseOpAttr(op_attr); -} - -inline OrtOp* CustomOpApi::CreateOp(_In_ const OrtKernelInfo* info, - _In_ const char* op_name, - _In_ const char* domain, - _In_ int version, - _In_opt_ const char** type_constraint_names, - _In_opt_ const ONNXTensorElementDataType* type_constraint_values, - _In_opt_ int type_constraint_count, - _In_opt_ const OrtOpAttr* const* attr_values, - _In_opt_ int attr_count, - _In_ int input_count, - _In_ int output_count) { - OrtOp* ort_op{}; - ThrowOnError(api_.CreateOp(info, op_name, domain, version, type_constraint_names, type_constraint_values, - type_constraint_count, attr_values, attr_count, input_count, output_count, &ort_op)); - return ort_op; -} - -inline void CustomOpApi::InvokeOp(_In_ const OrtKernelContext* context, - _In_ const OrtOp* ort_op, - _In_ const OrtValue* const* input_values, - _In_ int input_count, - _Inout_ OrtValue* const* output_values, - _In_ int output_count) { - ThrowOnError(api_.InvokeOp(context, ort_op, input_values, input_count, output_values, output_count)); -} - -inline void CustomOpApi::ReleaseOp(_Frees_ptr_opt_ OrtOp* ort_op) { - api_.ReleaseOp(ort_op); -} - -inline OrtKernelInfo* CustomOpApi::CopyKernelInfo(_In_ const OrtKernelInfo* info) { - OrtKernelInfo* info_copy{}; - ThrowOnError(api_.CopyKernelInfo(info, &info_copy)); - return info_copy; -} - -inline void CustomOpApi::ReleaseKernelInfo(_Frees_ptr_opt_ OrtKernelInfo* info_copy) { - api_.ReleaseKernelInfo(info_copy); -} - -inline SessionOptions& SessionOptions::DisablePerSessionThreads() { - ThrowOnError(GetApi().DisablePerSessionThreads(p_)); - return *this; -} - -inline std::vector GetAvailableProviders() { - int len; - char** providers; - const OrtApi& api = GetApi(); - ThrowOnError(api.GetAvailableProviders(&providers, &len)); - std::vector available_providers(providers, providers + len); - ThrowOnError(api.ReleaseAvailableProviders(providers, len)); - return available_providers; -} - -SessionOptions& AddInitializer(const char* name, const OrtValue* ort_val); - -} // namespace Ort