grpc_service.proto

// Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
//  * Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
//  * Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
//  * Neither the name of NVIDIA CORPORATION nor the names of its
//    contributors may be used to endorse or promote products derived
//    from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

syntax = "proto3";

package nvidia.inferenceserver;

//@@.. cpp:namespace:: nvidia::inferenceserver

import "api.proto";
import "request_status.proto";
import "server_status.proto";

//@@
//@@.. cpp:var:: service GRPCService
//@@
//@@   Inference Server GRPC endpoints.
//@@
service GRPCService
{
  //@@  .. cpp:var:: rpc Status(StatusRequest) returns (StatusResponse)
  //@@
  //@@     Get status for entire inference server or for a specified model.
  //@@
  rpc Status(StatusRequest) returns (StatusResponse) {}

  //@@  .. cpp:var:: rpc Health(HealthRequest) returns (HealthResponse)
  //@@
  //@@     Check liveness and readiness of the inference server.
  //@@
  rpc Health(HealthRequest) returns (HealthResponse) {}

  //@@  .. cpp:var:: rpc Infer(InferRequest) returns (InferResponse)
  //@@
  //@@     Request inference using a specific model. [ To handle large input
  //@@     tensors likely need to set the maximum message size to that they
  //@@     can be transmitted in one pass.
  //@@
  rpc Infer(InferRequest) returns (InferResponse) {}

  //@@  .. cpp:var:: rpc StreamInfer(stream InferRequest) returns (stream
  //@@     InferResponse)
  //@@
  //@@     Request inferences using a specific model in a streaming manner.
  //@@     Individual inference requests sent through the same stream will be
  //@@     processed in order and be returned on completion
  //@@
  rpc StreamInfer(stream InferRequest) returns (stream InferResponse) {}

  //@@  .. cpp:var:: rpc ModelControl(ModelControlRequest) returns
  //@@     (ModelControlResponse)
  //@@
  //@@     Request to load / unload a specified model.
  //@@
  rpc ModelControl(ModelControlRequest) returns (ModelControlResponse) {}

  //@@  .. cpp:var:: rpc SharedMemoryControl(SharedMemoryControlRequest) returns
  //@@     (SharedMemoryControlResponse)
  //@@
  //@@     Request to register / unregister a specified shared memory region.
  //@@
  rpc SharedMemoryControl(SharedMemoryControlRequest)
      returns (SharedMemoryControlResponse)
  {
  }

  //@@  .. cpp:var:: rpc Status(RepositoryRequest) returns (RepositoryResponse)
  //@@
  //@@     Get status associated with the model repository.
  //@@
  rpc Repository(RepositoryRequest) returns (RepositoryResponse) {}
}

//@@
//@@.. cpp:var:: message StatusRequest
//@@
//@@   Request message for Status gRPC endpoint.
//@@
message StatusRequest
{
  //@@
  //@@  .. cpp:var:: string model_name
  //@@
  //@@     The specific model status to be returned. If empty return status
  //@@     for all models.
  //@@
  string model_name = 1;
}

//@@
//@@.. cpp:var:: message StatusResponse
//@@
//@@   Response message for Status gRPC endpoint.
//@@
message StatusResponse
{
  //@@
  //@@  .. cpp:var:: RequestStatus request_status
  //@@
  //@@     The status of the request, indicating success or failure.
  //@@
  RequestStatus request_status = 1;

  //@@
  //@@  .. cpp:var:: ServerStatus server_status
  //@@
  //@@     The server and model status.
  //@@
  ServerStatus server_status = 2;
}

//@@
//@@.. cpp:var:: message HealthRequest
//@@
//@@   Request message for Health gRPC endpoint.
//@@
message HealthRequest
{
  //@@
  //@@  .. cpp:var:: string mode
  //@@
  //@@     The requested health action: 'live' requests the liveness
  //@@     state of the inference server; 'ready' requests the readiness state
  //@@     of the inference server.
  //@@
  string mode = 1;
}

//@@
//@@.. cpp:var:: message HealthResponse
//@@
//@@   Response message for Health gRPC endpoint.
//@@
message HealthResponse
{
  //@@
  //@@  .. cpp:var:: RequestStatus request_status
  //@@
  //@@     The status of the request, indicating success or failure.
  //@@
  RequestStatus request_status = 1;

  //@@
  //@@  .. cpp:var:: bool health
  //@@
  //@@     The result of the request. True indicates the inference server is
  //@@     live/ready, false indicates the inference server is not live/ready.
  //@@
  bool health = 2;
}

//@@
//@@.. cpp:var:: message ModelControlRequest
//@@
//@@   Request message for ModelControl gRPC endpoint.
//@@
message ModelControlRequest
{
  //@@  .. cpp:enum:: Type
  //@@
  //@@     Types of control operation
  //@@
  enum Type {
    //@@    .. cpp:enumerator:: Type::UNLOAD = 0
    //@@
    //@@       To unload the specified model.
    //@@
    UNLOAD = 0;

    //@@    .. cpp:enumerator:: Type::LOAD = 1
    //@@
    //@@       To load the specified model. If the model has been loaded,
    //@@       it will be reloaded to fetch the latest change.
    //@@
    LOAD = 1;
  }

  //@@
  //@@  .. cpp:var:: string model_name
  //@@
  //@@     The target model name.
  //@@
  string model_name = 1;

  //@@
  //@@  .. cpp:var:: Type type
  //@@
  //@@     The control type that is operated on the specified model.
  //@@
  Type type = 2;
}

//@@
//@@.. cpp:var:: message ModelControlResponse
//@@
//@@   Response message for ModelControl gRPC endpoint.
//@@
message ModelControlResponse
{
  //@@
  //@@  .. cpp:var:: RequestStatus request_status
  //@@
  //@@     The status of the request, indicating success or failure.
  //@@
  RequestStatus request_status = 1;
}

//@@
//@@.. cpp:var:: message SharedMemoryControlRequest
//@@
//@@   Request message for managing registered shared memory regions in TRTIS.
//@@
message SharedMemoryControlRequest
{
  //@@  .. cpp:var:: message Register
  //@@
  //@@     Register a shared memory region.
  //@@
  message Register
  {
    //@@
    //@@  .. cpp:var:: string name
    //@@
    //@@     The name for this shared memory region.
    //@@
    string name = 1;

    //@@
    //@@  .. cpp:var:: message SystemSharedMemoryIdentifier
    //@@
    //@@     The identifier for this system shared memory region.
    //@@
    message SystemSharedMemoryIdentifier
    {
      //@@  .. cpp:var:: string shared_memory_key
      //@@
      //@@     The name of the shared memory region that holds the input data
      //@@     (or where the output data should be written).
      //@@
      string shared_memory_key = 1;

      //@@  .. cpp:var:: uint64 offset
      //@@
      //@@     This is the offset of the shared memory block from the start
      //@@     of the shared memory region.
      //@@     start = offset, end = offset + byte_size;
      //@@
      uint64 offset = 2;
    }

    //@@
    //@@  .. cpp:var:: message CUDASharedMemoryIdentifier
    //@@
    //@@     The identifier for this system shared memory region.
    //@@
    message CUDASharedMemoryIdentifier
    {
      //@@  .. cpp:var:: bytes raw_handle
      //@@
      //@@     The raw serialized cudaIPC handle.
      //@@
      bytes raw_handle = 1;

      //@@  .. cpp:var:: int64 device_id
      //@@
      //@@     The GPU device ID on which the cudaIPC handle was created.
      //@@
      int64 device_id = 2;
    }

    //@@  .. cpp:var:: oneof shared_memory_types
    //@@
    //@@     Types of shared memory identifiers
    //@@
    oneof shared_memory_types
    {
      //@@
      //@@  .. cpp:var:: SystemSharedMemoryIdentifier system_shared_memory
      //@@
      //@@     The identifier for this system shared memory region.
      //@@
      SystemSharedMemoryIdentifier system_shared_memory = 2;

      //@@
      //@@  .. cpp:var:: CUDASharedMemoryIdentifier cuda_shared_memory
      //@@
      //@@     The identifier for this CUDA shared memory region.
      //@@
      CUDASharedMemoryIdentifier cuda_shared_memory = 3;
    }

    //@@  .. cpp:var:: uint64 byte_size
    //@@
    //@@     Size of the shared memory block, in bytes.
    //@@
    uint64 byte_size = 4;
  }

  //@@  .. cpp:var:: message Unregister
  //@@
  //@@     Unregister a specified shared memory region.
  //@@
  message Unregister
  {
    //@@
    //@@  .. cpp:var:: string name
    //@@
    //@@     The name for this shared memory region to unregister.
    //@@
    string name = 1;
  }

  //@@  .. cpp:var:: message UnregisterAll
  //@@
  //@@     Unregister all shared memory regions.
  //@@
  message UnregisterAll {}

  //@@  .. cpp:var:: message GetStatus
  //@@
  //@@     Get the status of all active shared memory regions.
  //@@
  message Status {}

  //@@  .. cpp:var:: oneof shared_memory_control
  //@@
  //@@     Types of control operations for shared memory
  //@@
  oneof shared_memory_control
  {
    //@@    .. cpp:var:: Register register
    //@@
    //@@       To register the specified shared memory region.
    //@@
    Register register = 1;

    //@@    .. cpp:var:: Unregister unregister
    //@@
    //@@       To unregister the specified shared memory region.
    //@@
    Unregister unregister = 2;

    //@@    .. cpp:var:: UnregisterAll unregister_all
    //@@
    //@@       To unregister all active shared memory regions.
    //@@
    UnregisterAll unregister_all = 3;

    //@@    .. cpp:var:: Status status
    //@@
    //@@       Get the status of all active shared memory regions.
    //@@
    Status status = 4;
  }
}

//@@
//@@.. cpp:var:: message SharedMemoryControlResponse
//@@
//@@   Response message for SharedMemoryControl gRPC endpoint.
//@@
message SharedMemoryControlResponse
{
  //@@
  //@@.. cpp:var:: message Status
  //@@
  //@@   Status of all active shared memory regions.
  //@@
  message Status
  {
    //@@
    //@@  .. cpp:var:: SharedMemoryRegion shared_memory_region
    //@@
    //@@     The list of active/registered shared memory regions.
    //@@
    repeated SharedMemoryRegion shared_memory_region = 1;
  }

  //@@
  //@@  .. cpp:var:: RequestStatus request_status
  //@@
  //@@     The status of the request, indicating success or failure.
  //@@
  RequestStatus request_status = 1;

  oneof shared_memory_control
  {
    //@@
    //@@  .. cpp:var:: Status shared_memory_status
    //@@
    //@@     The status of all active shared memory regions.
    //@@
    Status shared_memory_status = 2;
  }
}

//@@
//@@.. cpp:var:: message InferRequest
//@@
//@@   Request message for Infer gRPC endpoint.
//@@
message InferRequest
{
  //@@  .. cpp:var:: string model_name
  //@@
  //@@     The name of the model to use for inferencing.
  //@@
  string model_name = 1;

  //@@  .. cpp:var:: int64 version
  //@@
  //@@     The version of the model to use for inference. If -1
  //@@     the latest/most-recent version of the model is used.
  //@@
  int64 model_version = 2;

  //@@  .. cpp:var:: InferRequestHeader meta_data
  //@@
  //@@     Meta-data for the request: input tensors, output
  //@@     tensors, etc.
  //@@
  InferRequestHeader meta_data = 3;

  //@@  .. cpp:var:: bytes raw_input (repeated)
  //@@
  //@@     The raw input tensor data in the order specified in 'meta_data'.
  //@@
  repeated bytes raw_input = 4;
}

//@@
//@@.. cpp:var:: message InferResponse
//@@
//@@   Response message for Infer gRPC endpoint.
//@@
message InferResponse
{
  //@@
  //@@  .. cpp:var:: RequestStatus request_status
  //@@
  //@@     The status of the request, indicating success or failure.
  //@@
  RequestStatus request_status = 1;

  //@@  .. cpp:var:: InferResponseHeader meta_data
  //@@
  //@@     The response meta-data for the output tensors.
  //@@
  InferResponseHeader meta_data = 2;

  //@@  .. cpp:var:: bytes raw_output (repeated)
  //@@
  //@@     The raw output tensor data in the order specified in 'meta_data'.
  //@@
  repeated bytes raw_output = 3;
}

//@@
//@@.. cpp:var:: message RepositoryRequest
//@@
//@@   Request message for Repository gRPC endpoint.
//@@
message RepositoryRequest
{
  //@@  .. cpp:var:: oneof request_type
  //@@
  //@@     Types of the repository request
  //@@
  oneof request_type
  {
    //@@
    //@@    .. cpp:var:: bool index
    //@@
    //@@       Request for the index of the model repository.
    //@@
    bool index = 1;
  }
}

//@@
//@@.. cpp:var:: message RepositoryResponse
//@@
//@@   Response message for Repository gRPC endpoint.
//@@
message RepositoryResponse
{
  //@@
  //@@  .. cpp:var:: RequestStatus request_status
  //@@
  //@@     The status of the request, indicating success or failure.
  //@@
  RequestStatus request_status = 1;

  //@@  .. cpp:var:: oneof response_type
  //@@
  //@@     Types of the repository reponse, which is one-to-one mapping to
  //@@     the repository request type.
  //@@
  oneof response_type
  {
    //@@
    //@@    .. cpp:var:: bool index
    //@@
    //@@       The index of the model repository.
    //@@
    ModelRepositoryIndex index = 2;
  }
}