From ae4b9c4405ea230d4c4ea9ba88242d1546342660 Mon Sep 17 00:00:00 2001 From: Satvik Patil Date: Wed, 1 Jan 2025 15:40:38 +0530 Subject: [PATCH] add proto files to parent folder --- common.proto | 133 +++++++++++++++++++++++++++++++++++++++++ connector_sdk.proto | 109 ++++++++++++++++++++++++++++++++++ destination_sdk.proto | 134 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 376 insertions(+) create mode 100644 common.proto create mode 100644 connector_sdk.proto create mode 100644 destination_sdk.proto diff --git a/common.proto b/common.proto new file mode 100644 index 0000000..2f09d03 --- /dev/null +++ b/common.proto @@ -0,0 +1,133 @@ +syntax = "proto3"; +option optimize_for = SPEED; +option java_multiple_files = true; +option go_package = "fivetran.com/fivetran_sdk"; +package fivetran_sdk; + +import "google/protobuf/timestamp.proto"; + +message ConfigurationFormRequest {} + +message ConfigurationFormResponse { + bool schema_selection_supported = 1; + bool table_selection_supported = 2; + repeated FormField fields = 3; + repeated ConfigurationTest tests = 4; +} + +message FormField { + string name = 1; + string label = 2; + bool required = 3; + optional string description = 4; + oneof type { + TextField text_field = 5; + DropdownField dropdown_field = 6; + ToggleField toggle_field = 7; + } +} + +message DropdownField { + repeated string dropdown_field = 1; +} + +message ToggleField {} + +enum TextField { + PlainText = 0; + Password = 1; + Hidden = 2; +} + +message ConfigurationTest { + string name = 1; // unique identifier for the test + string label = 2; // A few words indicating what we are testing, e.g. 'Connecting to database' +} + +message TestRequest { + string name = 1; + map configuration = 2; +} + +message TestResponse { + oneof response { + bool success = 1; + string failure = 2; + // potential future warning + } +} + +message SchemaList { + repeated Schema schemas = 1; +} + +message TableList { + repeated Table tables = 1; +} + +message Schema { + string name = 1; + repeated Table tables = 2; +} + +enum DataType { + UNSPECIFIED = 0; + BOOLEAN = 1; + SHORT = 2; + INT = 3; + LONG = 4; + DECIMAL = 5; + FLOAT = 6; + DOUBLE = 7; + NAIVE_DATE = 8; + NAIVE_DATETIME = 9; + UTC_DATETIME = 10; + BINARY = 11; + XML = 12; + STRING = 13; + JSON = 14; +} + +message DecimalParams { + uint32 precision = 1; + uint32 scale = 2; +} + +enum OpType { + UPSERT = 0; + UPDATE = 1; + DELETE = 2; + TRUNCATE = 3; +} + +message ValueType { + oneof inner { + bool null = 1; + bool bool = 2; + int32 short = 3; + int32 int = 4; + int64 long = 5; + float float = 6; + double double = 7; + google.protobuf.Timestamp naive_date = 8; + google.protobuf.Timestamp naive_datetime = 9; + google.protobuf.Timestamp utc_datetime = 10; + string decimal = 11; + bytes binary = 12; + string string = 13; + string json = 14; + string xml = 15; + } +} + +message Table { + string name = 1; + repeated Column columns = 2; +} + +message Column { + string name = 1; + DataType type = 2; + bool primary_key = 3; + optional DecimalParams decimal = 4; +} diff --git a/connector_sdk.proto b/connector_sdk.proto new file mode 100644 index 0000000..4d906ee --- /dev/null +++ b/connector_sdk.proto @@ -0,0 +1,109 @@ +syntax = "proto3"; +option optimize_for = SPEED; +option java_multiple_files = true; +option go_package = "fivetran.com/fivetran_sdk"; +package fivetran_sdk; + +import "common.proto"; + +// Fivetran (grpc client) <> Connector (grpc server) +service Connector { + rpc ConfigurationForm (ConfigurationFormRequest) returns (ConfigurationFormResponse) {} + rpc Test (TestRequest) returns (TestResponse) {} + rpc Schema (SchemaRequest) returns (SchemaResponse) {} + rpc Update (UpdateRequest) returns (stream UpdateResponse) {} +} + +message SchemaRequest { + map configuration = 1; +} + +message SchemaResponse { + oneof response { + bool schema_response_not_supported = 1; + SchemaList with_schema = 2; + TableList without_schema = 3; + } + optional bool selection_not_supported = 4; +} + +message UpdateRequest { + map configuration = 1; + optional Selection selection = 2; + optional string state_json = 3; +} + +message Selection { + oneof selection { + TablesWithNoSchema without_schema = 1; + TablesWithSchema with_schema = 2; + } +} + +message TablesWithNoSchema { + repeated TableSelection tables = 1; + bool include_new_tables = 2; +} + +message TablesWithSchema { + repeated SchemaSelection schemas = 1; + bool include_new_schemas = 2; +} + +message SchemaSelection { + bool included = 1; + string schema_name = 2; + repeated TableSelection tables = 3; + bool include_new_tables = 4; +} + +message TableSelection { + bool included = 1; + string table_name = 2; + map columns = 3; + bool include_new_columns = 4; +} + +message UpdateResponse { + oneof response { + LogEntry log_entry = 1; + Operation operation = 2; + } +} + +enum LogLevel { + INFO = 0; + WARNING = 1; + SEVERE = 2; +} + +message LogEntry { + LogLevel level = 1; + string message = 2; +} + +message Operation { + oneof op { + Record record = 1; + SchemaChange schema_change = 2; + Checkpoint checkpoint = 3; + } +} + +message SchemaChange { + oneof change { + SchemaList with_schema = 1; + TableList without_schema = 2; + } +} + +message Record { + optional string schema_name = 1; + string table_name = 2; + OpType type = 3; + map data = 4; +} + +message Checkpoint { + string state_json = 1; +} \ No newline at end of file diff --git a/destination_sdk.proto b/destination_sdk.proto new file mode 100644 index 0000000..8c5635b --- /dev/null +++ b/destination_sdk.proto @@ -0,0 +1,134 @@ +syntax = "proto3"; +option optimize_for = SPEED; +option java_multiple_files = true; +option go_package = "fivetran.com/fivetran_sdk"; +package fivetran_sdk; + +import "google/protobuf/timestamp.proto"; +import "common.proto"; + +// Fivetran (grpc client) <> Destination (grpc server) +service Destination { + rpc ConfigurationForm (ConfigurationFormRequest) returns (ConfigurationFormResponse) {} + rpc Capabilities (CapabilitiesRequest) returns (CapabilitiesResponse) {} + rpc Test (TestRequest) returns (TestResponse) {} + rpc DescribeTable (DescribeTableRequest) returns (DescribeTableResponse) {} + rpc CreateTable(CreateTableRequest) returns (CreateTableResponse) {} + rpc AlterTable(AlterTableRequest) returns (AlterTableResponse) {} + rpc Truncate(TruncateRequest) returns (TruncateResponse) {} + rpc WriteBatch (WriteBatchRequest) returns (WriteBatchResponse) {} +} + +message CapabilitiesRequest {} + +message CapabilitiesResponse { + BatchFileFormat batch_file_format = 1; +} + +message DescribeTableRequest { + map configuration = 1; + string schema_name = 2; + string table_name = 3; +} + +message DescribeTableResponse { + oneof response { + bool not_found = 1; + string failure = 2; + Table table = 3; + } +} + +message CreateTableRequest { + map configuration = 1; + string schema_name = 2; + Table table = 3; +} + +message CreateTableResponse { + oneof response { + bool success = 1; + string failure = 2; + } +} + +message AlterTableRequest { + map configuration = 1; + string schema_name = 2; + Table table = 3; +} + +message AlterTableResponse { + oneof response { + bool success = 1; + string failure = 2; + } +} + +message TruncateRequest { + map configuration = 1; + string schema_name = 2; + string table_name = 3; + string synced_column = 4; + google.protobuf.Timestamp utc_delete_before = 5; + optional SoftTruncate soft = 6; +} + +message SoftTruncate { + string deleted_column = 3; +} + +message TruncateResponse { + oneof response { + bool success = 1; + string failure = 2; + } +} + +message WriteBatchRequest { + map configuration = 1; + string schema_name = 2; + Table table = 3; + map keys = 4; + repeated string replace_files = 5; + repeated string update_files = 6; + repeated string delete_files = 7; + oneof file_params { + CsvFileParams csv = 8; + ParquetFileParams parquet = 9; + } +} + +message CsvFileParams { + Compression compression = 1; + Encryption encryption = 2; + string null_string = 3; + string unmodified_string = 4; +} + +message ParquetFileParams { + Encryption encryption = 1; +} + +enum Encryption { + NONE = 0; + AES = 1; +} + +enum BatchFileFormat { + CSV = 0; + PARQUET = 1; +} + +enum Compression { + OFF = 0; + ZSTD = 1; + GZIP = 2; +} + +message WriteBatchResponse { + oneof response { + bool success = 1; + string failure = 2; + } +} \ No newline at end of file