Skip to content

Commit

Permalink
add proto files to parent folder
Browse files Browse the repository at this point in the history
  • Loading branch information
Satvik Patil authored and Satvik Patil committed Jan 1, 2025
1 parent fcac73f commit ae4b9c4
Show file tree
Hide file tree
Showing 3 changed files with 376 additions and 0 deletions.
133 changes: 133 additions & 0 deletions common.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
syntax = "proto3";
option optimize_for = SPEED;
option java_multiple_files = true;
option go_package = "fivetran.com/fivetran_sdk";
package fivetran_sdk;

import "google/protobuf/timestamp.proto";

message ConfigurationFormRequest {}

message ConfigurationFormResponse {
bool schema_selection_supported = 1;
bool table_selection_supported = 2;
repeated FormField fields = 3;
repeated ConfigurationTest tests = 4;
}

message FormField {
string name = 1;
string label = 2;
bool required = 3;
optional string description = 4;
oneof type {
TextField text_field = 5;
DropdownField dropdown_field = 6;
ToggleField toggle_field = 7;
}
}

message DropdownField {
repeated string dropdown_field = 1;
}

message ToggleField {}

enum TextField {
PlainText = 0;
Password = 1;
Hidden = 2;
}

message ConfigurationTest {
string name = 1; // unique identifier for the test
string label = 2; // A few words indicating what we are testing, e.g. 'Connecting to database'
}

message TestRequest {
string name = 1;
map<string, string> configuration = 2;
}

message TestResponse {
oneof response {
bool success = 1;
string failure = 2;
// potential future warning
}
}

message SchemaList {
repeated Schema schemas = 1;
}

message TableList {
repeated Table tables = 1;
}

message Schema {
string name = 1;
repeated Table tables = 2;
}

enum DataType {
UNSPECIFIED = 0;
BOOLEAN = 1;
SHORT = 2;
INT = 3;
LONG = 4;
DECIMAL = 5;
FLOAT = 6;
DOUBLE = 7;
NAIVE_DATE = 8;
NAIVE_DATETIME = 9;
UTC_DATETIME = 10;
BINARY = 11;
XML = 12;
STRING = 13;
JSON = 14;
}

message DecimalParams {
uint32 precision = 1;
uint32 scale = 2;
}

enum OpType {
UPSERT = 0;
UPDATE = 1;
DELETE = 2;
TRUNCATE = 3;
}

message ValueType {
oneof inner {
bool null = 1;
bool bool = 2;
int32 short = 3;
int32 int = 4;
int64 long = 5;
float float = 6;
double double = 7;
google.protobuf.Timestamp naive_date = 8;
google.protobuf.Timestamp naive_datetime = 9;
google.protobuf.Timestamp utc_datetime = 10;
string decimal = 11;
bytes binary = 12;
string string = 13;
string json = 14;
string xml = 15;
}
}

message Table {
string name = 1;
repeated Column columns = 2;
}

message Column {
string name = 1;
DataType type = 2;
bool primary_key = 3;
optional DecimalParams decimal = 4;
}
109 changes: 109 additions & 0 deletions connector_sdk.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
syntax = "proto3";
option optimize_for = SPEED;
option java_multiple_files = true;
option go_package = "fivetran.com/fivetran_sdk";
package fivetran_sdk;

import "common.proto";

// Fivetran (grpc client) <> Connector (grpc server)
service Connector {
rpc ConfigurationForm (ConfigurationFormRequest) returns (ConfigurationFormResponse) {}
rpc Test (TestRequest) returns (TestResponse) {}
rpc Schema (SchemaRequest) returns (SchemaResponse) {}
rpc Update (UpdateRequest) returns (stream UpdateResponse) {}
}

message SchemaRequest {
map<string, string> configuration = 1;
}

message SchemaResponse {
oneof response {
bool schema_response_not_supported = 1;
SchemaList with_schema = 2;
TableList without_schema = 3;
}
optional bool selection_not_supported = 4;
}

message UpdateRequest {
map<string, string> configuration = 1;
optional Selection selection = 2;
optional string state_json = 3;
}

message Selection {
oneof selection {
TablesWithNoSchema without_schema = 1;
TablesWithSchema with_schema = 2;
}
}

message TablesWithNoSchema {
repeated TableSelection tables = 1;
bool include_new_tables = 2;
}

message TablesWithSchema {
repeated SchemaSelection schemas = 1;
bool include_new_schemas = 2;
}

message SchemaSelection {
bool included = 1;
string schema_name = 2;
repeated TableSelection tables = 3;
bool include_new_tables = 4;
}

message TableSelection {
bool included = 1;
string table_name = 2;
map<string, bool> columns = 3;
bool include_new_columns = 4;
}

message UpdateResponse {
oneof response {
LogEntry log_entry = 1;
Operation operation = 2;
}
}

enum LogLevel {
INFO = 0;
WARNING = 1;
SEVERE = 2;
}

message LogEntry {
LogLevel level = 1;
string message = 2;
}

message Operation {
oneof op {
Record record = 1;
SchemaChange schema_change = 2;
Checkpoint checkpoint = 3;
}
}

message SchemaChange {
oneof change {
SchemaList with_schema = 1;
TableList without_schema = 2;
}
}

message Record {
optional string schema_name = 1;
string table_name = 2;
OpType type = 3;
map<string, ValueType> data = 4;
}

message Checkpoint {
string state_json = 1;
}
134 changes: 134 additions & 0 deletions destination_sdk.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
syntax = "proto3";
option optimize_for = SPEED;
option java_multiple_files = true;
option go_package = "fivetran.com/fivetran_sdk";
package fivetran_sdk;

import "google/protobuf/timestamp.proto";
import "common.proto";

// Fivetran (grpc client) <> Destination (grpc server)
service Destination {
rpc ConfigurationForm (ConfigurationFormRequest) returns (ConfigurationFormResponse) {}
rpc Capabilities (CapabilitiesRequest) returns (CapabilitiesResponse) {}
rpc Test (TestRequest) returns (TestResponse) {}
rpc DescribeTable (DescribeTableRequest) returns (DescribeTableResponse) {}
rpc CreateTable(CreateTableRequest) returns (CreateTableResponse) {}
rpc AlterTable(AlterTableRequest) returns (AlterTableResponse) {}
rpc Truncate(TruncateRequest) returns (TruncateResponse) {}
rpc WriteBatch (WriteBatchRequest) returns (WriteBatchResponse) {}
}

message CapabilitiesRequest {}

message CapabilitiesResponse {
BatchFileFormat batch_file_format = 1;
}

message DescribeTableRequest {
map<string, string> configuration = 1;
string schema_name = 2;
string table_name = 3;
}

message DescribeTableResponse {
oneof response {
bool not_found = 1;
string failure = 2;
Table table = 3;
}
}

message CreateTableRequest {
map<string, string> configuration = 1;
string schema_name = 2;
Table table = 3;
}

message CreateTableResponse {
oneof response {
bool success = 1;
string failure = 2;
}
}

message AlterTableRequest {
map<string, string> configuration = 1;
string schema_name = 2;
Table table = 3;
}

message AlterTableResponse {
oneof response {
bool success = 1;
string failure = 2;
}
}

message TruncateRequest {
map<string, string> configuration = 1;
string schema_name = 2;
string table_name = 3;
string synced_column = 4;
google.protobuf.Timestamp utc_delete_before = 5;
optional SoftTruncate soft = 6;
}

message SoftTruncate {
string deleted_column = 3;
}

message TruncateResponse {
oneof response {
bool success = 1;
string failure = 2;
}
}

message WriteBatchRequest {
map<string, string> configuration = 1;
string schema_name = 2;
Table table = 3;
map<string, bytes> keys = 4;
repeated string replace_files = 5;
repeated string update_files = 6;
repeated string delete_files = 7;
oneof file_params {
CsvFileParams csv = 8;
ParquetFileParams parquet = 9;
}
}

message CsvFileParams {
Compression compression = 1;
Encryption encryption = 2;
string null_string = 3;
string unmodified_string = 4;
}

message ParquetFileParams {
Encryption encryption = 1;
}

enum Encryption {
NONE = 0;
AES = 1;
}

enum BatchFileFormat {
CSV = 0;
PARQUET = 1;
}

enum Compression {
OFF = 0;
ZSTD = 1;
GZIP = 2;
}

message WriteBatchResponse {
oneof response {
bool success = 1;
string failure = 2;
}
}

0 comments on commit ae4b9c4

Please sign in to comment.