From f69879776a3093855ff24bb6e13aa4e3678e0aa8 Mon Sep 17 00:00:00 2001 From: Abhi Agarwal Date: Wed, 19 Jun 2024 16:45:23 -0400 Subject: [PATCH] initial server skeleton --- Cargo.toml | 9 ++- models/Cargo.toml | 9 +-- server/Cargo.toml | 27 ++++++++- server/src/main.rs | 29 +++++++++- server/src/routes/catalogs.rs | 97 +++++++++++++++++++++++++++++++ server/src/routes/functions.rs | 89 ++++++++++++++++++++++++++++ server/src/routes/mod.rs | 26 +++++++++ server/src/routes/schemas.rs | 103 +++++++++++++++++++++++++++++++++ server/src/routes/tables.rs | 86 +++++++++++++++++++++++++++ server/src/routes/volumes.rs | 86 +++++++++++++++++++++++++++ server/src/state.rs | 7 +++ 11 files changed, 560 insertions(+), 8 deletions(-) create mode 100644 server/src/routes/catalogs.rs create mode 100644 server/src/routes/functions.rs create mode 100644 server/src/routes/mod.rs create mode 100644 server/src/routes/schemas.rs create mode 100644 server/src/routes/tables.rs create mode 100644 server/src/routes/volumes.rs create mode 100644 server/src/state.rs diff --git a/Cargo.toml b/Cargo.toml index 1c656a3..20027a2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,13 @@ resolver = "2" [workspace.package] edition = "2021" license = "Apache-2.0" -version = "0.0.1" +version = "0.1.0" +authors = [ + "Alexander Brassel ", + "Abhi Agarwal ", + "R. Tyler Croy ", +] +publish = false [workspace.dependencies] chrono = { version = "0.4.38", features = ["serde"] } @@ -18,4 +24,5 @@ serde = { version = "^1.0", features = ["derive"] } serde_json = "^1.0" tracing = { version = "^0.1.40" } tracing-subscriber = { version = "^0.3.18", features = ["env-filter"] } +utoipa = { version = "4.2.3", features = ["openapi_extensions"] } uuid = { version = "^1.8", features = ["serde", "v4"] } \ No newline at end of file diff --git a/models/Cargo.toml b/models/Cargo.toml index 72f8f0b..73bd034 100644 --- a/models/Cargo.toml +++ b/models/Cargo.toml @@ -1,17 +1,18 @@ [package] name = "unitycatalog-models" -version = "0.1.0" description = "Generated structs for Unity Catalog models" edition.workspace = true +authors.workspace = true license.workspace = true - -publish = false +version.workspace = true +publish.workspace = true [dependencies] strum = { version = "0.26", features = ["derive"] } +chrono = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +utoipa = { workspace = true } uuid = { workspace = true } -chrono = { workspace = true } diff --git a/server/Cargo.toml b/server/Cargo.toml index 11de01a..ab0ab7c 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -1,8 +1,33 @@ [package] name = "unitycatalog-server" +description = "Unity catalog server, implemented in rust" + edition.workspace = true +authors.workspace = true license.workspace = true version.workspace = true -publish = false +publish.workspace = true [dependencies] +anyhow = "^1.0.86" +axum = "^0.7.5" +object_store = { version = "^0.10.1", features = ["serde"] } +thiserror = "1.0.61" +toml_edit = { version = "0.22.14", features = ["serde"] } + +unitycatalog-models = { path = "../models" } + +chrono = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +reqwest = { workspace = true } +utoipa = { workspace = true, features = ["axum_extras"] } +uuid = { workspace = true } + +[features] +aws = ["object_store/aws"] +azure = ["object_store/azure"] +gcp = ["object_store/gcp"] diff --git a/server/src/main.rs b/server/src/main.rs index e7a11a9..58e538e 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -1,3 +1,28 @@ -fn main() { - println!("Hello, world!"); +#![allow(unused)] + +use crate::routes::all_routes; +use crate::state::AppState; +use anyhow::Result; +use axum::Router; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; + +mod routes; +mod state; + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::registry() + .with( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| "unitycatalog=debug,tower_http=debug".into()), + ) + .with(tracing_subscriber::fmt::layer()) + .try_init()?; + + let state = AppState {}; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:8000").await?; + let app = Router::new().nest("/api/2.1/unity-catalog/", all_routes(state)); + axum::serve(listener, app).await?; + Ok(()) } diff --git a/server/src/routes/catalogs.rs b/server/src/routes/catalogs.rs new file mode 100644 index 0000000..00f16a1 --- /dev/null +++ b/server/src/routes/catalogs.rs @@ -0,0 +1,97 @@ +use axum::{ + extract::{Path, Query, State}, + response::IntoResponse, + routing::{get, Router}, + Json, +}; + +use unitycatalog_models::models::{ + CatalogInfo, CreateCatalog, ListCatalogsResponse, UpdateCatalog, +}; + +use crate::state::AppState; + +use super::PaginationParams; + +pub fn router(state: AppState) -> Router { + Router::new() + .route("/", get(get_catalogs).post(post_catalog)) + .route( + "/:name", + get(get_catalog).patch(get_catalog).delete(delete_catalog), + ) + .with_state(state) +} + +#[utoipa::path( + post, + path = "/", + operation_id = "listCatalogs", + responses( + (status = 200, description = "The catalog list was successfully retrieved.") + ) +)] +async fn get_catalogs( + State(state): State, + Query(pagination): Query, +) -> Json { + todo!() +} + +#[utoipa::path( + post, + path = "/", + operation_id = "createCatalog", + responses( + (status = 200, description = "The new catalog was successfully created.", body = CatalogInfo) + ) +)] +async fn post_catalog( + State(state): State, + Json(body): Json, +) -> Json { + todo!() +} + +#[utoipa::path( + post, + path = "/{name}", + operation_id = "getCatalog", + responses( + (status = 200, description = "The catalog was successfully retrieved.", body = CatalogInfo) + ) +)] +async fn get_catalog(State(state): State, Path(name): Path) -> Json { + todo!() +} + +#[utoipa::path( + post, + path = "/{name}", + operation_id = "updateCatalog", + responses( + (status = 200, description = "The catalog was successfully updated.", body = CatalogInfo) + ) +)] +async fn patch_catalog( + State(state): State, + Path(name): Path, + Json(body): Json, +) -> Json { + todo!() +} + +#[utoipa::path( + post, + path = "/{name}", + operation_id = "deleteCatalog", + responses( + (status = 200, description = "The catalog was successfully deleted.") + ) +)] +async fn delete_catalog( + State(state): State, + Path(name): Path, +) -> impl IntoResponse { + todo!() +} diff --git a/server/src/routes/functions.rs b/server/src/routes/functions.rs new file mode 100644 index 0000000..46cd428 --- /dev/null +++ b/server/src/routes/functions.rs @@ -0,0 +1,89 @@ +use axum::{ + extract::{Path, Query, State}, + response::IntoResponse, + routing::{get, Router}, + Json, +}; + +use serde::{Deserialize, Serialize}; +use unitycatalog_models::models::{CreateFunction, FunctionInfo, ListFunctionsResponse}; +use utoipa::IntoParams; + +use crate::state::AppState; + +use super::PaginationParams; + +pub fn router(state: AppState) -> Router { + Router::new() + .route("/", get(get_functions).post(post_function)) + .route("/:name", get(get_function).delete(delete_function)) + .with_state(state) +} + +#[derive(Debug, Clone, Deserialize, Serialize, IntoParams)] +pub struct ListFunctionParams { + catalog_name: String, + schema_name: String, + #[serde(flatten)] + pagination: PaginationParams, +} + +#[utoipa::path( + post, + path = "/", + operation_id = "listFunctions", + responses( + (status = 200, description = "The function list was successfully retrieved.") + ) +)] +async fn get_functions( + State(state): State, + Query(pagination): Query, +) -> Json { + todo!() +} + +#[utoipa::path( + post, + path = "/", + operation_id = "createFunction", + responses( + (status = 200, description = "The new function was successfully created.", body = FunctionInfo) + ) +)] +async fn post_function( + State(state): State, + Json(body): Json, +) -> Json { + todo!() +} + +#[utoipa::path( + post, + path = "/{name}", + operation_id = "getFunction", + responses( + (status = 200, description = "The function was successfully retrieved.", body = FunctionInfo) + ) +)] +async fn get_function( + State(state): State, + Path(name): Path, +) -> Json { + todo!() +} + +#[utoipa::path( + post, + path = "/{name}", + operation_id = "deleteFunction", + responses( + (status = 200, description = "The function was successfully deleted.") + ) +)] +async fn delete_function( + State(state): State, + Path(name): Path, +) -> impl IntoResponse { + todo!() +} diff --git a/server/src/routes/mod.rs b/server/src/routes/mod.rs new file mode 100644 index 0000000..028c8b3 --- /dev/null +++ b/server/src/routes/mod.rs @@ -0,0 +1,26 @@ +use axum::Router; +use serde::{Deserialize, Serialize}; +use utoipa::IntoParams; + +use crate::state::AppState; + +mod functions; +mod catalogs; +mod schemas; +mod tables; +mod volumes; + +#[derive(Debug, Clone, Deserialize, Serialize, IntoParams)] +pub struct PaginationParams { + pub page_token: String, + pub max_results: u32, +} + +pub fn all_routes(state: AppState) -> Router { + Router::new() + .nest("/functions", functions::router(state.clone())) + .nest("/catalogs", catalogs::router(state.clone())) + .nest("/schemas", schemas::router(state.clone())) + .nest("/tables", tables::router(state.clone())) + .nest("/volumes", volumes::router(state.clone())) +} \ No newline at end of file diff --git a/server/src/routes/schemas.rs b/server/src/routes/schemas.rs new file mode 100644 index 0000000..e8fef0e --- /dev/null +++ b/server/src/routes/schemas.rs @@ -0,0 +1,103 @@ +use axum::{ + extract::{Path, Query, State}, + response::IntoResponse, + routing::{get, Router}, + Json, +}; + +use serde::{Deserialize, Serialize}; +use unitycatalog_models::models::{CreateSchema, ListSchemasResponse, SchemaInfo, UpdateCatalog}; + +use crate::state::AppState; + +use super::PaginationParams; + +pub fn router(state: AppState) -> Router { + Router::new() + .route("/", get(get_schemas).post(post_schema)) + .route( + "/:name", + get(get_schema).patch(patch_schema).delete(delete_schema), + ) + .with_state(state) +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ListSchemaParams { + catalog_name: String, + #[serde(flatten)] + pagination: PaginationParams, +} + +#[utoipa::path( + post, + path = "/", + operation_id = "listSchemas", + responses( + (status = 200, description = "The schemas list was successfully retrieved.") + ) +)] +async fn get_schemas( + State(state): State, + Query(pagination): Query, +) -> Json { + todo!() +} + +#[utoipa::path( + post, + path = "/", + operation_id = "createSchema", + responses( + (status = 200, description = "The new schema was successfully created.", body = CatalogInfo) + ) +)] +async fn post_schema( + State(state): State, + Json(body): Json, +) -> Json { + todo!() +} + +#[utoipa::path( + post, + path = "/{full_name}", + operation_id = "getSchema", + responses( + (status = 200, description = "The schema was successfully retrieved.", body = SchemaInfo) + ) +)] +async fn get_schema(State(state): State, Path(name): Path) -> Json { + todo!() +} + +#[utoipa::path( + post, + path = "/{full_name}", + operation_id = "updateSchema", + responses( + (status = 200, description = "The schema was successfully updated.", body = SchemaInfo) + ) +)] +async fn patch_schema( + State(state): State, + Path(name): Path, + Json(body): Json, +) -> Json { + todo!() +} + +#[utoipa::path( + post, + path = "/{full_name}", + operation_id = "deleteSchema", + responses( + (status = 200, description = "The schema was successfully deleted.") + ) +)] +async fn delete_schema( + State(state): State, + Path(name): Path, +) -> impl IntoResponse { + todo!() +} diff --git a/server/src/routes/tables.rs b/server/src/routes/tables.rs new file mode 100644 index 0000000..4a82a6b --- /dev/null +++ b/server/src/routes/tables.rs @@ -0,0 +1,86 @@ +use axum::{ + extract::{Path, Query, State}, + response::IntoResponse, + routing::{get, Router}, + Json, +}; + +use serde::{Deserialize, Serialize}; +use unitycatalog_models::models::{CreateTable, ListTablesResponse, TableInfo}; +use utoipa::IntoParams; + +use crate::state::AppState; + +use super::PaginationParams; + +pub fn router(state: AppState) -> Router { + Router::new() + .route("/", get(get_tables).post(post_table)) + .route("/:name", get(get_table).delete(delete_table)) + .with_state(state) +} + +#[derive(Debug, Clone, Deserialize, Serialize, IntoParams)] +pub struct ListTableParams { + catalog_name: String, + schema_name: String, + #[serde(flatten)] + pagination: PaginationParams, +} + +#[utoipa::path( + post, + path = "/", + operation_id = "listTables", + responses( + (status = 200, description = "The tables list was successfully retrieved.", body = ListTablesResponse) + ) +)] +async fn get_tables( + State(state): State, + Query(pagination): Query, +) -> Json { + todo!() +} + +#[utoipa::path( + post, + path = "/", + operation_id = "createTable", + responses( + (status = 200, description = "The new table was successfully created.", body = TableInfo) + ) +)] +async fn post_table( + State(state): State, + Json(body): Json, +) -> Json { + todo!() +} + +#[utoipa::path( + post, + path = "/{full_name}", + operation_id = "getTable", + responses( + (status = 200, description = "The table was successfully retrieved.", body = TableInfo) + ) +)] +async fn get_table(State(state): State, Path(name): Path) -> Json { + todo!() +} + +#[utoipa::path( + post, + path = "/{full_name}", + operation_id = "deleteTable", + responses( + (status = 200, description = "The table was successfully deleted.") + ) +)] +async fn delete_table( + State(state): State, + Path(name): Path, +) -> impl IntoResponse { + todo!() +} diff --git a/server/src/routes/volumes.rs b/server/src/routes/volumes.rs new file mode 100644 index 0000000..0890f68 --- /dev/null +++ b/server/src/routes/volumes.rs @@ -0,0 +1,86 @@ +use axum::{ + extract::{Path, Query, State}, + response::IntoResponse, + routing::{get, Router}, + Json, +}; + +use serde::{Deserialize, Serialize}; +use unitycatalog_models::models::{CreateVolumeRequestContent, ListVolumesResponseContent, VolumeInfo}; +use utoipa::IntoParams; + +use crate::state::AppState; + +use super::PaginationParams; + +pub fn router(state: AppState) -> Router { + Router::new() + .route("/", get(get_volumes).post(post_volume)) + .route("/:name", get(get_volume).delete(delete_volume)) + .with_state(state) +} + +#[derive(Debug, Clone, Deserialize, Serialize, IntoParams)] +pub struct ListVolumeParams { + catalog_name: String, + schema_name: String, + #[serde(flatten)] + pagination: PaginationParams, +} + +#[utoipa::path( + post, + path = "/", + operation_id = "listVolumes", + responses( + (status = 200, description = "The volumes list was successfully retrieved.", body = ListVolumesResponse) + ) +)] +async fn get_volumes( + State(state): State, + Query(pagination): Query, +) -> Json { + todo!() +} + +#[utoipa::path( + post, + path = "/", + operation_id = "createVolume", + responses( + (status = 200, description = "The new volume was successfully created.", body = VolumeInfo) + ) +)] +async fn post_volume( + State(state): State, + Json(body): Json, +) -> Json { + todo!() +} + +#[utoipa::path( + post, + path = "/{full_name}", + operation_id = "getVolume", + responses( + (status = 200, description = "The volume was successfully retrieved.", body = VolumeInfo) + ) +)] +async fn get_volume(State(state): State, Path(name): Path) -> Json { + todo!() +} + +#[utoipa::path( + post, + path = "/{full_name}", + operation_id = "deleteVolume", + responses( + (status = 200, description = "The volume was successfully deleted.") + ) +)] +async fn delete_volume( + State(state): State, + Path(name): Path, +) -> impl IntoResponse { + todo!() +} diff --git a/server/src/state.rs b/server/src/state.rs new file mode 100644 index 0000000..904a395 --- /dev/null +++ b/server/src/state.rs @@ -0,0 +1,7 @@ +use serde::{Deserialize, Serialize}; + + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AppState { + +} \ No newline at end of file