From e6f73a37326babe228d318b04b5343692bd74efc Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 6 Sep 2024 09:47:35 +1000 Subject: [PATCH 1/5] refactor!: remove htsget-lambda library code and replace with axum router --- htsget-actix/src/lib.rs | 26 +- htsget-actix/src/main.rs | 4 +- htsget-axum/src/server/data.rs | 22 +- htsget-axum/src/server/ticket.rs | 31 +- htsget-lambda/src/handlers/get.rs | 30 - htsget-lambda/src/handlers/mod.rs | 140 ---- htsget-lambda/src/handlers/post.rs | 31 - htsget-lambda/src/handlers/service_info.rs | 24 - htsget-lambda/src/lib.rs | 781 --------------------- htsget-lambda/src/main.rs | 23 +- htsget-test/src/http/cors.rs | 27 +- htsget-test/src/http/mod.rs | 21 +- htsget-test/src/http/server.rs | 82 ++- 13 files changed, 152 insertions(+), 1090 deletions(-) delete mode 100644 htsget-lambda/src/handlers/get.rs delete mode 100644 htsget-lambda/src/handlers/mod.rs delete mode 100644 htsget-lambda/src/handlers/post.rs delete mode 100644 htsget-lambda/src/handlers/service_info.rs delete mode 100644 htsget-lambda/src/lib.rs diff --git a/htsget-actix/src/lib.rs b/htsget-actix/src/lib.rs index 9d12a28f0..98ea64c20 100644 --- a/htsget-actix/src/lib.rs +++ b/htsget-actix/src/lib.rs @@ -175,8 +175,16 @@ mod tests { struct ActixTestRequest(T); impl TestRequest for ActixTestRequest { - fn insert_header(self, header: TestHeader>) -> Self { - Self(self.0.insert_header(header.into_tuple())) + fn insert_header( + self, + header: TestHeader, impl Into>, + ) -> Self { + let (name, value) = header.into_tuple(); + Self( + self + .0 + .insert_header((name.to_string(), value.to_str().unwrap())), + ) } fn set_payload(self, payload: impl Into) -> Self { @@ -187,11 +195,15 @@ mod tests { Self(self.0.uri(&uri.into())) } - fn method(self, method: impl Into) -> Self { + fn method(self, method: impl Into) -> Self { Self( - self - .0 - .method(method.into().parse().expect("expected valid method")), + self.0.method( + method + .into() + .to_string() + .parse() + .expect("expected valid method"), + ), ) } } @@ -224,7 +236,7 @@ mod tests { &self.config } - fn get_request(&self) -> ActixTestRequest { + fn request(&self) -> ActixTestRequest { ActixTestRequest(test::TestRequest::default()) } diff --git a/htsget-actix/src/main.rs b/htsget-actix/src/main.rs index 1f8a60193..b71c5b0a0 100644 --- a/htsget-actix/src/main.rs +++ b/htsget-actix/src/main.rs @@ -8,9 +8,7 @@ use htsget_config::command; #[actix_web::main] async fn main() -> std::io::Result<()> { - if let Some(path) = - Config::parse_args_with_command(command!()).expect("expected valid command parsing") - { + if let Some(path) = Config::parse_args_with_command(command!())? { let config = Config::from_path(&path)?; config.setup_tracing()?; diff --git a/htsget-axum/src/server/data.rs b/htsget-axum/src/server/data.rs index bdadfed1f..ff641c816 100644 --- a/htsget-axum/src/server/data.rs +++ b/htsget-axum/src/server/data.rs @@ -88,7 +88,7 @@ mod tests { use async_trait::async_trait; use http::header::HeaderName; - use http::{HeaderMap, HeaderValue, Method}; + use http::{HeaderMap, Method}; use reqwest::{Client, ClientBuilder, RequestBuilder}; use tempfile::{tempdir, TempDir}; use tokio::fs::{create_dir, File}; @@ -118,11 +118,11 @@ mod tests { } impl TestRequest for DataTestRequest { - fn insert_header(mut self, header: Header>) -> Self { - self.headers.insert( - HeaderName::from_str(&header.name.into()).unwrap(), - HeaderValue::from_str(&header.value.into()).unwrap(), - ); + fn insert_header( + mut self, + header: Header, impl Into>, + ) -> Self { + self.headers.insert(header.name.into(), header.value.into()); self } @@ -136,8 +136,8 @@ mod tests { self } - fn method(mut self, method: impl Into) -> Self { - self.method = method.into().parse().unwrap(); + fn method(mut self, method: impl Into) -> Self { + self.method = method.into(); self } } @@ -186,7 +186,7 @@ mod tests { &self.config } - fn get_request(&self) -> DataTestRequest { + fn request(&self) -> DataTestRequest { DataTestRequest::default() } @@ -294,8 +294,8 @@ mod tests { let test_server = DataTestServer::default(); let request = test_server - .get_request() - .method(Method::GET.to_string()) + .request() + .method(Method::GET) .uri(format!("{scheme}://localhost:{port}/data/key1")); let response = test_server.test_server(request, "".to_string()).await; diff --git a/htsget-axum/src/server/ticket.rs b/htsget-axum/src/server/ticket.rs index 79eaa8ff7..0d50899ca 100644 --- a/htsget-axum/src/server/ticket.rs +++ b/htsget-axum/src/server/ticket.rs @@ -105,7 +105,6 @@ mod tests { use std::convert::Infallible; use std::path::Path; use std::result; - use std::str::FromStr; use super::*; use async_trait::async_trait; @@ -119,7 +118,7 @@ mod tests { TestRequest, TestServer, }; use http::header::HeaderName; - use http::Request; + use http::{Method, Request}; use tempfile::TempDir; use tower::ServiceExt; @@ -130,15 +129,14 @@ mod tests { struct AxumTestRequest(T); impl TestRequest for AxumTestRequest> { - fn insert_header(mut self, header: Header>) -> Self { - self.0.headers_mut().insert( - HeaderName::from_str(&header.name.into()).expect("expected valid header name"), - header - .value - .into() - .parse() - .expect("expected valid header value"), - ); + fn insert_header( + mut self, + header: Header, impl Into>, + ) -> Self { + self + .0 + .headers_mut() + .insert(header.name.into(), header.value.into()); self } @@ -153,8 +151,8 @@ mod tests { self } - fn method(mut self, method: impl Into) -> Self { - *self.0.method_mut() = method.into().parse().expect("expected valid method"); + fn method(mut self, method: impl Into) -> Self { + *self.0.method_mut() = method.into(); self } } @@ -187,7 +185,7 @@ mod tests { &self.config } - fn get_request(&self) -> AxumTestRequest> { + fn request(&self) -> AxumTestRequest> { AxumTestRequest(Request::default()) } @@ -306,4 +304,9 @@ mod tests { async fn cors_preflight_request() { cors::test_cors_preflight_request(&AxumTestServer::default()).await; } + + #[tokio::test] + async fn get_route_invalid_method() { + server::test_errors(&AxumTestServer::default()).await; + } } diff --git a/htsget-lambda/src/handlers/get.rs b/htsget-lambda/src/handlers/get.rs deleted file mode 100644 index 2a95e0ad7..000000000 --- a/htsget-lambda/src/handlers/get.rs +++ /dev/null @@ -1,30 +0,0 @@ -use std::collections::HashMap; -use std::sync::Arc; - -use lambda_http::http; -use lambda_http::http::HeaderMap; -use tracing::info; -use tracing::instrument; - -use htsget_config::types::Request; -use htsget_http::{get as htsget_get, Endpoint}; -use htsget_search::HtsGet; - -use crate::handlers::handle_response; -use crate::{Body, Response}; - -/// Get request reads endpoint -#[instrument(skip(searcher))] -pub async fn get( - id: String, - searcher: Arc, - query: HashMap, - headers: HeaderMap, - endpoint: Endpoint, -) -> http::Result> { - let request = Request::new(id, query, headers); - - info!(request = ?request, "GET request"); - - handle_response(htsget_get(searcher, request, endpoint).await) -} diff --git a/htsget-lambda/src/handlers/mod.rs b/htsget-lambda/src/handlers/mod.rs deleted file mode 100644 index 16a4e4a46..000000000 --- a/htsget-lambda/src/handlers/mod.rs +++ /dev/null @@ -1,140 +0,0 @@ -//! Module primarily providing http response functionality for the htsget endpoints. -//! - -use lambda_http::http; -use lambda_http::http::{header, StatusCode}; -use serde::Serialize; -use serde_json::Error; - -use htsget_config::types::JsonResponse; -use htsget_http::{HtsGetError, Result}; - -use crate::{Body, Response}; - -pub mod get; -pub mod post; -pub mod service_info; - -/// New type used for formatting a http response. -pub struct FormatJson(T); - -impl FormatJson { - pub fn into_inner(self) -> T { - self.0 - } -} - -impl TryFrom> for Response { - type Error = http::Error; - - fn try_from(value: FormatJson) -> http::Result { - let mut body = match serde_json::to_string_pretty(&value.into_inner()) { - Ok(body) => body, - Err(e) => return Ok(FormatJson::try_from(e)?.into_inner()), - }; - body.push('\n'); - - Response::builder() - .status(StatusCode::OK) - .header(header::CONTENT_TYPE, mime::APPLICATION_JSON.as_ref()) - .body(Body::from(body)) - } -} - -impl TryFrom for FormatJson> { - type Error = http::Error; - - fn try_from(error: Error) -> http::Result { - Ok(Self( - Response::builder() - .status(StatusCode::INTERNAL_SERVER_ERROR) - .header(header::CONTENT_TYPE, mime::TEXT_PLAIN_UTF_8.as_ref()) - .body(Body::from(error.to_string()))?, - )) - } -} - -impl TryFrom for FormatJson> { - type Error = http::Error; - - fn try_from(error: HtsGetError) -> http::Result { - let (json, status_code) = error.to_json_representation(); - let mut response: Response = FormatJson(json).try_into()?; - *response.status_mut() = status_code; - Ok(Self(response)) - } -} - -/// Handles a response, converting errors to json and using the proper HTTP status code. -fn handle_response(response: Result) -> http::Result> { - match response { - Err(error) => Ok(FormatJson::try_from(error)?.into_inner()), - Ok(json) => FormatJson(json).try_into(), - } -} - -#[cfg(test)] -mod tests { - use lambda_http::http::{header, HeaderMap, Response, StatusCode}; - use lambda_http::Body; - use mime::Mime; - use serde::ser::Error; - use serde::{Serialize, Serializer}; - use serde_json::{json, Value}; - - use crate::handlers::FormatJson; - - struct TestError; - - impl Serialize for TestError { - fn serialize(&self, _: S) -> Result - where - S: Serializer, - { - Err(Error::custom(json!({"value": "1"}))) - } - } - - #[test] - fn into_response() { - let expected_body = json!({"value": "1"}); - let json = FormatJson(expected_body.clone()); - test_into_response( - json.try_into().unwrap(), - expected_body, - StatusCode::OK, - mime::APPLICATION_JSON, - ); - } - - #[test] - fn into_response_error() { - let json = FormatJson(TestError); - test_into_response( - json.try_into().unwrap(), - json!({"value": "1"}), - StatusCode::INTERNAL_SERVER_ERROR, - mime::TEXT_PLAIN_UTF_8, - ); - } - - fn test_into_response( - response: Response, - expected_body: Value, - expected_status_code: StatusCode, - expected_content_type: Mime, - ) { - let mut expected_headers = HeaderMap::new(); - expected_headers.insert( - header::CONTENT_TYPE, - expected_content_type.as_ref().parse().unwrap(), - ); - - assert_eq!(response.status(), expected_status_code); - assert_eq!(response.headers(), &expected_headers); - - let bytes: &[u8] = response.body().as_ref(); - let value: Value = serde_json::from_slice(bytes).unwrap(); - assert_eq!(value, expected_body); - } -} diff --git a/htsget-lambda/src/handlers/post.rs b/htsget-lambda/src/handlers/post.rs deleted file mode 100644 index 2babde158..000000000 --- a/htsget-lambda/src/handlers/post.rs +++ /dev/null @@ -1,31 +0,0 @@ -use std::collections::HashMap; -use std::sync::Arc; - -use lambda_http::http; -use lambda_http::http::HeaderMap; -use tracing::info; -use tracing::instrument; - -use htsget_config::types::Request; -use htsget_http::{post as htsget_post, Endpoint, PostRequest}; -use htsget_search::HtsGet; - -use crate::handlers::handle_response; -use crate::{Body, Response}; - -/// Post request reads endpoint -#[instrument(skip(searcher))] -pub async fn post( - id: String, - searcher: Arc, - query: HashMap, - body: PostRequest, - headers: HeaderMap, - endpoint: Endpoint, -) -> http::Result> { - let request = Request::new(id, query, headers); - - info!(body = ?body, "POST request"); - - handle_response(htsget_post(searcher, body, request, endpoint).await) -} diff --git a/htsget-lambda/src/handlers/service_info.rs b/htsget-lambda/src/handlers/service_info.rs deleted file mode 100644 index f3d3c300a..000000000 --- a/htsget-lambda/src/handlers/service_info.rs +++ /dev/null @@ -1,24 +0,0 @@ -use std::sync::Arc; - -use lambda_http::http; -use tracing::info; -use tracing::instrument; - -use htsget_http::get_service_info_json as get_base_service_info_json; -use htsget_http::Endpoint; -use htsget_search::HtsGet; - -use crate::handlers::FormatJson; -use crate::ServiceInfo; -use crate::{Body, Response}; - -/// Service info endpoint. -#[instrument(skip(searcher))] -pub fn get_service_info_json( - searcher: Arc, - endpoint: Endpoint, - config: &ServiceInfo, -) -> http::Result> { - info!(endpoint = ?endpoint, "service info request"); - FormatJson(get_base_service_info_json(endpoint, searcher, config)).try_into() -} diff --git a/htsget-lambda/src/lib.rs b/htsget-lambda/src/lib.rs deleted file mode 100644 index 9dc249223..000000000 --- a/htsget-lambda/src/lib.rs +++ /dev/null @@ -1,781 +0,0 @@ -//! Library providing the routing and http responses for aws lambda requests. -//! - -use std::collections::HashMap; -use std::future::Future; -use std::sync::Arc; - -use lambda_http::ext::RequestExt; -use lambda_http::http::{Method, StatusCode, Uri}; -use lambda_http::tower::ServiceBuilder; -use lambda_http::{http, service_fn, Body, Request, RequestPayloadExt, Response}; -use lambda_runtime::Error; -use tracing::instrument; -use tracing::{debug, info}; - -use htsget_axum::server::configure_cors; -use htsget_config::config::cors::CorsConfig; -pub use htsget_config::config::{Config, DataServerConfig, ServiceInfo, TicketServerConfig}; -pub use htsget_config::storage::Storage; -use htsget_http::{Endpoint, PostRequest}; -use htsget_search::HtsGet; - -use crate::handlers::get::get; -use crate::handlers::post::post; -use crate::handlers::service_info::get_service_info_json; - -pub mod handlers; - -/// A request route, with a method, endpoint and route type. -#[derive(Debug, PartialEq, Eq)] -pub struct Route { - method: HtsgetMethod, - endpoint: Endpoint, - route_type: RouteType, -} - -/// Valid htsget http request methods. -#[derive(Debug, PartialEq, Eq, Clone, Copy)] -pub enum HtsgetMethod { - Get, - Post, -} - -/// A route type, which is either the service info endpoint, or an id represented by a string. -#[derive(Debug, PartialEq, Eq)] -pub enum RouteType { - ServiceInfo, - Id(String), -} - -impl Route { - pub fn new(method: HtsgetMethod, endpoint: Endpoint, route_type: RouteType) -> Self { - Self { - method, - endpoint, - route_type, - } - } - - /// Gets the Route if the request is valid, otherwise returns an error with a response. - pub fn get_route(method: &Method, uri: &Uri) -> Result>> { - let with_endpoint = |endpoint: Endpoint, endpoint_type: &str| { - if endpoint_type.is_empty() { - Err( - Response::builder() - .status(StatusCode::NOT_FOUND) - .body(Body::Empty), - ) - } else { - let method = match *method { - Method::GET => Ok(HtsgetMethod::Get), - Method::POST => Ok(HtsgetMethod::Post), - _ => Err( - Response::builder() - .status(StatusCode::METHOD_NOT_ALLOWED) - .body(Body::Empty), - ), - }?; - if endpoint_type == "service-info" { - Ok(Route::new(method, endpoint, RouteType::ServiceInfo)) - } else { - Ok(Route::new( - method, - endpoint, - RouteType::Id(endpoint_type.to_string()), - )) - } - } - }; - - uri.path().strip_prefix("/reads/").map_or_else( - || { - uri.path().strip_prefix("/variants/").map_or_else( - || { - Err( - Response::builder() - .status(StatusCode::NOT_FOUND) - .body(Body::Empty), - ) - }, - |variants| with_endpoint(Endpoint::Variants, variants), - ) - }, - |reads| with_endpoint(Endpoint::Reads, reads), - ) - } - - pub fn method(&self) -> HtsgetMethod { - self.method - } - - pub fn endpoint(&self) -> &Endpoint { - &self.endpoint - } - - pub fn route_type(&self) -> &RouteType { - &self.route_type - } -} - -impl TryFrom<&Request> for Route { - type Error = http::Result>; - - fn try_from(request: &Request) -> Result { - Self::get_route( - request.method(), - &request - .raw_http_path() - .parse::() - .map_err(|err| Err(err.into()))?, - ) - } -} - -/// A Router is a struct which handles routing any htsget requests to the htsget search, using the config. -pub struct Router<'a, H> { - searcher: Arc, - config_service_info: &'a ServiceInfo, -} - -impl<'a, H: HtsGet + Send + Sync + 'static> Router<'a, H> { - pub fn new(searcher: Arc, config_service_info: &'a ServiceInfo) -> Self { - Self { - searcher, - config_service_info, - } - } - - /// Routes the request to the relevant htsget search endpoint using the lambda request and route. - pub async fn route_request_with_route( - &self, - request: Request, - route: Route, - ) -> http::Result> { - match route { - Route { - endpoint, - route_type: RouteType::ServiceInfo, - .. - } => get_service_info_json(self.searcher.clone(), endpoint, self.config_service_info), - Route { - method: HtsgetMethod::Get, - endpoint, - route_type: RouteType::Id(id), - } => { - get( - id, - self.searcher.clone(), - Self::extract_query(&request), - request.headers().clone(), - endpoint, - ) - .await - } - Route { - method: HtsgetMethod::Post, - endpoint, - route_type: RouteType::Id(id), - } => match Self::extract_query_from_payload(&request) { - None => Ok( - Response::builder() - .status(StatusCode::UNSUPPORTED_MEDIA_TYPE) - .body(Body::Empty)?, - ), - Some(query) => { - post( - id, - self.searcher.clone(), - Self::extract_query(&request), - query, - request.headers().clone(), - endpoint, - ) - .await - } - }, - } - } - - /// Routes the request to the relevant htsget search endpoint using the lambda request, returning a http response. - pub async fn route_request(&self, request: Request) -> http::Result> { - match Route::try_from(&request) { - Ok(route) => self.route_request_with_route(request, route).await, - Err(err) => err, - } - } - - /// Extracts post request query parameters. - #[instrument(level = "debug", ret)] - fn extract_query_from_payload(request: &Request) -> Option { - if request.body().is_empty() { - Some(PostRequest::default()) - } else { - let payload = request.payload::(); - debug!(payload = ?payload, "POST request payload"); - // Allows null/empty bodies. - payload.ok()? - } - } - - /// Extract get request query parameters. - #[instrument(level = "debug", ret)] - fn extract_query(request: &Request) -> HashMap { - let mut query = HashMap::new(); - // Silently ignores all but the last query key, for keys that are present more than once. - // This is the way actix-web does it, but should we return an error instead if a key is present - // more than once? - for (key, value) in request.query_string_parameters().iter() { - query.insert(key.to_string(), value.to_string()); - } - debug!(query = ?query, "GET request query"); - query - } -} - -pub async fn handle_request_service_fn(cors: CorsConfig, service: F) -> Result<(), Error> -where - F: FnMut(Request) -> Fut, - Fut: Future, Error>> + Send, -{ - let cors_layer = configure_cors(cors); - - let handler = ServiceBuilder::new() - .layer(cors_layer) - .service(service_fn(service)); - - lambda_http::run(handler).await?; - - Ok(()) -} - -pub async fn handle_request(cors: CorsConfig, router: &Router<'_, H>) -> Result<(), Error> -where - H: HtsGet + Send + Sync + 'static, -{ - handle_request_service_fn(cors, |event: Request| async move { - info!(event = ?event, "received request"); - Ok(router.route_request(event).await?) - }) - .await -} - -#[cfg(test)] -mod tests { - use std::future::Future; - use std::path::Path; - use std::str::FromStr; - use std::sync::Arc; - - use async_trait::async_trait; - use lambda_http::http::header::HeaderName; - use lambda_http::http::Uri; - use lambda_http::tower::ServiceExt; - use lambda_http::Body::Text; - use lambda_http::{Request, RequestExt, Service}; - use query_map::QueryMap; - use tempfile::TempDir; - - use htsget_axum::server::configure_cors; - use htsget_axum::server::BindServer; - use htsget_config::resolver::Resolver; - use htsget_config::types::{Class, JsonResponse}; - use htsget_http::Endpoint; - use htsget_test::http::server::{expected_url_path, test_response, test_response_service_info}; - use htsget_test::http::{config_with_tls, default_test_config, get_test_file}; - use htsget_test::http::{cors, server}; - use htsget_test::http::{Header, Response as TestResponse, TestRequest, TestServer}; - - use super::*; - - struct LambdaTestServer { - config: Config, - } - - struct LambdaTestRequest(T); - - impl TestRequest for LambdaTestRequest { - fn insert_header(mut self, header: Header>) -> Self { - self.0.headers_mut().insert( - HeaderName::from_str(&header.name.into()).expect("expected valid header name"), - header - .value - .into() - .parse() - .expect("expected valid header value"), - ); - self - } - - fn set_payload(mut self, payload: impl Into) -> Self { - *self.0.body_mut() = Text(payload.into()); - self - } - - fn uri(mut self, uri: impl Into) -> Self { - let uri = uri.into(); - *self.0.uri_mut() = uri.parse().expect("expected valid uri"); - if let Some(query) = self.0.uri().query().map(|s| s.to_string()) { - Self( - self - .0 - .with_query_string_parameters( - query - .parse::() - .expect("expected valid query parameters"), - ) - .with_raw_http_path(&uri), - ) - } else { - Self(self.0.with_raw_http_path(&uri)) - } - } - - fn method(mut self, method: impl Into) -> Self { - *self.0.method_mut() = method.into().parse().expect("expected valid method"); - self - } - } - - impl Default for LambdaTestServer { - fn default() -> Self { - Self { - config: default_test_config(), - } - } - } - - #[async_trait(?Send)] - impl TestServer> for LambdaTestServer { - async fn get_expected_path(&self) -> String { - spawn_server(self.get_config()).await - } - - fn get_config(&self) -> &Config { - &self.config - } - - fn get_request(&self) -> LambdaTestRequest { - LambdaTestRequest(Request::default()) - } - - async fn test_server( - &self, - request: LambdaTestRequest, - expected_path: String, - ) -> TestResponse { - let router = Router::new( - Arc::new(self.config.clone().owned_resolvers()), - self.config.service_info(), - ); - - route_request_to_response(request.0, router, expected_path, &self.config).await - } - } - - impl LambdaTestServer { - fn new_with_tls>(path: P) -> Self { - Self { - config: config_with_tls(path), - } - } - } - - #[tokio::test] - async fn get_http_tickets() { - server::test_get::(&LambdaTestServer::default()).await; - } - - #[tokio::test] - async fn post_http_tickets() { - server::test_post::(&LambdaTestServer::default()).await; - } - - #[tokio::test] - async fn parameterized_get_http_tickets() { - server::test_parameterized_get::(&LambdaTestServer::default()).await; - } - - #[tokio::test] - async fn parameterized_post_http_tickets() { - server::test_parameterized_post::(&LambdaTestServer::default()).await; - } - - #[tokio::test] - async fn parameterized_post_class_header_http_tickets() { - server::test_parameterized_post_class_header::(&LambdaTestServer::default()) - .await; - } - - #[tokio::test] - async fn cors_simple_request() { - cors::test_cors_simple_request(&LambdaTestServer::default()).await; - } - - #[tokio::test] - async fn cors_preflight_request() { - cors::test_cors_preflight_request(&LambdaTestServer::default()).await; - } - - #[tokio::test] - async fn get_https_tickets() { - let base_path = TempDir::new().unwrap(); - server::test_get::(&LambdaTestServer::new_with_tls(base_path.path())).await; - } - - #[tokio::test] - async fn post_https_tickets() { - let base_path = TempDir::new().unwrap(); - server::test_post::(&LambdaTestServer::new_with_tls(base_path.path())).await; - } - - #[tokio::test] - async fn parameterized_get_https_tickets() { - let base_path = TempDir::new().unwrap(); - server::test_parameterized_get::(&LambdaTestServer::new_with_tls( - base_path.path(), - )) - .await; - } - - #[tokio::test] - async fn parameterized_post_https_tickets() { - let base_path = TempDir::new().unwrap(); - server::test_parameterized_post::(&LambdaTestServer::new_with_tls( - base_path.path(), - )) - .await; - } - - #[tokio::test] - async fn parameterized_post_class_header_https_tickets() { - let base_path = TempDir::new().unwrap(); - server::test_parameterized_post_class_header::( - &LambdaTestServer::new_with_tls(base_path.path()), - ) - .await; - } - - #[tokio::test] - async fn service_info() { - server::test_service_info(&LambdaTestServer::default()).await; - } - - #[tokio::test] - async fn get_from_file_http_tickets() { - let config = default_test_config(); - endpoint_from_file("events/event_get.json", Class::Body, &config).await; - } - - #[tokio::test] - async fn post_from_file_http_tickets() { - let config = default_test_config(); - endpoint_from_file("events/event_post.json", Class::Body, &config).await; - } - - #[tokio::test] - async fn parameterized_get_from_file_http_tickets() { - let config = default_test_config(); - endpoint_from_file( - "events/event_parameterized_get.json", - Class::Header, - &config, - ) - .await; - } - - #[tokio::test] - async fn parameterized_post_from_file_http_tickets() { - let config = default_test_config(); - endpoint_from_file("events/event_parameterized_post.json", Class::Body, &config).await; - } - - #[tokio::test] - async fn parameterized_post_class_header_from_file_http_tickets() { - let config = default_test_config(); - endpoint_from_file( - "events/event_parameterized_post_class_header.json", - Class::Header, - &config, - ) - .await; - } - - #[tokio::test] - async fn get_from_file_https_tickets() { - let base_path = TempDir::new().unwrap(); - let config = config_with_tls(base_path.path()); - endpoint_from_file("events/event_get.json", Class::Body, &config).await; - } - - #[tokio::test] - async fn post_from_file_https_tickets() { - let base_path = TempDir::new().unwrap(); - let config = config_with_tls(base_path.path()); - endpoint_from_file("events/event_post.json", Class::Body, &config).await; - } - - #[tokio::test] - async fn parameterized_get_from_file_https_tickets() { - let base_path = TempDir::new().unwrap(); - let config = config_with_tls(base_path.path()); - endpoint_from_file( - "events/event_parameterized_get.json", - Class::Header, - &config, - ) - .await; - } - - #[tokio::test] - async fn parameterized_post_from_file_https_tickets() { - let base_path = TempDir::new().unwrap(); - let config = config_with_tls(base_path.path()); - endpoint_from_file("events/event_parameterized_post.json", Class::Body, &config).await; - } - - #[tokio::test] - async fn parameterized_post_class_header_from_file_https_tickets() { - let base_path = TempDir::new().unwrap(); - let config = config_with_tls(base_path.path()); - endpoint_from_file( - "events/event_parameterized_post_class_header.json", - Class::Header, - &config, - ) - .await; - } - - #[tokio::test] - async fn service_info_from_file() { - let config = default_test_config(); - test_service_info_from_file("events/event_service_info.json", &config).await; - } - - #[test] - fn get_route_invalid_method() { - let uri = Uri::builder().path_and_query("/reads/id").build().unwrap(); - test_expected_invalid_method(&Method::DELETE, &uri, StatusCode::METHOD_NOT_ALLOWED); - } - - #[test] - fn get_route_no_path() { - let uri = Uri::builder().path_and_query("").build().unwrap(); - test_expected_invalid_method(&Method::GET, &uri, StatusCode::NOT_FOUND); - } - - #[test] - fn get_route_no_endpoint() { - let uri = Uri::builder().path_and_query("/path/").build().unwrap(); - test_expected_invalid_method(&Method::GET, &uri, StatusCode::NOT_FOUND); - } - - #[test] - fn get_route_reads_no_id() { - let uri = Uri::builder().path_and_query("/reads/").build().unwrap(); - test_expected_invalid_method(&Method::GET, &uri, StatusCode::NOT_FOUND); - } - - #[test] - fn get_route_variants_no_id() { - let uri = Uri::builder().path_and_query("/variants/").build().unwrap(); - test_expected_invalid_method(&Method::GET, &uri, StatusCode::NOT_FOUND); - } - - #[test] - fn get_route_reads_service_info() { - let uri = Uri::builder() - .path_and_query("/reads/service-info") - .build() - .unwrap(); - let route = Route::get_route(&Method::GET, &uri).unwrap(); - assert_eq!( - route, - Route { - method: HtsgetMethod::Get, - endpoint: Endpoint::Reads, - route_type: RouteType::ServiceInfo - } - ); - } - - #[test] - fn get_route_variants_service_info() { - let uri = Uri::builder() - .path_and_query("/variants/service-info") - .build() - .unwrap(); - let route = Route::get_route(&Method::GET, &uri).unwrap(); - assert_eq!( - route, - Route { - method: HtsgetMethod::Get, - endpoint: Endpoint::Variants, - route_type: RouteType::ServiceInfo - } - ); - } - - #[test] - fn route_get_reads_id() { - let uri = Uri::builder().path_and_query("/reads/id").build().unwrap(); - let route = Route::get_route(&Method::GET, &uri).unwrap(); - assert_eq!( - route, - Route { - method: HtsgetMethod::Get, - endpoint: Endpoint::Reads, - route_type: RouteType::Id("id".to_string()) - } - ); - } - - #[test] - fn route_post_reads_id() { - let uri = Uri::builder().path_and_query("/reads/id").build().unwrap(); - let route = Route::get_route(&Method::POST, &uri).unwrap(); - assert_eq!( - route, - Route { - method: HtsgetMethod::Post, - endpoint: Endpoint::Reads, - route_type: RouteType::Id("id".to_string()) - } - ); - } - - #[test] - fn route_get_variants_id() { - let uri = Uri::builder() - .path_and_query("/variants/id") - .build() - .unwrap(); - let route = Route::get_route(&Method::GET, &uri).unwrap(); - assert_eq!( - route, - Route { - method: HtsgetMethod::Get, - endpoint: Endpoint::Variants, - route_type: RouteType::Id("id".to_string()) - } - ); - } - - #[test] - fn route_post_variants_id() { - let uri = Uri::builder() - .path_and_query("/variants/id") - .build() - .unwrap(); - let route = Route::get_route(&Method::POST, &uri).unwrap(); - assert_eq!( - route, - Route { - method: HtsgetMethod::Post, - endpoint: Endpoint::Variants, - route_type: RouteType::Id("id".to_string()) - } - ); - } - - fn test_expected_invalid_method(method: &Method, uri: &Uri, expected_status_code: StatusCode) { - if let Err(err) = Route::get_route(method, uri) { - let err = err.unwrap(); - assert_eq!(err.status(), expected_status_code); - assert_eq!(err.body(), &Body::Empty); - }; - } - - async fn with_router<'a, F, Fut>(test: F, config: &'a Config) - where - F: FnOnce(Router<'a, Vec>) -> Fut, - Fut: Future, - { - let router = Router::new( - Arc::new(config.clone().owned_resolvers()), - config.service_info(), - ); - test(router).await; - } - - fn get_request_from_file(file_path: &str) -> Request { - let event = get_test_file(file_path); - lambda_http::request::from_str(&event).expect("Failed to create lambda request.") - } - - async fn spawn_server(config: &Config) -> String { - let mut bind_data_server = BindServer::from(config.data_server().clone()); - let server = bind_data_server - .bind_data_server("/data".to_string()) - .await - .unwrap(); - let addr = server.local_addr(); - - let path = config.data_server().local_path().to_path_buf(); - tokio::spawn(async move { server.serve(path).await.unwrap() }); - - expected_url_path(config, addr.unwrap()) - } - - async fn endpoint_from_file(file_path: &str, class: Class, config: &Config) { - let expected_path = spawn_server(config).await; - - with_router( - |router| async move { - let response = route_request_to_response( - get_request_from_file(file_path), - router, - expected_path, - config, - ) - .await; - test_response::(response, class).await; - }, - config, - ) - .await; - } - - async fn test_service_info_from_file(file_path: &str, config: &Config) { - let expected_path = expected_url_path(config, config.data_server().addr()); - - with_router( - |router| async { - let response = route_request_to_response( - get_request_from_file(file_path), - router, - expected_path, - config, - ) - .await; - test_response_service_info(&response); - }, - config, - ) - .await; - } - - async fn route_request_to_response( - request: Request, - router: Router<'_, T>, - expected_path: String, - config: &Config, - ) -> TestResponse { - let response = ServiceBuilder::new() - .layer(configure_cors(config.ticket_server().cors().clone())) - .service(service_fn(|event: Request| async { - router.route_request(event).await - })) - .ready() - .await - .unwrap() - .call(request) - .await - .expect("failed to route request"); - - let status: u16 = response.status().into(); - let body = response.body().to_vec(); - - TestResponse::new(status, response.headers().clone(), body, expected_path) - } -} diff --git a/htsget-lambda/src/main.rs b/htsget-lambda/src/main.rs index b8e012d3c..9187f09ea 100644 --- a/htsget-lambda/src/main.rs +++ b/htsget-lambda/src/main.rs @@ -1,17 +1,18 @@ -use std::sync::Arc; - -use lambda_http::Error; -use tracing::debug; +use std::env::set_var; +use htsget_axum::server::ticket::TicketServer; use htsget_config::command; -use htsget_lambda::Config; -use htsget_lambda::{handle_request, Router}; +use htsget_config::config::Config; +use lambda_http::{run, Error}; +use tracing::debug; #[tokio::main] async fn main() -> Result<(), Error> { - if let Some(path) = - Config::parse_args_with_command(command!()).expect("expected valid command parsing") - { + // Ignore the API gateway stage. + // See https://github.com/awslabs/aws-lambda-rust-runtime/tree/main/lambda-http#integration-with-api-gateway-stages + set_var("AWS_LAMBDA_HTTP_IGNORE_STAGE_IN_PATH", "true"); + + if let Some(path) = Config::parse_args_with_command(command!())? { let config = Config::from_path(&path)?; config.setup_tracing()?; @@ -20,9 +21,9 @@ async fn main() -> Result<(), Error> { let service_info = config.service_info().clone(); let cors = config.ticket_server().cors().clone(); - let router = &Router::new(Arc::new(config.owned_resolvers()), &service_info); + let router = TicketServer::router(config.owned_resolvers(), service_info, cors); - handle_request(cors, router).await + run(router).await } else { Ok(()) } diff --git a/htsget-test/src/http/cors.rs b/htsget-test/src/http/cors.rs index 4250d0e6c..a95c82821 100644 --- a/htsget-test/src/http/cors.rs +++ b/htsget-test/src/http/cors.rs @@ -1,11 +1,10 @@ +use crate::http::{Header, TestRequest, TestServer}; use http::header::{ ACCESS_CONTROL_ALLOW_HEADERS, ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN, ACCESS_CONTROL_REQUEST_HEADERS, ACCESS_CONTROL_REQUEST_METHOD, ORIGIN, }; use http::Method; -use crate::http::{Header, TestRequest, TestServer}; - /// A simple cors request test. pub async fn test_cors_simple_request(tester: &impl TestServer) { test_cors_simple_request_uri(tester, "/variants/service-info").await; @@ -14,12 +13,12 @@ pub async fn test_cors_simple_request(tester: &impl TestServer(tester: &impl TestServer, uri: &str) { let request = tester - .get_request() - .method(Method::GET.to_string()) + .request() + .method(Method::GET) .uri(uri) .insert_header(Header { - name: ORIGIN.to_string(), - value: "http://example.com".to_string(), + name: ORIGIN, + value: http::HeaderValue::from_static("http://example.com"), }); let response = tester .test_server(request, tester.get_expected_path().await) @@ -48,20 +47,20 @@ pub async fn test_cors_preflight_request_uri( uri: &str, ) { let request = tester - .get_request() - .method(Method::OPTIONS.to_string()) + .request() + .method(Method::OPTIONS) .uri(uri) .insert_header(Header { - name: ORIGIN.to_string(), - value: "http://example.com".to_string(), + name: ORIGIN, + value: http::HeaderValue::from_static("http://example.com"), }) .insert_header(Header { - name: ACCESS_CONTROL_REQUEST_HEADERS.to_string(), - value: "X-Requested-With".to_string(), + name: ACCESS_CONTROL_REQUEST_HEADERS, + value: http::HeaderValue::from_static("X-Requested-With"), }) .insert_header(Header { - name: ACCESS_CONTROL_REQUEST_METHOD.to_string(), - value: "POST".to_string(), + name: ACCESS_CONTROL_REQUEST_METHOD, + value: http::HeaderValue::from_static("POST"), }); let response = tester .test_server(request, tester.get_expected_path().await) diff --git a/htsget-test/src/http/mod.rs b/htsget-test/src/http/mod.rs index 140abe034..c4e7f068b 100644 --- a/htsget-test/src/http/mod.rs +++ b/htsget-test/src/http/mod.rs @@ -12,7 +12,7 @@ use std::str::FromStr; use async_trait::async_trait; use http::uri::Authority; -use http::HeaderMap; +use http::{HeaderMap, HeaderName, Method}; use serde::de; use htsget_config::config::cors::{AllowType, CorsConfig}; @@ -29,13 +29,13 @@ use crate::Config; /// Represents a http header. #[derive(Debug)] -pub struct Header> { - pub name: T, - pub value: T, +pub struct Header { + pub name: K, + pub value: V, } -impl> Header { - pub fn into_tuple(self) -> (String, String) { +impl, V: Into> Header { + pub fn into_tuple(self) -> (HeaderName, http::HeaderValue) { (self.name.into(), self.value.into()) } } @@ -75,10 +75,13 @@ impl Response { /// Mock request trait that should be implemented to use test functions. pub trait TestRequest { - fn insert_header(self, header: Header>) -> Self; + fn insert_header( + self, + header: Header, impl Into>, + ) -> Self; fn set_payload(self, payload: impl Into) -> Self; fn uri(self, uri: impl Into) -> Self; - fn method(self, method: impl Into) -> Self; + fn method(self, method: impl Into) -> Self; } /// Mock server trait that should be implemented to use test functions. @@ -86,7 +89,7 @@ pub trait TestRequest { pub trait TestServer { async fn get_expected_path(&self) -> String; fn get_config(&self) -> &Config; - fn get_request(&self) -> T; + fn request(&self) -> T; async fn test_server(&self, request: T, expected_path: String) -> Response; } diff --git a/htsget-test/src/http/server.rs b/htsget-test/src/http/server.rs index d721f65b7..26402bf49 100644 --- a/htsget-test/src/http/server.rs +++ b/htsget-test/src/http/server.rs @@ -1,7 +1,7 @@ use std::fmt::Debug; use std::net::SocketAddr; -use http::Method; +use http::{HeaderValue, Method, StatusCode}; use reqwest::ClientBuilder; use serde::Deserialize; use serde_json::{json, Value}; @@ -105,12 +105,12 @@ where tester, vec![ tester - .get_request() - .method(Method::GET.to_string()) + .request() + .method(Method::GET) .uri("/variants/1-vcf/sample1-bcbio-cancer"), tester - .get_request() - .method(Method::GET.to_string()) + .request() + .method(Method::GET) .uri("/variants/2-vcf/sample1-bcbio-cancer"), ], Class::Body, @@ -120,12 +120,15 @@ where fn post_request_one(tester: &impl TestServer) -> T { tester - .get_request() - .method(Method::POST.to_string()) + .request() + .method(Method::POST) .uri("/variants/1-vcf/sample1-bcbio-cancer") .insert_header(Header { - name: http::header::CONTENT_TYPE.to_string(), - value: mime::APPLICATION_JSON.to_string(), + name: http::header::CONTENT_TYPE, + value: mime::APPLICATION_JSON + .to_string() + .parse::() + .unwrap(), }) } @@ -147,6 +150,18 @@ where } } +/// Test an array of requests that are expected to return error status codes. +async fn test_error_response( + tester: &impl TestServer, + request: T, + expected_status: StatusCode, +) where + T: TestRequest, +{ + let response = tester.test_server(request, "".to_string()).await; + assert_eq!(response.status, expected_status); +} + /// A post test using the tester. pub async fn test_post(tester: &impl TestServer) where @@ -174,12 +189,12 @@ where tester, vec![ tester - .get_request() - .method(Method::GET.to_string()) + .request() + .method(Method::GET) .uri("/variants/1-vcf/sample1-bcbio-cancer?format=VCF&class=header"), tester - .get_request() - .method(Method::GET.to_string()) + .request() + .method(Method::GET) .uri("/variants/2-vcf/sample1-bcbio-cancer?format=VCF&class=header"), ], Class::Header, @@ -230,8 +245,8 @@ where /// A service info test. pub async fn test_service_info(tester: &impl TestServer) { let request = tester - .get_request() - .method(Method::GET.to_string()) + .request() + .method(Method::GET) .uri("/variants/service-info"); let response = tester .test_server(request, tester.get_expected_path().await) @@ -240,6 +255,43 @@ pub async fn test_service_info(tester: &impl TestServer) { test_response_service_info(&response); } +/// Test requests that should result in errors. +pub async fn test_errors(tester: &impl TestServer) +where + T: TestRequest, +{ + test_error_response( + tester, + tester.request().method(Method::DELETE).uri("/reads/id"), + StatusCode::METHOD_NOT_ALLOWED, + ) + .await; + test_error_response( + tester, + tester.request().method(Method::GET).uri("/"), + StatusCode::NOT_FOUND, + ) + .await; + test_error_response( + tester, + tester.request().method(Method::GET).uri("/path"), + StatusCode::NOT_FOUND, + ) + .await; + test_error_response( + tester, + tester.request().method(Method::GET).uri("/reads"), + StatusCode::NOT_FOUND, + ) + .await; + test_error_response( + tester, + tester.request().method(Method::DELETE).uri("/variants"), + StatusCode::NOT_FOUND, + ) + .await; +} + /// An example VCF search response. pub fn expected_response(class: Class, url_path: String) -> Value { let url = format!("{url_path}/data/vcf/sample1-bcbio-cancer.vcf.gz"); From dd5bf2f0cd143e0f3f08ad1b5a249744bfdd8019 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 6 Sep 2024 12:30:18 +1000 Subject: [PATCH 2/5] test: add more server tests and fix Dockerfile.dockerignore --- Cargo.lock | 55 ++++++++++++++++---------------- deploy/Dockerfile.dockerignore | 2 ++ htsget-axum/src/server/ticket.rs | 2 +- htsget-config/Cargo.toml | 2 +- htsget-lambda/README.md | 4 +-- htsget-lambda/src/lib.rs | 3 ++ htsget-search/Cargo.toml | 2 +- htsget-test/Cargo.toml | 2 +- htsget-test/src/http/concat.rs | 7 ++-- htsget-test/src/http/server.rs | 46 ++++++++++++++++++++++++++ 10 files changed, 86 insertions(+), 39 deletions(-) create mode 100644 htsget-lambda/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 9686837e0..8a94b3b44 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1041,9 +1041,9 @@ dependencies = [ [[package]] name = "bit-vec" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2c54ff287cfc0a34f38a6b832ea1bd8e448a330b3e40a50859e6488bee07f22" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" @@ -2786,9 +2786,9 @@ dependencies = [ [[package]] name = "noodles" -version = "0.78.0" +version = "0.80.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81e0702ff73390d2b8f97751bc34cfedcd3f5d39b4e32875e4c64a32d8cc670b" +checksum = "15ea7a4ffa2e3684ce476156d199e388b465d860d702b739f3e39257ded7d174" dependencies = [ "noodles-bam", "noodles-bcf", @@ -2806,11 +2806,10 @@ dependencies = [ [[package]] name = "noodles-bam" -version = "0.65.0" +version = "0.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "406d4768f21c73e3075c0c0d77a5b21bc8b8169c8f0963122607cc410427b727" +checksum = "53bc69bd00891e3e1c5faffe4f55d00c94d9e53d4bdbe63ec8c7e2b881f3bc85" dependencies = [ - "bit-vec", "bstr", "byteorder", "bytes", @@ -2825,9 +2824,9 @@ dependencies = [ [[package]] name = "noodles-bcf" -version = "0.58.0" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f217d811dde790866d3c4902ef713f9dc5c4b96e51935571ddf68399f97ad6" +checksum = "3206eec367e12e38ff2efb9f11478be5d79a117870f804abc2dec8f27dd10d18" dependencies = [ "byteorder", "futures", @@ -2841,9 +2840,9 @@ dependencies = [ [[package]] name = "noodles-bgzf" -version = "0.32.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2fba0f4a64cc897d9396d730a0c444d148daed7de31ad5904ecc673178fc9d" +checksum = "3b50aaa8f0a3c8a0b738b641a6d1a78d9fd30a899ab2d398779ee3c4eb80f1c1" dependencies = [ "byteorder", "bytes", @@ -2866,9 +2865,9 @@ dependencies = [ [[package]] name = "noodles-cram" -version = "0.66.0" +version = "0.68.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c7f58d900eb9fa1f0ef1a4834a6f71bfa792b25b2fa3b02e582239089907909" +checksum = "68d1ea4ae713b5e17321ebb20e2f5ca99cadbe2b9e08501a7043458f3a66406e" dependencies = [ "async-compression", "bitflags", @@ -2891,9 +2890,9 @@ dependencies = [ [[package]] name = "noodles-csi" -version = "0.37.0" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4bc8001c54f1d8e47e1ac6041a5f27edc99b68bacea3fade9c89059de285aea" +checksum = "a69e79dbc09bd0cb86d29469ed29066e9a163bce6640527b343bdea458144618" dependencies = [ "bit-vec", "byteorder", @@ -2905,9 +2904,9 @@ dependencies = [ [[package]] name = "noodles-fasta" -version = "0.41.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a1662ac3ace299515c982a322e378bbeb4c1bd90fb098d823ef0f3a6abcc00" +checksum = "0634eec06d20e899a5d99922c40fa5186064d8c675c78690a461ccbb2edc60d1" dependencies = [ "bstr", "bytes", @@ -2919,10 +2918,11 @@ dependencies = [ [[package]] name = "noodles-fastq" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1edf1f924acddeee36304c444e242b9bda52ef9383dc2d7f008fca190753207" +checksum = "c596792c857f37e6a85e2cf1e68578f5b70f867cad028bf95c1e2b5d7c9c84eb" dependencies = [ + "bstr", "futures", "memchr", "tokio", @@ -2930,9 +2930,9 @@ dependencies = [ [[package]] name = "noodles-gff" -version = "0.35.0" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adef59012090b5694b58cad0e4426cd18af404803f942d02e664af607d89ee28" +checksum = "83d0a43629762ce799147e0d60f80293abe731b65740b7a6a92c1279ef88f0b5" dependencies = [ "futures", "indexmap 2.2.6", @@ -2945,9 +2945,9 @@ dependencies = [ [[package]] name = "noodles-sam" -version = "0.62.0" +version = "0.64.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b80efc627501962e2ff15411d1c011fa9cf3db1b47ddd13dceb1d1134068d5b7" +checksum = "72da678e9332b32a916f8c5d5a7c4324da11891bc7148077744566981acaf00c" dependencies = [ "bitflags", "bstr", @@ -2963,11 +2963,10 @@ dependencies = [ [[package]] name = "noodles-tabix" -version = "0.43.0" +version = "0.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "545e16e229b7f8734b0a2a36bd4c98a5b70128663b16b5201ddadc0d09c28d4a" +checksum = "263e63f58871224d0cd30ffc4a8531fb4f8f8ec686807febde8e19a69673fe01" dependencies = [ - "bit-vec", "byteorder", "indexmap 2.2.6", "noodles-bgzf", @@ -2978,9 +2977,9 @@ dependencies = [ [[package]] name = "noodles-vcf" -version = "0.61.0" +version = "0.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6372f1df57c1826d083370b6eac586f331509feed15fd80dda306ef3e7ac68d" +checksum = "549043d6958379906b1e306804e3c463f24db880da9ab73bd7678592aafb9c89" dependencies = [ "futures", "indexmap 2.2.6", diff --git a/deploy/Dockerfile.dockerignore b/deploy/Dockerfile.dockerignore index da4ac473c..8b535d547 100644 --- a/deploy/Dockerfile.dockerignore +++ b/deploy/Dockerfile.dockerignore @@ -1,10 +1,12 @@ * !/htsget-actix +!/htsget-axum !/htsget-config !/htsget-http !/htsget-lambda !/htsget-search +!/htsget-storage !/htsget-test !/Cargo.toml !/Cargo.lock diff --git a/htsget-axum/src/server/ticket.rs b/htsget-axum/src/server/ticket.rs index 0d50899ca..5abb5cbc1 100644 --- a/htsget-axum/src/server/ticket.rs +++ b/htsget-axum/src/server/ticket.rs @@ -306,7 +306,7 @@ mod tests { } #[tokio::test] - async fn get_route_invalid_method() { + async fn test_errors() { server::test_errors(&AxumTestServer::default()).await; } } diff --git a/htsget-config/Cargo.toml b/htsget-config/Cargo.toml index d783817db..87a0e26e4 100644 --- a/htsget-config/Cargo.toml +++ b/htsget-config/Cargo.toml @@ -18,7 +18,7 @@ default = [] [dependencies] thiserror = "1" async-trait = "0.1" -noodles = { version = "0.78", features = ["core"] } +noodles = { version = "0.80", features = ["core"] } serde = { version = "1", features = ["derive"] } serde_with = "3" serde_regex = "1" diff --git a/htsget-lambda/README.md b/htsget-lambda/README.md index cb69330be..881ab236c 100644 --- a/htsget-lambda/README.md +++ b/htsget-lambda/README.md @@ -38,8 +38,8 @@ See [htsget-search] for details on how to structure files. ### As a library -There shouldn't be any need to interact with this crate as a library, however some functions which deal with -routing queries are exposed in the public API. +There is no need to interact with this crate as a library. Note that the Lambda function itself doesn't have any +library code, and it instead uses `htsget-axum`. Please use that crate for functionality related to routing. #### Feature flags diff --git a/htsget-lambda/src/lib.rs b/htsget-lambda/src/lib.rs new file mode 100644 index 000000000..5456a94e7 --- /dev/null +++ b/htsget-lambda/src/lib.rs @@ -0,0 +1,3 @@ +//! htsget-lambda no longer has any library functions. Please use `htsget-axum` for similar +//! functionality on routers and logic. +//! diff --git a/htsget-search/Cargo.toml b/htsget-search/Cargo.toml index ad3994c9f..ead90d0e5 100644 --- a/htsget-search/Cargo.toml +++ b/htsget-search/Cargo.toml @@ -32,7 +32,7 @@ futures-util = "0.3" async-trait = "0.1" # Noodles -noodles = { version = "0.78", features = ["async", "core", "bgzf", "bam", "bcf", "cram", "csi", "sam", "tabix", "vcf"] } +noodles = { version = "0.80", features = ["async", "core", "bgzf", "bam", "bcf", "cram", "csi", "sam", "tabix", "vcf"] } # Error control, tracing, config thiserror = "1" diff --git a/htsget-test/Cargo.toml b/htsget-test/Cargo.toml index db85b3ab8..00c64c34d 100644 --- a/htsget-test/Cargo.toml +++ b/htsget-test/Cargo.toml @@ -42,7 +42,7 @@ default = [] # Server tests dependencies htsget-config = { version = "0.10.1", path = "../htsget-config", default-features = false, optional = true } -noodles = { version = "0.78", optional = true, features = ["async", "bgzf", "vcf", "cram", "bcf", "bam", "fasta"] } +noodles = { version = "0.80", optional = true, features = ["async", "bgzf", "vcf", "cram", "bcf", "bam", "fasta"] } reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"], optional = true } tokio = { version = "1", features = ["rt-multi-thread", "fs"], optional = true } diff --git a/htsget-test/src/http/concat.rs b/htsget-test/src/http/concat.rs index 77c7eb65c..1235d5ea7 100644 --- a/htsget-test/src/http/concat.rs +++ b/htsget-test/src/http/concat.rs @@ -6,7 +6,7 @@ use futures::future::join_all; use futures::{Stream, TryStreamExt}; use htsget_config::types::{Class, Format, Response, Url}; use http::{HeaderMap, HeaderName, HeaderValue}; -use noodles::{bam, bcf, bgzf, cram, fasta, vcf}; +use noodles::{bam, bcf, bgzf, cram, vcf}; use reqwest::Client; use std::future::Future; use std::io; @@ -199,7 +199,6 @@ impl ReadRecords { .read_file_definition() .await .map_err(TestError::read_record)?; - let repository = fasta::Repository::default(); let header = reader .read_file_header() .await @@ -208,9 +207,7 @@ impl ReadRecords { .map_err(TestError::read_record)?; println!("{:#?}", header); - self - .iterate_records(reader.records(&repository, &header)) - .await + self.iterate_records(reader.records(&header)).await } Format::Vcf => { let mut reader = diff --git a/htsget-test/src/http/server.rs b/htsget-test/src/http/server.rs index 26402bf49..0b183650e 100644 --- a/htsget-test/src/http/server.rs +++ b/htsget-test/src/http/server.rs @@ -290,6 +290,52 @@ where StatusCode::NOT_FOUND, ) .await; + + test_error_response( + tester, + tester + .request() + .method(Method::GET) + .uri("/variants/1-vcf/sample1-bcbio-cancer?format=BED"), + StatusCode::BAD_REQUEST, + ) + .await; + test_error_response( + tester, + tester + .request() + .method(Method::GET) + .uri("/variants/1-vcf/sample1-bcbio-cancer?class=header&start=1"), + StatusCode::BAD_REQUEST, + ) + .await; + test_error_response( + tester, + tester + .request() + .method(Method::GET) + .uri("/variants/1-vcf/sample1-bcbio-cancer?referenceName=*&start=1"), + StatusCode::BAD_REQUEST, + ) + .await; + test_error_response( + tester, + tester + .request() + .method(Method::GET) + .uri("/variants/1-vcf/sample1-bcbio-cancer?referenceName=chr1&start=2&end=1"), + StatusCode::BAD_REQUEST, + ) + .await; + test_error_response( + tester, + tester + .request() + .method(Method::GET) + .uri("/variants/1-vcf/sample1-bcbio-cancer?referenceName=*&end=1"), + StatusCode::BAD_REQUEST, + ) + .await; } /// An example VCF search response. From d2b018f686d312eb944ae5c1af5e2ee9c32d5d0f Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 6 Sep 2024 12:33:18 +1000 Subject: [PATCH 3/5] build: update dockerfile to use axum instead of actix --- deploy/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 73edccfa0..4ef68edc4 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.76-slim AS builder +FROM rust:1.81-slim AS builder LABEL org.opencontainers.image.source=https://github.com/umccr/htsget-rs LABEL org.opencontainers.image.url=https://github.com/umccr/htsget-rs/pkgs/container/htsget-rs @@ -17,9 +17,9 @@ RUN cargo build --all-features --release && \ FROM gcr.io/distroless/cc-debian12 -COPY --from=builder /build/target/release/htsget-actix /usr/local/bin/htsget-actix +COPY --from=builder /build/target/release/htsget-axum /usr/local/bin/htsget-axum ENV HTSGET_TICKET_SERVER_ADDR 0.0.0.0:8080 ENV HTSGET_DATA_SERVER_ADDR 0.0.0.0:8081 -CMD [ "htsget-actix" ] +CMD [ "htsget-axum" ] From 59a4aa3672d6b2d3d1a3287bbf324781f679173c Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 6 Sep 2024 13:32:31 +1000 Subject: [PATCH 4/5] deploy: add option to create buckets with test data and clarify docs --- deploy/README.md | 25 ++- deploy/bin/settings.ts | 1 + deploy/config/dev_umccr.toml | 5 + deploy/examples/local_storage/README.md | 3 +- deploy/examples/minio/README.md | 18 +- deploy/lib/htsget-lambda-stack.ts | 47 +++- deploy/package-lock.json | 286 ++++++++++++++++-------- deploy/package.json | 14 +- 8 files changed, 266 insertions(+), 133 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 98f18f27b..44cee9599 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -13,24 +13,27 @@ The CDK code in this directory constructs a CDK app from [`HtsgetLambdaStack`][h [`bin/settings.ts`][htsget-settings]: #### HtsgetSettings + These are general settings for the CDK deployment. -| Name | Description | Type | -|----------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------| -| `config` | The location of the htsget-rs server config. This must be specified. This config file configures the htsget-rs server. See [htsget-config] for a list of available server configuration options. | `string` | -| `domain` | The domain name for the Route53 Hosted Zone that the htsget-rs server will be under. This must be specified. A hosted zone with this name will either be looked up or created depending on the value of [`lookupHostedZone?`](#lookupHostedZone). | `string` | -| `authorizer` | Deployment options related to the authorizer. Note that this option allows specifying an AWS [JWT authorizer][jwt-authorizer]. The JWT authorizer automatically verifies tokens issued by a Cognito user pool. | [`HtsgetJwtAuthSettings`](#htsgetjwtauthsettings) | -| `subDomain?` | The domain name prefix to use for the htsget-rs server. Together with the [`domain`](#domain), this specifies url that the htsget-rs server will be reachable under. Defaults to `"htsget"`. | `string` | -| `s3BucketResources?` | The resources that are affected by the bucket policy with actions: `["s3:List*", "s3:Get*"]`. If this is not specified, it defaults to `["arn:aws:s3:::*"]`. This affects which buckets are allowed to be accessed with the policy. | `string[]` | -| `lookupHostedZone?` | Whether to lookup the hosted zone with the domain name. Defaults to `true`. If `true`, attempts to lookup an existing hosted zone using the domain name. Set this to `false` if you want to create a new hosted zone with the domain name. | `boolean` | +| Name | Description | Type | +| ------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------- | +| `config` | The location of the htsget-rs server config. This must be specified. This config file configures the htsget-rs server. See [htsget-config] for a list of available server configuration options. | `string` | +| `domain` | The domain name for the Route53 Hosted Zone that the htsget-rs server will be under. This must be specified. A hosted zone with this name will either be looked up or created depending on the value of [`lookupHostedZone?`](#lookupHostedZone). | `string` | +| `authorizer` | Deployment options related to the authorizer. Note that this option allows specifying an AWS [JWT authorizer][jwt-authorizer]. The JWT authorizer automatically verifies tokens issued by a Cognito user pool. | [`HtsgetJwtAuthSettings`](#htsgetjwtauthsettings) | +| `subDomain?` | The domain name prefix to use for the htsget-rs server. Together with the [`domain`](#domain), this specifies url that the htsget-rs server will be reachable under. Defaults to `"htsget"`. | `string` | +| `s3BucketResources` | The buckets to serve data from. If this is not specified, this defaults to `[]`. This affects which buckets are allowed to be accessed by the policy actions which are `["s3:List*", "s3:Get*"]`. Note that this option alone does not create buckets, it only gives permission to access them, see the `createS3Buckets` option. This option must be specified to allow `htsget-rs` to access data in the buckets. | `string[]` | +| `lookupHostedZone?` | Whether to lookup the hosted zone with the domain name. Defaults to `true`. If `true`, attempts to lookup an existing hosted zone using the domain name. Set this to `false` if you want to create a new hosted zone with the domain name. | `boolean` | +| `createS3Buckets?` | A list of buckets to create. Defaults to no buckets. Buckets are created with [`RemovalPolicy.RETAIN`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.RemovalPolicy.html). This also copies the example data under the `data` directory to those buckets. | `string[]` | #### HtsgetJwtAuthSettings + These settings are used to determine if the htsget API gateway endpoint is configured to have a JWT authorizer or not. | Name | Description | Type | -|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------|------------| +| ------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | | `public` | Whether this deployment is public. If this is `true` then no authorizer is present on the API gateway and the options below have no effect. | `boolean` | -| `jwtAudience?` | A list of the intended recipients of the JWT. A valid JWT must provide an aud that matches at least one entry in this list. | `string[]` | +| `jwtAudience?` | A list of the intended recipients of the JWT. A valid JWT must provide an aud that matches at least one entry in this list. | `string[]` | | `cogUserPoolId?` | The cognito user pool id for the authorizer. If this is not set, then a new user pool is created. No user pool is created if [`public`](#public) is true. | `string` | The [`HtsgetSettings`](#htsgetsettings) are passed into [`HtsgetLambdaStack`][htsget-lambda-stack] in order to change the deployment config. An example of a public instance deployment @@ -49,7 +52,7 @@ After installing the basic dependencies, complete the following steps: 1. Login to AWS and define `CDK_DEFAULT_*` env variables (if not defined already). You must be authenticated with your AWS cloud to run this step. 2. Install [cargo-lambda], as it is used to compile artifacts that are uploaded to aws lambda. -3. Define which configuration to use for htsget-rs as stated in the configuration section. +3. Define which configuration to use for htsget-rs as stated in the configuration section. Below is a summary of commands to run in this directory: diff --git a/deploy/bin/settings.ts b/deploy/bin/settings.ts index 5f4f8a463..09a117a2a 100644 --- a/deploy/bin/settings.ts +++ b/deploy/bin/settings.ts @@ -12,6 +12,7 @@ export const SETTINGS: HtsgetSettings = { "arn:aws:s3:::org.umccr.demo.htsget-rs-data/*", ], lookupHostedZone: true, + createS3Buckets: [], jwtAuthorizer: { // Set this to true if you want a public instance. public: false, diff --git a/deploy/config/dev_umccr.toml b/deploy/config/dev_umccr.toml index 43f97485d..b92035245 100644 --- a/deploy/config/dev_umccr.toml +++ b/deploy/config/dev_umccr.toml @@ -23,6 +23,11 @@ contact_url = "https://umccr.org/" documentation_url = "https://github.com/umccr/htsget-rs" environment = "dev" +[[resolvers]] +regex = '^(org.umccr.dev.htsget-rs-test-data)/(?P.*)$' +substitution_string = '$key' +storage = 'S3' + [[resolvers]] regex = '^(umccr-10c-data-dev)/(?P.*)$' substitution_string = '$key' diff --git a/deploy/examples/local_storage/README.md b/deploy/examples/local_storage/README.md index 699a22e98..77f7af370 100644 --- a/deploy/examples/local_storage/README.md +++ b/deploy/examples/local_storage/README.md @@ -17,6 +17,7 @@ curl http://127.0.0.1:8080/reads/data/bam/htsnexus_test_NA12878 ``` Which outputs: + ```sh { "htsget": { @@ -41,4 +42,4 @@ default settings, and `curl http://127.0.0.1:8080/reads/data/`, noting the e [local]: ../../../htsget-config/README.md#resolvers [compose]: compose.yml -[data]: ../../../data \ No newline at end of file +[data]: ../../../data diff --git a/deploy/examples/minio/README.md b/deploy/examples/minio/README.md index a91cab184..8aaba0ba1 100644 --- a/deploy/examples/minio/README.md +++ b/deploy/examples/minio/README.md @@ -3,17 +3,16 @@ [MinIO][minio] can be used with htsget-rs by configuring the [storage type][storage] as `S3` and setting the `endpoint` to the MinIO server. There are a few specific configuration options that need to be considered to use MinIO with htsget-rs, and those include: -* The standard [AWS environment variables][env-variables] for connecting to AWS services must be set, and configured to match those -used by MinIO. - * This means that htsget-rs expects an `AWS_DEFAULT_REGION` to be set, which must match the region used by MinIO (by default us-east-1). - * It also means that the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` must be set to match the credentials used by MinIO. -* If using virtual-hosted style [addressing][virtual-addressing] instead of path style [addressing][path-addressing], `MINIO_DOMAIN` must be -set on the MinIO server and DNS resolution must allow accessing the MinIO server using `bucket.`. - * Path style addressing can be used instead by setting `path_style = true` under the htsget-rs resolvers storage type. +- The standard [AWS environment variables][env-variables] for connecting to AWS services must be set, and configured to match those + used by MinIO. + _ This means that htsget-rs expects an `AWS_DEFAULT_REGION` to be set, which must match the region used by MinIO (by default us-east-1). + _ It also means that the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` must be set to match the credentials used by MinIO. +- If using virtual-hosted style [addressing][virtual-addressing] instead of path style [addressing][path-addressing], `MINIO_DOMAIN` must be + set on the MinIO server and DNS resolution must allow accessing the MinIO server using `bucket.`. \* Path style addressing can be used instead by setting `path_style = true` under the htsget-rs resolvers storage type. The caveats around the addressing style occur because there are two different addressing styles for S3 buckets, path style, e.g. `http://minio:9000/bucket`, and virtual-hosted style, e.g. `http://bucket.minio:9000`. AWS has declared path style addressing -as [deprecated][path-style-deprecated], so this example sets up virtual-hosted style addressing as the default. +as [deprecated][path-style-deprecated], so this example sets up virtual-hosted style addressing as the default. ## Deployment using Docker @@ -36,6 +35,7 @@ curl http://127.0.0.1:8080/reads/bam/htsnexus_test_NA12878 ``` Outputs: + ```sh { "htsget": { @@ -68,4 +68,4 @@ docker exec -it minio curl -H "Range: bytes=0-2596770" "http://data.minio:9000/b [virtual-addressing]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#virtual-hosted-style-access [path-addressing]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#path-style-access [compose]: compose.yml -[data]: ../../../data \ No newline at end of file +[data]: ../../../data diff --git a/deploy/lib/htsget-lambda-stack.ts b/deploy/lib/htsget-lambda-stack.ts index 3f2164dba..53c75ba5d 100644 --- a/deploy/lib/htsget-lambda-stack.ts +++ b/deploy/lib/htsget-lambda-stack.ts @@ -2,15 +2,15 @@ import { STACK_NAME } from "../bin/htsget-lambda"; import * as TOML from "@iarna/toml"; import { readFileSync } from "fs"; -import { Duration, Stack, StackProps, Tags } from "aws-cdk-lib"; +import { Duration, RemovalPolicy, Stack, StackProps, Tags } from "aws-cdk-lib"; import { Construct } from "constructs"; import { UserPool } from "aws-cdk-lib/aws-cognito"; import { + ManagedPolicy, + PolicyStatement, Role, ServicePrincipal, - PolicyStatement, - ManagedPolicy, } from "aws-cdk-lib/aws-iam"; import { Architecture } from "aws-cdk-lib/aws-lambda"; import { @@ -29,6 +29,12 @@ import { HttpMethod, } from "aws-cdk-lib/aws-apigatewayv2"; import { HttpJwtAuthorizer } from "aws-cdk-lib/aws-apigatewayv2-authorizers"; +import { + BlockPublicAccess, + Bucket, + BucketEncryption, +} from "aws-cdk-lib/aws-s3"; +import { BucketDeployment, Source } from "aws-cdk-lib/aws-s3-deployment"; /** * Settings related to the htsget lambda stack. @@ -50,10 +56,12 @@ export type HtsgetSettings = { subDomain?: string; /** - * Policies to add to the bucket. If this is not specified, this defaults to `["arn:aws:s3:::*"]`. - * This affects which buckets are allowed to be accessed by the policy actions which are `["s3:List*", "s3:Get*"]`. + * The buckets to serve data from. If this is not specified, this defaults to `[]`. This affects which buckets are + * allowed to be accessed by the policy actions which are `["s3:List*", "s3:Get*"]`. Note that this option alone + * does not create buckets, it only gives permission to access them, see the `createS3Buckets` option. + * This option must be specified to allow `htsget-rs` to access data in the buckets. */ - s3BucketResources?: string[]; + s3BucketResources: string[]; /** * Whether this deployment is gated behind a JWT authorizer, or if its public. @@ -66,6 +74,13 @@ export type HtsgetSettings = { * domain name. */ lookupHostedZone?: boolean; + + /** + * A list of buckets to create. Defaults to no buckets. Buckets are created with + * [`RemovalPolicy.RETAIN`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.RemovalPolicy.html). + * This also copies the example data under the `data` directory to those buckets. + */ + createS3Buckets?: string[]; }; /** @@ -151,9 +166,27 @@ export class HtsgetLambdaStack extends Stack { const s3BucketPolicy = new PolicyStatement({ actions: ["s3:List*", "s3:Get*"], - resources: settings.s3BucketResources ?? ["arn:aws:s3:::*"], + resources: settings.s3BucketResources ?? [], }); + if (settings.createS3Buckets) { + for (const name of settings.createS3Buckets ?? []) { + const bucket = new Bucket(this, "Bucket", { + blockPublicAccess: BlockPublicAccess.BLOCK_ALL, + encryption: BucketEncryption.S3_MANAGED, + enforceSSL: true, + removalPolicy: RemovalPolicy.RETAIN, + bucketName: name, + }); + + const dataDir = path.join(__dirname, "..", "..", "data"); + new BucketDeployment(this, "DeployFiles", { + sources: [Source.asset(dataDir)], + destinationBucket: bucket, + }); + } + } + lambdaRole.addManagedPolicy( ManagedPolicy.fromAwsManagedPolicyName( "service-role/AWSLambdaBasicExecutionRole", diff --git a/deploy/package-lock.json b/deploy/package-lock.json index c930155ff..fafd0a98c 100644 --- a/deploy/package-lock.json +++ b/deploy/package-lock.json @@ -9,46 +9,87 @@ "version": "1.0", "dependencies": { "@iarna/toml": "^3.0.0", - "aws-cdk-lib": "^2.144.0", + "aws-cdk-lib": "^2.155.0", "cargo-lambda-cdk": "^0.0.22", - "constructs": "^10.1.148", - "glob": "^10.3.10", + "constructs": "^10.3.0", + "glob": "^11.0.0", "source-map-support": "^0.5.21" }, "bin": { "htsget_app": "bin/htsget-lambda.js" }, "devDependencies": { - "@types/node": "^18.11.18", - "aws-cdk": "^2.148.1", - "prettier": "2.8.3", - "typescript": "^4.9.4" + "@types/node": "^22.5.4", + "aws-cdk": "^2.155.0", + "prettier": "^3.3.3", + "typescript": "^5.5.4" } }, "node_modules/@aws-cdk/asset-awscli-v1": { "version": "2.2.202", "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.202.tgz", - "integrity": "sha512-JqlF0D4+EVugnG5dAsNZMqhu3HW7ehOXm5SDMxMbXNDMdsF0pxtQKNHRl52z1U9igsHmaFpUgSGjbhAJ+0JONg==" + "integrity": "sha512-JqlF0D4+EVugnG5dAsNZMqhu3HW7ehOXm5SDMxMbXNDMdsF0pxtQKNHRl52z1U9igsHmaFpUgSGjbhAJ+0JONg==", + "license": "Apache-2.0" }, "node_modules/@aws-cdk/asset-kubectl-v20": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/@aws-cdk/asset-kubectl-v20/-/asset-kubectl-v20-2.1.2.tgz", - "integrity": "sha512-3M2tELJOxQv0apCIiuKQ4pAbncz9GuLwnKFqxifWfe77wuMxyTRPmxssYHs42ePqzap1LT6GDcPygGs+hHstLg==" + "integrity": "sha512-3M2tELJOxQv0apCIiuKQ4pAbncz9GuLwnKFqxifWfe77wuMxyTRPmxssYHs42ePqzap1LT6GDcPygGs+hHstLg==", + "license": "Apache-2.0" }, "node_modules/@aws-cdk/asset-node-proxy-agent-v6": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-node-proxy-agent-v6/-/asset-node-proxy-agent-v6-2.0.3.tgz", - "integrity": "sha512-twhuEG+JPOYCYPx/xy5uH2+VUsIEhPTzDY0F1KuB+ocjWWB/KEDiOVL19nHvbPCB6fhWnkykXEMJ4HHcKvjtvg==" + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-node-proxy-agent-v6/-/asset-node-proxy-agent-v6-2.1.0.tgz", + "integrity": "sha512-7bY3J8GCVxLupn/kNmpPc5VJz8grx+4RKfnnJiO1LG+uxkZfANZG3RMHhE+qQxxwkyQ9/MfPtTpf748UhR425A==", + "license": "Apache-2.0" + }, + "node_modules/@aws-cdk/cloud-assembly-schema": { + "version": "36.0.24", + "resolved": "https://registry.npmjs.org/@aws-cdk/cloud-assembly-schema/-/cloud-assembly-schema-36.0.24.tgz", + "integrity": "sha512-dHyb4lvd6nbNHLVvdyxVPgwc0MyzN3VzIJnWwGJWKOIwVqL7hvU2NkQQrktY9T2MtdhzUdDFm9qluxuLRV5Cfw==", + "bundleDependencies": [ + "jsonschema", + "semver" + ], + "license": "Apache-2.0", + "dependencies": { + "jsonschema": "^1.4.1", + "semver": "^7.6.3" + }, + "engines": { + "node": ">= 18.18.0" + } + }, + "node_modules/@aws-cdk/cloud-assembly-schema/node_modules/jsonschema": { + "version": "1.4.1", + "inBundle": true, + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/@aws-cdk/cloud-assembly-schema/node_modules/semver": { + "version": "7.6.3", + "inBundle": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } }, "node_modules/@iarna/toml": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@iarna/toml/-/toml-3.0.0.tgz", - "integrity": "sha512-td6ZUkz2oS3VeleBcN+m//Q6HlCFCPrnI0FZhrt/h4XqLEdOyYp2u21nd8MdsR+WJy5r9PTDaHTDDfhf4H4l6Q==" + "integrity": "sha512-td6ZUkz2oS3VeleBcN+m//Q6HlCFCPrnI0FZhrt/h4XqLEdOyYp2u21nd8MdsR+WJy5r9PTDaHTDDfhf4H4l6Q==", + "license": "ISC" }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "license": "ISC", "dependencies": { "string-width": "^5.1.2", "string-width-cjs": "npm:string-width@^4.2.0", @@ -65,21 +106,27 @@ "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "license": "MIT", "optional": true, "engines": { "node": ">=14" } }, "node_modules/@types/node": { - "version": "18.13.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.13.0.tgz", - "integrity": "sha512-gC3TazRzGoOnoKAhUx+Q0t8S9Tzs74z7m0ipwGpSqQrleP14hKxP4/JUeEQcD3W1/aIpnWl8pHowI7WokuZpXg==", - "dev": true + "version": "22.5.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.5.4.tgz", + "integrity": "sha512-FDuKUJQm/ju9fT/SeX/6+gBzoPzlVCzfzmGkwKvRHQVxi4BntVbyIwf6a4Xn62mrvndLiml6z/UBXIdEVjQLXg==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.19.2" + } }, "node_modules/ansi-regex": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "license": "MIT", "engines": { "node": ">=12" }, @@ -91,6 +138,7 @@ "version": "6.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "license": "MIT", "engines": { "node": ">=12" }, @@ -99,10 +147,11 @@ } }, "node_modules/aws-cdk": { - "version": "2.148.1", - "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.148.1.tgz", - "integrity": "sha512-wiAi4vFJ52A42PpU3zRi2gVDqbTXSBVFrqKRqEd8wYL1mqa0qMv9FR35NsgbM1RL9s7g5ZljYvl+G2tXpcp5Eg==", + "version": "2.155.0", + "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.155.0.tgz", + "integrity": "sha512-AV7Ym/o7/xyDh6sqcGatWD6Bqa7Swe0OWJq+1srVww0MdBiy5yM3zYAA1+ZeqZNjFQThJPA+pYZQFTgojuaVBA==", "dev": true, + "license": "Apache-2.0", "bin": { "cdk": "bin/cdk" }, @@ -114,9 +163,9 @@ } }, "node_modules/aws-cdk-lib": { - "version": "2.144.0", - "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.144.0.tgz", - "integrity": "sha512-DpyIyTs8NHX6WgAyYM2mGorirIk+eTjWzXGQRfzAe40qkwcqsb5Ax4JEl5gz1OEo9QIJIgWDtmImgWN0tUbILA==", + "version": "2.155.0", + "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.155.0.tgz", + "integrity": "sha512-QGzDhLldBXsyOUmhgtZ98PiOUS2g1Mb5MO08FiOvQn3+KSyJjQdq0GoyxtDpCNGLaWmIfcyrtB9aDhod38fl9g==", "bundleDependencies": [ "@balena/dockerignore", "case", @@ -130,10 +179,12 @@ "yaml", "mime-types" ], + "license": "Apache-2.0", "dependencies": { "@aws-cdk/asset-awscli-v1": "^2.2.202", "@aws-cdk/asset-kubectl-v20": "^2.1.2", "@aws-cdk/asset-node-proxy-agent-v6": "^2.0.3", + "@aws-cdk/cloud-assembly-schema": "^36.0.5", "@balena/dockerignore": "^1.0.2", "case": "1.6.3", "fs-extra": "^11.2.0", @@ -142,7 +193,7 @@ "mime-types": "^2.1.35", "minimatch": "^3.1.2", "punycode": "^2.3.1", - "semver": "^7.6.0", + "semver": "^7.6.2", "table": "^6.8.2", "yaml": "1.10.2" }, @@ -159,7 +210,7 @@ "license": "Apache-2.0" }, "node_modules/aws-cdk-lib/node_modules/ajv": { - "version": "8.13.0", + "version": "8.16.0", "inBundle": true, "license": "MIT", "dependencies": { @@ -319,17 +370,6 @@ "inBundle": true, "license": "MIT" }, - "node_modules/aws-cdk-lib/node_modules/lru-cache": { - "version": "6.0.0", - "inBundle": true, - "license": "ISC", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/aws-cdk-lib/node_modules/mime-db": { "version": "1.52.0", "inBundle": true, @@ -377,12 +417,9 @@ } }, "node_modules/aws-cdk-lib/node_modules/semver": { - "version": "7.6.0", + "version": "7.6.2", "inBundle": true, "license": "ISC", - "dependencies": { - "lru-cache": "^6.0.0" - }, "bin": { "semver": "bin/semver.js" }, @@ -461,11 +498,6 @@ "punycode": "^2.1.0" } }, - "node_modules/aws-cdk-lib/node_modules/yallist": { - "version": "4.0.0", - "inBundle": true, - "license": "ISC" - }, "node_modules/aws-cdk-lib/node_modules/yaml": { "version": "1.10.2", "inBundle": true, @@ -477,12 +509,14 @@ "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" }, "node_modules/brace-expansion": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" } @@ -490,7 +524,8 @@ "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", - "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "license": "MIT" }, "node_modules/cargo-lambda-cdk": { "version": "0.0.22", @@ -499,6 +534,7 @@ "bundleDependencies": [ "js-toml" ], + "license": "MIT", "dependencies": { "js-toml": "^0.1.1" }, @@ -607,6 +643,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", "dependencies": { "color-name": "~1.1.4" }, @@ -617,20 +654,23 @@ "node_modules/color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" }, "node_modules/constructs": { - "version": "10.1.246", - "resolved": "https://registry.npmjs.org/constructs/-/constructs-10.1.246.tgz", - "integrity": "sha512-2U2hnAuA4tCGGjHk/TulZfSlPobTyokEh+Azuch9nivv2yGI7/5nXDHC14i2MU/K7HFnnkQOHRSrwKSmOZkT/w==", + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/constructs/-/constructs-10.3.0.tgz", + "integrity": "sha512-vbK8i3rIb/xwZxSpTjz3SagHn1qq9BChLEfy5Hf6fB3/2eFbrwt2n9kHwQcS0CPTRBesreeAcsJfMq2229FnbQ==", + "license": "Apache-2.0", "engines": { - "node": ">= 14.17.0" + "node": ">= 16.14.0" } }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "license": "MIT", "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -643,17 +683,20 @@ "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "license": "MIT" }, "node_modules/emoji-regex": { "version": "9.2.2", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "license": "MIT" }, "node_modules/foreground-child": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", - "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.0.tgz", + "integrity": "sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg==", + "license": "ISC", "dependencies": { "cross-spawn": "^7.0.0", "signal-exit": "^4.0.1" @@ -671,6 +714,7 @@ "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", "dev": true, "hasInstallScript": true, + "license": "MIT", "optional": true, "os": [ "darwin" @@ -680,21 +724,23 @@ } }, "node_modules/glob": { - "version": "10.3.10", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.10.tgz", - "integrity": "sha512-fa46+tv1Ak0UPK1TOy/pZrIybNNt4HCv7SDzwyfiOZkvZLEbjsZkJBPtDHVshZjbecAoAGSC20MjLDG/qr679g==", + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-11.0.0.tgz", + "integrity": "sha512-9UiX/Bl6J2yaBbxKoEBRm4Cipxgok8kQYcOPEhScPwebu2I0HoQOuYdIO6S3hLuWoZgpDpwQZMzTFxgpkyT76g==", + "license": "ISC", "dependencies": { "foreground-child": "^3.1.0", - "jackspeak": "^2.3.5", - "minimatch": "^9.0.1", - "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0", - "path-scurry": "^1.10.1" + "jackspeak": "^4.0.1", + "minimatch": "^10.0.0", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^2.0.0" }, "bin": { "glob": "dist/esm/bin.mjs" }, "engines": { - "node": ">=16 || 14 >=14.17" + "node": "20 || >=22" }, "funding": { "url": "https://github.com/sponsors/isaacs" @@ -704,6 +750,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", "engines": { "node": ">=8" } @@ -711,17 +758,19 @@ "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" }, "node_modules/jackspeak": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", - "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.0.1.tgz", + "integrity": "sha512-cub8rahkh0Q/bw1+GxP7aeSe29hHHn2V4m29nnDlvCdlgU+3UGxkZp7Z53jLUdpX3jdTO0nJZUDl3xvbWc2Xog==", + "license": "BlueOak-1.0.0", "dependencies": { "@isaacs/cliui": "^8.0.2" }, "engines": { - "node": ">=14" + "node": "20 || >=22" }, "funding": { "url": "https://github.com/sponsors/isaacs" @@ -731,68 +780,80 @@ } }, "node_modules/lru-cache": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.1.0.tgz", - "integrity": "sha512-/1clY/ui8CzjKFyjdvwPWJUYKiFVXG2I2cY0ssG7h4+hwk+XOIX7ZSG9Q7TW8TW3Kp3BUSqgFWBLgL4PJ+Blag==", + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.0.0.tgz", + "integrity": "sha512-Qv32eSV1RSCfhY3fpPE2GNZ8jgM9X7rdAfemLWqTUxwiyIC4jJ6Sy0fZ8H+oLWevO6i4/bizg7c8d8i6bxrzbA==", + "license": "ISC", "engines": { - "node": "14 || >=16.14" + "node": "20 || >=22" } }, "node_modules/minimatch": { - "version": "9.0.3", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz", - "integrity": "sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==", + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.0.1.tgz", + "integrity": "sha512-ethXTt3SGGR+95gudmqJ1eNhRO7eGEGIgYA9vnPatK4/etz2MEVDno5GMCibdMTuBMyElzIlgxMna3K94XDIDQ==", + "license": "ISC", "dependencies": { "brace-expansion": "^2.0.1" }, "engines": { - "node": ">=16 || 14 >=14.17" + "node": "20 || >=22" }, "funding": { "url": "https://github.com/sponsors/isaacs" } }, "node_modules/minipass": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz", - "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==", + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "license": "ISC", "engines": { "node": ">=16 || 14 >=14.17" } }, + "node_modules/package-json-from-dist": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.0.tgz", + "integrity": "sha512-dATvCeZN/8wQsGywez1mzHtTlP22H8OEfPrVMLNr4/eGa+ijtLn/6M5f0dY8UKNrC2O9UCU6SSoG3qRKnt7STw==", + "license": "BlueOak-1.0.0" + }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/path-scurry": { - "version": "1.10.1", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.1.tgz", - "integrity": "sha512-MkhCqzzBEpPvxxQ71Md0b1Kk51W01lrYvlMzSUaIzNsODdd7mqhiimSZlr+VegAz5Z6Vzt9Xg2ttE//XBhH3EQ==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.0.tgz", + "integrity": "sha512-ypGJsmGtdXUOeM5u93TyeIEfEhM6s+ljAhrk5vAvSx8uyY/02OvrZnA0YNGUrPXfpJMgI1ODd3nwz8Npx4O4cg==", + "license": "BlueOak-1.0.0", "dependencies": { - "lru-cache": "^9.1.1 || ^10.0.0", - "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" }, "engines": { - "node": ">=16 || 14 >=14.17" + "node": "20 || >=22" }, "funding": { "url": "https://github.com/sponsors/isaacs" } }, "node_modules/prettier": { - "version": "2.8.3", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.8.3.tgz", - "integrity": "sha512-tJ/oJ4amDihPoufT5sM0Z1SKEuKay8LfVAMlbbhnnkvt6BUserZylqo2PN+p9KeljLr0OHa2rXHU1T8reeoTrw==", + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.3.3.tgz", + "integrity": "sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew==", "dev": true, + "license": "MIT", "bin": { - "prettier": "bin-prettier.js" + "prettier": "bin/prettier.cjs" }, "engines": { - "node": ">=10.13.0" + "node": ">=14" }, "funding": { "url": "https://github.com/prettier/prettier?sponsor=1" @@ -802,6 +863,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", "dependencies": { "shebang-regex": "^3.0.0" }, @@ -813,6 +875,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", "engines": { "node": ">=8" } @@ -821,6 +884,7 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", "engines": { "node": ">=14" }, @@ -832,6 +896,7 @@ "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "license": "BSD-3-Clause", "engines": { "node": ">=0.10.0" } @@ -840,6 +905,7 @@ "version": "0.5.21", "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "license": "MIT", "dependencies": { "buffer-from": "^1.0.0", "source-map": "^0.6.0" @@ -849,6 +915,7 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "license": "MIT", "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", @@ -866,6 +933,7 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -879,6 +947,7 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", "engines": { "node": ">=8" } @@ -886,12 +955,14 @@ "node_modules/string-width-cjs/node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" }, "node_modules/string-width-cjs/node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", "dependencies": { "ansi-regex": "^5.0.1" }, @@ -903,6 +974,7 @@ "version": "7.1.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "license": "MIT", "dependencies": { "ansi-regex": "^6.0.1" }, @@ -918,6 +990,7 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", "dependencies": { "ansi-regex": "^5.0.1" }, @@ -929,27 +1002,37 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/typescript": { - "version": "4.9.5", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz", - "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==", + "version": "5.5.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.4.tgz", + "integrity": "sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==", "dev": true, + "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" }, "engines": { - "node": ">=4.2.0" + "node": ">=14.17" } }, + "node_modules/undici-types": { + "version": "6.19.8", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", + "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", + "dev": true, + "license": "MIT" + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", "dependencies": { "isexe": "^2.0.0" }, @@ -964,6 +1047,7 @@ "version": "8.1.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "license": "MIT", "dependencies": { "ansi-styles": "^6.1.0", "string-width": "^5.0.1", @@ -981,6 +1065,7 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", @@ -997,6 +1082,7 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", "engines": { "node": ">=8" } @@ -1005,6 +1091,7 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", "dependencies": { "color-convert": "^2.0.1" }, @@ -1018,12 +1105,14 @@ "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" }, "node_modules/wrap-ansi-cjs/node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -1037,6 +1126,7 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", "dependencies": { "ansi-regex": "^5.0.1" }, diff --git a/deploy/package.json b/deploy/package.json index e73a4ed62..f8def646f 100644 --- a/deploy/package.json +++ b/deploy/package.json @@ -10,17 +10,17 @@ "cdk": "cdk" }, "devDependencies": { - "@types/node": "^18.11.18", - "aws-cdk": "^2.148.1", - "prettier": "2.8.3", - "typescript": "^4.9.4" + "@types/node": "^22.5.4", + "aws-cdk": "^2.155.0", + "prettier": "^3.3.3", + "typescript": "^5.5.4" }, "dependencies": { "@iarna/toml": "^3.0.0", - "aws-cdk-lib": "^2.144.0", + "aws-cdk-lib": "^2.155.0", "cargo-lambda-cdk": "^0.0.22", - "constructs": "^10.1.148", - "glob": "^10.3.10", + "constructs": "^10.3.0", + "glob": "^11.0.0", "source-map-support": "^0.5.21" } } From c45b0920f3832557ea0718a6b298c00eedacda46 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 6 Sep 2024 18:23:33 +1000 Subject: [PATCH 5/5] deploy: make deployment example work out of the box --- deploy/README.md | 30 ++++++++---- deploy/bin/settings.ts | 17 +++---- deploy/config/example_deploy.toml | 20 ++++++++ deploy/examples/local_storage/README.md | 1 - deploy/examples/minio/README.md | 13 +++--- deploy/lib/htsget-lambda-stack.ts | 62 ++++++++++++++++++------- 6 files changed, 102 insertions(+), 41 deletions(-) create mode 100644 deploy/config/example_deploy.toml diff --git a/deploy/README.md b/deploy/README.md index 44cee9599..9fa71a32a 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -16,15 +16,17 @@ The CDK code in this directory constructs a CDK app from [`HtsgetLambdaStack`][h These are general settings for the CDK deployment. -| Name | Description | Type | -| ------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------- | -| `config` | The location of the htsget-rs server config. This must be specified. This config file configures the htsget-rs server. See [htsget-config] for a list of available server configuration options. | `string` | -| `domain` | The domain name for the Route53 Hosted Zone that the htsget-rs server will be under. This must be specified. A hosted zone with this name will either be looked up or created depending on the value of [`lookupHostedZone?`](#lookupHostedZone). | `string` | -| `authorizer` | Deployment options related to the authorizer. Note that this option allows specifying an AWS [JWT authorizer][jwt-authorizer]. The JWT authorizer automatically verifies tokens issued by a Cognito user pool. | [`HtsgetJwtAuthSettings`](#htsgetjwtauthsettings) | -| `subDomain?` | The domain name prefix to use for the htsget-rs server. Together with the [`domain`](#domain), this specifies url that the htsget-rs server will be reachable under. Defaults to `"htsget"`. | `string` | -| `s3BucketResources` | The buckets to serve data from. If this is not specified, this defaults to `[]`. This affects which buckets are allowed to be accessed by the policy actions which are `["s3:List*", "s3:Get*"]`. Note that this option alone does not create buckets, it only gives permission to access them, see the `createS3Buckets` option. This option must be specified to allow `htsget-rs` to access data in the buckets. | `string[]` | -| `lookupHostedZone?` | Whether to lookup the hosted zone with the domain name. Defaults to `true`. If `true`, attempts to lookup an existing hosted zone using the domain name. Set this to `false` if you want to create a new hosted zone with the domain name. | `boolean` | -| `createS3Buckets?` | A list of buckets to create. Defaults to no buckets. Buckets are created with [`RemovalPolicy.RETAIN`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.RemovalPolicy.html). This also copies the example data under the `data` directory to those buckets. | `string[]` | +| Name | Description | Type | +|--------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------| +| `config` | The location of the htsget-rs server config. This must be specified. This config file configures the htsget-rs server. See [htsget-config] for a list of available server configuration options. | `string` | +| `domain` | The domain name for the Route53 Hosted Zone that the htsget-rs server will be under. This must be specified. A hosted zone with this name will either be looked up or created depending on the value of [`lookupHostedZone?`](#lookupHostedZone). | `string` | +| `authorizer` | Deployment options related to the authorizer. Note that this option allows specifying an AWS [JWT authorizer][jwt-authorizer]. The JWT authorizer automatically verifies tokens issued by a Cognito user pool. | [`HtsgetJwtAuthSettings`](#htsgetjwtauthsettings) | +| `subDomain?` | The domain name prefix to use for the htsget-rs server. Together with the [`domain`](#domain), this specifies url that the htsget-rs server will be reachable under. Defaults to `"htsget"`. | `string` | +| `s3BucketResources` | The buckets to serve data from. If this is not specified, this defaults to `[]`. This affects which buckets are allowed to be accessed by the policy actions which are `["s3:List*", "s3:Get*"]`. Note that this option does not create buckets, it only gives permission to access them, see the `createS3Buckets` option. This option must be specified to allow `htsget-rs` to access data in buckets that are not created in this stack. | `string[]` | +| `lookupHostedZone?` | Whether to lookup the hosted zone with the domain name. Defaults to `true`. If `true`, attempts to lookup an existing hosted zone using the domain name. Set this to `false` if you want to create a new hosted zone with the domain name. | `boolean` | +| `createS3Bucket?` | Whether to create a test bucket. Defaults to true. Buckets are created with [`RemovalPolicy.RETAIN`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.RemovalPolicy.html). The correct access permissions are automatically added. | `boolean` | +| `bucketName?` | The name of the bucket created using `createS3Bucket`. The name defaults to an automatically generated CDK name, use this option to override that. This option only has an affect is `createS3Buckets` is true. | `string` | +| `copyTestData?` | Whether to copy test data into the bucket. Defaults to true. This copies the example data under the `data` directory to those buckets. This option only has an affect is `createS3Buckets` is true. | `boolean` | #### HtsgetJwtAuthSettings @@ -70,6 +72,11 @@ npm install ### Deploy to AWS +> [!IMPORTANT] +> The default deployment is designed to work out of the box. A bucket with a CDK-generated name is created with test +> data from the [`data`][data] directory. All deployment settings can be tweaked using the [`settings.ts`][htsget-settings]. +> The only option that must be specified in the `domain`, which determines the domain name to serve htsget-rs at. + CDK should be bootstrapped once, if this hasn't been done before: ```sh @@ -82,6 +89,10 @@ Then to deploy the stack, run: npx cdk deploy ``` +> [!WARNING] +> By default this deployment will create a public instance of htsget-rs. Anyone will be able to query the server +> without authorizing unless you modify the `HtsgetJwtAuthSettings` settings. + ### Testing the endpoint When the deployment is finished, the htsget endpoint can be tested by querying it. If a JWT authorizer is configured, @@ -176,3 +187,4 @@ and a [MinIO][minio] deployment. [rust]: https://www.rust-lang.org/tools/install [zig]: https://ziglang.org/ [zig-getting-started]: https://ziglang.org/learn/getting-started/ +[data]: ../data diff --git a/deploy/bin/settings.ts b/deploy/bin/settings.ts index 09a117a2a..e28d44874 100644 --- a/deploy/bin/settings.ts +++ b/deploy/bin/settings.ts @@ -4,17 +4,18 @@ import { HtsgetSettings } from "../lib/htsget-lambda-stack"; * Settings to use for the htsget deployment. */ export const SETTINGS: HtsgetSettings = { - config: "config/dev_umccr.toml", + config: "config/example_deploy.toml", + // Specify the domain to serve htsget-rs under. domain: "dev.umccr.org", subDomain: "htsget", - s3BucketResources: [ - "arn:aws:s3:::org.umccr.demo.sbeacon-data/*", - "arn:aws:s3:::org.umccr.demo.htsget-rs-data/*", - ], - lookupHostedZone: true, - createS3Buckets: [], + s3BucketResources: [], + lookupHostedZone: false, + createS3Bucket: true, + copyTestData: true, + // Override the bucket name. + // bucketName: "bucket", jwtAuthorizer: { - // Set this to true if you want a public instance. + // Set this to false if you want a private instance. public: false, // jwtAudience: ["audience"], // cogUserPoolId: "user-pool-id", diff --git a/deploy/config/example_deploy.toml b/deploy/config/example_deploy.toml new file mode 100644 index 000000000..6a487b47c --- /dev/null +++ b/deploy/config/example_deploy.toml @@ -0,0 +1,20 @@ +ticket_server_cors_allow_headers = "All" +ticket_server_cors_allow_origins = [] +ticket_server_cors_allow_methods = "All" +ticket_server_cors_allow_credentials = true +ticket_server_cors_max_age = 300 + +data_server_enabled = false + +name = "umccr-htsget-rs" +version = "0.1" +organization_name = "UMCCR" +organization_url = "https://umccr.org/" +contact_url = "https://umccr.org/" +documentation_url = "https://github.com/umccr/htsget-rs" +environment = "dev" + +[[resolvers]] +regex = '^(?P.*?)/(?P.*)$' +substitution_string = '$key' +storage = 'S3' \ No newline at end of file diff --git a/deploy/examples/local_storage/README.md b/deploy/examples/local_storage/README.md index 77f7af370..25ce7d554 100644 --- a/deploy/examples/local_storage/README.md +++ b/deploy/examples/local_storage/README.md @@ -17,7 +17,6 @@ curl http://127.0.0.1:8080/reads/data/bam/htsnexus_test_NA12878 ``` Which outputs: - ```sh { "htsget": { diff --git a/deploy/examples/minio/README.md b/deploy/examples/minio/README.md index 8aaba0ba1..97df6f08c 100644 --- a/deploy/examples/minio/README.md +++ b/deploy/examples/minio/README.md @@ -3,12 +3,13 @@ [MinIO][minio] can be used with htsget-rs by configuring the [storage type][storage] as `S3` and setting the `endpoint` to the MinIO server. There are a few specific configuration options that need to be considered to use MinIO with htsget-rs, and those include: -- The standard [AWS environment variables][env-variables] for connecting to AWS services must be set, and configured to match those - used by MinIO. - _ This means that htsget-rs expects an `AWS_DEFAULT_REGION` to be set, which must match the region used by MinIO (by default us-east-1). - _ It also means that the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` must be set to match the credentials used by MinIO. -- If using virtual-hosted style [addressing][virtual-addressing] instead of path style [addressing][path-addressing], `MINIO_DOMAIN` must be - set on the MinIO server and DNS resolution must allow accessing the MinIO server using `bucket.`. \* Path style addressing can be used instead by setting `path_style = true` under the htsget-rs resolvers storage type. +* The standard [AWS environment variables][env-variables] for connecting to AWS services must be set, and configured to match those +used by MinIO. + * This means that htsget-rs expects an `AWS_DEFAULT_REGION` to be set, which must match the region used by MinIO (by default us-east-1). + * It also means that the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` must be set to match the credentials used by MinIO. +* If using virtual-hosted style [addressing][virtual-addressing] instead of path style [addressing][path-addressing], `MINIO_DOMAIN` must be +set on the MinIO server and DNS resolution must allow accessing the MinIO server using `bucket.`. + * Path style addressing can be used instead by setting `path_style = true` under the htsget-rs resolvers storage type. The caveats around the addressing style occur because there are two different addressing styles for S3 buckets, path style, e.g. `http://minio:9000/bucket`, and virtual-hosted style, e.g. `http://bucket.minio:9000`. AWS has declared path style addressing diff --git a/deploy/lib/htsget-lambda-stack.ts b/deploy/lib/htsget-lambda-stack.ts index 53c75ba5d..fdd5b39b2 100644 --- a/deploy/lib/htsget-lambda-stack.ts +++ b/deploy/lib/htsget-lambda-stack.ts @@ -2,7 +2,14 @@ import { STACK_NAME } from "../bin/htsget-lambda"; import * as TOML from "@iarna/toml"; import { readFileSync } from "fs"; -import { Duration, RemovalPolicy, Stack, StackProps, Tags } from "aws-cdk-lib"; +import { + CfnOutput, + Duration, + RemovalPolicy, + Stack, + StackProps, + Tags, +} from "aws-cdk-lib"; import { Construct } from "constructs"; import { UserPool } from "aws-cdk-lib/aws-cognito"; @@ -56,10 +63,11 @@ export type HtsgetSettings = { subDomain?: string; /** - * The buckets to serve data from. If this is not specified, this defaults to `[]`. This affects which buckets are - * allowed to be accessed by the policy actions which are `["s3:List*", "s3:Get*"]`. Note that this option alone - * does not create buckets, it only gives permission to access them, see the `createS3Buckets` option. - * This option must be specified to allow `htsget-rs` to access data in the buckets. + * The buckets to serve data from. If this is not specified, this defaults to `[]`. + * This affects which buckets are allowed to be accessed by the policy actions which are `["s3:List*", "s3:Get*"]`. + * Note that this option does not create buckets, it only gives permission to access them, see the `createS3Buckets` + * option. This option must be specified to allow `htsget-rs` to access data in buckets that are not created in + * this stack. */ s3BucketResources: string[]; @@ -76,11 +84,23 @@ export type HtsgetSettings = { lookupHostedZone?: boolean; /** - * A list of buckets to create. Defaults to no buckets. Buckets are created with + * Whether to create a test bucket. Defaults to true. Buckets are created with * [`RemovalPolicy.RETAIN`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.RemovalPolicy.html). - * This also copies the example data under the `data` directory to those buckets. + * The correct access permissions are automatically added. + */ + createS3Bucket?: boolean; + + /** + * The name of the bucket created using `createS3Bucket`. The name defaults to an automatically generated CDK name, + * use this option to override that. This option only has an affect is `createS3Buckets` is true. + */ + bucketName?: string; + + /** + * Whether to copy test data into the bucket. Defaults to true. This copies the example data under the `data` + * directory to those buckets. This option only has an affect is `createS3Buckets` is true. */ - createS3Buckets?: string[]; + copyTestData?: boolean; }; /** @@ -169,22 +189,26 @@ export class HtsgetLambdaStack extends Stack { resources: settings.s3BucketResources ?? [], }); - if (settings.createS3Buckets) { - for (const name of settings.createS3Buckets ?? []) { - const bucket = new Bucket(this, "Bucket", { - blockPublicAccess: BlockPublicAccess.BLOCK_ALL, - encryption: BucketEncryption.S3_MANAGED, - enforceSSL: true, - removalPolicy: RemovalPolicy.RETAIN, - bucketName: name, - }); + if (settings.createS3Bucket) { + const bucket = new Bucket(this, "Bucket", { + blockPublicAccess: BlockPublicAccess.BLOCK_ALL, + encryption: BucketEncryption.S3_MANAGED, + enforceSSL: true, + removalPolicy: RemovalPolicy.RETAIN, + bucketName: settings.bucketName, + }); + if (settings.copyTestData) { const dataDir = path.join(__dirname, "..", "..", "data"); new BucketDeployment(this, "DeployFiles", { sources: [Source.asset(dataDir)], destinationBucket: bucket, }); } + + s3BucketPolicy.addResources(`arn:aws:s3:::${bucket.bucketName}/*`); + + new CfnOutput(this, "HtsgetBucketName", { value: bucket.bucketName }); } lambdaRole.addManagedPolicy( @@ -240,6 +264,10 @@ export class HtsgetLambdaStack extends Stack { jwtAudience: settings.jwtAuthorizer.jwtAudience ?? [], }, ); + } else { + console.warn( + "This will create an instance of htsget-rs that is public! Anyone will be able to query the server without authorization.", + ); } let hostedZone;