From 9f2017c43bdda362fa4bbe847ba03920fce993dc Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Mon, 23 Oct 2023 15:08:01 +1100 Subject: [PATCH 1/3] Merge --- lib/workload/stateless/filemanager/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/workload/stateless/filemanager/README.md b/lib/workload/stateless/filemanager/README.md index b6fb0a4cf..188e86019 100644 --- a/lib/workload/stateless/filemanager/README.md +++ b/lib/workload/stateless/filemanager/README.md @@ -57,9 +57,7 @@ Then deploy the cdk to localstack: ```sh cd deploy - npm install - npx cdklocal bootstrap npx cdklocal deploy ``` From e1d580bb8ac1fa97c8bbcc9d1589b06e050298f5 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Mon, 23 Oct 2023 15:12:48 +1100 Subject: [PATCH 2/3] Update requirements as discussed in the get-together /cc @victorskl @mmalenic @reisingerf @andrewpatto #reviewthis --- skel/rust-api/src/main.rs | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/skel/rust-api/src/main.rs b/skel/rust-api/src/main.rs index 2255a38c2..0b9484d6d 100644 --- a/skel/rust-api/src/main.rs +++ b/skel/rust-api/src/main.rs @@ -13,19 +13,16 @@ use utoipa_swagger_ui::SwaggerUi; use tower_http::trace::{self, TraceLayer}; use tracing::{info, Level}; -/// FileManager keeps track of files from many storage backend. The FileManager is responsible for: +/// FileManager keeps track of files from many storage backend. All files have an external, public, +/// file UUID so that they can be uniquely identified in our whole microservices environment. /// -/// 1. Listening and ingesting file creation events from the different storage backends. -/// 2. Querying on file attributes such as name, type of file, creation date, etc... -/// 3. Interfacing with htsget-rs for the biological-specific file formats: CRAM, BAM, VCF, BCF... -/// 4. Interacting with the metadata microservice to enrich the results of any particular query, returning -/// its associated metadata. A file should have a metadata id. -/// 5. Making sure the workflow run id is present and associated with any given file. -/// 6. Audit: Keep records for deleted objects even after the actual data is deleted. -/// 7. Audit: Record file ownership (on create, no chain of custody functionality needed). +/// The FileManager is responsible for: /// -/// All files have an external, public, file UUID so that they can be uniquely identified in our whole -/// microservices environment. +/// 1. Listening and ingesting (indexing) file creation events from the different storage backends. +/// 2. Querying on file attributes such as name, type of file, creation date, etc... +/// 3. Record file provenance (on create, on delete). +/// 4. Record lifecycle status. +/// 5. Checksumming of files is not a requirement at this stage. #[tokio::main] async fn main() -> Result<(), Error> { From eb92e207c76cb81932b57eafd8cff9fa931bc16e Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Mon, 23 Oct 2023 15:25:42 +1100 Subject: [PATCH 3/3] Refined requirements based on those laid out by @reisingerf on 'ICAv2 migration / OrcaBus focus meeting' document --- skel/rust-api/src/main.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/skel/rust-api/src/main.rs b/skel/rust-api/src/main.rs index 0b9484d6d..5d4711a80 100644 --- a/skel/rust-api/src/main.rs +++ b/skel/rust-api/src/main.rs @@ -21,8 +21,13 @@ use tracing::{info, Level}; /// 1. Listening and ingesting (indexing) file creation events from the different storage backends. /// 2. Querying on file attributes such as name, type of file, creation date, etc... /// 3. Record file provenance (on create, on delete). -/// 4. Record lifecycle status. -/// 5. Checksumming of files is not a requirement at this stage. +/// 4. Record object lifecycle status, i.e: Tier transition, backup to different storage, deletion. +/// 5. Manage object identity, i.e: UUID, path, etc... +/// +/// Non goals: +/// +/// 1. Perform checksumming of files: Too computationally expensive for our file sizes. +/// 2. Link objects/paths to metadata: That would tightly couple filemanager with metadata service. #[tokio::main] async fn main() -> Result<(), Error> {