From 66ac215d6957337d03949841c40350b008fb76de Mon Sep 17 00:00:00 2001 From: iamvigneshwars Date: Tue, 26 Mar 2024 15:49:07 +0000 Subject: [PATCH 1/2] Create download link for processed images from s3 bucket' --- Cargo.lock | 480 ++++++++++++++++++ README.md | 4 +- graph-data-processing/Cargo.toml | 3 + graph-data-processing/src/graphql/entities.rs | 13 +- graph-data-processing/src/graphql/mod.rs | 23 +- graph-data-processing/src/main.rs | 66 ++- 6 files changed, 582 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 754969a..a7fdddf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -288,6 +288,306 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "aws-credential-types" +version = "0.56.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70a66ac8ef5fa9cf01c2d999f39d16812e90ec1467bd382cbbb74ba23ea86201" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "fastrand", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-http" +version = "0.56.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e626370f9ba806ae4c439e49675fd871f5767b093075cdf4fef16cac42ba900" +dependencies = [ + "aws-credential-types", + "aws-smithy-http", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "http-body 0.4.6", + "lazy_static", + "percent-encoding", + "pin-project-lite", + "tracing", +] + +[[package]] +name = "aws-runtime" +version = "0.56.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07ac5cf0ff19c1bca0cea7932e11b239d1025a45696a4f44f72ea86e2b8bdd07" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "fastrand", + "http 0.2.12", + "percent-encoding", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-s3" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e30370b61599168d38190ad272bb91842cd81870a6ca035c05dd5726d22832c" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-client", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "http 0.2.12", + "http-body 0.4.6", + "once_cell", + "percent-encoding", + "regex", + "tokio-stream", + "tracing", + "url", +] + +[[package]] +name = "aws-sigv4" +version = "0.56.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7b28f4910bb956b7ab320b62e98096402354eca976c587d1eeccd523d9bac03" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-http", + "bytes", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "once_cell", + "percent-encoding", + "regex", + "sha2", + "time", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "0.56.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cdb73f85528b9d19c23a496034ac53703955a59323d581c06aa27b4e4e247af" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", + "tokio-stream", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.56.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afb15946af1b8d3beeff53ad991d9bff68ac22426b6d40372b958a75fa61eaed" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc32c", + "crc32fast", + "hex", + "http 0.2.12", + "http-body 0.4.6", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + +[[package]] +name = "aws-smithy-client" +version = "0.56.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c27b2756264c82f830a91cb4d2d485b2d19ad5bea476d9a966e03d27f27ba59a" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-types", + "bytes", + "fastrand", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", + "hyper-rustls", + "lazy_static", + "pin-project-lite", + "rustls", + "tokio", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.56.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "850233feab37b591b7377fd52063aa37af615687f5896807abe7f49bd4e1d25b" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.56.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54cdcf365d8eee60686885f750a34c190e513677db58bbc466c44c588abf4199" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "aws-smithy-http-tower" +version = "0.56.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "822de399d0ce62829a69dfa8c5cd08efdbe61a7426b953e2268f8b8b52a607bd" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http-body 0.4.6", + "pin-project-lite", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.56.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb1e7ab8fa7ad10c193af7ae56d2420989e9f4758bf03601a342573333ea34f" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-runtime" +version = "0.56.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "745e096b3553e7e0f40622aa04971ce52765af82bebdeeac53aa6fc82fe801e6" +dependencies = [ + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand", + "http 0.2.12", + "http-body 0.4.6", + "once_cell", + "pin-project-lite", + "pin-utils", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "0.56.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d0ae0c9cfd57944e9711ea610b48a963fb174a53aabacc08c5794a594b1d02" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-types" +version = "0.56.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d90dbc8da2f6be461fa3c1906b20af8f79d14968fe47f2b7d29d086f62a51728" +dependencies = [ + "base64-simd", + "itoa", + "num-integer", + "ryu", + "serde", + "time", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.56.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01d2dedcdd8023043716cfeeb3c6c59f2d447fce365d8e194838891794b23b6" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "0.56.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85aa0451bf8af1bf22a4f028d5d28054507a14be43cb8ac0597a8471fba9edfe" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-types", + "http 0.2.12", + "rustc_version", + "tracing", +] + [[package]] name = "axum" version = "0.6.20" @@ -412,6 +712,16 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "base64ct" version = "1.6.0" @@ -538,6 +848,16 @@ dependencies = [ "serde", ] +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "cc" version = "1.0.90" @@ -623,6 +943,22 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.6" @@ -653,6 +989,24 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crc32c" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89254598aa9b9fa608de44b3ae54c810f0f06d755e24c50177f1f8f31ff50ce2" +dependencies = [ + "rustc_version", +] + +[[package]] +name = "crc32fast" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-channel" version = "0.5.12" @@ -760,6 +1114,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_more" +version = "0.99.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn 1.0.109", +] + [[package]] name = "digest" version = "0.10.7" @@ -1013,10 +1380,13 @@ dependencies = [ "anyhow", "async-graphql", "async-graphql-axum", + "aws-credential-types", + "aws-sdk-s3", "axum 0.7.4", "built", "chrono", "clap", + "derive_more", "dotenvy", "models", "opentelemetry", @@ -1277,6 +1647,22 @@ dependencies = [ "tokio", ] +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.12", + "hyper 0.14.28", + "log", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", +] + [[package]] name = "hyper-timeout" version = "0.4.1" @@ -1657,6 +2043,12 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + [[package]] name = "opentelemetry" version = "0.22.0" @@ -1773,6 +2165,12 @@ dependencies = [ "syn 2.0.53", ] +[[package]] +name = "outref" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" + [[package]] name = "overload" version = "0.1.1" @@ -2222,6 +2620,15 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "0.38.32" @@ -2241,11 +2648,24 @@ version = "0.21.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ + "log", "ring", "rustls-webpki", "sct", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "1.0.4" @@ -2277,6 +2697,15 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -2440,6 +2869,35 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "security-framework" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" + [[package]] name = "serde" version = "1.0.197" @@ -3058,6 +3516,16 @@ dependencies = [ "syn 2.0.53", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.15" @@ -3403,6 +3871,12 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "want" version = "0.3.1" @@ -3685,6 +4159,12 @@ dependencies = [ "tap", ] +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "zerocopy" version = "0.7.32" diff --git a/README.md b/README.md index 6c53dad..ffee60c 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,3 @@ -# Dataset Service +# DatasetProcessing Service -A component of the `data-gateway` [federation](https://github.com/DiamondLightSource/data-gateway), exposing information from ISPyB about data collected during the Sessions (a.k.a. Visits) via GraphQL. +A component of the `data-gateway` [federation](https://github.com/DiamondLightSource/data-gateway), exposing information from ISPyB about processed data collected during the Sessions (a.k.a. Visits) via GraphQL. diff --git a/graph-data-processing/Cargo.toml b/graph-data-processing/Cargo.toml index 0b80879..8015464 100644 --- a/graph-data-processing/Cargo.toml +++ b/graph-data-processing/Cargo.toml @@ -13,9 +13,12 @@ async-graphql = { version = "7.0.2", default-features = false, features = [ "tracing", ] } async-graphql-axum = { version = "7.0.2" } +aws-credential-types = { version = "0.56.0" } +aws-sdk-s3 = { version = "0.29.0" } axum = { version = "0.7.4", features = ["ws"] } chrono = { version = "0.4.35" } clap = { version = "4.5.2", features = ["derive", "env"] } +derive_more = { version = "0.99.17" } dotenvy = { version = "0.15.7" } models = { path = "../models" } opentelemetry = { version = "0.22.0", features = ["metrics"] } diff --git a/graph-data-processing/src/graphql/entities.rs b/graph-data-processing/src/graphql/entities.rs index d67b7ed..6ac9fd4 100644 --- a/graph-data-processing/src/graphql/entities.rs +++ b/graph-data-processing/src/graphql/entities.rs @@ -2,23 +2,30 @@ use async_graphql::SimpleObject; use models::data_collection_file_attachment; #[derive(Clone, Debug, PartialEq, SimpleObject)] -#[graphql(name = "DataProcessing", unresolvable)] +#[graphql(name = "DataProcessing", unresolvable, complex)] pub struct DataProcessing { /// An opaque unique identifier for the collected file attachment - pub data_collection_file_attachment_id: u32, + pub id: u32, /// Full path where the processed image is stored + #[graphql(skip)] pub file_full_path: String, } impl From for DataProcessing { fn from(values: data_collection_file_attachment::Model) -> Self { Self { - data_collection_file_attachment_id: values.data_collection_file_attachment_id, + id: values.data_collection_file_attachment_id, file_full_path: values.file_full_path, } } } +impl DataProcessing { + pub fn object_key(&self) -> String { + format!("{}", self.file_full_path) + } +} + #[derive(SimpleObject)] #[graphql(name = "Datasets", complex)] pub struct DataCollection { diff --git a/graph-data-processing/src/graphql/mod.rs b/graph-data-processing/src/graphql/mod.rs index c5d1207..ddcb6f8 100644 --- a/graph-data-processing/src/graphql/mod.rs +++ b/graph-data-processing/src/graphql/mod.rs @@ -1,12 +1,15 @@ /// Collection of graphql entities mod entities; - +use crate::S3Bucket; use async_graphql::{ ComplexObject, Context, EmptyMutation, EmptySubscription, Object, Schema, SchemaBuilder, }; +use aws_sdk_s3::presigning::PresigningConfig; use entities::{DataCollection, DataProcessing}; use models::data_collection_file_attachment; use sea_orm::{ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter}; +use std::time::Duration; +use url::Url; /// The GraphQL schema exposed by the service pub type RootSchema = Schema; @@ -38,6 +41,24 @@ impl DataCollection { } } +#[ComplexObject] +impl DataProcessing { + async fn download_url(&self, ctx: &Context<'_>) -> async_graphql::Result { + let s3_client = ctx.data::()?; + let bucket = ctx.data::()?; + let object_uri = s3_client + .get_object() + .bucket(bucket.clone()) + .key(self.object_key()) + .presigned(PresigningConfig::expires_in(Duration::from_secs(10 * 60))?) + .await? + .uri() + .clone(); + let object_url = Url::parse(&object_uri.to_string())?; + Ok(object_url.to_string()) + } +} + #[Object] impl Query { /// Reference datasets resolver for the router diff --git a/graph-data-processing/src/main.rs b/graph-data-processing/src/main.rs index c50eb9b..b8dfb46 100644 --- a/graph-data-processing/src/main.rs +++ b/graph-data-processing/src/main.rs @@ -10,8 +10,11 @@ mod graphql; use async_graphql::{extensions::Tracing, http::GraphiQLSource, SDLExportOptions}; use async_graphql_axum::{GraphQL, GraphQLSubscription}; +use aws_credential_types::{provider::SharedCredentialsProvider, Credentials}; +use aws_sdk_s3::{config::Region, Client}; use axum::{response::Html, routing::get, Router}; -use clap::Parser; +use clap::{ArgAction::SetTrue, Parser}; +use derive_more::{Deref, FromStr, Into}; use graphql::{root_schema_builder, RootSchema}; use opentelemetry_otlp::WithExportConfig; use sea_orm::{ConnectOptions, Database, DatabaseConnection, DbErr, TransactionError}; @@ -46,6 +49,12 @@ struct ServeArgs { /// The URL of the ISPyB instance which should be connected to #[arg(long, env = "DATABASE_URL")] database_url: Url, + /// The S3 bucket which images are to be stored in. + #[arg(long, env)] + s3_bucket: S3Bucket, + /// Configuration argument of the S3 client. + #[command(flatten)] + s3_client: S3ClientArgs, /// The [`tracing::Level`] to log at #[arg(long, env = "LOG_LEVEL", default_value_t = tracing::Level::INFO)] log_level: tracing::Level, @@ -54,6 +63,58 @@ struct ServeArgs { otel_collector_url: Option, } +/// S3 bucket where the processed data is stored +#[derive(Debug, Clone, Deref, FromStr, Into)] +pub struct S3Bucket(String); + +/// Arguments for configuring the S3 Client. +#[derive(Debug, Parser)] +pub struct S3ClientArgs { + /// The URL of the S3 endpoint to retrieve images from. + #[arg(long, env)] + s3_endpoint_url: Option, + /// The ID of the access key used for S3 authorization. + #[arg(long, env)] + s3_access_key_id: Option, + /// The secret access key used for S3 authorization. + #[arg(long, env)] + s3_secret_access_key: Option, + /// Forces path style endpoint URIs for S3 queries. + #[arg(long, env, action = SetTrue)] + s3_force_path_style: bool, + /// The AWS region of the S3 bucket. + #[arg(long, env)] + s3_region: Option, +} + +/// S3 client argument trait +pub trait FromS3ClientArgs { + /// Creates a S3 [`Client`] with the supplied credentials using the supplied endpoint configuration. + fn from_s3_client_args(args: S3ClientArgs) -> Self; +} + +impl FromS3ClientArgs for Client { + fn from_s3_client_args(args: S3ClientArgs) -> Self { + let credentials = Credentials::new( + args.s3_access_key_id.unwrap_or_default(), + args.s3_secret_access_key.unwrap_or_default(), + None, + None, + "Other", + ); + let credentials_provider = SharedCredentialsProvider::new(credentials); + let mut config_builder = aws_sdk_s3::config::Builder::new(); + config_builder.set_credentials_provider(Some(credentials_provider)); + config_builder.set_endpoint_url(args.s3_endpoint_url.map(String::from)); + config_builder.set_force_path_style(Some(args.s3_force_path_style)); + config_builder.set_region(Some(Region::new( + args.s3_region.unwrap_or(String::from("undefined")), + ))); + let config = config_builder.build(); + Client::from_conf(config) + } +} + /// Arguments for produces the GraphQL schema #[derive(Debug, Parser)] struct SchemaArgs { @@ -170,9 +231,12 @@ async fn main() { Cli::Serve(args) => { setup_telemetry(args.log_level, args.otel_collector_url).unwrap(); let database = setup_database(args.database_url).await.unwrap(); + let s3_client = aws_sdk_s3::Client::from_s3_client_args(args.s3_client); let schema = root_schema_builder() .extension(Tracing) .data(database) + .data(s3_client) + .data(args.s3_bucket) .finish(); let router = setup_router(schema); serve(router, args.port).await.unwrap(); From c1487a5864ffcb08a2c5e86db31d8719d78765b1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Mar 2024 16:38:59 +0000 Subject: [PATCH 2/2] Bump the patch group with 4 updates Bumps the patch group with 4 updates: [prettyplease](https://github.com/dtolnay/prettyplease), [axum](https://github.com/tokio-rs/axum), [chrono](https://github.com/chronotope/chrono) and [clap](https://github.com/clap-rs/clap). Updates `prettyplease` from 0.2.16 to 0.2.17 - [Release notes](https://github.com/dtolnay/prettyplease/releases) - [Commits](https://github.com/dtolnay/prettyplease/compare/0.2.16...0.2.17) Updates `axum` from 0.7.4 to 0.7.5 - [Release notes](https://github.com/tokio-rs/axum/releases) - [Changelog](https://github.com/tokio-rs/axum/blob/main/CHANGELOG.md) - [Commits](https://github.com/tokio-rs/axum/compare/axum-v0.7.4...axum-v0.7.5) Updates `chrono` from 0.4.35 to 0.4.36 - [Release notes](https://github.com/chronotope/chrono/releases) - [Changelog](https://github.com/chronotope/chrono/blob/main/CHANGELOG.md) - [Commits](https://github.com/chronotope/chrono/compare/v0.4.35...v0.4.36) Updates `clap` from 4.5.3 to 4.5.4 - [Release notes](https://github.com/clap-rs/clap/releases) - [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md) - [Commits](https://github.com/clap-rs/clap/compare/v4.5.3...v4.5.4) --- updated-dependencies: - dependency-name: prettyplease dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patch - dependency-name: axum dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patch - dependency-name: chrono dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patch - dependency-name: clap dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 60 ++++++++++++-------------------- graph-data-processing/Cargo.toml | 6 ++-- models/Cargo.toml | 2 +- 3 files changed, 27 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a7fdddf..113de26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -189,7 +189,7 @@ checksum = "93605d26b9da33b4cf6541906a9eb9e74396f1accbbc0f066e06f3b0869b84fc" dependencies = [ "async-graphql", "async-trait", - "axum 0.7.4", + "axum 0.7.5", "bytes", "futures-util", "serde_json", @@ -610,7 +610,7 @@ dependencies = [ "pin-project-lite", "rustversion", "serde", - "sync_wrapper", + "sync_wrapper 0.1.2", "tower", "tower-layer", "tower-service", @@ -618,9 +618,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1236b4b292f6c4d6dc34604bb5120d85c3fe1d1aa596bd5cc52ca054d13e7b9e" +checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" dependencies = [ "async-trait", "axum-core 0.4.3", @@ -644,7 +644,7 @@ dependencies = [ "serde_path_to_error", "serde_urlencoded", "sha1", - "sync_wrapper", + "sync_wrapper 1.0.0", "tokio", "tokio-tungstenite", "tower", @@ -685,7 +685,7 @@ dependencies = [ "mime", "pin-project-lite", "rustversion", - "sync_wrapper", + "sync_wrapper 0.1.2", "tower-layer", "tower-service", "tracing", @@ -878,9 +878,9 @@ checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" [[package]] name = "chrono" -version = "0.4.35" +version = "0.4.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eaf5903dcbc0a39312feb77df2ff4c76387d591b9fc7b04a238dcf8bb62639a" +checksum = "3313f40d6fa9e783bbb01e814800bde9618da2207c72e9782f35e8dc7c77dabc" dependencies = [ "android-tzdata", "iana-time-zone", @@ -893,9 +893,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.3" +version = "4.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "949626d00e063efc93b6dca932419ceb5432f99769911c0b995f7e884c778813" +checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" dependencies = [ "clap_builder", "clap_derive", @@ -915,9 +915,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.3" +version = "4.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90239a040c80f5e14809ca132ddc4176ab33d5e17e49691793296e3fcb34d72f" +checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -1382,7 +1382,7 @@ dependencies = [ "async-graphql-axum", "aws-credential-types", "aws-sdk-s3", - "axum 0.7.4", + "axum 0.7.5", "built", "chrono", "clap", @@ -1420,25 +1420,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "h2" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51ee2dd2e4f378392eeff5d51618cd9a63166a2513846bbc55f21cfacd9199d4" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http 1.1.0", - "indexmap 2.2.5", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "handlebars" version = "4.5.0" @@ -1613,7 +1594,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.3.25", + "h2", "http 0.2.12", "http-body 0.4.6", "httparse", @@ -1636,7 +1617,6 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.3", "http 1.1.0", "http-body 1.0.0", "httparse", @@ -2339,9 +2319,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "prettyplease" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" +checksum = "8d3928fb5db768cb86f891ff014f0144589297e3c6a1aba6ed7cecfdace270c7" dependencies = [ "proc-macro2", "syn 2.0.53", @@ -3384,6 +3364,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "sync_wrapper" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384595c11a4e2969895cad5a8c4029115f5ab956a9e5ef4de79d11a426e5f20c" + [[package]] name = "tap" version = "1.0.1" @@ -3603,7 +3589,7 @@ dependencies = [ "axum 0.6.20", "base64", "bytes", - "h2 0.3.25", + "h2", "http 0.2.12", "http-body 0.4.6", "hyper 0.14.28", diff --git a/graph-data-processing/Cargo.toml b/graph-data-processing/Cargo.toml index 8015464..89cae9c 100644 --- a/graph-data-processing/Cargo.toml +++ b/graph-data-processing/Cargo.toml @@ -15,9 +15,9 @@ async-graphql = { version = "7.0.2", default-features = false, features = [ async-graphql-axum = { version = "7.0.2" } aws-credential-types = { version = "0.56.0" } aws-sdk-s3 = { version = "0.29.0" } -axum = { version = "0.7.4", features = ["ws"] } -chrono = { version = "0.4.35" } -clap = { version = "4.5.2", features = ["derive", "env"] } +axum = { version = "0.7.5", features = ["ws"] } +chrono = { version = "0.4.36" } +clap = { version = "4.5.4", features = ["derive", "env"] } derive_more = { version = "0.99.17" } dotenvy = { version = "0.15.7" } models = { path = "../models" } diff --git a/models/Cargo.toml b/models/Cargo.toml index 93fc057..599590e 100644 --- a/models/Cargo.toml +++ b/models/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" path = "src/lib.rs" [dependencies] -prettyplease = "0.2.16" +prettyplease = "0.2.17" sea-orm = { workspace = true } [build-dependencies]