From 8e05d5fe401e0dd2fedbbf80a5998c724c87408b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Greinhofer?= Date: Sat, 9 Dec 2023 15:07:42 -0600 Subject: [PATCH] Add enpoint to list BNA result bucket MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a new lambda function to list the content of the BNA S3 bucket. Signed-off-by: Rémy Greinhofer --- .github/workflows/deployment-staging.yml | 1 + lambdas/Cargo.toml | 6 ++ lambdas/examples/list-bna-s3-folder.rs | 35 ++++++++++ lambdas/src/bna-results/get-bna-results.rs | 60 +++++++++++++++++ lambdas/src/fixtures/get-bna_results.json | 78 ++++++++++++++++++++++ 5 files changed, 180 insertions(+) create mode 100644 lambdas/examples/list-bna-s3-folder.rs create mode 100644 lambdas/src/bna-results/get-bna-results.rs create mode 100644 lambdas/src/fixtures/get-bna_results.json diff --git a/.github/workflows/deployment-staging.yml b/.github/workflows/deployment-staging.yml index 53c2e8d..ffff570 100644 --- a/.github/workflows/deployment-staging.yml +++ b/.github/workflows/deployment-staging.yml @@ -37,6 +37,7 @@ jobs: run: | LAMBDAS="get-bnas get-bnas-cities + get-bna-results get-cities get-cities-bnas post-enqueue-city diff --git a/lambdas/Cargo.toml b/lambdas/Cargo.toml index 02f2d25..36b2b1d 100644 --- a/lambdas/Cargo.toml +++ b/lambdas/Cargo.toml @@ -5,6 +5,7 @@ edition = "2021" [dependencies] aws-config = "1.0.0" +aws-sdk-s3 = "1.5.0" aws-sdk-sqs = "1.3.0" aws_lambda_events = "0.12.0" bnacore = { git = "https://github.com/PeopleForBikes/brokenspoke.git", rev = "98f20d7" } @@ -51,6 +52,10 @@ path = "src/cities/get-cities.rs" name = "get-cities-bnas" path = "src/cities/get-cities-bnas.rs" +[[bin]] +name = "get-bna-results" +path = "src/bna-results/get-bna-results.rs" + [[bin]] name = "post-submissions-city" path = "src/submissions/post-submissions-city.rs" @@ -60,6 +65,7 @@ name = "post-enqueue-city" path = "src/enqueue/post-city.rs" [dev-dependencies] +color-eyre = "0.6.2" rstest = "0.18.1" [package.metadata.lambda.deploy] diff --git a/lambdas/examples/list-bna-s3-folder.rs b/lambdas/examples/list-bna-s3-folder.rs new file mode 100644 index 0000000..cad918f --- /dev/null +++ b/lambdas/examples/list-bna-s3-folder.rs @@ -0,0 +1,35 @@ +use aws_config::BehaviorVersion; +use aws_sdk_s3::Client; +use color_eyre::{eyre::Report, Result}; +use dotenv::dotenv; + +#[tokio::main] +async fn main() -> Result<(), Report> { + dotenv().ok(); + + let aws_config = aws_config::load_defaults(BehaviorVersion::latest()).await; + let s3_client = aws_sdk_s3::Client::new(&aws_config); + + list_objects(&s3_client, "brokenspoke-analyzer").await?; + + Ok(()) +} + +pub async fn list_objects(client: &Client, bucket_name: &str) -> Result> { + let objects = client.list_objects_v2().bucket(bucket_name).send().await?; + println!("Objects in bucket:"); + // for obj in objects.contents() { + // if obj.key.clone().unwrap().ends_with("/") { + // println!("{:?}", obj.key().unwrap()); + // } + // } + let mut objs = objects + .contents() + .iter() + .filter(|o| o.key.clone().unwrap().ends_with("/")) + .map(|o| o.key.clone().unwrap()) + .collect::>(); + objs.sort(); + + Ok(objs) +} diff --git a/lambdas/src/bna-results/get-bna-results.rs b/lambdas/src/bna-results/get-bna-results.rs new file mode 100644 index 0000000..11267bf --- /dev/null +++ b/lambdas/src/bna-results/get-bna-results.rs @@ -0,0 +1,60 @@ +use aws_config::BehaviorVersion; +use dotenv::dotenv; +use lambda_http::{run, service_fn, Body, Error, IntoResponse, Request, Response}; +use lambdas::{get_apigw_request_id, APIError, APIErrors}; +use serde::{Deserialize, Serialize}; +use serde_json::json; + +const BUCKET_NAME: &str = "brokenspoke-analyzer"; + +#[derive(Deserialize, Serialize, Debug)] +pub struct BNAResults { + results: Vec, +} + +async fn function_handler(event: Request) -> Result, Error> { + dotenv().ok(); + + // Get the API Gateway request ID. + let apigw_request_id = get_apigw_request_id(&event); + + // Configure the AWS S3 client. + let aws_config = aws_config::load_defaults(BehaviorVersion::latest()).await; + let s3_client = aws_sdk_s3::Client::new(&aws_config); + + // Collect the folders in the bucket. + let objects = s3_client.list_objects_v2().bucket(BUCKET_NAME).send().await; + match objects { + Ok(o) => { + let mut bna_results = o + .contents() + .iter() + .filter(|o| o.key.clone().unwrap().ends_with("/")) + .map(|o| o.key.clone().unwrap()) + .collect::>(); + bna_results.sort(); + Ok(json!(bna_results).into_response().await) + } + Err(e) => { + let api_error = APIError::no_content( + apigw_request_id, + event.uri().path(), + format!("Cannot retrieve the content of the {BUCKET_NAME} bucket: {e}").as_str(), + ); + Ok(APIErrors::new(&[api_error]).into()) + } + } +} + +#[tokio::main] +async fn main() -> Result<(), Error> { + tracing_subscriber::fmt() + .with_max_level(tracing::Level::INFO) + // disable printing the name of the module in every log line. + .with_target(false) + // disabling time is handy because CloudWatch will add the ingestion time. + .without_time() + .init(); + + run(service_fn(function_handler)).await +} diff --git a/lambdas/src/fixtures/get-bna_results.json b/lambdas/src/fixtures/get-bna_results.json new file mode 100644 index 0000000..4246a30 --- /dev/null +++ b/lambdas/src/fixtures/get-bna_results.json @@ -0,0 +1,78 @@ +{ + "resource": "/bna-results", + "path": "/bna-results", + "httpMethod": "GET", + "headers": { + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", + "accept-encoding": "gzip, deflate, br", + "accept-language": "en-US,en;q=0.9", + "cookie": "s_fid=7AAB6XMPLAFD9BBF-0643XMPL09956DE2; regStatus=pre-register", + "Host": "70ixmpl4fl.execute-api.us-east-2.amazonaws.com", + "sec-fetch-dest": "document", + "sec-fetch-mode": "navigate", + "sec-fetch-site": "none", + "upgrade-insecure-requests": "1", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "X-Amzn-Trace-Id": "Root=1-5e66d96f-7491f09xmpl79d18acf3d050", + "X-Forwarded-For": "52.255.255.12", + "X-Forwarded-Port": "443", + "X-Forwarded-Proto": "https" + }, + "multiValueHeaders": { + "accept": [ + "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" + ], + "accept-encoding": ["gzip, deflate, br"], + "accept-language": ["en-US,en;q=0.9"], + "cookie": [ + "s_fid=7AABXMPL1AFD9BBF-0643XMPL09956DE2; regStatus=pre-register;" + ], + "Host": ["70ixmpl4fl.execute-api.ca-central-1.amazonaws.com"], + "sec-fetch-dest": ["document"], + "sec-fetch-mode": ["navigate"], + "sec-fetch-site": ["none"], + "upgrade-insecure-requests": ["1"], + "User-Agent": [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36" + ], + "X-Amzn-Trace-Id": ["Root=1-5e66d96f-7491f09xmpl79d18acf3d050"], + "X-Forwarded-For": ["52.255.255.12"], + "X-Forwarded-Port": ["443"], + "X-Forwarded-Proto": ["https"] + }, + "queryStringParameters": null, + "multiValueQueryStringParameters": null, + "stageVariables": null, + "requestContext": { + "resourceId": "2gxmpl", + "resourcePath": "/", + "httpMethod": "GET", + "extendedRequestId": "JJbxmplHYosFVYQ=", + "requestTime": "10/Mar/2020:00:03:59 +0000", + "path": "/Prod/", + "accountId": "123456789012", + "protocol": "HTTP/1.1", + "stage": "Prod", + "domainPrefix": "70ixmpl4fl", + "requestTimeEpoch": 1583798639428, + "requestId": "77375676-xmpl-4b79-853a-f982474efe18", + "identity": { + "cognitoIdentityPoolId": null, + "accountId": null, + "cognitoIdentityId": null, + "caller": null, + "sourceIp": "52.255.255.12", + "principalOrgId": null, + "accessKey": null, + "cognitoAuthenticationType": null, + "cognitoAuthenticationProvider": null, + "userArn": null, + "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "user": null + }, + "domainName": "70ixmpl4fl.execute-api.us-east-2.amazonaws.com", + "apiId": "70ixmpl4fl" + }, + "body": null, + "isBase64Encoded": false +}