Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: basic filemanager query API #372

Merged
merged 14 commits into from
Jul 1, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 81 additions & 0 deletions lib/workload/stateless/stacks/filemanager/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
@@ -1,15 +1,17 @@
//! This module contains the crate's error types.
//!

use crate::error::ErrorKind::{IoError, LoadingEnvironment};
use crate::workspace_path;
use miette::{diagnostic, Diagnostic, NamedSource, SourceOffset};
use std::fmt::{Display, Formatter};
use std::fs::read_to_string;
use std::panic::Location;
use std::{io, result};

use miette::{diagnostic, Diagnostic, NamedSource, SourceOffset};
use thiserror::Error;

use crate::error::ErrorKind::{IoError, LoadingEnvironment};
use crate::workspace_path;

pub type Result<T> = result::Result<T, Error>;

/// Error types for the filemanager.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,16 @@
//! database schema.
//!

use crate::error::ErrorKind::EntityGeneration;
use crate::error::{Error, Result};
use crate::Config;
use std::ffi::OsStr;
use std::fs::write;

use clap_builder::Parser;
use quote::quote;
use sea_orm_cli::{run_generate_command, Cli, Commands};
use std::ffi::OsStr;
use std::fs::write;

use crate::error::ErrorKind::EntityGeneration;
use crate::error::{Error, Result};
use crate::Config;

pub async fn generate_entities() -> Result<()> {
let config = Config::load()?;
Expand All @@ -19,6 +21,14 @@ pub async fn generate_entities() -> Result<()> {
"sea-orm-cli",
"generate",
"entity",
"--with-serde",
"both",
"--enum-extra-derives",
"strum::FromRepr",
"--enum-extra-derives",
"strum::EnumCount",
"--enum-extra-attributes",
"repr(u8)",
"-u",
&config.database_url,
"-o",
Expand All @@ -44,6 +54,5 @@ pub async fn generate_entities() -> Result<()> {

let out_file = out_dir.join("generated.rs");
write(out_file, format!("{}\n\n{}", generated_comment, generated))?;

Ok(())
}
Original file line number Diff line number Diff line change
@@ -1,21 +1,26 @@
use crate::error::Error;
use crate::error::ErrorKind::LoadingEnvironment;
use envy::from_env;
use error::Result;
use serde::Deserialize;
use std::env::var;
use std::path::{Path, PathBuf};

use envy::from_env;
use serde::Deserialize;

use error::Result;

use crate::error::Error;
use crate::error::ErrorKind::LoadingEnvironment;

pub mod error;
pub mod gen_entities;

/// Configuration environment variables for the build process.
#[derive(Debug, Deserialize)]
pub struct Config {
database_url: String,
out_dir: PathBuf,
}

impl Config {
/// Load environment variables into a `Config` struct.
#[track_caller]
pub fn load() -> Result<Self> {
Ok(from_env::<Config>()?)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
use filemanager::env::Config;
use lambda_http::Error;
use lambda_runtime::{run, service_fn, LambdaEvent};

Expand All @@ -11,15 +12,17 @@ use filemanager::handlers::init_tracing;
async fn main() -> Result<(), Error> {
init_tracing();

let options = &create_database_pool().await?;
let config = &Config::load()?;
let options = &create_database_pool(config).await?;
run(service_fn(|_: LambdaEvent<()>| async move {
update_credentials(options).await?;
update_credentials(options, config).await?;

receive_and_ingest(
S3Client::with_defaults().await,
SQSClient::with_defaults().await,
None::<String>,
DbClient::from_ref(options),
DbClient::new(options.clone()),
config,
)
.await?;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,24 @@ use lambda_runtime::{run, service_fn, Error, LambdaEvent};

use filemanager::clients::aws::s3::Client;
use filemanager::database::Client as DbClient;
use filemanager::env::Config;
use filemanager::handlers::aws::{create_database_pool, ingest_event, update_credentials};
use filemanager::handlers::init_tracing;

#[tokio::main]
async fn main() -> Result<(), Error> {
init_tracing();

let options = &create_database_pool().await?;
let config = &Config::load()?;
let options = &create_database_pool(config).await?;
run(service_fn(|event: LambdaEvent<SqsEvent>| async move {
update_credentials(options).await?;
update_credentials(options, config).await?;

ingest_event(
event.payload,
Client::with_defaults().await,
DbClient::from_ref(options),
DbClient::new(options.clone()),
config,
)
.await?;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ use lambda_runtime::{run, service_fn, Error, LambdaEvent};
use serde::Deserialize;

use filemanager::database::Client as DbClient;
use filemanager::env::Config;
use filemanager::events::aws::inventory::Manifest;
use filemanager::handlers::aws::{create_database_pool, ingest_s3_inventory, update_credentials};
use filemanager::handlers::init_tracing;
Expand All @@ -26,12 +27,13 @@ pub struct BucketKey {
async fn main() -> Result<(), Error> {
init_tracing();

let options = &create_database_pool().await?;
let config = &Config::load()?;
let options = &create_database_pool(config).await?;
run(service_fn(|event: LambdaEvent<Request>| async move {
update_credentials(options).await?;
update_credentials(options, config).await?;

let client = Client::with_defaults().await;
let database = DbClient::from_ref(options);
let database = DbClient::new(options.clone());

match event.payload {
Request::BucketKey(bucket_key) => {
Expand All @@ -41,11 +43,12 @@ async fn main() -> Result<(), Error> {
Some(bucket_key.bucket),
Some(bucket_key.key),
None,
config,
)
.await?
}
Request::Manifest(manifest) => {
ingest_s3_inventory(client, database, None, None, Some(manifest)).await?
ingest_s3_inventory(client, database, None, None, Some(manifest), config).await?
}
};

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ use crate::Event::Provider;
use filemanager::database::aws::migration::Migration;
use filemanager::database::Client as DbClient;
use filemanager::database::Migrate;
use filemanager::env::Config;
use filemanager::handlers::aws::{create_database_pool, update_credentials};
use filemanager::handlers::init_tracing;

Expand All @@ -32,9 +33,10 @@ pub enum CloudFormationRequest {
async fn main() -> Result<(), Error> {
init_tracing();

let options = &create_database_pool().await?;
let config = &Config::load()?;
let options = &create_database_pool(config).await?;
run(service_fn(|event: LambdaEvent<Event>| async move {
update_credentials(options).await?;
update_credentials(options, config).await?;

// Migrate depending on the type of lifecycle event using the CDK provider framework:
// https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.custom_resources-readme.html#provider-framework
Expand All @@ -47,7 +49,7 @@ async fn main() -> Result<(), Error> {
_ => {
// If there's nothing to migrate, then this will just return Ok.
Ok::<_, Error>(
Migration::new(DbClient::from_ref(options))
Migration::new(DbClient::new(options.clone()))
.migrate()
.await?,
)
Expand Down
Loading