Refactor endpoints

This is a complete refactor of the API, allowing us to easily switching
from one framework to the other, in our case `lambda_http` and `axum`.

The refactor is done in a way where the endpoints would use the same
logic and same database requests.

This patch also adds a new implementation for using Axum either as a
standalone server (to facilitate local debugging) or as a lambda
function handler.

Signed-off-by: Rémy Greinhofer <[email protected]>

.gitignore

@@ -1,4 +1,5 @@
 .DS_Store
+.hypothesis/
 .vscode/
 *.pdb
 **/*.rs.bk

Cargo.toml

@@ -10,37 +10,42 @@ members = ["lambdas", "entity", "migration", "effortless"]
 [workspace.dependencies]
 async-std = "1"
 aws_lambda_events = "0.15.0"
-aws-config = "1.0.0"
-aws-sdk-s3 = "1.5.0"
-aws-sdk-sqs = "1.3.0"
+aws-config = "1.5.9"
+aws-sdk-s3 = "1.57.0"
+aws-sdk-sqs = "1.46.0"
+axum = "0.7.7"
+axum-extra = "0.9.4"
+bnacore = { git = "https://github.com/PeopleForBikes/brokenspoke", rev = "c20ec31" }
+bon = "2.3.0"
 chrono = "0.4.24"
 color-eyre = "0.6.2"
 csv = "1.3.0"
 dotenv = "0.15.0"
 entity = { path = "entity" }
-futures = "0.3.21"
+futures = "0.3.31"
 http-serde = "2.0.0"
 itertools = "0.13.0"
 lambda_http = "0.13.0"
 lambda_runtime = "0.13.0"
 migration = { path = "migration" }
 nom = "7.1.3"
-once_cell = "1.17.1"
-reqwest = "0.12.1"
+once_cell = "1.20.2"
+query_map = "0.7.0"
+reqwest = "0.12.8"
 rstest = "0.23.0"
-sea-orm = "1"
+sea-orm = "1.1.0"
 sea-orm-migration = "1.0.0"
-serde = "1.0.159"
-serde_json = "1.0.95"
+serde = "1.0.213"
+serde_json = "1.0.132"
 serde_plain = "1.0.2"
-serde_with = "3.4.0"
-thiserror = "1"
-tokio = "1"
+serde_with = "3.11.0"
+thiserror = "1.0.65"
+tokio = "1.41.0"
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", default-features = false }
 url = "2.3.1"
 urlencoding = "2.1.3"
-uuid = "1.7.0"
+uuid = "1.11.0"
 
 [dependencies]
 chrono = { workspace = true }
@@ -59,7 +64,6 @@ serde_with = { workspace = true }
 tokio = { workspace = true }
 
 [dev-dependencies]
-bnacore = { git = "https://github.com/PeopleForBikes/brokenspoke", rev = "b1f76eb" }
-# bnacore = { path = "../brokenspoke/bnacore" }
+bnacore = { workspace = true }
 csv = { workspace = true }
 itertools = { workspace = true }

effortless/Cargo.toml

@@ -8,10 +8,14 @@ name = "effortless"
 path = "src/lib.rs"
 
 [dependencies]
+bon = { workspace = true }
 http-serde = { workspace = true }
 lambda_http = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 serde_with = { workspace = true }
 thiserror = { workspace = true }
 urlencoding = { workspace = true }
+
+[lints.clippy]
+result_large_err = "allow"

effortless/src/api.rs

@@ -3,7 +3,7 @@ use crate::{
     fragment::{self, get_apigw_request_id, BnaRequestExt},
 };
 use lambda_http::{http::StatusCode, Body, Request, RequestExt, RequestPayloadExt, Response};
-use serde::de::DeserializeOwned;
+use serde::{de::DeserializeOwned, Deserialize};
 use std::{fmt::Display, str::FromStr};
 
 /// Parse the first path parameter found in the API Gateway request, into the provided type.
@@ -192,23 +192,29 @@ pub const MAX_PAGE_SIZE: u64 = 100;
 pub const DEFAULT_PAGE_SIZE: u64 = 50;
 
 /// The pagination details.
-#[derive(Debug)]
+#[derive(Debug, Deserialize)]
 pub struct PaginationParameters {
     /// The number of items per page.
-    pub page_size: u64,
+    pub page_size: Option<u64>,
     /// The result page being returned.
     pub page: u64,
 }
 
 impl Default for PaginationParameters {
     fn default() -> Self {
         Self {
-            page_size: DEFAULT_PAGE_SIZE,
+            page_size: Some(DEFAULT_PAGE_SIZE),
             page: 0,
         }
     }
 }
 
+impl PaginationParameters {
+    pub fn page_size(&self) -> u64 {
+        self.page_size.unwrap_or(DEFAULT_PAGE_SIZE)
+    }
+}
+
 // Retrieves the pagination parameters.
 ///
 /// If nothing is provided, the first page is returned and will contain up to
@@ -231,8 +237,8 @@ pub fn extract_pagination_parameters(
         match page_size.parse::<u64>() {
             Ok(page_size) => {
                 pagination.page_size = match page_size {
-                    0..=MAX_PAGE_SIZE => page_size,
-                    _ => MAX_PAGE_SIZE,
+                    0..=MAX_PAGE_SIZE => Some(page_size),
+                    _ => Some(MAX_PAGE_SIZE),
                 }
             }
             Err(e) => {
@@ -281,7 +287,7 @@ mod tests {
         let req = from_str(input).unwrap();
 
         let actual = extract_pagination_parameters(&req).unwrap();
-        assert_eq!(actual.page_size, DEFAULT_PAGE_SIZE);
+        assert_eq!(actual.page_size, Some(DEFAULT_PAGE_SIZE));
         assert_eq!(actual.page, 0);
     }
 
@@ -299,7 +305,7 @@ mod tests {
         let req = result.with_query_string_parameters(data);
 
         let actual = extract_pagination_parameters(&req).unwrap();
-        assert_eq!(actual.page_size, PAGE_SIZE);
+        assert_eq!(actual.page_size, Some(PAGE_SIZE));
         assert_eq!(actual.page, PAGE);
     }
 

effortless/src/error.rs

@@ -1,3 +1,4 @@
+use bon::Builder;
 use lambda_http::{http::header, http::StatusCode, Body, Response};
 use serde::{Deserialize, Serialize};
 use serde_json::json;
@@ -25,7 +26,7 @@ pub enum APIErrorSource {
 
 /// Single API Error object as described in <https://jsonapi.org/format/#error-objects>.
 #[skip_serializing_none]
-#[derive(Deserialize, Serialize, Clone, Debug, PartialEq)]
+#[derive(Deserialize, Serialize, Clone, Debug, PartialEq, Builder)]
 pub struct APIError {
     /// A unique identifier for this particular occurrence of the problem.
     id: Option<String>,
@@ -111,6 +112,11 @@ impl APIError {
             details: message.into(),
         }
     }
+
+    /// Returns the APIError status.
+    pub fn status(&self) -> StatusCode {
+        self.status
+    }
 }
 
 impl From<APIError> for Response<Body> {
@@ -154,6 +160,11 @@ impl APIErrors {
     pub fn is_empty(&self) -> bool {
         self.errors.is_empty()
     }
+
+    /// Returns the errors field.
+    pub fn errors(&self) -> &[APIError] {
+        self.errors.as_slice()
+    }
 }
 
 impl From<APIError> for APIErrors {

effortless/src/fragment.rs

@@ -157,6 +157,9 @@ pub trait BnaRequestExt {
 
     /// Returns true if there are query parameters available.
     fn has_query_parameters(&self) -> bool;
+
+    /// Returns the path and query or "/" if not available.
+    fn path_and_query(&self) -> String;
 }
 
 impl<B> BnaRequestExt for http::Request<B> {
@@ -198,6 +201,12 @@ impl<B> BnaRequestExt for http::Request<B> {
     fn has_query_parameters(&self) -> bool {
         !self.query_string_parameters().is_empty()
     }
+
+    fn path_and_query(&self) -> String {
+        self.uri()
+            .path_and_query()
+            .map_or(String::from("/"), |p| p.to_string())
+    }
 }
 
 #[cfg(test)]

examples/seeder.rs

@@ -7,8 +7,8 @@ use color_eyre::{eyre::Report, Result};
 use csv::Reader;
 use dotenv::dotenv;
 use entity::{
-    census, city, core_services, infrastructure, opportunity, people, prelude::*, recreation,
-    retail, speed_limit, summary, transit,
+    census, city, core_services, fargate_price, infrastructure, opportunity, people, prelude::*,
+    recreation, retail, speed_limit, summary, transit,
 };
 use sea_orm::{prelude::Uuid, ActiveValue, Database, EntityTrait};
 use serde::Deserialize;
@@ -42,6 +42,7 @@ async fn main() -> Result<(), Report> {
     let mut bna_infrastructure: Vec<infrastructure::ActiveModel> = Vec::new();
     let mut versions: HashMap<Uuid, Calver> = HashMap::new();
     let mut city_fips2limit: HashMap<u32, u32> = HashMap::new();
+    let mut fargate_price: Vec<fargate_price::ActiveModel> = Vec::new();
 
     // Set the database connection.
     let database_url = dotenv::var("DATABASE_URL")?;
@@ -138,7 +139,7 @@ async fn main() -> Result<(), Report> {
             fips_code: scorecard
                 .census_fips_code
                 .map_or(ActiveValue::NotSet, |v| ActiveValue::Set(v.to_string())),
-            pop_size: match scorecard.pop_size {
+            pop_size: match scorecard.pop_size.unwrap() {
                 bnacore::scorecard::Size::Small => ActiveValue::Set(0),
                 bnacore::scorecard::Size::Medium => ActiveValue::Set(1),
                 bnacore::scorecard::Size::Large => ActiveValue::Set(2),

lambdas/Cargo.toml

@@ -4,11 +4,13 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
+axum = { workspace = true, features = ["macros", "original-uri", "query"] }
+axum-extra = { workspace = true, features = ["query"] }
 aws-config = { workspace = true }
 aws-sdk-s3 = { workspace = true }
 aws-sdk-sqs = { workspace = true }
 aws_lambda_events = { workspace = true }
-bnacore = { git = "https://github.com/PeopleForBikes/brokenspoke.git", rev = "f1d9115" }
+bnacore = { workspace = true }
 dotenv = { workspace = true }
 effortless = { path = "../effortless" }
 entity = { path = "../entity" }
@@ -17,6 +19,7 @@ lambda_http = { workspace = true }
 lambda_runtime = { workspace = true }
 nom = { workspace = true }
 once_cell = { workspace = true }
+query_map = { workspace = true, features = ["url-query"] }
 reqwest = { workspace = true, features = ["json", "native-tls-vendored"] }
 sea-orm = { workspace = true, features = [
   "sqlx-postgres",
@@ -26,79 +29,91 @@ sea-orm = { workspace = true, features = [
 serde = { workspace = true }
 serde_json = { workspace = true }
 serde_plain = { workspace = true }
-tokio = { workspace = true }
+thiserror = { workspace = true }
+tokio = { workspace = true, features = ["macros"] }
 tracing = { workspace = true, features = ["log"] }
 tracing-subscriber = { workspace = true, features = ["fmt"] }
 url = { workspace = true }
 uuid = { workspace = true, features = ["v4", "serde"] }
 
+[lints.clippy]
+result_large_err = "allow"
+
+[lib]
+name = "lambdas"
+path = "src/lib.rs"
+
+[[bin]]
+name = "server"
+path = "src/main.rs"
+
 [[bin]]
 name = "get-ratings"
-path = "src/ratings/get-ratings.rs"
+path = "src/bin/ratings/get-ratings.rs"
 
 [[bin]]
 name = "post-ratings"
-path = "src/ratings/post-ratings.rs"
+path = "src/bin/ratings/post-ratings.rs"
 
 # [[bin]]
 # name = "patch-bnas"
 # path = "src/bnas/patch-bnas.rs"
 
 [[bin]]
 name = "get-ratings-cities"
-path = "src/ratings/get-ratings-cities.rs"
+path = "src/bin/ratings/get-ratings-cities.rs"
 
 [[bin]]
 name = "get-ratings-analysis"
-path = "src/ratings/get-ratings-analysis.rs"
+path = "src/bin/ratings/get-ratings-analysis.rs"
 
 [[bin]]
 name = "get-cities"
-path = "src/cities/get-cities.rs"
+path = "src/bin/cities/get-cities.rs"
 
 [[bin]]
 name = "get-cities-ratings"
-path = "src/cities/get-cities-ratings.rs"
+path = "src/bin/cities/get-cities-ratings.rs"
 
 [[bin]]
 name = "get-cities-census"
-path = "src/cities/get-cities-census.rs"
+path = "src/bin/cities/get-cities-census.rs"
 
 [[bin]]
 name = "get-cities-submissions"
-path = "src/cities/get-cities-submissions.rs"
+path = "src/bin/cities/get-cities-submissions.rs"
 
 [[bin]]
 name = "get-price-fargate"
-path = "src/price-fargate/get-price-fargate.rs"
+path = "src/bin/price-fargate/get-price-fargate.rs"
 
 [[bin]]
 name = "patch-ratings-analysis"
-path = "src/ratings/patch-ratings-analysis.rs"
+path = "src/bin/ratings/patch-ratings-analysis.rs"
 
 [[bin]]
 name = "patch-cities-submissions"
-path = "src/cities/patch-cities-submissions.rs"
+path = "src/bin/cities/patch-cities-submissions.rs"
 
 [[bin]]
 name = "post-cities-submissions"
-path = "src/cities/post-cities-submissions.rs"
+path = "src/bin/cities/post-cities-submissions.rs"
 
 [[bin]]
 name = "post-ratings-analysis"
-path = "src/ratings/post-ratings-analysis.rs"
+path = "src/bin/ratings/post-ratings-analysis.rs"
 
 [[bin]]
 name = "post-ratings-enqueue"
-path = "src/ratings/post-ratings-enqueue.rs"
+path = "src/bin/ratings/post-ratings-enqueue.rs"
 
 [[bin]]
 name = "get-ratings-results"
-path = "src/ratings/get-ratings-results.rs"
+path = "src/bin/ratings/get-ratings-results.rs"
 
 [[bin]]
 name = "post-cities"
-path = "src/cities/post-cities.rs"
+path = "src/bin/cities/post-cities.rs"
 
 [dev-dependencies]
 color-eyre = { workspace = true }