From 0efbbed5e2c5e986decbf774dfd22bc72924beac Mon Sep 17 00:00:00 2001 From: Carter Date: Sat, 25 Nov 2023 20:48:51 -0800 Subject: [PATCH 1/2] Add fields to OAuth (#769) * Add url and description fields to OAuthClient model * Add OAuth client icon editing and deleting endpoints * updated query data * fix missed queries * sqlx prep * update with tests builds --- ...e89dcfdd5a5ab9c7a96dc4008e74e06be914.json} | 18 +- ...c89e94cc7c54e0b7302c877586186bb128a2.json} | 6 +- ...c567b8724509d1fdbd5cc75f516c1811c9d3.json} | 18 +- .../20231122230639_oauth_client_metadata.sql | 7 + src/database/models/oauth_client_item.rs | 14 +- src/models/v3/oauth_clients.rs | 10 ++ src/routes/v3/oauth_clients.rs | 160 +++++++++++++++++- tests/oauth_clients.rs | 6 + 8 files changed, 228 insertions(+), 11 deletions(-) rename .sqlx/{query-8bfb350d4f539a110b05f42812ea2593a1556ef214f3bed519de6b6e21c7d477.json => query-467df7989a1f4f5f3bc3409a4060e89dcfdd5a5ab9c7a96dc4008e74e06be914.json} (65%) rename .sqlx/{query-e8cc8895ebc8b1904a43e00f1e123f75ffdaebc76d67a5d35218fa9273d46d53.json => query-781f54f36b8713bd36f5f89c98f8c89e94cc7c54e0b7302c877586186bb128a2.json} (54%) rename .sqlx/{query-fdfb2433a8e407d42cec1791d67549ab5c23306758168af38f955c06d251b0b7.json => query-93190af0fcfe2e795ed6b90eece7c567b8724509d1fdbd5cc75f516c1811c9d3.json} (64%) create mode 100644 migrations/20231122230639_oauth_client_metadata.sql diff --git a/.sqlx/query-8bfb350d4f539a110b05f42812ea2593a1556ef214f3bed519de6b6e21c7d477.json b/.sqlx/query-467df7989a1f4f5f3bc3409a4060e89dcfdd5a5ab9c7a96dc4008e74e06be914.json similarity index 65% rename from .sqlx/query-8bfb350d4f539a110b05f42812ea2593a1556ef214f3bed519de6b6e21c7d477.json rename to .sqlx/query-467df7989a1f4f5f3bc3409a4060e89dcfdd5a5ab9c7a96dc4008e74e06be914.json index 9be628e8..b3ecb9f5 100644 --- a/.sqlx/query-8bfb350d4f539a110b05f42812ea2593a1556ef214f3bed519de6b6e21c7d477.json +++ b/.sqlx/query-467df7989a1f4f5f3bc3409a4060e89dcfdd5a5ab9c7a96dc4008e74e06be914.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n clients.id as \"id!\",\n clients.name as \"name!\",\n clients.icon_url as \"icon_url?\",\n clients.max_scopes as \"max_scopes!\",\n clients.secret_hash as \"secret_hash!\",\n clients.created as \"created!\",\n clients.created_by as \"created_by!\",\n uris.uri_ids as \"uri_ids?\",\n uris.uri_vals as \"uri_vals?\"\n FROM oauth_clients clients\n LEFT JOIN (\n SELECT client_id, array_agg(id) as uri_ids, array_agg(uri) as uri_vals\n FROM oauth_client_redirect_uris\n GROUP BY client_id\n ) uris ON clients.id = uris.client_id\n WHERE created_by = $1", + "query": "\n SELECT\n clients.id as \"id!\",\n clients.name as \"name!\",\n clients.icon_url as \"icon_url?\",\n clients.max_scopes as \"max_scopes!\",\n clients.secret_hash as \"secret_hash!\",\n clients.created as \"created!\",\n clients.created_by as \"created_by!\",\n clients.url as \"url?\",\n clients.description as \"description?\",\n uris.uri_ids as \"uri_ids?\",\n uris.uri_vals as \"uri_vals?\"\n FROM oauth_clients clients\n LEFT JOIN (\n SELECT client_id, array_agg(id) as uri_ids, array_agg(uri) as uri_vals\n FROM oauth_client_redirect_uris\n GROUP BY client_id\n ) uris ON clients.id = uris.client_id\n WHERE created_by = $1", "describe": { "columns": [ { @@ -40,11 +40,21 @@ }, { "ordinal": 7, + "name": "url?", + "type_info": "Text" + }, + { + "ordinal": 8, + "name": "description?", + "type_info": "Text" + }, + { + "ordinal": 9, "name": "uri_ids?", "type_info": "Int8Array" }, { - "ordinal": 8, + "ordinal": 10, "name": "uri_vals?", "type_info": "TextArray" } @@ -62,9 +72,11 @@ false, false, false, + true, + true, null, null ] }, - "hash": "8bfb350d4f539a110b05f42812ea2593a1556ef214f3bed519de6b6e21c7d477" + "hash": "467df7989a1f4f5f3bc3409a4060e89dcfdd5a5ab9c7a96dc4008e74e06be914" } diff --git a/.sqlx/query-e8cc8895ebc8b1904a43e00f1e123f75ffdaebc76d67a5d35218fa9273d46d53.json b/.sqlx/query-781f54f36b8713bd36f5f89c98f8c89e94cc7c54e0b7302c877586186bb128a2.json similarity index 54% rename from .sqlx/query-e8cc8895ebc8b1904a43e00f1e123f75ffdaebc76d67a5d35218fa9273d46d53.json rename to .sqlx/query-781f54f36b8713bd36f5f89c98f8c89e94cc7c54e0b7302c877586186bb128a2.json index d01d5876..1eaac2f1 100644 --- a/.sqlx/query-e8cc8895ebc8b1904a43e00f1e123f75ffdaebc76d67a5d35218fa9273d46d53.json +++ b/.sqlx/query-781f54f36b8713bd36f5f89c98f8c89e94cc7c54e0b7302c877586186bb128a2.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE oauth_clients\n SET name = $1, icon_url = $2, max_scopes = $3\n WHERE (id = $4)\n ", + "query": "\n UPDATE oauth_clients\n SET name = $1, icon_url = $2, max_scopes = $3, url = $4, description = $5\n WHERE (id = $6)\n ", "describe": { "columns": [], "parameters": { @@ -8,10 +8,12 @@ "Text", "Text", "Int8", + "Text", + "Text", "Int8" ] }, "nullable": [] }, - "hash": "e8cc8895ebc8b1904a43e00f1e123f75ffdaebc76d67a5d35218fa9273d46d53" + "hash": "781f54f36b8713bd36f5f89c98f8c89e94cc7c54e0b7302c877586186bb128a2" } diff --git a/.sqlx/query-fdfb2433a8e407d42cec1791d67549ab5c23306758168af38f955c06d251b0b7.json b/.sqlx/query-93190af0fcfe2e795ed6b90eece7c567b8724509d1fdbd5cc75f516c1811c9d3.json similarity index 64% rename from .sqlx/query-fdfb2433a8e407d42cec1791d67549ab5c23306758168af38f955c06d251b0b7.json rename to .sqlx/query-93190af0fcfe2e795ed6b90eece7c567b8724509d1fdbd5cc75f516c1811c9d3.json index 08fa78f0..4179fde2 100644 --- a/.sqlx/query-fdfb2433a8e407d42cec1791d67549ab5c23306758168af38f955c06d251b0b7.json +++ b/.sqlx/query-93190af0fcfe2e795ed6b90eece7c567b8724509d1fdbd5cc75f516c1811c9d3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n clients.id as \"id!\",\n clients.name as \"name!\",\n clients.icon_url as \"icon_url?\",\n clients.max_scopes as \"max_scopes!\",\n clients.secret_hash as \"secret_hash!\",\n clients.created as \"created!\",\n clients.created_by as \"created_by!\",\n uris.uri_ids as \"uri_ids?\",\n uris.uri_vals as \"uri_vals?\"\n FROM oauth_clients clients\n LEFT JOIN (\n SELECT client_id, array_agg(id) as uri_ids, array_agg(uri) as uri_vals\n FROM oauth_client_redirect_uris\n GROUP BY client_id\n ) uris ON clients.id = uris.client_id\n WHERE clients.id = ANY($1::bigint[])", + "query": "\n SELECT\n clients.id as \"id!\",\n clients.name as \"name!\",\n clients.icon_url as \"icon_url?\",\n clients.max_scopes as \"max_scopes!\",\n clients.secret_hash as \"secret_hash!\",\n clients.created as \"created!\",\n clients.created_by as \"created_by!\",\n clients.url as \"url?\",\n clients.description as \"description?\",\n uris.uri_ids as \"uri_ids?\",\n uris.uri_vals as \"uri_vals?\"\n FROM oauth_clients clients\n LEFT JOIN (\n SELECT client_id, array_agg(id) as uri_ids, array_agg(uri) as uri_vals\n FROM oauth_client_redirect_uris\n GROUP BY client_id\n ) uris ON clients.id = uris.client_id\n WHERE clients.id = ANY($1::bigint[])", "describe": { "columns": [ { @@ -40,11 +40,21 @@ }, { "ordinal": 7, + "name": "url?", + "type_info": "Text" + }, + { + "ordinal": 8, + "name": "description?", + "type_info": "Text" + }, + { + "ordinal": 9, "name": "uri_ids?", "type_info": "Int8Array" }, { - "ordinal": 8, + "ordinal": 10, "name": "uri_vals?", "type_info": "TextArray" } @@ -62,9 +72,11 @@ true, true, true, + true, + true, null, null ] }, - "hash": "fdfb2433a8e407d42cec1791d67549ab5c23306758168af38f955c06d251b0b7" + "hash": "93190af0fcfe2e795ed6b90eece7c567b8724509d1fdbd5cc75f516c1811c9d3" } diff --git a/migrations/20231122230639_oauth_client_metadata.sql b/migrations/20231122230639_oauth_client_metadata.sql new file mode 100644 index 00000000..dd05dbba --- /dev/null +++ b/migrations/20231122230639_oauth_client_metadata.sql @@ -0,0 +1,7 @@ +-- Add migration script here +ALTER TABLE + oauth_clients +ADD + COLUMN url text NULL, +ADD + COLUMN description text NULL; \ No newline at end of file diff --git a/src/database/models/oauth_client_item.rs b/src/database/models/oauth_client_item.rs index 48870c67..c2abbea7 100644 --- a/src/database/models/oauth_client_item.rs +++ b/src/database/models/oauth_client_item.rs @@ -23,6 +23,8 @@ pub struct OAuthClient { pub redirect_uris: Vec, pub created: DateTime, pub created_by: UserId, + pub url: Option, + pub description: Option, } struct ClientQueryResult { @@ -33,6 +35,8 @@ struct ClientQueryResult { secret_hash: String, created: DateTime, created_by: i64, + url: Option, + description: Option, uri_ids: Option>, uri_vals: Option>, } @@ -53,6 +57,8 @@ macro_rules! select_clients_with_predicate { clients.secret_hash as "secret_hash!", clients.created as "created!", clients.created_by as "created_by!", + clients.url as "url?", + clients.description as "description?", uris.uri_ids as "uri_ids?", uris.uri_vals as "uri_vals?" FROM oauth_clients clients @@ -155,12 +161,14 @@ impl OAuthClient { sqlx::query!( " UPDATE oauth_clients - SET name = $1, icon_url = $2, max_scopes = $3 - WHERE (id = $4) + SET name = $1, icon_url = $2, max_scopes = $3, url = $4, description = $5 + WHERE (id = $6) ", self.name, self.icon_url, self.max_scopes.to_postgres(), + self.url, + self.description, self.id.0, ) .execute(exec) @@ -240,6 +248,8 @@ impl From for OAuthClient { redirect_uris: redirects, created: r.created, created_by: UserId(r.created_by), + url: r.url, + description: r.description, } } } diff --git a/src/models/v3/oauth_clients.rs b/src/models/v3/oauth_clients.rs index f13eb97b..e3979f68 100644 --- a/src/models/v3/oauth_clients.rs +++ b/src/models/v3/oauth_clients.rs @@ -54,6 +54,13 @@ pub struct OAuthClient { // The user that created (and thus controls) this client pub created_by: UserId, + + // When this client was created + pub created: DateTime, + + // (optional) Metadata about the client + pub url: Option, + pub description: Option, } #[derive(Deserialize, Serialize)] @@ -88,6 +95,9 @@ impl From for OAuthClient { max_scopes: value.max_scopes, redirect_uris: value.redirect_uris.into_iter().map(|r| r.into()).collect(), created_by: value.created_by.into(), + created: value.created, + url: value.url, + description: value.description, } } } diff --git a/src/routes/v3/oauth_clients.rs b/src/routes/v3/oauth_clients.rs index 277ff912..881cd040 100644 --- a/src/routes/v3/oauth_clients.rs +++ b/src/routes/v3/oauth_clients.rs @@ -1,4 +1,4 @@ -use std::{collections::HashSet, fmt::Display}; +use std::{collections::HashSet, fmt::Display, sync::Arc}; use actix_web::{ delete, get, patch, post, @@ -16,7 +16,9 @@ use validator::Validate; use super::ApiError; use crate::{ auth::checks::ValidateAllAuthorized, + file_hosting::FileHost, models::{ids::base62_impl::parse_base62, oauth_clients::DeleteOAuthClientQueryParam}, + util::routes::read_from_payload, }; use crate::{ auth::{checks::ValidateAuthorized, get_user_from_headers}, @@ -50,6 +52,8 @@ pub fn config(cfg: &mut web::ServiceConfig) { .service(oauth_client_create) .service(oauth_client_edit) .service(oauth_client_delete) + .service(oauth_client_icon_edit) + .service(oauth_client_icon_delete) .service(get_client) .service(get_clients) .service(get_user_oauth_authorizations), @@ -145,6 +149,15 @@ pub struct NewOAuthApp { pub max_scopes: Scopes, pub redirect_uris: Vec, + + #[validate( + custom(function = "crate::util::validate::validate_url"), + length(max = 255) + )] + pub url: Option, + + #[validate(length(max = 255))] + pub description: Option, } #[post("app")] @@ -187,6 +200,8 @@ pub async fn oauth_client_create<'a>( redirect_uris, created: Utc::now(), created_by: current_user.id.into(), + url: new_oauth_app.url.clone(), + description: new_oauth_app.description.clone(), secret_hash: client_secret_hash, }; client.clone().insert(&mut transaction).await?; @@ -248,6 +263,15 @@ pub struct OAuthClientEdit { #[validate(length(min = 1))] pub redirect_uris: Option>, + + #[validate( + custom(function = "crate::util::validate::validate_url"), + length(max = 255) + )] + pub url: Option>, + + #[validate(length(max = 255))] + pub description: Option>, } #[patch("app/{id}")] @@ -289,6 +313,8 @@ pub async fn oauth_client_edit( icon_url, max_scopes, redirect_uris, + url, + description, } = client_updates.into_inner(); if let Some(name) = name { updated_client.name = name; @@ -302,6 +328,14 @@ pub async fn oauth_client_edit( updated_client.max_scopes = max_scopes; } + if let Some(url) = url { + updated_client.url = url; + } + + if let Some(description) = description { + updated_client.description = description; + } + let mut transaction = pool.begin().await?; updated_client .update_editable_fields(&mut *transaction) @@ -319,6 +353,130 @@ pub async fn oauth_client_edit( } } +#[derive(Serialize, Deserialize)] +pub struct Extension { + pub ext: String, +} + +#[patch("app/{id}/icon")] +#[allow(clippy::too_many_arguments)] +pub async fn oauth_client_icon_edit( + web::Query(ext): web::Query, + req: HttpRequest, + client_id: web::Path, + pool: web::Data, + redis: web::Data, + file_host: web::Data>, + mut payload: web::Payload, + session_queue: web::Data, +) -> Result { + if let Some(content_type) = crate::util::ext::get_image_content_type(&ext.ext) { + let cdn_url = dotenvy::var("CDN_URL")?; + let user = get_user_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Some(&[Scopes::SESSION_ACCESS]), + ) + .await? + .1; + + let client = OAuthClient::get((*client_id).into(), &**pool) + .await? + .ok_or_else(|| { + ApiError::InvalidInput("The specified client does not exist!".to_string()) + })?; + + client.validate_authorized(Some(&user))?; + + if let Some(ref icon) = client.icon_url { + let name = icon.split(&format!("{cdn_url}/")).nth(1); + + if let Some(icon_path) = name { + file_host.delete_file_version("", icon_path).await?; + } + } + + let bytes = + read_from_payload(&mut payload, 262144, "Icons must be smaller than 256KiB").await?; + let hash = sha1::Sha1::from(&bytes).hexdigest(); + let upload_data = file_host + .upload_file( + content_type, + &format!("data/{}/{}.{}", client_id, hash, ext.ext), + bytes.freeze(), + ) + .await?; + + let mut transaction = pool.begin().await?; + + let mut editable_client = client.clone(); + editable_client.icon_url = Some(format!("{}/{}", cdn_url, upload_data.file_name)); + + editable_client + .update_editable_fields(&mut *transaction) + .await?; + + transaction.commit().await?; + + Ok(HttpResponse::NoContent().body("")) + } else { + Err(ApiError::InvalidInput(format!( + "Invalid format for project icon: {}", + ext.ext + ))) + } +} + +#[delete("app/{id}/icon")] +pub async fn oauth_client_icon_delete( + req: HttpRequest, + client_id: web::Path, + pool: web::Data, + redis: web::Data, + file_host: web::Data>, + session_queue: web::Data, +) -> Result { + let cdn_url = dotenvy::var("CDN_URL")?; + let user = get_user_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Some(&[Scopes::SESSION_ACCESS]), + ) + .await? + .1; + + let client = OAuthClient::get((*client_id).into(), &**pool) + .await? + .ok_or_else(|| { + ApiError::InvalidInput("The specified client does not exist!".to_string()) + })?; + client.validate_authorized(Some(&user))?; + + if let Some(ref icon) = client.icon_url { + let name = icon.split(&format!("{cdn_url}/")).nth(1); + + if let Some(icon_path) = name { + file_host.delete_file_version("", icon_path).await?; + } + } + + let mut transaction = pool.begin().await?; + + let mut editable_client = client.clone(); + editable_client.icon_url = None; + + editable_client + .update_editable_fields(&mut *transaction) + .await?; + transaction.commit().await?; + + Ok(HttpResponse::NoContent().body("")) +} + #[get("authorizations")] pub async fn get_user_oauth_authorizations( req: HttpRequest, diff --git a/tests/oauth_clients.rs b/tests/oauth_clients.rs index 6f42c433..e601b6c0 100644 --- a/tests/oauth_clients.rs +++ b/tests/oauth_clients.rs @@ -41,6 +41,8 @@ async fn can_create_edit_get_oauth_client() { let client_id = get_json_val_str(creation_result.client.id); let icon_url = Some("https://modrinth.com/icon".to_string()); + let url = Some("https://modrinth.com".to_string()); + let description = Some("test description".to_string()); let edited_redirect_uris = vec![ redirect_uris[0].clone(), "https://modrinth.com/b".to_string(), @@ -50,6 +52,8 @@ async fn can_create_edit_get_oauth_client() { icon_url: Some(icon_url.clone()), max_scopes: None, redirect_uris: Some(edited_redirect_uris.clone()), + url: Some(url.clone()), + description: Some(description.clone()), }; let resp = env .api @@ -63,6 +67,8 @@ async fn can_create_edit_get_oauth_client() { .await; assert_eq!(1, clients.len()); assert_eq!(icon_url, clients[0].icon_url); + assert_eq!(url, clients[0].url); + assert_eq!(description, clients[0].description); assert_eq!(client_name, clients[0].name); assert_eq!(2, clients[0].redirect_uris.len()); assert_eq!(edited_redirect_uris[0], clients[0].redirect_uris[0].uri); From fd18185ef00cabe9c81d43e69d9eeea22e11ed35 Mon Sep 17 00:00:00 2001 From: Wyatt Verchere Date: Sun, 26 Nov 2023 19:29:59 -0800 Subject: [PATCH 2/2] More staging fixes (#768) * Fixes issues * staging fixes * passes tests * fixes. fmt/clippy * drops datapack/plugin extras * fixed failing test --------- Co-authored-by: Geometrically <18202329+Geometrically@users.noreply.github.com> --- ...836a02ff5dc408c9fc233e8b5ad3b48125dc4.json | 56 ++++++++ .../20231125080100_drops_mods_dp_plugins.sql | 31 +++++ src/database/models/loader_fields.rs | 52 +++++++- src/models/v2/mod.rs | 1 + src/models/v2/projects.rs | 6 +- src/models/v2/search.rs | 124 ++++++++++++++++++ src/routes/v2/projects.rs | 3 +- src/routes/v3/tags.rs | 22 ++-- src/search/indexing/mod.rs | 6 + tests/common/api_common/generic.rs | 10 +- tests/common/api_common/mod.rs | 7 - tests/common/api_v2/project.rs | 59 +++++---- tests/common/api_v3/project.rs | 6 +- tests/search.rs | 6 +- tests/v2/search.rs | 6 +- tests/v2/version.rs | 2 +- 16 files changed, 324 insertions(+), 73 deletions(-) create mode 100644 .sqlx/query-846b66683e6abd40acd158195d8836a02ff5dc408c9fc233e8b5ad3b48125dc4.json create mode 100644 migrations/20231125080100_drops_mods_dp_plugins.sql create mode 100644 src/models/v2/search.rs diff --git a/.sqlx/query-846b66683e6abd40acd158195d8836a02ff5dc408c9fc233e8b5ad3b48125dc4.json b/.sqlx/query-846b66683e6abd40acd158195d8836a02ff5dc408c9fc233e8b5ad3b48125dc4.json new file mode 100644 index 00000000..8f42b441 --- /dev/null +++ b/.sqlx/query-846b66683e6abd40acd158195d8836a02ff5dc408c9fc233e8b5ad3b48125dc4.json @@ -0,0 +1,56 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT DISTINCT lf.id, lf.field, lf.field_type, lf.optional, lf.min_val, lf.max_val, lf.enum_type\n FROM loader_fields lf\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "field", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "field_type", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "optional", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "min_val", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "max_val", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "enum_type", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false, + false, + true, + true, + true + ] + }, + "hash": "846b66683e6abd40acd158195d8836a02ff5dc408c9fc233e8b5ad3b48125dc4" +} diff --git a/migrations/20231125080100_drops_mods_dp_plugins.sql b/migrations/20231125080100_drops_mods_dp_plugins.sql new file mode 100644 index 00000000..b9750bf2 --- /dev/null +++ b/migrations/20231125080100_drops_mods_dp_plugins.sql @@ -0,0 +1,31 @@ +-- For every loader that has a loaders_project_types entry that connects it to the project_types 'plugin', +-- remove all non plugin project_types entries for that loader. +-- This is to ensure that the plugin project_types is the only one that is used for the plugin loaders + +--plugin +DELETE FROM loaders_project_types +WHERE joining_loader_id IN ( + SELECT DISTINCT l.id + FROM loaders l + LEFT JOIN loaders_project_types lpt ON lpt.joining_loader_id = l.id + LEFT JOIN project_types pt ON pt.id = lpt.joining_project_type_id + WHERE pt.name = 'plugin' +) +AND joining_project_type_id NOT IN ( + SELECT id FROM project_types + WHERE name = 'plugin' +); + +--datapack +DELETE FROM loaders_project_types +WHERE joining_loader_id IN ( + SELECT DISTINCT l.id + FROM loaders l + LEFT JOIN loaders_project_types lpt ON lpt.joining_loader_id = l.id + LEFT JOIN project_types pt ON pt.id = lpt.joining_project_type_id + WHERE pt.name = 'datapack' +) +AND joining_project_type_id NOT IN ( + SELECT id FROM project_types + WHERE name = 'datapack' +); \ No newline at end of file diff --git a/src/database/models/loader_fields.rs b/src/database/models/loader_fields.rs index 87edf8cc..22ceb316 100644 --- a/src/database/models/loader_fields.rs +++ b/src/database/models/loader_fields.rs @@ -13,6 +13,7 @@ const GAMES_LIST_NAMESPACE: &str = "games"; const LOADER_ID: &str = "loader_id"; const LOADERS_LIST_NAMESPACE: &str = "loaders"; const LOADER_FIELDS_NAMESPACE: &str = "loader_fields"; +const LOADER_FIELDS_NAMESPACE_ALL: &str = "loader_fields_all"; const LOADER_FIELD_ENUMS_ID_NAMESPACE: &str = "loader_field_enums"; const LOADER_FIELD_ENUM_VALUES_NAMESPACE: &str = "loader_field_enum_values"; @@ -396,8 +397,57 @@ impl LoaderField { .collect(); Ok(result) } -} + // Gets all fields for a given loader(s) + // This is for tags, which need all fields for all loaders + // We want to return them even in testing situations where we dont have loaders or loader_fields_loaders set up + pub async fn get_fields_all<'a, E>( + exec: E, + redis: &RedisPool, + ) -> Result, DatabaseError> + where + E: sqlx::Executor<'a, Database = sqlx::Postgres>, + { + let mut redis = redis.connect().await?; + + let cached_fields: Option> = redis + .get(LOADER_FIELDS_NAMESPACE_ALL, "") + .await? + .and_then(|x| serde_json::from_str::>(&x).ok()); + + if let Some(cached_fields) = cached_fields { + return Ok(cached_fields); + } + + let result = sqlx::query!( + " + SELECT DISTINCT lf.id, lf.field, lf.field_type, lf.optional, lf.min_val, lf.max_val, lf.enum_type + FROM loader_fields lf + ", + ) + .fetch_many(exec) + .try_filter_map(|e| async { + Ok(e.right().and_then(|r| { + Some(LoaderField { + id: LoaderFieldId(r.id), + field_type: LoaderFieldType::build(&r.field_type, r.enum_type)?, + field: r.field, + optional: r.optional, + min_val: r.min_val, + max_val: r.max_val, + }) + })) + }) + .try_collect::>() + .await?; + + redis + .set_serialized_to_json(LOADER_FIELDS_NAMESPACE_ALL, "", &result, None) + .await?; + + Ok(result) + } +} impl LoaderFieldEnum { pub async fn get<'a, E>( enum_name: &str, // Note: NOT loader field name diff --git a/src/models/v2/mod.rs b/src/models/v2/mod.rs index 5df1866a..291432b9 100644 --- a/src/models/v2/mod.rs +++ b/src/models/v2/mod.rs @@ -1,2 +1,3 @@ // Legacy models from V2, where its useful to keep the struct for rerouting/conversion pub mod projects; +pub mod search; diff --git a/src/models/v2/projects.rs b/src/models/v2/projects.rs index 5e146c0b..5df1ecdb 100644 --- a/src/models/v2/projects.rs +++ b/src/models/v2/projects.rs @@ -77,7 +77,11 @@ impl LegacyProject { // V2 versions only have one project type- v3 versions can rarely have multiple. // We'll just use the first one. - let mut project_type = data.project_types.first().cloned().unwrap_or_default(); + let mut project_type = data + .project_types + .first() + .cloned() + .unwrap_or("unknown".to_string()); let mut loaders = data.loaders; if let Some(versions_item) = versions_item { diff --git a/src/models/v2/search.rs b/src/models/v2/search.rs new file mode 100644 index 00000000..5405245d --- /dev/null +++ b/src/models/v2/search.rs @@ -0,0 +1,124 @@ +use serde::{Deserialize, Serialize}; + +use crate::search::ResultSearchProject; + +#[derive(Serialize, Deserialize, Debug)] +pub struct LegacySearchResults { + pub hits: Vec, + pub offset: usize, + pub limit: usize, + pub total_hits: usize, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct LegacyResultSearchProject { + pub project_id: String, + pub project_type: String, + pub slug: Option, + pub author: String, + pub title: String, + pub description: String, + pub categories: Vec, + pub display_categories: Vec, + pub versions: Vec, + pub downloads: i32, + pub follows: i32, + pub icon_url: String, + /// RFC 3339 formatted creation date of the project + pub date_created: String, + /// RFC 3339 formatted modification date of the project + pub date_modified: String, + pub latest_version: String, + pub license: String, + pub client_side: String, + pub server_side: String, + pub gallery: Vec, + pub featured_gallery: Option, + pub color: Option, +} + +// TODO: In other PR, when these are merged, make sure the v2 search testing functions use these +impl LegacyResultSearchProject { + pub fn from(result_search_project: ResultSearchProject) -> Self { + let mut categories = result_search_project.categories; + if categories.contains(&"mrpack".to_string()) { + if let Some(mrpack_loaders) = result_search_project.loader_fields.get("mrpack_loaders") + { + categories.extend(mrpack_loaders.clone()); + categories.retain(|c| c != "mrpack"); + } + } + let mut display_categories = result_search_project.display_categories; + if display_categories.contains(&"mrpack".to_string()) { + if let Some(mrpack_loaders) = result_search_project.loader_fields.get("mrpack_loaders") + { + display_categories.extend(mrpack_loaders.clone()); + display_categories.retain(|c| c != "mrpack"); + } + } + + // Sort then remove duplicates + categories.sort(); + categories.dedup(); + display_categories.sort(); + display_categories.dedup(); + + Self { + project_type: result_search_project + .project_types + .first() + .cloned() + .unwrap_or_default(), + client_side: result_search_project + .loader_fields + .get("client_side") + .cloned() + .unwrap_or_default() + .join(","), + server_side: result_search_project + .loader_fields + .get("server_side") + .cloned() + .unwrap_or_default() + .join(","), + versions: result_search_project + .loader_fields + .get("game_versions") + .cloned() + .unwrap_or_default(), + latest_version: result_search_project.version_id, + categories, + + project_id: result_search_project.project_id, + slug: result_search_project.slug, + author: result_search_project.author, + title: result_search_project.title, + description: result_search_project.description, + display_categories, + downloads: result_search_project.downloads, + follows: result_search_project.follows, + icon_url: result_search_project.icon_url, + license: result_search_project.license, + date_created: result_search_project.date_created, + date_modified: result_search_project.date_modified, + gallery: result_search_project.gallery, + featured_gallery: result_search_project.featured_gallery, + color: result_search_project.color, + } + } +} + +impl LegacySearchResults { + pub fn from(search_results: crate::search::SearchResults) -> Self { + Self { + hits: search_results + .hits + .into_iter() + .map(LegacyResultSearchProject::from) + .collect(), + offset: search_results.offset, + limit: search_results.limit, + total_hits: search_results.total_hits, + } + } +} diff --git a/src/routes/v2/projects.rs b/src/routes/v2/projects.rs index c1a75db7..debb7a95 100644 --- a/src/routes/v2/projects.rs +++ b/src/routes/v2/projects.rs @@ -6,6 +6,7 @@ use crate::models::projects::{ DonationLink, MonetizationStatus, Project, ProjectStatus, SearchRequest, SideType, }; use crate::models::v2::projects::LegacyProject; +use crate::models::v2::search::LegacySearchResults; use crate::queue::session::AuthQueue; use crate::routes::v3::projects::ProjectIds; use crate::routes::{v2_reroute, v3, ApiError}; @@ -95,7 +96,7 @@ pub async fn project_search( let results = search_for_project(&info, &config).await?; - // TODO: convert to v2 format-we may need a new v2 struct for this for 'original' format + let results = LegacySearchResults::from(results); Ok(HttpResponse::Ok().json(results)) } diff --git a/src/routes/v3/tags.rs b/src/routes/v3/tags.rs index 8fca6b51..3e6940ae 100644 --- a/src/routes/v3/tags.rs +++ b/src/routes/v3/tags.rs @@ -7,7 +7,7 @@ use crate::database::models::loader_fields::{ }; use crate::database::redis::RedisPool; use actix_web::{web, HttpResponse}; -use itertools::Itertools; + use serde_json::Value; use sqlx::PgPool; @@ -121,20 +121,16 @@ pub async fn loader_fields_list( redis: web::Data, ) -> Result { let query = query.into_inner(); - let all_loader_ids = Loader::list(&**pool, &redis) + let loader_field = LoaderField::get_fields_all(&**pool, &redis) .await? .into_iter() - .map(|x| x.id) - .collect_vec(); - let loader_field = - LoaderField::get_field(&query.loader_field, &all_loader_ids, &**pool, &redis) - .await? - .ok_or_else(|| { - ApiError::InvalidInput(format!( - "'{}' was not a valid loader field.", - query.loader_field - )) - })?; + .find(|x| x.field == query.loader_field) + .ok_or_else(|| { + ApiError::InvalidInput(format!( + "'{}' was not a valid loader field.", + query.loader_field + )) + })?; let loader_field_enum_id = match loader_field.field_type { LoaderFieldType::Enum(enum_id) | LoaderFieldType::ArrayEnum(enum_id) => enum_id, diff --git a/src/search/indexing/mod.rs b/src/search/indexing/mod.rs index 97155916..675b499b 100644 --- a/src/search/indexing/mod.rs +++ b/src/search/indexing/mod.rs @@ -124,10 +124,16 @@ async fn create_and_add_to_index( let index = create_index(client, name, custom_rules).await?; let mut new_filterable_attributes = index.get_filterable_attributes().await?; + let mut new_displayed_attributes = index.get_displayed_attributes().await?; + new_filterable_attributes.extend(additional_fields.iter().map(|s| s.to_string())); + new_displayed_attributes.extend(additional_fields.iter().map(|s| s.to_string())); index .set_filterable_attributes(new_filterable_attributes) .await?; + index + .set_displayed_attributes(new_displayed_attributes) + .await?; add_to_index(client, index, projects).await?; Ok(()) diff --git a/tests/common/api_common/generic.rs b/tests/common/api_common/generic.rs index 19b682b4..76b84650 100644 --- a/tests/common/api_common/generic.rs +++ b/tests/common/api_common/generic.rs @@ -2,12 +2,9 @@ use std::collections::HashMap; use actix_web::dev::ServiceResponse; use async_trait::async_trait; -use labrinth::{ - models::{ - projects::{ProjectId, VersionType}, - teams::{OrganizationPermissions, ProjectPermissions}, - }, - search::SearchResults, +use labrinth::models::{ + projects::{ProjectId, VersionType}, + teams::{OrganizationPermissions, ProjectPermissions}, }; use crate::common::{api_v2::ApiV2, api_v3::ApiV3, dummy_data::TestFile}; @@ -76,7 +73,6 @@ delegate_api_variant!( [edit_project, ServiceResponse, id_or_slug: &str, patch: serde_json::Value, pat: &str], [edit_project_bulk, ServiceResponse, ids_or_slugs: &[&str], patch: serde_json::Value, pat: &str], [edit_project_icon, ServiceResponse, id_or_slug: &str, icon: Option, pat: &str], - [search_deserialized_common, SearchResults, query: Option<&str>, facets: Option, pat: &str], } ); diff --git a/tests/common/api_common/mod.rs b/tests/common/api_common/mod.rs index abf38293..48fa9d02 100644 --- a/tests/common/api_common/mod.rs +++ b/tests/common/api_common/mod.rs @@ -11,7 +11,6 @@ use labrinth::{ projects::{ProjectId, VersionType}, teams::{OrganizationPermissions, ProjectPermissions}, }, - search::SearchResults, LabrinthConfig, }; @@ -66,12 +65,6 @@ pub trait ApiProject { icon: Option, pat: &str, ) -> ServiceResponse; - async fn search_deserialized_common( - &self, - query: Option<&str>, - facets: Option, - pat: &str, - ) -> SearchResults; } #[async_trait(?Send)] diff --git a/tests/common/api_v2/project.rs b/tests/common/api_v2/project.rs index f28c048c..9e6e1702 100644 --- a/tests/common/api_v2/project.rs +++ b/tests/common/api_v2/project.rs @@ -13,7 +13,8 @@ use actix_web::{ use async_trait::async_trait; use bytes::Bytes; use labrinth::{ - models::v2::projects::LegacyProject, search::SearchResults, util::actix::AppendsMultipart, + models::v2::{projects::LegacyProject, search::LegacySearchResults}, + util::actix::AppendsMultipart, }; use serde_json::json; @@ -37,6 +38,34 @@ impl ApiV2 { assert_eq!(resp.status(), 200); test::read_body_json(resp).await } + + pub async fn search_deserialized( + &self, + query: Option<&str>, + facets: Option, + pat: &str, + ) -> LegacySearchResults { + let query_field = if let Some(query) = query { + format!("&query={}", urlencoding::encode(query)) + } else { + "".to_string() + }; + + let facets_field = if let Some(facets) = facets { + format!("&facets={}", urlencoding::encode(&facets.to_string())) + } else { + "".to_string() + }; + + let req = test::TestRequest::get() + .uri(&format!("/v2/search?{}{}", query_field, facets_field)) + .append_header(("Authorization", pat)) + .to_request(); + let resp = self.call(req).await; + let status = resp.status(); + assert_eq!(status, 200); + test::read_body_json(resp).await + } } #[async_trait(?Send)] @@ -195,32 +224,4 @@ impl ApiProject for ApiV2 { self.call(req).await } } - - async fn search_deserialized_common( - &self, - query: Option<&str>, - facets: Option, - pat: &str, - ) -> SearchResults { - let query_field = if let Some(query) = query { - format!("&query={}", urlencoding::encode(query)) - } else { - "".to_string() - }; - - let facets_field = if let Some(facets) = facets { - format!("&facets={}", urlencoding::encode(&facets.to_string())) - } else { - "".to_string() - }; - - let req = test::TestRequest::get() - .uri(&format!("/v2/search?{}{}", query_field, facets_field)) - .append_header(("Authorization", pat)) - .to_request(); - let resp = self.call(req).await; - let status = resp.status(); - assert_eq!(status, 200); - test::read_body_json(resp).await - } } diff --git a/tests/common/api_v3/project.rs b/tests/common/api_v3/project.rs index a2616c09..73981a64 100644 --- a/tests/common/api_v3/project.rs +++ b/tests/common/api_v3/project.rs @@ -179,8 +179,10 @@ impl ApiProject for ApiV3 { self.call(req).await } } +} - async fn search_deserialized_common( +impl ApiV3 { + pub async fn search_deserialized( &self, query: Option<&str>, facets: Option, @@ -207,9 +209,7 @@ impl ApiProject for ApiV3 { assert_eq!(status, 200); test::read_body_json(resp).await } -} -impl ApiV3 { pub async fn get_analytics_revenue( &self, id_or_slugs: Vec<&str>, diff --git a/tests/search.rs b/tests/search.rs index e0693125..4a3b6b73 100644 --- a/tests/search.rs +++ b/tests/search.rs @@ -289,11 +289,7 @@ async fn search_projects() { let test_name = test_name.clone(); async move { let projects = api - .search_deserialized_common( - Some(&test_name), - Some(facets.clone()), - USER_USER_PAT, - ) + .search_deserialized(Some(&test_name), Some(facets.clone()), USER_USER_PAT) .await; let mut found_project_ids: Vec = projects .hits diff --git a/tests/v2/search.rs b/tests/v2/search.rs index 4a365b05..944a4dd5 100644 --- a/tests/v2/search.rs +++ b/tests/v2/search.rs @@ -281,11 +281,7 @@ async fn search_projects() { let test_name = test_name.clone(); async move { let projects = api - .search_deserialized_common( - Some(&test_name), - Some(facets.clone()), - USER_USER_PAT, - ) + .search_deserialized(Some(&test_name), Some(facets.clone()), USER_USER_PAT) .await; let mut found_project_ids: Vec = projects .hits diff --git a/tests/v2/version.rs b/tests/v2/version.rs index 421aa3c4..ba8f0c35 100644 --- a/tests/v2/version.rs +++ b/tests/v2/version.rs @@ -427,7 +427,7 @@ async fn add_version_project_types_v2() { let test_project = api .get_project_deserialized(&test_project.slug.unwrap(), USER_USER_PAT) .await; - assert_eq!(test_project.project_type, ""); // No project_type set, as no versions are set + assert_eq!(test_project.project_type, "unknown"); // No project_type set, as no versions are set // This is a known difference between older v2 ,but is acceptable. // This would be the appropriate test on older v2: // assert_eq!(test_project.project_type, "modpack");