loaders to loader_fields, added tests

migrations/20231005230721_dynamic-fields.sql

@@ -6,41 +6,6 @@ CREATE TABLE games (
 INSERT INTO games(id, name) VALUES (1, 'minecraft-java');
 INSERT INTO games(id, name) VALUES (2, 'minecraft-bedrock');
 
--- we are creating a new loader type- 'mrpack'- for minecraft modpacks
-INSERT INTO loaders (loader) VALUES ('mrpack');
-INSERT INTO loaders_project_types (joining_loader_id, joining_project_type_id) SELECT DISTINCT l.id, pt.id FROM loaders l CROSS JOIN project_types pt WHERE pt.name = 'modpack' AND l.loader = 'mrpack';
-
--- We create 'modpack' categories for every loader 
--- That way we keep information like "this modpack is a fabric modpack"
-INSERT INTO categories (category, project_type) 
-SELECT DISTINCT l.loader, pt.id FROM loaders l CROSS JOIN project_types pt WHERE pt.name = 'modpack' AND l.loader != 'mrpack';
-
--- insert the loader of every modpack mod as a category
-INSERT INTO mods_categories (joining_mod_id, joining_category_id)
-SELECT DISTINCT m.id, c.id 
-FROM mods m
-LEFT JOIN versions v ON m.id = v.mod_id
-LEFT JOIN loaders_versions lv ON v.id = lv.version_id
-LEFT JOIN loaders l ON lv.loader_id = l.id
-CROSS JOIN categories c 
-WHERE m.project_type = (SELECT id FROM project_types WHERE name = 'modpack') AND c.category = l.loader;
-
--- Non mrpack loaders no longer support modpacks
-DELETE FROM loaders_project_types WHERE joining_loader_id != (SELECT id FROM loaders WHERE loader = 'mrpack') AND joining_project_type_id = (SELECT id FROM project_types WHERE name = 'modpack');
-
-CREATE TABLE loaders_project_types_games (
-  loader_id integer REFERENCES loaders NOT NULL,
-  project_type_id integer REFERENCES project_types NOT NULL,
-  game_id integer REFERENCES games NOT NULL,
-  PRIMARY KEY (loader_id, project_type_id, game_id)
-);
-
--- all past loader_project_types are minecraft-java as the only game before this migration is minecraft-java
-INSERT INTO loaders_project_types_games (loader_id, project_type_id, game_id) SELECT joining_loader_id, joining_project_type_id, 1 FROM loaders_project_types;
-
--- Now that loaders are inferred, we can drop the project_type column from mods
-ALTER TABLE mods DROP COLUMN project_type;
-
 ALTER TABLE loaders ADD CONSTRAINT unique_loader_name UNIQUE (loader);
 
 CREATE TABLE loader_field_enums (
@@ -129,7 +94,7 @@ INSERT INTO loader_field_enums (id, enum_name, hidable) VALUES (2, 'game_version
 INSERT INTO loader_field_enum_values (original_id, enum_id, value, created, metadata)
 SELECT id, 2, version, created, json_build_object('type', type, 'major', major) FROM game_versions;
 
-INSERT INTO loader_fields (field, field_type, enum_type, optional, min_val) VALUES('game_versions', 'array_enum', 2, false, 1);
+INSERT INTO loader_fields (field, field_type, enum_type, optional, min_val) VALUES('game_versions', 'array_enum', 2, false, 0);
 
 INSERT INTO version_fields(version_id, field_id, enum_value) 
 SELECT gvv.joining_version_id, 2, lfev.id 
@@ -141,5 +106,46 @@ ALTER TABLE mods DROP COLUMN game_versions;
 DROP TABLE game_versions_versions;
 DROP TABLE game_versions;
 
+-- Convert project types
+-- we are creating a new loader type- 'mrpack'- for minecraft modpacks
+INSERT INTO loaders (loader) VALUES ('mrpack');
+
+-- For the loader 'mrpack', we create loader fields for every loader
+-- That way we keep information like "this modpack is a fabric modpack"
+INSERT INTO loader_field_enums (id, enum_name, hidable) VALUES (3, 'mrpack_loaders', true);
+INSERT INTO loader_field_enum_values (original_id, enum_id, value) SELECT id, 2, loader FROM loaders WHERE loader != 'mrpack';
+INSERT INTO loader_fields (field, field_type, enum_type, optional, min_val) VALUES('mrpack_loaders', 'array_enum', 3, false, 0);
+INSERT INTO loader_fields_loaders (loader_id, loader_field_id) 
+SELECT l.id, lf.id FROM loaders l CROSS JOIN loader_fields lf  WHERE lf.field = 'mrpack_loaders' AND l.loader = 'mrpack';
+
+INSERT INTO version_fields(version_id, field_id, enum_value)
+SELECT v.id, lf.id, lfev.id
+FROM versions v
+INNER JOIN mods m ON v.mod_id = m.id
+INNER JOIN loaders_versions lv ON v.id = lv.version_id
+INNER JOIN loaders l ON lv.loader_id = l.id
+CROSS JOIN loader_fields lf
+LEFT JOIN loader_field_enum_values lfev ON lf.enum_type = lfev.enum_id 
+WHERE m.project_type = (SELECT id FROM project_types WHERE name = 'modpack') AND lf.field = 'mrpack_loaders';
+
+INSERT INTO loaders_project_types (joining_loader_id, joining_project_type_id) SELECT DISTINCT l.id, pt.id FROM loaders l CROSS JOIN project_types pt WHERE pt.name = 'modpack' AND l.loader = 'mrpack';
+
+--- Non-mrpack loaders no longer support modpacks
+DELETE FROM loaders_project_types WHERE joining_loader_id != (SELECT id FROM loaders WHERE loader = 'mrpack') AND joining_project_type_id = (SELECT id FROM project_types WHERE name = 'modpack');
+
+CREATE TABLE loaders_project_types_games (
+  loader_id integer REFERENCES loaders NOT NULL,
+  project_type_id integer REFERENCES project_types NOT NULL,
+  game_id integer REFERENCES games NOT NULL,
+  PRIMARY KEY (loader_id, project_type_id, game_id)
+);
+
+-- all past loader_project_types are minecraft-java as the only game before this migration is minecraft-java
+INSERT INTO loaders_project_types_games (loader_id, project_type_id, game_id) SELECT joining_loader_id, joining_project_type_id, 1 FROM loaders_project_types;
+
+-- Now that loaders are inferred, we can drop the project_type column from mods
+ALTER TABLE mods DROP COLUMN project_type;
+
+
 -- Drop original_id columns
 ALTER TABLE loader_field_enum_values DROP COLUMN original_id;

src/database/models/loader_fields.rs

@@ -650,14 +650,14 @@ impl VersionField {
         }
 
         let query_loader_fields: Vec<JsonLoaderField> = loader_fields
-            .and_then(|x| serde_json::from_value(x).unwrap())
+            .and_then(|x| serde_json::from_value(x).ok())
             .unwrap_or_default();
         let query_version_field_combined: Vec<JsonVersionField> = version_fields
-            .and_then(|x| serde_json::from_value(x).unwrap())
+            .and_then(|x| serde_json::from_value(x).ok())
             .unwrap_or_default();
         let query_loader_field_enum_values: Vec<JsonLoaderFieldEnumValue> =
             loader_field_enum_values
-                .and_then(|x| serde_json::from_value(x).unwrap())
+                .and_then(|x| serde_json::from_value(x).ok())
                 .unwrap_or_default();
         let version_id = VersionId(version_id);
         query_loader_fields
@@ -918,9 +918,9 @@ impl VersionFieldValue {
         }
     }
 
-    // For conversion to an interanl string, such as for search facets or filtering
+    // For conversion to an interanl string(s), such as for search facets, filtering, or direct hardcoding
     // No matter the type, it will be converted to a Vec<String>, whre the non-array types will have a single element
-    pub fn as_search_strings(&self) -> Vec<String> {
+    pub fn as_strings(&self) -> Vec<String> {
         match self {
             VersionFieldValue::Integer(i) => vec![i.to_string()],
             VersionFieldValue::Text(s) => vec![s.clone()],

src/models/v2/projects.rs

@@ -66,15 +66,19 @@ pub struct LegacyProject {
 impl LegacyProject {
     // Convert from a standard V3 project to a V2 project
     // Requires any queried versions to be passed in, to get access to certain version fields contained within.
-    // It's safe to use a db version_item for this as the only info is side types and game versions, which used to be public on project anyway.
+    // - This can be any version, because the fields are ones that used to be on the project itself.
+    // - Its conceivable that certain V3 projects that have many different ones may not have the same fields on all of them.
+    // TODO: Should this return an error instead for v2 users?
+    // It's safe to use a db version_item for this as the only info is side types, game versions, and loader fields (for loaders), which used to be public on project anyway.
     pub fn from(data: Project, versions_item: Option<version_item::QueryVersion>) -> Self {
         let mut client_side = LegacySideType::Unknown;
         let mut server_side = LegacySideType::Unknown;
         let mut game_versions = Vec::new();
 
-        // TODO: extract modpack changes
-        // - if loader is mrpack, this is a modpack
-        // the loaders are whatever the corresponding cateogires are
+        // V2 versions only have one project type- v3 versions can rarely have multiple.
+        // We'll just use the first one.
+        let mut project_type = data.project_types.get(0).cloned().unwrap_or_default();
+        let mut loaders = data.loaders;
 
         if let Some(versions_item) = versions_item {
             client_side = versions_item
@@ -104,11 +108,20 @@ impl LegacyProject {
                 .and_then(|f| MinecraftGameVersion::try_from_version_field(f).ok())
                 .map(|v| v.into_iter().map(|v| v.version).collect())
                 .unwrap_or(Vec::new());
-        }
 
-        // V2 projects only have one project type- v3 ones can rarely have multiple.
-        // We'll just use the first one.
-        let project_type = data.project_types.get(0).cloned().unwrap_or_default();
+            // - if loader is mrpack, this is a modpack
+            // the loaders are whatever the corresponding loader fields are
+            if versions_item.loaders == vec!["mrpack".to_string()] {
+                project_type = "modpack".to_string();
+                if let Some(mrpack_loaders) = versions_item
+                    .version_fields
+                    .iter()
+                    .find(|f| f.field_name == "mrpack_loaders")
+                {
+                    loaders = mrpack_loaders.value.as_strings();
+                }
+            }
+        }
 
         Self {
             id: data.id,
@@ -132,7 +145,7 @@ impl LegacyProject {
             followers: data.followers,
             categories: data.categories,
             additional_categories: data.additional_categories,
-            loaders: data.loaders,
+            loaders,
             versions: data.versions,
             icon_url: data.icon_url,
             issues_url: data.issues_url,
@@ -251,6 +264,22 @@ impl From<Version> for LegacyVersion {
             }
         }
 
+        // - if loader is mrpack, this is a modpack
+        // the v2 loaders are whatever the corresponding loader fields are
+        let mut loaders = data.loaders.into_iter().map(|l| l.0).collect::<Vec<_>>();
+        if loaders == vec!["mrpack".to_string()] {
+            if let Some((_, mrpack_loaders)) = data
+                .fields
+                .into_iter()
+                .find(|(key, _)| key == "mrpack_loaders")
+            {
+                if let Ok(mrpack_loaders) = serde_json::from_value(mrpack_loaders) {
+                    loaders = mrpack_loaders;
+                }
+            }
+        }
+        let loaders = loaders.into_iter().map(Loader).collect::<Vec<_>>();
+
         Self {
             id: data.id,
             project_id: data.project_id,
@@ -268,7 +297,7 @@ impl From<Version> for LegacyVersion {
             files: data.files,
             dependencies: data.dependencies,
             game_versions,
-            loaders: data.loaders,
+            loaders,
         }
     }
 }

src/routes/v2/project_creation.rs

@@ -12,7 +12,6 @@ use crate::routes::{v2_reroute, v3};
 use actix_multipart::Multipart;
 use actix_web::web::Data;
 use actix_web::{post, HttpRequest, HttpResponse};
-use itertools::Itertools;
 use serde::{Deserialize, Serialize};
 use serde_json::json;
 use sqlx::postgres::PgPool;
@@ -153,9 +152,6 @@ pub async fn project_create(
             let server_side = legacy_create.server_side;
 
             let project_type = legacy_create.project_type;
-            // Modpacks now use the "mrpack" loader, and loaders are converted to categories.
-            // Setting of 'project_type' directly is removed, it's loader-based now.
-            let mut additional_categories = legacy_create.additional_categories;
 
             let initial_versions = legacy_create
                 .initial_versions
@@ -166,8 +162,10 @@ pub async fn project_create(
                     fields.insert("server_side".to_string(), json!(server_side));
                     fields.insert("game_versions".to_string(), json!(v.game_versions));
 
+                    // Modpacks now use the "mrpack" loader, and loaders are converted to loader fields.
+                    // Setting of 'project_type' directly is removed, it's loader-based now.
                     if project_type == "modpack" {
-                        additional_categories.extend(v.loaders.iter().map(|l| l.0.clone()));
+                        fields.insert("mrpack_loaders".to_string(), json!(v.loaders.clone()));
                     }
 
                     let loaders = if project_type == "modpack" {
@@ -195,19 +193,14 @@ pub async fn project_create(
                 })
                 .collect();
 
-            let additional_categories = additional_categories
-                .into_iter()
-                .unique()
-                .collect::<Vec<_>>();
-            println!("additional_categories: {:?}", additional_categories);
             Ok(v3::project_creation::ProjectCreateData {
                 title: legacy_create.title,
                 slug: legacy_create.slug,
                 description: legacy_create.description,
                 body: legacy_create.body,
                 initial_versions,
                 categories: legacy_create.categories,
-                additional_categories,
+                additional_categories: legacy_create.additional_categories,
                 issues_url: legacy_create.issues_url,
                 source_url: legacy_create.source_url,
                 wiki_url: legacy_create.wiki_url,
@@ -236,14 +229,9 @@ pub async fn project_create(
     )
     .await?;
 
-    println!("did a little test <3");
     // Convert response to V2 format
     match v2_reroute::extract_ok_json::<Project>(response).await {
         Ok(project) => {
-            println!(
-                "Just finished doing a project create, looking at repsonse: {:?}",
-                serde_json::to_string(&project).unwrap()
-            );
             let version_item = match project.versions.first() {
                 Some(vid) => version_item::Version::get((*vid).into(), &**client, &redis).await?,
                 None => None,

src/routes/v2/version_creation.rs

@@ -99,7 +99,7 @@ pub async fn version_create(
 
             // Ideally this would, if the project 'should' be a modpack:
             // - change the loaders to mrpack only
-            // - add categories to the project for the corresponding loaders
+            // - add loader fields to the project for the corresponding loaders
 
             Ok(v3::version_creation::InitialVersionData {
                 project_id: legacy_create.project_id,

src/routes/v2_reroute.rs

@@ -102,7 +102,7 @@ where
             headers.insert(key, value);
         }
         Err(err) => {
-            panic!("Error inserting test header: {:?}.", err);
+            CreateError::InvalidInput(format!("Error inserting test header: {:?}.", err));
         }
     };
 

src/routes/v3/analytics_get.rs

@@ -340,7 +340,7 @@ pub async fn revenue_get(
 
     let duration: PgInterval = Duration::minutes(resolution_minutes as i64)
         .try_into()
-        .unwrap();
+        .map_err(|_| ApiError::InvalidInput("Invalid resolution_minutes".to_string()))?;
     // Get the revenue data
     let payouts_values = sqlx::query!(
         "

src/routes/v3/organizations.rs

@@ -479,7 +479,6 @@ pub async fn organization_delete(
     redis: web::Data<RedisPool>,
     session_queue: web::Data<AuthQueue>,
 ) -> Result<HttpResponse, ApiError> {
-    println!("DELETE ORGANIZATION");
     let user = get_user_from_headers(
         &req,
         &**pool,
@@ -490,15 +489,13 @@ pub async fn organization_delete(
     .await?
     .1;
     let string = info.into_inner().0;
-    println!("string: {}", string);
 
     let organization = database::models::Organization::get(&string, &**pool, &redis)
         .await?
         .ok_or_else(|| {
             ApiError::InvalidInput("The specified organization does not exist!".to_string())
         })?;
 
-    println!("organization: {:?}", organization);
     if !user.role.is_admin() {
         let team_member = database::models::TeamMember::get_from_user_id_organization(
             organization.id,

src/routes/v3/projects.rs

@@ -274,7 +274,6 @@ pub async fn project_edit(
     redis: web::Data<RedisPool>,
     session_queue: web::Data<AuthQueue>,
 ) -> Result<HttpResponse, ApiError> {
-    println!("project_edit");
     let user = get_user_from_headers(
         &req,
         &**pool,
@@ -284,17 +283,13 @@ pub async fn project_edit(
     )
     .await?
     .1;
-    println!("serde user {}", serde_json::to_string(&user)?);
 
     new_project
         .validate()
         .map_err(|err| ApiError::Validation(validation_errors_to_string(err, None)))?;
 
-    println!("new project {}", serde_json::to_string(&new_project)?);
     let string = info.into_inner().0;
-    println!("string {}", string);
     let result = db_models::Project::get(&string, &**pool, &redis).await?;
-    println!("result {}", serde_json::to_string(&result)?);
     if let Some(project_item) = result {
         let id = project_item.inner.id;
 

src/routes/v3/users.rs

@@ -676,10 +676,6 @@ pub async fn user_notifications(
             .collect();
 
         notifications.sort_by(|a, b| b.created.cmp(&a.created));
-        println!(
-            "notifications: {:?}",
-            serde_json::to_string(&notifications).unwrap()
-        );
         Ok(HttpResponse::Ok().json(notifications))
     } else {
         Ok(HttpResponse::NotFound().body(""))

src/search/indexing/local_import.rs

@@ -101,7 +101,7 @@ pub async fn index_local(
                     let version_fields = VersionField::from_query_json(m.id, m.loader_fields, m.version_fields, m.loader_field_enum_values);
 
                     let loader_fields : HashMap<String, Vec<String>> = version_fields.into_iter().map(|vf| {
-                        (vf.field_name, vf.value.as_search_strings())
+                        (vf.field_name, vf.value.as_strings())
                     }).collect();
 
                     for v in loader_fields.keys().cloned() {
@@ -121,6 +121,15 @@ pub async fn index_local(
                         _ => false,
                     };
 
+                    // SPECIAL BEHAVIOUR
+                    // Todo: revisit.
+                    // For consistency with v2 searching, we consider the loader field 'mrpack_loaders' to be a category.
+                    // These were previously considered the loader, and in v2, the loader is a category for searching.
+                    // So to avoid breakage or awkward conversions, we just consider those loader_fields to be categories.
+                    // The loaders are kept in loader_fields as well, so that no information is lost on retrieval.
+                    let mrpack_loaders = loader_fields.get("mrpack_loaders").cloned().unwrap_or_default();
+                    categories.extend(mrpack_loaders);
+
                     UploadSearchProject {
                         version_id: version_id.to_string(),
                         project_id: project_id.to_string(),