Side types overhaul (#762)

* side types overhaul

* fixes, fmt clippy

* migration fix for v3 bug

* fixed migration issues

* more tested migration changes

* fmt, clippy

* bump cicd

---------

Co-authored-by: Geometrically <[email protected]>

migrations/20231005230721_dynamic-fields.sql

@@ -72,18 +72,20 @@ INSERT INTO loader_fields_loaders (loader_id, loader_field_id) SELECT l.id, lf.i
 INSERT INTO loader_fields_loaders (loader_id, loader_field_id) SELECT l.id, lf.id FROM loaders l CROSS JOIN loader_fields lf  WHERE lf.field = 'server_side' AND l.loader = ANY( ARRAY['forge', 'fabric', 'quilt', 'modloader','rift','liteloader', 'neoforge']);
 
 INSERT INTO version_fields (version_id, field_id, enum_value) 
-SELECT v.id, 1, m.client_side 
+SELECT v.id, lf.id, lfev.id  -- Note: bug fix/edited 2023-11-27
 FROM versions v
 INNER JOIN mods m ON v.mod_id = m.id
 INNER JOIN loader_field_enum_values lfev ON m.client_side = lfev.original_id
-WHERE client_side IS NOT NULL AND lfev.enum_id = 1;
+CROSS JOIN loader_fields lf
+WHERE client_side IS NOT NULL AND lfev.enum_id = 1 AND lf.field = 'client_side';
 
 INSERT INTO version_fields (version_id, field_id, enum_value) 
-SELECT v.id, 1, m.server_side 
+SELECT v.id, lf.id, lfev.id   -- Note: bug fix/edited 2023-11-27
 FROM versions v
 INNER JOIN mods m ON v.mod_id = m.id
-INNER JOIN loader_field_enum_values lfev ON m.client_side = lfev.original_id
-WHERE server_side IS NOT NULL AND lfev.enum_id = 1;
+INNER JOIN loader_field_enum_values lfev ON m.server_side = lfev.original_id
+CROSS JOIN loader_fields lf
+WHERE server_side IS NOT NULL AND lfev.enum_id = 1 AND lf.field = 'server_side';
 
 ALTER TABLE mods DROP COLUMN client_side;
 ALTER TABLE mods DROP COLUMN server_side;
@@ -95,11 +97,13 @@ INSERT INTO loader_field_enum_values (original_id, enum_id, value, created, meta
 SELECT id, 2, version, created, json_build_object('type', type, 'major', major) FROM game_versions;
 
 INSERT INTO loader_fields (field, field_type, enum_type, optional, min_val) VALUES('game_versions', 'array_enum', 2, false, 0);
+INSERT INTO loader_fields_loaders (loader_id, loader_field_id) SELECT l.id, lf.id FROM loaders l CROSS JOIN loader_fields lf  WHERE lf.field = 'game_versions' AND l.loader = ANY( ARRAY['forge', 'fabric', 'quilt', 'modloader','rift','liteloader', 'neoforge']);
 
 INSERT INTO version_fields(version_id, field_id, enum_value) 
-SELECT gvv.joining_version_id, 2, lfev.id 
+SELECT gvv.joining_version_id, lf.id, lfev.id 
 FROM game_versions_versions gvv INNER JOIN loader_field_enum_values lfev ON gvv.game_version_id = lfev.original_id
-WHERE lfev.enum_id = 2;
+CROSS JOIN loader_fields lf
+WHERE lf.field = 'game_versions' AND lfev.enum_id = 2;
 
 ALTER TABLE mods DROP COLUMN loaders;
 ALTER TABLE mods DROP COLUMN game_versions;
@@ -108,12 +112,13 @@ DROP TABLE game_versions;
 
 -- Convert project types
 -- we are creating a new loader type- 'mrpack'- for minecraft modpacks
+SELECT setval('loaders_id_seq', (SELECT MAX(id) FROM loaders) + 1, false);
 INSERT INTO loaders (loader) VALUES ('mrpack');
 
 -- For the loader 'mrpack', we create loader fields for every loader
 -- That way we keep information like "this modpack is a fabric modpack"
 INSERT INTO loader_field_enums (id, enum_name, hidable) VALUES (3, 'mrpack_loaders', true);
-INSERT INTO loader_field_enum_values (original_id, enum_id, value) SELECT id, 2, loader FROM loaders WHERE loader != 'mrpack';
+INSERT INTO loader_field_enum_values (original_id, enum_id, value) SELECT id, 3, loader FROM loaders WHERE loader != 'mrpack';
 INSERT INTO loader_fields (field, field_type, enum_type, optional, min_val) VALUES('mrpack_loaders', 'array_enum', 3, false, 0);
 INSERT INTO loader_fields_loaders (loader_id, loader_field_id) 
 SELECT l.id, lf.id FROM loaders l CROSS JOIN loader_fields lf  WHERE lf.field = 'mrpack_loaders' AND l.loader = 'mrpack';
@@ -125,11 +130,31 @@ INNER JOIN mods m ON v.mod_id = m.id
 INNER JOIN loaders_versions lv ON v.id = lv.version_id
 INNER JOIN loaders l ON lv.loader_id = l.id
 CROSS JOIN loader_fields lf
-LEFT JOIN loader_field_enum_values lfev ON lf.enum_type = lfev.enum_id 
+LEFT JOIN loader_field_enum_values lfev ON lf.enum_type = lfev.enum_id AND lfev.original_id = l.id
 WHERE m.project_type = (SELECT id FROM project_types WHERE name = 'modpack') AND lf.field = 'mrpack_loaders';
 
 INSERT INTO loaders_project_types (joining_loader_id, joining_project_type_id) SELECT DISTINCT l.id, pt.id FROM loaders l CROSS JOIN project_types pt WHERE pt.name = 'modpack' AND l.loader = 'mrpack';
 
+-- Set those versions to mrpack as their version
+INSERT INTO loaders_versions (version_id, loader_id)
+SELECT DISTINCT vf.version_id, l.id
+FROM version_fields vf
+LEFT JOIN loader_fields lf ON lf.id = vf.field_id
+CROSS JOIN loaders l
+WHERE lf.field = 'mrpack_loaders'
+AND l.loader = 'mrpack'
+ON CONFLICT DO NOTHING;
+
+--  Delete the old versions that had mrpack added to them
+DELETE FROM loaders_versions lv
+WHERE lv.loader_id != (SELECT id FROM loaders WHERE loader = 'mrpack')
+AND lv.version_id IN (
+    SELECT version_id
+    FROM loaders_versions
+    WHERE loader_id = (SELECT id FROM loaders WHERE loader = 'mrpack')
+);
+
+
 --- Non-mrpack loaders no longer support modpacks
 DELETE FROM loaders_project_types WHERE joining_loader_id != (SELECT id FROM loaders WHERE loader = 'mrpack') AND joining_project_type_id = (SELECT id FROM project_types WHERE name = 'modpack');
 

migrations/20231116112800_side_types_overhaul.sql

@@ -0,0 +1,96 @@
+
+INSERT INTO loader_fields (field, field_type, optional) SELECT 'singleplayer', 'boolean', false;
+INSERT INTO loader_fields (field, field_type, optional) SELECT 'client_and_server', 'boolean', false;
+INSERT INTO loader_fields (field, field_type, optional) SELECT 'client_only', 'boolean', false;
+INSERT INTO loader_fields (field, field_type, optional) SELECT 'server_only', 'boolean', false;
+
+-- Create 4 temporary columns for the four booleans (makes queries easier)
+ALTER TABLE versions ADD COLUMN singleplayer boolean;
+ALTER TABLE versions ADD COLUMN client_and_server boolean;
+ALTER TABLE versions ADD COLUMN client_only boolean;
+ALTER TABLE versions ADD COLUMN server_only boolean;
+
+-- Set singleplayer to be true if either client_side or server_side is 'required' OR 'optional'
+UPDATE versions v SET singleplayer = true
+FROM version_fields vf
+INNER JOIN loader_fields lf ON vf.field_id = lf.id
+INNER JOIN loader_field_enum_values lfev ON lf.enum_type = lfev.id AND vf.enum_value = lfev.id
+WHERE v.id = vf.version_id 
+AND (lf.field = 'client_side' OR lf.field = 'server_side') AND (lfev.value = 'required' OR lfev.value = 'optional');
+
+-- Set client and server to be true if either client_side or server_side is 'required' OR 'optional'
+UPDATE versions v SET client_and_server = true
+FROM version_fields vf
+INNER JOIN loader_fields lf ON vf.field_id = lf.id
+INNER JOIN loader_field_enum_values lfev ON lf.enum_type = lfev.id AND vf.enum_value = lfev.id
+WHERE v.id = vf.version_id 
+AND (lf.field = 'client_side' OR lf.field = 'server_side') AND (lfev.value = 'required' OR lfev.value = 'optional');
+
+-- Set client_only to be true if client_side is 'required' or 'optional', and server_side is 'optional', 'unsupported', or 'unknown'
+UPDATE versions v SET client_only = true
+FROM version_fields vf
+INNER JOIN loader_fields lf ON vf.field_id = lf.id
+INNER JOIN loader_field_enum_values lfev ON lf.enum_type = lfev.enum_id AND vf.enum_value = lfev.id
+CROSS JOIN version_fields vf2
+INNER JOIN loader_fields lf2 ON vf2.field_id = lf2.id
+INNER JOIN loader_field_enum_values lfev2 ON lf2.enum_type = lfev2.enum_id AND vf2.enum_value = lfev2.id
+WHERE v.id = vf.version_id AND v.id = vf2.version_id
+AND lf.field = 'client_side' AND (lfev.value = 'required' OR lfev.value = 'optional') 
+AND lf2.field = 'server_side' AND (lfev2.value = 'optional' OR lfev2.value = 'unsupported' OR lfev2.value = 'unknown');
+
+-- Set server_only to be true if server_side is 'required' or 'optional', and client_side is 'optional', 'unsupported', or 'unknown'
+UPDATE versions v SET server_only = true
+FROM version_fields vf
+INNER JOIN loader_fields lf ON vf.field_id = lf.id
+INNER JOIN loader_field_enum_values lfev ON lf.enum_type = lfev.enum_id AND vf.enum_value = lfev.id
+CROSS JOIN version_fields vf2
+INNER JOIN loader_fields lf2 ON vf2.field_id = lf2.id
+INNER JOIN loader_field_enum_values lfev2 ON lf2.enum_type = lfev2.enum_id AND vf2.enum_value = lfev2.id
+WHERE v.id = vf.version_id  AND v.id = vf2.version_id
+AND lf.field = 'server_side' AND (lfev.value = 'required' OR lfev.value = 'optional')
+AND lf2.field = 'client_side' AND (lfev2.value = 'optional' OR lfev2.value = 'unsupported' OR lfev2.value = 'unknown');
+
+-- Insert the values into the version_fields table
+INSERT INTO version_fields (version_id, field_id, int_value) 
+SELECT v.id, lf.id, CASE WHEN v.singleplayer THEN 1 ELSE 0 END
+FROM versions v
+INNER JOIN loader_fields lf ON lf.field = 'singleplayer';
+
+INSERT INTO version_fields (version_id, field_id, int_value)
+SELECT v.id, lf.id, CASE WHEN v.client_and_server THEN 1 ELSE 0 END
+FROM versions v
+INNER JOIN loader_fields lf ON lf.field = 'client_and_server';
+
+INSERT INTO version_fields (version_id, field_id, int_value)
+SELECT v.id, lf.id, CASE WHEN v.client_only THEN 1 ELSE 0 END
+FROM versions v
+INNER JOIN loader_fields lf ON lf.field = 'client_only';
+
+INSERT INTO version_fields (version_id, field_id, int_value)
+SELECT v.id, lf.id, CASE WHEN v.server_only THEN 1 ELSE 0 END
+FROM versions v
+INNER JOIN loader_fields lf ON lf.field = 'server_only';
+
+-- Drop the temporary columns
+ALTER TABLE versions DROP COLUMN singleplayer;
+ALTER TABLE versions DROP COLUMN client_and_server;
+ALTER TABLE versions DROP COLUMN client_only;
+ALTER TABLE versions DROP COLUMN server_only;
+
+-- For each loader where loader_fields_loaders is 'client_side' or 'server_side', add the new fields
+INSERT INTO loader_fields_loaders (loader_id, loader_field_id) 
+SELECT lfl.loader_id, lf.id
+FROM loader_fields_loaders lfl
+CROSS JOIN loader_fields lf
+WHERE lfl.loader_field_id IN (SELECT id FROM loader_fields WHERE field = 'client_side' OR field = 'server_side')
+AND lf.field IN ('singleplayer', 'client_and_server', 'client_only', 'server_only')
+ON CONFLICT DO NOTHING;
+
+-- Drop the old loader_fields_loaders entries
+DELETE FROM loader_fields_loaders WHERE loader_field_id IN (SELECT id FROM loader_fields WHERE field = 'client_side' OR field = 'server_side');
+
+-- Drop client_side and server_side loader fields
+DELETE FROM version_fields WHERE field_id IN (SELECT id FROM loader_fields WHERE field = 'client_side' OR field = 'server_side');
+DELETE FROM loader_field_enum_values WHERE id IN (SELECT enum_type FROM loader_fields WHERE field = 'client_side' OR field = 'server_side');
+DELETE FROM loader_fields WHERE field = 'client_side' OR field = 'server_side';
+DELETE FROM loader_field_enums WHERE id IN (SELECT enum_type FROM loader_fields WHERE field = 'side_types');

src/database/models/loader_fields.rs

@@ -320,6 +320,23 @@ impl LoaderField {
         exec: E,
         redis: &RedisPool,
     ) -> Result<Vec<LoaderField>, DatabaseError>
+    where
+        E: sqlx::Executor<'a, Database = sqlx::Postgres>,
+    {
+        let found_loader_fields = Self::get_fields_per_loader(loader_ids, exec, redis).await?;
+        let result = found_loader_fields
+            .into_values()
+            .flatten()
+            .unique_by(|x| x.id)
+            .collect();
+        Ok(result)
+    }
+
+    pub async fn get_fields_per_loader<'a, E>(
+        loader_ids: &[LoaderId],
+        exec: E,
+        redis: &RedisPool,
+    ) -> Result<HashMap<LoaderId, Vec<LoaderField>>, DatabaseError>
     where
         E: sqlx::Executor<'a, Database = sqlx::Postgres>,
     {
@@ -336,11 +353,11 @@ impl LoaderField {
             .filter_map(|x: String| serde_json::from_str::<RedisLoaderFieldTuple>(&x).ok())
             .collect();
 
-        let mut found_loader_fields = vec![];
+        let mut found_loader_fields = HashMap::new();
         if !cached_fields.is_empty() {
             for (loader_id, fields) in cached_fields {
                 if loader_ids.contains(&loader_id) {
-                    found_loader_fields.extend(fields);
+                    found_loader_fields.insert(loader_id, fields);
                     loader_ids.retain(|x| x != &loader_id);
                 }
             }
@@ -388,14 +405,10 @@ impl LoaderField {
                 redis
                     .set_serialized_to_json(LOADER_FIELDS_NAMESPACE, k.0, (k, &v), None)
                     .await?;
-                found_loader_fields.extend(v);
+                found_loader_fields.insert(k, v);
             }
         }
-        let result = found_loader_fields
-            .into_iter()
-            .unique_by(|x| x.id)
-            .collect();
-        Ok(result)
+        Ok(found_loader_fields)
     }
 
     // Gets all fields for a given loader(s)

src/models/v2/projects.rs

@@ -1,3 +1,5 @@
+use std::collections::HashMap;
+
 use super::super::ids::OrganizationId;
 use super::super::teams::TeamId;
 use super::super::users::UserId;
@@ -10,6 +12,7 @@ use crate::models::projects::{
     Project, ProjectStatus, Version, VersionFile, VersionStatus, VersionType,
 };
 use crate::models::threads::ThreadId;
+use crate::routes::v2_reroute;
 use chrono::{DateTime, Utc};
 use serde::{Deserialize, Serialize};
 
@@ -85,26 +88,6 @@ impl LegacyProject {
         let mut loaders = data.loaders;
 
         if let Some(versions_item) = versions_item {
-            client_side = versions_item
-                .version_fields
-                .iter()
-                .find(|f| f.field_name == "client_side")
-                .and_then(|f| {
-                    Some(LegacySideType::from_string(
-                        f.value.serialize_internal().as_str()?,
-                    ))
-                })
-                .unwrap_or(LegacySideType::Unknown);
-            server_side = versions_item
-                .version_fields
-                .iter()
-                .find(|f| f.field_name == "server_side")
-                .and_then(|f| {
-                    Some(LegacySideType::from_string(
-                        f.value.serialize_internal().as_str()?,
-                    ))
-                })
-                .unwrap_or(LegacySideType::Unknown);
             game_versions = versions_item
                 .version_fields
                 .iter()
@@ -113,6 +96,14 @@ impl LegacyProject {
                 .map(|v| v.into_iter().map(|v| v.version).collect())
                 .unwrap_or(Vec::new());
 
+            // Extract side types from remaining fields (singleplayer, client_only, etc)
+            let fields = versions_item
+                .version_fields
+                .iter()
+                .map(|f| (f.field_name.clone(), f.value.clone().serialize_internal()))
+                .collect::<HashMap<_, _>>();
+            (client_side, server_side) = v2_reroute::convert_side_types_v2(&fields);
+
             // - if loader is mrpack, this is a modpack
             // the loaders are whatever the corresponding loader fields are
             if versions_item.loaders == vec!["mrpack".to_string()] {
@@ -194,7 +185,7 @@ impl LegacyProject {
     }
 }
 
-#[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq)]
+#[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq, Copy)]
 #[serde(rename_all = "kebab-case")]
 pub enum LegacySideType {
     Required,

src/models/v3/pack.rs

@@ -1,4 +1,4 @@
-use crate::{models::projects::SideType, util::env::parse_strings_from_var};
+use crate::{models::v2::projects::LegacySideType, util::env::parse_strings_from_var};
 use serde::{Deserialize, Serialize};
 use validator::Validate;
 
@@ -23,7 +23,7 @@ pub struct PackFormat {
 pub struct PackFile {
     pub path: String,
     pub hashes: std::collections::HashMap<PackFileHash, String>,
-    pub env: Option<std::collections::HashMap<EnvType, SideType>>,
+    pub env: Option<std::collections::HashMap<EnvType, LegacySideType>>, // TODO: Should this use LegacySideType? Will probably require a overhaul of mrpack format to change this
     #[validate(custom(function = "validate_download_url"))]
     pub downloads: Vec<String>,
     pub file_size: u32,