From 521ccab383f2ac7df780932814c44e9a6b9b49c3 Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 3 Oct 2023 18:52:21 -0700 Subject: [PATCH] search patch for accurate loader/gv filtering --- sqlx-data.json | 338 ++++++++++++++-------------- src/search/indexing/local_import.rs | 12 +- src/search/indexing/mod.rs | 6 +- src/search/mod.rs | 2 + 4 files changed, 185 insertions(+), 173 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index 46c3d051..0ffe44e4 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -1797,172 +1797,6 @@ }, "query": "\n UPDATE users\n SET password = $1\n WHERE (id = $2)\n " }, - "4514723bdc1eb8a781215075bec51af1cc6fabe88a469338d5a59533eabf80c5": { - "describe": { - "columns": [ - { - "name": "id", - "ordinal": 0, - "type_info": "Int8" - }, - { - "name": "project_type", - "ordinal": 1, - "type_info": "Int4" - }, - { - "name": "title", - "ordinal": 2, - "type_info": "Varchar" - }, - { - "name": "description", - "ordinal": 3, - "type_info": "Varchar" - }, - { - "name": "downloads", - "ordinal": 4, - "type_info": "Int4" - }, - { - "name": "follows", - "ordinal": 5, - "type_info": "Int4" - }, - { - "name": "icon_url", - "ordinal": 6, - "type_info": "Varchar" - }, - { - "name": "published", - "ordinal": 7, - "type_info": "Timestamptz" - }, - { - "name": "approved", - "ordinal": 8, - "type_info": "Timestamptz" - }, - { - "name": "updated", - "ordinal": 9, - "type_info": "Timestamptz" - }, - { - "name": "team_id", - "ordinal": 10, - "type_info": "Int8" - }, - { - "name": "license", - "ordinal": 11, - "type_info": "Varchar" - }, - { - "name": "slug", - "ordinal": 12, - "type_info": "Varchar" - }, - { - "name": "status_name", - "ordinal": 13, - "type_info": "Varchar" - }, - { - "name": "color", - "ordinal": 14, - "type_info": "Int4" - }, - { - "name": "client_side_type", - "ordinal": 15, - "type_info": "Varchar" - }, - { - "name": "server_side_type", - "ordinal": 16, - "type_info": "Varchar" - }, - { - "name": "project_type_name", - "ordinal": 17, - "type_info": "Varchar" - }, - { - "name": "username", - "ordinal": 18, - "type_info": "Varchar" - }, - { - "name": "categories", - "ordinal": 19, - "type_info": "VarcharArray" - }, - { - "name": "additional_categories", - "ordinal": 20, - "type_info": "VarcharArray" - }, - { - "name": "loaders", - "ordinal": 21, - "type_info": "VarcharArray" - }, - { - "name": "versions", - "ordinal": 22, - "type_info": "VarcharArray" - }, - { - "name": "gallery", - "ordinal": 23, - "type_info": "VarcharArray" - }, - { - "name": "featured_gallery", - "ordinal": 24, - "type_info": "VarcharArray" - } - ], - "nullable": [ - false, - false, - false, - false, - false, - false, - true, - false, - true, - false, - false, - false, - true, - false, - true, - false, - false, - false, - false, - null, - null, - null, - null, - null, - null - ], - "parameters": { - "Left": [ - "TextArray", - "TextArray", - "Text" - ] - } - }, - "query": "\n SELECT m.id id, m.project_type project_type, m.title title, m.description description, m.downloads downloads, m.follows follows,\n m.icon_url icon_url, m.published published, m.approved approved, m.updated updated,\n m.team_id team_id, m.license license, m.slug slug, m.status status_name, m.color color,\n cs.name client_side_type, ss.name server_side_type, pt.name project_type_name, u.username username,\n ARRAY_AGG(DISTINCT c.category) filter (where c.category is not null and mc.is_additional is false) categories,\n ARRAY_AGG(DISTINCT c.category) filter (where c.category is not null and mc.is_additional is true) additional_categories,\n ARRAY_AGG(DISTINCT lo.loader) filter (where lo.loader is not null) loaders,\n ARRAY_AGG(DISTINCT gv.version) filter (where gv.version is not null) versions,\n ARRAY_AGG(DISTINCT mg.image_url) filter (where mg.image_url is not null and mg.featured is false) gallery,\n ARRAY_AGG(DISTINCT mg.image_url) filter (where mg.image_url is not null and mg.featured is true) featured_gallery\n FROM mods m\n LEFT OUTER JOIN mods_categories mc ON joining_mod_id = m.id\n LEFT OUTER JOIN categories c ON mc.joining_category_id = c.id\n LEFT OUTER JOIN versions v ON v.mod_id = m.id AND v.status != ALL($1)\n LEFT OUTER JOIN game_versions_versions gvv ON gvv.joining_version_id = v.id\n LEFT OUTER JOIN game_versions gv ON gvv.game_version_id = gv.id\n LEFT OUTER JOIN loaders_versions lv ON lv.version_id = v.id\n LEFT OUTER JOIN loaders lo ON lo.id = lv.loader_id\n LEFT OUTER JOIN mods_gallery mg ON mg.mod_id = m.id\n INNER JOIN project_types pt ON pt.id = m.project_type\n INNER JOIN side_types cs ON m.client_side = cs.id\n INNER JOIN side_types ss ON m.server_side = ss.id\n INNER JOIN team_members tm ON tm.team_id = m.team_id AND tm.role = $3 AND tm.accepted = TRUE\n INNER JOIN users u ON tm.user_id = u.id\n WHERE m.status = ANY($2)\n GROUP BY m.id, cs.id, ss.id, pt.id, u.id;\n " - }, "4567790f0dc98ff20b596a33161d1f6ac8af73da67fe8c54192724626c6bf670": { "describe": { "columns": [], @@ -2320,6 +2154,178 @@ }, "query": "\n SELECT c.id id, c.user_id user_id FROM collections c\n WHERE c.user_id = $2 AND c.id = ANY($1)\n " }, + "56d0e2c6e37f97d5300ca783f475a770a3d0ab84ef4297b40f56a11ebb6053cc": { + "describe": { + "columns": [ + { + "name": "id", + "ordinal": 0, + "type_info": "Int8" + }, + { + "name": "version_id", + "ordinal": 1, + "type_info": "Int8" + }, + { + "name": "project_type", + "ordinal": 2, + "type_info": "Int4" + }, + { + "name": "title", + "ordinal": 3, + "type_info": "Varchar" + }, + { + "name": "description", + "ordinal": 4, + "type_info": "Varchar" + }, + { + "name": "downloads", + "ordinal": 5, + "type_info": "Int4" + }, + { + "name": "follows", + "ordinal": 6, + "type_info": "Int4" + }, + { + "name": "icon_url", + "ordinal": 7, + "type_info": "Varchar" + }, + { + "name": "published", + "ordinal": 8, + "type_info": "Timestamptz" + }, + { + "name": "approved", + "ordinal": 9, + "type_info": "Timestamptz" + }, + { + "name": "updated", + "ordinal": 10, + "type_info": "Timestamptz" + }, + { + "name": "team_id", + "ordinal": 11, + "type_info": "Int8" + }, + { + "name": "license", + "ordinal": 12, + "type_info": "Varchar" + }, + { + "name": "slug", + "ordinal": 13, + "type_info": "Varchar" + }, + { + "name": "status_name", + "ordinal": 14, + "type_info": "Varchar" + }, + { + "name": "color", + "ordinal": 15, + "type_info": "Int4" + }, + { + "name": "client_side_type", + "ordinal": 16, + "type_info": "Varchar" + }, + { + "name": "server_side_type", + "ordinal": 17, + "type_info": "Varchar" + }, + { + "name": "project_type_name", + "ordinal": 18, + "type_info": "Varchar" + }, + { + "name": "username", + "ordinal": 19, + "type_info": "Varchar" + }, + { + "name": "categories", + "ordinal": 20, + "type_info": "VarcharArray" + }, + { + "name": "additional_categories", + "ordinal": 21, + "type_info": "VarcharArray" + }, + { + "name": "loaders", + "ordinal": 22, + "type_info": "VarcharArray" + }, + { + "name": "versions", + "ordinal": 23, + "type_info": "VarcharArray" + }, + { + "name": "gallery", + "ordinal": 24, + "type_info": "VarcharArray" + }, + { + "name": "featured_gallery", + "ordinal": 25, + "type_info": "VarcharArray" + } + ], + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + true, + false, + true, + false, + false, + false, + true, + false, + true, + false, + false, + false, + false, + null, + null, + null, + null, + null, + null + ], + "parameters": { + "Left": [ + "TextArray", + "TextArray", + "Text" + ] + } + }, + "query": "\n SELECT m.id id, v.id version_id, m.project_type project_type, m.title title, m.description description, m.downloads downloads, m.follows follows,\n m.icon_url icon_url, m.published published, m.approved approved, m.updated updated,\n m.team_id team_id, m.license license, m.slug slug, m.status status_name, m.color color,\n cs.name client_side_type, ss.name server_side_type, pt.name project_type_name, u.username username,\n ARRAY_AGG(DISTINCT c.category) filter (where c.category is not null and mc.is_additional is false) categories,\n ARRAY_AGG(DISTINCT c.category) filter (where c.category is not null and mc.is_additional is true) additional_categories,\n ARRAY_AGG(DISTINCT lo.loader) filter (where lo.loader is not null) loaders,\n ARRAY_AGG(DISTINCT gv.version) filter (where gv.version is not null) versions,\n ARRAY_AGG(DISTINCT mg.image_url) filter (where mg.image_url is not null and mg.featured is false) gallery,\n ARRAY_AGG(DISTINCT mg.image_url) filter (where mg.image_url is not null and mg.featured is true) featured_gallery\n FROM versions v\n INNER JOIN mods m ON v.mod_id = m.id AND m.status = ANY($2)\n LEFT OUTER JOIN mods_categories mc ON joining_mod_id = m.id\n LEFT OUTER JOIN categories c ON mc.joining_category_id = c.id\n LEFT OUTER JOIN game_versions_versions gvv ON gvv.joining_version_id = v.id\n LEFT OUTER JOIN game_versions gv ON gvv.game_version_id = gv.id\n LEFT OUTER JOIN loaders_versions lv ON lv.version_id = v.id\n LEFT OUTER JOIN loaders lo ON lo.id = lv.loader_id\n LEFT OUTER JOIN mods_gallery mg ON mg.mod_id = m.id\n INNER JOIN project_types pt ON pt.id = m.project_type\n INNER JOIN side_types cs ON m.client_side = cs.id\n INNER JOIN side_types ss ON m.server_side = ss.id\n INNER JOIN team_members tm ON tm.team_id = m.team_id AND tm.role = $3 AND tm.accepted = TRUE\n INNER JOIN users u ON tm.user_id = u.id\n WHERE v.status != ANY($1)\n GROUP BY v.id, m.id, cs.id, ss.id, pt.id, u.id;\n " + }, "5944eb30a2bc0381c4d15eb1cf6ccf6e146a54381f2da8ab224960430e951976": { "describe": { "columns": [ diff --git a/src/search/indexing/local_import.rs b/src/search/indexing/local_import.rs index 7c612bee..ca8b0e54 100644 --- a/src/search/indexing/local_import.rs +++ b/src/search/indexing/local_import.rs @@ -12,7 +12,7 @@ pub async fn index_local(pool: PgPool) -> Result, Index Ok( sqlx::query!( " - SELECT m.id id, m.project_type project_type, m.title title, m.description description, m.downloads downloads, m.follows follows, + SELECT m.id id, v.id version_id, m.project_type project_type, m.title title, m.description description, m.downloads downloads, m.follows follows, m.icon_url icon_url, m.published published, m.approved approved, m.updated updated, m.team_id team_id, m.license license, m.slug slug, m.status status_name, m.color color, cs.name client_side_type, ss.name server_side_type, pt.name project_type_name, u.username username, @@ -22,10 +22,10 @@ pub async fn index_local(pool: PgPool) -> Result, Index ARRAY_AGG(DISTINCT gv.version) filter (where gv.version is not null) versions, ARRAY_AGG(DISTINCT mg.image_url) filter (where mg.image_url is not null and mg.featured is false) gallery, ARRAY_AGG(DISTINCT mg.image_url) filter (where mg.image_url is not null and mg.featured is true) featured_gallery - FROM mods m + FROM versions v + INNER JOIN mods m ON v.mod_id = m.id AND m.status = ANY($2) LEFT OUTER JOIN mods_categories mc ON joining_mod_id = m.id LEFT OUTER JOIN categories c ON mc.joining_category_id = c.id - LEFT OUTER JOIN versions v ON v.mod_id = m.id AND v.status != ALL($1) LEFT OUTER JOIN game_versions_versions gvv ON gvv.joining_version_id = v.id LEFT OUTER JOIN game_versions gv ON gvv.game_version_id = gv.id LEFT OUTER JOIN loaders_versions lv ON lv.version_id = v.id @@ -36,8 +36,8 @@ pub async fn index_local(pool: PgPool) -> Result, Index INNER JOIN side_types ss ON m.server_side = ss.id INNER JOIN team_members tm ON tm.team_id = m.team_id AND tm.role = $3 AND tm.accepted = TRUE INNER JOIN users u ON tm.user_id = u.id - WHERE m.status = ANY($2) - GROUP BY m.id, cs.id, ss.id, pt.id, u.id; + WHERE v.status != ANY($1) + GROUP BY v.id, m.id, cs.id, ss.id, pt.id, u.id; ", &*crate::models::projects::VersionStatus::iterator().filter(|x| x.is_hidden()).map(|x| x.to_string()).collect::>(), &*crate::models::projects::ProjectStatus::iterator().filter(|x| x.is_searchable()).map(|x| x.to_string()).collect::>(), @@ -57,6 +57,7 @@ pub async fn index_local(pool: PgPool) -> Result, Index let versions = m.versions.unwrap_or_default(); let project_id: crate::models::projects::ProjectId = ProjectId(m.id).into(); + let version_id: crate::models::projects::ProjectId = ProjectId(m.version_id).into(); let license = match m.license.split(' ').next() { Some(license) => license.to_string(), @@ -69,6 +70,7 @@ pub async fn index_local(pool: PgPool) -> Result, Index }; UploadSearchProject { + version_id: version_id.to_string(), project_id: project_id.to_string(), title: m.title, description: m.description, diff --git a/src/search/indexing/mod.rs b/src/search/indexing/mod.rs index d0cd58d7..bcb5abd5 100644 --- a/src/search/indexing/mod.rs +++ b/src/search/indexing/mod.rs @@ -69,7 +69,7 @@ async fn create_index( }, )) => { // Only create index and set settings if the index doesn't already exist - let task = client.create_index(name, Some("project_id")).await?; + let task = client.create_index(name, Some("version_id")).await?; let task = task.wait_for_completion(client, None, None).await?; let index = task .try_make_index(client) @@ -103,7 +103,7 @@ async fn add_to_index( ) -> Result<(), IndexingError> { for chunk in mods.chunks(MEILISEARCH_CHUNK_SIZE) { index - .add_documents(chunk, Some("project_id")) + .add_documents(chunk, Some("version_id")) .await? .wait_for_completion(client, None, None) .await?; @@ -150,6 +150,7 @@ pub async fn add_projects( fn default_settings() -> Settings { Settings::new() + .with_distinct_attribute("project_id") .with_displayed_attributes(DEFAULT_DISPLAYED_ATTRIBUTES) .with_searchable_attributes(DEFAULT_SEARCHABLE_ATTRIBUTES) .with_sortable_attributes(DEFAULT_SORTABLE_ATTRIBUTES) @@ -161,6 +162,7 @@ fn default_settings() -> Settings { const DEFAULT_DISPLAYED_ATTRIBUTES: &[&str] = &[ "project_id", + "version_id", "project_type", "slug", "author", diff --git a/src/search/mod.rs b/src/search/mod.rs index af8155a1..84dcc1c7 100644 --- a/src/search/mod.rs +++ b/src/search/mod.rs @@ -71,6 +71,7 @@ impl SearchConfig { /// This contains some extra data that is not returned by search results. #[derive(Serialize, Deserialize, Debug, Clone)] pub struct UploadSearchProject { + pub version_id: String, pub project_id: String, pub project_type: String, pub slug: Option, @@ -111,6 +112,7 @@ pub struct SearchResults { #[derive(Serialize, Deserialize, Debug, Clone)] pub struct ResultSearchProject { + pub version_id: String, pub project_id: String, pub project_type: String, pub slug: Option,