From f023804e07e789e013a8fe3495c8b3d685bd8dbf Mon Sep 17 00:00:00 2001 From: Boxedfruits <34636700+BoxedFruits@users.noreply.github.com> Date: Sat, 6 Feb 2021 18:06:52 -0500 Subject: [PATCH 1/3] Working on fixing search with weighted columns --- app/controllers/search_controller.rb | 2 +- app/models/post.rb | 2 +- .../20210123211307_add_fulltext_index_to_posts_title.rb | 5 +++++ db/schema.rb | 3 ++- 4 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 db/migrate/20210123211307_add_fulltext_index_to_posts_title.rb diff --git a/app/controllers/search_controller.rb b/app/controllers/search_controller.rb index b5e93aa71..dd5503ce6 100644 --- a/app/controllers/search_controller.rb +++ b/app/controllers/search_controller.rb @@ -8,7 +8,7 @@ def search .paginate(page: params[:page], per_page: 25) if search_data[:search].present? - posts.search(search_data[:search]).user_sort({ term: params[:sort], default: :search_score }, + posts.search(search_data[:search]).user_sort({ term: params[:sort] }, relevance: :search_score, score: :score, age: :created_at) else posts.user_sort({ term: params[:sort], default: :score }, diff --git a/app/models/post.rb b/app/models/post.rb index dac01f8b7..2d5d2b870 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -56,7 +56,7 @@ class Post < ApplicationRecord after_save :recalc_score def self.search(term) - match_search term, posts: :body_markdown + match_search term, posts: [:title, :body_markdown] end # Double-define: initial definitions are less efficient, so if we have a record of the post type we'll diff --git a/db/migrate/20210123211307_add_fulltext_index_to_posts_title.rb b/db/migrate/20210123211307_add_fulltext_index_to_posts_title.rb new file mode 100644 index 000000000..323cdde67 --- /dev/null +++ b/db/migrate/20210123211307_add_fulltext_index_to_posts_title.rb @@ -0,0 +1,5 @@ +class AddFulltextIndexToPostsTitle < ActiveRecord::Migration[5.2] + def change + add_index :posts, :title, type: :fulltext + end +end diff --git a/db/schema.rb b/db/schema.rb index 7f65c6e8f..d816659d6 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_01_12_225651) do +ActiveRecord::Schema.define(version: 2021_01_23_211307) do create_table "abilities", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci", force: :cascade do |t| t.bigint "community_id" @@ -376,6 +376,7 @@ t.index ["post_type_id"], name: "index_posts_on_post_type_id" t.index ["score"], name: "index_posts_on_score" t.index ["tags_cache"], name: "index_posts_on_tags_cache" + t.index ["title"], name: "index_posts_on_title", type: :fulltext t.index ["upvote_count"], name: "index_posts_on_upvote_count" t.index ["user_id"], name: "index_posts_on_user_id" end From 1e284cf4b5430614fd295a601c4e6fec6f4a5b4c Mon Sep 17 00:00:00 2001 From: Boxedfruits <34636700+BoxedFruits@users.noreply.github.com> Date: Sat, 6 Feb 2021 18:08:59 -0500 Subject: [PATCH 2/3] activeRecord changes --- app/models/application_record.rb | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/app/models/application_record.rb b/app/models/application_record.rb index f5a09f024..1366f798d 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -8,7 +8,12 @@ def self.fuzzy_search(term, **cols) def self.match_search(term, **cols) sanitized = sanitize_for_search term, **cols - select(Arel.sql("`#{table_name}`.*, #{sanitized} AS search_score")).where(sanitized) + relevancyMultiplier = 0 + mappedCols = sanitized.map{ |val| "#{val} AS search_score_#{sanitized.find_index(val)}" }.join(", ") + whereStatement = sanitized.map{ |val| "#{val}" }.join(" OR") + Rails.logger.debug(cols.values[0]) + byebug + select(Arel.sql("`#{table_name}`.*, #{mappedCols}")).where(whereStatement).order('search_score_0*5').order(' search_score_1 ') end def self.sanitize_name(name) @@ -26,13 +31,15 @@ def attributes_print def self.sanitize_for_search(term, **cols) cols = cols.map do |k, v| if v.is_a?(Array) - v.map { |vv| "#{sanitize_name k}.#{sanitize_name vv}" }.join(', ') + #need to sanitize here + v.map { |vv| " MATCH #{sanitize_name k}.#{sanitize_name vv} AGAINST ('#{term}' IN BOOLEAN MODE)" } else "#{sanitize_name k}.#{sanitize_name v}" end - end.join(', ') + end - ActiveRecord::Base.send(:sanitize_sql_array, ["MATCH (#{cols}) AGAINST (? IN BOOLEAN MODE)", term]) + return cols[0] + # ActiveRecord::Base.send(:sanitize_sql_array, cols) end def self.sanitize_sql_in(ary) @@ -113,4 +120,4 @@ def user_sort(term_opts, **field_mappings) end ActiveRecord::Base.extend UserSortable -klasses.each { |klass| klass.send(:include, UserSortable) } +klasses.each { |klass| klass.send(:include, UserSortable) } \ No newline at end of file From f5723353bebf58fa3db439ac98501c0b6fdd272b Mon Sep 17 00:00:00 2001 From: Boxedfruits <34636700+BoxedFruits@users.noreply.github.com> Date: Sun, 7 Feb 2021 11:20:35 -0500 Subject: [PATCH 3/3] working on search weights --- app/models/application_record.rb | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/app/models/application_record.rb b/app/models/application_record.rb index 1366f798d..e5787022c 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -8,12 +8,11 @@ def self.fuzzy_search(term, **cols) def self.match_search(term, **cols) sanitized = sanitize_for_search term, **cols - relevancyMultiplier = 0 - mappedCols = sanitized.map{ |val| "#{val} AS search_score_#{sanitized.find_index(val)}" }.join(", ") - whereStatement = sanitized.map{ |val| "#{val}" }.join(" OR") - Rails.logger.debug(cols.values[0]) - byebug - select(Arel.sql("`#{table_name}`.*, #{mappedCols}")).where(whereStatement).order('search_score_0*5').order(' search_score_1 ') + + mappedCols = sanitized.map { |val| "#{val} AS search_score_#{sanitized.find_index(val)}" }.join(', ') + whereClause = sanitized.map { |val| val.to_s }.join(' OR') + + select(Arel.sql("`#{table_name}`.*, #{mappedCols}")).where(whereClause).order('search_score_0 * 2 + search_score_1 * 1 DESC') end def self.sanitize_name(name) @@ -31,15 +30,16 @@ def attributes_print def self.sanitize_for_search(term, **cols) cols = cols.map do |k, v| if v.is_a?(Array) - #need to sanitize here - v.map { |vv| " MATCH #{sanitize_name k}.#{sanitize_name vv} AGAINST ('#{term}' IN BOOLEAN MODE)" } + v.map do |vv| + #prob can do this just once at end of map + ActiveRecord::Base.sanitize_sql([" MATCH #{sanitize_name k}.#{sanitize_name vv} AGAINST (? IN BOOLEAN MODE)", term]) + end else "#{sanitize_name k}.#{sanitize_name v}" end end - return cols[0] - # ActiveRecord::Base.send(:sanitize_sql_array, cols) + cols[0] end def self.sanitize_sql_in(ary) @@ -120,4 +120,4 @@ def user_sort(term_opts, **field_mappings) end ActiveRecord::Base.extend UserSortable -klasses.each { |klass| klass.send(:include, UserSortable) } \ No newline at end of file +klasses.each { |klass| klass.send(:include, UserSortable) }