Merge branch 'kb_development' into kb_migration

app/chewy/accounts_index.rb

@@ -157,6 +157,6 @@ class AccountsIndex < Chewy::Index
     field(:domain, type: 'keyword', value: ->(account) { account.domain || '' })
     field(:display_name, type: 'text', analyzer: 'verbatim') { field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'verbatim' }
     field(:username, type: 'text', analyzer: 'verbatim', value: ->(account) { [account.username, account.domain].compact.join('@') }) { field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'verbatim' }
-    field(:text, type: 'text', analyzer: 'sudachi_analyzer', value: ->(account) { account.searchable_text }) { field :stemmed, type: 'text', analyzer: 'natural' }
+    field(:text, type: 'text', analyzer: 'sudachi_analyzer', value: ->(account) { account.searchable_text })
   end
 end

app/chewy/public_statuses_index.rb

@@ -144,12 +144,12 @@ class PublicStatusesIndex < Chewy::Index
   index_scope ::Status.unscoped
                       .kept
                       .indexable
-                      .includes(:media_attachments, :preloadable_poll, :preview_cards, :tags)
+                      .includes(:media_attachments, :preloadable_poll, :preview_cards, :tags, :account)
 
   root date_detection: false do
     field(:id, type: 'long')
     field(:account_id, type: 'long')
-    field(:text, type: 'text', analyzer: 'sudachi_analyzer', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'sudachi_analyzer') }
+    field(:text, type: 'text', analyzer: 'sudachi_analyzer', value: ->(status) { status.searchable_text })
     field(:tags, type: 'text', analyzer: 'hashtag', value: ->(status) { status.tags.map(&:display_name) })
     field(:language, type: 'keyword')
     field(:domain, type: 'keyword', value: ->(status) { status.account.domain || '' })

app/chewy/statuses_index.rb

@@ -145,6 +145,7 @@ class StatusesIndex < Chewy::Index
   settings index: index_preset(refresh_interval: '30s', number_of_shards: 5), analysis: Rails.env.test? ? DEVELOPMENT_SETTINGS : PRODUCTION_SETTINGS
 
   index_scope ::Status.unscoped.kept.without_reblogs.includes(
+    :account,
     :media_attachments,
     :preview_cards,
     :local_mentioned,
@@ -160,17 +161,26 @@ class StatusesIndex < Chewy::Index
                            if status.searchability == 'direct'
                              status.searchable_by.empty?
                            else
-                             status.searchability == 'limited' ? status.account.domain.present? : false
+                             status.searchability == 'limited' ? !status.local? : false
                            end
                          }
 
   root date_detection: false do
     field(:id, type: 'long')
     field(:account_id, type: 'long')
-    field(:text, type: 'text', analyzer: 'sudachi_analyzer', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'sudachi_analyzer') }
+    field(:text, type: 'text', analyzer: 'sudachi_analyzer', value: ->(status) { status.searchable_text })
     field(:tags, type: 'text', analyzer: 'hashtag', value: ->(status) { status.tags.map(&:display_name) })
     field(:searchable_by, type: 'long', value: ->(status) { status.searchable_by })
+    field(:mentioned_by, type: 'long', value: ->(status) { status.mentioned_by })
+    field(:favourited_by, type: 'long', value: ->(status) { status.favourited_by })
+    field(:reblogged_by, type: 'long', value: ->(status) { status.reblogged_by })
+    field(:bookmarked_by, type: 'long', value: ->(status) { status.bookmarked_by })
+    field(:bookmark_categoried_by, type: 'long', value: ->(status) { status.bookmark_categoried_by })
+    field(:emoji_reacted_by, type: 'long', value: ->(status) { status.emoji_reacted_by })
+    field(:referenced_by, type: 'long', value: ->(status) { status.referenced_by })
+    field(:voted_by, type: 'long', value: ->(status) { status.voted_by })
     field(:searchability, type: 'keyword', value: ->(status) { status.compute_searchability })
+    field(:visibility, type: 'keyword', value: ->(status) { status.searchable_visibility })
     field(:language, type: 'keyword')
     field(:domain, type: 'keyword', value: ->(status) { status.account.domain || '' })
     field(:properties, type: 'keyword', value: ->(status) { status.searchable_properties })

app/javascript/mastodon/features/compose/components/search.jsx

@@ -59,13 +59,15 @@ class Search extends PureComponent {
   defaultOptions = [
     { label: <><mark>has:</mark> <FormattedList type='disjunction' value={['media', 'poll', 'embed']} /></>, action: e => { e.preventDefault(); this._insertText('has:') } },
     { label: <><mark>is:</mark> <FormattedList type='disjunction' value={['reply', 'sensitive']} /></>, action: e => { e.preventDefault(); this._insertText('is:') } },
+    { label: <><mark>my:</mark> <FormattedList type='disjunction' value={['favourited', 'bookmarked', 'boosted']} /></>, action: e => { e.preventDefault(); this._insertText('my:') } },
     { label: <><mark>language:</mark> <FormattedMessage id='search_popout.language_code' defaultMessage='ISO language code' /></>, action: e => { e.preventDefault(); this._insertText('language:') } },
     { label: <><mark>from:</mark> <FormattedMessage id='search_popout.user' defaultMessage='user' /></>, action: e => { e.preventDefault(); this._insertText('from:') } },
     { label: <><mark>domain:</mark> <FormattedMessage id='search_popout.domain' defaultMessage='domain' /></>, action: e => { e.preventDefault(); this._insertText('domain:') } },
     { label: <><mark>before:</mark> <FormattedMessage id='search_popout.specific_date' defaultMessage='specific date' /></>, action: e => { e.preventDefault(); this._insertText('before:') } },
     { label: <><mark>during:</mark> <FormattedMessage id='search_popout.specific_date' defaultMessage='specific date' /></>, action: e => { e.preventDefault(); this._insertText('during:') } },
     { label: <><mark>after:</mark> <FormattedMessage id='search_popout.specific_date' defaultMessage='specific date' /></>, action: e => { e.preventDefault(); this._insertText('after:') } },
-    { label: <><mark>in:</mark> <FormattedList type='disjunction' value={['all', 'library']} /></>, action: e => { e.preventDefault(); this._insertText('in:') } }
+    { label: <><mark>in:</mark> <FormattedList type='disjunction' value={['all', 'library']} /></>, action: e => { e.preventDefault(); this._insertText('in:') } },
+    { label: <><mark>order:</mark> <FormattedList type='disjunction' value={['desc', 'asc']} /></>, action: e => { e.preventDefault(); this._insertText('order:') } },
   ];
 
   setRef = c => {

app/javascript/mastodon/reducers/bookmark_categories.js

@@ -65,7 +65,10 @@ const appendToBookmarkCategoryStatusesById = (state, bookmarkCategoryId, statuse
 };
 
 const removeStatusFromBookmarkCategoryById = (state, bookmarkCategoryId, status) => {
-  return state.updateIn([bookmarkCategoryId, 'items'], items => items.delete(status));
+  if (state.getIn([bookmarkCategoryId, 'items'])) {
+    return state.updateIn([bookmarkCategoryId, 'items'], items => items.delete(status));
+  }
+  return state;
 };
 
 const removeStatusFromAllBookmarkCategories = (state, status) => {

app/lib/activitypub/activity/create.rb

@@ -493,7 +493,8 @@ def process_references!
     references = @object['references'].nil? ? [] : ActivityPub::FetchReferencesService.new.call(@status, @object['references'])
     quote = @object['quote'] || @object['quoteUrl'] || @object['quoteURL'] || @object['_misskey_quote']
     references << quote if quote
-    ProcessReferencesWorker.perform_async(@status.id, [], references)
+
+    ProcessReferencesService.perform_worker_async(@status, [], references)
   end
 
   def join_group!

app/lib/activitypub/activity/like.rb

@@ -9,7 +9,7 @@ def perform
 
     return if @original_status.nil? || delete_arrived_first?(@json['id']) || reject_favourite?
 
-    if shortcode.nil?
+    if shortcode.nil? || !Setting.enable_emoji_reaction
       process_favourite
     else
       process_emoji_reaction

app/lib/importer/base_importer.rb

@@ -3,9 +3,12 @@
 class Importer::BaseImporter
   # @param [Integer] batch_size
   # @param [Concurrent::ThreadPoolExecutor] executor
-  def initialize(batch_size:, executor:)
+  def initialize(batch_size:, executor:, full: true, from: nil, to: nil)
     @batch_size = batch_size
     @executor   = executor
+    @full       = full
+    @from       = from.to_date if from.present?
+    @to         = to.to_date if to.present?
     @wait_for   = Concurrent::Set.new
   end
 

app/lib/importer/public_statuses_index_importer.rb

@@ -27,6 +27,9 @@ def index
   end
 
   def scope
-    Status.indexable.reorder(nil)
+    to_index = Status.indexable.reorder(nil)
+    to_index = to_index.where('statuses.created_at >= ?', @from) if @from.present?
+    to_index = to_index.where('statuses.created_at < ?', @to) if @to.present?
+    to_index
   end
 end

app/lib/importer/statuses_index_importer.rb

@@ -17,11 +17,13 @@ def import!
 
           bulk = ActiveRecord::Base.connection_pool.with_connection do
             to_index = index.adapter.default_scope.where(id: status_ids)
+            to_index = to_index.where('created_at >= ?', @from) if @from.present?
+            to_index = to_index.where('created_at < ?', @to) if @to.present?
             crutches = Chewy::Index::Crutch::Crutches.new index, to_index
             to_index.map do |object|
               # This is unlikely to happen, but the post may have been
               # un-interacted with since it was queued for indexing
-              if object.searchable_by.empty?
+              if object.searchable_by.empty? && %w(public private).exclude?(object.searchability)
                 deleted += 1
                 { delete: { _id: object.id } }
               else
@@ -49,13 +51,15 @@ def index
   end
 
   def scopes
-    [
+    targets = [
       local_statuses_scope,
       local_mentions_scope,
       local_favourites_scope,
       local_votes_scope,
       local_bookmarks_scope,
     ]
+    targets << remote_searchable_scope if @full
+    targets
   end
 
   def local_mentions_scope
@@ -77,4 +81,8 @@ def local_votes_scope
   def local_statuses_scope
     Status.local.select('"statuses"."id", COALESCE("statuses"."reblog_of_id", "statuses"."id") AS status_id')
   end
+
+  def remote_searchable_scope
+    Status.remote_dynamic_searchability.select('"statuses"."id", COALESCE("statuses"."reblog_of_id", "statuses"."id") AS status_id')
+  end
 end

app/lib/search_query_transformer.rb

@@ -4,13 +4,16 @@ class SearchQueryTransformer < Parslet::Transform
   SUPPORTED_PREFIXES = %w(
     has
     is
+    my
     language
     from
     before
     after
     during
     in
     domain
+    order
+    searchability
   ).freeze
 
   class Query
@@ -34,6 +37,18 @@ def request
       search
     end
 
+    def order_by
+      return @order_by if @order_by
+
+      @order_by = 'desc'
+      order_clauses.each { |clause| @order_by = clause.term }
+      @order_by
+    end
+
+    def valid
+      must_clauses.any? || must_not_clauses.any? || filter_clauses.any?
+    end
+
     private
 
     def clauses_by_operator
@@ -56,6 +71,10 @@ def filter_clauses
       clauses_by_operator.fetch(:filter, [])
     end
 
+    def order_clauses
+      clauses_by_operator.fetch(:order, [])
+    end
+
     def indexes
       case @flags['in']
       when 'library'
@@ -214,6 +233,7 @@ def to_query
       if @term.start_with?('#')
         { match: { tags: { query: @term, operator: 'and' } } }
       else
+        # Memo for checking when manually merge
         # { multi_match: { type: 'most_fields', query: @term, fields: ['text', 'text.stemmed'], operator: 'and' } }
         { match_phrase: { text: { query: @term } } }
       end
@@ -236,11 +256,12 @@ def to_query
   class PrefixClause
     attr_reader :operator, :prefix, :term
 
-    def initialize(prefix, operator, term, options = {})
+    def initialize(prefix, operator, term, options = {}) # rubocop:disable Metrics/CyclomaticComplexity
       @prefix = prefix
       @negated = operator == '-'
       @options = options
       @operator = :filter
+      @statuses_index_only = false
 
       case prefix
       when 'has', 'is'
@@ -274,13 +295,67 @@ def initialize(prefix, operator, term, options = {})
       when 'in'
         @operator = :flag
         @term = term
+      when 'my'
+        @type = :term
+        @term = @options[:current_account]&.id
+        @statuses_index_only = true
+        case term
+        when 'favourited', 'favorited', 'fav'
+          @filter = :favourited_by
+        when 'boosted', 'bt'
+          @filter = :reblogged_by
+        when 'replied', 'mentioned', 're'
+          @filter = :mentioned_by
+        when 'referenced', 'ref'
+          @filter = :referenced_by
+        when 'emoji_reacted', 'stamped', 'stamp'
+          @filter = :emoji_reacted_by
+        when 'bookmarked', 'bm'
+          @filter = :bookmarked_by
+        when 'categoried', 'bmc'
+          @filter = :bookmark_categoried_by
+        when 'voted', 'vote'
+          @filter = :voted_by
+        when 'interacted', 'act'
+          @filter = :searchable_by
+        else
+          raise "Unknown prefix: my:#{term}"
+        end
+      when 'order'
+        @operator = :order
+        @term = case term
+                when 'asc'
+                  term
+                else
+                  'desc'
+                end
+      when 'searchability'
+        @filter = :searchablity
+        @type = :terms
+        @statuses_index_only = true
+        @term = case term
+                when 'public'
+                  %w(public private direct limited)
+                when 'private'
+                  %w(private direct limited)
+                when 'direct'
+                  %w(direct limited)
+                else
+                  %w(limited)
+                end
       else
         raise "Unknown prefix: #{prefix}"
       end
     end
 
     def to_query
-      if @negated
+      if @statuses_index_only
+        if @negated
+          { bool: { must_not: [{ term: { _index: StatusesIndex.index_name } }, { @type => { @filter => @term } }] } }
+        else
+          { bool: { must: [{ term: { _index: StatusesIndex.index_name } }, { @type => { @filter => @term } }] } }
+        end
+      elsif @negated
         { bool: { must_not: { @type => { @filter => @term } } } }
       else
         { @type => { @filter => @term } }
@@ -302,7 +377,7 @@ def account_id_from_term(term)
     end
 
     def domain_from_term(term)
-      return '' if %w(local me).include?(term)
+      return '' if ['local', 'me', Rails.configuration.x.local_domain].include?(term)
 
       term
     end

app/models/bookmark_category_status.rb

@@ -13,6 +13,10 @@
 #
 
 class BookmarkCategoryStatus < ApplicationRecord
+  include Paginable
+
+  update_index('statuses', :status) if Chewy.enabled?
+
   belongs_to :bookmark_category
   belongs_to :status
   belongs_to :bookmark

app/models/concerns/status_search_concern.rb

@@ -5,6 +5,7 @@ module StatusSearchConcern
 
   included do
     scope :indexable, -> { without_reblogs.where(visibility: [:public, :login], searchability: nil).joins(:account).where(account: { indexable: true }) }
+    scope :remote_dynamic_searchability, -> { remote.where(searchability: [:public, :private]) }
   end
 
   def searchable_by
@@ -13,18 +14,52 @@ def searchable_by
 
       ids << account_id if local?
 
-      ids += local_mentioned.pluck(:id)
-      ids += local_favorited.pluck(:id)
-      ids += local_reblogged.pluck(:id)
-      ids += local_bookmarked.pluck(:id)
-      ids += local_emoji_reacted.pluck(:id)
-      ids += local_referenced.pluck(:id)
-      ids += preloadable_poll.local_voters.pluck(:id) if preloadable_poll.present?
+      ids += mentioned_by
+      ids += favourited_by
+      ids += reblogged_by
+      ids += bookmarked_by
+      ids += emoji_reacted_by
+      ids += referenced_by
+      ids += voted_by if preloadable_poll.present?
 
       ids.uniq
     end
   end
 
+  def mentioned_by
+    @mentioned_by ||= local_mentioned.pluck(:id)
+  end
+
+  def favourited_by
+    @favourited_by ||= local_favorited.pluck(:id)
+  end
+
+  def reblogged_by
+    @reblogged_by ||= local_reblogged.pluck(:id)
+  end
+
+  def bookmarked_by
+    @bookmarked_by ||= local_bookmarked.pluck(:id)
+  end
+
+  def bookmark_categoried_by
+    @bookmark_categoried_by ||= local_bookmark_categoried.pluck(:id).uniq
+  end
+
+  def emoji_reacted_by
+    @emoji_reacted_by ||= local_emoji_reacted.pluck(:id)
+  end
+
+  def referenced_by
+    @referenced_by ||= local_referenced.pluck(:id)
+  end
+
+  def voted_by
+    return [] if preloadable_poll.blank?
+
+    @voted_by ||= preloadable_poll.local_voters.pluck(:id)
+  end
+
   def searchable_text
     [
       spoiler_text,