From 0ffd8acacd749599006744da0cc9b6d9672fc458 Mon Sep 17 00:00:00 2001 From: KMY Date: Sat, 16 Sep 2023 10:53:09 +0900 Subject: [PATCH] Add search deploy date options --- app/lib/importer/base_importer.rb | 4 +++- app/lib/importer/public_statuses_index_importer.rb | 5 ++++- app/lib/importer/statuses_index_importer.rb | 2 ++ lib/mastodon/cli/search.rb | 4 +++- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/app/lib/importer/base_importer.rb b/app/lib/importer/base_importer.rb index ef8c8b38d4191f..96cacfc07b9668 100644 --- a/app/lib/importer/base_importer.rb +++ b/app/lib/importer/base_importer.rb @@ -3,10 +3,12 @@ class Importer::BaseImporter # @param [Integer] batch_size # @param [Concurrent::ThreadPoolExecutor] executor - def initialize(batch_size:, executor:, full: true) + def initialize(batch_size:, executor:, full: true, from: nil, to: nil) @batch_size = batch_size @executor = executor @full = full + @from = from.to_date if from.present? + @to = to.to_date if to.present? @wait_for = Concurrent::Set.new end diff --git a/app/lib/importer/public_statuses_index_importer.rb b/app/lib/importer/public_statuses_index_importer.rb index 7dfe9988683db5..02134dc2517052 100644 --- a/app/lib/importer/public_statuses_index_importer.rb +++ b/app/lib/importer/public_statuses_index_importer.rb @@ -27,6 +27,9 @@ def index end def scope - Status.indexable.reorder(nil) + to_index = Status.indexable.reorder(nil) + to_index = to_index.where('statuses.created_at >= ?', @from) if @from.present? + to_index = to_index.where('statuses.created_at < ?', @to) if @to.present? + to_index end end diff --git a/app/lib/importer/statuses_index_importer.rb b/app/lib/importer/statuses_index_importer.rb index bb74bb8d89179a..ca4b01d0e86de6 100644 --- a/app/lib/importer/statuses_index_importer.rb +++ b/app/lib/importer/statuses_index_importer.rb @@ -17,6 +17,8 @@ def import! bulk = ActiveRecord::Base.connection_pool.with_connection do to_index = index.adapter.default_scope.where(id: status_ids) + to_index = to_index.where('created_at >= ?', @from) if @from.present? + to_index = to_index.where('created_at < ?', @to) if @to.present? crutches = Chewy::Index::Crutch::Crutches.new index, to_index to_index.map do |object| # This is unlikely to happen, but the post may have been diff --git a/lib/mastodon/cli/search.rb b/lib/mastodon/cli/search.rb index 57e28951304c0e..af70b6c2c72a95 100644 --- a/lib/mastodon/cli/search.rb +++ b/lib/mastodon/cli/search.rb @@ -21,6 +21,8 @@ class Search < Base option :clean, type: :boolean, default: true, desc: 'Remove outdated documents from the index' option :reset_chewy, type: :boolean, default: false, desc: "Reset Chewy's internal index" option :full, type: :boolean, default: false, desc: 'Import full data over Mastodon default importer' + option :from, type: :string, default: nil, desc: 'Statuses start date' + option :to, type: :string, default: nil, desc: 'Statuses end date' desc 'deploy', 'Create or upgrade Elasticsearch indices and populate them' long_desc <<~LONG_DESC If Elasticsearch is empty, this command will create the necessary indices @@ -42,7 +44,7 @@ def deploy end pool = Concurrent::FixedThreadPool.new(options[:concurrency], max_queue: options[:concurrency] * 10) - importers = indices.index_with { |index| "Importer::#{index.name}Importer".constantize.new(batch_size: options[:batch_size], executor: pool, full: options[:full]) } + importers = indices.index_with { |index| "Importer::#{index.name}Importer".constantize.new(batch_size: options[:batch_size], executor: pool, full: options[:full], from: options[:from], to: options[:to]) } progress = ProgressBar.create(total: nil, format: '%t%c/%u |%b%i| %e (%r docs/s)', autofinish: false) Chewy::Stash::Specification.reset! if options[:reset_chewy]