Skip to content

Commit

Permalink
Add search deploy date options
Browse files Browse the repository at this point in the history
  • Loading branch information
kmycode committed Sep 16, 2023
1 parent 173b9f4 commit 0ffd8ac
Show file tree
Hide file tree
Showing 4 changed files with 12 additions and 3 deletions.
4 changes: 3 additions & 1 deletion app/lib/importer/base_importer.rb
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,12 @@
class Importer::BaseImporter
# @param [Integer] batch_size
# @param [Concurrent::ThreadPoolExecutor] executor
def initialize(batch_size:, executor:, full: true)
def initialize(batch_size:, executor:, full: true, from: nil, to: nil)
@batch_size = batch_size
@executor = executor
@full = full
@from = from.to_date if from.present?
@to = to.to_date if to.present?
@wait_for = Concurrent::Set.new
end

Expand Down
5 changes: 4 additions & 1 deletion app/lib/importer/public_statuses_index_importer.rb
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ def index
end

def scope
Status.indexable.reorder(nil)
to_index = Status.indexable.reorder(nil)
to_index = to_index.where('statuses.created_at >= ?', @from) if @from.present?
to_index = to_index.where('statuses.created_at < ?', @to) if @to.present?
to_index
end
end
2 changes: 2 additions & 0 deletions app/lib/importer/statuses_index_importer.rb
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ def import!

bulk = ActiveRecord::Base.connection_pool.with_connection do
to_index = index.adapter.default_scope.where(id: status_ids)
to_index = to_index.where('created_at >= ?', @from) if @from.present?
to_index = to_index.where('created_at < ?', @to) if @to.present?
crutches = Chewy::Index::Crutch::Crutches.new index, to_index
to_index.map do |object|
# This is unlikely to happen, but the post may have been
Expand Down
4 changes: 3 additions & 1 deletion lib/mastodon/cli/search.rb
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ class Search < Base
option :clean, type: :boolean, default: true, desc: 'Remove outdated documents from the index'
option :reset_chewy, type: :boolean, default: false, desc: "Reset Chewy's internal index"
option :full, type: :boolean, default: false, desc: 'Import full data over Mastodon default importer'
option :from, type: :string, default: nil, desc: 'Statuses start date'
option :to, type: :string, default: nil, desc: 'Statuses end date'
desc 'deploy', 'Create or upgrade Elasticsearch indices and populate them'
long_desc <<~LONG_DESC
If Elasticsearch is empty, this command will create the necessary indices
Expand All @@ -42,7 +44,7 @@ def deploy
end

pool = Concurrent::FixedThreadPool.new(options[:concurrency], max_queue: options[:concurrency] * 10)
importers = indices.index_with { |index| "Importer::#{index.name}Importer".constantize.new(batch_size: options[:batch_size], executor: pool, full: options[:full]) }
importers = indices.index_with { |index| "Importer::#{index.name}Importer".constantize.new(batch_size: options[:batch_size], executor: pool, full: options[:full], from: options[:from], to: options[:to]) }
progress = ProgressBar.create(total: nil, format: '%t%c/%u |%b%i| %e (%r docs/s)', autofinish: false)

Chewy::Stash::Specification.reset! if options[:reset_chewy]
Expand Down

0 comments on commit 0ffd8ac

Please sign in to comment.