From 86dc5baed311e50af87c66f572611030678af8e3 Mon Sep 17 00:00:00 2001 From: Tim Morgan Date: Tue, 3 Sep 2024 04:10:39 +0200 Subject: [PATCH] Sidekiq -> GoodJob --- Gemfile | 2 +- Gemfile.lock | 21 ++-- README.md | 2 +- config/application.rb | 15 ++- config/environments/cypress.rb | 2 + config/routes.rb | 8 ++ config/sidekiq-cypress.yml | 7 -- db/migrate/20240903015857_create_good_jobs.rb | 102 ++++++++++++++++++ .../20240903015959_add_admin_to_pilots.rb | 7 ++ db/schema.rb | 91 +++++++++++++++- fly.toml | 2 +- 11 files changed, 239 insertions(+), 20 deletions(-) delete mode 100644 config/sidekiq-cypress.yml create mode 100644 db/migrate/20240903015857_create_good_jobs.rb create mode 100644 db/migrate/20240903015959_add_admin_to_pilots.rb diff --git a/Gemfile b/Gemfile index f0bfcb8..cf26aac 100644 --- a/Gemfile +++ b/Gemfile @@ -13,10 +13,10 @@ gem "responders" # FRAMEWORK gem "devise" gem "devise-jwt" +gem "good_job" gem "kredis" gem "rack-cors" gem "redis" -gem "sidekiq" # MODELS gem "pg" diff --git a/Gemfile.lock b/Gemfile.lock index 0867ffc..040e739 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -120,14 +120,26 @@ GEM concurrent-ruby (~> 1.0) zeitwerk (~> 2.6) erubi (1.13.0) + et-orbi (1.2.11) + tzinfo factory_bot (6.4.6) activesupport (>= 5.0.0) factory_bot_rails (6.4.3) factory_bot (~> 6.4) railties (>= 5.0.0) ffaker (2.23.0) + fugit (1.11.1) + et-orbi (~> 1, >= 1.2.11) + raabro (~> 1.4) globalid (1.2.1) activesupport (>= 6.1) + good_job (4.2.1) + activejob (>= 6.1.0) + activerecord (>= 6.1.0) + concurrent-ruby (>= 1.3.1) + fugit (>= 1.11.0) + railties (>= 6.1.0) + thor (>= 1.0.0) hashdiff (1.1.1) i18n (1.14.5) concurrent-ruby (~> 1.0) @@ -180,6 +192,7 @@ GEM public_suffix (6.0.1) puma (6.4.2) nio4r (~> 2.0) + raabro (1.4.0) racc (1.8.1) rack (3.1.7) rack-cors (2.0.2) @@ -257,12 +270,6 @@ GEM rspec-support (~> 3.13) rspec-support (3.13.1) securerandom (0.3.1) - sidekiq (7.3.1) - concurrent-ruby (< 2) - connection_pool (>= 2.3.0) - logger - rack (>= 2.2.4) - redis-client (>= 0.22.2) stringio (3.1.1) strscan (3.1.0) thor (1.3.2) @@ -304,6 +311,7 @@ DEPENDENCIES dockerfile-rails factory_bot_rails ffaker + good_job jbuilder json_expressions kredis @@ -316,7 +324,6 @@ DEPENDENCIES redis responders rspec-rails - sidekiq webmock yard diff --git a/README.md b/README.md index 93752a8..1c4b4ee 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ such as: ``` backend: cd Backend && rvm 3.3.4@flyweight exec rails server frontend: cd Frontend && yarn dev -jobs: cd Backend && rvm 3.3.4@flyweight exec bundle exec sidekiq -C config/sidekiq.yml +jobs: cd Backend && rvm 3.3.4@flyweight exec bundle exec good_job start cable: cd Backend && rvm 3.3.4@flyweight exec ./bin/cable ``` diff --git a/config/application.rb b/config/application.rb index 35a1f35..4fdac76 100644 --- a/config/application.rb +++ b/config/application.rb @@ -48,9 +48,20 @@ class Application < Rails::Application # Skip views, helpers and assets when generating a new resource. config.api_only = true - # Use a real queuing backend for Active Job (and separate queues per environment). - config.active_job.queue_adapter = :sidekiq + config.active_job.queue_adapter = :good_job config.active_job.queue_name_prefix = "flyweight_#{Rails.env}" + config.active_job.queue_adapter = :good_job + config.good_job.max_threads = 2 + config.good_job.poll_interval = 30 # seconds + config.good_job.enable_cron = true + config.good_job.dashboard_default_locale = :en + config.good_job.queues = "flyweight_#{Rails.env}_default" + + # for GoodJob dashboard + config.middleware.use Rack::MethodOverride + config.middleware.use ActionDispatch::Flash + config.middleware.use ActionDispatch::Cookies + config.middleware.use ActionDispatch::Session::CookieStore config.generators do |g| g.test_framework :rspec, fixture: true, views: false diff --git a/config/environments/cypress.rb b/config/environments/cypress.rb index 984c282..067d643 100644 --- a/config/environments/cypress.rb +++ b/config/environments/cypress.rb @@ -65,4 +65,6 @@ # Uncomment if you wish to allow Action Cable access from any origin. # config.action_cable.disable_request_forgery_protection = true + + config.good_job.poll_interval = 1 end diff --git a/config/routes.rb b/config/routes.rb index 6f3564a..76a3c66 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -30,6 +30,14 @@ get "__cypress__/last_email" => Cypress::LastEmail.new end + if Rails.env.production? + authenticate :pilot, -> { _1.admin? } do + mount GoodJob::Engine => "good_job" + end + else + mount GoodJob::Engine => "good_job" + end + # Reveal health status on /up that returns 200 if the app boots with no exceptions, otherwise 500. # Can be used by load balancers and uptime monitors to verify that the app is live. get "up" => "rails/health#show", as: :rails_health_check diff --git a/config/sidekiq-cypress.yml b/config/sidekiq-cypress.yml deleted file mode 100644 index 3a64537..0000000 --- a/config/sidekiq-cypress.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -:concurrency: 1 -:pidfile: tmp/pids/sidekiq.pid -:queues: -- flyweight_cypress_default -- flyweight_cypress_active_storage_analysis -- flyweight_cypress_active_storage_purge diff --git a/db/migrate/20240903015857_create_good_jobs.rb b/db/migrate/20240903015857_create_good_jobs.rb new file mode 100644 index 0000000..cdfd5d9 --- /dev/null +++ b/db/migrate/20240903015857_create_good_jobs.rb @@ -0,0 +1,102 @@ +# frozen_string_literal: true + +class CreateGoodJobs < ActiveRecord::Migration[7.2] + def change + # Uncomment for Postgres v12 or earlier to enable gen_random_uuid() support + # enable_extension 'pgcrypto' + + create_table :good_jobs, id: :uuid do |t| + t.text :queue_name + t.integer :priority + t.jsonb :serialized_params + t.datetime :scheduled_at + t.datetime :performed_at + t.datetime :finished_at + t.text :error + + t.timestamps + + t.uuid :active_job_id + t.text :concurrency_key + t.text :cron_key + t.uuid :retried_good_job_id + t.datetime :cron_at + + t.uuid :batch_id + t.uuid :batch_callback_id + + t.boolean :is_discrete + t.integer :executions_count + t.text :job_class + t.integer :error_event, limit: 2 + t.text :labels, array: true + t.uuid :locked_by_id + t.datetime :locked_at + end + + create_table :good_job_batches, id: :uuid do |t| + t.timestamps + t.text :description + t.jsonb :serialized_properties + t.text :on_finish + t.text :on_success + t.text :on_discard + t.text :callback_queue_name + t.integer :callback_priority + t.datetime :enqueued_at + t.datetime :discarded_at + t.datetime :finished_at + end + + create_table :good_job_executions, id: :uuid do |t| + t.timestamps + + t.uuid :active_job_id, null: false + t.text :job_class + t.text :queue_name + t.jsonb :serialized_params + t.datetime :scheduled_at + t.datetime :finished_at + t.text :error + t.integer :error_event, limit: 2 + t.text :error_backtrace, array: true + t.uuid :process_id + t.interval :duration + end + + create_table :good_job_processes, id: :uuid do |t| + t.timestamps + t.jsonb :state + t.integer :lock_type, limit: 2 + end + + create_table :good_job_settings, id: :uuid do |t| + t.timestamps + t.text :key + t.jsonb :value + t.index :key, unique: true + end + + add_index :good_jobs, :scheduled_at, where: "(finished_at IS NULL)", name: :index_good_jobs_on_scheduled_at + add_index :good_jobs, %i[queue_name scheduled_at], where: "(finished_at IS NULL)", name: :index_good_jobs_on_queue_name_and_scheduled_at + add_index :good_jobs, %i[active_job_id created_at], name: :index_good_jobs_on_active_job_id_and_created_at + add_index :good_jobs, :concurrency_key, where: "(finished_at IS NULL)", name: :index_good_jobs_on_concurrency_key_when_unfinished + add_index :good_jobs, %i[cron_key created_at], where: "(cron_key IS NOT NULL)", name: :index_good_jobs_on_cron_key_and_created_at_cond + add_index :good_jobs, %i[cron_key cron_at], where: "(cron_key IS NOT NULL)", unique: true, name: :index_good_jobs_on_cron_key_and_cron_at_cond + add_index :good_jobs, %i[finished_at], where: "retried_good_job_id IS NULL AND finished_at IS NOT NULL", name: :index_good_jobs_jobs_on_finished_at + add_index :good_jobs, %i[priority created_at], order: {priority: "DESC NULLS LAST", created_at: :asc}, + where: "finished_at IS NULL", name: :index_good_jobs_jobs_on_priority_created_at_when_unfinished + add_index :good_jobs, %i[priority created_at], order: {priority: "ASC NULLS LAST", created_at: :asc}, + where: "finished_at IS NULL", name: :index_good_job_jobs_for_candidate_lookup + add_index :good_jobs, %i[batch_id], where: "batch_id IS NOT NULL" + add_index :good_jobs, %i[batch_callback_id], where: "batch_callback_id IS NOT NULL" + add_index :good_jobs, :labels, using: :gin, where: "(labels IS NOT NULL)", name: :index_good_jobs_on_labels + + add_index :good_job_executions, %i[active_job_id created_at], name: :index_good_job_executions_on_active_job_id_and_created_at + add_index :good_jobs, %i[priority scheduled_at], order: {priority: "ASC NULLS LAST", scheduled_at: :asc}, + where: "finished_at IS NULL AND locked_by_id IS NULL", name: :index_good_jobs_on_priority_scheduled_at_unfinished_unlocked + add_index :good_jobs, :locked_by_id, + where: "locked_by_id IS NOT NULL", name: "index_good_jobs_on_locked_by_id" + add_index :good_job_executions, %i[process_id created_at], name: :index_good_job_executions_on_process_id_and_created_at + end +end diff --git a/db/migrate/20240903015959_add_admin_to_pilots.rb b/db/migrate/20240903015959_add_admin_to_pilots.rb new file mode 100644 index 0000000..302af34 --- /dev/null +++ b/db/migrate/20240903015959_add_admin_to_pilots.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class AddAdminToPilots < ActiveRecord::Migration[7.2] + def change + add_column :pilots, :admin, :boolean, default: false, null: false + end +end diff --git a/db/schema.rb b/db/schema.rb index 4849cb2..18d7918 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2022_03_07_051658) do +ActiveRecord::Schema[7.2].define(version: 2024_09_03_015959) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -25,6 +25,94 @@ t.index ["uuid"], name: "index_flights_on_uuid", unique: true end + create_table "good_job_batches", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.text "description" + t.jsonb "serialized_properties" + t.text "on_finish" + t.text "on_success" + t.text "on_discard" + t.text "callback_queue_name" + t.integer "callback_priority" + t.datetime "enqueued_at" + t.datetime "discarded_at" + t.datetime "finished_at" + end + + create_table "good_job_executions", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.uuid "active_job_id", null: false + t.text "job_class" + t.text "queue_name" + t.jsonb "serialized_params" + t.datetime "scheduled_at" + t.datetime "finished_at" + t.text "error" + t.integer "error_event", limit: 2 + t.text "error_backtrace", array: true + t.uuid "process_id" + t.interval "duration" + t.index ["active_job_id", "created_at"], name: "index_good_job_executions_on_active_job_id_and_created_at" + t.index ["process_id", "created_at"], name: "index_good_job_executions_on_process_id_and_created_at" + end + + create_table "good_job_processes", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.jsonb "state" + t.integer "lock_type", limit: 2 + end + + create_table "good_job_settings", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.text "key" + t.jsonb "value" + t.index ["key"], name: "index_good_job_settings_on_key", unique: true + end + + create_table "good_jobs", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.text "queue_name" + t.integer "priority" + t.jsonb "serialized_params" + t.datetime "scheduled_at" + t.datetime "performed_at" + t.datetime "finished_at" + t.text "error" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.uuid "active_job_id" + t.text "concurrency_key" + t.text "cron_key" + t.uuid "retried_good_job_id" + t.datetime "cron_at" + t.uuid "batch_id" + t.uuid "batch_callback_id" + t.boolean "is_discrete" + t.integer "executions_count" + t.text "job_class" + t.integer "error_event", limit: 2 + t.text "labels", array: true + t.uuid "locked_by_id" + t.datetime "locked_at" + t.index ["active_job_id", "created_at"], name: "index_good_jobs_on_active_job_id_and_created_at" + t.index ["batch_callback_id"], name: "index_good_jobs_on_batch_callback_id", where: "(batch_callback_id IS NOT NULL)" + t.index ["batch_id"], name: "index_good_jobs_on_batch_id", where: "(batch_id IS NOT NULL)" + t.index ["concurrency_key"], name: "index_good_jobs_on_concurrency_key_when_unfinished", where: "(finished_at IS NULL)" + t.index ["cron_key", "created_at"], name: "index_good_jobs_on_cron_key_and_created_at_cond", where: "(cron_key IS NOT NULL)" + t.index ["cron_key", "cron_at"], name: "index_good_jobs_on_cron_key_and_cron_at_cond", unique: true, where: "(cron_key IS NOT NULL)" + t.index ["finished_at"], name: "index_good_jobs_jobs_on_finished_at", where: "((retried_good_job_id IS NULL) AND (finished_at IS NOT NULL))" + t.index ["labels"], name: "index_good_jobs_on_labels", where: "(labels IS NOT NULL)", using: :gin + t.index ["locked_by_id"], name: "index_good_jobs_on_locked_by_id", where: "(locked_by_id IS NOT NULL)" + t.index ["priority", "created_at"], name: "index_good_job_jobs_for_candidate_lookup", where: "(finished_at IS NULL)" + t.index ["priority", "created_at"], name: "index_good_jobs_jobs_on_priority_created_at_when_unfinished", order: { priority: "DESC NULLS LAST" }, where: "(finished_at IS NULL)" + t.index ["priority", "scheduled_at"], name: "index_good_jobs_on_priority_scheduled_at_unfinished_unlocked", where: "((finished_at IS NULL) AND (locked_by_id IS NULL))" + t.index ["queue_name", "scheduled_at"], name: "index_good_jobs_on_queue_name_and_scheduled_at", where: "(finished_at IS NULL)" + t.index ["scheduled_at"], name: "index_good_jobs_on_scheduled_at", where: "(finished_at IS NULL)" + end + create_table "jwt_denylist", force: :cascade do |t| t.string "jti", null: false t.datetime "exp", null: false @@ -53,6 +141,7 @@ t.datetime "remember_created_at" t.datetime "created_at", null: false t.datetime "updated_at", null: false + t.boolean "admin", default: false, null: false t.index ["email"], name: "index_pilots_on_email", unique: true t.index ["reset_password_token"], name: "index_pilots_on_reset_password_token", unique: true end diff --git a/fly.toml b/fly.toml index ecfc6ba..2fffe62 100644 --- a/fly.toml +++ b/fly.toml @@ -14,7 +14,7 @@ console_command = '/rails/bin/rails console' [processes] app = './bin/rails server' - sidekiq = 'bundle exec sidekiq' + good_job = 'bundle exec good_job start' [http_service] internal_port = 3000