From fef262e79563b152a716bf5a87c2a2d5fb6df74a Mon Sep 17 00:00:00 2001 From: Bhumi Shah Date: Thu, 30 Mar 2023 10:29:45 -0500 Subject: [PATCH 1/6] job to purge outgoing webhooks data that is older thann days --- .../app/jobs/webhooks/outgoing/purge_job.rb | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 bullet_train-outgoing_webhooks/app/jobs/webhooks/outgoing/purge_job.rb diff --git a/bullet_train-outgoing_webhooks/app/jobs/webhooks/outgoing/purge_job.rb b/bullet_train-outgoing_webhooks/app/jobs/webhooks/outgoing/purge_job.rb new file mode 100644 index 000000000..fcd848b04 --- /dev/null +++ b/bullet_train-outgoing_webhooks/app/jobs/webhooks/outgoing/purge_job.rb @@ -0,0 +1,23 @@ +class Webhooks::Outgoing::PurgeJob < ApplicationJob + queue_as :default + + def perform(days_old = 90) + purge("Webhooks::Outgoing::Event", days_old) + purge("Webhooks::Outgoing::Delivery", days_old) + purge("Webhooks::Outgoing::DeliveryAttempt", days_old) + + # events = "Webhooks::Outgoing::Event".constantize.where("created_at < ?", days_old.days.ago) + # events.destroy_all + + # deliveries = "Webhooks::Outgoing::Delivery".constantize.where("created_at < ?", days_old.days.ago) + # deliveries.destroy_all + + # delivery_attempts = "Webhooks::Outgoing::DeliveryAttempt".constantize.where("created_at < ?", days_old.days.ago) + # delivery_attempts.destroy_all + end + + def purge(table_name, days_old) + records = table_name.constantize.where("created_at < ?", days_old.days.ago) + records.destroy_all + end +end From 61672eb1ea8984d7e98f1feb1d9824217c48b19f Mon Sep 17 00:00:00 2001 From: Bhumi Shah Date: Fri, 31 Mar 2023 09:40:33 -0500 Subject: [PATCH 2/6] testing the purge job and cleanup --- .../app/jobs/webhooks/outgoing/purge_job.rb | 9 --------- 1 file changed, 9 deletions(-) diff --git a/bullet_train-outgoing_webhooks/app/jobs/webhooks/outgoing/purge_job.rb b/bullet_train-outgoing_webhooks/app/jobs/webhooks/outgoing/purge_job.rb index fcd848b04..3a0f84606 100644 --- a/bullet_train-outgoing_webhooks/app/jobs/webhooks/outgoing/purge_job.rb +++ b/bullet_train-outgoing_webhooks/app/jobs/webhooks/outgoing/purge_job.rb @@ -5,15 +5,6 @@ def perform(days_old = 90) purge("Webhooks::Outgoing::Event", days_old) purge("Webhooks::Outgoing::Delivery", days_old) purge("Webhooks::Outgoing::DeliveryAttempt", days_old) - - # events = "Webhooks::Outgoing::Event".constantize.where("created_at < ?", days_old.days.ago) - # events.destroy_all - - # deliveries = "Webhooks::Outgoing::Delivery".constantize.where("created_at < ?", days_old.days.ago) - # deliveries.destroy_all - - # delivery_attempts = "Webhooks::Outgoing::DeliveryAttempt".constantize.where("created_at < ?", days_old.days.ago) - # delivery_attempts.destroy_all end def purge(table_name, days_old) From ec73d20653adc7832c3db7a99b0d70b8d93cd165 Mon Sep 17 00:00:00 2001 From: Bhumi Shah Date: Fri, 31 Mar 2023 10:36:21 -0500 Subject: [PATCH 3/6] udpate outgoing webhooks docs about purge job --- bullet_train/docs/webhooks/outgoing.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bullet_train/docs/webhooks/outgoing.md b/bullet_train/docs/webhooks/outgoing.md index 99285a256..d7c9b6934 100644 --- a/bullet_train/docs/webhooks/outgoing.md +++ b/bullet_train/docs/webhooks/outgoing.md @@ -27,6 +27,8 @@ payment.generate_webhook(:succeeded) ## Delivery Webhooks are delivered asyncronously in a background job by default. If the resulting HTTP request results in a status code other than those in the 2XX series, it will be considered a failed attempt and delivery will be reattempted a number of times. +All of the delivery attempts and events are stored in the database. There is a background job to purge records in these tables as they can grow large in size over time. You can schedule this job to purge records older than 90 days by default. + ## Future Plans - Allow users to filter webhooks to be generated by a given parent model. For example, they should be able to subscribe to `post.created`, but only for `Post` objects created within a certain `Project`. - Integrate [Hammerstone Refine](https://hammerstone.dev) to allow even greater configurability for filtering webhooks. From 486c8f4a8d6c66d0a680742f0e1594632c20b6e7 Mon Sep 17 00:00:00 2001 From: Andrew Culver Date: Sun, 2 Apr 2023 21:44:44 -0700 Subject: [PATCH 4/6] Update purge_job.rb --- .../app/jobs/webhooks/outgoing/purge_job.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bullet_train-outgoing_webhooks/app/jobs/webhooks/outgoing/purge_job.rb b/bullet_train-outgoing_webhooks/app/jobs/webhooks/outgoing/purge_job.rb index 3a0f84606..9b881389a 100644 --- a/bullet_train-outgoing_webhooks/app/jobs/webhooks/outgoing/purge_job.rb +++ b/bullet_train-outgoing_webhooks/app/jobs/webhooks/outgoing/purge_job.rb @@ -2,13 +2,13 @@ class Webhooks::Outgoing::PurgeJob < ApplicationJob queue_as :default def perform(days_old = 90) - purge("Webhooks::Outgoing::Event", days_old) - purge("Webhooks::Outgoing::Delivery", days_old) - purge("Webhooks::Outgoing::DeliveryAttempt", days_old) + purge(Webhooks::Outgoing::Event, days_old) + purge(Webhooks::Outgoing::Delivery, days_old) + purge(Webhooks::Outgoing::DeliveryAttempt, days_old) end - def purge(table_name, days_old) - records = table_name.constantize.where("created_at < ?", days_old.days.ago) + def purge(model, days_old) + records = model.where("created_at < ?", days_old.days.ago) records.destroy_all end end From 0c50b507e9a296a3efe2c70468e8fd6a2de50afb Mon Sep 17 00:00:00 2001 From: Andrew Culver Date: Sun, 2 Apr 2023 21:46:08 -0700 Subject: [PATCH 5/6] Update purge_job.rb --- .../app/jobs/webhooks/outgoing/purge_job.rb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/bullet_train-outgoing_webhooks/app/jobs/webhooks/outgoing/purge_job.rb b/bullet_train-outgoing_webhooks/app/jobs/webhooks/outgoing/purge_job.rb index 9b881389a..cbbdcdcaa 100644 --- a/bullet_train-outgoing_webhooks/app/jobs/webhooks/outgoing/purge_job.rb +++ b/bullet_train-outgoing_webhooks/app/jobs/webhooks/outgoing/purge_job.rb @@ -2,13 +2,12 @@ class Webhooks::Outgoing::PurgeJob < ApplicationJob queue_as :default def perform(days_old = 90) - purge(Webhooks::Outgoing::Event, days_old) - purge(Webhooks::Outgoing::Delivery, days_old) purge(Webhooks::Outgoing::DeliveryAttempt, days_old) + purge(Webhooks::Outgoing::Delivery, days_old) + purge(Webhooks::Outgoing::Event, days_old) end def purge(model, days_old) - records = model.where("created_at < ?", days_old.days.ago) - records.destroy_all + model.where("created_at < ?", days_old.days.ago).delete_all end end From f21bbc3b39d14614ccb60ef46854ea2feda54063 Mon Sep 17 00:00:00 2001 From: Andrew Culver Date: Sun, 2 Apr 2023 21:46:41 -0700 Subject: [PATCH 6/6] Update purge_job.rb --- .../app/jobs/webhooks/outgoing/purge_job.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/bullet_train-outgoing_webhooks/app/jobs/webhooks/outgoing/purge_job.rb b/bullet_train-outgoing_webhooks/app/jobs/webhooks/outgoing/purge_job.rb index cbbdcdcaa..b4a1028cf 100644 --- a/bullet_train-outgoing_webhooks/app/jobs/webhooks/outgoing/purge_job.rb +++ b/bullet_train-outgoing_webhooks/app/jobs/webhooks/outgoing/purge_job.rb @@ -2,6 +2,7 @@ class Webhooks::Outgoing::PurgeJob < ApplicationJob queue_as :default def perform(days_old = 90) + # Delete children first, then parents. purge(Webhooks::Outgoing::DeliveryAttempt, days_old) purge(Webhooks::Outgoing::Delivery, days_old) purge(Webhooks::Outgoing::Event, days_old)