Backfill remaining work_histories for choices

There are still about 90k application_choices that need their
work_histories backfilled.

This commit attempts to do this. It dups the work histories from the
application_form and associates them with the application_choice.

We do the insert in pure SQL, to avoid the issues we had before. Where a
lot of audits that were not necessary were created for each work history
and all the application_choices, forms and candidates have been touched
due to ActiveRecord callbacks.

No audits or touches will be generated by inserting the records via SQL.

app/services/data_migrations/backfill_remaining_application_choices_work_experiences.rb

@@ -0,0 +1,34 @@
+module DataMigrations
+  class BackfillRemainingApplicationChoicesWorkExperiences
+    TIMESTAMP = 20240911144139
+    MANUAL_RUN = true
+
+    def change
+      time_now = Time.zone.now
+
+      choices_without_work_histories.each_slice(5000).with_index do |ids, index|
+        next_batch_time = time_now + index.minutes
+        BackfillWorkHistoriesWorker.perform_at(next_batch_time, ids)
+      end
+    end
+
+    def choices_without_work_histories
+      sql = <<-SQL
+        SELECT "application_choices".id FROM "application_choices" LEFT OUTER JOIN "application_experiences" "work_experiences" ON
+        "work_experiences"."experienceable_type" = 'ApplicationChoice' AND "work_experiences"."experienceable_id" = "application_choices"."id" AND
+        "work_experiences"."type" = 'ApplicationWorkExperience' WHERE "application_choices"."status" != 'unsubmitted' AND "work_experiences"."id" IS NULL
+        union
+        SELECT "application_choices".id FROM "application_choices" LEFT OUTER JOIN "application_experiences" "volunteering_experiences" ON
+        "volunteering_experiences"."experienceable_type" = 'ApplicationChoice' AND "volunteering_experiences"."experienceable_id" = "application_choices"."id" AND
+        "volunteering_experiences"."type" = 'ApplicationVolunteeringExperience' WHERE "application_choices"."status" != 'unsubmitted' AND
+        "volunteering_experiences"."id" IS NULL
+        union
+        SELECT "application_choices".id FROM "application_choices" LEFT OUTER JOIN "application_work_history_breaks" "work_history_breaks" ON
+        "work_history_breaks"."breakable_type" = 'ApplicationChoice' AND "work_history_breaks"."breakable_id" = "application_choices"."id" WHERE
+        "application_choices"."status" != 'unsubmitted' AND "work_history_breaks"."id" IS NULL
+      SQL
+
+      ApplicationChoice.find_by_sql(sql).pluck(:id)
+    end
+  end
+end

app/workers/backfill_work_histories_worker.rb

@@ -0,0 +1,56 @@
+class BackfillWorkHistoriesWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: :low_priority
+
+  def perform(choice_ids)
+    application_experiences = []
+    work_history_breaks = []
+
+    ApplicationChoice.where(id: choice_ids).find_each(batch_size: 200) do |choice|
+      application_form = choice.application_form
+
+      if choice.work_experiences.blank? && application_form.application_work_experiences.any?
+        application_experiences += valid_attributes(
+          application_form.application_work_experiences.map(&:dup),
+          choice,
+          'experienceable',
+        )
+      end
+
+      if choice.volunteering_experiences.blank? && application_form.application_volunteering_experiences.any?
+        application_experiences += valid_attributes(
+          application_form.application_volunteering_experiences.map(&:dup),
+          choice,
+          'experienceable',
+        )
+      end
+
+      if choice.work_history_breaks.blank? && application_form.application_work_history_breaks.any?
+        work_history_breaks += valid_attributes(
+          application_form.application_work_history_breaks.map(&:dup),
+          choice,
+          'breakable',
+        )
+      end
+    end
+
+    ApplicationExperience.insert_all(application_experiences)
+    ApplicationWorkHistoryBreak.insert_all(work_history_breaks)
+  end
+
+private
+
+  def valid_attributes(records, choice, polymorphic_column)
+    records.map do |record|
+      attributes = record.attributes
+      attributes["#{polymorphic_column}_type"] = 'ApplicationChoice'
+      attributes["#{polymorphic_column}_id"] = choice.id
+      attributes['created_at'] = Time.zone.now
+      attributes['updated_at'] = Time.zone.now
+      attributes.delete('id')
+
+      attributes
+    end
+  end
+end

spec/services/data_migrations/backfill_remaining_application_choices_work_experiences_spec.rb

@@ -0,0 +1,69 @@
+require 'rails_helper'
+
+RSpec.describe DataMigrations::BackfillRemainingApplicationChoicesWorkExperiences do
+  describe '#change' do
+    it 'enques jobs to MigrateApplicationChoicesWorker' do
+      application_form = create(:application_form)
+      create(:application_choice, :awaiting_provider_decision, application_form:)
+
+      expect { described_class.new.change }.to change(
+        BackfillWorkHistoriesWorker.jobs, :size
+      ).by(1)
+    end
+  end
+
+  describe '#choices_without_work_histories' do
+    it 'returns all the application choices that need to be backfilled' do
+      application_form = create(:application_form)
+      choice_without_work_history = create(:application_choice, :awaiting_provider_decision, application_form:)
+      choice_with_work_experience_not_volunteering = create(
+        :application_choice,
+        :awaiting_provider_decision,
+        application_form:,
+      )
+      create(
+        :application_work_experience,
+        experienceable: choice_with_work_experience_not_volunteering,
+      )
+      create(
+        :application_work_history_break,
+        breakable: choice_with_work_experience_not_volunteering,
+      )
+      choice_with_work_experience_no_break = create(
+        :application_choice,
+        :awaiting_provider_decision,
+        application_form:,
+      )
+      create(
+        :application_work_experience,
+        experienceable: choice_with_work_experience_no_break,
+      )
+      create(
+        :application_volunteering_experience,
+        experienceable: choice_with_work_experience_no_break,
+      )
+      choice_with_work_history = create(:application_choice, :awaiting_provider_decision, application_form:)
+      create(
+        :application_work_experience,
+        experienceable: choice_with_work_history,
+      )
+      create(
+        :application_volunteering_experience,
+        experienceable: choice_with_work_history,
+      )
+      create(
+        :application_work_history_break,
+        breakable: choice_with_work_history,
+      )
+
+      expect(described_class.new.choices_without_work_histories).to include(
+        choice_without_work_history.id,
+        choice_with_work_experience_not_volunteering.id,
+        choice_with_work_experience_no_break.id,
+      )
+      expect(described_class.new.choices_without_work_histories).not_to include(
+        choice_with_work_history.id,
+      )
+    end
+  end
+end

spec/workers/backfill_work_histories_worker_spec.rb

@@ -0,0 +1,53 @@
+require 'rails_helper'
+
+RSpec.describe BackfillWorkHistoriesWorker do
+  describe '#perform' do
+    before do
+      TestSuiteTimeMachine.unfreeze!
+    end
+
+    it 'dups the working experiences and histories from application_form to choice', :with_audited do
+      application_form = create(
+        :completed_application_form,
+        volunteering_experiences_count: 1,
+        full_work_history: true,
+      )
+      application_form_2 = create(
+        :completed_application_form,
+        volunteering_experiences_count: 1,
+        full_work_history: true,
+      )
+      choice = create(:application_choice, application_form:)
+      choice_with_data_migrated = create(
+        :application_choice,
+        application_form: application_form_2,
+      )
+      create(
+        :application_work_experience,
+        experienceable: choice_with_data_migrated,
+      )
+      create(
+        :application_volunteering_experience,
+        experienceable: choice_with_data_migrated,
+      )
+      create(
+        :application_work_history_break,
+        breakable: choice_with_data_migrated,
+      )
+      choice_ids = [choice.id, choice_with_data_migrated.id]
+
+      expect {
+        described_class.new.perform(choice_ids)
+      }.to change(choice.work_experiences, :count).by(2)
+        .and change(choice.volunteering_experiences, :count).by(1)
+        .and change(choice.work_history_breaks, :count).by(1)
+        .and not_change(choice_with_data_migrated.work_experiences, :count)
+        .and not_change(choice_with_data_migrated.volunteering_experiences, :count)
+        .and not_change(choice_with_data_migrated.work_history_breaks, :count)
+        .and not_change(choice.own_and_associated_audits, :count)
+        .and not_change(choice.reload, :updated_at)
+        .and not_change(application_form.reload, :updated_at)
+        .and not_change(application_form.candidate, :updated_at)
+    end
+  end
+end