Merge pull request #1522 from alphagov/bulk-data-cache-nov

Load government data in an asynchronous task

app/controllers/application_controller.rb

@@ -28,6 +28,14 @@ class ApplicationController < ActionController::Base
     end
   end
 
+  rescue_from BulkData::LocalDataUnavailableError do |error|
+    GovukError.notify(error)
+
+    render "errors/local_data_unavailable",
+           locals: { error: error },
+           status: :service_unavailable
+  end
+
   def rendering_context
     request.headers["Content-Publisher-Rendering-Context"] || "application"
   end

app/jobs/application_job.rb

@@ -4,4 +4,22 @@ class ApplicationJob < ActiveJob::Base
   def self.discard_and_log(exception)
     discard_on(exception) { |_, error| Rails.logger.warn(error) }
   end
+
+  def run_exclusively(transaction: true, lock_name: self.class.name)
+    name = "content-publisher:#{lock_name}"
+    options = { timeout_seconds: 0, transaction: transaction }
+    result = if transaction
+               ApplicationRecord.transaction do
+                 ApplicationRecord.with_advisory_lock_result(name, options) { yield }
+               end
+             else
+               ApplicationRecord.with_advisory_lock_result(name, options) { yield }
+             end
+
+    unless result.lock_was_acquired?
+      logger.info("Job skipped as exclusive lock '#{name}' could not be acuqired")
+    end
+
+    result.result
+  end
 end

app/jobs/asset_cleanup_job.rb

@@ -2,7 +2,7 @@
 
 class AssetCleanupJob < ApplicationJob
   def perform
-    ApplicationRecord.with_advisory_lock("content-publisher_asset-cleanup", timeout_seconds: 0) do
+    run_exclusively do
       clean_image_assets
       clean_file_attachment_assets
     end

app/jobs/populate_bulk_data_job.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class PopulateBulkDataJob < ApplicationJob
+  retry_on BulkData::RemoteDataUnavailableError do |_job, error|
+    GovukError.notify(error) unless error.cause.is_a?(GdsApi::HTTPServerError)
+  end
+
+  def perform
+    run_exclusively do
+      BulkData::GovernmentRepository.new.populate_cache(older_than: 5.minutes.ago)
+    end
+  end
+end

app/models/concerns/initialize_with_hash.rb

@@ -3,9 +3,14 @@
 module InitializeWithHash
   include ActiveSupport::Concern
 
-  def initialize(params = {})
-    params.each do |key, value|
+  attr_reader :attributes
+
+  def initialize(attributes = {})
+    @attributes = attributes
+    @attributes.each do |key, value|
       instance_variable_set("@#{key}", value)
     end
   end
+
+  alias_method :to_h, :attributes
 end

app/models/edition.rb

@@ -104,11 +104,12 @@ class Edition < ApplicationRecord
   end
 
   scope :history_mode, ->(history_mode = true) do
+    repository = BulkData::GovernmentRepository.new
     if history_mode
-      political.where.not(government_id: [nil, Government.current.content_id])
+      political.where.not(government_id: [nil, repository.current.content_id])
     else
       political(false)
-        .or(political.where(government_id: [nil, Government.current.content_id]))
+        .or(political.where(government_id: [nil, repository.current.content_id]))
     end
   end
 
@@ -133,7 +134,8 @@ def history_mode?
   def government
     return unless government_id
 
-    Government.find(government_id)
+    @government_repository ||= BulkData::GovernmentRepository.new
+    @government_repository.find(government_id)
   end
 
   def public_first_published_at

app/models/government.rb

@@ -1,49 +1,35 @@
 # frozen_string_literal: true
 
 class Government
-  include ActiveModel::Model
+  include InitializeWithHash
 
-  def self.find(content_id)
-    government = all.find { |g| g.content_id == content_id }
-    government || (raise "Government #{content_id} not found")
-  end
-
-  def self.for_date(date)
-    all.find { |government| government.covers?(date) } if date
-  end
-
-  def self.current
-    for_date(Date.current)
-  end
-
-  def self.past
-    all.reject(&:current?)
-  end
-
-  def self.all
-    @all ||= YAML.load_file(Rails.root.join("config/governments.yml"))
-                 .map { |hash| new(hash) }
-  end
-
-  attr_accessor :content_id, :slug, :name, :start_date, :end_date
+  attr_accessor :content_id, :locale, :title, :details
 
   def ==(other)
-    content_id == other.content_id
+    content_id == other.content_id && locale == other.locale
   end
 
   alias_method :eql?, :==
 
   def covers?(date)
-    return false if date < start_date
+    return false if date < started_on
     # Most end dates in Whitehall are the last date of a government so we
     # treat the date as going up to 23:59:59 on the day by appending 1 day to
     # the date
-    return false if end_date && date >= (end_date + 1)
+    return false if ended_on && date >= (ended_on + 1)
 
     true
   end
 
+  def started_on
+    @started_on ||= Date.parse(details["started_on"])
+  end
+
+  def ended_on
+    @ended_on ||= Date.parse(details["ended_on"]) if details["ended_on"]
+  end
+
   def current?
-    self == self.class.current
+    details["current"] == true
   end
 end

app/services/edit_edition_service.rb

@@ -28,6 +28,8 @@ def determine_political
   end
 
   def associate_with_government
-    edition.government_id = Government.for_date(edition.public_first_published_at)&.content_id
+    repository = BulkData::GovernmentRepository.new
+    date = edition.public_first_published_at
+    edition.government_id = repository.for_date(date)&.content_id
   end
 end

app/services/publish_service.rb

@@ -33,10 +33,11 @@ def publish_assets(live_edition)
   def associate_with_government
     return if edition.government
 
+    repository = BulkData::GovernmentRepository.new
     government = if edition.public_first_published_at
-                   Government.for_date(edition.public_first_published_at)
+                   repository.for_date(edition.public_first_published_at)
                  else
-                   Government.current
+                   repository.current
                  end
     edition.assign_attributes(government_id: government&.content_id)
 

app/services/resync_service.rb

@@ -46,10 +46,13 @@ def sync_draft_edition
   end
 
   def set_political_and_government(edition)
+    repository = BulkData::GovernmentRepository.new
+    government = repository.for_date(edition.public_first_published_at)
+
     edition.update!(
       revision_synced: false,
       system_political: PoliticalEditionIdentifier.new(edition).political?,
-      government_id: Government.for_date(edition.public_first_published_at)&.content_id,
+      government_id: government&.content_id,
     )
   end
 

app/views/documents/show/_historical_notice.html.erb

@@ -1,12 +1,8 @@
 <%= render "govuk_publishing_components/components/notice",
   {
-    title: I18n.t(
-      "documents.show.historical.title",
-      document_type: @edition.document_type.label.downcase
-    ),
-    description_text: I18n.t(
-      "documents.show.historical.description",
-      government_name: @edition.government.name
-    ),
+    title: t("documents.show.historical.title",
+             document_type: @edition.document_type.label.downcase),
+    description_text: t("documents.show.historical.description",
+                        government_name: @edition.government.title),
   }
 %>

app/views/errors/local_data_unavailable.html.erb

@@ -0,0 +1,22 @@
+<% if Rails.env.development? %>
+  <%
+  title = "Expected cached data missing"
+  backtrace = error.backtrace.map { |line| "- <code>#{line}</code>" }.join("\n")
+  body_govspeak = <<~GOVSPEAK
+    Content Publisher relies on background jobs to load data from other GOV.UK
+    applications. To resolve this error ensure that the worker process is
+    running for Content Publisher.
+
+    Alternatively or if you are experiencing issues with the worker, you can
+    run the job directly in a rails console which should tell you more:
+    `PopulateBulkDataJob.perform_now`.
+
+    ## Backtrace
+
+    #{backtrace}
+  GOVSPEAK
+  %>
+  <%= render partial: "errors/error", locals: { title: title, body_govspeak: body_govspeak } %>
+<% else %>
+  <%= render partial: "errors/error", locals: t("errors.local_data_unavailable") %>
+<% end %>