[CPDNPQ-2242] admin bulk reject applications (#2088)

* terraform for azure storage

* upload file ready for bulk operation

* perform bulk revert

* use GRS azure storage replication for production

* add example file

* feature spec

* ability to bulk reject applications from admin

* show pages for bulk uploads

* file validation

* use background job for running bulk operations

* improvements

* specs for jobs

* better errors

* move rake task to correct folder

* improvements from PR comments

* change not_ran scope. add missing tests

* save result as JSON within service

* remove rake task - can now be peformed in admin console

* make migration the latest

Gemfile

@@ -4,6 +4,7 @@ git_source(:github) { |repo| "https://github.com/#{repo}.git" }
 ruby File.read(".ruby-version").chomp
 
 gem "activerecord-session_store"
+gem "azure-blob"
 gem "blueprinter"
 gem "bootsnap", ">= 1.1.0", require: false
 gem "canonical-rails"

Gemfile.lock

@@ -113,6 +113,8 @@ GEM
       descendants_tracker (~> 0.0.4)
       ice_nine (~> 0.11.0)
       thread_safe (~> 0.3, >= 0.3.1)
+    azure-blob (0.5.4)
+      rexml
     base64 (0.2.0)
     bcrypt (3.1.20)
     bigdecimal (3.1.8)
@@ -692,6 +694,7 @@ DEPENDENCIES
   amazing_print
   axe-core-capybara (~> 4.6)
   axe-core-rspec (~> 4.10)
+  azure-blob
   blueprinter
   bootsnap (>= 1.1.0)
   brakeman

app/components/npq_separation/navigation_structures/admin_navigation_structure.rb

@@ -62,6 +62,11 @@ def structure
             href: npq_separation_admin_lead_providers_path,
             prefix: "/npq-separation/admin/lead_providers",
           ) => [],
+          Node.new(
+            name: "Bulk operations",
+            href: npq_separation_admin_bulk_operations_path,
+            prefix: "/npq-separation/admin/bulk_operations",
+          ) => [],
           Node.new(
             name: "Settings",
             href: "#",

app/controllers/npq_separation/admin/bulk_operations/reject_applications_controller.rb

@@ -0,0 +1,44 @@
+module NpqSeparation::Admin::BulkOperations
+  class RejectApplicationsController < NpqSeparation::AdminController
+    before_action :set_bulk_operations, :set_bulk_operation, only: %i[index create]
+    before_action :find_bulk_operation, only: %i[run show]
+
+    def create
+      if (file = params.dig(:bulk_operation_reject_applications, :file))
+        BulkOperation::RejectApplications.not_started.destroy_all
+        @bulk_operation.file.attach(file)
+        if @bulk_operation.valid?
+          @bulk_operation.save!
+          @bulk_operation.update!(row_count: @bulk_operation.file.download.lines.count)
+          return redirect_to :npq_separation_admin_bulk_operations_reject_applications
+        end
+      end
+
+      render :index, status: :unprocessable_entity
+    end
+
+    def run
+      @bulk_operation.update!(started_at: Time.zone.now, ran_by_admin_id: current_admin.id)
+      BulkOperation::BulkRejectApplicationsJob.perform_later(bulk_operation_id: @bulk_operation.id)
+      redirect_to :npq_separation_admin_bulk_operations_reject_applications
+    end
+
+    def show
+      # empty method, because rubocop will complain in the before_action otherwise
+    end
+
+  private
+
+    def set_bulk_operations
+      @bulk_operations = BulkOperation::RejectApplications.all.includes([file_attachment: :blob]).order(:created_at)
+    end
+
+    def set_bulk_operation
+      @bulk_operation = BulkOperation::RejectApplications.new admin: current_admin
+    end
+
+    def find_bulk_operation
+      @bulk_operation = BulkOperation::RejectApplications.find(params[:id])
+    end
+  end
+end

app/controllers/npq_separation/admin/bulk_operations/revert_applications_to_pending_controller.rb

@@ -0,0 +1,44 @@
+module NpqSeparation::Admin::BulkOperations
+  class RevertApplicationsToPendingController < NpqSeparation::AdminController
+    before_action :set_bulk_operations, :set_bulk_operation, only: %i[index create]
+    before_action :find_bulk_operation, only: %i[run show]
+
+    def create
+      if (file = params.dig(:bulk_operation_revert_applications_to_pending, :file))
+        BulkOperation::RevertApplicationsToPending.not_started.destroy_all
+        @bulk_operation.file.attach(file)
+        if @bulk_operation.valid?
+          @bulk_operation.save!
+          @bulk_operation.update!(row_count: @bulk_operation.file.download.lines.count)
+          return redirect_to :npq_separation_admin_bulk_operations_revert_applications_to_pending_index
+        end
+      end
+
+      render :index, status: :unprocessable_entity
+    end
+
+    def run
+      @bulk_operation.update!(started_at: Time.zone.now, ran_by_admin_id: current_admin.id)
+      BulkOperation::BulkChangeApplicationsToPendingJob.perform_later(bulk_operation_id: @bulk_operation.id)
+      redirect_to :npq_separation_admin_bulk_operations_revert_applications_to_pending_index
+    end
+
+    def show
+      # empty method, because rubocop will complain in the before_action otherwise
+    end
+
+  private
+
+    def set_bulk_operations
+      @bulk_operations = BulkOperation::RevertApplicationsToPending.all.includes([file_attachment: :blob]).order(:created_at)
+    end
+
+    def set_bulk_operation
+      @bulk_operation = BulkOperation::RevertApplicationsToPending.new admin: current_admin
+    end
+
+    def find_bulk_operation
+      @bulk_operation = BulkOperation::RevertApplicationsToPending.find(params[:id])
+    end
+  end
+end

app/controllers/npq_separation/admin/bulk_operations_controller.rb

@@ -0,0 +1,2 @@
+class NpqSeparation::Admin::BulkOperationsController < NpqSeparation::AdminController
+end

app/jobs/bulk_operation/bulk_change_applications_to_pending_job.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class BulkOperation::BulkChangeApplicationsToPendingJob < ApplicationJob
+  def perform(bulk_operation_id:)
+    bulk_operation = BulkOperation::RevertApplicationsToPending.find(bulk_operation_id)
+    application_ecf_ids = CSV.parse(bulk_operation.file.download, headers: false).flatten
+    Rails.logger.info("Bulk Operation started - bulk_operation_id: #{bulk_operation_id}")
+    BulkOperation::BulkChangeApplicationsToPending.new(application_ecf_ids:, bulk_operation:).run!(dry_run: false)
+    Rails.logger.info("Bulk Operation finished - bulk_operation_id: #{bulk_operation_id}")
+  end
+end

app/jobs/bulk_operation/bulk_reject_applications_job.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class BulkOperation::BulkRejectApplicationsJob < ApplicationJob
+  def perform(bulk_operation_id:)
+    bulk_operation = BulkOperation::RejectApplications.find(bulk_operation_id)
+    application_ecf_ids = CSV.parse(bulk_operation.file.download, headers: false).flatten
+    Rails.logger.info("Bulk Operation started - bulk_operation_id: #{bulk_operation_id}")
+    BulkOperation::BulkRejectApplications.new(application_ecf_ids:, bulk_operation:).run!
+    Rails.logger.info("Bulk Operation finished - bulk_operation_id: #{bulk_operation_id}")
+  end
+end

app/models/admin.rb

@@ -1,4 +1,6 @@
 class Admin < ApplicationRecord
+  has_many :bulk_operations
+
   validates :full_name,
             presence: { message: "Enter a full name" },
             length: { maximum: 64, message: "Full name must be shorter than 64 characters" }

app/models/bulk_operation.rb

@@ -0,0 +1,32 @@
+class BulkOperation < ApplicationRecord
+  belongs_to :admin
+  has_one_attached :file
+
+  validate :file_valid
+
+  scope :not_started, -> { where(started_at: nil) }
+
+  def started?
+    started_at.present?
+  end
+
+private
+
+  def file_valid
+    return unless file.attached?
+
+    errors.add(:file, :empty) unless file.blob.byte_size.positive?
+    if attachment_changes["file"]
+      check_format(attachment_changes["file"].attachable.read)
+    end
+  end
+
+  def check_format(string)
+    CSV.parse(string) do |row|
+      if row.size > 1
+        errors.add(:file, :invalid)
+        break
+      end
+    end
+  end
+end

app/models/bulk_operation/reject_applications.rb

@@ -0,0 +1,2 @@
+class BulkOperation::RejectApplications < BulkOperation
+end

app/models/bulk_operation/revert_applications_to_pending.rb

@@ -0,0 +1,2 @@
+class BulkOperation::RevertApplicationsToPending < BulkOperation
+end

app/services/bulk_operation/bulk_change_applications_to_pending.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+class BulkOperation::BulkChangeApplicationsToPending
+  attr_reader :application_ecf_ids, :bulk_operation
+
+  def initialize(application_ecf_ids:, bulk_operation:)
+    @application_ecf_ids = application_ecf_ids
+    @bulk_operation = bulk_operation
+  end
+
+  def run!(dry_run: true)
+    result = {}
+    ActiveRecord::Base.transaction do
+      result = application_ecf_ids.each_with_object({}) do |application_ecf_id, hash|
+        application = Application.find_by(ecf_id: application_ecf_id)
+        revert_to_pending = Applications::RevertToPending.new(application:, change_status_to_pending: "yes")
+        success = revert_to_pending.revert
+        hash[application_ecf_id] = outcome(success, application, revert_to_pending.errors)
+      end
+      bulk_operation.update!(result: result.to_json, finished_at: Time.zone.now)
+
+      raise ActiveRecord::Rollback if dry_run
+    end
+
+    result
+  end
+
+private
+
+  def outcome(success, application, errors)
+    return "Not found" if application.nil?
+    return "Changed to pending" if success
+
+    errors.full_messages.to_sentence
+  end
+end

app/services/bulk_operation/bulk_reject_applications.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+class BulkOperation::BulkRejectApplications
+  attr_reader :application_ecf_ids, :bulk_operation
+
+  def initialize(application_ecf_ids:, bulk_operation:)
+    @application_ecf_ids = application_ecf_ids
+    @bulk_operation = bulk_operation
+  end
+
+  def run!
+    result = {}
+    ActiveRecord::Base.transaction do
+      result = application_ecf_ids.each_with_object({}) do |application_ecf_id, hash|
+        application = Application.find_by(ecf_id: application_ecf_id)
+        reject_service = Applications::Reject.new(application:)
+        success = reject_service.reject
+        hash[application_ecf_id] = outcome(success, application, reject_service.errors)
+      end
+      bulk_operation.update!(result: result.to_json, finished_at: Time.zone.now)
+    end
+
+    result
+  end
+
+private
+
+  def outcome(success, application, errors)
+    return "Not found" if application.nil?
+    return "Changed to rejected" if success
+
+    errors.full_messages.to_sentence
+  end
+end

app/views/npq_separation/admin/bulk_operations/index.html.erb

@@ -0,0 +1,6 @@
+<h1 class="govuk-heading-l">Bulk operations</h1>
+
+<ul class="govuk-list govuk-list--bullet">
+  <%= tag.li( govuk_link_to("Revert applications to pending", npq_separation_admin_bulk_operations_revert_applications_to_pending_index_path, no_visited_state: true,)) %>
+  <%= tag.li( govuk_link_to("Reject applications", npq_separation_admin_bulk_operations_reject_applications_path, no_visited_state: true,)) %>
+</ul>

app/views/npq_separation/admin/bulk_operations/reject_applications/index.html.erb

@@ -0,0 +1,59 @@
+<%= govuk_back_link(href: npq_separation_admin_bulk_operations_path) %>
+
+<h1 class="govuk-heading-l">Reject Applications</h1>
+
+<%=
+  govuk_table do |table|
+    table.with_head do |header|
+      header.with_row do |row|
+        row.with_cell(text: "Filename")
+        row.with_cell(text: "Rows")
+        row.with_cell(text: "Created at")
+        row.with_cell(text: "Started at")
+        row.with_cell
+      end
+    end
+
+    table.with_body do |body|
+      @bulk_operations.each do |bulk_operation|
+        body.with_row do |row|
+          row.with_cell(text: govuk_link_to(bulk_operation.file.filename, npq_separation_admin_bulk_operations_reject_application_path(bulk_operation)))
+          row.with_cell(text: bulk_operation.row_count)
+          row.with_cell(text: bulk_operation.created_at.to_formatted_s(:govuk_short))
+          row.with_cell(text: bulk_operation.started_at&.to_formatted_s(:govuk_short))
+          row.with_cell do
+            unless bulk_operation.started?
+              form_with url: run_npq_separation_admin_bulk_operations_reject_application_path(bulk_operation) do |f|
+                f.govuk_submit "Reject Applications"
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+%>
+
+<%= form_for @bulk_operation, url: npq_separation_admin_bulk_operations_reject_applications_path, method: :post, multipart: true do |f| %>
+  <%= f.govuk_error_summary %>
+  <div class="govuk-form-group">
+    <%= f.label "file", "Upload an application list file", class: "govuk-label" %>
+    <%= f.file_field "file", class: "govuk-file-upload" %>
+  </div>
+
+  <%= govuk_details(summary_text: "Example file") do %>
+    <p>
+      The file is a list of application IDs, one ID per row, with no header row.
+    </p>
+    <p>
+      e.g.
+      <pre>
+2f581c80-b5bb-4404-bcaf-4044d9c0c674
+21fe9549-28f0-492c-ae3a-d52969b40536
+e7e8a629-f75f-4d98-b157-269136110099
+      </pre>
+    </p>
+  <% end %>
+
+  <%= f.govuk_submit "Upload" %>
+<% end %>