WIP 2. Add background job

app/controllers/csv_uploads_controller.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+class CsvUploadsController < ApplicationController
+  before_action :set_csv_upload, only: %i[show edit update destroy]
+
+  # GET /csv_uploads or /csv_uploads.json
+  def index
+    @csv_uploads = CsvUpload.all
+  end
+
+  # GET /csv_uploads/1 or /csv_uploads/1.json
+  def show; end
+
+  # GET /csv_uploads/new
+  def new
+    @csv_upload = CsvUpload.new
+  end
+
+  # GET /csv_uploads/1/edit
+  def edit; end
+
+  # POST /csv_uploads or /csv_uploads.json
+  def create
+    @csv_upload = CsvUpload.new(csv_upload_params)
+
+    respond_to do |format|
+      if @csv_upload.save
+        format.html do
+          redirect_to csv_upload_url(@csv_upload), notice: I18n.t("upload_successfully_created_notice")
+        end
+        format.json { render :show, status: :created, location: @csv_upload }
+
+        start_job
+      else
+        format.html { render :new, status: :unprocessable_entity }
+        format.json { render json: @csv_upload.errors, status: :unprocessable_entity }
+      end
+    end
+  end
+
+  # PATCH/PUT /csv_uploads/1 or /csv_uploads/1.json
+  def update
+    respond_to do |format|
+      if @csv_upload.update(csv_upload_params)
+        format.html { redirect_to csv_upload_url(@csv_upload), notice: I18n.t("upload_successfully_updated_notice") }
+        format.json { render :show, status: :ok, location: @csv_upload }
+      else
+        format.html { render :edit, status: :unprocessable_entity }
+        format.json { render json: @csv_upload.errors, status: :unprocessable_entity }
+      end
+    end
+  end
+
+  # DELETE /csv_uploads/1 or /csv_uploads/1.json
+  def destroy
+    @csv_upload.destroy
+
+    respond_to do |format|
+      format.html { redirect_to csv_uploads_url, notice: I18n.t("upload_successfully_destroyed_notice") }
+      format.json { head :no_content }
+    end
+  end
+
+  private
+
+  # start first csv upload job
+  def start_job
+    # set a wait so front end can load
+    DownloadLocalCopyStartJob.set(wait: 2.seconds).perform_later @csv_upload
+  end
+
+  # Use callbacks to share common setup or constraints between actions.
+  def set_csv_upload
+    @csv_upload = CsvUpload.find(params[:id])
+  end
+
+  # Only allow a list of trusted parameters through.
+  def csv_upload_params
+    params.require(:csv_upload).permit(:title, :local_authority_id, csv_files: [])
+  end
+end

app/helpers/csv_uploads_helper.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+module CsvUploadsHelper
+end

app/javascript/activestorage-directupload.js

@@ -0,0 +1,94 @@
+import * as activestorage from "@rails/activestorage"
+import { DirectUpload } from "@rails/activestorage";
+
+activestorage.start();
+
+// only select 'data-direct-upload' file inputs
+const input = document.querySelector('input[type=file][data-direct-upload-url]:not([disabled])');
+
+// bound to file select
+input.addEventListener('change', (event) => {
+  const fileInput = event.target;
+
+  // if no title is given, put a title in from the provided file
+  let titleEl = document.getElementById('csv_upload_title');
+  if(titleEl) {
+    if(titleEl.value === null || titleEl.value.match(/^ *$/) !== null) {
+      // get the filename, without the extension
+      titleEl.value = fileInput.value.split(/(\\|\/)/g).pop().replace('.csv', '');
+    }
+  }
+  Array.from(input.files).forEach(file => uploadFile(file, fileInput))
+  // clear the selected files from the file input
+  input.value = null
+})
+
+const uploadFile = (file, fileInput) => {
+  const url = input.dataset.directUploadUrl;
+  new Uploader(file, url, fileInput);
+}
+
+class Uploader {
+  // constructs and starts upload
+  constructor(file, url, fileInput) {
+    // file object, with metadata
+    this.file = file;
+    // create a uid for general dom selection
+    this.uid = String(Date.now().toString(32) + Math.random().toString(16)).replace(/\./g, '');
+    this.url = url;
+    this.upload = new DirectUpload(this.file, this.url, this);
+    this.fileInput = fileInput;
+
+    // create the template in the capturing form
+    this.progressTemplate = this.fileInput.form.querySelector('template.directupload-progress').innerHTML
+    this.progressDiv = document.createElement('div')
+    this.fileInput.after(this.progressDiv);
+    this.fileInput.blur();
+
+    // add the uid as an ID to the new element
+    this.progressDiv.id = 'directupload-progress-' + this.uid;
+    if(this.progressTemplate != null) {
+      // create a new template containing the filename and immutable details
+      let templateWithFilename = document.createElement('div');
+      templateWithFilename.innerHTML = this.progressTemplate;
+      templateWithFilename.querySelector('span.directupload-progress-filename').textContent = file.name;
+      this.progressTemplate = templateWithFilename.innerHTML;
+
+      // initialise the new template at 0% progress, and insert in form
+      let parsedTemplate = this.progressTemplate.replaceAll('{{status}}', 'Uploading');
+      parsedTemplate.replaceAll('{{progress}}', 0);
+      this.progressDiv.innerHTML = parsedTemplate;
+    }
+
+    this.upload.create((error, blob) => {
+      if (error) {
+        // Notify that there was an error, set to 0%
+        this.progressDiv.innerHTML = this.progressTemplate
+          .replaceAll('{{status}}', 'Error')
+          .replaceAll('{{progress}}', 0)
+      } else {
+        // Add a hidden field to end of form containing details and signed id
+        const hiddenField = document.createElement('input')
+        hiddenField.setAttribute("type", "hidden");
+        hiddenField.setAttribute("value", blob.signed_id);
+        hiddenField.name = input.name;
+        this.fileInput.form.appendChild(hiddenField);
+        this.progressDiv.innerHTML = this.progressTemplate
+        .replaceAll('{{status}}', 'Uploaded')
+        .replaceAll('{{progress}}', 100);
+      }
+    })
+  }
+
+  directUploadWillStoreFileWithXHR(request) {
+    request.upload.addEventListener("progress",
+      event => this.directUploadDidProgress(event))
+  }
+
+  // update the progress element
+  directUploadDidProgress(event) {
+    this.progressDiv.innerHTML = this.progressTemplate
+      .replaceAll('{{status}}', 'Uploading')
+      .replaceAll('{{progress}}', Math.max(1, Math.floor((event.loaded / event.total) * 100)));
+  }
+}

app/jobs/application_job.rb

@@ -1,9 +1,15 @@
 # frozen_string_literal: true
 
 class ApplicationJob < ActiveJob::Base
-  # Automatically retry jobs that encountered a deadlock
-  # retry_on ActiveRecord::Deadlocked
+  queue_as :low_priority
 
-  # Most jobs are safe to ignore if the underlying records are no longer available
-  # discard_on ActiveJob::DeserializationError
+  def add_message(message_text, type = :success)
+    new_message = CsvProcessingMessage.new(
+      message_type: type,
+      body: message_text,
+      data: [],
+      csv_upload: @csv_upload
+    )
+    new_message.save!
+  end
 end

app/jobs/check_format_job.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "csv"
+
+class CheckFormatJob < ApplicationJob
+  def perform(csv_upload)
+    @csv_upload = csv_upload
+    add_message "Starting CSV Upload"
+    csv_upload.csv_files.each do |csv_file|
+      csv_file.open do |csv_file_info|
+        add_message "Checking rows for #{csv_file.filename}"
+        check_rows(csv_file_info)
+      end
+    end
+
+    InsertRowsJob.perform_later @csv_upload
+    add_message "Inserting rows"
+  end
+
+  def check_rows(file_info)
+    results = []
+    CSV.foreach(file_info.path, headers: true, header_converters: :symbol) do |row|
+      errors = check_row_for_errors(row)
+
+      if errors
+        results.push errors
+        add_message "Error in row #{results.count}", :error
+      else
+        add_message "No errors in row #{results.count}"
+      end
+    end
+
+    file_info.unlink
+    results
+  end
+
+  def check_row_for_errors(row)
+    planning_application = PlanningApplicationCreation.new(
+      **row.to_h.merge(local_authority:)
+    )
+    begin
+      planning_application.validate_property
+      false
+    rescue StandardError
+      true
+    end
+  end
+
+  def local_authority
+    @csv_upload.local_authority
+  end
+end

app/jobs/delete_local_copy.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class DeleteLocalCopyJob < ApplicationJob
+  queue_as :low_priority
+
+  def perform(*args)
+    # Do something later
+  end
+end

app/jobs/download_local_copy_start_job.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class DownloadLocalCopyStartJob < ApplicationJob
+  def perform(csv_upload)
+    @csv_upload = csv_upload
+    csv_file_count = csv_upload.csv_files.count
+    csv_files_downloaded = 0
+
+    csv_upload.csv_files.each do |csv_file|
+      csv_file.download
+      csv_files_downloaded += 1
+      add_message "Downloaded #{csv_files_downloaded} / #{csv_file_count} files"
+    end
+
+    CheckFormatJob.perform_later @csv_upload
+    add_message "Checking format"
+  end
+end

app/jobs/insert_rows_job.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class InsertRowsJob < ApplicationJob
+  queue_as :default
+
+  def perform(*_args)
+    # Do something later
+
+    DeleteLocalCopyJob.perform_later @csv_upload
+    add_message message_text: "Deleting CSV uploads"
+  end
+end

app/models/csv_processing_message.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+class CsvProcessingMessage < ApplicationRecord
+  belongs_to :csv_upload
+  enum message_type: { info: 0, success: 1, error: 2 }
+end

app/models/csv_upload.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class CsvUpload < ApplicationRecord
+  has_many_attached :csv_files
+  has_many :csv_processing_messages, dependent: :destroy
+  belongs_to :local_authority, optional: true
+
+  def filtered_csv_processing_messages
+    success_message = csv_processing_messages
+                      .where(message_type: 0)
+                      .order(created_at: :desc)
+                      .first
+
+    error_messages = csv_processing_messages
+                     .where(message_type: [1, 2])
+                     .order(created_at: :asc)
+
+    if success_message.present?
+      error_messages.or(
+        CsvProcessingMessage.where(id: success_message.id)
+      )
+    else
+      error_messages
+    end
+  end
+end

app/models/local_authority.rb

@@ -2,6 +2,7 @@
 
 class LocalAuthority < ApplicationRecord
   has_many :planning_applications, dependent: :destroy
+  has_many :csv_uploads, dependent: :nullify
   belongs_to :api_client, optional: true
 
   validates :name, presence: true

app/views/csv_uploads/_csv_upload.html.erb

@@ -0,0 +1,21 @@
+<div id="<%= dom_id csv_upload %>">
+  <h2 class="govuk-heading-m">
+    <%= csv_upload.title %>
+  </h2>
+  <p class="govuk-body">Uploaded <%= csv_upload.created_at.to_fs(:short) %></p>
+  <h3 class="govuk-heading-s">Progress</h3>
+  <ul class="govuk-list govuk-list--bullet">
+    <li>Starting import job...</li>
+    <% csv_upload.filtered_csv_processing_messages.each do |msg| %>
+    <li data-created-at="<%= msg.created_at %>" data-message-id="<%= msg.id %>"><%= msg.body %></li>
+    <% end %>
+  </ul>
+  <a href="" class="govuk-button">Refresh progress</a>
+  <h3 class="govuk-heading-s">Attached files</h3>
+  <ul class="govuk-list govuk-list--bullet">
+    <% csv_upload.csv_files.each do |csv| %>
+       <li><%= link_to csv.filename, csv %></li>
+    <% end %>
+  </ul>
+  </p>
+</div>

app/views/csv_uploads/_csv_upload.json.jbuilder

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+json.extract! csv_upload, :id, :title, :created_at, :updated_at
+json.url csv_upload_url(csv_upload, format: :json)