Merge pull request #959 from thecartercenter/9783-xlsform-export

9783: XLSForm export

.rubocop.yml

@@ -54,9 +54,6 @@ Layout/LineLength:
 Lint/RedundantCopDisableDirective:
   Enabled: false
 
-Lint/LastKeywordArgument:
-  Enabled: true
-
 Lint/UnusedMethodArgument:
   # Otherwise we have to remove named args from the method signature altogether which seems to reduce
   # clarity of the code.

Gemfile

@@ -20,6 +20,7 @@ gem "recaptcha", "~> 3.4", require: "recaptcha/rails" # Small change in v4, we s
 gem "responders", "~> 3.0"
 gem "rqrcode", "~> 1.1"
 gem "rubyzip", "~> 2.3", require: "zip" # Explicitly specify name (https://stackoverflow.com/a/32740666/763231)
+gem "spreadsheet" # For XLSForm export
 gem "term-ansicolor", "~> 1.3"
 gem "terrapin", "~> 0.6.0"
 gem "thor", "~> 1.0"

Gemfile.lock

@@ -516,6 +516,7 @@ GEM
     ruby-jmeter (3.1.08)
       nokogiri
       rest-client
+    ruby-ole (1.2.12.2)
     ruby-progressbar (1.13.0)
     ruby-vips (2.1.4)
       ffi (~> 1.12)
@@ -552,6 +553,8 @@ GEM
     sexp_processor (4.17.0)
     spinjs-rails (1.3)
       rails (>= 3.1)
+    spreadsheet (1.3.0)
+      ruby-ole
     sprockets (3.7.2)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
@@ -719,6 +722,7 @@ DEPENDENCIES
   sentry-rails (~> 5.0)
   sentry-ruby (~> 5.0)
   spinjs-rails (~> 1.3.0)
+  spreadsheet
   sprockets (~> 3)
   sys-filesystem (~> 1.4)
   term-ansicolor (~> 1.3)

app/controllers/forms_controller.rb

@@ -235,6 +235,12 @@ def export_xml
     send_data(@form.odk_xml.download, filename: "form-#{@form.name.dasherize}-#{Time.zone.today}.xml")
   end
 
+  # XLSForm export.
+  def export_xls
+    exporter = Forms::Export.new(@form)
+    send_data(exporter.to_xls.html_safe, filename: "xlsform-#{@form.name.dasherize}-#{Time.zone.today}.xls") # rubocop:disable Rails/OutputSafety
+  end
+
   # ODK XML export for all published forms.
   # Theoretically works for standard forms too, but they have no XML so can't be exported at this time.
   def export_all

app/decorators/action_links/form_link_builder.rb

@@ -8,6 +8,7 @@ def initialize(form)
       unless new_action?
         actions << [:export_csv, {url: h.export_form_path(form)}]
         actions << [:export_xml, {url: h.export_xml_form_path(form)}] if form.odk_xml.attached?
+        actions << [:export_xls, {url: h.export_xls_form_path(form)}]
       end
 
       unless h.admin_mode?

app/decorators/odk/condition_decorator.rb

@@ -38,6 +38,10 @@ def right_qing
       decorate(object.right_qing)
     end
 
+    def convert_op(operation)
+      OP_XPATH[operation]
+    end
+
     private
 
     def select_multiple_to_odk

app/helpers/icon_helper.rb

@@ -11,6 +11,7 @@ module IconHelper
     export: "download",
     export_csv: "download",
     export_xml: "download",
+    export_xls: "download",
     go_live: "play",
     import: "upload",
     index: "list",

app/models/forms/export.rb

@@ -8,6 +8,36 @@ class Export
       DisplayLogic DisplayConditions Default Hidden
     ].freeze
 
+    QTYPE_TO_XLS = {
+      # conversions
+      "location" => "geopoint",
+      "long_text" => "text",
+      "datetime" => "dateTime",
+      "annotated_image" => "image",
+      "counter" => "integer",
+
+      # no change
+      "text" => "text",
+      "select_one" => "select_one",
+      "select_multiple" => "select_multiple",
+      "decimal" => "decimal",
+      "time" => "time",
+      "date" => "date",
+      "image" => "image",
+      "barcode" => "barcode",
+      "audio" => "audio",
+      "video" => "video",
+      "integer" => "integer",
+
+      # TODO: Not yet supported in our XLSForm exporter
+      "sketch" => "sketch (WARNING: not yet supported)",
+      "signature" => "signature (WARNING: not yet supported)"
+
+      # Note: XLSForm qtypes not supported in NEMO:
+      #   range, geotrace, geoshape, note, file, select_one_from_file, select_multiple_from_file,
+      #   background-audio, calculate, acknowledge, hidden, xml-external
+    }.freeze
+
     def initialize(form)
       @form = form
     end
@@ -21,6 +51,131 @@ def to_csv
       end
     end
 
+    # rubocop:disable Metrics/MethodLength, Metrics/BlockLength, Metrics/AbcSize, Metrics/PerceivedComplexity, Style/Next
+    def to_xls
+      # TODO: option set "levels"?
+
+      book = Spreadsheet::Workbook.new
+
+      # Create sheets
+      questions = book.create_worksheet(name: "survey")
+      choices = book.create_worksheet(name: "choices")
+      settings = book.create_worksheet(name: "settings")
+
+      # Write sheet headings at row index 0
+      questions.row(0).push("type", "name", "label", "required", "relevant", "constraint")
+      choices.row(0).push("list_name", "name", "label")
+      settings.row(0).push("form_title", "form_id", "version", "default_language")
+
+      group_depth = 1 # assume base level
+      repeat_depth = 1
+      option_sets_used = []
+
+      # Define the below "index modifiers" which keep track of the line of the spreadsheet we are writing to.
+      # The for loop below (tracked by index i) loops through the list of form items, and so the index does not take into account rows that we need to write for when groups end. In XLSForm, these are written to a row all to themselves.
+      # This causes the index i to be de-synchronized with the row of the spreadsheet that we are writing to.
+      # Hence, we push to the row (i + index_mod)
+      index_mod = 1 # start at row index 1
+      choices_index_mod = 1
+
+      @form.preordered_items.each_with_index do |q, i|
+        # this variable keeps track of the spreadsheet row to be written during this loop iteration
+        row_index = i + index_mod
+
+        # did one or more groups just end?
+        # if so, the qing's depth will be smaller than the depth counter
+        while group_depth > q.ancestry_depth
+          # are we in a repeat group?
+          # we don't want to end the repeat if we are ending a nested non-repeat group within a repeat
+          if repeat_depth > 1 && repeat_depth >= group_depth
+            questions.row(row_index).push("end repeat")
+            repeat_depth -= 1
+          else
+            # end the group
+            questions.row(row_index).push("end group")
+          end
+
+          # update counters to accomodate additional "end group" lines
+          group_depth -= 1
+          index_mod += 1
+          row_index += 1
+        end
+
+        if q.group? # is this a group?
+          group_name = q.code.tr(" ", "_")
+
+          if q.repeatable?
+            questions.row(row_index).push("begin repeat", group_name, q.code)
+            repeat_depth += 1
+          else
+            questions.row(row_index).push("begin group", group_name, q.code)
+          end
+
+          # update counters
+          group_depth += 1
+        else # is this a question?
+          # do we have an option set?
+          if q.option_set_id.present?
+            os = OptionSet.find(q.option_set_id)
+
+            # include leading space to respect XLSForm format
+            # question name should be followed by the option set name (if applicable) separated by a space
+            # replace any spaces in the option set name with underscores to ensure the form is parsed correctly
+            os_name = os.name.tr(" ", "_")
+            os_already_logged = option_sets_used.include?(q.option_set_id)
+
+            # log the option set to the spreadsheet if we haven't yet
+            # ni = index for the option nodes loop
+            # node = the current option node
+            # TODO: support option set "levels" by creating a cascading sheet here
+            unless os_already_logged
+              os.option_nodes.each_with_index do |node, ni|
+                if node.option.present?
+                  choices
+                    .row(ni + choices_index_mod)
+                    .push(os_name, node.option.canonical_name, node.option.canonical_name)
+                end
+              end
+
+              # increment the choices index by how many nodes there are, so we start at this row next time
+              choices_index_mod += os.option_nodes.length
+
+              option_sets_used.push(q.option_set_id)
+            end
+          end
+
+          # convert question types
+          qtype_converted = QTYPE_TO_XLS[q.qtype_name]
+
+          type_to_push = "#{qtype_converted} #{os_name}"
+
+          # Write the question row
+          questions.row(row_index).push(type_to_push, q.code, q.name, q.required.to_s)
+        end
+
+        # if we have any relevant conditions, add them to the end of the row
+        if q.display_conditions.any?
+          questions.row(row_index).push(conditions_to_xls(q.display_conditions, q.display_if))
+        end
+
+        if q.constraints.any?
+          q.constraints.each do |c|
+            questions.row(row_index).push(conditions_to_xls(c.conditions, c.accept_if))
+          end
+        end
+      end
+
+      # Settings
+      lang = @form.mission.setting.preferred_locales[0].to_s
+      settings.row(1).push(@form.name, @form.id, @form.current_version.decorate.name, lang)
+
+      # Write
+      file = StringIO.new
+      book.write(file)
+      file.string
+    end
+    # rubocop:enable Metrics/MethodLength, Metrics/BlockLength, Metrics/AbcSize, Metrics/PerceivedComplexity, Style/Next
+
     private
 
     def human_readable(klass, qing)
@@ -48,5 +203,44 @@ def name(qing)
         "Group"
       end
     end
+
+    # Takes an array of conditions and outputs a single string
+    # concatenates by either "and" or "or" depending on form settings
+    def conditions_to_xls(conditions, true_if)
+      relevant_to_push = ""
+      concatenator = true_if == "all_met" ? "and" : "or"
+
+      # how many conditions?
+      dc_length = conditions.length
+
+      conditions.each_with_index do |dc, i|
+        # prep left side of expression
+        left_qing = Questioning.find(dc.left_qing_id)
+        left_to_push = "${#{left_qing.code}_#{left_qing.full_dotted_rank}}"
+
+        # prep right side of expression
+        if dc.right_side_is_qing?
+          right_qing = Questioning.find(dc.right_qing_id)
+          right_to_push = "${#{right_qing.code}_#{right_qing.full_dotted_rank}}"
+        elsif Float(dc.value, exception: false).nil? # it's not a number
+          # to respect XLSform rules, surround with single quotes unless it's a number
+          right_to_push = "'#{dc.value}'"
+        else
+          right_to_push = dc.value.to_s
+        end
+
+        op = ODK::ConditionDecorator::OP_XPATH[dc.op.to_sym]
+        raise "Operation not found: #{dc.op}" if op.blank?
+
+        # omit the concatenator on the last condition only
+        relevant_to_push = if i + 1 == dc_length
+                             "#{relevant_to_push}#{left_to_push} #{op} #{right_to_push}"
+                           else
+                             "#{relevant_to_push}#{left_to_push} #{op} #{right_to_push} #{concatenator} "
+                           end
+      end
+
+      relevant_to_push
+    end
   end
 end

config/locales/en/main.yml

@@ -921,6 +921,7 @@ en:
     import_from_csv: "Import from CSV"
     export_csv: "Export CSV"
     export_xml: "Export XML"
+    export_xls: "Export XLSForm (experimental)"
     export: "Export"
     models:
       form:

config/routes.rb

@@ -138,6 +138,7 @@
         get "sms-guide", as: "sms_guide", action: "sms_guide"
         get "export"
         get "export_xml"
+        get "export_xls"
       end
     end
 

spec/features/forms/form/form_export_spec.rb

@@ -19,6 +19,15 @@
       click_link("Export XML")
       expect(page.body).to match("<h:title>#{form.name}</h:title>")
     end
+
+    it "XLSForm exports successfully" do
+      visit(form_path(form, locale: "en", mode: "m", mission_name: get_mission.compact_name))
+      click_link("Export XLSForm")
+
+      expect(page.current_url).to match("export_xls")
+      expect(page.body).to match(form.name.to_s)
+      expect(page.body).to match("text")
+    end
   end
 
   context "multiple forms" do