From 4b01b2ef6dc5f6677384c39c9358d3af9e961f9f Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 5 Sep 2023 13:56:42 -0400 Subject: [PATCH 01/34] Create to_xls method and route, successfully get a test xlsx file --- Gemfile | 1 + Gemfile.lock | 4 +++ app/controllers/forms_controller.rb | 7 ++++ .../action_links/form_link_builder.rb | 1 + app/models/forms/export.rb | 32 +++++++++++++++++++ config/routes.rb | 1 + 6 files changed, 46 insertions(+) diff --git a/Gemfile b/Gemfile index 4736e14499..414a6a823c 100644 --- a/Gemfile +++ b/Gemfile @@ -24,6 +24,7 @@ gem "term-ansicolor", "~> 1.3" gem "terrapin", "~> 0.6.0" gem "thor", "~> 1.0" gem "twilio-ruby", "~> 4.2" # Does not use semver after v5, watch out! +gem "spreadsheet" # For XLSForm export # JS/CSS gem "bootstrap", "~> 4.3" diff --git a/Gemfile.lock b/Gemfile.lock index 92093eb352..e4265194d8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -516,6 +516,7 @@ GEM ruby-jmeter (3.1.08) nokogiri rest-client + ruby-ole (1.2.12.2) ruby-progressbar (1.11.0) ruby-vips (2.1.4) ffi (~> 1.12) @@ -551,6 +552,8 @@ GEM sexp_processor (4.16.1) spinjs-rails (1.3) rails (>= 3.1) + spreadsheet (1.3.0) + ruby-ole sprockets (3.7.2) concurrent-ruby (~> 1.0) rack (> 1, < 3) @@ -722,6 +725,7 @@ DEPENDENCIES sentry-rails (~> 5.0) sentry-ruby (~> 5.0) spinjs-rails (~> 1.3.0) + spreadsheet sprockets (~> 3) sys-filesystem (~> 1.4) term-ansicolor (~> 1.3) diff --git a/app/controllers/forms_controller.rb b/app/controllers/forms_controller.rb index 173d2062b4..9b6756d279 100644 --- a/app/controllers/forms_controller.rb +++ b/app/controllers/forms_controller.rb @@ -235,6 +235,13 @@ def export_xml send_data(@form.odk_xml.download, filename: "form-#{@form.name.dasherize}-#{Time.zone.today}.xml") end + # XLSForm export. + def export_xls + exporter = Forms::Export.new(@form) + send_data(exporter.to_xls, filename: "test.xlsx") + # send_file ... + end + # ODK XML export for all published forms. # Theoretically works for standard forms too, but they have no XML so can't be exported at this time. def export_all diff --git a/app/decorators/action_links/form_link_builder.rb b/app/decorators/action_links/form_link_builder.rb index d8c81907cc..91ed0036d6 100644 --- a/app/decorators/action_links/form_link_builder.rb +++ b/app/decorators/action_links/form_link_builder.rb @@ -8,6 +8,7 @@ def initialize(form) unless new_action? actions << [:export_csv, {url: h.export_form_path(form)}] actions << [:export_xml, {url: h.export_xml_form_path(form)}] if form.odk_xml.attached? + actions << [:export_xls, {url: h.export_xls_form_path(form)}] end unless h.admin_mode? diff --git a/app/models/forms/export.rb b/app/models/forms/export.rb index 4070aab308..966bf1e9b6 100644 --- a/app/models/forms/export.rb +++ b/app/models/forms/export.rb @@ -16,11 +16,43 @@ def to_csv CSV.generate do |csv| csv << COLUMNS @form.preordered_items.each do |q| + Rails.logger.debug("*****************") + Rails.logger.debug(q) csv << row(q) end end end + def to_xls + book = Spreadsheet::Workbook.new + + # Create sheets + questions = book.create_worksheet :name => "survey" + choices = book.create_worksheet :name => "choices" + settings = book.create_worksheet :name => "settings" + + # Questions + questions.row(0).push "type", "name", "label", "required", "relevant" + questions.row(1).push "integer", "age", "How old are you?", "yes", "" + questions.row(2).push "select_one yes_no","likes_pizza", "Do you like pizza?", "", "" + + # Choices + choices.row(0).push "list_name", "name", "label" + choices.row(1).push "yes_no", "yes" + choices.row(1).push "Yes" # Try adding onto the end of a row + choices.row(2).push "yes_no", "no", "No" + + # Settings + settings.row(0).push "form_title", "form_id", "version", "default_language" + settings.row(1).push "Test Form", 1, Time.now.strftime("%Y%m%d"), "English (en)" + + # Write + file = StringIO.new + book.write(file) + + file.string.html_safe + end + private def human_readable(klass, qing) diff --git a/config/routes.rb b/config/routes.rb index 0c8e8059df..eb6d2ceac4 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -138,6 +138,7 @@ get "sms-guide", as: "sms_guide", action: "sms_guide" get "export" get "export_xml" + get "export_xls" end end From 17d70c04935233135acc7e7b88d87c3108209b9d Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 5 Sep 2023 14:36:44 -0400 Subject: [PATCH 02/34] Get real form data in to_xls method --- app/models/forms/export.rb | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/app/models/forms/export.rb b/app/models/forms/export.rb index 966bf1e9b6..43b020e176 100644 --- a/app/models/forms/export.rb +++ b/app/models/forms/export.rb @@ -16,8 +16,6 @@ def to_csv CSV.generate do |csv| csv << COLUMNS @form.preordered_items.each do |q| - Rails.logger.debug("*****************") - Rails.logger.debug(q) csv << row(q) end end @@ -31,25 +29,25 @@ def to_xls choices = book.create_worksheet :name => "choices" settings = book.create_worksheet :name => "settings" - # Questions - questions.row(0).push "type", "name", "label", "required", "relevant" - questions.row(1).push "integer", "age", "How old are you?", "yes", "" - questions.row(2).push "select_one yes_no","likes_pizza", "Do you like pizza?", "", "" + # Write sheet headings at row index 0 + questions.row(0).push("type", "name", "label", "required", "relevant") + choices.row(0).push("list_name", "name", "label") + settings.row(0).push("form_title", "form_id", "version", "default_language") - # Choices - choices.row(0).push "list_name", "name", "label" - choices.row(1).push "yes_no", "yes" - choices.row(1).push "Yes" # Try adding onto the end of a row - choices.row(2).push "yes_no", "no", "No" + @form.preordered_items.each_with_index do |q, i| + questions.row(i+1).push(q.qtype.name, q.code, q.name, q.required.to_s, "TODO") + end + + # Choices TODO + choices.row(1).push("yes_no", "yes", "YES") + choices.row(2).push("yes_no", "no", "NO") # Settings - settings.row(0).push "form_title", "form_id", "version", "default_language" - settings.row(1).push "Test Form", 1, Time.now.strftime("%Y%m%d"), "English (en)" + settings.row(1).push(@form.name, @form.id, @form.updated_at.to_s, "English (en)") # Write file = StringIO.new book.write(file) - file.string.html_safe end From 72659acc5c52fb929659bcd97f8bcc9f0a504fdd Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 6 Sep 2023 11:38:00 -0400 Subject: [PATCH 03/34] Add groups to xls output --- app/controllers/forms_controller.rb | 2 +- app/models/forms/export.rb | 27 ++++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/app/controllers/forms_controller.rb b/app/controllers/forms_controller.rb index 9b6756d279..9b469d293a 100644 --- a/app/controllers/forms_controller.rb +++ b/app/controllers/forms_controller.rb @@ -238,7 +238,7 @@ def export_xml # XLSForm export. def export_xls exporter = Forms::Export.new(@form) - send_data(exporter.to_xls, filename: "test.xlsx") + send_data(exporter.to_xls, filename: "xlsform-#{@form.name.dasherize}-#{Time.zone.today}.xlsx") # send_file ... end diff --git a/app/models/forms/export.rb b/app/models/forms/export.rb index 43b020e176..b425c5395e 100644 --- a/app/models/forms/export.rb +++ b/app/models/forms/export.rb @@ -34,8 +34,28 @@ def to_xls choices.row(0).push("list_name", "name", "label") settings.row(0).push("form_title", "form_id", "version", "default_language") + group_tracker = false + index_mod = 1; @form.preordered_items.each_with_index do |q, i| - questions.row(i+1).push(q.qtype.name, q.code, q.name, q.required.to_s, "TODO") + if q.group? + questions.row(i+index_mod).push("begin group", q.code) + + # Toggle group tracker so we know we are in a group + group_tracker = true + else + # did a group just end? if so, the rank will be a whole number + if group_tracker && !q.full_dotted_rank.include?(".") + + # end the group and stop tracking it with the counter + questions.row(i+index_mod).push("end group") + group_tracker = false + + # increment our index modifier + index_mod += 1 + end + + questions.row(i+index_mod).push(q.qtype_name, q.full_dotted_rank + "_" + q.code, q.name, q.required.to_s, "TODO") + end end # Choices TODO @@ -78,5 +98,10 @@ def name(qing) "Group" end end + + def to_number(value) + return if value.blank? + (value.to_f % 1).positive? ? value.to_f : value.to_i + end end end From 0139d5eb03691cda89e4a85d8e80ed7b01a09d06 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 6 Sep 2023 12:27:56 -0400 Subject: [PATCH 04/34] Implement nested groups?? --- app/models/forms/export.rb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/app/models/forms/export.rb b/app/models/forms/export.rb index b425c5395e..9f355b878a 100644 --- a/app/models/forms/export.rb +++ b/app/models/forms/export.rb @@ -34,21 +34,21 @@ def to_xls choices.row(0).push("list_name", "name", "label") settings.row(0).push("form_title", "form_id", "version", "default_language") - group_tracker = false - index_mod = 1; + group_tracker = 1 # assume base level + index_mod = 1; # begin writing rows after headings @form.preordered_items.each_with_index do |q, i| if q.group? questions.row(i+index_mod).push("begin group", q.code) - # Toggle group tracker so we know we are in a group - group_tracker = true + # Increment our group tracker + group_tracker += 1 else - # did a group just end? if so, the rank will be a whole number - if group_tracker && !q.full_dotted_rank.include?(".") + # did a group just end? if so, the ancestry depth will be smaller than the tracker + if q.ancestry_depth < group_tracker - # end the group and stop tracking it with the counter + # end the group and decrement the tracker questions.row(i+index_mod).push("end group") - group_tracker = false + group_tracker -= 1 # increment our index modifier index_mod += 1 From 0f02fd23c3e3e08d5ff5bd67bede04abc3cc29d1 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 6 Sep 2023 14:42:02 -0400 Subject: [PATCH 05/34] Implement writing option sets to xls - almost there --- app/models/forms/export.rb | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/app/models/forms/export.rb b/app/models/forms/export.rb index 9f355b878a..473a80eeeb 100644 --- a/app/models/forms/export.rb +++ b/app/models/forms/export.rb @@ -35,7 +35,8 @@ def to_xls settings.row(0).push("form_title", "form_id", "version", "default_language") group_tracker = 1 # assume base level - index_mod = 1; # begin writing rows after headings + index_mod = 1 + choices_index_mod = 1 @form.preordered_items.each_with_index do |q, i| if q.group? questions.row(i+index_mod).push("begin group", q.code) @@ -54,14 +55,27 @@ def to_xls index_mod += 1 end - questions.row(i+index_mod).push(q.qtype_name, q.full_dotted_rank + "_" + q.code, q.name, q.required.to_s, "TODO") + # do we have an option set? + if q.option_set_id.present? + os = OptionSet.find(q.option_set_id) + os_name = " " + os.name # to respect XLSForm format + + os.option_nodes.each_with_index do |node, x| + if node.option.present? + choices.row(x + choices_index_mod).push(os.name, node.option.canonical_name, node.option.canonical_name) + end + end + + # increment the choices index by how many nodes there are, so we start at this row next time + choices_index_mod += os.option_nodes.length + else + os_name = "" + end + + questions.row(i+index_mod).push(q.qtype_name + os_name, q.full_dotted_rank + "_" + q.code, q.name, q.required.to_s, "TODO") end end - # Choices TODO - choices.row(1).push("yes_no", "yes", "YES") - choices.row(2).push("yes_no", "no", "NO") - # Settings settings.row(1).push(@form.name, @form.id, @form.updated_at.to_s, "English (en)") From 3b6e1014ecc236fbaa42173659e8283fba0e5639 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 7 Sep 2023 15:48:49 -0400 Subject: [PATCH 06/34] add begin/end repeat groups, end groups still buggy --- app/models/forms/export.rb | 64 ++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 20 deletions(-) diff --git a/app/models/forms/export.rb b/app/models/forms/export.rb index 473a80eeeb..ad266a1e32 100644 --- a/app/models/forms/export.rb +++ b/app/models/forms/export.rb @@ -21,44 +21,63 @@ def to_csv end end + # rubocop:disable Metrics/MethodLength, Metrics/BlockLength, Metrics/AbcSize, Metrics/PerceivedComplexity def to_xls + # TODO + # Repeat groups with nested "begin/end repeat" and "begin/end group" lines + # skip logic with formatted info in "relevant" column using ${name} = ... syntax + book = Spreadsheet::Workbook.new # Create sheets - questions = book.create_worksheet :name => "survey" - choices = book.create_worksheet :name => "choices" - settings = book.create_worksheet :name => "settings" + questions = book.create_worksheet(name: "survey") + choices = book.create_worksheet(name: "choices") + settings = book.create_worksheet(name: "settings") # Write sheet headings at row index 0 questions.row(0).push("type", "name", "label", "required", "relevant") choices.row(0).push("list_name", "name", "label") settings.row(0).push("form_title", "form_id", "version", "default_language") - group_tracker = 1 # assume base level - index_mod = 1 - choices_index_mod = 1 + group_depth = 1 # assume base level + repeat_depth = 0 # assume no repeat + index_mod = 1 # start at row index 1 + choices_index_mod = 1 # start at row index 1 + @form.preordered_items.each_with_index do |q, i| if q.group? - questions.row(i+index_mod).push("begin group", q.code) + if q.repeatable? + questions.row(i + index_mod).push("begin repeat", q.code) + repeat_depth += 1 + else + questions.row(i + index_mod).push("begin group", q.code) + end - # Increment our group tracker - group_tracker += 1 + # update counters + group_depth += 1 else - # did a group just end? if so, the ancestry depth will be smaller than the tracker - if q.ancestry_depth < group_tracker - - # end the group and decrement the tracker - questions.row(i+index_mod).push("end group") - group_tracker -= 1 + # did a group just end? + # if so, the qing's depth will be smaller than the depth counter + if q.ancestry_depth < group_depth + # are we in a repeat group? + # we don't want to end the repeat if we are ending a nested non-repeat group within a repeat + if repeat_depth.positive? + questions.row(i + index_mod).push("end repeat") + repeat_depth -= 1 + else + # end the group + questions.row(i + index_mod).push("end group") + end - # increment our index modifier - index_mod += 1 + # update counters + group_depth -= 1 + index_mod += 1 end # do we have an option set? if q.option_set_id.present? os = OptionSet.find(q.option_set_id) - os_name = " " + os.name # to respect XLSForm format + os_name = " #{os.name}" # to respect XLSForm format os.option_nodes.each_with_index do |node, x| if node.option.present? @@ -68,11 +87,15 @@ def to_xls # increment the choices index by how many nodes there are, so we start at this row next time choices_index_mod += os.option_nodes.length - else + else os_name = "" end - questions.row(i+index_mod).push(q.qtype_name + os_name, q.full_dotted_rank + "_" + q.code, q.name, q.required.to_s, "TODO") + type_to_push = "#{q.qtype_name}#{os_name}" + code_to_push = "#{q.full_dotted_rank}_#{q.code}" + + # Write the question row + questions.row(i + index_mod).push(type_to_push, code_to_push, q.name, q.required.to_s, "TODO") end end @@ -84,6 +107,7 @@ def to_xls book.write(file) file.string.html_safe end + # rubocop:enable Metrics/MethodLength, Metrics/BlockLength, Metrics/AbcSize, Metrics/PerceivedComplexity private From a83328b6d7cfdf910dada6b15f8333a6127c7862 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 8 Sep 2023 09:56:15 -0400 Subject: [PATCH 07/34] get end group working but in the wrong order --- app/models/forms/export.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/models/forms/export.rb b/app/models/forms/export.rb index ad266a1e32..0a55a43ce6 100644 --- a/app/models/forms/export.rb +++ b/app/models/forms/export.rb @@ -40,9 +40,9 @@ def to_xls settings.row(0).push("form_title", "form_id", "version", "default_language") group_depth = 1 # assume base level - repeat_depth = 0 # assume no repeat + repeat_depth = 1 index_mod = 1 # start at row index 1 - choices_index_mod = 1 # start at row index 1 + choices_index_mod = 1 @form.preordered_items.each_with_index do |q, i| if q.group? @@ -61,7 +61,8 @@ def to_xls if q.ancestry_depth < group_depth # are we in a repeat group? # we don't want to end the repeat if we are ending a nested non-repeat group within a repeat - if repeat_depth.positive? + # e.g., if group depth is deeper than repeat depth + if repeat_depth > 1 && repeat_depth >= group_depth questions.row(i + index_mod).push("end repeat") repeat_depth -= 1 else From a4d9d25363cc59f0b839ac6c1f57a2da0eeb48a5 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 8 Sep 2023 10:08:43 -0400 Subject: [PATCH 08/34] Implement end group while loop if multiple groups are ending --- app/models/forms/export.rb | 41 +++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/app/models/forms/export.rb b/app/models/forms/export.rb index 0a55a43ce6..33a72bf287 100644 --- a/app/models/forms/export.rb +++ b/app/models/forms/export.rb @@ -45,7 +45,25 @@ def to_xls choices_index_mod = 1 @form.preordered_items.each_with_index do |q, i| - if q.group? + # did one or more groups just end? + # if so, the qing's depth will be smaller than the depth counter + while group_depth > q.ancestry_depth + # are we in a repeat group? + # we don't want to end the repeat if we are ending a nested non-repeat group within a repeat + if repeat_depth > 1 && repeat_depth >= group_depth + questions.row(i + index_mod).push("end repeat") + repeat_depth -= 1 + else + # end the group + questions.row(i + index_mod).push("end group") + end + + # update counters + group_depth -= 1 + index_mod += 1 + end + + if q.group? #is this a group? if q.repeatable? questions.row(i + index_mod).push("begin repeat", q.code) repeat_depth += 1 @@ -55,26 +73,7 @@ def to_xls # update counters group_depth += 1 - else - # did a group just end? - # if so, the qing's depth will be smaller than the depth counter - if q.ancestry_depth < group_depth - # are we in a repeat group? - # we don't want to end the repeat if we are ending a nested non-repeat group within a repeat - # e.g., if group depth is deeper than repeat depth - if repeat_depth > 1 && repeat_depth >= group_depth - questions.row(i + index_mod).push("end repeat") - repeat_depth -= 1 - else - # end the group - questions.row(i + index_mod).push("end group") - end - - # update counters - group_depth -= 1 - index_mod += 1 - end - + else # is this a question? # do we have an option set? if q.option_set_id.present? os = OptionSet.find(q.option_set_id) From 606a0ffe9d8d7a53eaa9dac5984fa4462747dcc5 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 15 Sep 2023 10:29:41 -0400 Subject: [PATCH 09/34] Implement display logic --- app/models/forms/export.rb | 65 +++++++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/app/models/forms/export.rb b/app/models/forms/export.rb index 33a72bf287..bb592982fa 100644 --- a/app/models/forms/export.rb +++ b/app/models/forms/export.rb @@ -24,8 +24,9 @@ def to_csv # rubocop:disable Metrics/MethodLength, Metrics/BlockLength, Metrics/AbcSize, Metrics/PerceivedComplexity def to_xls # TODO - # Repeat groups with nested "begin/end repeat" and "begin/end group" lines - # skip logic with formatted info in "relevant" column using ${name} = ... syntax + # - Constraints + # - option set "levels"? + # - Make question types compatible with XLSForm, e.g., "long_text" should just be "text", "counter" does not exist, etc. book = Spreadsheet::Workbook.new @@ -35,7 +36,7 @@ def to_xls settings = book.create_worksheet(name: "settings") # Write sheet headings at row index 0 - questions.row(0).push("type", "name", "label", "required", "relevant") + questions.row(0).push("type", "name", "label", "required", "relevant", "constraint") choices.row(0).push("list_name", "name", "label") settings.row(0).push("form_title", "form_id", "version", "default_language") @@ -95,7 +96,63 @@ def to_xls code_to_push = "#{q.full_dotted_rank}_#{q.code}" # Write the question row - questions.row(i + index_mod).push(type_to_push, code_to_push, q.name, q.required.to_s, "TODO") + questions.row(i + index_mod).push(type_to_push, code_to_push, q.name, q.required.to_s) + end + + # if we have any relevant conditions, add them to the end of the row + if q.display_conditions.any? + # start building string of conditions to print + relevant_to_push = "" + concatenator = q.display_if == "all_met" ? "and" : "or" + + # how many conditions? + dc_length = q.display_conditions.length + + q.display_conditions.each_with_index do |dc, i| + # prep left side of expression + left_qing = Questioning.find(dc.left_qing_id) + left_to_push = "${#{left_qing.full_dotted_rank}_#{left_qing.code}}" + + # prep right side of expression + if dc.right_side_is_qing? + right_qing = Questioning.find(dc.right_qing_id) + right_to_push = "${#{right_qing.full_dotted_rank}_#{right_qing.code}}" + else + # to respect XLSform rules, surround with single quotes unless it's a number + if Float(dc.value, exception: false).nil? # it's not a number + right_to_push = "'#{dc.value}'" + else + right_to_push = "#{dc.value}" + end + end + + case dc.op + when "eq" + op = "=" + when "neq" + op = "!=" + when "lt" + op = "<" + when "leq" + op = "<=" + when "gt" + op = ">" + when "geq" + op = ">=" + end + + # omit the concatenator on the last condition only + if i + 1 == dc_length + relevant_to_push = "#{relevant_to_push}#{left_to_push} #{op} #{right_to_push}" + else + relevant_to_push = "#{relevant_to_push}#{left_to_push} #{op} #{right_to_push} #{concatenator} " + end + end + questions.row(i + index_mod).push(relevant_to_push) + end + + if q.constraints.any? + #TODO end end From 39c64fdb8cae60bb29cb9e15592018f5bd16764a Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 15 Sep 2023 12:12:04 -0400 Subject: [PATCH 10/34] implement constraints --- app/models/forms/export.rb | 107 ++++++++++++++++++++----------------- 1 file changed, 58 insertions(+), 49 deletions(-) diff --git a/app/models/forms/export.rb b/app/models/forms/export.rb index bb592982fa..6ba4d5ce27 100644 --- a/app/models/forms/export.rb +++ b/app/models/forms/export.rb @@ -101,58 +101,15 @@ def to_xls # if we have any relevant conditions, add them to the end of the row if q.display_conditions.any? - # start building string of conditions to print - relevant_to_push = "" - concatenator = q.display_if == "all_met" ? "and" : "or" - - # how many conditions? - dc_length = q.display_conditions.length - - q.display_conditions.each_with_index do |dc, i| - # prep left side of expression - left_qing = Questioning.find(dc.left_qing_id) - left_to_push = "${#{left_qing.full_dotted_rank}_#{left_qing.code}}" - - # prep right side of expression - if dc.right_side_is_qing? - right_qing = Questioning.find(dc.right_qing_id) - right_to_push = "${#{right_qing.full_dotted_rank}_#{right_qing.code}}" - else - # to respect XLSform rules, surround with single quotes unless it's a number - if Float(dc.value, exception: false).nil? # it's not a number - right_to_push = "'#{dc.value}'" - else - right_to_push = "#{dc.value}" - end - end - - case dc.op - when "eq" - op = "=" - when "neq" - op = "!=" - when "lt" - op = "<" - when "leq" - op = "<=" - when "gt" - op = ">" - when "geq" - op = ">=" - end - - # omit the concatenator on the last condition only - if i + 1 == dc_length - relevant_to_push = "#{relevant_to_push}#{left_to_push} #{op} #{right_to_push}" - else - relevant_to_push = "#{relevant_to_push}#{left_to_push} #{op} #{right_to_push} #{concatenator} " - end - end - questions.row(i + index_mod).push(relevant_to_push) + questions.row(i + index_mod).push(conditions_to_xls(q.display_conditions, q.display_if)) + else + questions.row(i + index_mod).push("") end if q.constraints.any? - #TODO + q.constraints.each do |c| + questions.row(i + index_mod).push(conditions_to_xls(c.conditions, c.accept_if)) + end end end @@ -198,5 +155,57 @@ def to_number(value) return if value.blank? (value.to_f % 1).positive? ? value.to_f : value.to_i end + + # Takes an array of conditions and outputs a single string + # concatenates by either "and" or "or" depending on form settings + def conditions_to_xls(conditions, true_if) + relevant_to_push = "" + concatenator = true_if == "all_met" ? "and" : "or" + + # how many conditions? + dc_length = conditions.length + + conditions.each_with_index do |dc, i| + # prep left side of expression + left_qing = Questioning.find(dc.left_qing_id) + left_to_push = "${#{left_qing.full_dotted_rank}_#{left_qing.code}}" + + # prep right side of expression + if dc.right_side_is_qing? + right_qing = Questioning.find(dc.right_qing_id) + right_to_push = "${#{right_qing.full_dotted_rank}_#{right_qing.code}}" + else + # to respect XLSform rules, surround with single quotes unless it's a number + if Float(dc.value, exception: false).nil? # it's not a number + right_to_push = "'#{dc.value}'" + else + right_to_push = "#{dc.value}" + end + end + + case dc.op + when "eq" + op = "=" + when "neq" + op = "!=" + when "lt" + op = "<" + when "leq" + op = "<=" + when "gt" + op = ">" + when "geq" + op = ">=" + end + + # omit the concatenator on the last condition only + if i + 1 == dc_length + relevant_to_push = "#{relevant_to_push}#{left_to_push} #{op} #{right_to_push}" + else + relevant_to_push = "#{relevant_to_push}#{left_to_push} #{op} #{right_to_push} #{concatenator} " + end + end + return relevant_to_push + end end end From 7f68fa5bf1aa4005ca25bf87029a177f0866320a Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 15 Sep 2023 13:08:13 -0400 Subject: [PATCH 11/34] Address Hound warnings --- Gemfile | 2 +- app/models/forms/export.rb | 39 ++++++++++++++++++-------------------- 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/Gemfile b/Gemfile index 414a6a823c..742631e6fd 100644 --- a/Gemfile +++ b/Gemfile @@ -20,11 +20,11 @@ gem "recaptcha", "~> 3.4", require: "recaptcha/rails" # Small change in v4, we s gem "responders", "~> 3.0" gem "rqrcode", "~> 1.1" gem "rubyzip", "~> 2.3", require: "zip" # Explicitly specify name (https://stackoverflow.com/a/32740666/763231) +gem "spreadsheet" # For XLSForm export gem "term-ansicolor", "~> 1.3" gem "terrapin", "~> 0.6.0" gem "thor", "~> 1.0" gem "twilio-ruby", "~> 4.2" # Does not use semver after v5, watch out! -gem "spreadsheet" # For XLSForm export # JS/CSS gem "bootstrap", "~> 4.3" diff --git a/app/models/forms/export.rb b/app/models/forms/export.rb index 6ba4d5ce27..24083926cc 100644 --- a/app/models/forms/export.rb +++ b/app/models/forms/export.rb @@ -24,9 +24,8 @@ def to_csv # rubocop:disable Metrics/MethodLength, Metrics/BlockLength, Metrics/AbcSize, Metrics/PerceivedComplexity def to_xls # TODO - # - Constraints - # - option set "levels"? - # - Make question types compatible with XLSForm, e.g., "long_text" should just be "text", "counter" does not exist, etc. + # option set "levels"? + # Make question types compatible with XLSForm, e.g., "long_text" should just be "text", "counter" does not exist, etc. book = Spreadsheet::Workbook.new @@ -43,7 +42,7 @@ def to_xls group_depth = 1 # assume base level repeat_depth = 1 index_mod = 1 # start at row index 1 - choices_index_mod = 1 + choices_index_mod = 0 @form.preordered_items.each_with_index do |q, i| # did one or more groups just end? @@ -64,7 +63,7 @@ def to_xls index_mod += 1 end - if q.group? #is this a group? + if q.group? # is this a group? if q.repeatable? questions.row(i + index_mod).push("begin repeat", q.code) repeat_depth += 1 @@ -174,35 +173,33 @@ def conditions_to_xls(conditions, true_if) if dc.right_side_is_qing? right_qing = Questioning.find(dc.right_qing_id) right_to_push = "${#{right_qing.full_dotted_rank}_#{right_qing.code}}" - else + elsif Float(dc.value, exception: false).nil? # it's not a number # to respect XLSform rules, surround with single quotes unless it's a number - if Float(dc.value, exception: false).nil? # it's not a number - right_to_push = "'#{dc.value}'" - else - right_to_push = "#{dc.value}" - end + right_to_push = "'#{dc.value}'" + else + right_to_push = dc.value.to_s end - case dc.op + op = case dc.op when "eq" - op = "=" + "=" when "neq" - op = "!=" + "!=" when "lt" - op = "<" + "<" when "leq" - op = "<=" + "<=" when "gt" - op = ">" + ">" when "geq" - op = ">=" + ">=" end # omit the concatenator on the last condition only - if i + 1 == dc_length - relevant_to_push = "#{relevant_to_push}#{left_to_push} #{op} #{right_to_push}" + relevant_to_push = if i + 1 == dc_length + "#{relevant_to_push}#{left_to_push} #{op} #{right_to_push}" else - relevant_to_push = "#{relevant_to_push}#{left_to_push} #{op} #{right_to_push} #{concatenator} " + "#{relevant_to_push}#{left_to_push} #{op} #{right_to_push} #{concatenator} " end end return relevant_to_push From 6cfaf72a46ec7e7ab42339e174e7c93d6a769b2d Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 15 Sep 2023 13:16:25 -0400 Subject: [PATCH 12/34] Address more hound errors --- app/models/forms/export.rb | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/app/models/forms/export.rb b/app/models/forms/export.rb index 24083926cc..8ebedec7a6 100644 --- a/app/models/forms/export.rb +++ b/app/models/forms/export.rb @@ -8,6 +8,15 @@ class Export DisplayLogic DisplayConditions Default Hidden ].freeze + OPERATIONS = { + "eq" => "=", + "neq" => "!=", + "lt" => "<", + "leq" => "<=", + "gt" => ">", + "geq" => ">=" + } + def initialize(form) @form = form end @@ -180,27 +189,14 @@ def conditions_to_xls(conditions, true_if) right_to_push = dc.value.to_s end - op = case dc.op - when "eq" - "=" - when "neq" - "!=" - when "lt" - "<" - when "leq" - "<=" - when "gt" - ">" - when "geq" - ">=" - end + op = OPERATIONS[dc.op] # omit the concatenator on the last condition only relevant_to_push = if i + 1 == dc_length - "#{relevant_to_push}#{left_to_push} #{op} #{right_to_push}" - else - "#{relevant_to_push}#{left_to_push} #{op} #{right_to_push} #{concatenator} " - end + "#{relevant_to_push}#{left_to_push} #{op} #{right_to_push}" + else + "#{relevant_to_push}#{left_to_push} #{op} #{right_to_push} #{concatenator} " + end end return relevant_to_push end From 9bbcf5c211887900d4e90847278fc15f81767b4b Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 15 Sep 2023 13:18:24 -0400 Subject: [PATCH 13/34] Further appease the hound --- app/models/forms/export.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/forms/export.rb b/app/models/forms/export.rb index 8ebedec7a6..80ca732719 100644 --- a/app/models/forms/export.rb +++ b/app/models/forms/export.rb @@ -15,7 +15,7 @@ class Export "leq" => "<=", "gt" => ">", "geq" => ">=" - } + }.freeze def initialize(form) @form = form From 6959a09abccc06eb0f7e87e43e700b501279edce Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 25 Sep 2023 09:00:51 -0400 Subject: [PATCH 14/34] Add translation and icon --- app/helpers/icon_helper.rb | 1 + config/locales/en/main.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/app/helpers/icon_helper.rb b/app/helpers/icon_helper.rb index e87877879f..7c2ec8908f 100644 --- a/app/helpers/icon_helper.rb +++ b/app/helpers/icon_helper.rb @@ -11,6 +11,7 @@ module IconHelper export: "download", export_csv: "download", export_xml: "download", + export_xls: "download", go_live: "play", import: "upload", index: "list", diff --git a/config/locales/en/main.yml b/config/locales/en/main.yml index 75f1757984..eee860b0d7 100644 --- a/config/locales/en/main.yml +++ b/config/locales/en/main.yml @@ -921,6 +921,7 @@ en: import_from_csv: "Import from CSV" export_csv: "Export CSV" export_xml: "Export XML" + export_xls: "Export XLSForm" export: "Export" models: form: From dd181707e549ce5f2af56dde9d641787ea43943c Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 25 Sep 2023 09:45:07 -0400 Subject: [PATCH 15/34] Change to .xls output rather than .xlsx, so that excel does not call the output file invalid --- app/controllers/forms_controller.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/controllers/forms_controller.rb b/app/controllers/forms_controller.rb index 9b469d293a..36e90191e6 100644 --- a/app/controllers/forms_controller.rb +++ b/app/controllers/forms_controller.rb @@ -238,8 +238,7 @@ def export_xml # XLSForm export. def export_xls exporter = Forms::Export.new(@form) - send_data(exporter.to_xls, filename: "xlsform-#{@form.name.dasherize}-#{Time.zone.today}.xlsx") - # send_file ... + send_data(exporter.to_xls, filename: "xlsform-#{@form.name.dasherize}-#{Time.zone.today}.xls") end # ODK XML export for all published forms. From 182a45debffe10f67da47037b68c20c96fdfe441 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 25 Sep 2023 10:19:57 -0400 Subject: [PATCH 16/34] Add NEMO -> XLSForm qtype conversion table --- app/models/forms/export.rb | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/app/models/forms/export.rb b/app/models/forms/export.rb index 80ca732719..9da18384a8 100644 --- a/app/models/forms/export.rb +++ b/app/models/forms/export.rb @@ -17,6 +17,34 @@ class Export "geq" => ">=" }.freeze + QTYPE_TO_XLS = { + # conversions + "location" => "geopoint", + "long_text" => "text", + "datetime" => "dateTime", + "annotated_image" => "image", + "counter" => "integer", + + # no change + "text" => "text", + "select_one" => "select_one", + "select_multiple" => "select_multiple", + "decimal" => "decimal", + "time" => "time", + "date" => "date", + "image" => "image", + "barcode" => "barcode", + "audio" => "audio", + "video" => "video", + "integer" => "integer", + + # Not supported in XLSForm + "sketch" => "sketch (WARNING: not supported)", + "signature" => "signature (WARNING: not supported)", + + # XLSForm qtypes not supported in NEMO: range, geotrace, geoshape, note, file, select_one_from_file, select_multiple_from_file, background-audio, calculate, acknowledge, hidden, xml-external + }.freeze + def initialize(form) @form = form end @@ -100,7 +128,10 @@ def to_xls os_name = "" end - type_to_push = "#{q.qtype_name}#{os_name}" + # convert question types + qtype_converted = QTYPE_TO_XLS[q.qtype_name] + + type_to_push = "#{qtype_converted}#{os_name}" code_to_push = "#{q.full_dotted_rank}_#{q.code}" # Write the question row From 9114c7f6b8b88caba57954a2c6731146b7b64e87 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 25 Sep 2023 10:26:25 -0400 Subject: [PATCH 17/34] Remove trailing comma and a comment --- app/models/forms/export.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/app/models/forms/export.rb b/app/models/forms/export.rb index 9da18384a8..0a54cd8182 100644 --- a/app/models/forms/export.rb +++ b/app/models/forms/export.rb @@ -40,7 +40,7 @@ class Export # Not supported in XLSForm "sketch" => "sketch (WARNING: not supported)", - "signature" => "signature (WARNING: not supported)", + "signature" => "signature (WARNING: not supported)" # XLSForm qtypes not supported in NEMO: range, geotrace, geoshape, note, file, select_one_from_file, select_multiple_from_file, background-audio, calculate, acknowledge, hidden, xml-external }.freeze @@ -60,9 +60,7 @@ def to_csv # rubocop:disable Metrics/MethodLength, Metrics/BlockLength, Metrics/AbcSize, Metrics/PerceivedComplexity def to_xls - # TODO - # option set "levels"? - # Make question types compatible with XLSForm, e.g., "long_text" should just be "text", "counter" does not exist, etc. + # TODO: option set "levels"? book = Spreadsheet::Workbook.new From 658df7ada8945183146e851a497ad2b005224b3a Mon Sep 17 00:00:00 2001 From: Kevin Cooper Date: Thu, 12 Oct 2023 17:49:55 -0400 Subject: [PATCH 18/34] shush a few hounds --- app/controllers/forms_controller.rb | 2 +- app/models/forms/export.rb | 23 ++++++++++++++--------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/app/controllers/forms_controller.rb b/app/controllers/forms_controller.rb index 36e90191e6..c8275277e4 100644 --- a/app/controllers/forms_controller.rb +++ b/app/controllers/forms_controller.rb @@ -238,7 +238,7 @@ def export_xml # XLSForm export. def export_xls exporter = Forms::Export.new(@form) - send_data(exporter.to_xls, filename: "xlsform-#{@form.name.dasherize}-#{Time.zone.today}.xls") + send_data(exporter.to_xls.html_safe, filename: "xlsform-#{@form.name.dasherize}-#{Time.zone.today}.xls") # rubocop:disable Rails/OutputSafety end # ODK XML export for all published forms. diff --git a/app/models/forms/export.rb b/app/models/forms/export.rb index 0a54cd8182..348dae92d3 100644 --- a/app/models/forms/export.rb +++ b/app/models/forms/export.rb @@ -38,11 +38,13 @@ class Export "video" => "video", "integer" => "integer", - # Not supported in XLSForm - "sketch" => "sketch (WARNING: not supported)", - "signature" => "signature (WARNING: not supported)" + # TODO: Not yet supported in our XLSForm exporter + "sketch" => "sketch (WARNING: not yet supported)", + "signature" => "signature (WARNING: not yet supported)" - # XLSForm qtypes not supported in NEMO: range, geotrace, geoshape, note, file, select_one_from_file, select_multiple_from_file, background-audio, calculate, acknowledge, hidden, xml-external + # Note: XLSForm qtypes not supported in NEMO: + # range, geotrace, geoshape, note, file, select_one_from_file, select_multiple_from_file, + # background-audio, calculate, acknowledge, hidden, xml-external }.freeze def initialize(form) @@ -58,7 +60,7 @@ def to_csv end end - # rubocop:disable Metrics/MethodLength, Metrics/BlockLength, Metrics/AbcSize, Metrics/PerceivedComplexity + # rubocop:disable Metrics/MethodLength, Metrics/BlockLength, Metrics/AbcSize, Metrics/PerceivedComplexity, Style/Next def to_xls # TODO: option set "levels"? @@ -116,7 +118,9 @@ def to_xls os.option_nodes.each_with_index do |node, x| if node.option.present? - choices.row(x + choices_index_mod).push(os.name, node.option.canonical_name, node.option.canonical_name) + choices + .row(x + choices_index_mod) + .push(os.name, node.option.canonical_name, node.option.canonical_name) end end @@ -156,9 +160,9 @@ def to_xls # Write file = StringIO.new book.write(file) - file.string.html_safe + file.string end - # rubocop:enable Metrics/MethodLength, Metrics/BlockLength, Metrics/AbcSize, Metrics/PerceivedComplexity + # rubocop:enable Metrics/MethodLength, Metrics/BlockLength, Metrics/AbcSize, Metrics/PerceivedComplexity, Style/Next private @@ -227,7 +231,8 @@ def conditions_to_xls(conditions, true_if) "#{relevant_to_push}#{left_to_push} #{op} #{right_to_push} #{concatenator} " end end - return relevant_to_push + + relevant_to_push end end end From 31bf094d28638e2bb0d076009711b0a7c7ac4ca0 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 24 Oct 2023 11:02:23 -0400 Subject: [PATCH 19/34] Resolve merge conflicts --- app/models/forms/export.rb | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/app/models/forms/export.rb b/app/models/forms/export.rb index 348dae92d3..fa1f892b09 100644 --- a/app/models/forms/export.rb +++ b/app/models/forms/export.rb @@ -80,6 +80,7 @@ def to_xls repeat_depth = 1 index_mod = 1 # start at row index 1 choices_index_mod = 0 + option_sets_used = Array.new @form.preordered_items.each_with_index do |q, i| # did one or more groups just end? @@ -114,18 +115,23 @@ def to_xls # do we have an option set? if q.option_set_id.present? os = OptionSet.find(q.option_set_id) - os_name = " #{os.name}" # to respect XLSForm format + os_name = " #{os.name}" + os_already_logged = option_sets_used.include? q.option_set_id - os.option_nodes.each_with_index do |node, x| - if node.option.present? - choices + # log the option set to the spreadsheet if we haven't yet + unless os_already_logged + os.option_nodes.each_with_index do |node, x| + if node.option.present? .row(x + choices_index_mod) .push(os.name, node.option.canonical_name, node.option.canonical_name) + end end - end - # increment the choices index by how many nodes there are, so we start at this row next time - choices_index_mod += os.option_nodes.length + # increment the choices index by how many nodes there are, so we start at this row next time + choices_index_mod += os.option_nodes.length + + option_sets_used.push(q.option_set_id) + end else os_name = "" end From 880af49c9f4d1906543fb099208eba52663e59fe Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 24 Oct 2023 11:12:54 -0400 Subject: [PATCH 20/34] Resolve conflicts --- app/models/forms/export.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/models/forms/export.rb b/app/models/forms/export.rb index fa1f892b09..9a624cd586 100644 --- a/app/models/forms/export.rb +++ b/app/models/forms/export.rb @@ -122,8 +122,9 @@ def to_xls unless os_already_logged os.option_nodes.each_with_index do |node, x| if node.option.present? - .row(x + choices_index_mod) - .push(os.name, node.option.canonical_name, node.option.canonical_name) + choices + .row(x + choices_index_mod) + .push(os.name, node.option.canonical_name, node.option.canonical_name) end end From 9999629093ee4f25dfe1c24ac7073e9fa5140564 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 24 Oct 2023 11:39:21 -0400 Subject: [PATCH 21/34] Resolve hound warnings --- app/models/forms/export.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/forms/export.rb b/app/models/forms/export.rb index 9a624cd586..2d2e0a66b7 100644 --- a/app/models/forms/export.rb +++ b/app/models/forms/export.rb @@ -80,7 +80,7 @@ def to_xls repeat_depth = 1 index_mod = 1 # start at row index 1 choices_index_mod = 0 - option_sets_used = Array.new + option_sets_used = [] @form.preordered_items.each_with_index do |q, i| # did one or more groups just end? @@ -116,7 +116,7 @@ def to_xls if q.option_set_id.present? os = OptionSet.find(q.option_set_id) os_name = " #{os.name}" - os_already_logged = option_sets_used.include? q.option_set_id + os_already_logged = option_sets_used.include?(q.option_set_id) # log the option set to the spreadsheet if we haven't yet unless os_already_logged From bbeb30ee7069787fbe3ff8a3bc9d1a9503ffc2d3 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 31 Oct 2023 12:20:16 -0400 Subject: [PATCH 22/34] Add spec for xlsform export --- .../forms/form/xlsform_export_spec.rb | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 spec/features/forms/form/xlsform_export_spec.rb diff --git a/spec/features/forms/form/xlsform_export_spec.rb b/spec/features/forms/form/xlsform_export_spec.rb new file mode 100644 index 0000000000..8308ac2318 --- /dev/null +++ b/spec/features/forms/form/xlsform_export_spec.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +require "rails_helper" +require "fileutils" +require "zip" + +feature "xlsform export" do + context "single form" do + let(:user) { create(:user, role_name: "coordinator") } + let(:form) { create(:form, name: "Export Test", question_types: %w[text]) } + + before do + login(user) + end + + it "XML exports successfully" do + visit(form_path(form, locale: "en", mode: "m", mission_name: get_mission.compact_name)) + click_link("Export XLSForm") + + expect(page.current_url).to match("export_xls") + expect(page.body).to match("#{form.name}") + expect(page.body).to match("text") + end + end +end From 19eb4b5abfeca4cf3e265c550362c8a11c2bbcfe Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 31 Oct 2023 14:24:05 -0400 Subject: [PATCH 23/34] Update test name and resolve hound warning --- spec/features/forms/form/xlsform_export_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/features/forms/form/xlsform_export_spec.rb b/spec/features/forms/form/xlsform_export_spec.rb index 8308ac2318..30e4f17977 100644 --- a/spec/features/forms/form/xlsform_export_spec.rb +++ b/spec/features/forms/form/xlsform_export_spec.rb @@ -13,12 +13,12 @@ login(user) end - it "XML exports successfully" do + it "exports to XLSForm successfully" do visit(form_path(form, locale: "en", mode: "m", mission_name: get_mission.compact_name)) click_link("Export XLSForm") expect(page.current_url).to match("export_xls") - expect(page.body).to match("#{form.name}") + expect(page.body).to match(form.name.to_s) expect(page.body).to match("text") end end From 17df9e2dc2f964d514cefdb0247a7c813afaa889 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 31 Oct 2023 16:57:39 -0400 Subject: [PATCH 24/34] Add explanatory comments; fix odk upload error --- app/models/forms/export.rb | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/app/models/forms/export.rb b/app/models/forms/export.rb index 2d2e0a66b7..1b21c19c92 100644 --- a/app/models/forms/export.rb +++ b/app/models/forms/export.rb @@ -78,9 +78,14 @@ def to_xls group_depth = 1 # assume base level repeat_depth = 1 + option_sets_used = [] + + # Define the below "index modifiers" which keep track of the line of the spreadsheet we are writing to. + # The for loop below (tracked by index i) loops through the list of form items, and so the index does not take into account rows that we need to write for when groups end. In XLSForm, these are written to a row all to themselves. + # This causes the index i to be de-synchronized with the row of the spreadsheet that we are writing to. + # Hence, we push to the row (i + index_mod) index_mod = 1 # start at row index 1 choices_index_mod = 0 - option_sets_used = [] @form.preordered_items.each_with_index do |q, i| # did one or more groups just end? @@ -115,6 +120,9 @@ def to_xls # do we have an option set? if q.option_set_id.present? os = OptionSet.find(q.option_set_id) + + # include leading space to respect XLSForm format + # question name should be followed by the option set name (if applicable) separated by a space os_name = " #{os.name}" os_already_logged = option_sets_used.include?(q.option_set_id) @@ -141,7 +149,7 @@ def to_xls qtype_converted = QTYPE_TO_XLS[q.qtype_name] type_to_push = "#{qtype_converted}#{os_name}" - code_to_push = "#{q.full_dotted_rank}_#{q.code}" + code_to_push = "#{q.code}_#{q.full_dotted_rank}" # Write the question row questions.row(i + index_mod).push(type_to_push, code_to_push, q.name, q.required.to_s) @@ -199,11 +207,6 @@ def name(qing) end end - def to_number(value) - return if value.blank? - (value.to_f % 1).positive? ? value.to_f : value.to_i - end - # Takes an array of conditions and outputs a single string # concatenates by either "and" or "or" depending on form settings def conditions_to_xls(conditions, true_if) From a99e4f06e956be2ff41ddbb747a228014ac8a64b Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 Nov 2023 11:21:11 -0400 Subject: [PATCH 25/34] Successful test upload to odk central --- app/models/forms/export.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/models/forms/export.rb b/app/models/forms/export.rb index 1b21c19c92..8a884fe6a6 100644 --- a/app/models/forms/export.rb +++ b/app/models/forms/export.rb @@ -158,8 +158,6 @@ def to_xls # if we have any relevant conditions, add them to the end of the row if q.display_conditions.any? questions.row(i + index_mod).push(conditions_to_xls(q.display_conditions, q.display_if)) - else - questions.row(i + index_mod).push("") end if q.constraints.any? @@ -170,7 +168,7 @@ def to_xls end # Settings - settings.row(1).push(@form.name, @form.id, @form.updated_at.to_s, "English (en)") + settings.row(1).push(@form.name, @form.id, @form.current_version.decorate.name, "English (en)") # Write file = StringIO.new From 94c82d8757c046897730ccbcb4776a56ee78a574 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 Nov 2023 11:59:59 -0400 Subject: [PATCH 26/34] replace spaces with underscores in option set name --- app/models/forms/export.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/models/forms/export.rb b/app/models/forms/export.rb index 8a884fe6a6..d8375294b2 100644 --- a/app/models/forms/export.rb +++ b/app/models/forms/export.rb @@ -123,7 +123,8 @@ def to_xls # include leading space to respect XLSForm format # question name should be followed by the option set name (if applicable) separated by a space - os_name = " #{os.name}" + # replace any spaces in the option set name with underscores to ensure the form is parsed correctly + os_name = " #{os.name.tr(" ", "_")}" os_already_logged = option_sets_used.include?(q.option_set_id) # log the option set to the spreadsheet if we haven't yet From 8bf88adc4682d29b6b39d3c0334f8c0b97b071ec Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 1 Nov 2023 12:26:03 -0400 Subject: [PATCH 27/34] Add explanatory comment --- app/models/forms/export.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/models/forms/export.rb b/app/models/forms/export.rb index d8375294b2..dd2cc91494 100644 --- a/app/models/forms/export.rb +++ b/app/models/forms/export.rb @@ -128,11 +128,14 @@ def to_xls os_already_logged = option_sets_used.include?(q.option_set_id) # log the option set to the spreadsheet if we haven't yet + # ni = index for the option nodes loop + # node = the current option node + # TODO: support option set "levels" by creating a cascading sheet here unless os_already_logged - os.option_nodes.each_with_index do |node, x| + os.option_nodes.each_with_index do |node, ni| if node.option.present? choices - .row(x + choices_index_mod) + .row(ni + choices_index_mod) .push(os.name, node.option.canonical_name, node.option.canonical_name) end end From af58a0f8c66ae43b1cf08ffb9f577054ab39736e Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 2 Nov 2023 10:36:26 -0400 Subject: [PATCH 28/34] fix more odk upload and parsing errors --- app/models/forms/export.rb | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/app/models/forms/export.rb b/app/models/forms/export.rb index dd2cc91494..2a0f3ca876 100644 --- a/app/models/forms/export.rb +++ b/app/models/forms/export.rb @@ -107,11 +107,13 @@ def to_xls end if q.group? # is this a group? + group_name = q.code.tr(" ", "_") + if q.repeatable? - questions.row(i + index_mod).push("begin repeat", q.code) + questions.row(i + index_mod).push("begin repeat", group_name, q.code) repeat_depth += 1 else - questions.row(i + index_mod).push("begin group", q.code) + questions.row(i + index_mod).push("begin group", group_name, q.code) end # update counters @@ -124,7 +126,7 @@ def to_xls # include leading space to respect XLSForm format # question name should be followed by the option set name (if applicable) separated by a space # replace any spaces in the option set name with underscores to ensure the form is parsed correctly - os_name = " #{os.name.tr(" ", "_")}" + os_name = os.name.tr(" ", "_") os_already_logged = option_sets_used.include?(q.option_set_id) # log the option set to the spreadsheet if we haven't yet @@ -136,7 +138,7 @@ def to_xls if node.option.present? choices .row(ni + choices_index_mod) - .push(os.name, node.option.canonical_name, node.option.canonical_name) + .push(os_name, node.option.canonical_name, node.option.canonical_name) end end @@ -152,7 +154,7 @@ def to_xls # convert question types qtype_converted = QTYPE_TO_XLS[q.qtype_name] - type_to_push = "#{qtype_converted}#{os_name}" + type_to_push = "#{qtype_converted} #{os_name}" code_to_push = "#{q.code}_#{q.full_dotted_rank}" # Write the question row @@ -221,12 +223,12 @@ def conditions_to_xls(conditions, true_if) conditions.each_with_index do |dc, i| # prep left side of expression left_qing = Questioning.find(dc.left_qing_id) - left_to_push = "${#{left_qing.full_dotted_rank}_#{left_qing.code}}" + left_to_push = "${#{left_qing.code}_#{left_qing.full_dotted_rank}}" # prep right side of expression if dc.right_side_is_qing? right_qing = Questioning.find(dc.right_qing_id) - right_to_push = "${#{right_qing.full_dotted_rank}_#{right_qing.code}}" + right_to_push = "${#{right_qing.code}_#{right_qing.full_dotted_rank}}" elsif Float(dc.value, exception: false).nil? # it's not a number # to respect XLSform rules, surround with single quotes unless it's a number right_to_push = "'#{dc.value}'" From cf836ad1e175ed3b3c110fdbe4bc6743e3252d7e Mon Sep 17 00:00:00 2001 From: Kevin Cooper Date: Thu, 2 Nov 2023 13:07:19 -0400 Subject: [PATCH 29/34] remove temp LastKeywordArgument rule to fix rubocop --- .rubocop.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 0786dc0875..010d1e2dcb 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -54,9 +54,6 @@ Layout/LineLength: Lint/RedundantCopDisableDirective: Enabled: false -Lint/LastKeywordArgument: - Enabled: true - Lint/UnusedMethodArgument: # Otherwise we have to remove named args from the method signature altogether which seems to reduce # clarity of the code. From 0abef1f3e9bd04ed3564a7f545b9b5b5ec88c030 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 3 Nov 2023 12:50:45 -0400 Subject: [PATCH 30/34] Consolidate form export specs. Add mission default language to settings tab --- app/models/forms/export.rb | 3 ++- spec/features/forms/form/form_export_spec.rb | 9 +++++++ .../forms/form/xlsform_export_spec.rb | 25 ------------------- 3 files changed, 11 insertions(+), 26 deletions(-) delete mode 100644 spec/features/forms/form/xlsform_export_spec.rb diff --git a/app/models/forms/export.rb b/app/models/forms/export.rb index 2a0f3ca876..1e39b1fddc 100644 --- a/app/models/forms/export.rb +++ b/app/models/forms/export.rb @@ -174,7 +174,8 @@ def to_xls end # Settings - settings.row(1).push(@form.name, @form.id, @form.current_version.decorate.name, "English (en)") + lang = @form.mission.setting.preferred_locales[0].to_s + settings.row(1).push(@form.name, @form.id, @form.current_version.decorate.name, lang) # Write file = StringIO.new diff --git a/spec/features/forms/form/form_export_spec.rb b/spec/features/forms/form/form_export_spec.rb index 6992de91aa..6995adc28f 100644 --- a/spec/features/forms/form/form_export_spec.rb +++ b/spec/features/forms/form/form_export_spec.rb @@ -19,6 +19,15 @@ click_link("Export XML") expect(page.body).to match("#{form.name}") end + + it "XLSForm exports successfully" do + visit(form_path(form, locale: "en", mode: "m", mission_name: get_mission.compact_name)) + click_link("Export XLSForm") + + expect(page.current_url).to match("export_xls") + expect(page.body).to match(form.name.to_s) + expect(page.body).to match("text") + end end context "multiple forms" do diff --git a/spec/features/forms/form/xlsform_export_spec.rb b/spec/features/forms/form/xlsform_export_spec.rb deleted file mode 100644 index 30e4f17977..0000000000 --- a/spec/features/forms/form/xlsform_export_spec.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -require "rails_helper" -require "fileutils" -require "zip" - -feature "xlsform export" do - context "single form" do - let(:user) { create(:user, role_name: "coordinator") } - let(:form) { create(:form, name: "Export Test", question_types: %w[text]) } - - before do - login(user) - end - - it "exports to XLSForm successfully" do - visit(form_path(form, locale: "en", mode: "m", mission_name: get_mission.compact_name)) - click_link("Export XLSForm") - - expect(page.current_url).to match("export_xls") - expect(page.body).to match(form.name.to_s) - expect(page.body).to match("text") - end - end -end From 2c79e47d01fe046e6230af0fc0da7c7055c3f6d9 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 3 Nov 2023 14:31:17 -0400 Subject: [PATCH 31/34] Use conditionDecorator to convert operations for DRY --- app/decorators/odk/condition_decorator.rb | 4 ++++ app/models/forms/export.rb | 18 +++++------------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/app/decorators/odk/condition_decorator.rb b/app/decorators/odk/condition_decorator.rb index 66dce078a1..a5c114f5ff 100644 --- a/app/decorators/odk/condition_decorator.rb +++ b/app/decorators/odk/condition_decorator.rb @@ -38,6 +38,10 @@ def right_qing decorate(object.right_qing) end + def convert_op(operation) + OP_XPATH[operation] + end + private def select_multiple_to_odk diff --git a/app/models/forms/export.rb b/app/models/forms/export.rb index 1e39b1fddc..f8a199b6b4 100644 --- a/app/models/forms/export.rb +++ b/app/models/forms/export.rb @@ -8,15 +8,6 @@ class Export DisplayLogic DisplayConditions Default Hidden ].freeze - OPERATIONS = { - "eq" => "=", - "neq" => "!=", - "lt" => "<", - "leq" => "<=", - "gt" => ">", - "geq" => ">=" - }.freeze - QTYPE_TO_XLS = { # conversions "location" => "geopoint", @@ -81,9 +72,9 @@ def to_xls option_sets_used = [] # Define the below "index modifiers" which keep track of the line of the spreadsheet we are writing to. - # The for loop below (tracked by index i) loops through the list of form items, and so the index does not take into account rows that we need to write for when groups end. In XLSForm, these are written to a row all to themselves. - # This causes the index i to be de-synchronized with the row of the spreadsheet that we are writing to. - # Hence, we push to the row (i + index_mod) + # The for loop below (tracked by index i) loops through the list of form items, and so the index does not take into account rows that we need to write for when groups end. In XLSForm, these are written to a row all to themselves. + # This causes the index i to be de-synchronized with the row of the spreadsheet that we are writing to. + # Hence, we push to the row (i + index_mod) index_mod = 1 # start at row index 1 choices_index_mod = 0 @@ -237,7 +228,8 @@ def conditions_to_xls(conditions, true_if) right_to_push = dc.value.to_s end - op = OPERATIONS[dc.op] + op = ODK::ConditionDecorator::OP_XPATH[dc.op.to_sym] + raise "Operation not found: #{dc.op}" if op.blank? # omit the concatenator on the last condition only relevant_to_push = if i + 1 == dc_length From e3e250f1456e20a484a62751c26a1897083497c0 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 14 Nov 2023 12:16:22 -0500 Subject: [PATCH 32/34] Tweaks to row indices, add comments --- app/models/forms/export.rb | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/app/models/forms/export.rb b/app/models/forms/export.rb index f8a199b6b4..ba51c22f69 100644 --- a/app/models/forms/export.rb +++ b/app/models/forms/export.rb @@ -76,35 +76,39 @@ def to_xls # This causes the index i to be de-synchronized with the row of the spreadsheet that we are writing to. # Hence, we push to the row (i + index_mod) index_mod = 1 # start at row index 1 - choices_index_mod = 0 + choices_index_mod = 1 @form.preordered_items.each_with_index do |q, i| + # this variable keeps track of the spreadsheet row to be written during this loop iteration + row_index = i + index_mod + # did one or more groups just end? # if so, the qing's depth will be smaller than the depth counter while group_depth > q.ancestry_depth # are we in a repeat group? # we don't want to end the repeat if we are ending a nested non-repeat group within a repeat if repeat_depth > 1 && repeat_depth >= group_depth - questions.row(i + index_mod).push("end repeat") + questions.row(row_index).push("end repeat") repeat_depth -= 1 else # end the group - questions.row(i + index_mod).push("end group") + questions.row(row_index).push("end group") end - # update counters + # update counters to accomodate additional "end group" lines group_depth -= 1 index_mod += 1 + row_index += 1 end if q.group? # is this a group? group_name = q.code.tr(" ", "_") if q.repeatable? - questions.row(i + index_mod).push("begin repeat", group_name, q.code) + questions.row(row_index).push("begin repeat", group_name, q.code) repeat_depth += 1 else - questions.row(i + index_mod).push("begin group", group_name, q.code) + questions.row(row_index).push("begin group", group_name, q.code) end # update counters @@ -138,28 +142,25 @@ def to_xls option_sets_used.push(q.option_set_id) end - else - os_name = "" end # convert question types qtype_converted = QTYPE_TO_XLS[q.qtype_name] type_to_push = "#{qtype_converted} #{os_name}" - code_to_push = "#{q.code}_#{q.full_dotted_rank}" # Write the question row - questions.row(i + index_mod).push(type_to_push, code_to_push, q.name, q.required.to_s) + questions.row(row_index).push(type_to_push, q.code, q.name, q.required.to_s, filter_to_push) end # if we have any relevant conditions, add them to the end of the row if q.display_conditions.any? - questions.row(i + index_mod).push(conditions_to_xls(q.display_conditions, q.display_if)) + questions.row(row_index).push(conditions_to_xls(q.display_conditions, q.display_if)) end if q.constraints.any? q.constraints.each do |c| - questions.row(i + index_mod).push(conditions_to_xls(c.conditions, c.accept_if)) + questions.row(row_index).push(conditions_to_xls(c.conditions, c.accept_if)) end end end From 7439036bb1e9fccc21bfad9cc81aa73efce050b1 Mon Sep 17 00:00:00 2001 From: Kevin Cooper Date: Wed, 22 Nov 2023 16:54:47 -0500 Subject: [PATCH 33/34] Remove accidental future code from current PR --- app/models/forms/export.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/forms/export.rb b/app/models/forms/export.rb index ba51c22f69..a7b440dda0 100644 --- a/app/models/forms/export.rb +++ b/app/models/forms/export.rb @@ -150,7 +150,7 @@ def to_xls type_to_push = "#{qtype_converted} #{os_name}" # Write the question row - questions.row(row_index).push(type_to_push, q.code, q.name, q.required.to_s, filter_to_push) + questions.row(row_index).push(type_to_push, q.code, q.name, q.required.to_s) end # if we have any relevant conditions, add them to the end of the row From 3caa265a39e156838a741d15e92f0add7b5194ec Mon Sep 17 00:00:00 2001 From: Kevin Cooper Date: Wed, 22 Nov 2023 17:11:30 -0500 Subject: [PATCH 34/34] Add 'experimental' notice to button for now --- config/locales/en/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/locales/en/main.yml b/config/locales/en/main.yml index eee860b0d7..570304cd2a 100644 --- a/config/locales/en/main.yml +++ b/config/locales/en/main.yml @@ -921,7 +921,7 @@ en: import_from_csv: "Import from CSV" export_csv: "Export CSV" export_xml: "Export XML" - export_xls: "Export XLSForm" + export_xls: "Export XLSForm (experimental)" export: "Export" models: form: