Merge remote-tracking branch 'origin/main' into update-csv

backend/Gemfile

@@ -18,7 +18,7 @@ gem 'pundit'
 gem 'rack', '~> 2.0'
 gem 'rack-cors', '~> 2.0'
 gem 'rails', '~> 7.1.4'
-gem 'rexml', '>= 3.3.3'
+gem 'rexml', '>= 3.3.9'
 gem 'sendgrid-actionmailer'
 gem 'sprockets-rails'
 gem 'sucker_punch', '~> 3.0'

backend/Gemfile.lock

@@ -242,8 +242,7 @@ GEM
     responders (3.1.1)
       actionpack (>= 5.2)
       railties (>= 5.2)
-    rexml (3.3.6)
-      strscan
+    rexml (3.3.9)
     rspec-core (3.13.0)
       rspec-support (~> 3.13.0)
     rspec-expectations (3.13.1)
@@ -296,7 +295,6 @@ GEM
       activesupport (>= 6.1)
       sprockets (>= 3.0.0)
     stringio (3.1.1)
-    strscan (3.1.0)
     sucker_punch (3.2.0)
       concurrent-ruby (~> 1.0)
     syntax_suggest (2.0.0)
@@ -347,7 +345,7 @@ DEPENDENCIES
   rack (~> 2.0)
   rack-cors (~> 2.0)
   rails (~> 7.1.4)
-  rexml (>= 3.3.3)
+  rexml (>= 3.3.9)
   rspec-rails (~> 6.0.0)
   rubocop
   rubocop-rails

backend/app/controllers/survey_visits_controller.rb

@@ -73,7 +73,8 @@ def survey_visit_params
     params.require(:survey_visit)
           .permit(:surveyor_id, :home_id, :latitude, :longitude,
                   survey_response_attributes: [:survey_id,
-                                               { survey_answers_attributes: %i[survey_question_id answer] }])
+                                               { survey_answers_attributes: [:survey_question_id,
+                                                                             :answer, { answers: [] }] }])
   end
 
   def search_params

backend/app/views/survey_visits/_survey_visit.json.jbuilder

@@ -26,6 +26,7 @@ if survey_visit.survey_response.present?
       json.id sa.id
       json.survey_question_id sa.survey_question_id
       json.answer sa.answer
+      json.answers sa.answers
     end
   end
 end

backend/db/migrate/20241021114918_add_cluster_id_to_assignments.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class AddClusterIdToAssignments < ActiveRecord::Migration[7.1]
+  def change
+    change_table :assignments, bulk: true do |t|
+      t.integer :cluster_id
+
+      t.string :latitude
+      t.string :longitude
+    end
+
+    add_index :assignments, :cluster_id, unique: true
+  end
+end

backend/db/migrate/20241029235813_add_answers_to_survey_answers.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddAnswersToSurveyAnswers < ActiveRecord::Migration[7.1]
+  def change
+    add_column :survey_answers, :answers, :string, array: true, default: []
+  end
+end

backend/db/seed_importer.rb

@@ -9,7 +9,7 @@
 
 #   Seed importer provides a reusable function to import a csv file
 module SeedImporter
-  def import_seed_data(path, clustered_ordered_parcels)
+  def self.import_seed_data(path, clustered_ordered_parcels)
     options = { downcase_header: true, verbose: true }
 
     # Seed survey
@@ -49,7 +49,10 @@ def import_seed_data(path, clustered_ordered_parcels)
       end
     end
 
-    SmarterCSV.process(File.join(path, 'test_surveyors.csv'), options) do |chunk|
+    # Prevent SmarterCSV from converting a zipcode like "02110" to "2110"
+    modified_options = options.merge(convert_values_to_numeric: { except: :zipcode })
+
+    SmarterCSV.process(File.join(path, 'test_surveyors.csv'), modified_options) do |chunk|
       chunk.each do |data_hash|
         surveyor = Surveyor.new(data_hash)
         surveyor.user = User.where(email: data_hash[:email]).first
@@ -58,24 +61,16 @@ def import_seed_data(path, clustered_ordered_parcels)
     end
 
     # Seed assignments
-    # This is slightly inelegant, but first goes through the dummy data
-    # and ensures that each assignment exists.
-    # We sort the associations below.
-    SmarterCSV.process(File.join(path, 'test_assignments.csv'), options) do |chunk|
-      chunk.each do |data_hash|
-        assignment = Assignment.where(group: data_hash[:group]).first
-        assignment = Assignment.new(data_hash.except(:surveyor_email)) if assignment.nil?
-        assignment.save!
-      end
-    end
-
-    # Seed assignments-surveyor joins
-    SmarterCSV.process(File.join(path, 'test_assignments.csv'), options) do |chunk|
+    SmarterCSV.process(File.join(path, 'cluster_centroids_with_regions.csv'), options) do |chunk|
       chunk.each do |data_hash|
-        surveyor = Surveyor.where(email: data_hash[:surveyor_email]).first
-        assignment = Assignment.where(group: data_hash[:group]).first
+        assignment_hash = {
+          cluster_id: data_hash[:cluster],
+          latitude: data_hash[:lat],
+          longitude: data_hash[:lon],
+          region_code: data_hash[:region]
+        }
 
-        assignment.surveyors.append(surveyor)
+        assignment = Assignment.new(assignment_hash)
         assignment.save!
       end
     end
@@ -89,12 +84,17 @@ def import_seed_data(path, clustered_ordered_parcels)
           unit_num: :unit_number,
           zipcode: :zip_code,
           lu_desc: :building_type,
-          cluster: :assignment_id,
           order: :visit_order
         }
+
         home = Home.new(data_hash.transform_keys(key_mapping).slice(*Home.new.attributes.symbolize_keys.keys))
         home.state = 'MA' if home.state.blank?
         home.zip_code = "0#{home.zip_code}" if home.zip_code.length == 4
+
+        # For performance, lookup assignment id without loading Assignment model
+        assignment_id = Assignment.where(cluster_id: data_hash[:cluster]).pick(:id)
+        home.assignment_id = assignment_id
+
         home.save!
       end
     end

backend/lib/seeds/cluster_centroids_with_regions.csv

@@ -0,0 +1,146 @@
+cluster,lat,lon,region
+2,42.32625609,-71.07912228,1
+3,42.33090622,-71.08566668,1
+4,42.32726624,-71.10307884,1
+6,42.32724399,-71.09061039,1
+8,42.32118596,-71.0844145,1
+9,42.32169388,-71.07944581,1
+10,42.3172238,-71.08016852,1
+11,42.3194878,-71.09067048,1
+13,42.31221553,-71.08596887,1
+14,42.30128747,-71.08143702,1
+20,42.32211056,-71.07487253,1
+31,42.31448808,-71.06931203,1
+58,42.29760236,-71.08459406,1
+109,42.31494001,-71.09912739,1
+144,42.33080503,-71.10266723,1
+165,42.31974952,-71.06558348,1
+167,42.31545514,-71.09310314,1
+168,42.32803307,-71.09411685,1
+174,42.3129811,-71.08128125,1
+181,42.31159188,-71.07602815,1
+182,42.30737168,-71.07885924,1
+189,42.32635636,-71.07477098,1
+190,42.3230258,-71.07064082,1
+191,42.31422714,-71.0720529,1
+192,42.31512262,-71.07540732,1
+193,42.31948073,-71.07299648,1
+196,42.3186393,-71.0759973,1
+198,42.31752073,-71.09906931,1
+199,42.32350875,-71.09368895,1
+22,42.32065148,-71.05514242,2
+23,42.32240067,-71.05918419,2
+24,42.30957476,-71.04830926,2
+25,42.31773961,-71.05396453,2
+26,42.31397073,-71.05355634,2
+27,42.31336397,-71.05551757,2
+28,42.31551228,-71.05804313,2
+35,42.3025683,-71.06585999,2
+36,42.30543035,-71.06534007,2
+38,42.30909669,-71.06850894,2
+39,42.30732502,-71.06982827,2
+40,42.30567683,-71.07107942,2
+42,42.29915657,-71.0777503,2
+48,42.3020068,-71.05757273,2
+51,42.297154,-71.06471785,2
+145,42.31136186,-71.04989451,2
+161,42.31764452,-71.06134413,2
+162,42.31138565,-71.06114436,2
+163,42.31413223,-71.06470306,2
+164,42.31074347,-71.06542488,2
+171,42.30131672,-71.06348249,2
+172,42.30293984,-71.06974475,2
+173,42.30110425,-71.06990603,2
+178,42.29822697,-71.06808187,2
+179,42.29804284,-71.07267812,2
+180,42.30583145,-71.05959739,2
+183,42.30261568,-71.07473359,2
+59,42.2883605,-71.09237721,3
+60,42.28796536,-71.08337476,3
+61,42.28742865,-71.08849198,3
+62,42.28593541,-71.08442298,3
+63,42.28452772,-71.0870007,3
+99,42.26915448,-71.0975582,3
+100,42.27014033,-71.10384059,3
+101,42.27479546,-71.09908843,3
+103,42.27155248,-71.08892654,3
+104,42.27642545,-71.0879934,3
+105,42.28300534,-71.09478099,3
+106,42.28324112,-71.09705244,3
+107,42.2796326,-71.09058674,3
+111,42.27058799,-71.08201291,3
+113,42.27694949,-71.08549717,3
+114,42.2733561,-71.09045321,3
+115,42.27191448,-71.08518254,3
+117,42.2732719,-71.08609361,3
+118,42.26624414,-71.09736444,3
+119,42.26683972,-71.09906247,3
+122,42.27952737,-71.09916546,3
+123,42.27329758,-71.10370608,3
+124,42.27702167,-71.10417749,3
+125,42.27683305,-71.10002378,3
+126,42.2720147,-71.09700159,3
+127,42.27189696,-71.10087441,3
+128,42.26900199,-71.10191832,3
+153,42.28624536,-71.0823606,3
+176,42.27909553,-71.09708748,3
+177,42.2769883,-71.09533843,3
+204,42.27826037,-71.08254931,3
+47,42.29435146,-71.05277826,4
+49,42.29984382,-71.05177332,4
+50,42.29677369,-71.05781061,4
+76,42.29077716,-71.05157242,4
+77,42.2897643,-71.05428296,4
+78,42.29081611,-71.0582697,4
+80,42.28956551,-71.04859111,4
+81,42.28900119,-71.04065256,4
+83,42.28024019,-71.05164466,4
+84,42.28223826,-71.05090307,4
+86,42.28170013,-71.05391162,4
+131,42.2810956,-71.05667765,4
+137,42.27324975,-71.06368927,4
+142,42.2794253,-71.06054521,4
+146,42.29167837,-71.05483628,4
+147,42.2879369,-71.05252155,4
+151,42.28359972,-71.05800383,4
+152,42.28367649,-71.06159605,4
+155,42.28633941,-71.05610396,4
+156,42.28603636,-71.05938764,4
+157,42.28770332,-71.06041237,4
+159,42.28066536,-71.06412906,4
+200,42.28654151,-71.05013416,4
+201,42.28530711,-71.04732788,4
+202,42.28523065,-71.05285413,4
+207,42.29391827,-71.06007539,4
+209,42.29058934,-71.06236486,4
+210,42.28880416,-71.06172179,4
+53,42.29502916,-71.06867644,5
+54,42.29513355,-71.07441063,5
+56,42.29460013,-71.07662542,5
+57,42.29427284,-71.08061441,5
+65,42.28077372,-71.07729012,5
+66,42.28042901,-71.08018146,5
+67,42.2823864,-71.08241955,5
+68,42.28394836,-71.06822438,5
+69,42.2808632,-71.0694266,5
+70,42.27893328,-71.07554379,5
+95,42.27526803,-71.07337927,5
+96,42.27267817,-71.07723079,5
+102,42.27493323,-71.07878064,5
+112,42.2725775,-71.07403367,5
+129,42.27543456,-71.07678514,5
+130,42.27728273,-71.07539654,5
+138,42.27746359,-71.06830539,5
+139,42.27864892,-71.07129283,5
+140,42.27481492,-71.07039788,5
+158,42.28203564,-71.06687731,5
+184,42.28515403,-71.07825401,5
+185,42.2882704,-71.07329247,5
+186,42.28853382,-71.07636904,5
+187,42.28115915,-71.07382116,5
+188,42.28348691,-71.07407476,5
+205,42.29251742,-71.06811416,5
+206,42.29241808,-71.07572857,5
+208,42.29414475,-71.06393998,5
+211,42.28637781,-71.06824369,5
+212,42.28989372,-71.06819576,5