Integrate mefs + wages 2024_2025 (#1133)

- [x] New Mef file (upsert)
- [x] New structure for csv files (keep all yearly files under same
folder)
- [x] Amendments to seeders to ensure usage of upsert (to preserve ids
and be idempotent)
- [x] New Wage file

app/models/mef.rb

@@ -8,14 +8,14 @@ class Mef < ApplicationRecord
   scope :with_wages, -> { joins("JOIN wages ON wages.mef_codes ? mefs.code") }
 
   validates :label, :code, :short, :mefstat11, :ministry, presence: true
-  validates :code, uniqueness: true
+  validates :code, uniqueness: { scope: :school_year_id }
 
   def mefstat4
     mefstat11.slice(0..3)
   end
 
   def wage
-    wages = Wage.where(mefstat4: mefstat4, ministry: ministry)
+    wages = Wage.where(mefstat4: mefstat4, ministry: ministry, school_year: school_year)
     return wages.first unless wages.many?
 
     wages.find { |wage| wage.mef_codes.include? code }

app/models/student/mappers/base.rb

@@ -70,7 +70,7 @@ def map_classe!(entry)
                         SchoolYear.find_by(start_year: year)
                       end
 
-        mef = Mef.find_by(code: mef_code)
+        mef = Mef.find_by(code: mef_code, school_year: school_year)
 
         return if label.nil? || mef.nil? || school_year.nil?
 

app/models/wage.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class Wage < ApplicationRecord
+  belongs_to :school_year
+
   enum :ministry, Mef.ministries.keys
 
   validates :mefstat4, :ministry, :daily_rate, :yearly_cap, presence: true

db/exclusion_seeder.rb

@@ -30,12 +30,10 @@ class ExclusionSeeder
   def self.seed
     logger = ActiveSupport::TaggedLogging.new(Logger.new($stdout))
 
-    logger.info { "[seeds] inserting exclusion" }
-
     EXCLUSIONS.each do |uai, mef_code|
       Exclusion.find_or_create_by!(uai: uai, mef_code: mef_code)
     end
 
-    logger.info { "[seeds] done inserting #{Exclusion.count} exclusions" }
+    logger.info { "[seeds] upserted #{Exclusion.count} exclusions" }
   end
 end

db/mef_seeder.rb

@@ -2,6 +2,9 @@
 
 require "csv"
 
+# For the introduction of a new SchoolYear
+# First create the new SchoolYear so that SchoolYear.current returns the latest one
+# Then run MefSeeder.seed
 class MefSeeder
   MAPPING = {
     code: "MEF",
@@ -12,10 +15,26 @@ class MefSeeder
   }.freeze
 
   def self.seed
-    logger = ActiveSupport::TaggedLogging.new(Logger.new($stdout))
-    logger.info "[seeds] inserting MEF codes..."
+    @@logger = ActiveSupport::TaggedLogging.new(Logger.new($stdout))
 
-    data = CSV.read(Rails.root.join("data/mefs.csv"), headers: true)
+    Mef.transaction do
+      Dir.glob(Rails.root.join("data/mefs/*.csv")).each do |file_path|
+        process_file(file_path)
+      end
+    end
+
+    @@logger.info "[seeds] upserted #{Mef.count} total MEFs"
+  end
+
+  private
+
+  def self.process_file(file_path)
+    file_name = File.basename(file_path, ".csv")
+    start_year = file_name.split("_").first.to_i
+
+    school_year = SchoolYear.find_by!(start_year: start_year)
+
+    data = CSV.read(file_path, headers: true)
 
     mefs = data.map do |entry|
       attributes = MAPPING.transform_values do |value|
@@ -26,11 +45,21 @@ def self.seed
         end
       end
 
-      attributes.merge("school_year_id" => SchoolYear.current.id)
+      attributes.merge(school_year_id: school_year.id)
+    end
+
+    duplicates = mefs.group_by { |mef| [mef[:code], mef[:school_year_id]] }
+                   .select { |_, group| group.size > 1 }
+
+    if duplicates.any?
+      @@logger.warn "[seeds] found duplicates in MEF data for school year #{school_year}"
+      duplicates.each do |key, group|
+        @@logger.warn "[seeds] duplicate found for code: #{key[0]} and school_year_id: #{key[1]}"
+      end
     end
 
-    Mef.upsert_all(mefs, unique_by: :code) # rubocop:disable Rails/SkipsModelValidations
+    Mef.upsert_all(mefs, unique_by: [:code, :school_year_id]) # rubocop:disable Rails/SkipsModelValidations
 
-    logger.info "[seeds] done inserting MEF codes."
+    @@logger.info "[seeds] upserted #{mefs.size} MEFs for school year: #{school_year}"
   end
 end

db/migrate/20241011120605_change_unique_index_on_mef.rb

@@ -0,0 +1,9 @@
+class ChangeUniqueIndexOnMef < ActiveRecord::Migration[7.2]
+  def change
+    remove_index :mefs, name: "index_mefs_on_code"
+
+    add_index :mefs, [:code, :school_year_id],
+              name: "index_mefs_on_code_and_school_year",
+              unique: true
+  end
+end

db/migrate/20241011131127_add_school_year_id_to_wages.rb

@@ -0,0 +1,5 @@
+class AddSchoolYearIdToWages < ActiveRecord::Migration[7.2]
+  def change
+    add_reference :wages, :school_year, foreign_key: true
+  end
+end

db/school_year_seeder.rb

@@ -3,13 +3,10 @@
 class SchoolYearSeeder
   def self.seed
     logger = ActiveSupport::TaggedLogging.new(Logger.new($stdout))
-    logger.info "[seeds] inserting school years..."
 
-    SchoolYear.delete_all
+    SchoolYear.find_or_create_by(start_year: 2023)
+    SchoolYear.find_or_create_by(start_year: 2024)
 
-    SchoolYear.create(start_year: 2023)
-    SchoolYear.create(start_year: 2024)
-
-    logger.info "[seeds] done inserting #{SchoolYear.count} school years."
+    logger.info "[seeds] upserted #{SchoolYear.count} school years."
   end
 end

db/wage_seeder.rb

@@ -10,30 +10,45 @@ class WageSeeder
     ministry: "BOP"
   }.freeze
 
-  # rubocop:disable Metrics/AbcSize
   def self.seed
-    logger = ActiveSupport::TaggedLogging.new(Logger.new($stdout))
-    logger.info "[seeds] inserting daily wages by mef..."
+    @@logger = ActiveSupport::TaggedLogging.new(Logger.new($stdout))
 
-    Wage.delete_all
+    Wage.transaction do
+      Dir.glob(Rails.root.join("data/wages/*.csv")).each do |file_path|
+        process_file(file_path)
+      end
+    end
+
+    @@logger.info "[seeds] upserted #{Wage.count} total wages"
+  end
 
-    data = CSV.read(Rails.root.join("data/mefs-amounts.csv"), headers: true)
+  private
 
-    data
-      .group_by { |d| d.fields(*WAGE_MAPPING.values) }
-      .each do |group, wages|
-        daily, yearly, mefstat4, ministry = group
+  def self.process_file(file_path)
+    file_name = File.basename(file_path, ".csv")
+    start_year = file_name.split("_").first.to_i
 
-        Wage.create(
-          mefstat4: mefstat4,
-          ministry: Wage.ministries[ministry.downcase],
-          daily_rate: daily,
-          yearly_cap: yearly,
-          mef_codes: wages.pluck("MEF")
-        )
-      end
+    school_year = SchoolYear.find_by!(start_year: start_year)
+
+    data = CSV.read(file_path, headers: true)
+
+    wages = data.group_by { |d| d.fields(*WAGE_MAPPING.values) }
+                .map do |group, entries|
+      daily, yearly, mefstat4, ministry = group
+      {
+        mefstat4: mefstat4,
+        ministry: Wage.ministries[ministry.downcase],
+        daily_rate: daily.to_i,
+        yearly_cap: yearly.to_i,
+        mef_codes: entries.pluck("MEF"),
+        school_year_id: school_year.id
+      }
+    end
+
+    Wage.upsert_all(
+      wages
+    )
 
-    logger.info "[seeds] done inserting #{Wage.count} daily wages by mef."
+    @@logger.info "[seeds] upserted wages for school year #{school_year.start_year}-#{school_year.start_year + 1}"
   end
-  # rubocop:enable Metrics/AbcSize
 end

spec/factories/wages.rb

@@ -6,5 +6,6 @@
     yearly_cap { 100 }
     mefstat4 { 123 }
     ministry { :menj }
+    school_year { SchoolYear.current }
   end
 end

spec/models/mef_spec.rb

@@ -47,7 +47,7 @@
       end
     end
 
-    context "when there are several wages with mefstat4 & ministry" do
+    context "when there are several wages with same mefstat4 & ministry" do
       let!(:correct_wage) { create(:wage, mefstat4: mef.mefstat4, ministry: mef.ministry, mef_codes: [mef.code]) }
 
       before do
@@ -68,7 +68,7 @@
       "2532210311" => { daily_rate: 15, yearly_cap: 1350 }
     }.each do |mef_code, amounts|
       context mef_code.to_s do
-        let(:wage) { described_class.find_by(code: mef_code).wage }
+        let(:wage) { described_class.find_by!(code: mef_code).wage }
 
         it "has daily_rate = #{amounts[:daily_rate]}" do
           expect(wage.daily_rate).to eq amounts[:daily_rate]