begin ingest (move to ingest-payroll feature branch)

config/settings/local.py

@@ -20,3 +20,8 @@
 DEBUG_TOOLBAR_CONFIG = {
     "SHOW_TOOLBAR_CALLBACK": lambda request: DEBUG,
 }
+
+FILE_UPLOAD_HANDLERS = (
+    "django.core.files.uploadhandler.MemoryFileUploadHandler",
+    "django.core.files.uploadhandler.TemporaryFileUploadHandler",
+)

payroll/migrations/0016_employee_has_left.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.1.4 on 2025-01-09 11:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("payroll", "0015_employee_basic_pay_employee_ernic_employee_pension"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="employee",
+            name="has_left",
+            field=models.BooleanField(default=False),
+        ),
+    ]

payroll/models.py

@@ -93,14 +93,15 @@ class Employee(Position):
     basic_pay = models.BigIntegerField(default=0, db_comment="pence")
     pension = models.BigIntegerField(default=0, db_comment="pence")
     ernic = models.BigIntegerField(default=0, db_comment="pence")
+    has_left = models.BooleanField(default=False)
 
     # TODO: Missing fields from Admin Tool which aren't required yet.
     # EU/Non-EU (from programme code model)
 
     objects = EmployeeQuerySet.as_manager()
 
     def __str__(self) -> str:
-        return f"{self.employee_no} - {self.first_name} {self.last_name}"
+        return f"{self.id} {self.employee_no} - {self.first_name} {self.last_name}"
 
     def get_full_name(self) -> str:
         return f"{self.first_name} {self.last_name}"

payroll/services/ingest.py

@@ -0,0 +1,166 @@
+import csv
+from collections import namedtuple
+from typing import TypedDict
+
+from django.core.files import File
+from django.db import transaction
+
+from chartofaccountDIT.models import ProgrammeCode
+from costcentre.models import CostCentre
+from gifthospitality.models import Grade
+from payroll.models import Employee
+
+
+PayrollRow = namedtuple(
+    "PayrollRow",
+    (
+        "employee_no",
+        "first_name",
+        "last_name",
+        "cost_centre",
+        "programme_code",
+        "grade",
+        "assignment_status",
+        "fte",
+        "basic_pay",
+        "ernic",
+        "pension",
+    ),
+)
+
+
+EmployeeDict = dict[str, object]
+
+
+class ImportPayrollReport(TypedDict):
+    failed_records: list[EmployeeDict]
+    created: list[EmployeeDict]
+    updated: list[EmployeeDict]
+
+
+@transaction.atomic()
+def import_payroll(payroll_csv: File) -> ImportPayrollReport:
+    csv_reader = csv.reader((row.decode("utf-8") for row in payroll_csv))
+
+    # Skip header row.
+    next(csv_reader)
+
+    employees = []
+    cost_centres = []
+    programme_codes = []
+    grades = []
+
+    for row in csv_reader:
+        employee = csv_row_employee_dict(PayrollRow(*row))
+
+        employees.append(employee)
+        cost_centres.append(employee["cost_centre"])
+        programme_codes.append(employee["programme_code"])
+        grades.append(employee["grade"])
+
+    uniq_cost_centre_codes = set(cost_centres)
+    uniq_grades = set(grades)
+    uniq_programme_codes = set(programme_codes)
+
+    cost_centers = {
+        centre.cost_centre_code: centre
+        for centre in CostCentre.objects.filter(
+            cost_centre_code__in=uniq_cost_centre_codes
+        )
+    }
+    cost_center_codes = [centre.cost_centre_code for centre in cost_centers.values()]
+
+    programme_codes = {
+        code.programme_code: code
+        for code in ProgrammeCode.objects.filter(
+            programme_code__in=uniq_programme_codes
+        )
+    }
+    existing_programme_codes = [
+        code.programme_code for code in programme_codes.values()
+    ]
+
+    grades = {
+        grade.grade: grade for grade in Grade.objects.filter(grade__in=uniq_grades)
+    }
+    existing_grades = [grade.grade for grade in grades.values()]
+
+    clean_records = []
+    failed_records = []
+
+    for emp in employees:
+        errors = []
+        if emp["cost_centre"] not in cost_center_codes:
+            errors.append(f"Cost centre '{emp["cost_centre"]}' doesn't exists")
+        if emp["programme_code"] not in existing_programme_codes:
+            errors.append(f"Programme code '{emp["programme_code"]}' doesn't exists")
+        if emp["grade"] not in existing_grades:
+            errors.append(f"Grade '{emp["grade"]}' doesn't exists")
+        if errors:
+            emp["errors"] = errors
+            failed_records.append(emp)
+        else:
+            emp["cost_centre"] = cost_centers[emp["cost_centre"]]
+            emp["programme_code"] = programme_codes[emp["programme_code"]]
+            emp["grade"] = grades[emp["grade"]]
+            clean_records.append(emp)
+    result = save_data(clean_records)
+
+    return {"failed_records": failed_records, **result}
+
+
+def csv_row_employee_dict(hr_row) -> EmployeeDict:
+    employee = {
+        "employee_no": hr_row.employee_no,
+        "first_name": hr_row.first_name,
+        "last_name": hr_row.last_name,
+        "cost_centre": hr_row.cost_centre,
+        "programme_code": hr_row.programme_code,
+        "grade": hr_row.grade,
+        "assignment_status": hr_row.assignment_status,
+        "fte": hr_row.fte,
+        "basic_pay": hr_row.basic_pay,
+        "ernic": hr_row.ernic,
+        "pension": hr_row.pension,
+        "has_left": False,
+    }
+    return employee
+
+
+def save_data(csv_data):
+    emp_nos = {employee["employee_no"] for employee in csv_data}
+    Employee.objects.exclude(employee_no__in=emp_nos).filter(has_left=False).update(
+        has_left=True
+    )
+    return bulk_update_or_create(csv_data)
+
+
+def bulk_update_or_create(data):
+    if data:
+        keys = list(data[0].keys())
+    existing_ids = {
+        emp.employee_no: emp.id
+        for emp in Employee.objects.filter(
+            employee_no__in=[emp["employee_no"] for emp in data]
+        )
+    }
+    to_update = []
+    to_create = []
+
+    for item in data:
+        emp = Employee(**item)
+        for key in keys:
+            setattr(emp, key, item[key])
+
+        if item["employee_no"] in existing_ids:
+            emp.id = existing_ids[item["employee_no"]]
+            to_update.append(emp)
+        else:
+            to_create.append(emp)
+
+    if to_create:
+        Employee.objects.bulk_create(to_create)
+    if to_update:
+        Employee.objects.bulk_update(to_update, keys)
+
+    return {"created": to_create, "updated": to_update}

payroll/services/payroll.py

@@ -76,8 +76,7 @@ def payroll_forecast_report(
     )
 
     employee_qs = Employee.objects.filter(
-        cost_centre=cost_centre,
-        pay_periods__year=financial_year,
+        cost_centre=cost_centre, pay_periods__year=financial_year, has_left=False
     )
     pay_uplift_obj = PayUplift.objects.filter(financial_year=financial_year).first()
     attrition_obj = get_attrition_instance(financial_year, cost_centre)
@@ -152,7 +151,12 @@ def get_average_salary_for_grade(grade: Grade, cost_centre: CostCentre) -> int:
     salaries: list[int] = []
 
     for filter in filters:
-        employee_qs = Employee.objects.payroll().filter(grade=grade).filter(filter)
+        employee_qs = (
+            Employee.objects.payroll()
+            .filter(grade=grade)
+            .filter(filter)
+            .filter(has_left=False)
+        )
 
         basic_pay = employee_qs.aggregate(
             count=Count("basic_pay"), avg=Avg("basic_pay")

payroll/templates/payroll/page/import_payroll.html

@@ -0,0 +1,59 @@
+{% extends "base_generic.html" %}
+{% load breadcrumbs vite %}
+
+{% block title %}Import payroll{% endblock title %}
+
+{% block breadcrumbs %}
+    {{ block.super }}
+    {% breadcrumb "Import payroll" "payroll:import" %}
+{% endblock breadcrumbs %}
+
+{% block content %}
+    <h1 class="govuk-heading-l">Import payroll</h1>
+    <form method="POST" enctype="multipart/form-data">
+        {% csrf_token %}
+        <div>
+            <label for="payroll_csv">Payroll CSV file:</label>
+            <input type="file" name="payroll_csv" id="payroll_csv">
+        </div>
+        <button type="submit">Submit</button>
+    </form>
+    <div>
+        <br/><br/>
+        {% if output %}
+        Inserted records : {{output.created|length}} <br/>
+        Updated records: {{output.updated|length}} <br/>
+        Failing records: {{output.failed_records|length}} <br/>
+        {% endif %}
+
+        {{ error }}
+        {% if output.failed_records %}
+        <h3>Failing records </h3>
+        <table  className="govuk-table finance-table">
+            <thead class="govuk-table__head">
+              <tr class="govuk-table__row">
+                <td class="govuk-table__cell">Employee No</td>
+                <td class="govuk-table__cell">Name</td>
+                <td class="govuk-table__cell">Error</td>
+              </tr>
+            </thead>
+            <tbody>
+              {% for item in output.failed_records %}
+                <tr class="govuk-table__row">
+                  <td class="govuk-table__cell">{{ item.employee_no }}</td>
+                  <td class="govuk-table__cell">{{ item.first_name }} {{ item.last_name }}</td>
+                  <td class="govuk-table__cell"> {% for error in  item.errors %}
+                    {{error}}<br/>
+                    {% endfor %}
+                  </td>
+                </tr>
+              {% endfor %}
+            </tbody>
+          </table>
+          {% endif %}
+    </div>
+{% endblock content %}
+
+{% block scripts %}
+    {{ block.super }}
+{% endblock scripts %}

payroll/urls.py

@@ -48,4 +48,6 @@
         views.PayModifierApiView.as_view(),
         name="api_pay_modifiers",
     ),
+    # TODO: Remove temporary views when ready.
+    path("import", views.import_payroll_page, name="import"),
 ]

payroll/views.py

@@ -1,7 +1,8 @@
 import json
 
 from django.contrib.auth.mixins import PermissionRequiredMixin, UserPassesTestMixin
-from django.http import HttpResponse, JsonResponse
+from django.core.exceptions import PermissionDenied
+from django.http import HttpRequest, HttpResponse, JsonResponse
 from django.shortcuts import get_object_or_404
 from django.template.response import TemplateResponse
 from django.urls import reverse
@@ -15,6 +16,7 @@
 from payroll.models import Vacancy
 
 from .services import payroll as payroll_service
+from .services.ingest import import_payroll
 
 
 class EditPayrollBaseView(UserPassesTestMixin, View):
@@ -182,3 +184,24 @@ def get_context_data(self, **kwargs):
             "vacancy_id": self.object.id,
         }
         return super().get_context_data(**kwargs) | context
+
+
+def import_payroll_page(request: HttpRequest) -> HttpResponse:
+    if not request.user.is_superuser:
+        raise PermissionDenied
+
+    output = ""
+    context = {}
+
+    if request.method == "POST":
+        if "payroll_csv" not in request.FILES:
+            context = {"error": "Payroll file is required"}
+        else:
+            payroll_csv = request.FILES["payroll_csv"]
+            output = import_payroll(payroll_csv)
+
+    context = {
+        "output": output,
+    }
+
+    return TemplateResponse(request, "payroll/page/import_payroll.html", context)