another set of hotfixes (#1010)

* lazy_gettext is not json serializable

* better handling of file uploads.

* fixed the issue with the survey dashboard display.

* don't strip the input fields because it could be None

* fix for situations were there's no extra data for an attribute.

apollo/frontend/dashboard.py

@@ -308,7 +308,11 @@ def get_stratified_daily_progress(query, event, location_type):
             query.join(location_closure, location_closure.descendant_id == Submission.location_id)
             .join(ancestor_location, ancestor_location.id == location_closure.ancestor_id)
             .filter(location_closure.depth == depth_info.depth)
-            .with_entities(ancestor_location.name, ancestor_location.code, Submission.participant_updated)
+            .with_entities(
+                ancestor_location.name.label("getter"),
+                ancestor_location.code.label("code"),
+                Submission.participant_updated,
+            )
             .options(
                 Load(ancestor_location).load_only(
                     ancestor_location.id, ancestor_location.code, ancestor_location.name_translations
@@ -362,10 +366,10 @@ def get_stratified_daily_progress(query, event, location_type):
             )
             progress = df_resampled.truncate(before=start, after=end)
             progress.loc[progress.index == start.strftime("%Y-%m-%d"), "count"] = int(
-                df_resampled[df_resampled.index <= start].sum().iloc[0]
+                df_resampled[df_resampled.index <= start].sum().iloc[2]
             )
             progress.loc[progress.index == end.strftime("%Y-%m-%d"), "count"] = int(
-                df_resampled[df_resampled.index >= end].sum().iloc[0]
+                df_resampled[df_resampled.index >= end].sum().iloc[2]
             )
             dp = {idx.date(): int(progress.loc[idx]["count"]) for idx in progress.index}
             dp.update({"total": progress["count"].sum()})

apollo/helpers.py

@@ -1,10 +1,11 @@
 # -*- coding: utf-8 -*-
 import importlib
+import pkgutil
 
-from flask import Blueprint
 import magic
 import pandas as pd
-import pkgutil
+from flask import Blueprint
+from loguru import logger
 
 CSV_MIMETYPES = [
     "text/csv",
@@ -24,16 +25,15 @@
 
 
 def register_blueprints(app, package_name, package_path):
-    """Register all Blueprint instances on the specified Flask application
-    found in all modules for the specified package.
+    """Register all Blueprint instances on the specified Flask application.
 
     :param app: the Flask application
     :param package_name: the package name
     :param package_path: the package path
     """
     rv = []
     for _, name, _ in pkgutil.iter_modules(package_path):
-        if name.startswith('views_'):
+        if name.startswith("views_"):
             m = importlib.import_module(f"{package_name}.{name}")
             for item in dir(m):
                 item = getattr(m, item)
@@ -44,29 +44,37 @@ def register_blueprints(app, package_name, package_path):
 
 
 def _make_choices(qs, placeholder=None):
-    """Helper method for generating choices for :class:`SelectField`
-    instances.
-    """
+    """Helper method for generating choices for :class:`SelectField` instances."""
     if placeholder:
-        return [['', placeholder]] + [[str(i[0]), i[1]] for i in list(qs)]
+        return [["", placeholder]] + [[str(i[0]), i[1]] for i in list(qs)]
     else:
-        return [['', '']] + [[str(i[0]), i[1]] for i in list(qs)]
+        return [["", ""]] + [[str(i[0]), i[1]] for i in list(qs)]
 
 
 def load_source_file(source_file):
+    """Loads an import file."""
     # peek into file and read first 1kB, then reset pointer
     mimetype = magic.from_buffer(source_file.read(), mime=True)
     source_file.seek(0)
 
     if mimetype in CSV_MIMETYPES:
         # likely a CSV file
-        df = pd.read_csv(source_file, dtype=str).fillna('')
+        try:
+            df = pd.read_csv(source_file, dtype=str).fillna("")
+        except Exception:
+            logger.info("could not parse the CSV file.")
+            df = pd.DataFrame()
     elif mimetype in EXCEL_MIMETYPES:
         # likely an Excel spreadsheet, read all data as strings
-        xl = pd.ExcelFile(source_file)
-        ncols = xl.book.sheet_by_index(0).ncols
-        df = xl.parse(0, converters={i: str for i in range(ncols)}).fillna('')
+        try:
+            xl = pd.ExcelFile(source_file)
+            ncols = xl.book.sheet_by_index(0).ncols
+            df = xl.parse(0, converters={i: str for i in range(ncols)}).fillna("")
+        except Exception:
+            logger.info("could not parse the Excel file.")
+            df = pd.DataFrame()
     else:
-        raise RuntimeError('Unknown file type')
+        logger.info("could not determine the mimetype or an unacceptable file was uploaded.")
+        df = pd.DataFrame()
 
     return df

apollo/security_ext_forms.py

@@ -13,8 +13,8 @@ def validate(self, **kwargs):
         if not super(DeploymentLoginForm, self).validate(**kwargs):
             participant = (
                 models.Participant.query.filter(
-                    models.Participant.participant_id == self.username.data.strip(),
-                    models.Participant.password == self.password.data.strip(),
+                    models.Participant.participant_id == self.username.data,
+                    models.Participant.password == self.password.data,
                     models.Participant.participant_set == g.event.participant_set,
                 )
                 .join(models.ParticipantRole)

apollo/submissions/models.py

@@ -2,6 +2,7 @@
 import sqlalchemy as sa
 from depot.fields.specialized.image import UploadedImageWithThumb
 from depot.fields.sqlalchemy import UploadedFileField
+from flask_babel import gettext
 from flask_babel import lazy_gettext as _
 from geoalchemy2 import Geometry
 from sqlalchemy.dialects.postgresql import ARRAY, JSONB
@@ -147,7 +148,9 @@ def init_submissions(cls, event, form, role, location_type, task=None):
                 error_log.append(
                     {
                         "label": "ERROR",
-                        "message": _("Participant ID %(part_id)s has no location", part_id=participant.participant_id),
+                        "message": gettext(
+                            "Participant ID %(part_id)s has no location", part_id=participant.participant_id
+                        ),
                     }
                 )
                 continue

apollo/submissions/qa/query_builder.py

@@ -175,7 +175,7 @@ def visit_lookup(self, node, children):
             if symbol == ".":
                 return getattr(attribute, name)
             else:
-                return attribute.extra_data.get(name)
+                return attribute.extra_data.get(name) if attribute.extra_data is not None else None
         else:
             return getattr(self.submission, name)