fix (logging): Error handling Raw Completions with no sheet named Sheet1

alembic/versions/323012e80841_create_gdrive_table.py

@@ -5,6 +5,7 @@
 Create Date: 2023-10-25 22:19:12.231204
 
 """
+
 from typing import Sequence, Union
 
 from alembic import op

alembic/versions/b5c8e1cfcb42_create_participant_table.py

@@ -5,6 +5,7 @@
 Create Date: 2023-10-25 22:22:37.857141
 
 """
+
 from typing import Sequence, Union
 
 from alembic import op
@@ -22,6 +23,7 @@ def upgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
     op.add_column("participant", sa.Column("survey_id", sa.String(), nullable=True))
     op.add_column("participant", sa.Column("response_id", sa.String(), nullable=True))
+
     op.add_column(
         "participant", sa.Column("rules_consent_id", sa.String(), nullable=True)
     )

gdrive/database/database.py

@@ -1,6 +1,7 @@
 """
 DB Connection for Gdrive
 """
+
 import logging
 import sqlalchemy
 from sqlalchemy import orm
@@ -17,6 +18,8 @@
         settings.DB_URI,
         connect_args={"options": "-csearch_path=%s" % (settings.SCHEMA)},
     )
+
+
 else:
     log.info("No database configuration found. Creating in memory DB.")
     engine = sqlalchemy.create_engine("sqlite+pysqlite:///:memory:")

gdrive/export_api.py

@@ -5,6 +5,7 @@
 import io
 import json
 import logging
+import sys
 
 import fastapi
 from pydantic import BaseModel, Field
@@ -20,12 +21,14 @@
 
 @router.post("/export")
 async def upload_file(interactionId):
+    log.info(f"Export interaction {interactionId}")
     export_data = export_client.export(interactionId)
     export_bytes = io.BytesIO(
         export_client.codename(json.dumps(export_data, indent=2)).encode()
     )
     parent = drive_client.create_folder(interactionId, settings.ROOT_DIRECTORY)
     drive_client.upload_basic("analytics.json", parent, export_bytes)
+    log.info(f"Uploading {sys.getsizeof(export_bytes)} bytes to drive folder {parent}")
 
 
 class ParticipantModel(BaseModel):
@@ -67,16 +70,17 @@ async def survey_upload_response_task(request):
     """
     Background task that handles qualtrics response fetching and exporting
     """
+    log.info(f"Gathering response {request.responseId}")
     try:
         response = export_client.get_qualtrics_response(
             request.surveyId, request.responseId
         )
 
-        log.info("Response found, beginning export.")
+        log.info(f"{request.responseId} response found, beginning export.")
 
         if response["status"] != "Complete":
-            raise error.ExportError(
-                f"Cannot upload incomplete survery response to raw completions spreadsheet: {request.responseId}"
+            log.warn(
+                f"Incomplete survery response to raw completions spreadsheet: {request.responseId}"
             )
 
         # By the time we get here, we can count on the response containing the demographic data
@@ -86,7 +90,7 @@ async def survey_upload_response_task(request):
 
         if request.participant:
             participant = request.participant
-            sheets_client.upload_participant(
+            upload_result = sheets_client.upload_participant(
                 participant.first,
                 participant.last,
                 participant.email,
@@ -103,6 +107,12 @@ async def survey_upload_response_task(request):
                 survey_resp["skin_tone"],
             )
 
+            result_sheet_id = upload_result["spreadsheetId"]
+            if upload_result:
+                log.info(
+                    f"Uploaded response: {request.responseId} to completions spreadsheet {result_sheet_id}"
+                )
+
             crud.create_participant(
                 models.ParticipantModel(
                     survey_id=request.surveyId,
@@ -120,17 +130,22 @@ async def survey_upload_response_task(request):
                     skin_tone=survey_resp["skin_tone"],
                 )
             )
+            log.info(f"Wrote {request.responseId} to database")
 
         # call function that queries ES for all analytics entries (flow interactionId) with responseId
         interactionIds = export_client.export_response(request.responseId, response)
-
-        log.info("Analytics updated, beginning gdrive export.")
+        log.info(
+            f"Elastic Search returned {len(interactionIds)} interaction ids for response: {request.responseId}"
+        )
 
         # export list of interactionIds to gdrive
         for id in interactionIds:
             await upload_file(id)
+            log.info(
+                f"Exported response: {request.responseId} interaction: {id} to gdrive"
+            )
     except error.ExportError as e:
-        log.error(e.args)
+        log.error(f"Response: {request.responseId} encountered an error: {e.args}")
 
 
 class FindModel(BaseModel):

gdrive/export_client.py

@@ -165,6 +165,7 @@ def get_qualtrics_response(surveyId: str, responseId: str):
         json={"surveyId": surveyId, "responseId": responseId},
         timeout=30,  # qualtrics microservice retries as it waits for response to become available
     )
+
     if r.status_code != 200:
         raise error.ExportError(
             f"No survey response found for responseId: {responseId}"

gdrive/main.py

@@ -1,9 +1,11 @@
 """
 GDrive Microservice FastAPI Web App.
 """
+
 import fastapi
 import starlette_prometheus
 
+
 from . import api, export_api, analytics_api, settings
 
 app = fastapi.FastAPI()

gdrive/settings.py

@@ -2,6 +2,7 @@
 Configuration for the gdrive microservice settings.
 Context is switched based on if the app is in debug mode.
 """
+
 import json
 import logging
 import os
@@ -36,6 +37,8 @@
 QUALTRICS_APP_URL = os.getenv("QUALTRICS_APP_URL")
 QUALTRICS_APP_PORT = os.getenv("QUALTRICS_APP_PORT")
 
+RAW_COMPLETIONS_SHEET_NAME = os.getenv("GDRIVE_RAW_COMPLETIONS_SHEET_NAME", "Sheet1")
+
 DB_URI = os.getenv("IDVA_DB_CONN_STR")
 SCHEMA = "idva"
 
@@ -52,6 +55,7 @@
             if service["name"] == "gdrive":
                 log.info("Loading credentials from env var")
                 config = service["credentials"]
+
                 break
     else:
         with open(SERVICE_ACCOUNT_FILE) as file:

gdrive/sheets_client.py

@@ -4,6 +4,7 @@
 
 from google.oauth2 import service_account
 from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
 
 from gdrive import settings, error
 
@@ -119,7 +120,7 @@ def add_pivot_tables(
 
 
 def add_new_pages(
-    page_names: [str], sheets_id: str, row_count: int = 1000, column_count: int = 26
+    page_names: List[str], sheets_id: str, row_count: int = 1000, column_count: int = 26
 ):
     new_sheets_reqs = []
     for label in page_names:
@@ -221,17 +222,23 @@ def upload_participant(
     ]
 
     body = {"values": values}
-    result = (
-        sheets_service.spreadsheets()
-        .values()
-        .append(
-            spreadsheetId=settings.SHEETS_ID,
-            range="Sheet1!A1",
-            valueInputOption="RAW",
-            body=body,
+
+    try:
+        result = (
+            sheets_service.spreadsheets()
+            .values()
+            .append(
+                spreadsheetId=settings.SHEETS_ID,
+                range=f"{settings.RAW_COMPLETIONS_SHEET_NAME}!A1",
+                valueInputOption="RAW",
+                body=body,
+            )
+            .execute()
         )
-        .execute()
-    )
-    if "error" in result:
-        raise error.ExportError(result["error"]["message"])
-    return result
+
+        if "error" in result:
+            raise error.ExportError(result["error"]["message"])
+
+        return result
+    except HttpError as e:
+        raise error.ExportError(e)