[CAI-162] Chatbot/Save masked questions and answers using Presidio (#…

…1213)

* chore(chatbot): ssm var llamaindex for local development

* chore: ignore pycache files

* feat(chatbot): save masked answers and questions

* fix(chatbot): remove test comments

* chore: changeset

.changeset/sixty-books-agree.md

@@ -0,0 +1,6 @@
+---
+"chatbot": major
+"infrastructure": patch
+---
+
+Presidio

.gitignore

@@ -408,3 +408,6 @@ apps/chatbot/.env
 apps/chatbot/empty_htmls.json
 
 /results
+
+# python
+__pycache__/

apps/chatbot/src/app/main.py

@@ -82,7 +82,7 @@ async def query_creation (
   else:
     queriedAt = query.queriedAt
 
-  body = {
+  bodyToReturn = {
     "id": f'{uuid.uuid4()}',
     "sessionId": session['id'],
     "question": query.question,
@@ -92,11 +92,14 @@ async def query_creation (
     "badAnswer": False
   }
 
+  bodyToSave = bodyToReturn.copy()
+  bodyToSave["question"] = chatbot.mask_pii(query.question)
+  bodyToSave["answer"] = chatbot.mask_pii(answer)
   try:
-    table_queries.put_item(Item = body)
+    table_queries.put_item(Item = bodyToSave)
   except (BotoCoreError, ClientError) as e:
     raise HTTPException(status_code=422, detail=f"[POST /queries] error: {e}")
-  return body
+  return bodyToReturn
 
 
 def current_user_id(authorization: str):

apps/chatbot/src/modules/chatbot.py

@@ -127,10 +127,13 @@ def _unmask_reference(self, response_str: str, nodes) -> str:
 
 
     def mask_pii(self, message: str) -> str:
-        try:
-            return self.pii.mask_pii(message)
-        except Exception as e:
-            logging.warning(f"[chatbot.py - mask_pii] exception in mask_pii: {e}")
+        if USE_PRESIDIO:
+            try:
+                return self.pii.mask_pii(message)
+            except Exception as e:
+                logging.warning(f"[chatbot.py - mask_pii] exception in mask_pii: {e}")
+        else:
+            return message
 
 
     def generate(self, query_str: str) -> str:

apps/infrastructure/src/modules/chatbot/lambda_chatbot.tf

@@ -105,6 +105,16 @@ module "index_id_ssm_parameter" {
   ignore_value_changes = true
 }
 
+module "index_id_ssm_parameter_local" {
+  source = "git::https://github.com/terraform-aws-modules/terraform-aws-ssm-parameter.git?ref=77d2c139784197febbc8f8e18a33d23eb4736879" # v1.1.0
+
+  name                 = "/chatbot/index_id_local"
+  value                = "49c13f0d-d164-49f1-b5d4-8bdc0632d0de"
+  type                 = "String"
+  secure_type          = true
+  ignore_value_changes = true
+}
+
 # Invoke the lambda function every 3 minutes from 6:00 am to 11:00 pm to keep it warm
 resource "aws_cloudwatch_event_rule" "lambda_invocation_rule" {
   name                = "${local.prefix}-lambda-invocation-rule"
@@ -130,4 +140,4 @@ resource "aws_lambda_permission" "allow_eventbridge" {
   principal     = "events.amazonaws.com"
   source_arn    = aws_cloudwatch_event_rule.lambda_invocation_rule.arn
   statement_id  = "AllowExecutionFromEventBridge"
-}
+}