Merge branch 'main' into chatbot/llamaindex-pipeline

.changeset/cuddly-cats-retire.md

@@ -0,0 +1,5 @@
+---
+"active-campaign-client": minor
+---
+
+Add resync user handler

.changeset/cyan-dolls-reply.md

@@ -0,0 +1,5 @@
+---
+"strapi-cms": patch
+---
+
+Refactor move ACTIVE_CAMPAIGN_INTEGRATION_ENABLED env var in getter function

.changeset/eighty-seals-worry.md

@@ -0,0 +1,5 @@
+---
+"infrastructure": patch
+---
+
+Change ac sync lambda handler to index.sqsQueue

.changeset/empty-lies-sniff.md

@@ -0,0 +1,5 @@
+---
+"strapi-cms": patch
+---
+
+Fix env variable in lifecycle

.changeset/gorgeous-ghosts-float.md

@@ -0,0 +1,5 @@
+---
+"infrastructure": patch
+---
+
+Use the ddb stream event id as message group id in sqs

.changeset/khaki-knives-unite.md

@@ -0,0 +1,5 @@
+---
+"active-campaign-client": minor
+---
+
+Refactor resyncUserHandler to align contacts and subscriptions in Active Campaign

.changeset/loud-otters-unite.md

@@ -0,0 +1,5 @@
+---
+"active-campaign-client": patch
+---
+
+Add bulk import of contacts

.changeset/modern-hairs-train.md

@@ -0,0 +1,5 @@
+---
+"active-campaign-client": minor
+---
+
+Add SSM client to mange secrets

.changeset/nice-coins-mix.md

@@ -0,0 +1,6 @@
+---
+"infrastructure": minor
+"chatbot": minor
+---
+
+Implemented llm monitoring with LangFuse

.changeset/plenty-buses-hope.md

@@ -0,0 +1,5 @@
+---
+"infrastructure": patch
+---
+
+Added environment variables that allow strapi to integrate with active campaign

.changeset/pretty-gorillas-bow.md

@@ -0,0 +1,5 @@
+---
+"strapi-cms": minor
+---
+
+Fix webinar validateSlug on update

.changeset/weak-monkeys-grab.md

@@ -0,0 +1,5 @@
+---
+"strapi-cms": minor
+---
+
+Add Active Campaign integration to create and delete lists when creating and deleting webinars

.github/workflows/deploy_ac_sync_lambda.yaml

@@ -0,0 +1,78 @@
+name: Deploy Lambda Active Campaign Syncer
+
+on:
+  push:
+    branches:
+      - "main"
+    paths:
+      - "packages/active-campaign-client/**"
+      - ".github/workflows/deploy_ac_sync_lambda.yaml"
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: 'Choose environment'
+        type: choice
+        required: true
+        default: dev
+        options:
+          - dev
+          - prod
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup node
+        uses: actions/setup-node@v3
+        with:
+          node-version: '20.x'
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Build
+        working-directory: packages/active-campaign-client
+        run: npm run build
+
+      - name: Package
+        working-directory: packages/active-campaign-client/dist
+        run: zip -r function.zip .
+
+      - name: Archive build artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: active-campaign-client
+          path: packages/active-campaign-client/dist/function.zip
+
+  deploy:
+    name: Deploy Lambda Active Campaign Syncer
+    runs-on: ubuntu-22.04
+    needs: build
+
+    environment: ${{ github.event.inputs.environment || 'dev' }}
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      - name: Download build artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: active-campaign-client
+          path: ./packages/active-campaign-client/target
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{ secrets.DEPLOY_IAM_ROLE }}
+          aws-region: eu-south-1
+
+      - name: Deploy Lambda function (${{ github.event.inputs.environment || 'dev' }})
+        run: |
+          aws lambda update-function-code \
+          --function-name ac-${{ github.event.inputs.environment || 'dev' }}-sync-lambda \
+          --zip-file fileb://packages/active-campaign-client/target/function.zip

.gitignore

@@ -409,5 +409,8 @@ apps/chatbot/empty_htmls.json
 
 /results
 
+# terraform
+apps/infrastructure/src/builds
+
 ## Python ##
 __pycache__/

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "apps/nextjs-website/.tmp-docs"]
 	path = apps/nextjs-website/.tmp-docs
-	url = https://github.com/pagopa/devportal-docs.git
+	url = https://github.com/pagopa/devportal-docs.git

apps/chatbot/.dockerignore

@@ -2,3 +2,4 @@
 build-devp
 .pytest_cache
 load-test
+*.ipynb

apps/chatbot/.env.example

@@ -1,31 +1,66 @@
-environment=local
-CORS_DOMAINS=["*"]
-PYTHONPATH=app-path
-LOG_LEVEL=DEBUG
+AUTH_COGNITO_ALLOW_ACCOUNT_LINKING=true
+AUTH_COGNITO_CLIENT_ID=...
+AUTH_COGNITO_CLIENT_SECRET=
+AUTH_COGNITO_ISSUER=https://cognito-idp.eu-south-1.amazonaws.com/eu-south-1_xxxxxxxx
+AUTH_DISABLE_SIGNUP=false
+AUTH_DISABLE_USERNAME_PASSWORD=true
 CHB_AWS_ACCESS_KEY_ID=...
-CHB_AWS_SECRET_ACCESS_KEY=...
+CHB_AWS_BEDROCK_EMBED_REGION=eu-central-1
+CHB_AWS_BEDROCK_LLM_REGION=eu-west-3
 CHB_AWS_DEFAULT_REGION=eu-south-1
-CHB_AWS_BEDROCK_REGION=eu-west-3
 CHB_AWS_GUARDRAIL_ID=...
 CHB_AWS_GUARDRAIL_VERSION=...
-CHB_REDIS_URL=...
-CHB_WEBSITE_URL=...
-CHB_REDIS_INDEX_NAME=...
-CHB_LLAMAINDEX_INDEX_ID=...
+CHB_AWS_SECRET_ACCESS_KEY=...
 CHB_DOCUMENTATION_DIR=...
-CHB_USE_PRESIDIO=...
-CHB_GOOGLE_API_KEY=...
-CHB_PROVIDER=...
-CHB_MODEL_ID=...
-CHB_MODEL_TEMPERATURE=...
-CHB_MODEL_MAXTOKENS=...
+CHB_DYNAMODB_URL=http://locahost:8080
 CHB_EMBED_MODEL_ID=...
-CHB_ENGINE_SIMILARITY_TOPK=...
 CHB_ENGINE_SIMILARITY_CUTOFF=...
+CHB_ENGINE_SIMILARITY_TOPK=...
 CHB_ENGINE_USE_ASYNC=True
 CHB_ENGINE_USE_STREAMING=...
-CHB_ENGINE_USE_CHAT_ENGINE=...
+CHB_GOOGLE_API_KEY=...
+CHB_LANGFUSE_HOST=http://localhost:3000
+CHB_LANGFUSE_PUBLIC_KEY=/nonexistent/ssmpath
+CHB_LANGFUSE_SECRET_KEY=/nonexistent/ssmpath
+CHB_LLAMAINDEX_INDEX_ID=...
+CHB_MODEL_ID=...
+CHB_MODEL_MAXTOKENS=...
+CHB_MODEL_TEMPERATURE=...
+CHB_PROVIDER=...
 CHB_QUERY_TABLE_PREFIX=chatbot-local
-CHB_DYNAMODB_URL=http://locahost:8080
-CHB_USE_PRESIDIO=True
+CHB_REDIS_INDEX_NAME=...
+CHB_REDIS_URL=...
 CHB_SESSION_MAX_DURATION_DAYS=1
+CHB_USE_CHAT_ENGINE=True
+CHB_USE_PRESIDIO=True
+CHB_WEBSITE_URL=...
+CORS_DOMAINS=["*"]
+ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000
+environment=local
+LAMBDA_TASK_ROOT=app-dir
+LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=false
+LANGFUSE_INIT_ORG_ID=...
+LANGFUSE_INIT_ORG_NAME=...
+LANGFUSE_INIT_PROJECT_ID=monitor-123
+LANGFUSE_INIT_PROJECT_NAME=Monitor
+LANGFUSE_INIT_PROJECT_PUBLIC_KEY=pk-xxx
+LANGFUSE_INIT_PROJECT_SECRET_KEY=sk-xxx
+LANGFUSE_INIT_USER_EMAIL=...
+LANGFUSE_INIT_USER_NAME=...
+LANGFUSE_INIT_USER_PASSWORD=...
+LANGFUSE_TAG=development
+LANGFUSE_USER_EMAIL=[email protected]
+LANGFUSE_USER_NAME=User
+LANGFUSE_USER_PASSWORD=...
+LOG_LEVEL=DEBUG
+NEXTAUTH_SECRET=mysecret
+NEXTAUTH_URL=http://localhost:3001
+POSTGRES_DB=postgres
+POSTGRES_HOST=localhost
+POSTGRES_PASSWORD=postgres
+POSTGRES_PORT=5432
+POSTGRES_USER=postgres
+DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}
+PYTHONPATH=app-path
+SALT=mysalt
+TELEMETRY_ENABLED=true

apps/chatbot/README.md

@@ -7,10 +7,16 @@ Even though the provider is the Google one, we stored its API key in AWS. So, be
 
 The Retrieval-Augmented Generation (RAG) was implemented using [llama-index](https://docs.llamaindex.ai/en/stable/). All the parameters and prompts used are stored in `config`.
 
+The monitoring is done using [Langfuse](https://langfuse.com/) deployed on AWS.
+
 ## Environment Variables
 
 Create a `.env` file inside this folder and store the environment variables listed in `.env.example`.
 
+cp .env.example .env
+
+Note that the envirables inside `.env` file should be pointing to the AWS infrastructure.s
+
 ## Virtual environment
 
 Before creating your virtual environment, install [Miniconda](https://docs.anaconda.com/miniconda/#quick-command-line-install) and [Poetry](https://python-poetry.org/docs/main#installation) on your device.
@@ -36,30 +42,44 @@ In this way, `PYTHONPATH` points to where the Python packages and modules are, n
 
 To reach the remote redis instance, it is necessary to open a tunnel:
 
-```
     ./scripts/redis-tunnel.sh
-```
 
 Verify that the HTML files that compose the Developer Portal documentation exist in a directory. Otherwise create the documentation. Once you have the documentation directory ready, put its path in `params` and, in the end, create the vector index doing:
 
-```
     python src/modules/create_vector_index.py --params config/params.yaml
-```
 
 This script reads the documentation, split it into chucks with gerarchical organization, and stores it on Redis.
 
 Check out the params in order to store your vector index accordingly.
 
-## test
+## Test
+
+In order to test the chatbot and its APIs, run:
+
+    pytest
+
+For more details, read [TESTBOOK.md](https://github.com/pagopa/developer-portal/blob/main/apps/chatbot/TESTBOOK.md).
+
+## Docker
+
+In order to run the chatbot locally for the first time, you need to:
+
+- install [Docker Compose](https://docs.docker.com/compose/install/),
+- create `.env.local` file running:
+
+        cp .env.example .env.local
+
+  and fill it in,
+
+- run the following bash scripts:
 
-```
-pytest
-```
+        ./docker/docker-compose-build-local.sh
+        ./docker/docker-compose-run-create_index.sh
 
-## Web App
+In this way, the docker images are built and the vector index is stored in Redis.
 
-    python src/webapp/app.py
+Now you can start the API running:
 
-This scripts uses [Gradio](https://www.gradio.app/) framework to lunch a web application at the [default link](http://127.0.0.1:7860) where the user can interact with the chatbot.
+    ./docker/docker-compose-up-api.sh
 
-Both [`user icon`](https://www.flaticon.com/free-icon/user_1077012) and [`chatbot icon`](https://www.flaticon.com/free-icon/chatbot_8943377) are made by [Freepick](https://www.freepik.com/) and they were downloaded from [Flaticon](https://www.flaticon.com/).
+Note that the `docker/compose.yaml` needs `.env.local` file with the correct environment variables.

apps/chatbot/TESTBOOK.md

@@ -0,0 +1,17 @@
+# Testbook
+
+In order to test the chatbot functions and its APIs, run:
+
+    pytest
+
+the command test the function explained in the table below.
+
+|             Function             |       Requirements        |            Masked Inputs            |                                                                               Description                                                                                |
+| :------------------------------: | :-----------------------: | :---------------------------------: | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------: |
+|    `test_connection_redis()`     |       Redis client        |                None                 |                                                                  Check the connection with Redis is up                                                                   |
+|   `test_connection_langfuse()`   |      Langfuse client      |                None                 |                                                                 Check the connection with Langfuse is up                                                                 |
+|    `test_cloud_connection()`     | AWS or Gemini credentials |     LLM and Embedding model ID      |                                                                        Check the models' loading                                                                         |
+|    `test_prompt_templates()`     |        Llama-index        |           `prompts.yaml`            |                                                    Check the prompts have the same variables of the prompts templates                                                    |
+|        `test_pii_mask()`         |         Presidio          |          a string to mask           |                                                                  Check that Presidio works as expected                                                                   |
+| `test_messages_to_chathistory()` |        Llama-index        | chat history from the local storage |                                                  Check the write functioning of creating a chat history in Llama-index                                                   |
+|     `test_chat_generation()`     |        Llama-index        |             two queries             | Check the chatbot generation given a query, then checks again its functioning generating a new answer usinng a second query and the previous interaction as chat history |