feat(embedding): add chroma db as a way to store service-public documents

targets/export-elasticsearch/package.json

@@ -29,16 +29,20 @@
     "@socialgouv/cdtn-logger": "^2.11.0",
     "axios": "^0.26.1",
     "body-parser": "^1.19.2",
+    "chromadb": "^1.5.3",
     "cors": "^2.8.5",
     "express": "^4.17.3",
     "inversify": "^6.0.1",
     "inversify-express-utils": "^6.4.3",
+    "langchain": "^0.0.105",
+    "openai": "^3.3.0",
     "reflect-metadata": "^0.1.13",
     "zod": "^3.14.2"
   },
   "devDependencies": {
     "@shared/eslint-config": "^2.11.0",
     "@shared/types": "^2.11.0",
+    "@socialgouv/cdtn-utils": "^4.104.2",
     "@swc/cli": "0.1.55",
     "@swc/core": "1.2.150",
     "@swc/jest": "0.2.20",

targets/export-elasticsearch/request.http

@@ -26,3 +26,25 @@ POST http://localhost:8787/sitemap
 
 ###
 POST http://localhost:8787/copy
+
+
+###
+POST http://localhost:8787/embedding/service-public
+
+###
+GET http://localhost:8787/embedding/service-public/infos
+
+###
+GET http://localhost:8787/embedding/service-public?q=service
+
+###
+GET http://localhost:8787/embedding/list
+
+###
+POST http://localhost:8787/chat
+content-type: application/json
+
+{
+  "question": "Peux-tu m'aider à comprendre ce qu'est la période d'essai ?",
+  "history": ""
+}

targets/export-elasticsearch/src/controllers/chat.ts

@@ -0,0 +1,20 @@
+import type { interfaces } from "inversify-express-utils";
+import { controller, httpPost, request } from "inversify-express-utils";
+import { getName } from "../utils";
+import { inject } from "inversify";
+import { ChatService } from "../services";
+import { ValidatorChatMiddleware, ValidatorChatType } from "./middlewares";
+
+@controller("/chat")
+export class ChatController implements interfaces.Controller {
+  constructor(
+    @inject(getName(ChatService))
+    private readonly service: ChatService
+  ) {}
+
+  @httpPost("/", getName(ValidatorChatMiddleware))
+  async sendMessage(@request() req: Request): Promise<Record<string, any>> {
+    const bdy: ValidatorChatType = req.body as any;
+    return await this.service.ask(bdy.question, bdy.history);
+  }
+}

targets/export-elasticsearch/src/controllers/embedding.ts

@@ -0,0 +1,43 @@
+import type { interfaces } from "inversify-express-utils";
+import {
+  controller,
+  httpGet,
+  httpPost,
+  queryParam,
+} from "inversify-express-utils";
+import { getName } from "../utils";
+import { inject } from "inversify";
+import { EmbeddingService } from "../services";
+
+@controller("/embedding")
+export class EmbeddingController implements interfaces.Controller {
+  constructor(
+    @inject(getName(EmbeddingService))
+    private readonly service: EmbeddingService
+  ) {}
+
+  @httpGet("/service-public")
+  async getServicePublic(
+    @queryParam("q") query: string
+  ): Promise<Record<string, any>> {
+    if (!query) {
+      return { error: "Missing query parameter" };
+    }
+    return await this.service.getServicePublicDocuments(query);
+  }
+
+  @httpPost("/service-public")
+  async ingestServicePublic(): Promise<Record<string, any>> {
+    return await this.service.ingestServicePublicDocuments();
+  }
+
+  @httpGet("/service-public/infos")
+  async infosServicePublic(): Promise<Record<string, any>> {
+    return await this.service.countAndPeekServicePublicDocuments();
+  }
+
+  @httpGet("/list")
+  async listDocuments(): Promise<Record<string, any>> {
+    return await this.service.listAllDocumentsMetadata();
+  }
+}

targets/export-elasticsearch/src/controllers/index.ts

@@ -1,2 +1,4 @@
 export * from "./export";
 export * from "./monitoring";
+export * from "./embedding";
+export * from "./chat";

targets/export-elasticsearch/src/controllers/middlewares/chat.ts

@@ -0,0 +1,26 @@
+import type { NextFunction, Request, Response } from "express";
+import { injectable } from "inversify";
+import { BaseMiddleware } from "inversify-express-utils";
+import { z } from "zod";
+
+import { name } from "../../utils";
+
+const ValidatorChat = z.object({
+  question: z.string(),
+  history: z.string(),
+});
+
+export type ValidatorChatType = z.infer<typeof ValidatorChat>;
+
+@injectable()
+@name("ValidatorChatMiddleware")
+export class ValidatorChatMiddleware extends BaseMiddleware {
+  public handler(req: Request, res: Response, next: NextFunction): void {
+    const parse = ValidatorChat.safeParse(req.body);
+    if (parse.success) {
+      next();
+    } else {
+      res.status(400).json({ errors: parse.error.issues });
+    }
+  }
+}

targets/export-elasticsearch/src/controllers/middlewares/index.ts

@@ -1 +1,2 @@
 export * from "./export";
+export * from "./chat";

targets/export-elasticsearch/src/repositories/documents.ts

@@ -0,0 +1,56 @@
+import { client } from "@shared/graphql-client";
+import { logger } from "@socialgouv/cdtn-logger";
+import { injectable } from "inversify";
+
+import { name } from "../utils";
+import { getDocumentBySource } from "./graphql";
+
+interface Document {
+  id: string;
+  cdtnId: string;
+  title: string;
+  slug: string;
+  source: string;
+  text: string;
+  isPublished: boolean;
+  isSearchable: boolean;
+  metaDescription: string;
+  document: {
+    raw: string;
+    url: string;
+    date: string;
+    description: string;
+    referencedTexts?:
+      | {
+          slug: string;
+          type: string;
+          title: string;
+        }[]
+      | null;
+  };
+  __typename: string;
+}
+
+@injectable()
+@name("DocumentsRepository")
+export class DocumentsRepository {
+  public async getBySource(source: string): Promise<Document[]> {
+    try {
+      const res = await client
+        .query<{ documents: Document[] }>(getDocumentBySource, {
+          source,
+        })
+        .toPromise();
+      if (res.error) {
+        throw res.error;
+      }
+      if (!res.data?.documents) {
+        throw new Error("Failed to get, undefined object");
+      }
+      return res.data.documents;
+    } catch (e) {
+      logger.error(e);
+      throw e;
+    }
+  }
+}

targets/export-elasticsearch/src/repositories/graphql/queries.ts

@@ -83,3 +83,19 @@ query getExportEsStatusByStatus($status: String!) {
     updated_at
   }
 }`;
+
+export const getDocumentBySource = `
+query getDocumentBySource($source: String!) {
+  documents(where: {source: {_eq: $source},  is_available: {_eq: true} }) {
+    id: initial_id
+    cdtnId: cdtn_id
+    title
+    slug
+    source
+    text
+    isPublished: is_published
+    isSearchable: is_searchable
+    metaDescription: meta_description
+    document
+  }
+}`;

targets/export-elasticsearch/src/repositories/index.ts

@@ -1,2 +1,3 @@
 export * from "./azure";
 export * from "./status";
+export * from "./documents";

targets/export-elasticsearch/src/server.ts

@@ -7,14 +7,20 @@ import type { NextFunction, Request, Response } from "express";
 import { Container } from "inversify";
 import { InversifyExpressServer } from "inversify-express-utils";
 
-import { ExportEsRunMiddleware } from "./controllers/middlewares";
+import {
+  ExportEsRunMiddleware,
+  ValidatorChatMiddleware,
+} from "./controllers/middlewares";
 import {
   AzureParameters,
   AzureRepository,
+  DocumentsRepository,
   ExportRepository,
 } from "./repositories";
 import {
+  ChatService,
   CopyContainerService,
+  EmbeddingService,
   ExportService,
   SitemapService,
 } from "./services";
@@ -51,19 +57,31 @@ rootContainer
     process.env.AZ_URL_TO ??
       `https://${process.env.AZ_ACCOUNT_NAME_TO}.blob.core.windows.net`
   );
+
 rootContainer
   .bind<ExportRepository>(getName(ExportRepository))
   .to(ExportRepository);
+rootContainer
+  .bind<DocumentsRepository>(getName(DocumentsRepository))
+  .to(DocumentsRepository);
+
 /* MIDDLEWARE */
 rootContainer
   .bind<ExportEsRunMiddleware>(getName(ExportEsRunMiddleware))
   .to(ExportEsRunMiddleware);
+rootContainer
+  .bind<ValidatorChatMiddleware>(getName(ValidatorChatMiddleware))
+  .to(ValidatorChatMiddleware);
 /* SERVICES */
 rootContainer.bind<ExportService>(getName(ExportService)).to(ExportService);
 rootContainer.bind<SitemapService>(getName(SitemapService)).to(SitemapService);
 rootContainer
   .bind<CopyContainerService>(getName(CopyContainerService))
   .to(CopyContainerService);
+rootContainer
+  .bind<EmbeddingService>(getName(EmbeddingService))
+  .to(EmbeddingService);
+rootContainer.bind<ChatService>(getName(ChatService)).to(ChatService);
 
 // create server
 const server = new InversifyExpressServer(rootContainer);

targets/export-elasticsearch/src/services/chat.ts

@@ -0,0 +1,63 @@
+import { injectable } from "inversify";
+import { name } from "../utils";
+import { ConversationalRetrievalQAChain } from "langchain/chains";
+import { OpenAIEmbeddings } from "langchain/embeddings/openai";
+import { ChatOpenAI } from "langchain/chat_models/openai";
+import { BaseLanguageModel } from "langchain/base_language";
+import { Chroma } from "langchain/vectorstores/chroma";
+
+@injectable()
+@name("ChatService")
+export class ChatService {
+  model: BaseLanguageModel;
+  private QA_PROMPT = `Vous êtes un assistant juridique. Utilisez uniquement les éléments de contexte pour répondre à la question.
+  Votre réponse doit uniquement être en français.
+  Si vous ne connaissez pas la réponse, dites simplement que vous ne savez pas. N'essayez PAS d'inventer une réponse.
+  Si la question n'est pas liée au contexte, répondez poliment que vous êtes réglé pour répondre uniquement aux questions liées au contexte.
+  S'il vous manque des éléments de contexte, demandez à l'utilisateur de vous les fournir dans le cas où il y a plusieurs réponses possibles.
+  N'hésitez pas à donner des examples pour agrémenter votre réponse. De plus, n'hésite pas à reformuler la réponse pour la rendre plus claire.
+  {context}
+  Question: {question}
+  `;
+  private CONDENSE_PROMPT = `Compte tenu de la conversation suivante et d'une question de suivi, reformulez la question de suivi pour en faire une question autonome.
+  Historique du chat:
+  {chat_history}
+  Entrée de suivi: {question}
+  Question autonome:
+  `;
+
+  constructor() {
+    this.model = new ChatOpenAI({
+      openAIApiKey: process.env.OPENAI_API_KEY,
+      streaming: true,
+      modelName: "gpt-3.5-turbo-16k-0613",
+      temperature: 0,
+    });
+  }
+
+  async ask(question: string, historyMessage: string) {
+    const vectorStore = await Chroma.fromExistingCollection(
+      new OpenAIEmbeddings({
+        openAIApiKey: process.env.OPENAI_API_KEY,
+      }),
+      { collectionName: "service-public" }
+    );
+
+    const chain = ConversationalRetrievalQAChain.fromLLM(
+      this.model,
+      vectorStore.asRetriever(),
+      {
+        returnSourceDocuments: true,
+        qaTemplate: this.QA_PROMPT,
+        questionGeneratorTemplate: this.CONDENSE_PROMPT,
+      }
+    );
+
+    const result = await chain.call({
+      question,
+      chat_history: historyMessage,
+    });
+
+    return result;
+  }
+}