Release 1.9.0

backend/src/main/java/com/cloudera/cai/rag/Types.java

@@ -103,6 +103,8 @@ public record RagDataSource(
       @Nullable Integer documentCount,
       @Nullable Long totalDocSize) {}
 
+  public record QueryConfiguration(boolean enableHyde, boolean enableSummaryFilter) {}
+
   @With
   @Builder
   public record Session(
@@ -116,7 +118,8 @@ public record Session(
       Instant lastInteractionTime,
       String inferenceModel,
       String rerankModel,
-      Integer responseChunks) {
+      Integer responseChunks,
+      QueryConfiguration queryConfiguration) {
 
     public static Session fromCreateRequest(CreateSession input, String username) {
       return new Session(
@@ -130,14 +133,20 @@ public static Session fromCreateRequest(CreateSession input, String username) {
           null,
           input.inferenceModel(),
           input.rerankModel(),
-          input.responseChunks());
+          input.responseChunks(),
+          input.queryConfiguration());
     }
   }
 
+  @With
   public record CreateSession(
       String name,
       @Singular List<Long> dataSourceIds,
       String inferenceModel,
       String rerankModel,
-      Integer responseChunks) {}
+      Integer responseChunks,
+      QueryConfiguration queryConfiguration) {}
+
+  public record MetadataMetrics(
+      int numberOfDataSources, int numberOfSessions, int numberOfDocuments) {}
 }

backend/src/main/java/com/cloudera/cai/rag/datasources/RagDataSourceRepository.java

@@ -152,6 +152,15 @@ public void deleteDataSource(Long id) {
         handle -> handle.execute("UPDATE RAG_DATA_SOURCE SET DELETED = ? where ID = ?", true, id));
   }
 
+  public int getNumberOfDataSources() {
+    return jdbi.withHandle(
+        handle -> {
+          try (var query = handle.createQuery("SELECT count(*) FROM RAG_DATA_SOURCE")) {
+            return query.mapTo(Integer.class).one();
+          }
+        });
+  }
+
   // Nullables stuff below here.
 
   public static RagDataSourceRepository createNull() {

backend/src/main/java/com/cloudera/cai/rag/files/RagFileRepository.java

@@ -146,4 +146,13 @@ public RagDocument getRagDocumentById(Long id) {
           }
         });
   }
+
+  public int getNumberOfRagDocuments() {
+    return jdbi.withHandle(
+        handle -> {
+          try (var query = handle.createQuery("SELECT count(*) FROM rag_data_source_document")) {
+            return query.mapTo(Integer.class).one();
+          }
+        });
+  }
 }

backend/src/main/java/com/cloudera/cai/rag/metrics/MetricsController.java

@@ -0,0 +1,61 @@
+/*
+ * CLOUDERA APPLIED MACHINE LEARNING PROTOTYPE (AMP)
+ * (C) Cloudera, Inc. 2024
+ * All rights reserved.
+ *
+ * Applicable Open Source License: Apache 2.0
+ *
+ * NOTE: Cloudera open source products are modular software products
+ * made up of hundreds of individual components, each of which was
+ * individually copyrighted.  Each Cloudera open source product is a
+ * collective work under U.S. Copyright Law. Your license to use the
+ * collective work is as provided in your written agreement with
+ * Cloudera.  Used apart from the collective work, this file is
+ * licensed for your use pursuant to the open source license
+ * identified above.
+ *
+ * This code is provided to you pursuant a written agreement with
+ * (i) Cloudera, Inc. or (ii) a third-party authorized to distribute
+ * this code. If you do not have a written agreement with Cloudera nor
+ * with an authorized and properly licensed third party, you do not
+ * have any rights to access nor to use this code.
+ *
+ * Absent a written agreement with Cloudera, Inc. ("Cloudera") to the
+ * contrary, A) CLOUDERA PROVIDES THIS CODE TO YOU WITHOUT WARRANTIES OF ANY
+ * KIND; (B) CLOUDERA DISCLAIMS ANY AND ALL EXPRESS AND IMPLIED
+ * WARRANTIES WITH RESPECT TO THIS CODE, INCLUDING BUT NOT LIMITED TO
+ * IMPLIED WARRANTIES OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE; (C) CLOUDERA IS NOT LIABLE TO YOU,
+ * AND WILL NOT DEFEND, INDEMNIFY, NOR HOLD YOU HARMLESS FOR ANY CLAIMS
+ * ARISING FROM OR RELATED TO THE CODE; AND (D)WITH RESPECT TO YOUR EXERCISE
+ * OF ANY RIGHTS GRANTED TO YOU FOR THE CODE, CLOUDERA IS NOT LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, PUNITIVE OR
+ * CONSEQUENTIAL DAMAGES INCLUDING, BUT NOT LIMITED TO, DAMAGES
+ * RELATED TO LOST REVENUE, LOST PROFITS, LOSS OF INCOME, LOSS OF
+ * BUSINESS ADVANTAGE OR UNAVAILABILITY, OR LOSS OR CORRUPTION OF
+ * DATA.
+ */
+
+package com.cloudera.cai.rag.metrics;
+
+import com.cloudera.cai.rag.Types;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+@Slf4j
+@RequestMapping("/api/v1/rag/metrics")
+public class MetricsController {
+  private final MetricsService metricsService;
+
+  public MetricsController(MetricsService metricsService) {
+    this.metricsService = metricsService;
+  }
+
+  @GetMapping(produces = "application/json")
+  public Types.MetadataMetrics getMetrics() {
+    return metricsService.getMetrics();
+  }
+}

backend/src/main/java/com/cloudera/cai/rag/metrics/MetricsService.java

@@ -0,0 +1,75 @@
+/*
+ * CLOUDERA APPLIED MACHINE LEARNING PROTOTYPE (AMP)
+ * (C) Cloudera, Inc. 2024
+ * All rights reserved.
+ *
+ * Applicable Open Source License: Apache 2.0
+ *
+ * NOTE: Cloudera open source products are modular software products
+ * made up of hundreds of individual components, each of which was
+ * individually copyrighted.  Each Cloudera open source product is a
+ * collective work under U.S. Copyright Law. Your license to use the
+ * collective work is as provided in your written agreement with
+ * Cloudera.  Used apart from the collective work, this file is
+ * licensed for your use pursuant to the open source license
+ * identified above.
+ *
+ * This code is provided to you pursuant a written agreement with
+ * (i) Cloudera, Inc. or (ii) a third-party authorized to distribute
+ * this code. If you do not have a written agreement with Cloudera nor
+ * with an authorized and properly licensed third party, you do not
+ * have any rights to access nor to use this code.
+ *
+ * Absent a written agreement with Cloudera, Inc. ("Cloudera") to the
+ * contrary, A) CLOUDERA PROVIDES THIS CODE TO YOU WITHOUT WARRANTIES OF ANY
+ * KIND; (B) CLOUDERA DISCLAIMS ANY AND ALL EXPRESS AND IMPLIED
+ * WARRANTIES WITH RESPECT TO THIS CODE, INCLUDING BUT NOT LIMITED TO
+ * IMPLIED WARRANTIES OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE; (C) CLOUDERA IS NOT LIABLE TO YOU,
+ * AND WILL NOT DEFEND, INDEMNIFY, NOR HOLD YOU HARMLESS FOR ANY CLAIMS
+ * ARISING FROM OR RELATED TO THE CODE; AND (D)WITH RESPECT TO YOUR EXERCISE
+ * OF ANY RIGHTS GRANTED TO YOU FOR THE CODE, CLOUDERA IS NOT LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, PUNITIVE OR
+ * CONSEQUENTIAL DAMAGES INCLUDING, BUT NOT LIMITED TO, DAMAGES
+ * RELATED TO LOST REVENUE, LOST PROFITS, LOSS OF INCOME, LOSS OF
+ * BUSINESS ADVANTAGE OR UNAVAILABILITY, OR LOSS OR CORRUPTION OF
+ * DATA.
+ */
+
+package com.cloudera.cai.rag.metrics;
+
+import com.cloudera.cai.rag.Types;
+import com.cloudera.cai.rag.datasources.RagDataSourceRepository;
+import com.cloudera.cai.rag.files.RagFileRepository;
+import com.cloudera.cai.rag.sessions.SessionRepository;
+import org.springframework.stereotype.Component;
+
+@Component
+public class MetricsService {
+  private final RagDataSourceRepository ragDataSourceRepository;
+  private final SessionRepository sessionRepository;
+  private final RagFileRepository ragFileRepository;
+
+  public MetricsService(
+      RagDataSourceRepository ragDataSourceRepository,
+      SessionRepository sessionRepository,
+      RagFileRepository ragFileRepository) {
+    this.ragDataSourceRepository = ragDataSourceRepository;
+    this.sessionRepository = sessionRepository;
+    this.ragFileRepository = ragFileRepository;
+  }
+
+  public Types.MetadataMetrics getMetrics() {
+    var numberOfDataSources = ragDataSourceRepository.getNumberOfDataSources();
+    var numberOfSessions = sessionRepository.getNumberOfSessions();
+    var numberOfDocuments = ragFileRepository.getNumberOfRagDocuments();
+    return new Types.MetadataMetrics(numberOfDataSources, numberOfSessions, numberOfDocuments);
+  }
+
+  public static MetricsService createNull() {
+    return new MetricsService(
+        RagDataSourceRepository.createNull(),
+        SessionRepository.createNull(),
+        RagFileRepository.createNull());
+  }
+}

backend/src/main/java/com/cloudera/cai/rag/sessions/SessionRepository.java

@@ -41,6 +41,8 @@
 import com.cloudera.cai.rag.Types;
 import com.cloudera.cai.rag.configuration.JdbiConfiguration;
 import com.cloudera.cai.util.exceptions.NotFound;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import java.time.Instant;
 import java.util.List;
 import java.util.Map;
@@ -54,7 +56,10 @@
 
 @Component
 public class SessionRepository {
+  public static final Types.QueryConfiguration DEFAULT_QUERY_CONFIGURATION =
+      new Types.QueryConfiguration(false, true);
   private final Jdbi jdbi;
+  private final ObjectMapper objectMapper = new ObjectMapper();
 
   public SessionRepository(Jdbi jdbi) {
     this.jdbi = jdbi;
@@ -65,8 +70,8 @@ public Long create(Types.Session input) {
         handle -> {
           var sql =
               """
-            INSERT INTO CHAT_SESSION (name, created_by_id, updated_by_id, inference_model, rerank_model, response_chunks)
-            VALUES (:name, :createdById, :updatedById, :inferenceModel, :rerankModel, :responseChunks)
+            INSERT INTO CHAT_SESSION (name, created_by_id, updated_by_id, inference_model, rerank_model, response_chunks, query_configuration)
+            VALUES (:name, :createdById, :updatedById, :inferenceModel, :rerankModel, :responseChunks, :queryConfiguration)
           """;
           Long id = insertSession(input, handle, sql);
           insertSessionDataSources(handle, id, input.dataSourceIds());
@@ -90,8 +95,13 @@ INSERT INTO CHAT_SESSION_DATA_SOURCE (chat_session_id, data_source_id)
 
   private Long insertSession(Types.Session input, Handle handle, String sql) {
     try (var update = handle.createUpdate(sql)) {
+      Types.QueryConfiguration queryConfiguration = input.queryConfiguration();
+      String json = objectMapper.writeValueAsString(queryConfiguration);
+      update.bind("queryConfiguration", json);
       update.bindMethods(input);
       return update.executeAndReturnGeneratedKeys("id").mapTo(Long.class).one();
+    } catch (JsonProcessingException e) {
+      throw new RuntimeException(e);
     }
   }
 
@@ -119,26 +129,47 @@ private Stream<Types.Session.SessionBuilder> querySessions(Query query) {
             Types.Session.SessionBuilder sessionBuilder =
                 map.computeIfAbsent(
                     rowView.getColumn("id", Long.class),
-                    sessionId ->
-                        Types.Session.builder()
+                    sessionId -> {
+                      try {
+                        var queryConfiguration = extractQueryConfiguration(rowView);
+                        return Types.Session.builder()
                             .id(sessionId)
                             .name(rowView.getColumn("name", String.class))
                             .inferenceModel(rowView.getColumn("inference_model", String.class))
                             .responseChunks(rowView.getColumn("response_chunks", Integer.class))
                             .rerankModel(rowView.getColumn("rerank_model", String.class))
+                            .queryConfiguration(queryConfiguration)
                             .createdById(rowView.getColumn("created_by_id", String.class))
                             .timeCreated(rowView.getColumn("time_created", Instant.class))
                             .updatedById(rowView.getColumn("updated_by_id", String.class))
                             .timeUpdated(rowView.getColumn("time_updated", Instant.class))
                             .lastInteractionTime(
-                                rowView.getColumn("last_interaction_time", Instant.class)));
+                                rowView.getColumn("last_interaction_time", Instant.class));
+                      } catch (JsonProcessingException e) {
+                        throw new RuntimeException(e);
+                      }
+                    });
             if (rowView.getColumn("data_source_id", Long.class) != null) {
               sessionBuilder.dataSourceId(rowView.getColumn("data_source_id", Long.class));
             }
           });
     }
   }
 
+  private Types.QueryConfiguration extractQueryConfiguration(RowView rowView)
+      throws JsonProcessingException {
+    String queryConfigurationJson = rowView.getColumn("query_configuration", String.class);
+    if (queryConfigurationJson == null) {
+      return DEFAULT_QUERY_CONFIGURATION;
+    }
+    Types.QueryConfiguration queryConfiguration =
+        objectMapper.readValue(queryConfigurationJson, Types.QueryConfiguration.class);
+    if (queryConfiguration == null) {
+      return DEFAULT_QUERY_CONFIGURATION;
+    }
+    return queryConfiguration;
+  }
+
   public List<Types.Session> getSessions() {
     return jdbi.withHandle(
         handle -> {
@@ -155,27 +186,49 @@ public List<Types.Session> getSessions() {
         });
   }
 
-  public static SessionRepository createNull() {
-    return new SessionRepository(JdbiConfiguration.createNull());
-  }
-
   public void delete(Long id) {
     jdbi.useHandle(
         handle -> handle.execute("UPDATE CHAT_SESSION SET DELETED = ? WHERE ID = ?", true, id));
   }
 
   public void update(Types.Session input) {
     var updatedInput = input.withTimeUpdated(Instant.now());
+    String json = serializeQueryConfiguration(input);
     jdbi.useHandle(
         handle -> {
           var sql =
               """
-            UPDATE CHAT_SESSION
-            SET name = :name, updated_by_id = :updatedById, inference_model = :inferenceModel,
-                response_chunks = :responseChunks, time_updated = :timeUpdated, rerank_model = :rerankModel
-            WHERE id = :id
-          """;
-          handle.createUpdate(sql).bindMethods(updatedInput).execute();
+                  UPDATE CHAT_SESSION
+                  SET name = :name, updated_by_id = :updatedById, inference_model = :inferenceModel, query_configuration = :queryConfiguration,
+                      response_chunks = :responseChunks, time_updated = :timeUpdated, rerank_model = :rerankModel
+                  WHERE id = :id
+                """;
+          handle
+              .createUpdate(sql)
+              .bind("queryConfiguration", json)
+              .bindMethods(updatedInput)
+              .execute();
         });
   }
+
+  private String serializeQueryConfiguration(Types.Session input) {
+    try {
+      return objectMapper.writeValueAsString(input.queryConfiguration());
+    } catch (JsonProcessingException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  public int getNumberOfSessions() {
+    return jdbi.withHandle(
+        handle -> {
+          try (var query = handle.createQuery("SELECT count(*) FROM CHAT_SESSION")) {
+            return query.mapTo(Integer.class).one();
+          }
+        });
+  }
+
+  public static SessionRepository createNull() {
+    return new SessionRepository(JdbiConfiguration.createNull());
+  }
 }

backend/src/main/java/com/cloudera/cai/rag/util/UserTokenCookieDecoder.java

@@ -1,4 +1,4 @@
-/*******************************************************************************
+/*
  * CLOUDERA APPLIED MACHINE LEARNING PROTOTYPE (AMP)
  * (C) Cloudera, Inc. 2024
  * All rights reserved.
@@ -76,5 +76,5 @@ public String extractUsername(Cookie[] cookies) {
     return "unknown";
   }
 
-  public record JwtCookie(String username) {}
+  record JwtCookie(String username) {}
 }