Merge branch 'main' into feature/ai-service/dataset-curation-app

README.md

@@ -106,6 +106,9 @@ Wren AI consists of three core services:
 
 Want to get our latest sharing? [Follow us on Medium!](https://blog.getwren.ai/)
 
+- [Easily query your database using LLMs without writing SQL and then import the data into Microsoft Excel.](https://blog.getwren.ai/easily-query-your-database-using-llms-without-writing-sql-and-then-import-the-data-into-microsoft-871c9672c656)
+- [Can AI Solve Data Silos Challenge? New Challenges To The Multi-AI Agents Era.](https://blog.getwren.ai/can-ai-solve-data-silos-challenge-new-challenges-to-the-multi-ai-agents-era-cec7fabc7c20)
+- [Wren AI in Kubernetes: Text-to-SQL](https://blog.getwren.ai/wren-ai-in-kubernetes-text-to-sql-39b82bda3d34)
 - [How to use Meta Llama 3 to query MySQL database using Ollama and Wren AI](https://blog.getwren.ai/how-to-use-meta-llama-3-to-query-mysql-database-using-ollama-on-your-machine-2c087b204e41)
 - [How do we rewrite Wren AI LLM Service to support 1500+ concurrent users online?](https://blog.getwren.ai/how-do-we-rewrite-wren-ai-llm-service-to-support-1500-concurrent-users-online-9ba5c121afc3)
 - [The new wave of Composable Data Systems and the Interface to LLM agents](https://blog.getwren.ai/the-new-wave-of-composable-data-systems-and-the-interface-to-llm-agents-ec8f0a2e7141)

deployment/kustomizations/base/cm.yaml

@@ -13,9 +13,9 @@ data:
   WREN_UI_ENDPOINT: http://wren-ui-svc:3000
 
   #Release version used by wren ui https://github.com/Canner/WrenAI/blob/main/docker/docker-compose.yaml#L85-L88
-  WREN_PRODUCT_VERSION: "0.7.1-rc.1"
+  WREN_PRODUCT_VERSION: "0.7.1"
   WREN_ENGINE_VERSION: "0.7.0"
-  WREN_AI_SERVICE_VERSION: "0.7.1"
+  WREN_AI_SERVICE_VERSION: "0.7.2"
   WREN_UI_VERSION: "0.10.0"
 
   # OpenAI

deployment/kustomizations/patches/cm.yaml

@@ -7,10 +7,10 @@
     WREN_AI_SERVICE_PORT: "5555"
 
     #Release version used by wren ui https://github.com/Canner/WrenAI/blob/main/docker/docker-compose.yaml#L85-L88
-    WREN_PRODUCT_VERSION: "0.7.1-rc.1"
+    WREN_PRODUCT_VERSION: "0.7.1"
     #fix:
     WREN_ENGINE_VERSION: "0.7.0"
-    WREN_AI_SERVICE_VERSION: "0.7.1"
+    WREN_AI_SERVICE_VERSION: "0.7.2"
     #fix:
     WREN_UI_VERSION: "0.10.0"
 

docker/.env.example

@@ -20,9 +20,9 @@ GENERATION_MODEL=gpt-3.5-turbo # gpt-3.5-turbo, gpt-4o, gpt-4-turbo
 
 # version
 # CHANGE THIS TO THE LATEST VERSION
-WREN_PRODUCT_VERSION=0.7.1-rc.1
+WREN_PRODUCT_VERSION=0.7.1
 WREN_ENGINE_VERSION=0.7.0
-WREN_AI_SERVICE_VERSION=0.7.1
+WREN_AI_SERVICE_VERSION=0.7.2
 IBIS_SERVER_VERSION=0.7.0
 WREN_UI_VERSION=0.10.0
 WREN_BOOTSTRAP_VERSION=0.1.4

wren-ai-service/README.md

@@ -4,7 +4,9 @@
 
 Please read the [documentation](https://docs.getwren.ai/concept/wren_ai_service) here to understand the concepts of Wren AI Service.
 
-## Environment Setup
+## Setup for Local Development
+
+### Environment Setup
 
 - Python 3.12.*, recommended to use [`pyenv`](https://github.com/pyenv/pyenv?tab=readme-ov-file#installation) to manage the Python versions
 - install `poetry` with version 1.8.3: `curl -sSL https://install.python-poetry.org | python3 - --version 1.8.3`
@@ -13,7 +15,7 @@ Please read the [documentation](https://docs.getwren.ai/concept/wren_ai_service)
 - [for development] execute `poetry run pre-commit install` to install the pre-commit hooks and `poetry run pre-commit run --all-files` to run the pre-commit checks at the first time to check if everything is set up correctly
 - [for development] to run the tests, execute `make test`
 
-## Start the service for development
+### Start the service for development
 
 The following commands can quickly start the service for development:
 
@@ -24,7 +26,9 @@ The following commands can quickly start the service for development:
     - go to `http://WREN_UI_HOST:WREN_UI_PORT`(default is http://localhost:3000) to interact interact from the UI
 - `make dev-down` to stop the needed containers
 
-## Pipeline Evaluation(Deprecated, will introduce new way to evaluate the speed in the future)
+## Others
+
+### Pipeline Evaluation(Deprecated, will introduce new way to evaluate the speed in the future)
 
 - go to the `eval` folder and run `poetry install` to install the dependencies
 - execute commands in the `wren-ai-service` folder
@@ -59,7 +63,7 @@ The following commands can quickly start the service for development:
     - .html: test report in html format, showing tables and charts
     - .log: test log
 
-## Demo(Deprecated)
+### Demo
 
 - go to the `demo` folder and run `poetry install` to install the dependencies
 - in the `wren-ai-service` folder, open three terminals

wren-launcher/utils/docker.go

@@ -24,7 +24,7 @@ import (
 
 const (
 	// please change the version when the version is updated
-	WREN_PRODUCT_VERSION        string = "0.7.1-rc.1"
+	WREN_PRODUCT_VERSION        string = "0.7.1"
 	DOCKER_COMPOSE_YAML_URL     string = "https://raw.githubusercontent.com/Canner/WrenAI/" + WREN_PRODUCT_VERSION + "/docker/docker-compose.yaml"
 	DOCKER_COMPOSE_LLM_YAML_URL string = "https://raw.githubusercontent.com/Canner/WrenAI/" + WREN_PRODUCT_VERSION + "/docker/docker-compose.llm.yaml"
 	DOCKER_COMPOSE_ENV_URL      string = "https://raw.githubusercontent.com/Canner/WrenAI/" + WREN_PRODUCT_VERSION + "/docker/.env.example"

wren-ui/src/apollo/server/adaptors/tests/ibisAdaptor.test.ts

@@ -48,7 +48,7 @@ describe('IbisAdaptor', () => {
   };
 
   const mockClickHouseConnectionInfo: CLICK_HOUSE_CONNECTION_INFO = {
-    host: 'localhost',
+    host: 'my-host',
     port: 8443,
     database: 'my-database',
     user: 'my-user',
@@ -129,7 +129,7 @@ describe('IbisAdaptor', () => {
 
     expect(result).toEqual([]);
     expect(mockedAxios.post).toHaveBeenCalledWith(
-      `${ibisServerEndpoint}/v2/ibis/mssql/metadata/constraints`,
+      `${ibisServerEndpoint}/v2/connector/mssql/metadata/constraints`,
       { connectionInfo: expectConnectionInfo },
     );
   });
@@ -153,11 +153,61 @@ describe('IbisAdaptor', () => {
 
     expect(result).toEqual([]);
     expect(mockedAxios.post).toHaveBeenCalledWith(
-      `${ibisServerEndpoint}/v2/ibis/mysql/metadata/constraints`,
+      `${ibisServerEndpoint}/v2/connector/mysql/metadata/constraints`,
       { connectionInfo: expectConnectionInfo },
     );
   });
 
+  // check clickhouse connection info
+  it.each([
+    [
+      {
+        host: 'my-host',
+        port: 8443,
+        database: 'my-database',
+        user: 'my-user',
+        password: 'my-password',
+        ssl: true,
+      },
+      `clickhouse://my-user:my-password@my-host:8443/my-database?secure=1`,
+    ],
+    [
+      {
+        host: 'my-host',
+        port: 8443,
+        database: 'my-database',
+        user: 'my-user',
+        password: 'my-password',
+        ssl: false,
+      },
+      `clickhouse://my-user:my-password@my-host:8443/my-database?`,
+    ],
+  ])(
+    'should get correct clickhouse connection info',
+    async (connectionInfo, expectConnectionUrl) => {
+      const mockResponse = { data: [] };
+      mockedAxios.post.mockResolvedValue(mockResponse);
+      // mock decrypt method in Encryptor to return the same password
+      mockedEncryptor.prototype.decrypt.mockReturnValue(
+        JSON.stringify({ password: connectionInfo.password }),
+      );
+
+      const result = await ibisAdaptor.getConstraints(
+        DataSourceName.CLICK_HOUSE,
+        connectionInfo,
+      );
+      const expectConnectionInfo = {
+        connectionUrl: expectConnectionUrl,
+      };
+
+      expect(result).toEqual([]);
+      expect(mockedAxios.post).toHaveBeenCalledWith(
+        `${ibisServerEndpoint}/v2/connector/clickhouse/metadata/constraints`,
+        { connectionInfo: expectConnectionInfo },
+      );
+    },
+  );
+
   it('should get click house constraints', async () => {
     const mockResponse = { data: [] };
     mockedAxios.post.mockResolvedValue(mockResponse);
@@ -170,13 +220,15 @@ describe('IbisAdaptor', () => {
       DataSourceName.CLICK_HOUSE,
       mockClickHouseConnectionInfo,
     );
-    const expectConnectionInfo = Object.entries(
-      mockClickHouseConnectionInfo,
-    ).reduce((acc, [key, value]) => ((acc[snakeCase(key)] = value), acc), {});
+    const { user, password, host, port, database, ssl } =
+      mockClickHouseConnectionInfo;
+    const expectConnectionInfo = {
+      connectionUrl: `clickhouse://${user}:${password}@${host}:${port}/${database}${ssl ? '?secure=1' : ''}`,
+    };
 
     expect(result).toEqual([]);
     expect(mockedAxios.post).toHaveBeenCalledWith(
-      `${ibisServerEndpoint}/v2/ibis/clickhouse/metadata/constraints`,
+      `${ibisServerEndpoint}/v2/connector/clickhouse/metadata/constraints`,
       { connectionInfo: expectConnectionInfo },
     );
   });
@@ -195,7 +247,7 @@ describe('IbisAdaptor', () => {
 
     expect(result).toEqual([]);
     expect(mockedAxios.post).toHaveBeenCalledWith(
-      `${ibisServerEndpoint}/v2/ibis/postgres/metadata/constraints`,
+      `${ibisServerEndpoint}/v2/connector/postgres/metadata/constraints`,
       {
         connectionInfo: {
           connectionUrl: postgresConnectionUrl,
@@ -229,7 +281,7 @@ describe('IbisAdaptor', () => {
 
     expect(result).toEqual([]);
     expect(mockedAxios.post).toHaveBeenCalledWith(
-      `${ibisServerEndpoint}/v2/ibis/bigquery/metadata/constraints`,
+      `${ibisServerEndpoint}/v2/connector/bigquery/metadata/constraints`,
       { connectionInfo: expectConnectionInfo },
     );
   });
@@ -254,7 +306,7 @@ describe('IbisAdaptor', () => {
 
     expect(result).toEqual({ valid: true, message: null });
     expect(mockedAxios.post).toHaveBeenCalledWith(
-      `${ibisServerEndpoint}/v2/ibis/postgres/validate/column_is_valid`,
+      `${ibisServerEndpoint}/v2/connector/postgres/validate/column_is_valid`,
       {
         connectionInfo: { connectionUrl: postgresConnectionUrl },
         manifestStr: Buffer.from(JSON.stringify(mockManifest)).toString(
@@ -286,7 +338,7 @@ describe('IbisAdaptor', () => {
 
     expect(result).toEqual({ valid: false, message: 'Error' });
     expect(mockedAxios.post).toHaveBeenCalledWith(
-      `${ibisServerEndpoint}/v2/ibis/postgres/validate/column_is_valid`,
+      `${ibisServerEndpoint}/v2/connector/postgres/validate/column_is_valid`,
       {
         connectionInfo: { connectionUrl: postgresConnectionUrl },
         manifestStr: Buffer.from(JSON.stringify(mockManifest)).toString(

wren-ui/src/apollo/server/services/modelService.ts

@@ -8,7 +8,12 @@ import {
   ModelColumn,
   Relation,
 } from '@server/repositories';
-import { getLogger } from '@server/utils';
+import {
+  getLogger,
+  parseJson,
+  replaceAllowableSyntax,
+  validateDisplayName,
+} from '@server/utils';
 import { RelationData, UpdateRelationData } from '@server/types';
 import { IProjectService } from './projectService';
 import {
@@ -21,14 +26,10 @@ import { IMDLService } from './mdlService';
 import { IWrenEngineAdaptor } from '../adaptors/wrenEngineAdaptor';
 import { ValidationRules } from '@server/adaptors/ibisAdaptor';
 import { isEmpty, capitalize } from 'lodash';
-import {
-  replaceAllowableSyntax,
-  validateDisplayName,
-} from '@server/utils/regex';
+import {} from '@server/utils/regex';
 import * as Errors from '@server/utils/error';
 import { DataSourceName } from '@server/types';
 import { IQueryService } from './queryService';
-import { canParseJson } from '@/utils/helper';
 
 const logger = getLogger('ModelService');
 logger.level = 'debug';
@@ -143,7 +144,7 @@ export class ModelService implements IModelService {
       } as CheckCalculatedFieldCanQueryData);
     logger.debug(`${logTitle} : checkCalculatedFieldCanQuery: ${canQuery}`);
     if (!canQuery) {
-      const parsedErrorMessage = canParseJson(errorMessage);
+      const parsedErrorMessage = parseJson(errorMessage);
       throw Errors.create(Errors.GeneralErrorCodes.INVALID_CALCULATED_FIELD, {
         customMessage: parsedErrorMessage?.message || errorMessage,
         originalError: parsedErrorMessage || null,

wren-ui/src/apollo/server/utils/helper.ts

@@ -0,0 +1,11 @@
+/**
+ * @function
+ * @description Retrieve json without error
+ */
+export const parseJson = (data) => {
+  try {
+    return JSON.parse(data);
+  } catch (_e) {
+    return false;
+  }
+};

wren-ui/src/apollo/server/utils/index.ts

@@ -4,3 +4,5 @@ export * from './encode';
 export * from './string';
 export * from './docker';
 export * from './model';
+export * from './helper';
+export * from './regex';

wren-ui/src/components/table/MultiSelectBox.tsx

@@ -41,8 +41,8 @@ interface Props {
 
 export default function MultiSelectBox(props: Props) {
   const { columns, loading, items, onChange, value } = props;
-  const [selectedRowKeys, setSelectedRowKeys] = useState<React.Key[]>(
-    value || [],
+  const [selectedRowKeys, setSelectedRowKeys] = useState<Set<string>>(
+    new Set(value),
   );
   const [searchValue, setSearchValue] = useState<string>('');
   const formItemContext =
@@ -59,9 +59,15 @@ export default function MultiSelectBox(props: Props) {
       : items;
   }, [items, searchValue]);
 
-  const onSelect = (rowKeys: React.Key[]) => {
-    setSelectedRowKeys(rowKeys);
-    onChange && onChange(rowKeys as string[]);
+  const onSelect = (rowKey: string) => {
+    const newSelectedRowKey = new Set(selectedRowKeys);
+    if (newSelectedRowKey.has(rowKey)) {
+      newSelectedRowKey.delete(rowKey);
+    } else {
+      newSelectedRowKey.add(rowKey);
+    }
+    setSelectedRowKeys(newSelectedRowKey);
+    onChange && onChange(Array.from(newSelectedRowKey));
   };
 
   const onSearchChange = (event) => {
@@ -71,9 +77,9 @@ export default function MultiSelectBox(props: Props) {
   };
 
   const total =
-    selectedRowKeys.length === 0
+    selectedRowKeys.size === 0
       ? items.length
-      : `${selectedRowKeys.length}/${items.length}`;
+      : `${selectedRowKeys.size}/${items.length}`;
 
   return (
     <StyledBox
@@ -91,8 +97,24 @@ export default function MultiSelectBox(props: Props) {
       <Table
         rowSelection={{
           type: 'checkbox',
-          selectedRowKeys,
-          onChange: onSelect,
+          selectedRowKeys: Array.from(selectedRowKeys),
+          onSelect: (record) => onSelect(record['value']),
+          onChange(keys) {
+            // if more than 1 rows selected and there is only one possibility, if user selects all rows.
+            if (keys.length !== 1) {
+              if (keys.length === 0) {
+                setSelectedRowKeys(new Set());
+                onChange && onChange([]);
+                return;
+              }
+              const newSelectedRowKeys = [
+                ...selectedRowKeys,
+                ...(keys as string[]),
+              ];
+              setSelectedRowKeys(new Set(newSelectedRowKeys));
+              onChange && onChange(newSelectedRowKeys);
+            }
+          },
         }}
         rowKey={(record) => record.value}
         columns={columns}

wren-ui/src/utils/helper.ts

@@ -19,11 +19,3 @@ export const parseJson = (data) => {
     return data;
   }
 };
-
-export const canParseJson = (data) => {
-  try {
-    return JSON.parse(data);
-  } catch (_e) {
-    return false;
-  }
-};