From 0c52af59015512a3c5c5c8ccf99f4cb4c42ceee3 Mon Sep 17 00:00:00 2001 From: Viduni Wickramarachchi Date: Fri, 13 Dec 2024 08:25:46 -0500 Subject: [PATCH 01/26] [Obs AI Assistant] Update evaluation framework readme --- .../observability_ai_assistant_app/scripts/evaluation/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/README.md b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/README.md index f8d10c679e622..5bb17f9972407 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/README.md +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/README.md @@ -14,7 +14,7 @@ This tool is developed for our team working on the Elastic Observability platfor Run the tool using: -`$ node x-pack/solutions/observability/plugins/observability_solution/observability_ai_assistant_app/scripts/evaluation/index.js` +`$ node x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/index.js` This will evaluate all existing scenarios, and write the evaluation results to the terminal. From e056f20d7b803c666e77c94612501c1fbaac3413 Mon Sep 17 00:00:00 2001 From: Viduni Wickramarachchi Date: Fri, 13 Dec 2024 08:53:20 -0500 Subject: [PATCH 02/26] [Obs AI Assistant] Fix auth for the kibana url when custom elasticsearch credentials are provided --- .../scripts/evaluation/get_service_urls.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/get_service_urls.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/get_service_urls.ts index 859b47987a470..4e0a089f2c295 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/get_service_urls.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/get_service_urls.ts @@ -136,7 +136,7 @@ export async function getServiceUrls({ const kibanaUrlWithAuth = format({ ...parsedKibanaUrl, - auth: parsedKibanaUrl.auth || auth, + auth: parsedKibanaUrl.auth || 'elastic:changeme', }); const validatedKibanaUrl = await getKibanaUrl({ kibana: kibanaUrlWithAuth, log }); From 5d043ae3aeb05361f8239f1a09079ffd120839e5 Mon Sep 17 00:00:00 2001 From: Viduni Wickramarachchi Date: Fri, 13 Dec 2024 12:45:37 -0500 Subject: [PATCH 03/26] [Obs AI Assistant] Create dataview if it doesn't exist --- .../evaluation/scenarios/alerts/index.spec.ts | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts index 8e466bbdbea4f..2ac82db452ee6 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts @@ -7,7 +7,7 @@ /// -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import { RuleResponse } from '@kbn/alerting-plugin/common/routes/rule/response/types/v1'; import moment from 'moment'; import { apm, timerange } from '@kbn/apm-synthtrace-client'; @@ -31,13 +31,27 @@ describe('alert function', () => { ruleIds.push(responseApmRule.data.id); logger.info('Creating dataview'); - - await kibanaClient.callKibana( + const dataViewId = customThresholdAIAssistantLogCount.dataViewParams.options.id; + const existingDataView = await kibanaClient.callKibana( 'post', - { pathname: '/api/content_management/rpc/create' }, - customThresholdAIAssistantLogCount.dataViewParams + { pathname: '/api/content_management/rpc/get' }, + { + contentTypeId: customThresholdAIAssistantLogCount.dataViewParams.contentTypeId, + id: dataViewId, + version: 1, + } ); + if (!existingDataView) { + await kibanaClient.callKibana( + 'post', + { pathname: '/api/content_management/rpc/create' }, + customThresholdAIAssistantLogCount.dataViewParams + ); + } else { + logger.info('Data view already exists, skipping creation'); + } + logger.info('Creating logs rule'); const responseLogsRule = await kibanaClient.callKibana( 'post', From f04074a951890fe6a0d05318f8a0fc77298379b4 Mon Sep 17 00:00:00 2001 From: Viduni Wickramarachchi Date: Fri, 13 Dec 2024 13:01:50 -0500 Subject: [PATCH 04/26] [Obs AI Assistant] Logs for service urls --- .../scripts/evaluation/evaluation.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/evaluation.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/evaluation.ts index a01b276c37bdf..34492e7013e8a 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/evaluation.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/evaluation.ts @@ -37,6 +37,9 @@ function runEvaluations() { kibana: argv.kibana, }); + log.info(`Elasticsearch URL: ${serviceUrls.esUrl}`); + log.info(`Kibana URL: ${serviceUrls.kibanaUrl}`); + const kibanaClient = new KibanaClient(log, serviceUrls.kibanaUrl, argv.spaceId); const esClient = new Client({ node: serviceUrls.esUrl, From 63e1618ecba63ac977e12806ca3e111ecd01aff1 Mon Sep 17 00:00:00 2001 From: Viduni Wickramarachchi Date: Fri, 13 Dec 2024 13:03:52 -0500 Subject: [PATCH 05/26] [Obs AI Assistant] Temp skip for scenarios except alerts --- .../scripts/evaluation/scenarios/apm/index.spec.ts | 4 ++-- .../scripts/evaluation/scenarios/elasticsearch/index.spec.ts | 4 ++-- .../scripts/evaluation/scenarios/esql/index.spec.ts | 4 ++-- .../scripts/evaluation/scenarios/kb/index.spec.ts | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/apm/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/apm/index.spec.ts index 361d436f8ecec..889305340898f 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/apm/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/apm/index.spec.ts @@ -7,7 +7,7 @@ /// -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import moment from 'moment'; import { apm, timerange, serviceMap } from '@kbn/apm-synthtrace-client'; import { RuleResponse } from '@kbn/alerting-plugin/common/routes/rule/response/types/v1'; @@ -15,7 +15,7 @@ import { MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; import { chatClient, kibanaClient, synthtraceEsClients } from '../../services'; import { apmErrorCountAIAssistant } from '../../alert_templates/templates'; -describe('apm', () => { +describe.skip('apm', () => { const ruleIds: any[] = []; before(async () => { const responseApmRule = await kibanaClient.callKibana( diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/elasticsearch/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/elasticsearch/index.spec.ts index 898489290cc10..0fa397dd3f818 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/elasticsearch/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/elasticsearch/index.spec.ts @@ -7,11 +7,11 @@ /// -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import { MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; import { chatClient, esClient } from '../../services'; -describe('elasticsearch functions', () => { +describe.skip('elasticsearch functions', () => { describe('health', () => { it('returns the cluster health state', async () => { const conversation = await chatClient.complete( diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/esql/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/esql/index.spec.ts index 37699876f6165..86024cf180635 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/esql/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/esql/index.spec.ts @@ -8,7 +8,7 @@ /// import { apm, timerange } from '@kbn/apm-synthtrace-client'; -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import moment from 'moment'; import { chatClient, esClient, synthtraceEsClients } from '../../services'; @@ -43,7 +43,7 @@ async function evaluateEsqlQuery({ return; } -describe('ES|QL query generation', () => { +describe.skip('ES|QL query generation', () => { describe('other queries', () => { describe('with packetbeat data', () => { before(async () => { diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts index 9e27ff94f3111..909ebf3c09f12 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts @@ -7,11 +7,11 @@ /// -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import { MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; import { chatClient, esClient } from '../../services'; -describe('kb functions', () => { +describe.skip('kb functions', () => { it('summarizes and recalls information', async () => { let conversation = await chatClient.complete( 'Remember that this cluster is used to test the AI Assistant using the Observability AI Evaluation Framework' From f2e8d572f0a470c1d68a37ba12db7add3fc0a025 Mon Sep 17 00:00:00 2001 From: Viduni Wickramarachchi Date: Fri, 13 Dec 2024 13:11:15 -0500 Subject: [PATCH 06/26] [Obs AI Assistant] Add header to enable accessing internal APIs --- .../scripts/evaluation/kibana_client.ts | 8 ++++++-- .../scripts/evaluation/scenarios/alerts/index.spec.ts | 4 +--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts index 69f6715da2dbe..9c7c5a2dc2c0a 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts @@ -378,7 +378,7 @@ export class KibanaClient { pathname: '/internal/observability_ai_assistant/chat', }), params, - { responseType: 'stream', timeout: NaN } + { responseType: 'stream', timeout: NaN, headers: { 'x-elastic-internal-origin': true } } ); }).pipe( switchMap((response) => streamIntoObservable(response.data)), @@ -462,7 +462,11 @@ export class KibanaClient { title: currentTitle, scopes, }, - { responseType: 'stream', timeout: NaN } + { + responseType: 'stream', + timeout: NaN, + headers: { 'x-elastic-internal-origin': true }, + } ) ); }).pipe( diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts index 2ac82db452ee6..a50729e1a0ba3 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts @@ -31,6 +31,7 @@ describe('alert function', () => { ruleIds.push(responseApmRule.data.id); logger.info('Creating dataview'); + // TODO: consider spaceId when checking for existingDataView const dataViewId = customThresholdAIAssistantLogCount.dataViewParams.options.id; const existingDataView = await kibanaClient.callKibana( 'post', @@ -61,13 +62,11 @@ describe('alert function', () => { ruleIds.push(responseLogsRule.data.id); logger.debug('Cleaning APM indices'); - await synthtraceEsClients.apmSynthtraceEsClient.clean(); const myServiceInstance = apm.service('my-service', 'production', 'go').instance('my-instance'); logger.debug('Indexing synthtrace data'); - await synthtraceEsClients.apmSynthtraceEsClient.index( timerange(moment().subtract(15, 'minutes'), moment()) .interval('1m') @@ -92,7 +91,6 @@ describe('alert function', () => { ); logger.debug('Triggering a rule run'); - await Promise.all( ruleIds.map((ruleId) => kibanaClient.callKibana('post', { From 405678d79df7508bff33b21dcbfc50f8ea53db25 Mon Sep 17 00:00:00 2001 From: Viduni Wickramarachchi Date: Fri, 13 Dec 2024 14:38:51 -0500 Subject: [PATCH 07/26] [Obs AI Assistant] Fix apm afterAll hook --- .../scripts/evaluation/kibana_client.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts index 9c7c5a2dc2c0a..059f72a11c718 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts @@ -124,7 +124,7 @@ export class KibanaClient { return this.axios({ method, url, - data: data || {}, + ...(method.toLowerCase() !== 'delete' ? { data: data || {} } : {}), headers: { 'kbn-xsrf': 'true', 'x-elastic-internal-origin': 'foo', From 5869768fa626d0dfa5ead36b243d2d9ee547ff4b Mon Sep 17 00:00:00 2001 From: Viduni Wickramarachchi Date: Tue, 17 Dec 2024 08:40:38 -0500 Subject: [PATCH 08/26] [Obs AI Assistant] Update error handling --- .../observability_ai_assistant/server/functions/index.ts | 1 + .../scripts/evaluation/evaluation.ts | 1 - .../scripts/evaluation/kibana_client.ts | 2 +- .../scripts/evaluation/scenarios/alerts/index.spec.ts | 2 +- .../scripts/evaluation/scenarios/apm/index.spec.ts | 2 +- .../scripts/evaluation/scenarios/elasticsearch/index.spec.ts | 4 ++-- .../scripts/evaluation/scenarios/esql/index.spec.ts | 2 +- .../scripts/evaluation/scenarios/kb/index.spec.ts | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/x-pack/platform/plugins/shared/observability_solution/observability_ai_assistant/server/functions/index.ts b/x-pack/platform/plugins/shared/observability_solution/observability_ai_assistant/server/functions/index.ts index 244c867e7f644..f06b397f31268 100644 --- a/x-pack/platform/plugins/shared/observability_solution/observability_ai_assistant/server/functions/index.ts +++ b/x-pack/platform/plugins/shared/observability_solution/observability_ai_assistant/server/functions/index.ts @@ -91,6 +91,7 @@ export const registerFunctions: RegistrationCallback = async ({ functions.registerInstruction(({ availableFunctionNames }) => { const instructions: string[] = []; + console.log('availableFunctionNames', availableFunctionNames); if ( availableFunctionNames.includes(QUERY_FUNCTION_NAME) && diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/evaluation.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/evaluation.ts index 34492e7013e8a..b7545f8636b9d 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/evaluation.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/evaluation.ts @@ -38,7 +38,6 @@ function runEvaluations() { }); log.info(`Elasticsearch URL: ${serviceUrls.esUrl}`); - log.info(`Kibana URL: ${serviceUrls.kibanaUrl}`); const kibanaClient = new KibanaClient(log, serviceUrls.kibanaUrl, argv.spaceId); const esClient = new Client({ diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts index 059f72a11c718..a317dcd758f0b 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts @@ -130,7 +130,7 @@ export class KibanaClient { 'x-elastic-internal-origin': 'foo', }, }).catch((error) => { - if (isAxiosError(error)) { + if (isAxiosError(error) && error.response?.status !== 404) { const interestingPartsOfError = { ...omit(error, 'request', 'response', 'config'), ...pick( diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts index a50729e1a0ba3..686d11e0b5728 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts @@ -155,7 +155,7 @@ describe('alert function', () => { 'post', { pathname: `/api/content_management/rpc/delete` }, { - contentTypeId: 'index-pattern', + contentTypeId: customThresholdAIAssistantLogCount.dataViewParams.contentTypeId, id: customThresholdAIAssistantLogCount.dataViewParams.options.id, options: { force: true }, version: 1, diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/apm/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/apm/index.spec.ts index 889305340898f..0612d8cffad66 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/apm/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/apm/index.spec.ts @@ -15,7 +15,7 @@ import { MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; import { chatClient, kibanaClient, synthtraceEsClients } from '../../services'; import { apmErrorCountAIAssistant } from '../../alert_templates/templates'; -describe.skip('apm', () => { +describe('apm', () => { const ruleIds: any[] = []; before(async () => { const responseApmRule = await kibanaClient.callKibana( diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/elasticsearch/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/elasticsearch/index.spec.ts index 0fa397dd3f818..52c69254d1a5e 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/elasticsearch/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/elasticsearch/index.spec.ts @@ -11,7 +11,7 @@ import expect from '@kbn/expect/expect'; import { MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; import { chatClient, esClient } from '../../services'; -describe.skip('elasticsearch functions', () => { +describe('elasticsearch functions', () => { describe('health', () => { it('returns the cluster health state', async () => { const conversation = await chatClient.complete( @@ -99,7 +99,7 @@ describe.skip('elasticsearch functions', () => { describe('assistant created index', () => { it('creates index, adds documents and deletes index', async () => { let conversation = await chatClient.complete( - 'Create a new index called testing_ai_assistant what will have two documents, one for the test_suite alerts with message "This test is for alerts" and another one for the test_suite esql with the message "This test is for esql"' + 'Create a new index called testing_ai_assistant that will have two documents, one for the test_suite alerts with message "This test is for alerts" and another one for the test_suite esql with the message "This test is for esql"' ); conversation = await chatClient.complete( diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/esql/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/esql/index.spec.ts index 86024cf180635..15e9b7d6a0507 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/esql/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/esql/index.spec.ts @@ -43,7 +43,7 @@ async function evaluateEsqlQuery({ return; } -describe.skip('ES|QL query generation', () => { +describe('ES|QL query generation', () => { describe('other queries', () => { describe('with packetbeat data', () => { before(async () => { diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts index 909ebf3c09f12..e88ef8e037cb9 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts @@ -11,7 +11,7 @@ import expect from '@kbn/expect/expect'; import { MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; import { chatClient, esClient } from '../../services'; -describe.skip('kb functions', () => { +describe('kb functions', () => { it('summarizes and recalls information', async () => { let conversation = await chatClient.complete( 'Remember that this cluster is used to test the AI Assistant using the Observability AI Evaluation Framework' From 6196f53813a59adee110b74652129e315cff14bd Mon Sep 17 00:00:00 2001 From: Viduni Wickramarachchi Date: Tue, 17 Dec 2024 08:41:48 -0500 Subject: [PATCH 09/26] [Obs AI Assistant] Update calls to internal urls --- .../observability_ai_assistant/server/functions/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/platform/plugins/shared/observability_solution/observability_ai_assistant/server/functions/index.ts b/x-pack/platform/plugins/shared/observability_solution/observability_ai_assistant/server/functions/index.ts index f06b397f31268..244c867e7f644 100644 --- a/x-pack/platform/plugins/shared/observability_solution/observability_ai_assistant/server/functions/index.ts +++ b/x-pack/platform/plugins/shared/observability_solution/observability_ai_assistant/server/functions/index.ts @@ -91,7 +91,6 @@ export const registerFunctions: RegistrationCallback = async ({ functions.registerInstruction(({ availableFunctionNames }) => { const instructions: string[] = []; - console.log('availableFunctionNames', availableFunctionNames); if ( availableFunctionNames.includes(QUERY_FUNCTION_NAME) && From 3ddb0de60701d044c5ffb8e8a425324eeabb7e0b Mon Sep 17 00:00:00 2001 From: Viduni Wickramarachchi Date: Tue, 17 Dec 2024 09:16:34 -0500 Subject: [PATCH 10/26] [Obs AI Assistant] Improve data view creation --- .../scripts/evaluation/kibana_client.ts | 2 +- .../evaluation/scenarios/alerts/index.spec.ts | 22 ++++++------------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts index a317dcd758f0b..531858de87092 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts @@ -130,7 +130,7 @@ export class KibanaClient { 'x-elastic-internal-origin': 'foo', }, }).catch((error) => { - if (isAxiosError(error) && error.response?.status !== 404) { + if (isAxiosError(error) && error.status !== 409) { const interestingPartsOfError = { ...omit(error, 'request', 'response', 'config'), ...pick( diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts index 686d11e0b5728..a422e50b57a12 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts @@ -31,26 +31,18 @@ describe('alert function', () => { ruleIds.push(responseApmRule.data.id); logger.info('Creating dataview'); - // TODO: consider spaceId when checking for existingDataView - const dataViewId = customThresholdAIAssistantLogCount.dataViewParams.options.id; - const existingDataView = await kibanaClient.callKibana( - 'post', - { pathname: '/api/content_management/rpc/get' }, - { - contentTypeId: customThresholdAIAssistantLogCount.dataViewParams.contentTypeId, - id: dataViewId, - version: 1, - } - ); - - if (!existingDataView) { + try { await kibanaClient.callKibana( 'post', { pathname: '/api/content_management/rpc/create' }, customThresholdAIAssistantLogCount.dataViewParams ); - } else { - logger.info('Data view already exists, skipping creation'); + } catch (error) { + if (error?.status === 409) { + logger.info('Data view already exists, skipping creation'); + } else { + throw error; + } } logger.info('Creating logs rule'); From 2e1b6e8b830265ce24a509b9a20022834c8bc6e4 Mon Sep 17 00:00:00 2001 From: Viduni Wickramarachchi Date: Tue, 17 Dec 2024 09:41:34 -0500 Subject: [PATCH 11/26] [Obs AI Assistant] Change internal origin to Kibana --- .../scripts/evaluation/kibana_client.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts index 531858de87092..c4e9f898f5dca 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts @@ -74,7 +74,6 @@ type CompleteFunction = ( export interface ChatClient { chat: (message: StringOrMessageList) => Promise; complete: CompleteFunction; - evaluate: ( {}: { conversationId?: string; messages: InnerMessage[]; errors: ChatCompletionErrorEvent[] }, criteria: string[] @@ -127,7 +126,7 @@ export class KibanaClient { ...(method.toLowerCase() !== 'delete' ? { data: data || {} } : {}), headers: { 'kbn-xsrf': 'true', - 'x-elastic-internal-origin': 'foo', + 'x-elastic-internal-origin': 'Kibana', }, }).catch((error) => { if (isAxiosError(error) && error.status !== 409) { @@ -378,7 +377,11 @@ export class KibanaClient { pathname: '/internal/observability_ai_assistant/chat', }), params, - { responseType: 'stream', timeout: NaN, headers: { 'x-elastic-internal-origin': true } } + { + responseType: 'stream', + timeout: NaN, + headers: { 'x-elastic-internal-origin': 'Kibana' }, + } ); }).pipe( switchMap((response) => streamIntoObservable(response.data)), @@ -465,7 +468,7 @@ export class KibanaClient { { responseType: 'stream', timeout: NaN, - headers: { 'x-elastic-internal-origin': true }, + headers: { 'x-elastic-internal-origin': 'Kibana' }, } ) ); From 2a10441ea4613b926ebf3d5ef95deec15ef2a8f3 Mon Sep 17 00:00:00 2001 From: Viduni Wickramarachchi Date: Tue, 17 Dec 2024 10:44:12 -0500 Subject: [PATCH 12/26] [Obs AI Assistant] Improve scopes handling in the chat client --- .../scripts/evaluation/evaluation.ts | 2 +- .../scripts/evaluation/kibana_client.ts | 41 +++++++++++-------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/evaluation.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/evaluation.ts index b7545f8636b9d..9b20402952583 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/evaluation.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/evaluation.ts @@ -102,7 +102,7 @@ function runEvaluations() { evaluationConnectorId: evaluationConnector.id!, persist: argv.persist, suite: mocha.suite, - scopes: ['all'], + scopes: ['observability'], }); const header: string[][] = [ diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts index c4e9f898f5dca..91635333ac9fe 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts @@ -80,6 +80,7 @@ export interface ChatClient { ) => Promise; getResults: () => EvaluationResult[]; onResult: (cb: (result: EvaluationResult) => void) => () => void; + setScopes: (scope: AssistantScope[]) => void; } export class KibanaClient { @@ -230,6 +231,18 @@ export class KibanaClient { } } + getMessages(message: string | Array): Array { + if (typeof message === 'string') { + return [ + { + content: message, + role: MessageRole.User, + }, + ]; + } + return message; + } + createChatClient({ connectorId, evaluationConnectorId, @@ -243,22 +256,11 @@ export class KibanaClient { suite?: Mocha.Suite; scopes: AssistantScope[]; }): ChatClient { - function getMessages(message: string | Array): Array { - if (typeof message === 'string') { - return [ - { - content: message, - role: MessageRole.User, - }, - ]; - } - return message; - } - const that = this; let currentTitle: string = ''; let firstSuiteName: string = ''; + let currentScopes = scopes; if (suite) { suite.beforeEach(function () { @@ -361,7 +363,7 @@ export class KibanaClient { that.log.info('Chat', name); const chat$ = defer(() => { - that.log.debug(`Calling chat API`); + that.log.info('Calling the chat API...'); const params: ObservabilityAIAssistantAPIClientRequestParamsOf<'POST /internal/observability_ai_assistant/chat'>['params']['body'] = { name, @@ -369,7 +371,7 @@ export class KibanaClient { connectorId: connectorIdOverride || connectorId, functions: functions.map((fn) => pick(fn, 'name', 'description', 'parameters')), functionCall, - scopes, + scopes: currentScopes, }; return that.axios.post( @@ -403,7 +405,7 @@ export class KibanaClient { return { chat: async (message) => { const messages = [ - ...getMessages(message).map((msg) => ({ + ...this.getMessages(message).map((msg) => ({ message: msg, '@timestamp': new Date().toISOString(), })), @@ -411,7 +413,7 @@ export class KibanaClient { return chat('chat', { messages, functions: [] }); }, complete: async (...args) => { - that.log.info(`Complete`); + that.log.debug('Calling the chat complete API...'); let messagesArg: StringOrMessageList | undefined; let conversationId: string | undefined; let options: Options = {}; @@ -443,7 +445,7 @@ export class KibanaClient { } const messages = [ - ...getMessages(messagesArg!).map((msg) => ({ + ...this.getMessages(messagesArg!).map((msg) => ({ message: msg, '@timestamp': new Date().toISOString(), })), @@ -463,7 +465,7 @@ export class KibanaClient { connectorId, persist, title: currentTitle, - scopes, + scopes: currentScopes, }, { responseType: 'stream', @@ -654,6 +656,9 @@ export class KibanaClient { onResultCallbacks.push({ callback, unregister }); return unregister; }, + setScopes: (newScope: AssistantScope[]) => { + currentScopes = newScope; + }, }; } From b837d896d1edc1aac92ce326cf3caae552a315b6 Mon Sep 17 00:00:00 2001 From: Viduni Wickramarachchi Date: Tue, 17 Dec 2024 17:40:05 -0500 Subject: [PATCH 13/26] [Obs AI Assistant] Update elasticsearch and es|ql scope before/after tests --- .../scripts/evaluation/scenarios/apm/index.spec.ts | 1 + .../scenarios/elasticsearch/index.spec.ts | 11 +++++++++++ .../evaluation/scenarios/esql/index.spec.ts | 14 ++++++++++++-- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/apm/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/apm/index.spec.ts index 0612d8cffad66..4e65814dca5cc 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/apm/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/apm/index.spec.ts @@ -17,6 +17,7 @@ import { apmErrorCountAIAssistant } from '../../alert_templates/templates'; describe('apm', () => { const ruleIds: any[] = []; + before(async () => { const responseApmRule = await kibanaClient.callKibana( 'post', diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/elasticsearch/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/elasticsearch/index.spec.ts index 52c69254d1a5e..63c13ae0f4f04 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/elasticsearch/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/elasticsearch/index.spec.ts @@ -12,6 +12,12 @@ import { MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; import { chatClient, esClient } from '../../services'; describe('elasticsearch functions', () => { + // using 'all' for elasticsearch scenarios enables the LLM correctly pick + // elasticsearch functions when querying for data + before(() => { + chatClient.setScopes(['all']); + }); + describe('health', () => { it('returns the cluster health state', async () => { const conversation = await chatClient.complete( @@ -129,6 +135,7 @@ describe('elasticsearch functions', () => { }); }); }); + describe('other', () => { it('returns clusters license', async () => { const conversation = await chatClient.complete('What is my clusters license?'); @@ -141,4 +148,8 @@ describe('elasticsearch functions', () => { expect(result.passed).to.be(true); }); }); + + after(() => { + chatClient.setScopes(['observability']); + }); }); diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/esql/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/esql/index.spec.ts index 15e9b7d6a0507..6f550d4ffcbe7 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/esql/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/esql/index.spec.ts @@ -33,8 +33,8 @@ async function evaluateEsqlQuery({ ] : []), ...(execute - ? [`The query successfully executed without an error`] - : [`The query was not executed, it was only explained`]), + ? ['The query successfully executed without an error'] + : ['The query was not executed, it was only explained']), ...criteria, ]); @@ -44,6 +44,10 @@ async function evaluateEsqlQuery({ } describe('ES|QL query generation', () => { + before(() => { + chatClient.setScopes(['all']); + }); + describe('other queries', () => { describe('with packetbeat data', () => { before(async () => { @@ -348,6 +352,7 @@ describe('ES|QL query generation', () => { execute: false, }); }); + it('prod_web length', async () => { await evaluateEsqlQuery({ question: `can you convert this SPL query to ESQL? index=prod_web | eval length=len(message) | eval k255=if((length>255),1,0) | eval k2=if((length>2048),1,0) | eval k4=if((length>4096),1,0) |eval k16=if((length>16384),1,0) | stats count, sum(k255), sum(k2),sum(k4),sum(k16), sum(length)`, @@ -360,6 +365,7 @@ describe('ES|QL query generation', () => { execute: false, }); }); + it('prod_web filter message and host', async () => { await evaluateEsqlQuery({ question: `can you convert this SPL query to ESQL? index=prod_web NOT "Connection reset" NOT "[acm-app] created a ThreadLocal" sourcetype!=prod_urlf_east_logs sourcetype!=prod_urlf_west_logs host!="dbs-tools-*" NOT "Public] in context with path [/global] " host!="*dev*" host!="*qa*" host!="*uat*"`, @@ -377,4 +383,8 @@ describe('ES|QL query generation', () => { }); }); }); + + after(() => { + chatClient.setScopes(['observability']); + }); }); From 9f8dc783b433804ed9f50cb184df5d6fb849ddb4 Mon Sep 17 00:00:00 2001 From: Viduni Wickramarachchi Date: Wed, 18 Dec 2024 08:27:59 -0500 Subject: [PATCH 14/26] [Obs AI Assistant] Fix eslint issues --- .../evaluation/scenarios/kb/index.spec.ts | 60 ++++++++++--------- .../tests/chat/chat.spec.ts | 2 +- .../tests/complete/complete.spec.ts | 2 +- .../tests/complete/functions/alerts.spec.ts | 2 +- .../complete/functions/elasticsearch.spec.ts | 2 +- .../complete/functions/summarize.spec.ts | 2 +- .../tests/connectors/connectors.spec.ts | 2 +- .../tests/conversations/conversations.spec.ts | 2 +- .../knowledge_base/knowledge_base.spec.ts | 2 +- .../knowledge_base_migration.spec.ts | 2 +- .../knowledge_base_setup.spec.ts | 2 +- .../knowledge_base_status.spec.ts | 2 +- .../knowledge_base_user_instructions.spec.ts | 2 +- .../public_complete/public_complete.spec.ts | 2 +- .../tests/contextual_insights/index.spec.ts | 2 +- .../tests/conversations/index.spec.ts | 2 +- .../settings_security.spec.ts | 2 +- .../knowledge_base_management/index.spec.ts | 2 +- 18 files changed, 48 insertions(+), 46 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts index e88ef8e037cb9..d0cc14bc9bca1 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts @@ -11,41 +11,43 @@ import expect from '@kbn/expect/expect'; import { MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; import { chatClient, esClient } from '../../services'; -describe('kb functions', () => { - it('summarizes and recalls information', async () => { - let conversation = await chatClient.complete( - 'Remember that this cluster is used to test the AI Assistant using the Observability AI Evaluation Framework' - ); +describe('knowledge base', () => { + describe('kb functions', () => { + it('summarizes and recalls information', async () => { + let conversation = await chatClient.complete( + 'Remember that this cluster is used to test the AI Assistant using the Observability AI Evaluation Framework' + ); - conversation = await chatClient.complete( - conversation.conversationId!, - conversation.messages.concat({ - content: 'What is this cluster used for?', - role: MessageRole.User, - }) - ); + conversation = await chatClient.complete( + conversation.conversationId!, + conversation.messages.concat({ + content: 'What is this cluster used for?', + role: MessageRole.User, + }) + ); - const result = await chatClient.evaluate(conversation, [ - 'Calls the summarize function', - 'Effectively summarizes and remembers that this cluster is used to test the AI Assistant using the Observability AI Evaluation Framework', - 'Calls the "context" function to respond to What is this cluster used for?', - 'Effectively recalls that this cluster is used to test the AI Assistant using the Observability AI Evaluation Framework', - ]); + const result = await chatClient.evaluate(conversation, [ + 'Calls the summarize function', + 'Effectively summarizes and remembers that this cluster is used to test the AI Assistant using the Observability AI Evaluation Framework', + 'Calls the "context" function to respond to What is this cluster used for?', + 'Effectively recalls that this cluster is used to test the AI Assistant using the Observability AI Evaluation Framework', + ]); - expect(result.passed).to.be(true); - }); + expect(result.passed).to.be(true); + }); - after(async () => { - await esClient.deleteByQuery({ - index: '.kibana-observability-ai-assistant-kb-*', - ignore_unavailable: true, - query: { - match: { - text: { - query: '*Observability AI Evaluation Framework*', + after(async () => { + await esClient.deleteByQuery({ + index: '.kibana-observability-ai-assistant-kb-*', + ignore_unavailable: true, + query: { + match: { + text: { + query: '*Observability AI Evaluation Framework*', + }, }, }, - }, + }); }); }); }); diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/chat/chat.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/chat/chat.spec.ts index 04139a574a216..d1e34e236ebe2 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/chat/chat.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/chat/chat.spec.ts @@ -5,7 +5,7 @@ * 2.0. */ -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import { MessageRole, type Message } from '@kbn/observability-ai-assistant-plugin/common'; import { PassThrough } from 'stream'; import { createLlmProxy, LlmProxy } from '../../common/create_llm_proxy'; diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts index ad4808ed8f03b..f1564327e0a26 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts @@ -8,7 +8,7 @@ import { Response } from 'supertest'; import { MessageRole, type Message } from '@kbn/observability-ai-assistant-plugin/common'; import { omit, pick } from 'lodash'; import { PassThrough } from 'stream'; -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import { ChatCompletionChunkEvent, ConversationCreateEvent, diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/alerts.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/alerts.spec.ts index 60c28aab6781b..2c3310cb1c15c 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/alerts.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/alerts.spec.ts @@ -6,7 +6,7 @@ */ import { MessageRole, MessageAddEvent } from '@kbn/observability-ai-assistant-plugin/common'; -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import { LlmProxy, createLlmProxy } from '../../../common/create_llm_proxy'; import { FtrProviderContext } from '../../../common/ftr_provider_context'; import { getMessageAddedEvents, invokeChatCompleteWithFunctionRequest } from './helpers'; diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts index 10db0e16cae77..a41e005f695ba 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts @@ -6,7 +6,7 @@ */ import { MessageAddEvent, MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import { apm, timerange } from '@kbn/apm-synthtrace-client'; import { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; import { ELASTICSEARCH_FUNCTION_NAME } from '@kbn/observability-ai-assistant-plugin/server/functions/elasticsearch'; diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts index ae5ebdc19d44f..95cf1a0a34dec 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts @@ -6,7 +6,7 @@ */ import { MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import { LlmProxy, createLlmProxy } from '../../../common/create_llm_proxy'; import { FtrProviderContext } from '../../../common/ftr_provider_context'; import { invokeChatCompleteWithFunctionRequest } from './helpers'; diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/connectors/connectors.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/connectors/connectors.spec.ts index 42e1f8751719e..50a5490c14838 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/connectors/connectors.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/connectors/connectors.spec.ts @@ -5,7 +5,7 @@ * 2.0. */ -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import type { Agent as SuperTestAgent } from 'supertest'; import { FtrProviderContext } from '../../common/ftr_provider_context'; import { createProxyActionConnector, deleteActionConnector } from '../../common/action_connectors'; diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/conversations/conversations.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/conversations/conversations.spec.ts index bb85e99b99500..e144fca4ffb35 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/conversations/conversations.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/conversations/conversations.spec.ts @@ -5,7 +5,7 @@ * 2.0. */ -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import { merge, omit } from 'lodash'; import { type ConversationCreateRequest, diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts index 9d80db3baeae6..550193aa6083e 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts @@ -5,7 +5,7 @@ * 2.0. */ -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import { type KnowledgeBaseEntry } from '@kbn/observability-ai-assistant-plugin/common'; import { FtrProviderContext } from '../../common/ftr_provider_context'; import { diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_migration.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_migration.spec.ts index 46638d8eebe19..6283c4b82dbb3 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_migration.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_migration.spec.ts @@ -6,7 +6,7 @@ */ import { orderBy } from 'lodash'; -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import { AI_ASSISTANT_KB_INFERENCE_ID } from '@kbn/observability-ai-assistant-plugin/server/service/inference_endpoint'; import { SearchResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { KnowledgeBaseEntry } from '@kbn/observability-ai-assistant-plugin/common'; diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_setup.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_setup.spec.ts index 0d7625bb63ed3..c360f495ea597 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_setup.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_setup.spec.ts @@ -5,7 +5,7 @@ * 2.0. */ -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import { FtrProviderContext } from '../../common/ftr_provider_context'; import { deleteKnowledgeBaseModel, diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_status.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_status.spec.ts index 3f66931ca0719..bf65ffe60dade 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_status.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_status.spec.ts @@ -5,7 +5,7 @@ * 2.0. */ -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import { FtrProviderContext } from '../../common/ftr_provider_context'; import { deleteKnowledgeBaseModel, diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts index d5022a052d781..d35aa054b5832 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts @@ -5,7 +5,7 @@ * 2.0. */ -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import { sortBy } from 'lodash'; import { Message, MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; import { CONTEXT_FUNCTION_NAME } from '@kbn/observability-ai-assistant-plugin/server/functions/context'; diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/public_complete/public_complete.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/public_complete/public_complete.spec.ts index a46266f1b4d06..6ed0b486aafcc 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/public_complete/public_complete.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/public_complete/public_complete.spec.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import { FunctionDefinition, MessageRole, diff --git a/x-pack/test/observability_ai_assistant_functional/tests/contextual_insights/index.spec.ts b/x-pack/test/observability_ai_assistant_functional/tests/contextual_insights/index.spec.ts index b49c9fca76cd3..766c0ad4e9e1a 100644 --- a/x-pack/test/observability_ai_assistant_functional/tests/contextual_insights/index.spec.ts +++ b/x-pack/test/observability_ai_assistant_functional/tests/contextual_insights/index.spec.ts @@ -6,7 +6,7 @@ */ import { apm, timerange } from '@kbn/apm-synthtrace-client'; -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import moment from 'moment'; import { createLlmProxy, diff --git a/x-pack/test/observability_ai_assistant_functional/tests/conversations/index.spec.ts b/x-pack/test/observability_ai_assistant_functional/tests/conversations/index.spec.ts index d3208e5f1ff56..4702c48baffd9 100644 --- a/x-pack/test/observability_ai_assistant_functional/tests/conversations/index.spec.ts +++ b/x-pack/test/observability_ai_assistant_functional/tests/conversations/index.spec.ts @@ -5,7 +5,7 @@ * 2.0. */ -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import { MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; import { ChatFeedback } from '@kbn/observability-ai-assistant-plugin/public/analytics/schemas/chat_feedback'; import { pick } from 'lodash'; diff --git a/x-pack/test/observability_ai_assistant_functional/tests/feature_controls/settings_security.spec.ts b/x-pack/test/observability_ai_assistant_functional/tests/feature_controls/settings_security.spec.ts index d466abfd552ea..e60f0eb49218d 100644 --- a/x-pack/test/observability_ai_assistant_functional/tests/feature_controls/settings_security.spec.ts +++ b/x-pack/test/observability_ai_assistant_functional/tests/feature_controls/settings_security.spec.ts @@ -5,7 +5,7 @@ * 2.0. */ -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import { FtrProviderContext } from '../../ftr_provider_context'; import { createAndLoginUserWithCustomRole, deleteAndLogoutUser } from './helpers'; diff --git a/x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts b/x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts index 300fec6aa45aa..2660f7a0507f3 100644 --- a/x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts +++ b/x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts @@ -5,7 +5,7 @@ * 2.0. */ -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import { subj as testSubjSelector } from '@kbn/test-subj-selector'; import { TINY_ELSER, From f6a7e21d3d9e67e35ea3871a9d94b90dd39f596e Mon Sep 17 00:00:00 2001 From: Viduni Wickramarachchi Date: Wed, 18 Dec 2024 08:28:28 -0500 Subject: [PATCH 15/26] [Obs AI Assistant] Fix eslint issues --- .../tests/feature_controls/assistant_security.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/observability_ai_assistant_functional/tests/feature_controls/assistant_security.spec.ts b/x-pack/test/observability_ai_assistant_functional/tests/feature_controls/assistant_security.spec.ts index 1108c7251b89d..0e2acf8af784e 100644 --- a/x-pack/test/observability_ai_assistant_functional/tests/feature_controls/assistant_security.spec.ts +++ b/x-pack/test/observability_ai_assistant_functional/tests/feature_controls/assistant_security.spec.ts @@ -5,7 +5,7 @@ * 2.0. */ -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import { FtrProviderContext } from '../../ftr_provider_context'; import { createLlmProxy, From 92c98b44b18e9431d802beabf73b22b5fd9c876c Mon Sep 17 00:00:00 2001 From: Viduni Wickramarachchi Date: Wed, 18 Dec 2024 09:34:33 -0500 Subject: [PATCH 16/26] [Obs AI Assistant] Add new scenario/test for KB retrieval --- .../scripts/evaluation/kibana_client.ts | 14 +-- .../evaluation/scenarios/kb/index.spec.ts | 86 ++++++++++++++++++- 2 files changed, 91 insertions(+), 9 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts index 91635333ac9fe..ab8e6e0406766 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts @@ -148,7 +148,7 @@ export class KibanaClient { } async installKnowledgeBase() { - this.log.debug('Checking to see whether knowledge base is installed'); + this.log.info('Checking whether the knowledge base is installed'); const { data: { ready }, @@ -157,7 +157,7 @@ export class KibanaClient { }); if (ready) { - this.log.info('Knowledge base is installed'); + this.log.info('Knowledge base is already installed'); return; } @@ -184,7 +184,7 @@ export class KibanaClient { return; } - this.log.debug(`Checking if space ${this.spaceId} exists`); + this.log.info(`Checking if space ${this.spaceId} exists`); const spaceExistsResponse = await this.callKibana<{ id?: string; @@ -204,7 +204,7 @@ export class KibanaClient { }); if (spaceExistsResponse.data.id) { - this.log.debug(`Space id ${this.spaceId} found`); + this.log.info(`Space id ${this.spaceId} found`); return; } @@ -363,7 +363,7 @@ export class KibanaClient { that.log.info('Chat', name); const chat$ = defer(() => { - that.log.info('Calling the chat API...'); + that.log.info('Calling the /chat API'); const params: ObservabilityAIAssistantAPIClientRequestParamsOf<'POST /internal/observability_ai_assistant/chat'>['params']['body'] = { name, @@ -413,7 +413,7 @@ export class KibanaClient { return chat('chat', { messages, functions: [] }); }, complete: async (...args) => { - that.log.debug('Calling the chat complete API...'); + that.log.info('Calling complete'); let messagesArg: StringOrMessageList | undefined; let conversationId: string | undefined; let options: Options = {}; @@ -452,7 +452,7 @@ export class KibanaClient { ]; const stream$ = defer(() => { - that.log.debug(`Calling /chat/complete API`); + that.log.info(`Calling /chat/complete API`); return from( that.axios.post( that.getUrl({ diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts index d0cc14bc9bca1..ae749a8efcd14 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts @@ -9,7 +9,9 @@ import expect from '@kbn/expect/expect'; import { MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; -import { chatClient, esClient } from '../../services'; +import { chatClient, esClient, kibanaClient } from '../../services'; + +const KB_INDEX = '.kibana-observability-ai-assistant-kb-*'; describe('knowledge base', () => { describe('kb functions', () => { @@ -38,7 +40,7 @@ describe('knowledge base', () => { after(async () => { await esClient.deleteByQuery({ - index: '.kibana-observability-ai-assistant-kb-*', + index: KB_INDEX, ignore_unavailable: true, query: { match: { @@ -50,4 +52,84 @@ describe('knowledge base', () => { }); }); }); + + describe('kb retrieval', () => { + const testDocs = [ + { + id: 'doc_invention_1', + title: 'Quantum Revectorization Engine', + text: 'The Quantum Revectorization Engine (QRE), invented by Dr. Eliana Stone at Acme Labs in 2023, uses advanced quantum fields to reorder the subatomic structure of materials, effectively reconfiguring matter at a fundamental level. Its main achievement was to transform ordinary silicon wafers into superconductive materials without traditional cooling methods.', + }, + { + id: 'doc_invention_2', + title: 'Constraints of QRE', + text: 'Current constraints on the Quantum Revectorization Engine technology limit its revectorization radius to approximately 2 nanometers. Additionally, the energy required to maintain the quantum fields is extraordinarily high, necessitating specialized fusion reactors to sustain the process.', + }, + ]; + + before(async () => { + await kibanaClient.installKnowledgeBase(); + try { + await esClient.deleteByQuery({ + index: KB_INDEX, + ignore_unavailable: true, + query: { match_all: {} }, + refresh: true, + }); + } catch (error) { + // ignore error + } + + // Insert the test documents into KB + await kibanaClient.callKibana( + 'post', + { pathname: '/internal/observability_ai_assistant/kb/entries/import' }, + { + entries: testDocs, + } + ); + }); + + it('retrieves inventor and purpose of the QRE', async () => { + const prompt = 'Who invented the Quantum Revectorization Engine and what does it do?'; + const conversation = await chatClient.complete(prompt); + + const result = await chatClient.evaluate(conversation, [ + 'Uses KB retrieval function to find information about the Quantum Revectorization Engine', + 'Correctly identifies Dr. Eliana Stone at Acme Labs in 2023 as the inventor', + 'Accurately describes that it reorders the subatomic structure of materials and can transform silicon wafers into superconductive materials', + 'Does not invent unrelated or hallucinated details not present in the KB', + ]); + + expect(result.passed).to.be(true); + }); + + it('retrieves constraints and energy requirements of the QRE', async () => { + const prompt = + 'What is the approximate revectorization radius of the QRE and what kind of reactor is required to power it?'; + const conversation = await chatClient.complete(prompt); + + const result = await chatClient.evaluate(conversation, [ + 'Uses KB retrieval function to find the correct document about QRE constraints', + 'Mentions the 2 nanometer limit on the revectorization radius', + 'Mentions that specialized fusion reactors are needed', + 'Does not mention information unrelated to constraints or energy (i.e., does not mention the inventor or silicon wafer transformation from doc-invention-1)', + ]); + + expect(result.passed).to.be(true); + }); + + after(async () => { + await esClient.deleteByQuery({ + index: KB_INDEX, + ignore_unavailable: true, + query: { + match: { + text: 'Quantum Revectorization Engine', + }, + }, + refresh: true, + }); + }); + }); }); From dfc502630661b55bd857ad9f7d564978089e69b9 Mon Sep 17 00:00:00 2001 From: Viduni Wickramarachchi Date: Wed, 18 Dec 2024 13:55:01 -0500 Subject: [PATCH 17/26] [Obs AI Assistant] Add new scenario for documentation and improve logging --- .../scripts/evaluation/get_service_urls.ts | 2 +- .../scripts/evaluation/kibana_client.ts | 10 +- .../evaluation/scenarios/alerts/index.spec.ts | 2 +- .../evaluation/scenarios/apm/index.spec.ts | 2 +- .../scenarios/documentation/index.spec.ts | 109 ++++++++++++++++++ .../scenarios/elasticsearch/index.spec.ts | 2 +- .../evaluation/scenarios/esql/index.spec.ts | 1 + .../evaluation/scenarios/kb/index.spec.ts | 2 +- 8 files changed, 120 insertions(+), 10 deletions(-) create mode 100644 x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/documentation/index.spec.ts diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/get_service_urls.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/get_service_urls.ts index 4e0a089f2c295..416bb8636fe67 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/get_service_urls.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/get_service_urls.ts @@ -23,7 +23,7 @@ async function discoverAuth(parsedTarget: Url, log: ToolingLog) { const response = await fetch(url); status = response.status; } catch (err) { - log.debug(`${url} resulted in ${err.message}`); + log.error(`${url} resulted in ${err.message}`); status = 0; } diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts index ab8e6e0406766..dcd303dd6d57c 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts @@ -157,7 +157,7 @@ export class KibanaClient { }); if (ready) { - this.log.info('Knowledge base is already installed'); + this.log.success('Knowledge base is already installed'); return; } @@ -176,7 +176,7 @@ export class KibanaClient { { retries: 10 } ); - this.log.info('Knowledge base installed'); + this.log.success('Knowledge base installed'); } async createSpaceIfNeeded() { @@ -204,7 +204,7 @@ export class KibanaClient { }); if (spaceExistsResponse.data.id) { - this.log.info(`Space id ${this.spaceId} found`); + this.log.success(`Space id ${this.spaceId} found`); return; } @@ -223,7 +223,7 @@ export class KibanaClient { ); if (spaceCreatedResponse.status === 200) { - this.log.info(`Created space ${this.spaceId}`); + this.log.success(`Created space ${this.spaceId}`); } else { throw new Error( `Error creating space: ${spaceCreatedResponse.status} - ${spaceCreatedResponse.data}` @@ -624,7 +624,7 @@ export class KibanaClient { }) .concat({ score: errors.length === 0 ? 1 : 0, - criterion: 'The conversation encountered errors', + criterion: 'The conversation did not encounter any errors', reasoning: errors.length ? `The following errors occurred: ${errors.map((error) => error.error.message)}` : 'No errors occurred', diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts index a422e50b57a12..c235b830fcdd8 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts @@ -18,7 +18,7 @@ import { customThresholdAIAssistantLogCount, } from '../../alert_templates/templates'; -describe('alert function', () => { +describe('Alert function', () => { const ruleIds: any[] = []; before(async () => { diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/apm/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/apm/index.spec.ts index 4e65814dca5cc..cd8b22587e97b 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/apm/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/apm/index.spec.ts @@ -15,7 +15,7 @@ import { MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; import { chatClient, kibanaClient, synthtraceEsClients } from '../../services'; import { apmErrorCountAIAssistant } from '../../alert_templates/templates'; -describe('apm', () => { +describe('APM', () => { const ruleIds: any[] = []; before(async () => { diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/documentation/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/documentation/index.spec.ts new file mode 100644 index 0000000000000..d58f36ad82691 --- /dev/null +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/documentation/index.spec.ts @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/// + +import expect from '@kbn/expect/expect'; +import { + InstallationStatusResponse, + PerformInstallResponse, + UninstallResponse, +} from '@kbn/product-doc-base-plugin/common/http_api/installation'; +import { RETRIEVE_DOCUMENTATION_NAME } from '../../../../server/functions/documentation'; +import { chatClient, kibanaClient, logger } from '../../services'; + +const ELASTIC_DOCS_INSTALLATION_STATUS_API_PATH = '/internal/product_doc_base/status'; +const ELASTIC_DOCS_INSTALL_ALL_API_PATH = '/internal/product_doc_base/install'; +const ELASTIC_DOCS_UNINSTALL_ALL_API_PATH = '/internal/product_doc_base/uninstall'; + +describe('Retrieve documentation function', () => { + before(async () => { + let statusResponse = await kibanaClient.callKibana('get', { + pathname: ELASTIC_DOCS_INSTALLATION_STATUS_API_PATH, + }); + + if (statusResponse.data.overall === 'installed') { + logger.success('Elastic documentation is already installed'); + } else { + logger.info('Installing Elastic documentation'); + const installResponse = await kibanaClient.callKibana('post', { + pathname: ELASTIC_DOCS_INSTALL_ALL_API_PATH, + }); + + if (!installResponse.data.installed) { + logger.error('Could not install Elastic documentation'); + throw new Error('Documentation did not install successfully before running tests.'); + } + + statusResponse = await kibanaClient.callKibana('get', { + pathname: ELASTIC_DOCS_INSTALLATION_STATUS_API_PATH, + }); + + if (statusResponse.data.overall !== 'installed') { + throw new Error('Documentation is not fully installed, cannot proceed with tests.'); + } else { + logger.success('Installed Elastic documentation'); + } + } + }); + + it('retrieves Elasticsearch documentation', async () => { + const prompt = 'How can I configure HTTPS in Elasticsearch?'; + const conversation = await chatClient.complete(prompt); + + const result = await chatClient.evaluate(conversation, [ + `Uses the ${RETRIEVE_DOCUMENTATION_NAME} function before answering the question about the Elastic stack`, + 'The assistant provides guidance on configuring HTTPS for Elasticsearch based on the retrieved documentation', + 'Does not hallucinate steps without first calling the retrieve_elastic_doc function', + 'Mentions Elasticsearch and HTTPS configuration steps consistent with the documentation', + ]); + + expect(result.passed).to.be(true); + }); + + it('retrieves Kibana documentation', async () => { + const prompt = 'What is Kibana Lens and how do I create a bar chart visualization with it?'; + const conversation = await chatClient.complete(prompt); + + const result = await chatClient.evaluate(conversation, [ + `Uses the ${RETRIEVE_DOCUMENTATION_NAME} function before answering the question about Kibana`, + 'Accurately explains what Kibana Lens is and provides doc-based steps for creating a bar chart visualization', + `Does not invent unsupported instructions, answers should reference what's found in the Kibana docs`, + ]); + + expect(result.passed).to.be(true); + }); + + it('retrieves Observability documentation', async () => { + const prompt = + 'How can I set up APM instrumentation for my Node.js service in Elastic Observability?'; + const conversation = await chatClient.complete(prompt); + + const result = await chatClient.evaluate(conversation, [ + `Uses the ${RETRIEVE_DOCUMENTATION_NAME} function before answering the question about Observability`, + 'Provides instructions based on the Observability docs for setting up APM instrumentation in a Node.js service', + 'Mentions steps like installing the APM agent, configuring it with the service name and APM Server URL, etc., as per the docs', + 'Does not provide hallucinated steps, should align with actual Observability documentation', + ]); + + expect(result.passed).to.be(true); + }); + + after(async () => { + // Uninstall all installed documentation + logger.info('Uninstalling Elastic documentation'); + const uninstallResponse = await kibanaClient.callKibana('post', { + pathname: ELASTIC_DOCS_UNINSTALL_ALL_API_PATH, + }); + + if (uninstallResponse.data.success) { + logger.success('Uninstalled Elastic documentation'); + } else { + logger.error('Could not uninstall Elastic documentation'); + } + }); +}); diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/elasticsearch/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/elasticsearch/index.spec.ts index 63c13ae0f4f04..e49ba46c9e734 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/elasticsearch/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/elasticsearch/index.spec.ts @@ -11,7 +11,7 @@ import expect from '@kbn/expect/expect'; import { MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; import { chatClient, esClient } from '../../services'; -describe('elasticsearch functions', () => { +describe('Elasticsearch functions', () => { // using 'all' for elasticsearch scenarios enables the LLM correctly pick // elasticsearch functions when querying for data before(() => { diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/esql/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/esql/index.spec.ts index 6f550d4ffcbe7..168714e4e355b 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/esql/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/esql/index.spec.ts @@ -77,6 +77,7 @@ describe('ES|QL query generation', () => { }, }, }); + await esClient.index({ index: 'packetbeat-8.11.3', document: { diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts index ae749a8efcd14..6cba8f95eae24 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts @@ -13,7 +13,7 @@ import { chatClient, esClient, kibanaClient } from '../../services'; const KB_INDEX = '.kibana-observability-ai-assistant-kb-*'; -describe('knowledge base', () => { +describe('Knowledge base', () => { describe('kb functions', () => { it('summarizes and recalls information', async () => { let conversation = await chatClient.complete( From da19ba3d52574f5079ba8c554dc3392240bb1b71 Mon Sep 17 00:00:00 2001 From: Viduni Wickramarachchi Date: Fri, 20 Dec 2024 11:39:11 -0500 Subject: [PATCH 18/26] [Obs AI Assistant] Improve readme --- .../scripts/evaluation/README.md | 52 ++++++++++++------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/README.md b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/README.md index 5bb17f9972407..506772a3746fd 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/README.md +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/README.md @@ -2,7 +2,7 @@ ## Overview -This tool is developed for our team working on the Elastic Observability platform, specifically focusing on evaluating the Observability AI Assistant. It simplifies scripting and evaluating various scenarios with the Large Language Model (LLM) integration. +This tool is developed for our team working on the Elastic Observability platform, specifically focusing on evaluating the Observability AI Assistant. It simplifies scripting and evaluating various scenarios with Large Language Model (LLM) integrations. ## Setup requirements @@ -12,26 +12,40 @@ This tool is developed for our team working on the Elastic Observability platfor ## Running evaluations -Run the tool using: - -`$ node x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/index.js` - -This will evaluate all existing scenarios, and write the evaluation results to the terminal. - ### Configuration -#### Kibana and Elasticsearch - -By default, the tool will look for a Kibana instance running locally (at `http://localhost:5601`, which is the default address for running Kibana in development mode). It will also attempt to read the Kibana config file for the Elasticsearch address & credentials. If you want to override these settings, use `--kibana` and `--es`. Only basic auth is supported, e.g. `--kibana http://username:password@localhost:5601`. If you want to use a specific space, use `--spaceId` +#### To run the evaluation using a local Elasticsearch and Kibana instance: -#### Connector - -Use `--connectorId` to specify a `.gen-ai` or `.bedrock` connector to use. If none are given, it will prompt you to select a connector based on the ones that are available. If only a single supported connector is found, it will be used without prompting. - -#### Persisting conversations - -By default, completed conversations are not persisted. If you do want to persist them, for instance for reviewing purposes, set the `--persist` flag to store them. This will also generate a clickable link in the output of the evaluation that takes you to the conversation. +- Run Elasticsearch locally: `yarn es snapshot --license trial` +- Start Kibana (Default address for Kibana in dev mode: `http://localhost:5601`) +- Run this command to start evaluating: +`$ node x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/index.js` -If you want to clear conversations on startup, use the `--clear` flag. This only works when `--persist` is enabled. If `--spaceId` is set, only conversations for the current space will be cleared. +This will evaluate all existing scenarios, and write the evaluation results to the terminal. -When storing conversations, the name of the scenario is used as a title. Set the `--autoTitle` flag to have the LLM generate a title for you. +#### To run the evaluation using a hosted deployment: +- Add the credentials of Elasticsearch to `kibana.dev.yml` as follows: +``` +elasticsearch.hosts: https://: +elasticsearch.username: +elasticsearch.password: +elasticsearch.ssl.verificationMode: none +elasticsearch.ignoreVersionMismatch: true +``` +- Start Kibana +- Run this command to start evaluating: `node x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/index.js --kibana http://:@localhost:5601` + +By default the script will use the Elasticsearch credentials specified in `kibana.dev.yml`, if you want to override it use the `--es` flag when running the evaluation script: +E.g.: `node x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/index.js --kibana http://:@localhost:5601 --es https://:@:` + +The `--kibana` and `--es` flags override the default credentials. Only basic auth is supported. + +## Other (optional) configuration flags +- `--connectorId` - Specify a `.gen-ai` or `.bedrock` connector to use. If none are given, it will prompt you to select a connector based on the ones that are available. If only a single supported connector is found, it will be used without prompting. +- `--evaluateWith`: The connector ID to evaluate with. Leave empty to use the same connector, use "other" to get a selection menu. +- `--spaceId` - Specify the space ID if you want to use a specific space. +- `--persist` - By default, completed conversations are not persisted. If you want to persist them, for instance for reviewing purposes, include this flag when running the evaluation script. This will also generate a clickable link in the output of the evaluation that takes you to the conversation in Kibana. +- `--clear` - If you want to clear conversations on startup, include this command when running the evaluation script. This only works when `--persist` is enabled. If `--spaceId` is set, only conversations for the current space will be cleared +- `--autoTitle`: When storing conversations, the name of the scenario is used as a title. Set this flag to have the LLM generate a title for you. This only works when `--persist` is enabled. +- `--files`: A file or list of files containing the scenarios to evaluate. Defaults to all. +- `--grep`: A string or regex to filter scenarios by. From 7c001582f418662c951f0199e60a6d3e50a26a4d Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 23 Dec 2024 14:00:03 +0000 Subject: [PATCH 19/26] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- .../plugins/observability_ai_assistant_app/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/tsconfig.json b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/tsconfig.json index 212a36a502441..763bf6659170e 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/tsconfig.json +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/tsconfig.json @@ -82,6 +82,7 @@ "@kbn/product-doc-common", "@kbn/charts-theme", "@kbn/ai-assistant-icon", + "@kbn/product-doc-base-plugin", ], "exclude": [ "target/**/*" From 2d97d5b9fc3b5967bda290663cff9bd5183a6cc0 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 23 Dec 2024 14:19:41 +0000 Subject: [PATCH 20/26] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../scripts/evaluation/scenarios/alerts/index.spec.ts | 2 +- .../scripts/evaluation/scenarios/apm/index.spec.ts | 2 +- .../scripts/evaluation/scenarios/documentation/index.spec.ts | 2 +- .../scripts/evaluation/scenarios/elasticsearch/index.spec.ts | 2 +- .../scripts/evaluation/scenarios/esql/index.spec.ts | 2 +- .../scripts/evaluation/scenarios/kb/index.spec.ts | 2 +- .../tests/chat/chat.spec.ts | 2 +- .../tests/complete/complete.spec.ts | 2 +- .../tests/complete/functions/alerts.spec.ts | 2 +- .../tests/complete/functions/elasticsearch.spec.ts | 2 +- .../tests/complete/functions/summarize.spec.ts | 2 +- .../tests/connectors/connectors.spec.ts | 2 +- .../tests/conversations/conversations.spec.ts | 2 +- .../tests/knowledge_base/knowledge_base.spec.ts | 2 +- .../tests/knowledge_base/knowledge_base_migration.spec.ts | 2 +- .../tests/knowledge_base/knowledge_base_setup.spec.ts | 2 +- .../tests/knowledge_base/knowledge_base_status.spec.ts | 2 +- .../knowledge_base/knowledge_base_user_instructions.spec.ts | 2 +- .../tests/public_complete/public_complete.spec.ts | 2 +- .../tests/contextual_insights/index.spec.ts | 2 +- .../tests/conversations/index.spec.ts | 2 +- .../tests/feature_controls/assistant_security.spec.ts | 2 +- .../tests/feature_controls/settings_security.spec.ts | 2 +- .../tests/knowledge_base_management/index.spec.ts | 2 +- 24 files changed, 24 insertions(+), 24 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts index c235b830fcdd8..e87d138b332fe 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts @@ -7,7 +7,7 @@ /// -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import { RuleResponse } from '@kbn/alerting-plugin/common/routes/rule/response/types/v1'; import moment from 'moment'; import { apm, timerange } from '@kbn/apm-synthtrace-client'; diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/apm/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/apm/index.spec.ts index cd8b22587e97b..cc16126cf75e1 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/apm/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/apm/index.spec.ts @@ -7,7 +7,7 @@ /// -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import moment from 'moment'; import { apm, timerange, serviceMap } from '@kbn/apm-synthtrace-client'; import { RuleResponse } from '@kbn/alerting-plugin/common/routes/rule/response/types/v1'; diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/documentation/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/documentation/index.spec.ts index d58f36ad82691..910b9eeab9bc0 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/documentation/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/documentation/index.spec.ts @@ -7,7 +7,7 @@ /// -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import { InstallationStatusResponse, PerformInstallResponse, diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/elasticsearch/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/elasticsearch/index.spec.ts index e49ba46c9e734..3133d361a258c 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/elasticsearch/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/elasticsearch/index.spec.ts @@ -7,7 +7,7 @@ /// -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import { MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; import { chatClient, esClient } from '../../services'; diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/esql/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/esql/index.spec.ts index 168714e4e355b..838fea6de0784 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/esql/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/esql/index.spec.ts @@ -8,7 +8,7 @@ /// import { apm, timerange } from '@kbn/apm-synthtrace-client'; -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import moment from 'moment'; import { chatClient, esClient, synthtraceEsClients } from '../../services'; diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts index 6cba8f95eae24..194f5db1fc917 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts @@ -7,7 +7,7 @@ /// -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import { MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; import { chatClient, esClient, kibanaClient } from '../../services'; diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/chat/chat.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/chat/chat.spec.ts index d1e34e236ebe2..04139a574a216 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/chat/chat.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/chat/chat.spec.ts @@ -5,7 +5,7 @@ * 2.0. */ -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import { MessageRole, type Message } from '@kbn/observability-ai-assistant-plugin/common'; import { PassThrough } from 'stream'; import { createLlmProxy, LlmProxy } from '../../common/create_llm_proxy'; diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts index f1564327e0a26..ad4808ed8f03b 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts @@ -8,7 +8,7 @@ import { Response } from 'supertest'; import { MessageRole, type Message } from '@kbn/observability-ai-assistant-plugin/common'; import { omit, pick } from 'lodash'; import { PassThrough } from 'stream'; -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import { ChatCompletionChunkEvent, ConversationCreateEvent, diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/alerts.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/alerts.spec.ts index 2c3310cb1c15c..60c28aab6781b 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/alerts.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/alerts.spec.ts @@ -6,7 +6,7 @@ */ import { MessageRole, MessageAddEvent } from '@kbn/observability-ai-assistant-plugin/common'; -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import { LlmProxy, createLlmProxy } from '../../../common/create_llm_proxy'; import { FtrProviderContext } from '../../../common/ftr_provider_context'; import { getMessageAddedEvents, invokeChatCompleteWithFunctionRequest } from './helpers'; diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts index a41e005f695ba..10db0e16cae77 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts @@ -6,7 +6,7 @@ */ import { MessageAddEvent, MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import { apm, timerange } from '@kbn/apm-synthtrace-client'; import { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; import { ELASTICSEARCH_FUNCTION_NAME } from '@kbn/observability-ai-assistant-plugin/server/functions/elasticsearch'; diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts index 95cf1a0a34dec..ae5ebdc19d44f 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts @@ -6,7 +6,7 @@ */ import { MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import { LlmProxy, createLlmProxy } from '../../../common/create_llm_proxy'; import { FtrProviderContext } from '../../../common/ftr_provider_context'; import { invokeChatCompleteWithFunctionRequest } from './helpers'; diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/connectors/connectors.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/connectors/connectors.spec.ts index 50a5490c14838..42e1f8751719e 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/connectors/connectors.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/connectors/connectors.spec.ts @@ -5,7 +5,7 @@ * 2.0. */ -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import type { Agent as SuperTestAgent } from 'supertest'; import { FtrProviderContext } from '../../common/ftr_provider_context'; import { createProxyActionConnector, deleteActionConnector } from '../../common/action_connectors'; diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/conversations/conversations.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/conversations/conversations.spec.ts index e144fca4ffb35..bb85e99b99500 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/conversations/conversations.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/conversations/conversations.spec.ts @@ -5,7 +5,7 @@ * 2.0. */ -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import { merge, omit } from 'lodash'; import { type ConversationCreateRequest, diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts index 550193aa6083e..9d80db3baeae6 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts @@ -5,7 +5,7 @@ * 2.0. */ -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import { type KnowledgeBaseEntry } from '@kbn/observability-ai-assistant-plugin/common'; import { FtrProviderContext } from '../../common/ftr_provider_context'; import { diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_migration.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_migration.spec.ts index 6283c4b82dbb3..46638d8eebe19 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_migration.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_migration.spec.ts @@ -6,7 +6,7 @@ */ import { orderBy } from 'lodash'; -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import { AI_ASSISTANT_KB_INFERENCE_ID } from '@kbn/observability-ai-assistant-plugin/server/service/inference_endpoint'; import { SearchResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { KnowledgeBaseEntry } from '@kbn/observability-ai-assistant-plugin/common'; diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_setup.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_setup.spec.ts index c360f495ea597..0d7625bb63ed3 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_setup.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_setup.spec.ts @@ -5,7 +5,7 @@ * 2.0. */ -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import { FtrProviderContext } from '../../common/ftr_provider_context'; import { deleteKnowledgeBaseModel, diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_status.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_status.spec.ts index bf65ffe60dade..3f66931ca0719 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_status.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_status.spec.ts @@ -5,7 +5,7 @@ * 2.0. */ -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import { FtrProviderContext } from '../../common/ftr_provider_context'; import { deleteKnowledgeBaseModel, diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts index d35aa054b5832..d5022a052d781 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts @@ -5,7 +5,7 @@ * 2.0. */ -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import { sortBy } from 'lodash'; import { Message, MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; import { CONTEXT_FUNCTION_NAME } from '@kbn/observability-ai-assistant-plugin/server/functions/context'; diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/public_complete/public_complete.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/public_complete/public_complete.spec.ts index 6ed0b486aafcc..a46266f1b4d06 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/public_complete/public_complete.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/public_complete/public_complete.spec.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import { FunctionDefinition, MessageRole, diff --git a/x-pack/test/observability_ai_assistant_functional/tests/contextual_insights/index.spec.ts b/x-pack/test/observability_ai_assistant_functional/tests/contextual_insights/index.spec.ts index 766c0ad4e9e1a..b49c9fca76cd3 100644 --- a/x-pack/test/observability_ai_assistant_functional/tests/contextual_insights/index.spec.ts +++ b/x-pack/test/observability_ai_assistant_functional/tests/contextual_insights/index.spec.ts @@ -6,7 +6,7 @@ */ import { apm, timerange } from '@kbn/apm-synthtrace-client'; -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import moment from 'moment'; import { createLlmProxy, diff --git a/x-pack/test/observability_ai_assistant_functional/tests/conversations/index.spec.ts b/x-pack/test/observability_ai_assistant_functional/tests/conversations/index.spec.ts index 4702c48baffd9..d3208e5f1ff56 100644 --- a/x-pack/test/observability_ai_assistant_functional/tests/conversations/index.spec.ts +++ b/x-pack/test/observability_ai_assistant_functional/tests/conversations/index.spec.ts @@ -5,7 +5,7 @@ * 2.0. */ -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import { MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; import { ChatFeedback } from '@kbn/observability-ai-assistant-plugin/public/analytics/schemas/chat_feedback'; import { pick } from 'lodash'; diff --git a/x-pack/test/observability_ai_assistant_functional/tests/feature_controls/assistant_security.spec.ts b/x-pack/test/observability_ai_assistant_functional/tests/feature_controls/assistant_security.spec.ts index 0e2acf8af784e..1108c7251b89d 100644 --- a/x-pack/test/observability_ai_assistant_functional/tests/feature_controls/assistant_security.spec.ts +++ b/x-pack/test/observability_ai_assistant_functional/tests/feature_controls/assistant_security.spec.ts @@ -5,7 +5,7 @@ * 2.0. */ -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import { FtrProviderContext } from '../../ftr_provider_context'; import { createLlmProxy, diff --git a/x-pack/test/observability_ai_assistant_functional/tests/feature_controls/settings_security.spec.ts b/x-pack/test/observability_ai_assistant_functional/tests/feature_controls/settings_security.spec.ts index e60f0eb49218d..d466abfd552ea 100644 --- a/x-pack/test/observability_ai_assistant_functional/tests/feature_controls/settings_security.spec.ts +++ b/x-pack/test/observability_ai_assistant_functional/tests/feature_controls/settings_security.spec.ts @@ -5,7 +5,7 @@ * 2.0. */ -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import { FtrProviderContext } from '../../ftr_provider_context'; import { createAndLoginUserWithCustomRole, deleteAndLogoutUser } from './helpers'; diff --git a/x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts b/x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts index 2660f7a0507f3..300fec6aa45aa 100644 --- a/x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts +++ b/x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts @@ -5,7 +5,7 @@ * 2.0. */ -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import { subj as testSubjSelector } from '@kbn/test-subj-selector'; import { TINY_ELSER, From 385f0203bf43efdc48cab608998618a3f78f1b56 Mon Sep 17 00:00:00 2001 From: Viduni Wickramarachchi Date: Tue, 24 Dec 2024 08:39:16 -0500 Subject: [PATCH 21/26] [Obs AI Assistant] Address PR comments --- .../observability_ai_assistant_app/scripts/evaluation/README.md | 2 +- .../scripts/evaluation/get_service_urls.ts | 2 +- .../scripts/evaluation/kibana_client.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/README.md b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/README.md index 506772a3746fd..5b16b7ba76abc 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/README.md +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/README.md @@ -41,7 +41,7 @@ E.g.: `node x-pack/solutions/observability/plugins/observability_ai_assistant_ap The `--kibana` and `--es` flags override the default credentials. Only basic auth is supported. ## Other (optional) configuration flags -- `--connectorId` - Specify a `.gen-ai` or `.bedrock` connector to use. If none are given, it will prompt you to select a connector based on the ones that are available. If only a single supported connector is found, it will be used without prompting. +- `--connectorId` - Specify a generative AI connector to use. If none are given, it will prompt you to select a connector based on the ones that are available. If only a single supported connector is found, it will be used without prompting. - `--evaluateWith`: The connector ID to evaluate with. Leave empty to use the same connector, use "other" to get a selection menu. - `--spaceId` - Specify the space ID if you want to use a specific space. - `--persist` - By default, completed conversations are not persisted. If you want to persist them, for instance for reviewing purposes, include this flag when running the evaluation script. This will also generate a clickable link in the output of the evaluation that takes you to the conversation in Kibana. diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/get_service_urls.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/get_service_urls.ts index 416bb8636fe67..4e0a089f2c295 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/get_service_urls.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/get_service_urls.ts @@ -23,7 +23,7 @@ async function discoverAuth(parsedTarget: Url, log: ToolingLog) { const response = await fetch(url); status = response.status; } catch (err) { - log.error(`${url} resulted in ${err.message}`); + log.debug(`${url} resulted in ${err.message}`); status = 0; } diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts index dcd303dd6d57c..0f39145bd91f4 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts @@ -130,7 +130,7 @@ export class KibanaClient { 'x-elastic-internal-origin': 'Kibana', }, }).catch((error) => { - if (isAxiosError(error) && error.status !== 409) { + if (isAxiosError(error)) { const interestingPartsOfError = { ...omit(error, 'request', 'response', 'config'), ...pick( From 620c5e246e6082cb363a56085007c73f71a996ef Mon Sep 17 00:00:00 2001 From: Viduni Wickramarachchi Date: Tue, 24 Dec 2024 08:54:03 -0500 Subject: [PATCH 22/26] [Obs AI Assistant] Revert auth change as it's not necessary --- .../scripts/evaluation/get_service_urls.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/get_service_urls.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/get_service_urls.ts index 4e0a089f2c295..859b47987a470 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/get_service_urls.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/get_service_urls.ts @@ -136,7 +136,7 @@ export async function getServiceUrls({ const kibanaUrlWithAuth = format({ ...parsedKibanaUrl, - auth: parsedKibanaUrl.auth || 'elastic:changeme', + auth: parsedKibanaUrl.auth || auth, }); const validatedKibanaUrl = await getKibanaUrl({ kibana: kibanaUrlWithAuth, log }); From 932ba2c418bfc2c60eaaa5c32f4dfd8b22361d95 Mon Sep 17 00:00:00 2001 From: Viduni Wickramarachchi Date: Tue, 24 Dec 2024 10:25:37 -0500 Subject: [PATCH 23/26] [Obs AI Assistant] Make scope a part of the complete function --- .../scripts/evaluation/kibana_client.ts | 59 +++++--------- .../evaluation/scenarios/alerts/index.spec.ts | 24 +++--- .../evaluation/scenarios/apm/index.spec.ts | 51 ++++++------ .../scenarios/documentation/index.spec.ts | 8 +- .../scenarios/elasticsearch/index.spec.ts | 77 ++++++++++--------- .../evaluation/scenarios/esql/index.spec.ts | 12 +-- .../evaluation/scenarios/kb/index.spec.ts | 23 +++--- 7 files changed, 116 insertions(+), 138 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts index 0f39145bd91f4..465c6db885c53 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts @@ -26,7 +26,7 @@ import { Message, MessageRole } from '@kbn/observability-ai-assistant-plugin/com import { streamIntoObservable } from '@kbn/observability-ai-assistant-plugin/server'; import { ToolingLog } from '@kbn/tooling-log'; import axios, { AxiosInstance, AxiosResponse, isAxiosError } from 'axios'; -import { isArray, omit, pick, remove } from 'lodash'; +import { omit, pick, remove } from 'lodash'; import pRetry from 'p-retry'; import { concatMap, @@ -59,13 +59,15 @@ interface Options { screenContexts?: ObservabilityAIAssistantScreenContext[]; } -type CompleteFunction = ( - ...args: - | [StringOrMessageList] - | [StringOrMessageList, Options] - | [string | undefined, StringOrMessageList] - | [string | undefined, StringOrMessageList, Options] -) => Promise<{ +interface CompleteFunctionParams { + messages: StringOrMessageList; + conversationId?: string; + options?: Options; + scope?: AssistantScope; +} + +// 2. Update the CompleteFunction type accordingly +type CompleteFunction = (params: CompleteFunctionParams) => Promise<{ conversationId?: string; messages: InnerMessage[]; errors: ChatCompletionErrorEvent[]; @@ -80,7 +82,6 @@ export interface ChatClient { ) => Promise; getResults: () => EvaluationResult[]; onResult: (cb: (result: EvaluationResult) => void) => () => void; - setScopes: (scope: AssistantScope[]) => void; } export class KibanaClient { @@ -412,37 +413,16 @@ export class KibanaClient { ]; return chat('chat', { messages, functions: [] }); }, - complete: async (...args) => { + complete: async ({ + messages: messagesArg, + conversationId, + options = {}, + scope: newScope, + }: CompleteFunctionParams) => { that.log.info('Calling complete'); - let messagesArg: StringOrMessageList | undefined; - let conversationId: string | undefined; - let options: Options = {}; - function isMessageList(arg: any): arg is StringOrMessageList { - return isArray(arg) || typeof arg === 'string'; - } - - // | [StringOrMessageList] - // | [StringOrMessageList, Options] - // | [string, StringOrMessageList] - // | [string, StringOrMessageList, Options] - if (args.length === 1) { - messagesArg = args[0]; - } else if (args.length === 2 && !isMessageList(args[1])) { - messagesArg = args[0]; - options = args[1]; - } else if ( - args.length === 2 && - (typeof args[0] === 'string' || typeof args[0] === 'undefined') && - isMessageList(args[1]) - ) { - conversationId = args[0]; - messagesArg = args[1]; - } else if (args.length === 3) { - conversationId = args[0]; - messagesArg = args[1]; - options = args[2]; - } + // set scope + currentScopes = [newScope || 'observability']; const messages = [ ...this.getMessages(messagesArg!).map((msg) => ({ @@ -656,9 +636,6 @@ export class KibanaClient { onResultCallbacks.push({ callback, unregister }); return unregister; }, - setScopes: (newScope: AssistantScope[]) => { - currentScopes = newScope; - }, }; } diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts index e87d138b332fe..6ee2e12585852 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts @@ -7,7 +7,7 @@ /// -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import { RuleResponse } from '@kbn/alerting-plugin/common/routes/rule/response/types/v1'; import moment from 'moment'; import { apm, timerange } from '@kbn/apm-synthtrace-client'; @@ -99,9 +99,9 @@ describe('Alert function', () => { }); it('summary of active alerts', async () => { - const conversation = await chatClient.complete( - 'Are there any active alerts over the last 4 hours?' - ); + const conversation = await chatClient.complete({ + messages: 'Are there any active alerts over the last 4 hours?', + }); const result = await chatClient.evaluate(conversation, [ 'Correctly uses the `alerts` function to fetch data for the current time range', @@ -113,17 +113,17 @@ describe('Alert function', () => { }); it('filtered alerts', async () => { - let conversation = await chatClient.complete( - 'Do I have any active threshold alerts related to the AI Assistant?' - ); + let conversation = await chatClient.complete({ + messages: 'Do I have any active threshold alerts related to the AI Assistant?', + }); - conversation = await chatClient.complete( - conversation.conversationId!, - conversation.messages.concat({ + conversation = await chatClient.complete({ + conversationId: conversation.conversationId!, + messages: conversation.messages.concat({ content: 'Do I have any alerts on the service my-service?', role: MessageRole.User, - }) - ); + }), + }); const result = await chatClient.evaluate(conversation, [ 'Uses the get_alerts_dataset_info function', diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/apm/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/apm/index.spec.ts index cc16126cf75e1..016e22ae006e6 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/apm/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/apm/index.spec.ts @@ -7,7 +7,7 @@ /// -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import moment from 'moment'; import { apm, timerange, serviceMap } from '@kbn/apm-synthtrace-client'; import { RuleResponse } from '@kbn/alerting-plugin/common/routes/rule/response/types/v1'; @@ -93,9 +93,10 @@ describe('APM', () => { }); it('service throughput', async () => { - const conversation = await chatClient.complete( - 'What is the average throughput per minute for the ai-assistant-service service over the past 4 hours?' - ); + const conversation = await chatClient.complete({ + messages: + 'What is the average throughput per minute for the ai-assistant-service service over the past 4 hours?', + }); const result = await chatClient.evaluate(conversation, [ 'Uses the get_apm_dataset_info function to get information about the APM data streams', @@ -110,9 +111,9 @@ describe('APM', () => { }); it('service dependencies', async () => { - const conversation = await chatClient.complete( - 'What are the downstream dependencies of the ai-assistant-service-front service?' - ); + const conversation = await chatClient.complete({ + messages: 'What are the downstream dependencies of the ai-assistant-service-front service?', + }); const result = await chatClient.evaluate(conversation, [ 'Uses the get_apm_downstream_dependencies function with the `service.name` parameter being "ai-assistant-service-front"', @@ -123,34 +124,34 @@ describe('APM', () => { }); it('services in environment', async () => { - let conversation = await chatClient.complete( - 'What are the active services in the environment "test"?' - ); + let conversation = await chatClient.complete({ + messages: 'What are the active services in the environment "test"?', + }); - conversation = await chatClient.complete( - conversation.conversationId!, - conversation.messages.concat({ + conversation = await chatClient.complete({ + conversationId: conversation.conversationId!, + messages: conversation.messages.concat({ content: 'What is the average error rate per service over the past 4 hours?', role: MessageRole.User, - }) - ); + }), + }); - conversation = await chatClient.complete( - conversation.conversationId!, - conversation.messages.concat({ + conversation = await chatClient.complete({ + conversationId: conversation.conversationId!, + messages: conversation.messages.concat({ content: 'What are the top 2 most frequent errors in the services in the test environment in the last hour?', role: MessageRole.User, - }) - ); + }), + }); - conversation = await chatClient.complete( - conversation.conversationId!, - conversation.messages.concat({ + conversation = await chatClient.complete({ + conversationId: conversation.conversationId!, + messages: conversation.messages.concat({ content: 'Are there any alert for those services?', role: MessageRole.User, - }) - ); + }), + }); const result = await chatClient.evaluate(conversation, [ 'Responds with the active services in the environment "test"', diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/documentation/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/documentation/index.spec.ts index 910b9eeab9bc0..7259d6ea7bc7d 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/documentation/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/documentation/index.spec.ts @@ -7,7 +7,7 @@ /// -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import { InstallationStatusResponse, PerformInstallResponse, @@ -53,7 +53,7 @@ describe('Retrieve documentation function', () => { it('retrieves Elasticsearch documentation', async () => { const prompt = 'How can I configure HTTPS in Elasticsearch?'; - const conversation = await chatClient.complete(prompt); + const conversation = await chatClient.complete({ messages: prompt }); const result = await chatClient.evaluate(conversation, [ `Uses the ${RETRIEVE_DOCUMENTATION_NAME} function before answering the question about the Elastic stack`, @@ -67,7 +67,7 @@ describe('Retrieve documentation function', () => { it('retrieves Kibana documentation', async () => { const prompt = 'What is Kibana Lens and how do I create a bar chart visualization with it?'; - const conversation = await chatClient.complete(prompt); + const conversation = await chatClient.complete({ messages: prompt }); const result = await chatClient.evaluate(conversation, [ `Uses the ${RETRIEVE_DOCUMENTATION_NAME} function before answering the question about Kibana`, @@ -81,7 +81,7 @@ describe('Retrieve documentation function', () => { it('retrieves Observability documentation', async () => { const prompt = 'How can I set up APM instrumentation for my Node.js service in Elastic Observability?'; - const conversation = await chatClient.complete(prompt); + const conversation = await chatClient.complete({ messages: prompt }); const result = await chatClient.evaluate(conversation, [ `Uses the ${RETRIEVE_DOCUMENTATION_NAME} function before answering the question about Observability`, diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/elasticsearch/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/elasticsearch/index.spec.ts index 3133d361a258c..35e121c996f15 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/elasticsearch/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/elasticsearch/index.spec.ts @@ -7,22 +7,19 @@ /// -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import { MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; import { chatClient, esClient } from '../../services'; describe('Elasticsearch functions', () => { - // using 'all' for elasticsearch scenarios enables the LLM correctly pick - // elasticsearch functions when querying for data - before(() => { - chatClient.setScopes(['all']); - }); - describe('health', () => { it('returns the cluster health state', async () => { - const conversation = await chatClient.complete( - 'Can you tell me what the state of my Elasticsearch cluster is?' - ); + const conversation = await chatClient.complete({ + messages: 'Can you tell me what the state of my Elasticsearch cluster is?', + // using 'all' for elasticsearch scenarios enables the LLM correctly pick + // elasticsearch functions when querying for data + scope: 'all', + }); const result = await chatClient.evaluate(conversation, [ 'Calls the Elasticsearch function with method: GET and path: _cluster/health', @@ -64,7 +61,10 @@ describe('Elasticsearch functions', () => { }); it('returns the count of docs in the KB', async () => { - const conversation = await chatClient.complete('How many documents are in the index kb?'); + const conversation = await chatClient.complete({ + messages: 'How many documents are in the index kb?', + scope: 'all', + }); const result = await chatClient.evaluate(conversation, [ 'Calls the `elasticsearch` function OR the `query` function', @@ -75,15 +75,19 @@ describe('Elasticsearch functions', () => { }); it('returns store and refresh stats of an index', async () => { - let conversation = await chatClient.complete('What are the store stats of the index kb?'); + let conversation = await chatClient.complete({ + messages: 'What are the store stats of the index kb?', + scope: 'all', + }); - conversation = await chatClient.complete( - conversation.conversationId!, - conversation.messages.concat({ + conversation = await chatClient.complete({ + conversationId: conversation.conversationId!, + messages: conversation.messages.concat({ content: 'What are the the refresh stats of the index?', role: MessageRole.User, - }) - ); + }), + scope: 'all', + }); const result = await chatClient.evaluate(conversation, [ 'Calls the Elasticsearch function with method: kb/_stats/store', @@ -104,25 +108,29 @@ describe('Elasticsearch functions', () => { describe('assistant created index', () => { it('creates index, adds documents and deletes index', async () => { - let conversation = await chatClient.complete( - 'Create a new index called testing_ai_assistant that will have two documents, one for the test_suite alerts with message "This test is for alerts" and another one for the test_suite esql with the message "This test is for esql"' - ); + let conversation = await chatClient.complete({ + messages: + 'Create a new index called testing_ai_assistant that will have two documents, one for the test_suite alerts with message "This test is for alerts" and another one for the test_suite esql with the message "This test is for esql"', + scope: 'all', + }); - conversation = await chatClient.complete( - conversation.conversationId!, - conversation.messages.concat({ + conversation = await chatClient.complete({ + conversationId: conversation.conversationId!, + messages: conversation.messages.concat({ content: 'What are the fields types for the index testing_ai_assistant?', role: MessageRole.User, - }) - ); + }), + scope: 'all', + }); - conversation = await chatClient.complete( - conversation.conversationId!, - conversation.messages.concat({ + conversation = await chatClient.complete({ + conversationId: conversation.conversationId!, + messages: conversation.messages.concat({ content: 'Delete the testing_ai_assistant index', role: MessageRole.User, - }) - ); + }), + scope: 'all', + }); const result = await chatClient.evaluate(conversation, [ 'Calls the Elasticsearch function to create the index testing_ai_assistant and add the documents to it', @@ -138,7 +146,10 @@ describe('Elasticsearch functions', () => { describe('other', () => { it('returns clusters license', async () => { - const conversation = await chatClient.complete('What is my clusters license?'); + const conversation = await chatClient.complete({ + messages: 'What is my clusters license?', + scope: 'all', + }); const result = await chatClient.evaluate(conversation, [ 'Calls the Elasticsearch function', @@ -148,8 +159,4 @@ describe('Elasticsearch functions', () => { expect(result.passed).to.be(true); }); }); - - after(() => { - chatClient.setScopes(['observability']); - }); }); diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/esql/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/esql/index.spec.ts index 838fea6de0784..a54e790fd9b7f 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/esql/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/esql/index.spec.ts @@ -8,7 +8,7 @@ /// import { apm, timerange } from '@kbn/apm-synthtrace-client'; -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import moment from 'moment'; import { chatClient, esClient, synthtraceEsClients } from '../../services'; @@ -23,7 +23,7 @@ async function evaluateEsqlQuery({ execute?: boolean; criteria?: string[]; }): Promise { - const conversation = await chatClient.complete(question); + const conversation = await chatClient.complete({ messages: question, scope: 'all' }); const evaluation = await chatClient.evaluate(conversation, [ ...(expected @@ -44,10 +44,6 @@ async function evaluateEsqlQuery({ } describe('ES|QL query generation', () => { - before(() => { - chatClient.setScopes(['all']); - }); - describe('other queries', () => { describe('with packetbeat data', () => { before(async () => { @@ -384,8 +380,4 @@ describe('ES|QL query generation', () => { }); }); }); - - after(() => { - chatClient.setScopes(['observability']); - }); }); diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts index 194f5db1fc917..5fdca896c5932 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts @@ -7,7 +7,7 @@ /// -import expect from '@kbn/expect'; +import expect from '@kbn/expect/expect'; import { MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; import { chatClient, esClient, kibanaClient } from '../../services'; @@ -16,17 +16,18 @@ const KB_INDEX = '.kibana-observability-ai-assistant-kb-*'; describe('Knowledge base', () => { describe('kb functions', () => { it('summarizes and recalls information', async () => { - let conversation = await chatClient.complete( - 'Remember that this cluster is used to test the AI Assistant using the Observability AI Evaluation Framework' - ); + let conversation = await chatClient.complete({ + messages: + 'Remember that this cluster is used to test the AI Assistant using the Observability AI Evaluation Framework', + }); - conversation = await chatClient.complete( - conversation.conversationId!, - conversation.messages.concat({ + conversation = await chatClient.complete({ + conversationId: conversation.conversationId!, + messages: conversation.messages.concat({ content: 'What is this cluster used for?', role: MessageRole.User, - }) - ); + }), + }); const result = await chatClient.evaluate(conversation, [ 'Calls the summarize function', @@ -92,7 +93,7 @@ describe('Knowledge base', () => { it('retrieves inventor and purpose of the QRE', async () => { const prompt = 'Who invented the Quantum Revectorization Engine and what does it do?'; - const conversation = await chatClient.complete(prompt); + const conversation = await chatClient.complete({ messages: prompt }); const result = await chatClient.evaluate(conversation, [ 'Uses KB retrieval function to find information about the Quantum Revectorization Engine', @@ -107,7 +108,7 @@ describe('Knowledge base', () => { it('retrieves constraints and energy requirements of the QRE', async () => { const prompt = 'What is the approximate revectorization radius of the QRE and what kind of reactor is required to power it?'; - const conversation = await chatClient.complete(prompt); + const conversation = await chatClient.complete({ messages: prompt }); const result = await chatClient.evaluate(conversation, [ 'Uses KB retrieval function to find the correct document about QRE constraints', From faec300d8a2b66de07ebd00e5d6d957edb0cb311 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 24 Dec 2024 15:45:29 +0000 Subject: [PATCH 24/26] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../scripts/evaluation/scenarios/alerts/index.spec.ts | 2 +- .../scripts/evaluation/scenarios/apm/index.spec.ts | 2 +- .../scripts/evaluation/scenarios/documentation/index.spec.ts | 2 +- .../scripts/evaluation/scenarios/elasticsearch/index.spec.ts | 2 +- .../scripts/evaluation/scenarios/esql/index.spec.ts | 2 +- .../scripts/evaluation/scenarios/kb/index.spec.ts | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts index 6ee2e12585852..c56b65f2dff29 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/alerts/index.spec.ts @@ -7,7 +7,7 @@ /// -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import { RuleResponse } from '@kbn/alerting-plugin/common/routes/rule/response/types/v1'; import moment from 'moment'; import { apm, timerange } from '@kbn/apm-synthtrace-client'; diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/apm/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/apm/index.spec.ts index 016e22ae006e6..e2c295a229c2a 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/apm/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/apm/index.spec.ts @@ -7,7 +7,7 @@ /// -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import moment from 'moment'; import { apm, timerange, serviceMap } from '@kbn/apm-synthtrace-client'; import { RuleResponse } from '@kbn/alerting-plugin/common/routes/rule/response/types/v1'; diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/documentation/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/documentation/index.spec.ts index 7259d6ea7bc7d..b91c66c02a742 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/documentation/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/documentation/index.spec.ts @@ -7,7 +7,7 @@ /// -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import { InstallationStatusResponse, PerformInstallResponse, diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/elasticsearch/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/elasticsearch/index.spec.ts index 35e121c996f15..722043dc29e3e 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/elasticsearch/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/elasticsearch/index.spec.ts @@ -7,7 +7,7 @@ /// -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import { MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; import { chatClient, esClient } from '../../services'; diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/esql/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/esql/index.spec.ts index a54e790fd9b7f..23b0af988035c 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/esql/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/esql/index.spec.ts @@ -8,7 +8,7 @@ /// import { apm, timerange } from '@kbn/apm-synthtrace-client'; -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import moment from 'moment'; import { chatClient, esClient, synthtraceEsClients } from '../../services'; diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts index 5fdca896c5932..e047a02f5c582 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/scenarios/kb/index.spec.ts @@ -7,7 +7,7 @@ /// -import expect from '@kbn/expect/expect'; +import expect from '@kbn/expect'; import { MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; import { chatClient, esClient, kibanaClient } from '../../services'; From 63e5be1d927f7363d44e852643152781ab4dec70 Mon Sep 17 00:00:00 2001 From: Viduni Wickramarachchi Date: Tue, 24 Dec 2024 10:55:58 -0500 Subject: [PATCH 25/26] [Obs AI Assistant] remove comment --- .../scripts/evaluation/kibana_client.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts index 465c6db885c53..d0523c465803e 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts @@ -66,7 +66,6 @@ interface CompleteFunctionParams { scope?: AssistantScope; } -// 2. Update the CompleteFunction type accordingly type CompleteFunction = (params: CompleteFunctionParams) => Promise<{ conversationId?: string; messages: InnerMessage[]; From dbb34cc0d67b5187cc37fc3fcc618547ba63fb2c Mon Sep 17 00:00:00 2001 From: Viduni Wickramarachchi Date: Mon, 30 Dec 2024 08:42:08 -0500 Subject: [PATCH 26/26] [Obs AI Assistant] Avoid passing data only if data is empty --- .../scripts/evaluation/kibana_client.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts index d0523c465803e..ef4d3988679fa 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/kibana_client.ts @@ -124,7 +124,7 @@ export class KibanaClient { return this.axios({ method, url, - ...(method.toLowerCase() !== 'delete' ? { data: data || {} } : {}), + ...(method.toLowerCase() === 'delete' && !data ? {} : { data: data || {} }), headers: { 'kbn-xsrf': 'true', 'x-elastic-internal-origin': 'Kibana',