Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Obs AI Assistant] fix knowledge base tests failing in MKI #205779

Closed
wants to merge 1 commit into from
Closed

[Obs AI Assistant] fix knowledge base tests failing in MKI #205779

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions ...ration/test_suites/observability/ai_assistant/tests/knowledge_base/knowledge_base.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
const es = getService('es');
const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantAPIClient');

describe('Knowledge base', function () {
// Fails on MKI: https://github.com/elastic/kibana/issues/205581
this.tags(['failsOnMKI']);

describe('Knowledge base', () => {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hey @neptunian
The below tests are failing for me in MKI

1)    Observability AI Assistant API tests
   │       knowledge_base/knowledge_base.spec.ts
   │         Knowledge base
   │           when managing a single entry
   │             returns 200 on create:
   │
   │      Error: expected 200 "OK", got 500 "Internal Server Error"
   │       at Context.apply (knowledge_base.spec.ts:58:12)
   │       at Object.apply (wrap_function.js:74:30)
   │   ----
   │       at Test._assertStatus (node_modules/supertest/lib/test.js:252:14)
   │       at /Users/viduni/Workspace/Elastic/kibana/node_modules/supertest/lib/test.js:308:13
   │       at Test._assertFunction (node_modules/supertest/lib/test.js:285:13)
   │       at Test.assert (node_modules/supertest/lib/test.js:164:23)
   │       at localAssert (node_modules/supertest/lib/test.js:120:14)
   │       at fn (node_modules/supertest/lib/test.js:125:7)
   │       at Test.callback (node_modules/supertest/node_modules/superagent/src/node/index.js:899:3)
   │       at fn (node_modules/supertest/node_modules/superagent/src/node/index.js:1165:18)
   │       at IncomingMessage.<anonymous> (node_modules/supertest/node_modules/superagent/src/node/parsers/json.js:19:7)
   │       at IncomingMessage.emit (node:events:531:35)
   │       at endReadableNT (node:internal/streams/readable:1696:12)
   │       at processTicksAndRejections (node:internal/process/task_queues:82:21)
   │ 
   │ 
   │2)    Observability AI Assistant API tests
   │       knowledge_base/knowledge_base.spec.ts
   │         Knowledge base
   │           when managing a single entry
   │             returns 200 on get entries and entry exists:
   │
   │      TypeError: Cannot read properties of undefined (reading 'id')
   │       at Context.<anonymous> (knowledge_base.spec.ts:89:22)
   │       at processTicksAndRejections (node:internal/process/task_queues:95:5)
   │       at Object.apply (wrap_function.js:74:16)
   │ 
   │ 
   │3)    Observability AI Assistant API tests
   │       knowledge_base/knowledge_base.spec.ts
   │         Knowledge base
   │           when managing a single entry
   │             returns 200 on delete:
   │
   │      Error: expected 200 "OK", got 500 "Internal Server Error"
   │       at Context.apply (knowledge_base.spec.ts:103:12)
   │       at Object.apply (wrap_function.js:74:30)
   │   ----
   │       at Test._assertStatus (node_modules/supertest/lib/test.js:252:14)
   │       at /Users/viduni/Workspace/Elastic/kibana/node_modules/supertest/lib/test.js:308:13
   │       at Test._assertFunction (node_modules/supertest/lib/test.js:285:13)
   │       at Test.assert (node_modules/supertest/lib/test.js:164:23)
   │       at localAssert (node_modules/supertest/lib/test.js:120:14)
   │       at fn (node_modules/supertest/lib/test.js:125:7)
   │       at Test.callback (node_modules/supertest/node_modules/superagent/src/node/index.js:899:3)
   │       at fn (node_modules/supertest/node_modules/superagent/src/node/index.js:1165:18)
   │       at IncomingMessage.<anonymous> (node_modules/supertest/node_modules/superagent/src/node/parsers/json.js:19:7)
   │       at IncomingMessage.emit (node:events:531:35)
   │       at endReadableNT (node:internal/streams/readable:1696:12)
   │       at processTicksAndRejections (node:internal/process/task_queues:82:21)
   │ 
   │ 
   │4)    Observability AI Assistant API tests
   │       knowledge_base/knowledge_base.spec.ts
   │         Knowledge base
   │           when managing multiple entries
   │             "before each" hook for "returns 200 on create":
   │
   │      Error: expected 200 "OK", got 500 "Internal Server Error"
   │       at Context.<anonymous> (knowledge_base.spec.ts:181:12)
   │       at processTicksAndRejections (node:internal/process/task_queues:95:5)
   │       at Object.apply (wrap_function.js:74:16)
   │   ----
   │       at Test._assertStatus (node_modules/supertest/lib/test.js:252:14)
   │       at /Users/viduni/Workspace/Elastic/kibana/node_modules/supertest/lib/test.js:308:13
   │       at Test._assertFunction (node_modules/supertest/lib/test.js:285:13)
   │       at Test.assert (node_modules/supertest/lib/test.js:164:23)
   │       at localAssert (node_modules/supertest/lib/test.js:120:14)
   │       at fn (node_modules/supertest/lib/test.js:125:7)
   │       at Test.callback (node_modules/supertest/node_modules/superagent/src/node/index.js:899:3)
   │       at fn (node_modules/supertest/node_modules/superagent/src/node/index.js:1165:18)
   │       at IncomingMessage.<anonymous> (node_modules/supertest/node_modules/superagent/src/node/parsers/json.js:19:7)
   │       at IncomingMessage.emit (node:events:531:35)
   │       at endReadableNT (node:internal/streams/readable:1696:12)
   │       at processTicksAndRejections (node:internal/process/task_queues:82:21)
   │ 
   │ 
   │5)    Observability AI Assistant API tests
   │       knowledge_base/knowledge_base_status.spec.ts
   │         /internal/observability_ai_assistant/kb/status
   │           returns correct status after knowledge base is setup:
   │
   │      Error: expected false to equal true
   │       at Assertion.assert (expect.js:100:11)
   │       at Assertion.apply (expect.js:227:8)
   │       at Assertion.be (expect.js:69:22)
   │       at Context.<anonymous> (knowledge_base_status.spec.ts:49:33)
   │       at processTicksAndRejections (node:internal/process/task_queues:95:5)
   │       at Object.apply (wrap_function.js:74:16)

Am I missing something?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Interesting. Thanks for testing. I initially was able to reproduce that, but after updating the other knowledge base tests in this PR, I no longer got them. Did you run all the tests together?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I used node --no-warnings scripts/functional_test_runner --exclude-tag skipMKI --config x-pack/test_serverless/api_integration/test_suites/observability/ai_assistant/config.ts

Here's the complete log:
MKI-test-run.log

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I just ran them again and am now getting the errors... Guess this is inconsistent on my end for some reason.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I did another run and knowledge_base_status.spec.ts passed this time. Only tests in knowledge_base.spec.ts failed.

before(async () => {
await createKnowledgeBaseModel(ml);

Expand Down
14 changes: 6 additions & 8 deletions .../test_suites/observability/ai_assistant/tests/knowledge_base/knowledge_base_setup.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
const es = getService('es');
const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantAPIClient');

describe('/internal/observability_ai_assistant/kb/setup', function () {
before(async () => {
await deleteKnowledgeBaseModel(ml).catch(() => {});
await deleteInferenceEndpoint({ es }).catch(() => {});
});

it('returns empty object when successful', async () => {
describe('/internal/observability_ai_assistant/kb/setup', () => {
it('returns model info when successful', async () => {
await createKnowledgeBaseModel(ml);
const res = await observabilityAIAssistantAPIClient
.slsAdmin({
Expand All @@ -48,7 +43,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
await deleteInferenceEndpoint({ es });
});

it('returns bad request if model cannot be installed', async () => {
it('returns error message if model is not deployed', async () => {
const res = await observabilityAIAssistantAPIClient
.slsAdmin({
endpoint: `POST ${KNOWLEDGE_BASE_SETUP_API_URL}`,
Expand All @@ -64,6 +59,9 @@ export default function ApiTest({ getService }: FtrProviderContext) {
expect(res.body.message).to.include.string(
'No known trained model with model_id [pt_tiny_elser]'
);

// @ts-expect-error
expect(res.body.statusCode).to.be(500);
});

describe('security roles and access privileges', () => {
Expand Down
44 changes: 28 additions & 16 deletions ...test_suites/observability/ai_assistant/tests/knowledge_base/knowledge_base_status.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,7 @@ import {
TINY_ELSER,
deleteKnowledgeBaseModel,
} from '@kbn/test-suites-xpack/observability_ai_assistant_api_integration/tests/knowledge_base/helpers';
import { AI_ASSISTANT_KB_INFERENCE_ID } from '@kbn/observability-ai-assistant-plugin/server/service/inference_endpoint';
import { FtrProviderContext } from '../../common/ftr_provider_context';
import { KNOWLEDGE_BASE_SETUP_API_URL } from './knowledge_base_setup.spec';

const KNOWLEDGE_BASE_STATUS_API_URL = '/internal/observability_ai_assistant/kb/status';

Expand All @@ -23,15 +21,12 @@ export default function ApiTest({ getService }: FtrProviderContext) {
const es = getService('es');
const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantAPIClient');

describe('/internal/observability_ai_assistant/kb/status', function () {
// Fails on MKI: https://github.com/elastic/kibana/issues/205677
this.tags(['failsOnMKI']);

before(async () => {
describe('/internal/observability_ai_assistant/kb/status', () => {
beforeEach(async () => {
await createKnowledgeBaseModel(ml);
await observabilityAIAssistantAPIClient
.slsAdmin({
endpoint: `POST ${KNOWLEDGE_BASE_SETUP_API_URL}`,
endpoint: 'POST /internal/observability_ai_assistant/kb/setup',
params: {
query: {
model_id: TINY_ELSER.id,
Expand All @@ -41,34 +36,51 @@ export default function ApiTest({ getService }: FtrProviderContext) {
.expect(200);
});

after(async () => {
await deleteKnowledgeBaseModel(ml);
await deleteInferenceEndpoint({ es, name: AI_ASSISTANT_KB_INFERENCE_ID }).catch((err) => {});
afterEach(async () => {
await deleteKnowledgeBaseModel(ml).catch((e) => {});
await deleteInferenceEndpoint({ es }).catch((e) => {});
});

it('returns correct status after knowledge base is setup', async () => {
const res = await observabilityAIAssistantAPIClient
.slsEditor({ endpoint: `GET ${KNOWLEDGE_BASE_STATUS_API_URL}` })
.expect(200);

expect(res.body.ready).to.be(true);
expect(res.body.enabled).to.be(true);
expect(res.body.endpoint?.service_settings?.model_id).to.eql(TINY_ELSER.id);
});

it('returns correct status after model is deleted', async () => {
await deleteKnowledgeBaseModel(ml);

const res = await observabilityAIAssistantAPIClient
.slsEditor({
endpoint: `GET ${KNOWLEDGE_BASE_STATUS_API_URL}`,
})
.expect(200);

expect(res.body.ready).to.be(false);
expect(res.body.enabled).to.be(true);
expect(res.body.ready).to.be(true);
expect(res.body.endpoint?.service_settings?.model_id).to.eql(TINY_ELSER.id);
expect(res.body.errorMessage).to.include.string(
'No known trained model with model_id [pt_tiny_elser]'
);
});

it('returns correct status after elser is stopped', async () => {
await deleteInferenceEndpoint({ es, name: AI_ASSISTANT_KB_INFERENCE_ID });
it('returns correct status after inference endpoint is deleted', async () => {
await deleteInferenceEndpoint({ es });

const res = await observabilityAIAssistantAPIClient
.slsEditor({
endpoint: `GET ${KNOWLEDGE_BASE_STATUS_API_URL}`,
})
.expect(200);

expect(res.body.enabled).to.be(true);
expect(res.body.ready).to.be(false);
expect(res.body.enabled).to.be(true);
expect(res.body.errorMessage).to.include.string(
'Inference endpoint not found [obs_ai_assistant_kb_inference]'
);
});

describe('security roles and access privileges', () => {
Expand Down