From 5fe10785ba2bf7adeb221303c2c55b3c7f841825 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Thu, 31 Oct 2024 15:21:53 -0600 Subject: [PATCH 01/95] Initial setup for streams plugin --- package.json | 1 + tsconfig.base.json | 2 + x-pack/plugins/streams/README.md | 3 + x-pack/plugins/streams/common/config.ts | 30 + x-pack/plugins/streams/common/constants.ts | 9 + x-pack/plugins/streams/common/types.ts | 123 ++ x-pack/plugins/streams/jest.config.js | 18 + x-pack/plugins/streams/kibana.jsonc | 29 + x-pack/plugins/streams/public/index.ts | 13 + x-pack/plugins/streams/public/plugin.ts | 32 + x-pack/plugins/streams/public/types.ts | 16 + x-pack/plugins/streams/server/index.ts | 19 + .../lib/streams/bootstrap_root_assets.ts | 30 + .../server/lib/streams/bootstrap_stream.ts | 62 + .../component_templates/generate_layer.ts | 36 + .../component_templates/logs_all_layer.ts | 1186 +++++++++++++++++ .../errors/component_template_not_found.ts | 13 + .../streams/errors/definition_id_invalid.ts | 13 + .../streams/errors/definition_not_found.ts | 13 + .../streams/errors/fork_condition_missing.ts | 13 + .../lib/streams/errors/id_conflict_error.ts | 18 + .../server/lib/streams/errors/index.ts | 15 + .../errors/index_template_not_found.ts | 13 + .../errors/ingest_pipeline_not_found.ts | 13 + .../lib/streams/errors/permission_denied.ts | 13 + .../lib/streams/errors/security_exception.ts | 13 + .../server/lib/streams/helpers/retry.ts | 58 + .../generate_index_template.ts | 38 + .../lib/streams/index_templates/logs_all.ts | 12 + .../generate_ingest_pipeline.ts | 33 + .../generate_reroute_pipeline.ts | 42 + .../logs_all_default_pipeline.ts | 51 + .../logs_all_json_pipeline.ts | 58 + .../lib/streams/root_stream_definition.ts | 14 + .../streams/server/lib/streams/stream_crud.ts | 109 ++ x-pack/plugins/streams/server/plugin.ts | 130 ++ .../server/routes/create_server_route.ts | 11 + x-pack/plugins/streams/server/routes/index.ts | 18 + .../streams/server/routes/streams/enable.ts | 45 + .../streams/server/routes/streams/fork.ts | 74 + .../streams/server/routes/streams/read.ts | 41 + x-pack/plugins/streams/server/routes/types.ts | 22 + .../server/templates/components/base.ts | 58 + .../templates/manage_index_templates.ts | 107 ++ .../templates/streams_index_template.ts | 43 + x-pack/plugins/streams/server/types.ts | 47 + x-pack/plugins/streams/tsconfig.json | 41 + yarn.lock | 4 + 48 files changed, 2802 insertions(+) create mode 100644 x-pack/plugins/streams/README.md create mode 100644 x-pack/plugins/streams/common/config.ts create mode 100644 x-pack/plugins/streams/common/constants.ts create mode 100644 x-pack/plugins/streams/common/types.ts create mode 100644 x-pack/plugins/streams/jest.config.js create mode 100644 x-pack/plugins/streams/kibana.jsonc create mode 100644 x-pack/plugins/streams/public/index.ts create mode 100644 x-pack/plugins/streams/public/plugin.ts create mode 100644 x-pack/plugins/streams/public/types.ts create mode 100644 x-pack/plugins/streams/server/index.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/bootstrap_root_assets.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/bootstrap_stream.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/component_templates/logs_all_layer.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/component_template_not_found.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/definition_id_invalid.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/definition_not_found.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/fork_condition_missing.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/id_conflict_error.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/index.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/index_template_not_found.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/ingest_pipeline_not_found.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/permission_denied.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/security_exception.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/helpers/retry.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/index_templates/logs_all.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_all_default_pipeline.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_all_json_pipeline.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/stream_crud.ts create mode 100644 x-pack/plugins/streams/server/plugin.ts create mode 100644 x-pack/plugins/streams/server/routes/create_server_route.ts create mode 100644 x-pack/plugins/streams/server/routes/index.ts create mode 100644 x-pack/plugins/streams/server/routes/streams/enable.ts create mode 100644 x-pack/plugins/streams/server/routes/streams/fork.ts create mode 100644 x-pack/plugins/streams/server/routes/streams/read.ts create mode 100644 x-pack/plugins/streams/server/routes/types.ts create mode 100644 x-pack/plugins/streams/server/templates/components/base.ts create mode 100644 x-pack/plugins/streams/server/templates/manage_index_templates.ts create mode 100644 x-pack/plugins/streams/server/templates/streams_index_template.ts create mode 100644 x-pack/plugins/streams/server/types.ts create mode 100644 x-pack/plugins/streams/tsconfig.json diff --git a/package.json b/package.json index 4a91266058ccb..4054d80f42312 100644 --- a/package.json +++ b/package.json @@ -928,6 +928,7 @@ "@kbn/status-plugin-a-plugin": "link:test/server_integration/plugins/status_plugin_a", "@kbn/status-plugin-b-plugin": "link:test/server_integration/plugins/status_plugin_b", "@kbn/std": "link:packages/kbn-std", + "@kbn/streams-plugin": "link:x-pack/plugins/streams", "@kbn/synthetics-plugin": "link:x-pack/plugins/observability_solution/synthetics", "@kbn/synthetics-private-location": "link:x-pack/packages/kbn-synthetics-private-location", "@kbn/task-manager-fixture-plugin": "link:x-pack/test/alerting_api_integration/common/plugins/task_manager_fixture", diff --git a/tsconfig.base.json b/tsconfig.base.json index 727cb930bc606..b06a2ab957a8b 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1826,6 +1826,8 @@ "@kbn/stdio-dev-helpers/*": ["packages/kbn-stdio-dev-helpers/*"], "@kbn/storybook": ["packages/kbn-storybook"], "@kbn/storybook/*": ["packages/kbn-storybook/*"], + "@kbn/streams-plugin": ["x-pack/plugins/streams"], + "@kbn/streams-plugin/*": ["x-pack/plugins/streams/*"], "@kbn/synthetics-e2e": ["x-pack/plugins/observability_solution/synthetics/e2e"], "@kbn/synthetics-e2e/*": ["x-pack/plugins/observability_solution/synthetics/e2e/*"], "@kbn/synthetics-plugin": ["x-pack/plugins/observability_solution/synthetics"], diff --git a/x-pack/plugins/streams/README.md b/x-pack/plugins/streams/README.md new file mode 100644 index 0000000000000..9a3539a33535d --- /dev/null +++ b/x-pack/plugins/streams/README.md @@ -0,0 +1,3 @@ +# Streams Plugin + +This plugin provides an interface to manage streams \ No newline at end of file diff --git a/x-pack/plugins/streams/common/config.ts b/x-pack/plugins/streams/common/config.ts new file mode 100644 index 0000000000000..3371b1b6d8cbc --- /dev/null +++ b/x-pack/plugins/streams/common/config.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema, TypeOf } from '@kbn/config-schema'; + +export const configSchema = schema.object({}); + +export type StreamsConfig = TypeOf; + +/** + * The following map is passed to the server plugin setup under the + * exposeToBrowser: option, and controls which of the above config + * keys are allow-listed to be available in the browser config. + * + * NOTE: anything exposed here will be visible in the UI dev tools, + * and therefore MUST NOT be anything that is sensitive information! + */ +export const exposeToBrowserConfig = {} as const; + +type ValidKeys = keyof { + [K in keyof typeof exposeToBrowserConfig as (typeof exposeToBrowserConfig)[K] extends true + ? K + : never]: true; +}; + +export type StreamsPublicConfig = Pick; diff --git a/x-pack/plugins/streams/common/constants.ts b/x-pack/plugins/streams/common/constants.ts new file mode 100644 index 0000000000000..acc66e9286f87 --- /dev/null +++ b/x-pack/plugins/streams/common/constants.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const ASSET_VERSION = 1; +export const STREAMS_INDEX = '.streams'; diff --git a/x-pack/plugins/streams/common/types.ts b/x-pack/plugins/streams/common/types.ts new file mode 100644 index 0000000000000..a0bf385ed03cd --- /dev/null +++ b/x-pack/plugins/streams/common/types.ts @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; + +export const metricNameSchema = z + .string() + .length(1) + .regex(/[a-zA-Z]/) + .toUpperCase(); + +const apiMetricSchema = z.object({ + name: z.string(), + metrics: z.array( + z.object({ + name: metricNameSchema, + path: z.string(), + }) + ), + equation: z.string(), +}); + +const metaDataSchemaObj = z.object({ + source: z.string(), + destination: z.string(), + fromRoot: z.boolean().default(false), + expand: z.optional( + z.object({ + regex: z.string(), + map: z.array(z.string()), + }) + ), +}); + +type MetadataSchema = z.infer; + +const metadataSchema = metaDataSchemaObj + .or( + z.string().transform( + (value) => + ({ + source: value, + destination: value, + fromRoot: false, + } as MetadataSchema) + ) + ) + .transform((metadata) => ({ + ...metadata, + destination: metadata.destination ?? metadata.source, + })) + .superRefine((value, ctx) => { + if (value.source.length === 0) { + ctx.addIssue({ + path: ['source'], + code: z.ZodIssueCode.custom, + message: 'source should not be empty', + }); + } + if (value.destination.length === 0) { + ctx.addIssue({ + path: ['destination'], + code: z.ZodIssueCode.custom, + message: 'destination should not be empty', + }); + } + }); + +export const apiScraperDefinitionSchema = z.object({ + id: z.string().regex(/^[\w-]+$/), + name: z.string(), + identityFields: z.array(z.string()), + metadata: z.array(metadataSchema), + metrics: z.array(apiMetricSchema), + source: z.object({ + type: z.literal('elasticsearch_api'), + endpoint: z.string(), + method: z.enum(['GET', 'POST']), + params: z.object({ + body: z.record(z.string(), z.any()), + query: z.record(z.string(), z.any()), + }), + collect: z.object({ + path: z.string(), + keyed: z.boolean().default(false), + }), + }), + managed: z.boolean().default(false), + apiKeyId: z.optional(z.string()), +}); + +export type ApiScraperDefinition = z.infer; + +/** + * Example of a "root" StreamEntity + * { + * "id": "logs-all", + * "type": "logs", + * "dataset": "all", + * } + * + * Example of a forked StreamEntity + * { + * "id": "logs-nginx", + * "type": "logs", + * "dataset": "nginx", + * "forked_from": "logs-all" + * } + */ + +export const streamDefinitonSchema = z.object({ + id: z.string(), + type: z.enum(['logs', 'metrics']), + dataset: z.string(), + forked_from: z.optional(z.string()), + condition: z.optional(z.string()), +}); + +export type StreamDefinition = z.infer; diff --git a/x-pack/plugins/streams/jest.config.js b/x-pack/plugins/streams/jest.config.js new file mode 100644 index 0000000000000..2bf0ab535784d --- /dev/null +++ b/x-pack/plugins/streams/jest.config.js @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../..', + roots: ['/x-pack/plugins/logsai/stream_entities_manager'], + coverageDirectory: + '/target/kibana-coverage/jest/x-pack/plugins/logsai/stream_entities_manager', + coverageReporters: ['text', 'html'], + collectCoverageFrom: [ + '/x-pack/plugins/logsai/stream_entities_manager/{common,public,server}/**/*.{js,ts,tsx}', + ], +}; diff --git a/x-pack/plugins/streams/kibana.jsonc b/x-pack/plugins/streams/kibana.jsonc new file mode 100644 index 0000000000000..d97a0b371bb27 --- /dev/null +++ b/x-pack/plugins/streams/kibana.jsonc @@ -0,0 +1,29 @@ +{ + "type": "plugin", + "id": "@kbn/streams-plugin", + "owner": "@elastic/obs-entities", + "description": "A manager for Streams", + "group": "observability", + "visibility": "private", + "plugin": { + "id": "streams", + "configPath": ["xpack", "streams"], + "browser": true, + "server": true, + "requiredPlugins": [ + "data", + "security", + "encryptedSavedObjects", + "usageCollection", + "licensing", + "taskManager", + "features" + ], + "optionalPlugins": [ + "cloud", + "serverless" + ], + "requiredBundles": [ + ] + } +} diff --git a/x-pack/plugins/streams/public/index.ts b/x-pack/plugins/streams/public/index.ts new file mode 100644 index 0000000000000..5b83ea1d297d3 --- /dev/null +++ b/x-pack/plugins/streams/public/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { PluginInitializer, PluginInitializerContext } from '@kbn/core/public'; +import { Plugin } from './plugin'; + +export const plugin: PluginInitializer<{}, {}> = (context: PluginInitializerContext) => { + return new Plugin(context); +}; diff --git a/x-pack/plugins/streams/public/plugin.ts b/x-pack/plugins/streams/public/plugin.ts new file mode 100644 index 0000000000000..f35d18e06ff70 --- /dev/null +++ b/x-pack/plugins/streams/public/plugin.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CoreSetup, CoreStart, PluginInitializerContext } from '@kbn/core/public'; +import { Logger } from '@kbn/logging'; + +import type { StreamsPublicConfig } from '../common/config'; +import { StreamsPluginClass, StreamsPluginSetup, StreamsPluginStart } from './types'; + +export class Plugin implements StreamsPluginClass { + public config: StreamsPublicConfig; + public logger: Logger; + + constructor(context: PluginInitializerContext<{}>) { + this.config = context.config.get(); + this.logger = context.logger.get(); + } + + setup(core: CoreSetup, pluginSetup: StreamsPluginSetup) { + return {}; + } + + start(core: CoreStart) { + return {}; + } + + stop() {} +} diff --git a/x-pack/plugins/streams/public/types.ts b/x-pack/plugins/streams/public/types.ts new file mode 100644 index 0000000000000..61e5fa94098f0 --- /dev/null +++ b/x-pack/plugins/streams/public/types.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Plugin as PluginClass } from '@kbn/core/public'; + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface StreamsPluginSetup {} + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface StreamsPluginStart {} + +export type StreamsPluginClass = PluginClass<{}, {}, StreamsPluginSetup, StreamsPluginStart>; diff --git a/x-pack/plugins/streams/server/index.ts b/x-pack/plugins/streams/server/index.ts new file mode 100644 index 0000000000000..bd8aee304ad15 --- /dev/null +++ b/x-pack/plugins/streams/server/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { PluginInitializerContext } from '@kbn/core-plugins-server'; +import { StreamsConfig } from '../common/config'; +import { StreamsPluginSetup, StreamsPluginStart, config } from './plugin'; +import { StreamsRouteRepository } from './routes'; + +export type { StreamsConfig, StreamsPluginSetup, StreamsPluginStart, StreamsRouteRepository }; +export { config }; + +export const plugin = async (context: PluginInitializerContext) => { + const { StreamsPlugin } = await import('./plugin'); + return new StreamsPlugin(context); +}; diff --git a/x-pack/plugins/streams/server/lib/streams/bootstrap_root_assets.ts b/x-pack/plugins/streams/server/lib/streams/bootstrap_root_assets.ts new file mode 100644 index 0000000000000..a2026c2105f37 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/bootstrap_root_assets.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { Logger } from '@kbn/logging'; +import { + upsertComponent, + upsertIngestPipeline, + upsertTemplate, +} from '../../templates/manage_index_templates'; +import { logsAllLayer } from './component_templates/logs_all_layer'; +import { logsAllDefaultPipeline } from './ingest_pipelines/logs_all_default_pipeline'; +import { logsAllIndexTemplate } from './index_templates/logs_all'; +import { logsAllJsonPipeline } from './ingest_pipelines/logs_all_json_pipeline'; + +interface BootstrapRootEntityParams { + esClient: ElasticsearchClient; + logger: Logger; +} + +export async function bootstrapRootEntity({ esClient, logger }: BootstrapRootEntityParams) { + await upsertComponent({ esClient, logger, component: logsAllLayer }); + await upsertIngestPipeline({ esClient, logger, pipeline: logsAllJsonPipeline }); + await upsertIngestPipeline({ esClient, logger, pipeline: logsAllDefaultPipeline }); + await upsertTemplate({ esClient, logger, template: logsAllIndexTemplate }); +} diff --git a/x-pack/plugins/streams/server/lib/streams/bootstrap_stream.ts b/x-pack/plugins/streams/server/lib/streams/bootstrap_stream.ts new file mode 100644 index 0000000000000..c609af5b49978 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/bootstrap_stream.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; +import { Logger } from '@kbn/logging'; +import { StreamDefinition } from '../../../common/types'; +import { generateLayer } from './component_templates/generate_layer'; +import { generateIngestPipeline } from './ingest_pipelines/generate_ingest_pipeline'; +import { + upsertComponent, + upsertIngestPipeline, + upsertTemplate, +} from '../../templates/manage_index_templates'; +import { generateIndexTemplate } from './index_templates/generate_index_template'; +import { getIndexTemplateComponents } from './stream_crud'; +import { generateReroutePipeline } from './ingest_pipelines/generate_reroute_pipeline'; + +interface BootstrapStreamParams { + scopedClusterClient: IScopedClusterClient; + definition: StreamDefinition; + rootDefinition: StreamDefinition; + logger: Logger; +} +export async function bootstrapStream({ + scopedClusterClient, + definition, + rootDefinition, + logger, +}: BootstrapStreamParams) { + const { composedOf, ignoreMissing } = await getIndexTemplateComponents({ + scopedClusterClient, + definition: rootDefinition, + }); + const reroutePipeline = await generateReroutePipeline({ + esClient: scopedClusterClient.asCurrentUser, + definition: rootDefinition, + }); + await upsertComponent({ + esClient: scopedClusterClient.asSecondaryAuthUser, + logger, + component: generateLayer(definition.id), + }); + await upsertIngestPipeline({ + esClient: scopedClusterClient.asSecondaryAuthUser, + logger, + pipeline: generateIngestPipeline(definition.id), + }); + await upsertIngestPipeline({ + esClient: scopedClusterClient.asSecondaryAuthUser, + logger, + pipeline: reroutePipeline, + }); + await upsertTemplate({ + esClient: scopedClusterClient.asSecondaryAuthUser, + logger, + template: generateIndexTemplate(definition.id, composedOf, ignoreMissing), + }); +} diff --git a/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts b/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts new file mode 100644 index 0000000000000..bd8ee24be4dc1 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { ASSET_VERSION } from '../../../../common/constants'; + +export function generateLayer(id: string): ClusterPutComponentTemplateRequest { + return { + name: `${id}@layer`, + template: { + settings: { + index: { + lifecycle: { + name: 'logs', + }, + codec: 'best_compression', + mapping: { + total_fields: { + ignore_dynamic_beyond_limit: true, + }, + ignore_malformed: true, + }, + }, + }, + }, + version: ASSET_VERSION, + _meta: { + managed: true, + description: `Default settings for the ${id} StreamEntity`, + }, + }; +} diff --git a/x-pack/plugins/streams/server/lib/streams/component_templates/logs_all_layer.ts b/x-pack/plugins/streams/server/lib/streams/component_templates/logs_all_layer.ts new file mode 100644 index 0000000000000..33e43bce29544 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/component_templates/logs_all_layer.ts @@ -0,0 +1,1186 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { ASSET_VERSION } from '../../../../common/constants'; + +export const logsAllLayer: ClusterPutComponentTemplateRequest = { + name: 'logs-all@layer', + template: { + settings: { + index: { + lifecycle: { + name: 'logs', + }, + codec: 'best_compression', + mapping: { + total_fields: { + ignore_dynamic_beyond_limit: true, + }, + ignore_malformed: true, + }, + }, + }, + mappings: { + dynamic: false, + date_detection: false, + properties: { + '@timestamp': { + type: 'date', + }, + 'data_stream.namespace': { + type: 'constant_keyword', + }, + 'data_stream.dataset': { + type: 'constant_keyword', + value: 'all', + }, + 'data_stream.type': { + type: 'constant_keyword', + value: 'logs', + }, + + // Base + labels: { + type: 'object', + }, + message: { + type: 'match_only_text', + }, + tags: { + ignore_above: 1024, + type: 'keyword', + }, + event: { + properties: { + ingested: { + type: 'date', + }, + }, + }, + + // file + file: { + properties: { + accessed: { + type: 'date', + }, + attributes: { + ignore_above: 1024, + type: 'keyword', + }, + code_signature: { + properties: { + digest_algorithm: { + ignore_above: 1024, + type: 'keyword', + }, + exists: { + type: 'boolean', + }, + signing_id: { + ignore_above: 1024, + type: 'keyword', + }, + status: { + ignore_above: 1024, + type: 'keyword', + }, + subject_name: { + ignore_above: 1024, + type: 'keyword', + }, + team_id: { + ignore_above: 1024, + type: 'keyword', + }, + timestamp: { + type: 'date', + }, + trusted: { + type: 'boolean', + }, + valid: { + type: 'boolean', + }, + }, + }, + created: { + type: 'date', + }, + ctime: { + type: 'date', + }, + device: { + ignore_above: 1024, + type: 'keyword', + }, + directory: { + ignore_above: 1024, + type: 'keyword', + }, + drive_letter: { + ignore_above: 1, + type: 'keyword', + }, + elf: { + properties: { + architecture: { + ignore_above: 1024, + type: 'keyword', + }, + byte_order: { + ignore_above: 1024, + type: 'keyword', + }, + cpu_type: { + ignore_above: 1024, + type: 'keyword', + }, + creation_date: { + type: 'date', + }, + exports: { + type: 'flattened', + }, + go_import_hash: { + ignore_above: 1024, + type: 'keyword', + }, + go_imports: { + type: 'flattened', + }, + go_imports_names_entropy: { + type: 'long', + }, + go_imports_names_var_entropy: { + type: 'long', + }, + go_stripped: { + type: 'boolean', + }, + header: { + properties: { + abi_version: { + ignore_above: 1024, + type: 'keyword', + }, + class: { + ignore_above: 1024, + type: 'keyword', + }, + data: { + ignore_above: 1024, + type: 'keyword', + }, + entrypoint: { + type: 'long', + }, + object_version: { + ignore_above: 1024, + type: 'keyword', + }, + os_abi: { + ignore_above: 1024, + type: 'keyword', + }, + type: { + ignore_above: 1024, + type: 'keyword', + }, + version: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + import_hash: { + ignore_above: 1024, + type: 'keyword', + }, + imports: { + type: 'flattened', + }, + imports_names_entropy: { + type: 'long', + }, + imports_names_var_entropy: { + type: 'long', + }, + sections: { + properties: { + chi2: { + type: 'long', + }, + entropy: { + type: 'long', + }, + flags: { + ignore_above: 1024, + type: 'keyword', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + physical_offset: { + ignore_above: 1024, + type: 'keyword', + }, + physical_size: { + type: 'long', + }, + type: { + ignore_above: 1024, + type: 'keyword', + }, + var_entropy: { + type: 'long', + }, + virtual_address: { + type: 'long', + }, + virtual_size: { + type: 'long', + }, + }, + type: 'nested', + }, + segments: { + properties: { + sections: { + ignore_above: 1024, + type: 'keyword', + }, + type: { + ignore_above: 1024, + type: 'keyword', + }, + }, + type: 'nested', + }, + shared_libraries: { + ignore_above: 1024, + type: 'keyword', + }, + telfhash: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + extension: { + ignore_above: 1024, + type: 'keyword', + }, + fork_name: { + ignore_above: 1024, + type: 'keyword', + }, + gid: { + ignore_above: 1024, + type: 'keyword', + }, + group: { + ignore_above: 1024, + type: 'keyword', + }, + hash: { + properties: { + md5: { + ignore_above: 1024, + type: 'keyword', + }, + sha1: { + ignore_above: 1024, + type: 'keyword', + }, + sha256: { + ignore_above: 1024, + type: 'keyword', + }, + sha384: { + ignore_above: 1024, + type: 'keyword', + }, + sha512: { + ignore_above: 1024, + type: 'keyword', + }, + ssdeep: { + ignore_above: 1024, + type: 'keyword', + }, + tlsh: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + inode: { + ignore_above: 1024, + type: 'keyword', + }, + macho: { + properties: { + go_import_hash: { + ignore_above: 1024, + type: 'keyword', + }, + go_imports: { + type: 'flattened', + }, + go_imports_names_entropy: { + type: 'long', + }, + go_imports_names_var_entropy: { + type: 'long', + }, + go_stripped: { + type: 'boolean', + }, + import_hash: { + ignore_above: 1024, + type: 'keyword', + }, + imports: { + type: 'flattened', + }, + imports_names_entropy: { + type: 'long', + }, + imports_names_var_entropy: { + type: 'long', + }, + sections: { + properties: { + entropy: { + type: 'long', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + physical_size: { + type: 'long', + }, + var_entropy: { + type: 'long', + }, + virtual_size: { + type: 'long', + }, + }, + type: 'nested', + }, + symhash: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + mime_type: { + ignore_above: 1024, + type: 'keyword', + }, + mode: { + ignore_above: 1024, + type: 'keyword', + }, + mtime: { + type: 'date', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + owner: { + ignore_above: 1024, + type: 'keyword', + }, + path: { + fields: { + text: { + type: 'match_only_text', + }, + }, + ignore_above: 1024, + type: 'keyword', + }, + pe: { + properties: { + architecture: { + ignore_above: 1024, + type: 'keyword', + }, + company: { + ignore_above: 1024, + type: 'keyword', + }, + description: { + ignore_above: 1024, + type: 'keyword', + }, + file_version: { + ignore_above: 1024, + type: 'keyword', + }, + go_import_hash: { + ignore_above: 1024, + type: 'keyword', + }, + go_imports: { + type: 'flattened', + }, + go_imports_names_entropy: { + type: 'long', + }, + go_imports_names_var_entropy: { + type: 'long', + }, + go_stripped: { + type: 'boolean', + }, + imphash: { + ignore_above: 1024, + type: 'keyword', + }, + import_hash: { + ignore_above: 1024, + type: 'keyword', + }, + imports: { + type: 'flattened', + }, + imports_names_entropy: { + type: 'long', + }, + imports_names_var_entropy: { + type: 'long', + }, + original_file_name: { + ignore_above: 1024, + type: 'keyword', + }, + pehash: { + ignore_above: 1024, + type: 'keyword', + }, + product: { + ignore_above: 1024, + type: 'keyword', + }, + sections: { + properties: { + entropy: { + type: 'long', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + physical_size: { + type: 'long', + }, + var_entropy: { + type: 'long', + }, + virtual_size: { + type: 'long', + }, + }, + type: 'nested', + }, + }, + }, + size: { + type: 'long', + }, + target_path: { + fields: { + text: { + type: 'match_only_text', + }, + }, + ignore_above: 1024, + type: 'keyword', + }, + type: { + ignore_above: 1024, + type: 'keyword', + }, + uid: { + ignore_above: 1024, + type: 'keyword', + }, + x509: { + properties: { + alternative_names: { + ignore_above: 1024, + type: 'keyword', + }, + issuer: { + properties: { + common_name: { + ignore_above: 1024, + type: 'keyword', + }, + country: { + ignore_above: 1024, + type: 'keyword', + }, + distinguished_name: { + ignore_above: 1024, + type: 'keyword', + }, + locality: { + ignore_above: 1024, + type: 'keyword', + }, + organization: { + ignore_above: 1024, + type: 'keyword', + }, + organizational_unit: { + ignore_above: 1024, + type: 'keyword', + }, + state_or_province: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + not_after: { + type: 'date', + }, + not_before: { + type: 'date', + }, + public_key_algorithm: { + ignore_above: 1024, + type: 'keyword', + }, + public_key_curve: { + ignore_above: 1024, + type: 'keyword', + }, + public_key_exponent: { + doc_values: false, + index: false, + type: 'long', + }, + public_key_size: { + type: 'long', + }, + serial_number: { + ignore_above: 1024, + type: 'keyword', + }, + signature_algorithm: { + ignore_above: 1024, + type: 'keyword', + }, + subject: { + properties: { + common_name: { + ignore_above: 1024, + type: 'keyword', + }, + country: { + ignore_above: 1024, + type: 'keyword', + }, + distinguished_name: { + ignore_above: 1024, + type: 'keyword', + }, + locality: { + ignore_above: 1024, + type: 'keyword', + }, + organization: { + ignore_above: 1024, + type: 'keyword', + }, + organizational_unit: { + ignore_above: 1024, + type: 'keyword', + }, + state_or_province: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + version_number: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + }, + }, + + // Host + host: { + properties: { + architecture: { + ignore_above: 1024, + type: 'keyword', + }, + cpu: { + properties: { + usage: { + scaling_factor: 1000, + type: 'scaled_float', + }, + }, + }, + disk: { + properties: { + read: { + properties: { + bytes: { + type: 'long', + }, + }, + }, + write: { + properties: { + bytes: { + type: 'long', + }, + }, + }, + }, + }, + domain: { + ignore_above: 1024, + type: 'keyword', + }, + geo: { + properties: { + city_name: { + ignore_above: 1024, + type: 'keyword', + }, + continent_code: { + ignore_above: 1024, + type: 'keyword', + }, + continent_name: { + ignore_above: 1024, + type: 'keyword', + }, + country_iso_code: { + ignore_above: 1024, + type: 'keyword', + }, + country_name: { + ignore_above: 1024, + type: 'keyword', + }, + location: { + type: 'geo_point', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + postal_code: { + ignore_above: 1024, + type: 'keyword', + }, + region_iso_code: { + ignore_above: 1024, + type: 'keyword', + }, + region_name: { + ignore_above: 1024, + type: 'keyword', + }, + timezone: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + hostname: { + ignore_above: 1024, + type: 'keyword', + }, + id: { + ignore_above: 1024, + type: 'keyword', + }, + ip: { + type: 'ip', + }, + mac: { + ignore_above: 1024, + type: 'keyword', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + network: { + properties: { + egress: { + properties: { + bytes: { + type: 'long', + }, + packets: { + type: 'long', + }, + }, + }, + ingress: { + properties: { + bytes: { + type: 'long', + }, + packets: { + type: 'long', + }, + }, + }, + }, + }, + os: { + properties: { + family: { + ignore_above: 1024, + type: 'keyword', + }, + full: { + fields: { + text: { + type: 'match_only_text', + }, + }, + ignore_above: 1024, + type: 'keyword', + }, + kernel: { + ignore_above: 1024, + type: 'keyword', + }, + name: { + fields: { + text: { + type: 'match_only_text', + }, + }, + ignore_above: 1024, + type: 'keyword', + }, + platform: { + ignore_above: 1024, + type: 'keyword', + }, + type: { + ignore_above: 1024, + type: 'keyword', + }, + version: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + type: { + ignore_above: 1024, + type: 'keyword', + }, + uptime: { + type: 'long', + }, + }, + }, + + // Orchestrator + orchestrator: { + properties: { + api_version: { + ignore_above: 1024, + type: 'keyword', + }, + cluster: { + properties: { + name: { + ignore_above: 1024, + type: 'keyword', + }, + url: { + ignore_above: 1024, + type: 'keyword', + }, + version: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + namespace: { + ignore_above: 1024, + type: 'keyword', + }, + organization: { + ignore_above: 1024, + type: 'keyword', + }, + resource: { + properties: { + name: { + ignore_above: 1024, + type: 'keyword', + }, + type: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + type: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + + log: { + properties: { + file: { + properties: { + path: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + level: { + ignore_above: 1024, + type: 'keyword', + }, + logger: { + ignore_above: 1024, + type: 'keyword', + }, + origin: { + properties: { + file: { + properties: { + line: { + type: 'long', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + function: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + syslog: { + properties: { + facility: { + properties: { + code: { + type: 'long', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + priority: { + type: 'long', + }, + severity: { + properties: { + code: { + type: 'long', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + }, + type: 'object', + }, + }, + }, + + // ECS + ecs: { + properties: { + version: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + + // Agent + agent: { + properties: { + build: { + properties: { + original: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + ephemeral_id: { + ignore_above: 1024, + type: 'keyword', + }, + id: { + ignore_above: 1024, + type: 'keyword', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + type: { + ignore_above: 1024, + type: 'keyword', + }, + version: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + + // Cloud + cloud: { + properties: { + account: { + properties: { + id: { + ignore_above: 1024, + type: 'keyword', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + availability_zone: { + ignore_above: 1024, + type: 'keyword', + }, + instance: { + properties: { + id: { + ignore_above: 1024, + type: 'keyword', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + machine: { + properties: { + type: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + origin: { + properties: { + account: { + properties: { + id: { + ignore_above: 1024, + type: 'keyword', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + availability_zone: { + ignore_above: 1024, + type: 'keyword', + }, + instance: { + properties: { + id: { + ignore_above: 1024, + type: 'keyword', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + machine: { + properties: { + type: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + project: { + properties: { + id: { + ignore_above: 1024, + type: 'keyword', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + provider: { + ignore_above: 1024, + type: 'keyword', + }, + region: { + ignore_above: 1024, + type: 'keyword', + }, + service: { + properties: { + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + }, + }, + project: { + properties: { + id: { + ignore_above: 1024, + type: 'keyword', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + provider: { + ignore_above: 1024, + type: 'keyword', + }, + region: { + ignore_above: 1024, + type: 'keyword', + }, + service: { + properties: { + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + target: { + properties: { + account: { + properties: { + id: { + ignore_above: 1024, + type: 'keyword', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + availability_zone: { + ignore_above: 1024, + type: 'keyword', + }, + instance: { + properties: { + id: { + ignore_above: 1024, + type: 'keyword', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + machine: { + properties: { + type: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + project: { + properties: { + id: { + ignore_above: 1024, + type: 'keyword', + }, + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + provider: { + ignore_above: 1024, + type: 'keyword', + }, + region: { + ignore_above: 1024, + type: 'keyword', + }, + service: { + properties: { + name: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + version: ASSET_VERSION, + _meta: { + managed: true, + description: 'Default layer for logs-all StreamEntity', + }, + deprecated: false, +}; diff --git a/x-pack/plugins/streams/server/lib/streams/errors/component_template_not_found.ts b/x-pack/plugins/streams/server/lib/streams/errors/component_template_not_found.ts new file mode 100644 index 0000000000000..a7e9cebf98507 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/component_template_not_found.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class ComponentTemplateNotFound extends Error { + constructor(message: string) { + super(message); + this.name = 'ComponentTemplateNotFound'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/definition_id_invalid.ts b/x-pack/plugins/streams/server/lib/streams/errors/definition_id_invalid.ts new file mode 100644 index 0000000000000..817e8f67bf25d --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/definition_id_invalid.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class DefinitionIdInvalid extends Error { + constructor(message: string) { + super(message); + this.name = 'DefinitionIdInvalid'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/definition_not_found.ts b/x-pack/plugins/streams/server/lib/streams/errors/definition_not_found.ts new file mode 100644 index 0000000000000..f7e60193baa5f --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/definition_not_found.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class DefinitionNotFound extends Error { + constructor(message: string) { + super(message); + this.name = 'DefinitionNotFound'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/fork_condition_missing.ts b/x-pack/plugins/streams/server/lib/streams/errors/fork_condition_missing.ts new file mode 100644 index 0000000000000..713751dbe4363 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/fork_condition_missing.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class ForkConditionMissing extends Error { + constructor(message: string) { + super(message); + this.name = 'ForkConditionMissing'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/id_conflict_error.ts b/x-pack/plugins/streams/server/lib/streams/errors/id_conflict_error.ts new file mode 100644 index 0000000000000..b2e23e03dd7ef --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/id_conflict_error.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ApiScraperDefinition } from '../../../../common/types'; + +export class IdConflict extends Error { + public definition: ApiScraperDefinition; + + constructor(message: string, def: ApiScraperDefinition) { + super(message); + this.name = 'IdConflict'; + this.definition = def; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/index.ts b/x-pack/plugins/streams/server/lib/streams/errors/index.ts new file mode 100644 index 0000000000000..73842ef3018fe --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './definition_id_invalid'; +export * from './definition_not_found'; +export * from './id_conflict_error'; +export * from './permission_denied'; +export * from './security_exception'; +export * from './index_template_not_found'; +export * from './fork_condition_missing'; +export * from './component_template_not_found'; diff --git a/x-pack/plugins/streams/server/lib/streams/errors/index_template_not_found.ts b/x-pack/plugins/streams/server/lib/streams/errors/index_template_not_found.ts new file mode 100644 index 0000000000000..4f4735dd15fa1 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/index_template_not_found.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class IndexTemplateNotFound extends Error { + constructor(message: string) { + super(message); + this.name = 'IndexTemplateNotFound'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/ingest_pipeline_not_found.ts b/x-pack/plugins/streams/server/lib/streams/errors/ingest_pipeline_not_found.ts new file mode 100644 index 0000000000000..8bf9bbd4933ce --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/ingest_pipeline_not_found.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class IngestPipelineNotFound extends Error { + constructor(message: string) { + super(message); + this.name = 'IngestPipelineNotFound'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/permission_denied.ts b/x-pack/plugins/streams/server/lib/streams/errors/permission_denied.ts new file mode 100644 index 0000000000000..f0133e28063ca --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/permission_denied.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class PermissionDenied extends Error { + constructor(message: string) { + super(message); + this.name = 'PermissionDenied'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/security_exception.ts b/x-pack/plugins/streams/server/lib/streams/errors/security_exception.ts new file mode 100644 index 0000000000000..0b4ae450c2530 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/security_exception.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class SecurityException extends Error { + constructor(message: string) { + super(message); + this.name = 'SecurityException'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/helpers/retry.ts b/x-pack/plugins/streams/server/lib/streams/helpers/retry.ts new file mode 100644 index 0000000000000..32604a22bf9be --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/helpers/retry.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { setTimeout } from 'timers/promises'; +import { errors as EsErrors } from '@elastic/elasticsearch'; +import type { Logger } from '@kbn/logging'; +import { SecurityException } from '../errors'; + +const MAX_ATTEMPTS = 5; + +const retryResponseStatuses = [ + 503, // ServiceUnavailable + 408, // RequestTimeout + 410, // Gone +]; + +const isRetryableError = (e: any) => + e instanceof EsErrors.NoLivingConnectionsError || + e instanceof EsErrors.ConnectionError || + e instanceof EsErrors.TimeoutError || + (e instanceof EsErrors.ResponseError && retryResponseStatuses.includes(e?.statusCode!)); + +/** + * Retries any transient network or configuration issues encountered from Elasticsearch with an exponential backoff. + * Should only be used to wrap operations that are idempotent and can be safely executed more than once. + */ +export const retryTransientEsErrors = async ( + esCall: () => Promise, + { logger, attempt = 0 }: { logger?: Logger; attempt?: number } = {} +): Promise => { + try { + return await esCall(); + } catch (e) { + if (attempt < MAX_ATTEMPTS && isRetryableError(e)) { + const retryCount = attempt + 1; + const retryDelaySec = Math.min(Math.pow(2, retryCount), 64); // 2s, 4s, 8s, 16s, 32s, 64s, 64s, 64s ... + + logger?.warn( + `Retrying Elasticsearch operation after [${retryDelaySec}s] due to error: ${e.toString()} ${ + e.stack + }` + ); + + await setTimeout(retryDelaySec * 1000); + return retryTransientEsErrors(esCall, { logger, attempt: retryCount }); + } + + if (e.meta?.body?.error?.type === 'security_exception') { + throw new SecurityException(e.meta.body.error.reason); + } + + throw e; + } +}; diff --git a/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts b/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts new file mode 100644 index 0000000000000..5052aacf99b6b --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ASSET_VERSION } from '../../../../common/constants'; + +export function generateIndexTemplate( + id: string, + composedOf: string[] = [], + ignoreMissing: string[] = [] +) { + return { + name: id, + index_patterns: [`${id}-*`], + composed_of: [...composedOf, `${id}@layer`], + priority: 200, + version: ASSET_VERSION, + _meta: { + managed: true, + description: `The index template for ${id} StreamEntity`, + }, + data_stream: { + hidden: false, + }, + template: { + settings: { + index: { + default_pipeline: `${id}@default-pipeline`, + }, + }, + }, + allow_auto_create: true, + ignore_missing_component_templates: [...ignoreMissing, `${id}@layer`], + }; +} diff --git a/x-pack/plugins/streams/server/lib/streams/index_templates/logs_all.ts b/x-pack/plugins/streams/server/lib/streams/index_templates/logs_all.ts new file mode 100644 index 0000000000000..b04b9b8e6841d --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/index_templates/logs_all.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { generateIndexTemplate } from './generate_index_template'; + +export const logsAllIndexTemplate: IndicesPutIndexTemplateRequest = + generateIndexTemplate('logs-all'); diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts new file mode 100644 index 0000000000000..5411579c285bc --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ASSET_VERSION } from '../../../../common/constants'; + +export function generateIngestPipeline(id: string) { + return { + id: `${id}@default-pipeline`, + processors: [ + { + append: { + field: 'labels.elastic.stream_entities', + value: [id], + }, + }, + { + pipeline: { + name: `${id}@reroutes`, + ignore_missing_pipeline: true, + }, + }, + ], + _meta: { + description: `Default pipeline for the ${id} StreamEntity`, + managed: true, + }, + version: ASSET_VERSION, + }; +} diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts new file mode 100644 index 0000000000000..a8f0a84ca27d2 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { StreamDefinition } from '../../../../common/types'; +import { ASSET_VERSION, STREAMS_INDEX } from '../../../../common/constants'; + +interface GenerateReroutePipelineParams { + esClient: ElasticsearchClient; + definition: StreamDefinition; +} + +export async function generateReroutePipeline({ + esClient, + definition, +}: GenerateReroutePipelineParams) { + const response = await esClient.search({ + index: STREAMS_INDEX, + query: { match: { forked_from: definition.id } }, + }); + + return { + id: `${definition.id}@reroutes`, + processors: response.hits.hits.map((doc) => { + return { + reroute: { + dataset: doc._source.dataset, + if: doc._source.condition, + }, + }; + }), + _meta: { + description: `Reoute pipeline for the ${definition.id} StreamEntity`, + managed: true, + }, + version: ASSET_VERSION, + }; +} diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_all_default_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_all_default_pipeline.ts new file mode 100644 index 0000000000000..333e752ec9762 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_all_default_pipeline.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ASSET_VERSION } from '../../../../common/constants'; + +export const logsAllDefaultPipeline = { + id: 'logs-all@default-pipeline', + processors: [ + { + set: { + description: "If '@timestamp' is missing, set it with the ingest timestamp", + field: '@timestamp', + override: false, + copy_from: '_ingest.timestamp', + }, + }, + { + set: { + field: 'event.ingested', + value: '{{{_ingest.timestamp}}}', + }, + }, + { + append: { + field: 'labels.elastic.stream_entities', + value: ['logs-all'], + }, + }, + { + pipeline: { + name: 'logs-all@json-pipeline', + ignore_missing_pipeline: true, + }, + }, + { + pipeline: { + name: 'logs-all@reroutes', + ignore_missing_pipeline: true, + }, + }, + ], + _meta: { + description: 'Default pipeline for the logs-all StreamEntity', + managed: true, + }, + version: ASSET_VERSION, +}; diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_all_json_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_all_json_pipeline.ts new file mode 100644 index 0000000000000..276b981aafbfd --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_all_json_pipeline.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ASSET_VERSION } from '../../../../common/constants'; + +export const logsAllJsonPipeline = { + id: 'logs-all@json-pipeline', + processors: [ + { + rename: { + if: "ctx.message instanceof String && ctx.message.startsWith('{') && ctx.message.endsWith('}')", + field: 'message', + target_field: '_tmp_json_message', + ignore_missing: true, + }, + }, + { + json: { + if: 'ctx._tmp_json_message != null', + field: '_tmp_json_message', + add_to_root: true, + add_to_root_conflict_strategy: 'merge' as const, + allow_duplicate_keys: true, + on_failure: [ + { + rename: { + field: '_tmp_json_message', + target_field: 'message', + ignore_missing: true, + }, + }, + ], + }, + }, + { + dot_expander: { + if: 'ctx._tmp_json_message != null', + field: '*', + override: true, + }, + }, + { + remove: { + field: '_tmp_json_message', + ignore_missing: true, + }, + }, + ], + _meta: { + description: 'automatic parsing of JSON log messages', + managed: true, + }, + version: ASSET_VERSION, +}; diff --git a/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts b/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts new file mode 100644 index 0000000000000..1a10a9d5ee807 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { StreamDefinition } from '../../../common/types'; + +export const rootStreamDefinition: StreamDefinition = { + id: 'logs-all', + type: 'logs', + dataset: 'all', +}; diff --git a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts new file mode 100644 index 0000000000000..0294524815d21 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; +import { get } from 'lodash'; +import { StreamDefinition } from '../../../common/types'; +import { STREAMS_INDEX } from '../../../common/constants'; +import { ComponentTemplateNotFound, DefinitionNotFound, IndexTemplateNotFound } from './errors'; + +interface BaseParams { + scopedClusterClient: IScopedClusterClient; +} + +interface BaseParamsWithDefinition extends BaseParams { + definition: StreamDefinition; +} + +export async function createStream({ definition, scopedClusterClient }: BaseParamsWithDefinition) { + return scopedClusterClient.asCurrentUser.index({ + id: definition.id, + index: STREAMS_INDEX, + document: definition, + refresh: 'wait_for', + }); +} + +interface ReadStreamParams extends BaseParams { + id: string; +} + +export async function readStream({ id, scopedClusterClient }: ReadStreamParams) { + const response = await scopedClusterClient.asCurrentUser.get({ + id, + index: STREAMS_INDEX, + }); + if (!response.found) { + throw new DefinitionNotFound(`Stream Entity definition for ${id} not found.`); + } + const definition = response._source as StreamDefinition; + const indexTemplate = await readIndexTemplate({ scopedClusterClient, definition }); + const componentTemplate = await readComponentTemplate({ scopedClusterClient, definition }); + const ingestPipelines = await readIngestPipelines({ scopedClusterClient, definition }); + return { + definition, + index_template: indexTemplate, + component_template: componentTemplate, + ingest_pipelines: ingestPipelines, + }; +} + +export async function readIndexTemplate({ + scopedClusterClient, + definition, +}: BaseParamsWithDefinition) { + const response = await scopedClusterClient.asSecondaryAuthUser.indices.getIndexTemplate({ + name: definition.id, + }); + const indexTemplate = response.index_templates.find((doc) => doc.name === definition.id); + if (!indexTemplate) { + throw new IndexTemplateNotFound(`Unable to find index_template for ${definition.id}`); + } + return indexTemplate; +} + +export async function readComponentTemplate({ + scopedClusterClient, + definition, +}: BaseParamsWithDefinition) { + const response = await scopedClusterClient.asSecondaryAuthUser.cluster.getComponentTemplate({ + name: `${definition.id}@layer`, + }); + const componentTemplate = response.component_templates.find( + (doc) => doc.name === `${definition.id}@layer` + ); + if (!componentTemplate) { + throw new ComponentTemplateNotFound(`Unable to find component_template for ${definition.id}`); + } + return componentTemplate; +} + +export async function readIngestPipelines({ + scopedClusterClient, + definition, +}: BaseParamsWithDefinition) { + const response = await scopedClusterClient.asSecondaryAuthUser.ingest.getPipeline({ + id: `${definition.id}*`, + }); + + return response; +} + +export async function getIndexTemplateComponents({ + scopedClusterClient, + definition, +}: BaseParamsWithDefinition) { + const indexTemplate = await readIndexTemplate({ scopedClusterClient, definition }); + return { + composedOf: indexTemplate.index_template.composed_of, + ignoreMissing: get( + indexTemplate, + 'index_template.ignore_missing_component_templates', + [] + ) as string[], + }; +} diff --git a/x-pack/plugins/streams/server/plugin.ts b/x-pack/plugins/streams/server/plugin.ts new file mode 100644 index 0000000000000..16473918e2cfa --- /dev/null +++ b/x-pack/plugins/streams/server/plugin.ts @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + CoreSetup, + CoreStart, + DEFAULT_APP_CATEGORIES, + KibanaRequest, + Logger, + Plugin, + PluginConfigDescriptor, + PluginInitializerContext, +} from '@kbn/core/server'; +import { registerRoutes } from '@kbn/server-route-repository'; +import { StreamsConfig, configSchema, exposeToBrowserConfig } from '../common/config'; +import { installStreamsTemplates } from './templates/manage_index_templates'; +import { StreamsRouteRepository } from './routes'; +import { RouteDependencies } from './routes/types'; +import { + StreamsPluginSetupDependencies, + StreamsPluginStartDependencies, + StreamsServer, +} from './types'; + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface StreamsPluginSetup {} +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface StreamsPluginStart {} + +export const config: PluginConfigDescriptor = { + schema: configSchema, + exposeToBrowser: exposeToBrowserConfig, +}; + +export class StreamsPlugin + implements + Plugin< + StreamsPluginSetup, + StreamsPluginStart, + StreamsPluginSetupDependencies, + StreamsPluginStartDependencies + > +{ + public config: StreamsConfig; + public logger: Logger; + public server?: StreamsServer; + + constructor(context: PluginInitializerContext) { + this.config = context.config.get(); + this.logger = context.logger.get(); + } + + public setup(core: CoreSetup, plugins: StreamsPluginSetupDependencies): StreamsPluginSetup { + this.server = { + config: this.config, + logger: this.logger, + } as StreamsServer; + + plugins.features.registerKibanaFeature({ + id: 'streams', + name: 'Streams', + order: 1500, + app: ['streams', 'kibana'], + catalogue: ['streams', 'observability'], + category: DEFAULT_APP_CATEGORIES.observability, + privileges: { + all: { + app: ['streams', 'kibana'], + catalogue: ['streams', 'observability'], + savedObject: { + all: [], + read: [], + }, + ui: ['read', 'write'], + api: ['streams_enable', 'streams_fork', 'streams_read'], + }, + read: { + app: ['streams', 'kibana'], + catalogue: ['streams', 'observability'], + api: ['streams_read'], + ui: ['read'], + savedObject: { + all: [], + read: [], + }, + }, + }, + }); + + registerRoutes({ + repository: StreamsRouteRepository, + dependencies: { + server: this.server, + getScopedClients: async ({ request }: { request: KibanaRequest }) => { + const [coreStart] = await core.getStartServices(); + const scopedClusterClient = coreStart.elasticsearch.client.asScoped(request); + const soClient = coreStart.savedObjects.getScopedClient(request); + return { scopedClusterClient, soClient }; + }, + }, + core, + logger: this.logger, + }); + + return {}; + } + + public start(core: CoreStart, plugins: StreamsPluginStartDependencies): StreamsPluginStart { + if (this.server) { + this.server.core = core; + this.server.isServerless = core.elasticsearch.getCapabilities().serverless; + this.server.security = plugins.security; + this.server.encryptedSavedObjects = plugins.encryptedSavedObjects; + this.server.taskManager = plugins.taskManager; + } + + const esClient = core.elasticsearch.client.asInternalUser; + installStreamsTemplates({ esClient, logger: this.logger }).catch((err) => + this.logger.error(err) + ); + + return {}; + } + + public stop() {} +} diff --git a/x-pack/plugins/streams/server/routes/create_server_route.ts b/x-pack/plugins/streams/server/routes/create_server_route.ts new file mode 100644 index 0000000000000..94d85a71c82bb --- /dev/null +++ b/x-pack/plugins/streams/server/routes/create_server_route.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createServerRouteFactory } from '@kbn/server-route-repository'; +import { StreamsRouteHandlerResources } from './types'; + +export const createServerRoute = createServerRouteFactory(); diff --git a/x-pack/plugins/streams/server/routes/index.ts b/x-pack/plugins/streams/server/routes/index.ts new file mode 100644 index 0000000000000..c24362f3ec8a9 --- /dev/null +++ b/x-pack/plugins/streams/server/routes/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { enableStreamsRoute } from './streams/enable'; +import { forkStreamsRoute } from './streams/fork'; +import { readStreamRoute } from './streams/read'; + +export const StreamsRouteRepository = { + ...enableStreamsRoute, + ...forkStreamsRoute, + ...readStreamRoute, +}; + +export type StreamsRouteRepository = typeof StreamsRouteRepository; diff --git a/x-pack/plugins/streams/server/routes/streams/enable.ts b/x-pack/plugins/streams/server/routes/streams/enable.ts new file mode 100644 index 0000000000000..1241434a975df --- /dev/null +++ b/x-pack/plugins/streams/server/routes/streams/enable.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; +import { SecurityException } from '../../lib/streams/errors'; +import { createServerRoute } from '../create_server_route'; +import { bootstrapRootEntity } from '../../lib/streams/bootstrap_root_assets'; +import { createStream } from '../../lib/streams/stream_crud'; +import { rootStreamDefinition } from '../../lib/streams/root_stream_definition'; + +export const enableStreamsRoute = createServerRoute({ + endpoint: 'POST /api/streams/_enable 2023-10-31', + params: z.object({}), + options: { + access: 'public', + security: { + authz: { + requiredPrivileges: ['streams_enable'], + }, + }, + }, + handler: async ({ request, response, logger, getScopedClients }) => { + try { + const { scopedClusterClient } = await getScopedClients({ request }); + await bootstrapRootEntity({ + esClient: scopedClusterClient.asSecondaryAuthUser, + logger, + }); + await createStream({ + scopedClusterClient, + definition: rootStreamDefinition, + }); + return response.ok({ body: { acknowledged: true } }); + } catch (e) { + if (e instanceof SecurityException) { + return response.customError({ body: e, statusCode: 400 }); + } + return response.customError({ body: e, statusCode: 500 }); + } + }, +}); diff --git a/x-pack/plugins/streams/server/routes/streams/fork.ts b/x-pack/plugins/streams/server/routes/streams/fork.ts new file mode 100644 index 0000000000000..1d21b4194f172 --- /dev/null +++ b/x-pack/plugins/streams/server/routes/streams/fork.ts @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; +import { + DefinitionNotFound, + ForkConditionMissing, + IndexTemplateNotFound, + SecurityException, +} from '../../lib/streams/errors'; +import { createServerRoute } from '../create_server_route'; +import { streamDefinitonSchema } from '../../../common/types'; +import { bootstrapStream } from '../../lib/streams/bootstrap_stream'; +import { createStream, readStream } from '../../lib/streams/stream_crud'; + +export const forkStreamsRoute = createServerRoute({ + endpoint: 'POST /api/streams/{id}/_fork 2023-10-31', + options: { + access: 'public', + security: { + authz: { + requiredPrivileges: ['streams_fork'], + }, + }, + }, + params: z.object({ + path: z.object({ + id: z.string(), + }), + body: streamDefinitonSchema, + }), + handler: async ({ response, params, logger, request, getScopedClients }) => { + try { + if (!params.body.condition) { + throw new ForkConditionMissing('You must provide a condition to fork a stream'); + } + + const { scopedClusterClient } = await getScopedClients({ request }); + + const { definition: rootDefinition } = await readStream({ + scopedClusterClient, + id: params.path.id, + }); + + await createStream({ + scopedClusterClient, + definition: { ...params.body, forked_from: rootDefinition.id }, + }); + + await bootstrapStream({ + scopedClusterClient, + definition: params.body, + rootDefinition, + logger, + }); + + return response.ok({ body: { acknowledged: true } }); + } catch (e) { + if (e instanceof IndexTemplateNotFound || e instanceof DefinitionNotFound) { + return response.notFound({ body: e }); + } + + if (e instanceof SecurityException || e instanceof ForkConditionMissing) { + return response.customError({ body: e, statusCode: 400 }); + } + + return response.customError({ body: e, statusCode: 500 }); + } + }, +}); diff --git a/x-pack/plugins/streams/server/routes/streams/read.ts b/x-pack/plugins/streams/server/routes/streams/read.ts new file mode 100644 index 0000000000000..c2f7cfe59b3ec --- /dev/null +++ b/x-pack/plugins/streams/server/routes/streams/read.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; +import { createServerRoute } from '../create_server_route'; +import { DefinitionNotFound } from '../../lib/streams/errors'; +import { readStream } from '../../lib/streams/stream_crud'; + +export const readStreamRoute = createServerRoute({ + endpoint: 'GET /api/streams/{id} 2023-10-31', + options: { + access: 'public', + security: { + authz: { + requiredPrivileges: ['streams_read'], + }, + }, + }, + params: z.object({ + path: z.object({ id: z.string() }), + }), + handler: async ({ response, params, request, getScopedClients }) => { + try { + const { scopedClusterClient } = await getScopedClients({ request }); + const streamEntity = await readStream({ + scopedClusterClient, + id: params.path.id, + }); + + return response.ok({ body: streamEntity }); + } catch (e) { + if (e instanceof DefinitionNotFound) { + return response.notFound({ body: e }); + } + } + }, +}); diff --git a/x-pack/plugins/streams/server/routes/types.ts b/x-pack/plugins/streams/server/routes/types.ts new file mode 100644 index 0000000000000..d547d56c088cd --- /dev/null +++ b/x-pack/plugins/streams/server/routes/types.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { KibanaRequest } from '@kbn/core-http-server'; +import { DefaultRouteHandlerResources } from '@kbn/server-route-repository'; +import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; +import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; +import { StreamsServer } from '../types'; + +export interface RouteDependencies { + server: StreamsServer; + getScopedClients: ({ request }: { request: KibanaRequest }) => Promise<{ + scopedClusterClient: IScopedClusterClient; + soClient: SavedObjectsClientContract; + }>; +} + +export type StreamsRouteHandlerResources = RouteDependencies & DefaultRouteHandlerResources; diff --git a/x-pack/plugins/streams/server/templates/components/base.ts b/x-pack/plugins/streams/server/templates/components/base.ts new file mode 100644 index 0000000000000..a4744c6962155 --- /dev/null +++ b/x-pack/plugins/streams/server/templates/components/base.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; + +export const STREAMS_BASE_COMPONENT = 'streams@mappings'; + +export const BaseComponentTemplateConfig: ClusterPutComponentTemplateRequest = { + name: STREAMS_BASE_COMPONENT, + _meta: { + description: 'Component template for the Stream Entities Manager data set', + managed: true, + }, + template: { + mappings: { + properties: { + labels: { + type: 'object', + }, + tags: { + ignore_above: 1024, + type: 'keyword', + }, + id: { + ignore_above: 1024, + type: 'keyword', + }, + dataset: { + ignore_above: 1024, + type: 'keyword', + }, + type: { + ignore_above: 1024, + type: 'keyword', + }, + forked_from: { + ignore_above: 1024, + type: 'keyword', + }, + condition: { + ignore_above: 1024, + type: 'keyword', + }, + event: { + properties: { + ingested: { + type: 'date', + }, + }, + }, + }, + }, + }, +}; diff --git a/x-pack/plugins/streams/server/templates/manage_index_templates.ts b/x-pack/plugins/streams/server/templates/manage_index_templates.ts new file mode 100644 index 0000000000000..f562c4dfe4183 --- /dev/null +++ b/x-pack/plugins/streams/server/templates/manage_index_templates.ts @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + ClusterPutComponentTemplateRequest, + IndicesPutIndexTemplateRequest, + IngestPutPipelineRequest, +} from '@elastic/elasticsearch/lib/api/types'; +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { BaseComponentTemplateConfig } from './components/base'; +import { retryTransientEsErrors } from '../lib/streams/helpers/retry'; +import { streamsIndexTemplate } from './streams_index_template'; + +interface TemplateManagementOptions { + esClient: ElasticsearchClient; + template: IndicesPutIndexTemplateRequest; + logger: Logger; +} + +interface PipelineManagementOptions { + esClient: ElasticsearchClient; + pipeline: IngestPutPipelineRequest; + logger: Logger; +} + +interface ComponentManagementOptions { + esClient: ElasticsearchClient; + component: ClusterPutComponentTemplateRequest; + logger: Logger; +} + +export const installStreamsTemplates = async ({ + esClient, + logger, +}: { + esClient: ElasticsearchClient; + logger: Logger; +}) => { + await upsertComponent({ + esClient, + logger, + component: BaseComponentTemplateConfig, + }); + await upsertTemplate({ + esClient, + logger, + template: streamsIndexTemplate, + }); +}; + +interface DeleteTemplateOptions { + esClient: ElasticsearchClient; + name: string; + logger: Logger; +} + +export async function upsertTemplate({ esClient, template, logger }: TemplateManagementOptions) { + try { + await retryTransientEsErrors(() => esClient.indices.putIndexTemplate(template), { logger }); + logger.debug(() => `Installed index template: ${JSON.stringify(template)}`); + } catch (error: any) { + logger.error(`Error updating index template: ${error.message}`); + throw error; + } +} + +export async function upsertIngestPipeline({ + esClient, + pipeline, + logger, +}: PipelineManagementOptions) { + try { + await retryTransientEsErrors(() => esClient.ingest.putPipeline(pipeline), { logger }); + logger.debug(() => `Installed index template: ${JSON.stringify(pipeline)}`); + } catch (error: any) { + logger.error(`Error updating index template: ${error.message}`); + throw error; + } +} + +export async function deleteTemplate({ esClient, name, logger }: DeleteTemplateOptions) { + try { + await retryTransientEsErrors( + () => esClient.indices.deleteIndexTemplate({ name }, { ignore: [404] }), + { logger } + ); + } catch (error: any) { + logger.error(`Error deleting index template: ${error.message}`); + throw error; + } +} + +export async function upsertComponent({ esClient, component, logger }: ComponentManagementOptions) { + try { + await retryTransientEsErrors(() => esClient.cluster.putComponentTemplate(component), { + logger, + }); + logger.debug(() => `Installed component template: ${JSON.stringify(component)}`); + } catch (error: any) { + logger.error(`Error updating component template: ${error.message}`); + throw error; + } +} diff --git a/x-pack/plugins/streams/server/templates/streams_index_template.ts b/x-pack/plugins/streams/server/templates/streams_index_template.ts new file mode 100644 index 0000000000000..0e843b2289cf3 --- /dev/null +++ b/x-pack/plugins/streams/server/templates/streams_index_template.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { STREAMS_BASE_COMPONENT } from './components/base'; +import { STREAMS_INDEX } from '../../common/constants'; + +export const streamsIndexTemplate: IndicesPutIndexTemplateRequest = { + name: 'stream-entities', + _meta: { + description: + 'Index template for indices managed by the Streams framework for the instance dataset', + ecs_version: '8.0.0', + managed: true, + managed_by: 'streams', + }, + composed_of: [STREAMS_BASE_COMPONENT], + index_patterns: [STREAMS_INDEX], + priority: 200, + template: { + mappings: { + _meta: { + version: '1.6.0', + }, + date_detection: false, + dynamic: false, + }, + settings: { + index: { + codec: 'best_compression', + mapping: { + total_fields: { + limit: 2000, + }, + }, + }, + }, + }, +}; diff --git a/x-pack/plugins/streams/server/types.ts b/x-pack/plugins/streams/server/types.ts new file mode 100644 index 0000000000000..f1cfcb08a2649 --- /dev/null +++ b/x-pack/plugins/streams/server/types.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CoreStart, ElasticsearchClient, Logger } from '@kbn/core/server'; +import { SecurityPluginStart } from '@kbn/security-plugin/server'; +import { + EncryptedSavedObjectsPluginSetup, + EncryptedSavedObjectsPluginStart, +} from '@kbn/encrypted-saved-objects-plugin/server'; +import { LicensingPluginStart } from '@kbn/licensing-plugin/server'; +import { + TaskManagerSetupContract, + TaskManagerStartContract, +} from '@kbn/task-manager-plugin/server'; +import { FeaturesPluginSetup } from '@kbn/features-plugin/server'; +import { StreamsConfig } from '../common/config'; + +export interface StreamsServer { + core: CoreStart; + config: StreamsConfig; + logger: Logger; + security: SecurityPluginStart; + encryptedSavedObjects: EncryptedSavedObjectsPluginStart; + isServerless: boolean; + taskManager: TaskManagerStartContract; +} + +export interface ElasticsearchAccessorOptions { + elasticsearchClient: ElasticsearchClient; +} + +export interface StreamsPluginSetupDependencies { + encryptedSavedObjects: EncryptedSavedObjectsPluginSetup; + taskManager: TaskManagerSetupContract; + features: FeaturesPluginSetup; +} + +export interface StreamsPluginStartDependencies { + security: SecurityPluginStart; + encryptedSavedObjects: EncryptedSavedObjectsPluginStart; + licensing: LicensingPluginStart; + taskManager: TaskManagerStartContract; +} diff --git a/x-pack/plugins/streams/tsconfig.json b/x-pack/plugins/streams/tsconfig.json new file mode 100644 index 0000000000000..98386b54ca9ea --- /dev/null +++ b/x-pack/plugins/streams/tsconfig.json @@ -0,0 +1,41 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types" + }, + "include": [ + "../../../typings/**/*", + "common/**/*", + "server/**/*", + "public/**/*", + "types/**/*" + ], + "exclude": [ + "target/**/*" + ], + "kbn_references": [ + "@kbn/entities-schema", + "@kbn/config-schema", + "@kbn/core", + "@kbn/server-route-repository-client", + "@kbn/logging", + "@kbn/core-plugins-server", + "@kbn/core-http-server", + "@kbn/security-plugin", + "@kbn/rison", + "@kbn/es-query", + "@kbn/core-elasticsearch-client-server-mocks", + "@kbn/core-saved-objects-api-server-mocks", + "@kbn/logging-mocks", + "@kbn/core-saved-objects-api-server", + "@kbn/core-elasticsearch-server", + "@kbn/task-manager-plugin", + "@kbn/datemath", + "@kbn/server-route-repository", + "@kbn/zod", + "@kbn/zod-helpers", + "@kbn/encrypted-saved-objects-plugin", + "@kbn/licensing-plugin", + "@kbn/alerting-plugin" + ] +} diff --git a/yarn.lock b/yarn.lock index 57ffec1e0a278..e7f4249969902 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6908,6 +6908,10 @@ version "0.0.0" uid "" +"@kbn/streams-plugin@link:x-pack/plugins/streams": + version "0.0.0" + uid "" + "@kbn/synthetics-e2e@link:x-pack/plugins/observability_solution/synthetics/e2e": version "0.0.0" uid "" From 6f940ebda01c9d5df6464591bf60e049594d9022 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Fri, 1 Nov 2024 11:21:33 -0600 Subject: [PATCH 02/95] migrating to the streams naming schema; adding an abstraction for the fork conditions --- x-pack/plugins/streams/common/types.ts | 120 ++++------------ x-pack/plugins/streams/jest.config.js | 9 +- .../lib/streams/bootstrap_root_assets.ts | 16 +-- .../component_templates/generate_layer.ts | 4 +- .../{logs_all_layer.ts => logs_layer.ts} | 17 +-- .../lib/streams/errors/id_conflict_error.ts | 7 +- .../lib/streams/errors/malformed_stream_id.ts | 13 ++ .../reroute_condition_to_painless.test.ts | 133 ++++++++++++++++++ .../helpers/reroute_condition_to_painless.ts | 85 +++++++++++ .../generate_index_template.ts | 12 +- .../index_templates/{logs_all.ts => logs.ts} | 3 +- .../generate_ingest_pipeline.ts | 12 +- .../generate_reroute_pipeline.ts | 17 ++- ...t_pipeline.ts => logs_default_pipeline.ts} | 16 +-- ...json_pipeline.ts => logs_json_pipeline.ts} | 4 +- .../lib/streams/root_stream_definition.ts | 5 +- .../streams/server/lib/streams/stream_crud.ts | 48 ++++--- .../streams/server/routes/streams/fork.ts | 13 +- .../streams/server/routes/streams/read.ts | 2 + .../server/templates/components/base.ts | 6 +- 20 files changed, 353 insertions(+), 189 deletions(-) rename x-pack/plugins/streams/server/lib/streams/component_templates/{logs_all_layer.ts => logs_layer.ts} (98%) create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/malformed_stream_id.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/helpers/reroute_condition_to_painless.test.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/helpers/reroute_condition_to_painless.ts rename x-pack/plugins/streams/server/lib/streams/index_templates/{logs_all.ts => logs.ts} (79%) rename x-pack/plugins/streams/server/lib/streams/ingest_pipelines/{logs_all_default_pipeline.ts => logs_default_pipeline.ts} (72%) rename x-pack/plugins/streams/server/lib/streams/ingest_pipelines/{logs_all_json_pipeline.ts => logs_json_pipeline.ts} (95%) diff --git a/x-pack/plugins/streams/common/types.ts b/x-pack/plugins/streams/common/types.ts index a0bf385ed03cd..5bdc018452376 100644 --- a/x-pack/plugins/streams/common/types.ts +++ b/x-pack/plugins/streams/common/types.ts @@ -7,117 +7,53 @@ import { z } from '@kbn/zod'; -export const metricNameSchema = z - .string() - .length(1) - .regex(/[a-zA-Z]/) - .toUpperCase(); +const stringOrNumberOrBoolean = z.union([z.string(), z.number(), z.boolean()]); -const apiMetricSchema = z.object({ - name: z.string(), - metrics: z.array( - z.object({ - name: metricNameSchema, - path: z.string(), - }) - ), - equation: z.string(), +export const rerouteFilterConditionSchema = z.object({ + field: z.string(), + operator: z.enum(['eq', 'neq', 'lt', 'lte', 'gt', 'gte', 'contains', 'startsWith', 'endsWith']), + value: stringOrNumberOrBoolean, }); -const metaDataSchemaObj = z.object({ - source: z.string(), - destination: z.string(), - fromRoot: z.boolean().default(false), - expand: z.optional( - z.object({ - regex: z.string(), - map: z.array(z.string()), - }) - ), -}); +export type RerouteFilterCondition = z.infer; -type MetadataSchema = z.infer; +export interface RerouteAndCondition { + and: RerouteCondition[]; +} -const metadataSchema = metaDataSchemaObj - .or( - z.string().transform( - (value) => - ({ - source: value, - destination: value, - fromRoot: false, - } as MetadataSchema) - ) - ) - .transform((metadata) => ({ - ...metadata, - destination: metadata.destination ?? metadata.source, - })) - .superRefine((value, ctx) => { - if (value.source.length === 0) { - ctx.addIssue({ - path: ['source'], - code: z.ZodIssueCode.custom, - message: 'source should not be empty', - }); - } - if (value.destination.length === 0) { - ctx.addIssue({ - path: ['destination'], - code: z.ZodIssueCode.custom, - message: 'destination should not be empty', - }); - } - }); +export interface RerouteOrCondition { + or: RerouteCondition[]; +} -export const apiScraperDefinitionSchema = z.object({ - id: z.string().regex(/^[\w-]+$/), - name: z.string(), - identityFields: z.array(z.string()), - metadata: z.array(metadataSchema), - metrics: z.array(apiMetricSchema), - source: z.object({ - type: z.literal('elasticsearch_api'), - endpoint: z.string(), - method: z.enum(['GET', 'POST']), - params: z.object({ - body: z.record(z.string(), z.any()), - query: z.record(z.string(), z.any()), - }), - collect: z.object({ - path: z.string(), - keyed: z.boolean().default(false), - }), - }), - managed: z.boolean().default(false), - apiKeyId: z.optional(z.string()), -}); +export type RerouteCondition = RerouteFilterCondition | RerouteAndCondition | RerouteOrCondition; -export type ApiScraperDefinition = z.infer; +export const rerouteConditionSchema: z.ZodType = z.lazy(() => + z.union([ + rerouteFilterConditionSchema, + z.object({ and: z.array(rerouteConditionSchema) }), + z.object({ or: z.array(rerouteConditionSchema) }), + ]) +); /** - * Example of a "root" StreamEntity + * Example of a "root" stream * { - * "id": "logs-all", - * "type": "logs", - * "dataset": "all", + * "id": "logs", * } * - * Example of a forked StreamEntity + * Example of a forked stream * { - * "id": "logs-nginx", - * "type": "logs", - * "dataset": "nginx", - * "forked_from": "logs-all" + * "id": "logs.nginx", + * "condition": { field: 'log.logger, operator: 'eq', value": "nginx_proxy" } + * "forked_from": "logs" * } */ export const streamDefinitonSchema = z.object({ id: z.string(), - type: z.enum(['logs', 'metrics']), - dataset: z.string(), forked_from: z.optional(z.string()), - condition: z.optional(z.string()), + condition: z.optional(rerouteConditionSchema), + root: z.boolean().default(false), }); export type StreamDefinition = z.infer; diff --git a/x-pack/plugins/streams/jest.config.js b/x-pack/plugins/streams/jest.config.js index 2bf0ab535784d..43d4fd28da9b5 100644 --- a/x-pack/plugins/streams/jest.config.js +++ b/x-pack/plugins/streams/jest.config.js @@ -8,11 +8,8 @@ module.exports = { preset: '@kbn/test', rootDir: '../../..', - roots: ['/x-pack/plugins/logsai/stream_entities_manager'], - coverageDirectory: - '/target/kibana-coverage/jest/x-pack/plugins/logsai/stream_entities_manager', + roots: ['/x-pack/plugins/streams'], + coverageDirectory: '/target/kibana-coverage/jest/x-pack/plugins/streams', coverageReporters: ['text', 'html'], - collectCoverageFrom: [ - '/x-pack/plugins/logsai/stream_entities_manager/{common,public,server}/**/*.{js,ts,tsx}', - ], + collectCoverageFrom: ['/x-pack/plugins/streams/{common,public,server}/**/*.{js,ts,tsx}'], }; diff --git a/x-pack/plugins/streams/server/lib/streams/bootstrap_root_assets.ts b/x-pack/plugins/streams/server/lib/streams/bootstrap_root_assets.ts index a2026c2105f37..e640339f1e456 100644 --- a/x-pack/plugins/streams/server/lib/streams/bootstrap_root_assets.ts +++ b/x-pack/plugins/streams/server/lib/streams/bootstrap_root_assets.ts @@ -12,10 +12,10 @@ import { upsertIngestPipeline, upsertTemplate, } from '../../templates/manage_index_templates'; -import { logsAllLayer } from './component_templates/logs_all_layer'; -import { logsAllDefaultPipeline } from './ingest_pipelines/logs_all_default_pipeline'; -import { logsAllIndexTemplate } from './index_templates/logs_all'; -import { logsAllJsonPipeline } from './ingest_pipelines/logs_all_json_pipeline'; +import { logsLayer } from './component_templates/logs_layer'; +import { logsDefaultPipeline } from './ingest_pipelines/logs_default_pipeline'; +import { logsIndexTemplate } from './index_templates/logs'; +import { logsJsonPipeline } from './ingest_pipelines/logs_json_pipeline'; interface BootstrapRootEntityParams { esClient: ElasticsearchClient; @@ -23,8 +23,8 @@ interface BootstrapRootEntityParams { } export async function bootstrapRootEntity({ esClient, logger }: BootstrapRootEntityParams) { - await upsertComponent({ esClient, logger, component: logsAllLayer }); - await upsertIngestPipeline({ esClient, logger, pipeline: logsAllJsonPipeline }); - await upsertIngestPipeline({ esClient, logger, pipeline: logsAllDefaultPipeline }); - await upsertTemplate({ esClient, logger, template: logsAllIndexTemplate }); + await upsertComponent({ esClient, logger, component: logsLayer }); + await upsertIngestPipeline({ esClient, logger, pipeline: logsJsonPipeline }); + await upsertIngestPipeline({ esClient, logger, pipeline: logsDefaultPipeline }); + await upsertTemplate({ esClient, logger, template: logsIndexTemplate }); } diff --git a/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts b/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts index bd8ee24be4dc1..b3a63bf5cc4c5 100644 --- a/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts +++ b/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts @@ -10,7 +10,7 @@ import { ASSET_VERSION } from '../../../../common/constants'; export function generateLayer(id: string): ClusterPutComponentTemplateRequest { return { - name: `${id}@layer`, + name: `${id}@stream.layer`, template: { settings: { index: { @@ -30,7 +30,7 @@ export function generateLayer(id: string): ClusterPutComponentTemplateRequest { version: ASSET_VERSION, _meta: { managed: true, - description: `Default settings for the ${id} StreamEntity`, + description: `Default settings for the ${id} stream`, }, }; } diff --git a/x-pack/plugins/streams/server/lib/streams/component_templates/logs_all_layer.ts b/x-pack/plugins/streams/server/lib/streams/component_templates/logs_layer.ts similarity index 98% rename from x-pack/plugins/streams/server/lib/streams/component_templates/logs_all_layer.ts rename to x-pack/plugins/streams/server/lib/streams/component_templates/logs_layer.ts index 33e43bce29544..2c2053fa6c82d 100644 --- a/x-pack/plugins/streams/server/lib/streams/component_templates/logs_all_layer.ts +++ b/x-pack/plugins/streams/server/lib/streams/component_templates/logs_layer.ts @@ -8,8 +8,8 @@ import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; import { ASSET_VERSION } from '../../../../common/constants'; -export const logsAllLayer: ClusterPutComponentTemplateRequest = { - name: 'logs-all@layer', +export const logsLayer: ClusterPutComponentTemplateRequest = { + name: 'logs@stream.layer', template: { settings: { index: { @@ -32,17 +32,6 @@ export const logsAllLayer: ClusterPutComponentTemplateRequest = { '@timestamp': { type: 'date', }, - 'data_stream.namespace': { - type: 'constant_keyword', - }, - 'data_stream.dataset': { - type: 'constant_keyword', - value: 'all', - }, - 'data_stream.type': { - type: 'constant_keyword', - value: 'logs', - }, // Base labels: { @@ -1180,7 +1169,7 @@ export const logsAllLayer: ClusterPutComponentTemplateRequest = { version: ASSET_VERSION, _meta: { managed: true, - description: 'Default layer for logs-all StreamEntity', + description: 'Default layer for logs stream', }, deprecated: false, }; diff --git a/x-pack/plugins/streams/server/lib/streams/errors/id_conflict_error.ts b/x-pack/plugins/streams/server/lib/streams/errors/id_conflict_error.ts index b2e23e03dd7ef..a24c7357379fa 100644 --- a/x-pack/plugins/streams/server/lib/streams/errors/id_conflict_error.ts +++ b/x-pack/plugins/streams/server/lib/streams/errors/id_conflict_error.ts @@ -5,14 +5,9 @@ * 2.0. */ -import { ApiScraperDefinition } from '../../../../common/types'; - export class IdConflict extends Error { - public definition: ApiScraperDefinition; - - constructor(message: string, def: ApiScraperDefinition) { + constructor(message: string) { super(message); this.name = 'IdConflict'; - this.definition = def; } } diff --git a/x-pack/plugins/streams/server/lib/streams/errors/malformed_stream_id.ts b/x-pack/plugins/streams/server/lib/streams/errors/malformed_stream_id.ts new file mode 100644 index 0000000000000..2f988204c74b0 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/malformed_stream_id.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class MalformedStreamId extends Error { + constructor(message: string) { + super(message); + this.name = 'MalformedStreamId'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/helpers/reroute_condition_to_painless.test.ts b/x-pack/plugins/streams/server/lib/streams/helpers/reroute_condition_to_painless.test.ts new file mode 100644 index 0000000000000..243ea404ba66d --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/helpers/reroute_condition_to_painless.test.ts @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { rerouteConditionToPainless } from './reroute_condition_to_painless'; + +const operatorConditionAndResutls = [ + { + condition: { field: 'log.logger', operator: 'eq' as const, value: 'nginx_proxy' }, + result: 'ctx.log?.logger == "nginx_proxy"', + }, + { + condition: { field: 'log.logger', operator: 'neq' as const, value: 'nginx_proxy' }, + result: 'ctx.log?.logger != "nginx_proxy"', + }, + { + condition: { field: 'http.response.status_code', operator: 'lt' as const, value: 500 }, + result: 'ctx.http?.response?.status_code < 500', + }, + { + condition: { field: 'http.response.status_code', operator: 'lte' as const, value: 500 }, + result: 'ctx.http?.response?.status_code <= 500', + }, + { + condition: { field: 'http.response.status_code', operator: 'gt' as const, value: 500 }, + result: 'ctx.http?.response?.status_code > 500', + }, + { + condition: { field: 'http.response.status_code', operator: 'gte' as const, value: 500 }, + result: 'ctx.http?.response?.status_code >= 500', + }, + { + condition: { field: 'log.logger', operator: 'startsWith' as const, value: 'nginx' }, + result: 'ctx.log?.logger.startsWith("nginx")', + }, + { + condition: { field: 'log.logger', operator: 'endsWith' as const, value: 'proxy' }, + result: 'ctx.log?.logger.endsWith("proxy")', + }, + { + condition: { field: 'log.logger', operator: 'contains' as const, value: 'proxy' }, + result: 'ctx.log?.logger.contains("proxy")', + }, +]; + +describe('rerouteConditionToPainless', () => { + describe('operators', () => { + operatorConditionAndResutls.forEach((setup) => { + test(`${setup.condition.operator}`, () => { + expect(rerouteConditionToPainless(setup.condition)).toEqual(setup.result); + }); + }); + }); + + describe('and', () => { + test('simple', () => { + const condition = { + and: [ + { field: 'log.logger', operator: 'eq' as const, value: 'nginx_proxy' }, + { field: 'log.level', operator: 'eq' as const, value: 'error' }, + ], + }; + expect( + expect(rerouteConditionToPainless(condition)).toEqual( + 'ctx.log?.logger == "nginx_proxy" && ctx.log?.level == "error"' + ) + ); + }); + }); + + describe('or', () => { + test('simple', () => { + const condition = { + or: [ + { field: 'log.logger', operator: 'eq' as const, value: 'nginx_proxy' }, + { field: 'log.level', operator: 'eq' as const, value: 'error' }, + ], + }; + expect( + expect(rerouteConditionToPainless(condition)).toEqual( + 'ctx.log?.logger == "nginx_proxy" || ctx.log?.level == "error"' + ) + ); + }); + }); + + describe('nested', () => { + test('and with a filter and or with 2 filters', () => { + const condition = { + and: [ + { field: 'log.logger', operator: 'eq' as const, value: 'nginx_proxy' }, + { + or: [ + { field: 'log.level', operator: 'eq' as const, value: 'error' }, + { field: 'log.level', operator: 'eq' as const, value: 'ERROR' }, + ], + }, + ], + }; + expect( + expect(rerouteConditionToPainless(condition)).toEqual( + 'ctx.log?.logger == "nginx_proxy" && (ctx.log?.level == "error" || ctx.log?.level == "ERROR")' + ) + ); + }); + test('and with 2 or with filters', () => { + const condition = { + and: [ + { + or: [ + { field: 'log.logger', operator: 'eq' as const, value: 'nginx_proxy' }, + { field: 'service.name', operator: 'eq' as const, value: 'nginx' }, + ], + }, + { + or: [ + { field: 'log.level', operator: 'eq' as const, value: 'error' }, + { field: 'log.level', operator: 'eq' as const, value: 'ERROR' }, + ], + }, + ], + }; + expect( + expect(rerouteConditionToPainless(condition)).toEqual( + '(ctx.log?.logger == "nginx_proxy" || ctx.service?.name == "nginx") && (ctx.log?.level == "error" || ctx.log?.level == "ERROR")' + ) + ); + }); + }); +}); diff --git a/x-pack/plugins/streams/server/lib/streams/helpers/reroute_condition_to_painless.ts b/x-pack/plugins/streams/server/lib/streams/helpers/reroute_condition_to_painless.ts new file mode 100644 index 0000000000000..5df1e30097b58 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/helpers/reroute_condition_to_painless.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isBoolean, isString } from 'lodash'; +import { + RerouteAndCondition, + RerouteCondition, + rerouteConditionSchema, + RerouteFilterCondition, + rerouteFilterConditionSchema, + RerouteOrCondition, +} from '../../../../common/types'; + +function isFilterCondition(subject: any): subject is RerouteFilterCondition { + const result = rerouteFilterConditionSchema.safeParse(subject); + return result.success; +} + +function isAndCondition(subject: any): subject is RerouteAndCondition { + const result = rerouteConditionSchema.safeParse(subject); + return result.success && subject.and != null; +} + +function isOrCondition(subject: any): subject is RerouteOrCondition { + const result = rerouteConditionSchema.safeParse(subject); + return result.success && subject.or != null; +} + +function safePainlessField(condition: RerouteFilterCondition) { + return `ctx.${condition.field.split('.').join('?.')}`; +} + +function encodeValue(value: string | number | boolean) { + if (isString(value)) { + return `"${value}"`; + } + if (isBoolean(value)) { + return value ? 'true' : 'false'; + } + return value; +} + +function toPainless(condition: RerouteFilterCondition) { + switch (condition.operator) { + case 'neq': + return `${safePainlessField(condition)} != ${encodeValue(condition.value)}`; + case 'lt': + return `${safePainlessField(condition)} < ${encodeValue(condition.value)}`; + case 'lte': + return `${safePainlessField(condition)} <= ${encodeValue(condition.value)}`; + case 'gt': + return `${safePainlessField(condition)} > ${encodeValue(condition.value)}`; + case 'gte': + return `${safePainlessField(condition)} >= ${encodeValue(condition.value)}`; + case 'startsWith': + return `${safePainlessField(condition)}.startsWith(${encodeValue(condition.value)})`; + case 'endsWith': + return `${safePainlessField(condition)}.endsWith(${encodeValue(condition.value)})`; + case 'contains': + return `${safePainlessField(condition)}.contains(${encodeValue(condition.value)})`; + default: + return `${safePainlessField(condition)} == ${encodeValue(condition.value)}`; + } +} + +export function rerouteConditionToPainless(condition: RerouteCondition, nested = false): string { + if (isFilterCondition(condition)) { + return toPainless(condition); + } + if (isAndCondition(condition)) { + const and = condition.and + .map((filter) => rerouteConditionToPainless(filter, true)) + .join(' && '); + return nested ? `(${and})` : and; + } + if (isOrCondition(condition)) { + const or = condition.or.map((filter) => rerouteConditionToPainless(filter, true)).join(' || '); + return nested ? `(${or})` : or; + } + return ''; +} diff --git a/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts b/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts index 5052aacf99b6b..14972ce8a6380 100644 --- a/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts +++ b/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts @@ -13,14 +13,14 @@ export function generateIndexTemplate( ignoreMissing: string[] = [] ) { return { - name: id, - index_patterns: [`${id}-*`], - composed_of: [...composedOf, `${id}@layer`], + name: `${id}@stream`, + index_patterns: [id], + composed_of: [...composedOf, `${id}@stream.layer`], priority: 200, version: ASSET_VERSION, _meta: { managed: true, - description: `The index template for ${id} StreamEntity`, + description: `The index template for ${id} stream`, }, data_stream: { hidden: false, @@ -28,11 +28,11 @@ export function generateIndexTemplate( template: { settings: { index: { - default_pipeline: `${id}@default-pipeline`, + default_pipeline: `${id}@stream.default-pipeline`, }, }, }, allow_auto_create: true, - ignore_missing_component_templates: [...ignoreMissing, `${id}@layer`], + ignore_missing_component_templates: [...ignoreMissing, `${id}@stream.layer`], }; } diff --git a/x-pack/plugins/streams/server/lib/streams/index_templates/logs_all.ts b/x-pack/plugins/streams/server/lib/streams/index_templates/logs.ts similarity index 79% rename from x-pack/plugins/streams/server/lib/streams/index_templates/logs_all.ts rename to x-pack/plugins/streams/server/lib/streams/index_templates/logs.ts index b04b9b8e6841d..58324213dae16 100644 --- a/x-pack/plugins/streams/server/lib/streams/index_templates/logs_all.ts +++ b/x-pack/plugins/streams/server/lib/streams/index_templates/logs.ts @@ -8,5 +8,4 @@ import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; import { generateIndexTemplate } from './generate_index_template'; -export const logsAllIndexTemplate: IndicesPutIndexTemplateRequest = - generateIndexTemplate('logs-all'); +export const logsIndexTemplate: IndicesPutIndexTemplateRequest = generateIndexTemplate('logs'); diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts index 5411579c285bc..1cd7067014a95 100644 --- a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts @@ -9,23 +9,17 @@ import { ASSET_VERSION } from '../../../../common/constants'; export function generateIngestPipeline(id: string) { return { - id: `${id}@default-pipeline`, + id: `${id}@stream.default-pipeline`, processors: [ - { - append: { - field: 'labels.elastic.stream_entities', - value: [id], - }, - }, { pipeline: { - name: `${id}@reroutes`, + name: `${id}@stream.reroutes`, ignore_missing_pipeline: true, }, }, ], _meta: { - description: `Default pipeline for the ${id} StreamEntity`, + description: `Default pipeline for the ${id} streams`, managed: true, }, version: ASSET_VERSION, diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts index a8f0a84ca27d2..43dc8cb787f5a 100644 --- a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts @@ -8,6 +8,7 @@ import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { StreamDefinition } from '../../../../common/types'; import { ASSET_VERSION, STREAMS_INDEX } from '../../../../common/constants'; +import { rerouteConditionToPainless } from '../helpers/reroute_condition_to_painless'; interface GenerateReroutePipelineParams { esClient: ElasticsearchClient; @@ -24,17 +25,25 @@ export async function generateReroutePipeline({ }); return { - id: `${definition.id}@reroutes`, + id: `${definition.id}@stream.reroutes`, processors: response.hits.hits.map((doc) => { + if (!doc._source) { + throw new Error('Source missing for stream definiton document'); + } + if (!doc._source.condition) { + throw new Error( + `Reroute condition missing from forked stream definition ${doc._source.id}` + ); + } return { reroute: { - dataset: doc._source.dataset, - if: doc._source.condition, + destination: doc._source.id, + if: rerouteConditionToPainless(doc._source.condition), }, }; }), _meta: { - description: `Reoute pipeline for the ${definition.id} StreamEntity`, + description: `Reoute pipeline for the ${definition.id} stream`, managed: true, }, version: ASSET_VERSION, diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_all_default_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_default_pipeline.ts similarity index 72% rename from x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_all_default_pipeline.ts rename to x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_default_pipeline.ts index 333e752ec9762..9d9edf3e92b4a 100644 --- a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_all_default_pipeline.ts +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_default_pipeline.ts @@ -7,8 +7,8 @@ import { ASSET_VERSION } from '../../../../common/constants'; -export const logsAllDefaultPipeline = { - id: 'logs-all@default-pipeline', +export const logsDefaultPipeline = { + id: 'logs@stream.default-pipeline', processors: [ { set: { @@ -24,27 +24,21 @@ export const logsAllDefaultPipeline = { value: '{{{_ingest.timestamp}}}', }, }, - { - append: { - field: 'labels.elastic.stream_entities', - value: ['logs-all'], - }, - }, { pipeline: { - name: 'logs-all@json-pipeline', + name: 'logs@stream.json-pipeline', ignore_missing_pipeline: true, }, }, { pipeline: { - name: 'logs-all@reroutes', + name: 'logs@stream.reroutes', ignore_missing_pipeline: true, }, }, ], _meta: { - description: 'Default pipeline for the logs-all StreamEntity', + description: 'Default pipeline for the logs stream', managed: true, }, version: ASSET_VERSION, diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_all_json_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_json_pipeline.ts similarity index 95% rename from x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_all_json_pipeline.ts rename to x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_json_pipeline.ts index 276b981aafbfd..2b6fb7d8d9344 100644 --- a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_all_json_pipeline.ts +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_json_pipeline.ts @@ -7,8 +7,8 @@ import { ASSET_VERSION } from '../../../../common/constants'; -export const logsAllJsonPipeline = { - id: 'logs-all@json-pipeline', +export const logsJsonPipeline = { + id: 'logs@stream.json-pipeline', processors: [ { rename: { diff --git a/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts b/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts index 1a10a9d5ee807..4d95632df9f9e 100644 --- a/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts +++ b/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts @@ -8,7 +8,6 @@ import { StreamDefinition } from '../../../common/types'; export const rootStreamDefinition: StreamDefinition = { - id: 'logs-all', - type: 'logs', - dataset: 'all', + id: 'logs', + root: true, }; diff --git a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts index 0294524815d21..47e2b4061c063 100644 --- a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts +++ b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts @@ -33,23 +33,27 @@ interface ReadStreamParams extends BaseParams { } export async function readStream({ id, scopedClusterClient }: ReadStreamParams) { - const response = await scopedClusterClient.asCurrentUser.get({ - id, - index: STREAMS_INDEX, - }); - if (!response.found) { - throw new DefinitionNotFound(`Stream Entity definition for ${id} not found.`); + try { + const response = await scopedClusterClient.asCurrentUser.get({ + id, + index: STREAMS_INDEX, + }); + const definition = response._source as StreamDefinition; + const indexTemplate = await readIndexTemplate({ scopedClusterClient, definition }); + const componentTemplate = await readComponentTemplate({ scopedClusterClient, definition }); + const ingestPipelines = await readIngestPipelines({ scopedClusterClient, definition }); + return { + definition, + index_template: indexTemplate, + component_template: componentTemplate, + ingest_pipelines: ingestPipelines, + }; + } catch (e) { + if (e.meta?.statusCode === 404) { + throw new DefinitionNotFound(`Stream definition for ${id} not found.`); + } + throw e; } - const definition = response._source as StreamDefinition; - const indexTemplate = await readIndexTemplate({ scopedClusterClient, definition }); - const componentTemplate = await readComponentTemplate({ scopedClusterClient, definition }); - const ingestPipelines = await readIngestPipelines({ scopedClusterClient, definition }); - return { - definition, - index_template: indexTemplate, - component_template: componentTemplate, - ingest_pipelines: ingestPipelines, - }; } export async function readIndexTemplate({ @@ -57,9 +61,11 @@ export async function readIndexTemplate({ definition, }: BaseParamsWithDefinition) { const response = await scopedClusterClient.asSecondaryAuthUser.indices.getIndexTemplate({ - name: definition.id, + name: `${definition.id}@stream`, }); - const indexTemplate = response.index_templates.find((doc) => doc.name === definition.id); + const indexTemplate = response.index_templates.find( + (doc) => doc.name === `${definition.id}@stream` + ); if (!indexTemplate) { throw new IndexTemplateNotFound(`Unable to find index_template for ${definition.id}`); } @@ -71,10 +77,10 @@ export async function readComponentTemplate({ definition, }: BaseParamsWithDefinition) { const response = await scopedClusterClient.asSecondaryAuthUser.cluster.getComponentTemplate({ - name: `${definition.id}@layer`, + name: `${definition.id}@stream.layer`, }); const componentTemplate = response.component_templates.find( - (doc) => doc.name === `${definition.id}@layer` + (doc) => doc.name === `${definition.id}@stream.layer` ); if (!componentTemplate) { throw new ComponentTemplateNotFound(`Unable to find component_template for ${definition.id}`); @@ -87,7 +93,7 @@ export async function readIngestPipelines({ definition, }: BaseParamsWithDefinition) { const response = await scopedClusterClient.asSecondaryAuthUser.ingest.getPipeline({ - id: `${definition.id}*`, + id: `${definition.id}@stream.*`, }); return response; diff --git a/x-pack/plugins/streams/server/routes/streams/fork.ts b/x-pack/plugins/streams/server/routes/streams/fork.ts index 1d21b4194f172..198c2e88c4c52 100644 --- a/x-pack/plugins/streams/server/routes/streams/fork.ts +++ b/x-pack/plugins/streams/server/routes/streams/fork.ts @@ -16,6 +16,7 @@ import { createServerRoute } from '../create_server_route'; import { streamDefinitonSchema } from '../../../common/types'; import { bootstrapStream } from '../../lib/streams/bootstrap_stream'; import { createStream, readStream } from '../../lib/streams/stream_crud'; +import { MalformedStreamId } from '../../lib/streams/errors/malformed_stream_id'; export const forkStreamsRoute = createServerRoute({ endpoint: 'POST /api/streams/{id}/_fork 2023-10-31', @@ -46,6 +47,12 @@ export const forkStreamsRoute = createServerRoute({ id: params.path.id, }); + if (!params.body.id.startsWith(rootDefinition.id)) { + throw new MalformedStreamId( + `The ID (${params.body.id}) from the new stream must start with the parent's id (${rootDefinition.id})` + ); + } + await createStream({ scopedClusterClient, definition: { ...params.body, forked_from: rootDefinition.id }, @@ -64,7 +71,11 @@ export const forkStreamsRoute = createServerRoute({ return response.notFound({ body: e }); } - if (e instanceof SecurityException || e instanceof ForkConditionMissing) { + if ( + e instanceof SecurityException || + e instanceof ForkConditionMissing || + e instanceof MalformedStreamId + ) { return response.customError({ body: e, statusCode: 400 }); } diff --git a/x-pack/plugins/streams/server/routes/streams/read.ts b/x-pack/plugins/streams/server/routes/streams/read.ts index c2f7cfe59b3ec..061522787de52 100644 --- a/x-pack/plugins/streams/server/routes/streams/read.ts +++ b/x-pack/plugins/streams/server/routes/streams/read.ts @@ -36,6 +36,8 @@ export const readStreamRoute = createServerRoute({ if (e instanceof DefinitionNotFound) { return response.notFound({ body: e }); } + + return response.customError({ body: e, statusCode: 500 }); } }, }); diff --git a/x-pack/plugins/streams/server/templates/components/base.ts b/x-pack/plugins/streams/server/templates/components/base.ts index a4744c6962155..fac220fb8be1f 100644 --- a/x-pack/plugins/streams/server/templates/components/base.ts +++ b/x-pack/plugins/streams/server/templates/components/base.ts @@ -37,13 +37,15 @@ export const BaseComponentTemplateConfig: ClusterPutComponentTemplateRequest = { ignore_above: 1024, type: 'keyword', }, + root: { + type: 'boolean', + }, forked_from: { ignore_above: 1024, type: 'keyword', }, condition: { - ignore_above: 1024, - type: 'keyword', + type: 'object', }, event: { properties: { From 9b338087e0852d8c329164cbff1a5aae9af2e9ac Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Fri, 1 Nov 2024 12:00:54 -0600 Subject: [PATCH 03/95] ensure forked streams are marked root:false --- x-pack/plugins/streams/server/routes/streams/fork.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/streams/server/routes/streams/fork.ts b/x-pack/plugins/streams/server/routes/streams/fork.ts index 198c2e88c4c52..c019abd17b58d 100644 --- a/x-pack/plugins/streams/server/routes/streams/fork.ts +++ b/x-pack/plugins/streams/server/routes/streams/fork.ts @@ -55,7 +55,7 @@ export const forkStreamsRoute = createServerRoute({ await createStream({ scopedClusterClient, - definition: { ...params.body, forked_from: rootDefinition.id }, + definition: { ...params.body, forked_from: rootDefinition.id, root: false }, }); await bootstrapStream({ From c8cd0f7734d8378a4b38cd3732a322a1079b3ce0 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Wed, 6 Nov 2024 12:24:38 +0100 Subject: [PATCH 04/95] Commit entities plugins --- x-pack/plugins/logsai/entities_api/README.md | 3 + .../logsai/entities_api/common/entities.ts | 115 +++++++ .../logsai/entities_api/common/index.ts | 16 + .../common/queries/entity_source_query.ts | 37 +++ .../common/queries/entity_time_range_query.ts | 27 ++ .../entities_api/common/utils/esql_escape.ts | 14 + .../utils/esql_result_to_plain_objects.ts | 20 ++ .../common/utils/get_esql_request.ts | 51 +++ .../utils/get_index_patterns_for_filters.ts | 17 + .../logsai/entities_api/jest.config.js | 23 ++ .../plugins/logsai/entities_api/kibana.jsonc | 28 ++ .../logsai/entities_api/public/api/index.tsx | 50 +++ .../logsai/entities_api/public/index.ts | 30 ++ .../logsai/entities_api/public/plugin.ts | 50 +++ .../entities_api/public/services/types.ts | 12 + .../logsai/entities_api/public/types.ts | 24 ++ .../server/built_in_definitions_stub.ts | 101 ++++++ .../logsai/entities_api/server/config.ts | 14 + .../logsai/entities_api/server/index.ts | 37 +++ .../lib/clients/create_alerts_client.ts | 17 + .../lib/clients/create_dashboards_client.ts | 50 +++ .../clients/create_entities_api_es_client.ts | 22 ++ .../lib/clients/create_entity_client.ts | 18 + .../server/lib/clients/create_rules_client.ts | 17 + .../server/lib/clients/create_slo_client.ts | 17 + .../lib/entities/entity_lookup_table.ts | 17 + .../entities/get_data_streams_for_filter.ts | 84 +++++ .../lib/entities/get_definition_entities.ts | 18 + .../lib/entities/get_type_definitions.ts | 18 + .../lib/entities/query_signals_as_entities.ts | 165 ++++++++++ .../lib/entities/query_sources_as_entities.ts | 309 ++++++++++++++++++ .../server/lib/with_entities_api_span.ts | 33 ++ .../logsai/entities_api/server/plugin.ts | 64 ++++ .../create_entities_api_server_route.ts | 13 + .../server/routes/entities/get_entities.ts | 135 ++++++++ .../entities/get_entity_from_type_and_key.ts | 59 ++++ .../entities/get_esql_identity_commands.ts | 236 +++++++++++++ .../server/routes/entities/route.ts | 253 ++++++++++++++ .../entities_api/server/routes/esql/route.ts | 57 ++++ ...et_global_entities_api_route_repository.ts | 22 ++ .../server/routes/register_routes.ts | 28 ++ .../entities_api/server/routes/types.ts | 39 +++ .../entities_api/server/routes/types/route.ts | 93 ++++++ .../logsai/entities_api/server/types.ts | 45 +++ .../plugins/logsai/entities_api/tsconfig.json | 36 ++ .../get_mock_entities_app_context.tsx | 32 ++ .../entities_app/.storybook/jest_setup.js | 11 + .../logsai/entities_app/.storybook/main.js | 8 + .../logsai/entities_app/.storybook/preview.js | 13 + .../.storybook/storybook_decorator.tsx | 18 + x-pack/plugins/logsai/entities_app/README.md | 3 + .../logsai/entities_app/jest.config.js | 23 ++ .../plugins/logsai/entities_app/kibana.jsonc | 24 ++ .../entities_app/public/application.tsx | 49 +++ .../components/all_entities_view/index.tsx | 27 ++ .../public/components/app_root/index.tsx | 67 ++++ .../data_stream_detail_view/index.tsx | 32 ++ .../entities_app_context_provider/index.tsx | 27 ++ .../entities_app_page_header_title.tsx | 28 ++ .../entities_app_page_header/index.tsx | 16 + .../entities_app_page_template/index.tsx | 50 +++ .../entities_app_router_breadcrumb/index.tsx | 11 + .../entities_app_search_bar/index.tsx | 73 +++++ .../entity_detail_overview/index.tsx | 292 +++++++++++++++++ .../components/entity_detail_view/index.tsx | 243 ++++++++++++++ .../index.tsx | 30 ++ .../entity_health_status_badge/index.tsx | 54 +++ .../entity_overview_tab_list/index.tsx | 24 ++ .../entity_pivot_type_view/index.tsx | 69 ++++ .../entity_table/controlled_entity_table.tsx | 217 ++++++++++++ .../public/components/entity_table/index.tsx | 149 +++++++++ .../esql_chart/controlled_esql_chart.tsx | 174 ++++++++++ .../esql_chart/uncontrolled_esql_chart.tsx | 29 ++ .../esql_grid/controlled_esql_grid.tsx | 95 ++++++ .../public/components/loading_panel/index.tsx | 35 ++ .../public/components/redirect_to/index.tsx | 27 ++ .../hooks/use_entities_app_breadcrumbs.ts | 11 + .../public/hooks/use_entities_app_fetch.ts | 73 +++++ .../public/hooks/use_entities_app_params.ts | 14 + .../hooks/use_entities_app_route_path.ts | 15 + .../public/hooks/use_entities_app_router.ts | 55 ++++ .../public/hooks/use_esql_query_result.ts | 56 ++++ .../entities_app/public/hooks/use_kibana.tsx | 36 ++ .../logsai/entities_app/public/index.ts | 26 ++ .../logsai/entities_app/public/plugin.ts | 115 +++++++ .../entities_app/public/routes/config.tsx | 114 +++++++ .../entities_app/public/services/types.ts | 9 + .../logsai/entities_app/public/types.ts | 47 +++ .../public/util/esql_result_to_timeseries.ts | 99 ++++++ .../util/get_initial_columns_for_logs.ts | 104 ++++++ .../logsai/entities_app/server/config.ts | 14 + .../logsai/entities_app/server/index.ts | 35 ++ .../logsai/entities_app/server/plugin.ts | 42 +++ .../logsai/entities_app/server/types.ts | 17 + .../plugins/logsai/entities_app/tsconfig.json | 38 +++ 95 files changed, 5404 insertions(+) create mode 100644 x-pack/plugins/logsai/entities_api/README.md create mode 100644 x-pack/plugins/logsai/entities_api/common/entities.ts create mode 100644 x-pack/plugins/logsai/entities_api/common/index.ts create mode 100644 x-pack/plugins/logsai/entities_api/common/queries/entity_source_query.ts create mode 100644 x-pack/plugins/logsai/entities_api/common/queries/entity_time_range_query.ts create mode 100644 x-pack/plugins/logsai/entities_api/common/utils/esql_escape.ts create mode 100644 x-pack/plugins/logsai/entities_api/common/utils/esql_result_to_plain_objects.ts create mode 100644 x-pack/plugins/logsai/entities_api/common/utils/get_esql_request.ts create mode 100644 x-pack/plugins/logsai/entities_api/common/utils/get_index_patterns_for_filters.ts create mode 100644 x-pack/plugins/logsai/entities_api/jest.config.js create mode 100644 x-pack/plugins/logsai/entities_api/kibana.jsonc create mode 100644 x-pack/plugins/logsai/entities_api/public/api/index.tsx create mode 100644 x-pack/plugins/logsai/entities_api/public/index.ts create mode 100644 x-pack/plugins/logsai/entities_api/public/plugin.ts create mode 100644 x-pack/plugins/logsai/entities_api/public/services/types.ts create mode 100644 x-pack/plugins/logsai/entities_api/public/types.ts create mode 100644 x-pack/plugins/logsai/entities_api/server/built_in_definitions_stub.ts create mode 100644 x-pack/plugins/logsai/entities_api/server/config.ts create mode 100644 x-pack/plugins/logsai/entities_api/server/index.ts create mode 100644 x-pack/plugins/logsai/entities_api/server/lib/clients/create_alerts_client.ts create mode 100644 x-pack/plugins/logsai/entities_api/server/lib/clients/create_dashboards_client.ts create mode 100644 x-pack/plugins/logsai/entities_api/server/lib/clients/create_entities_api_es_client.ts create mode 100644 x-pack/plugins/logsai/entities_api/server/lib/clients/create_entity_client.ts create mode 100644 x-pack/plugins/logsai/entities_api/server/lib/clients/create_rules_client.ts create mode 100644 x-pack/plugins/logsai/entities_api/server/lib/clients/create_slo_client.ts create mode 100644 x-pack/plugins/logsai/entities_api/server/lib/entities/entity_lookup_table.ts create mode 100644 x-pack/plugins/logsai/entities_api/server/lib/entities/get_data_streams_for_filter.ts create mode 100644 x-pack/plugins/logsai/entities_api/server/lib/entities/get_definition_entities.ts create mode 100644 x-pack/plugins/logsai/entities_api/server/lib/entities/get_type_definitions.ts create mode 100644 x-pack/plugins/logsai/entities_api/server/lib/entities/query_signals_as_entities.ts create mode 100644 x-pack/plugins/logsai/entities_api/server/lib/entities/query_sources_as_entities.ts create mode 100644 x-pack/plugins/logsai/entities_api/server/lib/with_entities_api_span.ts create mode 100644 x-pack/plugins/logsai/entities_api/server/plugin.ts create mode 100644 x-pack/plugins/logsai/entities_api/server/routes/create_entities_api_server_route.ts create mode 100644 x-pack/plugins/logsai/entities_api/server/routes/entities/get_entities.ts create mode 100644 x-pack/plugins/logsai/entities_api/server/routes/entities/get_entity_from_type_and_key.ts create mode 100644 x-pack/plugins/logsai/entities_api/server/routes/entities/get_esql_identity_commands.ts create mode 100644 x-pack/plugins/logsai/entities_api/server/routes/entities/route.ts create mode 100644 x-pack/plugins/logsai/entities_api/server/routes/esql/route.ts create mode 100644 x-pack/plugins/logsai/entities_api/server/routes/get_global_entities_api_route_repository.ts create mode 100644 x-pack/plugins/logsai/entities_api/server/routes/register_routes.ts create mode 100644 x-pack/plugins/logsai/entities_api/server/routes/types.ts create mode 100644 x-pack/plugins/logsai/entities_api/server/routes/types/route.ts create mode 100644 x-pack/plugins/logsai/entities_api/server/types.ts create mode 100644 x-pack/plugins/logsai/entities_api/tsconfig.json create mode 100644 x-pack/plugins/logsai/entities_app/.storybook/get_mock_entities_app_context.tsx create mode 100644 x-pack/plugins/logsai/entities_app/.storybook/jest_setup.js create mode 100644 x-pack/plugins/logsai/entities_app/.storybook/main.js create mode 100644 x-pack/plugins/logsai/entities_app/.storybook/preview.js create mode 100644 x-pack/plugins/logsai/entities_app/.storybook/storybook_decorator.tsx create mode 100644 x-pack/plugins/logsai/entities_app/README.md create mode 100644 x-pack/plugins/logsai/entities_app/jest.config.js create mode 100644 x-pack/plugins/logsai/entities_app/kibana.jsonc create mode 100644 x-pack/plugins/logsai/entities_app/public/application.tsx create mode 100644 x-pack/plugins/logsai/entities_app/public/components/all_entities_view/index.tsx create mode 100644 x-pack/plugins/logsai/entities_app/public/components/app_root/index.tsx create mode 100644 x-pack/plugins/logsai/entities_app/public/components/data_stream_detail_view/index.tsx create mode 100644 x-pack/plugins/logsai/entities_app/public/components/entities_app_context_provider/index.tsx create mode 100644 x-pack/plugins/logsai/entities_app/public/components/entities_app_page_header/entities_app_page_header_title.tsx create mode 100644 x-pack/plugins/logsai/entities_app/public/components/entities_app_page_header/index.tsx create mode 100644 x-pack/plugins/logsai/entities_app/public/components/entities_app_page_template/index.tsx create mode 100644 x-pack/plugins/logsai/entities_app/public/components/entities_app_router_breadcrumb/index.tsx create mode 100644 x-pack/plugins/logsai/entities_app/public/components/entities_app_search_bar/index.tsx create mode 100644 x-pack/plugins/logsai/entities_app/public/components/entity_detail_overview/index.tsx create mode 100644 x-pack/plugins/logsai/entities_app/public/components/entity_detail_view/index.tsx create mode 100644 x-pack/plugins/logsai/entities_app/public/components/entity_detail_view_header_section/index.tsx create mode 100644 x-pack/plugins/logsai/entities_app/public/components/entity_health_status_badge/index.tsx create mode 100644 x-pack/plugins/logsai/entities_app/public/components/entity_overview_tab_list/index.tsx create mode 100644 x-pack/plugins/logsai/entities_app/public/components/entity_pivot_type_view/index.tsx create mode 100644 x-pack/plugins/logsai/entities_app/public/components/entity_table/controlled_entity_table.tsx create mode 100644 x-pack/plugins/logsai/entities_app/public/components/entity_table/index.tsx create mode 100644 x-pack/plugins/logsai/entities_app/public/components/esql_chart/controlled_esql_chart.tsx create mode 100644 x-pack/plugins/logsai/entities_app/public/components/esql_chart/uncontrolled_esql_chart.tsx create mode 100644 x-pack/plugins/logsai/entities_app/public/components/esql_grid/controlled_esql_grid.tsx create mode 100644 x-pack/plugins/logsai/entities_app/public/components/loading_panel/index.tsx create mode 100644 x-pack/plugins/logsai/entities_app/public/components/redirect_to/index.tsx create mode 100644 x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_breadcrumbs.ts create mode 100644 x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_fetch.ts create mode 100644 x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_params.ts create mode 100644 x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_route_path.ts create mode 100644 x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_router.ts create mode 100644 x-pack/plugins/logsai/entities_app/public/hooks/use_esql_query_result.ts create mode 100644 x-pack/plugins/logsai/entities_app/public/hooks/use_kibana.tsx create mode 100644 x-pack/plugins/logsai/entities_app/public/index.ts create mode 100644 x-pack/plugins/logsai/entities_app/public/plugin.ts create mode 100644 x-pack/plugins/logsai/entities_app/public/routes/config.tsx create mode 100644 x-pack/plugins/logsai/entities_app/public/services/types.ts create mode 100644 x-pack/plugins/logsai/entities_app/public/types.ts create mode 100644 x-pack/plugins/logsai/entities_app/public/util/esql_result_to_timeseries.ts create mode 100644 x-pack/plugins/logsai/entities_app/public/util/get_initial_columns_for_logs.ts create mode 100644 x-pack/plugins/logsai/entities_app/server/config.ts create mode 100644 x-pack/plugins/logsai/entities_app/server/index.ts create mode 100644 x-pack/plugins/logsai/entities_app/server/plugin.ts create mode 100644 x-pack/plugins/logsai/entities_app/server/types.ts create mode 100644 x-pack/plugins/logsai/entities_app/tsconfig.json diff --git a/x-pack/plugins/logsai/entities_api/README.md b/x-pack/plugins/logsai/entities_api/README.md new file mode 100644 index 0000000000000..f69a2546e7464 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/README.md @@ -0,0 +1,3 @@ +# Entities API + +APIs for the Entities App. diff --git a/x-pack/plugins/logsai/entities_api/common/entities.ts b/x-pack/plugins/logsai/entities_api/common/entities.ts new file mode 100644 index 0000000000000..07fb7374187ab --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/common/entities.ts @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ValuesType } from 'utility-types'; + +type EntityPivotIdentity = Record; + +export interface EntityDataSource { + index: string | string[]; +} + +export interface IEntity { + id: string; + type: string; + key: string; + displayName: string; +} + +interface IPivotEntity extends IEntity { + identity: EntityPivotIdentity; +} + +export type StoredEntity = IEntity; + +export interface DefinitionEntity extends StoredEntity { + filters: EntityFilter[]; + pivot: Pivot; +} + +export interface Pivot { + type: string; + identityFields: string[]; +} + +export type PivotEntity = IPivotEntity; + +export interface StoredPivotEntity extends StoredEntity, IPivotEntity {} + +interface EntityDefinitionTermFilter { + term: { [x: string]: string }; +} + +interface EntityDefinitionIndexFilter { + index: string[]; +} + +interface EntityDefinitionMatchAllFilter { + match_all: {}; +} + +export interface EntityGrouping { + id: string; + filters: EntityFilter[]; + pivot: Pivot; +} + +export interface EntityDisplayNameTemplate { + concat: Array<{ field: string } | { literal: string }>; +} + +export interface EntityTypeDefinition { + pivot: Pivot; + displayName: string; + displayNameTemplate?: EntityDisplayNameTemplate; +} + +export type EntityFilter = + | EntityDefinitionTermFilter + | EntityDefinitionIndexFilter + | EntityDefinitionMatchAllFilter; + +export type Entity = DefinitionEntity | PivotEntity | StoredPivotEntity; + +export const ENTITY_HEALTH_STATUS = { + Healthy: 'Healthy' as const, + Degraded: 'Degraded' as const, + Violated: 'Violated' as const, + NoData: 'NoData' as const, +}; + +export type EntityHealthStatus = ValuesType; + +export const ENTITY_HEALTH_STATUS_INT = { + [ENTITY_HEALTH_STATUS.Violated]: 4 as const, + [ENTITY_HEALTH_STATUS.Degraded]: 3 as const, + [ENTITY_HEALTH_STATUS.NoData]: 2 as const, + [ENTITY_HEALTH_STATUS.Healthy]: 1 as const, +} satisfies Record; + +const HEALTH_STATUS_INT_TO_KEYWORD = Object.fromEntries( + Object.entries(ENTITY_HEALTH_STATUS_INT).map(([healthStatus, int]) => { + return [int, healthStatus as EntityHealthStatus]; + }) +); + +export const healthStatusIntToKeyword = (value: ValuesType) => { + return HEALTH_STATUS_INT_TO_KEYWORD[value]; +}; + +export type EntityWithSignalStatus = IEntity & { + alertsCount: number; + healthStatus: EntityHealthStatus | null; +}; + +export function isPivotEntity(entity: IEntity): entity is IPivotEntity { + return 'identity' in entity; +} + +export function isDefinitionEntity(entity: IEntity): entity is DefinitionEntity { + return 'filters' in entity; +} diff --git a/x-pack/plugins/logsai/entities_api/common/index.ts b/x-pack/plugins/logsai/entities_api/common/index.ts new file mode 100644 index 0000000000000..b759a5e7bc8a2 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/common/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export type { + Entity, + EntityWithSignalStatus, + EntityHealthStatus, + PivotEntity, + Pivot, +} from './entities'; + +export { getIndexPatternsForFilters } from './utils/get_index_patterns_for_filters'; +export { entitySourceQuery } from './queries/entity_source_query'; diff --git a/x-pack/plugins/logsai/entities_api/common/queries/entity_source_query.ts b/x-pack/plugins/logsai/entities_api/common/queries/entity_source_query.ts new file mode 100644 index 0000000000000..f91e5c627cc8e --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/common/queries/entity_source_query.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import { isDefinitionEntity, isPivotEntity, type IEntity } from '../entities'; + +export function entitySourceQuery({ entity }: { entity: IEntity }): QueryDslQueryContainer[] { + if (isPivotEntity(entity)) { + return Object.entries(entity.identity).map(([field, value]) => { + return { + term: { + [field]: value, + }, + }; + }); + } else if (isDefinitionEntity(entity)) { + return entity.filters + .flatMap((filter): QueryDslQueryContainer => { + if ('index' in filter) { + return { + bool: { + should: filter.index.map((index) => ({ wildcard: { _index: index } })), + minimum_should_match: 1, + }, + }; + } + return filter; + }) + .concat(entity.pivot.identityFields.map((field) => ({ exists: { field } }))); + } + + throw new Error(`Could not build query for unknown entity type`); +} diff --git a/x-pack/plugins/logsai/entities_api/common/queries/entity_time_range_query.ts b/x-pack/plugins/logsai/entities_api/common/queries/entity_time_range_query.ts new file mode 100644 index 0000000000000..72f65400377c8 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/common/queries/entity_time_range_query.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; + +export function entityTimeRangeQuery(start: number, end: number): QueryDslQueryContainer[] { + return [ + { + range: { + 'entity.lastSeenTimestamp': { + gte: start, + }, + }, + }, + { + range: { + 'entity.firstSeenTimestamp': { + lte: end, + }, + }, + }, + ]; +} diff --git a/x-pack/plugins/logsai/entities_api/common/utils/esql_escape.ts b/x-pack/plugins/logsai/entities_api/common/utils/esql_escape.ts new file mode 100644 index 0000000000000..460fda2d4f72c --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/common/utils/esql_escape.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function escapeString(value: string): string { + return `"${value.replaceAll(/[^\\]"/g, '\\"')}"`; +} + +export function escapeColumn(column: string): string { + return `\`${column.replaceAll(/[^\\]`/g, '\\`')}\``; +} diff --git a/x-pack/plugins/logsai/entities_api/common/utils/esql_result_to_plain_objects.ts b/x-pack/plugins/logsai/entities_api/common/utils/esql_result_to_plain_objects.ts new file mode 100644 index 0000000000000..ad48bcb311b25 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/common/utils/esql_result_to_plain_objects.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ESQLSearchResponse } from '@kbn/es-types'; + +export function esqlResultToPlainObjects>( + result: ESQLSearchResponse +): T[] { + return result.values.map((row) => { + return row.reduce>((acc, value, index) => { + const column = result.columns[index]; + acc[column.name] = value; + return acc; + }, {}); + }) as T[]; +} diff --git a/x-pack/plugins/logsai/entities_api/common/utils/get_esql_request.ts b/x-pack/plugins/logsai/entities_api/common/utils/get_esql_request.ts new file mode 100644 index 0000000000000..1dde049db8ade --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/common/utils/get_esql_request.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { excludeFrozenQuery } from '@kbn/observability-utils-common/es/queries/exclude_frozen_query'; +import { kqlQuery } from '@kbn/observability-utils-common/es/queries/kql_query'; + +export function getEsqlRequest({ + query, + start, + end, + kuery, + dslFilter, + timestampField, +}: { + query: string; + start?: number; + end?: number; + kuery?: string; + dslFilter?: QueryDslQueryContainer[]; + timestampField?: string; +}) { + return { + query, + filter: { + bool: { + filter: [ + ...(start && end + ? [ + { + range: { + [timestampField ?? '@timestamp']: { + gte: start, + lte: end, + }, + }, + }, + ] + : []), + ...excludeFrozenQuery(), + ...kqlQuery(kuery), + ...(dslFilter ?? []), + ], + }, + }, + }; +} diff --git a/x-pack/plugins/logsai/entities_api/common/utils/get_index_patterns_for_filters.ts b/x-pack/plugins/logsai/entities_api/common/utils/get_index_patterns_for_filters.ts new file mode 100644 index 0000000000000..7b513942745c1 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/common/utils/get_index_patterns_for_filters.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EntityFilter } from '../entities'; + +export function getIndexPatternsForFilters(filters: EntityFilter[]) { + return filters.flatMap((filter) => { + if ('index' in filter) { + return filter.index.flat(); + } + return []; + }); +} diff --git a/x-pack/plugins/logsai/entities_api/jest.config.js b/x-pack/plugins/logsai/entities_api/jest.config.js new file mode 100644 index 0000000000000..0e49642be1469 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/jest.config.js @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../../..', + roots: [ + '/x-pack/plugins/logsai/entities_api/public', + '/x-pack/plugins/logsai/entities_api/common', + '/x-pack/plugins/logsai/entities_api/server', + ], + setupFiles: [], + collectCoverage: true, + collectCoverageFrom: [ + '/x-pack/plugins/logsai/entities_api/{public,common,server}/**/*.{js,ts,tsx}', + ], + + coverageReporters: ['html'], +}; diff --git a/x-pack/plugins/logsai/entities_api/kibana.jsonc b/x-pack/plugins/logsai/entities_api/kibana.jsonc new file mode 100644 index 0000000000000..285bd1473b293 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/kibana.jsonc @@ -0,0 +1,28 @@ +{ + "type": "plugin", + "id": "@kbn/entities-api-plugin", + "owner": "@elastic/observability-ui", + "plugin": { + "id": "entitiesAPI", + "server": true, + "browser": true, + "configPath": ["xpack", "entitiesAPI"], + "requiredPlugins": [ + "observabilityShared", + "inference", + "dataViews", + "data", + "unifiedSearch", + "datasetQuality", + "share", + "alerting", + "ruleRegistry", + "slo", + "spaces" + ], + "requiredBundles": [ + ], + "optionalPlugins": [], + "extraPublicDirs": [] + } +} diff --git a/x-pack/plugins/logsai/entities_api/public/api/index.tsx b/x-pack/plugins/logsai/entities_api/public/api/index.tsx new file mode 100644 index 0000000000000..995a4ca0b0b80 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/public/api/index.tsx @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { CoreSetup, CoreStart, HttpFetchOptions } from '@kbn/core/public'; +import type { + ClientRequestParamsOf, + ReturnOf, + RouteRepositoryClient, +} from '@kbn/server-route-repository'; +import { createRepositoryClient } from '@kbn/server-route-repository-client'; +import type { EntitiesAPIServerRouteRepository as EntitiesAPIServerRouteRepository } from '../../server'; + +type FetchOptions = Omit & { + body?: any; +}; + +export type EntitiesAPIClientOptions = Omit< + FetchOptions, + 'query' | 'body' | 'pathname' | 'signal' +> & { + signal: AbortSignal | null; +}; + +export type EntitiesAPIClient = RouteRepositoryClient< + EntitiesAPIServerRouteRepository, + EntitiesAPIClientOptions +>; + +export type AutoAbortedEntitiesAPIClient = RouteRepositoryClient< + EntitiesAPIServerRouteRepository, + Omit +>; + +export type EntitiesAPIEndpoint = keyof EntitiesAPIServerRouteRepository; + +export type APIReturnType = ReturnOf< + EntitiesAPIServerRouteRepository, + TEndpoint +>; + +export type EntitiesAPIClientRequestParamsOf = + ClientRequestParamsOf; + +export function createEntitiesAPIClient(core: CoreStart | CoreSetup): EntitiesAPIClient { + return createRepositoryClient(core); +} diff --git a/x-pack/plugins/logsai/entities_api/public/index.ts b/x-pack/plugins/logsai/entities_api/public/index.ts new file mode 100644 index 0000000000000..87fd6b5519616 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/public/index.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { PluginInitializer, PluginInitializerContext } from '@kbn/core/public'; + +import { EntitiesAPIPlugin } from './plugin'; +import type { + EntitiesAPIPublicSetup, + EntitiesAPIPublicStart, + EntitiesAPISetupDependencies, + EntitiesAPIStartDependencies, + ConfigSchema, +} from './types'; + +export type { EntitiesAPIPublicSetup, EntitiesAPIPublicStart }; + +export const plugin: PluginInitializer< + EntitiesAPIPublicSetup, + EntitiesAPIPublicStart, + EntitiesAPISetupDependencies, + EntitiesAPIStartDependencies +> = (pluginInitializerContext: PluginInitializerContext) => + new EntitiesAPIPlugin(pluginInitializerContext); + +export { getIndexPatternsForFilters, entitySourceQuery } from '../common'; + +export type { Entity } from '../common'; diff --git a/x-pack/plugins/logsai/entities_api/public/plugin.ts b/x-pack/plugins/logsai/entities_api/public/plugin.ts new file mode 100644 index 0000000000000..a697c842986b3 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/public/plugin.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/public'; +import type { Logger } from '@kbn/logging'; +import { EntitiesAPIClient, createEntitiesAPIClient } from './api'; +import type { + ConfigSchema, + EntitiesAPIPublicSetup, + EntitiesAPIPublicStart, + EntitiesAPISetupDependencies, + EntitiesAPIStartDependencies, +} from './types'; + +export class EntitiesAPIPlugin + implements + Plugin< + EntitiesAPIPublicSetup, + EntitiesAPIPublicStart, + EntitiesAPISetupDependencies, + EntitiesAPIStartDependencies + > +{ + logger: Logger; + entitiesAPIClient!: EntitiesAPIClient; + + constructor(context: PluginInitializerContext) { + this.logger = context.logger.get(); + } + setup( + coreSetup: CoreSetup, + pluginsSetup: EntitiesAPISetupDependencies + ): EntitiesAPIPublicSetup { + const entitiesAPIClient = (this.entitiesAPIClient = createEntitiesAPIClient(coreSetup)); + + return { + entitiesAPIClient, + }; + } + + start(coreStart: CoreStart, pluginsStart: EntitiesAPIStartDependencies): EntitiesAPIPublicStart { + return { + entitiesAPIClient: this.entitiesAPIClient, + }; + } +} diff --git a/x-pack/plugins/logsai/entities_api/public/services/types.ts b/x-pack/plugins/logsai/entities_api/public/services/types.ts new file mode 100644 index 0000000000000..73ee15972873f --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/public/services/types.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EntitiesAPIClient } from '../api'; + +export interface EntitiesAPIServices { + entitiesAPIClient: EntitiesAPIClient; +} diff --git a/x-pack/plugins/logsai/entities_api/public/types.ts b/x-pack/plugins/logsai/entities_api/public/types.ts new file mode 100644 index 0000000000000..46c300ce6e3f4 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/public/types.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EntitiesAPIClient } from './api'; + +/* eslint-disable @typescript-eslint/no-empty-interface*/ + +export interface ConfigSchema {} + +export interface EntitiesAPISetupDependencies {} + +export interface EntitiesAPIStartDependencies {} + +export interface EntitiesAPIPublicSetup { + entitiesAPIClient: EntitiesAPIClient; +} + +export interface EntitiesAPIPublicStart { + entitiesAPIClient: EntitiesAPIClient; +} diff --git a/x-pack/plugins/logsai/entities_api/server/built_in_definitions_stub.ts b/x-pack/plugins/logsai/entities_api/server/built_in_definitions_stub.ts new file mode 100644 index 0000000000000..f3e58415b7d4a --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/server/built_in_definitions_stub.ts @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DefinitionEntity, EntityTypeDefinition } from '../common/entities'; + +export const allDataStreamsEntity: DefinitionEntity = { + id: 'data_streams', + key: 'data_streams', + displayName: 'Data streams', + type: 'data_stream', + pivot: { + type: 'data_stream', + identityFields: ['data_stream.dataset', 'data_stream.type', 'data_stream.namespace'], + }, + filters: [ + { + index: ['logs-*', 'metrics-*', 'traces-*', '.data_streams'], + }, + ], +}; + +export const allLogsEntity: DefinitionEntity = { + id: 'all_logs', + key: 'all_logs', + displayName: 'logs-*', + type: 'data_stream', + pivot: { + type: 'data_stream', + identityFields: ['data_stream.dataset', 'data_stream.type', 'data_stream.namespace'], + }, + filters: [ + { + index: ['logs-*', '.data_streams'], + }, + { + term: { + 'data_stream.type': 'logs', + }, + }, + ], +}; + +export const allMetricsEntity: DefinitionEntity = { + id: 'all_metrics', + key: 'all_metrics', + displayName: 'metrics-*', + type: 'data_stream', + pivot: { + type: 'data_stream', + identityFields: ['data_stream.dataset', 'data_stream.type', 'data_stream.namespace'], + }, + filters: [ + { + index: ['metrics-*', '.data_streams'], + }, + { + term: { + 'data_stream.type': 'metrics', + }, + }, + ], +}; + +// export const allLogsEntity: DefinitionEntity = { +// id: 'all_logs', +// displayName: 'All logs', +// type: 'data_stream', +// pivot: { +// identityFields: ['data_stream.dataset', 'data_stream.type', 'data_stream.namespace'], +// }, +// filters: [ +// { +// index: ['logs-*'], +// }, +// ], +// }; + +export const builtinEntityDefinitions = [allDataStreamsEntity, allLogsEntity, allMetricsEntity]; + +const dataStreamTypeDefinition: EntityTypeDefinition = { + displayName: 'Data streams', + displayNameTemplate: { + concat: [ + { field: 'data_stream.type' }, + { literal: '-' }, + { field: 'data_stream.dataset' }, + { literal: '-' }, + { field: 'data_stream.namespace' }, + ], + }, + pivot: { + type: 'data_stream', + identityFields: ['data_stream.type', 'data_stream.dataset', 'data_stream.namespace'], + }, +}; + +export const builtinTypeDefinitions = [dataStreamTypeDefinition]; diff --git a/x-pack/plugins/logsai/entities_api/server/config.ts b/x-pack/plugins/logsai/entities_api/server/config.ts new file mode 100644 index 0000000000000..4e943ec83302d --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/server/config.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema, type TypeOf } from '@kbn/config-schema'; + +export const config = schema.object({ + enabled: schema.boolean({ defaultValue: true }), +}); + +export type EntitiesAPIConfig = TypeOf; diff --git a/x-pack/plugins/logsai/entities_api/server/index.ts b/x-pack/plugins/logsai/entities_api/server/index.ts new file mode 100644 index 0000000000000..37a96a0f8ec9e --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/server/index.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { + PluginConfigDescriptor, + PluginInitializer, + PluginInitializerContext, +} from '@kbn/core/server'; +import type { EntitiesAPIConfig } from './config'; +import type { + EntitiesAPIServerStart, + EntitiesAPIServerSetup, + EntitiesAPISetupDependencies, + EntitiesAPIStartDependencies, +} from './types'; + +export type { EntitiesAPIServerRouteRepository } from './routes/get_global_entities_api_route_repository'; + +export type { EntitiesAPIServerSetup, EntitiesAPIServerStart }; + +import { config as configSchema } from './config'; +import { EntitiesAPIPlugin } from './plugin'; + +export const config: PluginConfigDescriptor = { + schema: configSchema, +}; + +export const plugin: PluginInitializer< + EntitiesAPIServerSetup, + EntitiesAPIServerStart, + EntitiesAPISetupDependencies, + EntitiesAPIStartDependencies +> = async (pluginInitializerContext: PluginInitializerContext) => + new EntitiesAPIPlugin(pluginInitializerContext); diff --git a/x-pack/plugins/logsai/entities_api/server/lib/clients/create_alerts_client.ts b/x-pack/plugins/logsai/entities_api/server/lib/clients/create_alerts_client.ts new file mode 100644 index 0000000000000..f8cf9f32b9933 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/server/lib/clients/create_alerts_client.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AlertsClient } from '@kbn/rule-registry-plugin/server'; +import type { EntitiesAPIRouteHandlerResources } from '../../routes/types'; + +export async function createAlertsClient( + resources: Pick +): Promise { + return await resources.plugins.ruleRegistry + .start() + .then((ruleRegistry) => ruleRegistry.getRacClientWithRequest(resources.request)); +} diff --git a/x-pack/plugins/logsai/entities_api/server/lib/clients/create_dashboards_client.ts b/x-pack/plugins/logsai/entities_api/server/lib/clients/create_dashboards_client.ts new file mode 100644 index 0000000000000..e94efb1d07244 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/server/lib/clients/create_dashboards_client.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObject, SavedObjectsFindResult } from '@kbn/core/server'; +import type { DashboardAttributes } from '@kbn/dashboard-plugin/common'; +import { EntitiesAPIRouteHandlerResources } from '../../routes/types'; + +interface DashboardsClient { + getAllDashboards: () => Promise>>; + getDashboardsById: (ids: string[]) => Promise>>; +} + +export async function createDashboardsClient( + resources: Pick +): Promise { + const soClient = (await resources.context.core).savedObjects.client; + async function getDashboards( + page: number + ): Promise>> { + const perPage = 1000; + + const response = await soClient.find({ + type: 'dashboard', + perPage, + page, + }); + + const fetchedUntil = (page - 1) * perPage + response.saved_objects.length; + + if (response.total <= fetchedUntil) { + return response.saved_objects; + } + return [...response.saved_objects, ...(await getDashboards(page + 1))]; + } + + return { + getDashboardsById: async (ids: string[]) => { + const response = await soClient.bulkGet(ids.map((id) => ({ type: 'dashboard', id }))); + + return response.saved_objects as Array>; + }, + getAllDashboards: async () => { + return await getDashboards(1); + }, + }; +} diff --git a/x-pack/plugins/logsai/entities_api/server/lib/clients/create_entities_api_es_client.ts b/x-pack/plugins/logsai/entities_api/server/lib/clients/create_entities_api_es_client.ts new file mode 100644 index 0000000000000..321c25c5d2214 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/server/lib/clients/create_entities_api_es_client.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createObservabilityEsClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; +import { EntitiesAPIRouteHandlerResources } from '../../routes/types'; + +export async function createEntitiesAPIEsClient({ + context, + logger, +}: Pick) { + const esClient = createObservabilityEsClient({ + client: (await context.core).elasticsearch.client.asCurrentUser, + logger, + plugin: 'entitiesApi', + }); + + return esClient; +} diff --git a/x-pack/plugins/logsai/entities_api/server/lib/clients/create_entity_client.ts b/x-pack/plugins/logsai/entities_api/server/lib/clients/create_entity_client.ts new file mode 100644 index 0000000000000..b7997227a9270 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/server/lib/clients/create_entity_client.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EntityClient } from '@kbn/entityManager-plugin/server/lib/entity_client'; +import type { EntitiesAPIRouteHandlerResources } from '../../routes/types'; + +export async function createEntityClient({ + plugins, + request, +}: Pick): Promise { + return await plugins.entityManager + .start() + .then((entityManagerStart) => entityManagerStart.getScopedClient({ request })); +} diff --git a/x-pack/plugins/logsai/entities_api/server/lib/clients/create_rules_client.ts b/x-pack/plugins/logsai/entities_api/server/lib/clients/create_rules_client.ts new file mode 100644 index 0000000000000..651f0a9f48ebb --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/server/lib/clients/create_rules_client.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RulesClient } from '@kbn/alerting-plugin/server'; +import type { EntitiesAPIRouteHandlerResources } from '../../routes/types'; + +export async function createRulesClient( + resources: Pick +): Promise { + return await resources.plugins.alerting + .start() + .then((alertingStart) => alertingStart.getRulesClientWithRequest(resources.request)); +} diff --git a/x-pack/plugins/logsai/entities_api/server/lib/clients/create_slo_client.ts b/x-pack/plugins/logsai/entities_api/server/lib/clients/create_slo_client.ts new file mode 100644 index 0000000000000..66f276e93292e --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/server/lib/clients/create_slo_client.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SloClient } from '@kbn/slo-plugin/server'; +import type { EntitiesAPIRouteHandlerResources } from '../../routes/types'; + +export async function createSloClient( + resources: Pick +): Promise { + return await resources.plugins.slo + .start() + .then((sloStart) => sloStart.getSloClientWithRequest(resources.request)); +} diff --git a/x-pack/plugins/logsai/entities_api/server/lib/entities/entity_lookup_table.ts b/x-pack/plugins/logsai/entities_api/server/lib/entities/entity_lookup_table.ts new file mode 100644 index 0000000000000..8ac42a17567df --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/server/lib/entities/entity_lookup_table.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ESQLLookupTableColumns } from '@kbn/es-types/src/search'; +import { ValuesType } from 'utility-types'; + +export interface EntityLookupTable { + name: string; + joins: string[]; + columns: Record> & { + 'entity.id': { keyword: Array }; + }; +} diff --git a/x-pack/plugins/logsai/entities_api/server/lib/entities/get_data_streams_for_filter.ts b/x-pack/plugins/logsai/entities_api/server/lib/entities/get_data_streams_for_filter.ts new file mode 100644 index 0000000000000..55a7670885c4c --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/server/lib/entities/get_data_streams_for_filter.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; +import { excludeFrozenQuery } from '@kbn/observability-utils-common/es/queries/exclude_frozen_query'; +import { kqlQuery } from '@kbn/observability-utils-common/es/queries/kql_query'; +import { chunk, compact, uniqBy } from 'lodash'; +import pLimit from 'p-limit'; +import { rangeQuery } from '@kbn/observability-utils-common/es/queries/range_query'; +import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; + +export async function getDataStreamsForFilter({ + esClient, + kql, + indexPatterns, + dslFilter, + start, + end, +}: { + esClient: ObservabilityElasticsearchClient; + kql?: string; + dslFilter?: QueryDslQueryContainer[]; + indexPatterns: string[]; + start: number; + end: number; +}): Promise> { + const indicesResponse = await esClient.search('get_data_streams_for_entities', { + index: indexPatterns, + timeout: '1ms', + terminate_after: 1, + size: 0, + track_total_hits: false, + request_cache: false, + query: { + bool: { + filter: [ + ...excludeFrozenQuery(), + ...kqlQuery(kql), + ...rangeQuery(start, end), + ...(dslFilter ?? []), + ], + }, + }, + aggs: { + indices: { + terms: { + field: '_index', + size: 50000, + }, + }, + }, + }); + + const allIndicesChunks = chunk( + indicesResponse.aggregations?.indices.buckets.map(({ key }) => key as string) ?? [], + 25 + ); + + const limiter = pLimit(5); + + const allDataStreams = await Promise.all( + allIndicesChunks.map(async (allIndices) => { + return limiter(async () => { + const resolveIndicesResponse = await esClient.client.indices.resolveIndex({ + name: allIndices.join(','), + }); + + return compact( + resolveIndicesResponse.indices + .filter((index) => index.data_stream) + .map( + (index) => + (index.name.includes(':') ? index.name.split(':')[0] + ':' : '') + index.data_stream + ) + ).map((dataStream) => ({ name: dataStream })); + }); + }) + ); + + return uniqBy(allDataStreams.flat(), (ds) => ds.name); +} diff --git a/x-pack/plugins/logsai/entities_api/server/lib/entities/get_definition_entities.ts b/x-pack/plugins/logsai/entities_api/server/lib/entities/get_definition_entities.ts new file mode 100644 index 0000000000000..6fd0c45cebb4a --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/server/lib/entities/get_definition_entities.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; +import { builtinEntityDefinitions } from '../../built_in_definitions_stub'; +import { DefinitionEntity } from '../../../common/entities'; + +export async function getDefinitionEntities({ + esClient, +}: { + esClient: ObservabilityElasticsearchClient; +}): Promise { + return builtinEntityDefinitions; +} diff --git a/x-pack/plugins/logsai/entities_api/server/lib/entities/get_type_definitions.ts b/x-pack/plugins/logsai/entities_api/server/lib/entities/get_type_definitions.ts new file mode 100644 index 0000000000000..13825e2c35f15 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/server/lib/entities/get_type_definitions.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; +import { builtinTypeDefinitions } from '../../built_in_definitions_stub'; +import { EntityTypeDefinition } from '../../../common/entities'; + +export async function getTypeDefinitions({ + esClient, +}: { + esClient: ObservabilityElasticsearchClient; +}): Promise { + return builtinTypeDefinitions; +} diff --git a/x-pack/plugins/logsai/entities_api/server/lib/entities/query_signals_as_entities.ts b/x-pack/plugins/logsai/entities_api/server/lib/entities/query_signals_as_entities.ts new file mode 100644 index 0000000000000..f6003294b3ed7 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/server/lib/entities/query_signals_as_entities.ts @@ -0,0 +1,165 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import { + ALERT_STATUS, + ALERT_STATUS_ACTIVE, + ALERT_TIME_RANGE, + ALERT_UUID, + AlertConsumers, +} from '@kbn/rule-registry-plugin/common/technical_rule_data_field_names'; +import { SloClient } from '@kbn/slo-plugin/server'; +import { AlertsClient } from '@kbn/rule-registry-plugin/server'; +import { pick } from 'lodash'; +import { ValuesType } from 'utility-types'; +import { Logger } from '@kbn/logging'; +import { querySourcesAsEntities } from './query_sources_as_entities'; +import { + ENTITY_HEALTH_STATUS_INT, + EntityGrouping, + EntityTypeDefinition, + IEntity, +} from '../../../common/entities'; + +export async function querySignalsAsEntities({ + logger, + start, + end, + esClient, + groupings, + typeDefinitions, + filters, + sloClient, + alertsClient, +}: { + logger: Logger; + esClient: ObservabilityElasticsearchClient; + start: number; + end: number; + groupings: EntityGrouping[]; + typeDefinitions: EntityTypeDefinition[]; + filters?: QueryDslQueryContainer[]; + spaceId: string; + sloClient: SloClient; + alertsClient: AlertsClient; +}) { + const consumersOfInterest: string[] = Object.values(AlertConsumers).filter( + (consumer) => consumer !== AlertConsumers.SIEM && consumer !== AlertConsumers.EXAMPLE + ); + + const [sloSummaryDataScope, authorizedAlertsIndices] = await Promise.all([ + sloClient.getDataScopeForSummarySlos({ + start, + end, + }), + alertsClient.getAuthorizedAlertsIndices(consumersOfInterest), + ]); + + const [entitiesFromAlerts = [], entitiesFromSlos] = await Promise.all([ + authorizedAlertsIndices + ? querySourcesAsEntities({ + logger, + groupings, + typeDefinitions, + esClient, + sources: [{ index: authorizedAlertsIndices }], + filters: [ + ...(filters ?? [ + { + term: { + [ALERT_STATUS]: ALERT_STATUS_ACTIVE, + }, + }, + ]), + ], + rangeQuery: { + range: { + [ALERT_TIME_RANGE]: { + gte: start, + lte: end, + }, + }, + }, + columns: { + alertsCount: { + expression: `COUNT_DISTINCT(${ALERT_UUID})`, + }, + }, + sortField: 'alertsCount', + sortOrder: 'desc', + size: 10_000, + postFilter: `WHERE alertsCount > 0`, + }) + : undefined, + querySourcesAsEntities({ + logger, + groupings, + typeDefinitions, + esClient, + sources: [{ index: sloSummaryDataScope.index }], + filters: [...(filters ?? []), sloSummaryDataScope.query], + rangeQuery: sloSummaryDataScope.query, + columns: { + healthStatus: { + expression: `CASE( + COUNT(status == "VIOLATED" OR NULL) > 0, + ${ENTITY_HEALTH_STATUS_INT.Violated}, + COUNT(status == "DEGRADED" OR NULL) > 0, + ${ENTITY_HEALTH_STATUS_INT.Degraded}, + COUNT(status == "NO_DATA" OR NULL) > 0, + ${ENTITY_HEALTH_STATUS_INT.NoData}, + COUNT(status == "HEALTHY" OR NULL) > 0, + ${ENTITY_HEALTH_STATUS_INT.Healthy}, + NULL + )`, + }, + }, + }), + ]); + + const entitiesById = new Map< + string, + IEntity & { + healthStatus: ValuesType | null; + alertsCount: number; + } + >(); + + entitiesFromAlerts.forEach((entity) => { + const existing = entitiesById.get(entity.id); + const alertsCount = entity.columns.alertsCount as number; + if (existing) { + existing.alertsCount = alertsCount; + } else { + entitiesById.set(entity.id, { + ...pick(entity, 'id', 'key', 'type', 'displayName'), + alertsCount, + healthStatus: null, + }); + } + }); + + entitiesFromSlos.forEach((entity) => { + const existing = entitiesById.get(entity.id); + const healthStatus = entity.columns.healthStatus as ValuesType< + typeof ENTITY_HEALTH_STATUS_INT + > | null; + if (existing) { + existing.healthStatus = healthStatus; + } else { + entitiesById.set(entity.id, { + ...pick(entity, 'id', 'key', 'type', 'displayName'), + alertsCount: 0, + healthStatus, + }); + } + }); + + return Array.from(entitiesById.values()); +} diff --git a/x-pack/plugins/logsai/entities_api/server/lib/entities/query_sources_as_entities.ts b/x-pack/plugins/logsai/entities_api/server/lib/entities/query_sources_as_entities.ts new file mode 100644 index 0000000000000..1c3aefeea13b9 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/server/lib/entities/query_sources_as_entities.ts @@ -0,0 +1,309 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { Logger } from '@kbn/logging'; +import { ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; +import { isEmpty, omit, partition, pick, pickBy, uniq } from 'lodash'; +import { + DefinitionEntity, + EntityDataSource, + EntityDisplayNameTemplate, + EntityFilter, + EntityGrouping, + EntityTypeDefinition, + IEntity, +} from '../../../common/entities'; +import { escapeColumn, escapeString } from '../../../common/utils/esql_escape'; +import { esqlResultToPlainObjects } from '../../../common/utils/esql_result_to_plain_objects'; +import { getEsqlRequest } from '../../../common/utils/get_esql_request'; +import { + ENTITY_ID_SEPARATOR, + getEsqlIdentityCommands, +} from '../../routes/entities/get_esql_identity_commands'; +import { EntityLookupTable } from './entity_lookup_table'; + +interface EntityColumnMap { + [columnName: string]: + | { + expression: string; + } + | { + metadata: {}; + }; +} + +const MAX_NUMBER_OF_ENTITIES = 500; + +function getFieldsFromFilters(filters: EntityFilter[]) { + return filters.flatMap((filter) => { + if ('term' in filter) { + return [Object.keys(filter.term)[0]]; + } + return []; + }); +} + +function getLookupCommands(table: EntityLookupTable) { + const entityIdAlias = `${table.name}.entity.id`; + + const joinKeys = table.joins.map((joinField) => { + if (joinField === 'entity.id') { + return entityIdAlias; + } + return joinField; + }); + + return [ + `EVAL ${escapeColumn(entityIdAlias)} = entity.id`, + `LOOKUP ${table.name} ON ${joinKeys.map((key) => escapeColumn(key)).join(', ')}`, + ...(joinKeys.includes('entity.id') + ? [ + `EVAL entity.id = CASE( + entity.id IS NULL, ${escapeColumn(entityIdAlias)}, + ${escapeColumn(entityIdAlias)} IS NOT NULL, MV_APPEND(entity.id, ${escapeColumn( + entityIdAlias + )}), + NULL + )`, + `MV_EXPAND entity.id`, + ] + : []), + `DROP ${escapeColumn(entityIdAlias)}`, + ]; +} + +function getConcatExpressionFromDisplayNameTemplate( + displayNameTemplate: EntityDisplayNameTemplate +) { + return `CONCAT( + ${displayNameTemplate.concat + .map((part) => ('literal' in part ? escapeString(part.literal) : escapeColumn(part.field))) + .join(', ')} + )`; +} + +export async function querySourcesAsEntities< + TEntityColumnMap extends EntityColumnMap | undefined = undefined, + TLookupColumnName extends string = never +>({ + esClient, + logger, + sources, + typeDefinitions, + groupings, + columns, + rangeQuery, + filters, + postFilter, + sortField = 'entity.displayName', + sortOrder = 'asc', + size = MAX_NUMBER_OF_ENTITIES, + tables, +}: { + esClient: ObservabilityElasticsearchClient; + logger: Logger; + sources: EntityDataSource[]; + groupings: Array; + typeDefinitions: EntityTypeDefinition[]; + columns?: TEntityColumnMap; + rangeQuery: QueryDslQueryContainer; + filters?: QueryDslQueryContainer[]; + postFilter?: string; + sortField?: + | Exclude + | (keyof TEntityColumnMap & string) + | 'entity.type' + | 'entity.displayName'; + sortOrder?: 'asc' | 'desc'; + size?: number; + tables?: Array>; +}): Promise< + Array< + IEntity & { + columns: Record< + Exclude | (keyof TEntityColumnMap & string), + unknown + >; + } + > +> { + const indexPatterns = sources.flatMap((source) => source.index); + + const commands = [`FROM ${indexPatterns.join(',')} METADATA _index`]; + + const [lookupBeforeTables, lookupAfterTables] = partition( + tables, + (table) => !table.joins.includes('entity.id') + ); + + lookupBeforeTables.forEach((table) => { + commands.push(...getLookupCommands(table)); + }); + + const allGroupingFields = uniq(groupings.flatMap((grouping) => grouping.pivot.identityFields)); + + const fieldsToFilterOn = uniq( + groupings.flatMap((grouping) => getFieldsFromFilters(grouping.filters)) + ); + + const fieldCapsResponse = await esClient.fieldCaps( + 'check_column_availability_for_source_indices', + { + fields: [...allGroupingFields, ...fieldsToFilterOn, 'entity.displayName'], + index: indexPatterns, + index_filter: { + bool: { + filter: [rangeQuery], + }, + }, + } + ); + + const [validGroupings, invalidGroupings] = partition(groupings, (grouping) => { + const allFields = grouping.pivot.identityFields.concat(getFieldsFromFilters(grouping.filters)); + + return allFields.every((field) => !isEmpty(fieldCapsResponse.fields[field])); + }); + + if (invalidGroupings.length) { + logger.debug( + `Some groups were not applicable because not all fields are available: ${invalidGroupings + .map((grouping) => grouping.id) + .join(', ')}` + ); + } + + if (!validGroupings.length) { + logger.debug(`No valid groupings were applicable, returning no results`); + return []; + } + + const groupColumns = uniq([ + ...validGroupings.flatMap(({ pivot }) => pivot.identityFields), + ...lookupAfterTables.flatMap((table) => table.joins), + ]).filter((fieldName) => fieldName !== 'entity.id'); + + const hasEntityDisplayName = !isEmpty(fieldCapsResponse.fields['entity.displayName']); + + if (hasEntityDisplayName) { + commands.push(`EVAL entity.displayName = entity.displayName.keyword`); + } + + const metadataColumns = { + ...pickBy(columns, (column): column is { metadata: {} } => 'metadata' in column), + ...Object.fromEntries(fieldsToFilterOn.map((fieldName) => [fieldName, { metadata: {} }])), + ...(hasEntityDisplayName ? { 'entity.displayName': { metadata: {} } } : {}), + }; + + const expressionColumns = pickBy( + columns, + (column): column is { expression: string } => 'expression' in column + ); + + const columnStatements = Object.entries(metadataColumns) + .map(([fieldName]) => `${escapeColumn(fieldName)} = MAX(${escapeColumn(fieldName)})`) + .concat( + Object.entries(expressionColumns).map( + ([fieldName, { expression }]) => `${escapeColumn(fieldName)} = ${expression}` + ) + ); + + const columnsInFinalStatsBy = columnStatements.concat( + groupColumns.map((column) => { + return `${escapeColumn(column)} = MAX(${escapeColumn(column)})`; + }) + ); + + const identityCommands = getEsqlIdentityCommands({ + groupings, + columns: columnsInFinalStatsBy, + preaggregate: isEmpty(expressionColumns), + entityIdExists: lookupBeforeTables.length > 0, + }); + + commands.push(...identityCommands); + + lookupAfterTables.forEach((table) => { + commands.push(...getLookupCommands(table)); + }); + + typeDefinitions.forEach((typeDefinition) => { + if (typeDefinition.displayNameTemplate) { + commands.push(`EVAL entity.displayName = CASE( + entity.type == ${escapeString( + typeDefinition.pivot.type + )} AND ${typeDefinition.pivot.identityFields + .map((field) => `${escapeColumn(field)} IS NOT NULL`) + .join(' AND ')}, + ${getConcatExpressionFromDisplayNameTemplate(typeDefinition.displayNameTemplate)}, + entity.displayName + )`); + } + }); + + if (postFilter) { + commands.push(postFilter); + } + + const sortOrderUC = sortOrder.toUpperCase(); + + commands.push(`SORT \`${sortField}\` ${sortOrderUC} NULLS LAST`); + + commands.push(`LIMIT ${size}`); + + const request = { + ...getEsqlRequest({ + query: commands.join('\n| '), + dslFilter: [rangeQuery, ...(filters ?? [])], + }), + ...(tables?.length + ? { + tables: Object.fromEntries( + tables.map((table) => { + const entityIdColumn = table.columns['entity.id']; + return [ + table.name, + { + ...omit(table.columns, 'entity.id'), + [`${table.name}.entity.id`]: entityIdColumn, + }, + ]; + }) + ), + } + : {}), + }; + + const response = await esClient.esql('search_source_indices_for_entities', request); + + // should actually be a lookup to properly sort, but multiple LOOKUPs break + const groupingsByEntityId = new Map( + groupings.map((grouping) => { + const parts = [grouping.pivot.type, ENTITY_ID_SEPARATOR, grouping.id]; + const id = parts.join(''); + return [id, grouping]; + }) + ); + + return esqlResultToPlainObjects(response).map((row) => { + const columnValues = omit(row, 'entity.id', 'entity.displayName', 'entity.type'); + + const entityId = row['entity.id']; + + const grouping = groupingsByEntityId.get(entityId); + + return { + id: entityId, + type: row['entity.type'], + key: row['entity.key'], + displayName: row['entity.displayName'], + ...pick(grouping, 'displayName', 'type', 'key'), + columns: columnValues as Record, + }; + }); +} diff --git a/x-pack/plugins/logsai/entities_api/server/lib/with_entities_api_span.ts b/x-pack/plugins/logsai/entities_api/server/lib/with_entities_api_span.ts new file mode 100644 index 0000000000000..d5f9b4e92efb1 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/server/lib/with_entities_api_span.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { withSpan, SpanOptions, parseSpanOptions } from '@kbn/apm-utils'; +import { Logger } from '@kbn/logging'; + +export function withEntitiesAPISpan( + optionsOrName: SpanOptions | string, + cb: () => Promise, + logger: Logger +): Promise { + const options = parseSpanOptions(optionsOrName); + + const optionsWithDefaults = { + ...(options.intercept ? {} : { type: 'plugin:entitiesApi' }), + ...options, + labels: { + plugin: 'entitiesApi', + ...options.labels, + }, + }; + + return withSpan(optionsWithDefaults, cb, logger); +} diff --git a/x-pack/plugins/logsai/entities_api/server/plugin.ts b/x-pack/plugins/logsai/entities_api/server/plugin.ts new file mode 100644 index 0000000000000..f407d22e81605 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/server/plugin.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/server'; +import type { Logger } from '@kbn/logging'; +import { mapValues } from 'lodash'; +import { registerServerRoutes } from './routes/register_routes'; +import { EntitiesAPIRouteHandlerResources } from './routes/types'; +import type { + ConfigSchema, + EntitiesAPIServerSetup, + EntitiesAPIServerStart, + EntitiesAPISetupDependencies, + EntitiesAPIStartDependencies, +} from './types'; + +export class EntitiesAPIPlugin + implements + Plugin< + EntitiesAPIServerSetup, + EntitiesAPIServerStart, + EntitiesAPISetupDependencies, + EntitiesAPIStartDependencies + > +{ + logger: Logger; + + constructor(context: PluginInitializerContext) { + this.logger = context.logger.get(); + } + setup( + coreSetup: CoreSetup, + pluginsSetup: EntitiesAPISetupDependencies + ): EntitiesAPIServerSetup { + const startServicesPromise = coreSetup + .getStartServices() + .then(([_coreStart, pluginsStart]) => pluginsStart); + + registerServerRoutes({ + core: coreSetup, + logger: this.logger, + dependencies: { + plugins: mapValues(pluginsSetup, (value, key) => { + return { + start: () => + startServicesPromise.then( + (startServices) => startServices[key as keyof typeof startServices] + ), + setup: () => value, + }; + }) as unknown as EntitiesAPIRouteHandlerResources['plugins'], + }, + }); + return {}; + } + + start(core: CoreStart, pluginsStart: EntitiesAPIStartDependencies): EntitiesAPIServerStart { + return {}; + } +} diff --git a/x-pack/plugins/logsai/entities_api/server/routes/create_entities_api_server_route.ts b/x-pack/plugins/logsai/entities_api/server/routes/create_entities_api_server_route.ts new file mode 100644 index 0000000000000..0b76e703d0531 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/server/routes/create_entities_api_server_route.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { createServerRouteFactory } from '@kbn/server-route-repository'; +import type { EntitiesAPIRouteCreateOptions, EntitiesAPIRouteHandlerResources } from './types'; + +export const createEntitiesAPIServerRoute = createServerRouteFactory< + EntitiesAPIRouteHandlerResources, + EntitiesAPIRouteCreateOptions +>(); diff --git a/x-pack/plugins/logsai/entities_api/server/routes/entities/get_entities.ts b/x-pack/plugins/logsai/entities_api/server/routes/entities/get_entities.ts new file mode 100644 index 0000000000000..47f103a481446 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/server/routes/entities/get_entities.ts @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types'; +import type { Logger } from '@kbn/logging'; +import type { ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; +import type { AlertsClient } from '@kbn/rule-registry-plugin/server'; +import { SloClient } from '@kbn/slo-plugin/server'; +import type { + DefinitionEntity, + EntityDataSource, + EntityTypeDefinition, + EntityWithSignalStatus, +} from '../../../common/entities'; +import { EntityGrouping, healthStatusIntToKeyword } from '../../../common/entities'; +import { querySignalsAsEntities } from '../../lib/entities/query_signals_as_entities'; +import { querySourcesAsEntities } from '../../lib/entities/query_sources_as_entities'; +import { withEntitiesAPISpan } from '../../lib/with_entities_api_span'; + +export async function getEntities({ + currentUserEsClient, + internalUserEsClient, + start, + end, + sourceRangeQuery, + groupings, + typeDefinitions, + sources, + logger, + filters, + alertsClient, + sloClient, + sortField, + sortOrder, + postFilter, + spaceId, +}: { + currentUserEsClient: ObservabilityElasticsearchClient; + internalUserEsClient: ObservabilityElasticsearchClient; + start: number; + end: number; + sourceRangeQuery?: QueryDslQueryContainer; + sources: EntityDataSource[]; + groupings: Array; + typeDefinitions: EntityTypeDefinition[]; + logger: Logger; + filters?: QueryDslQueryContainer[]; + alertsClient: AlertsClient; + sloClient: SloClient; + sortField: 'entity.type' | 'entity.displayName' | 'healthStatus' | 'alertsCount'; + sortOrder: 'asc' | 'desc'; + postFilter?: string; + spaceId: string; +}): Promise { + if (!groupings.length) { + throw new Error('No groupings were defined'); + } + + return withEntitiesAPISpan( + 'get_latest_entities', + async () => { + const entitiesWithSignals = await querySignalsAsEntities({ + logger, + start, + end, + spaceId, + alertsClient, + esClient: internalUserEsClient, + groupings, + typeDefinitions, + sloClient, + filters, + }); + + const entitiesFromSources = await querySourcesAsEntities({ + logger, + esClient: currentUserEsClient, + groupings, + typeDefinitions, + sources, + filters, + sortField, + sortOrder, + postFilter, + tables: [ + { + name: 'signals', + joins: ['entity.id'], + columns: entitiesWithSignals.reduce( + (prev, current) => { + prev['entity.id'].keyword.push(current.id); + prev.alertsCount.long.push(current.alertsCount); + prev.healthStatus.long.push(current.healthStatus); + return prev; + }, + { + 'entity.id': { keyword: [] as string[] }, + alertsCount: { + long: [] as Array, + }, + healthStatus: { long: [] as Array }, + } + ), + }, + ], + rangeQuery: sourceRangeQuery ?? { + range: { + '@timestamp': { + gte: start, + lte: end, + }, + }, + }, + }); + + return entitiesFromSources.map((entity) => { + const { columns, ...base } = entity; + + return { + ...base, + healthStatus: + entity.columns.healthStatus !== null + ? healthStatusIntToKeyword(entity.columns.healthStatus as 1 | 2 | 3 | 4) + : null, + alertsCount: entity.columns.alertsCount as number, + }; + }); + }, + logger + ); +} diff --git a/x-pack/plugins/logsai/entities_api/server/routes/entities/get_entity_from_type_and_key.ts b/x-pack/plugins/logsai/entities_api/server/routes/entities/get_entity_from_type_and_key.ts new file mode 100644 index 0000000000000..ce016ecec9e9c --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/server/routes/entities/get_entity_from_type_and_key.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { notFound } from '@hapi/boom'; +import { ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; +import { pivotEntityFromTypeAndKey } from './get_esql_identity_commands'; +import { DefinitionEntity, Entity, EntityTypeDefinition } from '../../../common/entities'; + +export async function getEntityFromTypeAndKey({ + esClient, + type, + key, + definitionEntities, + typeDefinitions, +}: { + esClient: ObservabilityElasticsearchClient; + type: string; + key: string; + definitionEntities: DefinitionEntity[]; + typeDefinitions: EntityTypeDefinition[]; +}): Promise<{ + entity: Entity; + typeDefinition: EntityTypeDefinition; +}> { + const typeDefinition = typeDefinitions.find((typeDef) => typeDef.pivot.type === type); + + if (!typeDefinition) { + throw notFound(`Could not find type definition for type ${type}`); + } + + const definitionsForType = definitionEntities.filter((definition) => definition.type === type); + + if (!definitionsForType.length) { + throw notFound(`Could not find definition for type ${type}`); + } + + const entityAsDefinition = definitionsForType.find((definition) => definition.key === key); + + if (entityAsDefinition) { + return { + entity: entityAsDefinition, + typeDefinition, + }; + } + + return { + entity: pivotEntityFromTypeAndKey({ + type, + key, + identityFields: typeDefinition.pivot.identityFields, + displayNameTemplate: typeDefinition.displayNameTemplate, + }), + typeDefinition, + }; +} diff --git a/x-pack/plugins/logsai/entities_api/server/routes/entities/get_esql_identity_commands.ts b/x-pack/plugins/logsai/entities_api/server/routes/entities/get_esql_identity_commands.ts new file mode 100644 index 0000000000000..90ce040b98a7d --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/server/routes/entities/get_esql_identity_commands.ts @@ -0,0 +1,236 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { compact, uniq, uniqBy } from 'lodash'; +import { + EntityGrouping, + Pivot, + EntityFilter, + PivotEntity, + EntityTypeDefinition, + DefinitionEntity, + Entity, + IEntity, + EntityDisplayNameTemplate, +} from '../../../common/entities'; +import { escapeColumn, escapeString } from '../../../common/utils/esql_escape'; + +const ENTITY_MISSING_VALUE_STRING = '__EMPTY__'; +export const ENTITY_ID_SEPARATOR = '@'; +const ENTITY_KEYS_SEPARATOR = '/'; +const ENTITY_ID_LIST_SEPARATOR = ';'; + +export function entityFromIdentifiers({ + entity, + typeDefinition, + definitionEntities, +}: { + entity: IEntity; + typeDefinition: EntityTypeDefinition | undefined; + definitionEntities: Map; +}): Entity | undefined { + if (definitionEntities.has(entity.key)) { + return definitionEntities.get(entity.key)!; + } + + if (!typeDefinition) { + return undefined; + } + + const next = pivotEntityFromTypeAndKey({ + type: entity.type, + key: entity.key, + identityFields: typeDefinition.pivot.identityFields, + displayNameTemplate: typeDefinition.displayNameTemplate, + }); + + return next; +} + +export function pivotEntityFromTypeAndKey({ + type, + key, + identityFields, + displayNameTemplate, +}: { + type: string; + key: string; + identityFields: string[]; + displayNameTemplate: EntityDisplayNameTemplate | undefined; +}): PivotEntity { + const sortedIdentityFields = identityFields.concat().sort(); + + const keys = key.split(ENTITY_KEYS_SEPARATOR); + + const identity = Object.fromEntries( + keys.map((value, index) => { + return [sortedIdentityFields[index], value]; + }) + ); + + const id = `${type}${ENTITY_ID_SEPARATOR}${key}}`; + + let displayName = key; + + if (displayNameTemplate) { + displayName = displayNameTemplate.concat + .map((part) => { + return 'literal' in part ? part.literal : part.field; + }) + .join(''); + } + + return { + id, + type, + key, + identity, + displayName, + }; +} + +function joinArray(source: T[], separator: T): T[] { + return source.flatMap((value, index, array) => { + if (index === array.length - 1) { + return [value]; + } + return [value, separator]; + }); +} + +function getExpressionForPivot(pivot: Pivot) { + const sortedFields = pivot.identityFields.concat().sort(); + + const restArguments = joinArray( + sortedFields.map((field) => { + return escapeColumn(field); + }), + `"${ENTITY_KEYS_SEPARATOR}"` + ).join(', '); + + // CONCAT() + // host@foo + // data_stream@logs;kubernetes-container-logs;elastic-apps + + return `CONCAT("${pivot.type}", "${ENTITY_ID_SEPARATOR}", ${restArguments})`; +} + +function getExpressionForFilter(filter: EntityFilter) { + if ('term' in filter) { + const fieldName = Object.keys(filter.term)[0]; + return `${escapeColumn(fieldName)} == ${escapeString(filter.term[fieldName])}`; + } + if ('index' in filter) { + return `${filter.index + .flatMap((index) => [index, `.ds-${index}-*`]) + .map((index) => `(_index LIKE ${escapeString(`${index}`)})`) + .join(' OR ')}`; + } +} + +function getMatchesExpressionForGrouping(grouping: EntityGrouping) { + const applicableFilters = grouping.filters.filter((filter) => { + return 'term' in filter || 'index' in filter; + }); + + if (applicableFilters.length) { + return `${compact( + applicableFilters.map((filter) => `(${getExpressionForFilter(filter)})`) + ).join(' AND ')}`; + } + + return `true`; +} + +export function getEsqlIdentityCommands({ + groupings, + entityIdExists, + columns, + preaggregate, +}: { + groupings: EntityGrouping[]; + entityIdExists: boolean; + columns: string[]; + preaggregate: boolean; +}): string[] { + const pivotIdExpressions = uniqBy( + groupings.map((grouping) => grouping.pivot), + (pivot) => pivot.type + ).map(getExpressionForPivot); + + const filterClauses = groupings.map((grouping) => { + return { + fieldName: `_matches_group_${grouping.id}`, + type: grouping.pivot.type, + key: grouping.id, + expression: getMatchesExpressionForGrouping(grouping), + }; + }); + + const filterColumnEvals = filterClauses.map(({ fieldName, expression }) => { + return `${escapeColumn(fieldName)} = ${expression}`; + }); + + const filterStatsByColumns = filterClauses.map(({ fieldName }) => { + return `${escapeColumn(fieldName)} = MAX(${escapeColumn(fieldName)})`; + }); + + const filterIdExpressions = filterClauses.map(({ fieldName, type, key }) => { + return `CASE( + ${escapeColumn(fieldName)}, + CONCAT(${escapeString(type)}, "${ENTITY_ID_SEPARATOR}", "${key}"), + NULL + )`; + }); + + const groupingFields = uniq(groupings.flatMap((grouping) => grouping.pivot.identityFields)); + + const groupExpressions = joinArray( + [ + ...(entityIdExists ? [`entity.id`] : []), + ...pivotIdExpressions.concat(filterIdExpressions).map((expression) => { + return `COALESCE(${expression}, "${ENTITY_MISSING_VALUE_STRING}")`; + }), + ], + `"${ENTITY_ID_LIST_SEPARATOR}"` + ).join(', '); + + const entityIdExpression = + groupExpressions.length === 1 + ? groupExpressions[0] + : `MV_DEDUPE( + SPLIT( + CONCAT(${groupExpressions}), + "${ENTITY_ID_LIST_SEPARATOR}" + ) + )`; + + const commands: string[] = [`EVAL ${filterColumnEvals.join(', ')}`]; + + const allColumns = filterStatsByColumns.concat(columns); + + if (preaggregate) { + commands.push(`STATS ${allColumns.join(', ')} BY ${groupingFields.join(', ')}`); + } + + const entityDisplayNameExists = columns.find((column) => column.includes('entity.displayName')); + + return [ + ...commands, + `EVAL entity.id = ${entityIdExpression}`, + `STATS ${columns.join(', ')} BY entity.id`, + `MV_EXPAND entity.id`, + `WHERE entity.id != "${ENTITY_MISSING_VALUE_STRING}"`, + `EVAL entity_identifier = SPLIT(entity.id, "${ENTITY_ID_SEPARATOR}")`, + `EVAL entity.type = MV_FIRST(entity_identifier)`, + `EVAL entity.key = MV_LAST(entity_identifier)`, + entityDisplayNameExists + ? `EVAL entity.displayName = COALESCE(entity.displayName, entity.key)` + : `EVAL entity.displayName = entity.key`, + `DROP entity_identifier`, + ]; +} diff --git a/x-pack/plugins/logsai/entities_api/server/routes/entities/route.ts b/x-pack/plugins/logsai/entities_api/server/routes/entities/route.ts new file mode 100644 index 0000000000000..d4875d83264ba --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/server/routes/entities/route.ts @@ -0,0 +1,253 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { kqlQuery } from '@kbn/observability-utils-common/es/queries/kql_query'; +import { rangeQuery } from '@kbn/observability-utils-common/es/queries/range_query'; +import { createObservabilityEsClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; +import { z } from '@kbn/zod'; +import { + Entity, + entitySourceQuery, + EntityWithSignalStatus, + getIndexPatternsForFilters, +} from '../../../common'; +import { EntityTypeDefinition } from '../../../common/entities'; +import { entityTimeRangeQuery } from '../../../common/queries/entity_time_range_query'; +import { createAlertsClient } from '../../lib/clients/create_alerts_client'; +import { createEntitiesAPIEsClient } from '../../lib/clients/create_entities_api_es_client'; +import { createSloClient } from '../../lib/clients/create_slo_client'; +import { getDataStreamsForFilter } from '../../lib/entities/get_data_streams_for_filter'; +import { getDefinitionEntities } from '../../lib/entities/get_definition_entities'; +import { getTypeDefinitions } from '../../lib/entities/get_type_definitions'; +import { createEntitiesAPIServerRoute } from '../create_entities_api_server_route'; +import { getEntities } from './get_entities'; +import { getEntityFromTypeAndKey } from './get_entity_from_type_and_key'; + +export const findEntitiesRoute = createEntitiesAPIServerRoute({ + endpoint: 'POST /internal/entities_api/entities', + options: { + tags: ['access:entities'], + }, + params: z.object({ + body: z.object({ + start: z.number(), + end: z.number(), + kuery: z.string(), + types: z.array(z.string()), + sortField: z.union([ + z.literal('entity.type'), + z.literal('entity.displayName'), + z.literal('alertsCount'), + z.literal('healthStatus'), + ]), + sortOrder: z.union([z.literal('asc'), z.literal('desc')]), + }), + }), + handler: async (resources): Promise<{ entities: EntityWithSignalStatus[] }> => { + const { context, logger, request } = resources; + + const { + body: { start, end, kuery, types, sortField, sortOrder }, + } = resources.params; + + const [currentUserEsClient, internalUserEsClient, sloClient, alertsClient, spaceId] = + await Promise.all([ + createEntitiesAPIEsClient(resources), + createObservabilityEsClient({ + client: (await context.core).elasticsearch.client.asInternalUser, + logger, + plugin: 'entitiesApi', + }), + createSloClient(resources), + createAlertsClient(resources), + (await resources.plugins.spaces.start()).spacesService.getSpaceId(request), + ]); + + const filters = [...kqlQuery(kuery)]; + + const [definitionEntities, typeDefinitions] = await Promise.all([ + getDefinitionEntities({ + esClient: currentUserEsClient, + }), + getTypeDefinitions({ + esClient: currentUserEsClient, + }), + ]); + + const groupings = definitionEntities + .filter((definitionEntity) => { + return types.includes('all') || types.includes(definitionEntity.type); + }) + .map((definitionEntity) => { + return { + id: definitionEntity.id, + type: definitionEntity.type, + key: definitionEntity.key, + pivot: { + type: definitionEntity.type, + identityFields: definitionEntity.pivot.identityFields, + }, + displayName: definitionEntity.displayName, + filters: definitionEntity.filters, + }; + }); + + const entities = await getEntities({ + start, + end, + alertsClient, + currentUserEsClient, + typeDefinitions, + groupings, + internalUserEsClient, + logger, + sloClient, + sortField, + sortOrder, + sources: [{ index: ['.data_streams'] }], + sourceRangeQuery: { + bool: { + should: [ + { bool: { filter: entityTimeRangeQuery(start, end) } }, + { bool: { filter: rangeQuery(start, end) } }, + { + bool: { + must_not: [ + { + exists: { + field: 'entity.firstSeenTimestamp', + }, + }, + { + exists: { + field: '@timestamp', + }, + }, + ], + }, + }, + ], + minimum_should_match: 1, + }, + }, + spaceId, + filters, + postFilter: undefined, + }); + + return { + entities, + }; + }, +}); + +export const getEntityRoute = createEntitiesAPIServerRoute({ + endpoint: 'GET /internal/entities_api/entity/{type}/{key}', + options: { + tags: ['access:entities'], + }, + params: z.object({ + path: z.object({ + type: z.string(), + key: z.string(), + }), + }), + handler: async ( + resources + ): Promise<{ + entity: Entity; + typeDefinition: EntityTypeDefinition; + }> => { + const { + path: { type, key }, + } = resources.params; + + const esClient = await createEntitiesAPIEsClient(resources); + + const [definitionEntities, typeDefinitions] = await Promise.all([ + getDefinitionEntities({ + esClient, + }), + getTypeDefinitions({ + esClient, + }), + ]); + + return await getEntityFromTypeAndKey({ + esClient, + type, + key, + typeDefinitions, + definitionEntities, + }); + }, +}); + +export const getDataStreamsForEntityRoute = createEntitiesAPIServerRoute({ + endpoint: 'GET /internal/entities_api/entity/{type}/{key}/data_streams', + options: { + tags: ['access:entities'], + }, + params: z.object({ + path: z.object({ + type: z.string(), + key: z.string(), + }), + query: z.object({ + start: z.string(), + end: z.string(), + }), + }), + handler: async (resources): Promise<{ dataStreams: Array<{ name: string }> }> => { + const { + path: { type, key }, + query: { start: startAsString, end: endAsString }, + } = resources.params; + + const start = Number(startAsString); + const end = Number(endAsString); + + const esClient = await createEntitiesAPIEsClient(resources); + + const [definitionEntities, typeDefinitions] = await Promise.all([ + getDefinitionEntities({ + esClient, + }), + getTypeDefinitions({ + esClient, + }), + ]); + + const { entity } = await getEntityFromTypeAndKey({ + esClient, + type, + key, + definitionEntities, + typeDefinitions, + }); + + const foundDataStreams = await getDataStreamsForFilter({ + start, + end, + esClient, + dslFilter: entitySourceQuery({ entity }), + indexPatterns: definitionEntities.flatMap((definition) => + getIndexPatternsForFilters(definition.filters) + ), + }); + + return { + dataStreams: foundDataStreams, + }; + }, +}); + +export const entitiesRoutes = { + ...findEntitiesRoute, + ...getEntityRoute, + ...getDataStreamsForEntityRoute, +}; diff --git a/x-pack/plugins/logsai/entities_api/server/routes/esql/route.ts b/x-pack/plugins/logsai/entities_api/server/routes/esql/route.ts new file mode 100644 index 0000000000000..efc321b2092f8 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/server/routes/esql/route.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ESQLSearchResponse } from '@kbn/es-types'; +import * as t from 'io-ts'; +import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { getEsqlRequest } from '../../../common/utils/get_esql_request'; +import { createEntitiesAPIServerRoute } from '../create_entities_api_server_route'; +import { createEntitiesAPIEsClient } from '../../lib/clients/create_entities_api_es_client'; + +const queryEsqlRoute = createEntitiesAPIServerRoute({ + endpoint: 'POST /internal/entities_api/esql', + params: t.type({ + body: t.intersection([ + t.type({ + query: t.string, + kuery: t.string, + operationName: t.string, + }), + t.partial({ + start: t.number, + end: t.number, + timestampField: t.string, + dslFilter: t.array(t.record(t.string, t.any)), + }), + ]), + }), + options: { + tags: ['access:entities'], + }, + handler: async ({ context, logger, params }): Promise => { + const esClient = await createEntitiesAPIEsClient({ context, logger }); + + const { + body: { query, kuery, start, end, dslFilter, timestampField, operationName }, + } = params; + + const request = getEsqlRequest({ + query, + start, + end, + timestampField, + kuery, + dslFilter: dslFilter as QueryDslQueryContainer[], + }); + + return await esClient.esql(operationName, request); + }, +}); + +export const esqlRoutes = { + ...queryEsqlRoute, +}; diff --git a/x-pack/plugins/logsai/entities_api/server/routes/get_global_entities_api_route_repository.ts b/x-pack/plugins/logsai/entities_api/server/routes/get_global_entities_api_route_repository.ts new file mode 100644 index 0000000000000..3a7a26cb11f13 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/server/routes/get_global_entities_api_route_repository.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { entitiesRoutes } from './entities/route'; +import { esqlRoutes } from './esql/route'; +import { typesRoutes } from './types/route'; + +export function getGlobalEntitiesAPIServerRouteRepository() { + return { + ...entitiesRoutes, + ...typesRoutes, + ...esqlRoutes, + }; +} + +export type EntitiesAPIServerRouteRepository = ReturnType< + typeof getGlobalEntitiesAPIServerRouteRepository +>; diff --git a/x-pack/plugins/logsai/entities_api/server/routes/register_routes.ts b/x-pack/plugins/logsai/entities_api/server/routes/register_routes.ts new file mode 100644 index 0000000000000..b4aa6af182919 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/server/routes/register_routes.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { CoreSetup } from '@kbn/core/server'; +import type { Logger } from '@kbn/logging'; +import { registerRoutes } from '@kbn/server-route-repository'; +import { getGlobalEntitiesAPIServerRouteRepository } from './get_global_entities_api_route_repository'; +import type { EntitiesAPIRouteHandlerResources } from './types'; + +export function registerServerRoutes({ + core, + logger, + dependencies, +}: { + core: CoreSetup; + logger: Logger; + dependencies: Omit; +}) { + registerRoutes({ + core, + logger, + repository: getGlobalEntitiesAPIServerRouteRepository(), + dependencies, + }); +} diff --git a/x-pack/plugins/logsai/entities_api/server/routes/types.ts b/x-pack/plugins/logsai/entities_api/server/routes/types.ts new file mode 100644 index 0000000000000..aeeeaf8a0ff19 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/server/routes/types.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { CustomRequestHandlerContext, KibanaRequest } from '@kbn/core/server'; +import type { LicensingApiRequestHandlerContext } from '@kbn/licensing-plugin/server/types'; +import type { Logger } from '@kbn/logging'; +import type { EntitiesAPISetupDependencies, EntitiesAPIStartDependencies } from '../types'; + +export type EntitiesAPIRequestHandlerContext = CustomRequestHandlerContext<{ + licensing: Pick; +}>; + +export interface EntitiesAPIRouteHandlerResources { + request: KibanaRequest; + context: EntitiesAPIRequestHandlerContext; + logger: Logger; + plugins: { + [key in keyof EntitiesAPISetupDependencies]: { + setup: Required[key]; + }; + } & { + [key in keyof EntitiesAPIStartDependencies]: { + start: () => Promise[key]>; + }; + }; +} + +export interface EntitiesAPIRouteCreateOptions { + options: { + timeout?: { + idleSocket?: number; + }; + tags: Array<'access:entities'>; + }; +} diff --git a/x-pack/plugins/logsai/entities_api/server/routes/types/route.ts b/x-pack/plugins/logsai/entities_api/server/routes/types/route.ts new file mode 100644 index 0000000000000..84d6172ab7437 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/server/routes/types/route.ts @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; +import { notFound } from '@hapi/boom'; +import { createEntitiesAPIServerRoute } from '../create_entities_api_server_route'; +import { DefinitionEntity, EntityTypeDefinition } from '../../../common/entities'; +import { getTypeDefinitions } from '../../lib/entities/get_type_definitions'; +import { createEntitiesAPIEsClient } from '../../lib/clients/create_entities_api_es_client'; +import { getDefinitionEntities } from '../../lib/entities/get_definition_entities'; + +export const getAllTypeDefinitionsRoute = createEntitiesAPIServerRoute({ + endpoint: 'GET /internal/entities_api/types', + options: { + tags: ['access:entities'], + }, + handler: async ( + resources + ): Promise<{ + typeDefinitions: EntityTypeDefinition[]; + definitionEntities: DefinitionEntity[]; + }> => { + const esClient = await createEntitiesAPIEsClient(resources); + + const [typeDefinitions, definitionEntities] = await Promise.all([ + getTypeDefinitions({ + esClient, + }), + getDefinitionEntities({ + esClient, + }), + ]); + + return { + typeDefinitions, + definitionEntities, + }; + }, +}); + +export const getTypeDefinitionRoute = createEntitiesAPIServerRoute({ + endpoint: 'GET /internal/entities_api/types/{type}', + options: { + tags: ['access:entities'], + }, + params: z.object({ + path: z.object({ + type: z.string(), + }), + }), + handler: async ( + resources + ): Promise<{ typeDefinition: EntityTypeDefinition; definitionEntities: DefinitionEntity[] }> => { + const esClient = await createEntitiesAPIEsClient(resources); + + const { + path: { type }, + } = resources.params; + + const [typeDefinitions, definitionEntities] = await Promise.all([ + getTypeDefinitions({ + esClient, + }), + getDefinitionEntities({ + esClient, + }), + ]); + + const typeDefinition = typeDefinitions.find((definition) => definition.pivot.type === type); + + if (!typeDefinition) { + throw notFound(); + } + + const definitionEntitiesForType = definitionEntities.filter( + (definitionEntity) => definitionEntity.type === type + ); + + return { + typeDefinition, + definitionEntities: definitionEntitiesForType, + }; + }, +}); + +export const typesRoutes = { + ...getTypeDefinitionRoute, + ...getAllTypeDefinitionsRoute, +}; diff --git a/x-pack/plugins/logsai/entities_api/server/types.ts b/x-pack/plugins/logsai/entities_api/server/types.ts new file mode 100644 index 0000000000000..3628392ba5bd7 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/server/types.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { + EntityManagerServerPluginSetup, + EntityManagerServerPluginStart, +} from '@kbn/entityManager-plugin/server'; +import type { + RuleRegistryPluginSetupContract, + RuleRegistryPluginStartContract, +} from '@kbn/rule-registry-plugin/server'; +import type { + PluginSetupContract as AlertingPluginSetup, + PluginStartContract as AlertingPluginStart, +} from '@kbn/alerting-plugin/server'; +import type { SloPluginStart, SloPluginSetup } from '@kbn/slo-plugin/server'; +import type { SpacesPluginSetup, SpacesPluginStart } from '@kbn/spaces-plugin/server'; +/* eslint-disable @typescript-eslint/no-empty-interface*/ + +export interface ConfigSchema {} + +export interface EntitiesAPISetupDependencies { + entityManager: EntityManagerServerPluginSetup; + ruleRegistry: RuleRegistryPluginSetupContract; + alerting: AlertingPluginSetup; + slo: SloPluginSetup; + spaces: SpacesPluginSetup; +} + +export interface EntitiesAPIStartDependencies { + entityManager: EntityManagerServerPluginStart; + ruleRegistry: RuleRegistryPluginStartContract; + alerting: AlertingPluginStart; + slo: SloPluginStart; + spaces: SpacesPluginStart; +} + +export interface EntitiesAPIServerSetup {} + +export interface EntitiesAPIClient {} + +export interface EntitiesAPIServerStart {} diff --git a/x-pack/plugins/logsai/entities_api/tsconfig.json b/x-pack/plugins/logsai/entities_api/tsconfig.json new file mode 100644 index 0000000000000..6594bc7977fc6 --- /dev/null +++ b/x-pack/plugins/logsai/entities_api/tsconfig.json @@ -0,0 +1,36 @@ +{ + "extends": "../../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types" + }, + "include": [ + "../../../typings/**/*", + "common/**/*", + "public/**/*", + "typings/**/*", + "public/**/*.json", + "server/**/*", + ".storybook/**/*" + ], + "exclude": ["target/**/*", ".storybook/**/*.js"], + "kbn_references": [ + "@kbn/core", + "@kbn/server-route-repository", + "@kbn/server-route-repository-client", + "@kbn/logging", + "@kbn/config-schema", + "@kbn/rule-registry-plugin", + "@kbn/observability-utils-server", + "@kbn/dashboard-plugin", + "@kbn/alerting-plugin", + "@kbn/apm-utils", + "@kbn/slo-plugin", + "@kbn/licensing-plugin", + "@kbn/zod", + "@kbn/es-types", + "@kbn/observability-utils-common", + "@kbn/data-views-plugin", + "@kbn/entityManager-plugin", + "@kbn/spaces-plugin", + ] +} diff --git a/x-pack/plugins/logsai/entities_app/.storybook/get_mock_entities_app_context.tsx b/x-pack/plugins/logsai/entities_app/.storybook/get_mock_entities_app_context.tsx new file mode 100644 index 0000000000000..bdece7025e442 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/.storybook/get_mock_entities_app_context.tsx @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { coreMock } from '@kbn/core/public/mocks'; +import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; +import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public'; +import type { EntitiesAPIPublicStart } from '@kbn/entities-api-plugin/public'; +import type { ObservabilitySharedPluginStart } from '@kbn/observability-shared-plugin/public'; +import type { UnifiedSearchPublicPluginStart } from '@kbn/unified-search-plugin/public'; +import type { EntitiesAppKibanaContext } from '../public/hooks/use_kibana'; + +export function getMockEntitiesAppContext(): EntitiesAppKibanaContext { + const core = coreMock.createStart(); + + return { + core, + dependencies: { + start: { + observabilityShared: {} as unknown as ObservabilitySharedPluginStart, + dataViews: {} as unknown as DataViewsPublicPluginStart, + data: {} as unknown as DataPublicPluginStart, + unifiedSearch: {} as unknown as UnifiedSearchPublicPluginStart, + entitiesAPI: {} as unknown as EntitiesAPIPublicStart, + }, + }, + services: {}, + }; +} diff --git a/x-pack/plugins/logsai/entities_app/.storybook/jest_setup.js b/x-pack/plugins/logsai/entities_app/.storybook/jest_setup.js new file mode 100644 index 0000000000000..32071b8aa3f62 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/.storybook/jest_setup.js @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { setGlobalConfig } from '@storybook/testing-react'; +import * as globalStorybookConfig from './preview'; + +setGlobalConfig(globalStorybookConfig); diff --git a/x-pack/plugins/logsai/entities_app/.storybook/main.js b/x-pack/plugins/logsai/entities_app/.storybook/main.js new file mode 100644 index 0000000000000..86b48c32f103e --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/.storybook/main.js @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = require('@kbn/storybook').defaultConfig; diff --git a/x-pack/plugins/logsai/entities_app/.storybook/preview.js b/x-pack/plugins/logsai/entities_app/.storybook/preview.js new file mode 100644 index 0000000000000..c8155e9c3d92c --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/.storybook/preview.js @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiThemeProviderDecorator } from '@kbn/kibana-react-plugin/common'; +import * as jest from 'jest-mock'; + +window.jest = jest; + +export const decorators = [EuiThemeProviderDecorator]; diff --git a/x-pack/plugins/logsai/entities_app/.storybook/storybook_decorator.tsx b/x-pack/plugins/logsai/entities_app/.storybook/storybook_decorator.tsx new file mode 100644 index 0000000000000..b6bf80135cec0 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/.storybook/storybook_decorator.tsx @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React, { ComponentType, useMemo } from 'react'; +import { EntitiesAppContextProvider } from '../public/components/entities_app_context_provider'; +import { getMockEntitiesAppContext } from './get_mock_entities_app_context'; + +export function KibanaReactStorybookDecorator(Story: ComponentType) { + const context = useMemo(() => getMockEntitiesAppContext(), []); + return ( + + + + ); +} diff --git a/x-pack/plugins/logsai/entities_app/README.md b/x-pack/plugins/logsai/entities_app/README.md new file mode 100644 index 0000000000000..790e874891669 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/README.md @@ -0,0 +1,3 @@ +# Entities App + +Home of the Entities app plugin, which renders ... _entities_! diff --git a/x-pack/plugins/logsai/entities_app/jest.config.js b/x-pack/plugins/logsai/entities_app/jest.config.js new file mode 100644 index 0000000000000..87f38e5717347 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/jest.config.js @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../../..', + roots: [ + '/x-pack/plugins/logsai/entities_app/public', + '/x-pack/plugins/logsai/entities_app/common', + '/x-pack/plugins/logsai/entities_app/server', + ], + setupFiles: ['/x-pack/plugins/logsai/entities_app/.storybook/jest_setup.js'], + collectCoverage: true, + collectCoverageFrom: [ + '/x-pack/plugins/logsai/entities_app/{public,common,server}/**/*.{js,ts,tsx}', + ], + + coverageReporters: ['html'], +}; diff --git a/x-pack/plugins/logsai/entities_app/kibana.jsonc b/x-pack/plugins/logsai/entities_app/kibana.jsonc new file mode 100644 index 0000000000000..7e019c9ba0d53 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/kibana.jsonc @@ -0,0 +1,24 @@ +{ + "type": "plugin", + "id": "@kbn/entities-app-plugin", + "owner": "@elastic/observability-ui", + "plugin": { + "id": "entitiesApp", + "server": true, + "browser": true, + "configPath": ["xpack", "entitiesApp"], + "requiredPlugins": [ + "entitiesAPI", + "observabilityShared", + "data", + "dataViews", + "unifiedSearch" + ], + "requiredBundles": [ + "kibanaReact", + "esqlDataGrid" + ], + "optionalPlugins": [], + "extraPublicDirs": [] + } +} diff --git a/x-pack/plugins/logsai/entities_app/public/application.tsx b/x-pack/plugins/logsai/entities_app/public/application.tsx new file mode 100644 index 0000000000000..6d4f9be2eca43 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/application.tsx @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React from 'react'; +import ReactDOM from 'react-dom'; +import { APP_WRAPPER_CLASS, type AppMountParameters, type CoreStart } from '@kbn/core/public'; +import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render'; +import { css } from '@emotion/css'; +import type { EntitiesAppStartDependencies } from './types'; +import { EntitiesAppServices } from './services/types'; +import { AppRoot } from './components/app_root'; + +export const renderApp = ({ + coreStart, + pluginsStart, + services, + appMountParameters, +}: { + coreStart: CoreStart; + pluginsStart: EntitiesAppStartDependencies; + services: EntitiesAppServices; +} & { appMountParameters: AppMountParameters }) => { + const { element } = appMountParameters; + + const appWrapperClassName = css` + overflow: auto; + `; + const appWrapperElement = document.getElementsByClassName(APP_WRAPPER_CLASS)[1]; + appWrapperElement.classList.add(appWrapperClassName); + + ReactDOM.render( + + + , + element + ); + return () => { + ReactDOM.unmountComponentAtNode(element); + appWrapperElement.classList.remove(APP_WRAPPER_CLASS); + }; +}; diff --git a/x-pack/plugins/logsai/entities_app/public/components/all_entities_view/index.tsx b/x-pack/plugins/logsai/entities_app/public/components/all_entities_view/index.tsx new file mode 100644 index 0000000000000..ec6e24bf286bd --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/components/all_entities_view/index.tsx @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { EuiFlexGroup } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import React from 'react'; +import { EntityTable } from '../entity_table'; +import { EntitiesAppPageHeader } from '../entities_app_page_header'; +import { EntitiesAppPageHeaderTitle } from '../entities_app_page_header/entities_app_page_header_title'; + +export function AllEntitiesView() { + return ( + + + + + + + ); +} diff --git a/x-pack/plugins/logsai/entities_app/public/components/app_root/index.tsx b/x-pack/plugins/logsai/entities_app/public/components/app_root/index.tsx new file mode 100644 index 0000000000000..078311b9a54a0 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/components/app_root/index.tsx @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RedirectAppLinks } from '@kbn/shared-ux-link-redirect-app'; +import React from 'react'; +import { type AppMountParameters, type CoreStart } from '@kbn/core/public'; +import { RouteRenderer, RouterProvider } from '@kbn/typed-react-router-config'; +import { HeaderMenuPortal } from '@kbn/observability-shared-plugin/public'; +import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import { EntitiesAppContextProvider } from '../entities_app_context_provider'; +import { entitiesAppRouter } from '../../routes/config'; +import { EntitiesAppStartDependencies } from '../../types'; +import { EntitiesAppServices } from '../../services/types'; + +export function AppRoot({ + coreStart, + pluginsStart, + services, + appMountParameters, +}: { + coreStart: CoreStart; + pluginsStart: EntitiesAppStartDependencies; + services: EntitiesAppServices; +} & { appMountParameters: AppMountParameters }) { + const { history } = appMountParameters; + + const context = { + core: coreStart, + dependencies: { + start: pluginsStart, + }, + services, + }; + + return ( + + + + + + + + + ); +} + +export function EntitiesAppHeaderActionMenu({ + appMountParameters, +}: { + appMountParameters: AppMountParameters; +}) { + const { setHeaderActionMenu, theme$ } = appMountParameters; + + return ( + + + + <> + + + + ); +} diff --git a/x-pack/plugins/logsai/entities_app/public/components/data_stream_detail_view/index.tsx b/x-pack/plugins/logsai/entities_app/public/components/data_stream_detail_view/index.tsx new file mode 100644 index 0000000000000..456fc6de083ea --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/components/data_stream_detail_view/index.tsx @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React from 'react'; +import { i18n } from '@kbn/i18n'; +import { EntityDetailViewWithoutParams } from '../entity_detail_view'; +import { useEntitiesAppParams } from '../../hooks/use_entities_app_params'; + +export function DataStreamDetailView() { + const { + path: { key, tab }, + } = useEntitiesAppParams('/data_stream/{key}/{tab}'); + return ( + [ + { + name: 'parsing', + label: i18n.translate('xpack.entities.dataStreamDetailView.parsingTab', { + defaultMessage: 'Parsing', + }), + content: <>, + }, + ]} + /> + ); +} diff --git a/x-pack/plugins/logsai/entities_app/public/components/entities_app_context_provider/index.tsx b/x-pack/plugins/logsai/entities_app/public/components/entities_app_context_provider/index.tsx new file mode 100644 index 0000000000000..457d53c5cc4a8 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/components/entities_app_context_provider/index.tsx @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React, { useMemo } from 'react'; +import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; +import type { EntitiesAppKibanaContext } from '../../hooks/use_kibana'; + +export function EntitiesAppContextProvider({ + context, + children, +}: { + context: EntitiesAppKibanaContext; + children: React.ReactNode; +}) { + const servicesForContext = useMemo(() => { + const { core, ...services } = context; + return { + ...core, + ...services, + }; + }, [context]); + + return {children}; +} diff --git a/x-pack/plugins/logsai/entities_app/public/components/entities_app_page_header/entities_app_page_header_title.tsx b/x-pack/plugins/logsai/entities_app/public/components/entities_app_page_header/entities_app_page_header_title.tsx new file mode 100644 index 0000000000000..60241118cc6a5 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/components/entities_app_page_header/entities_app_page_header_title.tsx @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { EuiFlexGroup, EuiHorizontalRule, EuiTitle } from '@elastic/eui'; +import React from 'react'; + +export function EntitiesAppPageHeaderTitle({ + title, + children, +}: { + title: string; + children?: React.ReactNode; +}) { + return ( + + + +

{title}

+ + {children} + + + + ); +} diff --git a/x-pack/plugins/logsai/entities_app/public/components/entities_app_page_header/index.tsx b/x-pack/plugins/logsai/entities_app/public/components/entities_app_page_header/index.tsx new file mode 100644 index 0000000000000..c0de2c80c7ef7 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/components/entities_app_page_header/index.tsx @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { EuiFlexGroup, EuiPageHeader } from '@elastic/eui'; +import React from 'react'; + +export function EntitiesAppPageHeader({ children }: { children: React.ReactNode }) { + return ( + + {children} + + ); +} diff --git a/x-pack/plugins/logsai/entities_app/public/components/entities_app_page_template/index.tsx b/x-pack/plugins/logsai/entities_app/public/components/entities_app_page_template/index.tsx new file mode 100644 index 0000000000000..91daed3591b46 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/components/entities_app_page_template/index.tsx @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { EuiPanel } from '@elastic/eui'; +import { css } from '@emotion/css'; +import React from 'react'; +import { useKibana } from '../../hooks/use_kibana'; + +export function EntitiesAppPageTemplate({ children }: { children: React.ReactNode }) { + const { + dependencies: { + start: { observabilityShared }, + }, + } = useKibana(); + + const { PageTemplate } = observabilityShared.navigation; + + return ( + + + {children} + + + ); +} diff --git a/x-pack/plugins/logsai/entities_app/public/components/entities_app_router_breadcrumb/index.tsx b/x-pack/plugins/logsai/entities_app/public/components/entities_app_router_breadcrumb/index.tsx new file mode 100644 index 0000000000000..5c2a3b876c5dd --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/components/entities_app_router_breadcrumb/index.tsx @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createRouterBreadcrumbComponent } from '@kbn/typed-react-router-config'; +import type { EntitiesAppRoutes } from '../../routes/config'; + +export const EntitiesAppRouterBreadcrumb = createRouterBreadcrumbComponent(); diff --git a/x-pack/plugins/logsai/entities_app/public/components/entities_app_search_bar/index.tsx b/x-pack/plugins/logsai/entities_app/public/components/entities_app_search_bar/index.tsx new file mode 100644 index 0000000000000..a9ffda0a02fbc --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/components/entities_app_search_bar/index.tsx @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { css } from '@emotion/css'; +import type { TimeRange } from '@kbn/es-query'; +import { SearchBar } from '@kbn/unified-search-plugin/public'; +import React, { useMemo } from 'react'; +import type { DataView } from '@kbn/data-views-plugin/common'; +import { useKibana } from '../../hooks/use_kibana'; + +const parentClassName = css` + width: 100%; +`; + +interface Props { + query: string; + dateRangeFrom?: string; + dateRangeTo?: string; + onQueryChange: (payload: { dateRange?: TimeRange; query: string }) => void; + onQuerySubmit: (payload: { dateRange?: TimeRange; query: string }, isUpdate?: boolean) => void; + onRefresh?: Required>['onRefresh']; + placeholder?: string; + dataViews?: DataView[]; +} + +export function EntitiesAppSearchBar({ + dateRangeFrom, + dateRangeTo, + onQueryChange, + onQuerySubmit, + onRefresh, + query, + placeholder, + dataViews, +}: Props) { + const { + dependencies: { + start: { unifiedSearch }, + }, + } = useKibana(); + + const queryObj = useMemo(() => ({ query, language: 'kuery' }), [query]); + + return ( +
+ { + onQuerySubmit({ dateRange, query: (nextQuery?.query as string | undefined) ?? '' }); + }} + onQueryChange={({ dateRange, query: nextQuery }) => { + onQueryChange({ dateRange, query: (nextQuery?.query as string | undefined) ?? '' }); + }} + query={queryObj} + showQueryInput + showFilterBar={false} + showQueryMenu={false} + showDatePicker={Boolean(dateRangeFrom && dateRangeTo)} + showSubmitButton={true} + dateRangeFrom={dateRangeFrom} + dateRangeTo={dateRangeTo} + onRefresh={onRefresh} + displayStyle="inPage" + disableQueryLanguageSwitcher + placeholder={placeholder} + indexPatterns={dataViews} + /> +
+ ); +} diff --git a/x-pack/plugins/logsai/entities_app/public/components/entity_detail_overview/index.tsx b/x-pack/plugins/logsai/entities_app/public/components/entity_detail_overview/index.tsx new file mode 100644 index 0000000000000..06913f6d6dc12 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/components/entity_detail_overview/index.tsx @@ -0,0 +1,292 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { + EuiBadge, + EuiFlexGroup, + EuiFlexItem, + EuiPanel, + EuiSuperSelect, + EuiText, + EuiTitle, +} from '@elastic/eui'; +import { css } from '@emotion/css'; +import { i18n } from '@kbn/i18n'; +import { useAbortableAsync } from '@kbn/observability-utils-browser/hooks/use_abortable_async'; +import { useDateRange } from '@kbn/observability-utils-browser/hooks/use_date_range'; +import { take, uniqueId } from 'lodash'; +import React, { useMemo, useState } from 'react'; +import { Entity, entitySourceQuery } from '@kbn/entities-api-plugin/public'; +import { EntityTypeDefinition } from '@kbn/entities-api-plugin/common/entities'; +import { useKibana } from '../../hooks/use_kibana'; +import { getInitialColumnsForLogs } from '../../util/get_initial_columns_for_logs'; +import { ControlledEsqlChart } from '../esql_chart/controlled_esql_chart'; +import { ControlledEsqlGrid } from '../esql_grid/controlled_esql_grid'; +import { EntitiesAppSearchBar } from '../entities_app_search_bar'; +import { useEsqlQueryResult } from '../../hooks/use_esql_query_result'; + +export function EntityDetailOverview({ + entity, + typeDefinition, + dataStreams, +}: { + entity: Entity; + typeDefinition: EntityTypeDefinition; + dataStreams: Array<{ name: string }>; +}) { + const { + dependencies: { + start: { + dataViews, + data, + entitiesAPI: { entitiesAPIClient }, + }, + }, + } = useKibana(); + + const { + timeRange, + absoluteTimeRange: { start, end }, + } = useDateRange({ data }); + + const [displayedKqlFilter, setDisplayedKqlFilter] = useState(''); + const [persistedKqlFilter, setPersistedKqlFilter] = useState(''); + + const [selectedDataStream, setSelectedDataStream] = useState(''); + + const queriedDataStreams = useMemo( + () => + selectedDataStream ? [selectedDataStream] : dataStreams.map((dataStream) => dataStream.name), + [selectedDataStream, dataStreams] + ); + + const queries = useMemo(() => { + if (!queriedDataStreams.length) { + return undefined; + } + + const baseDslFilter = entitySourceQuery({ + entity, + }); + + const indexPatterns = queriedDataStreams; + + const baseQuery = `FROM ${indexPatterns.join(', ')}`; + + const logsQuery = `${baseQuery} | LIMIT 100`; + + const histogramQuery = `${baseQuery} | STATS metric = COUNT(*) BY @timestamp = BUCKET(@timestamp, 1 minute)`; + + return { + logsQuery, + histogramQuery, + baseDslFilter: [...baseDslFilter], + }; + }, [queriedDataStreams, entity]); + + const logsQueryResult = useEsqlQueryResult({ + query: queries?.logsQuery, + start, + end, + kuery: persistedKqlFilter ?? '', + dslFilter: queries?.baseDslFilter, + operationName: 'get_logs_for_entity', + }); + + const histogramQueryFetch = useAbortableAsync( + async ({ signal }) => { + if (!queries?.histogramQuery) { + return undefined; + } + + return entitiesAPIClient.fetch('POST /internal/entities_api/esql', { + signal, + params: { + body: { + query: queries.histogramQuery, + kuery: persistedKqlFilter ?? '', + dslFilter: queries.baseDslFilter, + operationName: 'get_histogram_for_entity', + start, + end, + }, + }, + }); + }, + [ + queries?.histogramQuery, + persistedKqlFilter, + start, + end, + queries?.baseDslFilter, + entitiesAPIClient, + ] + ); + + const columnAnalysis = useMemo(() => { + if (logsQueryResult.value) { + return { + analysis: getInitialColumnsForLogs({ + response: logsQueryResult.value, + pivots: [typeDefinition.pivot], + }), + analysisId: uniqueId(), + }; + } + return undefined; + }, [logsQueryResult, typeDefinition]); + + const dataViewsFetch = useAbortableAsync(() => { + if (!queriedDataStreams.length) { + return Promise.resolve([]); + } + + return dataViews + .create( + { + title: queriedDataStreams.join(','), + timeFieldName: '@timestamp', + }, + false, // skip fetch fields + true // display errors + ) + .then((response) => { + return [response]; + }); + }, [dataViews, queriedDataStreams]); + + const fetchedDataViews = useMemo(() => dataViewsFetch.value ?? [], [dataViewsFetch.value]); + + return ( + <> + + + + { + setDisplayedKqlFilter(query); + }} + onQuerySubmit={() => { + setPersistedKqlFilter(displayedKqlFilter); + }} + onRefresh={() => { + logsQueryResult.refresh(); + histogramQueryFetch.refresh(); + }} + placeholder={i18n.translate( + 'xpack.entities.entityDetailOverview.searchBarPlaceholder', + { + defaultMessage: 'Filter data by using KQL', + } + )} + dataViews={fetchedDataViews} + dateRangeFrom={timeRange.from} + dateRangeTo={timeRange.to} + /> + + + ({ + value: dataStream.name, + inputDisplay: dataStream.name, + })) ?? []), + ]} + valueOfSelected={selectedDataStream} + onChange={(next) => { + setSelectedDataStream(next); + }} + /> + + + + + + + + + + {columnAnalysis?.analysis.constants.length ? ( + <> + + +

+ {i18n.translate('xpack.entities.entityDetailOverview.h3.constantsLabel', { + defaultMessage: 'Constants', + })} +

+ + + {take(columnAnalysis.analysis.constants, 10).map((constant) => ( + {`${constant.name}:${ + constant.value === '' || constant.value === 0 ? '(empty)' : constant.value + }`} + ))} + {columnAnalysis.analysis.constants.length > 10 ? ( + + {i18n.translate('xpack.entities.entityDetailOverview.moreTextLabel', { + defaultMessage: '{overflowCount} more', + values: { + overflowCount: columnAnalysis.analysis.constants.length - 20, + }, + })} + + ) : null} + + + + ) : null} + {queries?.logsQuery ? ( + + ) : null} + + + + + ); +} diff --git a/x-pack/plugins/logsai/entities_app/public/components/entity_detail_view/index.tsx b/x-pack/plugins/logsai/entities_app/public/components/entity_detail_view/index.tsx new file mode 100644 index 0000000000000..8d76b501cbc62 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/components/entity_detail_view/index.tsx @@ -0,0 +1,243 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { Entity } from '@kbn/entities-api-plugin/common'; +import React from 'react'; +import { i18n } from '@kbn/i18n'; +import { useDateRange } from '@kbn/observability-utils-browser/hooks/use_date_range'; +import { + EuiBadge, + EuiFlexGroup, + EuiFlexItem, + EuiHorizontalRule, + EuiPanel, + useEuiTheme, +} from '@elastic/eui'; +import { css } from '@emotion/css'; +import { useEntitiesAppParams } from '../../hooks/use_entities_app_params'; +import { useKibana } from '../../hooks/use_kibana'; +import { useEntitiesAppRouter } from '../../hooks/use_entities_app_router'; +import { useEntitiesAppFetch } from '../../hooks/use_entities_app_fetch'; +import { LoadingPanel } from '../loading_panel'; +import { useEntitiesAppBreadcrumbs } from '../../hooks/use_entities_app_breadcrumbs'; +import { EntitiesAppPageHeader } from '../entities_app_page_header'; +import { EntitiesAppPageHeaderTitle } from '../entities_app_page_header/entities_app_page_header_title'; +import { EntityDetailViewHeaderSection } from '../entity_detail_view_header_section'; +import { EntityOverviewTabList } from '../entity_overview_tab_list'; +import { EntityDetailOverview } from '../entity_detail_overview'; + +interface TabDependencies { + entity: Entity; + dataStreams: Array<{ name: string }>; +} + +interface Tab { + name: string; + label: string; + content: React.ReactElement; +} + +export function EntityDetailViewWithoutParams({ + tab, + entityKey: key, + type, + getAdditionalTabs, +}: { + tab: string; + entityKey: string; + type: string; + getAdditionalTabs?: (dependencies: TabDependencies) => Tab[]; +}) { + const { + dependencies: { + start: { + data, + entitiesAPI: { entitiesAPIClient }, + }, + }, + services: {}, + } = useKibana(); + + const { + absoluteTimeRange: { start, end }, + } = useDateRange({ data }); + + const router = useEntitiesAppRouter(); + + const theme = useEuiTheme().euiTheme; + + const entityFetch = useEntitiesAppFetch( + ({ signal }) => { + return entitiesAPIClient.fetch('GET /internal/entities_api/entity/{type}/{key}', { + signal, + params: { + path: { + type, + key: encodeURIComponent(key), + }, + }, + }); + }, + [type, key, entitiesAPIClient] + ); + + const typeDefinition = entityFetch.value?.typeDefinition; + + const entity = entityFetch.value?.entity; + + useEntitiesAppBreadcrumbs(() => { + if (!typeDefinition || !entity) { + return []; + } + + return [ + { + title: typeDefinition.displayName, + path: `/{type}`, + params: { path: { type } }, + } as const, + { + title: entity.displayName, + path: `/{type}/{key}`, + params: { path: { type, key } }, + } as const, + ]; + }, [type, key, entity?.displayName, typeDefinition]); + + const entityDataStreamsFetch = useEntitiesAppFetch( + async ({ signal }) => { + return entitiesAPIClient.fetch( + 'GET /internal/entities_api/entity/{type}/{key}/data_streams', + { + signal, + params: { + path: { + type, + key: encodeURIComponent(key), + }, + query: { + start: String(start), + end: String(end), + }, + }, + } + ); + }, + [key, type, entitiesAPIClient, start, end] + ); + + const dataStreams = entityDataStreamsFetch.value?.dataStreams; + + if (!entity || !typeDefinition || !dataStreams) { + return ; + } + + const tabs = { + overview: { + href: router.link('/{type}/{key}/{tab}', { + path: { type, key, tab: 'overview' }, + }), + label: i18n.translate('xpack.entities.entityDetailView.overviewTabLabel', { + defaultMessage: 'Overview', + }), + content: ( + + ), + }, + ...Object.fromEntries( + getAdditionalTabs?.({ + entity, + dataStreams, + }).map(({ name, ...rest }) => [ + name, + { + ...rest, + href: router.link(`/{type}/{key}/{tab}`, { + path: { + type, + key, + tab: name, + }, + }), + }, + ]) ?? [] + ), + }; + + const selectedTab = tabs[tab as keyof typeof tabs]; + + return ( + + + + + + div { + border: 0px solid ${theme.colors.lightShade}; + border-right-width: 1px; + width: 25%; + } + > div:last-child { + border-right-width: 0; + } + `} + > + + + + {type} + + + + + + + + { + return { + name: tabKey, + label, + href, + selected: tab === tabKey, + }; + })} + /> + {selectedTab.content} + + ); +} + +export function EntityDetailView() { + const { + path: { type, key, tab }, + } = useEntitiesAppParams('/{type}/{key}/{tab}'); + + return ; +} diff --git a/x-pack/plugins/logsai/entities_app/public/components/entity_detail_view_header_section/index.tsx b/x-pack/plugins/logsai/entities_app/public/components/entity_detail_view_header_section/index.tsx new file mode 100644 index 0000000000000..4aafb7a7e9bc6 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/components/entity_detail_view_header_section/index.tsx @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { EuiFlexGroup, EuiText } from '@elastic/eui'; +import { css } from '@emotion/css'; +import React from 'react'; + +export function EntityDetailViewHeaderSection({ + title, + children, +}: { + title: React.ReactNode; + children: React.ReactNode; +}) { + return ( + + + {title} + + {children} + + ); +} diff --git a/x-pack/plugins/logsai/entities_app/public/components/entity_health_status_badge/index.tsx b/x-pack/plugins/logsai/entities_app/public/components/entity_health_status_badge/index.tsx new file mode 100644 index 0000000000000..33fc375e63f87 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/components/entity_health_status_badge/index.tsx @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React from 'react'; +import { EntityHealthStatus } from '@kbn/entities-api-plugin/common'; +import { EuiBadge } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; + +export function EntityHealthStatusBadge({ + healthStatus, +}: { + healthStatus: EntityHealthStatus | null; +}) { + if (healthStatus === 'Violated') { + return ( + + {i18n.translate('xpack.entities.healthStatus.violatedBadgeLabel', { + defaultMessage: 'Violated', + })} + + ); + } + + if (healthStatus === 'Degraded') { + return ( + + {i18n.translate('xpack.entities.healthStatus.degradedBadgeLabel', { + defaultMessage: 'Degraded', + })} + + ); + } + + if (healthStatus === 'NoData') { + return ( + + {i18n.translate('xpack.entities.healthStatus.noDataBadgeLabel', { + defaultMessage: 'No data', + })} + + ); + } + + return ( + + {i18n.translate('xpack.entities.healthStatus.healthyBadgeLabel', { + defaultMessage: 'Healthy', + })} + + ); +} diff --git a/x-pack/plugins/logsai/entities_app/public/components/entity_overview_tab_list/index.tsx b/x-pack/plugins/logsai/entities_app/public/components/entity_overview_tab_list/index.tsx new file mode 100644 index 0000000000000..d0e4772d0993d --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/components/entity_overview_tab_list/index.tsx @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React from 'react'; +import { EuiTab, EuiTabs } from '@elastic/eui'; + +export function EntityOverviewTabList< + T extends { name: string; label: string; href: string; selected: boolean } +>({ tabs }: { tabs: T[] }) { + return ( + + {tabs.map((tab) => { + return ( + + {tab.label} + + ); + })} + + ); +} diff --git a/x-pack/plugins/logsai/entities_app/public/components/entity_pivot_type_view/index.tsx b/x-pack/plugins/logsai/entities_app/public/components/entity_pivot_type_view/index.tsx new file mode 100644 index 0000000000000..efb88a7b2813f --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/components/entity_pivot_type_view/index.tsx @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { EuiFlexGroup } from '@elastic/eui'; +import React from 'react'; +import { EntityTable } from '../entity_table'; +import { EntitiesAppPageHeader } from '../entities_app_page_header'; +import { EntitiesAppPageHeaderTitle } from '../entities_app_page_header/entities_app_page_header_title'; +import { useEntitiesAppParams } from '../../hooks/use_entities_app_params'; +import { useKibana } from '../../hooks/use_kibana'; +import { useEntitiesAppFetch } from '../../hooks/use_entities_app_fetch'; +import { useEntitiesAppBreadcrumbs } from '../../hooks/use_entities_app_breadcrumbs'; + +export function EntityPivotTypeView() { + const { + path: { type }, + } = useEntitiesAppParams('/{type}'); + + const { + dependencies: { + start: { + entitiesAPI: { entitiesAPIClient }, + }, + }, + } = useKibana(); + + const typeDefinitionsFetch = useEntitiesAppFetch( + ({ signal }) => { + return entitiesAPIClient.fetch('GET /internal/entities_api/types/{type}', { + signal, + params: { + path: { + type, + }, + }, + }); + }, + [entitiesAPIClient, type] + ); + + const typeDefinition = typeDefinitionsFetch.value?.typeDefinition; + + const title = typeDefinition?.displayName ?? ''; + + useEntitiesAppBreadcrumbs(() => { + if (!title) { + return []; + } + return [ + { + title, + path: `/{type}`, + params: { path: { type } }, + } as const, + ]; + }, [title, type]); + + return ( + + + + + + + ); +} diff --git a/x-pack/plugins/logsai/entities_app/public/components/entity_table/controlled_entity_table.tsx b/x-pack/plugins/logsai/entities_app/public/components/entity_table/controlled_entity_table.tsx new file mode 100644 index 0000000000000..4d407da954cc7 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/components/entity_table/controlled_entity_table.tsx @@ -0,0 +1,217 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { + CriteriaWithPagination, + EuiBadge, + EuiBasicTable, + EuiBasicTableColumn, + EuiFlexGroup, + EuiFlexItem, + EuiLink, + EuiSelect, +} from '@elastic/eui'; +import { css } from '@emotion/css'; +import type { TimeRange } from '@kbn/data-plugin/common'; +import type { DataView } from '@kbn/data-views-plugin/common'; +import { i18n } from '@kbn/i18n'; +import type { EntityWithSignalStatus } from '@kbn/entities-api-plugin/common'; +import React, { useMemo } from 'react'; +import { useEntitiesAppRouter } from '../../hooks/use_entities_app_router'; +import { EntitiesAppSearchBar } from '../entities_app_search_bar'; +import { EntityHealthStatusBadge } from '../entity_health_status_badge'; + +export function ControlledEntityTable({ + rows, + columns, + loading, + timeRange, + onTimeRangeChange, + kqlFilter, + onKqlFilterChange, + onKqlFilterSubmit, + pagination: { pageSize, pageIndex }, + onPaginationChange, + totalItemCount, + dataViews, + showTypeSelect, + selectedType, + availableTypes, + onSelectedTypeChange, + sort, + onSortChange, +}: { + rows: EntityWithSignalStatus[]; + columns: Array>; + kqlFilter: string; + timeRange: TimeRange; + onTimeRangeChange: (nextTimeRange: TimeRange) => void; + onKqlFilterChange: (nextKql: string) => void; + onKqlFilterSubmit: () => void; + loading: boolean; + pagination: { pageSize: number; pageIndex: number }; + onPaginationChange: (pagination: { pageSize: number; pageIndex: number }) => void; + totalItemCount: number; + dataViews?: DataView[]; + showTypeSelect?: boolean; + selectedType?: string; + onSelectedTypeChange?: (nextType: string) => void; + availableTypes?: Array<{ label: string; value: string }>; + sort?: { field: string; order: 'asc' | 'desc' }; + onSortChange?: (nextSort: { field: string; order: 'asc' | 'desc' }) => void; +}) { + const router = useEntitiesAppRouter(); + + const displayedColumns = useMemo>>(() => { + return [ + { + field: 'entity.type', + name: i18n.translate('xpack.entities.entityTable.typeColumnLabel', { + defaultMessage: 'Type', + }), + width: '96px', + render: (_, { type }) => { + return {type}; + }, + }, + { + field: 'entity.displayName', + name: i18n.translate('xpack.entities.entityTable.nameColumnLabel', { + defaultMessage: 'Name', + }), + sortable: true, + render: (_, { type, key, displayName }) => { + return ( + + {displayName} + + ); + }, + }, + { + field: 'slos', + name: i18n.translate('xpack.entities.entityTable.healthStatusColumnLabel', { + defaultMessage: 'Health status', + }), + sortable: true, + width: '96px', + render: (_, { healthStatus }) => { + if (healthStatus) { + return ; + } + + return <>; + }, + }, + { + field: 'alerts', + name: i18n.translate('xpack.entities.entityTable.alertsColumnLabel', { + defaultMessage: 'Alerts', + }), + sortable: true, + width: '96px', + render: (_, { alertsCount }) => { + if (!alertsCount) { + return <>; + } + return {alertsCount}; + }, + }, + ]; + }, [router]); + + const displayedRows = useMemo( + () => rows.slice(pageIndex * pageSize, pageIndex * pageSize + pageSize), + [rows, pageIndex, pageSize] + ); + + return ( + + + + { + onKqlFilterChange(query); + }} + onQuerySubmit={({ dateRange }) => { + onKqlFilterSubmit(); + if (dateRange) { + onTimeRangeChange(dateRange); + } + }} + placeholder={i18n.translate('xpack.entities.entityTable.filterEntitiesPlaceholder', { + defaultMessage: 'Filter entities', + })} + dateRangeFrom={timeRange.from} + dateRangeTo={timeRange.to} + dataViews={dataViews} + /> + + {showTypeSelect ? ( + + { + onSelectedTypeChange?.(event.currentTarget.value); + }} + isLoading={!availableTypes} + options={availableTypes?.map(({ value, label }) => ({ value, text: label }))} + /> + + ) : null} + + + columns={displayedColumns} + items={displayedRows} + itemId="name" + pagination={{ + pageSize, + pageIndex, + totalItemCount, + }} + sorting={ + sort + ? { + sort: { + direction: sort.order, + field: sort.field as any, + }, + } + : {} + } + loading={loading} + noItemsMessage={i18n.translate('xpack.entities.entityTable.noItemsMessage', { + defaultMessage: `No entities found`, + })} + onChange={(criteria: CriteriaWithPagination) => { + const { size, index } = criteria.page; + onPaginationChange({ pageIndex: index, pageSize: size }); + if (criteria.sort) { + onSortChange?.({ + field: criteria.sort.field, + order: criteria.sort.direction, + }); + } + }} + /> + + ); +} diff --git a/x-pack/plugins/logsai/entities_app/public/components/entity_table/index.tsx b/x-pack/plugins/logsai/entities_app/public/components/entity_table/index.tsx new file mode 100644 index 0000000000000..3bcafc336507e --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/components/entity_table/index.tsx @@ -0,0 +1,149 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { useDateRange } from '@kbn/observability-utils-browser/hooks/use_date_range'; +import React, { useMemo, useState } from 'react'; +import { useAbortableAsync } from '@kbn/observability-utils-browser/hooks/use_abortable_async'; +import { getIndexPatternsForFilters } from '@kbn/entities-api-plugin/public'; +import { DefinitionEntity } from '@kbn/entities-api-plugin/common/entities'; +import { useKibana } from '../../hooks/use_kibana'; +import { ControlledEntityTable } from './controlled_entity_table'; +import { useEntitiesAppFetch } from '../../hooks/use_entities_app_fetch'; + +export function EntityTable({ type }: { type: 'all' | string }) { + const { + dependencies: { + start: { + dataViews, + data, + entitiesAPI: { entitiesAPIClient }, + }, + }, + } = useKibana(); + + const { + timeRange, + setTimeRange, + absoluteTimeRange: { start, end }, + } = useDateRange({ data }); + + const [displayedKqlFilter, setDisplayedKqlFilter] = useState(''); + const [persistedKqlFilter, setPersistedKqlFilter] = useState(''); + + const [selectedType, setSelectedType] = useState(type); + + const [sort, setSort] = useState<{ field: string; order: 'asc' | 'desc' }>({ + field: 'entity.displayName', + order: 'desc', + }); + + const typeDefinitionsFetch = useEntitiesAppFetch( + ({ + signal, + }): Promise<{ + definitionEntities: DefinitionEntity[]; + }> => { + if (selectedType === 'all') { + return entitiesAPIClient.fetch('GET /internal/entities_api/types', { + signal, + }); + } + return entitiesAPIClient.fetch('GET /internal/entities_api/types/{type}', { + signal, + params: { + path: { + type: selectedType, + }, + }, + }); + }, + [entitiesAPIClient, selectedType] + ); + + const queryFetch = useEntitiesAppFetch( + ({ signal }) => { + return entitiesAPIClient.fetch('POST /internal/entities_api/entities', { + signal, + params: { + body: { + start, + end, + kuery: persistedKqlFilter, + types: [selectedType], + sortField: sort.field as any, + sortOrder: sort.order, + }, + }, + }); + }, + [entitiesAPIClient, selectedType, persistedKqlFilter, start, end, sort.field, sort.order] + ); + + const [pagination, setPagination] = useState<{ pageSize: number; pageIndex: number }>({ + pageSize: 10, + pageIndex: 0, + }); + + const entities = useMemo(() => { + return queryFetch.value?.entities ?? []; + }, [queryFetch.value]); + + const dataViewsFetch = useAbortableAsync(() => { + if (!typeDefinitionsFetch.value) { + return undefined; + } + + const allIndexPatterns = typeDefinitionsFetch.value.definitionEntities.flatMap((definition) => + getIndexPatternsForFilters(definition.filters) + ); + + return dataViews + .create( + { + title: allIndexPatterns.join(', '), + timeFieldName: '@timestamp', + }, + false, // skip fetch fields + true // display errors + ) + .then((response) => { + return [response]; + }); + }, [dataViews, typeDefinitionsFetch.value]); + + return ( + { + setTimeRange(nextTimeRange); + }} + rows={entities} + loading={queryFetch.loading} + kqlFilter={displayedKqlFilter} + onKqlFilterChange={(next) => { + setDisplayedKqlFilter(next); + }} + onKqlFilterSubmit={() => { + setPersistedKqlFilter(displayedKqlFilter); + }} + onPaginationChange={(next) => { + setPagination(next); + }} + pagination={pagination} + totalItemCount={entities.length} + columns={[]} + dataViews={dataViewsFetch.value} + showTypeSelect={type === 'all'} + onSelectedTypeChange={(nextType) => { + setSelectedType(nextType); + }} + onSortChange={(nextSort) => { + setSort(nextSort); + }} + sort={sort} + /> + ); +} diff --git a/x-pack/plugins/logsai/entities_app/public/components/esql_chart/controlled_esql_chart.tsx b/x-pack/plugins/logsai/entities_app/public/components/esql_chart/controlled_esql_chart.tsx new file mode 100644 index 0000000000000..aea3b7f577479 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/components/esql_chart/controlled_esql_chart.tsx @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React, { useMemo } from 'react'; +import { + AreaSeries, + Axis, + BarSeries, + Chart, + CurveType, + LineSeries, + Position, + ScaleType, + Settings, + Tooltip, + niceTimeFormatter, +} from '@elastic/charts'; +import { EuiFlexGroup, EuiFlexItem, EuiIcon, EuiSpacer } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { getTimeZone } from '@kbn/observability-utils-browser/utils/ui_settings/get_timezone'; +import { css } from '@emotion/css'; +import { AbortableAsyncState } from '@kbn/observability-utils-browser/hooks/use_abortable_async'; +import { ESQLSearchResponse } from '@kbn/es-types'; +import { esqlResultToTimeseries } from '../../util/esql_result_to_timeseries'; +import { useKibana } from '../../hooks/use_kibana'; +import { LoadingPanel } from '../loading_panel'; + +const END_ZONE_LABEL = i18n.translate('xpack.entities.esqlChart.endzone', { + defaultMessage: + 'The selected time range does not include this entire bucket. It might contain partial data.', +}); + +function getChartType(type: 'area' | 'bar' | 'line') { + switch (type) { + case 'area': + return AreaSeries; + case 'bar': + return BarSeries; + default: + return LineSeries; + } +} + +export function ControlledEsqlChart({ + id, + result, + metricNames, + chartType = 'line', + height, +}: { + id: string; + result: AbortableAsyncState; + metricNames: T[]; + chartType?: 'area' | 'bar' | 'line'; + height: number; +}) { + const { + core: { uiSettings }, + } = useKibana(); + + const allTimeseries = useMemo( + () => + esqlResultToTimeseries({ + result, + metricNames, + }), + // eslint-disable-next-line react-hooks/exhaustive-deps + [result, ...metricNames] + ); + + if (result.loading && !result.value?.values.length) { + return ( + + ); + } + + const xValues = allTimeseries.flatMap(({ data }) => data.map(({ x }) => x)); + + const min = Math.min(...xValues); + const max = Math.max(...xValues); + + const isEmpty = min === 0 && max === 0; + + const xFormatter = niceTimeFormatter([min, max]); + + const xDomain = isEmpty ? { min: 0, max: 1 } : { min, max }; + + const yTickFormat = (value: number | null) => (value === null ? '' : String(value)); + const yLabelFormat = (label: string) => label; + + const timeZone = getTimeZone(uiSettings); + + return ( + + { + const formattedValue = xFormatter(value); + if (max === value) { + return ( + <> + + + + + {END_ZONE_LABEL} + + + {formattedValue} + + ); + } + return formattedValue; + }} + /> + + + + {allTimeseries.map((serie) => { + const Series = getChartType(chartType); + + return ( + + ); + })} + + ); +} diff --git a/x-pack/plugins/logsai/entities_app/public/components/esql_chart/uncontrolled_esql_chart.tsx b/x-pack/plugins/logsai/entities_app/public/components/esql_chart/uncontrolled_esql_chart.tsx new file mode 100644 index 0000000000000..ebf0b3f16a630 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/components/esql_chart/uncontrolled_esql_chart.tsx @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React from 'react'; +import { useEsqlQueryResult } from '../../hooks/use_esql_query_result'; +import { ControlledEsqlChart } from './controlled_esql_chart'; + +export function UncontrolledEsqlChart({ + id, + query, + metricNames, + height, + start, + end, +}: { + id: string; + query: string; + metricNames: T[]; + height: number; + start: number; + end: number; +}) { + const result = useEsqlQueryResult({ query, start, end, operationName: 'visualize' }); + + return ; +} diff --git a/x-pack/plugins/logsai/entities_app/public/components/esql_grid/controlled_esql_grid.tsx b/x-pack/plugins/logsai/entities_app/public/components/esql_grid/controlled_esql_grid.tsx new file mode 100644 index 0000000000000..7957d99af796b --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/components/esql_grid/controlled_esql_grid.tsx @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React, { useMemo } from 'react'; +import { i18n } from '@kbn/i18n'; +import { ESQLDataGrid } from '@kbn/esql-datagrid/public'; +import { + AbortableAsyncState, + useAbortableAsync, +} from '@kbn/observability-utils-browser/hooks/use_abortable_async'; +import { getESQLAdHocDataview } from '@kbn/esql-utils'; +import { EuiCallOut } from '@elastic/eui'; +import { ESQLSearchResponse } from '@kbn/es-types'; +import { DatatableColumn } from '@kbn/expressions-plugin/common'; +import { LoadingPanel } from '../loading_panel'; +import { useKibana } from '../../hooks/use_kibana'; + +export function ControlledEsqlGrid({ + query, + result, + initialColumns, + analysisId, +}: { + query: string; + result: AbortableAsyncState; + initialColumns?: ESQLSearchResponse['columns']; + analysisId?: string; +}) { + const { + dependencies: { + start: { dataViews }, + }, + } = useKibana(); + + const response = result.value; + + const dataViewAsync = useAbortableAsync(() => { + return getESQLAdHocDataview(query, dataViews); + }, [query, dataViews]); + + const datatableColumns = useMemo(() => { + return ( + response?.columns.map((column): DatatableColumn => { + return { + id: column.name, + meta: { + type: 'string', + esType: column.type, + }, + name: column.name, + }; + }) ?? [] + ); + }, [response?.columns]); + + const initialDatatableColumns = useMemo(() => { + if (!initialColumns) { + return undefined; + } + + const initialColumnNames = new Set([...initialColumns.map((column) => column.name)]); + return datatableColumns.filter((column) => initialColumnNames.has(column.name)); + }, [datatableColumns, initialColumns]); + + if (!dataViewAsync.value || !response) { + return ; + } + + if (!result.loading && !result.error && !response.values.length) { + return ( + + {i18n.translate('xpack.entities.controlledEsqlGrid.noResultsCallOutLabel', { + defaultMessage: 'No results', + })} + + ); + } + + return ( + + ); +} diff --git a/x-pack/plugins/logsai/entities_app/public/components/loading_panel/index.tsx b/x-pack/plugins/logsai/entities_app/public/components/loading_panel/index.tsx new file mode 100644 index 0000000000000..58e432b84a4bd --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/components/loading_panel/index.tsx @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { EuiFlexGroup, EuiLoadingSpinner, EuiSpacer } from '@elastic/eui'; +import React from 'react'; + +export function LoadingPanel({ + loading = true, + size, + className, +}: { + loading?: boolean; + size?: React.ComponentProps['size']; + className?: string; +}) { + if (!loading) { + return null; + } + + return ( + + + + + + ); +} diff --git a/x-pack/plugins/logsai/entities_app/public/components/redirect_to/index.tsx b/x-pack/plugins/logsai/entities_app/public/components/redirect_to/index.tsx new file mode 100644 index 0000000000000..11ee321def8a5 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/components/redirect_to/index.tsx @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React, { useLayoutEffect } from 'react'; +import { PathsOf, TypeOf } from '@kbn/typed-react-router-config'; +import { DeepPartial } from 'utility-types'; +import { merge } from 'lodash'; +import { EntitiesAppRoutes } from '../../routes/config'; +import { useEntitiesAppRouter } from '../../hooks/use_entities_app_router'; +import { useEntitiesAppParams } from '../../hooks/use_entities_app_params'; + +export function RedirectTo< + TPath extends PathsOf, + TParams extends TypeOf +>({ path, params }: { path: TPath; params?: DeepPartial }) { + const router = useEntitiesAppRouter(); + const currentParams = useEntitiesAppParams('/*'); + useLayoutEffect(() => { + router.replace(path, ...([merge({}, currentParams, params)] as any)); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + return <>; +} diff --git a/x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_breadcrumbs.ts b/x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_breadcrumbs.ts new file mode 100644 index 0000000000000..d47ed5f78d40e --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_breadcrumbs.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createUseBreadcrumbs } from '@kbn/typed-react-router-config'; +import { EntitiesAppRoutes } from '../routes/config'; + +export const useEntitiesAppBreadcrumbs = createUseBreadcrumbs(); diff --git a/x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_fetch.ts b/x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_fetch.ts new file mode 100644 index 0000000000000..454325ad0dbbc --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_fetch.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import { + UseAbortableAsync, + useAbortableAsync, +} from '@kbn/observability-utils-browser/hooks/use_abortable_async'; +import { omit } from 'lodash'; +import { useKibana } from './use_kibana'; + +export const useEntitiesAppFetch: UseAbortableAsync<{}, { disableToastOnError?: boolean }> = ( + callback, + deps, + options +) => { + const { + core: { notifications }, + } = useKibana(); + + const onError = (error: Error) => { + let requestUrl: string | undefined; + + if (!options?.disableToastOnError) { + if ( + 'body' in error && + typeof error.body === 'object' && + !!error.body && + 'message' in error.body && + typeof error.body.message === 'string' + ) { + error.message = error.body.message; + } + + if ( + 'request' in error && + typeof error.request === 'object' && + !!error.request && + 'url' in error.request && + typeof error.request.url === 'string' + ) { + requestUrl = error.request.url; + } + + notifications.toasts.addError(error, { + title: i18n.translate('xpack.entities.failedToFetchError', { + defaultMessage: 'Failed to fetch data{requestUrlSuffix}', + values: { + requestUrlSuffix: requestUrl ? ` (${requestUrl})` : '', + }, + }), + }); + } + }; + + const optionsForHook = { + ...omit(options, 'disableToastOnError'), + onError, + }; + + return useAbortableAsync( + ({ signal }) => { + return callback({ signal }); + }, + // eslint-disable-next-line react-hooks/exhaustive-deps + deps, + optionsForHook + ); +}; diff --git a/x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_params.ts b/x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_params.ts new file mode 100644 index 0000000000000..cec3c83079198 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_params.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { type PathsOf, type TypeOf, useParams } from '@kbn/typed-react-router-config'; +import type { EntitiesAppRoutes } from '../routes/config'; + +export function useEntitiesAppParams>( + path: TPath +): TypeOf { + return useParams(path)! as TypeOf; +} diff --git a/x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_route_path.ts b/x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_route_path.ts new file mode 100644 index 0000000000000..49db4a8053155 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_route_path.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { PathsOf, useRoutePath } from '@kbn/typed-react-router-config'; +import type { EntitiesAppRoutes } from '../routes/config'; + +export function useEntitiesAppRoutePath() { + const path = useRoutePath(); + + return path as PathsOf; +} diff --git a/x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_router.ts b/x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_router.ts new file mode 100644 index 0000000000000..368166a85dbff --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_router.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { PathsOf, TypeAsArgs, TypeOf } from '@kbn/typed-react-router-config'; +import { useMemo } from 'react'; +import type { EntitiesAppRouter, EntitiesAppRoutes } from '../routes/config'; +import { entitiesAppRouter } from '../routes/config'; +import { useKibana } from './use_kibana'; + +interface StatefulEntitiesAppRouter extends EntitiesAppRouter { + push>( + path: T, + ...params: TypeAsArgs> + ): void; + replace>( + path: T, + ...params: TypeAsArgs> + ): void; +} + +export function useEntitiesAppRouter(): StatefulEntitiesAppRouter { + const { + core: { + http, + application: { navigateToApp }, + }, + } = useKibana(); + + const link = (...args: any[]) => { + // @ts-expect-error + return entitiesAppRouter.link(...args); + }; + + return useMemo( + () => ({ + ...entitiesAppRouter, + push: (...args) => { + const next = link(...args); + navigateToApp('entities', { path: next, replace: false }); + }, + replace: (path, ...args) => { + const next = link(path, ...args); + navigateToApp('entities', { path: next, replace: true }); + }, + link: (path, ...args) => { + return http.basePath.prepend('/app/entities' + link(path, ...args)); + }, + }), + [navigateToApp, http.basePath] + ); +} diff --git a/x-pack/plugins/logsai/entities_app/public/hooks/use_esql_query_result.ts b/x-pack/plugins/logsai/entities_app/public/hooks/use_esql_query_result.ts new file mode 100644 index 0000000000000..d47744ac67d77 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/hooks/use_esql_query_result.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useAbortableAsync } from '@kbn/observability-utils-browser/hooks/use_abortable_async'; +import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import { useKibana } from './use_kibana'; + +export function useEsqlQueryResult({ + query, + kuery, + start, + end, + operationName, + dslFilter, +}: { + query?: string; + kuery?: string; + start: number; + end: number; + operationName: string; + dslFilter?: QueryDslQueryContainer[]; +}) { + const { + dependencies: { + start: { + entitiesAPI: { entitiesAPIClient }, + }, + }, + } = useKibana(); + + return useAbortableAsync( + ({ signal }) => { + if (!query) { + return undefined; + } + return entitiesAPIClient.fetch('POST /internal/entities_api/esql', { + signal, + params: { + body: { + query, + start, + end, + kuery: kuery ?? '', + operationName, + dslFilter, + }, + }, + }); + }, + [entitiesAPIClient, query, start, end, kuery, operationName, dslFilter] + ); +} diff --git a/x-pack/plugins/logsai/entities_app/public/hooks/use_kibana.tsx b/x-pack/plugins/logsai/entities_app/public/hooks/use_kibana.tsx new file mode 100644 index 0000000000000..0f10255e80e44 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/hooks/use_kibana.tsx @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { CoreStart } from '@kbn/core/public'; +import { useMemo } from 'react'; +import { useKibana } from '@kbn/kibana-react-plugin/public'; +import type { EntitiesAppStartDependencies } from '../types'; +import type { EntitiesAppServices } from '../services/types'; + +export interface EntitiesAppKibanaContext { + core: CoreStart; + dependencies: { + start: EntitiesAppStartDependencies; + }; + services: EntitiesAppServices; +} + +const useTypedKibana = (): EntitiesAppKibanaContext => { + const context = useKibana>(); + + return useMemo(() => { + const { dependencies, services, ...core } = context.services; + + return { + core, + dependencies, + services, + }; + }, [context.services]); +}; + +export { useTypedKibana as useKibana }; diff --git a/x-pack/plugins/logsai/entities_app/public/index.ts b/x-pack/plugins/logsai/entities_app/public/index.ts new file mode 100644 index 0000000000000..961e3deb9a018 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/index.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { PluginInitializer, PluginInitializerContext } from '@kbn/core/public'; + +import { EntitiesAppPlugin } from './plugin'; +import type { + EntitiesAppPublicSetup, + EntitiesAppPublicStart, + EntitiesAppSetupDependencies, + EntitiesAppStartDependencies, + ConfigSchema, +} from './types'; + +export type { EntitiesAppPublicSetup, EntitiesAppPublicStart }; + +export const plugin: PluginInitializer< + EntitiesAppPublicSetup, + EntitiesAppPublicStart, + EntitiesAppSetupDependencies, + EntitiesAppStartDependencies +> = (pluginInitializerContext: PluginInitializerContext) => + new EntitiesAppPlugin(pluginInitializerContext); diff --git a/x-pack/plugins/logsai/entities_app/public/plugin.ts b/x-pack/plugins/logsai/entities_app/public/plugin.ts new file mode 100644 index 0000000000000..0dc5462e04b1d --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/plugin.ts @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import { from, map } from 'rxjs'; +import { + AppMountParameters, + CoreSetup, + CoreStart, + DEFAULT_APP_CATEGORIES, + Plugin, + PluginInitializerContext, +} from '@kbn/core/public'; +import type { Logger } from '@kbn/logging'; +import { ENTITY_APP_ID } from '@kbn/deeplinks-observability/constants'; +import type { + ConfigSchema, + EntitiesAppPublicSetup, + EntitiesAppPublicStart, + EntitiesAppSetupDependencies, + EntitiesAppStartDependencies, +} from './types'; +import { EntitiesAppServices } from './services/types'; + +export class EntitiesAppPlugin + implements + Plugin< + EntitiesAppPublicSetup, + EntitiesAppPublicStart, + EntitiesAppSetupDependencies, + EntitiesAppStartDependencies + > +{ + logger: Logger; + + constructor(context: PluginInitializerContext) { + this.logger = context.logger.get(); + } + setup( + coreSetup: CoreSetup, + pluginsSetup: EntitiesAppSetupDependencies + ): EntitiesAppPublicSetup { + pluginsSetup.observabilityShared.navigation.registerSections( + from(coreSetup.getStartServices()).pipe( + map(([coreStart, pluginsStart]) => { + return [ + { + label: '', + sortKey: 101, + entries: [ + { + label: i18n.translate('xpack.entities.entitiesAppLinkTitle', { + defaultMessage: 'Entities', + }), + app: ENTITY_APP_ID, + path: '/', + matchPath(currentPath: string) { + return ['/', ''].some((testPath) => currentPath.startsWith(testPath)); + }, + }, + ], + }, + ]; + }) + ) + ); + + coreSetup.application.register({ + id: ENTITY_APP_ID, + title: i18n.translate('xpack.entities.appTitle', { + defaultMessage: 'Entities', + }), + euiIconType: 'logoObservability', + appRoute: '/app/entities', + category: DEFAULT_APP_CATEGORIES.observability, + visibleIn: ['sideNav'], + order: 8001, + deepLinks: [ + { + id: 'entities', + title: i18n.translate('xpack.entities.entitiesAppDeepLinkTitle', { + defaultMessage: 'Entities', + }), + path: '/', + }, + ], + mount: async (appMountParameters: AppMountParameters) => { + // Load application bundle and Get start services + const [{ renderApp }, [coreStart, pluginsStart]] = await Promise.all([ + import('./application'), + coreSetup.getStartServices(), + ]); + + const services: EntitiesAppServices = {}; + + return renderApp({ + coreStart, + pluginsStart, + services, + appMountParameters, + }); + }, + }); + + return {}; + } + + start(coreStart: CoreStart, pluginsStart: EntitiesAppStartDependencies): EntitiesAppPublicStart { + return {}; + } +} diff --git a/x-pack/plugins/logsai/entities_app/public/routes/config.tsx b/x-pack/plugins/logsai/entities_app/public/routes/config.tsx new file mode 100644 index 0000000000000..1c832aff35faf --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/routes/config.tsx @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { i18n } from '@kbn/i18n'; +import { createRouter, Outlet, RouteMap } from '@kbn/typed-react-router-config'; +import * as t from 'io-ts'; +import React from 'react'; +import { AllEntitiesView } from '../components/all_entities_view'; +import { EntityDetailView } from '../components/entity_detail_view'; +import { EntitiesAppPageTemplate } from '../components/entities_app_page_template'; +import { EntitiesAppRouterBreadcrumb } from '../components/entities_app_router_breadcrumb'; +import { RedirectTo } from '../components/redirect_to'; +import { DataStreamDetailView } from '../components/data_stream_detail_view'; +import { EntityPivotTypeView } from '../components/entity_pivot_type_view'; + +/** + * The array of route definitions to be used when the application + * creates the routes. + */ +const entitiesAppRoutes = { + '/': { + element: ( + + + + + + ), + children: { + '/all': { + element: ( + + + + ), + }, + '/data_stream/{key}': { + element: , + params: t.type({ + path: t.type({ + key: t.string, + }), + }), + children: { + '/data_stream/{key}': { + element: ( + + ), + }, + '/data_stream/{key}/{tab}': { + element: , + params: t.type({ + path: t.type({ + tab: t.string, + }), + }), + }, + }, + }, + '/{type}': { + element: , + params: t.type({ + path: t.type({ type: t.string }), + }), + children: { + '/{type}': { + element: , + }, + '/{type}/{key}': { + params: t.type({ + path: t.type({ key: t.string }), + }), + element: , + children: { + '/{type}/{key}': { + element: ( + + ), + }, + '/{type}/{key}/{tab}': { + element: , + params: t.type({ + path: t.type({ tab: t.string }), + }), + }, + }, + }, + }, + }, + '/': { + element: , + }, + }, + }, +} satisfies RouteMap; + +export type EntitiesAppRoutes = typeof entitiesAppRoutes; + +export const entitiesAppRouter = createRouter(entitiesAppRoutes); + +export type EntitiesAppRouter = typeof entitiesAppRouter; diff --git a/x-pack/plugins/logsai/entities_app/public/services/types.ts b/x-pack/plugins/logsai/entities_app/public/services/types.ts new file mode 100644 index 0000000000000..7b6cda92161c0 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/services/types.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface EntitiesAppServices {} diff --git a/x-pack/plugins/logsai/entities_app/public/types.ts b/x-pack/plugins/logsai/entities_app/public/types.ts new file mode 100644 index 0000000000000..727f4e606186c --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/types.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { + EntitiesAPIPublicSetup, + EntitiesAPIPublicStart, +} from '@kbn/entities-api-plugin/public'; +import type { + ObservabilitySharedPluginSetup, + ObservabilitySharedPluginStart, +} from '@kbn/observability-shared-plugin/public'; +import type { DataPublicPluginSetup, DataPublicPluginStart } from '@kbn/data-plugin/public'; +import type { + DataViewsPublicPluginSetup, + DataViewsPublicPluginStart, +} from '@kbn/data-views-plugin/public'; +import type { + UnifiedSearchPluginSetup, + UnifiedSearchPublicPluginStart, +} from '@kbn/unified-search-plugin/public'; + +/* eslint-disable @typescript-eslint/no-empty-interface*/ + +export interface ConfigSchema {} + +export interface EntitiesAppSetupDependencies { + observabilityShared: ObservabilitySharedPluginSetup; + entitiesAPI: EntitiesAPIPublicSetup; + data: DataPublicPluginSetup; + dataViews: DataViewsPublicPluginSetup; + unifiedSearch: UnifiedSearchPluginSetup; +} + +export interface EntitiesAppStartDependencies { + observabilityShared: ObservabilitySharedPluginStart; + entitiesAPI: EntitiesAPIPublicStart; + data: DataPublicPluginStart; + dataViews: DataViewsPublicPluginStart; + unifiedSearch: UnifiedSearchPublicPluginStart; +} + +export interface EntitiesAppPublicSetup {} + +export interface EntitiesAppPublicStart {} diff --git a/x-pack/plugins/logsai/entities_app/public/util/esql_result_to_timeseries.ts b/x-pack/plugins/logsai/entities_app/public/util/esql_result_to_timeseries.ts new file mode 100644 index 0000000000000..11a6840baaedb --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/util/esql_result_to_timeseries.ts @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { orderBy } from 'lodash'; +import type { AbortableAsyncState } from '@kbn/observability-utils-browser/hooks/use_abortable_async'; +import type { ESQLSearchResponse } from '@kbn/es-types'; + +interface Timeseries { + id: string; + label: string; + metricNames: T[]; + data: Array<{ x: number } & Record>; +} + +export function esqlResultToTimeseries({ + result, + metricNames, +}: { + result: AbortableAsyncState; + metricNames: T[]; +}): Array> { + const columns = result.value?.columns; + + const rows = result.value?.values; + + if (!columns?.length || !rows?.length) { + return []; + } + + const timestampColumn = columns.find((col) => col.name === '@timestamp'); + + if (!timestampColumn) { + return []; + } + + const collectedSeries: Map> = new Map(); + + rows.forEach((columnsInRow) => { + const values = new Map(); + const labels = new Map(); + let timestamp: number; + + columnsInRow.forEach((value, index) => { + const column = columns[index]; + const isTimestamp = column.name === '@timestamp'; + const isMetric = metricNames.indexOf(column.name as T) !== -1; + + if (isTimestamp) { + timestamp = new Date(value as string | number).getTime(); + } else if (isMetric) { + values.set(column.name, value as number | null); + } else { + labels.set(column.name, String(value)); + } + }); + + const seriesKey = + Array.from(labels.entries()) + .map(([key, value]) => [key, value].join(':')) + .sort() + .join(',') || '-'; + + if (!collectedSeries.has(seriesKey)) { + collectedSeries.set(seriesKey, { + id: seriesKey, + data: [], + label: seriesKey, + metricNames, + }); + } + + const series = collectedSeries.get(seriesKey)!; + + const coordinate = { + x: timestamp!, + } as { x: number } & Record; + + values.forEach((value, key) => { + if (key !== 'x') { + // @ts-expect-error + coordinate[key as T] = value; + } + }); + + series.data.push(coordinate); + + return collectedSeries; + }); + + return Array.from(collectedSeries.entries()).map(([id, timeseries]) => { + return { + ...timeseries, + data: orderBy(timeseries.data, 'x', 'asc'), + }; + }); +} diff --git a/x-pack/plugins/logsai/entities_app/public/util/get_initial_columns_for_logs.ts b/x-pack/plugins/logsai/entities_app/public/util/get_initial_columns_for_logs.ts new file mode 100644 index 0000000000000..4e1daa16eca98 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/public/util/get_initial_columns_for_logs.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isArray } from 'lodash'; +import type { ESQLSearchResponse } from '@kbn/es-types'; +import { Pivot } from '@kbn/entities-api-plugin/common'; + +type Column = ESQLSearchResponse['columns'][number]; + +interface ColumnExtraction { + constants: Array<{ name: string; value: unknown }>; + initialColumns: Column[]; +} + +function analyzeColumnValues(response: ESQLSearchResponse): Array<{ + name: string; + unique: boolean; + constant: boolean; + empty: boolean; + index: number; + column: Column; +}> { + return response.columns.map((column, index) => { + const values = new Set(); + for (const row of response.values) { + const val = row[index]; + values.add(isArray(val) ? val.map(String).join(',') : val); + } + return { + name: column.name, + unique: values.size === response.values.length, + constant: values.size === 1, + empty: Array.from(values.values()).every((value) => !value), + index, + column, + }; + }); +} + +export function getInitialColumnsForLogs({ + response, + pivots, +}: { + response: ESQLSearchResponse; + pivots: Pivot[]; +}): ColumnExtraction { + const analyzedColumns = analyzeColumnValues(response); + + const withoutUselessColumns = analyzedColumns.filter(({ column, empty, constant, unique }) => { + return empty === false && constant === false && !(column.type === 'keyword' && unique); + }); + + const constantColumns = analyzedColumns.filter(({ constant }) => constant); + + const timestampColumnIndex = withoutUselessColumns.findIndex( + (column) => column.name === '@timestamp' + ); + + const messageColumnIndex = withoutUselessColumns.findIndex( + (column) => column.name === 'message' || column.name === 'msg' + ); + + const initialColumns = new Set(); + + if (timestampColumnIndex !== -1) { + initialColumns.add(withoutUselessColumns[timestampColumnIndex].column); + } + + if (messageColumnIndex !== -1) { + initialColumns.add(withoutUselessColumns[messageColumnIndex].column); + } + + const allIdentityFields = new Set([...pivots.flatMap((pivot) => pivot.identityFields)]); + + const columnsWithIdentityFields = analyzedColumns.filter((column) => + allIdentityFields.has(column.name) + ); + const columnsInOrderOfPreference = [ + ...columnsWithIdentityFields, + ...withoutUselessColumns, + ...constantColumns, + ]; + + for (const { column } of columnsInOrderOfPreference) { + if (initialColumns.size <= 8) { + initialColumns.add(column); + } else { + break; + } + } + + const constants = constantColumns.map(({ name, index, column }) => { + return { name, value: response.values[0][index] }; + }); + + return { + initialColumns: Array.from(initialColumns.values()), + constants, + }; +} diff --git a/x-pack/plugins/logsai/entities_app/server/config.ts b/x-pack/plugins/logsai/entities_app/server/config.ts new file mode 100644 index 0000000000000..6fcffd687a9dd --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/server/config.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema, type TypeOf } from '@kbn/config-schema'; + +export const config = schema.object({ + enabled: schema.boolean({ defaultValue: true }), +}); + +export type EntitiesAppConfig = TypeOf; diff --git a/x-pack/plugins/logsai/entities_app/server/index.ts b/x-pack/plugins/logsai/entities_app/server/index.ts new file mode 100644 index 0000000000000..cd970efad2c39 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/server/index.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { + PluginConfigDescriptor, + PluginInitializer, + PluginInitializerContext, +} from '@kbn/core/server'; +import type { EntitiesAppConfig } from './config'; +import { EntitiesAppPlugin } from './plugin'; +import type { + EntitiesAppServerSetup, + EntitiesAppServerStart, + EntitiesAppSetupDependencies, + EntitiesAppStartDependencies, +} from './types'; + +export type { EntitiesAppServerSetup, EntitiesAppServerStart }; + +import { config as configSchema } from './config'; + +export const config: PluginConfigDescriptor = { + schema: configSchema, +}; + +export const plugin: PluginInitializer< + EntitiesAppServerSetup, + EntitiesAppServerStart, + EntitiesAppSetupDependencies, + EntitiesAppStartDependencies +> = async (pluginInitializerContext: PluginInitializerContext) => + new EntitiesAppPlugin(pluginInitializerContext); diff --git a/x-pack/plugins/logsai/entities_app/server/plugin.ts b/x-pack/plugins/logsai/entities_app/server/plugin.ts new file mode 100644 index 0000000000000..3c039d8b83b88 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/server/plugin.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/server'; +import type { Logger } from '@kbn/logging'; +import type { + ConfigSchema, + EntitiesAppServerSetup, + EntitiesAppServerStart, + EntitiesAppSetupDependencies, + EntitiesAppStartDependencies, +} from './types'; + +export class EntitiesAppPlugin + implements + Plugin< + EntitiesAppServerSetup, + EntitiesAppServerStart, + EntitiesAppSetupDependencies, + EntitiesAppStartDependencies + > +{ + logger: Logger; + + constructor(context: PluginInitializerContext) { + this.logger = context.logger.get(); + } + setup( + coreSetup: CoreSetup, + pluginsSetup: EntitiesAppSetupDependencies + ): EntitiesAppServerSetup { + return {}; + } + + start(core: CoreStart, pluginsStart: EntitiesAppStartDependencies): EntitiesAppServerStart { + return {}; + } +} diff --git a/x-pack/plugins/logsai/entities_app/server/types.ts b/x-pack/plugins/logsai/entities_app/server/types.ts new file mode 100644 index 0000000000000..c56d169f7a8e6 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/server/types.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +/* eslint-disable @typescript-eslint/no-empty-interface*/ + +export interface ConfigSchema {} + +export interface EntitiesAppSetupDependencies {} + +export interface EntitiesAppStartDependencies {} + +export interface EntitiesAppServerSetup {} + +export interface EntitiesAppServerStart {} diff --git a/x-pack/plugins/logsai/entities_app/tsconfig.json b/x-pack/plugins/logsai/entities_app/tsconfig.json new file mode 100644 index 0000000000000..47156e97ad7a2 --- /dev/null +++ b/x-pack/plugins/logsai/entities_app/tsconfig.json @@ -0,0 +1,38 @@ +{ + "extends": "../../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types" + }, + "include": [ + "../../../typings/**/*", + "common/**/*", + "public/**/*", + "typings/**/*", + "public/**/*.json", + "server/**/*", + ".storybook/**/*" + ], + "exclude": ["target/**/*", ".storybook/**/*.js"], + "kbn_references": [ + "@kbn/core", + "@kbn/observability-shared-plugin", + "@kbn/data-views-plugin", + "@kbn/data-plugin", + "@kbn/unified-search-plugin", + "@kbn/entities-api-plugin", + "@kbn/react-kibana-context-render", + "@kbn/i18n", + "@kbn/shared-ux-link-redirect-app", + "@kbn/typed-react-router-config", + "@kbn/kibana-react-plugin", + "@kbn/observability-utils-browser", + "@kbn/es-query", + "@kbn/logging", + "@kbn/deeplinks-observability", + "@kbn/es-types", + "@kbn/config-schema", + "@kbn/esql-datagrid", + "@kbn/esql-utils", + "@kbn/expressions-plugin", + ] +} From 7e8f2d742c01d10fe4390ae7f7e6556907c421ad Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Wed, 6 Nov 2024 12:45:04 +0100 Subject: [PATCH 05/95] Move files into streams_* --- x-pack/plugins/logsai/{entities_api => streams_api}/README.md | 0 .../logsai/{entities_api => streams_api}/common/entities.ts | 0 .../plugins/logsai/{entities_api => streams_api}/common/index.ts | 0 .../common/queries/entity_source_query.ts | 0 .../common/queries/entity_time_range_query.ts | 0 .../{entities_api => streams_api}/common/utils/esql_escape.ts | 0 .../common/utils/esql_result_to_plain_objects.ts | 0 .../common/utils/get_esql_request.ts | 0 .../common/utils/get_index_patterns_for_filters.ts | 0 .../plugins/logsai/{entities_api => streams_api}/jest.config.js | 0 x-pack/plugins/logsai/{entities_api => streams_api}/kibana.jsonc | 0 .../logsai/{entities_api => streams_api}/public/api/index.tsx | 0 .../plugins/logsai/{entities_api => streams_api}/public/index.ts | 0 .../plugins/logsai/{entities_api => streams_api}/public/plugin.ts | 0 .../logsai/{entities_api => streams_api}/public/services/types.ts | 0 .../plugins/logsai/{entities_api => streams_api}/public/types.ts | 0 .../server/built_in_definitions_stub.ts | 0 .../plugins/logsai/{entities_api => streams_api}/server/config.ts | 0 .../plugins/logsai/{entities_api => streams_api}/server/index.ts | 0 .../server/lib/clients/create_alerts_client.ts | 0 .../server/lib/clients/create_dashboards_client.ts | 0 .../server/lib/clients/create_entities_api_es_client.ts | 0 .../server/lib/clients/create_entity_client.ts | 0 .../server/lib/clients/create_rules_client.ts | 0 .../server/lib/clients/create_slo_client.ts | 0 .../server/lib/entities/entity_lookup_table.ts | 0 .../server/lib/entities/get_data_streams_for_filter.ts | 0 .../server/lib/entities/get_definition_entities.ts | 0 .../server/lib/entities/get_type_definitions.ts | 0 .../server/lib/entities/query_signals_as_entities.ts | 0 .../server/lib/entities/query_sources_as_entities.ts | 0 .../server/lib/with_entities_api_span.ts | 0 .../plugins/logsai/{entities_api => streams_api}/server/plugin.ts | 0 .../server/routes/create_entities_api_server_route.ts | 0 .../server/routes/entities/get_entities.ts | 0 .../server/routes/entities/get_entity_from_type_and_key.ts | 0 .../server/routes/entities/get_esql_identity_commands.ts | 0 .../{entities_api => streams_api}/server/routes/entities/route.ts | 0 .../{entities_api => streams_api}/server/routes/esql/route.ts | 0 .../server/routes/get_global_entities_api_route_repository.ts | 0 .../server/routes/register_routes.ts | 0 .../logsai/{entities_api => streams_api}/server/routes/types.ts | 0 .../{entities_api => streams_api}/server/routes/types/route.ts | 0 .../plugins/logsai/{entities_api => streams_api}/server/types.ts | 0 x-pack/plugins/logsai/{entities_api => streams_api}/tsconfig.json | 0 .../.storybook/get_mock_entities_app_context.tsx | 0 .../logsai/{entities_app => streams_app}/.storybook/jest_setup.js | 0 .../logsai/{entities_app => streams_app}/.storybook/main.js | 0 .../logsai/{entities_app => streams_app}/.storybook/preview.js | 0 .../.storybook/storybook_decorator.tsx | 0 x-pack/plugins/logsai/{entities_app => streams_app}/README.md | 0 .../plugins/logsai/{entities_app => streams_app}/jest.config.js | 0 x-pack/plugins/logsai/{entities_app => streams_app}/kibana.jsonc | 0 .../logsai/{entities_app => streams_app}/public/application.tsx | 0 .../public/components/all_entities_view/index.tsx | 0 .../public/components/app_root/index.tsx | 0 .../public/components/data_stream_detail_view/index.tsx | 0 .../public/components/entities_app_context_provider/index.tsx | 0 .../entities_app_page_header/entities_app_page_header_title.tsx | 0 .../public/components/entities_app_page_header/index.tsx | 0 .../public/components/entities_app_page_template/index.tsx | 0 .../public/components/entities_app_router_breadcrumb/index.tsx | 0 .../public/components/entities_app_search_bar/index.tsx | 0 .../public/components/entity_detail_overview/index.tsx | 0 .../public/components/entity_detail_view/index.tsx | 0 .../public/components/entity_detail_view_header_section/index.tsx | 0 .../public/components/entity_health_status_badge/index.tsx | 0 .../public/components/entity_overview_tab_list/index.tsx | 0 .../public/components/entity_pivot_type_view/index.tsx | 0 .../public/components/entity_table/controlled_entity_table.tsx | 0 .../public/components/entity_table/index.tsx | 0 .../public/components/esql_chart/controlled_esql_chart.tsx | 0 .../public/components/esql_chart/uncontrolled_esql_chart.tsx | 0 .../public/components/esql_grid/controlled_esql_grid.tsx | 0 .../public/components/loading_panel/index.tsx | 0 .../public/components/redirect_to/index.tsx | 0 .../public/hooks/use_entities_app_breadcrumbs.ts | 0 .../public/hooks/use_entities_app_fetch.ts | 0 .../public/hooks/use_entities_app_params.ts | 0 .../public/hooks/use_entities_app_route_path.ts | 0 .../public/hooks/use_entities_app_router.ts | 0 .../public/hooks/use_esql_query_result.ts | 0 .../{entities_app => streams_app}/public/hooks/use_kibana.tsx | 0 .../plugins/logsai/{entities_app => streams_app}/public/index.ts | 0 .../plugins/logsai/{entities_app => streams_app}/public/plugin.ts | 0 .../logsai/{entities_app => streams_app}/public/routes/config.tsx | 0 .../logsai/{entities_app => streams_app}/public/services/types.ts | 0 .../plugins/logsai/{entities_app => streams_app}/public/types.ts | 0 .../public/util/esql_result_to_timeseries.ts | 0 .../public/util/get_initial_columns_for_logs.ts | 0 .../plugins/logsai/{entities_app => streams_app}/server/config.ts | 0 .../plugins/logsai/{entities_app => streams_app}/server/index.ts | 0 .../plugins/logsai/{entities_app => streams_app}/server/plugin.ts | 0 .../plugins/logsai/{entities_app => streams_app}/server/types.ts | 0 x-pack/plugins/logsai/{entities_app => streams_app}/tsconfig.json | 0 95 files changed, 0 insertions(+), 0 deletions(-) rename x-pack/plugins/logsai/{entities_api => streams_api}/README.md (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/common/entities.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/common/index.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/common/queries/entity_source_query.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/common/queries/entity_time_range_query.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/common/utils/esql_escape.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/common/utils/esql_result_to_plain_objects.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/common/utils/get_esql_request.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/common/utils/get_index_patterns_for_filters.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/jest.config.js (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/kibana.jsonc (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/public/api/index.tsx (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/public/index.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/public/plugin.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/public/services/types.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/public/types.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/server/built_in_definitions_stub.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/server/config.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/server/index.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/server/lib/clients/create_alerts_client.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/server/lib/clients/create_dashboards_client.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/server/lib/clients/create_entities_api_es_client.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/server/lib/clients/create_entity_client.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/server/lib/clients/create_rules_client.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/server/lib/clients/create_slo_client.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/server/lib/entities/entity_lookup_table.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/server/lib/entities/get_data_streams_for_filter.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/server/lib/entities/get_definition_entities.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/server/lib/entities/get_type_definitions.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/server/lib/entities/query_signals_as_entities.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/server/lib/entities/query_sources_as_entities.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/server/lib/with_entities_api_span.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/server/plugin.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/server/routes/create_entities_api_server_route.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/server/routes/entities/get_entities.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/server/routes/entities/get_entity_from_type_and_key.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/server/routes/entities/get_esql_identity_commands.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/server/routes/entities/route.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/server/routes/esql/route.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/server/routes/get_global_entities_api_route_repository.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/server/routes/register_routes.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/server/routes/types.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/server/routes/types/route.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/server/types.ts (100%) rename x-pack/plugins/logsai/{entities_api => streams_api}/tsconfig.json (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/.storybook/get_mock_entities_app_context.tsx (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/.storybook/jest_setup.js (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/.storybook/main.js (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/.storybook/preview.js (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/.storybook/storybook_decorator.tsx (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/README.md (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/jest.config.js (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/kibana.jsonc (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/application.tsx (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/components/all_entities_view/index.tsx (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/components/app_root/index.tsx (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/components/data_stream_detail_view/index.tsx (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/components/entities_app_context_provider/index.tsx (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/components/entities_app_page_header/entities_app_page_header_title.tsx (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/components/entities_app_page_header/index.tsx (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/components/entities_app_page_template/index.tsx (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/components/entities_app_router_breadcrumb/index.tsx (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/components/entities_app_search_bar/index.tsx (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/components/entity_detail_overview/index.tsx (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/components/entity_detail_view/index.tsx (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/components/entity_detail_view_header_section/index.tsx (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/components/entity_health_status_badge/index.tsx (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/components/entity_overview_tab_list/index.tsx (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/components/entity_pivot_type_view/index.tsx (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/components/entity_table/controlled_entity_table.tsx (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/components/entity_table/index.tsx (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/components/esql_chart/controlled_esql_chart.tsx (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/components/esql_chart/uncontrolled_esql_chart.tsx (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/components/esql_grid/controlled_esql_grid.tsx (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/components/loading_panel/index.tsx (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/components/redirect_to/index.tsx (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/hooks/use_entities_app_breadcrumbs.ts (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/hooks/use_entities_app_fetch.ts (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/hooks/use_entities_app_params.ts (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/hooks/use_entities_app_route_path.ts (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/hooks/use_entities_app_router.ts (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/hooks/use_esql_query_result.ts (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/hooks/use_kibana.tsx (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/index.ts (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/plugin.ts (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/routes/config.tsx (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/services/types.ts (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/types.ts (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/util/esql_result_to_timeseries.ts (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/public/util/get_initial_columns_for_logs.ts (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/server/config.ts (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/server/index.ts (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/server/plugin.ts (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/server/types.ts (100%) rename x-pack/plugins/logsai/{entities_app => streams_app}/tsconfig.json (100%) diff --git a/x-pack/plugins/logsai/entities_api/README.md b/x-pack/plugins/logsai/streams_api/README.md similarity index 100% rename from x-pack/plugins/logsai/entities_api/README.md rename to x-pack/plugins/logsai/streams_api/README.md diff --git a/x-pack/plugins/logsai/entities_api/common/entities.ts b/x-pack/plugins/logsai/streams_api/common/entities.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/common/entities.ts rename to x-pack/plugins/logsai/streams_api/common/entities.ts diff --git a/x-pack/plugins/logsai/entities_api/common/index.ts b/x-pack/plugins/logsai/streams_api/common/index.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/common/index.ts rename to x-pack/plugins/logsai/streams_api/common/index.ts diff --git a/x-pack/plugins/logsai/entities_api/common/queries/entity_source_query.ts b/x-pack/plugins/logsai/streams_api/common/queries/entity_source_query.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/common/queries/entity_source_query.ts rename to x-pack/plugins/logsai/streams_api/common/queries/entity_source_query.ts diff --git a/x-pack/plugins/logsai/entities_api/common/queries/entity_time_range_query.ts b/x-pack/plugins/logsai/streams_api/common/queries/entity_time_range_query.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/common/queries/entity_time_range_query.ts rename to x-pack/plugins/logsai/streams_api/common/queries/entity_time_range_query.ts diff --git a/x-pack/plugins/logsai/entities_api/common/utils/esql_escape.ts b/x-pack/plugins/logsai/streams_api/common/utils/esql_escape.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/common/utils/esql_escape.ts rename to x-pack/plugins/logsai/streams_api/common/utils/esql_escape.ts diff --git a/x-pack/plugins/logsai/entities_api/common/utils/esql_result_to_plain_objects.ts b/x-pack/plugins/logsai/streams_api/common/utils/esql_result_to_plain_objects.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/common/utils/esql_result_to_plain_objects.ts rename to x-pack/plugins/logsai/streams_api/common/utils/esql_result_to_plain_objects.ts diff --git a/x-pack/plugins/logsai/entities_api/common/utils/get_esql_request.ts b/x-pack/plugins/logsai/streams_api/common/utils/get_esql_request.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/common/utils/get_esql_request.ts rename to x-pack/plugins/logsai/streams_api/common/utils/get_esql_request.ts diff --git a/x-pack/plugins/logsai/entities_api/common/utils/get_index_patterns_for_filters.ts b/x-pack/plugins/logsai/streams_api/common/utils/get_index_patterns_for_filters.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/common/utils/get_index_patterns_for_filters.ts rename to x-pack/plugins/logsai/streams_api/common/utils/get_index_patterns_for_filters.ts diff --git a/x-pack/plugins/logsai/entities_api/jest.config.js b/x-pack/plugins/logsai/streams_api/jest.config.js similarity index 100% rename from x-pack/plugins/logsai/entities_api/jest.config.js rename to x-pack/plugins/logsai/streams_api/jest.config.js diff --git a/x-pack/plugins/logsai/entities_api/kibana.jsonc b/x-pack/plugins/logsai/streams_api/kibana.jsonc similarity index 100% rename from x-pack/plugins/logsai/entities_api/kibana.jsonc rename to x-pack/plugins/logsai/streams_api/kibana.jsonc diff --git a/x-pack/plugins/logsai/entities_api/public/api/index.tsx b/x-pack/plugins/logsai/streams_api/public/api/index.tsx similarity index 100% rename from x-pack/plugins/logsai/entities_api/public/api/index.tsx rename to x-pack/plugins/logsai/streams_api/public/api/index.tsx diff --git a/x-pack/plugins/logsai/entities_api/public/index.ts b/x-pack/plugins/logsai/streams_api/public/index.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/public/index.ts rename to x-pack/plugins/logsai/streams_api/public/index.ts diff --git a/x-pack/plugins/logsai/entities_api/public/plugin.ts b/x-pack/plugins/logsai/streams_api/public/plugin.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/public/plugin.ts rename to x-pack/plugins/logsai/streams_api/public/plugin.ts diff --git a/x-pack/plugins/logsai/entities_api/public/services/types.ts b/x-pack/plugins/logsai/streams_api/public/services/types.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/public/services/types.ts rename to x-pack/plugins/logsai/streams_api/public/services/types.ts diff --git a/x-pack/plugins/logsai/entities_api/public/types.ts b/x-pack/plugins/logsai/streams_api/public/types.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/public/types.ts rename to x-pack/plugins/logsai/streams_api/public/types.ts diff --git a/x-pack/plugins/logsai/entities_api/server/built_in_definitions_stub.ts b/x-pack/plugins/logsai/streams_api/server/built_in_definitions_stub.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/server/built_in_definitions_stub.ts rename to x-pack/plugins/logsai/streams_api/server/built_in_definitions_stub.ts diff --git a/x-pack/plugins/logsai/entities_api/server/config.ts b/x-pack/plugins/logsai/streams_api/server/config.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/server/config.ts rename to x-pack/plugins/logsai/streams_api/server/config.ts diff --git a/x-pack/plugins/logsai/entities_api/server/index.ts b/x-pack/plugins/logsai/streams_api/server/index.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/server/index.ts rename to x-pack/plugins/logsai/streams_api/server/index.ts diff --git a/x-pack/plugins/logsai/entities_api/server/lib/clients/create_alerts_client.ts b/x-pack/plugins/logsai/streams_api/server/lib/clients/create_alerts_client.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/server/lib/clients/create_alerts_client.ts rename to x-pack/plugins/logsai/streams_api/server/lib/clients/create_alerts_client.ts diff --git a/x-pack/plugins/logsai/entities_api/server/lib/clients/create_dashboards_client.ts b/x-pack/plugins/logsai/streams_api/server/lib/clients/create_dashboards_client.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/server/lib/clients/create_dashboards_client.ts rename to x-pack/plugins/logsai/streams_api/server/lib/clients/create_dashboards_client.ts diff --git a/x-pack/plugins/logsai/entities_api/server/lib/clients/create_entities_api_es_client.ts b/x-pack/plugins/logsai/streams_api/server/lib/clients/create_entities_api_es_client.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/server/lib/clients/create_entities_api_es_client.ts rename to x-pack/plugins/logsai/streams_api/server/lib/clients/create_entities_api_es_client.ts diff --git a/x-pack/plugins/logsai/entities_api/server/lib/clients/create_entity_client.ts b/x-pack/plugins/logsai/streams_api/server/lib/clients/create_entity_client.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/server/lib/clients/create_entity_client.ts rename to x-pack/plugins/logsai/streams_api/server/lib/clients/create_entity_client.ts diff --git a/x-pack/plugins/logsai/entities_api/server/lib/clients/create_rules_client.ts b/x-pack/plugins/logsai/streams_api/server/lib/clients/create_rules_client.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/server/lib/clients/create_rules_client.ts rename to x-pack/plugins/logsai/streams_api/server/lib/clients/create_rules_client.ts diff --git a/x-pack/plugins/logsai/entities_api/server/lib/clients/create_slo_client.ts b/x-pack/plugins/logsai/streams_api/server/lib/clients/create_slo_client.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/server/lib/clients/create_slo_client.ts rename to x-pack/plugins/logsai/streams_api/server/lib/clients/create_slo_client.ts diff --git a/x-pack/plugins/logsai/entities_api/server/lib/entities/entity_lookup_table.ts b/x-pack/plugins/logsai/streams_api/server/lib/entities/entity_lookup_table.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/server/lib/entities/entity_lookup_table.ts rename to x-pack/plugins/logsai/streams_api/server/lib/entities/entity_lookup_table.ts diff --git a/x-pack/plugins/logsai/entities_api/server/lib/entities/get_data_streams_for_filter.ts b/x-pack/plugins/logsai/streams_api/server/lib/entities/get_data_streams_for_filter.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/server/lib/entities/get_data_streams_for_filter.ts rename to x-pack/plugins/logsai/streams_api/server/lib/entities/get_data_streams_for_filter.ts diff --git a/x-pack/plugins/logsai/entities_api/server/lib/entities/get_definition_entities.ts b/x-pack/plugins/logsai/streams_api/server/lib/entities/get_definition_entities.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/server/lib/entities/get_definition_entities.ts rename to x-pack/plugins/logsai/streams_api/server/lib/entities/get_definition_entities.ts diff --git a/x-pack/plugins/logsai/entities_api/server/lib/entities/get_type_definitions.ts b/x-pack/plugins/logsai/streams_api/server/lib/entities/get_type_definitions.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/server/lib/entities/get_type_definitions.ts rename to x-pack/plugins/logsai/streams_api/server/lib/entities/get_type_definitions.ts diff --git a/x-pack/plugins/logsai/entities_api/server/lib/entities/query_signals_as_entities.ts b/x-pack/plugins/logsai/streams_api/server/lib/entities/query_signals_as_entities.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/server/lib/entities/query_signals_as_entities.ts rename to x-pack/plugins/logsai/streams_api/server/lib/entities/query_signals_as_entities.ts diff --git a/x-pack/plugins/logsai/entities_api/server/lib/entities/query_sources_as_entities.ts b/x-pack/plugins/logsai/streams_api/server/lib/entities/query_sources_as_entities.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/server/lib/entities/query_sources_as_entities.ts rename to x-pack/plugins/logsai/streams_api/server/lib/entities/query_sources_as_entities.ts diff --git a/x-pack/plugins/logsai/entities_api/server/lib/with_entities_api_span.ts b/x-pack/plugins/logsai/streams_api/server/lib/with_entities_api_span.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/server/lib/with_entities_api_span.ts rename to x-pack/plugins/logsai/streams_api/server/lib/with_entities_api_span.ts diff --git a/x-pack/plugins/logsai/entities_api/server/plugin.ts b/x-pack/plugins/logsai/streams_api/server/plugin.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/server/plugin.ts rename to x-pack/plugins/logsai/streams_api/server/plugin.ts diff --git a/x-pack/plugins/logsai/entities_api/server/routes/create_entities_api_server_route.ts b/x-pack/plugins/logsai/streams_api/server/routes/create_entities_api_server_route.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/server/routes/create_entities_api_server_route.ts rename to x-pack/plugins/logsai/streams_api/server/routes/create_entities_api_server_route.ts diff --git a/x-pack/plugins/logsai/entities_api/server/routes/entities/get_entities.ts b/x-pack/plugins/logsai/streams_api/server/routes/entities/get_entities.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/server/routes/entities/get_entities.ts rename to x-pack/plugins/logsai/streams_api/server/routes/entities/get_entities.ts diff --git a/x-pack/plugins/logsai/entities_api/server/routes/entities/get_entity_from_type_and_key.ts b/x-pack/plugins/logsai/streams_api/server/routes/entities/get_entity_from_type_and_key.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/server/routes/entities/get_entity_from_type_and_key.ts rename to x-pack/plugins/logsai/streams_api/server/routes/entities/get_entity_from_type_and_key.ts diff --git a/x-pack/plugins/logsai/entities_api/server/routes/entities/get_esql_identity_commands.ts b/x-pack/plugins/logsai/streams_api/server/routes/entities/get_esql_identity_commands.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/server/routes/entities/get_esql_identity_commands.ts rename to x-pack/plugins/logsai/streams_api/server/routes/entities/get_esql_identity_commands.ts diff --git a/x-pack/plugins/logsai/entities_api/server/routes/entities/route.ts b/x-pack/plugins/logsai/streams_api/server/routes/entities/route.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/server/routes/entities/route.ts rename to x-pack/plugins/logsai/streams_api/server/routes/entities/route.ts diff --git a/x-pack/plugins/logsai/entities_api/server/routes/esql/route.ts b/x-pack/plugins/logsai/streams_api/server/routes/esql/route.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/server/routes/esql/route.ts rename to x-pack/plugins/logsai/streams_api/server/routes/esql/route.ts diff --git a/x-pack/plugins/logsai/entities_api/server/routes/get_global_entities_api_route_repository.ts b/x-pack/plugins/logsai/streams_api/server/routes/get_global_entities_api_route_repository.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/server/routes/get_global_entities_api_route_repository.ts rename to x-pack/plugins/logsai/streams_api/server/routes/get_global_entities_api_route_repository.ts diff --git a/x-pack/plugins/logsai/entities_api/server/routes/register_routes.ts b/x-pack/plugins/logsai/streams_api/server/routes/register_routes.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/server/routes/register_routes.ts rename to x-pack/plugins/logsai/streams_api/server/routes/register_routes.ts diff --git a/x-pack/plugins/logsai/entities_api/server/routes/types.ts b/x-pack/plugins/logsai/streams_api/server/routes/types.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/server/routes/types.ts rename to x-pack/plugins/logsai/streams_api/server/routes/types.ts diff --git a/x-pack/plugins/logsai/entities_api/server/routes/types/route.ts b/x-pack/plugins/logsai/streams_api/server/routes/types/route.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/server/routes/types/route.ts rename to x-pack/plugins/logsai/streams_api/server/routes/types/route.ts diff --git a/x-pack/plugins/logsai/entities_api/server/types.ts b/x-pack/plugins/logsai/streams_api/server/types.ts similarity index 100% rename from x-pack/plugins/logsai/entities_api/server/types.ts rename to x-pack/plugins/logsai/streams_api/server/types.ts diff --git a/x-pack/plugins/logsai/entities_api/tsconfig.json b/x-pack/plugins/logsai/streams_api/tsconfig.json similarity index 100% rename from x-pack/plugins/logsai/entities_api/tsconfig.json rename to x-pack/plugins/logsai/streams_api/tsconfig.json diff --git a/x-pack/plugins/logsai/entities_app/.storybook/get_mock_entities_app_context.tsx b/x-pack/plugins/logsai/streams_app/.storybook/get_mock_entities_app_context.tsx similarity index 100% rename from x-pack/plugins/logsai/entities_app/.storybook/get_mock_entities_app_context.tsx rename to x-pack/plugins/logsai/streams_app/.storybook/get_mock_entities_app_context.tsx diff --git a/x-pack/plugins/logsai/entities_app/.storybook/jest_setup.js b/x-pack/plugins/logsai/streams_app/.storybook/jest_setup.js similarity index 100% rename from x-pack/plugins/logsai/entities_app/.storybook/jest_setup.js rename to x-pack/plugins/logsai/streams_app/.storybook/jest_setup.js diff --git a/x-pack/plugins/logsai/entities_app/.storybook/main.js b/x-pack/plugins/logsai/streams_app/.storybook/main.js similarity index 100% rename from x-pack/plugins/logsai/entities_app/.storybook/main.js rename to x-pack/plugins/logsai/streams_app/.storybook/main.js diff --git a/x-pack/plugins/logsai/entities_app/.storybook/preview.js b/x-pack/plugins/logsai/streams_app/.storybook/preview.js similarity index 100% rename from x-pack/plugins/logsai/entities_app/.storybook/preview.js rename to x-pack/plugins/logsai/streams_app/.storybook/preview.js diff --git a/x-pack/plugins/logsai/entities_app/.storybook/storybook_decorator.tsx b/x-pack/plugins/logsai/streams_app/.storybook/storybook_decorator.tsx similarity index 100% rename from x-pack/plugins/logsai/entities_app/.storybook/storybook_decorator.tsx rename to x-pack/plugins/logsai/streams_app/.storybook/storybook_decorator.tsx diff --git a/x-pack/plugins/logsai/entities_app/README.md b/x-pack/plugins/logsai/streams_app/README.md similarity index 100% rename from x-pack/plugins/logsai/entities_app/README.md rename to x-pack/plugins/logsai/streams_app/README.md diff --git a/x-pack/plugins/logsai/entities_app/jest.config.js b/x-pack/plugins/logsai/streams_app/jest.config.js similarity index 100% rename from x-pack/plugins/logsai/entities_app/jest.config.js rename to x-pack/plugins/logsai/streams_app/jest.config.js diff --git a/x-pack/plugins/logsai/entities_app/kibana.jsonc b/x-pack/plugins/logsai/streams_app/kibana.jsonc similarity index 100% rename from x-pack/plugins/logsai/entities_app/kibana.jsonc rename to x-pack/plugins/logsai/streams_app/kibana.jsonc diff --git a/x-pack/plugins/logsai/entities_app/public/application.tsx b/x-pack/plugins/logsai/streams_app/public/application.tsx similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/application.tsx rename to x-pack/plugins/logsai/streams_app/public/application.tsx diff --git a/x-pack/plugins/logsai/entities_app/public/components/all_entities_view/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/all_entities_view/index.tsx similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/components/all_entities_view/index.tsx rename to x-pack/plugins/logsai/streams_app/public/components/all_entities_view/index.tsx diff --git a/x-pack/plugins/logsai/entities_app/public/components/app_root/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/app_root/index.tsx similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/components/app_root/index.tsx rename to x-pack/plugins/logsai/streams_app/public/components/app_root/index.tsx diff --git a/x-pack/plugins/logsai/entities_app/public/components/data_stream_detail_view/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/data_stream_detail_view/index.tsx similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/components/data_stream_detail_view/index.tsx rename to x-pack/plugins/logsai/streams_app/public/components/data_stream_detail_view/index.tsx diff --git a/x-pack/plugins/logsai/entities_app/public/components/entities_app_context_provider/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/entities_app_context_provider/index.tsx similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/components/entities_app_context_provider/index.tsx rename to x-pack/plugins/logsai/streams_app/public/components/entities_app_context_provider/index.tsx diff --git a/x-pack/plugins/logsai/entities_app/public/components/entities_app_page_header/entities_app_page_header_title.tsx b/x-pack/plugins/logsai/streams_app/public/components/entities_app_page_header/entities_app_page_header_title.tsx similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/components/entities_app_page_header/entities_app_page_header_title.tsx rename to x-pack/plugins/logsai/streams_app/public/components/entities_app_page_header/entities_app_page_header_title.tsx diff --git a/x-pack/plugins/logsai/entities_app/public/components/entities_app_page_header/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/entities_app_page_header/index.tsx similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/components/entities_app_page_header/index.tsx rename to x-pack/plugins/logsai/streams_app/public/components/entities_app_page_header/index.tsx diff --git a/x-pack/plugins/logsai/entities_app/public/components/entities_app_page_template/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/entities_app_page_template/index.tsx similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/components/entities_app_page_template/index.tsx rename to x-pack/plugins/logsai/streams_app/public/components/entities_app_page_template/index.tsx diff --git a/x-pack/plugins/logsai/entities_app/public/components/entities_app_router_breadcrumb/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/entities_app_router_breadcrumb/index.tsx similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/components/entities_app_router_breadcrumb/index.tsx rename to x-pack/plugins/logsai/streams_app/public/components/entities_app_router_breadcrumb/index.tsx diff --git a/x-pack/plugins/logsai/entities_app/public/components/entities_app_search_bar/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/entities_app_search_bar/index.tsx similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/components/entities_app_search_bar/index.tsx rename to x-pack/plugins/logsai/streams_app/public/components/entities_app_search_bar/index.tsx diff --git a/x-pack/plugins/logsai/entities_app/public/components/entity_detail_overview/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/entity_detail_overview/index.tsx similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/components/entity_detail_overview/index.tsx rename to x-pack/plugins/logsai/streams_app/public/components/entity_detail_overview/index.tsx diff --git a/x-pack/plugins/logsai/entities_app/public/components/entity_detail_view/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/entity_detail_view/index.tsx similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/components/entity_detail_view/index.tsx rename to x-pack/plugins/logsai/streams_app/public/components/entity_detail_view/index.tsx diff --git a/x-pack/plugins/logsai/entities_app/public/components/entity_detail_view_header_section/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/entity_detail_view_header_section/index.tsx similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/components/entity_detail_view_header_section/index.tsx rename to x-pack/plugins/logsai/streams_app/public/components/entity_detail_view_header_section/index.tsx diff --git a/x-pack/plugins/logsai/entities_app/public/components/entity_health_status_badge/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/entity_health_status_badge/index.tsx similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/components/entity_health_status_badge/index.tsx rename to x-pack/plugins/logsai/streams_app/public/components/entity_health_status_badge/index.tsx diff --git a/x-pack/plugins/logsai/entities_app/public/components/entity_overview_tab_list/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/entity_overview_tab_list/index.tsx similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/components/entity_overview_tab_list/index.tsx rename to x-pack/plugins/logsai/streams_app/public/components/entity_overview_tab_list/index.tsx diff --git a/x-pack/plugins/logsai/entities_app/public/components/entity_pivot_type_view/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/entity_pivot_type_view/index.tsx similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/components/entity_pivot_type_view/index.tsx rename to x-pack/plugins/logsai/streams_app/public/components/entity_pivot_type_view/index.tsx diff --git a/x-pack/plugins/logsai/entities_app/public/components/entity_table/controlled_entity_table.tsx b/x-pack/plugins/logsai/streams_app/public/components/entity_table/controlled_entity_table.tsx similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/components/entity_table/controlled_entity_table.tsx rename to x-pack/plugins/logsai/streams_app/public/components/entity_table/controlled_entity_table.tsx diff --git a/x-pack/plugins/logsai/entities_app/public/components/entity_table/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/entity_table/index.tsx similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/components/entity_table/index.tsx rename to x-pack/plugins/logsai/streams_app/public/components/entity_table/index.tsx diff --git a/x-pack/plugins/logsai/entities_app/public/components/esql_chart/controlled_esql_chart.tsx b/x-pack/plugins/logsai/streams_app/public/components/esql_chart/controlled_esql_chart.tsx similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/components/esql_chart/controlled_esql_chart.tsx rename to x-pack/plugins/logsai/streams_app/public/components/esql_chart/controlled_esql_chart.tsx diff --git a/x-pack/plugins/logsai/entities_app/public/components/esql_chart/uncontrolled_esql_chart.tsx b/x-pack/plugins/logsai/streams_app/public/components/esql_chart/uncontrolled_esql_chart.tsx similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/components/esql_chart/uncontrolled_esql_chart.tsx rename to x-pack/plugins/logsai/streams_app/public/components/esql_chart/uncontrolled_esql_chart.tsx diff --git a/x-pack/plugins/logsai/entities_app/public/components/esql_grid/controlled_esql_grid.tsx b/x-pack/plugins/logsai/streams_app/public/components/esql_grid/controlled_esql_grid.tsx similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/components/esql_grid/controlled_esql_grid.tsx rename to x-pack/plugins/logsai/streams_app/public/components/esql_grid/controlled_esql_grid.tsx diff --git a/x-pack/plugins/logsai/entities_app/public/components/loading_panel/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/loading_panel/index.tsx similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/components/loading_panel/index.tsx rename to x-pack/plugins/logsai/streams_app/public/components/loading_panel/index.tsx diff --git a/x-pack/plugins/logsai/entities_app/public/components/redirect_to/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/redirect_to/index.tsx similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/components/redirect_to/index.tsx rename to x-pack/plugins/logsai/streams_app/public/components/redirect_to/index.tsx diff --git a/x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_breadcrumbs.ts b/x-pack/plugins/logsai/streams_app/public/hooks/use_entities_app_breadcrumbs.ts similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_breadcrumbs.ts rename to x-pack/plugins/logsai/streams_app/public/hooks/use_entities_app_breadcrumbs.ts diff --git a/x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_fetch.ts b/x-pack/plugins/logsai/streams_app/public/hooks/use_entities_app_fetch.ts similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_fetch.ts rename to x-pack/plugins/logsai/streams_app/public/hooks/use_entities_app_fetch.ts diff --git a/x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_params.ts b/x-pack/plugins/logsai/streams_app/public/hooks/use_entities_app_params.ts similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_params.ts rename to x-pack/plugins/logsai/streams_app/public/hooks/use_entities_app_params.ts diff --git a/x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_route_path.ts b/x-pack/plugins/logsai/streams_app/public/hooks/use_entities_app_route_path.ts similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_route_path.ts rename to x-pack/plugins/logsai/streams_app/public/hooks/use_entities_app_route_path.ts diff --git a/x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_router.ts b/x-pack/plugins/logsai/streams_app/public/hooks/use_entities_app_router.ts similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/hooks/use_entities_app_router.ts rename to x-pack/plugins/logsai/streams_app/public/hooks/use_entities_app_router.ts diff --git a/x-pack/plugins/logsai/entities_app/public/hooks/use_esql_query_result.ts b/x-pack/plugins/logsai/streams_app/public/hooks/use_esql_query_result.ts similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/hooks/use_esql_query_result.ts rename to x-pack/plugins/logsai/streams_app/public/hooks/use_esql_query_result.ts diff --git a/x-pack/plugins/logsai/entities_app/public/hooks/use_kibana.tsx b/x-pack/plugins/logsai/streams_app/public/hooks/use_kibana.tsx similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/hooks/use_kibana.tsx rename to x-pack/plugins/logsai/streams_app/public/hooks/use_kibana.tsx diff --git a/x-pack/plugins/logsai/entities_app/public/index.ts b/x-pack/plugins/logsai/streams_app/public/index.ts similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/index.ts rename to x-pack/plugins/logsai/streams_app/public/index.ts diff --git a/x-pack/plugins/logsai/entities_app/public/plugin.ts b/x-pack/plugins/logsai/streams_app/public/plugin.ts similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/plugin.ts rename to x-pack/plugins/logsai/streams_app/public/plugin.ts diff --git a/x-pack/plugins/logsai/entities_app/public/routes/config.tsx b/x-pack/plugins/logsai/streams_app/public/routes/config.tsx similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/routes/config.tsx rename to x-pack/plugins/logsai/streams_app/public/routes/config.tsx diff --git a/x-pack/plugins/logsai/entities_app/public/services/types.ts b/x-pack/plugins/logsai/streams_app/public/services/types.ts similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/services/types.ts rename to x-pack/plugins/logsai/streams_app/public/services/types.ts diff --git a/x-pack/plugins/logsai/entities_app/public/types.ts b/x-pack/plugins/logsai/streams_app/public/types.ts similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/types.ts rename to x-pack/plugins/logsai/streams_app/public/types.ts diff --git a/x-pack/plugins/logsai/entities_app/public/util/esql_result_to_timeseries.ts b/x-pack/plugins/logsai/streams_app/public/util/esql_result_to_timeseries.ts similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/util/esql_result_to_timeseries.ts rename to x-pack/plugins/logsai/streams_app/public/util/esql_result_to_timeseries.ts diff --git a/x-pack/plugins/logsai/entities_app/public/util/get_initial_columns_for_logs.ts b/x-pack/plugins/logsai/streams_app/public/util/get_initial_columns_for_logs.ts similarity index 100% rename from x-pack/plugins/logsai/entities_app/public/util/get_initial_columns_for_logs.ts rename to x-pack/plugins/logsai/streams_app/public/util/get_initial_columns_for_logs.ts diff --git a/x-pack/plugins/logsai/entities_app/server/config.ts b/x-pack/plugins/logsai/streams_app/server/config.ts similarity index 100% rename from x-pack/plugins/logsai/entities_app/server/config.ts rename to x-pack/plugins/logsai/streams_app/server/config.ts diff --git a/x-pack/plugins/logsai/entities_app/server/index.ts b/x-pack/plugins/logsai/streams_app/server/index.ts similarity index 100% rename from x-pack/plugins/logsai/entities_app/server/index.ts rename to x-pack/plugins/logsai/streams_app/server/index.ts diff --git a/x-pack/plugins/logsai/entities_app/server/plugin.ts b/x-pack/plugins/logsai/streams_app/server/plugin.ts similarity index 100% rename from x-pack/plugins/logsai/entities_app/server/plugin.ts rename to x-pack/plugins/logsai/streams_app/server/plugin.ts diff --git a/x-pack/plugins/logsai/entities_app/server/types.ts b/x-pack/plugins/logsai/streams_app/server/types.ts similarity index 100% rename from x-pack/plugins/logsai/entities_app/server/types.ts rename to x-pack/plugins/logsai/streams_app/server/types.ts diff --git a/x-pack/plugins/logsai/entities_app/tsconfig.json b/x-pack/plugins/logsai/streams_app/tsconfig.json similarity index 100% rename from x-pack/plugins/logsai/entities_app/tsconfig.json rename to x-pack/plugins/logsai/streams_app/tsconfig.json From 999dfefe1702c22ab214ac19aa6b5471e9a8ab8d Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Wed, 6 Nov 2024 13:09:29 +0100 Subject: [PATCH 06/95] Rename stuff & remove stuff --- package.json | 2 + tsconfig.base.json | 4 + .../logsai/streams_api/common/entities.ts | 115 -------- .../logsai/streams_api/common/index.ts | 16 -- .../common/queries/entity_source_query.ts | 37 --- .../common/queries/entity_time_range_query.ts | 27 -- .../common/utils/get_esql_request.ts | 51 ---- .../plugins/logsai/streams_api/jest.config.js | 8 +- .../plugins/logsai/streams_api/kibana.jsonc | 6 +- .../logsai/streams_api/public/api/index.tsx | 28 +- .../logsai/streams_api/public/index.ts | 26 +- .../logsai/streams_api/public/plugin.ts | 36 +-- .../streams_api/public/services/types.ts | 6 +- .../logsai/streams_api/public/types.ts | 14 +- .../server/built_in_definitions_stub.ts | 101 ------- .../logsai/streams_api/server/config.ts | 2 +- .../logsai/streams_api/server/index.ts | 30 +-- .../lib/clients/create_alerts_client.ts | 4 +- .../lib/clients/create_dashboards_client.ts | 4 +- .../lib/clients/create_entity_client.ts | 18 -- .../server/lib/clients/create_rules_client.ts | 4 +- .../server/lib/clients/create_slo_client.ts | 17 -- ...ent.ts => create_streams_api_es_client.ts} | 10 +- ...s_api_span.ts => with_streams_api_span.ts} | 12 +- .../logsai/streams_api/server/plugin.ts | 30 +-- ....ts => create_streams_api_server_route.ts} | 8 +- .../server/routes/entities/get_entities.ts | 135 ---------- .../entities/get_entity_from_type_and_key.ts | 59 ---- .../entities/get_esql_identity_commands.ts | 236 ---------------- .../server/routes/entities/route.ts | 253 ------------------ .../streams_api/server/routes/esql/route.ts | 57 ---- ...et_global_entities_api_route_repository.ts | 22 -- ...et_global_streams_api_route_repository.ts} | 15 +- .../server/routes/register_routes.ts | 8 +- .../logsai/streams_api/server/routes/types.ts | 18 +- .../streams_api/server/routes/types/route.ts | 93 ------- .../logsai/streams_api/server/types.ts | 27 +- .../plugins/logsai/streams_api/tsconfig.json | 18 -- ...t.tsx => get_mock_streams_app_context.tsx} | 8 +- .../.storybook/storybook_decorator.tsx | 10 +- .../plugins/logsai/streams_app/jest.config.js | 10 +- .../plugins/logsai/streams_app/kibana.jsonc | 8 +- .../logsai/streams_app/public/application.tsx | 8 +- .../components/all_entities_view/index.tsx | 10 +- .../public/components/app_root/index.tsx | 22 +- .../data_stream_detail_view/index.tsx | 4 +- .../entity_detail_overview/index.tsx | 16 +- .../components/entity_detail_view/index.tsx | 44 +-- .../entity_health_status_badge/index.tsx | 2 +- .../entity_pivot_type_view/index.tsx | 28 +- .../entity_table/controlled_entity_table.tsx | 10 +- .../public/components/entity_table/index.tsx | 22 +- .../public/components/redirect_to/index.tsx | 14 +- .../index.tsx | 6 +- .../index.tsx | 2 +- .../streams_app_page_header_title.tsx} | 2 +- .../index.tsx | 2 +- .../index.tsx | 4 +- .../index.tsx | 4 +- .../public/hooks/use_esql_query_result.ts | 6 +- .../streams_app/public/hooks/use_kibana.tsx | 14 +- ...umbs.ts => use_streams_app_breadcrumbs.ts} | 4 +- ..._app_fetch.ts => use_streams_app_fetch.ts} | 2 +- ...pp_params.ts => use_streams_app_params.ts} | 8 +- ..._path.ts => use_streams_app_route_path.ts} | 6 +- ...pp_router.ts => use_streams_app_router.ts} | 22 +- .../logsai/streams_app/public/index.ts | 22 +- .../logsai/streams_app/public/plugin.ts | 34 +-- .../streams_app/public/routes/config.tsx | 24 +- .../streams_app/public/services/types.ts | 2 +- .../logsai/streams_app/public/types.ts | 18 +- .../util/get_initial_columns_for_logs.ts | 2 +- .../logsai/streams_app/server/config.ts | 2 +- .../logsai/streams_app/server/index.ts | 28 +- .../logsai/streams_app/server/plugin.ts | 26 +- .../logsai/streams_app/server/types.ts | 8 +- .../plugins/logsai/streams_app/tsconfig.json | 20 -- yarn.lock | 8 + 78 files changed, 382 insertions(+), 1667 deletions(-) delete mode 100644 x-pack/plugins/logsai/streams_api/common/entities.ts delete mode 100644 x-pack/plugins/logsai/streams_api/common/index.ts delete mode 100644 x-pack/plugins/logsai/streams_api/common/queries/entity_source_query.ts delete mode 100644 x-pack/plugins/logsai/streams_api/common/queries/entity_time_range_query.ts delete mode 100644 x-pack/plugins/logsai/streams_api/common/utils/get_esql_request.ts delete mode 100644 x-pack/plugins/logsai/streams_api/server/built_in_definitions_stub.ts delete mode 100644 x-pack/plugins/logsai/streams_api/server/lib/clients/create_entity_client.ts delete mode 100644 x-pack/plugins/logsai/streams_api/server/lib/clients/create_slo_client.ts rename x-pack/plugins/logsai/streams_api/server/lib/clients/{create_entities_api_es_client.ts => create_streams_api_es_client.ts} (64%) rename x-pack/plugins/logsai/streams_api/server/lib/{with_entities_api_span.ts => with_streams_api_span.ts} (63%) rename x-pack/plugins/logsai/streams_api/server/routes/{create_entities_api_server_route.ts => create_streams_api_server_route.ts} (58%) delete mode 100644 x-pack/plugins/logsai/streams_api/server/routes/entities/get_entities.ts delete mode 100644 x-pack/plugins/logsai/streams_api/server/routes/entities/get_entity_from_type_and_key.ts delete mode 100644 x-pack/plugins/logsai/streams_api/server/routes/entities/get_esql_identity_commands.ts delete mode 100644 x-pack/plugins/logsai/streams_api/server/routes/entities/route.ts delete mode 100644 x-pack/plugins/logsai/streams_api/server/routes/esql/route.ts delete mode 100644 x-pack/plugins/logsai/streams_api/server/routes/get_global_entities_api_route_repository.ts rename x-pack/plugins/logsai/streams_api/{common/utils/get_index_patterns_for_filters.ts => server/routes/get_global_streams_api_route_repository.ts} (50%) delete mode 100644 x-pack/plugins/logsai/streams_api/server/routes/types/route.ts rename x-pack/plugins/logsai/streams_app/.storybook/{get_mock_entities_app_context.tsx => get_mock_streams_app_context.tsx} (78%) rename x-pack/plugins/logsai/streams_app/public/components/{entities_app_context_provider => streams_app_context_provider}/index.tsx (81%) rename x-pack/plugins/logsai/streams_app/public/components/{entities_app_page_header => streams_app_page_header}/index.tsx (84%) rename x-pack/plugins/logsai/streams_app/public/components/{entities_app_page_header/entities_app_page_header_title.tsx => streams_app_page_header/streams_app_page_header_title.tsx} (94%) rename x-pack/plugins/logsai/streams_app/public/components/{entities_app_page_template => streams_app_page_template}/index.tsx (93%) rename x-pack/plugins/logsai/streams_app/public/components/{entities_app_router_breadcrumb => streams_app_router_breadcrumb}/index.tsx (67%) rename x-pack/plugins/logsai/streams_app/public/components/{entities_app_search_bar => streams_app_search_bar}/index.tsx (96%) rename x-pack/plugins/logsai/streams_app/public/hooks/{use_entities_app_breadcrumbs.ts => use_streams_app_breadcrumbs.ts} (70%) rename x-pack/plugins/logsai/streams_app/public/hooks/{use_entities_app_fetch.ts => use_streams_app_fetch.ts} (94%) rename x-pack/plugins/logsai/streams_app/public/hooks/{use_entities_app_params.ts => use_streams_app_params.ts} (59%) rename x-pack/plugins/logsai/streams_app/public/hooks/{use_entities_app_route_path.ts => use_streams_app_route_path.ts} (70%) rename x-pack/plugins/logsai/streams_app/public/hooks/{use_entities_app_router.ts => use_streams_app_router.ts} (65%) diff --git a/package.json b/package.json index 4054d80f42312..a9a1a2d6845c8 100644 --- a/package.json +++ b/package.json @@ -928,6 +928,8 @@ "@kbn/status-plugin-a-plugin": "link:test/server_integration/plugins/status_plugin_a", "@kbn/status-plugin-b-plugin": "link:test/server_integration/plugins/status_plugin_b", "@kbn/std": "link:packages/kbn-std", + "@kbn/streams-api-plugin": "link:x-pack/plugins/logsai/streams_api", + "@kbn/streams-app-plugin": "link:x-pack/plugins/logsai/streams_app", "@kbn/streams-plugin": "link:x-pack/plugins/streams", "@kbn/synthetics-plugin": "link:x-pack/plugins/observability_solution/synthetics", "@kbn/synthetics-private-location": "link:x-pack/packages/kbn-synthetics-private-location", diff --git a/tsconfig.base.json b/tsconfig.base.json index b06a2ab957a8b..5dda5431f641a 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1826,6 +1826,10 @@ "@kbn/stdio-dev-helpers/*": ["packages/kbn-stdio-dev-helpers/*"], "@kbn/storybook": ["packages/kbn-storybook"], "@kbn/storybook/*": ["packages/kbn-storybook/*"], + "@kbn/streams-api-plugin": ["x-pack/plugins/logsai/streams_api"], + "@kbn/streams-api-plugin/*": ["x-pack/plugins/logsai/streams_api/*"], + "@kbn/streams-app-plugin": ["x-pack/plugins/logsai/streams_app"], + "@kbn/streams-app-plugin/*": ["x-pack/plugins/logsai/streams_app/*"], "@kbn/streams-plugin": ["x-pack/plugins/streams"], "@kbn/streams-plugin/*": ["x-pack/plugins/streams/*"], "@kbn/synthetics-e2e": ["x-pack/plugins/observability_solution/synthetics/e2e"], diff --git a/x-pack/plugins/logsai/streams_api/common/entities.ts b/x-pack/plugins/logsai/streams_api/common/entities.ts deleted file mode 100644 index 07fb7374187ab..0000000000000 --- a/x-pack/plugins/logsai/streams_api/common/entities.ts +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ValuesType } from 'utility-types'; - -type EntityPivotIdentity = Record; - -export interface EntityDataSource { - index: string | string[]; -} - -export interface IEntity { - id: string; - type: string; - key: string; - displayName: string; -} - -interface IPivotEntity extends IEntity { - identity: EntityPivotIdentity; -} - -export type StoredEntity = IEntity; - -export interface DefinitionEntity extends StoredEntity { - filters: EntityFilter[]; - pivot: Pivot; -} - -export interface Pivot { - type: string; - identityFields: string[]; -} - -export type PivotEntity = IPivotEntity; - -export interface StoredPivotEntity extends StoredEntity, IPivotEntity {} - -interface EntityDefinitionTermFilter { - term: { [x: string]: string }; -} - -interface EntityDefinitionIndexFilter { - index: string[]; -} - -interface EntityDefinitionMatchAllFilter { - match_all: {}; -} - -export interface EntityGrouping { - id: string; - filters: EntityFilter[]; - pivot: Pivot; -} - -export interface EntityDisplayNameTemplate { - concat: Array<{ field: string } | { literal: string }>; -} - -export interface EntityTypeDefinition { - pivot: Pivot; - displayName: string; - displayNameTemplate?: EntityDisplayNameTemplate; -} - -export type EntityFilter = - | EntityDefinitionTermFilter - | EntityDefinitionIndexFilter - | EntityDefinitionMatchAllFilter; - -export type Entity = DefinitionEntity | PivotEntity | StoredPivotEntity; - -export const ENTITY_HEALTH_STATUS = { - Healthy: 'Healthy' as const, - Degraded: 'Degraded' as const, - Violated: 'Violated' as const, - NoData: 'NoData' as const, -}; - -export type EntityHealthStatus = ValuesType; - -export const ENTITY_HEALTH_STATUS_INT = { - [ENTITY_HEALTH_STATUS.Violated]: 4 as const, - [ENTITY_HEALTH_STATUS.Degraded]: 3 as const, - [ENTITY_HEALTH_STATUS.NoData]: 2 as const, - [ENTITY_HEALTH_STATUS.Healthy]: 1 as const, -} satisfies Record; - -const HEALTH_STATUS_INT_TO_KEYWORD = Object.fromEntries( - Object.entries(ENTITY_HEALTH_STATUS_INT).map(([healthStatus, int]) => { - return [int, healthStatus as EntityHealthStatus]; - }) -); - -export const healthStatusIntToKeyword = (value: ValuesType) => { - return HEALTH_STATUS_INT_TO_KEYWORD[value]; -}; - -export type EntityWithSignalStatus = IEntity & { - alertsCount: number; - healthStatus: EntityHealthStatus | null; -}; - -export function isPivotEntity(entity: IEntity): entity is IPivotEntity { - return 'identity' in entity; -} - -export function isDefinitionEntity(entity: IEntity): entity is DefinitionEntity { - return 'filters' in entity; -} diff --git a/x-pack/plugins/logsai/streams_api/common/index.ts b/x-pack/plugins/logsai/streams_api/common/index.ts deleted file mode 100644 index b759a5e7bc8a2..0000000000000 --- a/x-pack/plugins/logsai/streams_api/common/index.ts +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -export type { - Entity, - EntityWithSignalStatus, - EntityHealthStatus, - PivotEntity, - Pivot, -} from './entities'; - -export { getIndexPatternsForFilters } from './utils/get_index_patterns_for_filters'; -export { entitySourceQuery } from './queries/entity_source_query'; diff --git a/x-pack/plugins/logsai/streams_api/common/queries/entity_source_query.ts b/x-pack/plugins/logsai/streams_api/common/queries/entity_source_query.ts deleted file mode 100644 index f91e5c627cc8e..0000000000000 --- a/x-pack/plugins/logsai/streams_api/common/queries/entity_source_query.ts +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; -import { isDefinitionEntity, isPivotEntity, type IEntity } from '../entities'; - -export function entitySourceQuery({ entity }: { entity: IEntity }): QueryDslQueryContainer[] { - if (isPivotEntity(entity)) { - return Object.entries(entity.identity).map(([field, value]) => { - return { - term: { - [field]: value, - }, - }; - }); - } else if (isDefinitionEntity(entity)) { - return entity.filters - .flatMap((filter): QueryDslQueryContainer => { - if ('index' in filter) { - return { - bool: { - should: filter.index.map((index) => ({ wildcard: { _index: index } })), - minimum_should_match: 1, - }, - }; - } - return filter; - }) - .concat(entity.pivot.identityFields.map((field) => ({ exists: { field } }))); - } - - throw new Error(`Could not build query for unknown entity type`); -} diff --git a/x-pack/plugins/logsai/streams_api/common/queries/entity_time_range_query.ts b/x-pack/plugins/logsai/streams_api/common/queries/entity_time_range_query.ts deleted file mode 100644 index 72f65400377c8..0000000000000 --- a/x-pack/plugins/logsai/streams_api/common/queries/entity_time_range_query.ts +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; - -export function entityTimeRangeQuery(start: number, end: number): QueryDslQueryContainer[] { - return [ - { - range: { - 'entity.lastSeenTimestamp': { - gte: start, - }, - }, - }, - { - range: { - 'entity.firstSeenTimestamp': { - lte: end, - }, - }, - }, - ]; -} diff --git a/x-pack/plugins/logsai/streams_api/common/utils/get_esql_request.ts b/x-pack/plugins/logsai/streams_api/common/utils/get_esql_request.ts deleted file mode 100644 index 1dde049db8ade..0000000000000 --- a/x-pack/plugins/logsai/streams_api/common/utils/get_esql_request.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { excludeFrozenQuery } from '@kbn/observability-utils-common/es/queries/exclude_frozen_query'; -import { kqlQuery } from '@kbn/observability-utils-common/es/queries/kql_query'; - -export function getEsqlRequest({ - query, - start, - end, - kuery, - dslFilter, - timestampField, -}: { - query: string; - start?: number; - end?: number; - kuery?: string; - dslFilter?: QueryDslQueryContainer[]; - timestampField?: string; -}) { - return { - query, - filter: { - bool: { - filter: [ - ...(start && end - ? [ - { - range: { - [timestampField ?? '@timestamp']: { - gte: start, - lte: end, - }, - }, - }, - ] - : []), - ...excludeFrozenQuery(), - ...kqlQuery(kuery), - ...(dslFilter ?? []), - ], - }, - }, - }; -} diff --git a/x-pack/plugins/logsai/streams_api/jest.config.js b/x-pack/plugins/logsai/streams_api/jest.config.js index 0e49642be1469..89a3c5927205a 100644 --- a/x-pack/plugins/logsai/streams_api/jest.config.js +++ b/x-pack/plugins/logsai/streams_api/jest.config.js @@ -9,14 +9,14 @@ module.exports = { preset: '@kbn/test', rootDir: '../../../..', roots: [ - '/x-pack/plugins/logsai/entities_api/public', - '/x-pack/plugins/logsai/entities_api/common', - '/x-pack/plugins/logsai/entities_api/server', + '/x-pack/plugins/logsai/streams_api/public', + '/x-pack/plugins/logsai/streams_api/common', + '/x-pack/plugins/logsai/streams_api/server', ], setupFiles: [], collectCoverage: true, collectCoverageFrom: [ - '/x-pack/plugins/logsai/entities_api/{public,common,server}/**/*.{js,ts,tsx}', + '/x-pack/plugins/logsai/streams_api/{public,common,server}/**/*.{js,ts,tsx}', ], coverageReporters: ['html'], diff --git a/x-pack/plugins/logsai/streams_api/kibana.jsonc b/x-pack/plugins/logsai/streams_api/kibana.jsonc index 285bd1473b293..74753c8b6841b 100644 --- a/x-pack/plugins/logsai/streams_api/kibana.jsonc +++ b/x-pack/plugins/logsai/streams_api/kibana.jsonc @@ -1,12 +1,12 @@ { "type": "plugin", - "id": "@kbn/entities-api-plugin", + "id": "@kbn/streams-api-plugin", "owner": "@elastic/observability-ui", "plugin": { - "id": "entitiesAPI", + "id": "streamsAPI", "server": true, "browser": true, - "configPath": ["xpack", "entitiesAPI"], + "configPath": ["xpack", "streamsAPI"], "requiredPlugins": [ "observabilityShared", "inference", diff --git a/x-pack/plugins/logsai/streams_api/public/api/index.tsx b/x-pack/plugins/logsai/streams_api/public/api/index.tsx index 995a4ca0b0b80..39f84b1618bcb 100644 --- a/x-pack/plugins/logsai/streams_api/public/api/index.tsx +++ b/x-pack/plugins/logsai/streams_api/public/api/index.tsx @@ -12,39 +12,39 @@ import type { RouteRepositoryClient, } from '@kbn/server-route-repository'; import { createRepositoryClient } from '@kbn/server-route-repository-client'; -import type { EntitiesAPIServerRouteRepository as EntitiesAPIServerRouteRepository } from '../../server'; +import type { StreamsAPIServerRouteRepository as StreamsAPIServerRouteRepository } from '../../server'; type FetchOptions = Omit & { body?: any; }; -export type EntitiesAPIClientOptions = Omit< +export type StreamsAPIClientOptions = Omit< FetchOptions, 'query' | 'body' | 'pathname' | 'signal' > & { signal: AbortSignal | null; }; -export type EntitiesAPIClient = RouteRepositoryClient< - EntitiesAPIServerRouteRepository, - EntitiesAPIClientOptions +export type StreamsAPIClient = RouteRepositoryClient< + StreamsAPIServerRouteRepository, + StreamsAPIClientOptions >; -export type AutoAbortedEntitiesAPIClient = RouteRepositoryClient< - EntitiesAPIServerRouteRepository, - Omit +export type AutoAbortedStreamsAPIClient = RouteRepositoryClient< + StreamsAPIServerRouteRepository, + Omit >; -export type EntitiesAPIEndpoint = keyof EntitiesAPIServerRouteRepository; +export type StreamsAPIEndpoint = keyof StreamsAPIServerRouteRepository; -export type APIReturnType = ReturnOf< - EntitiesAPIServerRouteRepository, +export type APIReturnType = ReturnOf< + StreamsAPIServerRouteRepository, TEndpoint >; -export type EntitiesAPIClientRequestParamsOf = - ClientRequestParamsOf; +export type StreamsAPIClientRequestParamsOf = + ClientRequestParamsOf; -export function createEntitiesAPIClient(core: CoreStart | CoreSetup): EntitiesAPIClient { +export function createStreamsAPIClient(core: CoreStart | CoreSetup): StreamsAPIClient { return createRepositoryClient(core); } diff --git a/x-pack/plugins/logsai/streams_api/public/index.ts b/x-pack/plugins/logsai/streams_api/public/index.ts index 87fd6b5519616..d921276420539 100644 --- a/x-pack/plugins/logsai/streams_api/public/index.ts +++ b/x-pack/plugins/logsai/streams_api/public/index.ts @@ -6,25 +6,21 @@ */ import type { PluginInitializer, PluginInitializerContext } from '@kbn/core/public'; -import { EntitiesAPIPlugin } from './plugin'; +import { StreamsAPIPlugin } from './plugin'; import type { - EntitiesAPIPublicSetup, - EntitiesAPIPublicStart, - EntitiesAPISetupDependencies, - EntitiesAPIStartDependencies, + StreamsAPIPublicSetup, + StreamsAPIPublicStart, + StreamsAPISetupDependencies, + StreamsAPIStartDependencies, ConfigSchema, } from './types'; -export type { EntitiesAPIPublicSetup, EntitiesAPIPublicStart }; +export type { StreamsAPIPublicSetup, StreamsAPIPublicStart }; export const plugin: PluginInitializer< - EntitiesAPIPublicSetup, - EntitiesAPIPublicStart, - EntitiesAPISetupDependencies, - EntitiesAPIStartDependencies + StreamsAPIPublicSetup, + StreamsAPIPublicStart, + StreamsAPISetupDependencies, + StreamsAPIStartDependencies > = (pluginInitializerContext: PluginInitializerContext) => - new EntitiesAPIPlugin(pluginInitializerContext); - -export { getIndexPatternsForFilters, entitySourceQuery } from '../common'; - -export type { Entity } from '../common'; + new StreamsAPIPlugin(pluginInitializerContext); diff --git a/x-pack/plugins/logsai/streams_api/public/plugin.ts b/x-pack/plugins/logsai/streams_api/public/plugin.ts index a697c842986b3..2b14c6b479716 100644 --- a/x-pack/plugins/logsai/streams_api/public/plugin.ts +++ b/x-pack/plugins/logsai/streams_api/public/plugin.ts @@ -7,44 +7,44 @@ import { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/public'; import type { Logger } from '@kbn/logging'; -import { EntitiesAPIClient, createEntitiesAPIClient } from './api'; +import { StreamsAPIClient, createStreamsAPIClient } from './api'; import type { ConfigSchema, - EntitiesAPIPublicSetup, - EntitiesAPIPublicStart, - EntitiesAPISetupDependencies, - EntitiesAPIStartDependencies, + StreamsAPIPublicSetup, + StreamsAPIPublicStart, + StreamsAPISetupDependencies, + StreamsAPIStartDependencies, } from './types'; -export class EntitiesAPIPlugin +export class StreamsAPIPlugin implements Plugin< - EntitiesAPIPublicSetup, - EntitiesAPIPublicStart, - EntitiesAPISetupDependencies, - EntitiesAPIStartDependencies + StreamsAPIPublicSetup, + StreamsAPIPublicStart, + StreamsAPISetupDependencies, + StreamsAPIStartDependencies > { logger: Logger; - entitiesAPIClient!: EntitiesAPIClient; + streamsAPIClient!: StreamsAPIClient; constructor(context: PluginInitializerContext) { this.logger = context.logger.get(); } setup( - coreSetup: CoreSetup, - pluginsSetup: EntitiesAPISetupDependencies - ): EntitiesAPIPublicSetup { - const entitiesAPIClient = (this.entitiesAPIClient = createEntitiesAPIClient(coreSetup)); + coreSetup: CoreSetup, + pluginsSetup: StreamsAPISetupDependencies + ): StreamsAPIPublicSetup { + const streamsAPIClient = (this.streamsAPIClient = createStreamsAPIClient(coreSetup)); return { - entitiesAPIClient, + streamsAPIClient, }; } - start(coreStart: CoreStart, pluginsStart: EntitiesAPIStartDependencies): EntitiesAPIPublicStart { + start(coreStart: CoreStart, pluginsStart: StreamsAPIStartDependencies): StreamsAPIPublicStart { return { - entitiesAPIClient: this.entitiesAPIClient, + streamsAPIClient: this.streamsAPIClient, }; } } diff --git a/x-pack/plugins/logsai/streams_api/public/services/types.ts b/x-pack/plugins/logsai/streams_api/public/services/types.ts index 73ee15972873f..5fdd8d61e23a6 100644 --- a/x-pack/plugins/logsai/streams_api/public/services/types.ts +++ b/x-pack/plugins/logsai/streams_api/public/services/types.ts @@ -5,8 +5,8 @@ * 2.0. */ -import type { EntitiesAPIClient } from '../api'; +import type { StreamsAPIClient } from '../api'; -export interface EntitiesAPIServices { - entitiesAPIClient: EntitiesAPIClient; +export interface StreamsAPIServices { + streamsAPIClient: StreamsAPIClient; } diff --git a/x-pack/plugins/logsai/streams_api/public/types.ts b/x-pack/plugins/logsai/streams_api/public/types.ts index 46c300ce6e3f4..577135a4ff439 100644 --- a/x-pack/plugins/logsai/streams_api/public/types.ts +++ b/x-pack/plugins/logsai/streams_api/public/types.ts @@ -5,20 +5,20 @@ * 2.0. */ -import { EntitiesAPIClient } from './api'; +import { StreamsAPIClient } from './api'; /* eslint-disable @typescript-eslint/no-empty-interface*/ export interface ConfigSchema {} -export interface EntitiesAPISetupDependencies {} +export interface StreamsAPISetupDependencies {} -export interface EntitiesAPIStartDependencies {} +export interface StreamsAPIStartDependencies {} -export interface EntitiesAPIPublicSetup { - entitiesAPIClient: EntitiesAPIClient; +export interface StreamsAPIPublicSetup { + streamsAPIClient: StreamsAPIClient; } -export interface EntitiesAPIPublicStart { - entitiesAPIClient: EntitiesAPIClient; +export interface StreamsAPIPublicStart { + streamsAPIClient: StreamsAPIClient; } diff --git a/x-pack/plugins/logsai/streams_api/server/built_in_definitions_stub.ts b/x-pack/plugins/logsai/streams_api/server/built_in_definitions_stub.ts deleted file mode 100644 index f3e58415b7d4a..0000000000000 --- a/x-pack/plugins/logsai/streams_api/server/built_in_definitions_stub.ts +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { DefinitionEntity, EntityTypeDefinition } from '../common/entities'; - -export const allDataStreamsEntity: DefinitionEntity = { - id: 'data_streams', - key: 'data_streams', - displayName: 'Data streams', - type: 'data_stream', - pivot: { - type: 'data_stream', - identityFields: ['data_stream.dataset', 'data_stream.type', 'data_stream.namespace'], - }, - filters: [ - { - index: ['logs-*', 'metrics-*', 'traces-*', '.data_streams'], - }, - ], -}; - -export const allLogsEntity: DefinitionEntity = { - id: 'all_logs', - key: 'all_logs', - displayName: 'logs-*', - type: 'data_stream', - pivot: { - type: 'data_stream', - identityFields: ['data_stream.dataset', 'data_stream.type', 'data_stream.namespace'], - }, - filters: [ - { - index: ['logs-*', '.data_streams'], - }, - { - term: { - 'data_stream.type': 'logs', - }, - }, - ], -}; - -export const allMetricsEntity: DefinitionEntity = { - id: 'all_metrics', - key: 'all_metrics', - displayName: 'metrics-*', - type: 'data_stream', - pivot: { - type: 'data_stream', - identityFields: ['data_stream.dataset', 'data_stream.type', 'data_stream.namespace'], - }, - filters: [ - { - index: ['metrics-*', '.data_streams'], - }, - { - term: { - 'data_stream.type': 'metrics', - }, - }, - ], -}; - -// export const allLogsEntity: DefinitionEntity = { -// id: 'all_logs', -// displayName: 'All logs', -// type: 'data_stream', -// pivot: { -// identityFields: ['data_stream.dataset', 'data_stream.type', 'data_stream.namespace'], -// }, -// filters: [ -// { -// index: ['logs-*'], -// }, -// ], -// }; - -export const builtinEntityDefinitions = [allDataStreamsEntity, allLogsEntity, allMetricsEntity]; - -const dataStreamTypeDefinition: EntityTypeDefinition = { - displayName: 'Data streams', - displayNameTemplate: { - concat: [ - { field: 'data_stream.type' }, - { literal: '-' }, - { field: 'data_stream.dataset' }, - { literal: '-' }, - { field: 'data_stream.namespace' }, - ], - }, - pivot: { - type: 'data_stream', - identityFields: ['data_stream.type', 'data_stream.dataset', 'data_stream.namespace'], - }, -}; - -export const builtinTypeDefinitions = [dataStreamTypeDefinition]; diff --git a/x-pack/plugins/logsai/streams_api/server/config.ts b/x-pack/plugins/logsai/streams_api/server/config.ts index 4e943ec83302d..356f19481709d 100644 --- a/x-pack/plugins/logsai/streams_api/server/config.ts +++ b/x-pack/plugins/logsai/streams_api/server/config.ts @@ -11,4 +11,4 @@ export const config = schema.object({ enabled: schema.boolean({ defaultValue: true }), }); -export type EntitiesAPIConfig = TypeOf; +export type StreamsAPIConfig = TypeOf; diff --git a/x-pack/plugins/logsai/streams_api/server/index.ts b/x-pack/plugins/logsai/streams_api/server/index.ts index 37a96a0f8ec9e..a33eedcdfda9a 100644 --- a/x-pack/plugins/logsai/streams_api/server/index.ts +++ b/x-pack/plugins/logsai/streams_api/server/index.ts @@ -9,29 +9,29 @@ import type { PluginInitializer, PluginInitializerContext, } from '@kbn/core/server'; -import type { EntitiesAPIConfig } from './config'; +import type { StreamsAPIConfig } from './config'; import type { - EntitiesAPIServerStart, - EntitiesAPIServerSetup, - EntitiesAPISetupDependencies, - EntitiesAPIStartDependencies, + StreamsAPIServerStart, + StreamsAPIServerSetup, + StreamsAPISetupDependencies, + StreamsAPIStartDependencies, } from './types'; -export type { EntitiesAPIServerRouteRepository } from './routes/get_global_entities_api_route_repository'; +export type { StreamsAPIServerRouteRepository } from './routes/get_global_streams_api_route_repository'; -export type { EntitiesAPIServerSetup, EntitiesAPIServerStart }; +export type { StreamsAPIServerSetup, StreamsAPIServerStart }; import { config as configSchema } from './config'; -import { EntitiesAPIPlugin } from './plugin'; +import { StreamsAPIPlugin } from './plugin'; -export const config: PluginConfigDescriptor = { +export const config: PluginConfigDescriptor = { schema: configSchema, }; export const plugin: PluginInitializer< - EntitiesAPIServerSetup, - EntitiesAPIServerStart, - EntitiesAPISetupDependencies, - EntitiesAPIStartDependencies -> = async (pluginInitializerContext: PluginInitializerContext) => - new EntitiesAPIPlugin(pluginInitializerContext); + StreamsAPIServerSetup, + StreamsAPIServerStart, + StreamsAPISetupDependencies, + StreamsAPIStartDependencies +> = async (pluginInitializerContext: PluginInitializerContext) => + new StreamsAPIPlugin(pluginInitializerContext); diff --git a/x-pack/plugins/logsai/streams_api/server/lib/clients/create_alerts_client.ts b/x-pack/plugins/logsai/streams_api/server/lib/clients/create_alerts_client.ts index f8cf9f32b9933..5fff67d75ede9 100644 --- a/x-pack/plugins/logsai/streams_api/server/lib/clients/create_alerts_client.ts +++ b/x-pack/plugins/logsai/streams_api/server/lib/clients/create_alerts_client.ts @@ -6,10 +6,10 @@ */ import type { AlertsClient } from '@kbn/rule-registry-plugin/server'; -import type { EntitiesAPIRouteHandlerResources } from '../../routes/types'; +import type { StreamsAPIRouteHandlerResources } from '../../routes/types'; export async function createAlertsClient( - resources: Pick + resources: Pick ): Promise { return await resources.plugins.ruleRegistry .start() diff --git a/x-pack/plugins/logsai/streams_api/server/lib/clients/create_dashboards_client.ts b/x-pack/plugins/logsai/streams_api/server/lib/clients/create_dashboards_client.ts index e94efb1d07244..ca29728195fb3 100644 --- a/x-pack/plugins/logsai/streams_api/server/lib/clients/create_dashboards_client.ts +++ b/x-pack/plugins/logsai/streams_api/server/lib/clients/create_dashboards_client.ts @@ -7,7 +7,7 @@ import type { SavedObject, SavedObjectsFindResult } from '@kbn/core/server'; import type { DashboardAttributes } from '@kbn/dashboard-plugin/common'; -import { EntitiesAPIRouteHandlerResources } from '../../routes/types'; +import { StreamsAPIRouteHandlerResources } from '../../routes/types'; interface DashboardsClient { getAllDashboards: () => Promise>>; @@ -15,7 +15,7 @@ interface DashboardsClient { } export async function createDashboardsClient( - resources: Pick + resources: Pick ): Promise { const soClient = (await resources.context.core).savedObjects.client; async function getDashboards( diff --git a/x-pack/plugins/logsai/streams_api/server/lib/clients/create_entity_client.ts b/x-pack/plugins/logsai/streams_api/server/lib/clients/create_entity_client.ts deleted file mode 100644 index b7997227a9270..0000000000000 --- a/x-pack/plugins/logsai/streams_api/server/lib/clients/create_entity_client.ts +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { EntityClient } from '@kbn/entityManager-plugin/server/lib/entity_client'; -import type { EntitiesAPIRouteHandlerResources } from '../../routes/types'; - -export async function createEntityClient({ - plugins, - request, -}: Pick): Promise { - return await plugins.entityManager - .start() - .then((entityManagerStart) => entityManagerStart.getScopedClient({ request })); -} diff --git a/x-pack/plugins/logsai/streams_api/server/lib/clients/create_rules_client.ts b/x-pack/plugins/logsai/streams_api/server/lib/clients/create_rules_client.ts index 651f0a9f48ebb..1add3009f2f75 100644 --- a/x-pack/plugins/logsai/streams_api/server/lib/clients/create_rules_client.ts +++ b/x-pack/plugins/logsai/streams_api/server/lib/clients/create_rules_client.ts @@ -6,10 +6,10 @@ */ import type { RulesClient } from '@kbn/alerting-plugin/server'; -import type { EntitiesAPIRouteHandlerResources } from '../../routes/types'; +import type { StreamsAPIRouteHandlerResources } from '../../routes/types'; export async function createRulesClient( - resources: Pick + resources: Pick ): Promise { return await resources.plugins.alerting .start() diff --git a/x-pack/plugins/logsai/streams_api/server/lib/clients/create_slo_client.ts b/x-pack/plugins/logsai/streams_api/server/lib/clients/create_slo_client.ts deleted file mode 100644 index 66f276e93292e..0000000000000 --- a/x-pack/plugins/logsai/streams_api/server/lib/clients/create_slo_client.ts +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { SloClient } from '@kbn/slo-plugin/server'; -import type { EntitiesAPIRouteHandlerResources } from '../../routes/types'; - -export async function createSloClient( - resources: Pick -): Promise { - return await resources.plugins.slo - .start() - .then((sloStart) => sloStart.getSloClientWithRequest(resources.request)); -} diff --git a/x-pack/plugins/logsai/streams_api/server/lib/clients/create_entities_api_es_client.ts b/x-pack/plugins/logsai/streams_api/server/lib/clients/create_streams_api_es_client.ts similarity index 64% rename from x-pack/plugins/logsai/streams_api/server/lib/clients/create_entities_api_es_client.ts rename to x-pack/plugins/logsai/streams_api/server/lib/clients/create_streams_api_es_client.ts index 321c25c5d2214..9d7037f56a76d 100644 --- a/x-pack/plugins/logsai/streams_api/server/lib/clients/create_entities_api_es_client.ts +++ b/x-pack/plugins/logsai/streams_api/server/lib/clients/create_streams_api_es_client.ts @@ -5,17 +5,17 @@ * 2.0. */ -import { createObservabilityEsClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; -import { EntitiesAPIRouteHandlerResources } from '../../routes/types'; +import { createObservabilityEsClient } from '@kbn/observability-utils/es/client/create_observability_es_client'; +import { StreamsAPIRouteHandlerResources } from '../../routes/types'; -export async function createEntitiesAPIEsClient({ +export async function createStreamsAPIEsClient({ context, logger, -}: Pick) { +}: Pick) { const esClient = createObservabilityEsClient({ client: (await context.core).elasticsearch.client.asCurrentUser, logger, - plugin: 'entitiesApi', + plugin: 'streamsApi', }); return esClient; diff --git a/x-pack/plugins/logsai/streams_api/server/lib/with_entities_api_span.ts b/x-pack/plugins/logsai/streams_api/server/lib/with_streams_api_span.ts similarity index 63% rename from x-pack/plugins/logsai/streams_api/server/lib/with_entities_api_span.ts rename to x-pack/plugins/logsai/streams_api/server/lib/with_streams_api_span.ts index d5f9b4e92efb1..f9c6c6ff23b6c 100644 --- a/x-pack/plugins/logsai/streams_api/server/lib/with_entities_api_span.ts +++ b/x-pack/plugins/logsai/streams_api/server/lib/with_streams_api_span.ts @@ -1,9 +1,3 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License @@ -13,7 +7,7 @@ import { withSpan, SpanOptions, parseSpanOptions } from '@kbn/apm-utils'; import { Logger } from '@kbn/logging'; -export function withEntitiesAPISpan( +export function withStreamsAPISpan( optionsOrName: SpanOptions | string, cb: () => Promise, logger: Logger @@ -21,10 +15,10 @@ export function withEntitiesAPISpan( const options = parseSpanOptions(optionsOrName); const optionsWithDefaults = { - ...(options.intercept ? {} : { type: 'plugin:entitiesApi' }), + ...(options.intercept ? {} : { type: 'plugin:streamsAPI' }), ...options, labels: { - plugin: 'entitiesApi', + plugin: 'streamsAPI', ...options.labels, }, }; diff --git a/x-pack/plugins/logsai/streams_api/server/plugin.ts b/x-pack/plugins/logsai/streams_api/server/plugin.ts index f407d22e81605..105a348c06b88 100644 --- a/x-pack/plugins/logsai/streams_api/server/plugin.ts +++ b/x-pack/plugins/logsai/streams_api/server/plugin.ts @@ -9,22 +9,22 @@ import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kb import type { Logger } from '@kbn/logging'; import { mapValues } from 'lodash'; import { registerServerRoutes } from './routes/register_routes'; -import { EntitiesAPIRouteHandlerResources } from './routes/types'; +import { StreamsAPIRouteHandlerResources } from './routes/types'; import type { ConfigSchema, - EntitiesAPIServerSetup, - EntitiesAPIServerStart, - EntitiesAPISetupDependencies, - EntitiesAPIStartDependencies, + StreamsAPIServerSetup, + StreamsAPIServerStart, + StreamsAPISetupDependencies, + StreamsAPIStartDependencies, } from './types'; -export class EntitiesAPIPlugin +export class StreamsAPIPlugin implements Plugin< - EntitiesAPIServerSetup, - EntitiesAPIServerStart, - EntitiesAPISetupDependencies, - EntitiesAPIStartDependencies + StreamsAPIServerSetup, + StreamsAPIServerStart, + StreamsAPISetupDependencies, + StreamsAPIStartDependencies > { logger: Logger; @@ -33,9 +33,9 @@ export class EntitiesAPIPlugin this.logger = context.logger.get(); } setup( - coreSetup: CoreSetup, - pluginsSetup: EntitiesAPISetupDependencies - ): EntitiesAPIServerSetup { + coreSetup: CoreSetup, + pluginsSetup: StreamsAPISetupDependencies + ): StreamsAPIServerSetup { const startServicesPromise = coreSetup .getStartServices() .then(([_coreStart, pluginsStart]) => pluginsStart); @@ -52,13 +52,13 @@ export class EntitiesAPIPlugin ), setup: () => value, }; - }) as unknown as EntitiesAPIRouteHandlerResources['plugins'], + }) as unknown as StreamsAPIRouteHandlerResources['plugins'], }, }); return {}; } - start(core: CoreStart, pluginsStart: EntitiesAPIStartDependencies): EntitiesAPIServerStart { + start(core: CoreStart, pluginsStart: StreamsAPIStartDependencies): StreamsAPIServerStart { return {}; } } diff --git a/x-pack/plugins/logsai/streams_api/server/routes/create_entities_api_server_route.ts b/x-pack/plugins/logsai/streams_api/server/routes/create_streams_api_server_route.ts similarity index 58% rename from x-pack/plugins/logsai/streams_api/server/routes/create_entities_api_server_route.ts rename to x-pack/plugins/logsai/streams_api/server/routes/create_streams_api_server_route.ts index 0b76e703d0531..d3d2cecb42d48 100644 --- a/x-pack/plugins/logsai/streams_api/server/routes/create_entities_api_server_route.ts +++ b/x-pack/plugins/logsai/streams_api/server/routes/create_streams_api_server_route.ts @@ -5,9 +5,9 @@ * 2.0. */ import { createServerRouteFactory } from '@kbn/server-route-repository'; -import type { EntitiesAPIRouteCreateOptions, EntitiesAPIRouteHandlerResources } from './types'; +import type { StreamsAPIRouteCreateOptions, StreamsAPIRouteHandlerResources } from './types'; -export const createEntitiesAPIServerRoute = createServerRouteFactory< - EntitiesAPIRouteHandlerResources, - EntitiesAPIRouteCreateOptions +export const createStreamsAPIServerRoute = createServerRouteFactory< + StreamsAPIRouteHandlerResources, + StreamsAPIRouteCreateOptions >(); diff --git a/x-pack/plugins/logsai/streams_api/server/routes/entities/get_entities.ts b/x-pack/plugins/logsai/streams_api/server/routes/entities/get_entities.ts deleted file mode 100644 index 47f103a481446..0000000000000 --- a/x-pack/plugins/logsai/streams_api/server/routes/entities/get_entities.ts +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types'; -import type { Logger } from '@kbn/logging'; -import type { ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; -import type { AlertsClient } from '@kbn/rule-registry-plugin/server'; -import { SloClient } from '@kbn/slo-plugin/server'; -import type { - DefinitionEntity, - EntityDataSource, - EntityTypeDefinition, - EntityWithSignalStatus, -} from '../../../common/entities'; -import { EntityGrouping, healthStatusIntToKeyword } from '../../../common/entities'; -import { querySignalsAsEntities } from '../../lib/entities/query_signals_as_entities'; -import { querySourcesAsEntities } from '../../lib/entities/query_sources_as_entities'; -import { withEntitiesAPISpan } from '../../lib/with_entities_api_span'; - -export async function getEntities({ - currentUserEsClient, - internalUserEsClient, - start, - end, - sourceRangeQuery, - groupings, - typeDefinitions, - sources, - logger, - filters, - alertsClient, - sloClient, - sortField, - sortOrder, - postFilter, - spaceId, -}: { - currentUserEsClient: ObservabilityElasticsearchClient; - internalUserEsClient: ObservabilityElasticsearchClient; - start: number; - end: number; - sourceRangeQuery?: QueryDslQueryContainer; - sources: EntityDataSource[]; - groupings: Array; - typeDefinitions: EntityTypeDefinition[]; - logger: Logger; - filters?: QueryDslQueryContainer[]; - alertsClient: AlertsClient; - sloClient: SloClient; - sortField: 'entity.type' | 'entity.displayName' | 'healthStatus' | 'alertsCount'; - sortOrder: 'asc' | 'desc'; - postFilter?: string; - spaceId: string; -}): Promise { - if (!groupings.length) { - throw new Error('No groupings were defined'); - } - - return withEntitiesAPISpan( - 'get_latest_entities', - async () => { - const entitiesWithSignals = await querySignalsAsEntities({ - logger, - start, - end, - spaceId, - alertsClient, - esClient: internalUserEsClient, - groupings, - typeDefinitions, - sloClient, - filters, - }); - - const entitiesFromSources = await querySourcesAsEntities({ - logger, - esClient: currentUserEsClient, - groupings, - typeDefinitions, - sources, - filters, - sortField, - sortOrder, - postFilter, - tables: [ - { - name: 'signals', - joins: ['entity.id'], - columns: entitiesWithSignals.reduce( - (prev, current) => { - prev['entity.id'].keyword.push(current.id); - prev.alertsCount.long.push(current.alertsCount); - prev.healthStatus.long.push(current.healthStatus); - return prev; - }, - { - 'entity.id': { keyword: [] as string[] }, - alertsCount: { - long: [] as Array, - }, - healthStatus: { long: [] as Array }, - } - ), - }, - ], - rangeQuery: sourceRangeQuery ?? { - range: { - '@timestamp': { - gte: start, - lte: end, - }, - }, - }, - }); - - return entitiesFromSources.map((entity) => { - const { columns, ...base } = entity; - - return { - ...base, - healthStatus: - entity.columns.healthStatus !== null - ? healthStatusIntToKeyword(entity.columns.healthStatus as 1 | 2 | 3 | 4) - : null, - alertsCount: entity.columns.alertsCount as number, - }; - }); - }, - logger - ); -} diff --git a/x-pack/plugins/logsai/streams_api/server/routes/entities/get_entity_from_type_and_key.ts b/x-pack/plugins/logsai/streams_api/server/routes/entities/get_entity_from_type_and_key.ts deleted file mode 100644 index ce016ecec9e9c..0000000000000 --- a/x-pack/plugins/logsai/streams_api/server/routes/entities/get_entity_from_type_and_key.ts +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { notFound } from '@hapi/boom'; -import { ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; -import { pivotEntityFromTypeAndKey } from './get_esql_identity_commands'; -import { DefinitionEntity, Entity, EntityTypeDefinition } from '../../../common/entities'; - -export async function getEntityFromTypeAndKey({ - esClient, - type, - key, - definitionEntities, - typeDefinitions, -}: { - esClient: ObservabilityElasticsearchClient; - type: string; - key: string; - definitionEntities: DefinitionEntity[]; - typeDefinitions: EntityTypeDefinition[]; -}): Promise<{ - entity: Entity; - typeDefinition: EntityTypeDefinition; -}> { - const typeDefinition = typeDefinitions.find((typeDef) => typeDef.pivot.type === type); - - if (!typeDefinition) { - throw notFound(`Could not find type definition for type ${type}`); - } - - const definitionsForType = definitionEntities.filter((definition) => definition.type === type); - - if (!definitionsForType.length) { - throw notFound(`Could not find definition for type ${type}`); - } - - const entityAsDefinition = definitionsForType.find((definition) => definition.key === key); - - if (entityAsDefinition) { - return { - entity: entityAsDefinition, - typeDefinition, - }; - } - - return { - entity: pivotEntityFromTypeAndKey({ - type, - key, - identityFields: typeDefinition.pivot.identityFields, - displayNameTemplate: typeDefinition.displayNameTemplate, - }), - typeDefinition, - }; -} diff --git a/x-pack/plugins/logsai/streams_api/server/routes/entities/get_esql_identity_commands.ts b/x-pack/plugins/logsai/streams_api/server/routes/entities/get_esql_identity_commands.ts deleted file mode 100644 index 90ce040b98a7d..0000000000000 --- a/x-pack/plugins/logsai/streams_api/server/routes/entities/get_esql_identity_commands.ts +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { compact, uniq, uniqBy } from 'lodash'; -import { - EntityGrouping, - Pivot, - EntityFilter, - PivotEntity, - EntityTypeDefinition, - DefinitionEntity, - Entity, - IEntity, - EntityDisplayNameTemplate, -} from '../../../common/entities'; -import { escapeColumn, escapeString } from '../../../common/utils/esql_escape'; - -const ENTITY_MISSING_VALUE_STRING = '__EMPTY__'; -export const ENTITY_ID_SEPARATOR = '@'; -const ENTITY_KEYS_SEPARATOR = '/'; -const ENTITY_ID_LIST_SEPARATOR = ';'; - -export function entityFromIdentifiers({ - entity, - typeDefinition, - definitionEntities, -}: { - entity: IEntity; - typeDefinition: EntityTypeDefinition | undefined; - definitionEntities: Map; -}): Entity | undefined { - if (definitionEntities.has(entity.key)) { - return definitionEntities.get(entity.key)!; - } - - if (!typeDefinition) { - return undefined; - } - - const next = pivotEntityFromTypeAndKey({ - type: entity.type, - key: entity.key, - identityFields: typeDefinition.pivot.identityFields, - displayNameTemplate: typeDefinition.displayNameTemplate, - }); - - return next; -} - -export function pivotEntityFromTypeAndKey({ - type, - key, - identityFields, - displayNameTemplate, -}: { - type: string; - key: string; - identityFields: string[]; - displayNameTemplate: EntityDisplayNameTemplate | undefined; -}): PivotEntity { - const sortedIdentityFields = identityFields.concat().sort(); - - const keys = key.split(ENTITY_KEYS_SEPARATOR); - - const identity = Object.fromEntries( - keys.map((value, index) => { - return [sortedIdentityFields[index], value]; - }) - ); - - const id = `${type}${ENTITY_ID_SEPARATOR}${key}}`; - - let displayName = key; - - if (displayNameTemplate) { - displayName = displayNameTemplate.concat - .map((part) => { - return 'literal' in part ? part.literal : part.field; - }) - .join(''); - } - - return { - id, - type, - key, - identity, - displayName, - }; -} - -function joinArray(source: T[], separator: T): T[] { - return source.flatMap((value, index, array) => { - if (index === array.length - 1) { - return [value]; - } - return [value, separator]; - }); -} - -function getExpressionForPivot(pivot: Pivot) { - const sortedFields = pivot.identityFields.concat().sort(); - - const restArguments = joinArray( - sortedFields.map((field) => { - return escapeColumn(field); - }), - `"${ENTITY_KEYS_SEPARATOR}"` - ).join(', '); - - // CONCAT() - // host@foo - // data_stream@logs;kubernetes-container-logs;elastic-apps - - return `CONCAT("${pivot.type}", "${ENTITY_ID_SEPARATOR}", ${restArguments})`; -} - -function getExpressionForFilter(filter: EntityFilter) { - if ('term' in filter) { - const fieldName = Object.keys(filter.term)[0]; - return `${escapeColumn(fieldName)} == ${escapeString(filter.term[fieldName])}`; - } - if ('index' in filter) { - return `${filter.index - .flatMap((index) => [index, `.ds-${index}-*`]) - .map((index) => `(_index LIKE ${escapeString(`${index}`)})`) - .join(' OR ')}`; - } -} - -function getMatchesExpressionForGrouping(grouping: EntityGrouping) { - const applicableFilters = grouping.filters.filter((filter) => { - return 'term' in filter || 'index' in filter; - }); - - if (applicableFilters.length) { - return `${compact( - applicableFilters.map((filter) => `(${getExpressionForFilter(filter)})`) - ).join(' AND ')}`; - } - - return `true`; -} - -export function getEsqlIdentityCommands({ - groupings, - entityIdExists, - columns, - preaggregate, -}: { - groupings: EntityGrouping[]; - entityIdExists: boolean; - columns: string[]; - preaggregate: boolean; -}): string[] { - const pivotIdExpressions = uniqBy( - groupings.map((grouping) => grouping.pivot), - (pivot) => pivot.type - ).map(getExpressionForPivot); - - const filterClauses = groupings.map((grouping) => { - return { - fieldName: `_matches_group_${grouping.id}`, - type: grouping.pivot.type, - key: grouping.id, - expression: getMatchesExpressionForGrouping(grouping), - }; - }); - - const filterColumnEvals = filterClauses.map(({ fieldName, expression }) => { - return `${escapeColumn(fieldName)} = ${expression}`; - }); - - const filterStatsByColumns = filterClauses.map(({ fieldName }) => { - return `${escapeColumn(fieldName)} = MAX(${escapeColumn(fieldName)})`; - }); - - const filterIdExpressions = filterClauses.map(({ fieldName, type, key }) => { - return `CASE( - ${escapeColumn(fieldName)}, - CONCAT(${escapeString(type)}, "${ENTITY_ID_SEPARATOR}", "${key}"), - NULL - )`; - }); - - const groupingFields = uniq(groupings.flatMap((grouping) => grouping.pivot.identityFields)); - - const groupExpressions = joinArray( - [ - ...(entityIdExists ? [`entity.id`] : []), - ...pivotIdExpressions.concat(filterIdExpressions).map((expression) => { - return `COALESCE(${expression}, "${ENTITY_MISSING_VALUE_STRING}")`; - }), - ], - `"${ENTITY_ID_LIST_SEPARATOR}"` - ).join(', '); - - const entityIdExpression = - groupExpressions.length === 1 - ? groupExpressions[0] - : `MV_DEDUPE( - SPLIT( - CONCAT(${groupExpressions}), - "${ENTITY_ID_LIST_SEPARATOR}" - ) - )`; - - const commands: string[] = [`EVAL ${filterColumnEvals.join(', ')}`]; - - const allColumns = filterStatsByColumns.concat(columns); - - if (preaggregate) { - commands.push(`STATS ${allColumns.join(', ')} BY ${groupingFields.join(', ')}`); - } - - const entityDisplayNameExists = columns.find((column) => column.includes('entity.displayName')); - - return [ - ...commands, - `EVAL entity.id = ${entityIdExpression}`, - `STATS ${columns.join(', ')} BY entity.id`, - `MV_EXPAND entity.id`, - `WHERE entity.id != "${ENTITY_MISSING_VALUE_STRING}"`, - `EVAL entity_identifier = SPLIT(entity.id, "${ENTITY_ID_SEPARATOR}")`, - `EVAL entity.type = MV_FIRST(entity_identifier)`, - `EVAL entity.key = MV_LAST(entity_identifier)`, - entityDisplayNameExists - ? `EVAL entity.displayName = COALESCE(entity.displayName, entity.key)` - : `EVAL entity.displayName = entity.key`, - `DROP entity_identifier`, - ]; -} diff --git a/x-pack/plugins/logsai/streams_api/server/routes/entities/route.ts b/x-pack/plugins/logsai/streams_api/server/routes/entities/route.ts deleted file mode 100644 index d4875d83264ba..0000000000000 --- a/x-pack/plugins/logsai/streams_api/server/routes/entities/route.ts +++ /dev/null @@ -1,253 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { kqlQuery } from '@kbn/observability-utils-common/es/queries/kql_query'; -import { rangeQuery } from '@kbn/observability-utils-common/es/queries/range_query'; -import { createObservabilityEsClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; -import { z } from '@kbn/zod'; -import { - Entity, - entitySourceQuery, - EntityWithSignalStatus, - getIndexPatternsForFilters, -} from '../../../common'; -import { EntityTypeDefinition } from '../../../common/entities'; -import { entityTimeRangeQuery } from '../../../common/queries/entity_time_range_query'; -import { createAlertsClient } from '../../lib/clients/create_alerts_client'; -import { createEntitiesAPIEsClient } from '../../lib/clients/create_entities_api_es_client'; -import { createSloClient } from '../../lib/clients/create_slo_client'; -import { getDataStreamsForFilter } from '../../lib/entities/get_data_streams_for_filter'; -import { getDefinitionEntities } from '../../lib/entities/get_definition_entities'; -import { getTypeDefinitions } from '../../lib/entities/get_type_definitions'; -import { createEntitiesAPIServerRoute } from '../create_entities_api_server_route'; -import { getEntities } from './get_entities'; -import { getEntityFromTypeAndKey } from './get_entity_from_type_and_key'; - -export const findEntitiesRoute = createEntitiesAPIServerRoute({ - endpoint: 'POST /internal/entities_api/entities', - options: { - tags: ['access:entities'], - }, - params: z.object({ - body: z.object({ - start: z.number(), - end: z.number(), - kuery: z.string(), - types: z.array(z.string()), - sortField: z.union([ - z.literal('entity.type'), - z.literal('entity.displayName'), - z.literal('alertsCount'), - z.literal('healthStatus'), - ]), - sortOrder: z.union([z.literal('asc'), z.literal('desc')]), - }), - }), - handler: async (resources): Promise<{ entities: EntityWithSignalStatus[] }> => { - const { context, logger, request } = resources; - - const { - body: { start, end, kuery, types, sortField, sortOrder }, - } = resources.params; - - const [currentUserEsClient, internalUserEsClient, sloClient, alertsClient, spaceId] = - await Promise.all([ - createEntitiesAPIEsClient(resources), - createObservabilityEsClient({ - client: (await context.core).elasticsearch.client.asInternalUser, - logger, - plugin: 'entitiesApi', - }), - createSloClient(resources), - createAlertsClient(resources), - (await resources.plugins.spaces.start()).spacesService.getSpaceId(request), - ]); - - const filters = [...kqlQuery(kuery)]; - - const [definitionEntities, typeDefinitions] = await Promise.all([ - getDefinitionEntities({ - esClient: currentUserEsClient, - }), - getTypeDefinitions({ - esClient: currentUserEsClient, - }), - ]); - - const groupings = definitionEntities - .filter((definitionEntity) => { - return types.includes('all') || types.includes(definitionEntity.type); - }) - .map((definitionEntity) => { - return { - id: definitionEntity.id, - type: definitionEntity.type, - key: definitionEntity.key, - pivot: { - type: definitionEntity.type, - identityFields: definitionEntity.pivot.identityFields, - }, - displayName: definitionEntity.displayName, - filters: definitionEntity.filters, - }; - }); - - const entities = await getEntities({ - start, - end, - alertsClient, - currentUserEsClient, - typeDefinitions, - groupings, - internalUserEsClient, - logger, - sloClient, - sortField, - sortOrder, - sources: [{ index: ['.data_streams'] }], - sourceRangeQuery: { - bool: { - should: [ - { bool: { filter: entityTimeRangeQuery(start, end) } }, - { bool: { filter: rangeQuery(start, end) } }, - { - bool: { - must_not: [ - { - exists: { - field: 'entity.firstSeenTimestamp', - }, - }, - { - exists: { - field: '@timestamp', - }, - }, - ], - }, - }, - ], - minimum_should_match: 1, - }, - }, - spaceId, - filters, - postFilter: undefined, - }); - - return { - entities, - }; - }, -}); - -export const getEntityRoute = createEntitiesAPIServerRoute({ - endpoint: 'GET /internal/entities_api/entity/{type}/{key}', - options: { - tags: ['access:entities'], - }, - params: z.object({ - path: z.object({ - type: z.string(), - key: z.string(), - }), - }), - handler: async ( - resources - ): Promise<{ - entity: Entity; - typeDefinition: EntityTypeDefinition; - }> => { - const { - path: { type, key }, - } = resources.params; - - const esClient = await createEntitiesAPIEsClient(resources); - - const [definitionEntities, typeDefinitions] = await Promise.all([ - getDefinitionEntities({ - esClient, - }), - getTypeDefinitions({ - esClient, - }), - ]); - - return await getEntityFromTypeAndKey({ - esClient, - type, - key, - typeDefinitions, - definitionEntities, - }); - }, -}); - -export const getDataStreamsForEntityRoute = createEntitiesAPIServerRoute({ - endpoint: 'GET /internal/entities_api/entity/{type}/{key}/data_streams', - options: { - tags: ['access:entities'], - }, - params: z.object({ - path: z.object({ - type: z.string(), - key: z.string(), - }), - query: z.object({ - start: z.string(), - end: z.string(), - }), - }), - handler: async (resources): Promise<{ dataStreams: Array<{ name: string }> }> => { - const { - path: { type, key }, - query: { start: startAsString, end: endAsString }, - } = resources.params; - - const start = Number(startAsString); - const end = Number(endAsString); - - const esClient = await createEntitiesAPIEsClient(resources); - - const [definitionEntities, typeDefinitions] = await Promise.all([ - getDefinitionEntities({ - esClient, - }), - getTypeDefinitions({ - esClient, - }), - ]); - - const { entity } = await getEntityFromTypeAndKey({ - esClient, - type, - key, - definitionEntities, - typeDefinitions, - }); - - const foundDataStreams = await getDataStreamsForFilter({ - start, - end, - esClient, - dslFilter: entitySourceQuery({ entity }), - indexPatterns: definitionEntities.flatMap((definition) => - getIndexPatternsForFilters(definition.filters) - ), - }); - - return { - dataStreams: foundDataStreams, - }; - }, -}); - -export const entitiesRoutes = { - ...findEntitiesRoute, - ...getEntityRoute, - ...getDataStreamsForEntityRoute, -}; diff --git a/x-pack/plugins/logsai/streams_api/server/routes/esql/route.ts b/x-pack/plugins/logsai/streams_api/server/routes/esql/route.ts deleted file mode 100644 index efc321b2092f8..0000000000000 --- a/x-pack/plugins/logsai/streams_api/server/routes/esql/route.ts +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ESQLSearchResponse } from '@kbn/es-types'; -import * as t from 'io-ts'; -import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { getEsqlRequest } from '../../../common/utils/get_esql_request'; -import { createEntitiesAPIServerRoute } from '../create_entities_api_server_route'; -import { createEntitiesAPIEsClient } from '../../lib/clients/create_entities_api_es_client'; - -const queryEsqlRoute = createEntitiesAPIServerRoute({ - endpoint: 'POST /internal/entities_api/esql', - params: t.type({ - body: t.intersection([ - t.type({ - query: t.string, - kuery: t.string, - operationName: t.string, - }), - t.partial({ - start: t.number, - end: t.number, - timestampField: t.string, - dslFilter: t.array(t.record(t.string, t.any)), - }), - ]), - }), - options: { - tags: ['access:entities'], - }, - handler: async ({ context, logger, params }): Promise => { - const esClient = await createEntitiesAPIEsClient({ context, logger }); - - const { - body: { query, kuery, start, end, dslFilter, timestampField, operationName }, - } = params; - - const request = getEsqlRequest({ - query, - start, - end, - timestampField, - kuery, - dslFilter: dslFilter as QueryDslQueryContainer[], - }); - - return await esClient.esql(operationName, request); - }, -}); - -export const esqlRoutes = { - ...queryEsqlRoute, -}; diff --git a/x-pack/plugins/logsai/streams_api/server/routes/get_global_entities_api_route_repository.ts b/x-pack/plugins/logsai/streams_api/server/routes/get_global_entities_api_route_repository.ts deleted file mode 100644 index 3a7a26cb11f13..0000000000000 --- a/x-pack/plugins/logsai/streams_api/server/routes/get_global_entities_api_route_repository.ts +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { entitiesRoutes } from './entities/route'; -import { esqlRoutes } from './esql/route'; -import { typesRoutes } from './types/route'; - -export function getGlobalEntitiesAPIServerRouteRepository() { - return { - ...entitiesRoutes, - ...typesRoutes, - ...esqlRoutes, - }; -} - -export type EntitiesAPIServerRouteRepository = ReturnType< - typeof getGlobalEntitiesAPIServerRouteRepository ->; diff --git a/x-pack/plugins/logsai/streams_api/common/utils/get_index_patterns_for_filters.ts b/x-pack/plugins/logsai/streams_api/server/routes/get_global_streams_api_route_repository.ts similarity index 50% rename from x-pack/plugins/logsai/streams_api/common/utils/get_index_patterns_for_filters.ts rename to x-pack/plugins/logsai/streams_api/server/routes/get_global_streams_api_route_repository.ts index 7b513942745c1..116de5f2a712c 100644 --- a/x-pack/plugins/logsai/streams_api/common/utils/get_index_patterns_for_filters.ts +++ b/x-pack/plugins/logsai/streams_api/server/routes/get_global_streams_api_route_repository.ts @@ -5,13 +5,10 @@ * 2.0. */ -import type { EntityFilter } from '../entities'; - -export function getIndexPatternsForFilters(filters: EntityFilter[]) { - return filters.flatMap((filter) => { - if ('index' in filter) { - return filter.index.flat(); - } - return []; - }); +export function getGlobalStreamsAPIServerRouteRepository() { + return {}; } + +export type StreamsAPIServerRouteRepository = ReturnType< + typeof getGlobalStreamsAPIServerRouteRepository +>; diff --git a/x-pack/plugins/logsai/streams_api/server/routes/register_routes.ts b/x-pack/plugins/logsai/streams_api/server/routes/register_routes.ts index b4aa6af182919..e8d0149f6f080 100644 --- a/x-pack/plugins/logsai/streams_api/server/routes/register_routes.ts +++ b/x-pack/plugins/logsai/streams_api/server/routes/register_routes.ts @@ -7,8 +7,8 @@ import type { CoreSetup } from '@kbn/core/server'; import type { Logger } from '@kbn/logging'; import { registerRoutes } from '@kbn/server-route-repository'; -import { getGlobalEntitiesAPIServerRouteRepository } from './get_global_entities_api_route_repository'; -import type { EntitiesAPIRouteHandlerResources } from './types'; +import { getGlobalStreamsAPIServerRouteRepository } from './get_global_streams_api_route_repository'; +import type { StreamsAPIRouteHandlerResources } from './types'; export function registerServerRoutes({ core, @@ -17,12 +17,12 @@ export function registerServerRoutes({ }: { core: CoreSetup; logger: Logger; - dependencies: Omit; + dependencies: Omit; }) { registerRoutes({ core, logger, - repository: getGlobalEntitiesAPIServerRouteRepository(), + repository: getGlobalStreamsAPIServerRouteRepository(), dependencies, }); } diff --git a/x-pack/plugins/logsai/streams_api/server/routes/types.ts b/x-pack/plugins/logsai/streams_api/server/routes/types.ts index aeeeaf8a0ff19..1194b5e1210fe 100644 --- a/x-pack/plugins/logsai/streams_api/server/routes/types.ts +++ b/x-pack/plugins/logsai/streams_api/server/routes/types.ts @@ -8,28 +8,28 @@ import type { CustomRequestHandlerContext, KibanaRequest } from '@kbn/core/server'; import type { LicensingApiRequestHandlerContext } from '@kbn/licensing-plugin/server/types'; import type { Logger } from '@kbn/logging'; -import type { EntitiesAPISetupDependencies, EntitiesAPIStartDependencies } from '../types'; +import type { StreamsAPISetupDependencies, StreamsAPIStartDependencies } from '../types'; -export type EntitiesAPIRequestHandlerContext = CustomRequestHandlerContext<{ +export type StreamsAPIRequestHandlerContext = CustomRequestHandlerContext<{ licensing: Pick; }>; -export interface EntitiesAPIRouteHandlerResources { +export interface StreamsAPIRouteHandlerResources { request: KibanaRequest; - context: EntitiesAPIRequestHandlerContext; + context: StreamsAPIRequestHandlerContext; logger: Logger; plugins: { - [key in keyof EntitiesAPISetupDependencies]: { - setup: Required[key]; + [key in keyof StreamsAPISetupDependencies]: { + setup: Required[key]; }; } & { - [key in keyof EntitiesAPIStartDependencies]: { - start: () => Promise[key]>; + [key in keyof StreamsAPIStartDependencies]: { + start: () => Promise[key]>; }; }; } -export interface EntitiesAPIRouteCreateOptions { +export interface StreamsAPIRouteCreateOptions { options: { timeout?: { idleSocket?: number; diff --git a/x-pack/plugins/logsai/streams_api/server/routes/types/route.ts b/x-pack/plugins/logsai/streams_api/server/routes/types/route.ts deleted file mode 100644 index 84d6172ab7437..0000000000000 --- a/x-pack/plugins/logsai/streams_api/server/routes/types/route.ts +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { z } from '@kbn/zod'; -import { notFound } from '@hapi/boom'; -import { createEntitiesAPIServerRoute } from '../create_entities_api_server_route'; -import { DefinitionEntity, EntityTypeDefinition } from '../../../common/entities'; -import { getTypeDefinitions } from '../../lib/entities/get_type_definitions'; -import { createEntitiesAPIEsClient } from '../../lib/clients/create_entities_api_es_client'; -import { getDefinitionEntities } from '../../lib/entities/get_definition_entities'; - -export const getAllTypeDefinitionsRoute = createEntitiesAPIServerRoute({ - endpoint: 'GET /internal/entities_api/types', - options: { - tags: ['access:entities'], - }, - handler: async ( - resources - ): Promise<{ - typeDefinitions: EntityTypeDefinition[]; - definitionEntities: DefinitionEntity[]; - }> => { - const esClient = await createEntitiesAPIEsClient(resources); - - const [typeDefinitions, definitionEntities] = await Promise.all([ - getTypeDefinitions({ - esClient, - }), - getDefinitionEntities({ - esClient, - }), - ]); - - return { - typeDefinitions, - definitionEntities, - }; - }, -}); - -export const getTypeDefinitionRoute = createEntitiesAPIServerRoute({ - endpoint: 'GET /internal/entities_api/types/{type}', - options: { - tags: ['access:entities'], - }, - params: z.object({ - path: z.object({ - type: z.string(), - }), - }), - handler: async ( - resources - ): Promise<{ typeDefinition: EntityTypeDefinition; definitionEntities: DefinitionEntity[] }> => { - const esClient = await createEntitiesAPIEsClient(resources); - - const { - path: { type }, - } = resources.params; - - const [typeDefinitions, definitionEntities] = await Promise.all([ - getTypeDefinitions({ - esClient, - }), - getDefinitionEntities({ - esClient, - }), - ]); - - const typeDefinition = typeDefinitions.find((definition) => definition.pivot.type === type); - - if (!typeDefinition) { - throw notFound(); - } - - const definitionEntitiesForType = definitionEntities.filter( - (definitionEntity) => definitionEntity.type === type - ); - - return { - typeDefinition, - definitionEntities: definitionEntitiesForType, - }; - }, -}); - -export const typesRoutes = { - ...getTypeDefinitionRoute, - ...getAllTypeDefinitionsRoute, -}; diff --git a/x-pack/plugins/logsai/streams_api/server/types.ts b/x-pack/plugins/logsai/streams_api/server/types.ts index 3628392ba5bd7..55e291caffa04 100644 --- a/x-pack/plugins/logsai/streams_api/server/types.ts +++ b/x-pack/plugins/logsai/streams_api/server/types.ts @@ -4,42 +4,31 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import type { - EntityManagerServerPluginSetup, - EntityManagerServerPluginStart, -} from '@kbn/entityManager-plugin/server'; +/* eslint-disable @typescript-eslint/no-empty-interface*/ + import type { RuleRegistryPluginSetupContract, RuleRegistryPluginStartContract, } from '@kbn/rule-registry-plugin/server'; import type { - PluginSetupContract as AlertingPluginSetup, PluginStartContract as AlertingPluginStart, + PluginSetupContract as AlertingPluginSetup, } from '@kbn/alerting-plugin/server'; -import type { SloPluginStart, SloPluginSetup } from '@kbn/slo-plugin/server'; -import type { SpacesPluginSetup, SpacesPluginStart } from '@kbn/spaces-plugin/server'; -/* eslint-disable @typescript-eslint/no-empty-interface*/ export interface ConfigSchema {} -export interface EntitiesAPISetupDependencies { - entityManager: EntityManagerServerPluginSetup; +export interface StreamsAPISetupDependencies { ruleRegistry: RuleRegistryPluginSetupContract; alerting: AlertingPluginSetup; - slo: SloPluginSetup; - spaces: SpacesPluginSetup; } -export interface EntitiesAPIStartDependencies { - entityManager: EntityManagerServerPluginStart; +export interface StreamsAPIStartDependencies { ruleRegistry: RuleRegistryPluginStartContract; alerting: AlertingPluginStart; - slo: SloPluginStart; - spaces: SpacesPluginStart; } -export interface EntitiesAPIServerSetup {} +export interface StreamsAPIServerSetup {} -export interface EntitiesAPIClient {} +export interface StreamsAPIClient {} -export interface EntitiesAPIServerStart {} +export interface StreamsAPIServerStart {} diff --git a/x-pack/plugins/logsai/streams_api/tsconfig.json b/x-pack/plugins/logsai/streams_api/tsconfig.json index 6594bc7977fc6..452af0f751f7e 100644 --- a/x-pack/plugins/logsai/streams_api/tsconfig.json +++ b/x-pack/plugins/logsai/streams_api/tsconfig.json @@ -14,23 +14,5 @@ ], "exclude": ["target/**/*", ".storybook/**/*.js"], "kbn_references": [ - "@kbn/core", - "@kbn/server-route-repository", - "@kbn/server-route-repository-client", - "@kbn/logging", - "@kbn/config-schema", - "@kbn/rule-registry-plugin", - "@kbn/observability-utils-server", - "@kbn/dashboard-plugin", - "@kbn/alerting-plugin", - "@kbn/apm-utils", - "@kbn/slo-plugin", - "@kbn/licensing-plugin", - "@kbn/zod", - "@kbn/es-types", - "@kbn/observability-utils-common", - "@kbn/data-views-plugin", - "@kbn/entityManager-plugin", - "@kbn/spaces-plugin", ] } diff --git a/x-pack/plugins/logsai/streams_app/.storybook/get_mock_entities_app_context.tsx b/x-pack/plugins/logsai/streams_app/.storybook/get_mock_streams_app_context.tsx similarity index 78% rename from x-pack/plugins/logsai/streams_app/.storybook/get_mock_entities_app_context.tsx rename to x-pack/plugins/logsai/streams_app/.storybook/get_mock_streams_app_context.tsx index bdece7025e442..afb5e7a88e066 100644 --- a/x-pack/plugins/logsai/streams_app/.storybook/get_mock_entities_app_context.tsx +++ b/x-pack/plugins/logsai/streams_app/.storybook/get_mock_streams_app_context.tsx @@ -8,12 +8,12 @@ import { coreMock } from '@kbn/core/public/mocks'; import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public'; -import type { EntitiesAPIPublicStart } from '@kbn/entities-api-plugin/public'; +import type { StreamsAPIPublicStart } from '@kbn/streams-api-plugin/public'; import type { ObservabilitySharedPluginStart } from '@kbn/observability-shared-plugin/public'; import type { UnifiedSearchPublicPluginStart } from '@kbn/unified-search-plugin/public'; -import type { EntitiesAppKibanaContext } from '../public/hooks/use_kibana'; +import type { StreamsAppKibanaContext } from '../public/hooks/use_kibana'; -export function getMockEntitiesAppContext(): EntitiesAppKibanaContext { +export function getMockStreamsAppContext(): StreamsAppKibanaContext { const core = coreMock.createStart(); return { @@ -24,7 +24,7 @@ export function getMockEntitiesAppContext(): EntitiesAppKibanaContext { dataViews: {} as unknown as DataViewsPublicPluginStart, data: {} as unknown as DataPublicPluginStart, unifiedSearch: {} as unknown as UnifiedSearchPublicPluginStart, - entitiesAPI: {} as unknown as EntitiesAPIPublicStart, + streamsAPI: {} as unknown as StreamsAPIPublicStart, }, }, services: {}, diff --git a/x-pack/plugins/logsai/streams_app/.storybook/storybook_decorator.tsx b/x-pack/plugins/logsai/streams_app/.storybook/storybook_decorator.tsx index b6bf80135cec0..617b5aee8128f 100644 --- a/x-pack/plugins/logsai/streams_app/.storybook/storybook_decorator.tsx +++ b/x-pack/plugins/logsai/streams_app/.storybook/storybook_decorator.tsx @@ -5,14 +5,14 @@ * 2.0. */ import React, { ComponentType, useMemo } from 'react'; -import { EntitiesAppContextProvider } from '../public/components/entities_app_context_provider'; -import { getMockEntitiesAppContext } from './get_mock_entities_app_context'; +import { StreamsAppContextProvider } from '../public/components/streams_app_context_provider'; +import { getMockStreamsAppContext } from './get_mock_streams_app_context'; export function KibanaReactStorybookDecorator(Story: ComponentType) { - const context = useMemo(() => getMockEntitiesAppContext(), []); + const context = useMemo(() => getMockStreamsAppContext(), []); return ( - + - + ); } diff --git a/x-pack/plugins/logsai/streams_app/jest.config.js b/x-pack/plugins/logsai/streams_app/jest.config.js index 87f38e5717347..0dde0422306bb 100644 --- a/x-pack/plugins/logsai/streams_app/jest.config.js +++ b/x-pack/plugins/logsai/streams_app/jest.config.js @@ -9,14 +9,14 @@ module.exports = { preset: '@kbn/test', rootDir: '../../../..', roots: [ - '/x-pack/plugins/logsai/entities_app/public', - '/x-pack/plugins/logsai/entities_app/common', - '/x-pack/plugins/logsai/entities_app/server', + '/x-pack/plugins/logsai/streams_app/public', + '/x-pack/plugins/logsai/streams_app/common', + '/x-pack/plugins/logsai/streams_app/server', ], - setupFiles: ['/x-pack/plugins/logsai/entities_app/.storybook/jest_setup.js'], + setupFiles: ['/x-pack/plugins/logsai/streams_app/.storybook/jest_setup.js'], collectCoverage: true, collectCoverageFrom: [ - '/x-pack/plugins/logsai/entities_app/{public,common,server}/**/*.{js,ts,tsx}', + '/x-pack/plugins/logsai/streams_app/{public,common,server}/**/*.{js,ts,tsx}', ], coverageReporters: ['html'], diff --git a/x-pack/plugins/logsai/streams_app/kibana.jsonc b/x-pack/plugins/logsai/streams_app/kibana.jsonc index 7e019c9ba0d53..68de86387f5fb 100644 --- a/x-pack/plugins/logsai/streams_app/kibana.jsonc +++ b/x-pack/plugins/logsai/streams_app/kibana.jsonc @@ -1,14 +1,14 @@ { "type": "plugin", - "id": "@kbn/entities-app-plugin", + "id": "@kbn/streams-app-plugin", "owner": "@elastic/observability-ui", "plugin": { - "id": "entitiesApp", + "id": "streamsApp", "server": true, "browser": true, - "configPath": ["xpack", "entitiesApp"], + "configPath": ["xpack", "streamsApp"], "requiredPlugins": [ - "entitiesAPI", + "streamsAPI", "observabilityShared", "data", "dataViews", diff --git a/x-pack/plugins/logsai/streams_app/public/application.tsx b/x-pack/plugins/logsai/streams_app/public/application.tsx index 6d4f9be2eca43..720f785ecde4b 100644 --- a/x-pack/plugins/logsai/streams_app/public/application.tsx +++ b/x-pack/plugins/logsai/streams_app/public/application.tsx @@ -9,8 +9,8 @@ import ReactDOM from 'react-dom'; import { APP_WRAPPER_CLASS, type AppMountParameters, type CoreStart } from '@kbn/core/public'; import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render'; import { css } from '@emotion/css'; -import type { EntitiesAppStartDependencies } from './types'; -import { EntitiesAppServices } from './services/types'; +import type { StreamsAppStartDependencies } from './types'; +import { StreamsAppServices } from './services/types'; import { AppRoot } from './components/app_root'; export const renderApp = ({ @@ -20,8 +20,8 @@ export const renderApp = ({ appMountParameters, }: { coreStart: CoreStart; - pluginsStart: EntitiesAppStartDependencies; - services: EntitiesAppServices; + pluginsStart: StreamsAppStartDependencies; + services: StreamsAppServices; } & { appMountParameters: AppMountParameters }) => { const { element } = appMountParameters; diff --git a/x-pack/plugins/logsai/streams_app/public/components/all_entities_view/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/all_entities_view/index.tsx index ec6e24bf286bd..ed346df834c36 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/all_entities_view/index.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/all_entities_view/index.tsx @@ -8,19 +8,19 @@ import { EuiFlexGroup } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import React from 'react'; import { EntityTable } from '../entity_table'; -import { EntitiesAppPageHeader } from '../entities_app_page_header'; -import { EntitiesAppPageHeaderTitle } from '../entities_app_page_header/entities_app_page_header_title'; +import { StreamsAppPageHeader } from '../streams_app_page_header'; +import { StreamsAppPageHeaderTitle } from '../streams_app_page_header/streams_app_page_header_title'; export function AllEntitiesView() { return ( - - + - + ); diff --git a/x-pack/plugins/logsai/streams_app/public/components/app_root/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/app_root/index.tsx index 078311b9a54a0..d92891e017f96 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/app_root/index.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/app_root/index.tsx @@ -11,10 +11,10 @@ import { type AppMountParameters, type CoreStart } from '@kbn/core/public'; import { RouteRenderer, RouterProvider } from '@kbn/typed-react-router-config'; import { HeaderMenuPortal } from '@kbn/observability-shared-plugin/public'; import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; -import { EntitiesAppContextProvider } from '../entities_app_context_provider'; -import { entitiesAppRouter } from '../../routes/config'; -import { EntitiesAppStartDependencies } from '../../types'; -import { EntitiesAppServices } from '../../services/types'; +import { StreamsAppContextProvider } from '../streams_app_context_provider'; +import { streamsAppRouter } from '../../routes/config'; +import { StreamsAppStartDependencies } from '../../types'; +import { StreamsAppServices } from '../../services/types'; export function AppRoot({ coreStart, @@ -23,8 +23,8 @@ export function AppRoot({ appMountParameters, }: { coreStart: CoreStart; - pluginsStart: EntitiesAppStartDependencies; - services: EntitiesAppServices; + pluginsStart: StreamsAppStartDependencies; + services: StreamsAppServices; } & { appMountParameters: AppMountParameters }) { const { history } = appMountParameters; @@ -37,18 +37,18 @@ export function AppRoot({ }; return ( - + - + - + - + ); } -export function EntitiesAppHeaderActionMenu({ +export function StreamsAppHeaderActionMenu({ appMountParameters, }: { appMountParameters: AppMountParameters; diff --git a/x-pack/plugins/logsai/streams_app/public/components/data_stream_detail_view/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/data_stream_detail_view/index.tsx index 456fc6de083ea..c718ef52c5a16 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/data_stream_detail_view/index.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/data_stream_detail_view/index.tsx @@ -7,12 +7,12 @@ import React from 'react'; import { i18n } from '@kbn/i18n'; import { EntityDetailViewWithoutParams } from '../entity_detail_view'; -import { useEntitiesAppParams } from '../../hooks/use_entities_app_params'; +import { useStreamsAppParams } from '../../hooks/use_streams_app_params'; export function DataStreamDetailView() { const { path: { key, tab }, - } = useEntitiesAppParams('/data_stream/{key}/{tab}'); + } = useStreamsAppParams('/data_stream/{key}/{tab}'); return ( - { setDisplayedKqlFilter(query); @@ -195,7 +195,7 @@ export function EntityDetailOverview({ `} > { - return entitiesAPIClient.fetch('GET /internal/entities_api/entity/{type}/{key}', { + return streamsAPIClient.fetch('GET /internal/streams_api/entity/{type}/{key}', { signal, params: { path: { @@ -81,14 +81,14 @@ export function EntityDetailViewWithoutParams({ }, }); }, - [type, key, entitiesAPIClient] + [type, key, streamsAPIClient] ); const typeDefinition = entityFetch.value?.typeDefinition; const entity = entityFetch.value?.entity; - useEntitiesAppBreadcrumbs(() => { + useStreamsAppBreadcrumbs(() => { if (!typeDefinition || !entity) { return []; } @@ -107,10 +107,10 @@ export function EntityDetailViewWithoutParams({ ]; }, [type, key, entity?.displayName, typeDefinition]); - const entityDataStreamsFetch = useEntitiesAppFetch( + const entityDataStreamsFetch = useStreamsAppFetch( async ({ signal }) => { - return entitiesAPIClient.fetch( - 'GET /internal/entities_api/entity/{type}/{key}/data_streams', + return streamsAPIClient.fetch( + 'GET /internal/streams_api/entity/{type}/{key}/data_streams', { signal, params: { @@ -126,7 +126,7 @@ export function EntityDetailViewWithoutParams({ } ); }, - [key, type, entitiesAPIClient, start, end] + [key, type, streamsAPIClient, start, end] ); const dataStreams = entityDataStreamsFetch.value?.dataStreams; @@ -175,8 +175,8 @@ export function EntityDetailViewWithoutParams({ return ( - - + + - - + + { return { @@ -237,7 +237,7 @@ export function EntityDetailViewWithoutParams({ export function EntityDetailView() { const { path: { type, key, tab }, - } = useEntitiesAppParams('/{type}/{key}/{tab}'); + } = useStreamsAppParams('/{type}/{key}/{tab}'); return ; } diff --git a/x-pack/plugins/logsai/streams_app/public/components/entity_health_status_badge/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/entity_health_status_badge/index.tsx index 33fc375e63f87..c66d444b2d695 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/entity_health_status_badge/index.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/entity_health_status_badge/index.tsx @@ -5,7 +5,7 @@ * 2.0. */ import React from 'react'; -import { EntityHealthStatus } from '@kbn/entities-api-plugin/common'; +import { EntityHealthStatus } from '@kbn/streams-api-plugin/common'; import { EuiBadge } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; diff --git a/x-pack/plugins/logsai/streams_app/public/components/entity_pivot_type_view/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/entity_pivot_type_view/index.tsx index efb88a7b2813f..5feeddb08d4fa 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/entity_pivot_type_view/index.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/entity_pivot_type_view/index.tsx @@ -7,29 +7,29 @@ import { EuiFlexGroup } from '@elastic/eui'; import React from 'react'; import { EntityTable } from '../entity_table'; -import { EntitiesAppPageHeader } from '../entities_app_page_header'; -import { EntitiesAppPageHeaderTitle } from '../entities_app_page_header/entities_app_page_header_title'; -import { useEntitiesAppParams } from '../../hooks/use_entities_app_params'; +import { StreamsAppPageHeader } from '../streams_app_page_header'; +import { StreamsAppPageHeaderTitle } from '../streams_app_page_header/streams_app_page_header_title'; +import { useStreamsAppParams } from '../../hooks/use_streams_app_params'; import { useKibana } from '../../hooks/use_kibana'; -import { useEntitiesAppFetch } from '../../hooks/use_entities_app_fetch'; -import { useEntitiesAppBreadcrumbs } from '../../hooks/use_entities_app_breadcrumbs'; +import { useStreamsAppFetch } from '../../hooks/use_streams_app_fetch'; +import { useStreamsAppBreadcrumbs } from '../../hooks/use_streams_app_breadcrumbs'; export function EntityPivotTypeView() { const { path: { type }, - } = useEntitiesAppParams('/{type}'); + } = useStreamsAppParams('/{type}'); const { dependencies: { start: { - entitiesAPI: { entitiesAPIClient }, + streamsAPI: { streamsAPIClient }, }, }, } = useKibana(); - const typeDefinitionsFetch = useEntitiesAppFetch( + const typeDefinitionsFetch = useStreamsAppFetch( ({ signal }) => { - return entitiesAPIClient.fetch('GET /internal/entities_api/types/{type}', { + return streamsAPIClient.fetch('GET /internal/streams_api/types/{type}', { signal, params: { path: { @@ -38,14 +38,14 @@ export function EntityPivotTypeView() { }, }); }, - [entitiesAPIClient, type] + [streamsAPIClient, type] ); const typeDefinition = typeDefinitionsFetch.value?.typeDefinition; const title = typeDefinition?.displayName ?? ''; - useEntitiesAppBreadcrumbs(() => { + useStreamsAppBreadcrumbs(() => { if (!title) { return []; } @@ -60,9 +60,9 @@ export function EntityPivotTypeView() { return ( - - - + + + ); diff --git a/x-pack/plugins/logsai/streams_app/public/components/entity_table/controlled_entity_table.tsx b/x-pack/plugins/logsai/streams_app/public/components/entity_table/controlled_entity_table.tsx index 4d407da954cc7..f611a425701e6 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/entity_table/controlled_entity_table.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/entity_table/controlled_entity_table.tsx @@ -18,10 +18,10 @@ import { css } from '@emotion/css'; import type { TimeRange } from '@kbn/data-plugin/common'; import type { DataView } from '@kbn/data-views-plugin/common'; import { i18n } from '@kbn/i18n'; -import type { EntityWithSignalStatus } from '@kbn/entities-api-plugin/common'; +import type { EntityWithSignalStatus } from '@kbn/streams-api-plugin/common'; import React, { useMemo } from 'react'; -import { useEntitiesAppRouter } from '../../hooks/use_entities_app_router'; -import { EntitiesAppSearchBar } from '../entities_app_search_bar'; +import { useStreamsAppRouter } from '../../hooks/use_streams_app_router'; +import { StreamsAppSearchBar } from '../streams_app_search_bar'; import { EntityHealthStatusBadge } from '../entity_health_status_badge'; export function ControlledEntityTable({ @@ -63,7 +63,7 @@ export function ControlledEntityTable({ sort?: { field: string; order: 'asc' | 'desc' }; onSortChange?: (nextSort: { field: string; order: 'asc' | 'desc' }) => void; }) { - const router = useEntitiesAppRouter(); + const router = useStreamsAppRouter(); const displayedColumns = useMemo>>(() => { return [ @@ -140,7 +140,7 @@ export function ControlledEntityTable({ - { onKqlFilterChange(query); diff --git a/x-pack/plugins/logsai/streams_app/public/components/entity_table/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/entity_table/index.tsx index 3bcafc336507e..38fb484e4ba1b 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/entity_table/index.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/entity_table/index.tsx @@ -7,11 +7,11 @@ import { useDateRange } from '@kbn/observability-utils-browser/hooks/use_date_range'; import React, { useMemo, useState } from 'react'; import { useAbortableAsync } from '@kbn/observability-utils-browser/hooks/use_abortable_async'; -import { getIndexPatternsForFilters } from '@kbn/entities-api-plugin/public'; -import { DefinitionEntity } from '@kbn/entities-api-plugin/common/entities'; +import { getIndexPatternsForFilters } from '@kbn/streams-api-plugin/public'; +import { DefinitionEntity } from '@kbn/streams-api-plugin/common/entities'; import { useKibana } from '../../hooks/use_kibana'; import { ControlledEntityTable } from './controlled_entity_table'; -import { useEntitiesAppFetch } from '../../hooks/use_entities_app_fetch'; +import { useStreamsAppFetch } from '../../hooks/use_streams_app_fetch'; export function EntityTable({ type }: { type: 'all' | string }) { const { @@ -19,7 +19,7 @@ export function EntityTable({ type }: { type: 'all' | string }) { start: { dataViews, data, - entitiesAPI: { entitiesAPIClient }, + streamsAPI: { streamsAPIClient }, }, }, } = useKibana(); @@ -40,18 +40,18 @@ export function EntityTable({ type }: { type: 'all' | string }) { order: 'desc', }); - const typeDefinitionsFetch = useEntitiesAppFetch( + const typeDefinitionsFetch = useStreamsAppFetch( ({ signal, }): Promise<{ definitionEntities: DefinitionEntity[]; }> => { if (selectedType === 'all') { - return entitiesAPIClient.fetch('GET /internal/entities_api/types', { + return streamsAPIClient.fetch('GET /internal/streams_api/types', { signal, }); } - return entitiesAPIClient.fetch('GET /internal/entities_api/types/{type}', { + return streamsAPIClient.fetch('GET /internal/streams_api/types/{type}', { signal, params: { path: { @@ -60,12 +60,12 @@ export function EntityTable({ type }: { type: 'all' | string }) { }, }); }, - [entitiesAPIClient, selectedType] + [streamsAPIClient, selectedType] ); - const queryFetch = useEntitiesAppFetch( + const queryFetch = useStreamsAppFetch( ({ signal }) => { - return entitiesAPIClient.fetch('POST /internal/entities_api/entities', { + return streamsAPIClient.fetch('POST /internal/streams_api/entities', { signal, params: { body: { @@ -79,7 +79,7 @@ export function EntityTable({ type }: { type: 'all' | string }) { }, }); }, - [entitiesAPIClient, selectedType, persistedKqlFilter, start, end, sort.field, sort.order] + [streamsAPIClient, selectedType, persistedKqlFilter, start, end, sort.field, sort.order] ); const [pagination, setPagination] = useState<{ pageSize: number; pageIndex: number }>({ diff --git a/x-pack/plugins/logsai/streams_app/public/components/redirect_to/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/redirect_to/index.tsx index 11ee321def8a5..2bde67faa3b98 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/redirect_to/index.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/redirect_to/index.tsx @@ -8,16 +8,16 @@ import React, { useLayoutEffect } from 'react'; import { PathsOf, TypeOf } from '@kbn/typed-react-router-config'; import { DeepPartial } from 'utility-types'; import { merge } from 'lodash'; -import { EntitiesAppRoutes } from '../../routes/config'; -import { useEntitiesAppRouter } from '../../hooks/use_entities_app_router'; -import { useEntitiesAppParams } from '../../hooks/use_entities_app_params'; +import { StreamsAppRoutes } from '../../routes/config'; +import { useStreamsAppRouter } from '../../hooks/use_streams_app_router'; +import { useStreamsAppParams } from '../../hooks/use_streams_app_params'; export function RedirectTo< - TPath extends PathsOf, - TParams extends TypeOf + TPath extends PathsOf, + TParams extends TypeOf >({ path, params }: { path: TPath; params?: DeepPartial }) { - const router = useEntitiesAppRouter(); - const currentParams = useEntitiesAppParams('/*'); + const router = useStreamsAppRouter(); + const currentParams = useStreamsAppParams('/*'); useLayoutEffect(() => { router.replace(path, ...([merge({}, currentParams, params)] as any)); // eslint-disable-next-line react-hooks/exhaustive-deps diff --git a/x-pack/plugins/logsai/streams_app/public/components/entities_app_context_provider/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/streams_app_context_provider/index.tsx similarity index 81% rename from x-pack/plugins/logsai/streams_app/public/components/entities_app_context_provider/index.tsx rename to x-pack/plugins/logsai/streams_app/public/components/streams_app_context_provider/index.tsx index 457d53c5cc4a8..fd5886c089396 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/entities_app_context_provider/index.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/streams_app_context_provider/index.tsx @@ -6,13 +6,13 @@ */ import React, { useMemo } from 'react'; import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; -import type { EntitiesAppKibanaContext } from '../../hooks/use_kibana'; +import type { StreamsAppKibanaContext } from '../../hooks/use_kibana'; -export function EntitiesAppContextProvider({ +export function StreamsAppContextProvider({ context, children, }: { - context: EntitiesAppKibanaContext; + context: StreamsAppKibanaContext; children: React.ReactNode; }) { const servicesForContext = useMemo(() => { diff --git a/x-pack/plugins/logsai/streams_app/public/components/entities_app_page_header/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_header/index.tsx similarity index 84% rename from x-pack/plugins/logsai/streams_app/public/components/entities_app_page_header/index.tsx rename to x-pack/plugins/logsai/streams_app/public/components/streams_app_page_header/index.tsx index c0de2c80c7ef7..51fac8af35a61 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/entities_app_page_header/index.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_header/index.tsx @@ -7,7 +7,7 @@ import { EuiFlexGroup, EuiPageHeader } from '@elastic/eui'; import React from 'react'; -export function EntitiesAppPageHeader({ children }: { children: React.ReactNode }) { +export function StreamsAppPageHeader({ children }: { children: React.ReactNode }) { return ( {children} diff --git a/x-pack/plugins/logsai/streams_app/public/components/entities_app_page_header/entities_app_page_header_title.tsx b/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_header/streams_app_page_header_title.tsx similarity index 94% rename from x-pack/plugins/logsai/streams_app/public/components/entities_app_page_header/entities_app_page_header_title.tsx rename to x-pack/plugins/logsai/streams_app/public/components/streams_app_page_header/streams_app_page_header_title.tsx index 60241118cc6a5..fd1014a615912 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/entities_app_page_header/entities_app_page_header_title.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_header/streams_app_page_header_title.tsx @@ -7,7 +7,7 @@ import { EuiFlexGroup, EuiHorizontalRule, EuiTitle } from '@elastic/eui'; import React from 'react'; -export function EntitiesAppPageHeaderTitle({ +export function StreamsAppPageHeaderTitle({ title, children, }: { diff --git a/x-pack/plugins/logsai/streams_app/public/components/entities_app_page_template/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_template/index.tsx similarity index 93% rename from x-pack/plugins/logsai/streams_app/public/components/entities_app_page_template/index.tsx rename to x-pack/plugins/logsai/streams_app/public/components/streams_app_page_template/index.tsx index 91daed3591b46..19f2de51ceb2a 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/entities_app_page_template/index.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_template/index.tsx @@ -9,7 +9,7 @@ import { css } from '@emotion/css'; import React from 'react'; import { useKibana } from '../../hooks/use_kibana'; -export function EntitiesAppPageTemplate({ children }: { children: React.ReactNode }) { +export function StreamsAppPageTemplate({ children }: { children: React.ReactNode }) { const { dependencies: { start: { observabilityShared }, diff --git a/x-pack/plugins/logsai/streams_app/public/components/entities_app_router_breadcrumb/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/streams_app_router_breadcrumb/index.tsx similarity index 67% rename from x-pack/plugins/logsai/streams_app/public/components/entities_app_router_breadcrumb/index.tsx rename to x-pack/plugins/logsai/streams_app/public/components/streams_app_router_breadcrumb/index.tsx index 5c2a3b876c5dd..88aab4662de61 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/entities_app_router_breadcrumb/index.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/streams_app_router_breadcrumb/index.tsx @@ -6,6 +6,6 @@ */ import { createRouterBreadcrumbComponent } from '@kbn/typed-react-router-config'; -import type { EntitiesAppRoutes } from '../../routes/config'; +import type { StreamsAppRoutes } from '../../routes/config'; -export const EntitiesAppRouterBreadcrumb = createRouterBreadcrumbComponent(); +export const StreamsAppRouterBreadcrumb = createRouterBreadcrumbComponent(); diff --git a/x-pack/plugins/logsai/streams_app/public/components/entities_app_search_bar/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/streams_app_search_bar/index.tsx similarity index 96% rename from x-pack/plugins/logsai/streams_app/public/components/entities_app_search_bar/index.tsx rename to x-pack/plugins/logsai/streams_app/public/components/streams_app_search_bar/index.tsx index a9ffda0a02fbc..6b8c257c772e7 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/entities_app_search_bar/index.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/streams_app_search_bar/index.tsx @@ -26,7 +26,7 @@ interface Props { dataViews?: DataView[]; } -export function EntitiesAppSearchBar({ +export function StreamsAppSearchBar({ dateRangeFrom, dateRangeTo, onQueryChange, @@ -47,7 +47,7 @@ export function EntitiesAppSearchBar({ return (
{ onQuerySubmit({ dateRange, query: (nextQuery?.query as string | undefined) ?? '' }); }} diff --git a/x-pack/plugins/logsai/streams_app/public/hooks/use_esql_query_result.ts b/x-pack/plugins/logsai/streams_app/public/hooks/use_esql_query_result.ts index d47744ac67d77..853f4bc3a41f3 100644 --- a/x-pack/plugins/logsai/streams_app/public/hooks/use_esql_query_result.ts +++ b/x-pack/plugins/logsai/streams_app/public/hooks/use_esql_query_result.ts @@ -27,7 +27,7 @@ export function useEsqlQueryResult({ const { dependencies: { start: { - entitiesAPI: { entitiesAPIClient }, + streamsAPI: { streamsAPIClient }, }, }, } = useKibana(); @@ -37,7 +37,7 @@ export function useEsqlQueryResult({ if (!query) { return undefined; } - return entitiesAPIClient.fetch('POST /internal/entities_api/esql', { + return streamsAPIClient.fetch('POST /internal/streams_api/esql', { signal, params: { body: { @@ -51,6 +51,6 @@ export function useEsqlQueryResult({ }, }); }, - [entitiesAPIClient, query, start, end, kuery, operationName, dslFilter] + [streamsAPIClient, query, start, end, kuery, operationName, dslFilter] ); } diff --git a/x-pack/plugins/logsai/streams_app/public/hooks/use_kibana.tsx b/x-pack/plugins/logsai/streams_app/public/hooks/use_kibana.tsx index 0f10255e80e44..9c6b23465fb11 100644 --- a/x-pack/plugins/logsai/streams_app/public/hooks/use_kibana.tsx +++ b/x-pack/plugins/logsai/streams_app/public/hooks/use_kibana.tsx @@ -8,19 +8,19 @@ import type { CoreStart } from '@kbn/core/public'; import { useMemo } from 'react'; import { useKibana } from '@kbn/kibana-react-plugin/public'; -import type { EntitiesAppStartDependencies } from '../types'; -import type { EntitiesAppServices } from '../services/types'; +import type { StreamsAppStartDependencies } from '../types'; +import type { StreamsAppServices } from '../services/types'; -export interface EntitiesAppKibanaContext { +export interface StreamsAppKibanaContext { core: CoreStart; dependencies: { - start: EntitiesAppStartDependencies; + start: StreamsAppStartDependencies; }; - services: EntitiesAppServices; + services: StreamsAppServices; } -const useTypedKibana = (): EntitiesAppKibanaContext => { - const context = useKibana>(); +const useTypedKibana = (): StreamsAppKibanaContext => { + const context = useKibana>(); return useMemo(() => { const { dependencies, services, ...core } = context.services; diff --git a/x-pack/plugins/logsai/streams_app/public/hooks/use_entities_app_breadcrumbs.ts b/x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_breadcrumbs.ts similarity index 70% rename from x-pack/plugins/logsai/streams_app/public/hooks/use_entities_app_breadcrumbs.ts rename to x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_breadcrumbs.ts index d47ed5f78d40e..e3ac760e3b779 100644 --- a/x-pack/plugins/logsai/streams_app/public/hooks/use_entities_app_breadcrumbs.ts +++ b/x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_breadcrumbs.ts @@ -6,6 +6,6 @@ */ import { createUseBreadcrumbs } from '@kbn/typed-react-router-config'; -import { EntitiesAppRoutes } from '../routes/config'; +import { StreamsAppRoutes } from '../routes/config'; -export const useEntitiesAppBreadcrumbs = createUseBreadcrumbs(); +export const useStreamsAppBreadcrumbs = createUseBreadcrumbs(); diff --git a/x-pack/plugins/logsai/streams_app/public/hooks/use_entities_app_fetch.ts b/x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_fetch.ts similarity index 94% rename from x-pack/plugins/logsai/streams_app/public/hooks/use_entities_app_fetch.ts rename to x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_fetch.ts index 454325ad0dbbc..6632fced59792 100644 --- a/x-pack/plugins/logsai/streams_app/public/hooks/use_entities_app_fetch.ts +++ b/x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_fetch.ts @@ -13,7 +13,7 @@ import { import { omit } from 'lodash'; import { useKibana } from './use_kibana'; -export const useEntitiesAppFetch: UseAbortableAsync<{}, { disableToastOnError?: boolean }> = ( +export const useStreamsAppFetch: UseAbortableAsync<{}, { disableToastOnError?: boolean }> = ( callback, deps, options diff --git a/x-pack/plugins/logsai/streams_app/public/hooks/use_entities_app_params.ts b/x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_params.ts similarity index 59% rename from x-pack/plugins/logsai/streams_app/public/hooks/use_entities_app_params.ts rename to x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_params.ts index cec3c83079198..2931a6fa64f8b 100644 --- a/x-pack/plugins/logsai/streams_app/public/hooks/use_entities_app_params.ts +++ b/x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_params.ts @@ -5,10 +5,10 @@ * 2.0. */ import { type PathsOf, type TypeOf, useParams } from '@kbn/typed-react-router-config'; -import type { EntitiesAppRoutes } from '../routes/config'; +import type { StreamsAppRoutes } from '../routes/config'; -export function useEntitiesAppParams>( +export function useStreamsAppParams>( path: TPath -): TypeOf { - return useParams(path)! as TypeOf; +): TypeOf { + return useParams(path)! as TypeOf; } diff --git a/x-pack/plugins/logsai/streams_app/public/hooks/use_entities_app_route_path.ts b/x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_route_path.ts similarity index 70% rename from x-pack/plugins/logsai/streams_app/public/hooks/use_entities_app_route_path.ts rename to x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_route_path.ts index 49db4a8053155..78e63ead57da6 100644 --- a/x-pack/plugins/logsai/streams_app/public/hooks/use_entities_app_route_path.ts +++ b/x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_route_path.ts @@ -6,10 +6,10 @@ */ import { PathsOf, useRoutePath } from '@kbn/typed-react-router-config'; -import type { EntitiesAppRoutes } from '../routes/config'; +import type { StreamsAppRoutes } from '../routes/config'; -export function useEntitiesAppRoutePath() { +export function useStreamsAppRoutePath() { const path = useRoutePath(); - return path as PathsOf; + return path as PathsOf; } diff --git a/x-pack/plugins/logsai/streams_app/public/hooks/use_entities_app_router.ts b/x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_router.ts similarity index 65% rename from x-pack/plugins/logsai/streams_app/public/hooks/use_entities_app_router.ts rename to x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_router.ts index 368166a85dbff..37a9386af1d05 100644 --- a/x-pack/plugins/logsai/streams_app/public/hooks/use_entities_app_router.ts +++ b/x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_router.ts @@ -7,22 +7,22 @@ import { PathsOf, TypeAsArgs, TypeOf } from '@kbn/typed-react-router-config'; import { useMemo } from 'react'; -import type { EntitiesAppRouter, EntitiesAppRoutes } from '../routes/config'; -import { entitiesAppRouter } from '../routes/config'; +import type { StreamsAppRouter, StreamsAppRoutes } from '../routes/config'; +import { streamsAppRouter } from '../routes/config'; import { useKibana } from './use_kibana'; -interface StatefulEntitiesAppRouter extends EntitiesAppRouter { - push>( +interface StatefulStreamsAppRouter extends StreamsAppRouter { + push>( path: T, - ...params: TypeAsArgs> + ...params: TypeAsArgs> ): void; - replace>( + replace>( path: T, - ...params: TypeAsArgs> + ...params: TypeAsArgs> ): void; } -export function useEntitiesAppRouter(): StatefulEntitiesAppRouter { +export function useStreamsAppRouter(): StatefulStreamsAppRouter { const { core: { http, @@ -32,12 +32,12 @@ export function useEntitiesAppRouter(): StatefulEntitiesAppRouter { const link = (...args: any[]) => { // @ts-expect-error - return entitiesAppRouter.link(...args); + return streamsAppRouter.link(...args); }; - return useMemo( + return useMemo( () => ({ - ...entitiesAppRouter, + ...streamsAppRouter, push: (...args) => { const next = link(...args); navigateToApp('entities', { path: next, replace: false }); diff --git a/x-pack/plugins/logsai/streams_app/public/index.ts b/x-pack/plugins/logsai/streams_app/public/index.ts index 961e3deb9a018..eea2d8b7452a8 100644 --- a/x-pack/plugins/logsai/streams_app/public/index.ts +++ b/x-pack/plugins/logsai/streams_app/public/index.ts @@ -6,21 +6,21 @@ */ import type { PluginInitializer, PluginInitializerContext } from '@kbn/core/public'; -import { EntitiesAppPlugin } from './plugin'; +import { StreamsAppPlugin } from './plugin'; import type { - EntitiesAppPublicSetup, - EntitiesAppPublicStart, - EntitiesAppSetupDependencies, - EntitiesAppStartDependencies, + StreamsAppPublicSetup, + StreamsAppPublicStart, + StreamsAppSetupDependencies, + StreamsAppStartDependencies, ConfigSchema, } from './types'; -export type { EntitiesAppPublicSetup, EntitiesAppPublicStart }; +export type { StreamsAppPublicSetup, StreamsAppPublicStart }; export const plugin: PluginInitializer< - EntitiesAppPublicSetup, - EntitiesAppPublicStart, - EntitiesAppSetupDependencies, - EntitiesAppStartDependencies + StreamsAppPublicSetup, + StreamsAppPublicStart, + StreamsAppSetupDependencies, + StreamsAppStartDependencies > = (pluginInitializerContext: PluginInitializerContext) => - new EntitiesAppPlugin(pluginInitializerContext); + new StreamsAppPlugin(pluginInitializerContext); diff --git a/x-pack/plugins/logsai/streams_app/public/plugin.ts b/x-pack/plugins/logsai/streams_app/public/plugin.ts index 0dc5462e04b1d..37e01c37a954d 100644 --- a/x-pack/plugins/logsai/streams_app/public/plugin.ts +++ b/x-pack/plugins/logsai/streams_app/public/plugin.ts @@ -19,20 +19,20 @@ import type { Logger } from '@kbn/logging'; import { ENTITY_APP_ID } from '@kbn/deeplinks-observability/constants'; import type { ConfigSchema, - EntitiesAppPublicSetup, - EntitiesAppPublicStart, - EntitiesAppSetupDependencies, - EntitiesAppStartDependencies, + StreamsAppPublicSetup, + StreamsAppPublicStart, + StreamsAppSetupDependencies, + StreamsAppStartDependencies, } from './types'; -import { EntitiesAppServices } from './services/types'; +import { StreamsAppServices } from './services/types'; -export class EntitiesAppPlugin +export class StreamsAppPlugin implements Plugin< - EntitiesAppPublicSetup, - EntitiesAppPublicStart, - EntitiesAppSetupDependencies, - EntitiesAppStartDependencies + StreamsAppPublicSetup, + StreamsAppPublicStart, + StreamsAppSetupDependencies, + StreamsAppStartDependencies > { logger: Logger; @@ -41,9 +41,9 @@ export class EntitiesAppPlugin this.logger = context.logger.get(); } setup( - coreSetup: CoreSetup, - pluginsSetup: EntitiesAppSetupDependencies - ): EntitiesAppPublicSetup { + coreSetup: CoreSetup, + pluginsSetup: StreamsAppSetupDependencies + ): StreamsAppPublicSetup { pluginsSetup.observabilityShared.navigation.registerSections( from(coreSetup.getStartServices()).pipe( map(([coreStart, pluginsStart]) => { @@ -53,7 +53,7 @@ export class EntitiesAppPlugin sortKey: 101, entries: [ { - label: i18n.translate('xpack.entities.entitiesAppLinkTitle', { + label: i18n.translate('xpack.entities.streamsAppLinkTitle', { defaultMessage: 'Entities', }), app: ENTITY_APP_ID, @@ -82,7 +82,7 @@ export class EntitiesAppPlugin deepLinks: [ { id: 'entities', - title: i18n.translate('xpack.entities.entitiesAppDeepLinkTitle', { + title: i18n.translate('xpack.entities.streamsAppDeepLinkTitle', { defaultMessage: 'Entities', }), path: '/', @@ -95,7 +95,7 @@ export class EntitiesAppPlugin coreSetup.getStartServices(), ]); - const services: EntitiesAppServices = {}; + const services: StreamsAppServices = {}; return renderApp({ coreStart, @@ -109,7 +109,7 @@ export class EntitiesAppPlugin return {}; } - start(coreStart: CoreStart, pluginsStart: EntitiesAppStartDependencies): EntitiesAppPublicStart { + start(coreStart: CoreStart, pluginsStart: StreamsAppStartDependencies): StreamsAppPublicStart { return {}; } } diff --git a/x-pack/plugins/logsai/streams_app/public/routes/config.tsx b/x-pack/plugins/logsai/streams_app/public/routes/config.tsx index 1c832aff35faf..a27a9d8c657a9 100644 --- a/x-pack/plugins/logsai/streams_app/public/routes/config.tsx +++ b/x-pack/plugins/logsai/streams_app/public/routes/config.tsx @@ -10,8 +10,8 @@ import * as t from 'io-ts'; import React from 'react'; import { AllEntitiesView } from '../components/all_entities_view'; import { EntityDetailView } from '../components/entity_detail_view'; -import { EntitiesAppPageTemplate } from '../components/entities_app_page_template'; -import { EntitiesAppRouterBreadcrumb } from '../components/entities_app_router_breadcrumb'; +import { StreamsAppPageTemplate } from '../components/streams_app_page_template'; +import { StreamsAppRouterBreadcrumb } from '../components/streams_app_router_breadcrumb'; import { RedirectTo } from '../components/redirect_to'; import { DataStreamDetailView } from '../components/data_stream_detail_view'; import { EntityPivotTypeView } from '../components/entity_pivot_type_view'; @@ -20,31 +20,31 @@ import { EntityPivotTypeView } from '../components/entity_pivot_type_view'; * The array of route definitions to be used when the application * creates the routes. */ -const entitiesAppRoutes = { +const streamsAppRoutes = { '/': { element: ( - - + - - + + ), children: { '/all': { element: ( - - + ), }, '/data_stream/{key}': { @@ -107,8 +107,8 @@ const entitiesAppRoutes = { }, } satisfies RouteMap; -export type EntitiesAppRoutes = typeof entitiesAppRoutes; +export type StreamsAppRoutes = typeof streamsAppRoutes; -export const entitiesAppRouter = createRouter(entitiesAppRoutes); +export const streamsAppRouter = createRouter(streamsAppRoutes); -export type EntitiesAppRouter = typeof entitiesAppRouter; +export type StreamsAppRouter = typeof streamsAppRouter; diff --git a/x-pack/plugins/logsai/streams_app/public/services/types.ts b/x-pack/plugins/logsai/streams_app/public/services/types.ts index 7b6cda92161c0..7f75493d2525c 100644 --- a/x-pack/plugins/logsai/streams_app/public/services/types.ts +++ b/x-pack/plugins/logsai/streams_app/public/services/types.ts @@ -6,4 +6,4 @@ */ // eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface EntitiesAppServices {} +export interface StreamsAppServices {} diff --git a/x-pack/plugins/logsai/streams_app/public/types.ts b/x-pack/plugins/logsai/streams_app/public/types.ts index 727f4e606186c..ef8f30c47dd75 100644 --- a/x-pack/plugins/logsai/streams_app/public/types.ts +++ b/x-pack/plugins/logsai/streams_app/public/types.ts @@ -5,9 +5,9 @@ * 2.0. */ import type { - EntitiesAPIPublicSetup, - EntitiesAPIPublicStart, -} from '@kbn/entities-api-plugin/public'; + StreamsAPIPublicSetup, + StreamsAPIPublicStart, +} from '@kbn/streams-api-plugin/public'; import type { ObservabilitySharedPluginSetup, ObservabilitySharedPluginStart, @@ -26,22 +26,22 @@ import type { export interface ConfigSchema {} -export interface EntitiesAppSetupDependencies { +export interface StreamsAppSetupDependencies { observabilityShared: ObservabilitySharedPluginSetup; - entitiesAPI: EntitiesAPIPublicSetup; + streamsAPI: StreamsAPIPublicSetup; data: DataPublicPluginSetup; dataViews: DataViewsPublicPluginSetup; unifiedSearch: UnifiedSearchPluginSetup; } -export interface EntitiesAppStartDependencies { +export interface StreamsAppStartDependencies { observabilityShared: ObservabilitySharedPluginStart; - entitiesAPI: EntitiesAPIPublicStart; + streamsAPI: StreamsAPIPublicStart; data: DataPublicPluginStart; dataViews: DataViewsPublicPluginStart; unifiedSearch: UnifiedSearchPublicPluginStart; } -export interface EntitiesAppPublicSetup {} +export interface StreamsAppPublicSetup {} -export interface EntitiesAppPublicStart {} +export interface StreamsAppPublicStart {} diff --git a/x-pack/plugins/logsai/streams_app/public/util/get_initial_columns_for_logs.ts b/x-pack/plugins/logsai/streams_app/public/util/get_initial_columns_for_logs.ts index 4e1daa16eca98..209766f3e99e1 100644 --- a/x-pack/plugins/logsai/streams_app/public/util/get_initial_columns_for_logs.ts +++ b/x-pack/plugins/logsai/streams_app/public/util/get_initial_columns_for_logs.ts @@ -7,7 +7,7 @@ import { isArray } from 'lodash'; import type { ESQLSearchResponse } from '@kbn/es-types'; -import { Pivot } from '@kbn/entities-api-plugin/common'; +import { Pivot } from '@kbn/streams-api-plugin/common'; type Column = ESQLSearchResponse['columns'][number]; diff --git a/x-pack/plugins/logsai/streams_app/server/config.ts b/x-pack/plugins/logsai/streams_app/server/config.ts index 6fcffd687a9dd..73e631c7f6d9f 100644 --- a/x-pack/plugins/logsai/streams_app/server/config.ts +++ b/x-pack/plugins/logsai/streams_app/server/config.ts @@ -11,4 +11,4 @@ export const config = schema.object({ enabled: schema.boolean({ defaultValue: true }), }); -export type EntitiesAppConfig = TypeOf; +export type StreamsAppConfig = TypeOf; diff --git a/x-pack/plugins/logsai/streams_app/server/index.ts b/x-pack/plugins/logsai/streams_app/server/index.ts index cd970efad2c39..1ca3b3c6e3de7 100644 --- a/x-pack/plugins/logsai/streams_app/server/index.ts +++ b/x-pack/plugins/logsai/streams_app/server/index.ts @@ -9,27 +9,27 @@ import type { PluginInitializer, PluginInitializerContext, } from '@kbn/core/server'; -import type { EntitiesAppConfig } from './config'; -import { EntitiesAppPlugin } from './plugin'; +import type { StreamsAppConfig } from './config'; +import { StreamsAppPlugin } from './plugin'; import type { - EntitiesAppServerSetup, - EntitiesAppServerStart, - EntitiesAppSetupDependencies, - EntitiesAppStartDependencies, + StreamsAppServerSetup, + StreamsAppServerStart, + StreamsAppSetupDependencies, + StreamsAppStartDependencies, } from './types'; -export type { EntitiesAppServerSetup, EntitiesAppServerStart }; +export type { StreamsAppServerSetup, StreamsAppServerStart }; import { config as configSchema } from './config'; -export const config: PluginConfigDescriptor = { +export const config: PluginConfigDescriptor = { schema: configSchema, }; export const plugin: PluginInitializer< - EntitiesAppServerSetup, - EntitiesAppServerStart, - EntitiesAppSetupDependencies, - EntitiesAppStartDependencies -> = async (pluginInitializerContext: PluginInitializerContext) => - new EntitiesAppPlugin(pluginInitializerContext); + StreamsAppServerSetup, + StreamsAppServerStart, + StreamsAppSetupDependencies, + StreamsAppStartDependencies +> = async (pluginInitializerContext: PluginInitializerContext) => + new StreamsAppPlugin(pluginInitializerContext); diff --git a/x-pack/plugins/logsai/streams_app/server/plugin.ts b/x-pack/plugins/logsai/streams_app/server/plugin.ts index 3c039d8b83b88..00fb61c1a9ccf 100644 --- a/x-pack/plugins/logsai/streams_app/server/plugin.ts +++ b/x-pack/plugins/logsai/streams_app/server/plugin.ts @@ -9,19 +9,19 @@ import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kb import type { Logger } from '@kbn/logging'; import type { ConfigSchema, - EntitiesAppServerSetup, - EntitiesAppServerStart, - EntitiesAppSetupDependencies, - EntitiesAppStartDependencies, + StreamsAppServerSetup, + StreamsAppServerStart, + StreamsAppSetupDependencies, + StreamsAppStartDependencies, } from './types'; -export class EntitiesAppPlugin +export class StreamsAppPlugin implements Plugin< - EntitiesAppServerSetup, - EntitiesAppServerStart, - EntitiesAppSetupDependencies, - EntitiesAppStartDependencies + StreamsAppServerSetup, + StreamsAppServerStart, + StreamsAppSetupDependencies, + StreamsAppStartDependencies > { logger: Logger; @@ -30,13 +30,13 @@ export class EntitiesAppPlugin this.logger = context.logger.get(); } setup( - coreSetup: CoreSetup, - pluginsSetup: EntitiesAppSetupDependencies - ): EntitiesAppServerSetup { + coreSetup: CoreSetup, + pluginsSetup: StreamsAppSetupDependencies + ): StreamsAppServerSetup { return {}; } - start(core: CoreStart, pluginsStart: EntitiesAppStartDependencies): EntitiesAppServerStart { + start(core: CoreStart, pluginsStart: StreamsAppStartDependencies): StreamsAppServerStart { return {}; } } diff --git a/x-pack/plugins/logsai/streams_app/server/types.ts b/x-pack/plugins/logsai/streams_app/server/types.ts index c56d169f7a8e6..0425b145b2746 100644 --- a/x-pack/plugins/logsai/streams_app/server/types.ts +++ b/x-pack/plugins/logsai/streams_app/server/types.ts @@ -8,10 +8,10 @@ export interface ConfigSchema {} -export interface EntitiesAppSetupDependencies {} +export interface StreamsAppSetupDependencies {} -export interface EntitiesAppStartDependencies {} +export interface StreamsAppStartDependencies {} -export interface EntitiesAppServerSetup {} +export interface StreamsAppServerSetup {} -export interface EntitiesAppServerStart {} +export interface StreamsAppServerStart {} diff --git a/x-pack/plugins/logsai/streams_app/tsconfig.json b/x-pack/plugins/logsai/streams_app/tsconfig.json index 47156e97ad7a2..452af0f751f7e 100644 --- a/x-pack/plugins/logsai/streams_app/tsconfig.json +++ b/x-pack/plugins/logsai/streams_app/tsconfig.json @@ -14,25 +14,5 @@ ], "exclude": ["target/**/*", ".storybook/**/*.js"], "kbn_references": [ - "@kbn/core", - "@kbn/observability-shared-plugin", - "@kbn/data-views-plugin", - "@kbn/data-plugin", - "@kbn/unified-search-plugin", - "@kbn/entities-api-plugin", - "@kbn/react-kibana-context-render", - "@kbn/i18n", - "@kbn/shared-ux-link-redirect-app", - "@kbn/typed-react-router-config", - "@kbn/kibana-react-plugin", - "@kbn/observability-utils-browser", - "@kbn/es-query", - "@kbn/logging", - "@kbn/deeplinks-observability", - "@kbn/es-types", - "@kbn/config-schema", - "@kbn/esql-datagrid", - "@kbn/esql-utils", - "@kbn/expressions-plugin", ] } diff --git a/yarn.lock b/yarn.lock index e7f4249969902..4dac6fb7759b7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6908,6 +6908,14 @@ version "0.0.0" uid "" +"@kbn/streams-api-plugin@link:x-pack/plugins/logsai/streams_api": + version "0.0.0" + uid "" + +"@kbn/streams-app-plugin@link:x-pack/plugins/logsai/streams_app": + version "0.0.0" + uid "" + "@kbn/streams-plugin@link:x-pack/plugins/streams": version "0.0.0" uid "" From d99a3ff546e6bcd716faad58a1f770ea859753e1 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 8 Nov 2024 17:41:32 +0100 Subject: [PATCH 07/95] stream management --- x-pack/plugins/streams/common/types.ts | 63 +- .../lib/streams/bootstrap_root_assets.ts | 30 - .../server/lib/streams/bootstrap_stream.ts | 62 - .../component_templates/generate_layer.ts | 35 +- .../streams/component_templates/logs_layer.ts | 1172 +---------------- ....test.ts => condition_to_painless.test.ts} | 14 +- ...o_painless.ts => condition_to_painless.ts} | 34 +- .../server/lib/streams/helpers/hierarchy.ts | 31 + .../lib/streams/index_templates/logs.ts | 11 - .../generate_ingest_pipeline.ts | 18 +- .../generate_reroute_pipeline.ts | 30 +- .../ingest_pipelines/logs_default_pipeline.ts | 48 +- .../ingest_pipelines/logs_json_pipeline.ts | 58 - .../lib/streams/root_stream_definition.ts | 20 + .../streams/server/lib/streams/stream_crud.ts | 121 +- x-pack/plugins/streams/server/routes/index.ts | 2 + .../streams/server/routes/streams/edit.ts | 162 +++ .../streams/server/routes/streams/enable.ts | 10 +- .../streams/server/routes/streams/fork.ts | 36 +- 19 files changed, 468 insertions(+), 1489 deletions(-) delete mode 100644 x-pack/plugins/streams/server/lib/streams/bootstrap_root_assets.ts delete mode 100644 x-pack/plugins/streams/server/lib/streams/bootstrap_stream.ts rename x-pack/plugins/streams/server/lib/streams/helpers/{reroute_condition_to_painless.test.ts => condition_to_painless.test.ts} (89%) rename x-pack/plugins/streams/server/lib/streams/helpers/{reroute_condition_to_painless.ts => condition_to_painless.ts} (68%) create mode 100644 x-pack/plugins/streams/server/lib/streams/helpers/hierarchy.ts delete mode 100644 x-pack/plugins/streams/server/lib/streams/index_templates/logs.ts delete mode 100644 x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_json_pipeline.ts create mode 100644 x-pack/plugins/streams/server/routes/streams/edit.ts diff --git a/x-pack/plugins/streams/common/types.ts b/x-pack/plugins/streams/common/types.ts index 5bdc018452376..0019eb8fee054 100644 --- a/x-pack/plugins/streams/common/types.ts +++ b/x-pack/plugins/streams/common/types.ts @@ -9,32 +9,62 @@ import { z } from '@kbn/zod'; const stringOrNumberOrBoolean = z.union([z.string(), z.number(), z.boolean()]); -export const rerouteFilterConditionSchema = z.object({ +export const filterConditionSchema = z.object({ field: z.string(), operator: z.enum(['eq', 'neq', 'lt', 'lte', 'gt', 'gte', 'contains', 'startsWith', 'endsWith']), value: stringOrNumberOrBoolean, }); -export type RerouteFilterCondition = z.infer; +export type FilterCondition = z.infer; -export interface RerouteAndCondition { - and: RerouteCondition[]; +export interface AndCondition { + and: Condition[]; } export interface RerouteOrCondition { - or: RerouteCondition[]; + or: Condition[]; } -export type RerouteCondition = RerouteFilterCondition | RerouteAndCondition | RerouteOrCondition; +export type Condition = FilterCondition | AndCondition | RerouteOrCondition | undefined; -export const rerouteConditionSchema: z.ZodType = z.lazy(() => +export const conditionSchema: z.ZodType = z.lazy(() => z.union([ - rerouteFilterConditionSchema, - z.object({ and: z.array(rerouteConditionSchema) }), - z.object({ or: z.array(rerouteConditionSchema) }), + filterConditionSchema, + z.object({ and: z.array(conditionSchema) }), + z.object({ or: z.array(conditionSchema) }), ]) ); +export const grokProcessingDefinitionSchema = z.object({ + type: z.literal('grok'), + field: z.string(), + patterns: z.array(z.string()), + pattern_definitions: z.optional(z.record(z.string())), +}); + +export const dissectProcessingDefinitionSchema = z.object({ + type: z.literal('dissect'), + field: z.string(), + pattern: z.string(), +}); + +export const processingDefinitionSchema = z.object({ + condition: z.optional(conditionSchema), + config: z.discriminatedUnion('type', [ + grokProcessingDefinitionSchema, + dissectProcessingDefinitionSchema, + ]), +}); + +export type ProcessingDefinition = z.infer; + +export const fieldDefinitionSchema = z.object({ + name: z.string(), + type: z.enum(['keyword', 'text', 'long', 'double', 'date', 'boolean', 'ip']), +}); + +export type FieldDefinition = z.infer; + /** * Example of a "root" stream * { @@ -51,9 +81,16 @@ export const rerouteConditionSchema: z.ZodType = z.lazy(() => export const streamDefinitonSchema = z.object({ id: z.string(), - forked_from: z.optional(z.string()), - condition: z.optional(rerouteConditionSchema), - root: z.boolean().default(false), + processing: z.array(processingDefinitionSchema).default([]), + fields: z.array(fieldDefinitionSchema).default([]), + children: z + .array( + z.object({ + id: z.string(), + condition: conditionSchema, + }) + ) + .default([]), }); export type StreamDefinition = z.infer; diff --git a/x-pack/plugins/streams/server/lib/streams/bootstrap_root_assets.ts b/x-pack/plugins/streams/server/lib/streams/bootstrap_root_assets.ts deleted file mode 100644 index e640339f1e456..0000000000000 --- a/x-pack/plugins/streams/server/lib/streams/bootstrap_root_assets.ts +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; -import { Logger } from '@kbn/logging'; -import { - upsertComponent, - upsertIngestPipeline, - upsertTemplate, -} from '../../templates/manage_index_templates'; -import { logsLayer } from './component_templates/logs_layer'; -import { logsDefaultPipeline } from './ingest_pipelines/logs_default_pipeline'; -import { logsIndexTemplate } from './index_templates/logs'; -import { logsJsonPipeline } from './ingest_pipelines/logs_json_pipeline'; - -interface BootstrapRootEntityParams { - esClient: ElasticsearchClient; - logger: Logger; -} - -export async function bootstrapRootEntity({ esClient, logger }: BootstrapRootEntityParams) { - await upsertComponent({ esClient, logger, component: logsLayer }); - await upsertIngestPipeline({ esClient, logger, pipeline: logsJsonPipeline }); - await upsertIngestPipeline({ esClient, logger, pipeline: logsDefaultPipeline }); - await upsertTemplate({ esClient, logger, template: logsIndexTemplate }); -} diff --git a/x-pack/plugins/streams/server/lib/streams/bootstrap_stream.ts b/x-pack/plugins/streams/server/lib/streams/bootstrap_stream.ts deleted file mode 100644 index c609af5b49978..0000000000000 --- a/x-pack/plugins/streams/server/lib/streams/bootstrap_stream.ts +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; -import { Logger } from '@kbn/logging'; -import { StreamDefinition } from '../../../common/types'; -import { generateLayer } from './component_templates/generate_layer'; -import { generateIngestPipeline } from './ingest_pipelines/generate_ingest_pipeline'; -import { - upsertComponent, - upsertIngestPipeline, - upsertTemplate, -} from '../../templates/manage_index_templates'; -import { generateIndexTemplate } from './index_templates/generate_index_template'; -import { getIndexTemplateComponents } from './stream_crud'; -import { generateReroutePipeline } from './ingest_pipelines/generate_reroute_pipeline'; - -interface BootstrapStreamParams { - scopedClusterClient: IScopedClusterClient; - definition: StreamDefinition; - rootDefinition: StreamDefinition; - logger: Logger; -} -export async function bootstrapStream({ - scopedClusterClient, - definition, - rootDefinition, - logger, -}: BootstrapStreamParams) { - const { composedOf, ignoreMissing } = await getIndexTemplateComponents({ - scopedClusterClient, - definition: rootDefinition, - }); - const reroutePipeline = await generateReroutePipeline({ - esClient: scopedClusterClient.asCurrentUser, - definition: rootDefinition, - }); - await upsertComponent({ - esClient: scopedClusterClient.asSecondaryAuthUser, - logger, - component: generateLayer(definition.id), - }); - await upsertIngestPipeline({ - esClient: scopedClusterClient.asSecondaryAuthUser, - logger, - pipeline: generateIngestPipeline(definition.id), - }); - await upsertIngestPipeline({ - esClient: scopedClusterClient.asSecondaryAuthUser, - logger, - pipeline: reroutePipeline, - }); - await upsertTemplate({ - esClient: scopedClusterClient.asSecondaryAuthUser, - logger, - template: generateIndexTemplate(definition.id, composedOf, ignoreMissing), - }); -} diff --git a/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts b/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts index b3a63bf5cc4c5..f7398cb1304a8 100644 --- a/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts +++ b/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts @@ -5,26 +5,31 @@ * 2.0. */ -import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { + ClusterPutComponentTemplateRequest, + MappingProperty, +} from '@elastic/elasticsearch/lib/api/types'; +import { StreamDefinition } from '../../../../common/types'; import { ASSET_VERSION } from '../../../../common/constants'; +import { logsSettings } from './logs_layer'; +import { isRoot } from '../helpers/hierarchy'; -export function generateLayer(id: string): ClusterPutComponentTemplateRequest { +export function generateLayer( + id: string, + definition: StreamDefinition +): ClusterPutComponentTemplateRequest { + const properties: Record = {}; + definition.fields.forEach((field) => { + properties[field.name] = { + type: field.type, + }; + }); return { name: `${id}@stream.layer`, template: { - settings: { - index: { - lifecycle: { - name: 'logs', - }, - codec: 'best_compression', - mapping: { - total_fields: { - ignore_dynamic_beyond_limit: true, - }, - ignore_malformed: true, - }, - }, + settings: isRoot(definition.id) ? logsSettings : {}, + mappings: { + properties, }, }, version: ASSET_VERSION, diff --git a/x-pack/plugins/streams/server/lib/streams/component_templates/logs_layer.ts b/x-pack/plugins/streams/server/lib/streams/component_templates/logs_layer.ts index 2c2053fa6c82d..6b41d04131c56 100644 --- a/x-pack/plugins/streams/server/lib/streams/component_templates/logs_layer.ts +++ b/x-pack/plugins/streams/server/lib/streams/component_templates/logs_layer.ts @@ -5,1171 +5,19 @@ * 2.0. */ -import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; -import { ASSET_VERSION } from '../../../../common/constants'; +import { IndicesIndexSettings } from '@elastic/elasticsearch/lib/api/types'; -export const logsLayer: ClusterPutComponentTemplateRequest = { - name: 'logs@stream.layer', - template: { - settings: { - index: { - lifecycle: { - name: 'logs', - }, - codec: 'best_compression', - mapping: { - total_fields: { - ignore_dynamic_beyond_limit: true, - }, - ignore_malformed: true, - }, - }, +export const logsSettings: IndicesIndexSettings = { + index: { + lifecycle: { + name: 'logs', }, - mappings: { - dynamic: false, - date_detection: false, - properties: { - '@timestamp': { - type: 'date', - }, - - // Base - labels: { - type: 'object', - }, - message: { - type: 'match_only_text', - }, - tags: { - ignore_above: 1024, - type: 'keyword', - }, - event: { - properties: { - ingested: { - type: 'date', - }, - }, - }, - - // file - file: { - properties: { - accessed: { - type: 'date', - }, - attributes: { - ignore_above: 1024, - type: 'keyword', - }, - code_signature: { - properties: { - digest_algorithm: { - ignore_above: 1024, - type: 'keyword', - }, - exists: { - type: 'boolean', - }, - signing_id: { - ignore_above: 1024, - type: 'keyword', - }, - status: { - ignore_above: 1024, - type: 'keyword', - }, - subject_name: { - ignore_above: 1024, - type: 'keyword', - }, - team_id: { - ignore_above: 1024, - type: 'keyword', - }, - timestamp: { - type: 'date', - }, - trusted: { - type: 'boolean', - }, - valid: { - type: 'boolean', - }, - }, - }, - created: { - type: 'date', - }, - ctime: { - type: 'date', - }, - device: { - ignore_above: 1024, - type: 'keyword', - }, - directory: { - ignore_above: 1024, - type: 'keyword', - }, - drive_letter: { - ignore_above: 1, - type: 'keyword', - }, - elf: { - properties: { - architecture: { - ignore_above: 1024, - type: 'keyword', - }, - byte_order: { - ignore_above: 1024, - type: 'keyword', - }, - cpu_type: { - ignore_above: 1024, - type: 'keyword', - }, - creation_date: { - type: 'date', - }, - exports: { - type: 'flattened', - }, - go_import_hash: { - ignore_above: 1024, - type: 'keyword', - }, - go_imports: { - type: 'flattened', - }, - go_imports_names_entropy: { - type: 'long', - }, - go_imports_names_var_entropy: { - type: 'long', - }, - go_stripped: { - type: 'boolean', - }, - header: { - properties: { - abi_version: { - ignore_above: 1024, - type: 'keyword', - }, - class: { - ignore_above: 1024, - type: 'keyword', - }, - data: { - ignore_above: 1024, - type: 'keyword', - }, - entrypoint: { - type: 'long', - }, - object_version: { - ignore_above: 1024, - type: 'keyword', - }, - os_abi: { - ignore_above: 1024, - type: 'keyword', - }, - type: { - ignore_above: 1024, - type: 'keyword', - }, - version: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - import_hash: { - ignore_above: 1024, - type: 'keyword', - }, - imports: { - type: 'flattened', - }, - imports_names_entropy: { - type: 'long', - }, - imports_names_var_entropy: { - type: 'long', - }, - sections: { - properties: { - chi2: { - type: 'long', - }, - entropy: { - type: 'long', - }, - flags: { - ignore_above: 1024, - type: 'keyword', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - physical_offset: { - ignore_above: 1024, - type: 'keyword', - }, - physical_size: { - type: 'long', - }, - type: { - ignore_above: 1024, - type: 'keyword', - }, - var_entropy: { - type: 'long', - }, - virtual_address: { - type: 'long', - }, - virtual_size: { - type: 'long', - }, - }, - type: 'nested', - }, - segments: { - properties: { - sections: { - ignore_above: 1024, - type: 'keyword', - }, - type: { - ignore_above: 1024, - type: 'keyword', - }, - }, - type: 'nested', - }, - shared_libraries: { - ignore_above: 1024, - type: 'keyword', - }, - telfhash: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - extension: { - ignore_above: 1024, - type: 'keyword', - }, - fork_name: { - ignore_above: 1024, - type: 'keyword', - }, - gid: { - ignore_above: 1024, - type: 'keyword', - }, - group: { - ignore_above: 1024, - type: 'keyword', - }, - hash: { - properties: { - md5: { - ignore_above: 1024, - type: 'keyword', - }, - sha1: { - ignore_above: 1024, - type: 'keyword', - }, - sha256: { - ignore_above: 1024, - type: 'keyword', - }, - sha384: { - ignore_above: 1024, - type: 'keyword', - }, - sha512: { - ignore_above: 1024, - type: 'keyword', - }, - ssdeep: { - ignore_above: 1024, - type: 'keyword', - }, - tlsh: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - inode: { - ignore_above: 1024, - type: 'keyword', - }, - macho: { - properties: { - go_import_hash: { - ignore_above: 1024, - type: 'keyword', - }, - go_imports: { - type: 'flattened', - }, - go_imports_names_entropy: { - type: 'long', - }, - go_imports_names_var_entropy: { - type: 'long', - }, - go_stripped: { - type: 'boolean', - }, - import_hash: { - ignore_above: 1024, - type: 'keyword', - }, - imports: { - type: 'flattened', - }, - imports_names_entropy: { - type: 'long', - }, - imports_names_var_entropy: { - type: 'long', - }, - sections: { - properties: { - entropy: { - type: 'long', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - physical_size: { - type: 'long', - }, - var_entropy: { - type: 'long', - }, - virtual_size: { - type: 'long', - }, - }, - type: 'nested', - }, - symhash: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - mime_type: { - ignore_above: 1024, - type: 'keyword', - }, - mode: { - ignore_above: 1024, - type: 'keyword', - }, - mtime: { - type: 'date', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - owner: { - ignore_above: 1024, - type: 'keyword', - }, - path: { - fields: { - text: { - type: 'match_only_text', - }, - }, - ignore_above: 1024, - type: 'keyword', - }, - pe: { - properties: { - architecture: { - ignore_above: 1024, - type: 'keyword', - }, - company: { - ignore_above: 1024, - type: 'keyword', - }, - description: { - ignore_above: 1024, - type: 'keyword', - }, - file_version: { - ignore_above: 1024, - type: 'keyword', - }, - go_import_hash: { - ignore_above: 1024, - type: 'keyword', - }, - go_imports: { - type: 'flattened', - }, - go_imports_names_entropy: { - type: 'long', - }, - go_imports_names_var_entropy: { - type: 'long', - }, - go_stripped: { - type: 'boolean', - }, - imphash: { - ignore_above: 1024, - type: 'keyword', - }, - import_hash: { - ignore_above: 1024, - type: 'keyword', - }, - imports: { - type: 'flattened', - }, - imports_names_entropy: { - type: 'long', - }, - imports_names_var_entropy: { - type: 'long', - }, - original_file_name: { - ignore_above: 1024, - type: 'keyword', - }, - pehash: { - ignore_above: 1024, - type: 'keyword', - }, - product: { - ignore_above: 1024, - type: 'keyword', - }, - sections: { - properties: { - entropy: { - type: 'long', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - physical_size: { - type: 'long', - }, - var_entropy: { - type: 'long', - }, - virtual_size: { - type: 'long', - }, - }, - type: 'nested', - }, - }, - }, - size: { - type: 'long', - }, - target_path: { - fields: { - text: { - type: 'match_only_text', - }, - }, - ignore_above: 1024, - type: 'keyword', - }, - type: { - ignore_above: 1024, - type: 'keyword', - }, - uid: { - ignore_above: 1024, - type: 'keyword', - }, - x509: { - properties: { - alternative_names: { - ignore_above: 1024, - type: 'keyword', - }, - issuer: { - properties: { - common_name: { - ignore_above: 1024, - type: 'keyword', - }, - country: { - ignore_above: 1024, - type: 'keyword', - }, - distinguished_name: { - ignore_above: 1024, - type: 'keyword', - }, - locality: { - ignore_above: 1024, - type: 'keyword', - }, - organization: { - ignore_above: 1024, - type: 'keyword', - }, - organizational_unit: { - ignore_above: 1024, - type: 'keyword', - }, - state_or_province: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - not_after: { - type: 'date', - }, - not_before: { - type: 'date', - }, - public_key_algorithm: { - ignore_above: 1024, - type: 'keyword', - }, - public_key_curve: { - ignore_above: 1024, - type: 'keyword', - }, - public_key_exponent: { - doc_values: false, - index: false, - type: 'long', - }, - public_key_size: { - type: 'long', - }, - serial_number: { - ignore_above: 1024, - type: 'keyword', - }, - signature_algorithm: { - ignore_above: 1024, - type: 'keyword', - }, - subject: { - properties: { - common_name: { - ignore_above: 1024, - type: 'keyword', - }, - country: { - ignore_above: 1024, - type: 'keyword', - }, - distinguished_name: { - ignore_above: 1024, - type: 'keyword', - }, - locality: { - ignore_above: 1024, - type: 'keyword', - }, - organization: { - ignore_above: 1024, - type: 'keyword', - }, - organizational_unit: { - ignore_above: 1024, - type: 'keyword', - }, - state_or_province: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - version_number: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - }, - }, - - // Host - host: { - properties: { - architecture: { - ignore_above: 1024, - type: 'keyword', - }, - cpu: { - properties: { - usage: { - scaling_factor: 1000, - type: 'scaled_float', - }, - }, - }, - disk: { - properties: { - read: { - properties: { - bytes: { - type: 'long', - }, - }, - }, - write: { - properties: { - bytes: { - type: 'long', - }, - }, - }, - }, - }, - domain: { - ignore_above: 1024, - type: 'keyword', - }, - geo: { - properties: { - city_name: { - ignore_above: 1024, - type: 'keyword', - }, - continent_code: { - ignore_above: 1024, - type: 'keyword', - }, - continent_name: { - ignore_above: 1024, - type: 'keyword', - }, - country_iso_code: { - ignore_above: 1024, - type: 'keyword', - }, - country_name: { - ignore_above: 1024, - type: 'keyword', - }, - location: { - type: 'geo_point', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - postal_code: { - ignore_above: 1024, - type: 'keyword', - }, - region_iso_code: { - ignore_above: 1024, - type: 'keyword', - }, - region_name: { - ignore_above: 1024, - type: 'keyword', - }, - timezone: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - hostname: { - ignore_above: 1024, - type: 'keyword', - }, - id: { - ignore_above: 1024, - type: 'keyword', - }, - ip: { - type: 'ip', - }, - mac: { - ignore_above: 1024, - type: 'keyword', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - network: { - properties: { - egress: { - properties: { - bytes: { - type: 'long', - }, - packets: { - type: 'long', - }, - }, - }, - ingress: { - properties: { - bytes: { - type: 'long', - }, - packets: { - type: 'long', - }, - }, - }, - }, - }, - os: { - properties: { - family: { - ignore_above: 1024, - type: 'keyword', - }, - full: { - fields: { - text: { - type: 'match_only_text', - }, - }, - ignore_above: 1024, - type: 'keyword', - }, - kernel: { - ignore_above: 1024, - type: 'keyword', - }, - name: { - fields: { - text: { - type: 'match_only_text', - }, - }, - ignore_above: 1024, - type: 'keyword', - }, - platform: { - ignore_above: 1024, - type: 'keyword', - }, - type: { - ignore_above: 1024, - type: 'keyword', - }, - version: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - type: { - ignore_above: 1024, - type: 'keyword', - }, - uptime: { - type: 'long', - }, - }, - }, - - // Orchestrator - orchestrator: { - properties: { - api_version: { - ignore_above: 1024, - type: 'keyword', - }, - cluster: { - properties: { - name: { - ignore_above: 1024, - type: 'keyword', - }, - url: { - ignore_above: 1024, - type: 'keyword', - }, - version: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - namespace: { - ignore_above: 1024, - type: 'keyword', - }, - organization: { - ignore_above: 1024, - type: 'keyword', - }, - resource: { - properties: { - name: { - ignore_above: 1024, - type: 'keyword', - }, - type: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - type: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - - log: { - properties: { - file: { - properties: { - path: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - level: { - ignore_above: 1024, - type: 'keyword', - }, - logger: { - ignore_above: 1024, - type: 'keyword', - }, - origin: { - properties: { - file: { - properties: { - line: { - type: 'long', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - function: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - syslog: { - properties: { - facility: { - properties: { - code: { - type: 'long', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - priority: { - type: 'long', - }, - severity: { - properties: { - code: { - type: 'long', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - }, - type: 'object', - }, - }, - }, - - // ECS - ecs: { - properties: { - version: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - - // Agent - agent: { - properties: { - build: { - properties: { - original: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - ephemeral_id: { - ignore_above: 1024, - type: 'keyword', - }, - id: { - ignore_above: 1024, - type: 'keyword', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - type: { - ignore_above: 1024, - type: 'keyword', - }, - version: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - - // Cloud - cloud: { - properties: { - account: { - properties: { - id: { - ignore_above: 1024, - type: 'keyword', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - availability_zone: { - ignore_above: 1024, - type: 'keyword', - }, - instance: { - properties: { - id: { - ignore_above: 1024, - type: 'keyword', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - machine: { - properties: { - type: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - origin: { - properties: { - account: { - properties: { - id: { - ignore_above: 1024, - type: 'keyword', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - availability_zone: { - ignore_above: 1024, - type: 'keyword', - }, - instance: { - properties: { - id: { - ignore_above: 1024, - type: 'keyword', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - machine: { - properties: { - type: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - project: { - properties: { - id: { - ignore_above: 1024, - type: 'keyword', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - provider: { - ignore_above: 1024, - type: 'keyword', - }, - region: { - ignore_above: 1024, - type: 'keyword', - }, - service: { - properties: { - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - }, - }, - project: { - properties: { - id: { - ignore_above: 1024, - type: 'keyword', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - provider: { - ignore_above: 1024, - type: 'keyword', - }, - region: { - ignore_above: 1024, - type: 'keyword', - }, - service: { - properties: { - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - target: { - properties: { - account: { - properties: { - id: { - ignore_above: 1024, - type: 'keyword', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - availability_zone: { - ignore_above: 1024, - type: 'keyword', - }, - instance: { - properties: { - id: { - ignore_above: 1024, - type: 'keyword', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - machine: { - properties: { - type: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - project: { - properties: { - id: { - ignore_above: 1024, - type: 'keyword', - }, - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - provider: { - ignore_above: 1024, - type: 'keyword', - }, - region: { - ignore_above: 1024, - type: 'keyword', - }, - service: { - properties: { - name: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - }, - }, - }, - }, + codec: 'best_compression', + mapping: { + total_fields: { + ignore_dynamic_beyond_limit: true, }, + ignore_malformed: true, }, }, - version: ASSET_VERSION, - _meta: { - managed: true, - description: 'Default layer for logs stream', - }, - deprecated: false, }; diff --git a/x-pack/plugins/streams/server/lib/streams/helpers/reroute_condition_to_painless.test.ts b/x-pack/plugins/streams/server/lib/streams/helpers/condition_to_painless.test.ts similarity index 89% rename from x-pack/plugins/streams/server/lib/streams/helpers/reroute_condition_to_painless.test.ts rename to x-pack/plugins/streams/server/lib/streams/helpers/condition_to_painless.test.ts index 243ea404ba66d..aab7f27f12d14 100644 --- a/x-pack/plugins/streams/server/lib/streams/helpers/reroute_condition_to_painless.test.ts +++ b/x-pack/plugins/streams/server/lib/streams/helpers/condition_to_painless.test.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { rerouteConditionToPainless } from './reroute_condition_to_painless'; +import { conditionToPainless } from './condition_to_painless'; const operatorConditionAndResutls = [ { @@ -46,11 +46,11 @@ const operatorConditionAndResutls = [ }, ]; -describe('rerouteConditionToPainless', () => { +describe('conditionToPainless', () => { describe('operators', () => { operatorConditionAndResutls.forEach((setup) => { test(`${setup.condition.operator}`, () => { - expect(rerouteConditionToPainless(setup.condition)).toEqual(setup.result); + expect(conditionToPainless(setup.condition)).toEqual(setup.result); }); }); }); @@ -64,7 +64,7 @@ describe('rerouteConditionToPainless', () => { ], }; expect( - expect(rerouteConditionToPainless(condition)).toEqual( + expect(conditionToPainless(condition)).toEqual( 'ctx.log?.logger == "nginx_proxy" && ctx.log?.level == "error"' ) ); @@ -80,7 +80,7 @@ describe('rerouteConditionToPainless', () => { ], }; expect( - expect(rerouteConditionToPainless(condition)).toEqual( + expect(conditionToPainless(condition)).toEqual( 'ctx.log?.logger == "nginx_proxy" || ctx.log?.level == "error"' ) ); @@ -101,7 +101,7 @@ describe('rerouteConditionToPainless', () => { ], }; expect( - expect(rerouteConditionToPainless(condition)).toEqual( + expect(conditionToPainless(condition)).toEqual( 'ctx.log?.logger == "nginx_proxy" && (ctx.log?.level == "error" || ctx.log?.level == "ERROR")' ) ); @@ -124,7 +124,7 @@ describe('rerouteConditionToPainless', () => { ], }; expect( - expect(rerouteConditionToPainless(condition)).toEqual( + expect(conditionToPainless(condition)).toEqual( '(ctx.log?.logger == "nginx_proxy" || ctx.service?.name == "nginx") && (ctx.log?.level == "error" || ctx.log?.level == "ERROR")' ) ); diff --git a/x-pack/plugins/streams/server/lib/streams/helpers/reroute_condition_to_painless.ts b/x-pack/plugins/streams/server/lib/streams/helpers/condition_to_painless.ts similarity index 68% rename from x-pack/plugins/streams/server/lib/streams/helpers/reroute_condition_to_painless.ts rename to x-pack/plugins/streams/server/lib/streams/helpers/condition_to_painless.ts index 5df1e30097b58..539ad3603535b 100644 --- a/x-pack/plugins/streams/server/lib/streams/helpers/reroute_condition_to_painless.ts +++ b/x-pack/plugins/streams/server/lib/streams/helpers/condition_to_painless.ts @@ -7,30 +7,30 @@ import { isBoolean, isString } from 'lodash'; import { - RerouteAndCondition, - RerouteCondition, - rerouteConditionSchema, - RerouteFilterCondition, - rerouteFilterConditionSchema, + AndCondition, + Condition, + conditionSchema, + FilterCondition, + filterConditionSchema, RerouteOrCondition, } from '../../../../common/types'; -function isFilterCondition(subject: any): subject is RerouteFilterCondition { - const result = rerouteFilterConditionSchema.safeParse(subject); +function isFilterCondition(subject: any): subject is FilterCondition { + const result = filterConditionSchema.safeParse(subject); return result.success; } -function isAndCondition(subject: any): subject is RerouteAndCondition { - const result = rerouteConditionSchema.safeParse(subject); +function isAndCondition(subject: any): subject is AndCondition { + const result = conditionSchema.safeParse(subject); return result.success && subject.and != null; } function isOrCondition(subject: any): subject is RerouteOrCondition { - const result = rerouteConditionSchema.safeParse(subject); + const result = conditionSchema.safeParse(subject); return result.success && subject.or != null; } -function safePainlessField(condition: RerouteFilterCondition) { +function safePainlessField(condition: FilterCondition) { return `ctx.${condition.field.split('.').join('?.')}`; } @@ -44,7 +44,7 @@ function encodeValue(value: string | number | boolean) { return value; } -function toPainless(condition: RerouteFilterCondition) { +function toPainless(condition: FilterCondition) { switch (condition.operator) { case 'neq': return `${safePainlessField(condition)} != ${encodeValue(condition.value)}`; @@ -67,19 +67,17 @@ function toPainless(condition: RerouteFilterCondition) { } } -export function rerouteConditionToPainless(condition: RerouteCondition, nested = false): string { +export function conditionToPainless(condition: Condition, nested = false): string { if (isFilterCondition(condition)) { return toPainless(condition); } if (isAndCondition(condition)) { - const and = condition.and - .map((filter) => rerouteConditionToPainless(filter, true)) - .join(' && '); + const and = condition.and.map((filter) => conditionToPainless(filter, true)).join(' && '); return nested ? `(${and})` : and; } if (isOrCondition(condition)) { - const or = condition.or.map((filter) => rerouteConditionToPainless(filter, true)).join(' || '); + const or = condition.or.map((filter) => conditionToPainless(filter, true)).join(' || '); return nested ? `(${or})` : or; } - return ''; + return 'false'; } diff --git a/x-pack/plugins/streams/server/lib/streams/helpers/hierarchy.ts b/x-pack/plugins/streams/server/lib/streams/helpers/hierarchy.ts new file mode 100644 index 0000000000000..eeb69f89473a1 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/helpers/hierarchy.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { StreamDefinition } from '../../../../common/types'; + +export function isDescendandOf(parent: StreamDefinition, child: StreamDefinition) { + return child.id.startsWith(parent.id); +} + +// streams are named by dots: logs.a.b.c - a child means there is only one level of dots added +export function isChildOf(parent: StreamDefinition, child: StreamDefinition) { + return ( + isDescendandOf(parent, child) && child.id.split('.').length === parent.id.split('.').length + 1 + ); +} + +export function getParentId(id: string) { + const parts = id.split('.'); + if (parts.length === 1) { + return undefined; + } + return parts.slice(0, parts.length - 1).join('.'); +} + +export function isRoot(id: string) { + return id.split('.').length === 1; +} diff --git a/x-pack/plugins/streams/server/lib/streams/index_templates/logs.ts b/x-pack/plugins/streams/server/lib/streams/index_templates/logs.ts deleted file mode 100644 index 58324213dae16..0000000000000 --- a/x-pack/plugins/streams/server/lib/streams/index_templates/logs.ts +++ /dev/null @@ -1,11 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; -import { generateIndexTemplate } from './generate_index_template'; - -export const logsIndexTemplate: IndicesPutIndexTemplateRequest = generateIndexTemplate('logs'); diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts index 1cd7067014a95..146671e79744f 100644 --- a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts @@ -5,12 +5,26 @@ * 2.0. */ +import { StreamDefinition } from '../../../../common/types'; import { ASSET_VERSION } from '../../../../common/constants'; +import { conditionToPainless } from '../helpers/condition_to_painless'; +import { logsDefaultPipelineProcessors } from './logs_default_pipeline'; +import { isRoot } from '../helpers/hierarchy'; -export function generateIngestPipeline(id: string) { +export function generateIngestPipeline(id: string, definition: StreamDefinition) { return { id: `${id}@stream.default-pipeline`, processors: [ + ...(isRoot(definition.id) ? logsDefaultPipelineProcessors : []), + ...definition.processing.map((processor) => { + const { type, ...config } = processor.config; + return { + [type]: { + ...config, + if: processor.condition ? conditionToPainless(processor.condition) : undefined, + }, + }; + }), { pipeline: { name: `${id}@stream.reroutes`, @@ -19,7 +33,7 @@ export function generateIngestPipeline(id: string) { }, ], _meta: { - description: `Default pipeline for the ${id} streams`, + description: `Default pipeline for the ${id} stream`, managed: true, }, version: ASSET_VERSION, diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts index 43dc8cb787f5a..b73dc61b148ed 100644 --- a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts @@ -5,40 +5,22 @@ * 2.0. */ -import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { StreamDefinition } from '../../../../common/types'; -import { ASSET_VERSION, STREAMS_INDEX } from '../../../../common/constants'; -import { rerouteConditionToPainless } from '../helpers/reroute_condition_to_painless'; +import { ASSET_VERSION } from '../../../../common/constants'; +import { conditionToPainless } from '../helpers/condition_to_painless'; interface GenerateReroutePipelineParams { - esClient: ElasticsearchClient; definition: StreamDefinition; } -export async function generateReroutePipeline({ - esClient, - definition, -}: GenerateReroutePipelineParams) { - const response = await esClient.search({ - index: STREAMS_INDEX, - query: { match: { forked_from: definition.id } }, - }); - +export async function generateReroutePipeline({ definition }: GenerateReroutePipelineParams) { return { id: `${definition.id}@stream.reroutes`, - processors: response.hits.hits.map((doc) => { - if (!doc._source) { - throw new Error('Source missing for stream definiton document'); - } - if (!doc._source.condition) { - throw new Error( - `Reroute condition missing from forked stream definition ${doc._source.id}` - ); - } + processors: definition.children.map((child) => { return { reroute: { - destination: doc._source.id, - if: rerouteConditionToPainless(doc._source.condition), + destination: child.id, + if: conditionToPainless(child.condition), }, }; }), diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_default_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_default_pipeline.ts index 9d9edf3e92b4a..762155ba5047c 100644 --- a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_default_pipeline.ts +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_default_pipeline.ts @@ -5,41 +5,19 @@ * 2.0. */ -import { ASSET_VERSION } from '../../../../common/constants'; - -export const logsDefaultPipeline = { - id: 'logs@stream.default-pipeline', - processors: [ - { - set: { - description: "If '@timestamp' is missing, set it with the ingest timestamp", - field: '@timestamp', - override: false, - copy_from: '_ingest.timestamp', - }, - }, - { - set: { - field: 'event.ingested', - value: '{{{_ingest.timestamp}}}', - }, +export const logsDefaultPipelineProcessors = [ + { + set: { + description: "If '@timestamp' is missing, set it with the ingest timestamp", + field: '@timestamp', + override: false, + copy_from: '_ingest.timestamp', }, - { - pipeline: { - name: 'logs@stream.json-pipeline', - ignore_missing_pipeline: true, - }, - }, - { - pipeline: { - name: 'logs@stream.reroutes', - ignore_missing_pipeline: true, - }, + }, + { + pipeline: { + name: 'logs@json-pipeline', + ignore_missing_pipeline: true, }, - ], - _meta: { - description: 'Default pipeline for the logs stream', - managed: true, }, - version: ASSET_VERSION, -}; +]; diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_json_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_json_pipeline.ts deleted file mode 100644 index 2b6fb7d8d9344..0000000000000 --- a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_json_pipeline.ts +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ASSET_VERSION } from '../../../../common/constants'; - -export const logsJsonPipeline = { - id: 'logs@stream.json-pipeline', - processors: [ - { - rename: { - if: "ctx.message instanceof String && ctx.message.startsWith('{') && ctx.message.endsWith('}')", - field: 'message', - target_field: '_tmp_json_message', - ignore_missing: true, - }, - }, - { - json: { - if: 'ctx._tmp_json_message != null', - field: '_tmp_json_message', - add_to_root: true, - add_to_root_conflict_strategy: 'merge' as const, - allow_duplicate_keys: true, - on_failure: [ - { - rename: { - field: '_tmp_json_message', - target_field: 'message', - ignore_missing: true, - }, - }, - ], - }, - }, - { - dot_expander: { - if: 'ctx._tmp_json_message != null', - field: '*', - override: true, - }, - }, - { - remove: { - field: '_tmp_json_message', - ignore_missing: true, - }, - }, - ], - _meta: { - description: 'automatic parsing of JSON log messages', - managed: true, - }, - version: ASSET_VERSION, -}; diff --git a/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts b/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts index 4d95632df9f9e..3555146ccc008 100644 --- a/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts +++ b/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts @@ -10,4 +10,24 @@ import { StreamDefinition } from '../../../common/types'; export const rootStreamDefinition: StreamDefinition = { id: 'logs', root: true, + processing: [], + children: [], + fields: [ + { + name: '@timestamp', + type: 'date', + }, + { + name: 'message', + type: 'text', + }, + { + name: 'host.name', + type: 'keyword', + }, + { + name: 'log.level', + type: 'keyword', + }, + ], }; diff --git a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts index 47e2b4061c063..5fe9e5f6de0df 100644 --- a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts +++ b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts @@ -7,9 +7,19 @@ import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; import { get } from 'lodash'; +import { Logger } from '@kbn/logging'; import { StreamDefinition } from '../../../common/types'; import { STREAMS_INDEX } from '../../../common/constants'; -import { ComponentTemplateNotFound, DefinitionNotFound, IndexTemplateNotFound } from './errors'; +import { DefinitionNotFound, IndexTemplateNotFound } from './errors'; +import { + upsertComponent, + upsertIngestPipeline, + upsertTemplate, +} from '../../templates/manage_index_templates'; +import { generateLayer } from './component_templates/generate_layer'; +import { generateIngestPipeline } from './ingest_pipelines/generate_ingest_pipeline'; +import { generateReroutePipeline } from './ingest_pipelines/generate_reroute_pipeline'; +import { generateIndexTemplate } from './index_templates/generate_index_template'; interface BaseParams { scopedClusterClient: IScopedClusterClient; @@ -19,7 +29,7 @@ interface BaseParamsWithDefinition extends BaseParams { definition: StreamDefinition; } -export async function createStream({ definition, scopedClusterClient }: BaseParamsWithDefinition) { +async function upsertStream({ definition, scopedClusterClient }: BaseParamsWithDefinition) { return scopedClusterClient.asCurrentUser.index({ id: definition.id, index: STREAMS_INDEX, @@ -39,14 +49,8 @@ export async function readStream({ id, scopedClusterClient }: ReadStreamParams) index: STREAMS_INDEX, }); const definition = response._source as StreamDefinition; - const indexTemplate = await readIndexTemplate({ scopedClusterClient, definition }); - const componentTemplate = await readComponentTemplate({ scopedClusterClient, definition }); - const ingestPipelines = await readIngestPipelines({ scopedClusterClient, definition }); return { definition, - index_template: indexTemplate, - component_template: componentTemplate, - ingest_pipelines: ingestPipelines, }; } catch (e) { if (e.meta?.statusCode === 404) { @@ -56,6 +60,18 @@ export async function readStream({ id, scopedClusterClient }: ReadStreamParams) } } +export async function checkStreamExists({ id, scopedClusterClient }: ReadStreamParams) { + try { + await readStream({ id, scopedClusterClient }); + return true; + } catch (e) { + if (e instanceof DefinitionNotFound) { + return false; + } + throw e; + } +} + export async function readIndexTemplate({ scopedClusterClient, definition, @@ -72,33 +88,6 @@ export async function readIndexTemplate({ return indexTemplate; } -export async function readComponentTemplate({ - scopedClusterClient, - definition, -}: BaseParamsWithDefinition) { - const response = await scopedClusterClient.asSecondaryAuthUser.cluster.getComponentTemplate({ - name: `${definition.id}@stream.layer`, - }); - const componentTemplate = response.component_templates.find( - (doc) => doc.name === `${definition.id}@stream.layer` - ); - if (!componentTemplate) { - throw new ComponentTemplateNotFound(`Unable to find component_template for ${definition.id}`); - } - return componentTemplate; -} - -export async function readIngestPipelines({ - scopedClusterClient, - definition, -}: BaseParamsWithDefinition) { - const response = await scopedClusterClient.asSecondaryAuthUser.ingest.getPipeline({ - id: `${definition.id}@stream.*`, - }); - - return response; -} - export async function getIndexTemplateComponents({ scopedClusterClient, definition, @@ -113,3 +102,65 @@ export async function getIndexTemplateComponents({ ) as string[], }; } + +interface SyncStreamParams { + scopedClusterClient: IScopedClusterClient; + definition: StreamDefinition; + rootDefinition?: StreamDefinition; + logger: Logger; +} + +export async function syncStream({ + scopedClusterClient, + definition, + rootDefinition, + logger, +}: SyncStreamParams) { + await upsertStream({ + scopedClusterClient, + definition, + }); + await upsertComponent({ + esClient: scopedClusterClient.asSecondaryAuthUser, + logger, + component: generateLayer(definition.id, definition), + }); + await upsertIngestPipeline({ + esClient: scopedClusterClient.asSecondaryAuthUser, + logger, + pipeline: generateIngestPipeline(definition.id, definition), + }); + const reroutePipeline = await generateReroutePipeline({ + definition, + }); + await upsertIngestPipeline({ + esClient: scopedClusterClient.asSecondaryAuthUser, + logger, + pipeline: reroutePipeline, + }); + if (rootDefinition) { + const { composedOf, ignoreMissing } = await getIndexTemplateComponents({ + scopedClusterClient, + definition: rootDefinition, + }); + const parentReroutePipeline = await generateReroutePipeline({ + definition: rootDefinition, + }); + await upsertIngestPipeline({ + esClient: scopedClusterClient.asSecondaryAuthUser, + logger, + pipeline: parentReroutePipeline, + }); + await upsertTemplate({ + esClient: scopedClusterClient.asSecondaryAuthUser, + logger, + template: generateIndexTemplate(definition.id, composedOf, ignoreMissing), + }); + } else { + await upsertTemplate({ + esClient: scopedClusterClient.asSecondaryAuthUser, + logger, + template: generateIndexTemplate(definition.id), + }); + } +} diff --git a/x-pack/plugins/streams/server/routes/index.ts b/x-pack/plugins/streams/server/routes/index.ts index c24362f3ec8a9..544b60ccd404c 100644 --- a/x-pack/plugins/streams/server/routes/index.ts +++ b/x-pack/plugins/streams/server/routes/index.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { editStreamRoute } from './streams/edit'; import { enableStreamsRoute } from './streams/enable'; import { forkStreamsRoute } from './streams/fork'; import { readStreamRoute } from './streams/read'; @@ -13,6 +14,7 @@ export const StreamsRouteRepository = { ...enableStreamsRoute, ...forkStreamsRoute, ...readStreamRoute, + ...editStreamRoute, }; export type StreamsRouteRepository = typeof StreamsRouteRepository; diff --git a/x-pack/plugins/streams/server/routes/streams/edit.ts b/x-pack/plugins/streams/server/routes/streams/edit.ts new file mode 100644 index 0000000000000..2ee75e32af730 --- /dev/null +++ b/x-pack/plugins/streams/server/routes/streams/edit.ts @@ -0,0 +1,162 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; +import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; +import { Logger } from '@kbn/logging'; +import { + DefinitionNotFound, + ForkConditionMissing, + IndexTemplateNotFound, + SecurityException, +} from '../../lib/streams/errors'; +import { createServerRoute } from '../create_server_route'; +import { StreamDefinition, streamDefinitonSchema } from '../../../common/types'; +import { syncStream, readStream, checkStreamExists } from '../../lib/streams/stream_crud'; +import { MalformedStreamId } from '../../lib/streams/errors/malformed_stream_id'; +import { getParentId } from '../../lib/streams/helpers/hierarchy'; + +export const editStreamRoute = createServerRoute({ + endpoint: 'PUT /api/streams/{id} 2023-10-31', + options: { + access: 'public', + security: { + authz: { + requiredPrivileges: ['streams_fork'], + }, + }, + }, + params: z.object({ + path: z.object({ + id: z.string(), + }), + body: z.object({ definition: streamDefinitonSchema }), + }), + handler: async ({ response, params, logger, request, getScopedClients }) => { + try { + const { scopedClusterClient } = await getScopedClients({ request }); + + await validateStreamChildren( + scopedClusterClient, + params.path.id, + params.body.definition.children + ); + + const parentId = getParentId(params.path.id); + let parentDefinition: StreamDefinition | undefined; + + if (parentId) { + parentDefinition = await updateParentStream( + scopedClusterClient, + parentId, + params.path.id, + logger + ); + } + const streamDefinition = { ...params.body.definition }; + + await syncStream({ + scopedClusterClient, + definition: streamDefinition, + rootDefinition: parentDefinition, + logger, + }); + + for (const child of streamDefinition.children) { + const streamExists = await checkStreamExists({ + scopedClusterClient, + id: child.id, + }); + if (streamExists) { + continue; + } + // create empty streams for each child if they don't exist + const childDefinition = { + id: child.id, + children: [], + fields: [], + processing: [], + root: false, + }; + + await syncStream({ + scopedClusterClient, + definition: childDefinition, + logger, + }); + } + + return response.ok({ body: { acknowledged: true } }); + } catch (e) { + if (e instanceof IndexTemplateNotFound || e instanceof DefinitionNotFound) { + return response.notFound({ body: e }); + } + + if ( + e instanceof SecurityException || + e instanceof ForkConditionMissing || + e instanceof MalformedStreamId + ) { + return response.customError({ body: e, statusCode: 400 }); + } + + return response.customError({ body: e, statusCode: 500 }); + } + }, +}); + +async function updateParentStream( + scopedClusterClient: IScopedClusterClient, + parentId: string, + id: string, + logger: Logger +) { + const { definition: parentDefinition } = await readStream({ + scopedClusterClient, + id: parentId, + }); + + if (!parentDefinition.children.some((child) => child.id === id)) { + // add the child to the parent stream with an empty condition for now + parentDefinition.children.push({ + id, + condition: undefined, + }); + + await syncStream({ + scopedClusterClient, + definition: parentDefinition, + logger, + }); + } + return parentDefinition; +} + +async function validateStreamChildren( + scopedClusterClient: IScopedClusterClient, + id: string, + children: Array<{ id: string }> +) { + try { + const { definition: oldDefinition } = await readStream({ + scopedClusterClient, + id, + }); + const oldChildren = oldDefinition.children.map((child) => child.id); + const newChildren = new Set(children.map((child) => child.id)); + if (oldChildren.some((child) => !newChildren.has(child))) { + throw new MalformedStreamId( + 'Cannot remove children from a stream, please delete the stream instead' + ); + } + } catch (e) { + // Ignore if the stream does not exist, but re-throw if it's another error + if (!(e instanceof DefinitionNotFound)) { + throw e; + } + } +} diff --git a/x-pack/plugins/streams/server/routes/streams/enable.ts b/x-pack/plugins/streams/server/routes/streams/enable.ts index 1241434a975df..111cd870c41a4 100644 --- a/x-pack/plugins/streams/server/routes/streams/enable.ts +++ b/x-pack/plugins/streams/server/routes/streams/enable.ts @@ -8,8 +8,7 @@ import { z } from '@kbn/zod'; import { SecurityException } from '../../lib/streams/errors'; import { createServerRoute } from '../create_server_route'; -import { bootstrapRootEntity } from '../../lib/streams/bootstrap_root_assets'; -import { createStream } from '../../lib/streams/stream_crud'; +import { syncStream } from '../../lib/streams/stream_crud'; import { rootStreamDefinition } from '../../lib/streams/root_stream_definition'; export const enableStreamsRoute = createServerRoute({ @@ -26,13 +25,10 @@ export const enableStreamsRoute = createServerRoute({ handler: async ({ request, response, logger, getScopedClients }) => { try { const { scopedClusterClient } = await getScopedClients({ request }); - await bootstrapRootEntity({ - esClient: scopedClusterClient.asSecondaryAuthUser, - logger, - }); - await createStream({ + await syncStream({ scopedClusterClient, definition: rootStreamDefinition, + logger, }); return response.ok({ body: { acknowledged: true } }); } catch (e) { diff --git a/x-pack/plugins/streams/server/routes/streams/fork.ts b/x-pack/plugins/streams/server/routes/streams/fork.ts index c019abd17b58d..ced96f465e3d3 100644 --- a/x-pack/plugins/streams/server/routes/streams/fork.ts +++ b/x-pack/plugins/streams/server/routes/streams/fork.ts @@ -13,10 +13,10 @@ import { SecurityException, } from '../../lib/streams/errors'; import { createServerRoute } from '../create_server_route'; -import { streamDefinitonSchema } from '../../../common/types'; -import { bootstrapStream } from '../../lib/streams/bootstrap_stream'; -import { createStream, readStream } from '../../lib/streams/stream_crud'; +import { conditionSchema, streamDefinitonSchema } from '../../../common/types'; +import { syncStream, readStream } from '../../lib/streams/stream_crud'; import { MalformedStreamId } from '../../lib/streams/errors/malformed_stream_id'; +import { isChildOf } from '../../lib/streams/helpers/hierarchy'; export const forkStreamsRoute = createServerRoute({ endpoint: 'POST /api/streams/{id}/_fork 2023-10-31', @@ -32,7 +32,7 @@ export const forkStreamsRoute = createServerRoute({ path: z.object({ id: z.string(), }), - body: streamDefinitonSchema, + body: z.object({ stream: streamDefinitonSchema, condition: conditionSchema }), }), handler: async ({ response, params, logger, request, getScopedClients }) => { try { @@ -47,20 +47,36 @@ export const forkStreamsRoute = createServerRoute({ id: params.path.id, }); - if (!params.body.id.startsWith(rootDefinition.id)) { + const childDefinition = { ...params.body.stream, root: false }; + + // check whether root stream has a child of the given name already + if (rootDefinition.children.some((child) => child.id === childDefinition.id)) { + throw new MalformedStreamId( + `The stream with ID (${params.body.stream.id}) already exists as a child of the parent stream` + ); + } + + if (!isChildOf(rootDefinition, childDefinition)) { throw new MalformedStreamId( - `The ID (${params.body.id}) from the new stream must start with the parent's id (${rootDefinition.id})` + `The ID (${params.body.stream.id}) from the new stream must start with the parent's id (${rootDefinition.id}), followed by a dot and a name` ); } - await createStream({ + rootDefinition.children.push({ + id: params.body.stream.id, + condition: params.body.condition, + }); + + await syncStream({ scopedClusterClient, - definition: { ...params.body, forked_from: rootDefinition.id, root: false }, + definition: rootDefinition, + rootDefinition, + logger, }); - await bootstrapStream({ + await syncStream({ scopedClusterClient, - definition: params.body, + definition: params.body.stream, rootDefinition, logger, }); From 0e893453e3401a0d5b19c1e3b48252528ae61c8a Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 8 Nov 2024 22:27:08 +0100 Subject: [PATCH 08/95] add delete and listing endpoint --- .../component_templates/generate_layer.ts | 3 +- .../manage_component_templates.ts | 28 +++++ .../lib/streams/component_templates/name.ts | 10 ++ .../server/lib/streams/helpers/hierarchy.ts | 1 - .../generate_index_template.ts | 6 +- .../lib/streams/index_templates/name.ts | 10 ++ .../generate_ingest_pipeline.ts | 3 +- .../generate_reroute_pipeline.ts | 3 +- .../manage_ingest_pipelines.ts | 27 +++++ .../lib/streams/ingest_pipelines/name.ts | 14 +++ .../lib/streams/root_stream_definition.ts | 1 - .../streams/server/lib/streams/stream_crud.ts | 52 +++++++++ x-pack/plugins/streams/server/routes/index.ts | 4 + .../streams/server/routes/streams/delete.ts | 104 ++++++++++++++++++ .../streams/server/routes/streams/list.ts | 65 +++++++++++ 15 files changed, 324 insertions(+), 7 deletions(-) create mode 100644 x-pack/plugins/streams/server/lib/streams/component_templates/manage_component_templates.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/component_templates/name.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/index_templates/name.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/ingest_pipelines/manage_ingest_pipelines.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/ingest_pipelines/name.ts create mode 100644 x-pack/plugins/streams/server/routes/streams/delete.ts create mode 100644 x-pack/plugins/streams/server/routes/streams/list.ts diff --git a/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts b/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts index f7398cb1304a8..4a39a78b38990 100644 --- a/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts +++ b/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts @@ -13,6 +13,7 @@ import { StreamDefinition } from '../../../../common/types'; import { ASSET_VERSION } from '../../../../common/constants'; import { logsSettings } from './logs_layer'; import { isRoot } from '../helpers/hierarchy'; +import { getComponentTemplateName } from './name'; export function generateLayer( id: string, @@ -25,7 +26,7 @@ export function generateLayer( }; }); return { - name: `${id}@stream.layer`, + name: getComponentTemplateName(id), template: { settings: isRoot(definition.id) ? logsSettings : {}, mappings: { diff --git a/x-pack/plugins/streams/server/lib/streams/component_templates/manage_component_templates.ts b/x-pack/plugins/streams/server/lib/streams/component_templates/manage_component_templates.ts new file mode 100644 index 0000000000000..0eed31ecab086 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/component_templates/manage_component_templates.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { Logger } from '@kbn/logging'; +import { retryTransientEsErrors } from '../helpers/retry'; + +interface DeleteComponentOptions { + esClient: ElasticsearchClient; + name: string; + logger: Logger; +} + +export async function deleteComponent({ esClient, name, logger }: DeleteComponentOptions) { + try { + await retryTransientEsErrors( + () => esClient.cluster.deleteComponentTemplate({ name }, { ignore: [404] }), + { logger } + ); + } catch (error: any) { + logger.error(`Error deleting component template: ${error.message}`); + throw error; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/component_templates/name.ts b/x-pack/plugins/streams/server/lib/streams/component_templates/name.ts new file mode 100644 index 0000000000000..6ea05b9a53b28 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/component_templates/name.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function getComponentTemplateName(id: string) { + return `${id}@stream.layer`; +} diff --git a/x-pack/plugins/streams/server/lib/streams/helpers/hierarchy.ts b/x-pack/plugins/streams/server/lib/streams/helpers/hierarchy.ts index eeb69f89473a1..f31846bab36db 100644 --- a/x-pack/plugins/streams/server/lib/streams/helpers/hierarchy.ts +++ b/x-pack/plugins/streams/server/lib/streams/helpers/hierarchy.ts @@ -11,7 +11,6 @@ export function isDescendandOf(parent: StreamDefinition, child: StreamDefinition return child.id.startsWith(parent.id); } -// streams are named by dots: logs.a.b.c - a child means there is only one level of dots added export function isChildOf(parent: StreamDefinition, child: StreamDefinition) { return ( isDescendandOf(parent, child) && child.id.split('.').length === parent.id.split('.').length + 1 diff --git a/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts b/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts index 14972ce8a6380..295c819a08a30 100644 --- a/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts +++ b/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts @@ -6,6 +6,8 @@ */ import { ASSET_VERSION } from '../../../../common/constants'; +import { getProcessingPipelineName } from '../ingest_pipelines/name'; +import { getIndexTemplateName } from './name'; export function generateIndexTemplate( id: string, @@ -13,7 +15,7 @@ export function generateIndexTemplate( ignoreMissing: string[] = [] ) { return { - name: `${id}@stream`, + name: getIndexTemplateName(id), index_patterns: [id], composed_of: [...composedOf, `${id}@stream.layer`], priority: 200, @@ -28,7 +30,7 @@ export function generateIndexTemplate( template: { settings: { index: { - default_pipeline: `${id}@stream.default-pipeline`, + default_pipeline: getProcessingPipelineName(id), }, }, }, diff --git a/x-pack/plugins/streams/server/lib/streams/index_templates/name.ts b/x-pack/plugins/streams/server/lib/streams/index_templates/name.ts new file mode 100644 index 0000000000000..ec8ea5519a6b4 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/index_templates/name.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function getIndexTemplateName(id: string) { + return `${id}@stream`; +} diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts index 146671e79744f..eb09df8831304 100644 --- a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts @@ -10,10 +10,11 @@ import { ASSET_VERSION } from '../../../../common/constants'; import { conditionToPainless } from '../helpers/condition_to_painless'; import { logsDefaultPipelineProcessors } from './logs_default_pipeline'; import { isRoot } from '../helpers/hierarchy'; +import { getProcessingPipelineName } from './name'; export function generateIngestPipeline(id: string, definition: StreamDefinition) { return { - id: `${id}@stream.default-pipeline`, + id: getProcessingPipelineName(id), processors: [ ...(isRoot(definition.id) ? logsDefaultPipelineProcessors : []), ...definition.processing.map((processor) => { diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts index b73dc61b148ed..9b46e0cf4ac92 100644 --- a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts @@ -8,6 +8,7 @@ import { StreamDefinition } from '../../../../common/types'; import { ASSET_VERSION } from '../../../../common/constants'; import { conditionToPainless } from '../helpers/condition_to_painless'; +import { getReroutePipelineName } from './name'; interface GenerateReroutePipelineParams { definition: StreamDefinition; @@ -15,7 +16,7 @@ interface GenerateReroutePipelineParams { export async function generateReroutePipeline({ definition }: GenerateReroutePipelineParams) { return { - id: `${definition.id}@stream.reroutes`, + id: getReroutePipelineName(definition.id), processors: definition.children.map((child) => { return { reroute: { diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/manage_ingest_pipelines.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/manage_ingest_pipelines.ts new file mode 100644 index 0000000000000..96449e142b68a --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/manage_ingest_pipelines.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { Logger } from '@kbn/logging'; +import { retryTransientEsErrors } from '../helpers/retry'; + +interface DeletePipelineOptions { + esClient: ElasticsearchClient; + id: string; + logger: Logger; +} + +export async function deleteIngestPipeline({ esClient, id, logger }: DeletePipelineOptions) { + try { + await retryTransientEsErrors(() => esClient.ingest.deletePipeline({ id }, { ignore: [404] }), { + logger, + }); + } catch (error: any) { + logger.error(`Error deleting ingest pipeline: ${error.message}`); + throw error; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/name.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/name.ts new file mode 100644 index 0000000000000..8d2a97ff3137f --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/name.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function getProcessingPipelineName(id: string) { + return `${id}@stream.processing`; +} + +export function getReroutePipelineName(id: string) { + return `${id}@stream.reroutes`; +} diff --git a/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts b/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts index 3555146ccc008..4930876ae9923 100644 --- a/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts +++ b/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts @@ -9,7 +9,6 @@ import { StreamDefinition } from '../../../common/types'; export const rootStreamDefinition: StreamDefinition = { id: 'logs', - root: true, processing: [], children: [], fields: [ diff --git a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts index 5fe9e5f6de0df..53d5a86e7cd83 100644 --- a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts +++ b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts @@ -12,6 +12,7 @@ import { StreamDefinition } from '../../../common/types'; import { STREAMS_INDEX } from '../../../common/constants'; import { DefinitionNotFound, IndexTemplateNotFound } from './errors'; import { + deleteTemplate, upsertComponent, upsertIngestPipeline, upsertTemplate, @@ -20,6 +21,11 @@ import { generateLayer } from './component_templates/generate_layer'; import { generateIngestPipeline } from './ingest_pipelines/generate_ingest_pipeline'; import { generateReroutePipeline } from './ingest_pipelines/generate_reroute_pipeline'; import { generateIndexTemplate } from './index_templates/generate_index_template'; +import { deleteComponent } from './component_templates/manage_component_templates'; +import { deleteIngestPipeline } from './ingest_pipelines/manage_ingest_pipelines'; +import { getIndexTemplateName } from './index_templates/name'; +import { getComponentTemplateName } from './component_templates/name'; +import { getProcessingPipelineName, getReroutePipelineName } from './ingest_pipelines/name'; interface BaseParams { scopedClusterClient: IScopedClusterClient; @@ -29,6 +35,39 @@ interface BaseParamsWithDefinition extends BaseParams { definition: StreamDefinition; } +interface DeleteStreamParams extends BaseParams { + id: string; + logger: Logger; +} + +export async function deleteStreamObjects({ id, scopedClusterClient, logger }: DeleteStreamParams) { + await scopedClusterClient.asCurrentUser.delete({ + id, + index: STREAMS_INDEX, + refresh: 'wait_for', + }); + await deleteTemplate({ + esClient: scopedClusterClient.asSecondaryAuthUser, + name: getIndexTemplateName(id), + logger, + }); + await deleteComponent({ + esClient: scopedClusterClient.asSecondaryAuthUser, + name: getComponentTemplateName(id), + logger, + }); + await deleteIngestPipeline({ + esClient: scopedClusterClient.asSecondaryAuthUser, + id: getProcessingPipelineName(id), + logger, + }); + await deleteIngestPipeline({ + esClient: scopedClusterClient.asSecondaryAuthUser, + id: getReroutePipelineName(id), + logger, + }); +} + async function upsertStream({ definition, scopedClusterClient }: BaseParamsWithDefinition) { return scopedClusterClient.asCurrentUser.index({ id: definition.id, @@ -38,6 +77,19 @@ async function upsertStream({ definition, scopedClusterClient }: BaseParamsWithD }); } +type ListStreamsParams = BaseParams; + +export async function listStreams({ scopedClusterClient }: ListStreamsParams) { + const response = await scopedClusterClient.asCurrentUser.search({ + index: STREAMS_INDEX, + size: 10000, + fields: ['id'], + _source: false, + }); + const definitions = response.hits.hits.map((hit) => hit.fields as { id: string[] }); + return definitions; +} + interface ReadStreamParams extends BaseParams { id: string; } diff --git a/x-pack/plugins/streams/server/routes/index.ts b/x-pack/plugins/streams/server/routes/index.ts index 544b60ccd404c..691b61e93aeab 100644 --- a/x-pack/plugins/streams/server/routes/index.ts +++ b/x-pack/plugins/streams/server/routes/index.ts @@ -5,9 +5,11 @@ * 2.0. */ +import { deleteStreamRoute } from './streams/delete'; import { editStreamRoute } from './streams/edit'; import { enableStreamsRoute } from './streams/enable'; import { forkStreamsRoute } from './streams/fork'; +import { listStreamsRoute } from './streams/list'; import { readStreamRoute } from './streams/read'; export const StreamsRouteRepository = { @@ -15,6 +17,8 @@ export const StreamsRouteRepository = { ...forkStreamsRoute, ...readStreamRoute, ...editStreamRoute, + ...deleteStreamRoute, + ...listStreamsRoute, }; export type StreamsRouteRepository = typeof StreamsRouteRepository; diff --git a/x-pack/plugins/streams/server/routes/streams/delete.ts b/x-pack/plugins/streams/server/routes/streams/delete.ts new file mode 100644 index 0000000000000..2176530a42518 --- /dev/null +++ b/x-pack/plugins/streams/server/routes/streams/delete.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; +import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; +import { Logger } from '@kbn/logging'; +import { + DefinitionNotFound, + ForkConditionMissing, + IndexTemplateNotFound, + SecurityException, +} from '../../lib/streams/errors'; +import { createServerRoute } from '../create_server_route'; +import { syncStream, readStream, deleteStreamObjects } from '../../lib/streams/stream_crud'; +import { MalformedStreamId } from '../../lib/streams/errors/malformed_stream_id'; +import { getParentId } from '../../lib/streams/helpers/hierarchy'; + +export const deleteStreamRoute = createServerRoute({ + endpoint: 'DELETE /api/streams/{id} 2023-10-31', + options: { + access: 'public', + security: { + authz: { + requiredPrivileges: ['streams_fork'], + }, + }, + }, + params: z.object({ + path: z.object({ + id: z.string(), + }), + }), + handler: async ({ response, params, logger, request, getScopedClients }) => { + try { + const { scopedClusterClient } = await getScopedClients({ request }); + + const parentId = getParentId(params.path.id); + if (!parentId) { + throw new MalformedStreamId('Cannot delete root stream'); + } + + await updateParentStream(scopedClusterClient, params.path.id, parentId, logger); + + await deleteStream(scopedClusterClient, params.path.id, logger); + + return response.ok({ body: { acknowledged: true } }); + } catch (e) { + if (e instanceof IndexTemplateNotFound || e instanceof DefinitionNotFound) { + return response.notFound({ body: e }); + } + + if ( + e instanceof SecurityException || + e instanceof ForkConditionMissing || + e instanceof MalformedStreamId + ) { + return response.customError({ body: e, statusCode: 400 }); + } + + return response.customError({ body: e, statusCode: 500 }); + } + }, +}); + +async function deleteStream(scopedClusterClient: IScopedClusterClient, id: string, logger: Logger) { + try { + const { definition } = await readStream({ scopedClusterClient, id }); + for (const child of definition.children) { + await deleteStream(scopedClusterClient, child.id, logger); + } + await deleteStreamObjects({ scopedClusterClient, id, logger }); + } catch (e) { + if (e instanceof DefinitionNotFound) { + logger.debug(`Stream definition for ${id} not found.`); + } else { + throw e; + } + } +} + +async function updateParentStream( + scopedClusterClient: IScopedClusterClient, + id: string, + parentId: string, + logger: Logger +) { + const { definition: parentDefinition } = await readStream({ + scopedClusterClient, + id: parentId, + }); + + parentDefinition.children = parentDefinition.children.filter((child) => child.id !== id); + + await syncStream({ + scopedClusterClient, + definition: parentDefinition, + logger, + }); + return parentDefinition; +} diff --git a/x-pack/plugins/streams/server/routes/streams/list.ts b/x-pack/plugins/streams/server/routes/streams/list.ts new file mode 100644 index 0000000000000..37446788ad1de --- /dev/null +++ b/x-pack/plugins/streams/server/routes/streams/list.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; +import { createServerRoute } from '../create_server_route'; +import { DefinitionNotFound } from '../../lib/streams/errors'; +import { listStreams } from '../../lib/streams/stream_crud'; + +export const listStreamsRoute = createServerRoute({ + endpoint: 'GET /api/streams 2023-10-31', + options: { + access: 'public', + security: { + authz: { + requiredPrivileges: ['streams_read'], + }, + }, + }, + params: z.object({}), + handler: async ({ response, request, getScopedClients }) => { + try { + const { scopedClusterClient } = await getScopedClients({ request }); + const definitions = await listStreams({ scopedClusterClient }); + + const trees = asTrees(definitions); + + return response.ok({ body: { streams: trees } }); + } catch (e) { + if (e instanceof DefinitionNotFound) { + return response.notFound({ body: e }); + } + + return response.customError({ body: e, statusCode: 500 }); + } + }, +}); + +interface ListStreamDefinition { + id: string; + children: ListStreamDefinition[]; +} + +function asTrees(definitions: Array<{ id: string[] }>) { + const trees: ListStreamDefinition[] = []; + definitions.forEach((definition) => { + const path = definition.id[0].split('.'); + let currentTree = trees; + path.forEach((_id, index) => { + const partialPath = path.slice(0, index + 1).join('.'); + const existingNode = currentTree.find((node) => node.id === partialPath); + if (existingNode) { + currentTree = existingNode.children; + } else { + const newNode = { id: partialPath, children: [] }; + currentTree.push(newNode); + currentTree = newNode.children; + } + }); + }); + return trees; +} From 9dd5fde2e977e7c265528e6aaaa5a87343cd354e Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 8 Nov 2024 22:34:45 +0100 Subject: [PATCH 09/95] cleanup --- .../manage_component_templates.ts | 19 ++++ .../index_templates/manage_index_templates.ts | 44 +++++++ .../manage_ingest_pipelines.ts | 21 ++++ .../streams/server/lib/streams/stream_crud.ts | 14 +-- x-pack/plugins/streams/server/plugin.ts | 6 - .../server/templates/components/base.ts | 60 ---------- .../templates/manage_index_templates.ts | 107 ------------------ .../templates/streams_index_template.ts | 43 ------- 8 files changed, 90 insertions(+), 224 deletions(-) create mode 100644 x-pack/plugins/streams/server/lib/streams/index_templates/manage_index_templates.ts delete mode 100644 x-pack/plugins/streams/server/templates/components/base.ts delete mode 100644 x-pack/plugins/streams/server/templates/manage_index_templates.ts delete mode 100644 x-pack/plugins/streams/server/templates/streams_index_template.ts diff --git a/x-pack/plugins/streams/server/lib/streams/component_templates/manage_component_templates.ts b/x-pack/plugins/streams/server/lib/streams/component_templates/manage_component_templates.ts index 0eed31ecab086..a7d707a4ce42a 100644 --- a/x-pack/plugins/streams/server/lib/streams/component_templates/manage_component_templates.ts +++ b/x-pack/plugins/streams/server/lib/streams/component_templates/manage_component_templates.ts @@ -7,6 +7,7 @@ import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { Logger } from '@kbn/logging'; +import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; import { retryTransientEsErrors } from '../helpers/retry'; interface DeleteComponentOptions { @@ -15,6 +16,12 @@ interface DeleteComponentOptions { logger: Logger; } +interface ComponentManagementOptions { + esClient: ElasticsearchClient; + component: ClusterPutComponentTemplateRequest; + logger: Logger; +} + export async function deleteComponent({ esClient, name, logger }: DeleteComponentOptions) { try { await retryTransientEsErrors( @@ -26,3 +33,15 @@ export async function deleteComponent({ esClient, name, logger }: DeleteComponen throw error; } } + +export async function upsertComponent({ esClient, component, logger }: ComponentManagementOptions) { + try { + await retryTransientEsErrors(() => esClient.cluster.putComponentTemplate(component), { + logger, + }); + logger.debug(() => `Installed component template: ${JSON.stringify(component)}`); + } catch (error: any) { + logger.error(`Error updating component template: ${error.message}`); + throw error; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/index_templates/manage_index_templates.ts b/x-pack/plugins/streams/server/lib/streams/index_templates/manage_index_templates.ts new file mode 100644 index 0000000000000..9383e698b3436 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/index_templates/manage_index_templates.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { retryTransientEsErrors } from '../helpers/retry'; + +interface TemplateManagementOptions { + esClient: ElasticsearchClient; + template: IndicesPutIndexTemplateRequest; + logger: Logger; +} + +interface DeleteTemplateOptions { + esClient: ElasticsearchClient; + name: string; + logger: Logger; +} + +export async function upsertTemplate({ esClient, template, logger }: TemplateManagementOptions) { + try { + await retryTransientEsErrors(() => esClient.indices.putIndexTemplate(template), { logger }); + logger.debug(() => `Installed index template: ${JSON.stringify(template)}`); + } catch (error: any) { + logger.error(`Error updating index template: ${error.message}`); + throw error; + } +} + +export async function deleteTemplate({ esClient, name, logger }: DeleteTemplateOptions) { + try { + await retryTransientEsErrors( + () => esClient.indices.deleteIndexTemplate({ name }, { ignore: [404] }), + { logger } + ); + } catch (error: any) { + logger.error(`Error deleting index template: ${error.message}`); + throw error; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/manage_ingest_pipelines.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/manage_ingest_pipelines.ts index 96449e142b68a..467e2efb48f0d 100644 --- a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/manage_ingest_pipelines.ts +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/manage_ingest_pipelines.ts @@ -7,6 +7,7 @@ import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { Logger } from '@kbn/logging'; +import { IngestPutPipelineRequest } from '@elastic/elasticsearch/lib/api/types'; import { retryTransientEsErrors } from '../helpers/retry'; interface DeletePipelineOptions { @@ -15,6 +16,12 @@ interface DeletePipelineOptions { logger: Logger; } +interface PipelineManagementOptions { + esClient: ElasticsearchClient; + pipeline: IngestPutPipelineRequest; + logger: Logger; +} + export async function deleteIngestPipeline({ esClient, id, logger }: DeletePipelineOptions) { try { await retryTransientEsErrors(() => esClient.ingest.deletePipeline({ id }, { ignore: [404] }), { @@ -25,3 +32,17 @@ export async function deleteIngestPipeline({ esClient, id, logger }: DeletePipel throw error; } } + +export async function upsertIngestPipeline({ + esClient, + pipeline, + logger, +}: PipelineManagementOptions) { + try { + await retryTransientEsErrors(() => esClient.ingest.putPipeline(pipeline), { logger }); + logger.debug(() => `Installed index template: ${JSON.stringify(pipeline)}`); + } catch (error: any) { + logger.error(`Error updating index template: ${error.message}`); + throw error; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts index 53d5a86e7cd83..b557714da1682 100644 --- a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts +++ b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts @@ -11,21 +11,19 @@ import { Logger } from '@kbn/logging'; import { StreamDefinition } from '../../../common/types'; import { STREAMS_INDEX } from '../../../common/constants'; import { DefinitionNotFound, IndexTemplateNotFound } from './errors'; -import { - deleteTemplate, - upsertComponent, - upsertIngestPipeline, - upsertTemplate, -} from '../../templates/manage_index_templates'; +import { deleteTemplate, upsertTemplate } from './index_templates/manage_index_templates'; import { generateLayer } from './component_templates/generate_layer'; import { generateIngestPipeline } from './ingest_pipelines/generate_ingest_pipeline'; import { generateReroutePipeline } from './ingest_pipelines/generate_reroute_pipeline'; import { generateIndexTemplate } from './index_templates/generate_index_template'; -import { deleteComponent } from './component_templates/manage_component_templates'; -import { deleteIngestPipeline } from './ingest_pipelines/manage_ingest_pipelines'; +import { deleteComponent, upsertComponent } from './component_templates/manage_component_templates'; import { getIndexTemplateName } from './index_templates/name'; import { getComponentTemplateName } from './component_templates/name'; import { getProcessingPipelineName, getReroutePipelineName } from './ingest_pipelines/name'; +import { + deleteIngestPipeline, + upsertIngestPipeline, +} from './ingest_pipelines/manage_ingest_pipelines'; interface BaseParams { scopedClusterClient: IScopedClusterClient; diff --git a/x-pack/plugins/streams/server/plugin.ts b/x-pack/plugins/streams/server/plugin.ts index 16473918e2cfa..951c1e3751d72 100644 --- a/x-pack/plugins/streams/server/plugin.ts +++ b/x-pack/plugins/streams/server/plugin.ts @@ -17,7 +17,6 @@ import { } from '@kbn/core/server'; import { registerRoutes } from '@kbn/server-route-repository'; import { StreamsConfig, configSchema, exposeToBrowserConfig } from '../common/config'; -import { installStreamsTemplates } from './templates/manage_index_templates'; import { StreamsRouteRepository } from './routes'; import { RouteDependencies } from './routes/types'; import { @@ -118,11 +117,6 @@ export class StreamsPlugin this.server.taskManager = plugins.taskManager; } - const esClient = core.elasticsearch.client.asInternalUser; - installStreamsTemplates({ esClient, logger: this.logger }).catch((err) => - this.logger.error(err) - ); - return {}; } diff --git a/x-pack/plugins/streams/server/templates/components/base.ts b/x-pack/plugins/streams/server/templates/components/base.ts deleted file mode 100644 index fac220fb8be1f..0000000000000 --- a/x-pack/plugins/streams/server/templates/components/base.ts +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; - -export const STREAMS_BASE_COMPONENT = 'streams@mappings'; - -export const BaseComponentTemplateConfig: ClusterPutComponentTemplateRequest = { - name: STREAMS_BASE_COMPONENT, - _meta: { - description: 'Component template for the Stream Entities Manager data set', - managed: true, - }, - template: { - mappings: { - properties: { - labels: { - type: 'object', - }, - tags: { - ignore_above: 1024, - type: 'keyword', - }, - id: { - ignore_above: 1024, - type: 'keyword', - }, - dataset: { - ignore_above: 1024, - type: 'keyword', - }, - type: { - ignore_above: 1024, - type: 'keyword', - }, - root: { - type: 'boolean', - }, - forked_from: { - ignore_above: 1024, - type: 'keyword', - }, - condition: { - type: 'object', - }, - event: { - properties: { - ingested: { - type: 'date', - }, - }, - }, - }, - }, - }, -}; diff --git a/x-pack/plugins/streams/server/templates/manage_index_templates.ts b/x-pack/plugins/streams/server/templates/manage_index_templates.ts deleted file mode 100644 index f562c4dfe4183..0000000000000 --- a/x-pack/plugins/streams/server/templates/manage_index_templates.ts +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { - ClusterPutComponentTemplateRequest, - IndicesPutIndexTemplateRequest, - IngestPutPipelineRequest, -} from '@elastic/elasticsearch/lib/api/types'; -import { ElasticsearchClient, Logger } from '@kbn/core/server'; -import { BaseComponentTemplateConfig } from './components/base'; -import { retryTransientEsErrors } from '../lib/streams/helpers/retry'; -import { streamsIndexTemplate } from './streams_index_template'; - -interface TemplateManagementOptions { - esClient: ElasticsearchClient; - template: IndicesPutIndexTemplateRequest; - logger: Logger; -} - -interface PipelineManagementOptions { - esClient: ElasticsearchClient; - pipeline: IngestPutPipelineRequest; - logger: Logger; -} - -interface ComponentManagementOptions { - esClient: ElasticsearchClient; - component: ClusterPutComponentTemplateRequest; - logger: Logger; -} - -export const installStreamsTemplates = async ({ - esClient, - logger, -}: { - esClient: ElasticsearchClient; - logger: Logger; -}) => { - await upsertComponent({ - esClient, - logger, - component: BaseComponentTemplateConfig, - }); - await upsertTemplate({ - esClient, - logger, - template: streamsIndexTemplate, - }); -}; - -interface DeleteTemplateOptions { - esClient: ElasticsearchClient; - name: string; - logger: Logger; -} - -export async function upsertTemplate({ esClient, template, logger }: TemplateManagementOptions) { - try { - await retryTransientEsErrors(() => esClient.indices.putIndexTemplate(template), { logger }); - logger.debug(() => `Installed index template: ${JSON.stringify(template)}`); - } catch (error: any) { - logger.error(`Error updating index template: ${error.message}`); - throw error; - } -} - -export async function upsertIngestPipeline({ - esClient, - pipeline, - logger, -}: PipelineManagementOptions) { - try { - await retryTransientEsErrors(() => esClient.ingest.putPipeline(pipeline), { logger }); - logger.debug(() => `Installed index template: ${JSON.stringify(pipeline)}`); - } catch (error: any) { - logger.error(`Error updating index template: ${error.message}`); - throw error; - } -} - -export async function deleteTemplate({ esClient, name, logger }: DeleteTemplateOptions) { - try { - await retryTransientEsErrors( - () => esClient.indices.deleteIndexTemplate({ name }, { ignore: [404] }), - { logger } - ); - } catch (error: any) { - logger.error(`Error deleting index template: ${error.message}`); - throw error; - } -} - -export async function upsertComponent({ esClient, component, logger }: ComponentManagementOptions) { - try { - await retryTransientEsErrors(() => esClient.cluster.putComponentTemplate(component), { - logger, - }); - logger.debug(() => `Installed component template: ${JSON.stringify(component)}`); - } catch (error: any) { - logger.error(`Error updating component template: ${error.message}`); - throw error; - } -} diff --git a/x-pack/plugins/streams/server/templates/streams_index_template.ts b/x-pack/plugins/streams/server/templates/streams_index_template.ts deleted file mode 100644 index 0e843b2289cf3..0000000000000 --- a/x-pack/plugins/streams/server/templates/streams_index_template.ts +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; -import { STREAMS_BASE_COMPONENT } from './components/base'; -import { STREAMS_INDEX } from '../../common/constants'; - -export const streamsIndexTemplate: IndicesPutIndexTemplateRequest = { - name: 'stream-entities', - _meta: { - description: - 'Index template for indices managed by the Streams framework for the instance dataset', - ecs_version: '8.0.0', - managed: true, - managed_by: 'streams', - }, - composed_of: [STREAMS_BASE_COMPONENT], - index_patterns: [STREAMS_INDEX], - priority: 200, - template: { - mappings: { - _meta: { - version: '1.6.0', - }, - date_detection: false, - dynamic: false, - }, - settings: { - index: { - codec: 'best_compression', - mapping: { - total_fields: { - limit: 2000, - }, - }, - }, - }, - }, -}; From 7400a3bbb356b0d71b98959b995e125f83a75c46 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 8 Nov 2024 22:45:15 +0100 Subject: [PATCH 10/95] add resync endpoint --- .../streams/server/lib/streams/stream_crud.ts | 1 + x-pack/plugins/streams/server/routes/index.ts | 2 + .../streams/server/routes/streams/resync.ts | 42 +++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 x-pack/plugins/streams/server/routes/streams/resync.ts diff --git a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts index b557714da1682..7ffadf552fcbc 100644 --- a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts +++ b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts @@ -83,6 +83,7 @@ export async function listStreams({ scopedClusterClient }: ListStreamsParams) { size: 10000, fields: ['id'], _source: false, + sort: [{ id: 'asc' }], }); const definitions = response.hits.hits.map((hit) => hit.fields as { id: string[] }); return definitions; diff --git a/x-pack/plugins/streams/server/routes/index.ts b/x-pack/plugins/streams/server/routes/index.ts index 691b61e93aeab..6fc734d3371b4 100644 --- a/x-pack/plugins/streams/server/routes/index.ts +++ b/x-pack/plugins/streams/server/routes/index.ts @@ -11,9 +11,11 @@ import { enableStreamsRoute } from './streams/enable'; import { forkStreamsRoute } from './streams/fork'; import { listStreamsRoute } from './streams/list'; import { readStreamRoute } from './streams/read'; +import { resyncStreamsRoute } from './streams/resync'; export const StreamsRouteRepository = { ...enableStreamsRoute, + ...resyncStreamsRoute, ...forkStreamsRoute, ...readStreamRoute, ...editStreamRoute, diff --git a/x-pack/plugins/streams/server/routes/streams/resync.ts b/x-pack/plugins/streams/server/routes/streams/resync.ts new file mode 100644 index 0000000000000..e2888328e9fba --- /dev/null +++ b/x-pack/plugins/streams/server/routes/streams/resync.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; +import { createServerRoute } from '../create_server_route'; +import { syncStream, readStream, listStreams } from '../../lib/streams/stream_crud'; + +export const resyncStreamsRoute = createServerRoute({ + endpoint: 'POST /api/streams/_resync 2023-10-31', + options: { + access: 'public', + security: { + authz: { + requiredPrivileges: ['streams_fork'], + }, + }, + }, + params: z.object({}), + handler: async ({ response, logger, request, getScopedClients }) => { + const { scopedClusterClient } = await getScopedClients({ request }); + + const streams = await listStreams({ scopedClusterClient }); + + for (const stream of streams) { + const { definition } = await readStream({ + scopedClusterClient, + id: stream.id[0], + }); + await syncStream({ + scopedClusterClient, + definition, + logger, + }); + } + + return response.ok({}); + }, +}); From 5140687c04ef26feee5059092f188bd70cebabea Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Sun, 10 Nov 2024 17:49:06 +0100 Subject: [PATCH 11/95] Remove cruft, handle package relocation --- .eslintrc.js | 5 +- package.json | 4 +- packages/deeplinks/observability/constants.ts | 2 + packages/kbn-apm-utils/index.ts | 39 ++- packages/kbn-es-types/src/search.ts | 2 +- .../src/kibana_json_v2_schema.ts | 38 ++- .../kbn-typed-react-router-config/index.ts | 1 + .../kibana.jsonc | 2 +- .../src/breadcrumbs/breadcrumb.tsx | 51 +++ .../src/breadcrumbs/context.tsx | 113 +++++++ .../create_router_breadcrumb_component.tsx | 17 + .../src/breadcrumbs/index.tsx | 11 + .../src/breadcrumbs/use_breadcrumbs.ts | 101 ++++++ .../src/breadcrumbs/use_router_breadcrumb.ts | 53 +++ .../src/create_router.ts | 8 +- .../src/router_provider.tsx | 5 +- .../src/types/index.ts | 6 + .../src/use_router.tsx | 2 +- .../tsconfig.json | 7 +- tsconfig.base.json | 8 +- x-pack/.i18nrc.json | 7 +- .../ai-infra/inference-common/index.ts | 2 + .../src/util/truncate_list.ts | 16 + .../observability_utils/README.md | 5 - .../client/create_observability_es_client.ts | 88 ----- .../es/utils/esql_result_to_plain_objects.ts | 20 -- .../chart/utils.ts | 0 .../hooks/use_abort_controller.ts | 3 + .../hooks/use_abortable_async.ts | 36 +- .../hooks/use_date_range.ts | 63 ++++ .../hooks/use_local_storage.ts | 60 ++++ .../hooks/use_theme.ts | 0 .../jest.config.js | 14 + .../observability_utils_browser/kibana.jsonc | 5 + .../observability_utils_browser/package.json | 6 + .../observability_utils_browser/tsconfig.json | 23 ++ .../utils/ui_settings/get_timezone.ts | 17 + .../array/join_by_key.test.ts | 0 .../array/join_by_key.ts | 0 .../entities/get_entity_kuery.ts | 14 + .../es/format_value_for_kql.ts | 10 + .../es/queries/entity_query.ts | 24 ++ .../es/queries/exclude_frozen_query.ts | 0 .../es/queries/exclude_tiers_query.ts | 0 .../es/queries/kql_query.ts | 0 .../es/queries/range_query.ts | 0 .../es/queries/term_query.ts | 0 .../format/integer.ts | 11 + .../jest.config.js | 4 +- .../kibana.jsonc | 2 +- .../llm/log_analysis/document_analysis.ts | 24 ++ .../highlight_patterns_from_regex.ts | 51 +++ .../merge_sample_documents_with_field_caps.ts | 78 +++++ .../sort_and_truncate_analyzed_fields.ts | 52 +++ .../llm/short_id_table.test.ts | 48 +++ .../llm/short_id_table.ts | 56 ++++ .../ml/p_value_to_label.ts | 19 ++ .../object/flatten_object.test.ts | 0 .../object/flatten_object.ts | 0 .../object/merge_plain_object.test.ts | 0 .../object/merge_plain_objects.ts | 0 .../object/unflatten_object.test.ts | 0 .../object/unflatten_object.ts | 1 - .../package.json | 4 +- .../tsconfig.json | 7 +- .../entities/analyze_documents.ts | 84 +++++ .../entities/get_data_streams_for_entity.ts | 63 ++++ .../entities/signals/get_alerts_for_entity.ts | 57 ++++ .../signals/get_anomalies_for_entity.ts | 16 + .../entities/signals/get_slos_for_entity.ts | 80 +++++ .../client/create_observability_es_client.ts | 130 ++++++++ .../es/esql_result_to_plain_objects.test.ts | 66 ++++ .../es}/esql_result_to_plain_objects.ts | 12 +- .../es/queries/exclude_frozen_query.ts | 23 ++ .../es/queries/kql_query.ts | 17 + .../es/queries/range_query.ts | 25 ++ .../es/queries/term_query.ts | 24 ++ .../observability_utils_server/jest.config.js | 12 + .../observability_utils_server/kibana.jsonc | 5 + .../observability_utils_server/package.json | 6 + .../observability_utils_server/tsconfig.json | 28 ++ .../plugins/logsai/streams_api/kibana.jsonc | 9 - .../public/api/{index.tsx => index.ts} | 0 .../clients/create_streams_api_es_client.ts | 2 +- .../lib/entities/entity_lookup_table.ts | 17 - .../entities/get_data_streams_for_filter.ts | 84 ----- .../lib/entities/get_definition_entities.ts | 18 - .../lib/entities/get_type_definitions.ts | 18 - .../lib/entities/query_signals_as_entities.ts | 165 ---------- .../lib/entities/query_sources_as_entities.ts | 309 ------------------ .../plugins/logsai/streams_api/tsconfig.json | 11 + .../get_mock_streams_app_context.tsx | 8 +- .../streams_app/common/entity_source_query.ts | 14 + .../logsai/streams_app/common/index.ts | 22 ++ .../plugins/logsai/streams_app/kibana.jsonc | 7 +- .../components/all_entities_view/index.tsx | 27 -- .../data_stream_detail_view/index.tsx | 32 -- .../entity_detail_overview/index.tsx | 292 ----------------- .../components/entity_detail_view/index.tsx | 178 +++------- .../entity_health_status_badge/index.tsx | 54 --- .../entity_pivot_type_view/index.tsx | 69 ---- .../entity_table/controlled_entity_table.tsx | 217 ------------ .../public/components/entity_table/index.tsx | 149 --------- .../esql_chart/controlled_esql_chart.tsx | 2 +- .../esql_chart/uncontrolled_esql_chart.tsx | 29 -- .../esql_grid/controlled_esql_grid.tsx | 95 ------ .../public/components/not_found/index.tsx | 24 ++ .../stream_detail_overview/index.tsx | 155 +++++++++ .../components/stream_detail_view/index.tsx | 67 ++++ .../components/stream_list_view/index.tsx | 30 ++ .../public/hooks/use_esql_query_result.ts | 56 ---- .../public/hooks/use_streams_app_fetch.ts | 2 +- .../public/hooks/use_streams_app_router.ts | 6 +- .../logsai/streams_app/public/plugin.ts | 22 +- .../streams_app/public/routes/config.tsx | 66 +--- .../logsai/streams_app/public/types.ts | 27 +- .../util/get_initial_columns_for_logs.ts | 104 ------ .../logsai/streams_app/server/types.ts | 10 +- .../plugins/logsai/streams_app/tsconfig.json | 19 ++ .../create_apm_event_client/index.ts | 2 +- .../server/lib/helpers/tier_filter.ts | 2 +- .../server/utils/unflatten_known_fields.ts | 4 +- .../apm_data_access/tsconfig.json | 1 - .../routes/entities/get_latest_entity.ts | 4 +- .../infra/server/routes/entities/index.ts | 2 +- .../infra/tsconfig.json | 1 - .../public/hooks/use_entity_manager.ts | 2 +- .../hooks/use_inventory_abortable_async.ts | 2 +- .../routes/entities/get_entity_groups.ts | 6 +- .../routes/entities/get_latest_entities.ts | 6 +- .../inventory/server/routes/entities/route.ts | 4 +- .../server/routes/has_data/get_has_data.ts | 4 +- .../inventory/server/routes/has_data/route.ts | 2 +- .../inventory/tsconfig.json | 1 - .../register_embeddable_item.tsx | 2 +- .../items/esql_item/register_esql_item.tsx | 2 +- .../esql_widget_preview.tsx | 2 +- .../events_timeline/events_timeline.tsx | 2 +- .../investigate_app/tsconfig.json | 1 - .../observability_shared/kibana.jsonc | 2 +- x-pack/plugins/streams/common/index.ts | 8 + x-pack/plugins/streams/public/api/index.ts | 52 +++ x-pack/plugins/streams/public/index.ts | 7 +- x-pack/plugins/streams/public/plugin.ts | 13 +- x-pack/plugins/streams/public/types.ts | 8 +- .../server/lib/streams/bootstrap_stream.ts | 1 + .../streams/server/lib/streams/stream_crud.ts | 55 +++- .../streams/server/routes/esql/route.ts | 62 ++++ x-pack/plugins/streams/server/routes/index.ts | 4 + .../streams/server/routes/streams/enable.ts | 4 +- .../streams/server/routes/streams/fork.ts | 2 + .../streams/server/routes/streams/list.ts | 37 +++ .../streams/server/routes/streams/read.ts | 17 +- x-pack/plugins/streams/tsconfig.json | 14 +- yarn.lock | 10 +- 155 files changed, 2458 insertions(+), 2199 deletions(-) create mode 100644 packages/kbn-typed-react-router-config/src/breadcrumbs/breadcrumb.tsx create mode 100644 packages/kbn-typed-react-router-config/src/breadcrumbs/context.tsx create mode 100644 packages/kbn-typed-react-router-config/src/breadcrumbs/create_router_breadcrumb_component.tsx create mode 100644 packages/kbn-typed-react-router-config/src/breadcrumbs/index.tsx create mode 100644 packages/kbn-typed-react-router-config/src/breadcrumbs/use_breadcrumbs.ts create mode 100644 packages/kbn-typed-react-router-config/src/breadcrumbs/use_router_breadcrumb.ts create mode 100644 x-pack/packages/ai-infra/inference-common/src/util/truncate_list.ts delete mode 100644 x-pack/packages/observability/observability_utils/README.md delete mode 100644 x-pack/packages/observability/observability_utils/es/client/create_observability_es_client.ts delete mode 100644 x-pack/packages/observability/observability_utils/es/utils/esql_result_to_plain_objects.ts rename x-pack/packages/observability/observability_utils/{ => observability_utils_browser}/chart/utils.ts (100%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_browser}/hooks/use_abort_controller.ts (92%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_browser}/hooks/use_abortable_async.ts (72%) create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_date_range.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_local_storage.ts rename x-pack/packages/observability/observability_utils/{ => observability_utils_browser}/hooks/use_theme.ts (100%) create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_browser/jest.config.js create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_browser/kibana.jsonc create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_browser/package.json create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_browser/tsconfig.json create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_browser/utils/ui_settings/get_timezone.ts rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/array/join_by_key.test.ts (100%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/array/join_by_key.ts (100%) create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_common/entities/get_entity_kuery.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_common/es/format_value_for_kql.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_common/es/queries/entity_query.ts rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/es/queries/exclude_frozen_query.ts (100%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/es/queries/exclude_tiers_query.ts (100%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/es/queries/kql_query.ts (100%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/es/queries/range_query.ts (100%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/es/queries/term_query.ts (100%) create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_common/format/integer.ts rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/jest.config.js (84%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/kibana.jsonc (61%) create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/document_analysis.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/highlight_patterns_from_regex.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/merge_sample_documents_with_field_caps.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/sort_and_truncate_analyzed_fields.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_common/llm/short_id_table.test.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_common/llm/short_id_table.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_common/ml/p_value_to_label.ts rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/object/flatten_object.test.ts (100%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/object/flatten_object.ts (100%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/object/merge_plain_object.test.ts (100%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/object/merge_plain_objects.ts (100%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/object/unflatten_object.test.ts (100%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/object/unflatten_object.ts (99%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/package.json (62%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/tsconfig.json (70%) create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/entities/analyze_documents.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/entities/get_data_streams_for_entity.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/entities/signals/get_alerts_for_entity.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/entities/signals/get_anomalies_for_entity.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/entities/signals/get_slos_for_entity.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/es/esql_result_to_plain_objects.test.ts rename x-pack/{plugins/logsai/streams_api/common/utils => packages/observability/observability_utils/observability_utils_server/es}/esql_result_to_plain_objects.ts (73%) create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/exclude_frozen_query.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/kql_query.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/range_query.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/term_query.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/jest.config.js create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/kibana.jsonc create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/package.json create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/tsconfig.json rename x-pack/plugins/logsai/streams_api/public/api/{index.tsx => index.ts} (100%) delete mode 100644 x-pack/plugins/logsai/streams_api/server/lib/entities/entity_lookup_table.ts delete mode 100644 x-pack/plugins/logsai/streams_api/server/lib/entities/get_data_streams_for_filter.ts delete mode 100644 x-pack/plugins/logsai/streams_api/server/lib/entities/get_definition_entities.ts delete mode 100644 x-pack/plugins/logsai/streams_api/server/lib/entities/get_type_definitions.ts delete mode 100644 x-pack/plugins/logsai/streams_api/server/lib/entities/query_signals_as_entities.ts delete mode 100644 x-pack/plugins/logsai/streams_api/server/lib/entities/query_sources_as_entities.ts create mode 100644 x-pack/plugins/logsai/streams_app/common/entity_source_query.ts create mode 100644 x-pack/plugins/logsai/streams_app/common/index.ts delete mode 100644 x-pack/plugins/logsai/streams_app/public/components/all_entities_view/index.tsx delete mode 100644 x-pack/plugins/logsai/streams_app/public/components/data_stream_detail_view/index.tsx delete mode 100644 x-pack/plugins/logsai/streams_app/public/components/entity_detail_overview/index.tsx delete mode 100644 x-pack/plugins/logsai/streams_app/public/components/entity_health_status_badge/index.tsx delete mode 100644 x-pack/plugins/logsai/streams_app/public/components/entity_pivot_type_view/index.tsx delete mode 100644 x-pack/plugins/logsai/streams_app/public/components/entity_table/controlled_entity_table.tsx delete mode 100644 x-pack/plugins/logsai/streams_app/public/components/entity_table/index.tsx delete mode 100644 x-pack/plugins/logsai/streams_app/public/components/esql_chart/uncontrolled_esql_chart.tsx delete mode 100644 x-pack/plugins/logsai/streams_app/public/components/esql_grid/controlled_esql_grid.tsx create mode 100644 x-pack/plugins/logsai/streams_app/public/components/not_found/index.tsx create mode 100644 x-pack/plugins/logsai/streams_app/public/components/stream_detail_overview/index.tsx create mode 100644 x-pack/plugins/logsai/streams_app/public/components/stream_detail_view/index.tsx create mode 100644 x-pack/plugins/logsai/streams_app/public/components/stream_list_view/index.tsx delete mode 100644 x-pack/plugins/logsai/streams_app/public/hooks/use_esql_query_result.ts delete mode 100644 x-pack/plugins/logsai/streams_app/public/util/get_initial_columns_for_logs.ts create mode 100644 x-pack/plugins/streams/common/index.ts create mode 100644 x-pack/plugins/streams/public/api/index.ts create mode 100644 x-pack/plugins/streams/server/routes/esql/route.ts create mode 100644 x-pack/plugins/streams/server/routes/streams/list.ts diff --git a/.eslintrc.js b/.eslintrc.js index 0e486a64c9440..2e5c812fbfb56 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -952,6 +952,7 @@ module.exports = { { files: [ 'x-pack/plugins/observability_solution/**/*.{ts,tsx}', + 'x-pack/plugins/logsai/**/*.{ts,tsx}', 'x-pack/packages/observability/**/*.{ts,tsx}', ], rules: { @@ -959,7 +960,7 @@ module.exports = { 'error', { additionalHooks: - '^(useAbortableAsync|useMemoWithAbortSignal|useFetcher|useProgressiveFetcher|useBreadcrumb|useAsync|useTimeRangeAsync|useAutoAbortedHttpClient)$', + '^(useAbortableAsync|useMemoWithAbortSignal|useFetcher|useProgressiveFetcher|useBreadcrumb|useAsync|useTimeRangeAsync|useAutoAbortedHttpClient|use.*Fetch)$', }, ], }, @@ -968,6 +969,7 @@ module.exports = { files: [ 'x-pack/plugins/aiops/**/*.tsx', 'x-pack/plugins/observability_solution/**/*.tsx', + 'x-pack/plugins/logsai/**/*.{ts,tsx}', 'src/plugins/ai_assistant_management/**/*.tsx', 'x-pack/packages/observability/**/*.{ts,tsx}', ], @@ -984,6 +986,7 @@ module.exports = { { files: [ 'x-pack/plugins/observability_solution/**/!(*.stories.tsx|*.test.tsx|*.storybook_decorator.tsx|*.mock.tsx)', + 'x-pack/plugins/logsai/**/!(*.stories.tsx|*.test.tsx|*.storybook_decorator.tsx|*.mock.tsx)', 'src/plugins/ai_assistant_management/**/!(*.stories.tsx|*.test.tsx|*.storybook_decorator.tsx|*.mock.tsx)', 'x-pack/packages/observability/logs_overview/**/!(*.stories.tsx|*.test.tsx|*.storybook_decorator.tsx|*.mock.tsx)', ], diff --git a/package.json b/package.json index a9a1a2d6845c8..d5c517fc7e47f 100644 --- a/package.json +++ b/package.json @@ -700,7 +700,9 @@ "@kbn/observability-plugin": "link:x-pack/plugins/observability_solution/observability", "@kbn/observability-shared-plugin": "link:x-pack/plugins/observability_solution/observability_shared", "@kbn/observability-synthetics-test-data": "link:x-pack/packages/observability/synthetics_test_data", - "@kbn/observability-utils": "link:x-pack/packages/observability/observability_utils", + "@kbn/observability-utils-browser": "link:x-pack/packages/observability/observability_utils/observability_utils_browser", + "@kbn/observability-utils-common": "link:x-pack/packages/observability/observability_utils/observability_utils_common", + "@kbn/observability-utils-server": "link:x-pack/packages/observability/observability_utils/observability_utils_server", "@kbn/oidc-provider-plugin": "link:x-pack/test/security_api_integration/plugins/oidc_provider", "@kbn/open-telemetry-instrumented-plugin": "link:test/common/plugins/otel_metrics", "@kbn/openapi-common": "link:packages/kbn-openapi-common", diff --git a/packages/deeplinks/observability/constants.ts b/packages/deeplinks/observability/constants.ts index 25642ba69613a..3dcc009481d0a 100644 --- a/packages/deeplinks/observability/constants.ts +++ b/packages/deeplinks/observability/constants.ts @@ -35,3 +35,5 @@ export const OBLT_UX_APP_ID = 'ux'; export const OBLT_PROFILING_APP_ID = 'profiling'; export const INVENTORY_APP_ID = 'inventory'; + +export const STREAMS_APP_ID = 'streams'; diff --git a/packages/kbn-apm-utils/index.ts b/packages/kbn-apm-utils/index.ts index 7ada02fe8173e..8bd26bc481352 100644 --- a/packages/kbn-apm-utils/index.ts +++ b/packages/kbn-apm-utils/index.ts @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import agent from 'elastic-apm-node'; +import agent, { Logger } from 'elastic-apm-node'; import asyncHooks from 'async_hooks'; export interface SpanOptions { @@ -34,14 +34,42 @@ const runInNewContext = any>(cb: T): ReturnType( optionsOrName: SpanOptions | string, - cb: (span?: Span) => Promise + cb: (span?: Span) => Promise, + logger?: Logger ): Promise { const options = parseSpanOptions(optionsOrName); const { name, type, subtype, labels, intercept } = options; + let time: number | undefined; + if (logger?.isLevelEnabled('debug')) { + time = performance.now(); + } + + function logTook(failed: boolean) { + if (time) { + logger?.debug( + () => + `Operation ${name}${failed ? ` (failed)` : ''} ${ + Math.round(performance.now() - time!) / 1000 + }s` + ); + } + } + + const withLogTook = [ + (res: TR): TR | Promise => { + logTook(false); + return res; + }, + (err: any): never => { + logTook(true); + throw err; + }, + ]; + if (!agent.isStarted()) { - return cb(); + return cb().then(...withLogTook); } let createdSpan: Span | undefined; @@ -57,7 +85,7 @@ export async function withSpan( createdSpan = agent.startSpan(name) ?? undefined; if (!createdSpan) { - return cb(); + return cb().then(...withLogTook); } } @@ -76,7 +104,7 @@ export async function withSpan( } if (!span) { - return promise; + return promise.then(...withLogTook); } const targetedSpan = span; @@ -98,6 +126,7 @@ export async function withSpan( } return promise + .then(...withLogTook) .then((res) => { if (!targetedSpan.outcome || targetedSpan.outcome === 'unknown') { targetedSpan.outcome = 'success'; diff --git a/packages/kbn-es-types/src/search.ts b/packages/kbn-es-types/src/search.ts index 87f9dd15517c9..d3675e04c2663 100644 --- a/packages/kbn-es-types/src/search.ts +++ b/packages/kbn-es-types/src/search.ts @@ -695,5 +695,5 @@ export interface ESQLSearchParams { locale?: string; include_ccs_metadata?: boolean; dropNullColumns?: boolean; - params?: Array>; + params?: estypesWithoutBodyKey.ScalarValue[] | Array>; } diff --git a/packages/kbn-kibana-manifest-schema/src/kibana_json_v2_schema.ts b/packages/kbn-kibana-manifest-schema/src/kibana_json_v2_schema.ts index 30682d763e0b0..e45e6bd3f05f5 100644 --- a/packages/kbn-kibana-manifest-schema/src/kibana_json_v2_schema.ts +++ b/packages/kbn-kibana-manifest-schema/src/kibana_json_v2_schema.ts @@ -55,13 +55,6 @@ export const MANIFEST_V2: JSONSchema = { `, default: 'common', }, - visibility: { - enum: ['private', 'shared'], - description: desc` - Specifies the visibility of this module, i.e. whether it can be accessed by everybody or only modules in the same group - `, - default: 'shared', - }, devOnly: { type: 'boolean', description: desc` @@ -112,6 +105,37 @@ export const MANIFEST_V2: JSONSchema = { type: 'string', }, }, + allOf: [ + { + if: { + properties: { group: { const: 'platform' } }, + }, + then: { + properties: { + visibility: { + enum: ['private', 'shared'], + description: desc` + Specifies the visibility of this module, i.e. whether it can be accessed by everybody or only modules in the same group + `, + default: 'shared', + }, + }, + required: ['visibility'], + }, + else: { + properties: { + visibility: { + const: 'private', + description: desc` + Specifies the visibility of this module, i.e. whether it can be accessed by everybody or only modules in the same group + `, + default: 'private', + }, + }, + required: ['visibility'], + }, + }, + ], oneOf: [ { type: 'object', diff --git a/packages/kbn-typed-react-router-config/index.ts b/packages/kbn-typed-react-router-config/index.ts index 3075cb48a951b..6aff73c55e6c3 100644 --- a/packages/kbn-typed-react-router-config/index.ts +++ b/packages/kbn-typed-react-router-config/index.ts @@ -17,3 +17,4 @@ export * from './src/use_match_routes'; export * from './src/use_params'; export * from './src/use_router'; export * from './src/use_route_path'; +export * from './src/breadcrumbs'; diff --git a/packages/kbn-typed-react-router-config/kibana.jsonc b/packages/kbn-typed-react-router-config/kibana.jsonc index 0462d28238890..4316fef7473cf 100644 --- a/packages/kbn-typed-react-router-config/kibana.jsonc +++ b/packages/kbn-typed-react-router-config/kibana.jsonc @@ -1,5 +1,5 @@ { - "type": "shared-common", + "type": "shared-browser", "id": "@kbn/typed-react-router-config", "owner": ["@elastic/obs-knowledge-team", "@elastic/obs-ux-management-team"] } diff --git a/packages/kbn-typed-react-router-config/src/breadcrumbs/breadcrumb.tsx b/packages/kbn-typed-react-router-config/src/breadcrumbs/breadcrumb.tsx new file mode 100644 index 0000000000000..94a32c76c21f2 --- /dev/null +++ b/packages/kbn-typed-react-router-config/src/breadcrumbs/breadcrumb.tsx @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import React from 'react'; +import { RequiredKeys } from 'utility-types'; +import { useRouterBreadcrumb } from './use_router_breadcrumb'; +import { PathsOf, RouteMap, TypeOf } from '../types'; + +type AsParamsProps> = RequiredKeys extends never + ? {} + : { params: TObject }; + +export type RouterBreadcrumb = < + TRoutePath extends PathsOf +>({}: { + title: string; + children: React.ReactNode; + path: TRoutePath; +} & AsParamsProps>) => React.ReactElement; + +export function RouterBreadcrumb< + TRouteMap extends RouteMap, + TRoutePath extends PathsOf +>({ + title, + path, + params, + children, +}: { + title: string; + path: TRoutePath; + children: React.ReactElement; + params?: Record; +}) { + useRouterBreadcrumb( + () => ({ + title, + path, + params, + }), + [] + ); + + return children; +} diff --git a/packages/kbn-typed-react-router-config/src/breadcrumbs/context.tsx b/packages/kbn-typed-react-router-config/src/breadcrumbs/context.tsx new file mode 100644 index 0000000000000..21d6a30567b18 --- /dev/null +++ b/packages/kbn-typed-react-router-config/src/breadcrumbs/context.tsx @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { ChromeBreadcrumb, ScopedHistory } from '@kbn/core/public'; +import { compact, isEqual } from 'lodash'; +import React, { createContext, useMemo, useState } from 'react'; +import { useHistory } from 'react-router-dom'; +import { useBreadcrumbs } from './use_breadcrumbs'; +import { + PathsOf, + Route, + RouteMap, + RouteMatch, + TypeAsArgs, + TypeAsParams, + TypeOf, + useMatchRoutes, + useRouter, +} from '../..'; + +export type Breadcrumb< + TRouteMap extends RouteMap = RouteMap, + TPath extends PathsOf = PathsOf +> = { + title: string; + path: TPath; +} & TypeAsParams>; + +interface BreadcrumbApi { + set>( + route: Route, + breadcrumb: Array> + ): void; + unset(route: Route): void; + getBreadcrumbs(matches: RouteMatch[]): Array>>; +} + +export const BreadcrumbsContext = createContext(undefined); + +export function BreadcrumbsContextProvider({ + children, +}: { + children: React.ReactNode; +}) { + const [, forceUpdate] = useState({}); + + const breadcrumbs = useMemo(() => { + return new Map>>(); + }, []); + + const history = useHistory() as ScopedHistory; + + const router = useRouter(); + + const matches: RouteMatch[] = useMatchRoutes(); + + const api = useMemo>( + () => ({ + set(route, breadcrumb) { + if (!isEqual(breadcrumbs.get(route), breadcrumb)) { + breadcrumbs.set(route, breadcrumb); + forceUpdate({}); + } + }, + unset(route) { + if (breadcrumbs.has(route)) { + breadcrumbs.delete(route); + forceUpdate({}); + } + }, + getBreadcrumbs(currentMatches: RouteMatch[]) { + return compact( + currentMatches.flatMap((match) => { + const breadcrumb = breadcrumbs.get(match.route); + + return breadcrumb; + }) + ); + }, + }), + [breadcrumbs] + ); + + const formattedBreadcrumbs: ChromeBreadcrumb[] = api + .getBreadcrumbs(matches) + .map((breadcrumb, index, array) => { + return { + text: breadcrumb.title, + ...(index === array.length - 1 + ? {} + : { + href: history.createHref({ + pathname: router.link( + breadcrumb.path, + ...(('params' in breadcrumb ? [breadcrumb.params] : []) as TypeAsArgs< + TypeOf, false> + >) + ), + }), + }), + }; + }); + + useBreadcrumbs(formattedBreadcrumbs); + + return {children}; +} diff --git a/packages/kbn-typed-react-router-config/src/breadcrumbs/create_router_breadcrumb_component.tsx b/packages/kbn-typed-react-router-config/src/breadcrumbs/create_router_breadcrumb_component.tsx new file mode 100644 index 0000000000000..04c8cf4e9d245 --- /dev/null +++ b/packages/kbn-typed-react-router-config/src/breadcrumbs/create_router_breadcrumb_component.tsx @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { RouteMap } from '../types'; +import { RouterBreadcrumb } from './breadcrumb'; + +export function createRouterBreadcrumbComponent< + TRouteMap extends RouteMap +>(): RouterBreadcrumb { + return RouterBreadcrumb as RouterBreadcrumb; +} diff --git a/packages/kbn-typed-react-router-config/src/breadcrumbs/index.tsx b/packages/kbn-typed-react-router-config/src/breadcrumbs/index.tsx new file mode 100644 index 0000000000000..6a01f9a1ef01c --- /dev/null +++ b/packages/kbn-typed-react-router-config/src/breadcrumbs/index.tsx @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +export { createRouterBreadcrumbComponent } from './create_router_breadcrumb_component'; +export { createUseBreadcrumbs } from './use_router_breadcrumb'; diff --git a/packages/kbn-typed-react-router-config/src/breadcrumbs/use_breadcrumbs.ts b/packages/kbn-typed-react-router-config/src/breadcrumbs/use_breadcrumbs.ts new file mode 100644 index 0000000000000..3d63c8a0f27d7 --- /dev/null +++ b/packages/kbn-typed-react-router-config/src/breadcrumbs/use_breadcrumbs.ts @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { i18n } from '@kbn/i18n'; +import { ApplicationStart, ChromeBreadcrumb, ChromeStart } from '@kbn/core/public'; +import { MouseEvent, useEffect, useMemo } from 'react'; +import { useKibana } from '@kbn/kibana-react-plugin/public'; +import { ChromeBreadcrumbsAppendExtension } from '@kbn/core-chrome-browser'; +import type { ServerlessPluginStart } from '@kbn/serverless/public'; + +function addClickHandlers( + breadcrumbs: ChromeBreadcrumb[], + navigateToHref?: (url: string) => Promise +) { + return breadcrumbs.map((bc) => ({ + ...bc, + ...(bc.href + ? { + onClick: (event: MouseEvent) => { + if (navigateToHref && bc.href) { + event.preventDefault(); + navigateToHref(bc.href); + } + }, + } + : {}), + })); +} + +function getTitleFromBreadCrumbs(breadcrumbs: ChromeBreadcrumb[]) { + return breadcrumbs.map(({ text }) => text?.toString() ?? '').reverse(); +} + +export const useBreadcrumbs = ( + extraCrumbs: ChromeBreadcrumb[], + options?: { + app?: { id: string; label: string }; + breadcrumbsAppendExtension?: ChromeBreadcrumbsAppendExtension; + serverless?: ServerlessPluginStart; + } +) => { + const { app, breadcrumbsAppendExtension, serverless } = options ?? {}; + + const { + services: { + chrome: { docTitle, setBreadcrumbs: chromeSetBreadcrumbs, setBreadcrumbsAppendExtension }, + application: { getUrlForApp, navigateToUrl }, + }, + } = useKibana<{ + application: ApplicationStart; + chrome: ChromeStart; + }>(); + + const setTitle = docTitle.change; + const appPath = getUrlForApp(app?.id ?? 'observability-overview') ?? ''; + + const setBreadcrumbs = useMemo( + () => serverless?.setBreadcrumbs ?? chromeSetBreadcrumbs, + [serverless, chromeSetBreadcrumbs] + ); + + useEffect(() => { + if (breadcrumbsAppendExtension) { + setBreadcrumbsAppendExtension(breadcrumbsAppendExtension); + } + return () => { + if (breadcrumbsAppendExtension) { + setBreadcrumbsAppendExtension(undefined); + } + }; + }, [breadcrumbsAppendExtension, setBreadcrumbsAppendExtension]); + + useEffect(() => { + const breadcrumbs = serverless + ? extraCrumbs + : [ + { + text: + app?.label ?? + i18n.translate('xpack.observabilityShared.breadcrumbs.observabilityLinkText', { + defaultMessage: 'Observability', + }), + href: appPath + '/overview', + }, + ...extraCrumbs, + ]; + + if (setBreadcrumbs) { + setBreadcrumbs(addClickHandlers(breadcrumbs, navigateToUrl)); + } + if (setTitle) { + setTitle(getTitleFromBreadCrumbs(breadcrumbs)); + } + }, [app?.label, appPath, extraCrumbs, navigateToUrl, serverless, setBreadcrumbs, setTitle]); +}; diff --git a/packages/kbn-typed-react-router-config/src/breadcrumbs/use_router_breadcrumb.ts b/packages/kbn-typed-react-router-config/src/breadcrumbs/use_router_breadcrumb.ts new file mode 100644 index 0000000000000..47003cbce5a26 --- /dev/null +++ b/packages/kbn-typed-react-router-config/src/breadcrumbs/use_router_breadcrumb.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { useContext, useEffect, useRef } from 'react'; +import { castArray } from 'lodash'; +import { PathsOf, RouteMap, useCurrentRoute } from '../..'; +import { Breadcrumb, BreadcrumbsContext } from './context'; + +type UseBreadcrumbs = >( + callback: () => Breadcrumb | Array>, + fnDeps: unknown[] +) => void; + +export function useRouterBreadcrumb(callback: () => Breadcrumb | Breadcrumb[], fnDeps: any[]) { + const api = useContext(BreadcrumbsContext); + + if (!api) { + throw new Error('Missing Breadcrumb API in context'); + } + + const { match } = useCurrentRoute(); + + const matchedRoute = useRef(match?.route); + + useEffect(() => { + if (matchedRoute.current && matchedRoute.current !== match?.route) { + api.unset(matchedRoute.current); + } + + matchedRoute.current = match?.route; + + if (matchedRoute.current) { + api.set(matchedRoute.current, castArray(callback())); + } + + return () => { + if (matchedRoute.current) { + api.unset(matchedRoute.current); + } + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [matchedRoute.current, match?.route, ...fnDeps]); +} + +export function createUseBreadcrumbs(): UseBreadcrumbs { + return useRouterBreadcrumb; +} diff --git a/packages/kbn-typed-react-router-config/src/create_router.ts b/packages/kbn-typed-react-router-config/src/create_router.ts index 467fe096ffece..4321f4c0744a7 100644 --- a/packages/kbn-typed-react-router-config/src/create_router.ts +++ b/packages/kbn-typed-react-router-config/src/create_router.ts @@ -11,7 +11,7 @@ import { deepExactRt, mergeRt } from '@kbn/io-ts-utils'; import { isLeft } from 'fp-ts/lib/Either'; import { Location } from 'history'; import { PathReporter } from 'io-ts/lib/PathReporter'; -import { compact, findLastIndex, merge, orderBy } from 'lodash'; +import { compact, findLastIndex, mapValues, merge, orderBy } from 'lodash'; import qs from 'query-string'; import { MatchedRoute, @@ -139,7 +139,9 @@ export function createRouter(routes: TRoutes): Router< if (route?.params) { const decoded = deepExactRt(route.params).decode( merge({}, route.defaults ?? {}, { - path: matchedRoute.match.params, + path: mapValues(matchedRoute.match.params, (value) => { + return decodeURIComponent(value); + }), query: qs.parse(location.search, { decode: true }), }) ); @@ -179,7 +181,7 @@ export function createRouter(routes: TRoutes): Router< .split('/') .map((part) => { const match = part.match(/(?:{([a-zA-Z]+)})/); - return match ? paramsWithBuiltInDefaults.path[match[1]] : part; + return match ? encodeURIComponent(paramsWithBuiltInDefaults.path[match[1]]) : part; }) .join('/'); diff --git a/packages/kbn-typed-react-router-config/src/router_provider.tsx b/packages/kbn-typed-react-router-config/src/router_provider.tsx index 329bf46046769..7608cb35fd6eb 100644 --- a/packages/kbn-typed-react-router-config/src/router_provider.tsx +++ b/packages/kbn-typed-react-router-config/src/router_provider.tsx @@ -13,6 +13,7 @@ import { Router as ReactRouter } from '@kbn/shared-ux-router'; import { RouteMap, Router } from './types'; import { RouterContextProvider } from './use_router'; +import { BreadcrumbsContextProvider } from './breadcrumbs/context'; export function RouterProvider({ children, @@ -25,7 +26,9 @@ export function RouterProvider({ }) { return ( - {children} + + {children} + ); } diff --git a/packages/kbn-typed-react-router-config/src/types/index.ts b/packages/kbn-typed-react-router-config/src/types/index.ts index 3b4c36c42af53..dbab588619db7 100644 --- a/packages/kbn-typed-react-router-config/src/types/index.ts +++ b/packages/kbn-typed-react-router-config/src/types/index.ts @@ -106,6 +106,12 @@ export type TypeAsArgs = keyof TObject extends never ? [TObject] | [] : [TObject]; +export type TypeAsParams = keyof TObject extends never + ? {} + : RequiredKeys extends never + ? never + : { params: TObject }; + export type FlattenRoutesOf = Array< ValuesType<{ [key in keyof MapRoutes]: ValuesType[key]>; diff --git a/packages/kbn-typed-react-router-config/src/use_router.tsx b/packages/kbn-typed-react-router-config/src/use_router.tsx index 531bc5aea53b3..af92e33b8952a 100644 --- a/packages/kbn-typed-react-router-config/src/use_router.tsx +++ b/packages/kbn-typed-react-router-config/src/use_router.tsx @@ -20,7 +20,7 @@ export const RouterContextProvider = ({ children: React.ReactNode; }) => {children}; -export function useRouter(): Router { +export function useRouter(): Router { const router = useContext(RouterContext); if (!router) { diff --git a/packages/kbn-typed-react-router-config/tsconfig.json b/packages/kbn-typed-react-router-config/tsconfig.json index efda701736093..f9a133f48e8d5 100644 --- a/packages/kbn-typed-react-router-config/tsconfig.json +++ b/packages/kbn-typed-react-router-config/tsconfig.json @@ -14,7 +14,12 @@ ], "kbn_references": [ "@kbn/io-ts-utils", - "@kbn/shared-ux-router" + "@kbn/shared-ux-router", + "@kbn/core", + "@kbn/i18n", + "@kbn/kibana-react-plugin", + "@kbn/core-chrome-browser", + "@kbn/serverless" ], "exclude": [ "target/**/*", diff --git a/tsconfig.base.json b/tsconfig.base.json index 5dda5431f641a..219e5a6247703 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1326,8 +1326,12 @@ "@kbn/observability-shared-plugin/*": ["x-pack/plugins/observability_solution/observability_shared/*"], "@kbn/observability-synthetics-test-data": ["x-pack/packages/observability/synthetics_test_data"], "@kbn/observability-synthetics-test-data/*": ["x-pack/packages/observability/synthetics_test_data/*"], - "@kbn/observability-utils": ["x-pack/packages/observability/observability_utils"], - "@kbn/observability-utils/*": ["x-pack/packages/observability/observability_utils/*"], + "@kbn/observability-utils-browser": ["x-pack/packages/observability/observability_utils/observability_utils_browser"], + "@kbn/observability-utils-browser/*": ["x-pack/packages/observability/observability_utils/observability_utils_browser/*"], + "@kbn/observability-utils-common": ["x-pack/packages/observability/observability_utils/observability_utils_common"], + "@kbn/observability-utils-common/*": ["x-pack/packages/observability/observability_utils/observability_utils_common/*"], + "@kbn/observability-utils-server": ["x-pack/packages/observability/observability_utils/observability_utils_server"], + "@kbn/observability-utils-server/*": ["x-pack/packages/observability/observability_utils/observability_utils_server/*"], "@kbn/oidc-provider-plugin": ["x-pack/test/security_api_integration/plugins/oidc_provider"], "@kbn/oidc-provider-plugin/*": ["x-pack/test/security_api_integration/plugins/oidc_provider/*"], "@kbn/open-telemetry-instrumented-plugin": ["test/common/plugins/otel_metrics"], diff --git a/x-pack/.i18nrc.json b/x-pack/.i18nrc.json index 7afbc9dc704c4..06dfb9f486136 100644 --- a/x-pack/.i18nrc.json +++ b/x-pack/.i18nrc.json @@ -111,7 +111,9 @@ "xpack.observabilityLogsOverview": [ "packages/observability/logs_overview/src/components" ], - "xpack.osquery": ["plugins/osquery"], + "xpack.osquery": [ + "plugins/osquery" + ], "xpack.painlessLab": "plugins/painless_lab", "xpack.profiling": [ "plugins/observability_solution/profiling" @@ -146,6 +148,9 @@ "xpack.securitySolutionEss": "plugins/security_solution_ess", "xpack.securitySolutionServerless": "plugins/security_solution_serverless", "xpack.sessionView": "plugins/session_view", + "xpack.streams": [ + "plugins/logsai/streams_app" + ], "xpack.slo": "plugins/observability_solution/slo", "xpack.snapshotRestore": "plugins/snapshot_restore", "xpack.spaces": "plugins/spaces", diff --git a/x-pack/packages/ai-infra/inference-common/index.ts b/x-pack/packages/ai-infra/inference-common/index.ts index 6de7ce3bb8008..fbb7aa7cf3fac 100644 --- a/x-pack/packages/ai-infra/inference-common/index.ts +++ b/x-pack/packages/ai-infra/inference-common/index.ts @@ -75,3 +75,5 @@ export { isInferenceInternalError, isInferenceRequestError, } from './src/errors'; + +export { truncateList } from './src/util/truncate_list'; diff --git a/x-pack/packages/ai-infra/inference-common/src/util/truncate_list.ts b/x-pack/packages/ai-infra/inference-common/src/util/truncate_list.ts new file mode 100644 index 0000000000000..59b5b1699a3b0 --- /dev/null +++ b/x-pack/packages/ai-infra/inference-common/src/util/truncate_list.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { take } from 'lodash'; + +export function truncateList(values: T[], limit: number): Array { + if (values.length <= limit) { + return values; + } + + return [...take(values, limit), `${values.length - limit} more values`]; +} diff --git a/x-pack/packages/observability/observability_utils/README.md b/x-pack/packages/observability/observability_utils/README.md deleted file mode 100644 index bd74c0bdffb47..0000000000000 --- a/x-pack/packages/observability/observability_utils/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# @kbn/observability-utils - -This package contains utilities for Observability plugins. It's a separate package to get out of dependency hell. You can put anything in here that is stateless and has no dependency on other plugins (either directly or via other packages). - -The utility functions should be used via direct imports to minimize impact on bundle size and limit the risk on importing browser code to the server and vice versa. diff --git a/x-pack/packages/observability/observability_utils/es/client/create_observability_es_client.ts b/x-pack/packages/observability/observability_utils/es/client/create_observability_es_client.ts deleted file mode 100644 index 0011e0f17c1c0..0000000000000 --- a/x-pack/packages/observability/observability_utils/es/client/create_observability_es_client.ts +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { ElasticsearchClient, Logger } from '@kbn/core/server'; -import type { ESQLSearchResponse, ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types'; -import { withSpan } from '@kbn/apm-utils'; -import type { EsqlQueryRequest } from '@elastic/elasticsearch/lib/api/types'; - -type SearchRequest = ESSearchRequest & { - index: string | string[]; - track_total_hits: number | boolean; - size: number | boolean; -}; - -/** - * An Elasticsearch Client with a fully typed `search` method and built-in - * APM instrumentation. - */ -export interface ObservabilityElasticsearchClient { - search( - operationName: string, - parameters: TSearchRequest - ): Promise>; - esql(operationName: string, parameters: EsqlQueryRequest): Promise; - client: ElasticsearchClient; -} - -export function createObservabilityEsClient({ - client, - logger, - plugin, -}: { - client: ElasticsearchClient; - logger: Logger; - plugin: string; -}): ObservabilityElasticsearchClient { - return { - client, - esql(operationName: string, parameters: EsqlQueryRequest) { - logger.trace(() => `Request (${operationName}):\n${JSON.stringify(parameters, null, 2)}`); - return withSpan({ name: operationName, labels: { plugin } }, () => { - return client.esql.query( - { ...parameters }, - { - querystring: { - drop_null_columns: true, - }, - } - ); - }) - .then((response) => { - logger.trace(() => `Response (${operationName}):\n${JSON.stringify(response, null, 2)}`); - return response as unknown as ESQLSearchResponse; - }) - .catch((error) => { - throw error; - }); - }, - search( - operationName: string, - parameters: SearchRequest - ) { - logger.trace(() => `Request (${operationName}):\n${JSON.stringify(parameters, null, 2)}`); - // wraps the search operation in a named APM span for better analysis - // (otherwise it would just be a _search span) - return withSpan( - { - name: operationName, - labels: { - plugin, - }, - }, - () => { - return client.search(parameters) as unknown as Promise< - InferSearchResponseOf - >; - } - ).then((response) => { - logger.trace(() => `Response (${operationName}):\n${JSON.stringify(response, null, 2)}`); - return response; - }); - }, - }; -} diff --git a/x-pack/packages/observability/observability_utils/es/utils/esql_result_to_plain_objects.ts b/x-pack/packages/observability/observability_utils/es/utils/esql_result_to_plain_objects.ts deleted file mode 100644 index ad48bcb311b25..0000000000000 --- a/x-pack/packages/observability/observability_utils/es/utils/esql_result_to_plain_objects.ts +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { ESQLSearchResponse } from '@kbn/es-types'; - -export function esqlResultToPlainObjects>( - result: ESQLSearchResponse -): T[] { - return result.values.map((row) => { - return row.reduce>((acc, value, index) => { - const column = result.columns[index]; - acc[column.name] = value; - return acc; - }, {}); - }) as T[]; -} diff --git a/x-pack/packages/observability/observability_utils/chart/utils.ts b/x-pack/packages/observability/observability_utils/observability_utils_browser/chart/utils.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/chart/utils.ts rename to x-pack/packages/observability/observability_utils/observability_utils_browser/chart/utils.ts diff --git a/x-pack/packages/observability/observability_utils/hooks/use_abort_controller.ts b/x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_abort_controller.ts similarity index 92% rename from x-pack/packages/observability/observability_utils/hooks/use_abort_controller.ts rename to x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_abort_controller.ts index a383e7b81b4d9..de5c70632b233 100644 --- a/x-pack/packages/observability/observability_utils/hooks/use_abort_controller.ts +++ b/x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_abort_controller.ts @@ -18,6 +18,9 @@ export function useAbortController() { return { signal: controller.signal, + abort: () => { + controller.abort(); + }, refresh: () => { setController(() => new AbortController()); }, diff --git a/x-pack/packages/observability/observability_utils/hooks/use_abortable_async.ts b/x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_abortable_async.ts similarity index 72% rename from x-pack/packages/observability/observability_utils/hooks/use_abortable_async.ts rename to x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_abortable_async.ts index 433ca877b0f62..d24a62ee125bc 100644 --- a/x-pack/packages/observability/observability_utils/hooks/use_abortable_async.ts +++ b/x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_abortable_async.ts @@ -17,10 +17,28 @@ export type AbortableAsyncState = (T extends Promise ? State : State) & { refresh: () => void }; +export type AbortableAsyncStateOf> = + T extends AbortableAsyncState ? Awaited : never; + +interface UseAbortableAsyncOptions { + clearValueOnNext?: boolean; + defaultValue?: () => T; + onError?: (error: Error) => void; +} + +export type UseAbortableAsync< + TAdditionalParameters extends Record = {}, + TAdditionalOptions extends Record = {} +> = ( + fn: ({}: { signal: AbortSignal } & TAdditionalParameters) => T | Promise, + deps: any[], + options?: UseAbortableAsyncOptions & TAdditionalOptions +) => AbortableAsyncState; + export function useAbortableAsync( fn: ({}: { signal: AbortSignal }) => T | Promise, deps: any[], - options?: { clearValueOnNext?: boolean; defaultValue?: () => T } + options?: UseAbortableAsyncOptions ): AbortableAsyncState { const clearValueOnNext = options?.clearValueOnNext; @@ -43,6 +61,13 @@ export function useAbortableAsync( setError(undefined); } + function handleError(err: Error) { + setError(err); + // setValue(undefined); + setLoading(false); + options?.onError?.(err); + } + try { const response = fn({ signal: controller.signal }); if (isPromise(response)) { @@ -52,10 +77,7 @@ export function useAbortableAsync( setError(undefined); setValue(nextValue); }) - .catch((err) => { - setValue(undefined); - setError(err); - }) + .catch(handleError) .finally(() => setLoading(false)); } else { setError(undefined); @@ -63,9 +85,7 @@ export function useAbortableAsync( setLoading(false); } } catch (err) { - setValue(undefined); - setError(err); - setLoading(false); + handleError(err); } return () => { diff --git a/x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_date_range.ts b/x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_date_range.ts new file mode 100644 index 0000000000000..941e106247b87 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_date_range.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { TimeRange } from '@kbn/data-plugin/common'; +import { DataPublicPluginStart } from '@kbn/data-plugin/public'; +import { useCallback, useEffect, useMemo, useState } from 'react'; + +export function useDateRange({ data }: { data: DataPublicPluginStart }): { + timeRange: TimeRange; + absoluteTimeRange: { + start: number; + end: number; + }; + setTimeRange: React.Dispatch>; +} { + const timefilter = data.query.timefilter.timefilter; + + const [timeRange, setTimeRange] = useState(() => timefilter.getTime()); + + const [absoluteTimeRange, setAbsoluteTimeRange] = useState(() => timefilter.getAbsoluteTime()); + + useEffect(() => { + const timeUpdateSubscription = timefilter.getTimeUpdate$().subscribe({ + next: () => { + setTimeRange(() => timefilter.getTime()); + setAbsoluteTimeRange(() => timefilter.getAbsoluteTime()); + }, + }); + + return () => { + timeUpdateSubscription.unsubscribe(); + }; + }, [timefilter]); + + const setTimeRangeMemoized: React.Dispatch> = useCallback( + (nextOrCallback) => { + const val = + typeof nextOrCallback === 'function' + ? nextOrCallback(timefilter.getTime()) + : nextOrCallback; + + timefilter.setTime(val); + }, + [timefilter] + ); + + const asEpoch = useMemo(() => { + return { + start: new Date(absoluteTimeRange.from).getTime(), + end: new Date(absoluteTimeRange.to).getTime(), + }; + }, [absoluteTimeRange]); + + return { + timeRange, + absoluteTimeRange: asEpoch, + setTimeRange: setTimeRangeMemoized, + }; +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_local_storage.ts b/x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_local_storage.ts new file mode 100644 index 0000000000000..ea9e13163e4b0 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_local_storage.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useState, useEffect, useMemo, useCallback } from 'react'; + +export function useLocalStorage(key: string, defaultValue: T) { + // This is necessary to fix a race condition issue. + // It guarantees that the latest value will be always returned after the value is updated + const [storageUpdate, setStorageUpdate] = useState(0); + + const item = useMemo(() => { + return getFromStorage(key, defaultValue); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [key, storageUpdate, defaultValue]); + + const saveToStorage = useCallback( + (value: T) => { + if (value === undefined) { + window.localStorage.removeItem(key); + } else { + window.localStorage.setItem(key, JSON.stringify(value)); + setStorageUpdate(storageUpdate + 1); + } + }, + [key, storageUpdate] + ); + + useEffect(() => { + function onUpdate(event: StorageEvent) { + if (event.key === key) { + setStorageUpdate(storageUpdate + 1); + } + } + window.addEventListener('storage', onUpdate); + return () => { + window.removeEventListener('storage', onUpdate); + }; + }, [key, setStorageUpdate, storageUpdate]); + + return useMemo(() => [item, saveToStorage] as const, [item, saveToStorage]); +} + +function getFromStorage(keyName: string, defaultValue: T) { + const storedItem = window.localStorage.getItem(keyName); + + if (storedItem !== null) { + try { + return JSON.parse(storedItem) as T; + } catch (err) { + window.localStorage.removeItem(keyName); + // eslint-disable-next-line no-console + console.log(`Unable to decode: ${keyName}`); + } + } + return defaultValue; +} diff --git a/x-pack/packages/observability/observability_utils/hooks/use_theme.ts b/x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_theme.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/hooks/use_theme.ts rename to x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_theme.ts diff --git a/x-pack/packages/observability/observability_utils/observability_utils_browser/jest.config.js b/x-pack/packages/observability/observability_utils/observability_utils_browser/jest.config.js new file mode 100644 index 0000000000000..33358c221fa1f --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_browser/jest.config.js @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../../../..', + roots: [ + '/x-pack/packages/observability/observability_utils/observability_utils_browser', + ], +}; diff --git a/x-pack/packages/observability/observability_utils/observability_utils_browser/kibana.jsonc b/x-pack/packages/observability/observability_utils/observability_utils_browser/kibana.jsonc new file mode 100644 index 0000000000000..dbee36828d080 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_browser/kibana.jsonc @@ -0,0 +1,5 @@ +{ + "type": "shared-browser", + "id": "@kbn/observability-utils-browser", + "owner": "@elastic/observability-ui" +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_browser/package.json b/x-pack/packages/observability/observability_utils/observability_utils_browser/package.json new file mode 100644 index 0000000000000..c72c8c0b45eb0 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_browser/package.json @@ -0,0 +1,6 @@ +{ + "name": "@kbn/observability-utils-browser", + "private": true, + "version": "1.0.0", + "license": "Elastic License 2.0" +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_browser/tsconfig.json b/x-pack/packages/observability/observability_utils/observability_utils_browser/tsconfig.json new file mode 100644 index 0000000000000..9cfa030bd901d --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_browser/tsconfig.json @@ -0,0 +1,23 @@ +{ + "extends": "../../../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + "types": [ + "jest", + "node", + "react" + ] + }, + "include": [ + "**/*.ts", + "**/*.tsx", + ], + "exclude": [ + "target/**/*" + ], + "kbn_references": [ + "@kbn/data-plugin", + "@kbn/core-ui-settings-browser", + "@kbn/std", + ] +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_browser/utils/ui_settings/get_timezone.ts b/x-pack/packages/observability/observability_utils/observability_utils_browser/utils/ui_settings/get_timezone.ts new file mode 100644 index 0000000000000..3ad5d17aa61bc --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_browser/utils/ui_settings/get_timezone.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { UI_SETTINGS } from '@kbn/data-plugin/public'; +import { IUiSettingsClient } from '@kbn/core-ui-settings-browser'; + +export function getTimeZone(uiSettings?: IUiSettingsClient) { + const kibanaTimeZone = uiSettings?.get<'Browser' | string>(UI_SETTINGS.DATEFORMAT_TZ); + if (!kibanaTimeZone || kibanaTimeZone === 'Browser') { + return 'local'; + } + + return kibanaTimeZone; +} diff --git a/x-pack/packages/observability/observability_utils/array/join_by_key.test.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/array/join_by_key.test.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/array/join_by_key.test.ts rename to x-pack/packages/observability/observability_utils/observability_utils_common/array/join_by_key.test.ts diff --git a/x-pack/packages/observability/observability_utils/array/join_by_key.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/array/join_by_key.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/array/join_by_key.ts rename to x-pack/packages/observability/observability_utils/observability_utils_common/array/join_by_key.ts diff --git a/x-pack/packages/observability/observability_utils/observability_utils_common/entities/get_entity_kuery.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/entities/get_entity_kuery.ts new file mode 100644 index 0000000000000..ba68e544379a4 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/entities/get_entity_kuery.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function getEntityKuery(entity: Record) { + return Object.entries(entity) + .map(([name, value]) => { + return `(${name}:"${value}")`; + }) + .join(' AND '); +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_common/es/format_value_for_kql.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/es/format_value_for_kql.ts new file mode 100644 index 0000000000000..a0fb5c15fd03e --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/es/format_value_for_kql.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function formatValueForKql(value: string) { + return `(${value.replaceAll(/((^|[^\\])):/g, '\\:')})`; +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_common/es/queries/entity_query.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/es/queries/entity_query.ts new file mode 100644 index 0000000000000..f2ae0991eecf4 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/es/queries/entity_query.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; + +export function entityQuery(entity: Record): QueryDslQueryContainer[] { + return [ + { + bool: { + filter: Object.entries(entity).map(([field, value]) => { + return { + term: { + [field]: value, + }, + }; + }), + }, + }, + ]; +} diff --git a/x-pack/packages/observability/observability_utils/es/queries/exclude_frozen_query.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/es/queries/exclude_frozen_query.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/es/queries/exclude_frozen_query.ts rename to x-pack/packages/observability/observability_utils/observability_utils_common/es/queries/exclude_frozen_query.ts diff --git a/x-pack/packages/observability/observability_utils/es/queries/exclude_tiers_query.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/es/queries/exclude_tiers_query.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/es/queries/exclude_tiers_query.ts rename to x-pack/packages/observability/observability_utils/observability_utils_common/es/queries/exclude_tiers_query.ts diff --git a/x-pack/packages/observability/observability_utils/es/queries/kql_query.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/es/queries/kql_query.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/es/queries/kql_query.ts rename to x-pack/packages/observability/observability_utils/observability_utils_common/es/queries/kql_query.ts diff --git a/x-pack/packages/observability/observability_utils/es/queries/range_query.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/es/queries/range_query.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/es/queries/range_query.ts rename to x-pack/packages/observability/observability_utils/observability_utils_common/es/queries/range_query.ts diff --git a/x-pack/packages/observability/observability_utils/es/queries/term_query.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/es/queries/term_query.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/es/queries/term_query.ts rename to x-pack/packages/observability/observability_utils/observability_utils_common/es/queries/term_query.ts diff --git a/x-pack/packages/observability/observability_utils/observability_utils_common/format/integer.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/format/integer.ts new file mode 100644 index 0000000000000..7cf202fb8c811 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/format/integer.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import numeral from '@elastic/numeral'; + +export function formatInteger(num: number) { + return numeral(num).format('0a'); +} diff --git a/x-pack/packages/observability/observability_utils/jest.config.js b/x-pack/packages/observability/observability_utils/observability_utils_common/jest.config.js similarity index 84% rename from x-pack/packages/observability/observability_utils/jest.config.js rename to x-pack/packages/observability/observability_utils/observability_utils_common/jest.config.js index c9dff28ed6cec..ee68881a5863b 100644 --- a/x-pack/packages/observability/observability_utils/jest.config.js +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/jest.config.js @@ -7,6 +7,6 @@ module.exports = { preset: '@kbn/test', - rootDir: '../../../..', - roots: ['/x-pack/packages/observability/observability_utils'], + rootDir: '../../../../..', + roots: ['/x-pack/packages/observability/observability_utils/observability_utils_common'], }; diff --git a/x-pack/packages/observability/observability_utils/kibana.jsonc b/x-pack/packages/observability/observability_utils/observability_utils_common/kibana.jsonc similarity index 61% rename from x-pack/packages/observability/observability_utils/kibana.jsonc rename to x-pack/packages/observability/observability_utils/observability_utils_common/kibana.jsonc index 096b2565d533f..eb120052e5b0e 100644 --- a/x-pack/packages/observability/observability_utils/kibana.jsonc +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/kibana.jsonc @@ -1,5 +1,5 @@ { "type": "shared-common", - "id": "@kbn/observability-utils", + "id": "@kbn/observability-utils-common", "owner": "@elastic/observability-ui" } diff --git a/x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/document_analysis.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/document_analysis.ts new file mode 100644 index 0000000000000..be896571ca217 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/document_analysis.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export interface DocumentAnalysis { + total: number; + sampled: number; + fields: Array<{ + name: string; + types: string[]; + cardinality: number | null; + values: Array; + empty: boolean; + }>; +} + +export interface TruncatedDocumentAnalysis { + fields: string[]; + total: number; + sampled: number; +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/highlight_patterns_from_regex.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/highlight_patterns_from_regex.ts new file mode 100644 index 0000000000000..11ab0c52f1795 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/highlight_patterns_from_regex.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +function addCapturingGroupsToRegex(regex: string): string { + // Match all parts of the regex that are not special characters + // We treat constant parts as sequences of characters that are not part of regex syntax + return regex.replaceAll(/((?:\.\*\?)|(?:\.\+\?)|(?:\+\?))/g, (...args) => { + return `(${args[1]})`; + }); +} + +export function highlightPatternFromRegex(pattern: string, str: string): string { + // First, add non-capturing groups to the regex around constant parts + const updatedPattern = addCapturingGroupsToRegex(pattern); + + const regex = new RegExp(updatedPattern, 'ds'); + + const matches = str.match(regex) as + | (RegExpMatchArray & { indices: Array<[number, number]> }) + | null; + + const slices: string[] = []; + + matches?.forEach((_, index) => { + if (index === 0) { + return; + } + + const [, prevEnd] = index > 1 ? matches?.indices[index - 1] : [undefined, undefined]; + const [start, end] = matches?.indices[index]; + + const literalSlice = prevEnd !== undefined ? str.slice(prevEnd, start) : undefined; + + if (literalSlice) { + slices.push(`${literalSlice}`); + } + + const slice = str.slice(start, end); + slices.push(slice); + + if (index === matches.length - 1) { + slices.push(str.slice(end)); + } + }); + + return slices.join(''); +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/merge_sample_documents_with_field_caps.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/merge_sample_documents_with_field_caps.ts new file mode 100644 index 0000000000000..58b6024aed046 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/merge_sample_documents_with_field_caps.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { castArray, sortBy, uniq } from 'lodash'; +import type { DocumentAnalysis } from './document_analysis'; + +export function mergeSampleDocumentsWithFieldCaps({ + total, + samples, + fieldCaps, +}: { + total: number; + samples: Array>; + fieldCaps: Array<{ name: string; esTypes?: string[] }>; +}): DocumentAnalysis { + const nonEmptyFields = new Set(); + const fieldValues = new Map>(); + + for (const document of samples) { + Object.keys(document).forEach((field) => { + if (!nonEmptyFields.has(field)) { + nonEmptyFields.add(field); + } + + const values = castArray(document[field]); + + const currentFieldValues = fieldValues.get(field) ?? []; + + values.forEach((value) => { + if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') { + currentFieldValues.push(value); + } + }); + + fieldValues.set(field, currentFieldValues); + }); + } + + const fields = fieldCaps.flatMap((spec) => { + const values = fieldValues.get(spec.name); + + const countByValues = new Map(); + + values?.forEach((value) => { + const currentCount = countByValues.get(value) ?? 0; + countByValues.set(value, currentCount + 1); + }); + + const sortedValues = sortBy( + Array.from(countByValues.entries()).map(([value, count]) => { + return { + value, + count, + }; + }), + 'count', + 'desc' + ); + + return { + name: spec.name, + types: spec.esTypes ?? [], + empty: !nonEmptyFields.has(spec.name), + cardinality: countByValues.size || null, + values: uniq(sortedValues.flatMap(({ value }) => value)), + }; + }); + + return { + total, + sampled: samples.length, + fields, + }; +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/sort_and_truncate_analyzed_fields.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/sort_and_truncate_analyzed_fields.ts new file mode 100644 index 0000000000000..c9a3e6a156601 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/sort_and_truncate_analyzed_fields.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { partition, shuffle } from 'lodash'; +import { truncateList } from '@kbn/inference-common'; +import type { DocumentAnalysis, TruncatedDocumentAnalysis } from './document_analysis'; + +export function sortAndTruncateAnalyzedFields( + analysis: DocumentAnalysis +): TruncatedDocumentAnalysis { + const { fields, ...meta } = analysis; + const [nonEmptyFields, emptyFields] = partition(analysis.fields, (field) => !field.empty); + + const sortedFields = [...shuffle(nonEmptyFields), ...shuffle(emptyFields)]; + + return { + ...meta, + fields: truncateList( + sortedFields.map((field) => { + let label = `${field.name}:${field.types.join(',')}`; + + if (field.empty) { + return `${name} (empty)`; + } + + label += ` - ${field.cardinality} distinct values`; + + if (field.name === '@timestamp' || field.name === 'event.ingested') { + return `${label}`; + } + + const shortValues = field.values.filter((value) => { + return String(value).length <= 1024; + }); + + if (shortValues.length) { + return `${label} (${truncateList( + shortValues.map((value) => '`' + value + '`'), + field.types.includes('text') || field.types.includes('match_only_text') ? 2 : 10 + ).join(', ')})`; + } + + return label; + }), + 500 + ).sort(), + }; +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_common/llm/short_id_table.test.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/llm/short_id_table.test.ts new file mode 100644 index 0000000000000..784cf67530652 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/llm/short_id_table.test.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { ShortIdTable } from './short_id_table'; + +describe('shortIdTable', () => { + it('generates at least 10k unique ids consistently', () => { + const ids = new Set(); + + const table = new ShortIdTable(); + + let i = 10_000; + while (i--) { + const id = table.take(String(i)); + ids.add(id); + } + + expect(ids.size).toBe(10_000); + }); + + it('returns the original id based on the generated id', () => { + const table = new ShortIdTable(); + + const idsByOriginal = new Map(); + + let i = 100; + while (i--) { + const id = table.take(String(i)); + idsByOriginal.set(String(i), id); + } + + expect(idsByOriginal.size).toBe(100); + + expect(() => { + Array.from(idsByOriginal.entries()).forEach(([originalId, shortId]) => { + const returnedOriginalId = table.lookup(shortId); + if (returnedOriginalId !== originalId) { + throw Error( + `Expected shortId ${shortId} to return ${originalId}, but ${returnedOriginalId} was returned instead` + ); + } + }); + }).not.toThrow(); + }); +}); diff --git a/x-pack/packages/observability/observability_utils/observability_utils_common/llm/short_id_table.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/llm/short_id_table.ts new file mode 100644 index 0000000000000..30049452ddf51 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/llm/short_id_table.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +const ALPHABET = 'abcdefghijklmnopqrstuvwxyz'; + +function generateShortId(size: number): string { + let id = ''; + let i = size; + while (i--) { + const index = Math.floor(Math.random() * ALPHABET.length); + id += ALPHABET[index]; + } + return id; +} + +const MAX_ATTEMPTS_AT_LENGTH = 100; + +export class ShortIdTable { + private byShortId: Map = new Map(); + private byOriginalId: Map = new Map(); + + constructor() {} + + take(originalId: string) { + if (this.byOriginalId.has(originalId)) { + return this.byOriginalId.get(originalId)!; + } + + let uniqueId: string | undefined; + let attemptsAtLength = 0; + let length = 4; + while (!uniqueId) { + const nextId = generateShortId(length); + attemptsAtLength++; + if (!this.byShortId.has(nextId)) { + uniqueId = nextId; + } else if (attemptsAtLength >= MAX_ATTEMPTS_AT_LENGTH) { + attemptsAtLength = 0; + length++; + } + } + + this.byShortId.set(uniqueId, originalId); + this.byOriginalId.set(originalId, uniqueId); + + return uniqueId; + } + + lookup(shortId: string) { + return this.byShortId.get(shortId); + } +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_common/ml/p_value_to_label.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/ml/p_value_to_label.ts new file mode 100644 index 0000000000000..3f6e0836d129b --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/ml/p_value_to_label.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const P_VALUE_SIGNIFICANCE_HIGH = 1e-6; +export const P_VALUE_SIGNIFICANCE_MEDIUM = 0.001; + +export function pValueToLabel(pValue: number): 'high' | 'medium' | 'low' { + if (pValue <= P_VALUE_SIGNIFICANCE_HIGH) { + return 'high'; + } else if (pValue <= P_VALUE_SIGNIFICANCE_MEDIUM) { + return 'medium'; + } else { + return 'low'; + } +} diff --git a/x-pack/packages/observability/observability_utils/object/flatten_object.test.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/object/flatten_object.test.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/object/flatten_object.test.ts rename to x-pack/packages/observability/observability_utils/observability_utils_common/object/flatten_object.test.ts diff --git a/x-pack/packages/observability/observability_utils/object/flatten_object.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/object/flatten_object.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/object/flatten_object.ts rename to x-pack/packages/observability/observability_utils/observability_utils_common/object/flatten_object.ts diff --git a/x-pack/packages/observability/observability_utils/object/merge_plain_object.test.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/object/merge_plain_object.test.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/object/merge_plain_object.test.ts rename to x-pack/packages/observability/observability_utils/observability_utils_common/object/merge_plain_object.test.ts diff --git a/x-pack/packages/observability/observability_utils/object/merge_plain_objects.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/object/merge_plain_objects.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/object/merge_plain_objects.ts rename to x-pack/packages/observability/observability_utils/observability_utils_common/object/merge_plain_objects.ts diff --git a/x-pack/packages/observability/observability_utils/object/unflatten_object.test.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/object/unflatten_object.test.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/object/unflatten_object.test.ts rename to x-pack/packages/observability/observability_utils/observability_utils_common/object/unflatten_object.test.ts diff --git a/x-pack/packages/observability/observability_utils/object/unflatten_object.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/object/unflatten_object.ts similarity index 99% rename from x-pack/packages/observability/observability_utils/object/unflatten_object.ts rename to x-pack/packages/observability/observability_utils/observability_utils_common/object/unflatten_object.ts index 142ea2eea6461..83508d5d2dbf5 100644 --- a/x-pack/packages/observability/observability_utils/object/unflatten_object.ts +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/object/unflatten_object.ts @@ -11,7 +11,6 @@ export function unflattenObject(source: Record, target: Record { if (item && typeof item === 'object' && !Array.isArray(item)) { diff --git a/x-pack/packages/observability/observability_utils/package.json b/x-pack/packages/observability/observability_utils/observability_utils_common/package.json similarity index 62% rename from x-pack/packages/observability/observability_utils/package.json rename to x-pack/packages/observability/observability_utils/observability_utils_common/package.json index 06f6e37858927..2f9be5f105279 100644 --- a/x-pack/packages/observability/observability_utils/package.json +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/package.json @@ -1,6 +1,6 @@ { - "name": "@kbn/observability-utils", + "name": "@kbn/observability-utils-common", "private": true, "version": "1.0.0", "license": "Elastic License 2.0" -} \ No newline at end of file +} diff --git a/x-pack/packages/observability/observability_utils/tsconfig.json b/x-pack/packages/observability/observability_utils/observability_utils_common/tsconfig.json similarity index 70% rename from x-pack/packages/observability/observability_utils/tsconfig.json rename to x-pack/packages/observability/observability_utils/observability_utils_common/tsconfig.json index b3f1a4a21c4e7..7954cdc946e9c 100644 --- a/x-pack/packages/observability/observability_utils/tsconfig.json +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/tsconfig.json @@ -1,5 +1,5 @@ { - "extends": "../../../../tsconfig.base.json", + "extends": "../../../../../tsconfig.base.json", "compilerOptions": { "outDir": "target/types", "types": [ @@ -16,11 +16,8 @@ "target/**/*" ], "kbn_references": [ - "@kbn/std", - "@kbn/core", - "@kbn/es-types", - "@kbn/apm-utils", "@kbn/es-query", "@kbn/safer-lodash-set", + "@kbn/inference-common", ] } diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/entities/analyze_documents.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/entities/analyze_documents.ts new file mode 100644 index 0000000000000..0cc1374d8b1d8 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/entities/analyze_documents.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { mapValues } from 'lodash'; +import { mergeSampleDocumentsWithFieldCaps } from '@kbn/observability-utils-common/llm/log_analysis/merge_sample_documents_with_field_caps'; +import { DocumentAnalysis } from '@kbn/observability-utils-common/llm/log_analysis/document_analysis'; +import type { ObservabilityElasticsearchClient } from '../es/client/create_observability_es_client'; +import { kqlQuery } from '../es/queries/kql_query'; +import { rangeQuery } from '../es/queries/range_query'; + +export async function analyzeDocuments({ + esClient, + kuery, + start, + end, + index, +}: { + esClient: ObservabilityElasticsearchClient; + kuery: string; + start: number; + end: number; + index: string | string[]; +}): Promise { + const [fieldCaps, hits] = await Promise.all([ + esClient.fieldCaps('get_field_caps_for_document_analysis', { + index, + fields: '*', + index_filter: { + bool: { + filter: rangeQuery(start, end), + }, + }, + }), + esClient + .search('get_document_samples', { + index, + size: 1000, + track_total_hits: true, + query: { + bool: { + must: [...kqlQuery(kuery), ...rangeQuery(start, end)], + should: [ + { + function_score: { + functions: [ + { + random_score: {}, + }, + ], + }, + }, + ], + }, + }, + sort: { + _score: { + order: 'desc', + }, + }, + _source: false, + fields: ['*' as const], + }) + .then((response) => ({ + hits: response.hits.hits.map((hit) => + mapValues(hit.fields!, (value) => (value.length === 1 ? value[0] : value)) + ), + total: response.hits.total, + })), + ]); + + const analysis = mergeSampleDocumentsWithFieldCaps({ + samples: hits.hits, + total: hits.total.value, + fieldCaps: Object.entries(fieldCaps.fields).map(([name, specs]) => { + return { name, esTypes: Object.keys(specs) }; + }), + }); + + return analysis; +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/entities/get_data_streams_for_entity.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/entities/get_data_streams_for_entity.ts new file mode 100644 index 0000000000000..43d9134c7aaf3 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/entities/get_data_streams_for_entity.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { compact, uniq } from 'lodash'; +import { ObservabilityElasticsearchClient } from '../es/client/create_observability_es_client'; +import { excludeFrozenQuery } from '../es/queries/exclude_frozen_query'; +import { kqlQuery } from '../es/queries/kql_query'; + +export async function getDataStreamsForEntity({ + esClient, + kuery, + index, +}: { + esClient: ObservabilityElasticsearchClient; + kuery: string; + index: string | string[]; +}) { + const response = await esClient.search('get_data_streams_for_entity', { + track_total_hits: false, + index, + size: 0, + terminate_after: 1, + timeout: '1ms', + aggs: { + indices: { + terms: { + field: '_index', + size: 10000, + }, + }, + }, + query: { + bool: { + filter: [...excludeFrozenQuery(), ...kqlQuery(kuery)], + }, + }, + }); + + const allIndices = + response.aggregations?.indices.buckets.map((bucket) => bucket.key as string) ?? []; + + if (!allIndices.length) { + return { + dataStreams: [], + }; + } + + const resolveIndexResponse = await esClient.client.indices.resolveIndex({ + name: allIndices, + }); + + const dataStreams = uniq( + compact(await resolveIndexResponse.indices.flatMap((idx) => idx.data_stream)) + ); + + return { + dataStreams, + }; +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/entities/signals/get_alerts_for_entity.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/entities/signals/get_alerts_for_entity.ts new file mode 100644 index 0000000000000..400aad8e94357 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/entities/signals/get_alerts_for_entity.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { RulesClient } from '@kbn/alerting-plugin/server'; +import { AlertsClient } from '@kbn/rule-registry-plugin/server'; +import { + ALERT_GROUP_FIELD, + ALERT_GROUP_VALUE, + ALERT_STATUS, + ALERT_STATUS_ACTIVE, + ALERT_TIME_RANGE, +} from '@kbn/rule-data-utils'; +import { kqlQuery } from '../../es/queries/kql_query'; +import { rangeQuery } from '../../es/queries/range_query'; + +export async function getAlertsForEntity({ + start, + end, + entity, + alertsClient, + rulesClient, + size, +}: { + start: number; + end: number; + entity: Record; + alertsClient: AlertsClient; + rulesClient: RulesClient; + size: number; +}) { + const alertsKuery = Object.entries(entity) + .map(([field, value]) => { + return `(${[ + `(${ALERT_GROUP_FIELD}:"${field}" AND ${ALERT_GROUP_VALUE}:"${value}")`, + `(${field}:"${value}")`, + ].join(' OR ')})`; + }) + .join(' AND '); + + const openAlerts = await alertsClient.find({ + size, + query: { + bool: { + filter: [ + ...kqlQuery(alertsKuery), + ...rangeQuery(start, end, ALERT_TIME_RANGE), + { term: { [ALERT_STATUS]: ALERT_STATUS_ACTIVE } }, + ], + }, + }, + }); + + return openAlerts; +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/entities/signals/get_anomalies_for_entity.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/entities/signals/get_anomalies_for_entity.ts new file mode 100644 index 0000000000000..b8802ed3c9045 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/entities/signals/get_anomalies_for_entity.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function getAnomaliesForEntity({ + start, + end, + entity, +}: { + start: number; + end: number; + entity: Record; +}) {} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/entities/signals/get_slos_for_entity.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/entities/signals/get_slos_for_entity.ts new file mode 100644 index 0000000000000..fc3a9d7b26d5c --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/entities/signals/get_slos_for_entity.ts @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ObservabilityElasticsearchClient } from '../../es/client/create_observability_es_client'; +import { kqlQuery } from '../../es/queries/kql_query'; + +export async function getSlosForEntity({ + start, + end, + entity, + esClient, + sloSummaryIndices, + size, + spaceId, +}: { + start: number; + end: number; + entity: Record; + esClient: ObservabilityElasticsearchClient; + sloSummaryIndices: string | string[]; + size: number; + spaceId: string; +}) { + const slosKuery = Object.entries(entity) + .map(([field, value]) => { + return `(slo.groupings.${field}:"${value}")`; + }) + .join(' AND '); + + const sloSummaryResponse = await esClient.search('get_slo_summaries_for_entity', { + index: sloSummaryIndices, + size, + track_total_hits: false, + query: { + bool: { + filter: [ + ...kqlQuery(slosKuery), + { + range: { + 'slo.createdAt': { + lte: end, + }, + }, + }, + { + range: { + summaryUpdatedAt: { + gte: start, + }, + }, + }, + { + term: { + spaceId, + }, + }, + ], + }, + }, + }); + + return { + ...sloSummaryResponse, + hits: { + ...sloSummaryResponse.hits, + hits: sloSummaryResponse.hits.hits.map((hit) => { + return { + ...hit, + _source: hit._source as Record & { + status: 'VIOLATED' | 'DEGRADED' | 'HEALTHY' | 'NO_DATA'; + }, + }; + }), + }, + }; +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts new file mode 100644 index 0000000000000..74b1d1805d71a --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + FieldCapsRequest, + FieldCapsResponse, + MsearchRequest, + SearchResponse, +} from '@elastic/elasticsearch/lib/api/types'; +import { withSpan } from '@kbn/apm-utils'; +import type { ElasticsearchClient, Logger } from '@kbn/core/server'; +import type { + ESQLSearchResponse, + ESSearchRequest, + InferSearchResponseOf, + ESQLSearchParams, +} from '@kbn/es-types'; +import { Required } from 'utility-types'; + +type SearchRequest = ESSearchRequest & { + index: string | string[]; + track_total_hits: number | boolean; + size: number | boolean; +}; +/** + * An Elasticsearch Client with a fully typed `search` method and built-in + * APM instrumentation. + */ +export interface ObservabilityElasticsearchClient { + search( + operationName: string, + parameters: TSearchRequest + ): Promise>; + msearch( + operationName: string, + parameters: MsearchRequest + ): Promise<{ + responses: Array>; + }>; + fieldCaps( + operationName: string, + request: Required + ): Promise; + esql(operationName: string, parameters: ESQLSearchParams): Promise; + client: ElasticsearchClient; +} + +export function createObservabilityEsClient({ + client, + logger, + plugin, +}: { + client: ElasticsearchClient; + logger: Logger; + plugin: string; +}): ObservabilityElasticsearchClient { + // wraps the ES calls in a named APM span for better analysis + // (otherwise it would just eg be a _search span) + const callWithLogger = ( + operationName: string, + request: Record, + callback: () => Promise + ) => { + logger.debug(() => `Request (${operationName}):\n${JSON.stringify(request)}`); + return withSpan( + { + name: operationName, + labels: { + plugin, + }, + }, + callback, + logger + ).then((response) => { + logger.trace(() => `Response (${operationName}):\n${JSON.stringify(response, null, 2)}`); + return response; + }); + }; + + return { + client, + fieldCaps(operationName, parameters) { + return callWithLogger(operationName, parameters, () => { + return client.fieldCaps({ + ...parameters, + }); + }); + }, + esql(operationName: string, parameters: ESQLSearchParams) { + return callWithLogger(operationName, parameters, () => { + return client.esql.transport.request( + { + path: '_query', + method: 'POST', + body: JSON.stringify(parameters), + querystring: { + drop_null_columns: true, + }, + }, + { + headers: { + 'content-type': 'application/json', + }, + } + ) as unknown as Promise; + }); + }, + search( + operationName: string, + parameters: SearchRequest + ) { + return callWithLogger(operationName, parameters, () => { + return client.search(parameters) as unknown as Promise< + InferSearchResponseOf + >; + }); + }, + msearch(operationName: string, parameters: MsearchRequest) { + return callWithLogger(operationName, parameters, () => { + return client.msearch(parameters) as unknown as Promise<{ + responses: Array>; + }>; + }); + }, + }; +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/esql_result_to_plain_objects.test.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/esql_result_to_plain_objects.test.ts new file mode 100644 index 0000000000000..4557d0ba0bdd5 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/esql_result_to_plain_objects.test.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ESQLSearchResponse } from '@kbn/es-types'; +import { esqlResultToPlainObjects } from './esql_result_to_plain_objects'; + +describe('esqlResultToPlainObjects', () => { + it('should return an empty array for an empty result', () => { + const result: ESQLSearchResponse = { + columns: [], + values: [], + }; + const output = esqlResultToPlainObjects(result); + expect(output).toEqual([]); + }); + + it('should return plain objects', () => { + const result: ESQLSearchResponse = { + columns: [{ name: 'name', type: 'keyword' }], + values: [['Foo Bar']], + }; + const output = esqlResultToPlainObjects(result); + expect(output).toEqual([{ name: 'Foo Bar' }]); + }); + + it('should return columns without "text" or "keyword" in their names', () => { + const result: ESQLSearchResponse = { + columns: [ + { name: 'name.text', type: 'text' }, + { name: 'age', type: 'keyword' }, + ], + values: [ + ['Foo Bar', 30], + ['Foo Qux', 25], + ], + }; + const output = esqlResultToPlainObjects(result); + expect(output).toEqual([ + { name: 'Foo Bar', age: 30 }, + { name: 'Foo Qux', age: 25 }, + ]); + }); + + it('should handle mixed columns correctly', () => { + const result: ESQLSearchResponse = { + columns: [ + { name: 'name', type: 'text' }, + { name: 'name.text', type: 'text' }, + { name: 'age', type: 'keyword' }, + ], + values: [ + ['Foo Bar', 'Foo Bar', 30], + ['Foo Qux', 'Foo Qux', 25], + ], + }; + const output = esqlResultToPlainObjects(result); + expect(output).toEqual([ + { name: 'Foo Bar', age: 30 }, + { name: 'Foo Qux', age: 25 }, + ]); + }); +}); diff --git a/x-pack/plugins/logsai/streams_api/common/utils/esql_result_to_plain_objects.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/esql_result_to_plain_objects.ts similarity index 73% rename from x-pack/plugins/logsai/streams_api/common/utils/esql_result_to_plain_objects.ts rename to x-pack/packages/observability/observability_utils/observability_utils_server/es/esql_result_to_plain_objects.ts index ad48bcb311b25..96049f75ef156 100644 --- a/x-pack/plugins/logsai/streams_api/common/utils/esql_result_to_plain_objects.ts +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/esql_result_to_plain_objects.ts @@ -13,7 +13,17 @@ export function esqlResultToPlainObjects>( return result.values.map((row) => { return row.reduce>((acc, value, index) => { const column = result.columns[index]; - acc[column.name] = value; + + if (!column) { + return acc; + } + + // Removes the type suffix from the column name + const name = column.name.replace(/\.(text|keyword)$/, ''); + if (!acc[name]) { + acc[name] = value; + } + return acc; }, {}); }) as T[]; diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/exclude_frozen_query.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/exclude_frozen_query.ts new file mode 100644 index 0000000000000..f348d925c41ca --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/exclude_frozen_query.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { estypes } from '@elastic/elasticsearch'; + +export function excludeFrozenQuery(): estypes.QueryDslQueryContainer[] { + return [ + { + bool: { + must_not: [ + { + term: { + _tier: 'data_frozen', + }, + }, + ], + }, + }, + ]; +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/kql_query.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/kql_query.ts new file mode 100644 index 0000000000000..2f560157cc8c6 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/kql_query.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { estypes } from '@elastic/elasticsearch'; +import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query'; + +export function kqlQuery(kql?: string): estypes.QueryDslQueryContainer[] { + if (!kql) { + return []; + } + + const ast = fromKueryExpression(kql); + return [toElasticsearchQuery(ast)]; +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/range_query.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/range_query.ts new file mode 100644 index 0000000000000..d73476354c377 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/range_query.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { estypes } from '@elastic/elasticsearch'; + +export function rangeQuery( + start?: number, + end?: number, + field = '@timestamp' +): estypes.QueryDslQueryContainer[] { + return [ + { + range: { + [field]: { + gte: start, + lte: end, + format: 'epoch_millis', + }, + }, + }, + ]; +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/term_query.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/term_query.ts new file mode 100644 index 0000000000000..dfaeb737bf8b7 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/term_query.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; + +interface TermQueryOpts { + queryEmptyString: boolean; +} + +export function termQuery( + field: T, + value: string | boolean | number | undefined | null, + opts: TermQueryOpts = { queryEmptyString: true } +): QueryDslQueryContainer[] { + if (value === null || value === undefined || (!opts.queryEmptyString && value === '')) { + return []; + } + + return [{ term: { [field]: value } }]; +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/jest.config.js b/x-pack/packages/observability/observability_utils/observability_utils_server/jest.config.js new file mode 100644 index 0000000000000..5a52de35fcd06 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/jest.config.js @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../../../..', + roots: ['/x-pack/packages/observability/observability_utils/observability_utils_server'], +}; diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/kibana.jsonc b/x-pack/packages/observability/observability_utils/observability_utils_server/kibana.jsonc new file mode 100644 index 0000000000000..4c2f20ef1491f --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/kibana.jsonc @@ -0,0 +1,5 @@ +{ + "type": "shared-server", + "id": "@kbn/observability-utils-server", + "owner": "@elastic/observability-ui" +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/package.json b/x-pack/packages/observability/observability_utils/observability_utils_server/package.json new file mode 100644 index 0000000000000..43abbbb757fea --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/package.json @@ -0,0 +1,6 @@ +{ + "name": "@kbn/observability-utils-server", + "private": true, + "version": "1.0.0", + "license": "Elastic License 2.0" +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/tsconfig.json b/x-pack/packages/observability/observability_utils/observability_utils_server/tsconfig.json new file mode 100644 index 0000000000000..f51d93089c627 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/tsconfig.json @@ -0,0 +1,28 @@ +{ + "extends": "../../../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + "types": [ + "jest", + "node", + "react" + ] + }, + "include": [ + "**/*.ts", + "**/*.tsx", + ], + "exclude": [ + "target/**/*" + ], + "kbn_references": [ + "@kbn/core", + "@kbn/es-types", + "@kbn/apm-utils", + "@kbn/es-query", + "@kbn/observability-utils-common", + "@kbn/alerting-plugin", + "@kbn/rule-registry-plugin", + "@kbn/rule-data-utils", + ] +} diff --git a/x-pack/plugins/logsai/streams_api/kibana.jsonc b/x-pack/plugins/logsai/streams_api/kibana.jsonc index 74753c8b6841b..e446a0210d5cb 100644 --- a/x-pack/plugins/logsai/streams_api/kibana.jsonc +++ b/x-pack/plugins/logsai/streams_api/kibana.jsonc @@ -8,17 +8,8 @@ "browser": true, "configPath": ["xpack", "streamsAPI"], "requiredPlugins": [ - "observabilityShared", - "inference", - "dataViews", - "data", - "unifiedSearch", - "datasetQuality", - "share", "alerting", "ruleRegistry", - "slo", - "spaces" ], "requiredBundles": [ ], diff --git a/x-pack/plugins/logsai/streams_api/public/api/index.tsx b/x-pack/plugins/logsai/streams_api/public/api/index.ts similarity index 100% rename from x-pack/plugins/logsai/streams_api/public/api/index.tsx rename to x-pack/plugins/logsai/streams_api/public/api/index.ts diff --git a/x-pack/plugins/logsai/streams_api/server/lib/clients/create_streams_api_es_client.ts b/x-pack/plugins/logsai/streams_api/server/lib/clients/create_streams_api_es_client.ts index 9d7037f56a76d..26c2693259252 100644 --- a/x-pack/plugins/logsai/streams_api/server/lib/clients/create_streams_api_es_client.ts +++ b/x-pack/plugins/logsai/streams_api/server/lib/clients/create_streams_api_es_client.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { createObservabilityEsClient } from '@kbn/observability-utils/es/client/create_observability_es_client'; +import { createObservabilityEsClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; import { StreamsAPIRouteHandlerResources } from '../../routes/types'; export async function createStreamsAPIEsClient({ diff --git a/x-pack/plugins/logsai/streams_api/server/lib/entities/entity_lookup_table.ts b/x-pack/plugins/logsai/streams_api/server/lib/entities/entity_lookup_table.ts deleted file mode 100644 index 8ac42a17567df..0000000000000 --- a/x-pack/plugins/logsai/streams_api/server/lib/entities/entity_lookup_table.ts +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ESQLLookupTableColumns } from '@kbn/es-types/src/search'; -import { ValuesType } from 'utility-types'; - -export interface EntityLookupTable { - name: string; - joins: string[]; - columns: Record> & { - 'entity.id': { keyword: Array }; - }; -} diff --git a/x-pack/plugins/logsai/streams_api/server/lib/entities/get_data_streams_for_filter.ts b/x-pack/plugins/logsai/streams_api/server/lib/entities/get_data_streams_for_filter.ts deleted file mode 100644 index 55a7670885c4c..0000000000000 --- a/x-pack/plugins/logsai/streams_api/server/lib/entities/get_data_streams_for_filter.ts +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; -import { excludeFrozenQuery } from '@kbn/observability-utils-common/es/queries/exclude_frozen_query'; -import { kqlQuery } from '@kbn/observability-utils-common/es/queries/kql_query'; -import { chunk, compact, uniqBy } from 'lodash'; -import pLimit from 'p-limit'; -import { rangeQuery } from '@kbn/observability-utils-common/es/queries/range_query'; -import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; - -export async function getDataStreamsForFilter({ - esClient, - kql, - indexPatterns, - dslFilter, - start, - end, -}: { - esClient: ObservabilityElasticsearchClient; - kql?: string; - dslFilter?: QueryDslQueryContainer[]; - indexPatterns: string[]; - start: number; - end: number; -}): Promise> { - const indicesResponse = await esClient.search('get_data_streams_for_entities', { - index: indexPatterns, - timeout: '1ms', - terminate_after: 1, - size: 0, - track_total_hits: false, - request_cache: false, - query: { - bool: { - filter: [ - ...excludeFrozenQuery(), - ...kqlQuery(kql), - ...rangeQuery(start, end), - ...(dslFilter ?? []), - ], - }, - }, - aggs: { - indices: { - terms: { - field: '_index', - size: 50000, - }, - }, - }, - }); - - const allIndicesChunks = chunk( - indicesResponse.aggregations?.indices.buckets.map(({ key }) => key as string) ?? [], - 25 - ); - - const limiter = pLimit(5); - - const allDataStreams = await Promise.all( - allIndicesChunks.map(async (allIndices) => { - return limiter(async () => { - const resolveIndicesResponse = await esClient.client.indices.resolveIndex({ - name: allIndices.join(','), - }); - - return compact( - resolveIndicesResponse.indices - .filter((index) => index.data_stream) - .map( - (index) => - (index.name.includes(':') ? index.name.split(':')[0] + ':' : '') + index.data_stream - ) - ).map((dataStream) => ({ name: dataStream })); - }); - }) - ); - - return uniqBy(allDataStreams.flat(), (ds) => ds.name); -} diff --git a/x-pack/plugins/logsai/streams_api/server/lib/entities/get_definition_entities.ts b/x-pack/plugins/logsai/streams_api/server/lib/entities/get_definition_entities.ts deleted file mode 100644 index 6fd0c45cebb4a..0000000000000 --- a/x-pack/plugins/logsai/streams_api/server/lib/entities/get_definition_entities.ts +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; -import { builtinEntityDefinitions } from '../../built_in_definitions_stub'; -import { DefinitionEntity } from '../../../common/entities'; - -export async function getDefinitionEntities({ - esClient, -}: { - esClient: ObservabilityElasticsearchClient; -}): Promise { - return builtinEntityDefinitions; -} diff --git a/x-pack/plugins/logsai/streams_api/server/lib/entities/get_type_definitions.ts b/x-pack/plugins/logsai/streams_api/server/lib/entities/get_type_definitions.ts deleted file mode 100644 index 13825e2c35f15..0000000000000 --- a/x-pack/plugins/logsai/streams_api/server/lib/entities/get_type_definitions.ts +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; -import { builtinTypeDefinitions } from '../../built_in_definitions_stub'; -import { EntityTypeDefinition } from '../../../common/entities'; - -export async function getTypeDefinitions({ - esClient, -}: { - esClient: ObservabilityElasticsearchClient; -}): Promise { - return builtinTypeDefinitions; -} diff --git a/x-pack/plugins/logsai/streams_api/server/lib/entities/query_signals_as_entities.ts b/x-pack/plugins/logsai/streams_api/server/lib/entities/query_signals_as_entities.ts deleted file mode 100644 index f6003294b3ed7..0000000000000 --- a/x-pack/plugins/logsai/streams_api/server/lib/entities/query_signals_as_entities.ts +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; -import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; -import { - ALERT_STATUS, - ALERT_STATUS_ACTIVE, - ALERT_TIME_RANGE, - ALERT_UUID, - AlertConsumers, -} from '@kbn/rule-registry-plugin/common/technical_rule_data_field_names'; -import { SloClient } from '@kbn/slo-plugin/server'; -import { AlertsClient } from '@kbn/rule-registry-plugin/server'; -import { pick } from 'lodash'; -import { ValuesType } from 'utility-types'; -import { Logger } from '@kbn/logging'; -import { querySourcesAsEntities } from './query_sources_as_entities'; -import { - ENTITY_HEALTH_STATUS_INT, - EntityGrouping, - EntityTypeDefinition, - IEntity, -} from '../../../common/entities'; - -export async function querySignalsAsEntities({ - logger, - start, - end, - esClient, - groupings, - typeDefinitions, - filters, - sloClient, - alertsClient, -}: { - logger: Logger; - esClient: ObservabilityElasticsearchClient; - start: number; - end: number; - groupings: EntityGrouping[]; - typeDefinitions: EntityTypeDefinition[]; - filters?: QueryDslQueryContainer[]; - spaceId: string; - sloClient: SloClient; - alertsClient: AlertsClient; -}) { - const consumersOfInterest: string[] = Object.values(AlertConsumers).filter( - (consumer) => consumer !== AlertConsumers.SIEM && consumer !== AlertConsumers.EXAMPLE - ); - - const [sloSummaryDataScope, authorizedAlertsIndices] = await Promise.all([ - sloClient.getDataScopeForSummarySlos({ - start, - end, - }), - alertsClient.getAuthorizedAlertsIndices(consumersOfInterest), - ]); - - const [entitiesFromAlerts = [], entitiesFromSlos] = await Promise.all([ - authorizedAlertsIndices - ? querySourcesAsEntities({ - logger, - groupings, - typeDefinitions, - esClient, - sources: [{ index: authorizedAlertsIndices }], - filters: [ - ...(filters ?? [ - { - term: { - [ALERT_STATUS]: ALERT_STATUS_ACTIVE, - }, - }, - ]), - ], - rangeQuery: { - range: { - [ALERT_TIME_RANGE]: { - gte: start, - lte: end, - }, - }, - }, - columns: { - alertsCount: { - expression: `COUNT_DISTINCT(${ALERT_UUID})`, - }, - }, - sortField: 'alertsCount', - sortOrder: 'desc', - size: 10_000, - postFilter: `WHERE alertsCount > 0`, - }) - : undefined, - querySourcesAsEntities({ - logger, - groupings, - typeDefinitions, - esClient, - sources: [{ index: sloSummaryDataScope.index }], - filters: [...(filters ?? []), sloSummaryDataScope.query], - rangeQuery: sloSummaryDataScope.query, - columns: { - healthStatus: { - expression: `CASE( - COUNT(status == "VIOLATED" OR NULL) > 0, - ${ENTITY_HEALTH_STATUS_INT.Violated}, - COUNT(status == "DEGRADED" OR NULL) > 0, - ${ENTITY_HEALTH_STATUS_INT.Degraded}, - COUNT(status == "NO_DATA" OR NULL) > 0, - ${ENTITY_HEALTH_STATUS_INT.NoData}, - COUNT(status == "HEALTHY" OR NULL) > 0, - ${ENTITY_HEALTH_STATUS_INT.Healthy}, - NULL - )`, - }, - }, - }), - ]); - - const entitiesById = new Map< - string, - IEntity & { - healthStatus: ValuesType | null; - alertsCount: number; - } - >(); - - entitiesFromAlerts.forEach((entity) => { - const existing = entitiesById.get(entity.id); - const alertsCount = entity.columns.alertsCount as number; - if (existing) { - existing.alertsCount = alertsCount; - } else { - entitiesById.set(entity.id, { - ...pick(entity, 'id', 'key', 'type', 'displayName'), - alertsCount, - healthStatus: null, - }); - } - }); - - entitiesFromSlos.forEach((entity) => { - const existing = entitiesById.get(entity.id); - const healthStatus = entity.columns.healthStatus as ValuesType< - typeof ENTITY_HEALTH_STATUS_INT - > | null; - if (existing) { - existing.healthStatus = healthStatus; - } else { - entitiesById.set(entity.id, { - ...pick(entity, 'id', 'key', 'type', 'displayName'), - alertsCount: 0, - healthStatus, - }); - } - }); - - return Array.from(entitiesById.values()); -} diff --git a/x-pack/plugins/logsai/streams_api/server/lib/entities/query_sources_as_entities.ts b/x-pack/plugins/logsai/streams_api/server/lib/entities/query_sources_as_entities.ts deleted file mode 100644 index 1c3aefeea13b9..0000000000000 --- a/x-pack/plugins/logsai/streams_api/server/lib/entities/query_sources_as_entities.ts +++ /dev/null @@ -1,309 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { Logger } from '@kbn/logging'; -import { ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; -import { isEmpty, omit, partition, pick, pickBy, uniq } from 'lodash'; -import { - DefinitionEntity, - EntityDataSource, - EntityDisplayNameTemplate, - EntityFilter, - EntityGrouping, - EntityTypeDefinition, - IEntity, -} from '../../../common/entities'; -import { escapeColumn, escapeString } from '../../../common/utils/esql_escape'; -import { esqlResultToPlainObjects } from '../../../common/utils/esql_result_to_plain_objects'; -import { getEsqlRequest } from '../../../common/utils/get_esql_request'; -import { - ENTITY_ID_SEPARATOR, - getEsqlIdentityCommands, -} from '../../routes/entities/get_esql_identity_commands'; -import { EntityLookupTable } from './entity_lookup_table'; - -interface EntityColumnMap { - [columnName: string]: - | { - expression: string; - } - | { - metadata: {}; - }; -} - -const MAX_NUMBER_OF_ENTITIES = 500; - -function getFieldsFromFilters(filters: EntityFilter[]) { - return filters.flatMap((filter) => { - if ('term' in filter) { - return [Object.keys(filter.term)[0]]; - } - return []; - }); -} - -function getLookupCommands(table: EntityLookupTable) { - const entityIdAlias = `${table.name}.entity.id`; - - const joinKeys = table.joins.map((joinField) => { - if (joinField === 'entity.id') { - return entityIdAlias; - } - return joinField; - }); - - return [ - `EVAL ${escapeColumn(entityIdAlias)} = entity.id`, - `LOOKUP ${table.name} ON ${joinKeys.map((key) => escapeColumn(key)).join(', ')}`, - ...(joinKeys.includes('entity.id') - ? [ - `EVAL entity.id = CASE( - entity.id IS NULL, ${escapeColumn(entityIdAlias)}, - ${escapeColumn(entityIdAlias)} IS NOT NULL, MV_APPEND(entity.id, ${escapeColumn( - entityIdAlias - )}), - NULL - )`, - `MV_EXPAND entity.id`, - ] - : []), - `DROP ${escapeColumn(entityIdAlias)}`, - ]; -} - -function getConcatExpressionFromDisplayNameTemplate( - displayNameTemplate: EntityDisplayNameTemplate -) { - return `CONCAT( - ${displayNameTemplate.concat - .map((part) => ('literal' in part ? escapeString(part.literal) : escapeColumn(part.field))) - .join(', ')} - )`; -} - -export async function querySourcesAsEntities< - TEntityColumnMap extends EntityColumnMap | undefined = undefined, - TLookupColumnName extends string = never ->({ - esClient, - logger, - sources, - typeDefinitions, - groupings, - columns, - rangeQuery, - filters, - postFilter, - sortField = 'entity.displayName', - sortOrder = 'asc', - size = MAX_NUMBER_OF_ENTITIES, - tables, -}: { - esClient: ObservabilityElasticsearchClient; - logger: Logger; - sources: EntityDataSource[]; - groupings: Array; - typeDefinitions: EntityTypeDefinition[]; - columns?: TEntityColumnMap; - rangeQuery: QueryDslQueryContainer; - filters?: QueryDslQueryContainer[]; - postFilter?: string; - sortField?: - | Exclude - | (keyof TEntityColumnMap & string) - | 'entity.type' - | 'entity.displayName'; - sortOrder?: 'asc' | 'desc'; - size?: number; - tables?: Array>; -}): Promise< - Array< - IEntity & { - columns: Record< - Exclude | (keyof TEntityColumnMap & string), - unknown - >; - } - > -> { - const indexPatterns = sources.flatMap((source) => source.index); - - const commands = [`FROM ${indexPatterns.join(',')} METADATA _index`]; - - const [lookupBeforeTables, lookupAfterTables] = partition( - tables, - (table) => !table.joins.includes('entity.id') - ); - - lookupBeforeTables.forEach((table) => { - commands.push(...getLookupCommands(table)); - }); - - const allGroupingFields = uniq(groupings.flatMap((grouping) => grouping.pivot.identityFields)); - - const fieldsToFilterOn = uniq( - groupings.flatMap((grouping) => getFieldsFromFilters(grouping.filters)) - ); - - const fieldCapsResponse = await esClient.fieldCaps( - 'check_column_availability_for_source_indices', - { - fields: [...allGroupingFields, ...fieldsToFilterOn, 'entity.displayName'], - index: indexPatterns, - index_filter: { - bool: { - filter: [rangeQuery], - }, - }, - } - ); - - const [validGroupings, invalidGroupings] = partition(groupings, (grouping) => { - const allFields = grouping.pivot.identityFields.concat(getFieldsFromFilters(grouping.filters)); - - return allFields.every((field) => !isEmpty(fieldCapsResponse.fields[field])); - }); - - if (invalidGroupings.length) { - logger.debug( - `Some groups were not applicable because not all fields are available: ${invalidGroupings - .map((grouping) => grouping.id) - .join(', ')}` - ); - } - - if (!validGroupings.length) { - logger.debug(`No valid groupings were applicable, returning no results`); - return []; - } - - const groupColumns = uniq([ - ...validGroupings.flatMap(({ pivot }) => pivot.identityFields), - ...lookupAfterTables.flatMap((table) => table.joins), - ]).filter((fieldName) => fieldName !== 'entity.id'); - - const hasEntityDisplayName = !isEmpty(fieldCapsResponse.fields['entity.displayName']); - - if (hasEntityDisplayName) { - commands.push(`EVAL entity.displayName = entity.displayName.keyword`); - } - - const metadataColumns = { - ...pickBy(columns, (column): column is { metadata: {} } => 'metadata' in column), - ...Object.fromEntries(fieldsToFilterOn.map((fieldName) => [fieldName, { metadata: {} }])), - ...(hasEntityDisplayName ? { 'entity.displayName': { metadata: {} } } : {}), - }; - - const expressionColumns = pickBy( - columns, - (column): column is { expression: string } => 'expression' in column - ); - - const columnStatements = Object.entries(metadataColumns) - .map(([fieldName]) => `${escapeColumn(fieldName)} = MAX(${escapeColumn(fieldName)})`) - .concat( - Object.entries(expressionColumns).map( - ([fieldName, { expression }]) => `${escapeColumn(fieldName)} = ${expression}` - ) - ); - - const columnsInFinalStatsBy = columnStatements.concat( - groupColumns.map((column) => { - return `${escapeColumn(column)} = MAX(${escapeColumn(column)})`; - }) - ); - - const identityCommands = getEsqlIdentityCommands({ - groupings, - columns: columnsInFinalStatsBy, - preaggregate: isEmpty(expressionColumns), - entityIdExists: lookupBeforeTables.length > 0, - }); - - commands.push(...identityCommands); - - lookupAfterTables.forEach((table) => { - commands.push(...getLookupCommands(table)); - }); - - typeDefinitions.forEach((typeDefinition) => { - if (typeDefinition.displayNameTemplate) { - commands.push(`EVAL entity.displayName = CASE( - entity.type == ${escapeString( - typeDefinition.pivot.type - )} AND ${typeDefinition.pivot.identityFields - .map((field) => `${escapeColumn(field)} IS NOT NULL`) - .join(' AND ')}, - ${getConcatExpressionFromDisplayNameTemplate(typeDefinition.displayNameTemplate)}, - entity.displayName - )`); - } - }); - - if (postFilter) { - commands.push(postFilter); - } - - const sortOrderUC = sortOrder.toUpperCase(); - - commands.push(`SORT \`${sortField}\` ${sortOrderUC} NULLS LAST`); - - commands.push(`LIMIT ${size}`); - - const request = { - ...getEsqlRequest({ - query: commands.join('\n| '), - dslFilter: [rangeQuery, ...(filters ?? [])], - }), - ...(tables?.length - ? { - tables: Object.fromEntries( - tables.map((table) => { - const entityIdColumn = table.columns['entity.id']; - return [ - table.name, - { - ...omit(table.columns, 'entity.id'), - [`${table.name}.entity.id`]: entityIdColumn, - }, - ]; - }) - ), - } - : {}), - }; - - const response = await esClient.esql('search_source_indices_for_entities', request); - - // should actually be a lookup to properly sort, but multiple LOOKUPs break - const groupingsByEntityId = new Map( - groupings.map((grouping) => { - const parts = [grouping.pivot.type, ENTITY_ID_SEPARATOR, grouping.id]; - const id = parts.join(''); - return [id, grouping]; - }) - ); - - return esqlResultToPlainObjects(response).map((row) => { - const columnValues = omit(row, 'entity.id', 'entity.displayName', 'entity.type'); - - const entityId = row['entity.id']; - - const grouping = groupingsByEntityId.get(entityId); - - return { - id: entityId, - type: row['entity.type'], - key: row['entity.key'], - displayName: row['entity.displayName'], - ...pick(grouping, 'displayName', 'type', 'key'), - columns: columnValues as Record, - }; - }); -} diff --git a/x-pack/plugins/logsai/streams_api/tsconfig.json b/x-pack/plugins/logsai/streams_api/tsconfig.json index 452af0f751f7e..418397a654b70 100644 --- a/x-pack/plugins/logsai/streams_api/tsconfig.json +++ b/x-pack/plugins/logsai/streams_api/tsconfig.json @@ -14,5 +14,16 @@ ], "exclude": ["target/**/*", ".storybook/**/*.js"], "kbn_references": [ + "@kbn/core", + "@kbn/server-route-repository", + "@kbn/server-route-repository-client", + "@kbn/logging", + "@kbn/rule-registry-plugin", + "@kbn/config-schema", + "@kbn/apm-utils", + "@kbn/dashboard-plugin", + "@kbn/alerting-plugin", + "@kbn/observability-utils-server", + "@kbn/licensing-plugin", ] } diff --git a/x-pack/plugins/logsai/streams_app/.storybook/get_mock_streams_app_context.tsx b/x-pack/plugins/logsai/streams_app/.storybook/get_mock_streams_app_context.tsx index afb5e7a88e066..318b4a8e2cf9b 100644 --- a/x-pack/plugins/logsai/streams_app/.storybook/get_mock_streams_app_context.tsx +++ b/x-pack/plugins/logsai/streams_app/.storybook/get_mock_streams_app_context.tsx @@ -8,7 +8,7 @@ import { coreMock } from '@kbn/core/public/mocks'; import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public'; -import type { StreamsAPIPublicStart } from '@kbn/streams-api-plugin/public'; +import type { StreamsPluginStart } from '@kbn/streams-plugin/public'; import type { ObservabilitySharedPluginStart } from '@kbn/observability-shared-plugin/public'; import type { UnifiedSearchPublicPluginStart } from '@kbn/unified-search-plugin/public'; import type { StreamsAppKibanaContext } from '../public/hooks/use_kibana'; @@ -24,9 +24,11 @@ export function getMockStreamsAppContext(): StreamsAppKibanaContext { dataViews: {} as unknown as DataViewsPublicPluginStart, data: {} as unknown as DataPublicPluginStart, unifiedSearch: {} as unknown as UnifiedSearchPublicPluginStart, - streamsAPI: {} as unknown as StreamsAPIPublicStart, + streams: {} as unknown as StreamsPluginStart, }, }, - services: {}, + services: { + query: jest.fn(), + }, }; } diff --git a/x-pack/plugins/logsai/streams_app/common/entity_source_query.ts b/x-pack/plugins/logsai/streams_app/common/entity_source_query.ts new file mode 100644 index 0000000000000..7c7176e774750 --- /dev/null +++ b/x-pack/plugins/logsai/streams_app/common/entity_source_query.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function entitySourceQuery({ entity }: { entity: Record }) { + return { + bool: { + filter: Object.entries(entity).map(([key, value]) => ({ [key]: value })), + }, + }; +} diff --git a/x-pack/plugins/logsai/streams_app/common/index.ts b/x-pack/plugins/logsai/streams_app/common/index.ts new file mode 100644 index 0000000000000..c41a05b84d307 --- /dev/null +++ b/x-pack/plugins/logsai/streams_app/common/index.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { StreamDefinition } from '@kbn/streams-plugin/common'; + +interface EntityBase { + type: string; + displayName: string; + properties: Record; +} + +export type StreamEntity = EntityBase & { type: 'stream'; properties: StreamDefinition }; + +export type Entity = StreamEntity; + +export interface EntityTypeDefinition { + displayName: string; +} diff --git a/x-pack/plugins/logsai/streams_app/kibana.jsonc b/x-pack/plugins/logsai/streams_app/kibana.jsonc index 68de86387f5fb..bfa718d7d00e3 100644 --- a/x-pack/plugins/logsai/streams_app/kibana.jsonc +++ b/x-pack/plugins/logsai/streams_app/kibana.jsonc @@ -2,21 +2,22 @@ "type": "plugin", "id": "@kbn/streams-app-plugin", "owner": "@elastic/observability-ui", + "group": "observability", + "visibility": "private", "plugin": { "id": "streamsApp", "server": true, "browser": true, "configPath": ["xpack", "streamsApp"], "requiredPlugins": [ - "streamsAPI", + "streams", "observabilityShared", "data", "dataViews", "unifiedSearch" ], "requiredBundles": [ - "kibanaReact", - "esqlDataGrid" + "kibanaReact" ], "optionalPlugins": [], "extraPublicDirs": [] diff --git a/x-pack/plugins/logsai/streams_app/public/components/all_entities_view/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/all_entities_view/index.tsx deleted file mode 100644 index ed346df834c36..0000000000000 --- a/x-pack/plugins/logsai/streams_app/public/components/all_entities_view/index.tsx +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { EuiFlexGroup } from '@elastic/eui'; -import { i18n } from '@kbn/i18n'; -import React from 'react'; -import { EntityTable } from '../entity_table'; -import { StreamsAppPageHeader } from '../streams_app_page_header'; -import { StreamsAppPageHeaderTitle } from '../streams_app_page_header/streams_app_page_header_title'; - -export function AllEntitiesView() { - return ( - - - - - - - ); -} diff --git a/x-pack/plugins/logsai/streams_app/public/components/data_stream_detail_view/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/data_stream_detail_view/index.tsx deleted file mode 100644 index c718ef52c5a16..0000000000000 --- a/x-pack/plugins/logsai/streams_app/public/components/data_stream_detail_view/index.tsx +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import React from 'react'; -import { i18n } from '@kbn/i18n'; -import { EntityDetailViewWithoutParams } from '../entity_detail_view'; -import { useStreamsAppParams } from '../../hooks/use_streams_app_params'; - -export function DataStreamDetailView() { - const { - path: { key, tab }, - } = useStreamsAppParams('/data_stream/{key}/{tab}'); - return ( - [ - { - name: 'parsing', - label: i18n.translate('xpack.entities.dataStreamDetailView.parsingTab', { - defaultMessage: 'Parsing', - }), - content: <>, - }, - ]} - /> - ); -} diff --git a/x-pack/plugins/logsai/streams_app/public/components/entity_detail_overview/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/entity_detail_overview/index.tsx deleted file mode 100644 index ae91e7fb7111d..0000000000000 --- a/x-pack/plugins/logsai/streams_app/public/components/entity_detail_overview/index.tsx +++ /dev/null @@ -1,292 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { - EuiBadge, - EuiFlexGroup, - EuiFlexItem, - EuiPanel, - EuiSuperSelect, - EuiText, - EuiTitle, -} from '@elastic/eui'; -import { css } from '@emotion/css'; -import { i18n } from '@kbn/i18n'; -import { useAbortableAsync } from '@kbn/observability-utils-browser/hooks/use_abortable_async'; -import { useDateRange } from '@kbn/observability-utils-browser/hooks/use_date_range'; -import { take, uniqueId } from 'lodash'; -import React, { useMemo, useState } from 'react'; -import { Entity, entitySourceQuery } from '@kbn/streams-api-plugin/public'; -import { EntityTypeDefinition } from '@kbn/streams-api-plugin/common/entities'; -import { useKibana } from '../../hooks/use_kibana'; -import { getInitialColumnsForLogs } from '../../util/get_initial_columns_for_logs'; -import { ControlledEsqlChart } from '../esql_chart/controlled_esql_chart'; -import { ControlledEsqlGrid } from '../esql_grid/controlled_esql_grid'; -import { StreamsAppSearchBar } from '../streams_app_search_bar'; -import { useEsqlQueryResult } from '../../hooks/use_esql_query_result'; - -export function EntityDetailOverview({ - entity, - typeDefinition, - dataStreams, -}: { - entity: Entity; - typeDefinition: EntityTypeDefinition; - dataStreams: Array<{ name: string }>; -}) { - const { - dependencies: { - start: { - dataViews, - data, - streamsAPI: { streamsAPIClient }, - }, - }, - } = useKibana(); - - const { - timeRange, - absoluteTimeRange: { start, end }, - } = useDateRange({ data }); - - const [displayedKqlFilter, setDisplayedKqlFilter] = useState(''); - const [persistedKqlFilter, setPersistedKqlFilter] = useState(''); - - const [selectedDataStream, setSelectedDataStream] = useState(''); - - const queriedDataStreams = useMemo( - () => - selectedDataStream ? [selectedDataStream] : dataStreams.map((dataStream) => dataStream.name), - [selectedDataStream, dataStreams] - ); - - const queries = useMemo(() => { - if (!queriedDataStreams.length) { - return undefined; - } - - const baseDslFilter = entitySourceQuery({ - entity, - }); - - const indexPatterns = queriedDataStreams; - - const baseQuery = `FROM ${indexPatterns.join(', ')}`; - - const logsQuery = `${baseQuery} | LIMIT 100`; - - const histogramQuery = `${baseQuery} | STATS metric = COUNT(*) BY @timestamp = BUCKET(@timestamp, 1 minute)`; - - return { - logsQuery, - histogramQuery, - baseDslFilter: [...baseDslFilter], - }; - }, [queriedDataStreams, entity]); - - const logsQueryResult = useEsqlQueryResult({ - query: queries?.logsQuery, - start, - end, - kuery: persistedKqlFilter ?? '', - dslFilter: queries?.baseDslFilter, - operationName: 'get_logs_for_entity', - }); - - const histogramQueryFetch = useAbortableAsync( - async ({ signal }) => { - if (!queries?.histogramQuery) { - return undefined; - } - - return streamsAPIClient.fetch('POST /internal/streams_api/esql', { - signal, - params: { - body: { - query: queries.histogramQuery, - kuery: persistedKqlFilter ?? '', - dslFilter: queries.baseDslFilter, - operationName: 'get_histogram_for_entity', - start, - end, - }, - }, - }); - }, - [ - queries?.histogramQuery, - persistedKqlFilter, - start, - end, - queries?.baseDslFilter, - streamsAPIClient, - ] - ); - - const columnAnalysis = useMemo(() => { - if (logsQueryResult.value) { - return { - analysis: getInitialColumnsForLogs({ - response: logsQueryResult.value, - pivots: [typeDefinition.pivot], - }), - analysisId: uniqueId(), - }; - } - return undefined; - }, [logsQueryResult, typeDefinition]); - - const dataViewsFetch = useAbortableAsync(() => { - if (!queriedDataStreams.length) { - return Promise.resolve([]); - } - - return dataViews - .create( - { - title: queriedDataStreams.join(','), - timeFieldName: '@timestamp', - }, - false, // skip fetch fields - true // display errors - ) - .then((response) => { - return [response]; - }); - }, [dataViews, queriedDataStreams]); - - const fetchedDataViews = useMemo(() => dataViewsFetch.value ?? [], [dataViewsFetch.value]); - - return ( - <> - - - - { - setDisplayedKqlFilter(query); - }} - onQuerySubmit={() => { - setPersistedKqlFilter(displayedKqlFilter); - }} - onRefresh={() => { - logsQueryResult.refresh(); - histogramQueryFetch.refresh(); - }} - placeholder={i18n.translate( - 'xpack.entities.entityDetailOverview.searchBarPlaceholder', - { - defaultMessage: 'Filter data by using KQL', - } - )} - dataViews={fetchedDataViews} - dateRangeFrom={timeRange.from} - dateRangeTo={timeRange.to} - /> - - - ({ - value: dataStream.name, - inputDisplay: dataStream.name, - })) ?? []), - ]} - valueOfSelected={selectedDataStream} - onChange={(next) => { - setSelectedDataStream(next); - }} - /> - - - - - - - - - - {columnAnalysis?.analysis.constants.length ? ( - <> - - -

- {i18n.translate('xpack.entities.entityDetailOverview.h3.constantsLabel', { - defaultMessage: 'Constants', - })} -

- - - {take(columnAnalysis.analysis.constants, 10).map((constant) => ( - {`${constant.name}:${ - constant.value === '' || constant.value === 0 ? '(empty)' : constant.value - }`} - ))} - {columnAnalysis.analysis.constants.length > 10 ? ( - - {i18n.translate('xpack.entities.entityDetailOverview.moreTextLabel', { - defaultMessage: '{overflowCount} more', - values: { - overflowCount: columnAnalysis.analysis.constants.length - 20, - }, - })} - - ) : null} - - - - ) : null} - {queries?.logsQuery ? ( - - ) : null} - - - - - ); -} diff --git a/x-pack/plugins/logsai/streams_app/public/components/entity_detail_view/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/entity_detail_view/index.tsx index e6d195b9382b7..5dc5a10209ea1 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/entity_detail_view/index.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/entity_detail_view/index.tsx @@ -4,10 +4,6 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { Entity } from '@kbn/streams-api-plugin/common'; -import React from 'react'; -import { i18n } from '@kbn/i18n'; -import { useDateRange } from '@kbn/observability-utils-browser/hooks/use_date_range'; import { EuiBadge, EuiFlexGroup, @@ -17,161 +13,81 @@ import { useEuiTheme, } from '@elastic/eui'; import { css } from '@emotion/css'; -import { useStreamsAppParams } from '../../hooks/use_streams_app_params'; -import { useKibana } from '../../hooks/use_kibana'; +import { i18n } from '@kbn/i18n'; +import React from 'react'; +import { useStreamsAppBreadcrumbs } from '../../hooks/use_streams_app_breadcrumbs'; import { useStreamsAppRouter } from '../../hooks/use_streams_app_router'; -import { useStreamsAppFetch } from '../../hooks/use_streams_app_fetch'; +import { EntityDetailViewHeaderSection } from '../entity_detail_view_header_section'; +import { EntityOverviewTabList } from '../entity_overview_tab_list'; import { LoadingPanel } from '../loading_panel'; -import { useStreamsAppBreadcrumbs } from '../../hooks/use_streams_app_breadcrumbs'; import { StreamsAppPageHeader } from '../streams_app_page_header'; import { StreamsAppPageHeaderTitle } from '../streams_app_page_header/streams_app_page_header_title'; -import { EntityDetailViewHeaderSection } from '../entity_detail_view_header_section'; -import { EntityOverviewTabList } from '../entity_overview_tab_list'; -import { EntityDetailOverview } from '../entity_detail_overview'; - -interface TabDependencies { - entity: Entity; - dataStreams: Array<{ name: string }>; -} -interface Tab { +export interface EntityViewTab { name: string; label: string; content: React.ReactElement; } export function EntityDetailViewWithoutParams({ - tab, - entityKey: key, + selectedTab, + tabs, type, - getAdditionalTabs, + entity, }: { - tab: string; - entityKey: string; - type: string; - getAdditionalTabs?: (dependencies: TabDependencies) => Tab[]; + selectedTab: string; + tabs: EntityViewTab[]; + type: { + displayName?: string; + id: string; + }; + entity: { + displayName?: string; + id: string; + }; }) { - const { - dependencies: { - start: { - data, - streamsAPI: { streamsAPIClient }, - }, - }, - services: {}, - } = useKibana(); - - const { - absoluteTimeRange: { start, end }, - } = useDateRange({ data }); - const router = useStreamsAppRouter(); const theme = useEuiTheme().euiTheme; - const entityFetch = useStreamsAppFetch( - ({ signal }) => { - return streamsAPIClient.fetch('GET /internal/streams_api/entity/{type}/{key}', { - signal, - params: { - path: { - type, - key: encodeURIComponent(key), - }, - }, - }); - }, - [type, key, streamsAPIClient] - ); - - const typeDefinition = entityFetch.value?.typeDefinition; - - const entity = entityFetch.value?.entity; - useStreamsAppBreadcrumbs(() => { - if (!typeDefinition || !entity) { + if (!type.displayName || !entity.displayName) { return []; } return [ { - title: typeDefinition.displayName, - path: `/{type}`, - params: { path: { type } }, + title: type.displayName, + path: `/`, } as const, { title: entity.displayName, - path: `/{type}/{key}`, - params: { path: { type, key } }, + path: `/{key}`, + params: { path: { key: entity.id } }, } as const, ]; - }, [type, key, entity?.displayName, typeDefinition]); + }, [type.displayName, type.id, entity.displayName, entity.id]); - const entityDataStreamsFetch = useStreamsAppFetch( - async ({ signal }) => { - return streamsAPIClient.fetch( - 'GET /internal/streams_api/entity/{type}/{key}/data_streams', - { - signal, - params: { - path: { - type, - key: encodeURIComponent(key), - }, - query: { - start: String(start), - end: String(end), - }, - }, - } - ); - }, - [key, type, streamsAPIClient, start, end] - ); - - const dataStreams = entityDataStreamsFetch.value?.dataStreams; - - if (!entity || !typeDefinition || !dataStreams) { + if (!type.displayName || !entity.displayName) { return ; } - const tabs = { - overview: { - href: router.link('/{type}/{key}/{tab}', { - path: { type, key, tab: 'overview' }, - }), - label: i18n.translate('xpack.entities.entityDetailView.overviewTabLabel', { - defaultMessage: 'Overview', - }), - content: ( - - ), - }, - ...Object.fromEntries( - getAdditionalTabs?.({ - entity, - dataStreams, - }).map(({ name, ...rest }) => [ - name, + const tabMap = Object.fromEntries( + tabs.map((tab) => { + return [ + tab.name, { - ...rest, - href: router.link(`/{type}/{key}/{tab}`, { - path: { - type, - key, - tab: name, - }, + href: router.link('/{key}/{tab}', { + path: { key: entity.id, tab: 'overview' }, }), + label: tab.label, + content: tab.content, }, - ]) ?? [] - ), - }; + ]; + }) + ); - const selectedTab = tabs[tab as keyof typeof tabs]; + const selectedTabObject = tabMap[selectedTab]; return ( @@ -202,7 +118,7 @@ export function EntityDetailViewWithoutParams({ > @@ -211,7 +127,7 @@ export function EntityDetailViewWithoutParams({ align-self: flex-start; `} > - {type} + {type.displayName} @@ -220,24 +136,16 @@ export function EntityDetailViewWithoutParams({ { + tabs={Object.entries(tabMap).map(([tabKey, { label, href }]) => { return { name: tabKey, label, href, - selected: tab === tabKey, + selected: selectedTab === tabKey, }; })} /> - {selectedTab.content} + {selectedTabObject.content} ); } - -export function EntityDetailView() { - const { - path: { type, key, tab }, - } = useStreamsAppParams('/{type}/{key}/{tab}'); - - return ; -} diff --git a/x-pack/plugins/logsai/streams_app/public/components/entity_health_status_badge/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/entity_health_status_badge/index.tsx deleted file mode 100644 index c66d444b2d695..0000000000000 --- a/x-pack/plugins/logsai/streams_app/public/components/entity_health_status_badge/index.tsx +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import React from 'react'; -import { EntityHealthStatus } from '@kbn/streams-api-plugin/common'; -import { EuiBadge } from '@elastic/eui'; -import { i18n } from '@kbn/i18n'; - -export function EntityHealthStatusBadge({ - healthStatus, -}: { - healthStatus: EntityHealthStatus | null; -}) { - if (healthStatus === 'Violated') { - return ( - - {i18n.translate('xpack.entities.healthStatus.violatedBadgeLabel', { - defaultMessage: 'Violated', - })} - - ); - } - - if (healthStatus === 'Degraded') { - return ( - - {i18n.translate('xpack.entities.healthStatus.degradedBadgeLabel', { - defaultMessage: 'Degraded', - })} - - ); - } - - if (healthStatus === 'NoData') { - return ( - - {i18n.translate('xpack.entities.healthStatus.noDataBadgeLabel', { - defaultMessage: 'No data', - })} - - ); - } - - return ( - - {i18n.translate('xpack.entities.healthStatus.healthyBadgeLabel', { - defaultMessage: 'Healthy', - })} - - ); -} diff --git a/x-pack/plugins/logsai/streams_app/public/components/entity_pivot_type_view/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/entity_pivot_type_view/index.tsx deleted file mode 100644 index 5feeddb08d4fa..0000000000000 --- a/x-pack/plugins/logsai/streams_app/public/components/entity_pivot_type_view/index.tsx +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { EuiFlexGroup } from '@elastic/eui'; -import React from 'react'; -import { EntityTable } from '../entity_table'; -import { StreamsAppPageHeader } from '../streams_app_page_header'; -import { StreamsAppPageHeaderTitle } from '../streams_app_page_header/streams_app_page_header_title'; -import { useStreamsAppParams } from '../../hooks/use_streams_app_params'; -import { useKibana } from '../../hooks/use_kibana'; -import { useStreamsAppFetch } from '../../hooks/use_streams_app_fetch'; -import { useStreamsAppBreadcrumbs } from '../../hooks/use_streams_app_breadcrumbs'; - -export function EntityPivotTypeView() { - const { - path: { type }, - } = useStreamsAppParams('/{type}'); - - const { - dependencies: { - start: { - streamsAPI: { streamsAPIClient }, - }, - }, - } = useKibana(); - - const typeDefinitionsFetch = useStreamsAppFetch( - ({ signal }) => { - return streamsAPIClient.fetch('GET /internal/streams_api/types/{type}', { - signal, - params: { - path: { - type, - }, - }, - }); - }, - [streamsAPIClient, type] - ); - - const typeDefinition = typeDefinitionsFetch.value?.typeDefinition; - - const title = typeDefinition?.displayName ?? ''; - - useStreamsAppBreadcrumbs(() => { - if (!title) { - return []; - } - return [ - { - title, - path: `/{type}`, - params: { path: { type } }, - } as const, - ]; - }, [title, type]); - - return ( - - - - - - - ); -} diff --git a/x-pack/plugins/logsai/streams_app/public/components/entity_table/controlled_entity_table.tsx b/x-pack/plugins/logsai/streams_app/public/components/entity_table/controlled_entity_table.tsx deleted file mode 100644 index f611a425701e6..0000000000000 --- a/x-pack/plugins/logsai/streams_app/public/components/entity_table/controlled_entity_table.tsx +++ /dev/null @@ -1,217 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { - CriteriaWithPagination, - EuiBadge, - EuiBasicTable, - EuiBasicTableColumn, - EuiFlexGroup, - EuiFlexItem, - EuiLink, - EuiSelect, -} from '@elastic/eui'; -import { css } from '@emotion/css'; -import type { TimeRange } from '@kbn/data-plugin/common'; -import type { DataView } from '@kbn/data-views-plugin/common'; -import { i18n } from '@kbn/i18n'; -import type { EntityWithSignalStatus } from '@kbn/streams-api-plugin/common'; -import React, { useMemo } from 'react'; -import { useStreamsAppRouter } from '../../hooks/use_streams_app_router'; -import { StreamsAppSearchBar } from '../streams_app_search_bar'; -import { EntityHealthStatusBadge } from '../entity_health_status_badge'; - -export function ControlledEntityTable({ - rows, - columns, - loading, - timeRange, - onTimeRangeChange, - kqlFilter, - onKqlFilterChange, - onKqlFilterSubmit, - pagination: { pageSize, pageIndex }, - onPaginationChange, - totalItemCount, - dataViews, - showTypeSelect, - selectedType, - availableTypes, - onSelectedTypeChange, - sort, - onSortChange, -}: { - rows: EntityWithSignalStatus[]; - columns: Array>; - kqlFilter: string; - timeRange: TimeRange; - onTimeRangeChange: (nextTimeRange: TimeRange) => void; - onKqlFilterChange: (nextKql: string) => void; - onKqlFilterSubmit: () => void; - loading: boolean; - pagination: { pageSize: number; pageIndex: number }; - onPaginationChange: (pagination: { pageSize: number; pageIndex: number }) => void; - totalItemCount: number; - dataViews?: DataView[]; - showTypeSelect?: boolean; - selectedType?: string; - onSelectedTypeChange?: (nextType: string) => void; - availableTypes?: Array<{ label: string; value: string }>; - sort?: { field: string; order: 'asc' | 'desc' }; - onSortChange?: (nextSort: { field: string; order: 'asc' | 'desc' }) => void; -}) { - const router = useStreamsAppRouter(); - - const displayedColumns = useMemo>>(() => { - return [ - { - field: 'entity.type', - name: i18n.translate('xpack.entities.entityTable.typeColumnLabel', { - defaultMessage: 'Type', - }), - width: '96px', - render: (_, { type }) => { - return {type}; - }, - }, - { - field: 'entity.displayName', - name: i18n.translate('xpack.entities.entityTable.nameColumnLabel', { - defaultMessage: 'Name', - }), - sortable: true, - render: (_, { type, key, displayName }) => { - return ( - - {displayName} - - ); - }, - }, - { - field: 'slos', - name: i18n.translate('xpack.entities.entityTable.healthStatusColumnLabel', { - defaultMessage: 'Health status', - }), - sortable: true, - width: '96px', - render: (_, { healthStatus }) => { - if (healthStatus) { - return ; - } - - return <>; - }, - }, - { - field: 'alerts', - name: i18n.translate('xpack.entities.entityTable.alertsColumnLabel', { - defaultMessage: 'Alerts', - }), - sortable: true, - width: '96px', - render: (_, { alertsCount }) => { - if (!alertsCount) { - return <>; - } - return {alertsCount}; - }, - }, - ]; - }, [router]); - - const displayedRows = useMemo( - () => rows.slice(pageIndex * pageSize, pageIndex * pageSize + pageSize), - [rows, pageIndex, pageSize] - ); - - return ( - - - - { - onKqlFilterChange(query); - }} - onQuerySubmit={({ dateRange }) => { - onKqlFilterSubmit(); - if (dateRange) { - onTimeRangeChange(dateRange); - } - }} - placeholder={i18n.translate('xpack.entities.entityTable.filterEntitiesPlaceholder', { - defaultMessage: 'Filter entities', - })} - dateRangeFrom={timeRange.from} - dateRangeTo={timeRange.to} - dataViews={dataViews} - /> - - {showTypeSelect ? ( - - { - onSelectedTypeChange?.(event.currentTarget.value); - }} - isLoading={!availableTypes} - options={availableTypes?.map(({ value, label }) => ({ value, text: label }))} - /> - - ) : null} - - - columns={displayedColumns} - items={displayedRows} - itemId="name" - pagination={{ - pageSize, - pageIndex, - totalItemCount, - }} - sorting={ - sort - ? { - sort: { - direction: sort.order, - field: sort.field as any, - }, - } - : {} - } - loading={loading} - noItemsMessage={i18n.translate('xpack.entities.entityTable.noItemsMessage', { - defaultMessage: `No entities found`, - })} - onChange={(criteria: CriteriaWithPagination) => { - const { size, index } = criteria.page; - onPaginationChange({ pageIndex: index, pageSize: size }); - if (criteria.sort) { - onSortChange?.({ - field: criteria.sort.field, - order: criteria.sort.direction, - }); - } - }} - /> - - ); -} diff --git a/x-pack/plugins/logsai/streams_app/public/components/entity_table/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/entity_table/index.tsx deleted file mode 100644 index 38fb484e4ba1b..0000000000000 --- a/x-pack/plugins/logsai/streams_app/public/components/entity_table/index.tsx +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { useDateRange } from '@kbn/observability-utils-browser/hooks/use_date_range'; -import React, { useMemo, useState } from 'react'; -import { useAbortableAsync } from '@kbn/observability-utils-browser/hooks/use_abortable_async'; -import { getIndexPatternsForFilters } from '@kbn/streams-api-plugin/public'; -import { DefinitionEntity } from '@kbn/streams-api-plugin/common/entities'; -import { useKibana } from '../../hooks/use_kibana'; -import { ControlledEntityTable } from './controlled_entity_table'; -import { useStreamsAppFetch } from '../../hooks/use_streams_app_fetch'; - -export function EntityTable({ type }: { type: 'all' | string }) { - const { - dependencies: { - start: { - dataViews, - data, - streamsAPI: { streamsAPIClient }, - }, - }, - } = useKibana(); - - const { - timeRange, - setTimeRange, - absoluteTimeRange: { start, end }, - } = useDateRange({ data }); - - const [displayedKqlFilter, setDisplayedKqlFilter] = useState(''); - const [persistedKqlFilter, setPersistedKqlFilter] = useState(''); - - const [selectedType, setSelectedType] = useState(type); - - const [sort, setSort] = useState<{ field: string; order: 'asc' | 'desc' }>({ - field: 'entity.displayName', - order: 'desc', - }); - - const typeDefinitionsFetch = useStreamsAppFetch( - ({ - signal, - }): Promise<{ - definitionEntities: DefinitionEntity[]; - }> => { - if (selectedType === 'all') { - return streamsAPIClient.fetch('GET /internal/streams_api/types', { - signal, - }); - } - return streamsAPIClient.fetch('GET /internal/streams_api/types/{type}', { - signal, - params: { - path: { - type: selectedType, - }, - }, - }); - }, - [streamsAPIClient, selectedType] - ); - - const queryFetch = useStreamsAppFetch( - ({ signal }) => { - return streamsAPIClient.fetch('POST /internal/streams_api/entities', { - signal, - params: { - body: { - start, - end, - kuery: persistedKqlFilter, - types: [selectedType], - sortField: sort.field as any, - sortOrder: sort.order, - }, - }, - }); - }, - [streamsAPIClient, selectedType, persistedKqlFilter, start, end, sort.field, sort.order] - ); - - const [pagination, setPagination] = useState<{ pageSize: number; pageIndex: number }>({ - pageSize: 10, - pageIndex: 0, - }); - - const entities = useMemo(() => { - return queryFetch.value?.entities ?? []; - }, [queryFetch.value]); - - const dataViewsFetch = useAbortableAsync(() => { - if (!typeDefinitionsFetch.value) { - return undefined; - } - - const allIndexPatterns = typeDefinitionsFetch.value.definitionEntities.flatMap((definition) => - getIndexPatternsForFilters(definition.filters) - ); - - return dataViews - .create( - { - title: allIndexPatterns.join(', '), - timeFieldName: '@timestamp', - }, - false, // skip fetch fields - true // display errors - ) - .then((response) => { - return [response]; - }); - }, [dataViews, typeDefinitionsFetch.value]); - - return ( - { - setTimeRange(nextTimeRange); - }} - rows={entities} - loading={queryFetch.loading} - kqlFilter={displayedKqlFilter} - onKqlFilterChange={(next) => { - setDisplayedKqlFilter(next); - }} - onKqlFilterSubmit={() => { - setPersistedKqlFilter(displayedKqlFilter); - }} - onPaginationChange={(next) => { - setPagination(next); - }} - pagination={pagination} - totalItemCount={entities.length} - columns={[]} - dataViews={dataViewsFetch.value} - showTypeSelect={type === 'all'} - onSelectedTypeChange={(nextType) => { - setSelectedType(nextType); - }} - onSortChange={(nextSort) => { - setSort(nextSort); - }} - sort={sort} - /> - ); -} diff --git a/x-pack/plugins/logsai/streams_app/public/components/esql_chart/controlled_esql_chart.tsx b/x-pack/plugins/logsai/streams_app/public/components/esql_chart/controlled_esql_chart.tsx index aea3b7f577479..0f10325c90234 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/esql_chart/controlled_esql_chart.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/esql_chart/controlled_esql_chart.tsx @@ -28,7 +28,7 @@ import { esqlResultToTimeseries } from '../../util/esql_result_to_timeseries'; import { useKibana } from '../../hooks/use_kibana'; import { LoadingPanel } from '../loading_panel'; -const END_ZONE_LABEL = i18n.translate('xpack.entities.esqlChart.endzone', { +const END_ZONE_LABEL = i18n.translate('xpack.streams.esqlChart.endzone', { defaultMessage: 'The selected time range does not include this entire bucket. It might contain partial data.', }); diff --git a/x-pack/plugins/logsai/streams_app/public/components/esql_chart/uncontrolled_esql_chart.tsx b/x-pack/plugins/logsai/streams_app/public/components/esql_chart/uncontrolled_esql_chart.tsx deleted file mode 100644 index ebf0b3f16a630..0000000000000 --- a/x-pack/plugins/logsai/streams_app/public/components/esql_chart/uncontrolled_esql_chart.tsx +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import React from 'react'; -import { useEsqlQueryResult } from '../../hooks/use_esql_query_result'; -import { ControlledEsqlChart } from './controlled_esql_chart'; - -export function UncontrolledEsqlChart({ - id, - query, - metricNames, - height, - start, - end, -}: { - id: string; - query: string; - metricNames: T[]; - height: number; - start: number; - end: number; -}) { - const result = useEsqlQueryResult({ query, start, end, operationName: 'visualize' }); - - return ; -} diff --git a/x-pack/plugins/logsai/streams_app/public/components/esql_grid/controlled_esql_grid.tsx b/x-pack/plugins/logsai/streams_app/public/components/esql_grid/controlled_esql_grid.tsx deleted file mode 100644 index 7957d99af796b..0000000000000 --- a/x-pack/plugins/logsai/streams_app/public/components/esql_grid/controlled_esql_grid.tsx +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import React, { useMemo } from 'react'; -import { i18n } from '@kbn/i18n'; -import { ESQLDataGrid } from '@kbn/esql-datagrid/public'; -import { - AbortableAsyncState, - useAbortableAsync, -} from '@kbn/observability-utils-browser/hooks/use_abortable_async'; -import { getESQLAdHocDataview } from '@kbn/esql-utils'; -import { EuiCallOut } from '@elastic/eui'; -import { ESQLSearchResponse } from '@kbn/es-types'; -import { DatatableColumn } from '@kbn/expressions-plugin/common'; -import { LoadingPanel } from '../loading_panel'; -import { useKibana } from '../../hooks/use_kibana'; - -export function ControlledEsqlGrid({ - query, - result, - initialColumns, - analysisId, -}: { - query: string; - result: AbortableAsyncState; - initialColumns?: ESQLSearchResponse['columns']; - analysisId?: string; -}) { - const { - dependencies: { - start: { dataViews }, - }, - } = useKibana(); - - const response = result.value; - - const dataViewAsync = useAbortableAsync(() => { - return getESQLAdHocDataview(query, dataViews); - }, [query, dataViews]); - - const datatableColumns = useMemo(() => { - return ( - response?.columns.map((column): DatatableColumn => { - return { - id: column.name, - meta: { - type: 'string', - esType: column.type, - }, - name: column.name, - }; - }) ?? [] - ); - }, [response?.columns]); - - const initialDatatableColumns = useMemo(() => { - if (!initialColumns) { - return undefined; - } - - const initialColumnNames = new Set([...initialColumns.map((column) => column.name)]); - return datatableColumns.filter((column) => initialColumnNames.has(column.name)); - }, [datatableColumns, initialColumns]); - - if (!dataViewAsync.value || !response) { - return ; - } - - if (!result.loading && !result.error && !response.values.length) { - return ( - - {i18n.translate('xpack.entities.controlledEsqlGrid.noResultsCallOutLabel', { - defaultMessage: 'No results', - })} - - ); - } - - return ( - - ); -} diff --git a/x-pack/plugins/logsai/streams_app/public/components/not_found/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/not_found/index.tsx new file mode 100644 index 0000000000000..c0c1d50000cfc --- /dev/null +++ b/x-pack/plugins/logsai/streams_app/public/components/not_found/index.tsx @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { EuiCallOut } from '@elastic/eui'; +import React from 'react'; +import { i18n } from '@kbn/i18n'; + +export function NotFound() { + return ( + + {i18n.translate('xpack.streams.notFound.calloutLabel', { + defaultMessage: 'The current page can not be found.', + })} + + ); +} diff --git a/x-pack/plugins/logsai/streams_app/public/components/stream_detail_overview/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/stream_detail_overview/index.tsx new file mode 100644 index 0000000000000..c1d91b6440b5d --- /dev/null +++ b/x-pack/plugins/logsai/streams_app/public/components/stream_detail_overview/index.tsx @@ -0,0 +1,155 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { EuiFlexGroup, EuiFlexItem, EuiPanel } from '@elastic/eui'; +import { calculateAuto } from '@kbn/calculate-auto'; +import { i18n } from '@kbn/i18n'; +import { useAbortableAsync } from '@kbn/observability-utils-browser/hooks/use_abortable_async'; +import { useDateRange } from '@kbn/observability-utils-browser/hooks/use_date_range'; +import moment from 'moment'; +import React, { useMemo, useState } from 'react'; +import { StreamDefinition } from '@kbn/streams-plugin/common'; +import { entitySourceQuery } from '../../../common/entity_source_query'; +import { useKibana } from '../../hooks/use_kibana'; +import { ControlledEsqlChart } from '../esql_chart/controlled_esql_chart'; +import { StreamsAppSearchBar } from '../streams_app_search_bar'; + +export function StreamDetailOverview({ definition }: { definition?: StreamDefinition }) { + const { + dependencies: { + start: { + data, + dataViews, + streams: { streamsRepositoryClient }, + }, + }, + } = useKibana(); + + const { + timeRange, + absoluteTimeRange: { start, end }, + } = useDateRange({ data }); + + const [displayedKqlFilter, setDisplayedKqlFilter] = useState(''); + const [persistedKqlFilter, setPersistedKqlFilter] = useState(''); + + const dataStream = definition?.id; + + const queries = useMemo(() => { + if (!dataStream) { + return undefined; + } + + const baseDslFilter = entitySourceQuery({ + entity: { + _index: dataStream, + }, + }); + + const indexPatterns = [dataStream]; + + const baseQuery = `FROM ${indexPatterns.join(', ')}`; + + const bucketSize = calculateAuto.atLeast(50, moment.duration(1, 'minute'))!.asMinutes(); + + const histogramQuery = `${baseQuery} | STATS metric = COUNT(*) BY @timestamp = BUCKET(@timestamp, ${bucketSize} minutes)`; + + return { + histogramQuery, + baseDslFilter, + }; + }, [dataStream]); + + const histogramQueryFetch = useAbortableAsync( + async ({ signal }) => { + if (!queries?.histogramQuery) { + return undefined; + } + + return streamsRepositoryClient.fetch('POST /internal/streams/esql', { + params: { + body: { + operationName: 'get_histogram_for_stream', + query: queries.histogramQuery, + filter: queries.baseDslFilter, + kuery: persistedKqlFilter, + start, + end, + }, + }, + signal, + }); + }, + [ + streamsRepositoryClient, + queries?.histogramQuery, + persistedKqlFilter, + start, + end, + queries?.baseDslFilter, + ] + ); + + const dataViewsFetch = useAbortableAsync(() => { + return dataViews + .create( + { + title: dataStream, + timeFieldName: '@timestamp', + }, + false, // skip fetch fields + true // display errors + ) + .then((response) => { + return [response]; + }); + }, [dataViews, dataStream]); + + const fetchedDataViews = useMemo(() => dataViewsFetch.value ?? [], [dataViewsFetch.value]); + + return ( + <> + + + + { + setDisplayedKqlFilter(nextQuery); + }} + onQuerySubmit={() => { + setPersistedKqlFilter(displayedKqlFilter); + }} + onRefresh={() => { + histogramQueryFetch.refresh(); + }} + placeholder={i18n.translate( + 'xpack.streams.entityDetailOverview.searchBarPlaceholder', + { + defaultMessage: 'Filter data by using KQL', + } + )} + dataViews={fetchedDataViews} + dateRangeFrom={timeRange.from} + dateRangeTo={timeRange.to} + /> + + + + + + + + + + ); +} diff --git a/x-pack/plugins/logsai/streams_app/public/components/stream_detail_view/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/stream_detail_view/index.tsx new file mode 100644 index 0000000000000..ca479b0fab414 --- /dev/null +++ b/x-pack/plugins/logsai/streams_app/public/components/stream_detail_view/index.tsx @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React from 'react'; +import { i18n } from '@kbn/i18n'; +import { EntityDetailViewWithoutParams, EntityViewTab } from '../entity_detail_view'; +import { useStreamsAppParams } from '../../hooks/use_streams_app_params'; +import { useStreamsAppFetch } from '../../hooks/use_streams_app_fetch'; +import { useKibana } from '../../hooks/use_kibana'; +import { StreamDetailOverview } from '../stream_detail_overview'; + +export function StreamDetailView() { + const { + path: { key, tab }, + } = useStreamsAppParams('/{key}/{tab}'); + + const { + dependencies: { + start: { + streams: { streamsRepositoryClient }, + }, + }, + } = useKibana(); + + const { value: streamEntity } = useStreamsAppFetch( + ({ signal }) => { + return streamsRepositoryClient.fetch('GET /api/streams/{id} 2023-10-31', { + signal, + params: { + path: { + id: key, + }, + }, + }); + }, + [streamsRepositoryClient, key] + ); + + const type = { + id: 'stream', + displayName: i18n.translate('xpack.streams.streamDetailView.streamsTypeDisplayName', { + defaultMessage: 'Streams', + }), + }; + + const entity = { + id: key, + displayName: key, + }; + + const tabs: EntityViewTab[] = [ + { + name: 'overview', + content: , + label: i18n.translate('xpack.streams.streamDetailView.parsingTab', { + defaultMessage: 'Parsing', + }), + }, + ]; + + return ( + + ); +} diff --git a/x-pack/plugins/logsai/streams_app/public/components/stream_list_view/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/stream_list_view/index.tsx new file mode 100644 index 0000000000000..2e3e5b819e5b9 --- /dev/null +++ b/x-pack/plugins/logsai/streams_app/public/components/stream_list_view/index.tsx @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React from 'react'; +import { useKibana } from '../../hooks/use_kibana'; +import { useStreamsAppFetch } from '../../hooks/use_streams_app_fetch'; + +export function StreamListView() { + const { + dependencies: { + start: { + streams: { streamsRepositoryClient }, + }, + }, + } = useKibana(); + + const queryFetch = useStreamsAppFetch( + ({ signal }) => { + return streamsRepositoryClient.fetch('GET /api/streams 2023-10-31', { + signal, + }); + }, + [streamsRepositoryClient] + ); + + return <>; +} diff --git a/x-pack/plugins/logsai/streams_app/public/hooks/use_esql_query_result.ts b/x-pack/plugins/logsai/streams_app/public/hooks/use_esql_query_result.ts deleted file mode 100644 index 853f4bc3a41f3..0000000000000 --- a/x-pack/plugins/logsai/streams_app/public/hooks/use_esql_query_result.ts +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { useAbortableAsync } from '@kbn/observability-utils-browser/hooks/use_abortable_async'; -import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; -import { useKibana } from './use_kibana'; - -export function useEsqlQueryResult({ - query, - kuery, - start, - end, - operationName, - dslFilter, -}: { - query?: string; - kuery?: string; - start: number; - end: number; - operationName: string; - dslFilter?: QueryDslQueryContainer[]; -}) { - const { - dependencies: { - start: { - streamsAPI: { streamsAPIClient }, - }, - }, - } = useKibana(); - - return useAbortableAsync( - ({ signal }) => { - if (!query) { - return undefined; - } - return streamsAPIClient.fetch('POST /internal/streams_api/esql', { - signal, - params: { - body: { - query, - start, - end, - kuery: kuery ?? '', - operationName, - dslFilter, - }, - }, - }); - }, - [streamsAPIClient, query, start, end, kuery, operationName, dslFilter] - ); -} diff --git a/x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_fetch.ts b/x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_fetch.ts index 6632fced59792..08b112d4f207a 100644 --- a/x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_fetch.ts +++ b/x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_fetch.ts @@ -47,7 +47,7 @@ export const useStreamsAppFetch: UseAbortableAsync<{}, { disableToastOnError?: b } notifications.toasts.addError(error, { - title: i18n.translate('xpack.entities.failedToFetchError', { + title: i18n.translate('xpack.streams.failedToFetchError', { defaultMessage: 'Failed to fetch data{requestUrlSuffix}', values: { requestUrlSuffix: requestUrl ? ` (${requestUrl})` : '', diff --git a/x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_router.ts b/x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_router.ts index 37a9386af1d05..17472044b7b4d 100644 --- a/x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_router.ts +++ b/x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_router.ts @@ -40,14 +40,14 @@ export function useStreamsAppRouter(): StatefulStreamsAppRouter { ...streamsAppRouter, push: (...args) => { const next = link(...args); - navigateToApp('entities', { path: next, replace: false }); + navigateToApp('streams', { path: next, replace: false }); }, replace: (path, ...args) => { const next = link(path, ...args); - navigateToApp('entities', { path: next, replace: true }); + navigateToApp('streams', { path: next, replace: true }); }, link: (path, ...args) => { - return http.basePath.prepend('/app/entities' + link(path, ...args)); + return http.basePath.prepend('/app/streams' + link(path, ...args)); }, }), [navigateToApp, http.basePath] diff --git a/x-pack/plugins/logsai/streams_app/public/plugin.ts b/x-pack/plugins/logsai/streams_app/public/plugin.ts index 37e01c37a954d..50d0e4cf88ac6 100644 --- a/x-pack/plugins/logsai/streams_app/public/plugin.ts +++ b/x-pack/plugins/logsai/streams_app/public/plugin.ts @@ -16,7 +16,7 @@ import { PluginInitializerContext, } from '@kbn/core/public'; import type { Logger } from '@kbn/logging'; -import { ENTITY_APP_ID } from '@kbn/deeplinks-observability/constants'; +import { STREAMS_APP_ID } from '@kbn/deeplinks-observability/constants'; import type { ConfigSchema, StreamsAppPublicSetup, @@ -53,10 +53,10 @@ export class StreamsAppPlugin sortKey: 101, entries: [ { - label: i18n.translate('xpack.entities.streamsAppLinkTitle', { - defaultMessage: 'Entities', + label: i18n.translate('xpack.streams.streamsAppLinkTitle', { + defaultMessage: 'Streams', }), - app: ENTITY_APP_ID, + app: STREAMS_APP_ID, path: '/', matchPath(currentPath: string) { return ['/', ''].some((testPath) => currentPath.startsWith(testPath)); @@ -70,20 +70,20 @@ export class StreamsAppPlugin ); coreSetup.application.register({ - id: ENTITY_APP_ID, - title: i18n.translate('xpack.entities.appTitle', { - defaultMessage: 'Entities', + id: STREAMS_APP_ID, + title: i18n.translate('xpack.streams.appTitle', { + defaultMessage: 'Streams', }), euiIconType: 'logoObservability', - appRoute: '/app/entities', + appRoute: '/app/streams', category: DEFAULT_APP_CATEGORIES.observability, visibleIn: ['sideNav'], order: 8001, deepLinks: [ { - id: 'entities', - title: i18n.translate('xpack.entities.streamsAppDeepLinkTitle', { - defaultMessage: 'Entities', + id: 'streams', + title: i18n.translate('xpack.streams.streamsAppDeepLinkTitle', { + defaultMessage: 'Streams', }), path: '/', }, diff --git a/x-pack/plugins/logsai/streams_app/public/routes/config.tsx b/x-pack/plugins/logsai/streams_app/public/routes/config.tsx index a27a9d8c657a9..e3efdc6d871e7 100644 --- a/x-pack/plugins/logsai/streams_app/public/routes/config.tsx +++ b/x-pack/plugins/logsai/streams_app/public/routes/config.tsx @@ -8,13 +8,11 @@ import { i18n } from '@kbn/i18n'; import { createRouter, Outlet, RouteMap } from '@kbn/typed-react-router-config'; import * as t from 'io-ts'; import React from 'react'; -import { AllEntitiesView } from '../components/all_entities_view'; -import { EntityDetailView } from '../components/entity_detail_view'; +import { StreamDetailView } from '../components/stream_detail_view'; import { StreamsAppPageTemplate } from '../components/streams_app_page_template'; import { StreamsAppRouterBreadcrumb } from '../components/streams_app_router_breadcrumb'; import { RedirectTo } from '../components/redirect_to'; -import { DataStreamDetailView } from '../components/data_stream_detail_view'; -import { EntityPivotTypeView } from '../components/entity_pivot_type_view'; +import { StreamListView } from '../components/stream_list_view'; /** * The array of route definitions to be used when the application @@ -24,8 +22,8 @@ const streamsAppRoutes = { '/': { element: ( @@ -35,19 +33,7 @@ const streamsAppRoutes = { ), children: { - '/all': { - element: ( - - - - ), - }, - '/data_stream/{key}': { + '/{key}': { element: , params: t.type({ path: t.type({ @@ -55,13 +41,11 @@ const streamsAppRoutes = { }), }), children: { - '/data_stream/{key}': { - element: ( - - ), + '/{key}': { + element: , }, - '/data_stream/{key}/{tab}': { - element: , + '/{key}/{tab}': { + element: , params: t.type({ path: t.type({ tab: t.string, @@ -70,38 +54,8 @@ const streamsAppRoutes = { }, }, }, - '/{type}': { - element: , - params: t.type({ - path: t.type({ type: t.string }), - }), - children: { - '/{type}': { - element: , - }, - '/{type}/{key}': { - params: t.type({ - path: t.type({ key: t.string }), - }), - element: , - children: { - '/{type}/{key}': { - element: ( - - ), - }, - '/{type}/{key}/{tab}': { - element: , - params: t.type({ - path: t.type({ tab: t.string }), - }), - }, - }, - }, - }, - }, '/': { - element: , + element: , }, }, }, diff --git a/x-pack/plugins/logsai/streams_app/public/types.ts b/x-pack/plugins/logsai/streams_app/public/types.ts index ef8f30c47dd75..9f9f6147bea49 100644 --- a/x-pack/plugins/logsai/streams_app/public/types.ts +++ b/x-pack/plugins/logsai/streams_app/public/types.ts @@ -4,41 +4,34 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import type { - StreamsAPIPublicSetup, - StreamsAPIPublicStart, -} from '@kbn/streams-api-plugin/public'; -import type { - ObservabilitySharedPluginSetup, - ObservabilitySharedPluginStart, -} from '@kbn/observability-shared-plugin/public'; import type { DataPublicPluginSetup, DataPublicPluginStart } from '@kbn/data-plugin/public'; import type { DataViewsPublicPluginSetup, DataViewsPublicPluginStart, } from '@kbn/data-views-plugin/public'; import type { - UnifiedSearchPluginSetup, - UnifiedSearchPublicPluginStart, -} from '@kbn/unified-search-plugin/public'; - + ObservabilitySharedPluginSetup, + ObservabilitySharedPluginStart, +} from '@kbn/observability-shared-plugin/public'; +import type { StreamsPluginSetup, StreamsPluginStart } from '@kbn/streams-plugin/public'; +import type { UnifiedSearchPublicPluginStart } from '@kbn/unified-search-plugin/public'; /* eslint-disable @typescript-eslint/no-empty-interface*/ export interface ConfigSchema {} export interface StreamsAppSetupDependencies { - observabilityShared: ObservabilitySharedPluginSetup; - streamsAPI: StreamsAPIPublicSetup; + streams: StreamsPluginSetup; data: DataPublicPluginSetup; dataViews: DataViewsPublicPluginSetup; - unifiedSearch: UnifiedSearchPluginSetup; + observabilityShared: ObservabilitySharedPluginSetup; + unifiedSearch: {}; } export interface StreamsAppStartDependencies { - observabilityShared: ObservabilitySharedPluginStart; - streamsAPI: StreamsAPIPublicStart; + streams: StreamsPluginStart; data: DataPublicPluginStart; dataViews: DataViewsPublicPluginStart; + observabilityShared: ObservabilitySharedPluginStart; unifiedSearch: UnifiedSearchPublicPluginStart; } diff --git a/x-pack/plugins/logsai/streams_app/public/util/get_initial_columns_for_logs.ts b/x-pack/plugins/logsai/streams_app/public/util/get_initial_columns_for_logs.ts deleted file mode 100644 index 209766f3e99e1..0000000000000 --- a/x-pack/plugins/logsai/streams_app/public/util/get_initial_columns_for_logs.ts +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { isArray } from 'lodash'; -import type { ESQLSearchResponse } from '@kbn/es-types'; -import { Pivot } from '@kbn/streams-api-plugin/common'; - -type Column = ESQLSearchResponse['columns'][number]; - -interface ColumnExtraction { - constants: Array<{ name: string; value: unknown }>; - initialColumns: Column[]; -} - -function analyzeColumnValues(response: ESQLSearchResponse): Array<{ - name: string; - unique: boolean; - constant: boolean; - empty: boolean; - index: number; - column: Column; -}> { - return response.columns.map((column, index) => { - const values = new Set(); - for (const row of response.values) { - const val = row[index]; - values.add(isArray(val) ? val.map(String).join(',') : val); - } - return { - name: column.name, - unique: values.size === response.values.length, - constant: values.size === 1, - empty: Array.from(values.values()).every((value) => !value), - index, - column, - }; - }); -} - -export function getInitialColumnsForLogs({ - response, - pivots, -}: { - response: ESQLSearchResponse; - pivots: Pivot[]; -}): ColumnExtraction { - const analyzedColumns = analyzeColumnValues(response); - - const withoutUselessColumns = analyzedColumns.filter(({ column, empty, constant, unique }) => { - return empty === false && constant === false && !(column.type === 'keyword' && unique); - }); - - const constantColumns = analyzedColumns.filter(({ constant }) => constant); - - const timestampColumnIndex = withoutUselessColumns.findIndex( - (column) => column.name === '@timestamp' - ); - - const messageColumnIndex = withoutUselessColumns.findIndex( - (column) => column.name === 'message' || column.name === 'msg' - ); - - const initialColumns = new Set(); - - if (timestampColumnIndex !== -1) { - initialColumns.add(withoutUselessColumns[timestampColumnIndex].column); - } - - if (messageColumnIndex !== -1) { - initialColumns.add(withoutUselessColumns[messageColumnIndex].column); - } - - const allIdentityFields = new Set([...pivots.flatMap((pivot) => pivot.identityFields)]); - - const columnsWithIdentityFields = analyzedColumns.filter((column) => - allIdentityFields.has(column.name) - ); - const columnsInOrderOfPreference = [ - ...columnsWithIdentityFields, - ...withoutUselessColumns, - ...constantColumns, - ]; - - for (const { column } of columnsInOrderOfPreference) { - if (initialColumns.size <= 8) { - initialColumns.add(column); - } else { - break; - } - } - - const constants = constantColumns.map(({ name, index, column }) => { - return { name, value: response.values[0][index] }; - }); - - return { - initialColumns: Array.from(initialColumns.values()), - constants, - }; -} diff --git a/x-pack/plugins/logsai/streams_app/server/types.ts b/x-pack/plugins/logsai/streams_app/server/types.ts index 0425b145b2746..e425ae7422d75 100644 --- a/x-pack/plugins/logsai/streams_app/server/types.ts +++ b/x-pack/plugins/logsai/streams_app/server/types.ts @@ -4,13 +4,19 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import type { StreamsPluginStart, StreamsPluginSetup } from '@kbn/streams-plugin/server'; + /* eslint-disable @typescript-eslint/no-empty-interface*/ export interface ConfigSchema {} -export interface StreamsAppSetupDependencies {} +export interface StreamsAppSetupDependencies { + streams: StreamsPluginSetup; +} -export interface StreamsAppStartDependencies {} +export interface StreamsAppStartDependencies { + streams: StreamsPluginStart; +} export interface StreamsAppServerSetup {} diff --git a/x-pack/plugins/logsai/streams_app/tsconfig.json b/x-pack/plugins/logsai/streams_app/tsconfig.json index 452af0f751f7e..2a3bdffecab9d 100644 --- a/x-pack/plugins/logsai/streams_app/tsconfig.json +++ b/x-pack/plugins/logsai/streams_app/tsconfig.json @@ -14,5 +14,24 @@ ], "exclude": ["target/**/*", ".storybook/**/*.js"], "kbn_references": [ + "@kbn/core", + "@kbn/data-plugin", + "@kbn/data-views-plugin", + "@kbn/streams-api-plugin", + "@kbn/observability-shared-plugin", + "@kbn/unified-search-plugin", + "@kbn/react-kibana-context-render", + "@kbn/shared-ux-link-redirect-app", + "@kbn/typed-react-router-config", + "@kbn/i18n", + "@kbn/observability-utils-browser", + "@kbn/es-types", + "@kbn/kibana-react-plugin", + "@kbn/es-query", + "@kbn/logging", + "@kbn/deeplinks-observability", + "@kbn/config-schema", + "@kbn/calculate-auto", + "@kbn/streams-plugin", ] } diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.ts b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.ts index cf376e7c78294..9f04bb9a750f3 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.ts @@ -23,7 +23,7 @@ import { ValuesType } from 'utility-types'; import type { APMError, Metric, Span, Transaction, Event } from '@kbn/apm-types/es_schemas_ui'; import type { InspectResponse } from '@kbn/observability-plugin/typings/common'; import type { DataTier } from '@kbn/observability-shared-plugin/common'; -import { excludeTiersQuery } from '@kbn/observability-utils/es/queries/exclude_tiers_query'; +import { excludeTiersQuery } from '@kbn/observability-utils-common/es/queries/exclude_tiers_query'; import { withApmSpan } from '../../../../utils'; import type { ApmDataSource } from '../../../../../common/data_source'; import { cancelEsRequestOnAbort } from '../cancel_es_request_on_abort'; diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts index cad0b03579e3d..c1f8d5e3fce1f 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts @@ -6,7 +6,7 @@ */ import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { DataTier } from '@kbn/observability-shared-plugin/common'; -import { excludeTiersQuery } from '@kbn/observability-utils/es/queries/exclude_tiers_query'; +import { excludeTiersQuery } from '@kbn/observability-utils-common/es/queries/exclude_tiers_query'; export function getDataTierFilterCombined({ filter, diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts b/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts index b9a4322269828..6c9fe4c39b001 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts @@ -8,8 +8,8 @@ import type { DedotObject } from '@kbn/utility-types'; import * as APM_EVENT_FIELDS_MAP from '@kbn/apm-types/es_fields'; import type { ValuesType } from 'utility-types'; -import { unflattenObject } from '@kbn/observability-utils/object/unflatten_object'; -import { mergePlainObjects } from '@kbn/observability-utils/object/merge_plain_objects'; +import { unflattenObject } from '@kbn/observability-utils-common/object/unflatten_object'; +import { mergePlainObjects } from '@kbn/observability-utils-common/object/merge_plain_objects'; import { castArray, isArray } from 'lodash'; import { AgentName } from '@kbn/elastic-agent-utils'; import { EventOutcome } from '@kbn/apm-types/src/es_schemas/raw/fields'; diff --git a/x-pack/plugins/observability_solution/apm_data_access/tsconfig.json b/x-pack/plugins/observability_solution/apm_data_access/tsconfig.json index aeeb73bee2857..bc8ecf9222121 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/tsconfig.json +++ b/x-pack/plugins/observability_solution/apm_data_access/tsconfig.json @@ -20,7 +20,6 @@ "@kbn/apm-utils", "@kbn/core-http-server", "@kbn/security-plugin-types-server", - "@kbn/observability-utils", "@kbn/utility-types", "@kbn/elastic-agent-utils" ] diff --git a/x-pack/plugins/observability_solution/infra/server/routes/entities/get_latest_entity.ts b/x-pack/plugins/observability_solution/infra/server/routes/entities/get_latest_entity.ts index 7bcce2964fd13..728925e5e067b 100644 --- a/x-pack/plugins/observability_solution/infra/server/routes/entities/get_latest_entity.ts +++ b/x-pack/plugins/observability_solution/infra/server/routes/entities/get_latest_entity.ts @@ -11,8 +11,8 @@ import { ENTITY_TYPE, SOURCE_DATA_STREAM_TYPE, } from '@kbn/observability-shared-plugin/common/field_names/elasticsearch'; -import type { ObservabilityElasticsearchClient } from '@kbn/observability-utils/es/client/create_observability_es_client'; -import { esqlResultToPlainObjects } from '@kbn/observability-utils/es/utils/esql_result_to_plain_objects'; +import type { ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; +import { esqlResultToPlainObjects } from '@kbn/observability-utils-server/es/esql_result_to_plain_objects'; const ENTITIES_LATEST_ALIAS = entitiesAliasPattern({ type: '*', diff --git a/x-pack/plugins/observability_solution/infra/server/routes/entities/index.ts b/x-pack/plugins/observability_solution/infra/server/routes/entities/index.ts index cb169f83f171d..7fdba72f2fed5 100644 --- a/x-pack/plugins/observability_solution/infra/server/routes/entities/index.ts +++ b/x-pack/plugins/observability_solution/infra/server/routes/entities/index.ts @@ -8,7 +8,7 @@ import { schema } from '@kbn/config-schema'; import { METRICS_APP_ID } from '@kbn/deeplinks-observability/constants'; import { entityCentricExperience } from '@kbn/observability-plugin/common'; -import { createObservabilityEsClient } from '@kbn/observability-utils/es/client/create_observability_es_client'; +import { createObservabilityEsClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; import { getInfraMetricsClient } from '../../lib/helpers/get_infra_metrics_client'; import { InfraBackendLibs } from '../../lib/infra_types'; import { getDataStreamTypes } from './get_data_stream_types'; diff --git a/x-pack/plugins/observability_solution/infra/tsconfig.json b/x-pack/plugins/observability_solution/infra/tsconfig.json index 2103350048e4b..950300e4f2bb1 100644 --- a/x-pack/plugins/observability_solution/infra/tsconfig.json +++ b/x-pack/plugins/observability_solution/infra/tsconfig.json @@ -114,7 +114,6 @@ "@kbn/management-settings-ids", "@kbn/core-ui-settings-common", "@kbn/entityManager-plugin", - "@kbn/observability-utils", "@kbn/entities-schema", "@kbn/zod" ], diff --git a/x-pack/plugins/observability_solution/inventory/public/hooks/use_entity_manager.ts b/x-pack/plugins/observability_solution/inventory/public/hooks/use_entity_manager.ts index 1082017e1ad7a..740c88eb8a9b0 100644 --- a/x-pack/plugins/observability_solution/inventory/public/hooks/use_entity_manager.ts +++ b/x-pack/plugins/observability_solution/inventory/public/hooks/use_entity_manager.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { useAbortableAsync } from '@kbn/observability-utils/hooks/use_abortable_async'; +import { useAbortableAsync } from '@kbn/observability-utils-browser/hooks/use_abortable_async'; import { useState } from 'react'; import { useKibana } from './use_kibana'; diff --git a/x-pack/plugins/observability_solution/inventory/public/hooks/use_inventory_abortable_async.ts b/x-pack/plugins/observability_solution/inventory/public/hooks/use_inventory_abortable_async.ts index 84cef842488e0..1db3b512bbdd6 100644 --- a/x-pack/plugins/observability_solution/inventory/public/hooks/use_inventory_abortable_async.ts +++ b/x-pack/plugins/observability_solution/inventory/public/hooks/use_inventory_abortable_async.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { useAbortableAsync } from '@kbn/observability-utils/hooks/use_abortable_async'; +import { useAbortableAsync } from '@kbn/observability-utils-browser/hooks/use_abortable_async'; import { i18n } from '@kbn/i18n'; import { IHttpFetchError, ResponseErrorBody } from '@kbn/core-http-browser'; import { useKibana } from './use_kibana'; diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_groups.ts b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_groups.ts index b61f245f1aaf2..3d4394e1efd41 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_groups.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_groups.ts @@ -5,9 +5,9 @@ * 2.0. */ -import type { ObservabilityElasticsearchClient } from '@kbn/observability-utils/es/client/create_observability_es_client'; -import { kqlQuery } from '@kbn/observability-utils/es/queries/kql_query'; -import { esqlResultToPlainObjects } from '@kbn/observability-utils/es/utils/esql_result_to_plain_objects'; +import type { ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; +import { kqlQuery } from '@kbn/observability-utils-common/es/queries/kql_query'; +import { esqlResultToPlainObjects } from '@kbn/observability-utils-server/es/esql_result_to_plain_objects'; import { ENTITY_TYPE } from '@kbn/observability-shared-plugin/common'; import { ScalarValue } from '@elastic/elasticsearch/lib/api/types'; import { diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts index c95a488ad49dd..cca0fda0cc3f8 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts @@ -5,9 +5,9 @@ * 2.0. */ -import { type ObservabilityElasticsearchClient } from '@kbn/observability-utils/es/client/create_observability_es_client'; -import { kqlQuery } from '@kbn/observability-utils/es/queries/kql_query'; -import { esqlResultToPlainObjects } from '@kbn/observability-utils/es/utils/esql_result_to_plain_objects'; +import { type ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; +import { kqlQuery } from '@kbn/observability-utils-server/es/queries/kql_query'; +import { esqlResultToPlainObjects } from '@kbn/observability-utils-server/es/esql_result_to_plain_objects'; import { ENTITY_LAST_SEEN, ENTITY_TYPE } from '@kbn/observability-shared-plugin/common'; import type { ScalarValue } from '@elastic/elasticsearch/lib/api/types'; import { diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/entities/route.ts b/x-pack/plugins/observability_solution/inventory/server/routes/entities/route.ts index 88d6cb68ee214..05aa6687f105f 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/entities/route.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/entities/route.ts @@ -6,11 +6,11 @@ */ import { INVENTORY_APP_ID } from '@kbn/deeplinks-observability/constants'; import { jsonRt } from '@kbn/io-ts-utils'; -import { createObservabilityEsClient } from '@kbn/observability-utils/es/client/create_observability_es_client'; +import { createObservabilityEsClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; import { ENTITY_TYPE } from '@kbn/observability-shared-plugin/common'; import * as t from 'io-ts'; import { orderBy } from 'lodash'; -import { joinByKey } from '@kbn/observability-utils/array/join_by_key'; +import { joinByKey } from '@kbn/observability-utils-common/array/join_by_key'; import { entityColumnIdsRt, Entity } from '../../../common/entities'; import { createInventoryServerRoute } from '../create_inventory_server_route'; import { getEntityTypes } from './get_entity_types'; diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/has_data/get_has_data.ts b/x-pack/plugins/observability_solution/inventory/server/routes/has_data/get_has_data.ts index 27ba8c0fe46c3..1918ce98c30af 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/has_data/get_has_data.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/has_data/get_has_data.ts @@ -5,8 +5,8 @@ * 2.0. */ import type { Logger } from '@kbn/core/server'; -import { esqlResultToPlainObjects } from '@kbn/observability-utils/es/utils/esql_result_to_plain_objects'; -import { type ObservabilityElasticsearchClient } from '@kbn/observability-utils/es/client/create_observability_es_client'; +import { esqlResultToPlainObjects } from '@kbn/observability-utils-server/es/esql_result_to_plain_objects'; +import { type ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; import { getBuiltinEntityDefinitionIdESQLWhereClause } from '../entities/query_helper'; import { ENTITIES_LATEST_ALIAS } from '../../../common/entities'; diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/has_data/route.ts b/x-pack/plugins/observability_solution/inventory/server/routes/has_data/route.ts index aae8be7f846f8..f0e582b396177 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/has_data/route.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/has_data/route.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { createObservabilityEsClient } from '@kbn/observability-utils/es/client/create_observability_es_client'; +import { createObservabilityEsClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; import { INVENTORY_APP_ID } from '@kbn/deeplinks-observability/constants'; import { createInventoryServerRoute } from '../create_inventory_server_route'; import { getHasData } from './get_has_data'; diff --git a/x-pack/plugins/observability_solution/inventory/tsconfig.json b/x-pack/plugins/observability_solution/inventory/tsconfig.json index d27d170b0990e..ad8e57991c745 100644 --- a/x-pack/plugins/observability_solution/inventory/tsconfig.json +++ b/x-pack/plugins/observability_solution/inventory/tsconfig.json @@ -20,7 +20,6 @@ "@kbn/server-route-repository", "@kbn/shared-ux-link-redirect-app", "@kbn/typed-react-router-config", - "@kbn/observability-utils", "@kbn/kibana-react-plugin", "@kbn/i18n", "@kbn/deeplinks-observability", diff --git a/x-pack/plugins/observability_solution/investigate_app/public/items/embeddable_item/register_embeddable_item.tsx b/x-pack/plugins/observability_solution/investigate_app/public/items/embeddable_item/register_embeddable_item.tsx index 29b2a1319feff..77833e80ec199 100644 --- a/x-pack/plugins/observability_solution/investigate_app/public/items/embeddable_item/register_embeddable_item.tsx +++ b/x-pack/plugins/observability_solution/investigate_app/public/items/embeddable_item/register_embeddable_item.tsx @@ -8,7 +8,7 @@ import { EuiLoadingSpinner, EuiFlexItem } from '@elastic/eui'; import { css } from '@emotion/css'; import { ReactEmbeddableRenderer } from '@kbn/embeddable-plugin/public'; import type { GlobalWidgetParameters } from '@kbn/investigate-plugin/public'; -import { useAbortableAsync } from '@kbn/observability-utils/hooks/use_abortable_async'; +import { useAbortableAsync } from '@kbn/observability-utils-browser/hooks/use_abortable_async'; import React, { useEffect, useMemo, useRef, useState } from 'react'; import { v4 } from 'uuid'; import { ErrorMessage } from '../../components/error_message'; diff --git a/x-pack/plugins/observability_solution/investigate_app/public/items/esql_item/register_esql_item.tsx b/x-pack/plugins/observability_solution/investigate_app/public/items/esql_item/register_esql_item.tsx index 7b88081ca5503..46fe9ea2d9dd2 100644 --- a/x-pack/plugins/observability_solution/investigate_app/public/items/esql_item/register_esql_item.tsx +++ b/x-pack/plugins/observability_solution/investigate_app/public/items/esql_item/register_esql_item.tsx @@ -10,7 +10,7 @@ import type { ESQLSearchResponse } from '@kbn/es-types'; import { i18n } from '@kbn/i18n'; import { type GlobalWidgetParameters } from '@kbn/investigate-plugin/public'; import type { Suggestion } from '@kbn/lens-plugin/public'; -import { useAbortableAsync } from '@kbn/observability-utils/hooks/use_abortable_async'; +import { useAbortableAsync } from '@kbn/observability-utils-browser/hooks/use_abortable_async'; import React, { useMemo } from 'react'; import { ErrorMessage } from '../../components/error_message'; import { useKibana } from '../../hooks/use_kibana'; diff --git a/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/add_investigation_item/esql_widget_preview.tsx b/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/add_investigation_item/esql_widget_preview.tsx index 8d0056dbd538d..469baf6e07f5c 100644 --- a/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/add_investigation_item/esql_widget_preview.tsx +++ b/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/add_investigation_item/esql_widget_preview.tsx @@ -11,7 +11,7 @@ import type { ESQLColumn, ESQLRow } from '@kbn/es-types'; import { GlobalWidgetParameters } from '@kbn/investigate-plugin/public'; import { Item } from '@kbn/investigation-shared'; import type { Suggestion } from '@kbn/lens-plugin/public'; -import { useAbortableAsync } from '@kbn/observability-utils/hooks/use_abortable_async'; +import { useAbortableAsync } from '@kbn/observability-utils-browser/hooks/use_abortable_async'; import React, { useEffect, useMemo, useState } from 'react'; import { ErrorMessage } from '../../../../components/error_message'; import { SuggestVisualizationList } from '../../../../components/suggest_visualization_list'; diff --git a/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/events_timeline/events_timeline.tsx b/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/events_timeline/events_timeline.tsx index befa50bcc0e8d..c3f92139bd936 100644 --- a/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/events_timeline/events_timeline.tsx +++ b/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/events_timeline/events_timeline.tsx @@ -10,7 +10,7 @@ import moment from 'moment'; import { Chart, Axis, AreaSeries, Position, ScaleType, Settings } from '@elastic/charts'; import { useActiveCursor } from '@kbn/charts-plugin/public'; import { EuiSkeletonText } from '@elastic/eui'; -import { getBrushData } from '@kbn/observability-utils/chart/utils'; +import { getBrushData } from '@kbn/observability-utils-browser/chart/utils'; import { Group } from '@kbn/observability-alerting-rule-utils'; import { ALERT_GROUP } from '@kbn/rule-data-utils'; import { SERVICE_NAME } from '@kbn/observability-shared-plugin/common'; diff --git a/x-pack/plugins/observability_solution/investigate_app/tsconfig.json b/x-pack/plugins/observability_solution/investigate_app/tsconfig.json index a853456b1156c..7cfd1565688a7 100644 --- a/x-pack/plugins/observability_solution/investigate_app/tsconfig.json +++ b/x-pack/plugins/observability_solution/investigate_app/tsconfig.json @@ -67,7 +67,6 @@ "@kbn/calculate-auto", "@kbn/ml-random-sampler-utils", "@kbn/charts-plugin", - "@kbn/observability-utils", "@kbn/observability-alerting-rule-utils", ], } diff --git a/x-pack/plugins/observability_solution/observability_shared/kibana.jsonc b/x-pack/plugins/observability_solution/observability_shared/kibana.jsonc index a5cde081c7c54..8e5a4c25af48c 100644 --- a/x-pack/plugins/observability_solution/observability_shared/kibana.jsonc +++ b/x-pack/plugins/observability_solution/observability_shared/kibana.jsonc @@ -33,4 +33,4 @@ "common" ] } -} \ No newline at end of file +} diff --git a/x-pack/plugins/streams/common/index.ts b/x-pack/plugins/streams/common/index.ts new file mode 100644 index 0000000000000..3a7306e46cae2 --- /dev/null +++ b/x-pack/plugins/streams/common/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export type { StreamDefinition } from './types'; diff --git a/x-pack/plugins/streams/public/api/index.ts b/x-pack/plugins/streams/public/api/index.ts new file mode 100644 index 0000000000000..f64fc7f2fdce8 --- /dev/null +++ b/x-pack/plugins/streams/public/api/index.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { CoreSetup, CoreStart, HttpFetchOptions } from '@kbn/core/public'; +import type { + ClientRequestParamsOf, + ReturnOf, + RouteRepositoryClient, +} from '@kbn/server-route-repository'; +import { createRepositoryClient } from '@kbn/server-route-repository-client'; +import type { StreamsRouteRepository } from '../../server'; + +type FetchOptions = Omit & { + body?: any; +}; + +export type StreamsRepositoryClientOptions = Omit< + FetchOptions, + 'query' | 'body' | 'pathname' | 'signal' +> & { + signal: AbortSignal | null; +}; + +export type StreamsRepositoryClient = RouteRepositoryClient< + StreamsRouteRepository, + StreamsRepositoryClientOptions +>; + +export type AutoAbortedStreamsRepositoryClient = RouteRepositoryClient< + StreamsRouteRepository, + Omit +>; + +export type StreamsRepositoryEndpoint = keyof StreamsRouteRepository; + +export type APIReturnType = ReturnOf< + StreamsRouteRepository, + TEndpoint +>; + +export type StreamsAPIClientRequestParamsOf = + ClientRequestParamsOf; + +export function createStreamsRepositoryClient( + core: CoreStart | CoreSetup +): StreamsRepositoryClient { + return createRepositoryClient(core); +} diff --git a/x-pack/plugins/streams/public/index.ts b/x-pack/plugins/streams/public/index.ts index 5b83ea1d297d3..bc90fb0f40066 100644 --- a/x-pack/plugins/streams/public/index.ts +++ b/x-pack/plugins/streams/public/index.ts @@ -7,7 +7,12 @@ import { PluginInitializer, PluginInitializerContext } from '@kbn/core/public'; import { Plugin } from './plugin'; +import { StreamsPluginSetup, StreamsPluginStart } from './types'; -export const plugin: PluginInitializer<{}, {}> = (context: PluginInitializerContext) => { +export type { StreamsPluginSetup, StreamsPluginStart }; + +export const plugin: PluginInitializer = ( + context: PluginInitializerContext +) => { return new Plugin(context); }; diff --git a/x-pack/plugins/streams/public/plugin.ts b/x-pack/plugins/streams/public/plugin.ts index f35d18e06ff70..8016e342b8c8e 100644 --- a/x-pack/plugins/streams/public/plugin.ts +++ b/x-pack/plugins/streams/public/plugin.ts @@ -8,24 +8,31 @@ import { CoreSetup, CoreStart, PluginInitializerContext } from '@kbn/core/public'; import { Logger } from '@kbn/logging'; +import { createRepositoryClient } from '@kbn/server-route-repository-client'; import type { StreamsPublicConfig } from '../common/config'; import { StreamsPluginClass, StreamsPluginSetup, StreamsPluginStart } from './types'; +import { StreamsRepositoryClient } from './api'; export class Plugin implements StreamsPluginClass { public config: StreamsPublicConfig; public logger: Logger; + private repositoryClient!: StreamsRepositoryClient; + constructor(context: PluginInitializerContext<{}>) { this.config = context.config.get(); this.logger = context.logger.get(); } - setup(core: CoreSetup, pluginSetup: StreamsPluginSetup) { + setup(core: CoreSetup<{}>, pluginSetup: {}): StreamsPluginSetup { + this.repositoryClient = createRepositoryClient(core); return {}; } - start(core: CoreStart) { - return {}; + start(core: CoreStart, pluginsStart: {}): StreamsPluginStart { + return { + streamsRepositoryClient: this.repositoryClient, + }; } stop() {} diff --git a/x-pack/plugins/streams/public/types.ts b/x-pack/plugins/streams/public/types.ts index 61e5fa94098f0..929c89fb6ec98 100644 --- a/x-pack/plugins/streams/public/types.ts +++ b/x-pack/plugins/streams/public/types.ts @@ -6,11 +6,13 @@ */ import type { Plugin as PluginClass } from '@kbn/core/public'; +import type { StreamsRepositoryClient } from './api'; // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface StreamsPluginSetup {} -// eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface StreamsPluginStart {} +export interface StreamsPluginStart { + streamsRepositoryClient: StreamsRepositoryClient; +} -export type StreamsPluginClass = PluginClass<{}, {}, StreamsPluginSetup, StreamsPluginStart>; +export type StreamsPluginClass = PluginClass; diff --git a/x-pack/plugins/streams/server/lib/streams/bootstrap_stream.ts b/x-pack/plugins/streams/server/lib/streams/bootstrap_stream.ts index c609af5b49978..2a4a5b99a02bc 100644 --- a/x-pack/plugins/streams/server/lib/streams/bootstrap_stream.ts +++ b/x-pack/plugins/streams/server/lib/streams/bootstrap_stream.ts @@ -34,6 +34,7 @@ export async function bootstrapStream({ const { composedOf, ignoreMissing } = await getIndexTemplateComponents({ scopedClusterClient, definition: rootDefinition, + logger, }); const reroutePipeline = await generateReroutePipeline({ esClient: scopedClusterClient.asCurrentUser, diff --git a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts index 47e2b4061c063..8d32321dcd46b 100644 --- a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts +++ b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts @@ -5,14 +5,17 @@ * 2.0. */ -import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; +import { SearchHit } from '@kbn/es-types'; +import { createObservabilityEsClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; import { get } from 'lodash'; -import { StreamDefinition } from '../../../common/types'; +import { IScopedClusterClient, Logger } from '@kbn/core/server'; import { STREAMS_INDEX } from '../../../common/constants'; +import { StreamDefinition } from '../../../common/types'; import { ComponentTemplateNotFound, DefinitionNotFound, IndexTemplateNotFound } from './errors'; interface BaseParams { scopedClusterClient: IScopedClusterClient; + logger: Logger; } interface BaseParamsWithDefinition extends BaseParams { @@ -32,16 +35,17 @@ interface ReadStreamParams extends BaseParams { id: string; } -export async function readStream({ id, scopedClusterClient }: ReadStreamParams) { +export async function readStream({ id, ...baseParams }: ReadStreamParams) { + const { scopedClusterClient } = baseParams; try { const response = await scopedClusterClient.asCurrentUser.get({ id, index: STREAMS_INDEX, }); const definition = response._source as StreamDefinition; - const indexTemplate = await readIndexTemplate({ scopedClusterClient, definition }); - const componentTemplate = await readComponentTemplate({ scopedClusterClient, definition }); - const ingestPipelines = await readIngestPipelines({ scopedClusterClient, definition }); + const indexTemplate = await readIndexTemplate({ ...baseParams, definition }); + const componentTemplate = await readComponentTemplate({ ...baseParams, definition }); + const ingestPipelines = await readIngestPipelines({ ...baseParams, definition }); return { definition, index_template: indexTemplate, @@ -56,6 +60,38 @@ export async function readStream({ id, scopedClusterClient }: ReadStreamParams) } } +type ListStreamsParams = BaseParams; + +export interface ListStreamResponse { + hits: Array>; + total: { + value: number; + }; +} + +export async function listStreams({ + scopedClusterClient, + logger, +}: ListStreamsParams): Promise { + const esClient = createObservabilityEsClient({ + client: scopedClusterClient.asCurrentUser, + logger, + plugin: 'streams', + }); + + const response = await esClient.search('list_streams', { + index: STREAMS_INDEX, + allow_no_indices: true, + track_total_hits: true, + size: 10_000, + }); + + return { + hits: response.hits.hits, + total: response.hits.total, + }; +} + export async function readIndexTemplate({ scopedClusterClient, definition, @@ -99,11 +135,8 @@ export async function readIngestPipelines({ return response; } -export async function getIndexTemplateComponents({ - scopedClusterClient, - definition, -}: BaseParamsWithDefinition) { - const indexTemplate = await readIndexTemplate({ scopedClusterClient, definition }); +export async function getIndexTemplateComponents(params: BaseParamsWithDefinition) { + const indexTemplate = await readIndexTemplate(params); return { composedOf: indexTemplate.index_template.composed_of, ignoreMissing: get( diff --git a/x-pack/plugins/streams/server/routes/esql/route.ts b/x-pack/plugins/streams/server/routes/esql/route.ts new file mode 100644 index 0000000000000..5f6131852bf7c --- /dev/null +++ b/x-pack/plugins/streams/server/routes/esql/route.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ESQLSearchResponse } from '@kbn/es-types'; +import { createObservabilityEsClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; +import { kqlQuery } from '@kbn/observability-utils-common/es/queries/kql_query'; +import { excludeFrozenQuery } from '@kbn/observability-utils-common/es/queries/exclude_frozen_query'; +import { rangeQuery } from '@kbn/observability-utils-common/es/queries/range_query'; +import { z } from '@kbn/zod'; +import { isNumber } from 'lodash'; +import { createServerRoute } from '../create_server_route'; + +export const executeEsqlRoute = createServerRoute({ + endpoint: 'POST /internal/streams/esql', + params: z.object({ + body: z.object({ + query: z.string(), + operationName: z.string(), + filter: z.object({}).passthrough().optional(), + kuery: z.string().optional(), + start: z.number().optional(), + end: z.number().optional(), + }), + }), + handler: async ({ params, request, logger, getScopedClients }): Promise => { + const { scopedClusterClient } = await getScopedClients({ request }); + const observabilityEsClient = createObservabilityEsClient({ + client: scopedClusterClient.asCurrentUser, + logger, + plugin: 'streams', + }); + + const { + body: { operationName, query, filter, kuery, start, end }, + } = params; + + const response = await observabilityEsClient.esql(operationName, { + dropNullColumns: true, + query, + filter: { + bool: { + filter: [ + filter || { match_all: {} }, + ...kqlQuery(kuery), + ...excludeFrozenQuery(), + ...(isNumber(start) && isNumber(end) ? rangeQuery(start, end) : []), + ], + }, + }, + }); + + return response; + }, +}); + +export const esqlRoutes = { + ...executeEsqlRoute, +}; diff --git a/x-pack/plugins/streams/server/routes/index.ts b/x-pack/plugins/streams/server/routes/index.ts index c24362f3ec8a9..b6b5feb7205ba 100644 --- a/x-pack/plugins/streams/server/routes/index.ts +++ b/x-pack/plugins/streams/server/routes/index.ts @@ -5,14 +5,18 @@ * 2.0. */ +import { esqlRoutes } from './esql/route'; import { enableStreamsRoute } from './streams/enable'; import { forkStreamsRoute } from './streams/fork'; +import { listStreamsRoute } from './streams/list'; import { readStreamRoute } from './streams/read'; export const StreamsRouteRepository = { ...enableStreamsRoute, ...forkStreamsRoute, ...readStreamRoute, + ...listStreamsRoute, + ...esqlRoutes, }; export type StreamsRouteRepository = typeof StreamsRouteRepository; diff --git a/x-pack/plugins/streams/server/routes/streams/enable.ts b/x-pack/plugins/streams/server/routes/streams/enable.ts index 1241434a975df..6ebcaeb67266f 100644 --- a/x-pack/plugins/streams/server/routes/streams/enable.ts +++ b/x-pack/plugins/streams/server/routes/streams/enable.ts @@ -13,10 +13,9 @@ import { createStream } from '../../lib/streams/stream_crud'; import { rootStreamDefinition } from '../../lib/streams/root_stream_definition'; export const enableStreamsRoute = createServerRoute({ - endpoint: 'POST /api/streams/_enable 2023-10-31', + endpoint: 'POST /internal/streams/_enable', params: z.object({}), options: { - access: 'public', security: { authz: { requiredPrivileges: ['streams_enable'], @@ -33,6 +32,7 @@ export const enableStreamsRoute = createServerRoute({ await createStream({ scopedClusterClient, definition: rootStreamDefinition, + logger, }); return response.ok({ body: { acknowledged: true } }); } catch (e) { diff --git a/x-pack/plugins/streams/server/routes/streams/fork.ts b/x-pack/plugins/streams/server/routes/streams/fork.ts index c019abd17b58d..87ee53816fbc3 100644 --- a/x-pack/plugins/streams/server/routes/streams/fork.ts +++ b/x-pack/plugins/streams/server/routes/streams/fork.ts @@ -45,6 +45,7 @@ export const forkStreamsRoute = createServerRoute({ const { definition: rootDefinition } = await readStream({ scopedClusterClient, id: params.path.id, + logger, }); if (!params.body.id.startsWith(rootDefinition.id)) { @@ -56,6 +57,7 @@ export const forkStreamsRoute = createServerRoute({ await createStream({ scopedClusterClient, definition: { ...params.body, forked_from: rootDefinition.id, root: false }, + logger, }); await bootstrapStream({ diff --git a/x-pack/plugins/streams/server/routes/streams/list.ts b/x-pack/plugins/streams/server/routes/streams/list.ts new file mode 100644 index 0000000000000..b3a0f66b3a2b0 --- /dev/null +++ b/x-pack/plugins/streams/server/routes/streams/list.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createServerRoute } from '../create_server_route'; +import { ListStreamResponse, listStreams } from '../../lib/streams/stream_crud'; + +export const listStreamsRoute = createServerRoute({ + endpoint: 'GET /api/streams 2023-10-31', + options: { + access: 'public', + security: { + authz: { + requiredPrivileges: ['streams_read'], + }, + }, + }, + handler: async ({ + response, + request, + getScopedClients, + logger, + }): Promise<{ + streams: ListStreamResponse; + }> => { + const { scopedClusterClient } = await getScopedClients({ request }); + const streams = await listStreams({ + scopedClusterClient, + logger, + }); + + return { streams }; + }, +}); diff --git a/x-pack/plugins/streams/server/routes/streams/read.ts b/x-pack/plugins/streams/server/routes/streams/read.ts index 061522787de52..e57dbb5955bcb 100644 --- a/x-pack/plugins/streams/server/routes/streams/read.ts +++ b/x-pack/plugins/streams/server/routes/streams/read.ts @@ -6,9 +6,11 @@ */ import { z } from '@kbn/zod'; +import { notFound, internal } from '@hapi/boom'; import { createServerRoute } from '../create_server_route'; import { DefinitionNotFound } from '../../lib/streams/errors'; import { readStream } from '../../lib/streams/stream_crud'; +import { StreamDefinition } from '../../../common'; export const readStreamRoute = createServerRoute({ endpoint: 'GET /api/streams/{id} 2023-10-31', @@ -23,21 +25,28 @@ export const readStreamRoute = createServerRoute({ params: z.object({ path: z.object({ id: z.string() }), }), - handler: async ({ response, params, request, getScopedClients }) => { + handler: async ({ + response, + params, + request, + logger, + getScopedClients, + }): Promise<{ definition: StreamDefinition }> => { try { const { scopedClusterClient } = await getScopedClients({ request }); const streamEntity = await readStream({ + logger, scopedClusterClient, id: params.path.id, }); - return response.ok({ body: streamEntity }); + return streamEntity; } catch (e) { if (e instanceof DefinitionNotFound) { - return response.notFound({ body: e }); + throw notFound(e); } - return response.customError({ body: e, statusCode: 500 }); + throw internal(e); } }, }); diff --git a/x-pack/plugins/streams/tsconfig.json b/x-pack/plugins/streams/tsconfig.json index 98386b54ca9ea..803ed0d4ec022 100644 --- a/x-pack/plugins/streams/tsconfig.json +++ b/x-pack/plugins/streams/tsconfig.json @@ -14,28 +14,22 @@ "target/**/*" ], "kbn_references": [ - "@kbn/entities-schema", "@kbn/config-schema", "@kbn/core", - "@kbn/server-route-repository-client", "@kbn/logging", "@kbn/core-plugins-server", "@kbn/core-http-server", "@kbn/security-plugin", - "@kbn/rison", - "@kbn/es-query", - "@kbn/core-elasticsearch-client-server-mocks", - "@kbn/core-saved-objects-api-server-mocks", - "@kbn/logging-mocks", "@kbn/core-saved-objects-api-server", "@kbn/core-elasticsearch-server", "@kbn/task-manager-plugin", - "@kbn/datemath", "@kbn/server-route-repository", "@kbn/zod", - "@kbn/zod-helpers", "@kbn/encrypted-saved-objects-plugin", "@kbn/licensing-plugin", - "@kbn/alerting-plugin" + "@kbn/features-plugin", + "@kbn/server-route-repository-client", + "@kbn/es-types", + "@kbn/observability-utils-server" ] } diff --git a/yarn.lock b/yarn.lock index 4dac6fb7759b7..0d173a462ca65 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5908,7 +5908,15 @@ version "0.0.0" uid "" -"@kbn/observability-utils@link:x-pack/packages/observability/observability_utils": +"@kbn/observability-utils-browser@link:x-pack/packages/observability/observability_utils/observability_utils_browser": + version "0.0.0" + uid "" + +"@kbn/observability-utils-common@link:x-pack/packages/observability/observability_utils/observability_utils_common": + version "0.0.0" + uid "" + +"@kbn/observability-utils-server@link:x-pack/packages/observability/observability_utils/observability_utils_server": version "0.0.0" uid "" From 797c93d4436ea2ab00253af352ea2d2e18a64111 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Sun, 10 Nov 2024 19:19:37 +0100 Subject: [PATCH 12/95] Streams table --- .../components/entity_detail_view/index.tsx | 111 +++++------------- .../entity_overview_tab_list/index.tsx | 12 +- .../components/stream_detail_view/index.tsx | 29 +++-- .../components/stream_list_view/index.tsx | 51 +++++++- .../streams_app_page_body/index.tsx | 26 ++++ .../streams_app_page_header/index.tsx | 23 +++- .../streams_app_page_header_title.tsx | 22 +--- .../streams_app_page_template/index.tsx | 12 +- .../public/components/streams_table/index.tsx | 70 +++++++++++ x-pack/plugins/streams/server/index.ts | 2 + 10 files changed, 226 insertions(+), 132 deletions(-) create mode 100644 x-pack/plugins/logsai/streams_app/public/components/streams_app_page_body/index.tsx create mode 100644 x-pack/plugins/logsai/streams_app/public/components/streams_table/index.tsx diff --git a/x-pack/plugins/logsai/streams_app/public/components/entity_detail_view/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/entity_detail_view/index.tsx index 5dc5a10209ea1..526262c892fbb 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/entity_detail_view/index.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/entity_detail_view/index.tsx @@ -4,22 +4,14 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { - EuiBadge, - EuiFlexGroup, - EuiFlexItem, - EuiHorizontalRule, - EuiPanel, - useEuiTheme, -} from '@elastic/eui'; -import { css } from '@emotion/css'; -import { i18n } from '@kbn/i18n'; +import { EuiFlexGroup, EuiIcon, EuiLink, EuiPanel } from '@elastic/eui'; import React from 'react'; +import { i18n } from '@kbn/i18n'; import { useStreamsAppBreadcrumbs } from '../../hooks/use_streams_app_breadcrumbs'; import { useStreamsAppRouter } from '../../hooks/use_streams_app_router'; -import { EntityDetailViewHeaderSection } from '../entity_detail_view_header_section'; import { EntityOverviewTabList } from '../entity_overview_tab_list'; import { LoadingPanel } from '../loading_panel'; +import { StreamsAppPageBody } from '../streams_app_page_body'; import { StreamsAppPageHeader } from '../streams_app_page_header'; import { StreamsAppPageHeaderTitle } from '../streams_app_page_header/streams_app_page_header_title'; @@ -32,15 +24,10 @@ export interface EntityViewTab { export function EntityDetailViewWithoutParams({ selectedTab, tabs, - type, entity, }: { selectedTab: string; tabs: EntityViewTab[]; - type: { - displayName?: string; - id: string; - }; entity: { displayName?: string; id: string; @@ -48,27 +35,21 @@ export function EntityDetailViewWithoutParams({ }) { const router = useStreamsAppRouter(); - const theme = useEuiTheme().euiTheme; - useStreamsAppBreadcrumbs(() => { - if (!type.displayName || !entity.displayName) { + if (!entity.displayName) { return []; } return [ - { - title: type.displayName, - path: `/`, - } as const, { title: entity.displayName, path: `/{key}`, params: { path: { key: entity.id } }, } as const, ]; - }, [type.displayName, type.id, entity.displayName, entity.id]); + }, [entity.displayName, entity.id]); - if (!type.displayName || !entity.displayName) { + if (!entity.displayName) { return ; } @@ -78,7 +59,7 @@ export function EntityDetailViewWithoutParams({ tab.name, { href: router.link('/{key}/{tab}', { - path: { key: entity.id, tab: 'overview' }, + path: { key: entity.id, tab: tab.name }, }), label: tab.label, content: tab.content, @@ -90,62 +71,30 @@ export function EntityDetailViewWithoutParams({ const selectedTabObject = tabMap[selectedTab]; return ( - - - - - - div { - border: 0px solid ${theme.colors.lightShade}; - border-right-width: 1px; - width: 25%; - } - > div:last-child { - border-right-width: 0; - } - `} - > - - - - {type.displayName} - - - - - - + + + + + + {i18n.translate('xpack.streams.entityDetailView.goBackLinkLabel', { + defaultMessage: 'Back', + })} + + + + }> + { + return { + name: tabKey, + label, + href, + selected: selectedTab === tabKey, + }; + })} + /> - { - return { - name: tabKey, - label, - href, - selected: selectedTab === tabKey, - }; - })} - /> - {selectedTabObject.content} + {selectedTabObject.content} ); } diff --git a/x-pack/plugins/logsai/streams_app/public/components/entity_overview_tab_list/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/entity_overview_tab_list/index.tsx index d0e4772d0993d..08502b26f7ca3 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/entity_overview_tab_list/index.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/entity_overview_tab_list/index.tsx @@ -5,13 +5,21 @@ * 2.0. */ import React from 'react'; -import { EuiTab, EuiTabs } from '@elastic/eui'; +import { EuiTab, EuiTabs, useEuiTheme } from '@elastic/eui'; +import { css } from '@emotion/css'; export function EntityOverviewTabList< T extends { name: string; label: string; href: string; selected: boolean } >({ tabs }: { tabs: T[] }) { + const theme = useEuiTheme().euiTheme; + return ( - + {tabs.map((tab) => { return ( diff --git a/x-pack/plugins/logsai/streams_app/public/components/stream_detail_view/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/stream_detail_view/index.tsx index ca479b0fab414..6d82b13061fd5 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/stream_detail_view/index.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/stream_detail_view/index.tsx @@ -39,13 +39,6 @@ export function StreamDetailView() { [streamsRepositoryClient, key] ); - const type = { - id: 'stream', - displayName: i18n.translate('xpack.streams.streamDetailView.streamsTypeDisplayName', { - defaultMessage: 'Streams', - }), - }; - const entity = { id: key, displayName: key, @@ -55,13 +48,25 @@ export function StreamDetailView() { { name: 'overview', content: , - label: i18n.translate('xpack.streams.streamDetailView.parsingTab', { - defaultMessage: 'Parsing', + label: i18n.translate('xpack.streams.streamDetailView.overviewTab', { + defaultMessage: 'Overview', + }), + }, + { + name: 'routing', + content: <>, + label: i18n.translate('xpack.streams.streamDetailView.routingTab', { + defaultMessage: 'Routing', + }), + }, + { + name: 'processing', + content: <>, + label: i18n.translate('xpack.streams.streamDetailView.processingTab', { + defaultMessage: 'Processing', }), }, ]; - return ( - - ); + return ; } diff --git a/x-pack/plugins/logsai/streams_app/public/components/stream_list_view/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/stream_list_view/index.tsx index 2e3e5b819e5b9..6dfb40b586f2a 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/stream_list_view/index.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/stream_list_view/index.tsx @@ -4,9 +4,16 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import React from 'react'; +import React, { useState } from 'react'; +import { i18n } from '@kbn/i18n'; +import { EuiFlexGroup } from '@elastic/eui'; import { useKibana } from '../../hooks/use_kibana'; import { useStreamsAppFetch } from '../../hooks/use_streams_app_fetch'; +import { StreamsAppPageHeader } from '../streams_app_page_header'; +import { StreamsAppPageHeaderTitle } from '../streams_app_page_header/streams_app_page_header_title'; +import { StreamsAppPageBody } from '../streams_app_page_body'; +import { StreamsAppSearchBar } from '../streams_app_search_bar'; +import { StreamsTable } from '../streams_table'; export function StreamListView() { const { @@ -17,14 +24,46 @@ export function StreamListView() { }, } = useKibana(); - const queryFetch = useStreamsAppFetch( + const [query, setQuery] = useState(''); + + const [submittedQuery, setSubmittedQuery] = useState(''); + + const streamsListFetch = useStreamsAppFetch( ({ signal }) => { - return streamsRepositoryClient.fetch('GET /api/streams 2023-10-31', { - signal, - }); + return streamsRepositoryClient + .fetch('GET /api/streams 2023-10-31', { + signal, + }) + .then((response) => response.streams); }, [streamsRepositoryClient] ); - return <>; + return ( + + + } + /> + + + { + setQuery(next.query); + }} + onQuerySubmit={(next) => { + setSubmittedQuery(next.query); + }} + query={query} + /> + + + + + ); } diff --git a/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_body/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_body/index.tsx new file mode 100644 index 0000000000000..0f13dc31e277b --- /dev/null +++ b/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_body/index.tsx @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React from 'react'; +import { EuiPanel, useEuiTheme } from '@elastic/eui'; +import { css } from '@emotion/css'; + +export function StreamsAppPageBody({ children }: { children: React.ReactNode }) { + const theme = useEuiTheme().euiTheme; + return ( + + {children} + + ); +} diff --git a/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_header/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_header/index.tsx index 51fac8af35a61..8c9bd84036b18 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_header/index.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_header/index.tsx @@ -4,13 +4,26 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { EuiFlexGroup, EuiPageHeader } from '@elastic/eui'; +import { EuiFlexGroup, EuiPageHeader, EuiPanel } from '@elastic/eui'; import React from 'react'; -export function StreamsAppPageHeader({ children }: { children: React.ReactNode }) { +export function StreamsAppPageHeader({ + title, + children, +}: { + title: React.ReactNode; + children?: React.ReactNode; +}) { return ( - - {children} - + + + + + {title} + + + + {children} + ); } diff --git a/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_header/streams_app_page_header_title.tsx b/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_header/streams_app_page_header_title.tsx index fd1014a615912..ff7d6581dea4f 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_header/streams_app_page_header_title.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_header/streams_app_page_header_title.tsx @@ -4,25 +4,13 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { EuiFlexGroup, EuiHorizontalRule, EuiTitle } from '@elastic/eui'; +import { EuiTitle } from '@elastic/eui'; import React from 'react'; -export function StreamsAppPageHeaderTitle({ - title, - children, -}: { - title: string; - children?: React.ReactNode; -}) { +export function StreamsAppPageHeaderTitle({ title }: { title: string }) { return ( - - - -

{title}

- - {children} - - - + +

{title}

+ ); } diff --git a/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_template/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_template/index.tsx index 19f2de51ceb2a..c474f54c22745 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_template/index.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_template/index.tsx @@ -4,9 +4,9 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { EuiPanel } from '@elastic/eui'; import { css } from '@emotion/css'; import React from 'react'; +import { EuiPanel, EuiSpacer } from '@elastic/eui'; import { useKibana } from '../../hooks/use_kibana'; export function StreamsAppPageTemplate({ children }: { children: React.ReactNode }) { @@ -35,14 +35,8 @@ export function StreamsAppPageTemplate({ children }: { children: React.ReactNode }, }} > - + + {children} diff --git a/x-pack/plugins/logsai/streams_app/public/components/streams_table/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/streams_table/index.tsx new file mode 100644 index 0000000000000..4998def1140e5 --- /dev/null +++ b/x-pack/plugins/logsai/streams_app/public/components/streams_table/index.tsx @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React, { useMemo } from 'react'; +import { i18n } from '@kbn/i18n'; +import type { AbortableAsyncState } from '@kbn/observability-utils-browser/hooks/use_abortable_async'; +import type { ListStreamResponse } from '@kbn/streams-plugin/server'; +import { + EuiBasicTable, + EuiBasicTableColumn, + EuiFlexGroup, + EuiIcon, + EuiLink, + EuiTitle, +} from '@elastic/eui'; +import { useStreamsAppRouter } from '../../hooks/use_streams_app_router'; + +type ListStreamItem = ListStreamResponse['hits'][number]['_source']; + +export function StreamsTable({ + listFetch, +}: { + listFetch: AbortableAsyncState; +}) { + const router = useStreamsAppRouter(); + + const items = useMemo(() => { + return listFetch.value?.hits.map((hit) => hit._source) ?? []; + }, [listFetch.value?.hits]); + + const columns = useMemo>>(() => { + return [ + { + field: 'id', + name: i18n.translate('xpack.streams.streamsTable.nameColumnTitle', { + defaultMessage: 'Name', + }), + render: (_, { id }) => { + return ( + + + + {id} + + + ); + }, + }, + ]; + }, [router]); + + return ( + + +

+ {i18n.translate('xpack.streams.streamsTable.tableTitle', { + defaultMessage: 'Streams', + })} +

+ + + + ); +} diff --git a/x-pack/plugins/streams/server/index.ts b/x-pack/plugins/streams/server/index.ts index bd8aee304ad15..9ef13c62d6b7b 100644 --- a/x-pack/plugins/streams/server/index.ts +++ b/x-pack/plugins/streams/server/index.ts @@ -17,3 +17,5 @@ export const plugin = async (context: PluginInitializerContext) = const { StreamsPlugin } = await import('./plugin'); return new StreamsPlugin(context); }; + +export type { ListStreamResponse } from './lib/streams/stream_crud'; From fcce6288c531cc766b0b2b6f703d7b928b713de8 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Sun, 10 Nov 2024 19:49:31 +0100 Subject: [PATCH 13/95] Only fetch histogram when DS exists --- .../streams_app/common/entity_source_query.ts | 2 +- .../components/entity_detail_view/index.tsx | 5 +++- .../stream_detail_overview/index.tsx | 19 +++++++++++---- .../streams_app_page_header/index.tsx | 24 ++++++++++++------- .../streams/server/routes/esql/route.ts | 1 - 5 files changed, 35 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/logsai/streams_app/common/entity_source_query.ts b/x-pack/plugins/logsai/streams_app/common/entity_source_query.ts index 7c7176e774750..c076d1f6bd87f 100644 --- a/x-pack/plugins/logsai/streams_app/common/entity_source_query.ts +++ b/x-pack/plugins/logsai/streams_app/common/entity_source_query.ts @@ -8,7 +8,7 @@ export function entitySourceQuery({ entity }: { entity: Record }) { return { bool: { - filter: Object.entries(entity).map(([key, value]) => ({ [key]: value })), + filter: Object.entries(entity).map(([key, value]) => ({ term: { [key]: value } })), }, }; } diff --git a/x-pack/plugins/logsai/streams_app/public/components/entity_detail_view/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/entity_detail_view/index.tsx index 526262c892fbb..a8ce5d72b8983 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/entity_detail_view/index.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/entity_detail_view/index.tsx @@ -82,7 +82,10 @@ export function EntityDetailViewWithoutParams({ - }> + } + > { return { diff --git a/x-pack/plugins/logsai/streams_app/public/components/stream_detail_overview/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/stream_detail_overview/index.tsx index c1d91b6440b5d..99eb501cdc2bd 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/stream_detail_overview/index.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/stream_detail_overview/index.tsx @@ -16,6 +16,7 @@ import { entitySourceQuery } from '../../../common/entity_source_query'; import { useKibana } from '../../hooks/use_kibana'; import { ControlledEsqlChart } from '../esql_chart/controlled_esql_chart'; import { StreamsAppSearchBar } from '../streams_app_search_bar'; +import { useStreamsAppFetch } from '../../hooks/use_streams_app_fetch'; export function StreamDetailOverview({ definition }: { definition?: StreamDefinition }) { const { @@ -53,9 +54,11 @@ export function StreamDetailOverview({ definition }: { definition?: StreamDefini const baseQuery = `FROM ${indexPatterns.join(', ')}`; - const bucketSize = calculateAuto.atLeast(50, moment.duration(1, 'minute'))!.asMinutes(); + const bucketSize = Math.round( + calculateAuto.atLeast(50, moment.duration(1, 'minute'))!.asSeconds() + ); - const histogramQuery = `${baseQuery} | STATS metric = COUNT(*) BY @timestamp = BUCKET(@timestamp, ${bucketSize} minutes)`; + const histogramQuery = `${baseQuery} | STATS metric = COUNT(*) BY @timestamp = BUCKET(@timestamp, ${bucketSize} seconds)`; return { histogramQuery, @@ -63,9 +66,15 @@ export function StreamDetailOverview({ definition }: { definition?: StreamDefini }; }, [dataStream]); - const histogramQueryFetch = useAbortableAsync( + const histogramQueryFetch = useStreamsAppFetch( async ({ signal }) => { - if (!queries?.histogramQuery) { + if (!queries?.histogramQuery || !dataStream) { + return undefined; + } + + const existingIndices = await dataViews.getExistingIndices([dataStream]); + + if (existingIndices.length === 0) { return undefined; } @@ -84,6 +93,8 @@ export function StreamDetailOverview({ definition }: { definition?: StreamDefini }); }, [ + dataStream, + dataViews, streamsRepositoryClient, queries?.histogramQuery, persistedKqlFilter, diff --git a/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_header/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_header/index.tsx index 8c9bd84036b18..1171772116f2a 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_header/index.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_header/index.tsx @@ -4,26 +4,32 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { EuiFlexGroup, EuiPageHeader, EuiPanel } from '@elastic/eui'; +import { EuiFlexGroup, EuiPageHeader, useEuiTheme } from '@elastic/eui'; +import { css } from '@emotion/css'; import React from 'react'; export function StreamsAppPageHeader({ title, children, + verticalPaddingSize = 'l', }: { title: React.ReactNode; children?: React.ReactNode; + verticalPaddingSize?: 'none' | 'l'; }) { + const theme = useEuiTheme().euiTheme; + return ( - - - - - {title} - - + + + {title} {children} - + ); } diff --git a/x-pack/plugins/streams/server/routes/esql/route.ts b/x-pack/plugins/streams/server/routes/esql/route.ts index 5f6131852bf7c..2ba215c6d1978 100644 --- a/x-pack/plugins/streams/server/routes/esql/route.ts +++ b/x-pack/plugins/streams/server/routes/esql/route.ts @@ -39,7 +39,6 @@ export const executeEsqlRoute = createServerRoute({ } = params; const response = await observabilityEsClient.esql(operationName, { - dropNullColumns: true, query, filter: { bool: { From 9d372194f7d839c095d164815c0560673c3d20fb Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Sun, 10 Nov 2024 21:00:19 +0100 Subject: [PATCH 14/95] more fixes --- x-pack/plugins/streams/common/types.ts | 23 ++-- .../lib/streams/errors/malformed_children.ts | 13 ++ .../lib/streams/errors/malformed_fields.ts | 13 ++ .../server/lib/streams/helpers/hierarchy.ts | 5 + .../generate_index_template.ts | 19 +-- .../lib/streams/internal_stream_mapping.ts | 35 ++++++ .../streams/server/lib/streams/stream_crud.ts | 117 +++++++++++++++--- .../streams/server/routes/streams/edit.ts | 28 +++-- .../streams/server/routes/streams/enable.ts | 2 + .../streams/server/routes/streams/fork.ts | 10 +- .../streams/server/routes/streams/read.ts | 16 ++- 11 files changed, 224 insertions(+), 57 deletions(-) create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/malformed_children.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/malformed_fields.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/internal_stream_mapping.ts diff --git a/x-pack/plugins/streams/common/types.ts b/x-pack/plugins/streams/common/types.ts index 0019eb8fee054..9e35e50581782 100644 --- a/x-pack/plugins/streams/common/types.ts +++ b/x-pack/plugins/streams/common/types.ts @@ -65,22 +65,7 @@ export const fieldDefinitionSchema = z.object({ export type FieldDefinition = z.infer; -/** - * Example of a "root" stream - * { - * "id": "logs", - * } - * - * Example of a forked stream - * { - * "id": "logs.nginx", - * "condition": { field: 'log.logger, operator: 'eq', value": "nginx_proxy" } - * "forked_from": "logs" - * } - */ - -export const streamDefinitonSchema = z.object({ - id: z.string(), +export const streamWithoutIdDefinitonSchema = z.object({ processing: z.array(processingDefinitionSchema).default([]), fields: z.array(fieldDefinitionSchema).default([]), children: z @@ -93,4 +78,10 @@ export const streamDefinitonSchema = z.object({ .default([]), }); +export type StreamWithoutIdDefinition = z.infer; + +export const streamDefinitonSchema = streamWithoutIdDefinitonSchema.extend({ + id: z.string(), +}); + export type StreamDefinition = z.infer; diff --git a/x-pack/plugins/streams/server/lib/streams/errors/malformed_children.ts b/x-pack/plugins/streams/server/lib/streams/errors/malformed_children.ts new file mode 100644 index 0000000000000..699c4cdd5b1ef --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/malformed_children.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class MalformedChildren extends Error { + constructor(message: string) { + super(message); + this.name = 'MalformedChildren'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/malformed_fields.ts b/x-pack/plugins/streams/server/lib/streams/errors/malformed_fields.ts new file mode 100644 index 0000000000000..b8f7ac1392610 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/malformed_fields.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class MalformedFields extends Error { + constructor(message: string) { + super(message); + this.name = 'MalformedFields'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/helpers/hierarchy.ts b/x-pack/plugins/streams/server/lib/streams/helpers/hierarchy.ts index f31846bab36db..6f1cd308f3c3d 100644 --- a/x-pack/plugins/streams/server/lib/streams/helpers/hierarchy.ts +++ b/x-pack/plugins/streams/server/lib/streams/helpers/hierarchy.ts @@ -28,3 +28,8 @@ export function getParentId(id: string) { export function isRoot(id: string) { return id.split('.').length === 1; } + +export function getAncestors(id: string) { + const parts = id.split('.'); + return parts.slice(0, parts.length - 1).map((_, index) => parts.slice(0, index + 1).join('.')); +} diff --git a/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts b/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts index 295c819a08a30..8f7773c455c4d 100644 --- a/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts +++ b/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts @@ -9,15 +9,19 @@ import { ASSET_VERSION } from '../../../../common/constants'; import { getProcessingPipelineName } from '../ingest_pipelines/name'; import { getIndexTemplateName } from './name'; -export function generateIndexTemplate( - id: string, - composedOf: string[] = [], - ignoreMissing: string[] = [] -) { +export function generateIndexTemplate(id: string) { + const composedOf = id.split('.').reduce((acc, _, index, array) => { + if (index === array.length - 1) { + return acc; + } + const parent = array.slice(0, index + 1).join('.'); + return [...acc, `${parent}@stream.layer`]; + }, [] as string[]); + return { name: getIndexTemplateName(id), index_patterns: [id], - composed_of: [...composedOf, `${id}@stream.layer`], + composed_of: composedOf, priority: 200, version: ASSET_VERSION, _meta: { @@ -35,6 +39,7 @@ export function generateIndexTemplate( }, }, allow_auto_create: true, - ignore_missing_component_templates: [...ignoreMissing, `${id}@stream.layer`], + // ignore missing component templates to be more robust against out-of-order syncs + ignore_missing_component_templates: composedOf, }; } diff --git a/x-pack/plugins/streams/server/lib/streams/internal_stream_mapping.ts b/x-pack/plugins/streams/server/lib/streams/internal_stream_mapping.ts new file mode 100644 index 0000000000000..2428cc798f006 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/internal_stream_mapping.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; +import { STREAMS_INDEX } from '../../../common/constants'; + +export function createStreamsIndex(scopedClusterClient: IScopedClusterClient) { + return scopedClusterClient.asCurrentUser.indices.create({ + index: STREAMS_INDEX, + mappings: { + dynamic: 'strict', + properties: { + processing: { + type: 'object', + enabled: false, + }, + fields: { + type: 'object', + enabled: false, + }, + children: { + type: 'object', + enabled: false, + }, + id: { + type: 'keyword', + }, + }, + }, + }); +} diff --git a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts index 7ffadf552fcbc..6b29bd830ab14 100644 --- a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts +++ b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts @@ -8,7 +8,7 @@ import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; import { get } from 'lodash'; import { Logger } from '@kbn/logging'; -import { StreamDefinition } from '../../../common/types'; +import { FieldDefinition, StreamDefinition } from '../../../common/types'; import { STREAMS_INDEX } from '../../../common/constants'; import { DefinitionNotFound, IndexTemplateNotFound } from './errors'; import { deleteTemplate, upsertTemplate } from './index_templates/manage_index_templates'; @@ -24,6 +24,8 @@ import { deleteIngestPipeline, upsertIngestPipeline, } from './ingest_pipelines/manage_ingest_pipelines'; +import { getAncestors } from './helpers/hierarchy'; +import { MalformedFields } from './errors/malformed_fields'; interface BaseParams { scopedClusterClient: IScopedClusterClient; @@ -66,7 +68,7 @@ export async function deleteStreamObjects({ id, scopedClusterClient, logger }: D }); } -async function upsertStream({ definition, scopedClusterClient }: BaseParamsWithDefinition) { +async function upsertInternalStream({ definition, scopedClusterClient }: BaseParamsWithDefinition) { return scopedClusterClient.asCurrentUser.index({ id: definition.id, index: STREAMS_INDEX, @@ -111,6 +113,95 @@ export async function readStream({ id, scopedClusterClient }: ReadStreamParams) } } +interface ReadAncestorsParams extends BaseParams { + id: string; +} + +export async function readAncestors({ id, scopedClusterClient }: ReadAncestorsParams) { + const ancestorIds = getAncestors(id); + + return await Promise.all( + ancestorIds.map((ancestorId) => readStream({ scopedClusterClient, id: ancestorId })) + ); +} + +interface ReadDescendantsParams extends BaseParams { + id: string; +} + +export async function readDescendants({ id, scopedClusterClient }: ReadDescendantsParams) { + const response = await scopedClusterClient.asCurrentUser.search({ + index: STREAMS_INDEX, + size: 10000, + body: { + query: { + bool: { + filter: { + prefix: { + id, + }, + }, + must_not: { + term: { + id, + }, + }, + }, + }, + }, + }); + return response.hits.hits.map((hit) => hit._source as StreamDefinition); +} + +export async function validateAncestorFields( + scopedClusterClient: IScopedClusterClient, + id: string, + fields: FieldDefinition[] +) { + const ancestors = await readAncestors({ + id, + scopedClusterClient, + }); + for (const ancestor of ancestors) { + for (const field of fields) { + if ( + ancestor.definition.fields.some( + (ancestorField) => ancestorField.type !== field.type && ancestorField.name === field.name + ) + ) { + throw new MalformedFields( + `Field ${field.name} is already defined with incompatible type in the parent stream ${ancestor.definition.id}` + ); + } + } + } +} + +export async function validateDescendantFields( + scopedClusterClient: IScopedClusterClient, + id: string, + fields: FieldDefinition[] +) { + const descendants = await readDescendants({ + id, + scopedClusterClient, + }); + for (const descendant of descendants) { + for (const field of fields) { + if ( + descendant.fields.some( + (descendantField) => + descendantField.type !== field.type && descendantField.name === field.name + ) + ) { + throw new MalformedFields( + `Field ${field.name} is already defined with incompatible type in the child stream ${descendant.id}` + ); + } + } + } +} + export async function checkStreamExists({ id, scopedClusterClient }: ReadStreamParams) { try { await readStream({ id, scopedClusterClient }); @@ -167,7 +258,7 @@ export async function syncStream({ rootDefinition, logger, }: SyncStreamParams) { - await upsertStream({ + await upsertInternalStream({ scopedClusterClient, definition, }); @@ -189,11 +280,12 @@ export async function syncStream({ logger, pipeline: reroutePipeline, }); + await upsertTemplate({ + esClient: scopedClusterClient.asSecondaryAuthUser, + logger, + template: generateIndexTemplate(definition.id), + }); if (rootDefinition) { - const { composedOf, ignoreMissing } = await getIndexTemplateComponents({ - scopedClusterClient, - definition: rootDefinition, - }); const parentReroutePipeline = await generateReroutePipeline({ definition: rootDefinition, }); @@ -202,16 +294,5 @@ export async function syncStream({ logger, pipeline: parentReroutePipeline, }); - await upsertTemplate({ - esClient: scopedClusterClient.asSecondaryAuthUser, - logger, - template: generateIndexTemplate(definition.id, composedOf, ignoreMissing), - }); - } else { - await upsertTemplate({ - esClient: scopedClusterClient.asSecondaryAuthUser, - logger, - template: generateIndexTemplate(definition.id), - }); } } diff --git a/x-pack/plugins/streams/server/routes/streams/edit.ts b/x-pack/plugins/streams/server/routes/streams/edit.ts index 2ee75e32af730..a3c41eef5a7a3 100644 --- a/x-pack/plugins/streams/server/routes/streams/edit.ts +++ b/x-pack/plugins/streams/server/routes/streams/edit.ts @@ -15,10 +15,17 @@ import { SecurityException, } from '../../lib/streams/errors'; import { createServerRoute } from '../create_server_route'; -import { StreamDefinition, streamDefinitonSchema } from '../../../common/types'; -import { syncStream, readStream, checkStreamExists } from '../../lib/streams/stream_crud'; +import { StreamDefinition, streamWithoutIdDefinitonSchema } from '../../../common/types'; +import { + syncStream, + readStream, + checkStreamExists, + validateAncestorFields, + validateDescendantFields, +} from '../../lib/streams/stream_crud'; import { MalformedStreamId } from '../../lib/streams/errors/malformed_stream_id'; import { getParentId } from '../../lib/streams/helpers/hierarchy'; +import { MalformedChildren } from '../../lib/streams/errors/malformed_children'; export const editStreamRoute = createServerRoute({ endpoint: 'PUT /api/streams/{id} 2023-10-31', @@ -34,17 +41,15 @@ export const editStreamRoute = createServerRoute({ path: z.object({ id: z.string(), }), - body: z.object({ definition: streamDefinitonSchema }), + body: streamWithoutIdDefinitonSchema, }), handler: async ({ response, params, logger, request, getScopedClients }) => { try { const { scopedClusterClient } = await getScopedClients({ request }); - await validateStreamChildren( - scopedClusterClient, - params.path.id, - params.body.definition.children - ); + await validateStreamChildren(scopedClusterClient, params.path.id, params.body.children); + await validateAncestorFields(scopedClusterClient, params.path.id, params.body.fields); + await validateDescendantFields(scopedClusterClient, params.path.id, params.body.fields); const parentId = getParentId(params.path.id); let parentDefinition: StreamDefinition | undefined; @@ -57,11 +62,11 @@ export const editStreamRoute = createServerRoute({ logger ); } - const streamDefinition = { ...params.body.definition }; + const streamDefinition = { ...params.body }; await syncStream({ scopedClusterClient, - definition: streamDefinition, + definition: { ...streamDefinition, id: params.path.id }, rootDefinition: parentDefinition, logger, }); @@ -80,7 +85,6 @@ export const editStreamRoute = createServerRoute({ children: [], fields: [], processing: [], - root: false, }; await syncStream({ @@ -149,7 +153,7 @@ async function validateStreamChildren( const oldChildren = oldDefinition.children.map((child) => child.id); const newChildren = new Set(children.map((child) => child.id)); if (oldChildren.some((child) => !newChildren.has(child))) { - throw new MalformedStreamId( + throw new MalformedChildren( 'Cannot remove children from a stream, please delete the stream instead' ); } diff --git a/x-pack/plugins/streams/server/routes/streams/enable.ts b/x-pack/plugins/streams/server/routes/streams/enable.ts index 111cd870c41a4..3657ef04fbbb7 100644 --- a/x-pack/plugins/streams/server/routes/streams/enable.ts +++ b/x-pack/plugins/streams/server/routes/streams/enable.ts @@ -10,6 +10,7 @@ import { SecurityException } from '../../lib/streams/errors'; import { createServerRoute } from '../create_server_route'; import { syncStream } from '../../lib/streams/stream_crud'; import { rootStreamDefinition } from '../../lib/streams/root_stream_definition'; +import { createStreamsIndex } from '../../lib/streams/internal_stream_mapping'; export const enableStreamsRoute = createServerRoute({ endpoint: 'POST /api/streams/_enable 2023-10-31', @@ -25,6 +26,7 @@ export const enableStreamsRoute = createServerRoute({ handler: async ({ request, response, logger, getScopedClients }) => { try { const { scopedClusterClient } = await getScopedClients({ request }); + await createStreamsIndex(scopedClusterClient); await syncStream({ scopedClusterClient, definition: rootStreamDefinition, diff --git a/x-pack/plugins/streams/server/routes/streams/fork.ts b/x-pack/plugins/streams/server/routes/streams/fork.ts index ced96f465e3d3..1d626705d304a 100644 --- a/x-pack/plugins/streams/server/routes/streams/fork.ts +++ b/x-pack/plugins/streams/server/routes/streams/fork.ts @@ -14,7 +14,7 @@ import { } from '../../lib/streams/errors'; import { createServerRoute } from '../create_server_route'; import { conditionSchema, streamDefinitonSchema } from '../../../common/types'; -import { syncStream, readStream } from '../../lib/streams/stream_crud'; +import { syncStream, readStream, validateAncestorFields } from '../../lib/streams/stream_crud'; import { MalformedStreamId } from '../../lib/streams/errors/malformed_stream_id'; import { isChildOf } from '../../lib/streams/helpers/hierarchy'; @@ -47,7 +47,7 @@ export const forkStreamsRoute = createServerRoute({ id: params.path.id, }); - const childDefinition = { ...params.body.stream, root: false }; + const childDefinition = { ...params.body.stream }; // check whether root stream has a child of the given name already if (rootDefinition.children.some((child) => child.id === childDefinition.id)) { @@ -62,6 +62,12 @@ export const forkStreamsRoute = createServerRoute({ ); } + await validateAncestorFields( + scopedClusterClient, + params.body.stream.id, + params.body.stream.fields + ); + rootDefinition.children.push({ id: params.body.stream.id, condition: params.body.condition, diff --git a/x-pack/plugins/streams/server/routes/streams/read.ts b/x-pack/plugins/streams/server/routes/streams/read.ts index 061522787de52..64914a3d9c741 100644 --- a/x-pack/plugins/streams/server/routes/streams/read.ts +++ b/x-pack/plugins/streams/server/routes/streams/read.ts @@ -8,7 +8,7 @@ import { z } from '@kbn/zod'; import { createServerRoute } from '../create_server_route'; import { DefinitionNotFound } from '../../lib/streams/errors'; -import { readStream } from '../../lib/streams/stream_crud'; +import { readAncestors, readStream } from '../../lib/streams/stream_crud'; export const readStreamRoute = createServerRoute({ endpoint: 'GET /api/streams/{id} 2023-10-31', @@ -31,7 +31,19 @@ export const readStreamRoute = createServerRoute({ id: params.path.id, }); - return response.ok({ body: streamEntity }); + const ancestors = await readAncestors({ + id: streamEntity.definition.id, + scopedClusterClient, + }); + + const body = { + ...streamEntity.definition, + inheritedFields: ancestors.flatMap(({ definition: { id, fields } }) => + fields.map((field) => ({ ...field, from: id })) + ), + }; + + return response.ok({ body }); } catch (e) { if (e instanceof DefinitionNotFound) { return response.notFound({ body: e }); From d622fa1e29826e0d51f8db4a4fbe985bc4ec5ec6 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 11 Nov 2024 14:51:00 +0100 Subject: [PATCH 15/95] more adjustments --- x-pack/plugins/streams/common/constants.ts | 2 +- .../component_templates/generate_layer.ts | 1 + .../data_streams/manage_data_streams.ts | 93 ++++++++++++++++++ .../generate_index_template.ts | 3 - .../lib/streams/internal_stream_mapping.ts | 2 +- .../streams/server/lib/streams/stream_crud.ts | 98 ++++++++----------- x-pack/plugins/streams/server/plugin.ts | 2 +- .../streams/server/routes/streams/delete.ts | 2 +- .../streams/server/routes/streams/edit.ts | 2 +- .../streams/server/routes/streams/enable.ts | 2 +- .../streams/server/routes/streams/fork.ts | 2 +- .../streams/server/routes/streams/resync.ts | 2 +- 12 files changed, 145 insertions(+), 66 deletions(-) create mode 100644 x-pack/plugins/streams/server/lib/streams/data_streams/manage_data_streams.ts diff --git a/x-pack/plugins/streams/common/constants.ts b/x-pack/plugins/streams/common/constants.ts index acc66e9286f87..d7595990ded6e 100644 --- a/x-pack/plugins/streams/common/constants.ts +++ b/x-pack/plugins/streams/common/constants.ts @@ -6,4 +6,4 @@ */ export const ASSET_VERSION = 1; -export const STREAMS_INDEX = '.streams'; +export const STREAMS_INDEX = '.kibana_streams'; diff --git a/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts b/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts index 4a39a78b38990..82c89c9ab9171 100644 --- a/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts +++ b/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts @@ -30,6 +30,7 @@ export function generateLayer( template: { settings: isRoot(definition.id) ? logsSettings : {}, mappings: { + subobjects: false, properties, }, }, diff --git a/x-pack/plugins/streams/server/lib/streams/data_streams/manage_data_streams.ts b/x-pack/plugins/streams/server/lib/streams/data_streams/manage_data_streams.ts new file mode 100644 index 0000000000000..812739db56c73 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/data_streams/manage_data_streams.ts @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { retryTransientEsErrors } from '../helpers/retry'; + +interface DataStreamManagementOptions { + esClient: ElasticsearchClient; + name: string; + logger: Logger; +} + +interface DeleteDataStreamOptions { + esClient: ElasticsearchClient; + name: string; + logger: Logger; +} + +interface RolloverDataStreamOptions { + esClient: ElasticsearchClient; + name: string; + logger: Logger; +} + +export async function upsertDataStream({ esClient, name, logger }: DataStreamManagementOptions) { + const dataStreamExists = await esClient.indices.exists({ index: name }); + if (dataStreamExists) { + return; + } + try { + await retryTransientEsErrors(() => esClient.indices.createDataStream({ name }), { logger }); + logger.debug(() => `Installed data stream: ${name}`); + } catch (error: any) { + logger.error(`Error creating data stream: ${error.message}`); + throw error; + } +} + +export async function deleteDataStream({ esClient, name, logger }: DeleteDataStreamOptions) { + try { + await retryTransientEsErrors( + () => esClient.indices.deleteDataStream({ name }, { ignore: [404] }), + { logger } + ); + } catch (error: any) { + logger.error(`Error deleting data stream: ${error.message}`); + throw error; + } +} + +export async function rolloverDataStreamIfNecessary({ + esClient, + name, + logger, +}: RolloverDataStreamOptions) { + const dataStreams = await esClient.indices.getDataStream({ name: `${name},${name}.*` }); + for (const dataStream of dataStreams.data_streams) { + const currentMappings = + Object.values( + await esClient.indices.getMapping({ + index: dataStream.indices.at(-1)?.index_name, + }) + )[0].mappings.properties || {}; + const simulatedIndex = await esClient.indices.simulateIndexTemplate({ name: dataStream.name }); + const simulatedMappings = simulatedIndex.template.mappings.properties || {}; + + // check whether the same fields and same types are listed (don't check for other mapping attributes) + const isDifferent = + Object.values(simulatedMappings).length !== Object.values(currentMappings).length || + Object.entries(simulatedMappings || {}).some(([fieldName, { type }]) => { + const currentType = currentMappings[fieldName]?.type; + return currentType !== type; + }); + + if (!isDifferent) { + continue; + } + + try { + await retryTransientEsErrors(() => esClient.indices.rollover({ alias: dataStream.name }), { + logger, + }); + logger.debug(() => `Rolled over data stream: ${dataStream.name}`); + } catch (error: any) { + logger.error(`Error rolling over data stream: ${error.message}`); + throw error; + } + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts b/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts index 8f7773c455c4d..7a16534a618da 100644 --- a/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts +++ b/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts @@ -11,9 +11,6 @@ import { getIndexTemplateName } from './name'; export function generateIndexTemplate(id: string) { const composedOf = id.split('.').reduce((acc, _, index, array) => { - if (index === array.length - 1) { - return acc; - } const parent = array.slice(0, index + 1).join('.'); return [...acc, `${parent}@stream.layer`]; }, [] as string[]); diff --git a/x-pack/plugins/streams/server/lib/streams/internal_stream_mapping.ts b/x-pack/plugins/streams/server/lib/streams/internal_stream_mapping.ts index 2428cc798f006..8e88eeef8cd84 100644 --- a/x-pack/plugins/streams/server/lib/streams/internal_stream_mapping.ts +++ b/x-pack/plugins/streams/server/lib/streams/internal_stream_mapping.ts @@ -9,7 +9,7 @@ import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; import { STREAMS_INDEX } from '../../../common/constants'; export function createStreamsIndex(scopedClusterClient: IScopedClusterClient) { - return scopedClusterClient.asCurrentUser.indices.create({ + return scopedClusterClient.asInternalUser.indices.create({ index: STREAMS_INDEX, mappings: { dynamic: 'strict', diff --git a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts index 6b29bd830ab14..b5021b2aebeb4 100644 --- a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts +++ b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts @@ -6,11 +6,10 @@ */ import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; -import { get } from 'lodash'; import { Logger } from '@kbn/logging'; import { FieldDefinition, StreamDefinition } from '../../../common/types'; import { STREAMS_INDEX } from '../../../common/constants'; -import { DefinitionNotFound, IndexTemplateNotFound } from './errors'; +import { DefinitionNotFound } from './errors'; import { deleteTemplate, upsertTemplate } from './index_templates/manage_index_templates'; import { generateLayer } from './component_templates/generate_layer'; import { generateIngestPipeline } from './ingest_pipelines/generate_ingest_pipeline'; @@ -26,6 +25,11 @@ import { } from './ingest_pipelines/manage_ingest_pipelines'; import { getAncestors } from './helpers/hierarchy'; import { MalformedFields } from './errors/malformed_fields'; +import { + deleteDataStream, + rolloverDataStreamIfNecessary, + upsertDataStream, +} from './data_streams/manage_data_streams'; interface BaseParams { scopedClusterClient: IScopedClusterClient; @@ -41,35 +45,40 @@ interface DeleteStreamParams extends BaseParams { } export async function deleteStreamObjects({ id, scopedClusterClient, logger }: DeleteStreamParams) { - await scopedClusterClient.asCurrentUser.delete({ - id, - index: STREAMS_INDEX, - refresh: 'wait_for', - }); await deleteTemplate({ - esClient: scopedClusterClient.asSecondaryAuthUser, + esClient: scopedClusterClient.asCurrentUser, name: getIndexTemplateName(id), logger, }); await deleteComponent({ - esClient: scopedClusterClient.asSecondaryAuthUser, + esClient: scopedClusterClient.asCurrentUser, name: getComponentTemplateName(id), logger, }); await deleteIngestPipeline({ - esClient: scopedClusterClient.asSecondaryAuthUser, + esClient: scopedClusterClient.asCurrentUser, id: getProcessingPipelineName(id), logger, }); await deleteIngestPipeline({ - esClient: scopedClusterClient.asSecondaryAuthUser, + esClient: scopedClusterClient.asCurrentUser, id: getReroutePipelineName(id), logger, }); + await deleteDataStream({ + esClient: scopedClusterClient.asCurrentUser, + name: id, + logger, + }); + await scopedClusterClient.asInternalUser.delete({ + id, + index: STREAMS_INDEX, + refresh: 'wait_for', + }); } async function upsertInternalStream({ definition, scopedClusterClient }: BaseParamsWithDefinition) { - return scopedClusterClient.asCurrentUser.index({ + return scopedClusterClient.asInternalUser.index({ id: definition.id, index: STREAMS_INDEX, document: definition, @@ -80,7 +89,7 @@ async function upsertInternalStream({ definition, scopedClusterClient }: BasePar type ListStreamsParams = BaseParams; export async function listStreams({ scopedClusterClient }: ListStreamsParams) { - const response = await scopedClusterClient.asCurrentUser.search({ + const response = await scopedClusterClient.asInternalUser.search({ index: STREAMS_INDEX, size: 10000, fields: ['id'], @@ -97,7 +106,7 @@ interface ReadStreamParams extends BaseParams { export async function readStream({ id, scopedClusterClient }: ReadStreamParams) { try { - const response = await scopedClusterClient.asCurrentUser.get({ + const response = await scopedClusterClient.asInternalUser.get({ id, index: STREAMS_INDEX, }); @@ -130,7 +139,7 @@ interface ReadDescendantsParams extends BaseParams { } export async function readDescendants({ id, scopedClusterClient }: ReadDescendantsParams) { - const response = await scopedClusterClient.asCurrentUser.search({ + const response = await scopedClusterClient.asInternalUser.search({ index: STREAMS_INDEX, size: 10000, body: { @@ -214,37 +223,6 @@ export async function checkStreamExists({ id, scopedClusterClient }: ReadStreamP } } -export async function readIndexTemplate({ - scopedClusterClient, - definition, -}: BaseParamsWithDefinition) { - const response = await scopedClusterClient.asSecondaryAuthUser.indices.getIndexTemplate({ - name: `${definition.id}@stream`, - }); - const indexTemplate = response.index_templates.find( - (doc) => doc.name === `${definition.id}@stream` - ); - if (!indexTemplate) { - throw new IndexTemplateNotFound(`Unable to find index_template for ${definition.id}`); - } - return indexTemplate; -} - -export async function getIndexTemplateComponents({ - scopedClusterClient, - definition, -}: BaseParamsWithDefinition) { - const indexTemplate = await readIndexTemplate({ scopedClusterClient, definition }); - return { - composedOf: indexTemplate.index_template.composed_of, - ignoreMissing: get( - indexTemplate, - 'index_template.ignore_missing_component_templates', - [] - ) as string[], - }; -} - interface SyncStreamParams { scopedClusterClient: IScopedClusterClient; definition: StreamDefinition; @@ -258,17 +236,13 @@ export async function syncStream({ rootDefinition, logger, }: SyncStreamParams) { - await upsertInternalStream({ - scopedClusterClient, - definition, - }); await upsertComponent({ - esClient: scopedClusterClient.asSecondaryAuthUser, + esClient: scopedClusterClient.asCurrentUser, logger, component: generateLayer(definition.id, definition), }); await upsertIngestPipeline({ - esClient: scopedClusterClient.asSecondaryAuthUser, + esClient: scopedClusterClient.asCurrentUser, logger, pipeline: generateIngestPipeline(definition.id, definition), }); @@ -276,12 +250,12 @@ export async function syncStream({ definition, }); await upsertIngestPipeline({ - esClient: scopedClusterClient.asSecondaryAuthUser, + esClient: scopedClusterClient.asCurrentUser, logger, pipeline: reroutePipeline, }); await upsertTemplate({ - esClient: scopedClusterClient.asSecondaryAuthUser, + esClient: scopedClusterClient.asCurrentUser, logger, template: generateIndexTemplate(definition.id), }); @@ -290,9 +264,23 @@ export async function syncStream({ definition: rootDefinition, }); await upsertIngestPipeline({ - esClient: scopedClusterClient.asSecondaryAuthUser, + esClient: scopedClusterClient.asCurrentUser, logger, pipeline: parentReroutePipeline, }); } + await upsertDataStream({ + esClient: scopedClusterClient.asCurrentUser, + logger, + name: definition.id, + }); + await upsertInternalStream({ + scopedClusterClient, + definition, + }); + await rolloverDataStreamIfNecessary({ + esClient: scopedClusterClient.asCurrentUser, + name: definition.id, + logger, + }); } diff --git a/x-pack/plugins/streams/server/plugin.ts b/x-pack/plugins/streams/server/plugin.ts index 951c1e3751d72..478c77c6ca691 100644 --- a/x-pack/plugins/streams/server/plugin.ts +++ b/x-pack/plugins/streams/server/plugin.ts @@ -75,7 +75,7 @@ export class StreamsPlugin read: [], }, ui: ['read', 'write'], - api: ['streams_enable', 'streams_fork', 'streams_read'], + api: ['streams_write', 'streams_read'], }, read: { app: ['streams', 'kibana'], diff --git a/x-pack/plugins/streams/server/routes/streams/delete.ts b/x-pack/plugins/streams/server/routes/streams/delete.ts index 2176530a42518..cea5275e9b409 100644 --- a/x-pack/plugins/streams/server/routes/streams/delete.ts +++ b/x-pack/plugins/streams/server/routes/streams/delete.ts @@ -25,7 +25,7 @@ export const deleteStreamRoute = createServerRoute({ access: 'public', security: { authz: { - requiredPrivileges: ['streams_fork'], + requiredPrivileges: ['streams_write'], }, }, }, diff --git a/x-pack/plugins/streams/server/routes/streams/edit.ts b/x-pack/plugins/streams/server/routes/streams/edit.ts index a3c41eef5a7a3..5c027a9985a07 100644 --- a/x-pack/plugins/streams/server/routes/streams/edit.ts +++ b/x-pack/plugins/streams/server/routes/streams/edit.ts @@ -33,7 +33,7 @@ export const editStreamRoute = createServerRoute({ access: 'public', security: { authz: { - requiredPrivileges: ['streams_fork'], + requiredPrivileges: ['streams_write'], }, }, }, diff --git a/x-pack/plugins/streams/server/routes/streams/enable.ts b/x-pack/plugins/streams/server/routes/streams/enable.ts index 3657ef04fbbb7..3e122b855575d 100644 --- a/x-pack/plugins/streams/server/routes/streams/enable.ts +++ b/x-pack/plugins/streams/server/routes/streams/enable.ts @@ -19,7 +19,7 @@ export const enableStreamsRoute = createServerRoute({ access: 'public', security: { authz: { - requiredPrivileges: ['streams_enable'], + requiredPrivileges: ['streams_write'], }, }, }, diff --git a/x-pack/plugins/streams/server/routes/streams/fork.ts b/x-pack/plugins/streams/server/routes/streams/fork.ts index 1d626705d304a..f0ebb2a03ce52 100644 --- a/x-pack/plugins/streams/server/routes/streams/fork.ts +++ b/x-pack/plugins/streams/server/routes/streams/fork.ts @@ -24,7 +24,7 @@ export const forkStreamsRoute = createServerRoute({ access: 'public', security: { authz: { - requiredPrivileges: ['streams_fork'], + requiredPrivileges: ['streams_write'], }, }, }, diff --git a/x-pack/plugins/streams/server/routes/streams/resync.ts b/x-pack/plugins/streams/server/routes/streams/resync.ts index e2888328e9fba..badba1fe90346 100644 --- a/x-pack/plugins/streams/server/routes/streams/resync.ts +++ b/x-pack/plugins/streams/server/routes/streams/resync.ts @@ -15,7 +15,7 @@ export const resyncStreamsRoute = createServerRoute({ access: 'public', security: { authz: { - requiredPrivileges: ['streams_fork'], + requiredPrivileges: ['streams_write'], }, }, }, From fbc6c2a19e72d2e603c54e19f08232c837d708ad Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 11 Nov 2024 16:00:41 +0100 Subject: [PATCH 16/95] some more fixes --- x-pack/plugins/streams/common/types.ts | 6 +++++- .../server/lib/streams/root_stream_definition.ts | 2 +- .../plugins/streams/server/lib/streams/stream_crud.ts | 10 +++++----- x-pack/plugins/streams/server/routes/streams/fork.ts | 8 ++++---- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/streams/common/types.ts b/x-pack/plugins/streams/common/types.ts index 9e35e50581782..6cdb2f923f6f4 100644 --- a/x-pack/plugins/streams/common/types.ts +++ b/x-pack/plugins/streams/common/types.ts @@ -60,7 +60,7 @@ export type ProcessingDefinition = z.infer; export const fieldDefinitionSchema = z.object({ name: z.string(), - type: z.enum(['keyword', 'text', 'long', 'double', 'date', 'boolean', 'ip']), + type: z.enum(['keyword', 'match_only_text', 'long', 'double', 'date', 'boolean', 'ip']), }); export type FieldDefinition = z.infer; @@ -85,3 +85,7 @@ export const streamDefinitonSchema = streamWithoutIdDefinitonSchema.extend({ }); export type StreamDefinition = z.infer; + +export const streamDefinitonWithoutChildrenSchema = streamDefinitonSchema.omit({ children: true }); + +export type StreamWithoutChildrenDefinition = z.infer; diff --git a/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts b/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts index 4930876ae9923..2b7deed877309 100644 --- a/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts +++ b/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts @@ -18,7 +18,7 @@ export const rootStreamDefinition: StreamDefinition = { }, { name: 'message', - type: 'text', + type: 'match_only_text', }, { name: 'host.name', diff --git a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts index b5021b2aebeb4..78a126905d9a4 100644 --- a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts +++ b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts @@ -45,6 +45,11 @@ interface DeleteStreamParams extends BaseParams { } export async function deleteStreamObjects({ id, scopedClusterClient, logger }: DeleteStreamParams) { + await deleteDataStream({ + esClient: scopedClusterClient.asCurrentUser, + name: id, + logger, + }); await deleteTemplate({ esClient: scopedClusterClient.asCurrentUser, name: getIndexTemplateName(id), @@ -65,11 +70,6 @@ export async function deleteStreamObjects({ id, scopedClusterClient, logger }: D id: getReroutePipelineName(id), logger, }); - await deleteDataStream({ - esClient: scopedClusterClient.asCurrentUser, - name: id, - logger, - }); await scopedClusterClient.asInternalUser.delete({ id, index: STREAMS_INDEX, diff --git a/x-pack/plugins/streams/server/routes/streams/fork.ts b/x-pack/plugins/streams/server/routes/streams/fork.ts index f0ebb2a03ce52..1ff507e530065 100644 --- a/x-pack/plugins/streams/server/routes/streams/fork.ts +++ b/x-pack/plugins/streams/server/routes/streams/fork.ts @@ -13,7 +13,7 @@ import { SecurityException, } from '../../lib/streams/errors'; import { createServerRoute } from '../create_server_route'; -import { conditionSchema, streamDefinitonSchema } from '../../../common/types'; +import { conditionSchema, streamDefinitonWithoutChildrenSchema } from '../../../common/types'; import { syncStream, readStream, validateAncestorFields } from '../../lib/streams/stream_crud'; import { MalformedStreamId } from '../../lib/streams/errors/malformed_stream_id'; import { isChildOf } from '../../lib/streams/helpers/hierarchy'; @@ -32,7 +32,7 @@ export const forkStreamsRoute = createServerRoute({ path: z.object({ id: z.string(), }), - body: z.object({ stream: streamDefinitonSchema, condition: conditionSchema }), + body: z.object({ stream: streamDefinitonWithoutChildrenSchema, condition: conditionSchema }), }), handler: async ({ response, params, logger, request, getScopedClients }) => { try { @@ -47,7 +47,7 @@ export const forkStreamsRoute = createServerRoute({ id: params.path.id, }); - const childDefinition = { ...params.body.stream }; + const childDefinition = { ...params.body.stream, children: [] }; // check whether root stream has a child of the given name already if (rootDefinition.children.some((child) => child.id === childDefinition.id)) { @@ -82,7 +82,7 @@ export const forkStreamsRoute = createServerRoute({ await syncStream({ scopedClusterClient, - definition: params.body.stream, + definition: childDefinition, rootDefinition, logger, }); From e810f537bd60ec172d6fd14d3165c31aab21c1ef Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 12 Nov 2024 08:32:38 +0000 Subject: [PATCH 17/95] [CI] Auto-commit changed files from 'node scripts/build_plugin_list_docs' --- .github/CODEOWNERS | 1 + docs/developer/plugin-list.asciidoc | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index abc6749d52ea9..85999705f2a64 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -966,6 +966,7 @@ x-pack/plugins/snapshot_restore @elastic/kibana-management x-pack/plugins/spaces @elastic/kibana-security x-pack/plugins/stack_alerts @elastic/response-ops x-pack/plugins/stack_connectors @elastic/response-ops +x-pack/plugins/streams @elastic/obs-entities x-pack/plugins/task_manager @elastic/response-ops x-pack/plugins/telemetry_collection_xpack @elastic/kibana-core x-pack/plugins/threat_intelligence @elastic/security-threat-hunting-investigations diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index b6ba24df78976..71ab26400f496 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -897,6 +897,10 @@ routes, etc. |The stack_connectors plugin provides connector types shipped with Kibana, built on top of the framework provided in the actions plugin. +|{kib-repo}blob/{branch}/x-pack/plugins/streams/README.md[streams] +|This plugin provides an interface to manage streams + + |{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/synthetics/README.md[synthetics] |The purpose of this plugin is to provide users of Heartbeat more visibility of what's happening in their infrastructure. From e93ac173c857b4183e2f06fcd3284c082a51ccbd Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 12 Nov 2024 08:44:23 +0000 Subject: [PATCH 18/95] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/plugins/streams/tsconfig.json | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/x-pack/plugins/streams/tsconfig.json b/x-pack/plugins/streams/tsconfig.json index 98386b54ca9ea..e0ceaf8197fa6 100644 --- a/x-pack/plugins/streams/tsconfig.json +++ b/x-pack/plugins/streams/tsconfig.json @@ -14,28 +14,19 @@ "target/**/*" ], "kbn_references": [ - "@kbn/entities-schema", "@kbn/config-schema", "@kbn/core", - "@kbn/server-route-repository-client", "@kbn/logging", "@kbn/core-plugins-server", "@kbn/core-http-server", "@kbn/security-plugin", - "@kbn/rison", - "@kbn/es-query", - "@kbn/core-elasticsearch-client-server-mocks", - "@kbn/core-saved-objects-api-server-mocks", - "@kbn/logging-mocks", "@kbn/core-saved-objects-api-server", "@kbn/core-elasticsearch-server", "@kbn/task-manager-plugin", - "@kbn/datemath", "@kbn/server-route-repository", "@kbn/zod", - "@kbn/zod-helpers", "@kbn/encrypted-saved-objects-plugin", "@kbn/licensing-plugin", - "@kbn/alerting-plugin" + "@kbn/features-plugin" ] } From cf9f349bf2e893aa6152ca325c8ac75e6d8069ad Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 12 Nov 2024 10:36:51 +0100 Subject: [PATCH 19/95] add limits --- packages/kbn-optimizer/limits.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index df8a077e844f6..bdfb27369a372 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -159,6 +159,7 @@ pageLoadAssetSize: spaces: 57868 stackAlerts: 58316 stackConnectors: 67227 + streams: 16742 synthetics: 55971 telemetry: 51957 telemetryManagementSection: 38586 From 7b0a4e52e54a783401e2d8d3edc7d4361f2afadc Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Tue, 12 Nov 2024 11:43:32 +0100 Subject: [PATCH 20/95] Add streams to left nav --- .../deeplinks/observability/deep_links.ts | 13 +- .../get_mock_streams_app_context.tsx | 2 + .../plugins/logsai/streams_app/kibana.jsonc | 3 +- .../components/entity_detail_view/index.tsx | 3 +- .../stream_detail_overview/index.tsx | 99 ++-- .../components/stream_list_view/index.tsx | 19 +- .../streams_app_search_bar/index.tsx | 21 +- .../public/components/streams_table/index.tsx | 12 +- .../logsai/streams_app/public/plugin.ts | 1 + .../logsai/streams_app/public/types.ts | 3 + .../observability/kibana.jsonc | 5 +- .../observability/public/navigation_tree.ts | 15 +- .../observability/public/plugin.ts | 3 + .../serverless_observability/kibana.jsonc | 6 +- .../public/navigation_tree.ts | 507 +++++++++--------- .../serverless_observability/public/plugin.ts | 10 +- .../serverless_observability/public/types.ts | 11 +- x-pack/plugins/streams/public/plugin.ts | 19 +- x-pack/plugins/streams/public/types.ts | 7 +- x-pack/plugins/streams/server/routes/index.ts | 2 + .../streams/server/routes/streams/settings.ts | 33 ++ 21 files changed, 467 insertions(+), 327 deletions(-) create mode 100644 x-pack/plugins/streams/server/routes/streams/settings.ts diff --git a/packages/deeplinks/observability/deep_links.ts b/packages/deeplinks/observability/deep_links.ts index 1253b4e889fcf..256350feb2e21 100644 --- a/packages/deeplinks/observability/deep_links.ts +++ b/packages/deeplinks/observability/deep_links.ts @@ -21,6 +21,7 @@ import { OBLT_UX_APP_ID, OBLT_PROFILING_APP_ID, INVENTORY_APP_ID, + STREAMS_APP_ID, } from './constants'; type LogsApp = typeof LOGS_APP_ID; @@ -36,6 +37,7 @@ type AiAssistantApp = typeof AI_ASSISTANT_APP_ID; type ObltUxApp = typeof OBLT_UX_APP_ID; type ObltProfilingApp = typeof OBLT_PROFILING_APP_ID; type InventoryApp = typeof INVENTORY_APP_ID; +type StreamsApp = typeof STREAMS_APP_ID; export type AppId = | LogsApp @@ -50,7 +52,8 @@ export type AppId = | AiAssistantApp | ObltUxApp | ObltProfilingApp - | InventoryApp; + | InventoryApp + | StreamsApp; export type LogsLinkId = 'log-categories' | 'settings' | 'anomalies' | 'stream'; @@ -83,13 +86,16 @@ export type SyntheticsLinkId = 'certificates' | 'overview'; export type ProfilingLinkId = 'stacktraces' | 'flamegraphs' | 'functions'; +export type StreamsLinkId = 'overview'; + export type LinkId = | LogsLinkId | ObservabilityOverviewLinkId | MetricsLinkId | ApmLinkId | SyntheticsLinkId - | ProfilingLinkId; + | ProfilingLinkId + | StreamsLinkId; export type DeepLinkId = | AppId @@ -99,4 +105,5 @@ export type DeepLinkId = | `${ApmApp}:${ApmLinkId}` | `${SyntheticsApp}:${SyntheticsLinkId}` | `${ObltProfilingApp}:${ProfilingLinkId}` - | `${InventoryApp}:${InventoryLinkId}`; + | `${InventoryApp}:${InventoryLinkId}` + | `${StreamsApp}:${StreamsLinkId}`; diff --git a/x-pack/plugins/logsai/streams_app/.storybook/get_mock_streams_app_context.tsx b/x-pack/plugins/logsai/streams_app/.storybook/get_mock_streams_app_context.tsx index 318b4a8e2cf9b..1660042b2cb66 100644 --- a/x-pack/plugins/logsai/streams_app/.storybook/get_mock_streams_app_context.tsx +++ b/x-pack/plugins/logsai/streams_app/.storybook/get_mock_streams_app_context.tsx @@ -11,6 +11,7 @@ import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public'; import type { StreamsPluginStart } from '@kbn/streams-plugin/public'; import type { ObservabilitySharedPluginStart } from '@kbn/observability-shared-plugin/public'; import type { UnifiedSearchPublicPluginStart } from '@kbn/unified-search-plugin/public'; +import type { SharePublicStart } from '@kbn/share-plugin/public/plugin'; import type { StreamsAppKibanaContext } from '../public/hooks/use_kibana'; export function getMockStreamsAppContext(): StreamsAppKibanaContext { @@ -25,6 +26,7 @@ export function getMockStreamsAppContext(): StreamsAppKibanaContext { data: {} as unknown as DataPublicPluginStart, unifiedSearch: {} as unknown as UnifiedSearchPublicPluginStart, streams: {} as unknown as StreamsPluginStart, + share: {} as unknown as SharePublicStart, }, }, services: { diff --git a/x-pack/plugins/logsai/streams_app/kibana.jsonc b/x-pack/plugins/logsai/streams_app/kibana.jsonc index bfa718d7d00e3..413e68082f45d 100644 --- a/x-pack/plugins/logsai/streams_app/kibana.jsonc +++ b/x-pack/plugins/logsai/streams_app/kibana.jsonc @@ -14,7 +14,8 @@ "observabilityShared", "data", "dataViews", - "unifiedSearch" + "unifiedSearch", + "share" ], "requiredBundles": [ "kibanaReact" diff --git a/x-pack/plugins/logsai/streams_app/public/components/entity_detail_view/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/entity_detail_view/index.tsx index a8ce5d72b8983..d2ce4859e66d1 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/entity_detail_view/index.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/entity_detail_view/index.tsx @@ -5,8 +5,8 @@ * 2.0. */ import { EuiFlexGroup, EuiIcon, EuiLink, EuiPanel } from '@elastic/eui'; -import React from 'react'; import { i18n } from '@kbn/i18n'; +import React from 'react'; import { useStreamsAppBreadcrumbs } from '../../hooks/use_streams_app_breadcrumbs'; import { useStreamsAppRouter } from '../../hooks/use_streams_app_router'; import { EntityOverviewTabList } from '../entity_overview_tab_list'; @@ -34,7 +34,6 @@ export function EntityDetailViewWithoutParams({ }; }) { const router = useStreamsAppRouter(); - useStreamsAppBreadcrumbs(() => { if (!entity.displayName) { return []; diff --git a/x-pack/plugins/logsai/streams_app/public/components/stream_detail_overview/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/stream_detail_overview/index.tsx index 99eb501cdc2bd..44761641a38ff 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/stream_detail_overview/index.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/stream_detail_overview/index.tsx @@ -4,19 +4,18 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { EuiFlexGroup, EuiFlexItem, EuiPanel } from '@elastic/eui'; +import { EuiButton, EuiFlexGroup, EuiFlexItem, EuiPanel } from '@elastic/eui'; import { calculateAuto } from '@kbn/calculate-auto'; import { i18n } from '@kbn/i18n'; import { useAbortableAsync } from '@kbn/observability-utils-browser/hooks/use_abortable_async'; import { useDateRange } from '@kbn/observability-utils-browser/hooks/use_date_range'; -import moment from 'moment'; -import React, { useMemo, useState } from 'react'; import { StreamDefinition } from '@kbn/streams-plugin/common'; -import { entitySourceQuery } from '../../../common/entity_source_query'; +import moment from 'moment'; +import React, { useMemo } from 'react'; import { useKibana } from '../../hooks/use_kibana'; +import { useStreamsAppFetch } from '../../hooks/use_streams_app_fetch'; import { ControlledEsqlChart } from '../esql_chart/controlled_esql_chart'; import { StreamsAppSearchBar } from '../streams_app_search_bar'; -import { useStreamsAppFetch } from '../../hooks/use_streams_app_fetch'; export function StreamDetailOverview({ definition }: { definition?: StreamDefinition }) { const { @@ -25,6 +24,7 @@ export function StreamDetailOverview({ definition }: { definition?: StreamDefini data, dataViews, streams: { streamsRepositoryClient }, + share, }, }, } = useKibana(); @@ -32,25 +32,34 @@ export function StreamDetailOverview({ definition }: { definition?: StreamDefini const { timeRange, absoluteTimeRange: { start, end }, + setTimeRange, } = useDateRange({ data }); - const [displayedKqlFilter, setDisplayedKqlFilter] = useState(''); - const [persistedKqlFilter, setPersistedKqlFilter] = useState(''); - const dataStream = definition?.id; - const queries = useMemo(() => { - if (!dataStream) { + const indexPatterns = useMemo(() => { + if (!definition?.id) { return undefined; } - const baseDslFilter = entitySourceQuery({ - entity: { - _index: dataStream, - }, - }); + const isRoot = definition.id.indexOf('.') === -1; + + const dataStreamOfDefinition = definition.id; + + return isRoot + ? [dataStreamOfDefinition, `${dataStreamOfDefinition}.*`] + : [`${dataStreamOfDefinition}*`]; + }, [definition?.id]); - const indexPatterns = [dataStream]; + const discoverLocator = useMemo( + () => share.url.locators.get('DISCOVER_APP_LOCATOR'), + [share.url.locators] + ); + + const queries = useMemo(() => { + if (!indexPatterns) { + return undefined; + } const baseQuery = `FROM ${indexPatterns.join(', ')}`; @@ -61,18 +70,30 @@ export function StreamDetailOverview({ definition }: { definition?: StreamDefini const histogramQuery = `${baseQuery} | STATS metric = COUNT(*) BY @timestamp = BUCKET(@timestamp, ${bucketSize} seconds)`; return { + baseQuery, histogramQuery, - baseDslFilter, }; - }, [dataStream]); + }, [indexPatterns]); + + const discoverLink = useMemo(() => { + if (!discoverLocator || !queries?.baseQuery) { + return undefined; + } + + return discoverLocator.getRedirectUrl({ + query: { + esql: queries.baseQuery, + }, + }); + }, [queries?.baseQuery, discoverLocator]); const histogramQueryFetch = useStreamsAppFetch( async ({ signal }) => { - if (!queries?.histogramQuery || !dataStream) { + if (!queries?.histogramQuery || !indexPatterns) { return undefined; } - const existingIndices = await dataViews.getExistingIndices([dataStream]); + const existingIndices = await dataViews.getExistingIndices(indexPatterns); if (existingIndices.length === 0) { return undefined; @@ -83,8 +104,6 @@ export function StreamDetailOverview({ definition }: { definition?: StreamDefini body: { operationName: 'get_histogram_for_stream', query: queries.histogramQuery, - filter: queries.baseDslFilter, - kuery: persistedKqlFilter, start, end, }, @@ -92,16 +111,7 @@ export function StreamDetailOverview({ definition }: { definition?: StreamDefini signal, }); }, - [ - dataStream, - dataViews, - streamsRepositoryClient, - queries?.histogramQuery, - persistedKqlFilter, - start, - end, - queries?.baseDslFilter, - ] + [indexPatterns, dataViews, streamsRepositoryClient, queries?.histogramQuery, start, end] ); const dataViewsFetch = useAbortableAsync(() => { @@ -127,12 +137,15 @@ export function StreamDetailOverview({ definition }: { definition?: StreamDefini { - setDisplayedKqlFilter(nextQuery); - }} - onQuerySubmit={() => { - setPersistedKqlFilter(displayedKqlFilter); + onQuerySubmit={({ dateRange }, isUpdate) => { + if (!isUpdate) { + histogramQueryFetch.refresh(); + return; + } + + if (dateRange) { + setTimeRange({ from: dateRange.from, to: dateRange?.to, mode: dateRange.mode }); + } }} onRefresh={() => { histogramQueryFetch.refresh(); @@ -148,6 +161,16 @@ export function StreamDetailOverview({ definition }: { definition?: StreamDefini dateRangeTo={timeRange.to} /> + + {i18n.translate('xpack.streams.streamDetailOverview.openInDiscoverButtonLabel', { + defaultMessage: 'Open in Discover', + })} + diff --git a/x-pack/plugins/logsai/streams_app/public/components/stream_list_view/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/stream_list_view/index.tsx index 6dfb40b586f2a..c67fbca57fa1e 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/stream_list_view/index.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/stream_list_view/index.tsx @@ -6,13 +6,12 @@ */ import React, { useState } from 'react'; import { i18n } from '@kbn/i18n'; -import { EuiFlexGroup } from '@elastic/eui'; +import { EuiFlexGroup, EuiSearchBar } from '@elastic/eui'; import { useKibana } from '../../hooks/use_kibana'; import { useStreamsAppFetch } from '../../hooks/use_streams_app_fetch'; import { StreamsAppPageHeader } from '../streams_app_page_header'; import { StreamsAppPageHeaderTitle } from '../streams_app_page_header/streams_app_page_header_title'; import { StreamsAppPageBody } from '../streams_app_page_body'; -import { StreamsAppSearchBar } from '../streams_app_search_bar'; import { StreamsTable } from '../streams_table'; export function StreamListView() { @@ -26,8 +25,6 @@ export function StreamListView() { const [query, setQuery] = useState(''); - const [submittedQuery, setSubmittedQuery] = useState(''); - const streamsListFetch = useStreamsAppFetch( ({ signal }) => { return streamsRepositoryClient @@ -52,16 +49,16 @@ export function StreamListView() { /> - { - setQuery(next.query); + { - setSubmittedQuery(next.query); + onChange={(nextQuery) => { + setQuery(nextQuery.queryText); }} - query={query} /> - + diff --git a/x-pack/plugins/logsai/streams_app/public/components/streams_app_search_bar/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/streams_app_search_bar/index.tsx index 6b8c257c772e7..563fb752efbd5 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/streams_app_search_bar/index.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/streams_app_search_bar/index.tsx @@ -16,11 +16,11 @@ const parentClassName = css` `; interface Props { - query: string; + query?: string; dateRangeFrom?: string; dateRangeTo?: string; - onQueryChange: (payload: { dateRange?: TimeRange; query: string }) => void; - onQuerySubmit: (payload: { dateRange?: TimeRange; query: string }, isUpdate?: boolean) => void; + onQueryChange?: (payload: { dateRange?: TimeRange; query: string }) => void; + onQuerySubmit?: (payload: { dateRange?: TimeRange; query: string }, isUpdate?: boolean) => void; onRefresh?: Required>['onRefresh']; placeholder?: string; dataViews?: DataView[]; @@ -42,20 +42,25 @@ export function StreamsAppSearchBar({ }, } = useKibana(); - const queryObj = useMemo(() => ({ query, language: 'kuery' }), [query]); + const queryObj = useMemo(() => (query ? { query, language: 'kuery' } : undefined), [query]); + + const showQueryInput = query === undefined; return (
{ - onQuerySubmit({ dateRange, query: (nextQuery?.query as string | undefined) ?? '' }); + onQuerySubmit={({ dateRange, query: nextQuery }, isUpdate) => { + onQuerySubmit?.( + { dateRange, query: (nextQuery?.query as string | undefined) ?? '' }, + isUpdate + ); }} onQueryChange={({ dateRange, query: nextQuery }) => { - onQueryChange({ dateRange, query: (nextQuery?.query as string | undefined) ?? '' }); + onQueryChange?.({ dateRange, query: (nextQuery?.query as string | undefined) ?? '' }); }} query={queryObj} - showQueryInput + showQueryInput={showQueryInput} showFilterBar={false} showQueryMenu={false} showDatePicker={Boolean(dateRangeFrom && dateRangeTo)} diff --git a/x-pack/plugins/logsai/streams_app/public/components/streams_table/index.tsx b/x-pack/plugins/logsai/streams_app/public/components/streams_table/index.tsx index 4998def1140e5..858f0516b417a 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/streams_table/index.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/streams_table/index.tsx @@ -22,8 +22,10 @@ type ListStreamItem = ListStreamResponse['hits'][number]['_source']; export function StreamsTable({ listFetch, + query, }: { listFetch: AbortableAsyncState; + query: string; }) { const router = useStreamsAppRouter(); @@ -31,6 +33,14 @@ export function StreamsTable({ return listFetch.value?.hits.map((hit) => hit._source) ?? []; }, [listFetch.value?.hits]); + const filteredItems = useMemo(() => { + if (!query) { + return items; + } + + return items.filter((item) => item.id.toLowerCase().startsWith(query.toLowerCase())); + }, [query, items]); + const columns = useMemo>>(() => { return [ { @@ -64,7 +74,7 @@ export function StreamsTable({ })} - + ); } diff --git a/x-pack/plugins/logsai/streams_app/public/plugin.ts b/x-pack/plugins/logsai/streams_app/public/plugin.ts index 50d0e4cf88ac6..b8c0e52a0be3e 100644 --- a/x-pack/plugins/logsai/streams_app/public/plugin.ts +++ b/x-pack/plugins/logsai/streams_app/public/plugin.ts @@ -58,6 +58,7 @@ export class StreamsAppPlugin }), app: STREAMS_APP_ID, path: '/', + isTechnicalPreview: true, matchPath(currentPath: string) { return ['/', ''].some((testPath) => currentPath.startsWith(testPath)); }, diff --git a/x-pack/plugins/logsai/streams_app/public/types.ts b/x-pack/plugins/logsai/streams_app/public/types.ts index 9f9f6147bea49..58d44784fe031 100644 --- a/x-pack/plugins/logsai/streams_app/public/types.ts +++ b/x-pack/plugins/logsai/streams_app/public/types.ts @@ -15,6 +15,7 @@ import type { } from '@kbn/observability-shared-plugin/public'; import type { StreamsPluginSetup, StreamsPluginStart } from '@kbn/streams-plugin/public'; import type { UnifiedSearchPublicPluginStart } from '@kbn/unified-search-plugin/public'; +import type { SharePublicSetup, SharePublicStart } from '@kbn/share-plugin/public/plugin'; /* eslint-disable @typescript-eslint/no-empty-interface*/ export interface ConfigSchema {} @@ -25,6 +26,7 @@ export interface StreamsAppSetupDependencies { dataViews: DataViewsPublicPluginSetup; observabilityShared: ObservabilitySharedPluginSetup; unifiedSearch: {}; + share: SharePublicSetup; } export interface StreamsAppStartDependencies { @@ -33,6 +35,7 @@ export interface StreamsAppStartDependencies { dataViews: DataViewsPublicPluginStart; observabilityShared: ObservabilitySharedPluginStart; unifiedSearch: UnifiedSearchPublicPluginStart; + share: SharePublicStart; } export interface StreamsAppPublicSetup {} diff --git a/x-pack/plugins/observability_solution/observability/kibana.jsonc b/x-pack/plugins/observability_solution/observability/kibana.jsonc index 1c09efd7dd6e1..3a888ce14e5ef 100644 --- a/x-pack/plugins/observability_solution/observability/kibana.jsonc +++ b/x-pack/plugins/observability_solution/observability/kibana.jsonc @@ -56,7 +56,8 @@ "serverless", "guidedOnboarding", "observabilityAIAssistant", - "investigate" + "investigate", + "streams" ], "requiredBundles": [ "data", @@ -70,4 +71,4 @@ "common" ] } -} \ No newline at end of file +} diff --git a/x-pack/plugins/observability_solution/observability/public/navigation_tree.ts b/x-pack/plugins/observability_solution/observability/public/navigation_tree.ts index 07bb33ebb5a98..16718648c9724 100644 --- a/x-pack/plugins/observability_solution/observability/public/navigation_tree.ts +++ b/x-pack/plugins/observability_solution/observability/public/navigation_tree.ts @@ -7,7 +7,7 @@ import { i18n } from '@kbn/i18n'; import type { NavigationTreeDefinition } from '@kbn/core-chrome-browser'; import type { AddSolutionNavigationArg } from '@kbn/navigation-plugin/public'; -import { of } from 'rxjs'; +import { map, of } from 'rxjs'; import type { ObservabilityPublicPluginsStart } from './plugin'; const title = i18n.translate( @@ -18,7 +18,7 @@ const title = i18n.translate( ); const icon = 'logoObservability'; -export function createNavTree(pluginsStart: ObservabilityPublicPluginsStart) { +function createNavTree({ streamsAvailable }: { streamsAvailable?: boolean }) { const navTree: NavigationTreeDefinition = { body: [ { @@ -87,6 +87,13 @@ export function createNavTree(pluginsStart: ObservabilityPublicPluginsStart) { link: 'inventory', spaceBefore: 'm', }, + ...(streamsAvailable + ? [ + { + link: 'streams' as const, + }, + ] + : []), { id: 'apm', title: i18n.translate('xpack.observability.obltNav.applications', { @@ -558,6 +565,8 @@ export const createDefinition = ( title, icon: 'logoObservability', homePage: 'observabilityOnboarding', - navigationTree$: of(createNavTree(pluginsStart)), + navigationTree$: (pluginsStart.streams?.status$ || of({ status: 'disabled' as const })).pipe( + map(({ status }) => createNavTree({ streamsAvailable: status === 'enabled' })) + ), dataTestSubj: 'observabilitySideNav', }); diff --git a/x-pack/plugins/observability_solution/observability/public/plugin.ts b/x-pack/plugins/observability_solution/observability/public/plugin.ts index 5866a082556bb..c37f3cc2f624a 100644 --- a/x-pack/plugins/observability_solution/observability/public/plugin.ts +++ b/x-pack/plugins/observability_solution/observability/public/plugin.ts @@ -70,6 +70,7 @@ import type { import type { UiActionsSetup, UiActionsStart } from '@kbn/ui-actions-plugin/public'; import type { UnifiedSearchPublicPluginStart } from '@kbn/unified-search-plugin/public'; import type { UsageCollectionSetup } from '@kbn/usage-collection-plugin/public'; +import type { StreamsPluginStart, StreamsPluginSetup } from '@kbn/streams-plugin/public'; import { observabilityAppId, observabilityFeatureId } from '../common'; import { ALERTS_PATH, @@ -124,6 +125,7 @@ export interface ObservabilityPublicPluginsSetup { licensing: LicensingPluginSetup; serverless?: ServerlessPluginSetup; presentationUtil?: PresentationUtilPluginStart; + streams?: StreamsPluginSetup; } export interface ObservabilityPublicPluginsStart { actionTypeRegistry: ActionTypeRegistryContract; @@ -162,6 +164,7 @@ export interface ObservabilityPublicPluginsStart { dataViewFieldEditor: DataViewFieldEditorStart; toastNotifications: ToastsStart; investigate?: InvestigatePublicStart; + streams?: StreamsPluginStart; } export type ObservabilityPublicStart = ReturnType; diff --git a/x-pack/plugins/serverless_observability/kibana.jsonc b/x-pack/plugins/serverless_observability/kibana.jsonc index fce943c44865a..ad2c1f76ce563 100644 --- a/x-pack/plugins/serverless_observability/kibana.jsonc +++ b/x-pack/plugins/serverless_observability/kibana.jsonc @@ -25,7 +25,9 @@ "discover", "security" ], - "optionalPlugins": [], + "optionalPlugins": [ + "streams" + ], "requiredBundles": [] } -} \ No newline at end of file +} diff --git a/x-pack/plugins/serverless_observability/public/navigation_tree.ts b/x-pack/plugins/serverless_observability/public/navigation_tree.ts index 5df900ee46812..2cd30aaf62fc5 100644 --- a/x-pack/plugins/serverless_observability/public/navigation_tree.ts +++ b/x-pack/plugins/serverless_observability/public/navigation_tree.ts @@ -8,263 +8,278 @@ import { i18n } from '@kbn/i18n'; import type { NavigationTreeDefinition } from '@kbn/core-chrome-browser'; -export const navigationTree: NavigationTreeDefinition = { - body: [ - { type: 'recentlyAccessed' }, - { - type: 'navGroup', - id: 'observability_project_nav', - title: 'Observability', - icon: 'logoObservability', - defaultIsCollapsed: false, - isCollapsible: false, - breadcrumbStatus: 'hidden', - children: [ - { - title: i18n.translate('xpack.serverlessObservability.nav.discover', { - defaultMessage: 'Discover', - }), - link: 'last-used-logs-viewer', - // avoid duplicate "Discover" breadcrumbs - breadcrumbStatus: 'hidden', - renderAs: 'item', - children: [ - { - link: 'discover', - children: [ - { - link: 'observability-logs-explorer', - }, - ], - }, - ], - }, - { - title: i18n.translate('xpack.serverlessObservability.nav.dashboards', { - defaultMessage: 'Dashboards', - }), - link: 'dashboards', - getIsActive: ({ pathNameSerialized, prepend }) => { - return pathNameSerialized.startsWith(prepend('/app/dashboards')); +export const createNavigationTree = ({ + streamsAvailable, +}: { + streamsAvailable?: boolean; +}): NavigationTreeDefinition => { + return { + body: [ + { type: 'recentlyAccessed' }, + { + type: 'navGroup', + id: 'observability_project_nav', + title: 'Observability', + icon: 'logoObservability', + defaultIsCollapsed: false, + isCollapsible: false, + breadcrumbStatus: 'hidden', + children: [ + { + title: i18n.translate('xpack.serverlessObservability.nav.discover', { + defaultMessage: 'Discover', + }), + link: 'last-used-logs-viewer', + // avoid duplicate "Discover" breadcrumbs + breadcrumbStatus: 'hidden', + renderAs: 'item', + children: [ + { + link: 'discover', + children: [ + { + link: 'observability-logs-explorer', + }, + ], + }, + ], }, - }, - { - link: 'observability-overview:alerts', - }, - { - link: 'observability-overview:cases', - renderAs: 'item', - children: [ - { - link: 'observability-overview:cases_configure', + { + title: i18n.translate('xpack.serverlessObservability.nav.dashboards', { + defaultMessage: 'Dashboards', + }), + link: 'dashboards', + getIsActive: ({ pathNameSerialized, prepend }) => { + return pathNameSerialized.startsWith(prepend('/app/dashboards')); }, - { - link: 'observability-overview:cases_create', - }, - ], - }, - { - title: i18n.translate('xpack.serverlessObservability.nav.slo', { - defaultMessage: 'SLOs', - }), - link: 'slo', - }, - { - id: 'aiops', - title: 'AIOps', - link: 'ml:anomalyDetection', - renderAs: 'accordion', - spaceBefore: null, - children: [ - { - title: i18n.translate('xpack.serverlessObservability.nav.ml.jobs', { - defaultMessage: 'Anomaly detection', - }), - link: 'ml:anomalyDetection', - id: 'ml:anomalyDetection', - renderAs: 'item', - children: [ - { - link: 'ml:singleMetricViewer', + }, + { + link: 'observability-overview:alerts', + }, + { + link: 'observability-overview:cases', + renderAs: 'item', + children: [ + { + link: 'observability-overview:cases_configure', + }, + { + link: 'observability-overview:cases_create', + }, + ], + }, + { + title: i18n.translate('xpack.serverlessObservability.nav.slo', { + defaultMessage: 'SLOs', + }), + link: 'slo', + }, + { + id: 'aiops', + title: 'AIOps', + link: 'ml:anomalyDetection', + renderAs: 'accordion', + spaceBefore: null, + children: [ + { + title: i18n.translate('xpack.serverlessObservability.nav.ml.jobs', { + defaultMessage: 'Anomaly detection', + }), + link: 'ml:anomalyDetection', + id: 'ml:anomalyDetection', + renderAs: 'item', + children: [ + { + link: 'ml:singleMetricViewer', + }, + { + link: 'ml:anomalyExplorer', + }, + { + link: 'ml:settings', + }, + ], + }, + { + title: i18n.translate('xpack.serverlessObservability.ml.logRateAnalysis', { + defaultMessage: 'Log rate analysis', + }), + link: 'ml:logRateAnalysis', + getIsActive: ({ pathNameSerialized, prepend }) => { + return pathNameSerialized.includes(prepend('/app/ml/aiops/log_rate_analysis')); }, - { - link: 'ml:anomalyExplorer', + }, + { + title: i18n.translate('xpack.serverlessObservability.ml.changePointDetection', { + defaultMessage: 'Change point detection', + }), + link: 'ml:changePointDetections', + getIsActive: ({ pathNameSerialized, prepend }) => { + return pathNameSerialized.includes( + prepend('/app/ml/aiops/change_point_detection') + ); }, + }, + { + title: i18n.translate('xpack.serverlessObservability.nav.ml.job.notifications', { + defaultMessage: 'Job notifications', + }), + link: 'ml:notifications', + }, + ], + }, + { link: 'inventory', spaceBefore: 'm' }, + ...(streamsAvailable + ? [ { - link: 'ml:settings', + link: 'streams' as const, + }, + ] + : []), + { + id: 'apm', + title: i18n.translate('xpack.serverlessObservability.nav.applications', { + defaultMessage: 'Applications', + }), + link: 'apm:services', + renderAs: 'accordion', + children: [ + { + link: 'apm:services', + getIsActive: ({ pathNameSerialized }) => { + const regex = /app\/apm\/.*service.*/; + return regex.test(pathNameSerialized); }, - ], - }, - { - title: i18n.translate('xpack.serverlessObservability.ml.logRateAnalysis', { - defaultMessage: 'Log rate analysis', - }), - link: 'ml:logRateAnalysis', - getIsActive: ({ pathNameSerialized, prepend }) => { - return pathNameSerialized.includes(prepend('/app/ml/aiops/log_rate_analysis')); }, - }, - { - title: i18n.translate('xpack.serverlessObservability.ml.changePointDetection', { - defaultMessage: 'Change point detection', - }), - link: 'ml:changePointDetections', - getIsActive: ({ pathNameSerialized, prepend }) => { - return pathNameSerialized.includes(prepend('/app/ml/aiops/change_point_detection')); + { + link: 'apm:traces', + getIsActive: ({ pathNameSerialized, prepend }) => { + return pathNameSerialized.startsWith(prepend('/app/apm/traces')); + }, }, - }, - { - title: i18n.translate('xpack.serverlessObservability.nav.ml.job.notifications', { - defaultMessage: 'Job notifications', - }), - link: 'ml:notifications', - }, - ], - }, - { link: 'inventory', spaceBefore: 'm' }, - { - id: 'apm', - title: i18n.translate('xpack.serverlessObservability.nav.applications', { - defaultMessage: 'Applications', - }), - link: 'apm:services', - renderAs: 'accordion', - children: [ - { - link: 'apm:services', - getIsActive: ({ pathNameSerialized }) => { - const regex = /app\/apm\/.*service.*/; - return regex.test(pathNameSerialized); + { + link: 'apm:dependencies', + getIsActive: ({ pathNameSerialized, prepend }) => { + return pathNameSerialized.startsWith(prepend('/app/apm/dependencies')); + }, }, - }, - { - link: 'apm:traces', - getIsActive: ({ pathNameSerialized, prepend }) => { - return pathNameSerialized.startsWith(prepend('/app/apm/traces')); + { + link: 'apm:settings', + sideNavStatus: 'hidden', // only to be considered in the breadcrumbs }, - }, - { - link: 'apm:dependencies', - getIsActive: ({ pathNameSerialized, prepend }) => { - return pathNameSerialized.startsWith(prepend('/app/apm/dependencies')); + ], + }, + { + id: 'metrics', + title: i18n.translate('xpack.serverlessObservability.nav.infrastructure', { + defaultMessage: 'Infrastructure', + }), + link: 'metrics:inventory', + renderAs: 'accordion', + children: [ + { + link: 'metrics:inventory', + getIsActive: ({ pathNameSerialized, prepend }) => { + return pathNameSerialized.startsWith(prepend('/app/metrics/inventory')); + }, }, - }, - { - link: 'apm:settings', - sideNavStatus: 'hidden', // only to be considered in the breadcrumbs - }, - ], - }, - { - id: 'metrics', - title: i18n.translate('xpack.serverlessObservability.nav.infrastructure', { - defaultMessage: 'Infrastructure', - }), - link: 'metrics:inventory', - renderAs: 'accordion', - children: [ - { - link: 'metrics:inventory', - getIsActive: ({ pathNameSerialized, prepend }) => { - return pathNameSerialized.startsWith(prepend('/app/metrics/inventory')); + { + link: 'metrics:hosts', + getIsActive: ({ pathNameSerialized, prepend }) => { + return pathNameSerialized.startsWith(prepend('/app/metrics/hosts')); + }, }, - }, - { - link: 'metrics:hosts', - getIsActive: ({ pathNameSerialized, prepend }) => { - return pathNameSerialized.startsWith(prepend('/app/metrics/hosts')); + { + link: 'metrics:settings', + sideNavStatus: 'hidden', // only to be considered in the breadcrumbs }, - }, - { - link: 'metrics:settings', - sideNavStatus: 'hidden', // only to be considered in the breadcrumbs - }, - { - link: 'metrics:assetDetails', - sideNavStatus: 'hidden', // only to be considered in the breadcrumbs - }, - ], - }, - { - id: 'synthetics', - title: i18n.translate('xpack.serverlessObservability.nav.synthetics', { - defaultMessage: 'Synthetics', - }), - renderAs: 'accordion', - breadcrumbStatus: 'hidden', - children: [ - { - title: i18n.translate('xpack.serverlessObservability.nav.synthetics.overviewItem', { - defaultMessage: 'Overview', - }), - id: 'synthetics-overview', - link: 'synthetics:overview', - breadcrumbStatus: 'hidden', - }, - { - link: 'synthetics:certificates', - title: i18n.translate( - 'xpack.serverlessObservability.nav.synthetics.certificatesItem', - { - defaultMessage: 'TLS Certificates', - } - ), - id: 'synthetics-certificates', - breadcrumbStatus: 'hidden', - }, - ], - }, - ], - }, - ], - footer: [ - { - type: 'navItem', - title: i18n.translate('xpack.serverlessObservability.nav.getStarted', { - defaultMessage: 'Add data', - }), - link: 'observabilityOnboarding', - icon: 'launch', - }, - { - type: 'navItem', - id: 'devTools', - title: i18n.translate('xpack.serverlessObservability.nav.devTools', { - defaultMessage: 'Developer tools', - }), - link: 'dev_tools', - icon: 'editorCodeBlock', - }, - { - type: 'navGroup', - id: 'project_settings_project_nav', - title: i18n.translate('xpack.serverlessObservability.nav.projectSettings', { - defaultMessage: 'Project settings', - }), - icon: 'gear', - breadcrumbStatus: 'hidden', - children: [ - { - link: 'management', - title: i18n.translate('xpack.serverlessObservability.nav.mngt', { - defaultMessage: 'Management', - }), - }, - { - link: 'integrations', - }, - { - link: 'fleet', - }, - { - id: 'cloudLinkUserAndRoles', - cloudLink: 'userAndRoles', - }, - { - id: 'cloudLinkBilling', - cloudLink: 'billingAndSub', - }, - ], - }, - ], + { + link: 'metrics:assetDetails', + sideNavStatus: 'hidden', // only to be considered in the breadcrumbs + }, + ], + }, + { + id: 'synthetics', + title: i18n.translate('xpack.serverlessObservability.nav.synthetics', { + defaultMessage: 'Synthetics', + }), + renderAs: 'accordion', + breadcrumbStatus: 'hidden', + children: [ + { + title: i18n.translate('xpack.serverlessObservability.nav.synthetics.overviewItem', { + defaultMessage: 'Overview', + }), + id: 'synthetics-overview', + link: 'synthetics:overview', + breadcrumbStatus: 'hidden', + }, + { + link: 'synthetics:certificates', + title: i18n.translate( + 'xpack.serverlessObservability.nav.synthetics.certificatesItem', + { + defaultMessage: 'TLS Certificates', + } + ), + id: 'synthetics-certificates', + breadcrumbStatus: 'hidden', + }, + ], + }, + ], + }, + ], + footer: [ + { + type: 'navItem', + title: i18n.translate('xpack.serverlessObservability.nav.getStarted', { + defaultMessage: 'Add data', + }), + link: 'observabilityOnboarding', + icon: 'launch', + }, + { + type: 'navItem', + id: 'devTools', + title: i18n.translate('xpack.serverlessObservability.nav.devTools', { + defaultMessage: 'Developer tools', + }), + link: 'dev_tools', + icon: 'editorCodeBlock', + }, + { + type: 'navGroup', + id: 'project_settings_project_nav', + title: i18n.translate('xpack.serverlessObservability.nav.projectSettings', { + defaultMessage: 'Project settings', + }), + icon: 'gear', + breadcrumbStatus: 'hidden', + children: [ + { + link: 'management', + title: i18n.translate('xpack.serverlessObservability.nav.mngt', { + defaultMessage: 'Management', + }), + }, + { + link: 'integrations', + }, + { + link: 'fleet', + }, + { + id: 'cloudLinkUserAndRoles', + cloudLink: 'userAndRoles', + }, + { + id: 'cloudLinkBilling', + cloudLink: 'billingAndSub', + }, + ], + }, + ], + }; }; diff --git a/x-pack/plugins/serverless_observability/public/plugin.ts b/x-pack/plugins/serverless_observability/public/plugin.ts index 05d598b2b3a7e..774a76749f8d6 100644 --- a/x-pack/plugins/serverless_observability/public/plugin.ts +++ b/x-pack/plugins/serverless_observability/public/plugin.ts @@ -8,8 +8,8 @@ import { CoreSetup, CoreStart, Plugin } from '@kbn/core/public'; import { i18n } from '@kbn/i18n'; import { appCategories, appIds } from '@kbn/management-cards-navigation'; -import { of } from 'rxjs'; -import { navigationTree } from './navigation_tree'; +import { map, of } from 'rxjs'; +import { createNavigationTree } from './navigation_tree'; import { createObservabilityDashboardRegistration } from './logs_signal/overview_registration'; import { ServerlessObservabilityPublicSetup, @@ -50,7 +50,11 @@ export class ServerlessObservabilityPlugin setupDeps: ServerlessObservabilityPublicStartDependencies ): ServerlessObservabilityPublicStart { const { serverless, management, security } = setupDeps; - const navigationTree$ = of(navigationTree); + const navigationTree$ = (setupDeps.streams?.status$ || of({ status: 'disabled' })).pipe( + map(({ status }) => { + return createNavigationTree({ streamsAvailable: status === 'enabled' }); + }) + ); serverless.setProjectHome('/app/observability/landing'); serverless.initNavigation('oblt', navigationTree$, { dataTestSubj: 'svlObservabilitySideNav' }); const aiAssistantIsEnabled = core.application.capabilities.observabilityAIAssistant?.show; diff --git a/x-pack/plugins/serverless_observability/public/types.ts b/x-pack/plugins/serverless_observability/public/types.ts index c93865f0f596e..23da6c12637d3 100644 --- a/x-pack/plugins/serverless_observability/public/types.ts +++ b/x-pack/plugins/serverless_observability/public/types.ts @@ -8,13 +8,14 @@ import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; import type { DiscoverSetup } from '@kbn/discover-plugin/public'; import type { ManagementSetup, ManagementStart } from '@kbn/management-plugin/public'; -import { ObservabilityPublicSetup } from '@kbn/observability-plugin/public'; -import { +import type { ObservabilityPublicSetup } from '@kbn/observability-plugin/public'; +import type { ObservabilitySharedPluginSetup, ObservabilitySharedPluginStart, } from '@kbn/observability-shared-plugin/public'; -import { SecurityPluginStart } from '@kbn/security-plugin/public'; -import { ServerlessPluginSetup, ServerlessPluginStart } from '@kbn/serverless/public'; +import type { SecurityPluginStart } from '@kbn/security-plugin/public'; +import type { ServerlessPluginSetup, ServerlessPluginStart } from '@kbn/serverless/public'; +import type { StreamsPluginStart, StreamsPluginSetup } from '@kbn/streams-plugin/public'; // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface ServerlessObservabilityPublicSetup {} @@ -28,6 +29,7 @@ export interface ServerlessObservabilityPublicSetupDependencies { serverless: ServerlessPluginSetup; management: ManagementSetup; discover: DiscoverSetup; + streams?: StreamsPluginSetup; } export interface ServerlessObservabilityPublicStartDependencies { @@ -36,4 +38,5 @@ export interface ServerlessObservabilityPublicStartDependencies { management: ManagementStart; data: DataPublicPluginStart; security: SecurityPluginStart; + streams?: StreamsPluginStart; } diff --git a/x-pack/plugins/streams/public/plugin.ts b/x-pack/plugins/streams/public/plugin.ts index 8016e342b8c8e..f404a026dd7b2 100644 --- a/x-pack/plugins/streams/public/plugin.ts +++ b/x-pack/plugins/streams/public/plugin.ts @@ -9,6 +9,8 @@ import { CoreSetup, CoreStart, PluginInitializerContext } from '@kbn/core/public import { Logger } from '@kbn/logging'; import { createRepositoryClient } from '@kbn/server-route-repository-client'; +import { from, share, startWith } from 'rxjs'; +import { once } from 'lodash'; import type { StreamsPublicConfig } from '../common/config'; import { StreamsPluginClass, StreamsPluginSetup, StreamsPluginStart } from './types'; import { StreamsRepositoryClient } from './api'; @@ -26,14 +28,29 @@ export class Plugin implements StreamsPluginClass { setup(core: CoreSetup<{}>, pluginSetup: {}): StreamsPluginSetup { this.repositoryClient = createRepositoryClient(core); - return {}; + return { + status$: createStatusObservable(this.repositoryClient), + }; } start(core: CoreStart, pluginsStart: {}): StreamsPluginStart { return { streamsRepositoryClient: this.repositoryClient, + status$: createStatusObservable(this.repositoryClient), }; } stop() {} } + +const createStatusObservable = once((repositoryClient: StreamsRepositoryClient) => { + return from( + repositoryClient + .fetch('GET /internal/streams/_settings', { + signal: new AbortController().signal, + }) + .then((response) => ({ + status: response.enabled ? ('enabled' as const) : ('disabled' as const), + })) + ).pipe(startWith({ status: 'unknown' as const }), share()); +}); diff --git a/x-pack/plugins/streams/public/types.ts b/x-pack/plugins/streams/public/types.ts index 929c89fb6ec98..fc88f2a6c20fe 100644 --- a/x-pack/plugins/streams/public/types.ts +++ b/x-pack/plugins/streams/public/types.ts @@ -6,13 +6,16 @@ */ import type { Plugin as PluginClass } from '@kbn/core/public'; +import { Observable } from 'rxjs'; import type { StreamsRepositoryClient } from './api'; -// eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface StreamsPluginSetup {} +export interface StreamsPluginSetup { + status$: Observable<{ status: 'unknown' | 'enabled' | 'disabled' }>; +} export interface StreamsPluginStart { streamsRepositoryClient: StreamsRepositoryClient; + status$: Observable<{ status: 'unknown' | 'enabled' | 'disabled' }>; } export type StreamsPluginClass = PluginClass; diff --git a/x-pack/plugins/streams/server/routes/index.ts b/x-pack/plugins/streams/server/routes/index.ts index b6b5feb7205ba..6dcb5f70d588b 100644 --- a/x-pack/plugins/streams/server/routes/index.ts +++ b/x-pack/plugins/streams/server/routes/index.ts @@ -10,6 +10,7 @@ import { enableStreamsRoute } from './streams/enable'; import { forkStreamsRoute } from './streams/fork'; import { listStreamsRoute } from './streams/list'; import { readStreamRoute } from './streams/read'; +import { streamsSettingsRoutes } from './streams/settings'; export const StreamsRouteRepository = { ...enableStreamsRoute, @@ -17,6 +18,7 @@ export const StreamsRouteRepository = { ...readStreamRoute, ...listStreamsRoute, ...esqlRoutes, + ...streamsSettingsRoutes, }; export type StreamsRouteRepository = typeof StreamsRouteRepository; diff --git a/x-pack/plugins/streams/server/routes/streams/settings.ts b/x-pack/plugins/streams/server/routes/streams/settings.ts new file mode 100644 index 0000000000000..7ca5779d61ca0 --- /dev/null +++ b/x-pack/plugins/streams/server/routes/streams/settings.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { STREAMS_INDEX } from '../../../common/constants'; +import { createServerRoute } from '../create_server_route'; + +export const getStreamsSettingsRoute = createServerRoute({ + endpoint: 'GET /internal/streams/_settings', + options: { + security: { + authz: { + requiredPrivileges: ['streams_read'], + }, + }, + }, + handler: async ({ request, getScopedClients }): Promise<{ enabled: boolean }> => { + const { scopedClusterClient } = await getScopedClients({ request }); + + return { + enabled: await scopedClusterClient.asInternalUser.indices.exists({ + index: STREAMS_INDEX, + }), + }; + }, +}); + +export const streamsSettingsRoutes = { + ...getStreamsSettingsRoute, +}; From 34f3c44fe707d860c77f038fb148fe40b05463ae Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 12 Nov 2024 12:36:45 +0100 Subject: [PATCH 21/95] fix permissions --- x-pack/plugins/streams/server/plugin.ts | 23 +------------------ .../streams/server/routes/streams/delete.ts | 7 +++++- .../streams/server/routes/streams/edit.ts | 7 +++++- .../streams/server/routes/streams/enable.ts | 7 +++++- .../streams/server/routes/streams/fork.ts | 7 +++++- .../streams/server/routes/streams/list.ts | 7 +++++- .../streams/server/routes/streams/read.ts | 7 +++++- .../streams/server/routes/streams/resync.ts | 7 +++++- 8 files changed, 43 insertions(+), 29 deletions(-) diff --git a/x-pack/plugins/streams/server/plugin.ts b/x-pack/plugins/streams/server/plugin.ts index 478c77c6ca691..2d9f911440b1a 100644 --- a/x-pack/plugins/streams/server/plugin.ts +++ b/x-pack/plugins/streams/server/plugin.ts @@ -66,28 +66,7 @@ export class StreamsPlugin app: ['streams', 'kibana'], catalogue: ['streams', 'observability'], category: DEFAULT_APP_CATEGORIES.observability, - privileges: { - all: { - app: ['streams', 'kibana'], - catalogue: ['streams', 'observability'], - savedObject: { - all: [], - read: [], - }, - ui: ['read', 'write'], - api: ['streams_write', 'streams_read'], - }, - read: { - app: ['streams', 'kibana'], - catalogue: ['streams', 'observability'], - api: ['streams_read'], - ui: ['read'], - savedObject: { - all: [], - read: [], - }, - }, - }, + privileges: null, }); registerRoutes({ diff --git a/x-pack/plugins/streams/server/routes/streams/delete.ts b/x-pack/plugins/streams/server/routes/streams/delete.ts index cea5275e9b409..3820975dbe16a 100644 --- a/x-pack/plugins/streams/server/routes/streams/delete.ts +++ b/x-pack/plugins/streams/server/routes/streams/delete.ts @@ -23,9 +23,14 @@ export const deleteStreamRoute = createServerRoute({ endpoint: 'DELETE /api/streams/{id} 2023-10-31', options: { access: 'public', + availability: { + stability: 'experimental', + }, security: { authz: { - requiredPrivileges: ['streams_write'], + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', }, }, }, diff --git a/x-pack/plugins/streams/server/routes/streams/edit.ts b/x-pack/plugins/streams/server/routes/streams/edit.ts index 5c027a9985a07..b82b4d54044da 100644 --- a/x-pack/plugins/streams/server/routes/streams/edit.ts +++ b/x-pack/plugins/streams/server/routes/streams/edit.ts @@ -31,9 +31,14 @@ export const editStreamRoute = createServerRoute({ endpoint: 'PUT /api/streams/{id} 2023-10-31', options: { access: 'public', + availability: { + stability: 'experimental', + }, security: { authz: { - requiredPrivileges: ['streams_write'], + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', }, }, }, diff --git a/x-pack/plugins/streams/server/routes/streams/enable.ts b/x-pack/plugins/streams/server/routes/streams/enable.ts index 3e122b855575d..27d8929b28e50 100644 --- a/x-pack/plugins/streams/server/routes/streams/enable.ts +++ b/x-pack/plugins/streams/server/routes/streams/enable.ts @@ -17,9 +17,14 @@ export const enableStreamsRoute = createServerRoute({ params: z.object({}), options: { access: 'public', + availability: { + stability: 'experimental', + }, security: { authz: { - requiredPrivileges: ['streams_write'], + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', }, }, }, diff --git a/x-pack/plugins/streams/server/routes/streams/fork.ts b/x-pack/plugins/streams/server/routes/streams/fork.ts index 1ff507e530065..44f4052878003 100644 --- a/x-pack/plugins/streams/server/routes/streams/fork.ts +++ b/x-pack/plugins/streams/server/routes/streams/fork.ts @@ -22,9 +22,14 @@ export const forkStreamsRoute = createServerRoute({ endpoint: 'POST /api/streams/{id}/_fork 2023-10-31', options: { access: 'public', + availability: { + stability: 'experimental', + }, security: { authz: { - requiredPrivileges: ['streams_write'], + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', }, }, }, diff --git a/x-pack/plugins/streams/server/routes/streams/list.ts b/x-pack/plugins/streams/server/routes/streams/list.ts index 37446788ad1de..2e4f13a89bb41 100644 --- a/x-pack/plugins/streams/server/routes/streams/list.ts +++ b/x-pack/plugins/streams/server/routes/streams/list.ts @@ -14,9 +14,14 @@ export const listStreamsRoute = createServerRoute({ endpoint: 'GET /api/streams 2023-10-31', options: { access: 'public', + availability: { + stability: 'experimental', + }, security: { authz: { - requiredPrivileges: ['streams_read'], + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', }, }, }, diff --git a/x-pack/plugins/streams/server/routes/streams/read.ts b/x-pack/plugins/streams/server/routes/streams/read.ts index 64914a3d9c741..5ea2aaf5f2542 100644 --- a/x-pack/plugins/streams/server/routes/streams/read.ts +++ b/x-pack/plugins/streams/server/routes/streams/read.ts @@ -14,9 +14,14 @@ export const readStreamRoute = createServerRoute({ endpoint: 'GET /api/streams/{id} 2023-10-31', options: { access: 'public', + availability: { + stability: 'experimental', + }, security: { authz: { - requiredPrivileges: ['streams_read'], + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', }, }, }, diff --git a/x-pack/plugins/streams/server/routes/streams/resync.ts b/x-pack/plugins/streams/server/routes/streams/resync.ts index badba1fe90346..2365252ab00e6 100644 --- a/x-pack/plugins/streams/server/routes/streams/resync.ts +++ b/x-pack/plugins/streams/server/routes/streams/resync.ts @@ -13,9 +13,14 @@ export const resyncStreamsRoute = createServerRoute({ endpoint: 'POST /api/streams/_resync 2023-10-31', options: { access: 'public', + availability: { + stability: 'experimental', + }, security: { authz: { - requiredPrivileges: ['streams_write'], + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', }, }, }, From 0b7de2475f6f87ea4f70b950d05c16b2163ac15d Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 12 Nov 2024 12:48:58 +0100 Subject: [PATCH 22/95] remove feature completely --- x-pack/plugins/streams/server/plugin.ts | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/x-pack/plugins/streams/server/plugin.ts b/x-pack/plugins/streams/server/plugin.ts index 2d9f911440b1a..ef070984803d5 100644 --- a/x-pack/plugins/streams/server/plugin.ts +++ b/x-pack/plugins/streams/server/plugin.ts @@ -8,7 +8,6 @@ import { CoreSetup, CoreStart, - DEFAULT_APP_CATEGORIES, KibanaRequest, Logger, Plugin, @@ -59,16 +58,6 @@ export class StreamsPlugin logger: this.logger, } as StreamsServer; - plugins.features.registerKibanaFeature({ - id: 'streams', - name: 'Streams', - order: 1500, - app: ['streams', 'kibana'], - catalogue: ['streams', 'observability'], - category: DEFAULT_APP_CATEGORIES.observability, - privileges: null, - }); - registerRoutes({ repository: StreamsRouteRepository, dependencies: { From 76ed81d675d60daf7aa510d572bf21f0fa2e5b78 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 12 Nov 2024 12:59:55 +0100 Subject: [PATCH 23/95] cleanup --- x-pack/plugins/streams/kibana.jsonc | 1 - x-pack/plugins/streams/server/types.ts | 2 -- 2 files changed, 3 deletions(-) diff --git a/x-pack/plugins/streams/kibana.jsonc b/x-pack/plugins/streams/kibana.jsonc index d97a0b371bb27..8e428c0a285a1 100644 --- a/x-pack/plugins/streams/kibana.jsonc +++ b/x-pack/plugins/streams/kibana.jsonc @@ -17,7 +17,6 @@ "usageCollection", "licensing", "taskManager", - "features" ], "optionalPlugins": [ "cloud", diff --git a/x-pack/plugins/streams/server/types.ts b/x-pack/plugins/streams/server/types.ts index f1cfcb08a2649..f119faa0ed010 100644 --- a/x-pack/plugins/streams/server/types.ts +++ b/x-pack/plugins/streams/server/types.ts @@ -16,7 +16,6 @@ import { TaskManagerSetupContract, TaskManagerStartContract, } from '@kbn/task-manager-plugin/server'; -import { FeaturesPluginSetup } from '@kbn/features-plugin/server'; import { StreamsConfig } from '../common/config'; export interface StreamsServer { @@ -36,7 +35,6 @@ export interface ElasticsearchAccessorOptions { export interface StreamsPluginSetupDependencies { encryptedSavedObjects: EncryptedSavedObjectsPluginSetup; taskManager: TaskManagerSetupContract; - features: FeaturesPluginSetup; } export interface StreamsPluginStartDependencies { From 2427d2e5089c478d25dd8c9aa7dabb6520b9eeb2 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 12 Nov 2024 12:11:29 +0000 Subject: [PATCH 24/95] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/plugins/streams/tsconfig.json | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/streams/tsconfig.json b/x-pack/plugins/streams/tsconfig.json index e0ceaf8197fa6..c2fde35f9ca22 100644 --- a/x-pack/plugins/streams/tsconfig.json +++ b/x-pack/plugins/streams/tsconfig.json @@ -27,6 +27,5 @@ "@kbn/zod", "@kbn/encrypted-saved-objects-plugin", "@kbn/licensing-plugin", - "@kbn/features-plugin" ] } From e6bb640afef2ec62400b17bb9c60d20c67e62ce0 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Tue, 12 Nov 2024 15:32:06 +0100 Subject: [PATCH 25/95] [Observability] Split up observability-utils package --- package.json | 4 +- tsconfig.base.json | 8 +- .../observability_utils/README.md | 5 - .../client/create_observability_es_client.ts | 88 ------------ .../chart/utils.ts | 0 .../hooks/use_abort_controller.ts | 3 + .../hooks/use_abortable_async.ts | 38 +++-- .../hooks/use_date_range.ts | 63 +++++++++ .../hooks/use_local_storage.ts | 60 ++++++++ .../hooks/use_theme.ts | 0 .../jest.config.js | 14 ++ .../observability_utils_browser/kibana.jsonc | 5 + .../observability_utils_browser/package.json | 6 + .../observability_utils_browser/tsconfig.json | 23 ++++ .../utils/ui_settings/get_timezone.ts | 17 +++ .../array/join_by_key.test.ts | 0 .../array/join_by_key.ts | 0 .../entities/get_entity_kuery.ts | 14 ++ .../es/format_value_for_kql.ts | 10 ++ .../es/queries/entity_query.ts | 24 ++++ .../es/queries/exclude_frozen_query.ts | 0 .../es/queries/exclude_tiers_query.ts | 0 .../es/queries/kql_query.ts | 0 .../es/queries/range_query.ts | 0 .../es/queries/term_query.ts | 0 .../format/integer.ts | 11 ++ .../jest.config.js | 4 +- .../kibana.jsonc | 2 +- .../llm/log_analysis/document_analysis.ts | 24 ++++ .../highlight_patterns_from_regex.ts | 51 +++++++ .../merge_sample_documents_with_field_caps.ts | 78 +++++++++++ .../sort_and_truncate_analyzed_fields.ts | 52 +++++++ .../llm/short_id_table.test.ts | 48 +++++++ .../llm/short_id_table.ts | 56 ++++++++ .../ml/p_value_to_label.ts | 19 +++ .../object/flatten_object.test.ts | 0 .../object/flatten_object.ts | 0 .../object/merge_plain_object.test.ts | 0 .../object/merge_plain_objects.ts | 0 .../object/unflatten_object.test.ts | 0 .../object/unflatten_object.ts | 1 - .../package.json | 4 +- .../tsconfig.json | 7 +- .../entities/analyze_documents.ts | 84 +++++++++++ .../entities/get_data_streams_for_entity.ts | 63 +++++++++ .../entities/signals/get_alerts_for_entity.ts | 57 ++++++++ .../signals/get_anomalies_for_entity.ts | 16 +++ .../entities/signals/get_slos_for_entity.ts | 80 +++++++++++ .../client/create_observability_es_client.ts | 130 ++++++++++++++++++ .../es}/esql_result_to_plain_objects.test.ts | 0 .../es}/esql_result_to_plain_objects.ts | 0 .../es/queries/exclude_frozen_query.ts | 23 ++++ .../es/queries/kql_query.ts | 17 +++ .../es/queries/range_query.ts | 25 ++++ .../es/queries/term_query.ts | 24 ++++ .../observability_utils_server/jest.config.js | 12 ++ .../observability_utils_server/kibana.jsonc | 5 + .../observability_utils_server/package.json | 6 + .../observability_utils_server/tsconfig.json | 28 ++++ .../create_apm_event_client/index.ts | 2 +- .../server/lib/helpers/tier_filter.ts | 2 +- .../server/utils/unflatten_known_fields.ts | 4 +- .../entities/get_data_stream_types.test.ts | 2 +- .../routes/entities/get_data_stream_types.ts | 2 +- .../routes/entities/get_latest_entity.ts | 4 +- .../infra/server/routes/entities/index.ts | 2 +- .../common/utils/unflatten_entity.ts | 2 +- .../public/hooks/use_entity_manager.ts | 2 +- .../hooks/use_inventory_abortable_async.ts | 2 +- .../routes/entities/get_entity_groups.ts | 4 +- .../routes/entities/get_entity_types.ts | 2 +- .../routes/entities/get_latest_entities.ts | 4 +- .../inventory/server/routes/entities/route.ts | 4 +- .../server/routes/has_data/get_has_data.ts | 4 +- .../inventory/server/routes/has_data/route.ts | 2 +- .../register_embeddable_item.tsx | 2 +- .../items/esql_item/register_esql_item.tsx | 2 +- .../esql_widget_preview.tsx | 2 +- .../events_timeline/events_timeline.tsx | 2 +- yarn.lock | 10 +- 80 files changed, 1227 insertions(+), 144 deletions(-) delete mode 100644 x-pack/packages/observability/observability_utils/README.md delete mode 100644 x-pack/packages/observability/observability_utils/es/client/create_observability_es_client.ts rename x-pack/packages/observability/observability_utils/{ => observability_utils_browser}/chart/utils.ts (100%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_browser}/hooks/use_abort_controller.ts (92%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_browser}/hooks/use_abortable_async.ts (72%) create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_date_range.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_local_storage.ts rename x-pack/packages/observability/observability_utils/{ => observability_utils_browser}/hooks/use_theme.ts (100%) create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_browser/jest.config.js create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_browser/kibana.jsonc create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_browser/package.json create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_browser/tsconfig.json create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_browser/utils/ui_settings/get_timezone.ts rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/array/join_by_key.test.ts (100%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/array/join_by_key.ts (100%) create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_common/entities/get_entity_kuery.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_common/es/format_value_for_kql.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_common/es/queries/entity_query.ts rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/es/queries/exclude_frozen_query.ts (100%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/es/queries/exclude_tiers_query.ts (100%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/es/queries/kql_query.ts (100%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/es/queries/range_query.ts (100%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/es/queries/term_query.ts (100%) create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_common/format/integer.ts rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/jest.config.js (84%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/kibana.jsonc (61%) create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/document_analysis.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/highlight_patterns_from_regex.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/merge_sample_documents_with_field_caps.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/sort_and_truncate_analyzed_fields.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_common/llm/short_id_table.test.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_common/llm/short_id_table.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_common/ml/p_value_to_label.ts rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/object/flatten_object.test.ts (100%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/object/flatten_object.ts (100%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/object/merge_plain_object.test.ts (100%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/object/merge_plain_objects.ts (100%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/object/unflatten_object.test.ts (100%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/object/unflatten_object.ts (99%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/package.json (62%) rename x-pack/packages/observability/observability_utils/{ => observability_utils_common}/tsconfig.json (70%) create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/entities/analyze_documents.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/entities/get_data_streams_for_entity.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/entities/signals/get_alerts_for_entity.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/entities/signals/get_anomalies_for_entity.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/entities/signals/get_slos_for_entity.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts rename x-pack/packages/observability/observability_utils/{es/utils => observability_utils_server/es}/esql_result_to_plain_objects.test.ts (100%) rename x-pack/packages/observability/observability_utils/{es/utils => observability_utils_server/es}/esql_result_to_plain_objects.ts (100%) create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/exclude_frozen_query.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/kql_query.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/range_query.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/term_query.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/jest.config.js create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/kibana.jsonc create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/package.json create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/tsconfig.json diff --git a/package.json b/package.json index 87905c955d2d8..3efe7b2725ab2 100644 --- a/package.json +++ b/package.json @@ -701,7 +701,9 @@ "@kbn/observability-plugin": "link:x-pack/plugins/observability_solution/observability", "@kbn/observability-shared-plugin": "link:x-pack/plugins/observability_solution/observability_shared", "@kbn/observability-synthetics-test-data": "link:x-pack/packages/observability/synthetics_test_data", - "@kbn/observability-utils": "link:x-pack/packages/observability/observability_utils", + "@kbn/observability-utils-browser": "link:x-pack/packages/observability/observability_utils/observability_utils_browser", + "@kbn/observability-utils-common": "link:x-pack/packages/observability/observability_utils/observability_utils_common", + "@kbn/observability-utils-server": "link:x-pack/packages/observability/observability_utils/observability_utils_server", "@kbn/oidc-provider-plugin": "link:x-pack/test/security_api_integration/plugins/oidc_provider", "@kbn/open-telemetry-instrumented-plugin": "link:test/common/plugins/otel_metrics", "@kbn/openapi-common": "link:packages/kbn-openapi-common", diff --git a/tsconfig.base.json b/tsconfig.base.json index 68faf44ed74d4..381f425a904cd 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1328,8 +1328,12 @@ "@kbn/observability-shared-plugin/*": ["x-pack/plugins/observability_solution/observability_shared/*"], "@kbn/observability-synthetics-test-data": ["x-pack/packages/observability/synthetics_test_data"], "@kbn/observability-synthetics-test-data/*": ["x-pack/packages/observability/synthetics_test_data/*"], - "@kbn/observability-utils": ["x-pack/packages/observability/observability_utils"], - "@kbn/observability-utils/*": ["x-pack/packages/observability/observability_utils/*"], + "@kbn/observability-utils-browser": ["x-pack/packages/observability/observability_utils/observability_utils_browser"], + "@kbn/observability-utils-browser/*": ["x-pack/packages/observability/observability_utils/observability_utils_browser/*"], + "@kbn/observability-utils-common": ["x-pack/packages/observability/observability_utils/observability_utils_common"], + "@kbn/observability-utils-common/*": ["x-pack/packages/observability/observability_utils/observability_utils_common/*"], + "@kbn/observability-utils-server": ["x-pack/packages/observability/observability_utils/observability_utils_server"], + "@kbn/observability-utils-server/*": ["x-pack/packages/observability/observability_utils/observability_utils_server/*"], "@kbn/oidc-provider-plugin": ["x-pack/test/security_api_integration/plugins/oidc_provider"], "@kbn/oidc-provider-plugin/*": ["x-pack/test/security_api_integration/plugins/oidc_provider/*"], "@kbn/open-telemetry-instrumented-plugin": ["test/common/plugins/otel_metrics"], diff --git a/x-pack/packages/observability/observability_utils/README.md b/x-pack/packages/observability/observability_utils/README.md deleted file mode 100644 index bd74c0bdffb47..0000000000000 --- a/x-pack/packages/observability/observability_utils/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# @kbn/observability-utils - -This package contains utilities for Observability plugins. It's a separate package to get out of dependency hell. You can put anything in here that is stateless and has no dependency on other plugins (either directly or via other packages). - -The utility functions should be used via direct imports to minimize impact on bundle size and limit the risk on importing browser code to the server and vice versa. diff --git a/x-pack/packages/observability/observability_utils/es/client/create_observability_es_client.ts b/x-pack/packages/observability/observability_utils/es/client/create_observability_es_client.ts deleted file mode 100644 index 0011e0f17c1c0..0000000000000 --- a/x-pack/packages/observability/observability_utils/es/client/create_observability_es_client.ts +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { ElasticsearchClient, Logger } from '@kbn/core/server'; -import type { ESQLSearchResponse, ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types'; -import { withSpan } from '@kbn/apm-utils'; -import type { EsqlQueryRequest } from '@elastic/elasticsearch/lib/api/types'; - -type SearchRequest = ESSearchRequest & { - index: string | string[]; - track_total_hits: number | boolean; - size: number | boolean; -}; - -/** - * An Elasticsearch Client with a fully typed `search` method and built-in - * APM instrumentation. - */ -export interface ObservabilityElasticsearchClient { - search( - operationName: string, - parameters: TSearchRequest - ): Promise>; - esql(operationName: string, parameters: EsqlQueryRequest): Promise; - client: ElasticsearchClient; -} - -export function createObservabilityEsClient({ - client, - logger, - plugin, -}: { - client: ElasticsearchClient; - logger: Logger; - plugin: string; -}): ObservabilityElasticsearchClient { - return { - client, - esql(operationName: string, parameters: EsqlQueryRequest) { - logger.trace(() => `Request (${operationName}):\n${JSON.stringify(parameters, null, 2)}`); - return withSpan({ name: operationName, labels: { plugin } }, () => { - return client.esql.query( - { ...parameters }, - { - querystring: { - drop_null_columns: true, - }, - } - ); - }) - .then((response) => { - logger.trace(() => `Response (${operationName}):\n${JSON.stringify(response, null, 2)}`); - return response as unknown as ESQLSearchResponse; - }) - .catch((error) => { - throw error; - }); - }, - search( - operationName: string, - parameters: SearchRequest - ) { - logger.trace(() => `Request (${operationName}):\n${JSON.stringify(parameters, null, 2)}`); - // wraps the search operation in a named APM span for better analysis - // (otherwise it would just be a _search span) - return withSpan( - { - name: operationName, - labels: { - plugin, - }, - }, - () => { - return client.search(parameters) as unknown as Promise< - InferSearchResponseOf - >; - } - ).then((response) => { - logger.trace(() => `Response (${operationName}):\n${JSON.stringify(response, null, 2)}`); - return response; - }); - }, - }; -} diff --git a/x-pack/packages/observability/observability_utils/chart/utils.ts b/x-pack/packages/observability/observability_utils/observability_utils_browser/chart/utils.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/chart/utils.ts rename to x-pack/packages/observability/observability_utils/observability_utils_browser/chart/utils.ts diff --git a/x-pack/packages/observability/observability_utils/hooks/use_abort_controller.ts b/x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_abort_controller.ts similarity index 92% rename from x-pack/packages/observability/observability_utils/hooks/use_abort_controller.ts rename to x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_abort_controller.ts index a383e7b81b4d9..de5c70632b233 100644 --- a/x-pack/packages/observability/observability_utils/hooks/use_abort_controller.ts +++ b/x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_abort_controller.ts @@ -18,6 +18,9 @@ export function useAbortController() { return { signal: controller.signal, + abort: () => { + controller.abort(); + }, refresh: () => { setController(() => new AbortController()); }, diff --git a/x-pack/packages/observability/observability_utils/hooks/use_abortable_async.ts b/x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_abortable_async.ts similarity index 72% rename from x-pack/packages/observability/observability_utils/hooks/use_abortable_async.ts rename to x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_abortable_async.ts index 477d765ef7a7f..d24a62ee125bc 100644 --- a/x-pack/packages/observability/observability_utils/hooks/use_abortable_async.ts +++ b/x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_abortable_async.ts @@ -17,10 +17,28 @@ export type AbortableAsyncState = (T extends Promise ? State : State) & { refresh: () => void }; +export type AbortableAsyncStateOf> = + T extends AbortableAsyncState ? Awaited : never; + +interface UseAbortableAsyncOptions { + clearValueOnNext?: boolean; + defaultValue?: () => T; + onError?: (error: Error) => void; +} + +export type UseAbortableAsync< + TAdditionalParameters extends Record = {}, + TAdditionalOptions extends Record = {} +> = ( + fn: ({}: { signal: AbortSignal } & TAdditionalParameters) => T | Promise, + deps: any[], + options?: UseAbortableAsyncOptions & TAdditionalOptions +) => AbortableAsyncState; + export function useAbortableAsync( fn: ({}: { signal: AbortSignal }) => T | Promise, deps: any[], - options?: { clearValueOnNext?: boolean; defaultValue?: () => T } + options?: UseAbortableAsyncOptions ): AbortableAsyncState { const clearValueOnNext = options?.clearValueOnNext; @@ -43,6 +61,13 @@ export function useAbortableAsync( setError(undefined); } + function handleError(err: Error) { + setError(err); + // setValue(undefined); + setLoading(false); + options?.onError?.(err); + } + try { const response = fn({ signal: controller.signal }); if (isPromise(response)) { @@ -52,12 +77,7 @@ export function useAbortableAsync( setError(undefined); setValue(nextValue); }) - .catch((err) => { - setValue(undefined); - if (!controller.signal.aborted) { - setError(err); - } - }) + .catch(handleError) .finally(() => setLoading(false)); } else { setError(undefined); @@ -65,9 +85,7 @@ export function useAbortableAsync( setLoading(false); } } catch (err) { - setValue(undefined); - setError(err); - setLoading(false); + handleError(err); } return () => { diff --git a/x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_date_range.ts b/x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_date_range.ts new file mode 100644 index 0000000000000..941e106247b87 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_date_range.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { TimeRange } from '@kbn/data-plugin/common'; +import { DataPublicPluginStart } from '@kbn/data-plugin/public'; +import { useCallback, useEffect, useMemo, useState } from 'react'; + +export function useDateRange({ data }: { data: DataPublicPluginStart }): { + timeRange: TimeRange; + absoluteTimeRange: { + start: number; + end: number; + }; + setTimeRange: React.Dispatch>; +} { + const timefilter = data.query.timefilter.timefilter; + + const [timeRange, setTimeRange] = useState(() => timefilter.getTime()); + + const [absoluteTimeRange, setAbsoluteTimeRange] = useState(() => timefilter.getAbsoluteTime()); + + useEffect(() => { + const timeUpdateSubscription = timefilter.getTimeUpdate$().subscribe({ + next: () => { + setTimeRange(() => timefilter.getTime()); + setAbsoluteTimeRange(() => timefilter.getAbsoluteTime()); + }, + }); + + return () => { + timeUpdateSubscription.unsubscribe(); + }; + }, [timefilter]); + + const setTimeRangeMemoized: React.Dispatch> = useCallback( + (nextOrCallback) => { + const val = + typeof nextOrCallback === 'function' + ? nextOrCallback(timefilter.getTime()) + : nextOrCallback; + + timefilter.setTime(val); + }, + [timefilter] + ); + + const asEpoch = useMemo(() => { + return { + start: new Date(absoluteTimeRange.from).getTime(), + end: new Date(absoluteTimeRange.to).getTime(), + }; + }, [absoluteTimeRange]); + + return { + timeRange, + absoluteTimeRange: asEpoch, + setTimeRange: setTimeRangeMemoized, + }; +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_local_storage.ts b/x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_local_storage.ts new file mode 100644 index 0000000000000..ea9e13163e4b0 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_local_storage.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useState, useEffect, useMemo, useCallback } from 'react'; + +export function useLocalStorage(key: string, defaultValue: T) { + // This is necessary to fix a race condition issue. + // It guarantees that the latest value will be always returned after the value is updated + const [storageUpdate, setStorageUpdate] = useState(0); + + const item = useMemo(() => { + return getFromStorage(key, defaultValue); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [key, storageUpdate, defaultValue]); + + const saveToStorage = useCallback( + (value: T) => { + if (value === undefined) { + window.localStorage.removeItem(key); + } else { + window.localStorage.setItem(key, JSON.stringify(value)); + setStorageUpdate(storageUpdate + 1); + } + }, + [key, storageUpdate] + ); + + useEffect(() => { + function onUpdate(event: StorageEvent) { + if (event.key === key) { + setStorageUpdate(storageUpdate + 1); + } + } + window.addEventListener('storage', onUpdate); + return () => { + window.removeEventListener('storage', onUpdate); + }; + }, [key, setStorageUpdate, storageUpdate]); + + return useMemo(() => [item, saveToStorage] as const, [item, saveToStorage]); +} + +function getFromStorage(keyName: string, defaultValue: T) { + const storedItem = window.localStorage.getItem(keyName); + + if (storedItem !== null) { + try { + return JSON.parse(storedItem) as T; + } catch (err) { + window.localStorage.removeItem(keyName); + // eslint-disable-next-line no-console + console.log(`Unable to decode: ${keyName}`); + } + } + return defaultValue; +} diff --git a/x-pack/packages/observability/observability_utils/hooks/use_theme.ts b/x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_theme.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/hooks/use_theme.ts rename to x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_theme.ts diff --git a/x-pack/packages/observability/observability_utils/observability_utils_browser/jest.config.js b/x-pack/packages/observability/observability_utils/observability_utils_browser/jest.config.js new file mode 100644 index 0000000000000..33358c221fa1f --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_browser/jest.config.js @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../../../..', + roots: [ + '/x-pack/packages/observability/observability_utils/observability_utils_browser', + ], +}; diff --git a/x-pack/packages/observability/observability_utils/observability_utils_browser/kibana.jsonc b/x-pack/packages/observability/observability_utils/observability_utils_browser/kibana.jsonc new file mode 100644 index 0000000000000..dbee36828d080 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_browser/kibana.jsonc @@ -0,0 +1,5 @@ +{ + "type": "shared-browser", + "id": "@kbn/observability-utils-browser", + "owner": "@elastic/observability-ui" +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_browser/package.json b/x-pack/packages/observability/observability_utils/observability_utils_browser/package.json new file mode 100644 index 0000000000000..c72c8c0b45eb0 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_browser/package.json @@ -0,0 +1,6 @@ +{ + "name": "@kbn/observability-utils-browser", + "private": true, + "version": "1.0.0", + "license": "Elastic License 2.0" +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_browser/tsconfig.json b/x-pack/packages/observability/observability_utils/observability_utils_browser/tsconfig.json new file mode 100644 index 0000000000000..9cfa030bd901d --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_browser/tsconfig.json @@ -0,0 +1,23 @@ +{ + "extends": "../../../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + "types": [ + "jest", + "node", + "react" + ] + }, + "include": [ + "**/*.ts", + "**/*.tsx", + ], + "exclude": [ + "target/**/*" + ], + "kbn_references": [ + "@kbn/data-plugin", + "@kbn/core-ui-settings-browser", + "@kbn/std", + ] +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_browser/utils/ui_settings/get_timezone.ts b/x-pack/packages/observability/observability_utils/observability_utils_browser/utils/ui_settings/get_timezone.ts new file mode 100644 index 0000000000000..3ad5d17aa61bc --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_browser/utils/ui_settings/get_timezone.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { UI_SETTINGS } from '@kbn/data-plugin/public'; +import { IUiSettingsClient } from '@kbn/core-ui-settings-browser'; + +export function getTimeZone(uiSettings?: IUiSettingsClient) { + const kibanaTimeZone = uiSettings?.get<'Browser' | string>(UI_SETTINGS.DATEFORMAT_TZ); + if (!kibanaTimeZone || kibanaTimeZone === 'Browser') { + return 'local'; + } + + return kibanaTimeZone; +} diff --git a/x-pack/packages/observability/observability_utils/array/join_by_key.test.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/array/join_by_key.test.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/array/join_by_key.test.ts rename to x-pack/packages/observability/observability_utils/observability_utils_common/array/join_by_key.test.ts diff --git a/x-pack/packages/observability/observability_utils/array/join_by_key.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/array/join_by_key.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/array/join_by_key.ts rename to x-pack/packages/observability/observability_utils/observability_utils_common/array/join_by_key.ts diff --git a/x-pack/packages/observability/observability_utils/observability_utils_common/entities/get_entity_kuery.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/entities/get_entity_kuery.ts new file mode 100644 index 0000000000000..ba68e544379a4 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/entities/get_entity_kuery.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function getEntityKuery(entity: Record) { + return Object.entries(entity) + .map(([name, value]) => { + return `(${name}:"${value}")`; + }) + .join(' AND '); +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_common/es/format_value_for_kql.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/es/format_value_for_kql.ts new file mode 100644 index 0000000000000..a0fb5c15fd03e --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/es/format_value_for_kql.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function formatValueForKql(value: string) { + return `(${value.replaceAll(/((^|[^\\])):/g, '\\:')})`; +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_common/es/queries/entity_query.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/es/queries/entity_query.ts new file mode 100644 index 0000000000000..f2ae0991eecf4 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/es/queries/entity_query.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; + +export function entityQuery(entity: Record): QueryDslQueryContainer[] { + return [ + { + bool: { + filter: Object.entries(entity).map(([field, value]) => { + return { + term: { + [field]: value, + }, + }; + }), + }, + }, + ]; +} diff --git a/x-pack/packages/observability/observability_utils/es/queries/exclude_frozen_query.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/es/queries/exclude_frozen_query.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/es/queries/exclude_frozen_query.ts rename to x-pack/packages/observability/observability_utils/observability_utils_common/es/queries/exclude_frozen_query.ts diff --git a/x-pack/packages/observability/observability_utils/es/queries/exclude_tiers_query.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/es/queries/exclude_tiers_query.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/es/queries/exclude_tiers_query.ts rename to x-pack/packages/observability/observability_utils/observability_utils_common/es/queries/exclude_tiers_query.ts diff --git a/x-pack/packages/observability/observability_utils/es/queries/kql_query.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/es/queries/kql_query.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/es/queries/kql_query.ts rename to x-pack/packages/observability/observability_utils/observability_utils_common/es/queries/kql_query.ts diff --git a/x-pack/packages/observability/observability_utils/es/queries/range_query.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/es/queries/range_query.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/es/queries/range_query.ts rename to x-pack/packages/observability/observability_utils/observability_utils_common/es/queries/range_query.ts diff --git a/x-pack/packages/observability/observability_utils/es/queries/term_query.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/es/queries/term_query.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/es/queries/term_query.ts rename to x-pack/packages/observability/observability_utils/observability_utils_common/es/queries/term_query.ts diff --git a/x-pack/packages/observability/observability_utils/observability_utils_common/format/integer.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/format/integer.ts new file mode 100644 index 0000000000000..7cf202fb8c811 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/format/integer.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import numeral from '@elastic/numeral'; + +export function formatInteger(num: number) { + return numeral(num).format('0a'); +} diff --git a/x-pack/packages/observability/observability_utils/jest.config.js b/x-pack/packages/observability/observability_utils/observability_utils_common/jest.config.js similarity index 84% rename from x-pack/packages/observability/observability_utils/jest.config.js rename to x-pack/packages/observability/observability_utils/observability_utils_common/jest.config.js index c9dff28ed6cec..ee68881a5863b 100644 --- a/x-pack/packages/observability/observability_utils/jest.config.js +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/jest.config.js @@ -7,6 +7,6 @@ module.exports = { preset: '@kbn/test', - rootDir: '../../../..', - roots: ['/x-pack/packages/observability/observability_utils'], + rootDir: '../../../../..', + roots: ['/x-pack/packages/observability/observability_utils/observability_utils_common'], }; diff --git a/x-pack/packages/observability/observability_utils/kibana.jsonc b/x-pack/packages/observability/observability_utils/observability_utils_common/kibana.jsonc similarity index 61% rename from x-pack/packages/observability/observability_utils/kibana.jsonc rename to x-pack/packages/observability/observability_utils/observability_utils_common/kibana.jsonc index 096b2565d533f..eb120052e5b0e 100644 --- a/x-pack/packages/observability/observability_utils/kibana.jsonc +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/kibana.jsonc @@ -1,5 +1,5 @@ { "type": "shared-common", - "id": "@kbn/observability-utils", + "id": "@kbn/observability-utils-common", "owner": "@elastic/observability-ui" } diff --git a/x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/document_analysis.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/document_analysis.ts new file mode 100644 index 0000000000000..be896571ca217 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/document_analysis.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export interface DocumentAnalysis { + total: number; + sampled: number; + fields: Array<{ + name: string; + types: string[]; + cardinality: number | null; + values: Array; + empty: boolean; + }>; +} + +export interface TruncatedDocumentAnalysis { + fields: string[]; + total: number; + sampled: number; +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/highlight_patterns_from_regex.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/highlight_patterns_from_regex.ts new file mode 100644 index 0000000000000..11ab0c52f1795 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/highlight_patterns_from_regex.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +function addCapturingGroupsToRegex(regex: string): string { + // Match all parts of the regex that are not special characters + // We treat constant parts as sequences of characters that are not part of regex syntax + return regex.replaceAll(/((?:\.\*\?)|(?:\.\+\?)|(?:\+\?))/g, (...args) => { + return `(${args[1]})`; + }); +} + +export function highlightPatternFromRegex(pattern: string, str: string): string { + // First, add non-capturing groups to the regex around constant parts + const updatedPattern = addCapturingGroupsToRegex(pattern); + + const regex = new RegExp(updatedPattern, 'ds'); + + const matches = str.match(regex) as + | (RegExpMatchArray & { indices: Array<[number, number]> }) + | null; + + const slices: string[] = []; + + matches?.forEach((_, index) => { + if (index === 0) { + return; + } + + const [, prevEnd] = index > 1 ? matches?.indices[index - 1] : [undefined, undefined]; + const [start, end] = matches?.indices[index]; + + const literalSlice = prevEnd !== undefined ? str.slice(prevEnd, start) : undefined; + + if (literalSlice) { + slices.push(`${literalSlice}`); + } + + const slice = str.slice(start, end); + slices.push(slice); + + if (index === matches.length - 1) { + slices.push(str.slice(end)); + } + }); + + return slices.join(''); +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/merge_sample_documents_with_field_caps.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/merge_sample_documents_with_field_caps.ts new file mode 100644 index 0000000000000..58b6024aed046 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/merge_sample_documents_with_field_caps.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { castArray, sortBy, uniq } from 'lodash'; +import type { DocumentAnalysis } from './document_analysis'; + +export function mergeSampleDocumentsWithFieldCaps({ + total, + samples, + fieldCaps, +}: { + total: number; + samples: Array>; + fieldCaps: Array<{ name: string; esTypes?: string[] }>; +}): DocumentAnalysis { + const nonEmptyFields = new Set(); + const fieldValues = new Map>(); + + for (const document of samples) { + Object.keys(document).forEach((field) => { + if (!nonEmptyFields.has(field)) { + nonEmptyFields.add(field); + } + + const values = castArray(document[field]); + + const currentFieldValues = fieldValues.get(field) ?? []; + + values.forEach((value) => { + if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') { + currentFieldValues.push(value); + } + }); + + fieldValues.set(field, currentFieldValues); + }); + } + + const fields = fieldCaps.flatMap((spec) => { + const values = fieldValues.get(spec.name); + + const countByValues = new Map(); + + values?.forEach((value) => { + const currentCount = countByValues.get(value) ?? 0; + countByValues.set(value, currentCount + 1); + }); + + const sortedValues = sortBy( + Array.from(countByValues.entries()).map(([value, count]) => { + return { + value, + count, + }; + }), + 'count', + 'desc' + ); + + return { + name: spec.name, + types: spec.esTypes ?? [], + empty: !nonEmptyFields.has(spec.name), + cardinality: countByValues.size || null, + values: uniq(sortedValues.flatMap(({ value }) => value)), + }; + }); + + return { + total, + sampled: samples.length, + fields, + }; +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/sort_and_truncate_analyzed_fields.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/sort_and_truncate_analyzed_fields.ts new file mode 100644 index 0000000000000..c9a3e6a156601 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/llm/log_analysis/sort_and_truncate_analyzed_fields.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { partition, shuffle } from 'lodash'; +import { truncateList } from '@kbn/inference-common'; +import type { DocumentAnalysis, TruncatedDocumentAnalysis } from './document_analysis'; + +export function sortAndTruncateAnalyzedFields( + analysis: DocumentAnalysis +): TruncatedDocumentAnalysis { + const { fields, ...meta } = analysis; + const [nonEmptyFields, emptyFields] = partition(analysis.fields, (field) => !field.empty); + + const sortedFields = [...shuffle(nonEmptyFields), ...shuffle(emptyFields)]; + + return { + ...meta, + fields: truncateList( + sortedFields.map((field) => { + let label = `${field.name}:${field.types.join(',')}`; + + if (field.empty) { + return `${name} (empty)`; + } + + label += ` - ${field.cardinality} distinct values`; + + if (field.name === '@timestamp' || field.name === 'event.ingested') { + return `${label}`; + } + + const shortValues = field.values.filter((value) => { + return String(value).length <= 1024; + }); + + if (shortValues.length) { + return `${label} (${truncateList( + shortValues.map((value) => '`' + value + '`'), + field.types.includes('text') || field.types.includes('match_only_text') ? 2 : 10 + ).join(', ')})`; + } + + return label; + }), + 500 + ).sort(), + }; +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_common/llm/short_id_table.test.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/llm/short_id_table.test.ts new file mode 100644 index 0000000000000..784cf67530652 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/llm/short_id_table.test.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { ShortIdTable } from './short_id_table'; + +describe('shortIdTable', () => { + it('generates at least 10k unique ids consistently', () => { + const ids = new Set(); + + const table = new ShortIdTable(); + + let i = 10_000; + while (i--) { + const id = table.take(String(i)); + ids.add(id); + } + + expect(ids.size).toBe(10_000); + }); + + it('returns the original id based on the generated id', () => { + const table = new ShortIdTable(); + + const idsByOriginal = new Map(); + + let i = 100; + while (i--) { + const id = table.take(String(i)); + idsByOriginal.set(String(i), id); + } + + expect(idsByOriginal.size).toBe(100); + + expect(() => { + Array.from(idsByOriginal.entries()).forEach(([originalId, shortId]) => { + const returnedOriginalId = table.lookup(shortId); + if (returnedOriginalId !== originalId) { + throw Error( + `Expected shortId ${shortId} to return ${originalId}, but ${returnedOriginalId} was returned instead` + ); + } + }); + }).not.toThrow(); + }); +}); diff --git a/x-pack/packages/observability/observability_utils/observability_utils_common/llm/short_id_table.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/llm/short_id_table.ts new file mode 100644 index 0000000000000..30049452ddf51 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/llm/short_id_table.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +const ALPHABET = 'abcdefghijklmnopqrstuvwxyz'; + +function generateShortId(size: number): string { + let id = ''; + let i = size; + while (i--) { + const index = Math.floor(Math.random() * ALPHABET.length); + id += ALPHABET[index]; + } + return id; +} + +const MAX_ATTEMPTS_AT_LENGTH = 100; + +export class ShortIdTable { + private byShortId: Map = new Map(); + private byOriginalId: Map = new Map(); + + constructor() {} + + take(originalId: string) { + if (this.byOriginalId.has(originalId)) { + return this.byOriginalId.get(originalId)!; + } + + let uniqueId: string | undefined; + let attemptsAtLength = 0; + let length = 4; + while (!uniqueId) { + const nextId = generateShortId(length); + attemptsAtLength++; + if (!this.byShortId.has(nextId)) { + uniqueId = nextId; + } else if (attemptsAtLength >= MAX_ATTEMPTS_AT_LENGTH) { + attemptsAtLength = 0; + length++; + } + } + + this.byShortId.set(uniqueId, originalId); + this.byOriginalId.set(originalId, uniqueId); + + return uniqueId; + } + + lookup(shortId: string) { + return this.byShortId.get(shortId); + } +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_common/ml/p_value_to_label.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/ml/p_value_to_label.ts new file mode 100644 index 0000000000000..3f6e0836d129b --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/ml/p_value_to_label.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const P_VALUE_SIGNIFICANCE_HIGH = 1e-6; +export const P_VALUE_SIGNIFICANCE_MEDIUM = 0.001; + +export function pValueToLabel(pValue: number): 'high' | 'medium' | 'low' { + if (pValue <= P_VALUE_SIGNIFICANCE_HIGH) { + return 'high'; + } else if (pValue <= P_VALUE_SIGNIFICANCE_MEDIUM) { + return 'medium'; + } else { + return 'low'; + } +} diff --git a/x-pack/packages/observability/observability_utils/object/flatten_object.test.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/object/flatten_object.test.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/object/flatten_object.test.ts rename to x-pack/packages/observability/observability_utils/observability_utils_common/object/flatten_object.test.ts diff --git a/x-pack/packages/observability/observability_utils/object/flatten_object.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/object/flatten_object.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/object/flatten_object.ts rename to x-pack/packages/observability/observability_utils/observability_utils_common/object/flatten_object.ts diff --git a/x-pack/packages/observability/observability_utils/object/merge_plain_object.test.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/object/merge_plain_object.test.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/object/merge_plain_object.test.ts rename to x-pack/packages/observability/observability_utils/observability_utils_common/object/merge_plain_object.test.ts diff --git a/x-pack/packages/observability/observability_utils/object/merge_plain_objects.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/object/merge_plain_objects.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/object/merge_plain_objects.ts rename to x-pack/packages/observability/observability_utils/observability_utils_common/object/merge_plain_objects.ts diff --git a/x-pack/packages/observability/observability_utils/object/unflatten_object.test.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/object/unflatten_object.test.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/object/unflatten_object.test.ts rename to x-pack/packages/observability/observability_utils/observability_utils_common/object/unflatten_object.test.ts diff --git a/x-pack/packages/observability/observability_utils/object/unflatten_object.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/object/unflatten_object.ts similarity index 99% rename from x-pack/packages/observability/observability_utils/object/unflatten_object.ts rename to x-pack/packages/observability/observability_utils/observability_utils_common/object/unflatten_object.ts index 142ea2eea6461..83508d5d2dbf5 100644 --- a/x-pack/packages/observability/observability_utils/object/unflatten_object.ts +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/object/unflatten_object.ts @@ -11,7 +11,6 @@ export function unflattenObject(source: Record, target: Record { if (item && typeof item === 'object' && !Array.isArray(item)) { diff --git a/x-pack/packages/observability/observability_utils/package.json b/x-pack/packages/observability/observability_utils/observability_utils_common/package.json similarity index 62% rename from x-pack/packages/observability/observability_utils/package.json rename to x-pack/packages/observability/observability_utils/observability_utils_common/package.json index 06f6e37858927..2f9be5f105279 100644 --- a/x-pack/packages/observability/observability_utils/package.json +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/package.json @@ -1,6 +1,6 @@ { - "name": "@kbn/observability-utils", + "name": "@kbn/observability-utils-common", "private": true, "version": "1.0.0", "license": "Elastic License 2.0" -} \ No newline at end of file +} diff --git a/x-pack/packages/observability/observability_utils/tsconfig.json b/x-pack/packages/observability/observability_utils/observability_utils_common/tsconfig.json similarity index 70% rename from x-pack/packages/observability/observability_utils/tsconfig.json rename to x-pack/packages/observability/observability_utils/observability_utils_common/tsconfig.json index b3f1a4a21c4e7..7954cdc946e9c 100644 --- a/x-pack/packages/observability/observability_utils/tsconfig.json +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/tsconfig.json @@ -1,5 +1,5 @@ { - "extends": "../../../../tsconfig.base.json", + "extends": "../../../../../tsconfig.base.json", "compilerOptions": { "outDir": "target/types", "types": [ @@ -16,11 +16,8 @@ "target/**/*" ], "kbn_references": [ - "@kbn/std", - "@kbn/core", - "@kbn/es-types", - "@kbn/apm-utils", "@kbn/es-query", "@kbn/safer-lodash-set", + "@kbn/inference-common", ] } diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/entities/analyze_documents.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/entities/analyze_documents.ts new file mode 100644 index 0000000000000..0cc1374d8b1d8 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/entities/analyze_documents.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { mapValues } from 'lodash'; +import { mergeSampleDocumentsWithFieldCaps } from '@kbn/observability-utils-common/llm/log_analysis/merge_sample_documents_with_field_caps'; +import { DocumentAnalysis } from '@kbn/observability-utils-common/llm/log_analysis/document_analysis'; +import type { ObservabilityElasticsearchClient } from '../es/client/create_observability_es_client'; +import { kqlQuery } from '../es/queries/kql_query'; +import { rangeQuery } from '../es/queries/range_query'; + +export async function analyzeDocuments({ + esClient, + kuery, + start, + end, + index, +}: { + esClient: ObservabilityElasticsearchClient; + kuery: string; + start: number; + end: number; + index: string | string[]; +}): Promise { + const [fieldCaps, hits] = await Promise.all([ + esClient.fieldCaps('get_field_caps_for_document_analysis', { + index, + fields: '*', + index_filter: { + bool: { + filter: rangeQuery(start, end), + }, + }, + }), + esClient + .search('get_document_samples', { + index, + size: 1000, + track_total_hits: true, + query: { + bool: { + must: [...kqlQuery(kuery), ...rangeQuery(start, end)], + should: [ + { + function_score: { + functions: [ + { + random_score: {}, + }, + ], + }, + }, + ], + }, + }, + sort: { + _score: { + order: 'desc', + }, + }, + _source: false, + fields: ['*' as const], + }) + .then((response) => ({ + hits: response.hits.hits.map((hit) => + mapValues(hit.fields!, (value) => (value.length === 1 ? value[0] : value)) + ), + total: response.hits.total, + })), + ]); + + const analysis = mergeSampleDocumentsWithFieldCaps({ + samples: hits.hits, + total: hits.total.value, + fieldCaps: Object.entries(fieldCaps.fields).map(([name, specs]) => { + return { name, esTypes: Object.keys(specs) }; + }), + }); + + return analysis; +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/entities/get_data_streams_for_entity.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/entities/get_data_streams_for_entity.ts new file mode 100644 index 0000000000000..43d9134c7aaf3 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/entities/get_data_streams_for_entity.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { compact, uniq } from 'lodash'; +import { ObservabilityElasticsearchClient } from '../es/client/create_observability_es_client'; +import { excludeFrozenQuery } from '../es/queries/exclude_frozen_query'; +import { kqlQuery } from '../es/queries/kql_query'; + +export async function getDataStreamsForEntity({ + esClient, + kuery, + index, +}: { + esClient: ObservabilityElasticsearchClient; + kuery: string; + index: string | string[]; +}) { + const response = await esClient.search('get_data_streams_for_entity', { + track_total_hits: false, + index, + size: 0, + terminate_after: 1, + timeout: '1ms', + aggs: { + indices: { + terms: { + field: '_index', + size: 10000, + }, + }, + }, + query: { + bool: { + filter: [...excludeFrozenQuery(), ...kqlQuery(kuery)], + }, + }, + }); + + const allIndices = + response.aggregations?.indices.buckets.map((bucket) => bucket.key as string) ?? []; + + if (!allIndices.length) { + return { + dataStreams: [], + }; + } + + const resolveIndexResponse = await esClient.client.indices.resolveIndex({ + name: allIndices, + }); + + const dataStreams = uniq( + compact(await resolveIndexResponse.indices.flatMap((idx) => idx.data_stream)) + ); + + return { + dataStreams, + }; +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/entities/signals/get_alerts_for_entity.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/entities/signals/get_alerts_for_entity.ts new file mode 100644 index 0000000000000..400aad8e94357 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/entities/signals/get_alerts_for_entity.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { RulesClient } from '@kbn/alerting-plugin/server'; +import { AlertsClient } from '@kbn/rule-registry-plugin/server'; +import { + ALERT_GROUP_FIELD, + ALERT_GROUP_VALUE, + ALERT_STATUS, + ALERT_STATUS_ACTIVE, + ALERT_TIME_RANGE, +} from '@kbn/rule-data-utils'; +import { kqlQuery } from '../../es/queries/kql_query'; +import { rangeQuery } from '../../es/queries/range_query'; + +export async function getAlertsForEntity({ + start, + end, + entity, + alertsClient, + rulesClient, + size, +}: { + start: number; + end: number; + entity: Record; + alertsClient: AlertsClient; + rulesClient: RulesClient; + size: number; +}) { + const alertsKuery = Object.entries(entity) + .map(([field, value]) => { + return `(${[ + `(${ALERT_GROUP_FIELD}:"${field}" AND ${ALERT_GROUP_VALUE}:"${value}")`, + `(${field}:"${value}")`, + ].join(' OR ')})`; + }) + .join(' AND '); + + const openAlerts = await alertsClient.find({ + size, + query: { + bool: { + filter: [ + ...kqlQuery(alertsKuery), + ...rangeQuery(start, end, ALERT_TIME_RANGE), + { term: { [ALERT_STATUS]: ALERT_STATUS_ACTIVE } }, + ], + }, + }, + }); + + return openAlerts; +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/entities/signals/get_anomalies_for_entity.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/entities/signals/get_anomalies_for_entity.ts new file mode 100644 index 0000000000000..b8802ed3c9045 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/entities/signals/get_anomalies_for_entity.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function getAnomaliesForEntity({ + start, + end, + entity, +}: { + start: number; + end: number; + entity: Record; +}) {} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/entities/signals/get_slos_for_entity.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/entities/signals/get_slos_for_entity.ts new file mode 100644 index 0000000000000..fc3a9d7b26d5c --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/entities/signals/get_slos_for_entity.ts @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ObservabilityElasticsearchClient } from '../../es/client/create_observability_es_client'; +import { kqlQuery } from '../../es/queries/kql_query'; + +export async function getSlosForEntity({ + start, + end, + entity, + esClient, + sloSummaryIndices, + size, + spaceId, +}: { + start: number; + end: number; + entity: Record; + esClient: ObservabilityElasticsearchClient; + sloSummaryIndices: string | string[]; + size: number; + spaceId: string; +}) { + const slosKuery = Object.entries(entity) + .map(([field, value]) => { + return `(slo.groupings.${field}:"${value}")`; + }) + .join(' AND '); + + const sloSummaryResponse = await esClient.search('get_slo_summaries_for_entity', { + index: sloSummaryIndices, + size, + track_total_hits: false, + query: { + bool: { + filter: [ + ...kqlQuery(slosKuery), + { + range: { + 'slo.createdAt': { + lte: end, + }, + }, + }, + { + range: { + summaryUpdatedAt: { + gte: start, + }, + }, + }, + { + term: { + spaceId, + }, + }, + ], + }, + }, + }); + + return { + ...sloSummaryResponse, + hits: { + ...sloSummaryResponse.hits, + hits: sloSummaryResponse.hits.hits.map((hit) => { + return { + ...hit, + _source: hit._source as Record & { + status: 'VIOLATED' | 'DEGRADED' | 'HEALTHY' | 'NO_DATA'; + }, + }; + }), + }, + }; +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts new file mode 100644 index 0000000000000..74b1d1805d71a --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + FieldCapsRequest, + FieldCapsResponse, + MsearchRequest, + SearchResponse, +} from '@elastic/elasticsearch/lib/api/types'; +import { withSpan } from '@kbn/apm-utils'; +import type { ElasticsearchClient, Logger } from '@kbn/core/server'; +import type { + ESQLSearchResponse, + ESSearchRequest, + InferSearchResponseOf, + ESQLSearchParams, +} from '@kbn/es-types'; +import { Required } from 'utility-types'; + +type SearchRequest = ESSearchRequest & { + index: string | string[]; + track_total_hits: number | boolean; + size: number | boolean; +}; +/** + * An Elasticsearch Client with a fully typed `search` method and built-in + * APM instrumentation. + */ +export interface ObservabilityElasticsearchClient { + search( + operationName: string, + parameters: TSearchRequest + ): Promise>; + msearch( + operationName: string, + parameters: MsearchRequest + ): Promise<{ + responses: Array>; + }>; + fieldCaps( + operationName: string, + request: Required + ): Promise; + esql(operationName: string, parameters: ESQLSearchParams): Promise; + client: ElasticsearchClient; +} + +export function createObservabilityEsClient({ + client, + logger, + plugin, +}: { + client: ElasticsearchClient; + logger: Logger; + plugin: string; +}): ObservabilityElasticsearchClient { + // wraps the ES calls in a named APM span for better analysis + // (otherwise it would just eg be a _search span) + const callWithLogger = ( + operationName: string, + request: Record, + callback: () => Promise + ) => { + logger.debug(() => `Request (${operationName}):\n${JSON.stringify(request)}`); + return withSpan( + { + name: operationName, + labels: { + plugin, + }, + }, + callback, + logger + ).then((response) => { + logger.trace(() => `Response (${operationName}):\n${JSON.stringify(response, null, 2)}`); + return response; + }); + }; + + return { + client, + fieldCaps(operationName, parameters) { + return callWithLogger(operationName, parameters, () => { + return client.fieldCaps({ + ...parameters, + }); + }); + }, + esql(operationName: string, parameters: ESQLSearchParams) { + return callWithLogger(operationName, parameters, () => { + return client.esql.transport.request( + { + path: '_query', + method: 'POST', + body: JSON.stringify(parameters), + querystring: { + drop_null_columns: true, + }, + }, + { + headers: { + 'content-type': 'application/json', + }, + } + ) as unknown as Promise; + }); + }, + search( + operationName: string, + parameters: SearchRequest + ) { + return callWithLogger(operationName, parameters, () => { + return client.search(parameters) as unknown as Promise< + InferSearchResponseOf + >; + }); + }, + msearch(operationName: string, parameters: MsearchRequest) { + return callWithLogger(operationName, parameters, () => { + return client.msearch(parameters) as unknown as Promise<{ + responses: Array>; + }>; + }); + }, + }; +} diff --git a/x-pack/packages/observability/observability_utils/es/utils/esql_result_to_plain_objects.test.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/esql_result_to_plain_objects.test.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/es/utils/esql_result_to_plain_objects.test.ts rename to x-pack/packages/observability/observability_utils/observability_utils_server/es/esql_result_to_plain_objects.test.ts diff --git a/x-pack/packages/observability/observability_utils/es/utils/esql_result_to_plain_objects.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/esql_result_to_plain_objects.ts similarity index 100% rename from x-pack/packages/observability/observability_utils/es/utils/esql_result_to_plain_objects.ts rename to x-pack/packages/observability/observability_utils/observability_utils_server/es/esql_result_to_plain_objects.ts diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/exclude_frozen_query.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/exclude_frozen_query.ts new file mode 100644 index 0000000000000..f348d925c41ca --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/exclude_frozen_query.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { estypes } from '@elastic/elasticsearch'; + +export function excludeFrozenQuery(): estypes.QueryDslQueryContainer[] { + return [ + { + bool: { + must_not: [ + { + term: { + _tier: 'data_frozen', + }, + }, + ], + }, + }, + ]; +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/kql_query.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/kql_query.ts new file mode 100644 index 0000000000000..2f560157cc8c6 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/kql_query.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { estypes } from '@elastic/elasticsearch'; +import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query'; + +export function kqlQuery(kql?: string): estypes.QueryDslQueryContainer[] { + if (!kql) { + return []; + } + + const ast = fromKueryExpression(kql); + return [toElasticsearchQuery(ast)]; +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/range_query.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/range_query.ts new file mode 100644 index 0000000000000..d73476354c377 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/range_query.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { estypes } from '@elastic/elasticsearch'; + +export function rangeQuery( + start?: number, + end?: number, + field = '@timestamp' +): estypes.QueryDslQueryContainer[] { + return [ + { + range: { + [field]: { + gte: start, + lte: end, + format: 'epoch_millis', + }, + }, + }, + ]; +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/term_query.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/term_query.ts new file mode 100644 index 0000000000000..dfaeb737bf8b7 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/queries/term_query.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; + +interface TermQueryOpts { + queryEmptyString: boolean; +} + +export function termQuery( + field: T, + value: string | boolean | number | undefined | null, + opts: TermQueryOpts = { queryEmptyString: true } +): QueryDslQueryContainer[] { + if (value === null || value === undefined || (!opts.queryEmptyString && value === '')) { + return []; + } + + return [{ term: { [field]: value } }]; +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/jest.config.js b/x-pack/packages/observability/observability_utils/observability_utils_server/jest.config.js new file mode 100644 index 0000000000000..5a52de35fcd06 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/jest.config.js @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../../../..', + roots: ['/x-pack/packages/observability/observability_utils/observability_utils_server'], +}; diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/kibana.jsonc b/x-pack/packages/observability/observability_utils/observability_utils_server/kibana.jsonc new file mode 100644 index 0000000000000..4c2f20ef1491f --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/kibana.jsonc @@ -0,0 +1,5 @@ +{ + "type": "shared-server", + "id": "@kbn/observability-utils-server", + "owner": "@elastic/observability-ui" +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/package.json b/x-pack/packages/observability/observability_utils/observability_utils_server/package.json new file mode 100644 index 0000000000000..43abbbb757fea --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/package.json @@ -0,0 +1,6 @@ +{ + "name": "@kbn/observability-utils-server", + "private": true, + "version": "1.0.0", + "license": "Elastic License 2.0" +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/tsconfig.json b/x-pack/packages/observability/observability_utils/observability_utils_server/tsconfig.json new file mode 100644 index 0000000000000..f51d93089c627 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/tsconfig.json @@ -0,0 +1,28 @@ +{ + "extends": "../../../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + "types": [ + "jest", + "node", + "react" + ] + }, + "include": [ + "**/*.ts", + "**/*.tsx", + ], + "exclude": [ + "target/**/*" + ], + "kbn_references": [ + "@kbn/core", + "@kbn/es-types", + "@kbn/apm-utils", + "@kbn/es-query", + "@kbn/observability-utils-common", + "@kbn/alerting-plugin", + "@kbn/rule-registry-plugin", + "@kbn/rule-data-utils", + ] +} diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.ts b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.ts index cf376e7c78294..104515899fe26 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.ts @@ -23,7 +23,7 @@ import { ValuesType } from 'utility-types'; import type { APMError, Metric, Span, Transaction, Event } from '@kbn/apm-types/es_schemas_ui'; import type { InspectResponse } from '@kbn/observability-plugin/typings/common'; import type { DataTier } from '@kbn/observability-shared-plugin/common'; -import { excludeTiersQuery } from '@kbn/observability-utils/es/queries/exclude_tiers_query'; +import { excludeTiersQuery } from '@kbn/observability-utils-server/es/queries/exclude_tiers_query'; import { withApmSpan } from '../../../../utils'; import type { ApmDataSource } from '../../../../../common/data_source'; import { cancelEsRequestOnAbort } from '../cancel_es_request_on_abort'; diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts index cad0b03579e3d..c1f8d5e3fce1f 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/tier_filter.ts @@ -6,7 +6,7 @@ */ import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { DataTier } from '@kbn/observability-shared-plugin/common'; -import { excludeTiersQuery } from '@kbn/observability-utils/es/queries/exclude_tiers_query'; +import { excludeTiersQuery } from '@kbn/observability-utils-common/es/queries/exclude_tiers_query'; export function getDataTierFilterCombined({ filter, diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts b/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts index b9a4322269828..6c9fe4c39b001 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/utils/unflatten_known_fields.ts @@ -8,8 +8,8 @@ import type { DedotObject } from '@kbn/utility-types'; import * as APM_EVENT_FIELDS_MAP from '@kbn/apm-types/es_fields'; import type { ValuesType } from 'utility-types'; -import { unflattenObject } from '@kbn/observability-utils/object/unflatten_object'; -import { mergePlainObjects } from '@kbn/observability-utils/object/merge_plain_objects'; +import { unflattenObject } from '@kbn/observability-utils-common/object/unflatten_object'; +import { mergePlainObjects } from '@kbn/observability-utils-common/object/merge_plain_objects'; import { castArray, isArray } from 'lodash'; import { AgentName } from '@kbn/elastic-agent-utils'; import { EventOutcome } from '@kbn/apm-types/src/es_schemas/raw/fields'; diff --git a/x-pack/plugins/observability_solution/infra/server/routes/entities/get_data_stream_types.test.ts b/x-pack/plugins/observability_solution/infra/server/routes/entities/get_data_stream_types.test.ts index c66416331e4d0..7896285139930 100644 --- a/x-pack/plugins/observability_solution/infra/server/routes/entities/get_data_stream_types.test.ts +++ b/x-pack/plugins/observability_solution/infra/server/routes/entities/get_data_stream_types.test.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import type { ObservabilityElasticsearchClient } from '@kbn/observability-utils/es/client/create_observability_es_client'; +import type { ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; import { type EntityClient } from '@kbn/entityManager-plugin/server/lib/entity_client'; import { type InfraMetricsClient } from '../../lib/helpers/get_infra_metrics_client'; import { getDataStreamTypes } from './get_data_stream_types'; diff --git a/x-pack/plugins/observability_solution/infra/server/routes/entities/get_data_stream_types.ts b/x-pack/plugins/observability_solution/infra/server/routes/entities/get_data_stream_types.ts index 3218ae257f1a2..b18c76da1fd4e 100644 --- a/x-pack/plugins/observability_solution/infra/server/routes/entities/get_data_stream_types.ts +++ b/x-pack/plugins/observability_solution/infra/server/routes/entities/get_data_stream_types.ts @@ -11,7 +11,7 @@ import { EntityDataStreamType, SOURCE_DATA_STREAM_TYPE, } from '@kbn/observability-shared-plugin/common'; -import type { ObservabilityElasticsearchClient } from '@kbn/observability-utils/es/client/create_observability_es_client'; +import type { ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; import { type InfraMetricsClient } from '../../lib/helpers/get_infra_metrics_client'; import { getHasMetricsData } from './get_has_metrics_data'; import { getLatestEntity } from './get_latest_entity'; diff --git a/x-pack/plugins/observability_solution/infra/server/routes/entities/get_latest_entity.ts b/x-pack/plugins/observability_solution/infra/server/routes/entities/get_latest_entity.ts index 7bcce2964fd13..728925e5e067b 100644 --- a/x-pack/plugins/observability_solution/infra/server/routes/entities/get_latest_entity.ts +++ b/x-pack/plugins/observability_solution/infra/server/routes/entities/get_latest_entity.ts @@ -11,8 +11,8 @@ import { ENTITY_TYPE, SOURCE_DATA_STREAM_TYPE, } from '@kbn/observability-shared-plugin/common/field_names/elasticsearch'; -import type { ObservabilityElasticsearchClient } from '@kbn/observability-utils/es/client/create_observability_es_client'; -import { esqlResultToPlainObjects } from '@kbn/observability-utils/es/utils/esql_result_to_plain_objects'; +import type { ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; +import { esqlResultToPlainObjects } from '@kbn/observability-utils-server/es/esql_result_to_plain_objects'; const ENTITIES_LATEST_ALIAS = entitiesAliasPattern({ type: '*', diff --git a/x-pack/plugins/observability_solution/infra/server/routes/entities/index.ts b/x-pack/plugins/observability_solution/infra/server/routes/entities/index.ts index 1a8707678e8f7..30be4fc9da498 100644 --- a/x-pack/plugins/observability_solution/infra/server/routes/entities/index.ts +++ b/x-pack/plugins/observability_solution/infra/server/routes/entities/index.ts @@ -8,7 +8,7 @@ import { schema } from '@kbn/config-schema'; import { METRICS_APP_ID } from '@kbn/deeplinks-observability/constants'; import { entityCentricExperience } from '@kbn/observability-plugin/common'; -import { createObservabilityEsClient } from '@kbn/observability-utils/es/client/create_observability_es_client'; +import { createObservabilityEsClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; import { ENTITY_TYPES } from '@kbn/observability-shared-plugin/common'; import { getInfraMetricsClient } from '../../lib/helpers/get_infra_metrics_client'; import { InfraBackendLibs } from '../../lib/infra_types'; diff --git a/x-pack/plugins/observability_solution/inventory/common/utils/unflatten_entity.ts b/x-pack/plugins/observability_solution/inventory/common/utils/unflatten_entity.ts index 758d185a5753b..251790cd649a5 100644 --- a/x-pack/plugins/observability_solution/inventory/common/utils/unflatten_entity.ts +++ b/x-pack/plugins/observability_solution/inventory/common/utils/unflatten_entity.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { unflattenObject } from '@kbn/observability-utils/object/unflatten_object'; +import { unflattenObject } from '@kbn/observability-utils-common/object/unflatten_object'; import type { Entity, InventoryEntityLatest } from '../entities'; export function unflattenEntity(entity: Entity) { diff --git a/x-pack/plugins/observability_solution/inventory/public/hooks/use_entity_manager.ts b/x-pack/plugins/observability_solution/inventory/public/hooks/use_entity_manager.ts index 1082017e1ad7a..740c88eb8a9b0 100644 --- a/x-pack/plugins/observability_solution/inventory/public/hooks/use_entity_manager.ts +++ b/x-pack/plugins/observability_solution/inventory/public/hooks/use_entity_manager.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { useAbortableAsync } from '@kbn/observability-utils/hooks/use_abortable_async'; +import { useAbortableAsync } from '@kbn/observability-utils-browser/hooks/use_abortable_async'; import { useState } from 'react'; import { useKibana } from './use_kibana'; diff --git a/x-pack/plugins/observability_solution/inventory/public/hooks/use_inventory_abortable_async.ts b/x-pack/plugins/observability_solution/inventory/public/hooks/use_inventory_abortable_async.ts index 84cef842488e0..1db3b512bbdd6 100644 --- a/x-pack/plugins/observability_solution/inventory/public/hooks/use_inventory_abortable_async.ts +++ b/x-pack/plugins/observability_solution/inventory/public/hooks/use_inventory_abortable_async.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { useAbortableAsync } from '@kbn/observability-utils/hooks/use_abortable_async'; +import { useAbortableAsync } from '@kbn/observability-utils-browser/hooks/use_abortable_async'; import { i18n } from '@kbn/i18n'; import { IHttpFetchError, ResponseErrorBody } from '@kbn/core-http-browser'; import { useKibana } from './use_kibana'; diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_groups.ts b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_groups.ts index 8c72e18bc0740..71cf07282342b 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_groups.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_groups.ts @@ -6,8 +6,8 @@ */ import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; -import type { ObservabilityElasticsearchClient } from '@kbn/observability-utils/es/client/create_observability_es_client'; -import { esqlResultToPlainObjects } from '@kbn/observability-utils/es/utils/esql_result_to_plain_objects'; +import type { ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; +import { esqlResultToPlainObjects } from '@kbn/observability-utils-server/es/esql_result_to_plain_objects'; import { ENTITIES_LATEST_ALIAS, type EntityGroup, diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_types.ts b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_types.ts index 2dfc9b8ccfdf3..36b8e398de5f6 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_types.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_types.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { type ObservabilityElasticsearchClient } from '@kbn/observability-utils/es/client/create_observability_es_client'; +import { type ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; import { ENTITY_TYPE } from '@kbn/observability-shared-plugin/common'; import { ENTITIES_LATEST_ALIAS } from '../../../common/entities'; import { getBuiltinEntityDefinitionIdESQLWhereClause } from './query_helper'; diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts index 402d11720a9da..247f3983766ea 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts @@ -7,8 +7,8 @@ import type { QueryDslQueryContainer, ScalarValue } from '@elastic/elasticsearch/lib/api/types'; import { ENTITY_LAST_SEEN, ENTITY_TYPE } from '@kbn/observability-shared-plugin/common'; -import { type ObservabilityElasticsearchClient } from '@kbn/observability-utils/es/client/create_observability_es_client'; -import { esqlResultToPlainObjects } from '@kbn/observability-utils/es/utils/esql_result_to_plain_objects'; +import { type ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; +import { esqlResultToPlainObjects } from '@kbn/observability-utils-server/es/esql_result_to_plain_objects'; import { ENTITIES_LATEST_ALIAS, MAX_NUMBER_OF_ENTITIES, diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/entities/route.ts b/x-pack/plugins/observability_solution/inventory/server/routes/entities/route.ts index ae99713375b19..731dce0f420bd 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/entities/route.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/entities/route.ts @@ -6,11 +6,11 @@ */ import { INVENTORY_APP_ID } from '@kbn/deeplinks-observability/constants'; import { jsonRt } from '@kbn/io-ts-utils'; -import { createObservabilityEsClient } from '@kbn/observability-utils/es/client/create_observability_es_client'; +import { createObservabilityEsClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; import { ENTITY_TYPE } from '@kbn/observability-shared-plugin/common'; import * as t from 'io-ts'; import { orderBy } from 'lodash'; -import { joinByKey } from '@kbn/observability-utils/array/join_by_key'; +import { joinByKey } from '@kbn/observability-utils-common/array/join_by_key'; import { entityColumnIdsRt, Entity } from '../../../common/entities'; import { createInventoryServerRoute } from '../create_inventory_server_route'; import { getEntityTypes } from './get_entity_types'; diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/has_data/get_has_data.ts b/x-pack/plugins/observability_solution/inventory/server/routes/has_data/get_has_data.ts index c1e4a82c343b0..4d4ab50d6779b 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/has_data/get_has_data.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/has_data/get_has_data.ts @@ -5,8 +5,8 @@ * 2.0. */ import type { Logger } from '@kbn/core/server'; -import { esqlResultToPlainObjects } from '@kbn/observability-utils/es/utils/esql_result_to_plain_objects'; -import { type ObservabilityElasticsearchClient } from '@kbn/observability-utils/es/client/create_observability_es_client'; +import { esqlResultToPlainObjects } from '@kbn/observability-utils-server/es/esql_result_to_plain_objects'; +import { type ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; import { getBuiltinEntityDefinitionIdESQLWhereClause } from '../entities/query_helper'; import { ENTITIES_LATEST_ALIAS } from '../../../common/entities'; diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/has_data/route.ts b/x-pack/plugins/observability_solution/inventory/server/routes/has_data/route.ts index aae8be7f846f8..f0e582b396177 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/has_data/route.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/has_data/route.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { createObservabilityEsClient } from '@kbn/observability-utils/es/client/create_observability_es_client'; +import { createObservabilityEsClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; import { INVENTORY_APP_ID } from '@kbn/deeplinks-observability/constants'; import { createInventoryServerRoute } from '../create_inventory_server_route'; import { getHasData } from './get_has_data'; diff --git a/x-pack/plugins/observability_solution/investigate_app/public/items/embeddable_item/register_embeddable_item.tsx b/x-pack/plugins/observability_solution/investigate_app/public/items/embeddable_item/register_embeddable_item.tsx index 29b2a1319feff..77833e80ec199 100644 --- a/x-pack/plugins/observability_solution/investigate_app/public/items/embeddable_item/register_embeddable_item.tsx +++ b/x-pack/plugins/observability_solution/investigate_app/public/items/embeddable_item/register_embeddable_item.tsx @@ -8,7 +8,7 @@ import { EuiLoadingSpinner, EuiFlexItem } from '@elastic/eui'; import { css } from '@emotion/css'; import { ReactEmbeddableRenderer } from '@kbn/embeddable-plugin/public'; import type { GlobalWidgetParameters } from '@kbn/investigate-plugin/public'; -import { useAbortableAsync } from '@kbn/observability-utils/hooks/use_abortable_async'; +import { useAbortableAsync } from '@kbn/observability-utils-browser/hooks/use_abortable_async'; import React, { useEffect, useMemo, useRef, useState } from 'react'; import { v4 } from 'uuid'; import { ErrorMessage } from '../../components/error_message'; diff --git a/x-pack/plugins/observability_solution/investigate_app/public/items/esql_item/register_esql_item.tsx b/x-pack/plugins/observability_solution/investigate_app/public/items/esql_item/register_esql_item.tsx index 7b88081ca5503..46fe9ea2d9dd2 100644 --- a/x-pack/plugins/observability_solution/investigate_app/public/items/esql_item/register_esql_item.tsx +++ b/x-pack/plugins/observability_solution/investigate_app/public/items/esql_item/register_esql_item.tsx @@ -10,7 +10,7 @@ import type { ESQLSearchResponse } from '@kbn/es-types'; import { i18n } from '@kbn/i18n'; import { type GlobalWidgetParameters } from '@kbn/investigate-plugin/public'; import type { Suggestion } from '@kbn/lens-plugin/public'; -import { useAbortableAsync } from '@kbn/observability-utils/hooks/use_abortable_async'; +import { useAbortableAsync } from '@kbn/observability-utils-browser/hooks/use_abortable_async'; import React, { useMemo } from 'react'; import { ErrorMessage } from '../../components/error_message'; import { useKibana } from '../../hooks/use_kibana'; diff --git a/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/add_investigation_item/esql_widget_preview.tsx b/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/add_investigation_item/esql_widget_preview.tsx index 8d0056dbd538d..469baf6e07f5c 100644 --- a/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/add_investigation_item/esql_widget_preview.tsx +++ b/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/add_investigation_item/esql_widget_preview.tsx @@ -11,7 +11,7 @@ import type { ESQLColumn, ESQLRow } from '@kbn/es-types'; import { GlobalWidgetParameters } from '@kbn/investigate-plugin/public'; import { Item } from '@kbn/investigation-shared'; import type { Suggestion } from '@kbn/lens-plugin/public'; -import { useAbortableAsync } from '@kbn/observability-utils/hooks/use_abortable_async'; +import { useAbortableAsync } from '@kbn/observability-utils-browser/hooks/use_abortable_async'; import React, { useEffect, useMemo, useState } from 'react'; import { ErrorMessage } from '../../../../components/error_message'; import { SuggestVisualizationList } from '../../../../components/suggest_visualization_list'; diff --git a/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/events_timeline/events_timeline.tsx b/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/events_timeline/events_timeline.tsx index befa50bcc0e8d..c3f92139bd936 100644 --- a/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/events_timeline/events_timeline.tsx +++ b/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/events_timeline/events_timeline.tsx @@ -10,7 +10,7 @@ import moment from 'moment'; import { Chart, Axis, AreaSeries, Position, ScaleType, Settings } from '@elastic/charts'; import { useActiveCursor } from '@kbn/charts-plugin/public'; import { EuiSkeletonText } from '@elastic/eui'; -import { getBrushData } from '@kbn/observability-utils/chart/utils'; +import { getBrushData } from '@kbn/observability-utils-browser/chart/utils'; import { Group } from '@kbn/observability-alerting-rule-utils'; import { ALERT_GROUP } from '@kbn/rule-data-utils'; import { SERVICE_NAME } from '@kbn/observability-shared-plugin/common'; diff --git a/yarn.lock b/yarn.lock index ca28b48ef37c4..076ee5808cd30 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5910,7 +5910,15 @@ version "0.0.0" uid "" -"@kbn/observability-utils@link:x-pack/packages/observability/observability_utils": +"@kbn/observability-utils-browser@link:x-pack/packages/observability/observability_utils/observability_utils_browser": + version "0.0.0" + uid "" + +"@kbn/observability-utils-common@link:x-pack/packages/observability/observability_utils/observability_utils_common": + version "0.0.0" + uid "" + +"@kbn/observability-utils-server@link:x-pack/packages/observability/observability_utils/observability_utils_server": version "0.0.0" uid "" From 79925390182e32386628a164e38dd1f2c47fd6b8 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Tue, 12 Nov 2024 15:35:01 +0100 Subject: [PATCH 26/95] Fix references --- .../observability_solution/apm_data_access/tsconfig.json | 5 +++-- x-pack/plugins/observability_solution/infra/tsconfig.json | 4 ++-- .../plugins/observability_solution/inventory/tsconfig.json | 6 ++++-- .../observability_solution/investigate_app/tsconfig.json | 2 +- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm_data_access/tsconfig.json b/x-pack/plugins/observability_solution/apm_data_access/tsconfig.json index aeeb73bee2857..bfe0726115291 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/tsconfig.json +++ b/x-pack/plugins/observability_solution/apm_data_access/tsconfig.json @@ -20,8 +20,9 @@ "@kbn/apm-utils", "@kbn/core-http-server", "@kbn/security-plugin-types-server", - "@kbn/observability-utils", "@kbn/utility-types", - "@kbn/elastic-agent-utils" + "@kbn/elastic-agent-utils", + "@kbn/observability-utils-server", + "@kbn/observability-utils-common" ] } diff --git a/x-pack/plugins/observability_solution/infra/tsconfig.json b/x-pack/plugins/observability_solution/infra/tsconfig.json index 2103350048e4b..efd8be77b688c 100644 --- a/x-pack/plugins/observability_solution/infra/tsconfig.json +++ b/x-pack/plugins/observability_solution/infra/tsconfig.json @@ -114,9 +114,9 @@ "@kbn/management-settings-ids", "@kbn/core-ui-settings-common", "@kbn/entityManager-plugin", - "@kbn/observability-utils", "@kbn/entities-schema", - "@kbn/zod" + "@kbn/zod", + "@kbn/observability-utils-server" ], "exclude": ["target/**/*"] } diff --git a/x-pack/plugins/observability_solution/inventory/tsconfig.json b/x-pack/plugins/observability_solution/inventory/tsconfig.json index 5cb95e8ac6de5..9e1bc6fad7e4f 100644 --- a/x-pack/plugins/observability_solution/inventory/tsconfig.json +++ b/x-pack/plugins/observability_solution/inventory/tsconfig.json @@ -20,7 +20,6 @@ "@kbn/server-route-repository", "@kbn/shared-ux-link-redirect-app", "@kbn/typed-react-router-config", - "@kbn/observability-utils", "@kbn/kibana-react-plugin", "@kbn/i18n", "@kbn/deeplinks-observability", @@ -58,6 +57,9 @@ "@kbn/deeplinks-analytics", "@kbn/controls-plugin", "@kbn/securitysolution-io-ts-types", - "@kbn/react-hooks" + "@kbn/react-hooks", + "@kbn/observability-utils-common", + "@kbn/observability-utils-browser", + "@kbn/observability-utils-server" ] } diff --git a/x-pack/plugins/observability_solution/investigate_app/tsconfig.json b/x-pack/plugins/observability_solution/investigate_app/tsconfig.json index a853456b1156c..6f3741d98da69 100644 --- a/x-pack/plugins/observability_solution/investigate_app/tsconfig.json +++ b/x-pack/plugins/observability_solution/investigate_app/tsconfig.json @@ -67,7 +67,7 @@ "@kbn/calculate-auto", "@kbn/ml-random-sampler-utils", "@kbn/charts-plugin", - "@kbn/observability-utils", "@kbn/observability-alerting-rule-utils", + "@kbn/observability-utils-browser", ], } From 95ae9f8279485b3fa512f8eb20a1b95ec4713fc3 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 12 Nov 2024 14:53:39 +0000 Subject: [PATCH 27/95] [CI] Auto-commit changed files from 'node scripts/generate codeowners' --- .github/CODEOWNERS | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ef265cf7c569a..eb3e06897c62a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -830,7 +830,9 @@ x-pack/packages/observability/alerting_rule_utils @elastic/obs-ux-management-tea x-pack/packages/observability/alerting_test_data @elastic/obs-ux-management-team x-pack/packages/observability/get_padded_alert_time_range_util @elastic/obs-ux-management-team x-pack/packages/observability/logs_overview @elastic/obs-ux-logs-team -x-pack/packages/observability/observability_utils @elastic/observability-ui +x-pack/packages/observability/observability_utils/observability_utils_browser @elastic/observability-ui +x-pack/packages/observability/observability_utils/observability_utils_common @elastic/observability-ui +x-pack/packages/observability/observability_utils/observability_utils_server @elastic/observability-ui x-pack/packages/observability/synthetics_test_data @elastic/obs-ux-management-team x-pack/packages/rollup @elastic/kibana-management x-pack/packages/search/shared_ui @elastic/search-kibana From 7a88cc52f3551c165c855ba379752bffb51bfd9c Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Tue, 12 Nov 2024 16:11:25 +0100 Subject: [PATCH 28/95] Fix issues in APM plugin --- packages/kbn-apm-utils/index.ts | 39 ++++++++++++++++--- .../ai-infra/inference-common/index.ts | 2 + .../inference-common/src}/truncate_list.ts | 0 .../create_apm_event_client/index.ts | 2 +- 4 files changed, 37 insertions(+), 6 deletions(-) rename x-pack/{plugins/inference/common/utils => packages/ai-infra/inference-common/src}/truncate_list.ts (100%) diff --git a/packages/kbn-apm-utils/index.ts b/packages/kbn-apm-utils/index.ts index 7ada02fe8173e..8bd26bc481352 100644 --- a/packages/kbn-apm-utils/index.ts +++ b/packages/kbn-apm-utils/index.ts @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import agent from 'elastic-apm-node'; +import agent, { Logger } from 'elastic-apm-node'; import asyncHooks from 'async_hooks'; export interface SpanOptions { @@ -34,14 +34,42 @@ const runInNewContext = any>(cb: T): ReturnType( optionsOrName: SpanOptions | string, - cb: (span?: Span) => Promise + cb: (span?: Span) => Promise, + logger?: Logger ): Promise { const options = parseSpanOptions(optionsOrName); const { name, type, subtype, labels, intercept } = options; + let time: number | undefined; + if (logger?.isLevelEnabled('debug')) { + time = performance.now(); + } + + function logTook(failed: boolean) { + if (time) { + logger?.debug( + () => + `Operation ${name}${failed ? ` (failed)` : ''} ${ + Math.round(performance.now() - time!) / 1000 + }s` + ); + } + } + + const withLogTook = [ + (res: TR): TR | Promise => { + logTook(false); + return res; + }, + (err: any): never => { + logTook(true); + throw err; + }, + ]; + if (!agent.isStarted()) { - return cb(); + return cb().then(...withLogTook); } let createdSpan: Span | undefined; @@ -57,7 +85,7 @@ export async function withSpan( createdSpan = agent.startSpan(name) ?? undefined; if (!createdSpan) { - return cb(); + return cb().then(...withLogTook); } } @@ -76,7 +104,7 @@ export async function withSpan( } if (!span) { - return promise; + return promise.then(...withLogTook); } const targetedSpan = span; @@ -98,6 +126,7 @@ export async function withSpan( } return promise + .then(...withLogTook) .then((res) => { if (!targetedSpan.outcome || targetedSpan.outcome === 'unknown') { targetedSpan.outcome = 'success'; diff --git a/x-pack/packages/ai-infra/inference-common/index.ts b/x-pack/packages/ai-infra/inference-common/index.ts index 502d8e86a0beb..2791896c801ef 100644 --- a/x-pack/packages/ai-infra/inference-common/index.ts +++ b/x-pack/packages/ai-infra/inference-common/index.ts @@ -81,3 +81,5 @@ export { isInferenceInternalError, isInferenceRequestError, } from './src/errors'; + +export { truncateList } from './src/truncate_list'; diff --git a/x-pack/plugins/inference/common/utils/truncate_list.ts b/x-pack/packages/ai-infra/inference-common/src/truncate_list.ts similarity index 100% rename from x-pack/plugins/inference/common/utils/truncate_list.ts rename to x-pack/packages/ai-infra/inference-common/src/truncate_list.ts diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.ts b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.ts index 104515899fe26..9f04bb9a750f3 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.ts @@ -23,7 +23,7 @@ import { ValuesType } from 'utility-types'; import type { APMError, Metric, Span, Transaction, Event } from '@kbn/apm-types/es_schemas_ui'; import type { InspectResponse } from '@kbn/observability-plugin/typings/common'; import type { DataTier } from '@kbn/observability-shared-plugin/common'; -import { excludeTiersQuery } from '@kbn/observability-utils-server/es/queries/exclude_tiers_query'; +import { excludeTiersQuery } from '@kbn/observability-utils-common/es/queries/exclude_tiers_query'; import { withApmSpan } from '../../../../utils'; import type { ApmDataSource } from '../../../../../common/data_source'; import { cancelEsRequestOnAbort } from '../cancel_es_request_on_abort'; From dab77c18cfb0bd5788d31479da69cc8623c99c49 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 12 Nov 2024 16:02:29 +0000 Subject: [PATCH 29/95] [CI] Auto-commit changed files from 'node scripts/yarn_deduplicate' --- .../plugins/observability_solution/apm_data_access/tsconfig.json | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/apm_data_access/tsconfig.json b/x-pack/plugins/observability_solution/apm_data_access/tsconfig.json index bfe0726115291..d4c38fddf967e 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/tsconfig.json +++ b/x-pack/plugins/observability_solution/apm_data_access/tsconfig.json @@ -22,7 +22,6 @@ "@kbn/security-plugin-types-server", "@kbn/utility-types", "@kbn/elastic-agent-utils", - "@kbn/observability-utils-server", "@kbn/observability-utils-common" ] } From a60a16523c07286f78c05137b0d4898d9a050988 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Tue, 12 Nov 2024 14:14:08 -0700 Subject: [PATCH 30/95] Updating the owners and CODEOWNERS to @simianhacker @flash1293 @dgieselaar since we don't have an official team for this plugin yet. --- .github/CODEOWNERS | 2 +- x-pack/plugins/streams/kibana.jsonc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ecdb994e3d140..91eee74db6b03 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -966,7 +966,7 @@ x-pack/plugins/snapshot_restore @elastic/kibana-management x-pack/plugins/spaces @elastic/kibana-security x-pack/plugins/stack_alerts @elastic/response-ops x-pack/plugins/stack_connectors @elastic/response-ops -x-pack/plugins/streams @elastic/obs-entities +x-pack/plugins/streams @simianhacker @flash1293 @dgieselaar x-pack/plugins/task_manager @elastic/response-ops x-pack/plugins/telemetry_collection_xpack @elastic/kibana-core x-pack/plugins/threat_intelligence @elastic/security-threat-hunting-investigations diff --git a/x-pack/plugins/streams/kibana.jsonc b/x-pack/plugins/streams/kibana.jsonc index 8e428c0a285a1..06c37ed245cf1 100644 --- a/x-pack/plugins/streams/kibana.jsonc +++ b/x-pack/plugins/streams/kibana.jsonc @@ -1,7 +1,7 @@ { "type": "plugin", "id": "@kbn/streams-plugin", - "owner": "@elastic/obs-entities", + "owner": "@simianhacker @flash1293 @dgieselaar", "description": "A manager for Streams", "group": "observability", "visibility": "private", @@ -16,7 +16,7 @@ "encryptedSavedObjects", "usageCollection", "licensing", - "taskManager", + "taskManager" ], "optionalPlugins": [ "cloud", From 2bef5128599fc79398e641e75379f9b364c183e5 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Wed, 13 Nov 2024 10:38:24 +0100 Subject: [PATCH 31/95] Fix references --- .../observability_solution/investigate_app/tsconfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/investigate_app/tsconfig.json b/x-pack/plugins/observability_solution/investigate_app/tsconfig.json index 293809ad6ff30..0851a13367091 100644 --- a/x-pack/plugins/observability_solution/investigate_app/tsconfig.json +++ b/x-pack/plugins/observability_solution/investigate_app/tsconfig.json @@ -62,7 +62,6 @@ "@kbn/licensing-plugin", "@kbn/rule-data-utils", "@kbn/entities-schema", - "@kbn/inference-plugin", "@kbn/core-elasticsearch-server", "@kbn/calculate-auto", "@kbn/ml-random-sampler-utils", @@ -70,5 +69,6 @@ "@kbn/observability-alerting-rule-utils", "@kbn/observability-utils-browser", "@kbn/usage-collection-plugin", + "@kbn/inference-common", ], } From 5c1019846d6ccd6a96627f85aa15e280d411e9a3 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Wed, 13 Nov 2024 12:16:21 +0100 Subject: [PATCH 32/95] Option to unset value on error --- .../hooks/use_abortable_async.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_abortable_async.ts b/x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_abortable_async.ts index d24a62ee125bc..f0d2bf4a05872 100644 --- a/x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_abortable_async.ts +++ b/x-pack/packages/observability/observability_utils/observability_utils_browser/hooks/use_abortable_async.ts @@ -22,6 +22,7 @@ export type AbortableAsyncStateOf> = interface UseAbortableAsyncOptions { clearValueOnNext?: boolean; + unsetValueOnError?: boolean; defaultValue?: () => T; onError?: (error: Error) => void; } @@ -41,6 +42,7 @@ export function useAbortableAsync( options?: UseAbortableAsyncOptions ): AbortableAsyncState { const clearValueOnNext = options?.clearValueOnNext; + const unsetValueOnError = options?.unsetValueOnError; const controllerRef = useRef(new AbortController()); @@ -63,7 +65,9 @@ export function useAbortableAsync( function handleError(err: Error) { setError(err); - // setValue(undefined); + if (unsetValueOnError) { + setValue(undefined); + } setLoading(false); options?.onError?.(err); } From 9ca2da26f658d5a44059262a4a69a67d34297de7 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Wed, 13 Nov 2024 12:52:32 +0100 Subject: [PATCH 33/95] Fix lingering import --- .../inventory/public/components/grouped_inventory/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/inventory/public/components/grouped_inventory/index.tsx b/x-pack/plugins/observability_solution/inventory/public/components/grouped_inventory/index.tsx index 0964b7bb39465..6cfdc079be299 100644 --- a/x-pack/plugins/observability_solution/inventory/public/components/grouped_inventory/index.tsx +++ b/x-pack/plugins/observability_solution/inventory/public/components/grouped_inventory/index.tsx @@ -8,7 +8,7 @@ import { EuiSpacer } from '@elastic/eui'; import { ENTITY_TYPE } from '@kbn/observability-shared-plugin/common'; import React from 'react'; import useEffectOnce from 'react-use/lib/useEffectOnce'; -import { flattenObject } from '@kbn/observability-utils/object/flatten_object'; +import { flattenObject } from '@kbn/observability-utils-common/object/flatten_object'; import { useInventoryAbortableAsync } from '../../hooks/use_inventory_abortable_async'; import { useKibana } from '../../hooks/use_kibana'; import { useUnifiedSearchContext } from '../../hooks/use_unified_search_context'; From d655c0a63aac8364d7670204e9237d4f3d3eb7be Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Wed, 13 Nov 2024 14:17:02 +0100 Subject: [PATCH 34/95] Clean up esql signature --- .../client/create_observability_es_client.ts | 75 ++++++++++++------- .../esql_chart/controlled_esql_chart.tsx | 4 +- .../public/util/esql_result_to_timeseries.ts | 6 +- .../server/routes/has_data/get_has_data.ts | 2 +- .../streams/server/lib/streams/stream_crud.ts | 2 - .../streams/server/routes/esql/route.ts | 39 +++++----- 6 files changed, 75 insertions(+), 53 deletions(-) diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts index 3bcb138465cf8..8fc899ac254f0 100644 --- a/x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts @@ -10,12 +10,13 @@ import type { FieldCapsRequest, FieldCapsResponse, MsearchRequest, + ScalarValue, SearchResponse, } from '@elastic/elasticsearch/lib/api/types'; import { withSpan } from '@kbn/apm-utils'; import type { ElasticsearchClient, Logger } from '@kbn/core/server'; -import type { ESQLSearchResponse, ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types'; -import { Required } from 'utility-types'; +import type { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types'; +import { Required, ValuesType } from 'utility-types'; import { esqlResultToPlainObjects } from '../esql_result_to_plain_objects'; type SearchRequest = ESSearchRequest & { @@ -24,20 +25,32 @@ type SearchRequest = ESSearchRequest & { size: number | boolean; }; -type EsqlQueryParameters = EsqlQueryRequest & { parseOutput?: boolean }; -type EsqlOutputParameters = Omit & { - parseOutput?: boolean; - format?: 'json'; - columnar?: false; -}; +interface EsqlOptions { + asPlainObjects?: boolean; +} -type EsqlParameters = EsqlOutputParameters | EsqlQueryParameters; +type EsqlValue = ScalarValue | ScalarValue[]; -export type InferEsqlResponseOf< - TOutput = unknown, - TParameters extends EsqlParameters = EsqlParameters -> = TParameters extends EsqlOutputParameters ? TOutput[] : ESQLSearchResponse; +type EsqlOutput = Record; +type InferEsqlResponseOf< + TOutput extends EsqlOutput, + TOptions extends EsqlOptions | undefined = { asPlainObjects: true } +> = TOptions extends { asPlainObjects: true } + ? { + objects: Array<{ + [key in keyof TOutput]: TOutput[key]; + }>; + } + : { + columns: Array<{ name: keyof TOutput; type: string }>; + values: Array>>; + }; + +export interface EsqlQueryResponse { + columns: Array<{ name: string; type: string }>; + values: EsqlValue[][]; +} /** * An Elasticsearch Client with a fully typed `search` method and built-in * APM instrumentation. @@ -57,14 +70,14 @@ export interface ObservabilityElasticsearchClient { operationName: string, request: Required ): Promise; - esql( + esql< + TOutput extends EsqlOutput = EsqlOutput, + TEsqlOptions extends EsqlOptions | undefined = { asPlainObjects: true } + >( operationName: string, - parameters: TQueryParams - ): Promise>; - esql( - operationName: string, - parameters: TQueryParams - ): Promise>; + parameters: EsqlQueryRequest, + options?: TEsqlOptions + ): Promise>; client: ElasticsearchClient; } @@ -109,14 +122,18 @@ export function createObservabilityEsClient({ }); }); }, - esql( + esql< + TOutput extends EsqlOutput = EsqlOutput, + TEsqlOptions extends EsqlOptions | undefined = { asPlainObjects: true } + >( operationName: string, - { parseOutput = true, format = 'json', columnar = false, ...parameters }: TSearchRequest - ) { + parameters: EsqlQueryRequest, + options?: EsqlOptions + ): Promise> { return callWithLogger(operationName, parameters, () => { return client.esql .query( - { ...parameters, format, columnar }, + { ...parameters }, { querystring: { drop_null_columns: true, @@ -124,12 +141,14 @@ export function createObservabilityEsClient({ } ) .then((response) => { - const esqlResponse = response as unknown as ESQLSearchResponse; + const esqlResponse = response as unknown as EsqlQueryResponse; - const shouldParseOutput = parseOutput && !columnar && format === 'json'; - return shouldParseOutput - ? esqlResultToPlainObjects(esqlResponse) + const shouldParseOutput = options?.asPlainObjects !== false; + const finalResponse = shouldParseOutput + ? { objects: esqlResultToPlainObjects(esqlResponse) } : esqlResponse; + + return finalResponse as InferEsqlResponseOf; }); }); }, diff --git a/x-pack/plugins/logsai/streams_app/public/components/esql_chart/controlled_esql_chart.tsx b/x-pack/plugins/logsai/streams_app/public/components/esql_chart/controlled_esql_chart.tsx index 0f10325c90234..535a60266b4fe 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/esql_chart/controlled_esql_chart.tsx +++ b/x-pack/plugins/logsai/streams_app/public/components/esql_chart/controlled_esql_chart.tsx @@ -23,7 +23,7 @@ import { i18n } from '@kbn/i18n'; import { getTimeZone } from '@kbn/observability-utils-browser/utils/ui_settings/get_timezone'; import { css } from '@emotion/css'; import { AbortableAsyncState } from '@kbn/observability-utils-browser/hooks/use_abortable_async'; -import { ESQLSearchResponse } from '@kbn/es-types'; +import type { EsqlQueryResponse } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; import { esqlResultToTimeseries } from '../../util/esql_result_to_timeseries'; import { useKibana } from '../../hooks/use_kibana'; import { LoadingPanel } from '../loading_panel'; @@ -52,7 +52,7 @@ export function ControlledEsqlChart({ height, }: { id: string; - result: AbortableAsyncState; + result: AbortableAsyncState; metricNames: T[]; chartType?: 'area' | 'bar' | 'line'; height: number; diff --git a/x-pack/plugins/logsai/streams_app/public/util/esql_result_to_timeseries.ts b/x-pack/plugins/logsai/streams_app/public/util/esql_result_to_timeseries.ts index 11a6840baaedb..f5af978922ffb 100644 --- a/x-pack/plugins/logsai/streams_app/public/util/esql_result_to_timeseries.ts +++ b/x-pack/plugins/logsai/streams_app/public/util/esql_result_to_timeseries.ts @@ -4,9 +4,9 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { orderBy } from 'lodash'; import type { AbortableAsyncState } from '@kbn/observability-utils-browser/hooks/use_abortable_async'; -import type { ESQLSearchResponse } from '@kbn/es-types'; +import type { EsqlQueryResponse } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; +import { orderBy } from 'lodash'; interface Timeseries { id: string; @@ -19,7 +19,7 @@ export function esqlResultToTimeseries({ result, metricNames, }: { - result: AbortableAsyncState; + result: AbortableAsyncState; metricNames: T[]; }): Array> { const columns = result.value?.columns; diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/has_data/get_has_data.ts b/x-pack/plugins/observability_solution/inventory/server/routes/has_data/get_has_data.ts index 2f59478f17c02..a2c62b2472862 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/has_data/get_has_data.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/has_data/get_has_data.ts @@ -24,7 +24,7 @@ export async function getHasData({ | LIMIT 1`, }); - const totalCount = esqlResults[0]._count; + const totalCount = esqlResults.objects[0]._count; return { hasData: totalCount > 0 }; } catch (e) { diff --git a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts index 199a42a746f49..a74540cdcc62a 100644 --- a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts +++ b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts @@ -99,8 +99,6 @@ export async function listStreams({ const response = await scopedClusterClient.asInternalUser.search({ index: STREAMS_INDEX, size: 10000, - fields: ['id'], - _source: false, sort: [{ id: 'asc' }], }); const definitions = response.hits.hits.map((hit) => hit._source!); diff --git a/x-pack/plugins/streams/server/routes/esql/route.ts b/x-pack/plugins/streams/server/routes/esql/route.ts index 4f60d40898f4b..c226e982cf402 100644 --- a/x-pack/plugins/streams/server/routes/esql/route.ts +++ b/x-pack/plugins/streams/server/routes/esql/route.ts @@ -5,11 +5,13 @@ * 2.0. */ -import { ESQLSearchResponse } from '@kbn/es-types'; -import { createObservabilityEsClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; -import { kqlQuery } from '@kbn/observability-utils-common/es/queries/kql_query'; import { excludeFrozenQuery } from '@kbn/observability-utils-common/es/queries/exclude_frozen_query'; +import { kqlQuery } from '@kbn/observability-utils-common/es/queries/kql_query'; import { rangeQuery } from '@kbn/observability-utils-common/es/queries/range_query'; +import { + EsqlQueryResponse, + createObservabilityEsClient, +} from '@kbn/observability-utils-server/es/client/create_observability_es_client'; import { z } from '@kbn/zod'; import { isNumber } from 'lodash'; import { createServerRoute } from '../create_server_route'; @@ -26,7 +28,7 @@ export const executeEsqlRoute = createServerRoute({ end: z.number().optional(), }), }), - handler: async ({ params, request, logger, getScopedClients }): Promise => { + handler: async ({ params, request, logger, getScopedClients }): Promise => { const { scopedClusterClient } = await getScopedClients({ request }); const observabilityEsClient = createObservabilityEsClient({ client: scopedClusterClient.asCurrentUser, @@ -38,22 +40,25 @@ export const executeEsqlRoute = createServerRoute({ body: { operationName, query, filter, kuery, start, end }, } = params; - const response = await observabilityEsClient.esql(operationName, { - parseOutput: false, - query, - filter: { - bool: { - filter: [ - filter || { match_all: {} }, - ...kqlQuery(kuery), - ...excludeFrozenQuery(), - ...(isNumber(start) && isNumber(end) ? rangeQuery(start, end) : []), - ], + const response = await observabilityEsClient.esql( + operationName, + { + query, + filter: { + bool: { + filter: [ + filter || { match_all: {} }, + ...kqlQuery(kuery), + ...excludeFrozenQuery(), + ...(isNumber(start) && isNumber(end) ? rangeQuery(start, end) : []), + ], + }, }, }, - }); + { asPlainObjects: false } + ); - return response as any; + return response; }, }); From 8e0eba8a4b694f97292a094deebaabe57de8abed Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Wed, 13 Nov 2024 14:43:03 +0100 Subject: [PATCH 35/95] Handle sync callbacks in withSpan --- packages/kbn-apm-utils/index.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/kbn-apm-utils/index.ts b/packages/kbn-apm-utils/index.ts index 8bd26bc481352..4d551c3b9f037 100644 --- a/packages/kbn-apm-utils/index.ts +++ b/packages/kbn-apm-utils/index.ts @@ -69,7 +69,13 @@ export async function withSpan( ]; if (!agent.isStarted()) { - return cb().then(...withLogTook); + const promise = cb(); + // make sure tests that mock out the callback with a sync + // function don't fail. + if (typeof promise === 'object' && 'then' in promise) { + return promise.then(...withLogTook); + } + return promise; } let createdSpan: Span | undefined; From 5e90008035d1f353feda1326a84fa44c8026249b Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Wed, 13 Nov 2024 18:29:56 +0100 Subject: [PATCH 36/95] Make APIs internal --- .../src/create_repository_client.ts | 4 +- .../src/parse_endpoint.ts | 4 -- .../src/typings.ts | 61 +++++++++++++------ .../src/create_server_route_factory.ts | 3 +- .../src/register_routes.ts | 15 +++-- .../src/test_types.ts | 30 ++++++--- .../apm/server/routes/typings.ts | 4 +- x-pack/plugins/streams/public/plugin.ts | 2 +- x-pack/plugins/streams/server/routes/index.ts | 2 + .../streams/server/routes/streams/delete.ts | 21 ++++--- .../streams/server/routes/streams/disable.ts | 47 ++++++++++++++ .../streams/server/routes/streams/edit.ts | 13 ++-- .../streams/server/routes/streams/enable.ts | 17 ++++-- .../streams/server/routes/streams/fork.ts | 21 ++++--- .../streams/server/routes/streams/list.ts | 6 +- .../streams/server/routes/streams/read.ts | 19 ++++-- .../streams/server/routes/streams/resync.ts | 13 ++-- .../streams/server/routes/streams/settings.ts | 3 +- 18 files changed, 203 insertions(+), 82 deletions(-) create mode 100644 x-pack/plugins/streams/server/routes/streams/disable.ts diff --git a/packages/kbn-server-route-repository-client/src/create_repository_client.ts b/packages/kbn-server-route-repository-client/src/create_repository_client.ts index 015db2b9948d8..325f484103d09 100644 --- a/packages/kbn-server-route-repository-client/src/create_repository_client.ts +++ b/packages/kbn-server-route-repository-client/src/create_repository_client.ts @@ -15,12 +15,12 @@ import { } from '@kbn/server-route-repository-utils'; import { httpResponseIntoObservable } from '@kbn/sse-utils-client'; import { from } from 'rxjs'; -import { HttpFetchOptions, HttpFetchQuery, HttpResponse } from '@kbn/core-http-browser'; +import { HttpFetchQuery, HttpResponse } from '@kbn/core-http-browser'; import { omit } from 'lodash'; export function createRepositoryClient< TRepository extends ServerRouteRepository, - TClientOptions extends HttpFetchOptions = {} + TClientOptions extends Record = {} >(core: CoreStart | CoreSetup): RouteRepositoryClient { const fetch = ( endpoint: string, diff --git a/packages/kbn-server-route-repository-utils/src/parse_endpoint.ts b/packages/kbn-server-route-repository-utils/src/parse_endpoint.ts index e16590c9a666f..72d170a0c05a8 100644 --- a/packages/kbn-server-route-repository-utils/src/parse_endpoint.ts +++ b/packages/kbn-server-route-repository-utils/src/parse_endpoint.ts @@ -20,9 +20,5 @@ export function parseEndpoint(endpoint: string) { throw new Error(`Endpoint ${endpoint} was not prefixed with a valid HTTP method`); } - if (!version && pathname.startsWith('/api')) { - throw new Error(`Missing version for public endpoint ${endpoint}`); - } - return { method, pathname, version }; } diff --git a/packages/kbn-server-route-repository-utils/src/typings.ts b/packages/kbn-server-route-repository-utils/src/typings.ts index 35a2f41054c99..77dfd8f051bf6 100644 --- a/packages/kbn-server-route-repository-utils/src/typings.ts +++ b/packages/kbn-server-route-repository-utils/src/typings.ts @@ -8,7 +8,7 @@ */ import type { HttpFetchOptions } from '@kbn/core-http-browser'; -import type { IKibanaResponse } from '@kbn/core-http-server'; +import type { IKibanaResponse, RouteAccess } from '@kbn/core-http-server'; import type { KibanaRequest, KibanaResponseFactory, @@ -56,18 +56,36 @@ export interface RouteState { } export type ServerRouteHandlerResources = Record; -export type ServerRouteCreateOptions = Record; -type ValidateEndpoint = string extends TEndpoint +export interface ServerRouteCreateOptions { + [x: string]: any; +} + +type RouteMethodOf = TEndpoint extends `${infer TRouteMethod} ${string}` + ? TRouteMethod extends RouteMethod + ? TRouteMethod + : RouteMethod + : RouteMethod; + +type IsPublicEndpoint< + TEndpoint extends string, + TRouteAccess extends RouteAccess | undefined +> = TRouteAccess extends 'public' ? true - : TEndpoint extends `${string} ${string} ${string}` + : TRouteAccess extends 'internal' + ? false + : TEndpoint extends `${string} /api${string}` ? true - : TEndpoint extends `${string} ${infer TPathname}` - ? TPathname extends `/internal/${string}` - ? true - : false : false; +type IsVersionSpecified = + TEndpoint extends `${string} ${string} ${string}` ? true : false; + +type ValidateEndpoint< + TEndpoint extends string, + TRouteAccess extends RouteAccess | undefined +> = IsPublicEndpoint extends true ? IsVersionSpecified : true; + type IsAny = 1 | 0 extends (T extends never ? 1 : 0) ? true : false; // this ensures only plain objects can be returned, if it's not one @@ -131,12 +149,16 @@ export type CreateServerRouteFactory< > = < TEndpoint extends string, TReturnType extends ServerRouteHandlerReturnType, - TRouteParamsRT extends RouteParamsRT | undefined = undefined + TRouteParamsRT extends RouteParamsRT | undefined = undefined, + TRouteAccess extends RouteAccess | undefined = undefined >( options: { - endpoint: ValidateEndpoint extends true ? TEndpoint : never; + endpoint: ValidateEndpoint extends true ? TEndpoint : never; handler: ServerRouteHandler; params?: TRouteParamsRT; + options?: RouteConfigOptions> & { + access?: TRouteAccess; + }; } & TRouteCreateOptions ) => Record< TEndpoint, @@ -154,16 +176,15 @@ export type ServerRoute< TRouteParamsRT extends RouteParamsRT | undefined, TRouteHandlerResources extends ServerRouteHandlerResources, TReturnType extends ServerRouteHandlerReturnType, - TRouteCreateOptions extends ServerRouteCreateOptions -> = { + TRouteCreateOptions extends ServerRouteCreateOptions | undefined +> = TRouteCreateOptions & { endpoint: TEndpoint; handler: ServerRouteHandler; -} & TRouteCreateOptions & - (TRouteParamsRT extends RouteParamsRT ? { params: TRouteParamsRT } : {}); +} & (TRouteParamsRT extends RouteParamsRT ? { params: TRouteParamsRT } : {}); export type ServerRouteRepository = Record< string, - ServerRoute> + ServerRoute >; type ClientRequestParamsOfType = @@ -229,7 +250,7 @@ export type ClientRequestParamsOf< infer TRouteParamsRT, any, any, - ServerRouteCreateOptions + ServerRouteCreateOptions | undefined > ? TRouteParamsRT extends RouteParamsRT ? ClientRequestParamsOfType @@ -249,13 +270,17 @@ export interface RouteRepositoryClient< fetch>( endpoint: TEndpoint, ...args: MaybeOptionalArgs< - ClientRequestParamsOf & TAdditionalClientOptions + ClientRequestParamsOf & + TAdditionalClientOptions & + HttpFetchOptions > ): Promise>; stream>( endpoint: TEndpoint, ...args: MaybeOptionalArgs< - ClientRequestParamsOf & TAdditionalClientOptions + ClientRequestParamsOf & + TAdditionalClientOptions & + HttpFetchOptions > ): ReturnOf extends Observable ? TReturnType extends ServerSentEvent diff --git a/packages/kbn-server-route-repository/src/create_server_route_factory.ts b/packages/kbn-server-route-repository/src/create_server_route_factory.ts index be375bd069480..bbf45bcd3b442 100644 --- a/packages/kbn-server-route-repository/src/create_server_route_factory.ts +++ b/packages/kbn-server-route-repository/src/create_server_route_factory.ts @@ -8,7 +8,6 @@ */ import type { - DefaultRouteCreateOptions, DefaultRouteHandlerResources, ServerRouteCreateOptions, ServerRouteHandlerResources, @@ -17,7 +16,7 @@ import type { CreateServerRouteFactory } from '@kbn/server-route-repository-util export function createServerRouteFactory< TRouteHandlerResources extends ServerRouteHandlerResources = DefaultRouteHandlerResources, - TRouteCreateOptions extends ServerRouteCreateOptions = DefaultRouteCreateOptions + TRouteCreateOptions extends ServerRouteCreateOptions = {} >(): CreateServerRouteFactory { return (route) => ({ [route.endpoint]: route } as any); } diff --git a/packages/kbn-server-route-repository/src/register_routes.ts b/packages/kbn-server-route-repository/src/register_routes.ts index 5e0fa51a4544f..a2f8addc70b0c 100644 --- a/packages/kbn-server-route-repository/src/register_routes.ts +++ b/packages/kbn-server-route-repository/src/register_routes.ts @@ -22,7 +22,7 @@ import { } from '@kbn/server-route-repository-utils'; import { observableIntoEventSourceStream } from '@kbn/sse-utils-server'; import { isZod } from '@kbn/zod'; -import { merge } from 'lodash'; +import { merge, omit } from 'lodash'; import { Observable, isObservable } from 'rxjs'; import { ServerSentEvent } from '@kbn/sse-utils'; import { passThroughValidationObject, noParamsValidationObject } from './validation_objects'; @@ -52,7 +52,10 @@ export function registerRoutes>({ const router = core.http.createRouter(); routes.forEach((route) => { - const { params, endpoint, options, handler } = route; + const { endpoint, handler } = route; + + const params = 'params' in route ? route.params : undefined; + const options = 'options' in route ? route.options! : {}; const { method, pathname, version } = parseEndpoint(endpoint); @@ -141,16 +144,18 @@ export function registerRoutes>({ router[method]( { path: pathname, - options, + options: omit(options, 'security'), validate: validationObject, + security: options.security, }, wrappedHandler ); } else { router.versioned[method]({ path: pathname, - access: pathname.startsWith('/internal/') ? 'internal' : 'public', - options, + access: options.access ?? (pathname.startsWith('/internal/') ? 'internal' : 'public'), + options: omit(options, 'access', 'security'), + security: options.security, }).addVersion( { version, diff --git a/packages/kbn-server-route-repository/src/test_types.ts b/packages/kbn-server-route-repository/src/test_types.ts index acef5a524eb6b..4ee8ce65beb56 100644 --- a/packages/kbn-server-route-repository/src/test_types.ts +++ b/packages/kbn-server-route-repository/src/test_types.ts @@ -83,36 +83,46 @@ createServerRouteFactory<{ context: { getSpaceId: () => string } }, {}>()({ }); // Create options are available when registering a route. -createServerRouteFactory<{}, { options: { tags: string[] } }>()({ +createServerRouteFactory<{}, {}>()({ endpoint: 'GET /internal/endpoint_with_params', params: t.type({ path: t.type({ serviceName: t.string, }), }), - options: { - tags: [], - }, handler: async (resources) => { assertType<{ params: { path: { serviceName: string } } }>(resources); }, }); // Public APIs should be versioned -createServerRouteFactory<{}, { options: { tags: string[] } }>()({ +createServerRouteFactory<{}, { tags: string[] }>()({ // @ts-expect-error + endpoint: 'GET /api/endpoint_with_params', + tags: [], + handler: async (resources) => {}, +}); + +// `access` is respected +createServerRouteFactory<{}, { tags: string[] }>()({ endpoint: 'GET /api/endpoint_with_params', options: { - tags: [], + access: 'internal', }, + tags: [], handler: async (resources) => {}, }); -createServerRouteFactory<{}, { options: { tags: string[] } }>()({ +// specifying additional options makes them required +// @ts-expect-error +createServerRouteFactory<{}, { tags: string[] }>()({ endpoint: 'GET /api/endpoint_with_params 2023-10-31', - options: { - tags: [], - }, + handler: async (resources) => {}, +}); + +createServerRouteFactory<{}, { tags: string[] }>()({ + endpoint: 'GET /api/endpoint_with_params 2023-10-31', + tags: [], handler: async (resources) => {}, }); diff --git a/x-pack/plugins/observability_solution/apm/server/routes/typings.ts b/x-pack/plugins/observability_solution/apm/server/routes/typings.ts index f9ea085a11e6b..c3ea63801540c 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/typings.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/typings.ts @@ -9,7 +9,6 @@ import type { CoreSetup, CustomRequestHandlerContext, CoreStart, - RouteConfigOptions, IScopedClusterClient, IUiSettingsClient, SavedObjectsClientContract, @@ -60,9 +59,8 @@ export interface APMRouteCreateOptions { | 'oas-tag:APM agent keys' | 'oas-tag:APM annotations' >; - body?: { accepts: Array<'application/json' | 'multipart/form-data'> }; disableTelemetry?: boolean; - } & RouteConfigOptions; + }; } export type TelemetryUsageCounter = ReturnType; diff --git a/x-pack/plugins/streams/public/plugin.ts b/x-pack/plugins/streams/public/plugin.ts index a461281f0822e..532340dcafdf7 100644 --- a/x-pack/plugins/streams/public/plugin.ts +++ b/x-pack/plugins/streams/public/plugin.ts @@ -46,7 +46,7 @@ export class Plugin implements StreamsPluginClass { const createStatusObservable = once((repositoryClient: StreamsRepositoryClient) => { return from( repositoryClient - .fetch('GET /internal/streams/_status', { + .fetch('GET /api/streams/_status', { signal: new AbortController().signal, }) .then((response) => ({ diff --git a/x-pack/plugins/streams/server/routes/index.ts b/x-pack/plugins/streams/server/routes/index.ts index 31c0e452e891c..fb676ce94b9f8 100644 --- a/x-pack/plugins/streams/server/routes/index.ts +++ b/x-pack/plugins/streams/server/routes/index.ts @@ -7,6 +7,7 @@ import { esqlRoutes } from './esql/route'; import { deleteStreamRoute } from './streams/delete'; +import { disableStreamsRoute } from './streams/disable'; import { editStreamRoute } from './streams/edit'; import { enableStreamsRoute } from './streams/enable'; import { forkStreamsRoute } from './streams/fork'; @@ -25,6 +26,7 @@ export const StreamsRouteRepository = { ...listStreamsRoute, ...streamsStatusRoutes, ...esqlRoutes, + ...disableStreamsRoute, }; export type StreamsRouteRepository = typeof StreamsRouteRepository; diff --git a/x-pack/plugins/streams/server/routes/streams/delete.ts b/x-pack/plugins/streams/server/routes/streams/delete.ts index cea5275e9b409..048ec67499c82 100644 --- a/x-pack/plugins/streams/server/routes/streams/delete.ts +++ b/x-pack/plugins/streams/server/routes/streams/delete.ts @@ -8,6 +8,7 @@ import { z } from '@kbn/zod'; import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; import { Logger } from '@kbn/logging'; +import { badRequest, internal, notFound } from '@hapi/boom'; import { DefinitionNotFound, ForkConditionMissing, @@ -20,9 +21,9 @@ import { MalformedStreamId } from '../../lib/streams/errors/malformed_stream_id' import { getParentId } from '../../lib/streams/helpers/hierarchy'; export const deleteStreamRoute = createServerRoute({ - endpoint: 'DELETE /api/streams/{id} 2023-10-31', + endpoint: 'DELETE /api/streams/{id}', options: { - access: 'public', + access: 'internal', security: { authz: { requiredPrivileges: ['streams_write'], @@ -34,7 +35,13 @@ export const deleteStreamRoute = createServerRoute({ id: z.string(), }), }), - handler: async ({ response, params, logger, request, getScopedClients }) => { + handler: async ({ + response, + params, + logger, + request, + getScopedClients, + }): Promise<{ acknowledged: true }> => { try { const { scopedClusterClient } = await getScopedClients({ request }); @@ -47,10 +54,10 @@ export const deleteStreamRoute = createServerRoute({ await deleteStream(scopedClusterClient, params.path.id, logger); - return response.ok({ body: { acknowledged: true } }); + return { acknowledged: true }; } catch (e) { if (e instanceof IndexTemplateNotFound || e instanceof DefinitionNotFound) { - return response.notFound({ body: e }); + throw notFound(e); } if ( @@ -58,10 +65,10 @@ export const deleteStreamRoute = createServerRoute({ e instanceof ForkConditionMissing || e instanceof MalformedStreamId ) { - return response.customError({ body: e, statusCode: 400 }); + throw badRequest(e); } - return response.customError({ body: e, statusCode: 500 }); + throw internal(e); } }, }); diff --git a/x-pack/plugins/streams/server/routes/streams/disable.ts b/x-pack/plugins/streams/server/routes/streams/disable.ts new file mode 100644 index 0000000000000..d6fb79b2b0058 --- /dev/null +++ b/x-pack/plugins/streams/server/routes/streams/disable.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { badRequest, internal } from '@hapi/boom'; +import { z } from '@kbn/zod'; +import { STREAMS_INDEX } from '../../../common/constants'; +import { SecurityException } from '../../lib/streams/errors'; +import { createServerRoute } from '../create_server_route'; + +export const disableStreamsRoute = createServerRoute({ + endpoint: 'POST /api/streams/_disable', + params: z.object({}), + options: { + access: 'internal', + security: { + authz: { + requiredPrivileges: ['streams_write'], + }, + }, + }, + handler: async ({ + request, + response, + logger, + getScopedClients, + }): Promise<{ acknowledged: true }> => { + try { + const { scopedClusterClient } = await getScopedClients({ request }); + + await scopedClusterClient.asInternalUser.indices.delete({ + index: STREAMS_INDEX, + allow_no_indices: true, + }); + + return { acknowledged: true }; + } catch (e) { + if (e instanceof SecurityException) { + throw badRequest(e); + } + throw internal(e); + } + }, +}); diff --git a/x-pack/plugins/streams/server/routes/streams/edit.ts b/x-pack/plugins/streams/server/routes/streams/edit.ts index 5c027a9985a07..f0fa0aa881d60 100644 --- a/x-pack/plugins/streams/server/routes/streams/edit.ts +++ b/x-pack/plugins/streams/server/routes/streams/edit.ts @@ -8,6 +8,7 @@ import { z } from '@kbn/zod'; import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; import { Logger } from '@kbn/logging'; +import { badRequest, internal, notFound } from '@hapi/boom'; import { DefinitionNotFound, ForkConditionMissing, @@ -28,9 +29,9 @@ import { getParentId } from '../../lib/streams/helpers/hierarchy'; import { MalformedChildren } from '../../lib/streams/errors/malformed_children'; export const editStreamRoute = createServerRoute({ - endpoint: 'PUT /api/streams/{id} 2023-10-31', + endpoint: 'PUT /api/streams/{id}', options: { - access: 'public', + access: 'internal', security: { authz: { requiredPrivileges: ['streams_write'], @@ -94,10 +95,10 @@ export const editStreamRoute = createServerRoute({ }); } - return response.ok({ body: { acknowledged: true } }); + return { acknowledged: true }; } catch (e) { if (e instanceof IndexTemplateNotFound || e instanceof DefinitionNotFound) { - return response.notFound({ body: e }); + throw notFound(e); } if ( @@ -105,10 +106,10 @@ export const editStreamRoute = createServerRoute({ e instanceof ForkConditionMissing || e instanceof MalformedStreamId ) { - return response.customError({ body: e, statusCode: 400 }); + throw badRequest(e); } - return response.customError({ body: e, statusCode: 500 }); + throw internal(e); } }, }); diff --git a/x-pack/plugins/streams/server/routes/streams/enable.ts b/x-pack/plugins/streams/server/routes/streams/enable.ts index a0bd322e60845..15814a777aff2 100644 --- a/x-pack/plugins/streams/server/routes/streams/enable.ts +++ b/x-pack/plugins/streams/server/routes/streams/enable.ts @@ -6,6 +6,7 @@ */ import { z } from '@kbn/zod'; +import { badRequest, internal } from '@hapi/boom'; import { SecurityException } from '../../lib/streams/errors'; import { createServerRoute } from '../create_server_route'; import { syncStream } from '../../lib/streams/stream_crud'; @@ -13,16 +14,22 @@ import { rootStreamDefinition } from '../../lib/streams/root_stream_definition'; import { createStreamsIndex } from '../../lib/streams/internal_stream_mapping'; export const enableStreamsRoute = createServerRoute({ - endpoint: 'POST /internal/streams/_enable', + endpoint: 'POST /api/streams/_enable', params: z.object({}), options: { + access: 'internal', security: { authz: { requiredPrivileges: ['streams_write'], }, }, }, - handler: async ({ request, response, logger, getScopedClients }) => { + handler: async ({ + request, + response, + logger, + getScopedClients, + }): Promise<{ acknowledged: true }> => { try { const { scopedClusterClient } = await getScopedClients({ request }); await createStreamsIndex(scopedClusterClient); @@ -31,12 +38,12 @@ export const enableStreamsRoute = createServerRoute({ definition: rootStreamDefinition, logger, }); - return response.ok({ body: { acknowledged: true } }); + return { acknowledged: true }; } catch (e) { if (e instanceof SecurityException) { - return response.customError({ body: e, statusCode: 400 }); + throw badRequest(e); } - return response.customError({ body: e, statusCode: 500 }); + throw internal(e); } }, }); diff --git a/x-pack/plugins/streams/server/routes/streams/fork.ts b/x-pack/plugins/streams/server/routes/streams/fork.ts index 1ff507e530065..7f98cdce56c88 100644 --- a/x-pack/plugins/streams/server/routes/streams/fork.ts +++ b/x-pack/plugins/streams/server/routes/streams/fork.ts @@ -6,6 +6,7 @@ */ import { z } from '@kbn/zod'; +import { badRequest, internal, notFound } from '@hapi/boom'; import { DefinitionNotFound, ForkConditionMissing, @@ -19,9 +20,9 @@ import { MalformedStreamId } from '../../lib/streams/errors/malformed_stream_id' import { isChildOf } from '../../lib/streams/helpers/hierarchy'; export const forkStreamsRoute = createServerRoute({ - endpoint: 'POST /api/streams/{id}/_fork 2023-10-31', + endpoint: 'POST /api/streams/{id}/_fork', options: { - access: 'public', + access: 'internal', security: { authz: { requiredPrivileges: ['streams_write'], @@ -34,7 +35,13 @@ export const forkStreamsRoute = createServerRoute({ }), body: z.object({ stream: streamDefinitonWithoutChildrenSchema, condition: conditionSchema }), }), - handler: async ({ response, params, logger, request, getScopedClients }) => { + handler: async ({ + response, + params, + logger, + request, + getScopedClients, + }): Promise<{ acknowledged: true }> => { try { if (!params.body.condition) { throw new ForkConditionMissing('You must provide a condition to fork a stream'); @@ -87,10 +94,10 @@ export const forkStreamsRoute = createServerRoute({ logger, }); - return response.ok({ body: { acknowledged: true } }); + return { acknowledged: true }; } catch (e) { if (e instanceof IndexTemplateNotFound || e instanceof DefinitionNotFound) { - return response.notFound({ body: e }); + throw notFound(e); } if ( @@ -98,10 +105,10 @@ export const forkStreamsRoute = createServerRoute({ e instanceof ForkConditionMissing || e instanceof MalformedStreamId ) { - return response.customError({ body: e, statusCode: 400 }); + throw badRequest(e); } - return response.customError({ body: e, statusCode: 500 }); + throw internal(e); } }, }); diff --git a/x-pack/plugins/streams/server/routes/streams/list.ts b/x-pack/plugins/streams/server/routes/streams/list.ts index 44e6de1ebddd9..57b165b5ce5b6 100644 --- a/x-pack/plugins/streams/server/routes/streams/list.ts +++ b/x-pack/plugins/streams/server/routes/streams/list.ts @@ -13,9 +13,9 @@ import { listStreams } from '../../lib/streams/stream_crud'; import { StreamDefinition } from '../../../common'; export const listStreamsRoute = createServerRoute({ - endpoint: 'GET /api/streams 2023-10-31', + endpoint: 'GET /api/streams', options: { - access: 'public', + access: 'internal', security: { authz: { requiredPrivileges: ['streams_read'], @@ -40,7 +40,7 @@ export const listStreamsRoute = createServerRoute({ throw notFound(e); } - throw internal(e, 500); + throw internal(e); } }, }); diff --git a/x-pack/plugins/streams/server/routes/streams/read.ts b/x-pack/plugins/streams/server/routes/streams/read.ts index 6e70f55a8866c..ca6a7b4245538 100644 --- a/x-pack/plugins/streams/server/routes/streams/read.ts +++ b/x-pack/plugins/streams/server/routes/streams/read.ts @@ -10,11 +10,12 @@ import { notFound, internal } from '@hapi/boom'; import { createServerRoute } from '../create_server_route'; import { DefinitionNotFound } from '../../lib/streams/errors'; import { readAncestors, readStream } from '../../lib/streams/stream_crud'; +import { StreamDefinition } from '../../../common'; export const readStreamRoute = createServerRoute({ - endpoint: 'GET /api/streams/{id} 2023-10-31', + endpoint: 'GET /api/streams/{id}', options: { - access: 'public', + access: 'internal', security: { authz: { requiredPrivileges: ['streams_read'], @@ -24,7 +25,17 @@ export const readStreamRoute = createServerRoute({ params: z.object({ path: z.object({ id: z.string() }), }), - handler: async ({ response, params, request, logger, getScopedClients }) => { + handler: async ({ + response, + params, + request, + logger, + getScopedClients, + }): Promise< + StreamDefinition & { + inheritedFields: Array; + } + > => { try { const { scopedClusterClient } = await getScopedClients({ request }); const streamEntity = await readStream({ @@ -44,7 +55,7 @@ export const readStreamRoute = createServerRoute({ ), }; - return response.ok({ body }); + return body; } catch (e) { if (e instanceof DefinitionNotFound) { throw notFound(e); diff --git a/x-pack/plugins/streams/server/routes/streams/resync.ts b/x-pack/plugins/streams/server/routes/streams/resync.ts index 0b01e226641d6..0f88ed023890b 100644 --- a/x-pack/plugins/streams/server/routes/streams/resync.ts +++ b/x-pack/plugins/streams/server/routes/streams/resync.ts @@ -10,9 +10,9 @@ import { createServerRoute } from '../create_server_route'; import { syncStream, readStream, listStreams } from '../../lib/streams/stream_crud'; export const resyncStreamsRoute = createServerRoute({ - endpoint: 'POST /api/streams/_resync 2023-10-31', + endpoint: 'POST /api/streams/_resync', options: { - access: 'public', + access: 'internal', security: { authz: { requiredPrivileges: ['streams_write'], @@ -20,7 +20,12 @@ export const resyncStreamsRoute = createServerRoute({ }, }, params: z.object({}), - handler: async ({ response, logger, request, getScopedClients }) => { + handler: async ({ + response, + logger, + request, + getScopedClients, + }): Promise<{ acknowledged: true }> => { const { scopedClusterClient } = await getScopedClients({ request }); const { definitions: streams } = await listStreams({ scopedClusterClient }); @@ -37,6 +42,6 @@ export const resyncStreamsRoute = createServerRoute({ }); } - return response.ok({}); + return { acknowledged: true }; }, }); diff --git a/x-pack/plugins/streams/server/routes/streams/settings.ts b/x-pack/plugins/streams/server/routes/streams/settings.ts index c8ea44e8ec1fd..4cff13c41fc0d 100644 --- a/x-pack/plugins/streams/server/routes/streams/settings.ts +++ b/x-pack/plugins/streams/server/routes/streams/settings.ts @@ -9,8 +9,9 @@ import { STREAMS_INDEX } from '../../../common/constants'; import { createServerRoute } from '../create_server_route'; export const getStreamsStatusRoute = createServerRoute({ - endpoint: 'GET /internal/streams/_status', + endpoint: 'GET /api/streams/_status', options: { + access: 'internal', security: { authz: { requiredPrivileges: ['streams_read'], From b556194d229e990700699252011dcdc4fb64eed6 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Wed, 13 Nov 2024 20:06:17 +0100 Subject: [PATCH 37/95] Move to x-pack/plugins/streams_app --- .eslintrc.js | 6 +++--- package.json | 2 +- tsconfig.base.json | 4 ++-- x-pack/plugins/streams/tsconfig.json | 2 -- .../.storybook/get_mock_streams_app_context.tsx | 0 .../{logsai => }/streams_app/.storybook/jest_setup.js | 0 .../plugins/{logsai => }/streams_app/.storybook/main.js | 0 .../{logsai => }/streams_app/.storybook/preview.js | 0 .../streams_app/.storybook/storybook_decorator.tsx | 0 x-pack/plugins/{logsai => }/streams_app/README.md | 0 .../streams_app/common/entity_source_query.ts | 0 x-pack/plugins/{logsai => }/streams_app/common/index.ts | 0 x-pack/plugins/{logsai => }/streams_app/jest.config.js | 0 x-pack/plugins/{logsai => }/streams_app/kibana.jsonc | 0 .../{logsai => }/streams_app/public/application.tsx | 0 .../streams_app/public/components/app_root/index.tsx | 0 .../public/components/entity_detail_view/index.tsx | 0 .../entity_detail_view_header_section/index.tsx | 0 .../public/components/entity_overview_tab_list/index.tsx | 0 .../components/esql_chart/controlled_esql_chart.tsx | 0 .../streams_app/public/components/loading_panel/index.tsx | 0 .../streams_app/public/components/not_found/index.tsx | 0 .../streams_app/public/components/redirect_to/index.tsx | 0 .../public/components/stream_detail_overview/index.tsx | 0 .../public/components/stream_detail_view/index.tsx | 8 ++++---- .../public/components/stream_list_view/index.tsx | 4 ++-- .../components/streams_app_context_provider/index.tsx | 0 .../public/components/streams_app_page_body/index.tsx | 0 .../public/components/streams_app_page_header/index.tsx | 0 .../streams_app_page_header_title.tsx | 0 .../public/components/streams_app_page_template/index.tsx | 0 .../components/streams_app_router_breadcrumb/index.tsx | 0 .../public/components/streams_app_search_bar/index.tsx | 0 .../streams_app/public/components/streams_table/index.tsx | 0 .../{logsai => }/streams_app/public/hooks/use_kibana.tsx | 0 .../public/hooks/use_streams_app_breadcrumbs.ts | 0 .../streams_app/public/hooks/use_streams_app_fetch.ts | 0 .../streams_app/public/hooks/use_streams_app_params.ts | 0 .../public/hooks/use_streams_app_route_path.ts | 0 .../streams_app/public/hooks/use_streams_app_router.ts | 0 x-pack/plugins/{logsai => }/streams_app/public/index.ts | 0 x-pack/plugins/{logsai => }/streams_app/public/plugin.ts | 0 .../{logsai => }/streams_app/public/routes/config.tsx | 0 .../{logsai => }/streams_app/public/services/types.ts | 0 x-pack/plugins/{logsai => }/streams_app/public/types.ts | 0 .../streams_app/public/util/esql_result_to_timeseries.ts | 0 x-pack/plugins/{logsai => }/streams_app/server/config.ts | 0 x-pack/plugins/{logsai => }/streams_app/server/index.ts | 0 x-pack/plugins/{logsai => }/streams_app/server/plugin.ts | 0 x-pack/plugins/{logsai => }/streams_app/server/types.ts | 0 x-pack/plugins/{logsai => }/streams_app/tsconfig.json | 4 ++-- yarn.lock | 2 +- 52 files changed, 15 insertions(+), 17 deletions(-) rename x-pack/plugins/{logsai => }/streams_app/.storybook/get_mock_streams_app_context.tsx (100%) rename x-pack/plugins/{logsai => }/streams_app/.storybook/jest_setup.js (100%) rename x-pack/plugins/{logsai => }/streams_app/.storybook/main.js (100%) rename x-pack/plugins/{logsai => }/streams_app/.storybook/preview.js (100%) rename x-pack/plugins/{logsai => }/streams_app/.storybook/storybook_decorator.tsx (100%) rename x-pack/plugins/{logsai => }/streams_app/README.md (100%) rename x-pack/plugins/{logsai => }/streams_app/common/entity_source_query.ts (100%) rename x-pack/plugins/{logsai => }/streams_app/common/index.ts (100%) rename x-pack/plugins/{logsai => }/streams_app/jest.config.js (100%) rename x-pack/plugins/{logsai => }/streams_app/kibana.jsonc (100%) rename x-pack/plugins/{logsai => }/streams_app/public/application.tsx (100%) rename x-pack/plugins/{logsai => }/streams_app/public/components/app_root/index.tsx (100%) rename x-pack/plugins/{logsai => }/streams_app/public/components/entity_detail_view/index.tsx (100%) rename x-pack/plugins/{logsai => }/streams_app/public/components/entity_detail_view_header_section/index.tsx (100%) rename x-pack/plugins/{logsai => }/streams_app/public/components/entity_overview_tab_list/index.tsx (100%) rename x-pack/plugins/{logsai => }/streams_app/public/components/esql_chart/controlled_esql_chart.tsx (100%) rename x-pack/plugins/{logsai => }/streams_app/public/components/loading_panel/index.tsx (100%) rename x-pack/plugins/{logsai => }/streams_app/public/components/not_found/index.tsx (100%) rename x-pack/plugins/{logsai => }/streams_app/public/components/redirect_to/index.tsx (100%) rename x-pack/plugins/{logsai => }/streams_app/public/components/stream_detail_overview/index.tsx (100%) rename x-pack/plugins/{logsai => }/streams_app/public/components/stream_detail_view/index.tsx (86%) rename x-pack/plugins/{logsai => }/streams_app/public/components/stream_list_view/index.tsx (91%) rename x-pack/plugins/{logsai => }/streams_app/public/components/streams_app_context_provider/index.tsx (100%) rename x-pack/plugins/{logsai => }/streams_app/public/components/streams_app_page_body/index.tsx (100%) rename x-pack/plugins/{logsai => }/streams_app/public/components/streams_app_page_header/index.tsx (100%) rename x-pack/plugins/{logsai => }/streams_app/public/components/streams_app_page_header/streams_app_page_header_title.tsx (100%) rename x-pack/plugins/{logsai => }/streams_app/public/components/streams_app_page_template/index.tsx (100%) rename x-pack/plugins/{logsai => }/streams_app/public/components/streams_app_router_breadcrumb/index.tsx (100%) rename x-pack/plugins/{logsai => }/streams_app/public/components/streams_app_search_bar/index.tsx (100%) rename x-pack/plugins/{logsai => }/streams_app/public/components/streams_table/index.tsx (100%) rename x-pack/plugins/{logsai => }/streams_app/public/hooks/use_kibana.tsx (100%) rename x-pack/plugins/{logsai => }/streams_app/public/hooks/use_streams_app_breadcrumbs.ts (100%) rename x-pack/plugins/{logsai => }/streams_app/public/hooks/use_streams_app_fetch.ts (100%) rename x-pack/plugins/{logsai => }/streams_app/public/hooks/use_streams_app_params.ts (100%) rename x-pack/plugins/{logsai => }/streams_app/public/hooks/use_streams_app_route_path.ts (100%) rename x-pack/plugins/{logsai => }/streams_app/public/hooks/use_streams_app_router.ts (100%) rename x-pack/plugins/{logsai => }/streams_app/public/index.ts (100%) rename x-pack/plugins/{logsai => }/streams_app/public/plugin.ts (100%) rename x-pack/plugins/{logsai => }/streams_app/public/routes/config.tsx (100%) rename x-pack/plugins/{logsai => }/streams_app/public/services/types.ts (100%) rename x-pack/plugins/{logsai => }/streams_app/public/types.ts (100%) rename x-pack/plugins/{logsai => }/streams_app/public/util/esql_result_to_timeseries.ts (100%) rename x-pack/plugins/{logsai => }/streams_app/server/config.ts (100%) rename x-pack/plugins/{logsai => }/streams_app/server/index.ts (100%) rename x-pack/plugins/{logsai => }/streams_app/server/plugin.ts (100%) rename x-pack/plugins/{logsai => }/streams_app/server/types.ts (100%) rename x-pack/plugins/{logsai => }/streams_app/tsconfig.json (91%) diff --git a/.eslintrc.js b/.eslintrc.js index 905e386fa00cd..a1f86d675dde9 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -952,7 +952,7 @@ module.exports = { { files: [ 'x-pack/plugins/observability_solution/**/*.{ts,tsx}', - 'x-pack/plugins/logsai/**/*.{ts,tsx}', + 'x-pack/plugins/{streams,streams_app}/**/*.{ts,tsx}', 'x-pack/packages/observability/**/*.{ts,tsx}', ], rules: { @@ -969,7 +969,7 @@ module.exports = { files: [ 'x-pack/plugins/aiops/**/*.tsx', 'x-pack/plugins/observability_solution/**/*.tsx', - 'x-pack/plugins/logsai/**/*.{ts,tsx}', + 'x-pack/plugins/{streams,streams_app}/**/*.{ts,tsx}', 'src/plugins/ai_assistant_management/**/*.tsx', 'x-pack/packages/observability/**/*.{ts,tsx}', ], @@ -986,7 +986,7 @@ module.exports = { { files: [ 'x-pack/plugins/observability_solution/**/!(*.stories.tsx|*.test.tsx|*.storybook_decorator.tsx|*.mock.tsx)', - 'x-pack/plugins/logsai/**/!(*.stories.tsx|*.test.tsx|*.storybook_decorator.tsx|*.mock.tsx)', + 'x-pack/plugins/{streams,streams_app}/**/!(*.stories.tsx|*.test.tsx|*.storybook_decorator.tsx|*.mock.tsx)', 'src/plugins/ai_assistant_management/**/!(*.stories.tsx|*.test.tsx|*.storybook_decorator.tsx|*.mock.tsx)', 'x-pack/packages/observability/logs_overview/**/!(*.stories.tsx|*.test.tsx|*.storybook_decorator.tsx|*.mock.tsx)', ], diff --git a/package.json b/package.json index 59594be83e458..742c436dcd963 100644 --- a/package.json +++ b/package.json @@ -931,7 +931,7 @@ "@kbn/status-plugin-a-plugin": "link:test/server_integration/plugins/status_plugin_a", "@kbn/status-plugin-b-plugin": "link:test/server_integration/plugins/status_plugin_b", "@kbn/std": "link:packages/kbn-std", - "@kbn/streams-app-plugin": "link:x-pack/plugins/logsai/streams_app", + "@kbn/streams-app-plugin": "link:x-pack/plugins/streams_app", "@kbn/streams-plugin": "link:x-pack/plugins/streams", "@kbn/synthetics-plugin": "link:x-pack/plugins/observability_solution/synthetics", "@kbn/synthetics-private-location": "link:x-pack/packages/kbn-synthetics-private-location", diff --git a/tsconfig.base.json b/tsconfig.base.json index afb161f280e45..4ec8ad4b33897 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1832,8 +1832,8 @@ "@kbn/stdio-dev-helpers/*": ["packages/kbn-stdio-dev-helpers/*"], "@kbn/storybook": ["packages/kbn-storybook"], "@kbn/storybook/*": ["packages/kbn-storybook/*"], - "@kbn/streams-app-plugin": ["x-pack/plugins/logsai/streams_app"], - "@kbn/streams-app-plugin/*": ["x-pack/plugins/logsai/streams_app/*"], + "@kbn/streams-app-plugin": ["x-pack/plugins/streams_app"], + "@kbn/streams-app-plugin/*": ["x-pack/plugins/streams_app/*"], "@kbn/streams-plugin": ["x-pack/plugins/streams"], "@kbn/streams-plugin/*": ["x-pack/plugins/streams/*"], "@kbn/synthetics-e2e": ["x-pack/plugins/observability_solution/synthetics/e2e"], diff --git a/x-pack/plugins/streams/tsconfig.json b/x-pack/plugins/streams/tsconfig.json index ecf79c4f49dbc..3f863145f4d22 100644 --- a/x-pack/plugins/streams/tsconfig.json +++ b/x-pack/plugins/streams/tsconfig.json @@ -27,9 +27,7 @@ "@kbn/zod", "@kbn/encrypted-saved-objects-plugin", "@kbn/licensing-plugin", - "@kbn/features-plugin", "@kbn/server-route-repository-client", - "@kbn/es-types", "@kbn/observability-utils-server", "@kbn/observability-utils-common" ] diff --git a/x-pack/plugins/logsai/streams_app/.storybook/get_mock_streams_app_context.tsx b/x-pack/plugins/streams_app/.storybook/get_mock_streams_app_context.tsx similarity index 100% rename from x-pack/plugins/logsai/streams_app/.storybook/get_mock_streams_app_context.tsx rename to x-pack/plugins/streams_app/.storybook/get_mock_streams_app_context.tsx diff --git a/x-pack/plugins/logsai/streams_app/.storybook/jest_setup.js b/x-pack/plugins/streams_app/.storybook/jest_setup.js similarity index 100% rename from x-pack/plugins/logsai/streams_app/.storybook/jest_setup.js rename to x-pack/plugins/streams_app/.storybook/jest_setup.js diff --git a/x-pack/plugins/logsai/streams_app/.storybook/main.js b/x-pack/plugins/streams_app/.storybook/main.js similarity index 100% rename from x-pack/plugins/logsai/streams_app/.storybook/main.js rename to x-pack/plugins/streams_app/.storybook/main.js diff --git a/x-pack/plugins/logsai/streams_app/.storybook/preview.js b/x-pack/plugins/streams_app/.storybook/preview.js similarity index 100% rename from x-pack/plugins/logsai/streams_app/.storybook/preview.js rename to x-pack/plugins/streams_app/.storybook/preview.js diff --git a/x-pack/plugins/logsai/streams_app/.storybook/storybook_decorator.tsx b/x-pack/plugins/streams_app/.storybook/storybook_decorator.tsx similarity index 100% rename from x-pack/plugins/logsai/streams_app/.storybook/storybook_decorator.tsx rename to x-pack/plugins/streams_app/.storybook/storybook_decorator.tsx diff --git a/x-pack/plugins/logsai/streams_app/README.md b/x-pack/plugins/streams_app/README.md similarity index 100% rename from x-pack/plugins/logsai/streams_app/README.md rename to x-pack/plugins/streams_app/README.md diff --git a/x-pack/plugins/logsai/streams_app/common/entity_source_query.ts b/x-pack/plugins/streams_app/common/entity_source_query.ts similarity index 100% rename from x-pack/plugins/logsai/streams_app/common/entity_source_query.ts rename to x-pack/plugins/streams_app/common/entity_source_query.ts diff --git a/x-pack/plugins/logsai/streams_app/common/index.ts b/x-pack/plugins/streams_app/common/index.ts similarity index 100% rename from x-pack/plugins/logsai/streams_app/common/index.ts rename to x-pack/plugins/streams_app/common/index.ts diff --git a/x-pack/plugins/logsai/streams_app/jest.config.js b/x-pack/plugins/streams_app/jest.config.js similarity index 100% rename from x-pack/plugins/logsai/streams_app/jest.config.js rename to x-pack/plugins/streams_app/jest.config.js diff --git a/x-pack/plugins/logsai/streams_app/kibana.jsonc b/x-pack/plugins/streams_app/kibana.jsonc similarity index 100% rename from x-pack/plugins/logsai/streams_app/kibana.jsonc rename to x-pack/plugins/streams_app/kibana.jsonc diff --git a/x-pack/plugins/logsai/streams_app/public/application.tsx b/x-pack/plugins/streams_app/public/application.tsx similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/application.tsx rename to x-pack/plugins/streams_app/public/application.tsx diff --git a/x-pack/plugins/logsai/streams_app/public/components/app_root/index.tsx b/x-pack/plugins/streams_app/public/components/app_root/index.tsx similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/components/app_root/index.tsx rename to x-pack/plugins/streams_app/public/components/app_root/index.tsx diff --git a/x-pack/plugins/logsai/streams_app/public/components/entity_detail_view/index.tsx b/x-pack/plugins/streams_app/public/components/entity_detail_view/index.tsx similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/components/entity_detail_view/index.tsx rename to x-pack/plugins/streams_app/public/components/entity_detail_view/index.tsx diff --git a/x-pack/plugins/logsai/streams_app/public/components/entity_detail_view_header_section/index.tsx b/x-pack/plugins/streams_app/public/components/entity_detail_view_header_section/index.tsx similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/components/entity_detail_view_header_section/index.tsx rename to x-pack/plugins/streams_app/public/components/entity_detail_view_header_section/index.tsx diff --git a/x-pack/plugins/logsai/streams_app/public/components/entity_overview_tab_list/index.tsx b/x-pack/plugins/streams_app/public/components/entity_overview_tab_list/index.tsx similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/components/entity_overview_tab_list/index.tsx rename to x-pack/plugins/streams_app/public/components/entity_overview_tab_list/index.tsx diff --git a/x-pack/plugins/logsai/streams_app/public/components/esql_chart/controlled_esql_chart.tsx b/x-pack/plugins/streams_app/public/components/esql_chart/controlled_esql_chart.tsx similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/components/esql_chart/controlled_esql_chart.tsx rename to x-pack/plugins/streams_app/public/components/esql_chart/controlled_esql_chart.tsx diff --git a/x-pack/plugins/logsai/streams_app/public/components/loading_panel/index.tsx b/x-pack/plugins/streams_app/public/components/loading_panel/index.tsx similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/components/loading_panel/index.tsx rename to x-pack/plugins/streams_app/public/components/loading_panel/index.tsx diff --git a/x-pack/plugins/logsai/streams_app/public/components/not_found/index.tsx b/x-pack/plugins/streams_app/public/components/not_found/index.tsx similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/components/not_found/index.tsx rename to x-pack/plugins/streams_app/public/components/not_found/index.tsx diff --git a/x-pack/plugins/logsai/streams_app/public/components/redirect_to/index.tsx b/x-pack/plugins/streams_app/public/components/redirect_to/index.tsx similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/components/redirect_to/index.tsx rename to x-pack/plugins/streams_app/public/components/redirect_to/index.tsx diff --git a/x-pack/plugins/logsai/streams_app/public/components/stream_detail_overview/index.tsx b/x-pack/plugins/streams_app/public/components/stream_detail_overview/index.tsx similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/components/stream_detail_overview/index.tsx rename to x-pack/plugins/streams_app/public/components/stream_detail_overview/index.tsx diff --git a/x-pack/plugins/logsai/streams_app/public/components/stream_detail_view/index.tsx b/x-pack/plugins/streams_app/public/components/stream_detail_view/index.tsx similarity index 86% rename from x-pack/plugins/logsai/streams_app/public/components/stream_detail_view/index.tsx rename to x-pack/plugins/streams_app/public/components/stream_detail_view/index.tsx index 29d2b33ad1ff0..8cfe9e661372e 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/stream_detail_view/index.tsx +++ b/x-pack/plugins/streams_app/public/components/stream_detail_view/index.tsx @@ -27,7 +27,7 @@ export function StreamDetailView() { const { value: streamEntity } = useStreamsAppFetch( ({ signal }) => { - return streamsRepositoryClient.fetch('GET /api/streams/{id} 2023-10-31', { + return streamsRepositoryClient.fetch('GET /api/streams/{id}', { signal, params: { path: { @@ -48,21 +48,21 @@ export function StreamDetailView() { { name: 'overview', content: , - label: i18n.translate('xpack.streams.streamDetailView.overviewTab', { + label: i18n.translate('app_not_found_in_i18nrc.streamDetailView.overviewTab', { defaultMessage: 'Overview', }), }, { name: 'routing', content: <>, - label: i18n.translate('xpack.streams.streamDetailView.routingTab', { + label: i18n.translate('app_not_found_in_i18nrc.streamDetailView.routingTab', { defaultMessage: 'Routing', }), }, { name: 'processing', content: <>, - label: i18n.translate('xpack.streams.streamDetailView.processingTab', { + label: i18n.translate('app_not_found_in_i18nrc.streamDetailView.processingTab', { defaultMessage: 'Processing', }), }, diff --git a/x-pack/plugins/logsai/streams_app/public/components/stream_list_view/index.tsx b/x-pack/plugins/streams_app/public/components/stream_list_view/index.tsx similarity index 91% rename from x-pack/plugins/logsai/streams_app/public/components/stream_list_view/index.tsx rename to x-pack/plugins/streams_app/public/components/stream_list_view/index.tsx index 762fb35fed2e2..5effd1dedfd3f 100644 --- a/x-pack/plugins/logsai/streams_app/public/components/stream_list_view/index.tsx +++ b/x-pack/plugins/streams_app/public/components/stream_list_view/index.tsx @@ -27,7 +27,7 @@ export function StreamListView() { const streamsListFetch = useStreamsAppFetch( ({ signal }) => { - return streamsRepositoryClient.fetch('GET /api/streams 2023-10-31', { + return streamsRepositoryClient.fetch('GET /api/streams', { signal, }); }, @@ -39,7 +39,7 @@ export function StreamListView() { diff --git a/x-pack/plugins/logsai/streams_app/public/components/streams_app_context_provider/index.tsx b/x-pack/plugins/streams_app/public/components/streams_app_context_provider/index.tsx similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/components/streams_app_context_provider/index.tsx rename to x-pack/plugins/streams_app/public/components/streams_app_context_provider/index.tsx diff --git a/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_body/index.tsx b/x-pack/plugins/streams_app/public/components/streams_app_page_body/index.tsx similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/components/streams_app_page_body/index.tsx rename to x-pack/plugins/streams_app/public/components/streams_app_page_body/index.tsx diff --git a/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_header/index.tsx b/x-pack/plugins/streams_app/public/components/streams_app_page_header/index.tsx similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/components/streams_app_page_header/index.tsx rename to x-pack/plugins/streams_app/public/components/streams_app_page_header/index.tsx diff --git a/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_header/streams_app_page_header_title.tsx b/x-pack/plugins/streams_app/public/components/streams_app_page_header/streams_app_page_header_title.tsx similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/components/streams_app_page_header/streams_app_page_header_title.tsx rename to x-pack/plugins/streams_app/public/components/streams_app_page_header/streams_app_page_header_title.tsx diff --git a/x-pack/plugins/logsai/streams_app/public/components/streams_app_page_template/index.tsx b/x-pack/plugins/streams_app/public/components/streams_app_page_template/index.tsx similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/components/streams_app_page_template/index.tsx rename to x-pack/plugins/streams_app/public/components/streams_app_page_template/index.tsx diff --git a/x-pack/plugins/logsai/streams_app/public/components/streams_app_router_breadcrumb/index.tsx b/x-pack/plugins/streams_app/public/components/streams_app_router_breadcrumb/index.tsx similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/components/streams_app_router_breadcrumb/index.tsx rename to x-pack/plugins/streams_app/public/components/streams_app_router_breadcrumb/index.tsx diff --git a/x-pack/plugins/logsai/streams_app/public/components/streams_app_search_bar/index.tsx b/x-pack/plugins/streams_app/public/components/streams_app_search_bar/index.tsx similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/components/streams_app_search_bar/index.tsx rename to x-pack/plugins/streams_app/public/components/streams_app_search_bar/index.tsx diff --git a/x-pack/plugins/logsai/streams_app/public/components/streams_table/index.tsx b/x-pack/plugins/streams_app/public/components/streams_table/index.tsx similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/components/streams_table/index.tsx rename to x-pack/plugins/streams_app/public/components/streams_table/index.tsx diff --git a/x-pack/plugins/logsai/streams_app/public/hooks/use_kibana.tsx b/x-pack/plugins/streams_app/public/hooks/use_kibana.tsx similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/hooks/use_kibana.tsx rename to x-pack/plugins/streams_app/public/hooks/use_kibana.tsx diff --git a/x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_breadcrumbs.ts b/x-pack/plugins/streams_app/public/hooks/use_streams_app_breadcrumbs.ts similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_breadcrumbs.ts rename to x-pack/plugins/streams_app/public/hooks/use_streams_app_breadcrumbs.ts diff --git a/x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_fetch.ts b/x-pack/plugins/streams_app/public/hooks/use_streams_app_fetch.ts similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_fetch.ts rename to x-pack/plugins/streams_app/public/hooks/use_streams_app_fetch.ts diff --git a/x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_params.ts b/x-pack/plugins/streams_app/public/hooks/use_streams_app_params.ts similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_params.ts rename to x-pack/plugins/streams_app/public/hooks/use_streams_app_params.ts diff --git a/x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_route_path.ts b/x-pack/plugins/streams_app/public/hooks/use_streams_app_route_path.ts similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_route_path.ts rename to x-pack/plugins/streams_app/public/hooks/use_streams_app_route_path.ts diff --git a/x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_router.ts b/x-pack/plugins/streams_app/public/hooks/use_streams_app_router.ts similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/hooks/use_streams_app_router.ts rename to x-pack/plugins/streams_app/public/hooks/use_streams_app_router.ts diff --git a/x-pack/plugins/logsai/streams_app/public/index.ts b/x-pack/plugins/streams_app/public/index.ts similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/index.ts rename to x-pack/plugins/streams_app/public/index.ts diff --git a/x-pack/plugins/logsai/streams_app/public/plugin.ts b/x-pack/plugins/streams_app/public/plugin.ts similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/plugin.ts rename to x-pack/plugins/streams_app/public/plugin.ts diff --git a/x-pack/plugins/logsai/streams_app/public/routes/config.tsx b/x-pack/plugins/streams_app/public/routes/config.tsx similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/routes/config.tsx rename to x-pack/plugins/streams_app/public/routes/config.tsx diff --git a/x-pack/plugins/logsai/streams_app/public/services/types.ts b/x-pack/plugins/streams_app/public/services/types.ts similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/services/types.ts rename to x-pack/plugins/streams_app/public/services/types.ts diff --git a/x-pack/plugins/logsai/streams_app/public/types.ts b/x-pack/plugins/streams_app/public/types.ts similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/types.ts rename to x-pack/plugins/streams_app/public/types.ts diff --git a/x-pack/plugins/logsai/streams_app/public/util/esql_result_to_timeseries.ts b/x-pack/plugins/streams_app/public/util/esql_result_to_timeseries.ts similarity index 100% rename from x-pack/plugins/logsai/streams_app/public/util/esql_result_to_timeseries.ts rename to x-pack/plugins/streams_app/public/util/esql_result_to_timeseries.ts diff --git a/x-pack/plugins/logsai/streams_app/server/config.ts b/x-pack/plugins/streams_app/server/config.ts similarity index 100% rename from x-pack/plugins/logsai/streams_app/server/config.ts rename to x-pack/plugins/streams_app/server/config.ts diff --git a/x-pack/plugins/logsai/streams_app/server/index.ts b/x-pack/plugins/streams_app/server/index.ts similarity index 100% rename from x-pack/plugins/logsai/streams_app/server/index.ts rename to x-pack/plugins/streams_app/server/index.ts diff --git a/x-pack/plugins/logsai/streams_app/server/plugin.ts b/x-pack/plugins/streams_app/server/plugin.ts similarity index 100% rename from x-pack/plugins/logsai/streams_app/server/plugin.ts rename to x-pack/plugins/streams_app/server/plugin.ts diff --git a/x-pack/plugins/logsai/streams_app/server/types.ts b/x-pack/plugins/streams_app/server/types.ts similarity index 100% rename from x-pack/plugins/logsai/streams_app/server/types.ts rename to x-pack/plugins/streams_app/server/types.ts diff --git a/x-pack/plugins/logsai/streams_app/tsconfig.json b/x-pack/plugins/streams_app/tsconfig.json similarity index 91% rename from x-pack/plugins/logsai/streams_app/tsconfig.json rename to x-pack/plugins/streams_app/tsconfig.json index 1b469ccc36ab5..39acb94665ae5 100644 --- a/x-pack/plugins/logsai/streams_app/tsconfig.json +++ b/x-pack/plugins/streams_app/tsconfig.json @@ -1,5 +1,5 @@ { - "extends": "../../../../tsconfig.base.json", + "extends": "../../../tsconfig.base.json", "compilerOptions": { "outDir": "target/types" }, @@ -24,7 +24,6 @@ "@kbn/typed-react-router-config", "@kbn/i18n", "@kbn/observability-utils-browser", - "@kbn/es-types", "@kbn/kibana-react-plugin", "@kbn/es-query", "@kbn/logging", @@ -33,5 +32,6 @@ "@kbn/calculate-auto", "@kbn/streams-plugin", "@kbn/share-plugin", + "@kbn/observability-utils-server", ] } diff --git a/yarn.lock b/yarn.lock index 6cc38e97153c8..bcf1cd4488685 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6918,7 +6918,7 @@ version "0.0.0" uid "" -"@kbn/streams-app-plugin@link:x-pack/plugins/logsai/streams_app": +"@kbn/streams-app-plugin@link:x-pack/plugins/streams_app": version "0.0.0" uid "" From aa4a2e048fcd14af4fb1283113c6bb68d6c3b21c Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Wed, 13 Nov 2024 20:13:20 +0100 Subject: [PATCH 38/95] Improve types --- packages/kbn-server-route-repository-utils/src/typings.ts | 4 ++-- x-pack/packages/ai-infra/inference-common/index.ts | 2 +- .../ai-infra/inference-common/src/{util => }/truncate_list.ts | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename x-pack/packages/ai-infra/inference-common/src/{util => }/truncate_list.ts (100%) diff --git a/packages/kbn-server-route-repository-utils/src/typings.ts b/packages/kbn-server-route-repository-utils/src/typings.ts index 77dfd8f051bf6..e982d0a0b529a 100644 --- a/packages/kbn-server-route-repository-utils/src/typings.ts +++ b/packages/kbn-server-route-repository-utils/src/typings.ts @@ -62,8 +62,8 @@ export interface ServerRouteCreateOptions { } type RouteMethodOf = TEndpoint extends `${infer TRouteMethod} ${string}` - ? TRouteMethod extends RouteMethod - ? TRouteMethod + ? Lowercase extends RouteMethod + ? Lowercase : RouteMethod : RouteMethod; diff --git a/x-pack/packages/ai-infra/inference-common/index.ts b/x-pack/packages/ai-infra/inference-common/index.ts index af103ac2e95ab..2791896c801ef 100644 --- a/x-pack/packages/ai-infra/inference-common/index.ts +++ b/x-pack/packages/ai-infra/inference-common/index.ts @@ -82,4 +82,4 @@ export { isInferenceRequestError, } from './src/errors'; -export { truncateList } from './src/util/truncate_list'; +export { truncateList } from './src/truncate_list'; diff --git a/x-pack/packages/ai-infra/inference-common/src/util/truncate_list.ts b/x-pack/packages/ai-infra/inference-common/src/truncate_list.ts similarity index 100% rename from x-pack/packages/ai-infra/inference-common/src/util/truncate_list.ts rename to x-pack/packages/ai-infra/inference-common/src/truncate_list.ts From f0276dc7363eb3423baccb71a17cb8248d37ab91 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Wed, 13 Nov 2024 20:19:03 +0100 Subject: [PATCH 39/95] Improve type strictness --- packages/kbn-server-route-repository-utils/src/typings.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/kbn-server-route-repository-utils/src/typings.ts b/packages/kbn-server-route-repository-utils/src/typings.ts index e982d0a0b529a..ed3ae0e834be0 100644 --- a/packages/kbn-server-route-repository-utils/src/typings.ts +++ b/packages/kbn-server-route-repository-utils/src/typings.ts @@ -64,8 +64,8 @@ export interface ServerRouteCreateOptions { type RouteMethodOf = TEndpoint extends `${infer TRouteMethod} ${string}` ? Lowercase extends RouteMethod ? Lowercase - : RouteMethod - : RouteMethod; + : never + : never; type IsPublicEndpoint< TEndpoint extends string, From adcd660ff97bb34f26d0cb762ddaedff081054a3 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 14 Nov 2024 08:45:01 +0100 Subject: [PATCH 40/95] Update .i18nrc.json --- x-pack/.i18nrc.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/.i18nrc.json b/x-pack/.i18nrc.json index 67697eda51e1c..0b76050724732 100644 --- a/x-pack/.i18nrc.json +++ b/x-pack/.i18nrc.json @@ -150,7 +150,7 @@ "xpack.securitySolutionServerless": "plugins/security_solution_serverless", "xpack.sessionView": "plugins/session_view", "xpack.streams": [ - "plugins/logsai/streams_app" + "plugins/streams_app" ], "xpack.slo": "plugins/observability_solution/slo", "xpack.snapshotRestore": "plugins/snapshot_restore", From bb4dc6efe910d58907ad9f71b354876fd94faa3b Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 14 Nov 2024 15:52:10 +0100 Subject: [PATCH 41/95] Conditionally register items in side nav --- x-pack/plugins/streams_app/public/plugin.ts | 49 +++++++++++++++------ 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/streams_app/public/plugin.ts b/x-pack/plugins/streams_app/public/plugin.ts index b8c0e52a0be3e..9df399693d02c 100644 --- a/x-pack/plugins/streams_app/public/plugin.ts +++ b/x-pack/plugins/streams_app/public/plugin.ts @@ -6,9 +6,10 @@ */ import { i18n } from '@kbn/i18n'; -import { from, map } from 'rxjs'; +import { map } from 'rxjs'; import { AppMountParameters, + AppUpdater, CoreSetup, CoreStart, DEFAULT_APP_CATEGORIES, @@ -45,8 +46,12 @@ export class StreamsAppPlugin pluginsSetup: StreamsAppSetupDependencies ): StreamsAppPublicSetup { pluginsSetup.observabilityShared.navigation.registerSections( - from(coreSetup.getStartServices()).pipe( - map(([coreStart, pluginsStart]) => { + pluginsSetup.streams.status$.pipe( + map(({ status }) => { + if (status !== 'enabled') { + return []; + } + return [ { label: '', @@ -78,17 +83,35 @@ export class StreamsAppPlugin euiIconType: 'logoObservability', appRoute: '/app/streams', category: DEFAULT_APP_CATEGORIES.observability, - visibleIn: ['sideNav'], order: 8001, - deepLinks: [ - { - id: 'streams', - title: i18n.translate('xpack.streams.streamsAppDeepLinkTitle', { - defaultMessage: 'Streams', - }), - path: '/', - }, - ], + updater$: pluginsSetup.streams.status$.pipe( + map(({ status }): AppUpdater => { + return (app) => { + if (status !== 'enabled') { + return { + visibleIn: [], + deepLinks: [], + }; + } + + return { + visibleIn: ['sideNav', 'globalSearch'], + deepLinks: + status === 'enabled' + ? [ + { + id: 'streams', + title: i18n.translate('xpack.streams.streamsAppDeepLinkTitle', { + defaultMessage: 'Streams', + }), + path: '/', + }, + ] + : [], + }; + }; + }) + ), mount: async (appMountParameters: AppMountParameters) => { // Load application bundle and Get start services const [{ renderApp }, [coreStart, pluginsStart]] = await Promise.all([ From 2b74f6cffb8f1b60b763efbff9e29c359013398d Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 14 Nov 2024 16:03:14 +0100 Subject: [PATCH 42/95] Clean up --- .../es/client/create_observability_es_client.ts | 5 +++-- .../inventory/server/routes/has_data/get_has_data.ts | 2 +- x-pack/plugins/streams_app/README.md | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts index 8fc899ac254f0..40b9fcbb6a130 100644 --- a/x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts @@ -38,7 +38,7 @@ type InferEsqlResponseOf< TOptions extends EsqlOptions | undefined = { asPlainObjects: true } > = TOptions extends { asPlainObjects: true } ? { - objects: Array<{ + hits: Array<{ [key in keyof TOutput]: TOutput[key]; }>; } @@ -51,6 +51,7 @@ export interface EsqlQueryResponse { columns: Array<{ name: string; type: string }>; values: EsqlValue[][]; } + /** * An Elasticsearch Client with a fully typed `search` method and built-in * APM instrumentation. @@ -145,7 +146,7 @@ export function createObservabilityEsClient({ const shouldParseOutput = options?.asPlainObjects !== false; const finalResponse = shouldParseOutput - ? { objects: esqlResultToPlainObjects(esqlResponse) } + ? { hits: esqlResultToPlainObjects(esqlResponse) } : esqlResponse; return finalResponse as InferEsqlResponseOf; diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/has_data/get_has_data.ts b/x-pack/plugins/observability_solution/inventory/server/routes/has_data/get_has_data.ts index a2c62b2472862..2f48202f31837 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/has_data/get_has_data.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/has_data/get_has_data.ts @@ -24,7 +24,7 @@ export async function getHasData({ | LIMIT 1`, }); - const totalCount = esqlResults.objects[0]._count; + const totalCount = esqlResults.hits[0]._count; return { hasData: totalCount > 0 }; } catch (e) { diff --git a/x-pack/plugins/streams_app/README.md b/x-pack/plugins/streams_app/README.md index 790e874891669..6e03524f9274b 100644 --- a/x-pack/plugins/streams_app/README.md +++ b/x-pack/plugins/streams_app/README.md @@ -1,3 +1,3 @@ -# Entities App +# Streams app -Home of the Entities app plugin, which renders ... _entities_! +Home of the Streams app plugin, which allows users to manage Streams via the UI. From 46ffca7f598c7f443c5410bb2b55497b976dde6d Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 14 Nov 2024 15:35:06 +0000 Subject: [PATCH 43/95] [CI] Auto-commit changed files from 'node scripts/build_plugin_list_docs' --- .github/CODEOWNERS | 1 + docs/developer/plugin-list.asciidoc | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 06f8a9c6d05a1..29f75b5cf2a41 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -970,6 +970,7 @@ x-pack/plugins/spaces @elastic/kibana-security x-pack/plugins/stack_alerts @elastic/response-ops x-pack/plugins/stack_connectors @elastic/response-ops x-pack/plugins/streams @simianhacker @flash1293 @dgieselaar +x-pack/plugins/streams_app @elastic/observability-ui x-pack/plugins/task_manager @elastic/response-ops x-pack/plugins/telemetry_collection_xpack @elastic/kibana-core x-pack/plugins/threat_intelligence @elastic/security-threat-hunting-investigations diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index 71ab26400f496..b4f19d9a30734 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -901,6 +901,10 @@ routes, etc. |This plugin provides an interface to manage streams +|{kib-repo}blob/{branch}/x-pack/plugins/streams_app/README.md[streamsApp] +|Home of the Streams app plugin, which allows users to manage Streams via the UI. + + |{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/synthetics/README.md[synthetics] |The purpose of this plugin is to provide users of Heartbeat more visibility of what's happening in their infrastructure. From 922d1b4c441ecd417d4d9c4b2bb583a88e636651 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Tue, 19 Nov 2024 09:23:04 +0100 Subject: [PATCH 44/95] Review feedback --- packages/kbn-optimizer/limits.yml | 1 + .../object/unflatten_object.ts | 11 +++-- .../routes/entities/get_latest_entity.ts | 6 +-- .../routes/entities/get_entity_groups.ts | 6 ++- .../routes/entities/get_entity_types.ts | 5 +-- .../routes/entities/get_latest_entities.ts | 44 +++++++++++-------- .../streams/server/routes/streams/delete.ts | 6 ++- .../streams/server/routes/streams/disable.ts | 7 +-- .../stream_detail_overview/index.tsx | 21 --------- .../components/stream_detail_view/index.tsx | 15 ++----- .../public/components/streams_table/index.tsx | 2 +- 11 files changed, 55 insertions(+), 69 deletions(-) diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index 58424700d9bf6..59bfa5b2faa5a 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -160,6 +160,7 @@ pageLoadAssetSize: stackAlerts: 58316 stackConnectors: 67227 streams: 16742 + streamsApp: 20537 synthetics: 55971 telemetry: 51957 telemetryManagementSection: 38586 diff --git a/x-pack/packages/observability/observability_utils/observability_utils_common/object/unflatten_object.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/object/unflatten_object.ts index 83508d5d2dbf5..8a4493905f1d4 100644 --- a/x-pack/packages/observability/observability_utils/observability_utils_common/object/unflatten_object.ts +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/object/unflatten_object.ts @@ -6,13 +6,17 @@ */ import { set } from '@kbn/safer-lodash-set'; +import { DedotObject } from '@kbn/utility-types'; -export function unflattenObject(source: Record, target: Record = {}) { +export function unflattenObject>( + source: T, + target: Record = {} +): DedotObject { // eslint-disable-next-line guard-for-in for (const key in source) { const val = source[key as keyof typeof source]; if (Array.isArray(val)) { - const unflattenedArray = val.map((item) => { + const unflattenedArray = val.map((item: unknown) => { if (item && typeof item === 'object' && !Array.isArray(item)) { return unflattenObject(item); } @@ -23,5 +27,6 @@ export function unflattenObject(source: Record, target: Record; } diff --git a/x-pack/plugins/observability_solution/infra/server/routes/entities/get_latest_entity.ts b/x-pack/plugins/observability_solution/infra/server/routes/entities/get_latest_entity.ts index 0756bc3d52c8f..9758b18580eb2 100644 --- a/x-pack/plugins/observability_solution/infra/server/routes/entities/get_latest_entity.ts +++ b/x-pack/plugins/observability_solution/infra/server/routes/entities/get_latest_entity.ts @@ -40,8 +40,8 @@ export async function getLatestEntity({ return undefined; } - const response = await inventoryEsClient.esql<{ - source_data_stream?: { type?: string | string[] }; + const { hits } = await inventoryEsClient.esql<{ + 'source_data_stream.type': string | string[]; }>('get_latest_entities', { query: `FROM ${ENTITIES_LATEST_ALIAS} | WHERE ${ENTITY_TYPE} == ? @@ -51,5 +51,5 @@ export async function getLatestEntity({ params: [entityType, entityId], }); - return { sourceDataStreamType: response[0].source_data_stream?.type }; + return { sourceDataStreamType: hits[0]?.['source_data_stream.type'] }; } diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_groups.ts b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_groups.ts index 87d0c375149e0..9a52ec7f0228d 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_groups.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_groups.ts @@ -22,7 +22,7 @@ export async function getEntityGroupsBy({ inventoryEsClient: ObservabilityElasticsearchClient; field: string; esQuery?: QueryDslQueryContainer; -}) { +}): Promise { const from = `FROM ${ENTITIES_LATEST_ALIAS}`; const where = [getBuiltinEntityDefinitionIdESQLWhereClause()]; @@ -31,8 +31,10 @@ export async function getEntityGroupsBy({ const limit = `LIMIT ${MAX_NUMBER_OF_ENTITIES}`; const query = [from, ...where, group, sort, limit].join(' | '); - return inventoryEsClient.esql('get_entities_groups', { + const { hits } = await inventoryEsClient.esql('get_entities_groups', { query, filter: esQuery, }); + + return hits; } diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_types.ts b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_types.ts index e944e27379ab5..f5d400504007d 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_types.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_types.ts @@ -7,7 +7,6 @@ import { type ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; import { ENTITY_TYPE } from '@kbn/observability-shared-plugin/common'; -import type { EntityInstance } from '@kbn/entities-schema'; import { ENTITIES_LATEST_ALIAS } from '../../../common/entities'; import { getBuiltinEntityDefinitionIdESQLWhereClause } from './query_helper'; @@ -17,7 +16,7 @@ export async function getEntityTypes({ inventoryEsClient: ObservabilityElasticsearchClient; }) { const entityTypesEsqlResponse = await inventoryEsClient.esql<{ - entity: Pick; + 'entity.type': string; }>('get_entity_types', { query: `FROM ${ENTITIES_LATEST_ALIAS} | ${getBuiltinEntityDefinitionIdESQLWhereClause()} @@ -25,5 +24,5 @@ export async function getEntityTypes({ `, }); - return entityTypesEsqlResponse.map((response) => response.entity.type); + return entityTypesEsqlResponse.hits.map((response) => response['entity.type']); } diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts index c4bf13c5ec140..c6a0d2d2d4973 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts @@ -6,13 +6,13 @@ */ import type { QueryDslQueryContainer, ScalarValue } from '@elastic/elasticsearch/lib/api/types'; -import type { EntityInstance } from '@kbn/entities-schema'; import { ENTITY_DISPLAY_NAME, ENTITY_LAST_SEEN, ENTITY_TYPE, } from '@kbn/observability-shared-plugin/common'; import type { ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; +import { unflattenObject } from '@kbn/observability-utils-common/object/unflatten_object'; import { ENTITIES_LATEST_ALIAS, InventoryEntity, @@ -62,27 +62,33 @@ export async function getLatestEntities({ const query = [from, ...where, sort, limit].join(' | '); - const latestEntitiesEsqlResponse = await inventoryEsClient.esql( - 'get_latest_entities', - { - query, - filter: esQuery, - params, - } - ); + const latestEntitiesEsqlResponse = await inventoryEsClient.esql<{ + 'entity.id': string; + 'entity.type': string; + 'entity.definition_id': string; + 'entity.display_name': string; + 'entity.identity_fields': string | string[]; + 'entity.last_seen_timestamp': string; + 'entity.definition_version': string; + 'entity.schema_version': string; + }>('get_latest_entities', { + query, + filter: esQuery, + params, + }); - return latestEntitiesEsqlResponse.map((lastestEntity) => { - const { entity, ...metadata } = lastestEntity; + return latestEntitiesEsqlResponse.hits.map((latestEntity) => { + const { entity, ...metadata } = unflattenObject(latestEntity); return { - entityId: entity.id, - entityType: entity.type, - entityDefinitionId: entity.definition_id, - entityDisplayName: entity.display_name, - entityIdentityFields: entity.identity_fields, - entityLastSeenTimestamp: entity.last_seen_timestamp, - entityDefinitionVersion: entity.definition_version, - entitySchemaVersion: entity.schema_version, + entityId: latestEntity['entity.id'], + entityType: latestEntity['entity.type'], + entityDefinitionId: latestEntity['entity.definition_id'], + entityDisplayName: latestEntity['entity.display_name'], + entityIdentityFields: latestEntity['entity.identity_fields'], + entityLastSeenTimestamp: latestEntity['entity.last_seen_timestamp'], + entityDefinitionVersion: latestEntity['entity.definition_version'], + entitySchemaVersion: latestEntity['entity.schema_version'], ...metadata, }; }); diff --git a/x-pack/plugins/streams/server/routes/streams/delete.ts b/x-pack/plugins/streams/server/routes/streams/delete.ts index f6501f49ac4bf..456947a49c066 100644 --- a/x-pack/plugins/streams/server/routes/streams/delete.ts +++ b/x-pack/plugins/streams/server/routes/streams/delete.ts @@ -75,7 +75,11 @@ export const deleteStreamRoute = createServerRoute({ }, }); -async function deleteStream(scopedClusterClient: IScopedClusterClient, id: string, logger: Logger) { +export async function deleteStream( + scopedClusterClient: IScopedClusterClient, + id: string, + logger: Logger +) { try { const { definition } = await readStream({ scopedClusterClient, id }); for (const child of definition.children) { diff --git a/x-pack/plugins/streams/server/routes/streams/disable.ts b/x-pack/plugins/streams/server/routes/streams/disable.ts index d6fb79b2b0058..9cc1324a6b9d3 100644 --- a/x-pack/plugins/streams/server/routes/streams/disable.ts +++ b/x-pack/plugins/streams/server/routes/streams/disable.ts @@ -7,9 +7,9 @@ import { badRequest, internal } from '@hapi/boom'; import { z } from '@kbn/zod'; -import { STREAMS_INDEX } from '../../../common/constants'; import { SecurityException } from '../../lib/streams/errors'; import { createServerRoute } from '../create_server_route'; +import { deleteStream } from './delete'; export const disableStreamsRoute = createServerRoute({ endpoint: 'POST /api/streams/_disable', @@ -31,10 +31,7 @@ export const disableStreamsRoute = createServerRoute({ try { const { scopedClusterClient } = await getScopedClients({ request }); - await scopedClusterClient.asInternalUser.indices.delete({ - index: STREAMS_INDEX, - allow_no_indices: true, - }); + await deleteStream(scopedClusterClient, 'logs', logger); return { acknowledged: true }; } catch (e) { diff --git a/x-pack/plugins/streams_app/public/components/stream_detail_overview/index.tsx b/x-pack/plugins/streams_app/public/components/stream_detail_overview/index.tsx index 44761641a38ff..93a573fd4c01f 100644 --- a/x-pack/plugins/streams_app/public/components/stream_detail_overview/index.tsx +++ b/x-pack/plugins/streams_app/public/components/stream_detail_overview/index.tsx @@ -7,7 +7,6 @@ import { EuiButton, EuiFlexGroup, EuiFlexItem, EuiPanel } from '@elastic/eui'; import { calculateAuto } from '@kbn/calculate-auto'; import { i18n } from '@kbn/i18n'; -import { useAbortableAsync } from '@kbn/observability-utils-browser/hooks/use_abortable_async'; import { useDateRange } from '@kbn/observability-utils-browser/hooks/use_date_range'; import { StreamDefinition } from '@kbn/streams-plugin/common'; import moment from 'moment'; @@ -35,8 +34,6 @@ export function StreamDetailOverview({ definition }: { definition?: StreamDefini setTimeRange, } = useDateRange({ data }); - const dataStream = definition?.id; - const indexPatterns = useMemo(() => { if (!definition?.id) { return undefined; @@ -114,23 +111,6 @@ export function StreamDetailOverview({ definition }: { definition?: StreamDefini [indexPatterns, dataViews, streamsRepositoryClient, queries?.histogramQuery, start, end] ); - const dataViewsFetch = useAbortableAsync(() => { - return dataViews - .create( - { - title: dataStream, - timeFieldName: '@timestamp', - }, - false, // skip fetch fields - true // display errors - ) - .then((response) => { - return [response]; - }); - }, [dataViews, dataStream]); - - const fetchedDataViews = useMemo(() => dataViewsFetch.value ?? [], [dataViewsFetch.value]); - return ( <> @@ -156,7 +136,6 @@ export function StreamDetailOverview({ definition }: { definition?: StreamDefini defaultMessage: 'Filter data by using KQL', } )} - dataViews={fetchedDataViews} dateRangeFrom={timeRange.from} dateRangeTo={timeRange.to} /> diff --git a/x-pack/plugins/streams_app/public/components/stream_detail_view/index.tsx b/x-pack/plugins/streams_app/public/components/stream_detail_view/index.tsx index 8cfe9e661372e..ebf72a58d32a8 100644 --- a/x-pack/plugins/streams_app/public/components/stream_detail_view/index.tsx +++ b/x-pack/plugins/streams_app/public/components/stream_detail_view/index.tsx @@ -48,22 +48,15 @@ export function StreamDetailView() { { name: 'overview', content: , - label: i18n.translate('app_not_found_in_i18nrc.streamDetailView.overviewTab', { + label: i18n.translate('xpack.streams.streamDetailView.overviewTab', { defaultMessage: 'Overview', }), }, { - name: 'routing', + name: 'management', content: <>, - label: i18n.translate('app_not_found_in_i18nrc.streamDetailView.routingTab', { - defaultMessage: 'Routing', - }), - }, - { - name: 'processing', - content: <>, - label: i18n.translate('app_not_found_in_i18nrc.streamDetailView.processingTab', { - defaultMessage: 'Processing', + label: i18n.translate('xpack.streams.streamDetailView.managementTab', { + defaultMessage: 'Management', }), }, ]; diff --git a/x-pack/plugins/streams_app/public/components/streams_table/index.tsx b/x-pack/plugins/streams_app/public/components/streams_table/index.tsx index 93c06853a93f8..f92c94f115e9b 100644 --- a/x-pack/plugins/streams_app/public/components/streams_table/index.tsx +++ b/x-pack/plugins/streams_app/public/components/streams_table/index.tsx @@ -36,7 +36,7 @@ export function StreamsTable({ return items; } - return items.filter((item) => item.id.toLowerCase().startsWith(query.toLowerCase())); + return items.filter((item) => item.id.toLowerCase().includes(query.toLowerCase())); }, [query, items]); const columns = useMemo>>(() => { From d6ee23e5d06144cb4b233b831fb6458c1f0fa7e6 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Tue, 19 Nov 2024 09:29:42 +0100 Subject: [PATCH 45/95] Remove unflattening/field resolution --- .../es/esql_result_to_plain_objects.test.ts | 35 +++---------------- .../es/esql_result_to_plain_objects.ts | 34 ++++++++---------- 2 files changed, 20 insertions(+), 49 deletions(-) diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/esql_result_to_plain_objects.test.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/esql_result_to_plain_objects.test.ts index 4557d0ba0bdd5..55d77368cfdfb 100644 --- a/x-pack/packages/observability/observability_utils/observability_utils_server/es/esql_result_to_plain_objects.test.ts +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/esql_result_to_plain_objects.test.ts @@ -27,40 +27,15 @@ describe('esqlResultToPlainObjects', () => { expect(output).toEqual([{ name: 'Foo Bar' }]); }); - it('should return columns without "text" or "keyword" in their names', () => { + it('should not unflatten objects', () => { const result: ESQLSearchResponse = { columns: [ - { name: 'name.text', type: 'text' }, - { name: 'age', type: 'keyword' }, - ], - values: [ - ['Foo Bar', 30], - ['Foo Qux', 25], - ], - }; - const output = esqlResultToPlainObjects(result); - expect(output).toEqual([ - { name: 'Foo Bar', age: 30 }, - { name: 'Foo Qux', age: 25 }, - ]); - }); - - it('should handle mixed columns correctly', () => { - const result: ESQLSearchResponse = { - columns: [ - { name: 'name', type: 'text' }, - { name: 'name.text', type: 'text' }, - { name: 'age', type: 'keyword' }, - ], - values: [ - ['Foo Bar', 'Foo Bar', 30], - ['Foo Qux', 'Foo Qux', 25], + { name: 'name', type: 'keyword' }, + { name: 'name.nested', type: 'keyword' }, ], + values: [['Foo Bar', 'Bar Foo']], }; const output = esqlResultToPlainObjects(result); - expect(output).toEqual([ - { name: 'Foo Bar', age: 30 }, - { name: 'Foo Qux', age: 25 }, - ]); + expect(output).toEqual([{ name: 'Foo Bar', 'name.nested': 'Bar Foo' }]); }); }); diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/esql_result_to_plain_objects.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/esql_result_to_plain_objects.ts index 34781153532c5..53f54b608ca35 100644 --- a/x-pack/packages/observability/observability_utils/observability_utils_server/es/esql_result_to_plain_objects.ts +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/esql_result_to_plain_objects.ts @@ -6,28 +6,24 @@ */ import type { ESQLSearchResponse } from '@kbn/es-types'; -import { unflattenObject } from '@kbn/observability-utils-common/object/unflatten_object'; -export function esqlResultToPlainObjects( - result: ESQLSearchResponse -): TDocument[] { - return result.values.map((row) => { - return unflattenObject( - row.reduce>((acc, value, index) => { - const column = result.columns[index]; +export function esqlResultToPlainObjects< + TDocument extends Record = Record +>(result: ESQLSearchResponse): TDocument[] { + return result.values.map((row): TDocument => { + return row.reduce>((acc, value, index) => { + const column = result.columns[index]; - if (!column) { - return acc; - } + if (!column) { + return acc; + } - // Removes the type suffix from the column name - const name = column.name.replace(/\.(text|keyword)$/, ''); - if (!acc[name]) { - acc[name] = value; - } + const name = column.name; + if (!acc[name]) { + acc[name] = value; + } - return acc; - }, {}) - ) as TDocument; + return acc; + }, {}) as TDocument; }); } From ad2cc5e50966b6dc51d1cf4b69dd7b2e3eb219ba Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 19 Nov 2024 08:40:24 +0000 Subject: [PATCH 46/95] [CI] Auto-commit changed files from 'node scripts/notice' --- .../observability_utils/observability_utils_common/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/packages/observability/observability_utils/observability_utils_common/tsconfig.json b/x-pack/packages/observability/observability_utils/observability_utils_common/tsconfig.json index 7954cdc946e9c..e2226268918a7 100644 --- a/x-pack/packages/observability/observability_utils/observability_utils_common/tsconfig.json +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/tsconfig.json @@ -19,5 +19,6 @@ "@kbn/es-query", "@kbn/safer-lodash-set", "@kbn/inference-common", + "@kbn/utility-types", ] } From 26f5554234c76c270f40039c06855fb4d120178d Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Tue, 19 Nov 2024 10:03:08 +0100 Subject: [PATCH 47/95] Update paths in jest.config.js --- x-pack/plugins/streams_app/jest.config.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/streams_app/jest.config.js b/x-pack/plugins/streams_app/jest.config.js index 0dde0422306bb..4bf7ed18f8e8c 100644 --- a/x-pack/plugins/streams_app/jest.config.js +++ b/x-pack/plugins/streams_app/jest.config.js @@ -9,14 +9,14 @@ module.exports = { preset: '@kbn/test', rootDir: '../../../..', roots: [ - '/x-pack/plugins/logsai/streams_app/public', - '/x-pack/plugins/logsai/streams_app/common', - '/x-pack/plugins/logsai/streams_app/server', + '/x-pack/plugins/streams_app/public', + '/x-pack/plugins/streams_app/common', + '/x-pack/plugins/streams_app/server', ], - setupFiles: ['/x-pack/plugins/logsai/streams_app/.storybook/jest_setup.js'], + setupFiles: ['/x-pack/plugins/streams_app/.storybook/jest_setup.js'], collectCoverage: true, collectCoverageFrom: [ - '/x-pack/plugins/logsai/streams_app/{public,common,server}/**/*.{js,ts,tsx}', + '/x-pack/plugins/streams_app/{public,common,server}/**/*.{js,ts,tsx}', ], coverageReporters: ['html'], From 8dc61f34c38a03516ebe53ef2f3ed9a401cd9048 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Tue, 19 Nov 2024 10:19:59 +0100 Subject: [PATCH 48/95] Update i18n label --- .../streams_app/public/components/stream_list_view/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/streams_app/public/components/stream_list_view/index.tsx b/x-pack/plugins/streams_app/public/components/stream_list_view/index.tsx index 5effd1dedfd3f..e0530fc6bf5f0 100644 --- a/x-pack/plugins/streams_app/public/components/stream_list_view/index.tsx +++ b/x-pack/plugins/streams_app/public/components/stream_list_view/index.tsx @@ -39,7 +39,7 @@ export function StreamListView() { From ab533bacab65d88cfc7631985b0520be6dcd13d5 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 21 Nov 2024 09:41:48 +0100 Subject: [PATCH 49/95] [Streams] API for asset links --- .../client/create_observability_es_client.ts | 9 +- .../es/storage/index.ts | 46 +++ .../es/storage/index_adapter/errors.ts | 39 +++ .../es/storage/index_adapter/index.ts | 300 ++++++++++++++++++ .../es/storage/storage_client.ts | 59 ++++ .../es/storage/types.ts | 138 ++++++++ x-pack/plugins/streams/common/assets.ts | 26 ++ x-pack/plugins/streams/kibana.jsonc | 3 +- .../server/lib/streams/assets/asset_client.ts | 181 +++++++++++ .../lib/streams/assets/asset_service.ts | 48 +++ .../server/lib/streams/assets/fields.ts | 11 + .../lib/streams/assets/storage_settings.ts | 24 ++ x-pack/plugins/streams/server/plugin.ts | 9 +- .../streams/server/routes/assets/route.ts | 124 ++++++++ x-pack/plugins/streams/server/routes/index.ts | 2 + x-pack/plugins/streams/server/routes/types.ts | 2 + x-pack/plugins/streams/server/types.ts | 18 +- .../stream_detail_asset_view/index.tsx | 85 +++++ .../components/stream_detail_view/index.tsx | 8 + 19 files changed, 1122 insertions(+), 10 deletions(-) create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/errors.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/index.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/storage_client.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/types.ts create mode 100644 x-pack/plugins/streams/common/assets.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/assets/asset_client.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/assets/asset_service.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/assets/fields.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/assets/storage_settings.ts create mode 100644 x-pack/plugins/streams/server/routes/assets/route.ts create mode 100644 x-pack/plugins/streams_app/public/components/stream_detail_asset_view/index.tsx diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts index 40b9fcbb6a130..a33b8e2bfe93b 100644 --- a/x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts @@ -52,6 +52,8 @@ export interface EsqlQueryResponse { values: EsqlValue[][]; } +export type ObservabilitySearchRequest = SearchRequest; + /** * An Elasticsearch Client with a fully typed `search` method and built-in * APM instrumentation. @@ -86,10 +88,12 @@ export function createObservabilityEsClient({ client, logger, plugin, + labels, }: { client: ElasticsearchClient; logger: Logger; - plugin: string; + plugin?: string; + labels?: Record; }): ObservabilityElasticsearchClient { // wraps the ES calls in a named APM span for better analysis // (otherwise it would just eg be a _search span) @@ -103,7 +107,8 @@ export function createObservabilityEsClient({ { name: operationName, labels: { - plugin, + ...labels, + ...(plugin ? { plugin } : {}), }, }, callback, diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index.ts new file mode 100644 index 0000000000000..ccd91ca9d479a --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { StorageFieldTypeOf, StorageMappingProperty } from './types'; + +interface StorageSchemaProperties { + [x: string]: StorageMappingProperty; +} + +export interface StorageSchema { + properties: StorageSchemaProperties; +} + +interface StorageSettingsBase { + schema: StorageSchema; +} + +export interface IndexStorageSettings extends StorageSettingsBase { + name: string; +} + +export type StorageSettings = IndexStorageSettings; + +export interface IStorageAdapter { + getSearchIndexPattern(): string; +} + +export type StorageSettingsOf> = + TStorageAdapter extends IStorageAdapter + ? TStorageSettings extends StorageSettings + ? TStorageSettings + : never + : never; + +export type StorageDocumentOf = { + [TKey in keyof TStorageSettings['schema']['properties']]: StorageFieldTypeOf< + TStorageSettings['schema']['properties'][TKey] + >; +} & { _id: string }; + +export { StorageIndexAdapter } from './index_adapter'; +export { StorageClient } from './storage_client'; +export { types } from './types'; diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/errors.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/errors.ts new file mode 100644 index 0000000000000..39942a5c06c68 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/errors.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MappingProperty } from '@elastic/elasticsearch/lib/api/types'; +import { StorageSchema } from '..'; + +export class IncompatibleSchemaUpdateError extends Error { + constructor({ + existingProperties, + incompatibleProperties, + missingProperties, + nextProperties, + }: { + existingProperties: Record; + nextProperties: StorageSchema['properties']; + missingProperties: string[]; + incompatibleProperties: string[]; + }) { + const missingErrorMessage = missingProperties.length + ? `\nmissing properties: ${missingProperties.join(', ')}` + : ''; + + const incompatibleErrorMessage = incompatibleProperties.length + ? `\nincompatible properties: + + ${incompatibleProperties + .map((property) => { + return `\t${property}: expected ${existingProperties[property].type}, but got ${nextProperties[property].type}`; + }) + .join('\n')}` + : ''; + + super(`Incompatible schema update: ${missingErrorMessage} ${incompatibleErrorMessage}`); + } +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/index.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/index.ts new file mode 100644 index 0000000000000..5b2ba744c8167 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/index.ts @@ -0,0 +1,300 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import objectHash from 'object-hash'; +import stringify from 'json-stable-stringify'; +import { + IndicesPutIndexTemplateIndexTemplateMapping, + MappingProperty, +} from '@elastic/elasticsearch/lib/api/types'; +import { last, mapValues, orderBy, padStart } from 'lodash'; +import { isResponseError } from '@kbn/es-errors'; +import { IStorageAdapter, IndexStorageSettings, StorageSchema } from '..'; +import { StorageClient } from '../storage_client'; +import { IncompatibleSchemaUpdateError } from './errors'; +import { StorageMappingProperty } from '../types'; + +function getAliasName(name: string) { + return name; +} + +function getBackingIndexPattern(name: string) { + return `${name}-*`; +} + +function getBackingIndexName(name: string, count: number) { + const countId = padStart(count.toString(), 6, '0'); + return `${name}-${countId}`; +} + +function getIndexTemplateName(name: string) { + return `${name}`; +} + +function getSchemaVersion(storage: IndexStorageSettings): string { + const version = objectHash(stringify(storage.schema.properties)); + return version; +} + +function isCompatibleOrThrow( + existingProperties: Record, + nextProperties: StorageSchema['properties'] +): void { + const missingProperties: string[] = []; + const incompatibleProperties: string[] = []; + Object.entries(existingProperties).forEach(([key, propertyInExisting]) => { + const propertyInNext = toElasticsearchMappingProperty(nextProperties[key]); + if (!propertyInNext) { + missingProperties.push(key); + } else if ( + propertyInNext.type !== propertyInExisting.type || + propertyInExisting.meta?.required !== propertyInNext.meta?.required || + propertyInExisting.meta?.multi_value !== propertyInNext.meta?.multi_value + ) { + incompatibleProperties.push(key); + } + }); + + const totalErrors = missingProperties.length + incompatibleProperties.length; + + if (totalErrors > 0) { + throw new IncompatibleSchemaUpdateError({ + existingProperties, + nextProperties, + missingProperties, + incompatibleProperties, + }); + } +} + +function toElasticsearchMappingProperty(property: StorageMappingProperty): MappingProperty { + const { required, multi_value: multiValue, enum: enums, ...rest } = property; + + return { + ...rest, + meta: { + ...property.meta, + required: JSON.stringify(required ?? false), + multi_value: JSON.stringify(multiValue ?? false), + ...(enums ? { enum: JSON.stringify(enums) } : {}), + }, + }; +} + +export class StorageIndexAdapter + implements IStorageAdapter +{ + constructor( + private readonly esClient: ElasticsearchClient, + private readonly logger: Logger, + private readonly storage: TStorageSettings + ) {} + + getSearchIndexPattern(): string { + return getAliasName(this.storage.name); + } + + private async createIndexTemplate(create: boolean = true): Promise { + this.logger.debug(`Creating index template (create = ${create})`); + + const version = getSchemaVersion(this.storage); + + const template: IndicesPutIndexTemplateIndexTemplateMapping = { + mappings: { + _meta: { + version, + }, + properties: mapValues(this.storage.schema.properties, toElasticsearchMappingProperty), + }, + }; + + await this.esClient.indices.putIndexTemplate({ + name: getIndexTemplateName(this.storage.name), + create, + allow_auto_create: false, + index_patterns: getBackingIndexPattern(this.storage.name), + _meta: { + version, + }, + template, + }); + } + + private async updateIndexTemplateIfNeeded(): Promise { + const version = getSchemaVersion(this.storage); + const indexTemplate = await this.esClient.indices + .getIndexTemplate({ + name: getIndexTemplateName(this.storage.name), + }) + .then((templates) => templates.index_templates[0].index_template); + + const currentVersion = indexTemplate._meta?.version; + + this.logger.debug( + `updateIndexTemplateIfNeeded: Current version = ${currentVersion}, next version = ${version}` + ); + + if (currentVersion === version) { + return; + } + + isCompatibleOrThrow( + indexTemplate.template!.mappings!.properties!, + this.storage.schema.properties + ); + + this.logger.debug(`Updating index template due to version mismatch`); + + await this.createIndexTemplate(false); + } + + private async getCurrentWriteIndexName(): Promise { + const aliasName = getAliasName(this.storage.name); + + const aliases = await this.esClient.indices + .getAlias({ + name: getAliasName(this.storage.name), + }) + .catch((error) => { + if (isResponseError(error) && error.statusCode === 404) { + return {}; + } + throw error; + }); + + const writeIndex = Object.entries(aliases) + .map(([name, alias]) => { + return { + name, + isWriteIndex: alias.aliases[aliasName]?.is_write_index === true, + }; + }) + .find(({ isWriteIndex }) => { + return isWriteIndex; + }); + + return writeIndex?.name; + } + + private async createNextBackingIndex(): Promise { + const writeIndex = await this.getCurrentWriteIndexName(); + + this.logger.debug(`Creating next backing index, current write index = ${writeIndex}`); + + const nextIndexName = getBackingIndexName( + this.storage.name, + writeIndex ? parseInt(last(writeIndex.split('-'))!, 10) : 1 + ); + + await this.esClient.indices.create({ + index: nextIndexName, + }); + } + + private async createAlias(): Promise { + const aliasName = getAliasName(this.storage.name); + + let indexName = await this.getCurrentWriteIndexName(); + + if (!indexName) { + const indices = await this.esClient.indices.get({ + index: getBackingIndexPattern(this.storage.name), + }); + + indexName = orderBy(Object.keys(indices), 'desc')[0]; + } + + if (!indexName) { + throw new Error(`Could not find backing index for ${aliasName}`); + } + + await this.esClient.indices.putAlias({ + index: indexName, + name: aliasName, + is_write_index: true, + }); + } + + private async rolloverIfNeeded() { + const [writeIndexName, indexTemplate, indices] = await Promise.all([ + this.getCurrentWriteIndexName(), + this.esClient.indices + .getIndexTemplate({ + name: getIndexTemplateName(this.storage.name), + }) + .then((templates) => templates.index_templates[0].index_template), + this.esClient.indices.get({ + index: getBackingIndexPattern(this.storage.name), + }), + ]); + + if (!writeIndexName) { + throw new Error(`No write index found for ${getAliasName(this.storage.name)}`); + } + + if (!indexTemplate) { + throw new Error(`No index template found for ${getIndexTemplateName(this.storage.name)}`); + } + + const writeIndex = indices[writeIndexName]; + + const isSameVersion = writeIndex.mappings?._meta?.version === indexTemplate._meta?.version; + + if (!isSameVersion) { + await this.createNextBackingIndex(); + } + } + + async bootstrap() { + const { name } = this.storage; + + this.logger.debug('Retrieving existing Elasticsearch components'); + + const [indexTemplateExists, aliasExists, backingIndexExists] = await Promise.all([ + this.esClient.indices.existsIndexTemplate({ + name: getIndexTemplateName(name), + }), + this.esClient.indices.existsAlias({ + name: getAliasName(name), + }), + this.esClient.indices.exists({ + index: getBackingIndexPattern(name), + allow_no_indices: false, + }), + ]); + + this.logger.debug( + () => + `Existing components: ${JSON.stringify({ + indexTemplateExists, + aliasExists, + backingIndexExists, + })}` + ); + + if (!indexTemplateExists) { + await this.createIndexTemplate(); + } else { + await this.updateIndexTemplateIfNeeded(); + } + + if (!backingIndexExists) { + await this.createNextBackingIndex(); + } + + if (!aliasExists) { + await this.createAlias(); + } + + await this.rolloverIfNeeded(); + } + + getClient(): StorageClient { + return new StorageClient(this, this.esClient, this.logger); + } +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/storage_client.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/storage_client.ts new file mode 100644 index 0000000000000..b516a5314d2ce --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/storage_client.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { IStorageAdapter, StorageDocumentOf, StorageSettings } from '.'; +import { + ObservabilityElasticsearchClient, + ObservabilitySearchRequest, + createObservabilityEsClient, +} from '../client/create_observability_es_client'; + +export class StorageClient { + private readonly esClient: ObservabilityElasticsearchClient; + constructor( + private readonly storage: IStorageAdapter, + esClient: ElasticsearchClient, + private readonly logger: Logger + ) { + this.esClient = createObservabilityEsClient({ + client: esClient, + logger, + }); + } + + search>( + operationName: string, + request: TSearchRequest + ) { + return this.esClient.search< + StorageDocumentOf, + TSearchRequest & { index: string } + >(operationName, { ...request, index: this.storage.getSearchIndexPattern() }); + } + + async index({ + id, + document, + }: { + id: string; + document: Omit, '_id'>; + }) { + await this.esClient.client.index({ + index: this.storage.getSearchIndexPattern(), + document, + id, + }); + } + + async delete(id: string) { + await this.esClient.client.delete({ + id, + index: this.storage.getSearchIndexPattern(), + }); + } +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/types.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/types.ts new file mode 100644 index 0000000000000..cc9471b63e2f8 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/types.ts @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MappingProperty } from '@elastic/elasticsearch/lib/api/types'; +import { merge } from 'lodash'; + +type AllMappingPropertyType = Required['type']; + +type StorageMappingPropertyType = AllMappingPropertyType & + ( + | 'text' + | 'match_only_text' + | 'keyword' + | 'boolean' + | 'date' + | 'byte' + | 'float' + | 'double' + | 'long' + ); + +interface Metadata { + [x: string]: any; +} + +type WithMeta = T extends any + ? Omit & { + meta?: Metadata; + required?: boolean; + multi_value?: boolean; + enum?: string[]; + } + : never; + +export type StorageMappingProperty = WithMeta< + Extract +>; + +type MappingPropertyOf = Extract< + StorageMappingProperty, + { type: TType } +>; + +type MappingPropertyFactory< + TType extends StorageMappingPropertyType, + TDefaults extends Partial | undefined> +> = > | undefined>( + overrides?: TOverrides +) => MappingPropertyOf & Exclude & Exclude; + +function createFactory< + TType extends StorageMappingPropertyType, + TDefaults extends Partial> | undefined +>(type: TType, defaults?: TDefaults): MappingPropertyFactory; + +function createFactory( + type: StorageMappingPropertyType, + defaults?: Partial +) { + return (overrides: Partial) => { + return { + ...defaults, + ...overrides, + type, + }; + }; +} + +const baseTypes = { + keyword: createFactory('keyword', { ignore_above: 1024 }), + match_only_text: createFactory('match_only_text'), + text: createFactory('text'), + double: createFactory('double'), + long: createFactory('long'), + boolean: createFactory('boolean'), + date: createFactory('date', { format: 'strict_date_optional_time' }), + byte: createFactory('byte'), + float: createFactory('float'), +} satisfies { + [TKey in StorageMappingPropertyType]: MappingPropertyFactory; +}; + +function enumFactory< + TEnum extends string, + TOverrides extends Partial> | undefined +>( + enums: TEnum[], + overrides?: TOverrides +): MappingPropertyOf<'keyword'> & { enum: TEnum[] } & Exclude; + +function enumFactory(enums: string[], overrides?: Partial>) { + const nextOverrides = merge({ enum: enums }, overrides); + const prop = baseTypes.keyword(nextOverrides); + return prop; +} + +const types = { + ...baseTypes, + enum: enumFactory, +}; + +type PrimitiveOf = { + keyword: TProperty extends { enum: infer TEnums } + ? TEnums extends Array + ? TEnum + : never + : string; + match_only_text: string; + text: string; + boolean: boolean; + date: TProperty extends { format: 'strict_date_optional_time' } ? string : string | number; + double: number; + long: number; + byte: number; + float: number; +}[TProperty['type']]; + +type MaybeMultiValue = TProperty extends { + multi_value: true; +} + ? TPrimitive[] + : TPrimitive; +type MaybeRequired = TProperty extends { + required: true; +} + ? TPrimitive + : TPrimitive | undefined; + +export type StorageFieldTypeOf = MaybeRequired< + TProperty, + MaybeMultiValue> +>; + +export { types }; diff --git a/x-pack/plugins/streams/common/assets.ts b/x-pack/plugins/streams/common/assets.ts new file mode 100644 index 0000000000000..df873174dcff9 --- /dev/null +++ b/x-pack/plugins/streams/common/assets.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ValuesType } from 'utility-types'; + +export const ASSET_TYPES = { + Dashboard: 'dashboard' as const, + Rule: 'rule' as const, + Slo: 'slo' as const, +}; + +export type AssetType = ValuesType; + +export interface AssetLink { + type: AssetType; + assetId: string; +} + +export interface Asset extends AssetLink { + label: string; + tags: string[]; +} diff --git a/x-pack/plugins/streams/kibana.jsonc b/x-pack/plugins/streams/kibana.jsonc index 06c37ed245cf1..00563714e231b 100644 --- a/x-pack/plugins/streams/kibana.jsonc +++ b/x-pack/plugins/streams/kibana.jsonc @@ -16,7 +16,8 @@ "encryptedSavedObjects", "usageCollection", "licensing", - "taskManager" + "taskManager", + "alerting" ], "optionalPlugins": [ "cloud", diff --git a/x-pack/plugins/streams/server/lib/streams/assets/asset_client.ts b/x-pack/plugins/streams/server/lib/streams/assets/asset_client.ts new file mode 100644 index 0000000000000..9861ac8c268e1 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/assets/asset_client.ts @@ -0,0 +1,181 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import pLimit from 'p-limit'; +import { StorageClient } from '@kbn/observability-utils-server/es/storage'; +import { termQuery } from '@kbn/observability-utils-server/es/queries/term_query'; +import { RulesClient } from '@kbn/alerting-plugin/server'; +import { SavedObjectsClientContract } from '@kbn/core/server'; +import { keyBy } from 'lodash'; +import objectHash from 'object-hash'; +import { AssetStorageSettings } from './storage_settings'; +import { ASSET_TYPES, Asset, AssetType } from '../../../../common/assets'; +import { ASSET_ENTITY_ID, ASSET_ENTITY_TYPE } from './fields'; + +export class AssetClient { + constructor( + private readonly clients: { + storageClient: StorageClient; + soClient: SavedObjectsClientContract; + rulesClient: RulesClient; + } + ) {} + + async linkAsset({ + entityId, + entityType, + assetId, + assetType, + }: { + entityId: string; + entityType: string; + assetId: string; + assetType: AssetType; + }) { + const assetDoc = { + 'asset.id': assetId, + 'asset.type': assetType, + 'entity.id': entityId, + 'entity.type': entityType, + }; + + const id = objectHash(assetDoc); + + await this.clients.storageClient.index({ + id, + document: assetDoc, + }); + } + + async unlinkAsset({ + entityId, + entityType, + assetId, + assetType, + }: { + entityId: string; + entityType: string; + assetId: string; + assetType: string; + }) { + const id = objectHash({ + 'asset.id': assetId, + 'asset.type': assetType, + 'entity.id': entityId, + 'entity.type': entityType, + }); + + await this.clients.storageClient.delete(id); + } + + async getAssets({ + entityId, + entityType, + }: { + entityId: string; + entityType: 'stream'; + }): Promise { + const assetsResponse = await this.clients.storageClient.search('get_assets_for_entity', { + size: 10_000, + track_total_hits: false, + query: { + bool: { + filter: [ + ...termQuery(ASSET_ENTITY_ID, entityId), + ...termQuery(ASSET_ENTITY_TYPE, entityType), + ], + }, + }, + }); + + const assetLinks = assetsResponse.hits.hits.map((hit) => hit._source); + + if (!assetLinks.length) { + return []; + } + + const idsByType = Object.fromEntries( + Object.values(ASSET_TYPES).map((type) => [type, [] as string[]]) + ) as Record; + + assetLinks.forEach((assetLink) => { + const assetType = assetLink['asset.type']; + const assetId = assetLink['asset.id']; + idsByType[assetType].push(assetId); + }); + + const limiter = pLimit(10); + + const [dashboards, rules, slos] = await Promise.all([ + idsByType.dashboard.length + ? this.clients.soClient + .bulkGet<{ title: string; tags: string[] }>( + idsByType.dashboard.map((dashboardId) => ({ type: 'dashboard', id: dashboardId })) + ) + .then((response) => { + const dashboardsById = keyBy(response.saved_objects, 'id'); + + return idsByType.dashboard.flatMap((dashboardId): Asset[] => { + const dashboard = dashboardsById[dashboardId]; + if (dashboard) { + return [ + { + assetId: dashboardId, + label: dashboard.attributes.title, + tags: [], + type: 'dashboard', + }, + ]; + } + return []; + }); + }) + : [], + Promise.all( + idsByType.rule.map((ruleId) => { + return limiter(() => + this.clients.rulesClient.get({ id: ruleId }).then((rule): Asset => { + return { + type: 'rule', + assetId: ruleId, + label: rule.name, + tags: rule.tags, + }; + }) + ); + }) + ), + idsByType.slo.length + ? this.clients.soClient + .find<{ name: string; tags: string[] }>({ + type: 'slo', + filter: `slo.attributes.id:(${idsByType.slo.join(' OR ')})`, + perPage: idsByType.slo.length, + }) + .then((soResponse) => { + const sloDefinitionsById = keyBy(soResponse.saved_objects, 'slo.attributes.id'); + + return idsByType.slo.flatMap((sloId): Asset[] => { + const sloDefinition = sloDefinitionsById[sloId]; + if (sloDefinition) { + return [ + { + assetId: sloId, + label: sloDefinition.attributes.name, + tags: sloDefinition.attributes.tags, + type: 'slo', + }, + ]; + } + return []; + }); + }) + : [], + ]); + + return [...dashboards, ...rules, ...slos]; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/assets/asset_service.ts b/x-pack/plugins/streams/server/lib/streams/assets/asset_service.ts new file mode 100644 index 0000000000000..491761d314dd0 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/assets/asset_service.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CoreSetup, KibanaRequest, Logger } from '@kbn/core/server'; +import { StorageIndexAdapter } from '@kbn/observability-utils-server/es/storage'; +import { Observable, defer, firstValueFrom, from } from 'rxjs'; +import { StreamsPluginStartDependencies } from '../../../types'; +import { AssetClient } from './asset_client'; +import { assetStorageSettings } from './storage_settings'; + +export class AssetService { + private adapter$: Observable>; + constructor( + private readonly coreSetup: CoreSetup, + private readonly logger: Logger + ) { + this.adapter$ = defer(() => from(this.prepareIndex())); + } + + async prepareIndex(): Promise> { + const [coreStart] = await this.coreSetup.getStartServices(); + const esClient = coreStart.elasticsearch.client.asInternalUser; + + const adapter = new StorageIndexAdapter( + esClient, + this.logger.get('assets'), + assetStorageSettings + ); + await adapter.bootstrap(); + return adapter; + } + + async getClientWithRequest({ request }: { request: KibanaRequest }): Promise { + const [coreStart, pluginsStart] = await this.coreSetup.getStartServices(); + + return firstValueFrom(this.adapter$).then((adapter) => { + return new AssetClient({ + storageClient: adapter.getClient(), + soClient: coreStart.savedObjects.getScopedClient(request), + rulesClient: pluginsStart.alerting.getRulesClientWithRequest(request), + }); + }); + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/assets/fields.ts b/x-pack/plugins/streams/server/lib/streams/assets/fields.ts new file mode 100644 index 0000000000000..9dc57594829f7 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/assets/fields.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const ASSET_ENTITY_ID = 'entity.id'; +export const ASSET_ENTITY_TYPE = 'entity.type'; +export const ASSET_ASSET_ID = 'asset.id'; +export const ASSET_TYPE = 'asset.type'; diff --git a/x-pack/plugins/streams/server/lib/streams/assets/storage_settings.ts b/x-pack/plugins/streams/server/lib/streams/assets/storage_settings.ts new file mode 100644 index 0000000000000..b3609ee311500 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/assets/storage_settings.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IndexStorageSettings, types } from '@kbn/observability-utils-server/es/storage'; +import { ASSET_ASSET_ID, ASSET_ENTITY_ID, ASSET_ENTITY_TYPE, ASSET_TYPE } from './fields'; +import { ASSET_TYPES } from '../../../../common/assets'; + +export const assetStorageSettings = { + name: '.kibana_stream_assets', + schema: { + properties: { + [ASSET_ASSET_ID]: types.keyword({ required: true }), + [ASSET_TYPE]: types.enum(Object.values(ASSET_TYPES), { required: true }), + [ASSET_ENTITY_ID]: types.keyword(), + [ASSET_ENTITY_TYPE]: types.keyword(), + }, + }, +} satisfies IndexStorageSettings; + +export type AssetStorageSettings = typeof assetStorageSettings; diff --git a/x-pack/plugins/streams/server/plugin.ts b/x-pack/plugins/streams/server/plugin.ts index ef070984803d5..07f41304a7a4f 100644 --- a/x-pack/plugins/streams/server/plugin.ts +++ b/x-pack/plugins/streams/server/plugin.ts @@ -23,6 +23,7 @@ import { StreamsPluginStartDependencies, StreamsServer, } from './types'; +import { AssetService } from './lib/streams/assets/asset_service'; // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface StreamsPluginSetup {} @@ -52,15 +53,21 @@ export class StreamsPlugin this.logger = context.logger.get(); } - public setup(core: CoreSetup, plugins: StreamsPluginSetupDependencies): StreamsPluginSetup { + public setup( + core: CoreSetup, + plugins: StreamsPluginSetupDependencies + ): StreamsPluginSetup { this.server = { config: this.config, logger: this.logger, } as StreamsServer; + const assetService = new AssetService(core, this.logger); + registerRoutes({ repository: StreamsRouteRepository, dependencies: { + assets: assetService, server: this.server, getScopedClients: async ({ request }: { request: KibanaRequest }) => { const [coreStart] = await core.getStartServices(); diff --git a/x-pack/plugins/streams/server/routes/assets/route.ts b/x-pack/plugins/streams/server/routes/assets/route.ts new file mode 100644 index 0000000000000..c22ed84fcaea8 --- /dev/null +++ b/x-pack/plugins/streams/server/routes/assets/route.ts @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; +import { ASSET_TYPES, Asset } from '../../../common/assets'; +import { createServerRoute } from '../create_server_route'; + +interface ListAssetsResponse { + assets: Asset[]; +} + +interface LinkAssetResponse { + acknowledged: boolean; +} + +interface UnlinkAssetResponse { + acknowledged: boolean; +} + +const assetTypeSchema = z.union([ + z.literal(ASSET_TYPES.Dashboard), + z.literal(ASSET_TYPES.Rule), + z.literal(ASSET_TYPES.Slo), +]); + +export const listAssetsRoute = createServerRoute({ + endpoint: 'GET /api/streams/{id}/assets', + options: { + access: 'internal', + }, + params: z.object({ + path: z.object({ + id: z.string(), + }), + }), + async handler({ params, request, assets }): Promise { + const assetsClient = await assets.getClientWithRequest({ request }); + + const { + path: { id: streamId }, + } = params; + + return { + assets: await assetsClient.getAssets({ + entityId: streamId, + entityType: 'stream', + }), + }; + }, +}); + +export const linkAssetRoute = createServerRoute({ + endpoint: 'PUT /api/streams/{id}/assets/{type}/{assetId}', + options: { + access: 'internal', + }, + params: z.object({ + path: z.object({ + type: assetTypeSchema, + id: z.string(), + assetId: z.string(), + }), + }), + handler: async ({ params, request, assets }): Promise => { + const assetsClient = await assets.getClientWithRequest({ request }); + + const { + path: { assetId, id: streamId, type: assetType }, + } = params; + + await assetsClient.linkAsset({ + entityId: streamId, + entityType: 'stream', + assetId, + assetType, + }); + + return { + acknowledged: true, + }; + }, +}); + +export const unlinkAssetRoute = createServerRoute({ + endpoint: 'DELETE /api/streams/{id}/assets/{type}/{assetId}', + options: { + access: 'internal', + }, + params: z.object({ + path: z.object({ + type: assetTypeSchema, + id: z.string(), + assetId: z.string(), + }), + }), + handler: async ({ params, request, assets }): Promise => { + const assetsClient = await assets.getClientWithRequest({ request }); + + const { + path: { assetId, id: streamId, type: assetType }, + } = params; + + await assetsClient.unlinkAsset({ + entityId: streamId, + entityType: 'stream', + assetId, + assetType, + }); + + return { + acknowledged: true, + }; + }, +}); + +export const assetsRoutes = { + ...listAssetsRoute, + ...linkAssetRoute, + ...unlinkAssetRoute, +}; diff --git a/x-pack/plugins/streams/server/routes/index.ts b/x-pack/plugins/streams/server/routes/index.ts index fb676ce94b9f8..ce93f6069a8ea 100644 --- a/x-pack/plugins/streams/server/routes/index.ts +++ b/x-pack/plugins/streams/server/routes/index.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { assetsRoutes } from './assets/route'; import { esqlRoutes } from './esql/route'; import { deleteStreamRoute } from './streams/delete'; import { disableStreamsRoute } from './streams/disable'; @@ -27,6 +28,7 @@ export const StreamsRouteRepository = { ...streamsStatusRoutes, ...esqlRoutes, ...disableStreamsRoute, + ...assetsRoutes, }; export type StreamsRouteRepository = typeof StreamsRouteRepository; diff --git a/x-pack/plugins/streams/server/routes/types.ts b/x-pack/plugins/streams/server/routes/types.ts index d547d56c088cd..77248238fcb47 100644 --- a/x-pack/plugins/streams/server/routes/types.ts +++ b/x-pack/plugins/streams/server/routes/types.ts @@ -10,8 +10,10 @@ import { DefaultRouteHandlerResources } from '@kbn/server-route-repository'; import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; import { StreamsServer } from '../types'; +import { AssetService } from '../lib/streams/assets/asset_service'; export interface RouteDependencies { + assets: AssetService; server: StreamsServer; getScopedClients: ({ request }: { request: KibanaRequest }) => Promise<{ scopedClusterClient: IScopedClusterClient; diff --git a/x-pack/plugins/streams/server/types.ts b/x-pack/plugins/streams/server/types.ts index f119faa0ed010..85073f027f143 100644 --- a/x-pack/plugins/streams/server/types.ts +++ b/x-pack/plugins/streams/server/types.ts @@ -5,18 +5,22 @@ * 2.0. */ -import { CoreStart, ElasticsearchClient, Logger } from '@kbn/core/server'; -import { SecurityPluginStart } from '@kbn/security-plugin/server'; -import { +import type { CoreStart, ElasticsearchClient, Logger } from '@kbn/core/server'; +import type { SecurityPluginStart } from '@kbn/security-plugin/server'; +import type { EncryptedSavedObjectsPluginSetup, EncryptedSavedObjectsPluginStart, } from '@kbn/encrypted-saved-objects-plugin/server'; -import { LicensingPluginStart } from '@kbn/licensing-plugin/server'; -import { +import type { LicensingPluginStart } from '@kbn/licensing-plugin/server'; +import type { TaskManagerSetupContract, TaskManagerStartContract, } from '@kbn/task-manager-plugin/server'; -import { StreamsConfig } from '../common/config'; +import type { + PluginSetupContract as AlertingPluginSetup, + PluginStartContract as AlertingPluginStart, +} from '@kbn/alerting-plugin/server'; +import type { StreamsConfig } from '../common/config'; export interface StreamsServer { core: CoreStart; @@ -35,6 +39,7 @@ export interface ElasticsearchAccessorOptions { export interface StreamsPluginSetupDependencies { encryptedSavedObjects: EncryptedSavedObjectsPluginSetup; taskManager: TaskManagerSetupContract; + alerting: AlertingPluginSetup; } export interface StreamsPluginStartDependencies { @@ -42,4 +47,5 @@ export interface StreamsPluginStartDependencies { encryptedSavedObjects: EncryptedSavedObjectsPluginStart; licensing: LicensingPluginStart; taskManager: TaskManagerStartContract; + alerting: AlertingPluginStart; } diff --git a/x-pack/plugins/streams_app/public/components/stream_detail_asset_view/index.tsx b/x-pack/plugins/streams_app/public/components/stream_detail_asset_view/index.tsx new file mode 100644 index 0000000000000..525d4372cfc99 --- /dev/null +++ b/x-pack/plugins/streams_app/public/components/stream_detail_asset_view/index.tsx @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React, { useState } from 'react'; +import { i18n } from '@kbn/i18n'; +import { StreamDefinition } from '@kbn/streams-plugin/common'; +import { + EuiButton, + EuiFilterButton, + EuiFilterGroup, + EuiFlexGroup, + EuiFlexItem, + EuiPopover, + EuiSearchBar, + EuiSelectable, +} from '@elastic/eui'; +import { useStreamsAppFetch } from '../../hooks/use_streams_app_fetch'; +import { useKibana } from '../../hooks/use_kibana'; + +export function StreamDetailAssetView({ definition }: { definition?: StreamDefinition }) { + const { + dependencies: { + start: { + streams: { streamsRepositoryClient }, + }, + }, + } = useKibana(); + const [query, setQuery] = useState(''); + + const [isTagsPopoverOpen, setIsTagsPopoverOpen] = useState(false); + + const tagsButton = ( + + {i18n.translate('xpack.streams.streamDetailAssetView.tagsFilterButtonLabel', { + defaultMessage: 'Tags', + })} + + ); + + const assetsFetch = useStreamsAppFetch( + ({ signal }) => { + if (!definition?.id) { + return Promise.resolve(undefined); + } + return streamsRepositoryClient.fetch('GET /api/streams/{id}/assets', { + signal, + params: { + path: { + id: definition.id, + }, + }, + }); + }, + [definition?.id, streamsRepositoryClient] + ); + + return ( + + + { + setQuery(nextQuery.queryText); + }} + /> + + + + + + + {i18n.translate('xpack.streams.streamDetailAssetView.addAnAssetButtonLabel', { + defaultMessage: 'Add an asset', + })} + + + + ); +} diff --git a/x-pack/plugins/streams_app/public/components/stream_detail_view/index.tsx b/x-pack/plugins/streams_app/public/components/stream_detail_view/index.tsx index ebf72a58d32a8..c452c1b87600d 100644 --- a/x-pack/plugins/streams_app/public/components/stream_detail_view/index.tsx +++ b/x-pack/plugins/streams_app/public/components/stream_detail_view/index.tsx @@ -11,6 +11,7 @@ import { useStreamsAppParams } from '../../hooks/use_streams_app_params'; import { useStreamsAppFetch } from '../../hooks/use_streams_app_fetch'; import { useKibana } from '../../hooks/use_kibana'; import { StreamDetailOverview } from '../stream_detail_overview'; +import { StreamDetailAssetView } from '../stream_detail_asset_view'; export function StreamDetailView() { const { @@ -52,6 +53,13 @@ export function StreamDetailView() { defaultMessage: 'Overview', }), }, + { + name: 'assets', + content: , + label: i18n.translate('xpack.streams.streamDetailView.assetTab', { + defaultMessage: 'Assets', + }), + }, { name: 'management', content: <>, From ac426f558ae60345190abd15e8aef88af66bc773 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 21 Nov 2024 10:54:09 +0100 Subject: [PATCH 50/95] Add transform option --- .../client/create_observability_es_client.ts | 102 ++++++++++++------ .../routes/entities/get_latest_entity.ts | 19 ++-- .../routes/entities/get_entity_groups.ts | 12 ++- .../routes/entities/get_entity_types.ts | 17 ++- .../routes/entities/get_latest_entities.ts | 51 +++++---- .../server/routes/has_data/get_has_data.ts | 10 +- .../streams/server/routes/esql/route.ts | 6 +- 7 files changed, 139 insertions(+), 78 deletions(-) diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts index 40b9fcbb6a130..92c7b8d19e531 100644 --- a/x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts @@ -17,6 +17,8 @@ import { withSpan } from '@kbn/apm-utils'; import type { ElasticsearchClient, Logger } from '@kbn/core/server'; import type { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types'; import { Required, ValuesType } from 'utility-types'; +import { DedotObject } from '@kbn/utility-types'; +import { unflattenObject } from '@kbn/task-manager-plugin/server/metrics/lib'; import { esqlResultToPlainObjects } from '../esql_result_to_plain_objects'; type SearchRequest = ESSearchRequest & { @@ -25,33 +27,53 @@ type SearchRequest = ESSearchRequest & { size: number | boolean; }; -interface EsqlOptions { - asPlainObjects?: boolean; +export interface EsqlOptions { + transform?: 'none' | 'plain' | 'unflatten'; } -type EsqlValue = ScalarValue | ScalarValue[]; +export type EsqlValue = ScalarValue | ScalarValue[]; -type EsqlOutput = Record; +export type EsqlOutput = Record; -type InferEsqlResponseOf< +type MaybeUnflatten, TApply> = TApply extends true + ? DedotObject + : T; + +interface UnparsedEsqlResponseOf { + columns: Array<{ name: keyof TOutput; type: string }>; + values: Array>>; +} + +interface ParsedEsqlResponseOf< TOutput extends EsqlOutput, - TOptions extends EsqlOptions | undefined = { asPlainObjects: true } -> = TOptions extends { asPlainObjects: true } - ? { - hits: Array<{ + TOptions extends EsqlOptions | undefined = { transform: 'none' } +> { + hits: Array< + MaybeUnflatten< + { [key in keyof TOutput]: TOutput[key]; - }>; - } - : { - columns: Array<{ name: keyof TOutput; type: string }>; - values: Array>>; - }; - -export interface EsqlQueryResponse { - columns: Array<{ name: string; type: string }>; - values: EsqlValue[][]; + }, + TOptions extends { transform: 'unflatten' } ? true : false + > + >; } +export type InferEsqlResponseOf< + TOutput extends EsqlOutput, + TOptions extends EsqlOptions | undefined = { transform: 'none' } +> = TOptions extends { transform: 'plain' | 'unflatten' } + ? ParsedEsqlResponseOf + : UnparsedEsqlResponseOf; + +export type ObservabilityESSearchRequest = SearchRequest; + +export type ObservabilityEsQueryRequest = Omit; + +export type ParsedEsqlResponse = ParsedEsqlResponseOf; +export type UnparsedEsqlResponse = UnparsedEsqlResponseOf; + +export type EsqlQueryResponse = UnparsedEsqlResponse | ParsedEsqlResponse; + /** * An Elasticsearch Client with a fully typed `search` method and built-in * APM instrumentation. @@ -71,13 +93,17 @@ export interface ObservabilityElasticsearchClient { operationName: string, request: Required ): Promise; + esql( + operationName: string, + parameters: ObservabilityEsQueryRequest + ): Promise>; esql< TOutput extends EsqlOutput = EsqlOutput, - TEsqlOptions extends EsqlOptions | undefined = { asPlainObjects: true } + TEsqlOptions extends EsqlOptions = { transform: 'none' } >( operationName: string, - parameters: EsqlQueryRequest, - options?: TEsqlOptions + parameters: ObservabilityEsQueryRequest, + options: TEsqlOptions ): Promise>; client: ElasticsearchClient; } @@ -123,14 +149,11 @@ export function createObservabilityEsClient({ }); }); }, - esql< - TOutput extends EsqlOutput = EsqlOutput, - TEsqlOptions extends EsqlOptions | undefined = { asPlainObjects: true } - >( + esql( operationName: string, - parameters: EsqlQueryRequest, + parameters: ObservabilityEsQueryRequest, options?: EsqlOptions - ): Promise> { + ): Promise> { return callWithLogger(operationName, parameters, () => { return client.esql .query( @@ -142,15 +165,24 @@ export function createObservabilityEsClient({ } ) .then((response) => { - const esqlResponse = response as unknown as EsqlQueryResponse; + const esqlResponse = response as unknown as UnparsedEsqlResponseOf; + + const transform = options?.transform ?? 'none'; - const shouldParseOutput = options?.asPlainObjects !== false; - const finalResponse = shouldParseOutput - ? { hits: esqlResultToPlainObjects(esqlResponse) } - : esqlResponse; + if (transform === 'none') { + return esqlResponse; + } + + const parsedResponse = { hits: esqlResultToPlainObjects(esqlResponse) }; + + if (transform === 'plain') { + return parsedResponse; + } - return finalResponse as InferEsqlResponseOf; - }); + return { + hits: parsedResponse.hits.map((hit) => unflattenObject(hit)), + }; + }) as Promise>; }); }, search( diff --git a/x-pack/plugins/observability_solution/infra/server/routes/entities/get_latest_entity.ts b/x-pack/plugins/observability_solution/infra/server/routes/entities/get_latest_entity.ts index 9758b18580eb2..5a705aa93dca4 100644 --- a/x-pack/plugins/observability_solution/infra/server/routes/entities/get_latest_entity.ts +++ b/x-pack/plugins/observability_solution/infra/server/routes/entities/get_latest_entity.ts @@ -40,16 +40,23 @@ export async function getLatestEntity({ return undefined; } - const { hits } = await inventoryEsClient.esql<{ - 'source_data_stream.type': string | string[]; - }>('get_latest_entities', { - query: `FROM ${ENTITIES_LATEST_ALIAS} + const { hits } = await inventoryEsClient.esql< + { + 'source_data_stream.type': string | string[]; + }, + { transform: 'plain' } + >( + 'get_latest_entities', + { + query: `FROM ${ENTITIES_LATEST_ALIAS} | WHERE ${ENTITY_TYPE} == ? | WHERE ${hostOrContainerIdentityField} == ? | KEEP ${SOURCE_DATA_STREAM_TYPE} `, - params: [entityType, entityId], - }); + params: [entityType, entityId], + }, + { transform: 'plain' } + ); return { sourceDataStreamType: hits[0]?.['source_data_stream.type'] }; } diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_groups.ts b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_groups.ts index 9a52ec7f0228d..816b3c6af6ec2 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_groups.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_groups.ts @@ -31,10 +31,14 @@ export async function getEntityGroupsBy({ const limit = `LIMIT ${MAX_NUMBER_OF_ENTITIES}`; const query = [from, ...where, group, sort, limit].join(' | '); - const { hits } = await inventoryEsClient.esql('get_entities_groups', { - query, - filter: esQuery, - }); + const { hits } = await inventoryEsClient.esql( + 'get_entities_groups', + { + query, + filter: esQuery, + }, + { transform: 'plain' } + ); return hits; } diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_types.ts b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_types.ts index f5d400504007d..c1f7894a178b1 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_types.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_entity_types.ts @@ -15,14 +15,21 @@ export async function getEntityTypes({ }: { inventoryEsClient: ObservabilityElasticsearchClient; }) { - const entityTypesEsqlResponse = await inventoryEsClient.esql<{ - 'entity.type': string; - }>('get_entity_types', { - query: `FROM ${ENTITIES_LATEST_ALIAS} + const entityTypesEsqlResponse = await inventoryEsClient.esql< + { + 'entity.type': string; + }, + { transform: 'plain' } + >( + 'get_entity_types', + { + query: `FROM ${ENTITIES_LATEST_ALIAS} | ${getBuiltinEntityDefinitionIdESQLWhereClause()} | STATS count = COUNT(${ENTITY_TYPE}) BY ${ENTITY_TYPE} `, - }); + }, + { transform: 'plain' } + ); return entityTypesEsqlResponse.hits.map((response) => response['entity.type']); } diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts index c6a0d2d2d4973..5d88f85058edc 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts @@ -62,33 +62,40 @@ export async function getLatestEntities({ const query = [from, ...where, sort, limit].join(' | '); - const latestEntitiesEsqlResponse = await inventoryEsClient.esql<{ - 'entity.id': string; - 'entity.type': string; - 'entity.definition_id': string; - 'entity.display_name': string; - 'entity.identity_fields': string | string[]; - 'entity.last_seen_timestamp': string; - 'entity.definition_version': string; - 'entity.schema_version': string; - }>('get_latest_entities', { - query, - filter: esQuery, - params, - }); + const latestEntitiesEsqlResponse = await inventoryEsClient.esql< + { + 'entity.id': string; + 'entity.type': string; + 'entity.definition_id': string; + 'entity.display_name': string; + 'entity.identity_fields': string | string[]; + 'entity.last_seen_timestamp': string; + 'entity.definition_version': string; + 'entity.schema_version': string; + }, + { transform: 'unflatten' } + >( + 'get_latest_entities', + { + query, + filter: esQuery, + params, + }, + { transform: 'unflatten' } + ); return latestEntitiesEsqlResponse.hits.map((latestEntity) => { const { entity, ...metadata } = unflattenObject(latestEntity); return { - entityId: latestEntity['entity.id'], - entityType: latestEntity['entity.type'], - entityDefinitionId: latestEntity['entity.definition_id'], - entityDisplayName: latestEntity['entity.display_name'], - entityIdentityFields: latestEntity['entity.identity_fields'], - entityLastSeenTimestamp: latestEntity['entity.last_seen_timestamp'], - entityDefinitionVersion: latestEntity['entity.definition_version'], - entitySchemaVersion: latestEntity['entity.schema_version'], + entityId: entity.id, + entityType: entity.type, + entityDefinitionId: entity.definition_id, + entityDisplayName: entity.display_name, + entityIdentityFields: entity.identity_fields, + entityLastSeenTimestamp: entity.last_seen_timestamp, + entityDefinitionVersion: entity.definition_version, + entitySchemaVersion: entity.schema_version, ...metadata, }; }); diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/has_data/get_has_data.ts b/x-pack/plugins/observability_solution/inventory/server/routes/has_data/get_has_data.ts index 2f48202f31837..c3fd3971f09d2 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/has_data/get_has_data.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/has_data/get_has_data.ts @@ -17,12 +17,16 @@ export async function getHasData({ logger: Logger; }) { try { - const esqlResults = await inventoryEsClient.esql<{ _count: number }>('get_has_data', { - query: `FROM ${ENTITIES_LATEST_ALIAS} + const esqlResults = await inventoryEsClient.esql<{ _count: number }, { transform: 'plain' }>( + 'get_has_data', + { + query: `FROM ${ENTITIES_LATEST_ALIAS} | ${getBuiltinEntityDefinitionIdESQLWhereClause()} | STATS _count = COUNT(*) | LIMIT 1`, - }); + }, + { transform: 'plain' } + ); const totalCount = esqlResults.hits[0]._count; diff --git a/x-pack/plugins/streams/server/routes/esql/route.ts b/x-pack/plugins/streams/server/routes/esql/route.ts index c226e982cf402..780171bcfee24 100644 --- a/x-pack/plugins/streams/server/routes/esql/route.ts +++ b/x-pack/plugins/streams/server/routes/esql/route.ts @@ -9,7 +9,7 @@ import { excludeFrozenQuery } from '@kbn/observability-utils-common/es/queries/e import { kqlQuery } from '@kbn/observability-utils-common/es/queries/kql_query'; import { rangeQuery } from '@kbn/observability-utils-common/es/queries/range_query'; import { - EsqlQueryResponse, + ParsedEsqlResponse, createObservabilityEsClient, } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; import { z } from '@kbn/zod'; @@ -28,7 +28,7 @@ export const executeEsqlRoute = createServerRoute({ end: z.number().optional(), }), }), - handler: async ({ params, request, logger, getScopedClients }): Promise => { + handler: async ({ params, request, logger, getScopedClients }): Promise => { const { scopedClusterClient } = await getScopedClients({ request }); const observabilityEsClient = createObservabilityEsClient({ client: scopedClusterClient.asCurrentUser, @@ -55,7 +55,7 @@ export const executeEsqlRoute = createServerRoute({ }, }, }, - { asPlainObjects: false } + { transform: 'plain' } ); return response; From 97510ca229ab366bb78eb662fb7889beebd5d2bb Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 21 Nov 2024 10:05:41 +0000 Subject: [PATCH 51/95] [CI] Auto-commit changed files from 'node scripts/notice' --- .../observability_utils_server/tsconfig.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/tsconfig.json b/x-pack/packages/observability/observability_utils/observability_utils_server/tsconfig.json index f51d93089c627..f6dd781184b86 100644 --- a/x-pack/packages/observability/observability_utils/observability_utils_server/tsconfig.json +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/tsconfig.json @@ -24,5 +24,7 @@ "@kbn/alerting-plugin", "@kbn/rule-registry-plugin", "@kbn/rule-data-utils", + "@kbn/utility-types", + "@kbn/task-manager-plugin", ] } From a950210f51c059ca591333d056b87ef2a069ac29 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 21 Nov 2024 11:21:15 +0100 Subject: [PATCH 52/95] Fix types in investigate_app --- .../investigate_app/server/routes/types.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/investigate_app/server/routes/types.ts b/x-pack/plugins/observability_solution/investigate_app/server/routes/types.ts index afb022cdc9b7f..ffb3bd1351bca 100644 --- a/x-pack/plugins/observability_solution/investigate_app/server/routes/types.ts +++ b/x-pack/plugins/observability_solution/investigate_app/server/routes/types.ts @@ -57,6 +57,5 @@ export interface InvestigateAppRouteCreateOptions { timeout?: { idleSocket?: number; }; - tags: []; }; } From d2ba6577cbe9b54486a877427cc9e9b9e5cf1b36 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 21 Nov 2024 11:53:53 +0100 Subject: [PATCH 53/95] Fix jest.config.js --- x-pack/plugins/streams_app/jest.config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/streams_app/jest.config.js b/x-pack/plugins/streams_app/jest.config.js index 4bf7ed18f8e8c..d9c01c40a322d 100644 --- a/x-pack/plugins/streams_app/jest.config.js +++ b/x-pack/plugins/streams_app/jest.config.js @@ -7,7 +7,7 @@ module.exports = { preset: '@kbn/test', - rootDir: '../../../..', + rootDir: '../../..', roots: [ '/x-pack/plugins/streams_app/public', '/x-pack/plugins/streams_app/common', From 12507230587c26561abfd768bc73a4d49dc4c9cd Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 22 Nov 2024 11:33:23 +0000 Subject: [PATCH 54/95] [CI] Auto-commit changed files from 'node scripts/yarn_deduplicate' --- x-pack/plugins/observability_solution/slo/tsconfig.json | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/slo/tsconfig.json b/x-pack/plugins/observability_solution/slo/tsconfig.json index be74e370a1fc1..3b6a6984d33ad 100644 --- a/x-pack/plugins/observability_solution/slo/tsconfig.json +++ b/x-pack/plugins/observability_solution/slo/tsconfig.json @@ -100,6 +100,5 @@ "@kbn/observability-alerting-rule-utils", "@kbn/discover-shared-plugin", "@kbn/server-route-repository-client", - "@kbn/server-route-repository-utils" ] } From 9a487e41eca232befd8af228618f722d10f62449 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Fri, 22 Nov 2024 12:41:42 +0100 Subject: [PATCH 55/95] Use top-level security property --- packages/kbn-server-route-repository/src/register_routes.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-server-route-repository/src/register_routes.ts b/packages/kbn-server-route-repository/src/register_routes.ts index 36b840e0fe145..6201ffcd869ea 100644 --- a/packages/kbn-server-route-repository/src/register_routes.ts +++ b/packages/kbn-server-route-repository/src/register_routes.ts @@ -164,7 +164,7 @@ export function registerRoutes>({ access, // @ts-expect-error we are essentially calling multiple methods at the same type so TS gets confused options: omit(options, 'access', 'description', 'summary', 'deprecated', 'discontinued'), - security: options.security, + security, }).addVersion( { version, From 5245055416917c18e94d34bd06e7d5e7abd6387b Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Fri, 22 Nov 2024 13:29:17 +0100 Subject: [PATCH 56/95] Fix types in onboarding --- .../observability_onboarding/server/plugin.ts | 20 ++- .../server/routes/register_routes.ts | 126 ------------------ .../server/routes/types.ts | 6 +- 3 files changed, 16 insertions(+), 136 deletions(-) delete mode 100644 x-pack/plugins/observability_solution/observability_onboarding/server/routes/register_routes.ts diff --git a/x-pack/plugins/observability_solution/observability_onboarding/server/plugin.ts b/x-pack/plugins/observability_solution/observability_onboarding/server/plugin.ts index ccb260a002cf2..993f211f48434 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/server/plugin.ts +++ b/x-pack/plugins/observability_solution/observability_onboarding/server/plugin.ts @@ -14,8 +14,8 @@ import type { } from '@kbn/core/server'; import { mapValues } from 'lodash'; import { i18n } from '@kbn/i18n'; +import { registerRoutes } from '@kbn/server-route-repository'; import { getObservabilityOnboardingServerRouteRepository } from './routes'; -import { registerRoutes } from './routes/register_routes'; import { ObservabilityOnboardingRouteHandlerResources } from './routes/types'; import { ObservabilityOnboardingPluginSetup, @@ -71,16 +71,24 @@ export class ObservabilityOnboardingPlugin }) as ObservabilityOnboardingRouteHandlerResources['plugins']; const config = this.initContext.config.get(); - registerRoutes({ - core, - logger: this.logger, - repository: getObservabilityOnboardingServerRouteRepository(), - plugins: resourcePlugins, + + const dependencies: Omit< + ObservabilityOnboardingRouteHandlerResources, + 'core' | 'logger' | 'request' | 'context' | 'response' + > = { config, kibanaVersion: this.initContext.env.packageInfo.version, + plugins: resourcePlugins, services: { esLegacyConfigService: this.esLegacyConfigService, }, + }; + + registerRoutes({ + core, + logger: this.logger, + repository: getObservabilityOnboardingServerRouteRepository(), + dependencies, }); plugins.customIntegrations.registerCustomIntegration({ diff --git a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/register_routes.ts b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/register_routes.ts deleted file mode 100644 index 8fe51623510eb..0000000000000 --- a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/register_routes.ts +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { errors } from '@elastic/elasticsearch'; -import Boom from '@hapi/boom'; -import type { IKibanaResponse } from '@kbn/core/server'; -import { CoreSetup, Logger, RouteRegistrar } from '@kbn/core/server'; -import { - IoTsParamsObject, - ServerRouteRepository, - decodeRequestParams, - stripNullishRequestParameters, - parseEndpoint, - passThroughValidationObject, -} from '@kbn/server-route-repository'; -import * as t from 'io-ts'; -import { ObservabilityOnboardingConfig } from '..'; -import { EsLegacyConfigService } from '../services/es_legacy_config_service'; -import { ObservabilityOnboardingRequestHandlerContext } from '../types'; -import { ObservabilityOnboardingRouteHandlerResources } from './types'; - -interface RegisterRoutes { - core: CoreSetup; - repository: ServerRouteRepository; - logger: Logger; - plugins: ObservabilityOnboardingRouteHandlerResources['plugins']; - config: ObservabilityOnboardingConfig; - kibanaVersion: string; - services: { - esLegacyConfigService: EsLegacyConfigService; - }; -} - -export function registerRoutes({ - repository, - core, - logger, - plugins, - config, - kibanaVersion, - services, -}: RegisterRoutes) { - const routes = Object.values(repository); - - const router = core.http.createRouter(); - - routes.forEach((route) => { - const { endpoint, options, handler, params } = route; - const { pathname, method } = parseEndpoint(endpoint); - - (router[method] as RouteRegistrar)( - { - path: pathname, - validate: passThroughValidationObject, - options, - }, - async (context, request, response) => { - try { - const decodedParams = decodeRequestParams( - stripNullishRequestParameters({ - params: request.params, - body: request.body, - query: request.query, - }), - (params as IoTsParamsObject) ?? t.strict({}) - ); - - const data = (await handler({ - context, - request, - response, - logger, - params: decodedParams, - plugins, - core: { - setup: core, - start: async () => { - const [coreStart] = await core.getStartServices(); - return coreStart; - }, - }, - config, - kibanaVersion, - services, - })) as any; - - if (data === undefined) { - return response.noContent(); - } - - if (data instanceof response.noContent().constructor) { - return data as IKibanaResponse; - } - - return response.ok({ body: data }); - } catch (error) { - if (Boom.isBoom(error)) { - logger.error(error.output.payload.message); - return response.customError({ - statusCode: error.output.statusCode, - body: { message: error.output.payload.message }, - }); - } - - logger.error(error); - const opts = { - statusCode: 500, - body: { - message: error.message, - }, - }; - - if (error instanceof errors.RequestAbortedError) { - opts.statusCode = 499; - opts.body.message = 'Client closed request'; - } - - return response.customError(opts); - } - } - ); - }); -} diff --git a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/types.ts b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/types.ts index 4b35272eaa330..689ab14739818 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/types.ts +++ b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/types.ts @@ -46,10 +46,8 @@ export interface ObservabilityOnboardingRouteHandlerResources { } export interface ObservabilityOnboardingRouteCreateOptions { - options: { - tags: string[]; - xsrfRequired?: boolean; - }; + tags: string[]; + xsrfRequired?: boolean; } export const IntegrationRT = t.intersection([ From 90389fc59d56f6ae6a144553cd41f7c28b914949 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Fri, 22 Nov 2024 15:33:44 +0100 Subject: [PATCH 57/95] Fix type issues in Streams app --- x-pack/plugins/streams/server/routes/esql/route.ts | 6 +++--- .../public/components/esql_chart/controlled_esql_chart.tsx | 4 ++-- .../streams_app/public/util/esql_result_to_timeseries.ts | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/streams/server/routes/esql/route.ts b/x-pack/plugins/streams/server/routes/esql/route.ts index 780171bcfee24..0e0e41eee3c7e 100644 --- a/x-pack/plugins/streams/server/routes/esql/route.ts +++ b/x-pack/plugins/streams/server/routes/esql/route.ts @@ -9,7 +9,7 @@ import { excludeFrozenQuery } from '@kbn/observability-utils-common/es/queries/e import { kqlQuery } from '@kbn/observability-utils-common/es/queries/kql_query'; import { rangeQuery } from '@kbn/observability-utils-common/es/queries/range_query'; import { - ParsedEsqlResponse, + UnparsedEsqlResponse, createObservabilityEsClient, } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; import { z } from '@kbn/zod'; @@ -28,7 +28,7 @@ export const executeEsqlRoute = createServerRoute({ end: z.number().optional(), }), }), - handler: async ({ params, request, logger, getScopedClients }): Promise => { + handler: async ({ params, request, logger, getScopedClients }): Promise => { const { scopedClusterClient } = await getScopedClients({ request }); const observabilityEsClient = createObservabilityEsClient({ client: scopedClusterClient.asCurrentUser, @@ -55,7 +55,7 @@ export const executeEsqlRoute = createServerRoute({ }, }, }, - { transform: 'plain' } + { transform: 'none' } ); return response; diff --git a/x-pack/plugins/streams_app/public/components/esql_chart/controlled_esql_chart.tsx b/x-pack/plugins/streams_app/public/components/esql_chart/controlled_esql_chart.tsx index 535a60266b4fe..9008f8ee47098 100644 --- a/x-pack/plugins/streams_app/public/components/esql_chart/controlled_esql_chart.tsx +++ b/x-pack/plugins/streams_app/public/components/esql_chart/controlled_esql_chart.tsx @@ -23,7 +23,7 @@ import { i18n } from '@kbn/i18n'; import { getTimeZone } from '@kbn/observability-utils-browser/utils/ui_settings/get_timezone'; import { css } from '@emotion/css'; import { AbortableAsyncState } from '@kbn/observability-utils-browser/hooks/use_abortable_async'; -import type { EsqlQueryResponse } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; +import type { UnparsedEsqlResponse } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; import { esqlResultToTimeseries } from '../../util/esql_result_to_timeseries'; import { useKibana } from '../../hooks/use_kibana'; import { LoadingPanel } from '../loading_panel'; @@ -52,7 +52,7 @@ export function ControlledEsqlChart({ height, }: { id: string; - result: AbortableAsyncState; + result: AbortableAsyncState; metricNames: T[]; chartType?: 'area' | 'bar' | 'line'; height: number; diff --git a/x-pack/plugins/streams_app/public/util/esql_result_to_timeseries.ts b/x-pack/plugins/streams_app/public/util/esql_result_to_timeseries.ts index f5af978922ffb..c32bbf89135bd 100644 --- a/x-pack/plugins/streams_app/public/util/esql_result_to_timeseries.ts +++ b/x-pack/plugins/streams_app/public/util/esql_result_to_timeseries.ts @@ -5,7 +5,7 @@ * 2.0. */ import type { AbortableAsyncState } from '@kbn/observability-utils-browser/hooks/use_abortable_async'; -import type { EsqlQueryResponse } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; +import type { UnparsedEsqlResponse } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; import { orderBy } from 'lodash'; interface Timeseries { @@ -19,7 +19,7 @@ export function esqlResultToTimeseries({ result, metricNames, }: { - result: AbortableAsyncState; + result: AbortableAsyncState; metricNames: T[]; }): Array> { const columns = result.value?.columns; From 76d204201bfbeba5468d1b5f8217f778d678cedd Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Sat, 23 Nov 2024 11:12:59 +0100 Subject: [PATCH 58/95] Add tests --- .../src/register_routes.test.ts | 65 +++++++++++++++---- 1 file changed, 52 insertions(+), 13 deletions(-) diff --git a/packages/kbn-server-route-repository/src/register_routes.test.ts b/packages/kbn-server-route-repository/src/register_routes.test.ts index 249f81df3a8a9..b13592c57ba59 100644 --- a/packages/kbn-server-route-repository/src/register_routes.test.ts +++ b/packages/kbn-server-route-repository/src/register_routes.test.ts @@ -15,6 +15,7 @@ import { NEVER } from 'rxjs'; import * as makeZodValidationObject from './make_zod_validation_object'; import { registerRoutes } from './register_routes'; import { passThroughValidationObject, noParamsValidationObject } from './validation_objects'; +import { ServerRouteRepository } from '@kbn/server-route-repository-utils'; describe('registerRoutes', () => { const post = jest.fn(); @@ -54,44 +55,82 @@ describe('registerRoutes', () => { 'POST /internal/route': { endpoint: 'POST /internal/route', handler: jest.fn(), - options: { - internal: true, - }, }, 'POST /api/public_route version': { endpoint: 'POST /api/public_route version', handler: jest.fn(), + }, + 'POST /api/internal_but_looks_like_public version': { + endpoint: 'POST /api/internal_but_looks_like_public version', options: { - public: true, + access: 'internal', }, + handler: jest.fn(), }, - }); + 'POST /internal/route_with_security': { + endpoint: `POST /internal/route_with_security`, + handler: jest.fn(), + security: { + authz: { + enabled: false, + reason: 'whatever', + }, + }, + }, + 'POST /api/route_with_security version': { + endpoint: `POST /api/route_with_security version`, + handler: jest.fn(), + security: { + authz: { + enabled: false, + reason: 'whatever', + }, + }, + }, + } satisfies ServerRouteRepository); expect(createRouter).toHaveBeenCalledTimes(1); - expect(post).toHaveBeenCalledTimes(1); - const [internalRoute] = post.mock.calls[0]; expect(internalRoute.path).toEqual('/internal/route'); expect(internalRoute.options).toEqual({ - internal: true, + access: 'internal', }); expect(internalRoute.validate).toEqual(noParamsValidationObject); - expect(postWithVersion).toHaveBeenCalledTimes(1); + const [internalRouteWithSecurity] = post.mock.calls[1]; + + expect(internalRouteWithSecurity.path).toEqual('/internal/route_with_security'); + expect(internalRouteWithSecurity.security).toEqual({ + authz: { + enabled: false, + reason: 'whatever', + }, + }); + const [publicRoute] = postWithVersion.mock.calls[0]; expect(publicRoute.path).toEqual('/api/public_route'); - expect(publicRoute.options).toEqual({ - public: true, - }); expect(publicRoute.access).toEqual('public'); - expect(postAddVersion).toHaveBeenCalledTimes(1); + const [apiInternalRoute] = postWithVersion.mock.calls[1]; + expect(apiInternalRoute.path).toEqual('/api/internal_but_looks_like_public'); + expect(apiInternalRoute.access).toEqual('internal'); + const [versionedRoute] = postAddVersion.mock.calls[0]; expect(versionedRoute.version).toEqual('version'); expect(versionedRoute.validate).toEqual({ request: noParamsValidationObject, }); + + const [publicRouteWithSecurity] = postWithVersion.mock.calls[2]; + + expect(publicRouteWithSecurity.path).toEqual('/api/route_with_security'); + expect(publicRouteWithSecurity.security).toEqual({ + authz: { + enabled: false, + reason: 'whatever', + }, + }); }); it('does not allow any params if no schema is provided', () => { From e12aaece2c5d14d7f336dc6ed4084faa385f2fa2 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Sat, 23 Nov 2024 11:46:23 +0100 Subject: [PATCH 59/95] Migrate options.security, fix issues with routing --- .../src/register_routes.ts | 2 +- .../src/breadcrumbs/index.tsx | 1 + .../src/router_provider.tsx | 5 +---- .../entity_manager/server/routes/entities/create.ts | 12 +++++------- .../entity_manager/server/routes/entities/delete.ts | 12 +++++------- .../entity_manager/server/routes/entities/get.ts | 12 +++++------- .../entity_manager/server/routes/entities/reset.ts | 12 +++++------- .../entity_manager/server/routes/entities/update.ts | 12 +++++------- .../streams_app/public/components/app_root/index.tsx | 10 ++++++++-- 9 files changed, 36 insertions(+), 42 deletions(-) diff --git a/packages/kbn-server-route-repository/src/register_routes.ts b/packages/kbn-server-route-repository/src/register_routes.ts index 6201ffcd869ea..a0438529f05bd 100644 --- a/packages/kbn-server-route-repository/src/register_routes.ts +++ b/packages/kbn-server-route-repository/src/register_routes.ts @@ -150,7 +150,7 @@ export function registerRoutes>({ path: pathname, // @ts-expect-error we are essentially calling multiple methods at the same type so TS gets confused options: { - ...options, + ...omit(options, 'security'), access, }, security, diff --git a/packages/kbn-typed-react-router-config/src/breadcrumbs/index.tsx b/packages/kbn-typed-react-router-config/src/breadcrumbs/index.tsx index 6a01f9a1ef01c..721ba433cc01b 100644 --- a/packages/kbn-typed-react-router-config/src/breadcrumbs/index.tsx +++ b/packages/kbn-typed-react-router-config/src/breadcrumbs/index.tsx @@ -9,3 +9,4 @@ export { createRouterBreadcrumbComponent } from './create_router_breadcrumb_component'; export { createUseBreadcrumbs } from './use_router_breadcrumb'; +export { BreadcrumbsContextProvider } from './context'; diff --git a/packages/kbn-typed-react-router-config/src/router_provider.tsx b/packages/kbn-typed-react-router-config/src/router_provider.tsx index 7608cb35fd6eb..329bf46046769 100644 --- a/packages/kbn-typed-react-router-config/src/router_provider.tsx +++ b/packages/kbn-typed-react-router-config/src/router_provider.tsx @@ -13,7 +13,6 @@ import { Router as ReactRouter } from '@kbn/shared-ux-router'; import { RouteMap, Router } from './types'; import { RouterContextProvider } from './use_router'; -import { BreadcrumbsContextProvider } from './breadcrumbs/context'; export function RouterProvider({ children, @@ -26,9 +25,7 @@ export function RouterProvider({ }) { return ( - - {children} - + {children} ); } diff --git a/x-pack/plugins/entity_manager/server/routes/entities/create.ts b/x-pack/plugins/entity_manager/server/routes/entities/create.ts index a22916f3e69f7..fc0f4c6e9a1ed 100644 --- a/x-pack/plugins/entity_manager/server/routes/entities/create.ts +++ b/x-pack/plugins/entity_manager/server/routes/entities/create.ts @@ -50,13 +50,11 @@ import { canManageEntityDefinition } from '../../lib/auth'; */ export const createEntityDefinitionRoute = createEntityManagerServerRoute({ endpoint: 'POST /internal/entities/definition', - options: { - security: { - authz: { - enabled: false, - reason: - 'This endpoint mainly manages Elasticsearch resources using the requesting users credentials', - }, + security: { + authz: { + enabled: false, + reason: + 'This endpoint mainly manages Elasticsearch resources using the requesting users credentials', }, }, params: z.object({ diff --git a/x-pack/plugins/entity_manager/server/routes/entities/delete.ts b/x-pack/plugins/entity_manager/server/routes/entities/delete.ts index ff5b9624dbb3c..ec5b4ada3039f 100644 --- a/x-pack/plugins/entity_manager/server/routes/entities/delete.ts +++ b/x-pack/plugins/entity_manager/server/routes/entities/delete.ts @@ -52,13 +52,11 @@ import { canDeleteEntityDefinition } from '../../lib/auth/privileges'; */ export const deleteEntityDefinitionRoute = createEntityManagerServerRoute({ endpoint: 'DELETE /internal/entities/definition/{id}', - options: { - security: { - authz: { - enabled: false, - reason: - 'This endpoint mainly manages Elasticsearch resources using the requesting users credentials', - }, + security: { + authz: { + enabled: false, + reason: + 'This endpoint mainly manages Elasticsearch resources using the requesting users credentials', }, }, params: z.object({ diff --git a/x-pack/plugins/entity_manager/server/routes/entities/get.ts b/x-pack/plugins/entity_manager/server/routes/entities/get.ts index f22e0890e60ad..738ed0f440643 100644 --- a/x-pack/plugins/entity_manager/server/routes/entities/get.ts +++ b/x-pack/plugins/entity_manager/server/routes/entities/get.ts @@ -50,13 +50,11 @@ import { createEntityManagerServerRoute } from '../create_entity_manager_server_ */ export const getEntityDefinitionRoute = createEntityManagerServerRoute({ endpoint: 'GET /internal/entities/definition/{id?}', - options: { - security: { - authz: { - enabled: false, - reason: - 'This endpoint mainly manages Elasticsearch resources using the requesting users credentials', - }, + security: { + authz: { + enabled: false, + reason: + 'This endpoint mainly manages Elasticsearch resources using the requesting users credentials', }, }, params: z.object({ diff --git a/x-pack/plugins/entity_manager/server/routes/entities/reset.ts b/x-pack/plugins/entity_manager/server/routes/entities/reset.ts index ab4ba29fa1483..5da7a608aed84 100644 --- a/x-pack/plugins/entity_manager/server/routes/entities/reset.ts +++ b/x-pack/plugins/entity_manager/server/routes/entities/reset.ts @@ -25,13 +25,11 @@ import { stopTransforms } from '../../lib/entities/stop_transforms'; export const resetEntityDefinitionRoute = createEntityManagerServerRoute({ endpoint: 'POST /internal/entities/definition/{id}/_reset', - options: { - security: { - authz: { - enabled: false, - reason: - 'This endpoint mainly manages Elasticsearch resources using the requesting users credentials', - }, + security: { + authz: { + enabled: false, + reason: + 'This endpoint mainly manages Elasticsearch resources using the requesting users credentials', }, }, params: z.object({ diff --git a/x-pack/plugins/entity_manager/server/routes/entities/update.ts b/x-pack/plugins/entity_manager/server/routes/entities/update.ts index f1118028cda93..4f23486375f92 100644 --- a/x-pack/plugins/entity_manager/server/routes/entities/update.ts +++ b/x-pack/plugins/entity_manager/server/routes/entities/update.ts @@ -54,13 +54,11 @@ import { canManageEntityDefinition } from '../../lib/auth'; */ export const updateEntityDefinitionRoute = createEntityManagerServerRoute({ endpoint: 'PATCH /internal/entities/definition/{id}', - options: { - security: { - authz: { - enabled: false, - reason: - 'This endpoint mainly manages Elasticsearch resources using the requesting users credentials', - }, + security: { + authz: { + enabled: false, + reason: + 'This endpoint mainly manages Elasticsearch resources using the requesting users credentials', }, }, params: z.object({ diff --git a/x-pack/plugins/streams_app/public/components/app_root/index.tsx b/x-pack/plugins/streams_app/public/components/app_root/index.tsx index d92891e017f96..c5e8b78ae2155 100644 --- a/x-pack/plugins/streams_app/public/components/app_root/index.tsx +++ b/x-pack/plugins/streams_app/public/components/app_root/index.tsx @@ -8,7 +8,11 @@ import { RedirectAppLinks } from '@kbn/shared-ux-link-redirect-app'; import React from 'react'; import { type AppMountParameters, type CoreStart } from '@kbn/core/public'; -import { RouteRenderer, RouterProvider } from '@kbn/typed-react-router-config'; +import { + BreadcrumbsContextProvider, + RouteRenderer, + RouterProvider, +} from '@kbn/typed-react-router-config'; import { HeaderMenuPortal } from '@kbn/observability-shared-plugin/public'; import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; import { StreamsAppContextProvider } from '../streams_app_context_provider'; @@ -40,7 +44,9 @@ export function AppRoot({ - + + + From 1baac8f853ce4d548ee1f0e4f6e172242fc5fd7c Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Sat, 23 Nov 2024 11:54:33 +0100 Subject: [PATCH 60/95] Migrate options.security --- .../src/register_routes.ts | 2 +- .../plugins/streams/server/routes/streams/delete.ts | 12 ++++++------ .../plugins/streams/server/routes/streams/disable.ts | 8 ++++---- x-pack/plugins/streams/server/routes/streams/edit.ts | 12 ++++++------ .../plugins/streams/server/routes/streams/enable.ts | 12 ++++++------ x-pack/plugins/streams/server/routes/streams/fork.ts | 12 ++++++------ x-pack/plugins/streams/server/routes/streams/list.ts | 12 ++++++------ x-pack/plugins/streams/server/routes/streams/read.ts | 12 ++++++------ .../plugins/streams/server/routes/streams/resync.ts | 12 ++++++------ .../streams/server/routes/streams/settings.ts | 8 ++++---- 10 files changed, 51 insertions(+), 51 deletions(-) diff --git a/packages/kbn-server-route-repository/src/register_routes.ts b/packages/kbn-server-route-repository/src/register_routes.ts index a0438529f05bd..6201ffcd869ea 100644 --- a/packages/kbn-server-route-repository/src/register_routes.ts +++ b/packages/kbn-server-route-repository/src/register_routes.ts @@ -150,7 +150,7 @@ export function registerRoutes>({ path: pathname, // @ts-expect-error we are essentially calling multiple methods at the same type so TS gets confused options: { - ...omit(options, 'security'), + ...options, access, }, security, diff --git a/x-pack/plugins/streams/server/routes/streams/delete.ts b/x-pack/plugins/streams/server/routes/streams/delete.ts index 456947a49c066..369568ff9b7f0 100644 --- a/x-pack/plugins/streams/server/routes/streams/delete.ts +++ b/x-pack/plugins/streams/server/routes/streams/delete.ts @@ -24,12 +24,12 @@ export const deleteStreamRoute = createServerRoute({ endpoint: 'DELETE /api/streams/{id}', options: { access: 'internal', - security: { - authz: { - enabled: false, - reason: - 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', - }, + }, + security: { + authz: { + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', }, }, params: z.object({ diff --git a/x-pack/plugins/streams/server/routes/streams/disable.ts b/x-pack/plugins/streams/server/routes/streams/disable.ts index 9cc1324a6b9d3..b760b58f1fafd 100644 --- a/x-pack/plugins/streams/server/routes/streams/disable.ts +++ b/x-pack/plugins/streams/server/routes/streams/disable.ts @@ -16,10 +16,10 @@ export const disableStreamsRoute = createServerRoute({ params: z.object({}), options: { access: 'internal', - security: { - authz: { - requiredPrivileges: ['streams_write'], - }, + }, + security: { + authz: { + requiredPrivileges: ['streams_write'], }, }, handler: async ({ diff --git a/x-pack/plugins/streams/server/routes/streams/edit.ts b/x-pack/plugins/streams/server/routes/streams/edit.ts index 26b02cebf1976..378f1ba3c8f01 100644 --- a/x-pack/plugins/streams/server/routes/streams/edit.ts +++ b/x-pack/plugins/streams/server/routes/streams/edit.ts @@ -32,12 +32,12 @@ export const editStreamRoute = createServerRoute({ endpoint: 'PUT /api/streams/{id}', options: { access: 'internal', - security: { - authz: { - enabled: false, - reason: - 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', - }, + }, + security: { + authz: { + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', }, }, params: z.object({ diff --git a/x-pack/plugins/streams/server/routes/streams/enable.ts b/x-pack/plugins/streams/server/routes/streams/enable.ts index 656cea4e28201..e163c6cbc8bb2 100644 --- a/x-pack/plugins/streams/server/routes/streams/enable.ts +++ b/x-pack/plugins/streams/server/routes/streams/enable.ts @@ -18,12 +18,12 @@ export const enableStreamsRoute = createServerRoute({ params: z.object({}), options: { access: 'internal', - security: { - authz: { - enabled: false, - reason: - 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', - }, + }, + security: { + authz: { + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', }, }, handler: async ({ diff --git a/x-pack/plugins/streams/server/routes/streams/fork.ts b/x-pack/plugins/streams/server/routes/streams/fork.ts index 3530023143df5..8294aebb0b0e9 100644 --- a/x-pack/plugins/streams/server/routes/streams/fork.ts +++ b/x-pack/plugins/streams/server/routes/streams/fork.ts @@ -23,12 +23,12 @@ export const forkStreamsRoute = createServerRoute({ endpoint: 'POST /api/streams/{id}/_fork', options: { access: 'internal', - security: { - authz: { - enabled: false, - reason: - 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', - }, + }, + security: { + authz: { + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', }, }, params: z.object({ diff --git a/x-pack/plugins/streams/server/routes/streams/list.ts b/x-pack/plugins/streams/server/routes/streams/list.ts index d50c4f98b9d48..bd6a5200fe9ba 100644 --- a/x-pack/plugins/streams/server/routes/streams/list.ts +++ b/x-pack/plugins/streams/server/routes/streams/list.ts @@ -16,12 +16,12 @@ export const listStreamsRoute = createServerRoute({ endpoint: 'GET /api/streams', options: { access: 'internal', - security: { - authz: { - enabled: false, - reason: - 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', - }, + }, + security: { + authz: { + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', }, }, params: z.object({}), diff --git a/x-pack/plugins/streams/server/routes/streams/read.ts b/x-pack/plugins/streams/server/routes/streams/read.ts index 5c8e2aba1b7d5..b9d21ef25b673 100644 --- a/x-pack/plugins/streams/server/routes/streams/read.ts +++ b/x-pack/plugins/streams/server/routes/streams/read.ts @@ -16,12 +16,12 @@ export const readStreamRoute = createServerRoute({ endpoint: 'GET /api/streams/{id}', options: { access: 'internal', - security: { - authz: { - enabled: false, - reason: - 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', - }, + }, + security: { + authz: { + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', }, }, params: z.object({ diff --git a/x-pack/plugins/streams/server/routes/streams/resync.ts b/x-pack/plugins/streams/server/routes/streams/resync.ts index a45552fecea60..e34bd290108a1 100644 --- a/x-pack/plugins/streams/server/routes/streams/resync.ts +++ b/x-pack/plugins/streams/server/routes/streams/resync.ts @@ -13,12 +13,12 @@ export const resyncStreamsRoute = createServerRoute({ endpoint: 'POST /api/streams/_resync', options: { access: 'internal', - security: { - authz: { - enabled: false, - reason: - 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', - }, + }, + security: { + authz: { + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', }, }, params: z.object({}), diff --git a/x-pack/plugins/streams/server/routes/streams/settings.ts b/x-pack/plugins/streams/server/routes/streams/settings.ts index 4cff13c41fc0d..6e133b3948dd0 100644 --- a/x-pack/plugins/streams/server/routes/streams/settings.ts +++ b/x-pack/plugins/streams/server/routes/streams/settings.ts @@ -12,10 +12,10 @@ export const getStreamsStatusRoute = createServerRoute({ endpoint: 'GET /api/streams/_status', options: { access: 'internal', - security: { - authz: { - requiredPrivileges: ['streams_read'], - }, + }, + security: { + authz: { + requiredPrivileges: ['streams_read'], }, }, handler: async ({ request, getScopedClients }): Promise<{ enabled: boolean }> => { From 3947668a7d358a2955edaebd129d25861dc3269b Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Sat, 23 Nov 2024 12:27:33 +0100 Subject: [PATCH 61/95] Migrate options.security for entityManager --- .../entity_manager/server/routes/enablement/check.ts | 12 +++++------- .../server/routes/enablement/disable.ts | 12 +++++------- .../server/routes/enablement/enable.ts | 12 +++++------- .../plugins/streams/server/routes/streams/resync.ts | 1 + 4 files changed, 16 insertions(+), 21 deletions(-) diff --git a/x-pack/plugins/entity_manager/server/routes/enablement/check.ts b/x-pack/plugins/entity_manager/server/routes/enablement/check.ts index 5373ac9df50f5..100b0ac382dcf 100644 --- a/x-pack/plugins/entity_manager/server/routes/enablement/check.ts +++ b/x-pack/plugins/entity_manager/server/routes/enablement/check.ts @@ -45,13 +45,11 @@ import { createEntityManagerServerRoute } from '../create_entity_manager_server_ */ export const checkEntityDiscoveryEnabledRoute = createEntityManagerServerRoute({ endpoint: 'GET /internal/entities/managed/enablement', - options: { - security: { - authz: { - enabled: false, - reason: - 'This endpoint leverages the security plugin to evaluate the privileges needed as part of its core flow', - }, + security: { + authz: { + enabled: false, + reason: + 'This endpoint leverages the security plugin to evaluate the privileges needed as part of its core flow', }, }, handler: async ({ response, logger, server }) => { diff --git a/x-pack/plugins/entity_manager/server/routes/enablement/disable.ts b/x-pack/plugins/entity_manager/server/routes/enablement/disable.ts index a71a317045c44..1c8755c682f7f 100644 --- a/x-pack/plugins/entity_manager/server/routes/enablement/disable.ts +++ b/x-pack/plugins/entity_manager/server/routes/enablement/disable.ts @@ -44,13 +44,11 @@ import { createEntityManagerServerRoute } from '../create_entity_manager_server_ */ export const disableEntityDiscoveryRoute = createEntityManagerServerRoute({ endpoint: 'DELETE /internal/entities/managed/enablement', - options: { - security: { - authz: { - enabled: false, - reason: - 'This endpoint leverages the security plugin to evaluate the privileges needed as part of its core flow', - }, + security: { + authz: { + enabled: false, + reason: + 'This endpoint leverages the security plugin to evaluate the privileges needed as part of its core flow', }, }, params: z.object({ diff --git a/x-pack/plugins/entity_manager/server/routes/enablement/enable.ts b/x-pack/plugins/entity_manager/server/routes/enablement/enable.ts index 562b798a598a6..6ddb65804b90f 100644 --- a/x-pack/plugins/entity_manager/server/routes/enablement/enable.ts +++ b/x-pack/plugins/entity_manager/server/routes/enablement/enable.ts @@ -63,13 +63,11 @@ import { startTransforms } from '../../lib/entities/start_transforms'; */ export const enableEntityDiscoveryRoute = createEntityManagerServerRoute({ endpoint: 'PUT /internal/entities/managed/enablement', - options: { - security: { - authz: { - enabled: false, - reason: - 'This endpoint leverages the security plugin to evaluate the privileges needed as part of its core flow', - }, + security: { + authz: { + enabled: false, + reason: + 'This endpoint leverages the security plugin to evaluate the privileges needed as part of its core flow', }, }, params: z.object({ diff --git a/x-pack/plugins/streams/server/routes/streams/resync.ts b/x-pack/plugins/streams/server/routes/streams/resync.ts index e34bd290108a1..8e520410ca5c2 100644 --- a/x-pack/plugins/streams/server/routes/streams/resync.ts +++ b/x-pack/plugins/streams/server/routes/streams/resync.ts @@ -37,6 +37,7 @@ export const resyncStreamsRoute = createServerRoute({ scopedClusterClient, id: stream.id, }); + await syncStream({ scopedClusterClient, definition, From ef896625cacb734e383bc94977b4f943056bccb1 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Sat, 23 Nov 2024 20:17:13 +0100 Subject: [PATCH 62/95] Catch error in _status call --- x-pack/plugins/streams/public/plugin.ts | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/streams/public/plugin.ts b/x-pack/plugins/streams/public/plugin.ts index 532340dcafdf7..5a2ae3e066845 100644 --- a/x-pack/plugins/streams/public/plugin.ts +++ b/x-pack/plugins/streams/public/plugin.ts @@ -9,7 +9,7 @@ import { CoreSetup, CoreStart, PluginInitializerContext } from '@kbn/core/public import { Logger } from '@kbn/logging'; import { createRepositoryClient } from '@kbn/server-route-repository-client'; -import { from, share, startWith } from 'rxjs'; +import { from, shareReplay, startWith } from 'rxjs'; import { once } from 'lodash'; import type { StreamsPublicConfig } from '../common/config'; import { StreamsPluginClass, StreamsPluginSetup, StreamsPluginStart } from './types'; @@ -29,28 +29,34 @@ export class Plugin implements StreamsPluginClass { setup(core: CoreSetup<{}>, pluginSetup: {}): StreamsPluginSetup { this.repositoryClient = createRepositoryClient(core); return { - status$: createStatusObservable(this.repositoryClient), + status$: createStatusObservable(this.logger, this.repositoryClient), }; } start(core: CoreStart, pluginsStart: {}): StreamsPluginStart { return { streamsRepositoryClient: this.repositoryClient, - status$: createStatusObservable(this.repositoryClient), + status$: createStatusObservable(this.logger, this.repositoryClient), }; } stop() {} } -const createStatusObservable = once((repositoryClient: StreamsRepositoryClient) => { +const createStatusObservable = once((logger: Logger, repositoryClient: StreamsRepositoryClient) => { return from( repositoryClient .fetch('GET /api/streams/_status', { signal: new AbortController().signal, }) - .then((response) => ({ - status: response.enabled ? ('enabled' as const) : ('disabled' as const), - })) - ).pipe(startWith({ status: 'unknown' as const }), share()); + .then( + (response) => ({ + status: response.enabled ? ('enabled' as const) : ('disabled' as const), + }), + (error) => { + logger.error(error); + return { status: 'unknown' as const }; + } + ) + ).pipe(startWith({ status: 'unknown' as const }), shareReplay(1)); }); From 8ed5d23d717c2981c14cf76f936f40778368f322 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Sat, 23 Nov 2024 20:17:13 +0100 Subject: [PATCH 63/95] Catch error in _status call --- x-pack/plugins/streams/public/plugin.ts | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/streams/public/plugin.ts b/x-pack/plugins/streams/public/plugin.ts index 532340dcafdf7..5a2ae3e066845 100644 --- a/x-pack/plugins/streams/public/plugin.ts +++ b/x-pack/plugins/streams/public/plugin.ts @@ -9,7 +9,7 @@ import { CoreSetup, CoreStart, PluginInitializerContext } from '@kbn/core/public import { Logger } from '@kbn/logging'; import { createRepositoryClient } from '@kbn/server-route-repository-client'; -import { from, share, startWith } from 'rxjs'; +import { from, shareReplay, startWith } from 'rxjs'; import { once } from 'lodash'; import type { StreamsPublicConfig } from '../common/config'; import { StreamsPluginClass, StreamsPluginSetup, StreamsPluginStart } from './types'; @@ -29,28 +29,34 @@ export class Plugin implements StreamsPluginClass { setup(core: CoreSetup<{}>, pluginSetup: {}): StreamsPluginSetup { this.repositoryClient = createRepositoryClient(core); return { - status$: createStatusObservable(this.repositoryClient), + status$: createStatusObservable(this.logger, this.repositoryClient), }; } start(core: CoreStart, pluginsStart: {}): StreamsPluginStart { return { streamsRepositoryClient: this.repositoryClient, - status$: createStatusObservable(this.repositoryClient), + status$: createStatusObservable(this.logger, this.repositoryClient), }; } stop() {} } -const createStatusObservable = once((repositoryClient: StreamsRepositoryClient) => { +const createStatusObservable = once((logger: Logger, repositoryClient: StreamsRepositoryClient) => { return from( repositoryClient .fetch('GET /api/streams/_status', { signal: new AbortController().signal, }) - .then((response) => ({ - status: response.enabled ? ('enabled' as const) : ('disabled' as const), - })) - ).pipe(startWith({ status: 'unknown' as const }), share()); + .then( + (response) => ({ + status: response.enabled ? ('enabled' as const) : ('disabled' as const), + }), + (error) => { + logger.error(error); + return { status: 'unknown' as const }; + } + ) + ).pipe(startWith({ status: 'unknown' as const }), shareReplay(1)); }); From 1ce43ab067ceeaf1c0d9c79a11d13e3b09b8c365 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Sat, 23 Nov 2024 20:50:57 +0100 Subject: [PATCH 64/95] Add streams to app usage schema --- .../server/collectors/application_usage/schema.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts b/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts index 0cc56676137e5..ad2dce80fb650 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts @@ -183,6 +183,7 @@ export const applicationUsageSchema = { */ siem: commonSchema, space_selector: commonSchema, + streams: commonSchema, uptime: commonSchema, synthetics: commonSchema, ux: commonSchema, From 7065f3d24da6ee678816c2ab0344bbb3ccdcd43a Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Sat, 23 Nov 2024 21:31:48 +0100 Subject: [PATCH 65/95] fix telemetry --- src/plugins/telemetry/schema/oss_plugins.json | 131 ++++++++++++++++++ 1 file changed, 131 insertions(+) diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index 44fcda4f28581..fe0f599dd6ca1 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -7731,6 +7731,137 @@ } } }, + "streams": { + "properties": { + "appId": { + "type": "keyword", + "_meta": { + "description": "The application being tracked" + } + }, + "viewId": { + "type": "keyword", + "_meta": { + "description": "Always `main`" + } + }, + "clicks_total": { + "type": "long", + "_meta": { + "description": "General number of clicks in the application since we started counting them" + } + }, + "clicks_7_days": { + "type": "long", + "_meta": { + "description": "General number of clicks in the application over the last 7 days" + } + }, + "clicks_30_days": { + "type": "long", + "_meta": { + "description": "General number of clicks in the application over the last 30 days" + } + }, + "clicks_90_days": { + "type": "long", + "_meta": { + "description": "General number of clicks in the application over the last 90 days" + } + }, + "minutes_on_screen_total": { + "type": "float", + "_meta": { + "description": "Minutes the application is active and on-screen since we started counting them." + } + }, + "minutes_on_screen_7_days": { + "type": "float", + "_meta": { + "description": "Minutes the application is active and on-screen over the last 7 days" + } + }, + "minutes_on_screen_30_days": { + "type": "float", + "_meta": { + "description": "Minutes the application is active and on-screen over the last 30 days" + } + }, + "minutes_on_screen_90_days": { + "type": "float", + "_meta": { + "description": "Minutes the application is active and on-screen over the last 90 days" + } + }, + "views": { + "type": "array", + "items": { + "properties": { + "appId": { + "type": "keyword", + "_meta": { + "description": "The application being tracked" + } + }, + "viewId": { + "type": "keyword", + "_meta": { + "description": "The application view being tracked" + } + }, + "clicks_total": { + "type": "long", + "_meta": { + "description": "General number of clicks in the application sub view since we started counting them" + } + }, + "clicks_7_days": { + "type": "long", + "_meta": { + "description": "General number of clicks in the active application sub view over the last 7 days" + } + }, + "clicks_30_days": { + "type": "long", + "_meta": { + "description": "General number of clicks in the active application sub view over the last 30 days" + } + }, + "clicks_90_days": { + "type": "long", + "_meta": { + "description": "General number of clicks in the active application sub view over the last 90 days" + } + }, + "minutes_on_screen_total": { + "type": "float", + "_meta": { + "description": "Minutes the application sub view is active and on-screen since we started counting them." + } + }, + "minutes_on_screen_7_days": { + "type": "float", + "_meta": { + "description": "Minutes the application is active and on-screen active application sub view over the last 7 days" + } + }, + "minutes_on_screen_30_days": { + "type": "float", + "_meta": { + "description": "Minutes the application is active and on-screen active application sub view over the last 30 days" + } + }, + "minutes_on_screen_90_days": { + "type": "float", + "_meta": { + "description": "Minutes the application is active and on-screen active application sub view over the last 90 days" + } + } + } + } + } + } + }, "uptime": { "properties": { "appId": { From ddaabc18ab07aa253178c97046c7bdbb7e53d0fa Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Sun, 24 Nov 2024 09:47:54 +0100 Subject: [PATCH 66/95] Type changes --- .../src/register_routes.ts | 12 +++- .../server/functions/context.ts | 2 +- .../server/routes/chat/route.ts | 2 +- .../server/routes/register_routes.ts | 5 +- .../server/routes/types.ts | 62 ++++++++++++------- .../observability_onboarding/server/plugin.ts | 8 ++- x-pack/plugins/streams/server/plugin.ts | 7 +-- x-pack/plugins/streams/server/routes/index.ts | 4 +- 8 files changed, 66 insertions(+), 36 deletions(-) diff --git a/packages/kbn-server-route-repository/src/register_routes.ts b/packages/kbn-server-route-repository/src/register_routes.ts index 6201ffcd869ea..3a0fc1c8482da 100644 --- a/packages/kbn-server-route-repository/src/register_routes.ts +++ b/packages/kbn-server-route-repository/src/register_routes.ts @@ -16,6 +16,7 @@ import type { CoreSetup } from '@kbn/core-lifecycle-server'; import type { Logger } from '@kbn/logging'; import { DefaultRouteCreateOptions, + DefaultRouteHandlerResources, RouteParamsRT, ServerRoute, ZodParamsObject, @@ -44,7 +45,16 @@ export function registerRoutes>({ dependencies, }: { core: CoreSetup; - repository: Record>; + repository: Record< + string, + ServerRoute< + string, + RouteParamsRT | undefined, + DefaultRouteHandlerResources & TDependencies, + any, + any + > + >; logger: Logger; dependencies: TDependencies; }) { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/context.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/context.ts index 80ddf3cbc0a0d..1bf892c0d40ee 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/context.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/context.ts @@ -34,7 +34,7 @@ export function registerContextFunction({ visibility: FunctionVisibility.Internal, }, async ({ messages, screenContexts, chat }, signal) => { - const { analytics } = (await resources.context.core).coreStart; + const { analytics } = await resources.plugins.core.start(); async function getContext() { const screenDescription = compact( diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/chat/route.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/chat/route.ts index a6fe57cb58adc..e80e6fa156b06 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/chat/route.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/chat/route.ts @@ -194,7 +194,7 @@ const chatRecallRoute = createObservabilityAIAssistantServerRoute({ const response$ = from( recallAndScore({ - analytics: (await resources.context.core).coreStart.analytics, + analytics: (await resources.plugins.core.start()).analytics, chat: (name, params) => client .chat(name, { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/register_routes.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/register_routes.ts index 1a6140968c925..7f95e98860622 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/register_routes.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/register_routes.ts @@ -6,7 +6,7 @@ */ import type { CoreSetup } from '@kbn/core/server'; import type { Logger } from '@kbn/logging'; -import { registerRoutes } from '@kbn/server-route-repository'; +import { DefaultRouteHandlerResources, registerRoutes } from '@kbn/server-route-repository'; import { getGlobalObservabilityAIAssistantServerRouteRepository } from './get_global_observability_ai_assistant_route_repository'; import type { ObservabilityAIAssistantRouteHandlerResources } from './types'; import { ObservabilityAIAssistantPluginStartDependencies } from '../types'; @@ -20,12 +20,13 @@ export function registerServerRoutes({ logger: Logger; dependencies: Omit< ObservabilityAIAssistantRouteHandlerResources, - 'request' | 'context' | 'logger' | 'params' + keyof DefaultRouteHandlerResources >; }) { registerRoutes({ core, logger, + // @ts-expect-error request context is not assignable to that of DefaultRouteHandlerResources repository: getGlobalObservabilityAIAssistantServerRouteRepository(), dependencies, }); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/types.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/types.ts index 11f6dd6abed72..0e59ac5a214b9 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/types.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/types.ts @@ -6,32 +6,36 @@ */ import type { + CoreSetup, CoreStart, CustomRequestHandlerContext, IScopedClusterClient, IUiSettingsClient, - KibanaRequest, SavedObjectsClientContract, } from '@kbn/core/server'; -import type { Logger } from '@kbn/logging'; import type { LicensingApiRequestHandlerContext } from '@kbn/licensing-plugin/server/types'; import type { RacApiRequestHandlerContext } from '@kbn/rule-registry-plugin/server'; import type { RulesClientApi } from '@kbn/alerting-plugin/server/types'; +import { DefaultRouteHandlerResources } from '@kbn/server-route-repository-utils'; import type { ObservabilityAIAssistantService } from '../service'; import type { ObservabilityAIAssistantPluginSetupDependencies, ObservabilityAIAssistantPluginStartDependencies, } from '../types'; +type ObservabilityAIAssistantRequestHandlerContextBase = CustomRequestHandlerContext<{ + licensing: Pick; + // these two are here for compatibility with APM functions + rac: Pick; + alerting: { + getRulesClient: () => RulesClientApi; + }; +}>; + +// this is the type used across methods, it's stripped down for compatibility +// with the context that's available when executing as an action export type ObservabilityAIAssistantRequestHandlerContext = Omit< - CustomRequestHandlerContext<{ - licensing: Pick; - // these two are here for compatibility with APM functions - rac: Pick; - alerting: { - getRulesClient: () => RulesClientApi; - }; - }>, + ObservabilityAIAssistantRequestHandlerContextBase, 'core' | 'resolve' > & { core: Promise<{ @@ -49,22 +53,34 @@ export type ObservabilityAIAssistantRequestHandlerContext = Omit< }>; }; -export interface ObservabilityAIAssistantRouteHandlerResources { - request: KibanaRequest; - context: ObservabilityAIAssistantRequestHandlerContext; - logger: Logger; - service: ObservabilityAIAssistantService; - plugins: { - [key in keyof ObservabilityAIAssistantPluginSetupDependencies]: { - setup: Required[key]; - }; - } & { - [key in keyof ObservabilityAIAssistantPluginStartDependencies]: { - start: () => Promise[key]>; - }; +interface PluginContractResolveCore { + core: { + setup: CoreSetup; + start: () => Promise; }; } +type PluginContractResolveDependenciesStart = { + [key in keyof ObservabilityAIAssistantPluginStartDependencies]: { + start: () => Promise[key]>; + }; +}; + +type PluginContractResolveDependenciesSetup = { + [key in keyof ObservabilityAIAssistantPluginSetupDependencies]: { + setup: Required[key]; + }; +}; + +export interface ObservabilityAIAssistantRouteHandlerResources + extends DefaultRouteHandlerResources { + context: ObservabilityAIAssistantRequestHandlerContextBase; + service: ObservabilityAIAssistantService; + plugins: PluginContractResolveCore & + PluginContractResolveDependenciesSetup & + PluginContractResolveDependenciesStart; +} + export interface ObservabilityAIAssistantRouteCreateOptions { timeout?: { payload?: number; diff --git a/x-pack/plugins/observability_solution/observability_onboarding/server/plugin.ts b/x-pack/plugins/observability_solution/observability_onboarding/server/plugin.ts index 993f211f48434..30aaaf2588388 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/server/plugin.ts +++ b/x-pack/plugins/observability_solution/observability_onboarding/server/plugin.ts @@ -14,7 +14,7 @@ import type { } from '@kbn/core/server'; import { mapValues } from 'lodash'; import { i18n } from '@kbn/i18n'; -import { registerRoutes } from '@kbn/server-route-repository'; +import { DefaultRouteHandlerResources, registerRoutes } from '@kbn/server-route-repository'; import { getObservabilityOnboardingServerRouteRepository } from './routes'; import { ObservabilityOnboardingRouteHandlerResources } from './routes/types'; import { @@ -74,7 +74,7 @@ export class ObservabilityOnboardingPlugin const dependencies: Omit< ObservabilityOnboardingRouteHandlerResources, - 'core' | 'logger' | 'request' | 'context' | 'response' + keyof DefaultRouteHandlerResources > = { config, kibanaVersion: this.initContext.env.packageInfo.version, @@ -82,6 +82,10 @@ export class ObservabilityOnboardingPlugin services: { esLegacyConfigService: this.esLegacyConfigService, }, + core: { + setup: core, + start: () => core.getStartServices().then(([coreStart]) => coreStart), + }, }; registerRoutes({ diff --git a/x-pack/plugins/streams/server/plugin.ts b/x-pack/plugins/streams/server/plugin.ts index ef070984803d5..937f8c22b5be0 100644 --- a/x-pack/plugins/streams/server/plugin.ts +++ b/x-pack/plugins/streams/server/plugin.ts @@ -16,8 +16,7 @@ import { } from '@kbn/core/server'; import { registerRoutes } from '@kbn/server-route-repository'; import { StreamsConfig, configSchema, exposeToBrowserConfig } from '../common/config'; -import { StreamsRouteRepository } from './routes'; -import { RouteDependencies } from './routes/types'; +import { streamsRouteRepository } from './routes'; import { StreamsPluginSetupDependencies, StreamsPluginStartDependencies, @@ -58,8 +57,8 @@ export class StreamsPlugin logger: this.logger, } as StreamsServer; - registerRoutes({ - repository: StreamsRouteRepository, + registerRoutes({ + repository: streamsRouteRepository, dependencies: { server: this.server, getScopedClients: async ({ request }: { request: KibanaRequest }) => { diff --git a/x-pack/plugins/streams/server/routes/index.ts b/x-pack/plugins/streams/server/routes/index.ts index fb676ce94b9f8..7267dbedeacff 100644 --- a/x-pack/plugins/streams/server/routes/index.ts +++ b/x-pack/plugins/streams/server/routes/index.ts @@ -16,7 +16,7 @@ import { readStreamRoute } from './streams/read'; import { resyncStreamsRoute } from './streams/resync'; import { streamsStatusRoutes } from './streams/settings'; -export const StreamsRouteRepository = { +export const streamsRouteRepository = { ...enableStreamsRoute, ...resyncStreamsRoute, ...forkStreamsRoute, @@ -29,4 +29,4 @@ export const StreamsRouteRepository = { ...disableStreamsRoute, }; -export type StreamsRouteRepository = typeof StreamsRouteRepository; +export type StreamsRouteRepository = typeof streamsRouteRepository; From d57be239cc2310e2f094c6a8a0e5f5919736ea9d Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Sun, 24 Nov 2024 09:53:25 +0100 Subject: [PATCH 67/95] Update CODEOWNERS --- .github/CODEOWNERS | 2 +- x-pack/plugins/streams_app/kibana.jsonc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9b72f88e56b00..a476a9ee6efb2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -977,7 +977,7 @@ x-pack/plugins/spaces @elastic/kibana-security x-pack/plugins/stack_alerts @elastic/response-ops x-pack/plugins/stack_connectors @elastic/response-ops x-pack/plugins/streams @simianhacker @flash1293 @dgieselaar -x-pack/plugins/streams_app @elastic/observability-ui +x-pack/plugins/streams_app @simianhacker @flash1293 @dgieselaar x-pack/plugins/task_manager @elastic/response-ops x-pack/plugins/telemetry_collection_xpack @elastic/kibana-core x-pack/plugins/threat_intelligence @elastic/security-threat-hunting-investigations diff --git a/x-pack/plugins/streams_app/kibana.jsonc b/x-pack/plugins/streams_app/kibana.jsonc index 413e68082f45d..16666084c53e5 100644 --- a/x-pack/plugins/streams_app/kibana.jsonc +++ b/x-pack/plugins/streams_app/kibana.jsonc @@ -1,7 +1,7 @@ { "type": "plugin", "id": "@kbn/streams-app-plugin", - "owner": "@elastic/observability-ui", + "owner": "@simianhacker @flash1293 @dgieselaar", "group": "observability", "visibility": "private", "plugin": { From 003c3e6b562e29a637ebaf13731b62dd7e4f41e7 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Sun, 24 Nov 2024 09:09:06 +0000 Subject: [PATCH 68/95] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- .../observability_ai_assistant/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/tsconfig.json b/x-pack/plugins/observability_solution/observability_ai_assistant/tsconfig.json index d5acd7a365b50..709b3117d575d 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/tsconfig.json +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/tsconfig.json @@ -47,6 +47,7 @@ "@kbn/ai-assistant-common", "@kbn/inference-common", "@kbn/core-lifecycle-server", + "@kbn/server-route-repository-utils", ], "exclude": ["target/**/*"] } From 830fc5de2145b35e529b3c207e5fd5a9fae98bdf Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Sun, 24 Nov 2024 16:07:40 +0100 Subject: [PATCH 69/95] More flexible types --- .../src/register_routes.ts | 12 +----------- .../server/routes/register_routes.ts | 1 - .../server/routes/types.ts | 4 ++-- .../server/plugin.ts | 18 +++++++++++++----- .../server/rule_connector/index.ts | 2 +- 5 files changed, 17 insertions(+), 20 deletions(-) diff --git a/packages/kbn-server-route-repository/src/register_routes.ts b/packages/kbn-server-route-repository/src/register_routes.ts index 3a0fc1c8482da..6201ffcd869ea 100644 --- a/packages/kbn-server-route-repository/src/register_routes.ts +++ b/packages/kbn-server-route-repository/src/register_routes.ts @@ -16,7 +16,6 @@ import type { CoreSetup } from '@kbn/core-lifecycle-server'; import type { Logger } from '@kbn/logging'; import { DefaultRouteCreateOptions, - DefaultRouteHandlerResources, RouteParamsRT, ServerRoute, ZodParamsObject, @@ -45,16 +44,7 @@ export function registerRoutes>({ dependencies, }: { core: CoreSetup; - repository: Record< - string, - ServerRoute< - string, - RouteParamsRT | undefined, - DefaultRouteHandlerResources & TDependencies, - any, - any - > - >; + repository: Record>; logger: Logger; dependencies: TDependencies; }) { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/register_routes.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/register_routes.ts index 7f95e98860622..27c7361e8a7fb 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/register_routes.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/register_routes.ts @@ -26,7 +26,6 @@ export function registerServerRoutes({ registerRoutes({ core, logger, - // @ts-expect-error request context is not assignable to that of DefaultRouteHandlerResources repository: getGlobalObservabilityAIAssistantServerRouteRepository(), dependencies, }); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/types.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/types.ts index 0e59ac5a214b9..7b1f6a4f4cf95 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/types.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/types.ts @@ -73,8 +73,8 @@ type PluginContractResolveDependenciesSetup = { }; export interface ObservabilityAIAssistantRouteHandlerResources - extends DefaultRouteHandlerResources { - context: ObservabilityAIAssistantRequestHandlerContextBase; + extends Omit { + context: ObservabilityAIAssistantRequestHandlerContext; service: ObservabilityAIAssistantService; plugins: PluginContractResolveCore & PluginContractResolveDependenciesSetup & diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/plugin.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/plugin.ts index 63e06818a2b70..2ed5f161f2900 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/plugin.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/plugin.ts @@ -17,7 +17,6 @@ import { ObservabilityAIAssistantRequestHandlerContext, ObservabilityAIAssistantRouteHandlerResources, } from '@kbn/observability-ai-assistant-plugin/server/routes/types'; -import { ObservabilityAIAssistantPluginStartDependencies } from '@kbn/observability-ai-assistant-plugin/server/types'; import { mapValues } from 'lodash'; import { firstValueFrom } from 'rxjs'; import type { ObservabilityAIAssistantAppConfig } from './config'; @@ -59,13 +58,22 @@ export class ObservabilityAIAssistantAppPlugin setup: value, start: () => core.getStartServices().then((services) => { - const [, pluginsStartContracts] = services; + const [_, pluginsStartContracts] = services; + return pluginsStartContracts[ - key as keyof ObservabilityAIAssistantPluginStartDependencies + key as keyof ObservabilityAIAssistantAppPluginStartDependencies ]; }), }; - }) as ObservabilityAIAssistantRouteHandlerResources['plugins']; + }) as Omit; + + const withCore = { + ...routeHandlerPlugins, + core: { + setup: core, + start: () => core.getStartServices().then(([coreStart]) => coreStart), + }, + }; const initResources = async ( request: KibanaRequest @@ -110,7 +118,7 @@ export class ObservabilityAIAssistantAppPlugin context, service: plugins.observabilityAIAssistant.service, logger: this.logger.get('connector'), - plugins: routeHandlerPlugins, + plugins: withCore, }; }; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/rule_connector/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/rule_connector/index.ts index 33f3bdd2c98f8..1f5a097f8f7cd 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/rule_connector/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/rule_connector/index.ts @@ -248,7 +248,7 @@ If available, include the link of the conversation at the end of your answer.` isPublic: true, connectorId: execOptions.params.connector, signal: new AbortController().signal, - kibanaPublicUrl: (await resources.context.core).coreStart.http.basePath.publicBaseUrl, + kibanaPublicUrl: (await resources.plugins.core.start()).http.basePath.publicBaseUrl, instructions: [backgroundInstruction], messages: [ { From c46d94ca8d5f783ace821064e1482d2fbdc6ab7c Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Sun, 24 Nov 2024 18:12:39 +0100 Subject: [PATCH 70/95] Properly pass plugins to routes --- .../observability_ai_assistant/server/plugin.ts | 15 +++++++++++++-- .../server/routes/types.ts | 1 - .../server/plugin.ts | 1 - .../server/rule_connector/index.test.ts | 12 +++++++----- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts index f693fa53c06cc..7949276ac6aba 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts @@ -111,7 +111,18 @@ export class ObservabilityAIAssistantPlugin ]; }), }; - }) as ObservabilityAIAssistantRouteHandlerResources['plugins']; + }) as Pick< + ObservabilityAIAssistantRouteHandlerResources['plugins'], + keyof ObservabilityAIAssistantPluginStartDependencies + >; + + const withCore = { + ...routeHandlerPlugins, + core: { + setup: core, + start: () => core.getStartServices().then(([coreStart]) => coreStart), + }, + }; const service = (this.service = new ObservabilityAIAssistantService({ logger: this.logger.get('service'), @@ -133,7 +144,7 @@ export class ObservabilityAIAssistantPlugin core, logger: this.logger, dependencies: { - plugins: routeHandlerPlugins, + plugins: withCore, service: this.service, }, }); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/types.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/types.ts index 7b1f6a4f4cf95..62365536f3823 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/types.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/types.ts @@ -49,7 +49,6 @@ export type ObservabilityAIAssistantRequestHandlerContext = Omit< savedObjects: { client: SavedObjectsClientContract; }; - coreStart: CoreStart; }>; }; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/plugin.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/plugin.ts index 2ed5f161f2900..97fdc01069b97 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/plugin.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/plugin.ts @@ -98,7 +98,6 @@ export class ObservabilityAIAssistantAppPlugin }; }), core: Promise.resolve({ - coreStart, elasticsearch: { client: coreStart.elasticsearch.client.asScoped(request), }, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/rule_connector/index.test.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/rule_connector/index.test.ts index de02e4cf841ce..04fd10c3e506f 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/rule_connector/index.test.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/rule_connector/index.test.ts @@ -94,12 +94,14 @@ describe('observabilityAIAssistant rule_connector', () => { getAdhocInstructions: () => [], }), }, - context: { - core: Promise.resolve({ - coreStart: { http: { basePath: { publicBaseUrl: 'http://kibana.com' } } }, - }), - }, + context: {}, plugins: { + core: { + start: () => + Promise.resolve({ + http: { basePath: { publicBaseUrl: 'http://kibana.com' } }, + }), + }, actions: { start: async () => { return { From cebd9d0c4044b8257c1edcef5d536b8e7a2b2356 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Mon, 25 Nov 2024 11:41:39 +0100 Subject: [PATCH 71/95] Strip .text/.keyword suffixes --- .../object/unflatten_object.test.ts | 12 ++++++++++++ .../routes/entities/get_latest_entities.ts | 17 +++++++++++++---- .../server/routes/register_routes.ts | 2 +- .../observability/server/routes/types.ts | 2 -- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/x-pack/packages/observability/observability_utils/observability_utils_common/object/unflatten_object.test.ts b/x-pack/packages/observability/observability_utils/observability_utils_common/object/unflatten_object.test.ts index 22cee17bb1a64..aa25821b4ac14 100644 --- a/x-pack/packages/observability/observability_utils/observability_utils_common/object/unflatten_object.test.ts +++ b/x-pack/packages/observability/observability_utils/observability_utils_common/object/unflatten_object.test.ts @@ -37,4 +37,16 @@ describe('unflattenObject', () => { }, }); }); + + it('handles null values correctly', () => { + expect( + unflattenObject({ + 'agent.name': null, + }) + ).toEqual({ + agent: { + name: null, + }, + }); + }); }); diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts index 7cf0423ffae0b..9dcf17250ad68 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts @@ -12,6 +12,7 @@ import { ENTITY_TYPE, } from '@kbn/observability-shared-plugin/common'; import type { ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client'; +import { unflattenObject } from '@kbn/observability-utils-common/object/unflatten_object'; import { ENTITIES_LATEST_ALIAS, InventoryEntity, @@ -71,8 +72,8 @@ export async function getLatestEntities({ 'entity.last_seen_timestamp': string; 'entity.definition_version': string; 'entity.schema_version': string; - }, - { transform: 'unflatten' } + } & Record, + { transform: 'plain' } >( 'get_latest_entities', { @@ -80,11 +81,19 @@ export async function getLatestEntities({ filter: esQuery, params, }, - { transform: 'unflatten' } + { transform: 'plain' } ); return latestEntitiesEsqlResponse.hits.map((latestEntity) => { - const { entity, ...metadata } = latestEntity; + Object.keys(latestEntity).forEach((key) => { + const keyOfObject = key as keyof typeof latestEntity; + // strip out multi-field aliases + if (keyOfObject.endsWith('.text') || keyOfObject.endsWith('.keyword')) { + delete latestEntity[keyOfObject]; + } + }); + + const { entity, ...metadata } = unflattenObject(latestEntity); return { entityId: entity.id, diff --git a/x-pack/plugins/observability_solution/observability/server/routes/register_routes.ts b/x-pack/plugins/observability_solution/observability/server/routes/register_routes.ts index 370473879fdb0..9ce2d7c9f1829 100644 --- a/x-pack/plugins/observability_solution/observability/server/routes/register_routes.ts +++ b/x-pack/plugins/observability_solution/observability/server/routes/register_routes.ts @@ -36,7 +36,7 @@ export interface RegisterRoutesDependencies { export function registerRoutes({ repository, core, logger, dependencies }: RegisterRoutes) { registerServerRoutes({ core, - dependencies, + dependencies: { dependencies }, logger, repository, }); diff --git a/x-pack/plugins/observability_solution/observability/server/routes/types.ts b/x-pack/plugins/observability_solution/observability/server/routes/types.ts index 9039a66dc2dee..111bc4e714119 100644 --- a/x-pack/plugins/observability_solution/observability/server/routes/types.ts +++ b/x-pack/plugins/observability_solution/observability/server/routes/types.ts @@ -13,7 +13,6 @@ import { } from './get_global_observability_server_route_repository'; import { ObservabilityRequestHandlerContext } from '../types'; import { RegisterRoutesDependencies } from './register_routes'; -import { ObservabilityConfig } from '..'; export type { ObservabilityServerRouteRepository, APIEndpoint }; @@ -22,7 +21,6 @@ export interface ObservabilityRouteHandlerResources { dependencies: RegisterRoutesDependencies; logger: Logger; request: KibanaRequest; - config: ObservabilityConfig; } export interface ObservabilityRouteCreateOptions { From 44d5b2471f9f98e12636852609e31781c7c94b87 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Mon, 25 Nov 2024 11:54:00 +0100 Subject: [PATCH 72/95] Set security for route registration in dataset_quality --- .../dataset_quality/server/routes/register_routes.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/register_routes.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/register_routes.ts index b3881f1466993..db5620e0778f0 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/register_routes.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/register_routes.ts @@ -50,6 +50,7 @@ export function registerRoutes({ path: pathname, validate: passThroughValidationObject, options, + security: route.security, }, async (context, request, response) => { try { From 1fc2da99af0138e31dc185e863927e473d272b52 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 2 Dec 2024 17:27:46 +0100 Subject: [PATCH 73/95] make specific to dashboards --- .../es/storage/storage_client.ts | 2 + x-pack/plugins/streams/common/assets.ts | 22 ++ .../server/lib/streams/assets/asset_client.ts | 70 +++--- .../streams/server/routes/assets/route.ts | 164 -------------- .../streams/server/routes/dashboards/route.ts | 167 ++++++++++++++ x-pack/plugins/streams/server/routes/index.ts | 4 +- .../streams/server/routes/streams/fork.ts | 7 - .../streams/server/routes/streams/list.ts | 41 +--- .../asset_type_display.tsx | 42 ---- .../stream_detail_asset_view/index.tsx | 109 ---------- .../add_dashboard_flyout.tsx} | 78 ++++--- .../dashboard_table.tsx} | 56 ++--- .../stream_detail_dashboards_view/index.tsx | 205 ++++++++++++++++++ .../components/stream_detail_view/index.tsx | 10 +- 14 files changed, 512 insertions(+), 465 deletions(-) delete mode 100644 x-pack/plugins/streams/server/routes/assets/route.ts create mode 100644 x-pack/plugins/streams/server/routes/dashboards/route.ts delete mode 100644 x-pack/plugins/streams_app/public/components/stream_detail_asset_view/asset_type_display.tsx delete mode 100644 x-pack/plugins/streams_app/public/components/stream_detail_asset_view/index.tsx rename x-pack/plugins/streams_app/public/components/{stream_detail_asset_view/add_asset_flyout.tsx => stream_detail_dashboards_view/add_dashboard_flyout.tsx} (54%) rename x-pack/plugins/streams_app/public/components/{stream_detail_asset_view/asset_table.tsx => stream_detail_dashboards_view/dashboard_table.tsx} (50%) create mode 100644 x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/index.tsx diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/storage_client.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/storage_client.ts index 3868bbcc52b8c..fc10846b88723 100644 --- a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/storage_client.ts +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/storage_client.ts @@ -46,6 +46,7 @@ export class StorageClient { await this.esClient.client.index({ index: this.storage.getSearchIndexPattern(), document, + refresh: 'wait_for', id, }); } @@ -53,6 +54,7 @@ export class StorageClient { async delete(id: string) { await this.esClient.client.delete({ id, + refresh: 'wait_for', index: this.storage.getSearchIndexPattern(), }); } diff --git a/x-pack/plugins/streams/common/assets.ts b/x-pack/plugins/streams/common/assets.ts index df873174dcff9..f66e48f2ff96c 100644 --- a/x-pack/plugins/streams/common/assets.ts +++ b/x-pack/plugins/streams/common/assets.ts @@ -20,7 +20,29 @@ export interface AssetLink { assetId: string; } +export interface DashboardLink extends AssetLink { + type: typeof ASSET_TYPES.Dashboard; +} + +export interface SloLink extends AssetLink { + type: typeof ASSET_TYPES.Slo; +} + export interface Asset extends AssetLink { label: string; tags: string[]; } + +export interface Dashboard extends Asset { + type: typeof ASSET_TYPES.Dashboard; +} + +export interface Slo extends Asset { + type: typeof ASSET_TYPES.Slo; +} + +export interface AssetTypeToAssetMap { + [ASSET_TYPES.Dashboard]: Dashboard; + [ASSET_TYPES.Slo]: Slo; + [ASSET_TYPES.Rule]: Asset; +} diff --git a/x-pack/plugins/streams/server/lib/streams/assets/asset_client.ts b/x-pack/plugins/streams/server/lib/streams/assets/asset_client.ts index c1bdc6a0ae7fd..bb5b174d71e13 100644 --- a/x-pack/plugins/streams/server/lib/streams/assets/asset_client.ts +++ b/x-pack/plugins/streams/server/lib/streams/assets/asset_client.ts @@ -13,13 +13,20 @@ import { keyBy } from 'lodash'; import objectHash from 'object-hash'; import { SanitizedRule } from '@kbn/alerting-plugin/common'; import { AssetStorageSettings } from './storage_settings'; -import { ASSET_TYPES, Asset, AssetType } from '../../../../common/assets'; +import { + ASSET_TYPES, + Asset, + AssetType, + AssetTypeToAssetMap, + Dashboard, + Slo, +} from '../../../../common/assets'; import { ASSET_ENTITY_ID, ASSET_ENTITY_TYPE } from './fields'; function sloSavedObjectToAsset( sloId: string, savedObject: SavedObject<{ name: string; tags: string[] }> -): Asset { +): Slo { return { assetId: sloId, label: savedObject.attributes.name, @@ -33,7 +40,7 @@ function sloSavedObjectToAsset( function dashboardSavedObjectToAsset( dashboardId: string, savedObject: SavedObject<{ title: string }> -): Asset { +): Dashboard { return { assetId: dashboardId, label: savedObject.attributes.title, @@ -198,40 +205,43 @@ export class AssetClient { return [...dashboards, ...rules, ...slos]; } - async getSuggestions({ + async getSuggestions({ entityId, entityType, query, + assetType, }: { entityId: string; entityType: string; query: string; - }): Promise> { - const [suggestionsFromSlosAndDashboards, ruleResponse] = await Promise.all([ - this.clients.soClient - .find({ - type: ['dashboard', 'slo'], - search: query, - }) - .then((results) => { - return results.saved_objects.map((savedObject) => { - if (savedObject.type === 'slo') { - const sloSavedObject = savedObject as SavedObject<{ - id: string; - name: string; - tags: string[]; - }>; - return sloSavedObjectToAsset(sloSavedObject.attributes.id, sloSavedObject); - } - - const dashboardSavedObject = savedObject as SavedObject<{ title: string }>; - - return dashboardSavedObjectToAsset(dashboardSavedObject.id, dashboardSavedObject); - }); - }), - [], - ]); + assetType: T; + }): Promise> { + if (assetType === 'dashboard') { + const dashboardSavedObjects = await this.clients.soClient.find<{ title: string }>({ + type: 'dashboard', + search: query, + }); + + return dashboardSavedObjects.saved_objects.map((dashboardSavedObject) => { + return { + asset: dashboardSavedObjectToAsset(dashboardSavedObject.id, dashboardSavedObject), + }; + }) as Array<{ asset: AssetTypeToAssetMap[T] }>; + } + if (assetType === 'rule') { + return []; + } + if (assetType === 'slo') { + const sloSavedObjects = await this.clients.soClient.find<{ name: string; tags: string[] }>({ + type: 'slo', + search: query, + }); + + return sloSavedObjects.saved_objects.map((sloSavedObject) => { + return { asset: sloSavedObjectToAsset(sloSavedObject.id, sloSavedObject) }; + }) as Array<{ asset: AssetTypeToAssetMap[T] }>; + } - return suggestionsFromSlosAndDashboards.map((asset) => ({ asset })); + throw new Error(`Unsupported asset type: ${assetType}`); } } diff --git a/x-pack/plugins/streams/server/routes/assets/route.ts b/x-pack/plugins/streams/server/routes/assets/route.ts deleted file mode 100644 index cf3016cabf3ae..0000000000000 --- a/x-pack/plugins/streams/server/routes/assets/route.ts +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { z } from '@kbn/zod'; -import { ASSET_TYPES, Asset } from '../../../common/assets'; -import { createServerRoute } from '../create_server_route'; - -export interface ListAssetsResponse { - assets: Asset[]; -} - -export interface LinkAssetResponse { - acknowledged: boolean; -} - -export interface UnlinkAssetResponse { - acknowledged: boolean; -} - -export interface SuggestAssetsResponse { - suggestions: Array<{ - asset: Asset; - }>; -} - -const assetTypeSchema = z.union([ - z.literal(ASSET_TYPES.Dashboard), - z.literal(ASSET_TYPES.Rule), - z.literal(ASSET_TYPES.Slo), -]); - -export const listAssetsRoute = createServerRoute({ - endpoint: 'GET /api/streams/{id}/assets', - options: { - access: 'internal', - }, - params: z.object({ - path: z.object({ - id: z.string(), - }), - }), - async handler({ params, request, assets }): Promise { - const assetsClient = await assets.getClientWithRequest({ request }); - - const { - path: { id: streamId }, - } = params; - - return { - assets: await assetsClient.getAssets({ - entityId: streamId, - entityType: 'stream', - }), - }; - }, -}); - -export const linkAssetRoute = createServerRoute({ - endpoint: 'PUT /api/streams/{id}/assets/{type}/{assetId}', - options: { - access: 'internal', - }, - params: z.object({ - path: z.object({ - type: assetTypeSchema, - id: z.string(), - assetId: z.string(), - }), - }), - handler: async ({ params, request, assets }): Promise => { - const assetsClient = await assets.getClientWithRequest({ request }); - - const { - path: { assetId, id: streamId, type: assetType }, - } = params; - - await assetsClient.linkAsset({ - entityId: streamId, - entityType: 'stream', - assetId, - assetType, - }); - - return { - acknowledged: true, - }; - }, -}); - -export const unlinkAssetRoute = createServerRoute({ - endpoint: 'DELETE /api/streams/{id}/assets/{type}/{assetId}', - options: { - access: 'internal', - }, - params: z.object({ - path: z.object({ - type: assetTypeSchema, - id: z.string(), - assetId: z.string(), - }), - }), - handler: async ({ params, request, assets }): Promise => { - const assetsClient = await assets.getClientWithRequest({ request }); - - const { - path: { assetId, id: streamId, type: assetType }, - } = params; - - await assetsClient.unlinkAsset({ - entityId: streamId, - entityType: 'stream', - assetId, - assetType, - }); - - return { - acknowledged: true, - }; - }, -}); - -export const suggestAssetsRoute = createServerRoute({ - endpoint: 'GET /api/streams/{id}/assets/_suggestions', - options: { - access: 'internal', - }, - params: z.object({ - path: z.object({ - id: z.string(), - }), - query: z.object({ - query: z.string(), - }), - }), - handler: async ({ params, request, assets }): Promise => { - const assetsClient = await assets.getClientWithRequest({ request }); - - const { - path: { id: streamId }, - query: { query }, - } = params; - - const suggestions = await assetsClient.getSuggestions({ - entityId: streamId, - entityType: 'stream', - query, - }); - - return { - suggestions, - }; - }, -}); - -export const assetsRoutes = { - ...listAssetsRoute, - ...linkAssetRoute, - ...unlinkAssetRoute, - ...suggestAssetsRoute, -}; diff --git a/x-pack/plugins/streams/server/routes/dashboards/route.ts b/x-pack/plugins/streams/server/routes/dashboards/route.ts new file mode 100644 index 0000000000000..991c6d4bd5f47 --- /dev/null +++ b/x-pack/plugins/streams/server/routes/dashboards/route.ts @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; +import { Asset, Dashboard } from '../../../common/assets'; +import { createServerRoute } from '../create_server_route'; + +export interface ListDashboardsResponse { + dashboards: Dashboard[]; +} + +export interface LinkDashboardResponse { + acknowledged: boolean; +} + +export interface UnlinkDashboardResponse { + acknowledged: boolean; +} + +export interface SuggestDashboardResponse { + suggestions: Array<{ + dashboard: Dashboard; + }>; +} + +export const listDashboardsRoute = createServerRoute({ + endpoint: 'GET /api/streams/{id}/dashboards', + options: { + access: 'internal', + }, + params: z.object({ + path: z.object({ + id: z.string(), + }), + }), + async handler({ params, request, assets }): Promise { + const assetsClient = await assets.getClientWithRequest({ request }); + + const { + path: { id: streamId }, + } = params; + + function isDashboard(asset: Asset): asset is Dashboard { + return asset.type === 'dashboard'; + } + + return { + dashboards: ( + await assetsClient.getAssets({ + entityId: streamId, + entityType: 'stream', + }) + ).filter(isDashboard), + }; + }, +}); + +export const linkDashboardRoute = createServerRoute({ + endpoint: 'PUT /api/streams/{id}/dashboards/{dashboardId}', + options: { + access: 'internal', + }, + params: z.object({ + path: z.object({ + id: z.string(), + dashboardId: z.string(), + }), + }), + handler: async ({ params, request, assets }): Promise => { + const assetsClient = await assets.getClientWithRequest({ request }); + + const { + path: { dashboardId, id: streamId }, + } = params; + + await assetsClient.linkAsset({ + entityId: streamId, + entityType: 'stream', + assetId: dashboardId, + assetType: 'dashboard', + }); + + return { + acknowledged: true, + }; + }, +}); + +export const unlinkDashboardRoute = createServerRoute({ + endpoint: 'DELETE /api/streams/{id}/dashboards/{dashboardId}', + options: { + access: 'internal', + }, + params: z.object({ + path: z.object({ + id: z.string(), + dashboardId: z.string(), + }), + }), + handler: async ({ params, request, assets }): Promise => { + const assetsClient = await assets.getClientWithRequest({ request }); + + const { + path: { dashboardId, id: streamId }, + } = params; + + await assetsClient.unlinkAsset({ + entityId: streamId, + entityType: 'stream', + assetId: dashboardId, + assetType: 'dashboard', + }); + + return { + acknowledged: true, + }; + }, +}); + +export const suggestDashboardsRoute = createServerRoute({ + endpoint: 'GET /api/streams/{id}/dashboards/_suggestions', + options: { + access: 'internal', + }, + params: z.object({ + path: z.object({ + id: z.string(), + }), + query: z.object({ + query: z.string(), + }), + }), + handler: async ({ params, request, assets }): Promise => { + const assetsClient = await assets.getClientWithRequest({ request }); + + const { + path: { id: streamId }, + query: { query }, + } = params; + + const suggestions = ( + await assetsClient.getSuggestions({ + entityId: streamId, + entityType: 'stream', + assetType: 'dashboard', + query, + }) + ).map(({ asset: dashboard }) => ({ + dashboard, + })); + + return { + suggestions, + }; + }, +}); + +export const dashboardRoutes = { + ...listDashboardsRoute, + ...linkDashboardRoute, + ...unlinkDashboardRoute, + ...suggestDashboardsRoute, +}; diff --git a/x-pack/plugins/streams/server/routes/index.ts b/x-pack/plugins/streams/server/routes/index.ts index d58016fa97ac7..b085a56281584 100644 --- a/x-pack/plugins/streams/server/routes/index.ts +++ b/x-pack/plugins/streams/server/routes/index.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { assetsRoutes } from './assets/route'; +import { dashboardRoutes } from './dashboards/route'; import { esqlRoutes } from './esql/route'; import { deleteStreamRoute } from './streams/delete'; import { disableStreamsRoute } from './streams/disable'; @@ -28,7 +28,7 @@ export const streamsRouteRepository = { ...streamsStatusRoutes, ...esqlRoutes, ...disableStreamsRoute, - ...assetsRoutes, + ...dashboardRoutes, }; export type StreamsRouteRepository = typeof streamsRouteRepository; diff --git a/x-pack/plugins/streams/server/routes/streams/fork.ts b/x-pack/plugins/streams/server/routes/streams/fork.ts index 043bcdb0aabe2..12dce248dcdd1 100644 --- a/x-pack/plugins/streams/server/routes/streams/fork.ts +++ b/x-pack/plugins/streams/server/routes/streams/fork.ts @@ -96,13 +96,6 @@ export const forkStreamsRoute = createServerRoute({ logger, }); - await syncStream({ - scopedClusterClient, - definition: childDefinition, - rootDefinition, - logger, - }); - return { acknowledged: true }; } catch (e) { if (e instanceof IndexTemplateNotFound || e instanceof DefinitionNotFound) { diff --git a/x-pack/plugins/streams/server/routes/streams/list.ts b/x-pack/plugins/streams/server/routes/streams/list.ts index bd6a5200fe9ba..acdf73aa6888c 100644 --- a/x-pack/plugins/streams/server/routes/streams/list.ts +++ b/x-pack/plugins/streams/server/routes/streams/list.ts @@ -29,14 +29,12 @@ export const listStreamsRoute = createServerRoute({ response, request, getScopedClients, - }): Promise<{ definitions: StreamDefinition[]; trees: StreamTree[] }> => { + }): Promise<{ definitions: StreamDefinition[] }> => { try { const { scopedClusterClient } = await getScopedClients({ request }); const { definitions } = await listStreams({ scopedClusterClient }); - const trees = asTrees(definitions); - - return { definitions, trees }; + return { definitions }; } catch (e) { if (e instanceof DefinitionNotFound) { throw notFound(e); @@ -46,38 +44,3 @@ export const listStreamsRoute = createServerRoute({ } }, }); - -export interface StreamTree { - id: string; - children: StreamTree[]; -} - -function asTrees(definitions: StreamDefinition[]): StreamTree[] { - const nodes = new Map(); - - const rootNodes = new Set(); - - function getNode(id: string) { - let node = nodes.get(id); - if (!node) { - node = { id, children: [] }; - nodes.set(id, node); - } - return node; - } - - definitions.forEach((definition) => { - const path = definition.id.split('.'); - const parentId = path.slice(0, path.length - 1).join('.'); - const parentNode = parentId.length ? getNode(parentId) : undefined; - const selfNode = getNode(definition.id); - - if (parentNode) { - parentNode.children.push(selfNode); - } else { - rootNodes.add(selfNode); - } - }); - - return Array.from(rootNodes.values()); -} diff --git a/x-pack/plugins/streams_app/public/components/stream_detail_asset_view/asset_type_display.tsx b/x-pack/plugins/streams_app/public/components/stream_detail_asset_view/asset_type_display.tsx deleted file mode 100644 index 33ffb49ed61ff..0000000000000 --- a/x-pack/plugins/streams_app/public/components/stream_detail_asset_view/asset_type_display.tsx +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import React from 'react'; -import { AssetType } from '@kbn/streams-plugin/common'; -import { i18n } from '@kbn/i18n'; -import { EuiFlexGroup, EuiIcon, EuiText } from '@elastic/eui'; - -export function AssetTypeDisplay({ type }: { type: AssetType }) { - let label: string = ''; - let icon: string = ''; - if (type === 'dashboard') { - label = i18n.translate('xpack.streams.assetType.dashboard', { - defaultMessage: 'Dashboard', - }); - icon = 'dashboardApp'; - } else if (type === 'rule') { - label = i18n.translate('xpack.streams.assetType.rule', { - defaultMessage: 'Rule', - }); - icon = 'bell'; - } else if (type === 'slo') { - label = i18n.translate('xpack.streams.assetType.slo', { - defaultMessage: 'SLO', - }); - icon = 'visGauge'; - } - - if (!icon || !label) { - throw new Error(`Unknown type ${type}`); - } - - return ( - - - {label} - - ); -} diff --git a/x-pack/plugins/streams_app/public/components/stream_detail_asset_view/index.tsx b/x-pack/plugins/streams_app/public/components/stream_detail_asset_view/index.tsx deleted file mode 100644 index 3eed355886137..0000000000000 --- a/x-pack/plugins/streams_app/public/components/stream_detail_asset_view/index.tsx +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { - EuiButton, - EuiFilterButton, - EuiFilterGroup, - EuiFlexGroup, - EuiPopover, - EuiSearchBar, - EuiSelectable, -} from '@elastic/eui'; -import { i18n } from '@kbn/i18n'; -import { StreamDefinition } from '@kbn/streams-plugin/common'; -import React, { useMemo, useState } from 'react'; -import { useKibana } from '../../hooks/use_kibana'; -import { useStreamsAppFetch } from '../../hooks/use_streams_app_fetch'; -import { AddAssetFlyout } from './add_asset_flyout'; -import { AssetTable } from './asset_table'; - -export function StreamDetailAssetView({ definition }: { definition?: StreamDefinition }) { - const { - dependencies: { - start: { - streams: { streamsRepositoryClient }, - }, - }, - } = useKibana(); - const [query, setQuery] = useState(''); - - const [isTagsPopoverOpen, setIsTagsPopoverOpen] = useState(false); - - const [isAddAssetFlyoutOpen, setIsAddAssetFlyoutOpen] = useState(false); - - const tagsButton = ( - - {i18n.translate('xpack.streams.streamDetailAssetView.tagsFilterButtonLabel', { - defaultMessage: 'Tags', - })} - - ); - - const assetsFetch = useStreamsAppFetch( - ({ signal }) => { - if (!definition?.id) { - return Promise.resolve(undefined); - } - return streamsRepositoryClient.fetch('GET /api/streams/{id}/assets', { - signal, - params: { - path: { - id: definition.id, - }, - }, - }); - }, - [definition?.id, streamsRepositoryClient] - ); - - const selectedAssets = useMemo(() => { - return assetsFetch.value?.assets ?? []; - }, [assetsFetch.value?.assets]); - - return ( - - - { - setQuery(nextQuery.queryText); - }} - /> - - - - - - { - setIsAddAssetFlyoutOpen(true); - }} - > - {i18n.translate('xpack.streams.streamDetailAssetView.addAnAssetButtonLabel', { - defaultMessage: 'Add an asset', - })} - - - - {definition && isAddAssetFlyoutOpen ? ( - {}} - onClose={() => { - setIsAddAssetFlyoutOpen(false); - }} - /> - ) : null} - - ); -} diff --git a/x-pack/plugins/streams_app/public/components/stream_detail_asset_view/add_asset_flyout.tsx b/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/add_dashboard_flyout.tsx similarity index 54% rename from x-pack/plugins/streams_app/public/components/stream_detail_asset_view/add_asset_flyout.tsx rename to x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/add_dashboard_flyout.tsx index 04559094d68c5..d00f6d8b6855c 100644 --- a/x-pack/plugins/streams_app/public/components/stream_detail_asset_view/add_asset_flyout.tsx +++ b/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/add_dashboard_flyout.tsx @@ -6,8 +6,6 @@ */ import { EuiButton, - EuiFilterButton, - EuiFilterGroup, EuiFlexGroup, EuiFlyout, EuiFlyoutBody, @@ -18,22 +16,22 @@ import { EuiTitle, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import type { Asset } from '@kbn/streams-plugin/common'; +import type { Dashboard } from '@kbn/streams-plugin/common/assets'; import { debounce } from 'lodash'; -import React, { useMemo, useState } from 'react'; +import React, { useMemo, useState, useEffect } from 'react'; import { useKibana } from '../../hooks/use_kibana'; import { useStreamsAppFetch } from '../../hooks/use_streams_app_fetch'; -import { AssetTable } from './asset_table'; +import { DashboardsTable } from './dashboard_table'; -export function AddAssetFlyout({ +export function AddDashboardFlyout({ entityId, - onAssetsChange, - selectedAssets, + onAddDashboards, + linkedDashboards, onClose, }: { entityId: string; - onAssetsChange: (asset: Asset[]) => void; - selectedAssets: Asset[]; + onAddDashboards: (dashboard: Dashboard[]) => void; + linkedDashboards: Dashboard[]; onClose: () => void; }) { const { @@ -52,10 +50,10 @@ export function AddAssetFlyout({ return debounce(setSubmittedQuery, 150); }, []); - const assetSuggestionsFetch = useStreamsAppFetch( + const dashboardSuggestionsFetch = useStreamsAppFetch( ({ signal }) => { return streamsRepositoryClient - .fetch('GET /api/streams/{id}/assets/_suggestions', { + .fetch('GET /api/streams/{id}/dashboards/_suggestions', { signal, params: { path: { @@ -68,27 +66,36 @@ export function AddAssetFlyout({ }) .then(({ suggestions }) => { return { - assets: suggestions - .map((suggestion) => suggestion.asset) - .filter((asset) => { - return !selectedAssets.find( - (selectedAsset) => - selectedAsset.assetId === asset.assetId && selectedAsset.type === asset.type + dashboards: suggestions + .map((suggestion) => suggestion.dashboard) + .filter((dashboard) => { + return !linkedDashboards.find( + (linkedDashboard) => linkedDashboard.assetId === dashboard.assetId ); }), }; }); }, - [streamsRepositoryClient, entityId, submittedQuery, selectedAssets] + [streamsRepositoryClient, entityId, submittedQuery, linkedDashboards] ); + const [selectedDashboards, setSelectedDashboards] = useState([]); + + useEffect(() => { + setSelectedDashboards([]); + }, [linkedDashboards]); + + const allDashboards = useMemo(() => { + return dashboardSuggestionsFetch.value?.dashboards || []; + }, [dashboardSuggestionsFetch.value]); + return (

- {i18n.translate('xpack.streams.addAssetFlyout.flyoutHeaderLabel', { - defaultMessage: 'Add assets', + {i18n.translate('xpack.streams.addDashboardFlyout.flyoutHeaderLabel', { + defaultMessage: 'Add dashboards', })}

@@ -96,9 +103,9 @@ export function AddAssetFlyout({ - {i18n.translate('xpack.streams.addAssetFlyout.helpLabel', { + {i18n.translate('xpack.streams.addDashboardFlyout.helpLabel', { defaultMessage: - 'Select assets which you want to add and assign to the {stream} stream', + 'Select dashboards which you want to add and assign to the {stream} stream', values: { stream: entityId, }, @@ -115,21 +122,24 @@ export function AddAssetFlyout({ setSubmittedQueryDebounced(queryText); }} /> - - - {i18n.translate('xpack.streams.addAssetFlyout.typeFilterButtonLabel', { - defaultMessage: 'Type', - })} - - - + - {}}> - {i18n.translate('xpack.streams.addAssetFlyout.addAssetsButtonLabel', { - defaultMessage: 'Add assets', + { + onAddDashboards(selectedDashboards); + }} + > + {i18n.translate('xpack.streams.addDashboardFlyout.addDashboardsButtonLabel', { + defaultMessage: 'Add dashboards', })} diff --git a/x-pack/plugins/streams_app/public/components/stream_detail_asset_view/asset_table.tsx b/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/dashboard_table.tsx similarity index 50% rename from x-pack/plugins/streams_app/public/components/stream_detail_asset_view/asset_table.tsx rename to x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/dashboard_table.tsx index deb2d8628f1e4..20bacf1013eae 100644 --- a/x-pack/plugins/streams_app/public/components/stream_detail_asset_view/asset_table.tsx +++ b/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/dashboard_table.tsx @@ -12,40 +12,35 @@ import { EuiFlexItem, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import type { AbortableAsyncState } from '@kbn/observability-utils-browser/hooks/use_abortable_async'; -import type { Asset } from '@kbn/streams-plugin/common'; -import React, { useEffect, useMemo, useState } from 'react'; -import { AssetTypeDisplay } from './asset_type_display'; +import { Dashboard } from '@kbn/streams-plugin/common/assets'; +import React, { useMemo } from 'react'; -export function AssetTable({ - assetsFetch, +export function DashboardsTable({ + dashboards, compact = false, + selecedDashboards: selectedDashboards, + setSelectedDashboards, + loading, }: { - assetsFetch: AbortableAsyncState<{ assets: Asset[] } | undefined>; + loading: boolean; + dashboards: Dashboard[] | undefined; compact?: boolean; + selecedDashboards: Dashboard[]; + setSelectedDashboards: (dashboards: Dashboard[]) => void; }) { - const columns = useMemo((): Array> => { + const columns = useMemo((): Array> => { return [ { field: 'label', - name: i18n.translate('xpack.streams.assetTable.assetNameColumnTitle', { - defaultMessage: 'Asset name', + name: i18n.translate('xpack.streams.dashboardTable.dashboardNameColumnTitle', { + defaultMessage: 'Dashboard name', }), }, - { - field: 'type', - name: i18n.translate('xpack.streams.assetTable.assetTypeColumnTitle', { - defaultMessage: 'Type', - }), - render: (_, { type }) => { - return ; - }, - }, ...(!compact ? ([ { field: 'tags', - name: i18n.translate('xpack.streams.assetTable.tagsColumnTitle', { + name: i18n.translate('xpack.streams.dashboardTable.tagsColumnTitle', { defaultMessage: 'Tags', }), render: (_, { tags }) => { @@ -60,33 +55,28 @@ export function AssetTable({ ); }, }, - ] satisfies Array>) + ] satisfies Array>) : []), ]; }, [compact]); - const [selectedAssets, setSelectedAssets] = useState([]); - const items = useMemo(() => { - return assetsFetch.value?.assets ?? []; - }, [assetsFetch.value]); - - useEffect(() => { - setSelectedAssets([]); - }, [assetsFetch.value?.assets]); + return dashboards ?? []; + }, [dashboards]); return ( { - setSelectedAssets(selection); + onSelectionChange: (newSelection: Dashboard[]) => { + setSelectedDashboards(newSelection); }, - selected: selectedAssets, + selected: selectedDashboards, }} /> diff --git a/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/index.tsx b/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/index.tsx new file mode 100644 index 0000000000000..43c8297d7e29d --- /dev/null +++ b/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/index.tsx @@ -0,0 +1,205 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { + EuiButton, + EuiFilterButton, + EuiFilterGroup, + EuiFlexGroup, + EuiPopover, + EuiSearchBar, + EuiSelectable, + EuiFlexItem, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { StreamDefinition } from '@kbn/streams-plugin/common'; +import React, { useMemo, useState, useCallback } from 'react'; +import { Dashboard } from '@kbn/streams-plugin/common/assets'; +import { useAbortController } from '@kbn/observability-utils-browser/hooks/use_abort_controller'; +import { useKibana } from '../../hooks/use_kibana'; +import { useStreamsAppFetch } from '../../hooks/use_streams_app_fetch'; +import { AddDashboardFlyout } from './add_dashboard_flyout'; +import { DashboardsTable } from './dashboard_table'; + +const useDashboardCrud = (id?: string) => { + const { signal } = useAbortController(); + const { + dependencies: { + start: { + streams: { streamsRepositoryClient }, + }, + }, + } = useKibana(); + + const dashboardsFetch = useStreamsAppFetch(() => { + if (!id) { + return Promise.resolve(undefined); + } + return streamsRepositoryClient.fetch('GET /api/streams/{id}/dashboards', { + signal, + params: { + path: { + id, + }, + }, + }); + }, [id, signal, streamsRepositoryClient]); + + const addDashboards = useCallback( + async (dashboards: Dashboard[]) => { + if (!id) { + return; + } + await Promise.all( + dashboards.map((dashboard) => { + return streamsRepositoryClient.fetch('PUT /api/streams/{id}/dashboards/{dashboardId}', { + signal, + params: { + path: { + id, + dashboardId: dashboard.assetId, + }, + }, + }); + }) + ); + await dashboardsFetch.refresh(); + }, + [dashboardsFetch, id, signal, streamsRepositoryClient] + ); + + const removeDashboards = useCallback( + async (dashboards: Dashboard[]) => { + if (!id) { + return; + } + await Promise.all( + dashboards.map((dashboard) => { + return streamsRepositoryClient.fetch( + 'DELETE /api/streams/{id}/dashboards/{dashboardId}', + { + signal, + params: { + path: { + id, + dashboardId: dashboard.assetId, + }, + }, + } + ); + }) + ); + await dashboardsFetch.refresh(); + }, + [dashboardsFetch, id, signal, streamsRepositoryClient] + ); + + return { + dashboardsFetch, + addDashboards, + removeDashboards, + }; +}; + +export function StreamDetailDashboardsView({ definition }: { definition?: StreamDefinition }) { + const [query, setQuery] = useState(''); + + const [isTagsPopoverOpen, setIsTagsPopoverOpen] = useState(false); + + const [isAddDashboardFlyoutOpen, setIsAddDashboardFlyoutOpen] = useState(false); + + const { dashboardsFetch, addDashboards, removeDashboards } = useDashboardCrud(definition?.id); + + const tagsButton = ( + + {i18n.translate('xpack.streams.streamDetailDashboardView.tagsFilterButtonLabel', { + defaultMessage: 'Tags', + })} + + ); + + const linkedDashboards = useMemo(() => { + return dashboardsFetch.value?.dashboards ?? []; + }, [dashboardsFetch.value?.dashboards]); + + const filteredDashboards = useMemo(() => { + return linkedDashboards.filter((dashboard) => { + return dashboard.label.toLowerCase().includes(query.toLowerCase()); + }); + }, [linkedDashboards, query]); + + const [selectedDashboards, setSelectedDashboards] = useState([]); + + return ( + + + + {selectedDashboards.length > 0 && ( + { + await removeDashboards(selectedDashboards); + setSelectedDashboards([]); + }} + color="danger" + > + {i18n.translate('xpack.streams.streamDetailDashboardView.removeSelectedButtonLabel', { + defaultMessage: 'Unlink selected', + })} + + )} + { + setQuery(nextQuery.queryText); + }} + /> + + + + + + { + setIsAddDashboardFlyoutOpen(true); + }} + > + {i18n.translate('xpack.streams.streamDetailDashboardView.addADashboardButtonLabel', { + defaultMessage: 'Add a dashboard', + })} + + + + + + {definition && isAddDashboardFlyoutOpen ? ( + { + await addDashboards(dashboards); + setIsAddDashboardFlyoutOpen(false); + }} + onClose={() => { + setIsAddDashboardFlyoutOpen(false); + }} + /> + ) : null} + + + ); +} diff --git a/x-pack/plugins/streams_app/public/components/stream_detail_view/index.tsx b/x-pack/plugins/streams_app/public/components/stream_detail_view/index.tsx index 3e726d154599b..9567552ecf1ed 100644 --- a/x-pack/plugins/streams_app/public/components/stream_detail_view/index.tsx +++ b/x-pack/plugins/streams_app/public/components/stream_detail_view/index.tsx @@ -11,7 +11,7 @@ import { useStreamsAppParams } from '../../hooks/use_streams_app_params'; import { useStreamsAppFetch } from '../../hooks/use_streams_app_fetch'; import { useKibana } from '../../hooks/use_kibana'; import { StreamDetailOverview } from '../stream_detail_overview'; -import { StreamDetailAssetView } from '../stream_detail_asset_view'; +import { StreamDetailDashboardsView } from '../stream_detail_dashboards_view'; import { StreamDetailManagement } from '../stream_detail_management'; export function StreamDetailView() { @@ -56,10 +56,10 @@ export function StreamDetailView() { }), }, { - name: 'assets', - content: , - label: i18n.translate('xpack.streams.streamDetailView.assetTab', { - defaultMessage: 'Assets', + name: 'dashboards', + content: , + label: i18n.translate('xpack.streams.streamDetailView.dashboardsTab', { + defaultMessage: 'Dashboards', }), }, { From 17b955dcdddfd7d3f1a1bbd631065ea420905284 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 3 Dec 2024 16:58:17 +0100 Subject: [PATCH 74/95] add assets everywhere --- .../saved_objects_tagging/kibana.jsonc | 2 +- x-pack/plugins/streams/common/assets.ts | 6 + x-pack/plugins/streams/common/types.ts | 1 + .../server/lib/streams/assets/asset_client.ts | 88 +++++++++++- .../streams/server/lib/streams/stream_crud.ts | 41 +++++- x-pack/plugins/streams/server/plugin.ts | 3 +- .../streams/server/routes/dashboards/route.ts | 27 +++- .../streams/server/routes/streams/delete.ts | 14 +- .../streams/server/routes/streams/edit.ts | 8 +- .../streams/server/routes/streams/enable.ts | 3 +- .../streams/server/routes/streams/fork.ts | 4 +- .../streams/server/routes/streams/read.ts | 9 +- .../streams/server/routes/streams/resync.ts | 3 +- x-pack/plugins/streams/server/routes/types.ts | 2 + x-pack/plugins/streams_app/kibana.jsonc | 3 +- .../add_dashboard_flyout.tsx | 125 +++++++++++++++--- .../dashboard_table.tsx | 45 ++++--- .../stream_detail_dashboards_view/index.tsx | 49 +++---- .../to_reference_list.ts | 16 +++ x-pack/plugins/streams_app/public/types.ts | 2 + 20 files changed, 355 insertions(+), 96 deletions(-) create mode 100644 x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/to_reference_list.ts diff --git a/x-pack/plugins/saved_objects_tagging/kibana.jsonc b/x-pack/plugins/saved_objects_tagging/kibana.jsonc index 3a2cdef308de0..1968ff076047e 100644 --- a/x-pack/plugins/saved_objects_tagging/kibana.jsonc +++ b/x-pack/plugins/saved_objects_tagging/kibana.jsonc @@ -5,7 +5,7 @@ "@elastic/appex-sharedux" ], "group": "platform", - "visibility": "private", + "visibility": "shared", "plugin": { "id": "savedObjectsTagging", "browser": true, diff --git a/x-pack/plugins/streams/common/assets.ts b/x-pack/plugins/streams/common/assets.ts index f66e48f2ff96c..ac0a003460abc 100644 --- a/x-pack/plugins/streams/common/assets.ts +++ b/x-pack/plugins/streams/common/assets.ts @@ -37,6 +37,12 @@ export interface Dashboard extends Asset { type: typeof ASSET_TYPES.Dashboard; } +export interface ReadDashboard { + id: string; + label: string; + tags: string[]; +} + export interface Slo extends Asset { type: typeof ASSET_TYPES.Slo; } diff --git a/x-pack/plugins/streams/common/types.ts b/x-pack/plugins/streams/common/types.ts index c2d99d4ba1d89..f051b5b5eb356 100644 --- a/x-pack/plugins/streams/common/types.ts +++ b/x-pack/plugins/streams/common/types.ts @@ -88,6 +88,7 @@ export const streamWithoutIdDefinitonSchema = z.object({ }) ) .default([]), + dashboards: z.optional(z.array(z.string()).default([])), }); export type StreamWithoutIdDefinition = z.infer; diff --git a/x-pack/plugins/streams/server/lib/streams/assets/asset_client.ts b/x-pack/plugins/streams/server/lib/streams/assets/asset_client.ts index bb5b174d71e13..578a1ea3ee9b2 100644 --- a/x-pack/plugins/streams/server/lib/streams/assets/asset_client.ts +++ b/x-pack/plugins/streams/server/lib/streams/assets/asset_client.ts @@ -21,7 +21,7 @@ import { Dashboard, Slo, } from '../../../../common/assets'; -import { ASSET_ENTITY_ID, ASSET_ENTITY_TYPE } from './fields'; +import { ASSET_ENTITY_ID, ASSET_ENTITY_TYPE, ASSET_TYPE } from './fields'; function sloSavedObjectToAsset( sloId: string, @@ -93,6 +93,62 @@ export class AssetClient { }); } + async syncAssetList({ + entityId, + entityType, + assetType, + assetIds, + }: { + entityId: string; + entityType: string; + assetType: AssetType; + assetIds: string[]; + }) { + const assetsResponse = await this.clients.storageClient.search('get_assets_for_entity', { + size: 10_000, + track_total_hits: false, + query: { + bool: { + filter: [ + ...termQuery(ASSET_ENTITY_ID, entityId), + ...termQuery(ASSET_ENTITY_TYPE, entityType), + ...termQuery(ASSET_TYPE, assetType), + ], + }, + }, + }); + + const existingAssetLinks = assetsResponse.hits.hits.map((hit) => hit._source); + + const missingAssetIds = assetIds.filter( + (assetId) => + !existingAssetLinks.some((existingAssetLink) => existingAssetLink['asset.id'] === assetId) + ); + + const tooMuchAssetIds = existingAssetLinks + .map((existingAssetLink) => existingAssetLink['asset.id']) + .filter((assetId) => !assetIds.includes(assetId)); + + await Promise.all([ + ...missingAssetIds.map((assetId) => + this.linkAsset({ + entityId, + entityType, + assetId, + assetType, + }) + ), + ...tooMuchAssetIds.map((assetId) => + this.unlinkAsset({ + entityId, + entityType, + assetId, + assetType, + }) + ), + ]); + } + async unlinkAsset({ entityId, entityType, @@ -114,6 +170,32 @@ export class AssetClient { await this.clients.storageClient.delete(id); } + async getAssetIds({ + entityId, + entityType, + assetType, + }: { + entityId: string; + entityType: 'stream'; + assetType: AssetType; + }): Promise { + const assetsResponse = await this.clients.storageClient.search('get_assets_for_entity', { + size: 10_000, + track_total_hits: false, + query: { + bool: { + filter: [ + ...termQuery(ASSET_ENTITY_ID, entityId), + ...termQuery(ASSET_ENTITY_TYPE, entityType), + ...termQuery(ASSET_TYPE, assetType), + ], + }, + }, + }); + + return assetsResponse.hits.hits.map((hit) => hit._source['asset.id']); + } + async getAssets({ entityId, entityType, @@ -210,16 +292,20 @@ export class AssetClient { entityType, query, assetType, + tags, }: { entityId: string; entityType: string; query: string; + tags?: string[]; assetType: T; }): Promise> { if (assetType === 'dashboard') { const dashboardSavedObjects = await this.clients.soClient.find<{ title: string }>({ type: 'dashboard', search: query, + hasReferenceOperator: 'OR', + hasReference: tags?.map((tag) => ({ type: 'tag', id: tag })), }); return dashboardSavedObjects.saved_objects.map((dashboardSavedObject) => { diff --git a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts index 245e06e8b4573..e136f57b7df2d 100644 --- a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts +++ b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts @@ -30,6 +30,7 @@ import { upsertIngestPipeline, } from './ingest_pipelines/manage_ingest_pipelines'; import { getProcessingPipelineName, getReroutePipelineName } from './ingest_pipelines/name'; +import { AssetClient } from './assets/asset_client'; interface BaseParams { scopedClusterClient: IScopedClusterClient; @@ -42,9 +43,15 @@ interface BaseParamsWithDefinition extends BaseParams { interface DeleteStreamParams extends BaseParams { id: string; logger: Logger; + assetClient: AssetClient; } -export async function deleteStreamObjects({ id, scopedClusterClient, logger }: DeleteStreamParams) { +export async function deleteStreamObjects({ + id, + scopedClusterClient, + logger, + assetClient, +}: DeleteStreamParams) { await deleteDataStream({ esClient: scopedClusterClient.asCurrentUser, name: id, @@ -70,6 +77,12 @@ export async function deleteStreamObjects({ id, scopedClusterClient, logger }: D id: getReroutePipelineName(id), logger, }); + await assetClient.syncAssetList({ + entityId: id, + entityType: 'stream', + assetType: 'dashboard', + assetIds: [], + }); await scopedClusterClient.asInternalUser.delete({ id, index: STREAMS_INDEX, @@ -77,7 +90,10 @@ export async function deleteStreamObjects({ id, scopedClusterClient, logger }: D }); } -async function upsertInternalStream({ definition, scopedClusterClient }: BaseParamsWithDefinition) { +async function upsertInternalStream({ + definition: { dashboards, ...definition }, + scopedClusterClient, +}: BaseParamsWithDefinition) { return scopedClusterClient.asInternalUser.index({ id: definition.id, index: STREAMS_INDEX, @@ -86,6 +102,21 @@ async function upsertInternalStream({ definition, scopedClusterClient }: BasePar }); } +async function syncAssets({ + definition, + assetClient, +}: { + definition: StreamDefinition; + assetClient: AssetClient; +}) { + await assetClient.syncAssetList({ + entityId: definition.id, + entityType: 'stream', + assetType: 'dashboard', + assetIds: definition.dashboards ?? [], + }); +} + type ListStreamsParams = BaseParams; export interface ListStreamResponse { @@ -355,6 +386,7 @@ export async function checkReadAccess({ interface SyncStreamParams { scopedClusterClient: IScopedClusterClient; + assetClient: AssetClient; definition: StreamDefinition; rootDefinition?: StreamDefinition; logger: Logger; @@ -362,6 +394,7 @@ interface SyncStreamParams { export async function syncStream({ scopedClusterClient, + assetClient, definition, rootDefinition, logger, @@ -413,6 +446,10 @@ export async function syncStream({ scopedClusterClient, definition, }); + await syncAssets({ + definition, + assetClient, + }); await rolloverDataStreamIfNecessary({ esClient: scopedClusterClient.asCurrentUser, name: definition.id, diff --git a/x-pack/plugins/streams/server/plugin.ts b/x-pack/plugins/streams/server/plugin.ts index 32bc253897bce..a40f1a9f2af2a 100644 --- a/x-pack/plugins/streams/server/plugin.ts +++ b/x-pack/plugins/streams/server/plugin.ts @@ -70,9 +70,10 @@ export class StreamsPlugin server: this.server, getScopedClients: async ({ request }: { request: KibanaRequest }) => { const [coreStart] = await core.getStartServices(); + const assetClient = await assetService.getClientWithRequest({ request }); const scopedClusterClient = coreStart.elasticsearch.client.asScoped(request); const soClient = coreStart.savedObjects.getScopedClient(request); - return { scopedClusterClient, soClient }; + return { scopedClusterClient, soClient, assetClient }; }, }, core, diff --git a/x-pack/plugins/streams/server/routes/dashboards/route.ts b/x-pack/plugins/streams/server/routes/dashboards/route.ts index 991c6d4bd5f47..28f95d9c9a479 100644 --- a/x-pack/plugins/streams/server/routes/dashboards/route.ts +++ b/x-pack/plugins/streams/server/routes/dashboards/route.ts @@ -6,11 +6,11 @@ */ import { z } from '@kbn/zod'; -import { Asset, Dashboard } from '../../../common/assets'; +import { Asset, Dashboard, ReadDashboard } from '../../../common/assets'; import { createServerRoute } from '../create_server_route'; export interface ListDashboardsResponse { - dashboards: Dashboard[]; + dashboards: ReadDashboard[]; } export interface LinkDashboardResponse { @@ -23,7 +23,7 @@ export interface UnlinkDashboardResponse { export interface SuggestDashboardResponse { suggestions: Array<{ - dashboard: Dashboard; + dashboard: ReadDashboard; }>; } @@ -54,7 +54,13 @@ export const listDashboardsRoute = createServerRoute({ entityId: streamId, entityType: 'stream', }) - ).filter(isDashboard), + ) + .filter(isDashboard) + .map((asset) => ({ + id: asset.assetId, + label: asset.label, + tags: asset.tags, + })), }; }, }); @@ -122,7 +128,7 @@ export const unlinkDashboardRoute = createServerRoute({ }); export const suggestDashboardsRoute = createServerRoute({ - endpoint: 'GET /api/streams/{id}/dashboards/_suggestions', + endpoint: 'POST /api/streams/{id}/dashboards/_suggestions', options: { access: 'internal', }, @@ -133,6 +139,9 @@ export const suggestDashboardsRoute = createServerRoute({ query: z.object({ query: z.string(), }), + body: z.object({ + tags: z.optional(z.array(z.string())), + }), }), handler: async ({ params, request, assets }): Promise => { const assetsClient = await assets.getClientWithRequest({ request }); @@ -140,6 +149,7 @@ export const suggestDashboardsRoute = createServerRoute({ const { path: { id: streamId }, query: { query }, + body: { tags }, } = params; const suggestions = ( @@ -148,9 +158,14 @@ export const suggestDashboardsRoute = createServerRoute({ entityType: 'stream', assetType: 'dashboard', query, + tags, }) ).map(({ asset: dashboard }) => ({ - dashboard, + dashboard: { + id: dashboard.assetId, + label: dashboard.label, + tags: dashboard.tags, + }, })); return { diff --git a/x-pack/plugins/streams/server/routes/streams/delete.ts b/x-pack/plugins/streams/server/routes/streams/delete.ts index a2092838792cf..49b081e3cc65d 100644 --- a/x-pack/plugins/streams/server/routes/streams/delete.ts +++ b/x-pack/plugins/streams/server/routes/streams/delete.ts @@ -19,6 +19,7 @@ import { createServerRoute } from '../create_server_route'; import { syncStream, readStream, deleteStreamObjects } from '../../lib/streams/stream_crud'; import { MalformedStreamId } from '../../lib/streams/errors/malformed_stream_id'; import { getParentId } from '../../lib/streams/helpers/hierarchy'; +import { AssetClient } from '../../lib/streams/assets/asset_client'; export const deleteStreamRoute = createServerRoute({ endpoint: 'DELETE /api/streams/{id}', @@ -45,7 +46,7 @@ export const deleteStreamRoute = createServerRoute({ getScopedClients, }): Promise<{ acknowledged: true }> => { try { - const { scopedClusterClient } = await getScopedClients({ request }); + const { scopedClusterClient, assetClient } = await getScopedClients({ request }); const parentId = getParentId(params.path.id); if (!parentId) { @@ -53,9 +54,9 @@ export const deleteStreamRoute = createServerRoute({ } // need to update parent first to cut off documents streaming down - await updateParentStream(scopedClusterClient, params.path.id, parentId, logger); + await updateParentStream(scopedClusterClient, assetClient, params.path.id, parentId, logger); - await deleteStream(scopedClusterClient, params.path.id, logger); + await deleteStream(scopedClusterClient, assetClient, params.path.id, logger); return { acknowledged: true }; } catch (e) { @@ -78,15 +79,16 @@ export const deleteStreamRoute = createServerRoute({ export async function deleteStream( scopedClusterClient: IScopedClusterClient, + assetClient: AssetClient, id: string, logger: Logger ) { try { const { definition } = await readStream({ scopedClusterClient, id }); for (const child of definition.children) { - await deleteStream(scopedClusterClient, child.id, logger); + await deleteStream(scopedClusterClient, assetClient, child.id, logger); } - await deleteStreamObjects({ scopedClusterClient, id, logger }); + await deleteStreamObjects({ scopedClusterClient, id, logger, assetClient }); } catch (e) { if (e instanceof DefinitionNotFound) { logger.debug(`Stream definition for ${id} not found.`); @@ -98,6 +100,7 @@ export async function deleteStream( async function updateParentStream( scopedClusterClient: IScopedClusterClient, + assetClient: AssetClient, id: string, parentId: string, logger: Logger @@ -111,6 +114,7 @@ async function updateParentStream( await syncStream({ scopedClusterClient, + assetClient, definition: parentDefinition, logger, }); diff --git a/x-pack/plugins/streams/server/routes/streams/edit.ts b/x-pack/plugins/streams/server/routes/streams/edit.ts index 6125aa2470b94..816193814b3a1 100644 --- a/x-pack/plugins/streams/server/routes/streams/edit.ts +++ b/x-pack/plugins/streams/server/routes/streams/edit.ts @@ -27,6 +27,7 @@ import { import { MalformedStreamId } from '../../lib/streams/errors/malformed_stream_id'; import { getParentId } from '../../lib/streams/helpers/hierarchy'; import { MalformedChildren } from '../../lib/streams/errors/malformed_children'; +import { AssetClient } from '../../lib/streams/assets/asset_client'; export const editStreamRoute = createServerRoute({ endpoint: 'PUT /api/streams/{id}', @@ -48,7 +49,7 @@ export const editStreamRoute = createServerRoute({ }), handler: async ({ response, params, logger, request, getScopedClients }) => { try { - const { scopedClusterClient } = await getScopedClients({ request }); + const { scopedClusterClient, assetClient } = await getScopedClients({ request }); await validateStreamChildren(scopedClusterClient, params.path.id, params.body.children); await validateAncestorFields(scopedClusterClient, params.path.id, params.body.fields); @@ -81,6 +82,7 @@ export const editStreamRoute = createServerRoute({ await syncStream({ scopedClusterClient, + assetClient, definition: childDefinition, logger, }); @@ -88,6 +90,7 @@ export const editStreamRoute = createServerRoute({ await syncStream({ scopedClusterClient, + assetClient, definition: { ...streamDefinition, id: params.path.id, managed: true }, rootDefinition: parentDefinition, logger, @@ -96,6 +99,7 @@ export const editStreamRoute = createServerRoute({ if (parentId) { parentDefinition = await updateParentStream( scopedClusterClient, + assetClient, parentId, params.path.id, logger @@ -123,6 +127,7 @@ export const editStreamRoute = createServerRoute({ async function updateParentStream( scopedClusterClient: IScopedClusterClient, + assetClient: AssetClient, parentId: string, id: string, logger: Logger @@ -141,6 +146,7 @@ async function updateParentStream( await syncStream({ scopedClusterClient, + assetClient, definition: parentDefinition, logger, }); diff --git a/x-pack/plugins/streams/server/routes/streams/enable.ts b/x-pack/plugins/streams/server/routes/streams/enable.ts index cfcb97f9b3581..8cbd2d99d3fed 100644 --- a/x-pack/plugins/streams/server/routes/streams/enable.ts +++ b/x-pack/plugins/streams/server/routes/streams/enable.ts @@ -33,7 +33,7 @@ export const enableStreamsRoute = createServerRoute({ getScopedClients, }): Promise<{ acknowledged: true }> => { try { - const { scopedClusterClient } = await getScopedClients({ request }); + const { scopedClusterClient, assetClient } = await getScopedClients({ request }); const alreadyEnabled = await streamsEnabled({ scopedClusterClient }); if (alreadyEnabled) { return { acknowledged: true }; @@ -41,6 +41,7 @@ export const enableStreamsRoute = createServerRoute({ await createStreamsIndex(scopedClusterClient); await syncStream({ scopedClusterClient, + assetClient, definition: rootStreamDefinition, logger, }); diff --git a/x-pack/plugins/streams/server/routes/streams/fork.ts b/x-pack/plugins/streams/server/routes/streams/fork.ts index a4d846ceccb35..015acaf3ccecc 100644 --- a/x-pack/plugins/streams/server/routes/streams/fork.ts +++ b/x-pack/plugins/streams/server/routes/streams/fork.ts @@ -48,7 +48,7 @@ export const forkStreamsRoute = createServerRoute({ throw new ForkConditionMissing('You must provide a condition to fork a stream'); } - const { scopedClusterClient } = await getScopedClients({ request }); + const { scopedClusterClient, assetClient } = await getScopedClients({ request }); const { definition: rootDefinition } = await readStream({ scopedClusterClient, @@ -79,6 +79,7 @@ export const forkStreamsRoute = createServerRoute({ // need to create the child first, otherwise we risk streaming data even though the child data stream is not ready await syncStream({ scopedClusterClient, + assetClient, definition: childDefinition, rootDefinition, logger, @@ -91,6 +92,7 @@ export const forkStreamsRoute = createServerRoute({ await syncStream({ scopedClusterClient, + assetClient, definition: rootDefinition, rootDefinition, logger, diff --git a/x-pack/plugins/streams/server/routes/streams/read.ts b/x-pack/plugins/streams/server/routes/streams/read.ts index 5c503e2b7e625..fa874298af36e 100644 --- a/x-pack/plugins/streams/server/routes/streams/read.ts +++ b/x-pack/plugins/streams/server/routes/streams/read.ts @@ -39,15 +39,21 @@ export const readStreamRoute = createServerRoute({ } > => { try { - const { scopedClusterClient } = await getScopedClients({ request }); + const { scopedClusterClient, assetClient } = await getScopedClients({ request }); const streamEntity = await readStream({ scopedClusterClient, id: params.path.id, }); + const dashboards = await assetClient.getAssetIds({ + entityId: streamEntity.definition.id, + entityType: 'stream', + assetType: 'dashboard', + }); if (streamEntity.definition.managed === false) { return { ...streamEntity.definition, + dashboards, inheritedFields: [], }; } @@ -62,6 +68,7 @@ export const readStreamRoute = createServerRoute({ inheritedFields: ancestors.flatMap(({ definition: { id, fields } }) => fields.map((field) => ({ ...field, from: id })) ), + dashboards, }; return body; diff --git a/x-pack/plugins/streams/server/routes/streams/resync.ts b/x-pack/plugins/streams/server/routes/streams/resync.ts index 8e520410ca5c2..a10a3846fd2d5 100644 --- a/x-pack/plugins/streams/server/routes/streams/resync.ts +++ b/x-pack/plugins/streams/server/routes/streams/resync.ts @@ -28,7 +28,7 @@ export const resyncStreamsRoute = createServerRoute({ request, getScopedClients, }): Promise<{ acknowledged: true }> => { - const { scopedClusterClient } = await getScopedClients({ request }); + const { scopedClusterClient, assetClient } = await getScopedClients({ request }); const { definitions: streams } = await listStreams({ scopedClusterClient }); @@ -40,6 +40,7 @@ export const resyncStreamsRoute = createServerRoute({ await syncStream({ scopedClusterClient, + assetClient, definition, logger, }); diff --git a/x-pack/plugins/streams/server/routes/types.ts b/x-pack/plugins/streams/server/routes/types.ts index 77248238fcb47..3a9f903120e73 100644 --- a/x-pack/plugins/streams/server/routes/types.ts +++ b/x-pack/plugins/streams/server/routes/types.ts @@ -11,6 +11,7 @@ import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; import { StreamsServer } from '../types'; import { AssetService } from '../lib/streams/assets/asset_service'; +import { AssetClient } from '../lib/streams/assets/asset_client'; export interface RouteDependencies { assets: AssetService; @@ -18,6 +19,7 @@ export interface RouteDependencies { getScopedClients: ({ request }: { request: KibanaRequest }) => Promise<{ scopedClusterClient: IScopedClusterClient; soClient: SavedObjectsClientContract; + assetClient: AssetClient; }>; } diff --git a/x-pack/plugins/streams_app/kibana.jsonc b/x-pack/plugins/streams_app/kibana.jsonc index 16666084c53e5..65dddc0f92266 100644 --- a/x-pack/plugins/streams_app/kibana.jsonc +++ b/x-pack/plugins/streams_app/kibana.jsonc @@ -15,7 +15,8 @@ "data", "dataViews", "unifiedSearch", - "share" + "share", + "savedObjectsTagging" ], "requiredBundles": [ "kibanaReact" diff --git a/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/add_dashboard_flyout.tsx b/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/add_dashboard_flyout.tsx index d00f6d8b6855c..09b9f7e1e7e44 100644 --- a/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/add_dashboard_flyout.tsx +++ b/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/add_dashboard_flyout.tsx @@ -6,17 +6,24 @@ */ import { EuiButton, + EuiFilterButton, + EuiFilterGroup, EuiFlexGroup, + EuiFlexItem, EuiFlyout, EuiFlyoutBody, EuiFlyoutFooter, EuiFlyoutHeader, + EuiPopover, + EuiPopoverTitle, EuiSearchBar, + EuiSelectable, EuiText, EuiTitle, + useGeneratedHtmlId, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import type { Dashboard } from '@kbn/streams-plugin/common/assets'; +import type { ReadDashboard } from '@kbn/streams-plugin/common/assets'; import { debounce } from 'lodash'; import React, { useMemo, useState, useEffect } from 'react'; import { useKibana } from '../../hooks/use_kibana'; @@ -30,14 +37,15 @@ export function AddDashboardFlyout({ onClose, }: { entityId: string; - onAddDashboards: (dashboard: Dashboard[]) => void; - linkedDashboards: Dashboard[]; + onAddDashboards: (dashboard: ReadDashboard[]) => Promise; + linkedDashboards: ReadDashboard[]; onClose: () => void; }) { const { dependencies: { start: { streams: { streamsRepositoryClient }, + savedObjectsTagging: { ui: savedObjectsTaggingUi }, }, }, } = useKibana(); @@ -45,6 +53,10 @@ export function AddDashboardFlyout({ const [query, setQuery] = useState(''); const [submittedQuery, setSubmittedQuery] = useState(query); + const [selectedDashboards, setSelectedDashboards] = useState([]); + const [isLoading, setIsLoading] = useState(false); + const [selectedTags, setSelectedTags] = useState([]); + const [isPopoverOpen, setIsPopoverOpen] = useState(false); const setSubmittedQueryDebounced = useMemo(() => { return debounce(setSubmittedQuery, 150); @@ -53,7 +65,7 @@ export function AddDashboardFlyout({ const dashboardSuggestionsFetch = useStreamsAppFetch( ({ signal }) => { return streamsRepositoryClient - .fetch('GET /api/streams/{id}/dashboards/_suggestions', { + .fetch('POST /api/streams/{id}/dashboards/_suggestions', { signal, params: { path: { @@ -62,6 +74,9 @@ export function AddDashboardFlyout({ query: { query: submittedQuery, }, + body: { + tags: selectedTags, + }, }, }) .then(({ suggestions }) => { @@ -70,16 +85,36 @@ export function AddDashboardFlyout({ .map((suggestion) => suggestion.dashboard) .filter((dashboard) => { return !linkedDashboards.find( - (linkedDashboard) => linkedDashboard.assetId === dashboard.assetId + (linkedDashboard) => linkedDashboard.id === dashboard.id ); }), }; }); }, - [streamsRepositoryClient, entityId, submittedQuery, linkedDashboards] + [streamsRepositoryClient, entityId, submittedQuery, selectedTags, linkedDashboards] + ); + + const tagList = savedObjectsTaggingUi.getTagList(); + + const button = ( + setIsPopoverOpen(!isPopoverOpen)} + isSelected={isPopoverOpen} + numFilters={tagList.length} + hasActiveFilters={selectedTags.length > 0} + numActiveFilters={selectedTags.length} + > + {i18n.translate('xpack.streams.addDashboardFlyout.filterButtonLabel', { + defaultMessage: 'Tags', + })} + ); - const [selectedDashboards, setSelectedDashboards] = useState([]); + const filterGroupPopoverId = useGeneratedHtmlId({ + prefix: 'filterGroupPopover', + }); useEffect(() => { setSelectedDashboards([]); @@ -112,16 +147,61 @@ export function AddDashboardFlyout({ })} - { - setQuery(queryText); - setSubmittedQueryDebounced(queryText); - }} - /> + + { + setQuery(queryText); + setSubmittedQueryDebounced(queryText); + }} + /> + + + + setIsPopoverOpen(false)} + panelPaddingSize="none" + > + ({ + label: tag.name, + checked: selectedTags.includes(tag.id) ? 'on' : undefined, + }))} + onChange={(newOptions) => { + setSelectedTags( + newOptions + .filter((option) => option.checked === 'on') + .map((option) => savedObjectsTaggingUi.getTagIdFromName(option.label)!) + ); + }} + > + {(list, search) => ( +
+ {search} + {list} +
+ )} + + + + { - onAddDashboards(selectedDashboards); + onClick={async () => { + setIsLoading(true); + try { + await onAddDashboards(selectedDashboards); + } finally { + setIsLoading(false); + } }} > {i18n.translate('xpack.streams.addDashboardFlyout.addDashboardsButtonLabel', { diff --git a/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/dashboard_table.tsx b/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/dashboard_table.tsx index 20bacf1013eae..f4f1bb953a891 100644 --- a/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/dashboard_table.tsx +++ b/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/dashboard_table.tsx @@ -4,16 +4,12 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { - EuiBadge, - EuiBasicTable, - EuiBasicTableColumn, - EuiFlexGroup, - EuiFlexItem, -} from '@elastic/eui'; +import { EuiBasicTable, EuiBasicTableColumn, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import { Dashboard } from '@kbn/streams-plugin/common/assets'; +import { ReadDashboard } from '@kbn/streams-plugin/common/assets'; import React, { useMemo } from 'react'; +import { useKibana } from '../../hooks/use_kibana'; +import { tagListToReferenceList } from './to_reference_list'; export function DashboardsTable({ dashboards, @@ -23,12 +19,19 @@ export function DashboardsTable({ loading, }: { loading: boolean; - dashboards: Dashboard[] | undefined; + dashboards: ReadDashboard[] | undefined; compact?: boolean; - selecedDashboards: Dashboard[]; - setSelectedDashboards: (dashboards: Dashboard[]) => void; + selecedDashboards: ReadDashboard[]; + setSelectedDashboards: (dashboards: ReadDashboard[]) => void; }) { - const columns = useMemo((): Array> => { + const { + dependencies: { + start: { + savedObjectsTagging: { ui: savedObjectsTaggingUi }, + }, + }, + } = useKibana(); + const columns = useMemo((): Array> => { return [ { field: 'label', @@ -45,20 +48,18 @@ export function DashboardsTable({ }), render: (_, { tags }) => { return ( - - {tags.map((tag) => ( - - {tag} - - ))} + + ); }, }, - ] satisfies Array>) + ] satisfies Array>) : []), ]; - }, [compact]); + }, [compact, savedObjectsTaggingUi]); const items = useMemo(() => { return dashboards ?? []; @@ -69,11 +70,11 @@ export function DashboardsTable({ { + onSelectionChange: (newSelection: ReadDashboard[]) => { setSelectedDashboards(newSelection); }, selected: selectedDashboards, diff --git a/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/index.tsx b/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/index.tsx index 43c8297d7e29d..6f28c642ac559 100644 --- a/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/index.tsx +++ b/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/index.tsx @@ -4,20 +4,11 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { - EuiButton, - EuiFilterButton, - EuiFilterGroup, - EuiFlexGroup, - EuiPopover, - EuiSearchBar, - EuiSelectable, - EuiFlexItem, -} from '@elastic/eui'; +import { EuiButton, EuiFlexGroup, EuiSearchBar, EuiFlexItem } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { StreamDefinition } from '@kbn/streams-plugin/common'; import React, { useMemo, useState, useCallback } from 'react'; -import { Dashboard } from '@kbn/streams-plugin/common/assets'; +import { ReadDashboard } from '@kbn/streams-plugin/common/assets'; import { useAbortController } from '@kbn/observability-utils-browser/hooks/use_abort_controller'; import { useKibana } from '../../hooks/use_kibana'; import { useStreamsAppFetch } from '../../hooks/use_streams_app_fetch'; @@ -49,7 +40,7 @@ const useDashboardCrud = (id?: string) => { }, [id, signal, streamsRepositoryClient]); const addDashboards = useCallback( - async (dashboards: Dashboard[]) => { + async (dashboards: ReadDashboard[]) => { if (!id) { return; } @@ -60,7 +51,7 @@ const useDashboardCrud = (id?: string) => { params: { path: { id, - dashboardId: dashboard.assetId, + dashboardId: dashboard.id, }, }, }); @@ -72,7 +63,7 @@ const useDashboardCrud = (id?: string) => { ); const removeDashboards = useCallback( - async (dashboards: Dashboard[]) => { + async (dashboards: ReadDashboard[]) => { if (!id) { return; } @@ -85,7 +76,7 @@ const useDashboardCrud = (id?: string) => { params: { path: { id, - dashboardId: dashboard.assetId, + dashboardId: dashboard.id, }, }, } @@ -107,20 +98,11 @@ const useDashboardCrud = (id?: string) => { export function StreamDetailDashboardsView({ definition }: { definition?: StreamDefinition }) { const [query, setQuery] = useState(''); - const [isTagsPopoverOpen, setIsTagsPopoverOpen] = useState(false); - const [isAddDashboardFlyoutOpen, setIsAddDashboardFlyoutOpen] = useState(false); const { dashboardsFetch, addDashboards, removeDashboards } = useDashboardCrud(definition?.id); - const tagsButton = ( - - {i18n.translate('xpack.streams.streamDetailDashboardView.tagsFilterButtonLabel', { - defaultMessage: 'Tags', - })} - - ); - + const [isUnlinkLoading, setIsUnlinkLoading] = useState(false); const linkedDashboards = useMemo(() => { return dashboardsFetch.value?.dashboards ?? []; }, [dashboardsFetch.value?.dashboards]); @@ -131,7 +113,7 @@ export function StreamDetailDashboardsView({ definition }: { definition?: Stream }); }, [linkedDashboards, query]); - const [selectedDashboards, setSelectedDashboards] = useState([]); + const [selectedDashboards, setSelectedDashboards] = useState([]); return ( @@ -141,9 +123,15 @@ export function StreamDetailDashboardsView({ definition }: { definition?: Stream { - await removeDashboards(selectedDashboards); - setSelectedDashboards([]); + try { + setIsUnlinkLoading(true); + await removeDashboards(selectedDashboards); + setSelectedDashboards([]); + } finally { + setIsUnlinkLoading(false); + } }} color="danger" > @@ -161,11 +149,6 @@ export function StreamDetailDashboardsView({ definition }: { definition?: Stream setQuery(nextQuery.queryText); }} /> - - - - - ({ + id: tag, + name: 'tag', + type: 'tag', + })); +} diff --git a/x-pack/plugins/streams_app/public/types.ts b/x-pack/plugins/streams_app/public/types.ts index 58d44784fe031..47093ba013530 100644 --- a/x-pack/plugins/streams_app/public/types.ts +++ b/x-pack/plugins/streams_app/public/types.ts @@ -16,6 +16,7 @@ import type { import type { StreamsPluginSetup, StreamsPluginStart } from '@kbn/streams-plugin/public'; import type { UnifiedSearchPublicPluginStart } from '@kbn/unified-search-plugin/public'; import type { SharePublicSetup, SharePublicStart } from '@kbn/share-plugin/public/plugin'; +import type { SavedObjectTaggingPluginStart } from '@kbn/saved-objects-tagging-plugin/public'; /* eslint-disable @typescript-eslint/no-empty-interface*/ export interface ConfigSchema {} @@ -36,6 +37,7 @@ export interface StreamsAppStartDependencies { observabilityShared: ObservabilitySharedPluginStart; unifiedSearch: UnifiedSearchPublicPluginStart; share: SharePublicStart; + savedObjectsTagging: SavedObjectTaggingPluginStart; } export interface StreamsAppPublicSetup {} From a819748ce8d76c19adcfb4cf326b8e2e0ed300ca Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 3 Dec 2024 16:27:14 +0000 Subject: [PATCH 75/95] [CI] Auto-commit changed files from 'node scripts/notice' --- .../observability_utils/observability_utils_server/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/tsconfig.json b/x-pack/packages/observability/observability_utils/observability_utils_server/tsconfig.json index f6dd781184b86..92c4635cd90ff 100644 --- a/x-pack/packages/observability/observability_utils/observability_utils_server/tsconfig.json +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/tsconfig.json @@ -26,5 +26,6 @@ "@kbn/rule-data-utils", "@kbn/utility-types", "@kbn/task-manager-plugin", + "@kbn/es-errors", ] } From 1abbf61b1e7ea4c8f61fdc0879ddee6a8506bdd8 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 3 Dec 2024 16:28:12 +0000 Subject: [PATCH 76/95] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/plugins/streams/tsconfig.json | 3 ++- x-pack/plugins/streams_app/tsconfig.json | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/streams/tsconfig.json b/x-pack/plugins/streams/tsconfig.json index 3f863145f4d22..31cfdcf7e99ae 100644 --- a/x-pack/plugins/streams/tsconfig.json +++ b/x-pack/plugins/streams/tsconfig.json @@ -29,6 +29,7 @@ "@kbn/licensing-plugin", "@kbn/server-route-repository-client", "@kbn/observability-utils-server", - "@kbn/observability-utils-common" + "@kbn/observability-utils-common", + "@kbn/alerting-plugin" ] } diff --git a/x-pack/plugins/streams_app/tsconfig.json b/x-pack/plugins/streams_app/tsconfig.json index 39acb94665ae5..47bfb2df6f1cc 100644 --- a/x-pack/plugins/streams_app/tsconfig.json +++ b/x-pack/plugins/streams_app/tsconfig.json @@ -33,5 +33,7 @@ "@kbn/streams-plugin", "@kbn/share-plugin", "@kbn/observability-utils-server", + "@kbn/server-route-repository-client", + "@kbn/saved-objects-tagging-plugin", ] } From e5e2cc03064122dab3f0fc30332ae60623becfde Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 4 Dec 2024 13:35:07 +0100 Subject: [PATCH 77/95] fix --- x-pack/plugins/streams/server/types.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/streams/server/types.ts b/x-pack/plugins/streams/server/types.ts index 85073f027f143..63ed5328082a7 100644 --- a/x-pack/plugins/streams/server/types.ts +++ b/x-pack/plugins/streams/server/types.ts @@ -16,10 +16,7 @@ import type { TaskManagerSetupContract, TaskManagerStartContract, } from '@kbn/task-manager-plugin/server'; -import type { - PluginSetupContract as AlertingPluginSetup, - PluginStartContract as AlertingPluginStart, -} from '@kbn/alerting-plugin/server'; +import type { AlertingServerSetup, AlertingServerStart } from '@kbn/alerting-plugin/server'; import type { StreamsConfig } from '../common/config'; export interface StreamsServer { @@ -39,7 +36,7 @@ export interface ElasticsearchAccessorOptions { export interface StreamsPluginSetupDependencies { encryptedSavedObjects: EncryptedSavedObjectsPluginSetup; taskManager: TaskManagerSetupContract; - alerting: AlertingPluginSetup; + alerting: AlertingServerSetup; } export interface StreamsPluginStartDependencies { @@ -47,5 +44,5 @@ export interface StreamsPluginStartDependencies { encryptedSavedObjects: EncryptedSavedObjectsPluginStart; licensing: LicensingPluginStart; taskManager: TaskManagerStartContract; - alerting: AlertingPluginStart; + alerting: AlertingServerStart; } From d20fff0b40fdfe91d34c6560ab614d4969095e39 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 4 Dec 2024 13:43:50 +0100 Subject: [PATCH 78/95] fix types --- x-pack/plugins/streams/common/types.ts | 1 + .../server/lib/streams/assets/asset_service.ts | 11 +++++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/streams/common/types.ts b/x-pack/plugins/streams/common/types.ts index 59cdd1cf9c4b9..981a452e5402c 100644 --- a/x-pack/plugins/streams/common/types.ts +++ b/x-pack/plugins/streams/common/types.ts @@ -88,6 +88,7 @@ export const streamWithoutIdDefinitonSchema = z.object({ processing: z.array(processingDefinitionSchema).default([]), fields: z.array(fieldDefinitionSchema).default([]), children: z.array(streamChildSchema).default([]), + dashboards: z.optional(z.array(z.string())), }); export type StreamWithoutIdDefinition = z.infer; diff --git a/x-pack/plugins/streams/server/lib/streams/assets/asset_service.ts b/x-pack/plugins/streams/server/lib/streams/assets/asset_service.ts index 7190334822a78..f405f426f8f3f 100644 --- a/x-pack/plugins/streams/server/lib/streams/assets/asset_service.ts +++ b/x-pack/plugins/streams/server/lib/streams/assets/asset_service.ts @@ -37,12 +37,11 @@ export class AssetService { async getClientWithRequest({ request }: { request: KibanaRequest }): Promise { const [coreStart, pluginsStart] = await this.coreSetup.getStartServices(); - return lastValueFrom(this.adapter$).then((adapter) => { - return new AssetClient({ - storageClient: adapter.getClient(), - soClient: coreStart.savedObjects.getScopedClient(request), - rulesClient: pluginsStart.alerting.getRulesClientWithRequest(request), - }); + const adapter = await lastValueFrom(this.adapter$); + return new AssetClient({ + storageClient: adapter.getClient(), + soClient: coreStart.savedObjects.getScopedClient(request), + rulesClient: await pluginsStart.alerting.getRulesClientWithRequest(request), }); } } From 7452b90f2dc7a25bc43cc784140a5125d210e38c Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 4 Dec 2024 15:35:04 +0100 Subject: [PATCH 79/95] fix type --- x-pack/plugins/streams/server/routes/streams/disable.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/streams/server/routes/streams/disable.ts b/x-pack/plugins/streams/server/routes/streams/disable.ts index b760b58f1fafd..f1534b9b9e555 100644 --- a/x-pack/plugins/streams/server/routes/streams/disable.ts +++ b/x-pack/plugins/streams/server/routes/streams/disable.ts @@ -29,9 +29,9 @@ export const disableStreamsRoute = createServerRoute({ getScopedClients, }): Promise<{ acknowledged: true }> => { try { - const { scopedClusterClient } = await getScopedClients({ request }); + const { scopedClusterClient, assetClient } = await getScopedClients({ request }); - await deleteStream(scopedClusterClient, 'logs', logger); + await deleteStream(scopedClusterClient, assetClient, 'logs', logger); return { acknowledged: true }; } catch (e) { From 092881eec8d09bb80f51430af6b68812894c6173 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 4 Dec 2024 16:10:09 +0100 Subject: [PATCH 80/95] fix type --- .../streams_app/.storybook/get_mock_streams_app_context.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugins/streams_app/.storybook/get_mock_streams_app_context.tsx b/x-pack/plugins/streams_app/.storybook/get_mock_streams_app_context.tsx index cb3148a7f6644..8e5dad100fc31 100644 --- a/x-pack/plugins/streams_app/.storybook/get_mock_streams_app_context.tsx +++ b/x-pack/plugins/streams_app/.storybook/get_mock_streams_app_context.tsx @@ -13,6 +13,7 @@ import type { ObservabilitySharedPluginStart } from '@kbn/observability-shared-p import type { UnifiedSearchPublicPluginStart } from '@kbn/unified-search-plugin/public'; import type { SharePublicStart } from '@kbn/share-plugin/public/plugin'; import { NavigationPublicStart } from '@kbn/navigation-plugin/public/types'; +import type { SavedObjectTaggingPluginStart } from '@kbn/saved-objects-tagging-plugin/public'; import type { StreamsAppKibanaContext } from '../public/hooks/use_kibana'; export function getMockStreamsAppContext(): StreamsAppKibanaContext { @@ -29,6 +30,7 @@ export function getMockStreamsAppContext(): StreamsAppKibanaContext { streams: {} as unknown as StreamsPluginStart, share: {} as unknown as SharePublicStart, navigation: {} as unknown as NavigationPublicStart, + savedObjectsTagging: {} as unknown as SavedObjectTaggingPluginStart, }, }, services: { From ca13422aac4af640f67ba757c530350269a71a79 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 12 Dec 2024 11:38:15 +0100 Subject: [PATCH 81/95] Use _has_privileges --- .../server/lib/streams/assets/asset_client.ts | 8 +- .../streams/server/lib/streams/stream_crud.ts | 79 +++++++++++++------ .../streams/server/routes/streams/sample.ts | 19 +++-- 3 files changed, 73 insertions(+), 33 deletions(-) diff --git a/x-pack/plugins/streams/server/lib/streams/assets/asset_client.ts b/x-pack/plugins/streams/server/lib/streams/assets/asset_client.ts index 578a1ea3ee9b2..07d804e19f0c4 100644 --- a/x-pack/plugins/streams/server/lib/streams/assets/asset_client.ts +++ b/x-pack/plugins/streams/server/lib/streams/assets/asset_client.ts @@ -120,17 +120,17 @@ export class AssetClient { const existingAssetLinks = assetsResponse.hits.hits.map((hit) => hit._source); - const missingAssetIds = assetIds.filter( + const newAssetIds = assetIds.filter( (assetId) => !existingAssetLinks.some((existingAssetLink) => existingAssetLink['asset.id'] === assetId) ); - const tooMuchAssetIds = existingAssetLinks + const assetIdsToRemove = existingAssetLinks .map((existingAssetLink) => existingAssetLink['asset.id']) .filter((assetId) => !assetIds.includes(assetId)); await Promise.all([ - ...missingAssetIds.map((assetId) => + ...newAssetIds.map((assetId) => this.linkAsset({ entityId, entityType, @@ -138,7 +138,7 @@ export class AssetClient { assetType, }) ), - ...tooMuchAssetIds.map((assetId) => + ...assetIdsToRemove.map((assetId) => this.unlinkAsset({ entityId, entityType, diff --git a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts index babf63ac875c0..7c6c4df8c39eb 100644 --- a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts +++ b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts @@ -126,28 +126,44 @@ export interface ListStreamResponse { export async function listStreams({ scopedClusterClient, }: ListStreamsParams): Promise { - const response = await scopedClusterClient.asInternalUser.search({ + const [managedStreams, unmanagedStreams] = await Promise.all([ + listManagedStreams({ scopedClusterClient }), + listDataStreamsAsStreams({ scopedClusterClient }), + ]); + + return { + definitions: [...managedStreams, ...unmanagedStreams], + }; +} + +async function listManagedStreams({ + scopedClusterClient, +}: ListStreamsParams): Promise { + const streamsSearchResponse = await scopedClusterClient.asInternalUser.search({ index: STREAMS_INDEX, size: 10000, sort: [{ id: 'asc' }], }); - const dataStreams = await listDataStreamsAsStreams({ scopedClusterClient }); - let definitions = response.hits.hits.map((hit) => ({ ...hit._source!, managed: true })); - const hasAccess = await Promise.all( - definitions.map((definition) => checkReadAccess({ id: definition.id, scopedClusterClient })) - ); - definitions = definitions.filter((_, index) => hasAccess[index]); + const streams = streamsSearchResponse.hits.hits.map((hit) => ({ + ...hit._source!, + managed: true, + })); - return { - definitions: [...definitions, ...dataStreams], - }; + const privileges = await scopedClusterClient.asCurrentUser.security.hasPrivileges({ + index: [{ names: streams.map((stream) => stream.id), privileges: ['read'] }], + }); + + return streams.filter((stream) => { + return privileges.index[stream.id]?.read === true; + }); } export async function listDataStreamsAsStreams({ scopedClusterClient, }: ListStreamsParams): Promise { - const response = await scopedClusterClient.asInternalUser.indices.getDataStream(); + const response = await scopedClusterClient.asCurrentUser.indices.getDataStream(); + return response.data_streams .filter((dataStream) => dataStream.template.endsWith('@stream') === false) .map((dataStream) => ({ @@ -180,7 +196,7 @@ export async function readStream({ }); const definition = response._source as StreamDefinition; if (!skipAccessCheck) { - const hasAccess = await checkReadAccess({ id, scopedClusterClient }); + const hasAccess = await checkAccess({ id, scopedClusterClient }); if (!hasAccess) { throw new DefinitionNotFound(`Stream definition for ${id} not found.`); } @@ -214,8 +230,8 @@ export async function readDataStreamAsStream({ processing: [], }; if (!skipAccessCheck) { - const hasAccess = await checkReadAccess({ id, scopedClusterClient }); - if (!hasAccess) { + const { read } = await checkAccess({ id, scopedClusterClient }); + if (!read) { throw new DefinitionNotFound(`Stream definition for ${id} not found.`); } } @@ -372,21 +388,40 @@ export async function checkStreamExists({ id, scopedClusterClient }: ReadStreamP } } -interface CheckReadAccessParams extends BaseParams { +interface CheckAccessParams extends BaseParams { id: string; } -export async function checkReadAccess({ +export async function checkAccess({ id, scopedClusterClient, -}: CheckReadAccessParams): Promise { - try { - return await scopedClusterClient.asCurrentUser.indices.exists({ index: id }); - } catch (e) { - return false; - } +}: CheckAccessParams): Promise<{ read: boolean; write: boolean }> { + return checkAccessBulk({ + ids: [id], + scopedClusterClient, + }).then((privileges) => privileges[id]); } +interface CheckAccessBulkParams extends BaseParams { + ids: string[]; +} + +export async function checkAccessBulk({ + ids, + scopedClusterClient, +}: CheckAccessBulkParams): Promise> { + const hasPrivilegesResponse = await scopedClusterClient.asCurrentUser.security.hasPrivileges({ + index: [{ names: ids, privileges: ['read', 'write'] }], + }); + + return Object.fromEntries( + ids.map((id) => { + const hasReadAccess = hasPrivilegesResponse.index[id].read === true; + const hasWriteAccess = hasPrivilegesResponse.index[id].write === true; + return [id, { read: hasReadAccess, write: hasWriteAccess }]; + }) + ); +} interface SyncStreamParams { scopedClusterClient: IScopedClusterClient; assetClient: AssetClient; diff --git a/x-pack/plugins/streams/server/routes/streams/sample.ts b/x-pack/plugins/streams/server/routes/streams/sample.ts index cd3a989c29109..55c21c4db9396 100644 --- a/x-pack/plugins/streams/server/routes/streams/sample.ts +++ b/x-pack/plugins/streams/server/routes/streams/sample.ts @@ -7,10 +7,11 @@ import { z } from '@kbn/zod'; import { notFound, internal } from '@hapi/boom'; +import { errors } from '@elastic/elasticsearch'; import { conditionSchema } from '../../../common/types'; import { createServerRoute } from '../create_server_route'; import { DefinitionNotFound } from '../../lib/streams/errors'; -import { checkReadAccess } from '../../lib/streams/stream_crud'; +import { checkAccess } from '../../lib/streams/stream_crud'; import { conditionToQueryDsl } from '../../lib/streams/helpers/condition_to_query_dsl'; import { getFields, isComplete } from '../../lib/streams/helpers/condition_fields'; @@ -45,8 +46,8 @@ export const sampleStreamRoute = createServerRoute({ try { const { scopedClusterClient } = await getScopedClients({ request }); - const hasAccess = await checkReadAccess({ id: params.path.id, scopedClusterClient }); - if (!hasAccess) { + const { read } = await checkAccess({ id: params.path.id, scopedClusterClient }); + if (!read) { throw new DefinitionNotFound(`Stream definition for ${params.path.id} not found.`); } const searchBody = { @@ -90,16 +91,20 @@ export const sampleStreamRoute = createServerRoute({ }; const results = await scopedClusterClient.asCurrentUser.search({ index: params.path.id, + allow_no_indices: true, ...searchBody, }); return { documents: results.hits.hits.map((hit) => hit._source) }; - } catch (e) { - if (e instanceof DefinitionNotFound) { - throw notFound(e); + } catch (error) { + if (error instanceof errors.ResponseError && error.meta.statusCode === 404) { + throw notFound(error); + } + if (error instanceof DefinitionNotFound) { + throw notFound(error); } - throw internal(e); + throw internal(error); } }, }); From 1a63b3555d223721eab8d66c1d04c074fb08820f Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Fri, 13 Dec 2024 10:00:35 +0100 Subject: [PATCH 82/95] Use _bulk API --- .../es/storage/index.ts | 1 + .../es/storage/index_adapter/index.ts | 4 + .../es/storage/storage_client.ts | 47 ++++ .../find/schemas/find_rules_schemas.ts | 10 +- x-pack/plugins/streams/common/assets.ts | 38 +-- .../server/lib/streams/assets/asset_client.ts | 249 ++++++++++++------ .../lib/streams/assets/asset_service.ts | 4 +- .../streams/server/routes/dashboards/route.ts | 134 ++++++++-- .../add_dashboard_flyout.tsx | 22 +- .../dashboard_table.tsx | 16 +- .../stream_detail_dashboards_view/index.tsx | 96 +------ .../public/hooks/use_dashboards_api.ts | 91 +++++++ 12 files changed, 457 insertions(+), 255 deletions(-) create mode 100644 x-pack/plugins/streams_app/public/hooks/use_dashboards_api.ts diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index.ts index ccd91ca9d479a..51c1bf4d516e2 100644 --- a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index.ts +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index.ts @@ -26,6 +26,7 @@ export type StorageSettings = IndexStorageSettings; export interface IStorageAdapter { getSearchIndexPattern(): string; + getWriteTarget(): string; } export type StorageSettingsOf> = diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/index.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/index.ts index 5b2ba744c8167..3630fbb812a8b 100644 --- a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/index.ts +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/index.ts @@ -99,6 +99,10 @@ export class StorageIndexAdapter return getAliasName(this.storage.name); } + getWriteTarget(): string { + return getAliasName(this.storage.name); + } + private async createIndexTemplate(create: boolean = true): Promise { this.logger.debug(`Creating index template (create = ${create})`); diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/storage_client.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/storage_client.ts index fc10846b88723..ba8938a489361 100644 --- a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/storage_client.ts +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/storage_client.ts @@ -6,6 +6,8 @@ */ import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { BulkRequest } from '@elastic/elasticsearch/lib/api/types'; +import { compact } from 'lodash'; import { IStorageAdapter, StorageDocumentOf, StorageSettings } from '.'; import { ObservabilityESSearchRequest, @@ -13,6 +15,12 @@ import { createObservabilityEsClient, } from '../client/create_observability_es_client'; +type StorageBulkOperation = + | { + create: { document: Omit; _id: string }; + } + | { delete: { _id: string } }; + export class StorageClient { private readonly esClient: ObservabilityElasticsearchClient; constructor( @@ -58,4 +66,43 @@ export class StorageClient { index: this.storage.getSearchIndexPattern(), }); } + + async bulk(operations: Array>>) { + const index = this.storage.getWriteTarget(); + + const result = await this.esClient.client.bulk({ + index, + refresh: 'wait_for', + operations: operations.flatMap((operation): BulkRequest['operations'] => { + if ('create' in operation) { + return [ + { + create: { + _id: operation.create._id, + }, + }, + operation.create.document, + ]; + } + + return [operation]; + }), + }); + + if (result.errors) { + const errors = compact( + result.items.map((item) => { + const error = Object.values(item).find((operation) => operation.error)?.error; + return error; + }) + ); + return { + errors, + }; + } + + return { + acknowledged: true, + }; + } } diff --git a/x-pack/plugins/alerting/server/application/rule/methods/find/schemas/find_rules_schemas.ts b/x-pack/plugins/alerting/server/application/rule/methods/find/schemas/find_rules_schemas.ts index aec95d7f2c061..4ea4afa8b0c79 100644 --- a/x-pack/plugins/alerting/server/application/rule/methods/find/schemas/find_rules_schemas.ts +++ b/x-pack/plugins/alerting/server/application/rule/methods/find/schemas/find_rules_schemas.ts @@ -7,6 +7,11 @@ import { schema } from '@kbn/config-schema'; +const savedObjectReferenceSchema = schema.object({ + type: schema.string(), + id: schema.string(), +}); + export const findRulesOptionsSchema = schema.object( { perPage: schema.maybe(schema.number()), @@ -19,10 +24,7 @@ export const findRulesOptionsSchema = schema.object( sortField: schema.maybe(schema.string()), sortOrder: schema.maybe(schema.oneOf([schema.literal('asc'), schema.literal('desc')])), hasReference: schema.maybe( - schema.object({ - type: schema.string(), - id: schema.string(), - }) + schema.oneOf([savedObjectReferenceSchema, schema.arrayOf(savedObjectReferenceSchema)]) ), fields: schema.maybe(schema.arrayOf(schema.string())), filter: schema.maybe( diff --git a/x-pack/plugins/streams/common/assets.ts b/x-pack/plugins/streams/common/assets.ts index ac0a003460abc..9353418f14618 100644 --- a/x-pack/plugins/streams/common/assets.ts +++ b/x-pack/plugins/streams/common/assets.ts @@ -15,40 +15,26 @@ export const ASSET_TYPES = { export type AssetType = ValuesType; -export interface AssetLink { - type: AssetType; +export interface AssetLink { + assetType: TAssetType; assetId: string; } -export interface DashboardLink extends AssetLink { - type: typeof ASSET_TYPES.Dashboard; -} - -export interface SloLink extends AssetLink { - type: typeof ASSET_TYPES.Slo; -} - -export interface Asset extends AssetLink { - label: string; - tags: string[]; -} +export type DashboardLink = AssetLink<'dashboard'>; +export type SloLink = AssetLink<'slo'>; +export type RuleLink = AssetLink<'rule'>; -export interface Dashboard extends Asset { - type: typeof ASSET_TYPES.Dashboard; -} - -export interface ReadDashboard { - id: string; +export interface Asset extends AssetLink { label: string; tags: string[]; } -export interface Slo extends Asset { - type: typeof ASSET_TYPES.Slo; -} +export type DashboardAsset = Asset<'dashboard'>; +export type SloAsset = Asset<'slo'>; +export type RuleAsset = Asset<'rule'>; export interface AssetTypeToAssetMap { - [ASSET_TYPES.Dashboard]: Dashboard; - [ASSET_TYPES.Slo]: Slo; - [ASSET_TYPES.Rule]: Asset; + [ASSET_TYPES.Dashboard]: DashboardAsset; + [ASSET_TYPES.Slo]: SloAsset; + [ASSET_TYPES.Rule]: RuleAsset; } diff --git a/x-pack/plugins/streams/server/lib/streams/assets/asset_client.ts b/x-pack/plugins/streams/server/lib/streams/assets/asset_client.ts index 07d804e19f0c4..2231c293643cf 100644 --- a/x-pack/plugins/streams/server/lib/streams/assets/asset_client.ts +++ b/x-pack/plugins/streams/server/lib/streams/assets/asset_client.ts @@ -4,60 +4,89 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import pLimit from 'p-limit'; -import { StorageClient } from '@kbn/observability-utils-server/es/storage'; -import { termQuery } from '@kbn/observability-utils-server/es/queries/term_query'; +import { SanitizedRule } from '@kbn/alerting-plugin/common'; import { RulesClient } from '@kbn/alerting-plugin/server'; import { SavedObject, SavedObjectsClientContract } from '@kbn/core/server'; +import { termQuery } from '@kbn/observability-utils-server/es/queries/term_query'; +import { StorageClient, StorageDocumentOf } from '@kbn/observability-utils-server/es/storage'; import { keyBy } from 'lodash'; import objectHash from 'object-hash'; -import { SanitizedRule } from '@kbn/alerting-plugin/common'; -import { AssetStorageSettings } from './storage_settings'; +import pLimit from 'p-limit'; import { ASSET_TYPES, Asset, + AssetLink, AssetType, - AssetTypeToAssetMap, - Dashboard, - Slo, + DashboardAsset, + SloAsset, + RuleAsset, } from '../../../../common/assets'; import { ASSET_ENTITY_ID, ASSET_ENTITY_TYPE, ASSET_TYPE } from './fields'; +import { AssetStorageSettings } from './storage_settings'; function sloSavedObjectToAsset( sloId: string, savedObject: SavedObject<{ name: string; tags: string[] }> -): Slo { +): SloAsset { return { assetId: sloId, label: savedObject.attributes.name, tags: savedObject.attributes.tags.concat( savedObject.references.filter((ref) => ref.type === 'tag').map((ref) => ref.id) ), - type: 'slo', + assetType: 'slo', }; } function dashboardSavedObjectToAsset( dashboardId: string, savedObject: SavedObject<{ title: string }> -): Dashboard { +): DashboardAsset { return { assetId: dashboardId, label: savedObject.attributes.title, tags: savedObject.references.filter((ref) => ref.type === 'tag').map((ref) => ref.id), - type: 'dashboard', + assetType: 'dashboard', }; } -function ruleToAsset(ruleId: string, rule: SanitizedRule): Asset { +function ruleToAsset(ruleId: string, rule: SanitizedRule): RuleAsset { return { - type: 'rule', + assetType: 'rule', assetId: ruleId, label: rule.name, tags: rule.tags, }; } +function getAssetDocument({ + assetId, + entityId, + entityType, + assetType, +}: AssetLink & { entityId: string; entityType: string }): StorageDocumentOf { + const doc = { + 'asset.id': assetId, + 'asset.type': assetType, + 'entity.id': entityId, + 'entity.type': entityType, + }; + + return { + _id: objectHash(doc), + ...doc, + }; +} + +interface AssetBulkCreateOperation { + create: { asset: AssetLink }; +} +interface AssetBulkDeleteOperation { + delete: { asset: AssetLink }; +} + +export type AssetBulkOperation = AssetBulkCreateOperation | AssetBulkDeleteOperation; + export class AssetClient { constructor( private readonly clients: { @@ -67,29 +96,17 @@ export class AssetClient { } ) {} - async linkAsset({ - entityId, - entityType, - assetId, - assetType, - }: { - entityId: string; - entityType: string; - assetId: string; - assetType: AssetType; - }) { - const assetDoc = { - 'asset.id': assetId, - 'asset.type': assetType, - 'entity.id': entityId, - 'entity.type': entityType, - }; - - const id = objectHash(assetDoc); + async linkAsset( + properties: { + entityId: string; + entityType: string; + } & AssetLink + ) { + const { _id: id, ...document } = getAssetDocument(properties); await this.clients.storageClient.index({ id, - document: assetDoc, + document, }); } @@ -149,23 +166,13 @@ export class AssetClient { ]); } - async unlinkAsset({ - entityId, - entityType, - assetId, - assetType, - }: { - entityId: string; - entityType: string; - assetId: string; - assetType: string; - }) { - const id = objectHash({ - 'asset.id': assetId, - 'asset.type': assetType, - 'entity.id': entityId, - 'entity.type': entityType, - }); + async unlinkAsset( + properties: { + entityId: string; + entityType: string; + } & AssetLink + ) { + const { _id: id } = getAssetDocument(properties); await this.clients.storageClient.delete(id); } @@ -196,6 +203,36 @@ export class AssetClient { return assetsResponse.hits.hits.map((hit) => hit._source['asset.id']); } + async bulk( + { entityId, entityType }: { entityId: string; entityType: string }, + operations: AssetBulkOperation[] + ) { + return await this.clients.storageClient.bulk( + operations.map((operation) => { + const { _id, ...document } = getAssetDocument({ + ...Object.values(operation)[0].asset, + entityId, + entityType, + }); + + if ('create' in operation) { + return { + create: { + document, + _id, + }, + }; + } + + return { + delete: { + _id, + }, + }; + }) + ); + } + async getAssets({ entityId, entityType, @@ -287,47 +324,85 @@ export class AssetClient { return [...dashboards, ...rules, ...slos]; } - async getSuggestions({ - entityId, - entityType, + async getSuggestions({ query, - assetType, + assetTypes, tags, }: { - entityId: string; - entityType: string; query: string; + assetTypes?: AssetType[]; tags?: string[]; - assetType: T; - }): Promise> { - if (assetType === 'dashboard') { - const dashboardSavedObjects = await this.clients.soClient.find<{ title: string }>({ - type: 'dashboard', - search: query, - hasReferenceOperator: 'OR', - hasReference: tags?.map((tag) => ({ type: 'tag', id: tag })), - }); - - return dashboardSavedObjects.saved_objects.map((dashboardSavedObject) => { - return { - asset: dashboardSavedObjectToAsset(dashboardSavedObject.id, dashboardSavedObject), - }; - }) as Array<{ asset: AssetTypeToAssetMap[T] }>; - } - if (assetType === 'rule') { - return []; - } - if (assetType === 'slo') { - const sloSavedObjects = await this.clients.soClient.find<{ name: string; tags: string[] }>({ - type: 'slo', - search: query, - }); - - return sloSavedObjects.saved_objects.map((sloSavedObject) => { - return { asset: sloSavedObjectToAsset(sloSavedObject.id, sloSavedObject) }; - }) as Array<{ asset: AssetTypeToAssetMap[T] }>; - } + }): Promise<{ hasMore: boolean; assets: Asset[] }> { + const perPage = 101; + + const searchAll = !assetTypes; + + const searchDashboardsOrSlos = + searchAll || assetTypes.includes('dashboard') || assetTypes.includes('slo'); + + const searchRules = searchAll || assetTypes.includes('rule'); + + const [suggestionsFromSlosAndDashboards, suggestionsFromRules] = await Promise.all([ + searchDashboardsOrSlos + ? this.clients.soClient + .find({ + type: ['dashboard' as const, 'slo' as const].filter( + (type) => searchAll || assetTypes.includes(type) + ), + search: query, + perPage, + ...(tags + ? { + hasReferenceOperator: 'OR', + hasReference: tags.map((tag) => ({ type: 'tag', id: tag })), + } + : {}), + }) + .then((results) => { + return results.saved_objects.map((savedObject) => { + if (savedObject.type === 'slo') { + const sloSavedObject = savedObject as SavedObject<{ + id: string; + name: string; + tags: string[]; + }>; + return sloSavedObjectToAsset(sloSavedObject.attributes.id, sloSavedObject); + } - throw new Error(`Unsupported asset type: ${assetType}`); + const dashboardSavedObject = savedObject as SavedObject<{ + title: string; + }>; + + return dashboardSavedObjectToAsset(dashboardSavedObject.id, dashboardSavedObject); + }); + }) + : Promise.resolve([]), + searchRules + ? this.clients.rulesClient + .find({ + options: { + perPage, + ...(tags + ? { + hasReferenceOperator: 'OR', + hasReference: tags.map((tag) => ({ type: 'tag', id: tag })), + } + : {}), + }, + }) + .then((results) => { + return results.data.map((rule) => { + return ruleToAsset(rule.id, rule); + }); + }) + : Promise.resolve([]), + ]); + + return { + assets: [...suggestionsFromRules, ...suggestionsFromSlosAndDashboards], + hasMore: + Math.max(suggestionsFromSlosAndDashboards.length, suggestionsFromRules.length) > + perPage - 1, + }; } } diff --git a/x-pack/plugins/streams/server/lib/streams/assets/asset_service.ts b/x-pack/plugins/streams/server/lib/streams/assets/asset_service.ts index f405f426f8f3f..3d27b6dd182af 100644 --- a/x-pack/plugins/streams/server/lib/streams/assets/asset_service.ts +++ b/x-pack/plugins/streams/server/lib/streams/assets/asset_service.ts @@ -7,7 +7,7 @@ import { CoreSetup, KibanaRequest, Logger } from '@kbn/core/server'; import { StorageIndexAdapter } from '@kbn/observability-utils-server/es/storage'; -import { Observable, defer, from, lastValueFrom, share } from 'rxjs'; +import { Observable, defer, from, lastValueFrom, shareReplay } from 'rxjs'; import { StreamsPluginStartDependencies } from '../../../types'; import { AssetClient } from './asset_client'; import { assetStorageSettings } from './storage_settings'; @@ -18,7 +18,7 @@ export class AssetService { private readonly coreSetup: CoreSetup, private readonly logger: Logger ) { - this.adapter$ = defer(() => from(this.prepareIndex())).pipe(share()); + this.adapter$ = defer(() => from(this.prepareIndex())).pipe(shareReplay(1)); } async prepareIndex(): Promise> { diff --git a/x-pack/plugins/streams/server/routes/dashboards/route.ts b/x-pack/plugins/streams/server/routes/dashboards/route.ts index 28f95d9c9a479..eda8e409e7270 100644 --- a/x-pack/plugins/streams/server/routes/dashboards/route.ts +++ b/x-pack/plugins/streams/server/routes/dashboards/route.ts @@ -6,11 +6,19 @@ */ import { z } from '@kbn/zod'; -import { Asset, Dashboard, ReadDashboard } from '../../../common/assets'; +import { ErrorCause } from '@elastic/elasticsearch/lib/api/types'; +import { internal } from '@hapi/boom'; +import { Asset, DashboardAsset } from '../../../common/assets'; import { createServerRoute } from '../create_server_route'; +export interface SanitizedDashboardAsset { + id: string; + label: string; + tags: string[]; +} + export interface ListDashboardsResponse { - dashboards: ReadDashboard[]; + dashboards: SanitizedDashboardAsset[]; } export interface LinkDashboardResponse { @@ -22,12 +30,24 @@ export interface UnlinkDashboardResponse { } export interface SuggestDashboardResponse { - suggestions: Array<{ - dashboard: ReadDashboard; - }>; + suggestions: SanitizedDashboardAsset[]; +} + +export type BulkUpdateAssetsResponse = + | { + acknowledged: boolean; + } + | { errors: ErrorCause[] }; + +function sanitizeDashboardAsset(asset: DashboardAsset): SanitizedDashboardAsset { + return { + id: asset.assetId, + label: asset.label, + tags: asset.tags, + }; } -export const listDashboardsRoute = createServerRoute({ +const listDashboardsRoute = createServerRoute({ endpoint: 'GET /api/streams/{id}/dashboards', options: { access: 'internal', @@ -44,8 +64,8 @@ export const listDashboardsRoute = createServerRoute({ path: { id: streamId }, } = params; - function isDashboard(asset: Asset): asset is Dashboard { - return asset.type === 'dashboard'; + function isDashboard(asset: Asset): asset is DashboardAsset { + return asset.assetType === 'dashboard'; } return { @@ -56,16 +76,12 @@ export const listDashboardsRoute = createServerRoute({ }) ) .filter(isDashboard) - .map((asset) => ({ - id: asset.assetId, - label: asset.label, - tags: asset.tags, - })), + .map(sanitizeDashboardAsset), }; }, }); -export const linkDashboardRoute = createServerRoute({ +const linkDashboardRoute = createServerRoute({ endpoint: 'PUT /api/streams/{id}/dashboards/{dashboardId}', options: { access: 'internal', @@ -96,7 +112,7 @@ export const linkDashboardRoute = createServerRoute({ }, }); -export const unlinkDashboardRoute = createServerRoute({ +const unlinkDashboardRoute = createServerRoute({ endpoint: 'DELETE /api/streams/{id}/dashboards/{dashboardId}', options: { access: 'internal', @@ -127,7 +143,7 @@ export const unlinkDashboardRoute = createServerRoute({ }, }); -export const suggestDashboardsRoute = createServerRoute({ +const suggestDashboardsRoute = createServerRoute({ endpoint: 'POST /api/streams/{id}/dashboards/_suggestions', options: { access: 'internal', @@ -147,26 +163,19 @@ export const suggestDashboardsRoute = createServerRoute({ const assetsClient = await assets.getClientWithRequest({ request }); const { - path: { id: streamId }, query: { query }, body: { tags }, } = params; const suggestions = ( await assetsClient.getSuggestions({ - entityId: streamId, - entityType: 'stream', - assetType: 'dashboard', + assetTypes: ['dashboard'], query, tags, }) - ).map(({ asset: dashboard }) => ({ - dashboard: { - id: dashboard.assetId, - label: dashboard.label, - tags: dashboard.tags, - }, - })); + ).assets.map((asset) => { + return sanitizeDashboardAsset(asset as DashboardAsset); + }); return { suggestions, @@ -174,9 +183,80 @@ export const suggestDashboardsRoute = createServerRoute({ }, }); +const dashboardSchema = z.object({ + id: z.string(), +}); + +const bulkDashboardsRoute = createServerRoute({ + endpoint: `POST /api/streams/{id}/dashboards/_bulk`, + options: { + access: 'internal', + }, + params: z.object({ + path: z.object({ + id: z.string(), + }), + body: z.object({ + operations: z.array( + z.union([ + z.object({ + create: dashboardSchema, + }), + z.object({ + delete: dashboardSchema, + }), + ]) + ), + }), + }), + handler: async ({ params, request, assets, logger }): Promise => { + const assetsClient = await assets.getClientWithRequest({ request }); + + const { + path: { id: streamId }, + body: { operations }, + } = params; + + const result = await assetsClient.bulk( + { + entityId: streamId, + entityType: 'stream', + }, + operations.map((operation) => { + if ('create' in operation) { + return { + create: { + asset: { + assetType: 'dashboard', + assetId: operation.create.id, + }, + }, + }; + } + return { + delete: { + asset: { + assetType: 'dashboard', + assetId: operation.delete.id, + }, + }, + }; + }) + ); + + if (result.errors) { + logger.error(`Error indexing ${result.errors.length} items`); + throw internal(`Could not index all items`, { errors: result.errors }); + } + + return { acknowledged: true }; + }, +}); + export const dashboardRoutes = { ...listDashboardsRoute, ...linkDashboardRoute, ...unlinkDashboardRoute, ...suggestDashboardsRoute, + ...bulkDashboardsRoute, }; diff --git a/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/add_dashboard_flyout.tsx b/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/add_dashboard_flyout.tsx index 09b9f7e1e7e44..4e3918748e467 100644 --- a/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/add_dashboard_flyout.tsx +++ b/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/add_dashboard_flyout.tsx @@ -23,9 +23,9 @@ import { useGeneratedHtmlId, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import type { ReadDashboard } from '@kbn/streams-plugin/common/assets'; import { debounce } from 'lodash'; import React, { useMemo, useState, useEffect } from 'react'; +import type { SanitizedDashboardAsset } from '@kbn/streams-plugin/server/routes/dashboards/route'; import { useKibana } from '../../hooks/use_kibana'; import { useStreamsAppFetch } from '../../hooks/use_streams_app_fetch'; import { DashboardsTable } from './dashboard_table'; @@ -37,8 +37,8 @@ export function AddDashboardFlyout({ onClose, }: { entityId: string; - onAddDashboards: (dashboard: ReadDashboard[]) => Promise; - linkedDashboards: ReadDashboard[]; + onAddDashboards: (dashboard: SanitizedDashboardAsset[]) => Promise; + linkedDashboards: SanitizedDashboardAsset[]; onClose: () => void; }) { const { @@ -53,7 +53,7 @@ export function AddDashboardFlyout({ const [query, setQuery] = useState(''); const [submittedQuery, setSubmittedQuery] = useState(query); - const [selectedDashboards, setSelectedDashboards] = useState([]); + const [selectedDashboards, setSelectedDashboards] = useState([]); const [isLoading, setIsLoading] = useState(false); const [selectedTags, setSelectedTags] = useState([]); const [isPopoverOpen, setIsPopoverOpen] = useState(false); @@ -81,13 +81,11 @@ export function AddDashboardFlyout({ }) .then(({ suggestions }) => { return { - dashboards: suggestions - .map((suggestion) => suggestion.dashboard) - .filter((dashboard) => { - return !linkedDashboards.find( - (linkedDashboard) => linkedDashboard.id === dashboard.id - ); - }), + dashboards: suggestions.filter((dashboard) => { + return !linkedDashboards.find( + (linkedDashboard) => linkedDashboard.id === dashboard.id + ); + }), }; }); }, @@ -206,7 +204,7 @@ export function AddDashboardFlyout({ diff --git a/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/dashboard_table.tsx b/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/dashboard_table.tsx index f4f1bb953a891..eb04553ad88b1 100644 --- a/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/dashboard_table.tsx +++ b/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/dashboard_table.tsx @@ -6,23 +6,23 @@ */ import { EuiBasicTable, EuiBasicTableColumn, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import { ReadDashboard } from '@kbn/streams-plugin/common/assets'; import React, { useMemo } from 'react'; +import type { SanitizedDashboardAsset } from '@kbn/streams-plugin/server/routes/dashboards/route'; import { useKibana } from '../../hooks/use_kibana'; import { tagListToReferenceList } from './to_reference_list'; export function DashboardsTable({ dashboards, compact = false, - selecedDashboards: selectedDashboards, + selectedDashboards, setSelectedDashboards, loading, }: { loading: boolean; - dashboards: ReadDashboard[] | undefined; + dashboards: SanitizedDashboardAsset[] | undefined; compact?: boolean; - selecedDashboards: ReadDashboard[]; - setSelectedDashboards: (dashboards: ReadDashboard[]) => void; + selectedDashboards: SanitizedDashboardAsset[]; + setSelectedDashboards: (dashboards: SanitizedDashboardAsset[]) => void; }) { const { dependencies: { @@ -31,7 +31,7 @@ export function DashboardsTable({ }, }, } = useKibana(); - const columns = useMemo((): Array> => { + const columns = useMemo((): Array> => { return [ { field: 'label', @@ -56,7 +56,7 @@ export function DashboardsTable({ ); }, }, - ] satisfies Array>) + ] satisfies Array>) : []), ]; }, [compact, savedObjectsTaggingUi]); @@ -74,7 +74,7 @@ export function DashboardsTable({ items={items} loading={loading} selection={{ - onSelectionChange: (newSelection: ReadDashboard[]) => { + onSelectionChange: (newSelection: SanitizedDashboardAsset[]) => { setSelectedDashboards(newSelection); }, selected: selectedDashboards, diff --git a/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/index.tsx b/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/index.tsx index 6f28c642ac559..a9242b2e8ef80 100644 --- a/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/index.tsx +++ b/x-pack/plugins/streams_app/public/components/stream_detail_dashboards_view/index.tsx @@ -4,103 +4,21 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { EuiButton, EuiFlexGroup, EuiSearchBar, EuiFlexItem } from '@elastic/eui'; +import { EuiButton, EuiFlexGroup, EuiFlexItem, EuiSearchBar } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { StreamDefinition } from '@kbn/streams-plugin/common'; -import React, { useMemo, useState, useCallback } from 'react'; -import { ReadDashboard } from '@kbn/streams-plugin/common/assets'; -import { useAbortController } from '@kbn/observability-utils-browser/hooks/use_abort_controller'; -import { useKibana } from '../../hooks/use_kibana'; -import { useStreamsAppFetch } from '../../hooks/use_streams_app_fetch'; +import React, { useMemo, useState } from 'react'; +import type { SanitizedDashboardAsset } from '@kbn/streams-plugin/server/routes/dashboards/route'; import { AddDashboardFlyout } from './add_dashboard_flyout'; import { DashboardsTable } from './dashboard_table'; - -const useDashboardCrud = (id?: string) => { - const { signal } = useAbortController(); - const { - dependencies: { - start: { - streams: { streamsRepositoryClient }, - }, - }, - } = useKibana(); - - const dashboardsFetch = useStreamsAppFetch(() => { - if (!id) { - return Promise.resolve(undefined); - } - return streamsRepositoryClient.fetch('GET /api/streams/{id}/dashboards', { - signal, - params: { - path: { - id, - }, - }, - }); - }, [id, signal, streamsRepositoryClient]); - - const addDashboards = useCallback( - async (dashboards: ReadDashboard[]) => { - if (!id) { - return; - } - await Promise.all( - dashboards.map((dashboard) => { - return streamsRepositoryClient.fetch('PUT /api/streams/{id}/dashboards/{dashboardId}', { - signal, - params: { - path: { - id, - dashboardId: dashboard.id, - }, - }, - }); - }) - ); - await dashboardsFetch.refresh(); - }, - [dashboardsFetch, id, signal, streamsRepositoryClient] - ); - - const removeDashboards = useCallback( - async (dashboards: ReadDashboard[]) => { - if (!id) { - return; - } - await Promise.all( - dashboards.map((dashboard) => { - return streamsRepositoryClient.fetch( - 'DELETE /api/streams/{id}/dashboards/{dashboardId}', - { - signal, - params: { - path: { - id, - dashboardId: dashboard.id, - }, - }, - } - ); - }) - ); - await dashboardsFetch.refresh(); - }, - [dashboardsFetch, id, signal, streamsRepositoryClient] - ); - - return { - dashboardsFetch, - addDashboards, - removeDashboards, - }; -}; +import { useDashboardsApi } from '../../hooks/use_dashboards_api'; export function StreamDetailDashboardsView({ definition }: { definition?: StreamDefinition }) { const [query, setQuery] = useState(''); const [isAddDashboardFlyoutOpen, setIsAddDashboardFlyoutOpen] = useState(false); - const { dashboardsFetch, addDashboards, removeDashboards } = useDashboardCrud(definition?.id); + const { dashboardsFetch, addDashboards, removeDashboards } = useDashboardsApi(definition?.id); const [isUnlinkLoading, setIsUnlinkLoading] = useState(false); const linkedDashboards = useMemo(() => { @@ -113,7 +31,7 @@ export function StreamDetailDashboardsView({ definition }: { definition?: Stream }); }, [linkedDashboards, query]); - const [selectedDashboards, setSelectedDashboards] = useState([]); + const [selectedDashboards, setSelectedDashboards] = useState([]); return ( @@ -166,7 +84,7 @@ export function StreamDetailDashboardsView({ definition }: { definition?: Stream {definition && isAddDashboardFlyoutOpen ? ( diff --git a/x-pack/plugins/streams_app/public/hooks/use_dashboards_api.ts b/x-pack/plugins/streams_app/public/hooks/use_dashboards_api.ts new file mode 100644 index 0000000000000..3cdb78e285e93 --- /dev/null +++ b/x-pack/plugins/streams_app/public/hooks/use_dashboards_api.ts @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { useCallback } from 'react'; +import { useAbortController } from '@kbn/observability-utils-browser/hooks/use_abort_controller'; +import type { SanitizedDashboardAsset } from '@kbn/streams-plugin/server/routes/dashboards/route'; +import { useKibana } from './use_kibana'; +import { useStreamsAppFetch } from './use_streams_app_fetch'; + +export const useDashboardsApi = (id?: string) => { + const { signal } = useAbortController(); + const { + dependencies: { + start: { + streams: { streamsRepositoryClient }, + }, + }, + } = useKibana(); + + const dashboardsFetch = useStreamsAppFetch(() => { + if (!id) { + return Promise.resolve(undefined); + } + return streamsRepositoryClient.fetch('GET /api/streams/{id}/dashboards', { + signal, + params: { + path: { + id, + }, + }, + }); + }, [id, signal, streamsRepositoryClient]); + + const addDashboards = useCallback( + async (dashboards: SanitizedDashboardAsset[]) => { + if (!id) { + return; + } + + await streamsRepositoryClient.fetch('POST /api/streams/{id}/dashboards/_bulk', { + signal, + params: { + path: { + id, + }, + body: { + operations: dashboards.map((dashboard) => { + return { create: { id: dashboard.id } }; + }), + }, + }, + }); + + await dashboardsFetch.refresh(); + }, + [dashboardsFetch, id, signal, streamsRepositoryClient] + ); + + const removeDashboards = useCallback( + async (dashboards: SanitizedDashboardAsset[]) => { + if (!id) { + return; + } + await streamsRepositoryClient.fetch('POST /api/streams/{id}/dashboards/_bulk', { + signal, + params: { + path: { + id, + }, + body: { + operations: dashboards.map((dashboard) => { + return { delete: { id: dashboard.id } }; + }), + }, + }, + }); + + await dashboardsFetch.refresh(); + }, + [dashboardsFetch, id, signal, streamsRepositoryClient] + ); + + return { + dashboardsFetch, + addDashboards, + removeDashboards, + }; +}; From 8faf3f2f8f750c7e0e5c1c746f16aaa377e4b560 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Sat, 14 Dec 2024 10:02:22 +0100 Subject: [PATCH 83/95] Fix access checks --- .../server/routes/streams/schema/fields_simulation.ts | 6 +++--- .../streams/server/routes/streams/schema/unmapped_fields.ts | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/solutions/observability/plugins/streams/server/routes/streams/schema/fields_simulation.ts b/x-pack/solutions/observability/plugins/streams/server/routes/streams/schema/fields_simulation.ts index 01aa61a302a39..dca507a91e2c7 100644 --- a/x-pack/solutions/observability/plugins/streams/server/routes/streams/schema/fields_simulation.ts +++ b/x-pack/solutions/observability/plugins/streams/server/routes/streams/schema/fields_simulation.ts @@ -11,7 +11,7 @@ import { getFlattenedObject } from '@kbn/std'; import { fieldDefinitionSchema } from '../../../../common/types'; import { createServerRoute } from '../../create_server_route'; import { DefinitionNotFound } from '../../../lib/streams/errors'; -import { checkReadAccess } from '../../../lib/streams/stream_crud'; +import { checkAccess } from '../../../lib/streams/stream_crud'; const SAMPLE_SIZE = 200; @@ -47,8 +47,8 @@ export const schemaFieldsSimulationRoute = createServerRoute({ try { const { scopedClusterClient } = await getScopedClients({ request }); - const hasAccess = await checkReadAccess({ id: params.path.id, scopedClusterClient }); - if (!hasAccess) { + const { read } = await checkAccess({ id: params.path.id, scopedClusterClient }); + if (!read) { throw new DefinitionNotFound(`Stream definition for ${params.path.id} not found.`); } diff --git a/x-pack/solutions/observability/plugins/streams/server/routes/streams/schema/unmapped_fields.ts b/x-pack/solutions/observability/plugins/streams/server/routes/streams/schema/unmapped_fields.ts index 15bcb964b8fd6..c9db4d7b35754 100644 --- a/x-pack/solutions/observability/plugins/streams/server/routes/streams/schema/unmapped_fields.ts +++ b/x-pack/solutions/observability/plugins/streams/server/routes/streams/schema/unmapped_fields.ts @@ -9,7 +9,7 @@ import { z } from '@kbn/zod'; import { internal, notFound } from '@hapi/boom'; import { getFlattenedObject } from '@kbn/std'; import { DefinitionNotFound } from '../../../lib/streams/errors'; -import { checkReadAccess, readAncestors, readStream } from '../../../lib/streams/stream_crud'; +import { checkAccess, readAncestors, readStream } from '../../../lib/streams/stream_crud'; import { createServerRoute } from '../../create_server_route'; const SAMPLE_SIZE = 500; @@ -39,8 +39,8 @@ export const unmappedFieldsRoute = createServerRoute({ try { const { scopedClusterClient } = await getScopedClients({ request }); - const hasAccess = await checkReadAccess({ id: params.path.id, scopedClusterClient }); - if (!hasAccess) { + const { read } = await checkAccess({ id: params.path.id, scopedClusterClient }); + if (!read) { throw new DefinitionNotFound(`Stream definition for ${params.path.id} not found.`); } From 76cb698a4b68f74fe657bd5b5fa5105d98d79a68 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Mon, 16 Dec 2024 09:52:31 +0100 Subject: [PATCH 84/95] Split up dashboards hook --- .../stream_detail_dashboards_view/index.tsx | 8 ++++- .../public/hooks/use_dashboards_api.ts | 24 ++----------- .../public/hooks/use_dashboards_fetch.ts | 36 +++++++++++++++++++ 3 files changed, 45 insertions(+), 23 deletions(-) create mode 100644 x-pack/solutions/observability/plugins/streams_app/public/hooks/use_dashboards_fetch.ts diff --git a/x-pack/solutions/observability/plugins/streams_app/public/components/stream_detail_dashboards_view/index.tsx b/x-pack/solutions/observability/plugins/streams_app/public/components/stream_detail_dashboards_view/index.tsx index a9242b2e8ef80..1a80e82cf0c5f 100644 --- a/x-pack/solutions/observability/plugins/streams_app/public/components/stream_detail_dashboards_view/index.tsx +++ b/x-pack/solutions/observability/plugins/streams_app/public/components/stream_detail_dashboards_view/index.tsx @@ -12,13 +12,15 @@ import type { SanitizedDashboardAsset } from '@kbn/streams-plugin/server/routes/ import { AddDashboardFlyout } from './add_dashboard_flyout'; import { DashboardsTable } from './dashboard_table'; import { useDashboardsApi } from '../../hooks/use_dashboards_api'; +import { useDashboardsFetch } from '../../hooks/use_dashboards_fetch'; export function StreamDetailDashboardsView({ definition }: { definition?: StreamDefinition }) { const [query, setQuery] = useState(''); const [isAddDashboardFlyoutOpen, setIsAddDashboardFlyoutOpen] = useState(false); - const { dashboardsFetch, addDashboards, removeDashboards } = useDashboardsApi(definition?.id); + const dashboardsFetch = useDashboardsFetch(definition?.id); + const { addDashboards, removeDashboards } = useDashboardsApi(definition?.id); const [isUnlinkLoading, setIsUnlinkLoading] = useState(false); const linkedDashboards = useMemo(() => { @@ -45,7 +47,10 @@ export function StreamDetailDashboardsView({ definition }: { definition?: Stream onClick={async () => { try { setIsUnlinkLoading(true); + await removeDashboards(selectedDashboards); + await dashboardsFetch.refresh(); + setSelectedDashboards([]); } finally { setIsUnlinkLoading(false); @@ -93,6 +98,7 @@ export function StreamDetailDashboardsView({ definition }: { definition?: Stream entityId={definition.id} onAddDashboards={async (dashboards) => { await addDashboards(dashboards); + await dashboardsFetch.refresh(); setIsAddDashboardFlyoutOpen(false); }} onClose={() => { diff --git a/x-pack/solutions/observability/plugins/streams_app/public/hooks/use_dashboards_api.ts b/x-pack/solutions/observability/plugins/streams_app/public/hooks/use_dashboards_api.ts index 3cdb78e285e93..43029d9cf97fe 100644 --- a/x-pack/solutions/observability/plugins/streams_app/public/hooks/use_dashboards_api.ts +++ b/x-pack/solutions/observability/plugins/streams_app/public/hooks/use_dashboards_api.ts @@ -8,7 +8,6 @@ import { useCallback } from 'react'; import { useAbortController } from '@kbn/observability-utils-browser/hooks/use_abort_controller'; import type { SanitizedDashboardAsset } from '@kbn/streams-plugin/server/routes/dashboards/route'; import { useKibana } from './use_kibana'; -import { useStreamsAppFetch } from './use_streams_app_fetch'; export const useDashboardsApi = (id?: string) => { const { signal } = useAbortController(); @@ -20,20 +19,6 @@ export const useDashboardsApi = (id?: string) => { }, } = useKibana(); - const dashboardsFetch = useStreamsAppFetch(() => { - if (!id) { - return Promise.resolve(undefined); - } - return streamsRepositoryClient.fetch('GET /api/streams/{id}/dashboards', { - signal, - params: { - path: { - id, - }, - }, - }); - }, [id, signal, streamsRepositoryClient]); - const addDashboards = useCallback( async (dashboards: SanitizedDashboardAsset[]) => { if (!id) { @@ -53,10 +38,8 @@ export const useDashboardsApi = (id?: string) => { }, }, }); - - await dashboardsFetch.refresh(); }, - [dashboardsFetch, id, signal, streamsRepositoryClient] + [id, signal, streamsRepositoryClient] ); const removeDashboards = useCallback( @@ -77,14 +60,11 @@ export const useDashboardsApi = (id?: string) => { }, }, }); - - await dashboardsFetch.refresh(); }, - [dashboardsFetch, id, signal, streamsRepositoryClient] + [id, signal, streamsRepositoryClient] ); return { - dashboardsFetch, addDashboards, removeDashboards, }; diff --git a/x-pack/solutions/observability/plugins/streams_app/public/hooks/use_dashboards_fetch.ts b/x-pack/solutions/observability/plugins/streams_app/public/hooks/use_dashboards_fetch.ts new file mode 100644 index 0000000000000..8dbca5f64c29c --- /dev/null +++ b/x-pack/solutions/observability/plugins/streams_app/public/hooks/use_dashboards_fetch.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { useAbortController } from '@kbn/observability-utils-browser/hooks/use_abort_controller'; +import { useKibana } from './use_kibana'; +import { useStreamsAppFetch } from './use_streams_app_fetch'; + +export const useDashboardsFetch = (id?: string) => { + const { signal } = useAbortController(); + const { + dependencies: { + start: { + streams: { streamsRepositoryClient }, + }, + }, + } = useKibana(); + + const dashboardsFetch = useStreamsAppFetch(() => { + if (!id) { + return Promise.resolve(undefined); + } + return streamsRepositoryClient.fetch('GET /api/streams/{id}/dashboards', { + signal, + params: { + path: { + id, + }, + }, + }); + }, [id, signal, streamsRepositoryClient]); + + return dashboardsFetch; +}; From 7cf986b5bc1932bbc163e00ad2221d6d7378f8eb Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Mon, 23 Dec 2024 18:08:26 +0100 Subject: [PATCH 85/95] API tests --- .../src/typings.ts | 2 +- .../client/create_observability_es_client.ts | 2 +- .../es/storage/index.ts | 40 ++- .../es/storage/index_adapter/index.ts | 187 +++++++++- .../es/storage/storage_client.ts | 79 +++-- .../server/lib/streams/assets/asset_client.ts | 10 +- .../lib/streams/assets/asset_service.ts | 6 +- .../lib/streams/assets/storage_settings.ts | 2 +- .../streams/server/lib/streams/stream_crud.ts | 17 +- .../streams/server/routes/dashboards/route.ts | 8 +- .../streams/server/routes/streams/delete.ts | 14 +- .../apis/streams/assets/dashboard.ts | 332 ++++++++++++++++++ .../api_integration/apis/streams/classic.ts | 149 +++++--- .../api_integration/apis/streams/config.ts | 16 +- .../apis/streams/enrichment.ts | 11 +- .../apis/streams/flush_config.ts | 102 ++++-- .../api_integration/apis/streams/full_flow.ts | 12 +- .../apis/streams/helpers/repository_client.ts | 13 + .../apis/streams/helpers/requests.ts | 6 +- .../api_integration/apis/streams/index.ts | 1 + ...reate_supertest_service_from_repository.ts | 211 +++++++++++ 21 files changed, 1037 insertions(+), 183 deletions(-) create mode 100644 x-pack/test/api_integration/apis/streams/assets/dashboard.ts create mode 100644 x-pack/test/api_integration/apis/streams/helpers/repository_client.ts create mode 100644 x-pack/test/common/utils/server_route_repository/create_supertest_service_from_repository.ts diff --git a/src/platform/packages/shared/kbn-server-route-repository-utils/src/typings.ts b/src/platform/packages/shared/kbn-server-route-repository-utils/src/typings.ts index 5db4a87b8b326..6cc176113a590 100644 --- a/src/platform/packages/shared/kbn-server-route-repository-utils/src/typings.ts +++ b/src/platform/packages/shared/kbn-server-route-repository-utils/src/typings.ts @@ -213,7 +213,7 @@ type DecodedRequestParamsOfType = : never; export type EndpointOf = - keyof TServerRouteRepository; + keyof TServerRouteRepository & string; export type ReturnOf< TServerRouteRepository extends ServerRouteRepository, diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts index eaff732702c98..7731d72ffd0fe 100644 --- a/x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/client/create_observability_es_client.ts @@ -24,7 +24,7 @@ import { esqlResultToPlainObjects } from '../esql_result_to_plain_objects'; type SearchRequest = ESSearchRequest & { index: string | string[]; track_total_hits: number | boolean; - size: number | boolean; + size: number; }; export interface EsqlOptions { diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index.ts index 51c1bf4d516e2..b5b6f28cb25d2 100644 --- a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index.ts +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index.ts @@ -4,6 +4,16 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import type { + BulkRequest, + BulkResponse, + DeleteRequest, + DeleteResponse, + IndexRequest, + IndexResponse, + SearchRequest, +} from '@elastic/elasticsearch/lib/api/types'; +import { InferSearchResponseOf } from '@kbn/es-types'; import { StorageFieldTypeOf, StorageMappingProperty } from './types'; interface StorageSchemaProperties { @@ -24,9 +34,35 @@ export interface IndexStorageSettings extends StorageSettingsBase { export type StorageSettings = IndexStorageSettings; +export type StorageAdapterSearchRequest = Omit; +export type StorageAdapterSearchResponse< + TDocument, + TSearchRequest extends Omit +> = InferSearchResponseOf; + +export type StorageAdapterBulkRequest = Omit, 'index'>; +export type StorageAdapterBulkResponse = BulkResponse; + +export type StorageAdapterDeleteRequest = DeleteRequest; +export type StorageAdapterDeleteResponse = DeleteResponse; + +export type StorageAdapterIndexRequest = Omit< + IndexRequest, + 'index' +>; +export type StorageAdapterIndexResponse = IndexResponse; + export interface IStorageAdapter { - getSearchIndexPattern(): string; - getWriteTarget(): string; + bulk( + request: StorageAdapterBulkRequest + ): Promise; + search>( + request: StorageAdapterSearchRequest + ): Promise>; + index( + request: StorageAdapterIndexRequest + ): Promise; + delete(request: StorageAdapterDeleteRequest): Promise; } export type StorageSettingsOf> = diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/index.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/index.ts index 3630fbb812a8b..cb2542c605cd3 100644 --- a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/index.ts +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/index.ts @@ -5,19 +5,36 @@ * 2.0. */ -import { ElasticsearchClient, Logger } from '@kbn/core/server'; -import objectHash from 'object-hash'; -import stringify from 'json-stable-stringify'; import { + BulkResponseItem, + IndexResponse, IndicesPutIndexTemplateIndexTemplateMapping, MappingProperty, + Refresh, + SearchRequest, } from '@elastic/elasticsearch/lib/api/types'; -import { last, mapValues, orderBy, padStart } from 'lodash'; +import { ElasticsearchClient, Logger } from '@kbn/core/server'; import { isResponseError } from '@kbn/es-errors'; -import { IStorageAdapter, IndexStorageSettings, StorageSchema } from '..'; +import { InferSearchResponseOf } from '@kbn/es-types'; +import stringify from 'json-stable-stringify'; +import { last, mapValues, orderBy, padStart } from 'lodash'; +import objectHash from 'object-hash'; +import { + IStorageAdapter, + IndexStorageSettings, + StorageAdapterBulkRequest, + StorageAdapterBulkResponse, + StorageAdapterDeleteRequest, + StorageAdapterDeleteResponse, + StorageAdapterIndexRequest, + StorageAdapterIndexResponse, + StorageAdapterSearchRequest, + StorageAdapterSearchResponse, + StorageSchema, +} from '..'; import { StorageClient } from '../storage_client'; -import { IncompatibleSchemaUpdateError } from './errors'; import { StorageMappingProperty } from '../types'; +import { IncompatibleSchemaUpdateError } from './errors'; function getAliasName(name: string) { return name; @@ -87,7 +104,7 @@ function toElasticsearchMappingProperty(property: StorageMappingProperty): Mappi } export class StorageIndexAdapter - implements IStorageAdapter + implements IStorageAdapter { constructor( private readonly esClient: ElasticsearchClient, @@ -95,11 +112,11 @@ export class StorageIndexAdapter private readonly storage: TStorageSettings ) {} - getSearchIndexPattern(): string { - return getAliasName(this.storage.name); + private getSearchIndexPattern(): string { + return `${getAliasName(this.storage.name)}*`; } - getWriteTarget(): string { + private getWriteTarget(): string { return getAliasName(this.storage.name); } @@ -254,7 +271,7 @@ export class StorageIndexAdapter } } - async bootstrap() { + private async bootstrap() { const { name } = this.storage; this.logger.debug('Retrieving existing Elasticsearch components'); @@ -298,7 +315,153 @@ export class StorageIndexAdapter await this.rolloverIfNeeded(); } + private async retryAfterBootstrap(cb: () => Promise): Promise { + return cb().catch(async (error) => { + if (isResponseError(error) && error.statusCode === 404) { + this.logger.info(`Write target for ${this.storage.name} not found, bootstrapping`); + await this.bootstrap(); + return cb(); + } + throw error; + }); + } + + async search>( + request: StorageAdapterSearchRequest + ): Promise> { + return this.esClient.search({ + ...request, + index: this.getSearchIndexPattern(), + allow_no_indices: true, + }) as unknown as Promise>; + } + + private async removeDanglingItems({ ids, refresh }: { ids: string[]; refresh?: Refresh }) { + const writeIndex = await this.getCurrentWriteIndexName(); + + this.logger.debug( + () => `Removing dangling items for ${ids.join(', ')}, write index is ${writeIndex}` + ); + + if (writeIndex && ids.length) { + const danglingItemsResponse = await this.search({ + query: { + bool: { + filter: [{ terms: { _id: ids } }], + must_not: [ + { + term: { + _index: writeIndex, + }, + }, + ], + }, + }, + size: 10_000, + }); + + const danglingItemsToDelete = danglingItemsResponse.hits.hits.map((hit) => ({ + id: hit._id!, + index: hit._index, + })); + + if (danglingItemsToDelete.length > 0) { + const shouldRefresh = refresh === true || refresh === 'true' || refresh === 'wait_for'; + + this.logger.debug(() => `Deleting ${danglingItemsToDelete.length} dangling items`); + + await this.esClient.deleteByQuery({ + index: this.getSearchIndexPattern(), + refresh: shouldRefresh, + wait_for_completion: shouldRefresh, + query: { + bool: { + should: danglingItemsToDelete.map((item) => { + return { + bool: { + filter: [ + { + term: { + _index: item.index, + }, + }, + { + term: { + _id: item.id, + }, + }, + ], + }, + }; + }), + minimum_should_match: 1, + }, + }, + }); + } + } + } + + async index(request: StorageAdapterIndexRequest): Promise { + const attemptIndex = (): Promise => { + return this.esClient.index({ + ...request, + index: this.getWriteTarget(), + require_alias: true, + }); + }; + + return this.retryAfterBootstrap(attemptIndex).then(async (response) => { + this.logger.debug(() => `Indexed document ${request.id} into ${response._index}`); + if (request.id) { + await this.removeDanglingItems({ + ids: [request.id], + refresh: request.refresh, + }); + } + + return response; + }); + } + + async bulk(request: StorageAdapterBulkRequest): Promise { + const attemptBulk = () => { + return this.esClient.bulk({ + ...request, + index: this.getWriteTarget(), + require_alias: true, + }); + }; + + return this.retryAfterBootstrap(attemptBulk).then(async (response) => { + const ids = response.items + .filter( + (item): item is { index: BulkResponseItem & { _id: string } } => + !!item.index && !!item.index._id && !item.index.error + ) + .map((item) => item.index._id); + + if (ids.length) { + await this.removeDanglingItems({ ids, refresh: request.refresh }); + } + + return response; + }); + } + + async delete({ + id, + index, + refresh, + }: StorageAdapterDeleteRequest): Promise { + return await this.esClient.delete({ + index, + id, + refresh, + }); + } + getClient(): StorageClient { - return new StorageClient(this, this.esClient, this.logger); + return new StorageClient(this, this.logger); } } diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/storage_client.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/storage_client.ts index ba8938a489361..eb7aecdbb6100 100644 --- a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/storage_client.ts +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/storage_client.ts @@ -5,54 +5,41 @@ * 2.0. */ -import { ElasticsearchClient, Logger } from '@kbn/core/server'; import { BulkRequest } from '@elastic/elasticsearch/lib/api/types'; +import { withSpan } from '@kbn/apm-utils'; +import { Logger } from '@kbn/core/server'; import { compact } from 'lodash'; import { IStorageAdapter, StorageDocumentOf, StorageSettings } from '.'; -import { - ObservabilityESSearchRequest, - ObservabilityElasticsearchClient, - createObservabilityEsClient, -} from '../client/create_observability_es_client'; +import { ObservabilityESSearchRequest } from '../client/create_observability_es_client'; -type StorageBulkOperation = +type StorageBulkOperation = | { - create: { document: Omit; _id: string }; + index: { document: Omit; _id?: string }; } | { delete: { _id: string } }; export class StorageClient { - private readonly esClient: ObservabilityElasticsearchClient; - constructor( - private readonly storage: IStorageAdapter, - esClient: ElasticsearchClient, - logger: Logger - ) { - this.esClient = createObservabilityEsClient({ - client: esClient, - logger, - }); - } + constructor(private readonly storage: IStorageAdapter, logger: Logger) {} search>( operationName: string, request: TSearchRequest ) { - return this.esClient.search< - StorageDocumentOf, - TSearchRequest & { index: string } - >(operationName, { ...request, index: this.storage.getSearchIndexPattern() }); + return withSpan(operationName, () => + this.storage.search, Omit>( + request + ) + ); } async index({ id, document, }: { - id: string; + id?: string; document: Omit, '_id'>; }) { - await this.esClient.client.index({ - index: this.storage.getSearchIndexPattern(), + await this.storage.index, '_id'>>({ document, refresh: 'wait_for', id, @@ -60,28 +47,44 @@ export class StorageClient { } async delete(id: string) { - await this.esClient.client.delete({ - id, - refresh: 'wait_for', - index: this.storage.getSearchIndexPattern(), + const searchResponse = await this.storage.search({ + query: { + bool: { + filter: [ + { + term: { + id, + }, + }, + ], + }, + }, }); + + const document = searchResponse.hits.hits[0]; + + let deleted: boolean = false; + + if (document) { + await this.storage.delete({ id, index: document._index }); + deleted = true; + } + + return { acknowledged: true, deleted }; } async bulk(operations: Array>>) { - const index = this.storage.getWriteTarget(); - - const result = await this.esClient.client.bulk({ - index, + const result = await this.storage.bulk({ refresh: 'wait_for', operations: operations.flatMap((operation): BulkRequest['operations'] => { - if ('create' in operation) { + if ('index' in operation) { return [ { - create: { - _id: operation.create._id, + index: { + _id: operation.index._id, }, }, - operation.create.document, + operation.index.document, ]; } diff --git a/x-pack/solutions/observability/plugins/streams/server/lib/streams/assets/asset_client.ts b/x-pack/solutions/observability/plugins/streams/server/lib/streams/assets/asset_client.ts index 2231c293643cf..7a1f8434e8d5c 100644 --- a/x-pack/solutions/observability/plugins/streams/server/lib/streams/assets/asset_client.ts +++ b/x-pack/solutions/observability/plugins/streams/server/lib/streams/assets/asset_client.ts @@ -78,14 +78,14 @@ function getAssetDocument({ }; } -interface AssetBulkCreateOperation { - create: { asset: AssetLink }; +interface AssetBulkIndexOperation { + index: { asset: AssetLink }; } interface AssetBulkDeleteOperation { delete: { asset: AssetLink }; } -export type AssetBulkOperation = AssetBulkCreateOperation | AssetBulkDeleteOperation; +export type AssetBulkOperation = AssetBulkIndexOperation | AssetBulkDeleteOperation; export class AssetClient { constructor( @@ -215,9 +215,9 @@ export class AssetClient { entityType, }); - if ('create' in operation) { + if ('index' in operation) { return { - create: { + index: { document, _id, }, diff --git a/x-pack/solutions/observability/plugins/streams/server/lib/streams/assets/asset_service.ts b/x-pack/solutions/observability/plugins/streams/server/lib/streams/assets/asset_service.ts index 3d27b6dd182af..c83418aeda743 100644 --- a/x-pack/solutions/observability/plugins/streams/server/lib/streams/assets/asset_service.ts +++ b/x-pack/solutions/observability/plugins/streams/server/lib/streams/assets/asset_service.ts @@ -18,10 +18,10 @@ export class AssetService { private readonly coreSetup: CoreSetup, private readonly logger: Logger ) { - this.adapter$ = defer(() => from(this.prepareIndex())).pipe(shareReplay(1)); + this.adapter$ = defer(() => from(this.getAdapter())).pipe(shareReplay(1)); } - async prepareIndex(): Promise> { + async getAdapter(): Promise> { const [coreStart] = await this.coreSetup.getStartServices(); const esClient = coreStart.elasticsearch.client.asInternalUser; @@ -30,7 +30,7 @@ export class AssetService { this.logger.get('assets'), assetStorageSettings ); - await adapter.bootstrap(); + return adapter; } diff --git a/x-pack/solutions/observability/plugins/streams/server/lib/streams/assets/storage_settings.ts b/x-pack/solutions/observability/plugins/streams/server/lib/streams/assets/storage_settings.ts index b3609ee311500..92ac034c45353 100644 --- a/x-pack/solutions/observability/plugins/streams/server/lib/streams/assets/storage_settings.ts +++ b/x-pack/solutions/observability/plugins/streams/server/lib/streams/assets/storage_settings.ts @@ -10,7 +10,7 @@ import { ASSET_ASSET_ID, ASSET_ENTITY_ID, ASSET_ENTITY_TYPE, ASSET_TYPE } from ' import { ASSET_TYPES } from '../../../../common/assets'; export const assetStorageSettings = { - name: '.kibana_stream_assets', + name: '.kibana_streams_assets', schema: { properties: { [ASSET_ASSET_ID]: types.keyword({ required: true }), diff --git a/x-pack/solutions/observability/plugins/streams/server/lib/streams/stream_crud.ts b/x-pack/solutions/observability/plugins/streams/server/lib/streams/stream_crud.ts index 950896f37cf34..4c0ccf7ee8890 100644 --- a/x-pack/solutions/observability/plugins/streams/server/lib/streams/stream_crud.ts +++ b/x-pack/solutions/observability/plugins/streams/server/lib/streams/stream_crud.ts @@ -310,21 +310,26 @@ async function getUnmanagedElasticsearchAssets({ name, scopedClusterClient, }: ReadUnmanagedAssetsParams) { - let dataStream: IndicesDataStream; + let dataStream: IndicesDataStream | undefined; try { - const response = await scopedClusterClient.asInternalUser.indices.getDataStream({ name }); + const response = await scopedClusterClient.asCurrentUser.indices.getDataStream({ name }); dataStream = response.data_streams[0]; } catch (e) { if (e.meta?.statusCode === 404) { - throw new DefinitionNotFound(`Stream definition for ${name} not found.`); + // fall through and throw not found + } else { + throw e; } - throw e; + } + + if (!dataStream) { + throw new DefinitionNotFound(`Stream definition for ${name} not found.`); } // retrieve linked index template, component template and ingest pipeline const templateName = dataStream.template; const componentTemplates: string[] = []; - const template = await scopedClusterClient.asInternalUser.indices.getIndexTemplate({ + const template = await scopedClusterClient.asCurrentUser.indices.getIndexTemplate({ name: templateName, }); if (template.index_templates.length) { @@ -333,7 +338,7 @@ async function getUnmanagedElasticsearchAssets({ }); } const writeIndexName = dataStream.indices.at(-1)?.index_name!; - const currentIndex = await scopedClusterClient.asInternalUser.indices.get({ + const currentIndex = await scopedClusterClient.asCurrentUser.indices.get({ index: writeIndexName, }); const ingestPipelineId = currentIndex[writeIndexName].settings?.index?.default_pipeline!; diff --git a/x-pack/solutions/observability/plugins/streams/server/routes/dashboards/route.ts b/x-pack/solutions/observability/plugins/streams/server/routes/dashboards/route.ts index eda8e409e7270..b5ea8646daf6b 100644 --- a/x-pack/solutions/observability/plugins/streams/server/routes/dashboards/route.ts +++ b/x-pack/solutions/observability/plugins/streams/server/routes/dashboards/route.ts @@ -200,7 +200,7 @@ const bulkDashboardsRoute = createServerRoute({ operations: z.array( z.union([ z.object({ - create: dashboardSchema, + index: dashboardSchema, }), z.object({ delete: dashboardSchema, @@ -223,12 +223,12 @@ const bulkDashboardsRoute = createServerRoute({ entityType: 'stream', }, operations.map((operation) => { - if ('create' in operation) { + if ('index' in operation) { return { - create: { + index: { asset: { assetType: 'dashboard', - assetId: operation.create.id, + assetId: operation.index.id, }, }, }; diff --git a/x-pack/solutions/observability/plugins/streams/server/routes/streams/delete.ts b/x-pack/solutions/observability/plugins/streams/server/routes/streams/delete.ts index bd863538e0e6a..1f3e9569755a1 100644 --- a/x-pack/solutions/observability/plugins/streams/server/routes/streams/delete.ts +++ b/x-pack/solutions/observability/plugins/streams/server/routes/streams/delete.ts @@ -54,13 +54,17 @@ export const deleteStreamRoute = createServerRoute({ const { scopedClusterClient, assetClient } = await getScopedClients({ request }); const parentId = getParentId(params.path.id); - if (!parentId) { - throw new MalformedStreamId('Cannot delete root stream'); + if (parentId) { + // need to update parent first to cut off documents streaming down + await updateParentStream( + scopedClusterClient, + assetClient, + params.path.id, + parentId, + logger + ); } - // need to update parent first to cut off documents streaming down - await updateParentStream(scopedClusterClient, assetClient, params.path.id, parentId, logger); - await deleteStream(scopedClusterClient, assetClient, params.path.id, logger); return { acknowledged: true }; diff --git a/x-pack/test/api_integration/apis/streams/assets/dashboard.ts b/x-pack/test/api_integration/apis/streams/assets/dashboard.ts new file mode 100644 index 0000000000000..cffd6a6a60ba6 --- /dev/null +++ b/x-pack/test/api_integration/apis/streams/assets/dashboard.ts @@ -0,0 +1,332 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import expect from '@kbn/expect'; +import { enableStreams, indexDocument } from '../helpers/requests'; +import { createStreamsRepositorySupertestClient } from '../helpers/repository_client'; +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { cleanUpRootStream } from '../helpers/cleanup'; + +export default function ({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const esClient = getService('es'); + + const kibanaServer = getService('kibanaServer'); + + const apiClient = createStreamsRepositorySupertestClient(supertest); + + const SPACE_ID = 'default'; + const ARCHIVES = [ + 'test/api_integration/fixtures/kbn_archiver/saved_objects/search.json', + 'test/api_integration/fixtures/kbn_archiver/saved_objects/basic.json', + ]; + + const SEARCH_DASHBOARD_ID = 'b70c7ae0-3224-11e8-a572-ffca06da1357'; + const BASIC_DASHBOARD_ID = 'be3733a0-9efe-11e7-acb3-3dab96693fab'; + const BASIC_DASHBOARD_TITLE = 'Requests'; + + async function loadDashboards() { + for (const archive of ARCHIVES) { + await kibanaServer.importExport.load(archive, { space: SPACE_ID }); + } + } + + async function unloadDashboards() { + for (const archive of ARCHIVES) { + await kibanaServer.importExport.unload(archive, { space: SPACE_ID }); + } + } + + async function linkDashboard(id: string) { + const response = await apiClient.fetch('PUT /api/streams/{id}/dashboards/{dashboardId}', { + params: { path: { id: 'logs', dashboardId: id } }, + }); + + expect(response.status).to.be(200); + } + + async function unlinkDashboard(id: string) { + const response = await apiClient.fetch('DELETE /api/streams/{id}/dashboards/{dashboardId}', { + params: { path: { id: 'logs', dashboardId: id } }, + }); + + expect(response.status).to.be(200); + } + + async function bulkLinkDashboard(...ids: string[]) { + const response = await apiClient.fetch('POST /api/streams/{id}/dashboards/_bulk', { + params: { + path: { id: 'logs' }, + body: { + operations: ids.map((id) => { + return { + index: { + id, + }, + }; + }), + }, + }, + }); + + expect(response.status).to.be(200); + } + + async function bulkUnlinkDashboard(...ids: string[]) { + const response = await apiClient.fetch('POST /api/streams/{id}/dashboards/_bulk', { + params: { + path: { id: 'logs' }, + body: { + operations: ids.map((id) => { + return { + delete: { + id, + }, + }; + }), + }, + }, + }); + + expect(response.status).to.be(200); + } + + async function deleteAssetIndices() { + const concreteIndices = await esClient.indices.resolveIndex({ + name: '.kibana_streams_assets*', + }); + + if (concreteIndices.indices.length) { + await esClient.indices.delete({ + index: concreteIndices.indices.map((index) => index.name), + }); + } + } + + describe('Asset links', () => { + before(async () => { + await enableStreams(supertest); + + await indexDocument(esClient, 'logs', { + '@timestamp': '2024-01-01T00:00:10.000Z', + message: '2023-01-01T00:00:10.000Z error test', + }); + }); + + after(async () => { + await cleanUpRootStream(esClient); + + await esClient.indices.deleteDataStream({ + name: ['logs*'], + }); + + await deleteAssetIndices(); + }); + + describe('without writing', () => { + it('creates no indices initially', async () => { + const exists = await esClient.indices.exists({ index: '.kibana_streams_assets' }); + + expect(exists).to.eql(false); + }); + + it('creates no indices after reading the assets', async () => { + const response = await apiClient.fetch('GET /api/streams/{id}/dashboards', { + params: { path: { id: 'logs' } }, + }); + + expect(response.status).to.be(200); + + const exists = await esClient.indices.exists({ index: '.kibana_streams_assets' }); + + expect(exists).to.eql(false); + }); + }); + + describe('after linking a dashboard', () => { + before(async () => { + await loadDashboards(); + + await linkDashboard(SEARCH_DASHBOARD_ID); + }); + + after(async () => { + await unloadDashboards(); + await unlinkDashboard(SEARCH_DASHBOARD_ID); + }); + + it('creates the index', async () => { + const exists = await esClient.indices.exists({ index: '.kibana_streams_assets' }); + + expect(exists).to.be(true); + }); + + it('lists the dashboard in the stream response', async () => { + const response = await apiClient.fetch('GET /api/streams/{id}', { + params: { path: { id: 'logs' } }, + }); + + expect(response.status).to.eql(200); + + expect(response.body.dashboards?.length).to.eql(1); + }); + + it('lists the dashboard in the dashboards get response', async () => { + const response = await apiClient.fetch('GET /api/streams/{id}/dashboards', { + params: { path: { id: 'logs' } }, + }); + + expect(response.status).to.eql(200); + + expect(response.body.dashboards.length).to.eql(1); + }); + + describe('after manually rolling over the index and relinking the dashboard', () => { + before(async () => { + await esClient.indices.create({ + index: `.kibana_streams_assets-000002`, + }); + + await esClient.indices.updateAliases({ + actions: [ + { + add: { + index: `.kibana_streams_assets-000001`, + alias: `.kibana_streams_assets`, + is_write_index: false, + }, + }, + { + add: { + index: `.kibana_streams_assets-000002`, + alias: `.kibana_streams_assets`, + is_write_index: true, + }, + }, + ], + }); + + await unlinkDashboard(SEARCH_DASHBOARD_ID); + await linkDashboard(SEARCH_DASHBOARD_ID); + }); + + it('there are no duplicates', async () => { + const response = await apiClient.fetch('GET /api/streams/{id}/dashboards', { + params: { path: { id: 'logs' } }, + }); + + expect(response.status).to.eql(200); + + expect(response.body.dashboards.length).to.eql(1); + + const esResponse = await esClient.search({ + index: `.kibana_streams_assets`, + }); + + expect(esResponse.hits.hits.length).to.eql(1); + }); + }); + + describe('after deleting the indices and relinking the dashboard', () => { + before(async () => { + await deleteAssetIndices(); + + await unlinkDashboard(SEARCH_DASHBOARD_ID); + await linkDashboard(SEARCH_DASHBOARD_ID); + }); + + it('recovers on write and lists the linked dashboard ', async () => { + const response = await apiClient.fetch('GET /api/streams/{id}/dashboards', { + params: { path: { id: 'logs' } }, + }); + + expect(response.status).to.eql(200); + + expect(response.body.dashboards.length).to.eql(1); + }); + }); + }); + + describe('after using the bulk API', () => { + before(async () => { + await loadDashboards(); + + await bulkLinkDashboard(SEARCH_DASHBOARD_ID, BASIC_DASHBOARD_ID); + }); + + after(async () => { + await bulkUnlinkDashboard(SEARCH_DASHBOARD_ID, BASIC_DASHBOARD_ID); + await unloadDashboards(); + }); + + it('shows the linked dashboards', async () => { + const response = await apiClient.fetch('GET /api/streams/{id}/dashboards', { + params: { path: { id: 'logs' } }, + }); + + expect(response.body.dashboards.length).to.eql(2); + }); + + describe('after unlinking one dashboard', () => { + before(async () => { + await bulkUnlinkDashboard(SEARCH_DASHBOARD_ID); + }); + + it('only shows the remaining linked dashboard', async () => { + const response = await apiClient.fetch('GET /api/streams/{id}/dashboards', { + params: { path: { id: 'logs' } }, + }); + + expect(response.body.dashboards.length).to.eql(1); + + expect(response.body.dashboards[0].id).to.eql(BASIC_DASHBOARD_ID); + }); + }); + }); + + describe('suggestions', () => { + before(async () => { + await loadDashboards(); + + await linkDashboard(SEARCH_DASHBOARD_ID); + }); + + after(async () => { + await unlinkDashboard(SEARCH_DASHBOARD_ID); + await unloadDashboards(); + }); + + describe('after creating multiple dashboards', () => { + it('suggests dashboards to link', async () => { + const response = await apiClient.fetch('POST /api/streams/{id}/dashboards/_suggestions', { + params: { path: { id: 'logs' }, body: { tags: [] }, query: { query: '' } }, + }); + + expect(response.status).to.eql(200); + expect(response.body.suggestions.length).to.eql(2); + }); + + // TODO: needs a dataset with dashboards with tags + it.skip('filters suggested dashboards based on tags', () => {}); + + it('filters suggested dashboards based on the query', async () => { + const response = await apiClient.fetch('POST /api/streams/{id}/dashboards/_suggestions', { + params: { + path: { id: 'logs' }, + body: { tags: [] }, + query: { query: BASIC_DASHBOARD_TITLE }, + }, + }); + + expect(response.status).to.eql(200); + expect(response.body.suggestions.length).to.eql(1); + + expect(response.body.suggestions[0].id).to.eql(BASIC_DASHBOARD_ID); + }); + }); + }); + }); +} diff --git a/x-pack/test/api_integration/apis/streams/classic.ts b/x-pack/test/api_integration/apis/streams/classic.ts index 25a7238a757ca..c260beae8d23f 100644 --- a/x-pack/test/api_integration/apis/streams/classic.ts +++ b/x-pack/test/api_integration/apis/streams/classic.ts @@ -6,19 +6,11 @@ */ import expect from '@kbn/expect'; -import { JsonObject } from '@kbn/utility-types'; -import { - deleteStream, - enableStreams, - fetchDocument, - getStream, - indexDocument, - listStreams, - putStream, -} from './helpers/requests'; -import { FtrProviderContext } from '../../ftr_provider_context'; import { waitForDocumentInIndex } from '../../../alerting_api_integration/observability/helpers/alerting_wait_for_helpers'; +import { FtrProviderContext } from '../../ftr_provider_context'; import { cleanUpRootStream } from './helpers/cleanup'; +import { createStreamsRepositorySupertestClient } from './helpers/repository_client'; +import { fetchDocument, indexDocument } from './helpers/requests'; export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); @@ -26,27 +18,37 @@ export default function ({ getService }: FtrProviderContext) { const retryService = getService('retry'); const logger = getService('log'); + const TEST_STREAM_NAME = 'logs-test-default'; + + const apiClient = createStreamsRepositorySupertestClient(supertest); + describe('Classic streams', () => { - after(async () => { - await cleanUpRootStream(esClient); + before(async () => { + await apiClient.fetch('POST /api/streams/_enable'); }); - before(async () => { - await enableStreams(supertest); + after(async () => { + await cleanUpRootStream(esClient); + await esClient.indices.deleteDataStream({ + name: ['logs*'], + }); }); it('Shows non-wired data streams', async () => { const doc = { message: '2023-01-01T00:00:10.000Z error test', }; - const response = await indexDocument(esClient, 'logs-test-default', doc); + const response = await indexDocument(esClient, TEST_STREAM_NAME, doc); expect(response.result).to.eql('created'); - const streams = await listStreams(supertest); - const classicStream = streams.definitions.find( - (stream: JsonObject) => stream.id === 'logs-test-default' - ); + + const { body: streams, status } = await apiClient.fetch('GET /api/streams'); + + expect(status).to.eql(200); + + const classicStream = streams.definitions.find((stream) => stream.id === TEST_STREAM_NAME); + expect(classicStream).to.eql({ - id: 'logs-test-default', + id: TEST_STREAM_NAME, managed: false, children: [], fields: [], @@ -55,30 +57,47 @@ export default function ({ getService }: FtrProviderContext) { }); it('Allows setting processing on classic streams', async () => { - const response = await putStream(supertest, 'logs-test-default', { - managed: false, - children: [], - fields: [], - processing: [ - { - config: { - type: 'grok', - field: 'message', - patterns: [ - '%{TIMESTAMP_ISO8601:inner_timestamp} %{LOGLEVEL:log.level} %{GREEDYDATA:message2}', - ], - }, + const putResponse = await apiClient.fetch('PUT /api/streams/{id}', { + params: { + path: { + id: TEST_STREAM_NAME, }, - ], + body: { + managed: false, + children: [], + fields: [], + processing: [ + { + config: { + type: 'grok', + field: 'message', + patterns: [ + '%{TIMESTAMP_ISO8601:inner_timestamp} %{LOGLEVEL:log.level} %{GREEDYDATA:message2}', + ], + }, + }, + ], + }, + }, }); - expect(response).to.have.property('acknowledged', true); - const streamBody = await getStream(supertest, 'logs-test-default'); - expect(streamBody).to.eql({ - id: 'logs-test-default', + + expect(putResponse.status).to.eql(200); + + expect(putResponse.body).to.have.property('acknowledged', true); + + const getResponse = await apiClient.fetch('GET /api/streams/{id}', { + params: { path: { id: TEST_STREAM_NAME } }, + }); + + expect(getResponse.status).to.eql(200); + + expect(getResponse.body).to.eql({ + id: TEST_STREAM_NAME, managed: false, children: [], inheritedFields: [], fields: [], + dashboards: [], processing: [ { config: { @@ -98,16 +117,16 @@ export default function ({ getService }: FtrProviderContext) { '@timestamp': '2024-01-01T00:00:10.000Z', message: '2023-01-01T00:00:10.000Z error test', }; - const response = await indexDocument(esClient, 'logs-test-default', doc); + const response = await indexDocument(esClient, TEST_STREAM_NAME, doc); expect(response.result).to.eql('created'); await waitForDocumentInIndex({ esClient, - indexName: 'logs-test-default', + indexName: TEST_STREAM_NAME, retryService, logger, docCountTarget: 2, }); - const result = await fetchDocument(esClient, 'logs-test-default', response._id); + const result = await fetchDocument(esClient, TEST_STREAM_NAME, response._id); expect(result._source).to.eql({ '@timestamp': '2024-01-01T00:00:10.000Z', message: '2023-01-01T00:00:10.000Z error test', @@ -120,13 +139,21 @@ export default function ({ getService }: FtrProviderContext) { }); it('Allows removing processing on classic streams', async () => { - const response = await putStream(supertest, 'logs-test-default', { - managed: false, - children: [], - fields: [], - processing: [], + const response = await apiClient.fetch('PUT /api/streams/{id}', { + params: { + path: { id: TEST_STREAM_NAME }, + body: { + managed: false, + children: [], + fields: [], + processing: [], + }, + }, }); - expect(response).to.have.property('acknowledged', true); + + expect(response.status).to.eql(200); + + expect(response.body).to.have.property('acknowledged', true); }); it('Executes processing on classic streams after removing processing', async () => { @@ -134,16 +161,16 @@ export default function ({ getService }: FtrProviderContext) { // default logs pipeline fills in timestamp with current date if not set message: '2023-01-01T00:00:10.000Z info mylogger this is the message', }; - const response = await indexDocument(esClient, 'logs-test-default', doc); + const response = await indexDocument(esClient, TEST_STREAM_NAME, doc); expect(response.result).to.eql('created'); await waitForDocumentInIndex({ esClient, - indexName: 'logs-test-default', + indexName: TEST_STREAM_NAME, retryService, logger, docCountTarget: 3, }); - const result = await fetchDocument(esClient, 'logs-test-default', response._id); + const result = await fetchDocument(esClient, TEST_STREAM_NAME, response._id); expect(result._source).to.eql({ // accept any date '@timestamp': (result._source as { [key: string]: unknown })['@timestamp'], @@ -152,10 +179,22 @@ export default function ({ getService }: FtrProviderContext) { }); it('Allows deleting classic streams', async () => { - await deleteStream(supertest, 'logs-test-default'); - const streams = await listStreams(supertest); - const classicStream = streams.definitions.find( - (stream: JsonObject) => stream.id === 'logs-test-default' + const deleteStreamResponse = await apiClient.fetch('DELETE /api/streams/{id}', { + params: { + path: { + id: TEST_STREAM_NAME, + }, + }, + }); + + expect(deleteStreamResponse.status).to.eql(200); + + const getStreamsResponse = await apiClient.fetch('GET /api/streams'); + + expect(getStreamsResponse.status).to.eql(200); + + const classicStream = getStreamsResponse.body.definitions.find( + (stream) => stream.id === TEST_STREAM_NAME ); expect(classicStream).to.eql(undefined); }); diff --git a/x-pack/test/api_integration/apis/streams/config.ts b/x-pack/test/api_integration/apis/streams/config.ts index c737db9499836..2fbcac9ff3b8d 100644 --- a/x-pack/test/api_integration/apis/streams/config.ts +++ b/x-pack/test/api_integration/apis/streams/config.ts @@ -5,12 +5,26 @@ * 2.0. */ -import { FtrConfigProviderContext } from '@kbn/test'; +import { FtrConfigProviderContext, getKibanaCliLoggers } from '@kbn/test'; export default async function ({ readConfigFile }: FtrConfigProviderContext) { const baseIntegrationTestsConfig = await readConfigFile(require.resolve('../../config.ts')); return { ...baseIntegrationTestsConfig.getAll(), + kbnTestServer: { + ...baseIntegrationTestsConfig.get('kbnTestServer'), + serverArgs: [ + ...baseIntegrationTestsConfig.get('kbnTestServer.serverArgs'), + `--logging.loggers=${JSON.stringify([ + ...getKibanaCliLoggers(baseIntegrationTestsConfig.get('kbnTestServer.serverArgs')), + { + name: 'plugins.streams', + level: 'debug', + appenders: ['default'], + }, + ])}`, + ], + }, testFiles: [require.resolve('.')], }; } diff --git a/x-pack/test/api_integration/apis/streams/enrichment.ts b/x-pack/test/api_integration/apis/streams/enrichment.ts index 22293b09fbbbb..2310a09286938 100644 --- a/x-pack/test/api_integration/apis/streams/enrichment.ts +++ b/x-pack/test/api_integration/apis/streams/enrichment.ts @@ -20,14 +20,17 @@ export default function ({ getService }: FtrProviderContext) { const logger = getService('log'); describe('Enrichment', () => { - after(async () => { - await cleanUpRootStream(esClient); - }); - before(async () => { await enableStreams(supertest); }); + after(async () => { + await cleanUpRootStream(esClient); + await esClient.indices.deleteDataStream({ + name: ['logs*'], + }); + }); + it('Place processing steps', async () => { const body = { fields: [ diff --git a/x-pack/test/api_integration/apis/streams/flush_config.ts b/x-pack/test/api_integration/apis/streams/flush_config.ts index f3fa79e92d457..2925cd54e06ae 100644 --- a/x-pack/test/api_integration/apis/streams/flush_config.ts +++ b/x-pack/test/api_integration/apis/streams/flush_config.ts @@ -6,11 +6,17 @@ */ import expect from '@kbn/expect'; -import { SearchTotalHits } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { deleteStream, enableStreams, indexDocument } from './helpers/requests'; +import { ClientRequestParamsOf } from '@kbn/server-route-repository-utils'; +import { StreamsRouteRepository } from '@kbn/streams-plugin/server'; import { FtrProviderContext } from '../../ftr_provider_context'; -import { waitForDocumentInIndex } from '../../../alerting_api_integration/observability/helpers/alerting_wait_for_helpers'; import { cleanUpRootStream } from './helpers/cleanup'; +import { createStreamsRepositorySupertestClient } from './helpers/repository_client'; +import { enableStreams, indexDocument } from './helpers/requests'; + +type StreamPutItem = ClientRequestParamsOf< + StreamsRouteRepository, + 'PUT /api/streams/{id}' +>['params']['body'] & { id: string }; const streams = [ { @@ -64,7 +70,12 @@ const streams = [ { id: 'logs.test', processing: [], - fields: [], + fields: [ + { + name: 'numberfield', + type: 'long', + }, + ], children: [], }, { @@ -80,45 +91,79 @@ const streams = [ ], fields: [ { - name: 'numberfield', - type: 'long', + name: 'field2', + type: 'keyword', }, ], children: [], }, -]; +] satisfies StreamPutItem[]; export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); const esClient = getService('es'); - const retryService = getService('retry'); - const logger = getService('log'); + + const apiClient = createStreamsRepositorySupertestClient(supertest); // An anticipated use case is that a user will want to flush a tree of streams from a config file describe('Flush from config file', () => { after(async () => { - await deleteStream(supertest, 'logs.nginx'); await cleanUpRootStream(esClient); + await esClient.indices.deleteDataStream({ + name: ['logs*'], + }); }); - // Note: Each step is dependent on the previous - it('Enable streams', async () => { + before(async () => { await enableStreams(supertest); + await createStreams(); + await indexDocuments(); }); - it('PUTs all streams one by one without errors', async () => { - for (const { id: streamId, ...stream } of streams) { - const response = await supertest - .put(`/api/streams/${streamId}`) - .set('kbn-xsrf', 'xxx') - .send(stream) - .expect(200); + it('puts the data in the right data streams', async () => { + const logsResponse = await esClient.search({ + index: 'logs', + query: { + match: { 'log.level': 'info' }, + }, + }); - expect(response.body).to.have.property('acknowledged', true); - } + expect(logsResponse.hits.total).to.eql({ value: 1, relation: 'eq' }); + + const logsTestResponse = await esClient.search({ + index: 'logs.test', + query: { + match: { numberfield: 20 }, + }, + }); + + expect(logsTestResponse.hits.total).to.eql({ value: 1, relation: 'eq' }); + + const logsTest2Response = await esClient.search({ + index: 'logs.test2', + query: { + match: { field2: 'abc' }, + }, + }); + + expect(logsTest2Response.hits.total).to.eql({ value: 1, relation: 'eq' }); }); - it('send data and it is handled properly', async () => { + async function createStreams() { + for (const { id: streamId, ...stream } of streams) { + await apiClient + .fetch('PUT /api/streams/{id}', { + params: { + body: stream, + path: { id: streamId }, + }, + }) + .expect(200) + .then((response) => expect(response.body.acknowledged).to.eql(true)); + } + } + + async function indexDocuments() { // send data that stays in logs const doc = { '@timestamp': '2024-01-01T00:00:00.000Z', @@ -127,7 +172,6 @@ export default function ({ getService }: FtrProviderContext) { }; const response = await indexDocument(esClient, 'logs', doc); expect(response.result).to.eql('created'); - await waitForDocumentInIndex({ esClient, indexName: 'logs', retryService, logger }); // send data that lands in logs.test const doc2 = { @@ -137,7 +181,6 @@ export default function ({ getService }: FtrProviderContext) { }; const response2 = await indexDocument(esClient, 'logs', doc2); expect(response2.result).to.eql('created'); - await waitForDocumentInIndex({ esClient, indexName: 'logs.test', retryService, logger }); // send data that lands in logs.test2 const doc3 = { @@ -147,15 +190,6 @@ export default function ({ getService }: FtrProviderContext) { }; const response3 = await indexDocument(esClient, 'logs', doc3); expect(response3.result).to.eql('created'); - await waitForDocumentInIndex({ esClient, indexName: 'logs.test2', retryService, logger }); - }); - - it('makes data searchable as expected', async () => { - const query = { - match: { numberfield: 123 }, - }; - const response = await esClient.search({ index: 'logs.test2', query }); - expect((response.hits.total as SearchTotalHits).value).to.eql(1); - }); + } }); } diff --git a/x-pack/test/api_integration/apis/streams/full_flow.ts b/x-pack/test/api_integration/apis/streams/full_flow.ts index aad931ab11816..b4aaf5b69743c 100644 --- a/x-pack/test/api_integration/apis/streams/full_flow.ts +++ b/x-pack/test/api_integration/apis/streams/full_flow.ts @@ -6,13 +6,7 @@ */ import expect from '@kbn/expect'; -import { - deleteStream, - enableStreams, - fetchDocument, - forkStream, - indexDocument, -} from './helpers/requests'; +import { enableStreams, fetchDocument, forkStream, indexDocument } from './helpers/requests'; import { FtrProviderContext } from '../../ftr_provider_context'; import { waitForDocumentInIndex } from '../../../alerting_api_integration/observability/helpers/alerting_wait_for_helpers'; import { cleanUpRootStream } from './helpers/cleanup'; @@ -25,8 +19,10 @@ export default function ({ getService }: FtrProviderContext) { describe('Basic functionality', () => { after(async () => { - await deleteStream(supertest, 'logs.nginx'); await cleanUpRootStream(esClient); + await esClient.indices.deleteDataStream({ + name: ['logs*'], + }); }); // Note: Each step is dependent on the previous diff --git a/x-pack/test/api_integration/apis/streams/helpers/repository_client.ts b/x-pack/test/api_integration/apis/streams/helpers/repository_client.ts new file mode 100644 index 0000000000000..a8de14234b6cf --- /dev/null +++ b/x-pack/test/api_integration/apis/streams/helpers/repository_client.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { StreamsRouteRepository } from '@kbn/streams-plugin/server'; +import supertest from 'supertest'; +import { getApiClientFromSupertest } from '../../../../common/utils/server_route_repository/create_supertest_service_from_repository'; + +export function createStreamsRepositorySupertestClient(st: supertest.Agent) { + return getApiClientFromSupertest(st); +} diff --git a/x-pack/test/api_integration/apis/streams/helpers/requests.ts b/x-pack/test/api_integration/apis/streams/helpers/requests.ts index 43e7f02b7a750..8342778f8ab18 100644 --- a/x-pack/test/api_integration/apis/streams/helpers/requests.ts +++ b/x-pack/test/api_integration/apis/streams/helpers/requests.ts @@ -17,7 +17,7 @@ export async function enableStreams(supertest: Agent) { } export async function indexDocument(esClient: Client, index: string, document: JsonObject) { - const response = await esClient.index({ index, document }); + const response = await esClient.index({ index, document, refresh: 'wait_for' }); return response; } @@ -37,13 +37,13 @@ export async function forkStream(supertest: Agent, root: string, body: JsonObjec } export async function putStream(supertest: Agent, name: string, body: JsonObject) { - const req = supertest.put(`/api/streams/${name}`).set('kbn-xsrf', 'xxx'); + const req = supertest.put(`/api/streams/${encodeURIComponent(name)}`).set('kbn-xsrf', 'xxx'); const response = await req.send(body).expect(200); return response.body; } export async function getStream(supertest: Agent, name: string) { - const req = supertest.get(`/api/streams/${name}`).set('kbn-xsrf', 'xxx'); + const req = supertest.get(`/api/streams/${encodeURIComponent(name)}`).set('kbn-xsrf', 'xxx'); const response = await req.send().expect(200); return response.body; } diff --git a/x-pack/test/api_integration/apis/streams/index.ts b/x-pack/test/api_integration/apis/streams/index.ts index 14decb2400196..6b619661909ac 100644 --- a/x-pack/test/api_integration/apis/streams/index.ts +++ b/x-pack/test/api_integration/apis/streams/index.ts @@ -13,5 +13,6 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./enrichment')); loadTestFile(require.resolve('./classic')); loadTestFile(require.resolve('./flush_config')); + loadTestFile(require.resolve('./assets/dashboard')); }); } diff --git a/x-pack/test/common/utils/server_route_repository/create_supertest_service_from_repository.ts b/x-pack/test/common/utils/server_route_repository/create_supertest_service_from_repository.ts new file mode 100644 index 0000000000000..4272ac2626a26 --- /dev/null +++ b/x-pack/test/common/utils/server_route_repository/create_supertest_service_from_repository.ts @@ -0,0 +1,211 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + formatRequest, + ServerRouteRepository, + EndpointOf, + ReturnOf, + ClientRequestParamsOf, +} from '@kbn/server-route-repository'; +import supertest from 'supertest'; +import { Subtract, RequiredKeys } from 'utility-types'; +import { format, UrlObject } from 'url'; +import { kbnTestConfig } from '@kbn/test'; + +type MaybeOptional> = RequiredKeys extends never + ? [TArgs] | [] + : [TArgs]; + +interface RepositorySupertestClient { + fetch: >( + endpoint: TEndpoint, + ...options: MaybeOptional< + { + type?: 'form-data'; + } & ClientRequestParamsOf + > + ) => RepositorySupertestReturnOf; +} + +type RepositorySupertestReturnOf< + TServerRouteRepository extends ServerRouteRepository, + TEndpoint extends EndpointOf +> = OverwriteThisMethods< + WithoutPromise, + Promise<{ + text: string; + status: number; + body: ReturnOf; + }> +>; + +type ScopedApiClientWithBasicAuthFactory = ( + kibanaServer: UrlObject, + username: string +) => RepositorySupertestClient; + +type ApiClientFromSupertestFactory = ( + st: supertest.Agent +) => RepositorySupertestClient; + +interface RepositorySupertestClientFactory { + getScopedApiClientWithBasicAuth: ScopedApiClientWithBasicAuthFactory; + getApiClientFromSupertest: ApiClientFromSupertestFactory; +} + +export function createSupertestClientFactoryFromRepository< + TServerRouteRepository extends ServerRouteRepository +>(): RepositorySupertestClientFactory { + return { + getScopedApiClientWithBasicAuth: (kibanaServer, username) => { + return getScopedApiClientWithBasicAuth(kibanaServer, username); + }, + getApiClientFromSupertest: (st) => { + return getApiClientFromSupertest(st); + }, + }; +} + +function getScopedApiClientWithBasicAuth( + kibanaServer: UrlObject, + username: string +): RepositorySupertestClient { + const { password } = kbnTestConfig.getUrlParts(); + const baseUrlWithAuth = format({ + ...kibanaServer, + auth: `${username}:${password}`, + }); + + return getApiClientFromSupertest(supertest(baseUrlWithAuth)); +} + +export function getApiClientFromSupertest( + st: supertest.Agent +): RepositorySupertestClient { + return { + fetch: (endpoint, ...rest) => { + const options = rest.length ? rest[0] : { type: undefined }; + + const { type } = options; + + const params = 'params' in options ? (options.params as Record) : {}; + + const { method, pathname, version } = formatRequest(endpoint, params.path); + const url = format({ pathname, query: params?.query }); + + const headers: Record = { 'kbn-xsrf': 'foo' }; + + if (version) { + headers['Elastic-Api-Version'] = version; + } + + let res: supertest.Test; + if (type === 'form-data') { + const fields: Array<[string, any]> = Object.entries(params.body); + const formDataRequest = st[method](url) + .set(headers) + .set('Content-type', 'multipart/form-data'); + + for (const field of fields) { + void formDataRequest.field(field[0], field[1]); + } + + res = formDataRequest; + } else if (params.body) { + res = st[method](url).send(params.body).set(headers); + } else { + res = st[method](url).set(headers); + } + + return res as RepositorySupertestReturnOf< + TServerRouteRepository, + EndpointOf + >; + }, + }; +} + +type WithoutPromise> = Subtract>; + +// this is a little intense, but without it, method overrides are lost +// e.g., { +// end(one:string) +// end(one:string, two:string) +// } +// would lose the first signature. This keeps up to eight signatures. +type OverloadedParameters = T extends { + (...args: infer A1): any; + (...args: infer A2): any; + (...args: infer A3): any; + (...args: infer A4): any; + (...args: infer A5): any; + (...args: infer A6): any; + (...args: infer A7): any; + (...args: infer A8): any; +} + ? A1 | A2 | A3 | A4 | A5 | A6 | A7 | A8 + : T extends { + (...args: infer A1): any; + (...args: infer A2): any; + (...args: infer A3): any; + (...args: infer A4): any; + (...args: infer A5): any; + (...args: infer A6): any; + (...args: infer A7): any; + } + ? A1 | A2 | A3 | A4 | A5 | A6 | A7 + : T extends { + (...args: infer A1): any; + (...args: infer A2): any; + (...args: infer A3): any; + (...args: infer A4): any; + (...args: infer A5): any; + (...args: infer A6): any; + } + ? A1 | A2 | A3 | A4 | A5 | A6 + : T extends { + (...args: infer A1): any; + (...args: infer A2): any; + (...args: infer A3): any; + (...args: infer A4): any; + (...args: infer A5): any; + } + ? A1 | A2 | A3 | A4 | A5 + : T extends { + (...args: infer A1): any; + (...args: infer A2): any; + (...args: infer A3): any; + (...args: infer A4): any; + } + ? A1 | A2 | A3 | A4 + : T extends { + (...args: infer A1): any; + (...args: infer A2): any; + (...args: infer A3): any; + } + ? A1 | A2 | A3 + : T extends { + (...args: infer A1): any; + (...args: infer A2): any; + } + ? A1 | A2 + : T extends (...args: infer A) => any + ? A + : any; + +type OverrideReturnType any, TNextReturnType> = ( + ...args: OverloadedParameters +) => WithoutPromise> & TNextReturnType; + +type OverwriteThisMethods, TNextReturnType> = TNextReturnType & { + [key in keyof T]: T[key] extends (...args: infer TArgs) => infer TReturnType + ? TReturnType extends Promise + ? OverrideReturnType + : (...args: TArgs) => TReturnType + : T[key]; +}; From 746a068665b51c0d151e8562f0175db29307c5aa Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Fri, 27 Dec 2024 13:04:25 +0100 Subject: [PATCH 86/95] Storage adapter tests --- .../es/storage/README.md | 43 ++ .../es/storage/get_schema_version.ts | 15 + .../es/storage/index.ts | 12 +- .../es/storage/index_adapter/errors.ts | 39 -- .../es/storage/index_adapter/index.test.ts | 596 ++++++++++++++++++ .../es/storage/index_adapter/index.ts | 390 +++++------- .../es/storage/storage_client.ts | 10 +- 7 files changed, 822 insertions(+), 283 deletions(-) create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/README.md create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/get_schema_version.ts delete mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/errors.ts create mode 100644 x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/index.test.ts diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/README.md b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/README.md new file mode 100644 index 0000000000000..2b83a7e3ea1b8 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/README.md @@ -0,0 +1,43 @@ +# Storage adapter + +Storage adapters are an abstraction for managing & writing data into Elasticsearch, from Kibana plugins. + +There are several ways one can use Elasticsearch in Kibana, for instance: + +- a simple id-based CRUD table +- timeseries data with regular indices +- timeseries data with data streams + +But then there are many choices to be made that make this a very complex problem: + +- Elasticsearch asset managmeent +- Authentication +- Schema changes +- Kibana's distributed nature +- Stateful versus serverless + +The intent of storage adapters is to come up with an abstraction that allows Kibana developers to have a common interface for writing to and reading data from Elasticsearch. For instance, for setting up your data store, it should not matter how you authenticate (internal user? current user? API keys?). + +## Saved objects + +Some of these problems are solved by Saved Objects. But Saved Objects come with a lot of baggage - Kibana RBAC, relationships, spaces, all of which might not be +needed for your use case but are still restrictive. One could consider Saved Objects to be the target of an adapter, but Storage Adapters aim to address a wider set of use-cases. + +## Philosophy + +Storage adapters should largely adhere to the following principles: + +- Interfaces are as close to Elasticsearch as possible. Meaning, the `search` method is practically a pass-through for `_search`. +- Strongly-typed. TypeScript types are inferred from the schema. This makes it easy to create fully-typed clients for any storage. +- Lazy writes. No Elasticsearch assets (templates, indices, aliases) get installed unless necessary. Anything that gets persisted to Elasticsearch raises questions (in SDHs, UIs, APIs) and should be avoided when possible. This also helps avoidable upgrade issues (e.g. conflicting mappings for something that never got used). +- Recoverable. If somehow Elasticsearch assets get borked, the adapters should make a best-effort attempt to recover, or log warnings with clear remediation steps. + +## Future goals + +Currently, we only have the StorageIndexAdapter which writes to plain indices. In the future, we'll want more: + +- A StorageDataStreamAdapter or StorageSavedObjectAdapter +- Federated search +- Data/Index Lifecycle Management +- Migration scripts +- Runtime mappings for older versions diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/get_schema_version.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/get_schema_version.ts new file mode 100644 index 0000000000000..0be986c168cba --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/get_schema_version.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import stringify from 'json-stable-stringify'; +import objectHash from 'object-hash'; +import { IndexStorageSettings } from '.'; + +export function getSchemaVersion(storage: IndexStorageSettings): string { + const version = objectHash(stringify(storage.schema.properties)); + return version; +} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index.ts index b5b6f28cb25d2..913f079e93313 100644 --- a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index.ts +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index.ts @@ -5,6 +5,7 @@ * 2.0. */ import type { + BulkOperationContainer, BulkRequest, BulkResponse, DeleteRequest, @@ -40,7 +41,14 @@ export type StorageAdapterSearchResponse< TSearchRequest extends Omit > = InferSearchResponseOf; -export type StorageAdapterBulkRequest = Omit, 'index'>; +export type StorageAdapterBulkOperation = Pick; + +export type StorageAdapterBulkRequest> = Omit< + BulkRequest, + 'operations' | 'index' +> & { + operations: Array; +}; export type StorageAdapterBulkResponse = BulkResponse; export type StorageAdapterDeleteRequest = DeleteRequest; @@ -53,7 +61,7 @@ export type StorageAdapterIndexRequest = Omit< export type StorageAdapterIndexResponse = IndexResponse; export interface IStorageAdapter { - bulk( + bulk>( request: StorageAdapterBulkRequest ): Promise; search>( diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/errors.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/errors.ts deleted file mode 100644 index 39942a5c06c68..0000000000000 --- a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/errors.ts +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { MappingProperty } from '@elastic/elasticsearch/lib/api/types'; -import { StorageSchema } from '..'; - -export class IncompatibleSchemaUpdateError extends Error { - constructor({ - existingProperties, - incompatibleProperties, - missingProperties, - nextProperties, - }: { - existingProperties: Record; - nextProperties: StorageSchema['properties']; - missingProperties: string[]; - incompatibleProperties: string[]; - }) { - const missingErrorMessage = missingProperties.length - ? `\nmissing properties: ${missingProperties.join(', ')}` - : ''; - - const incompatibleErrorMessage = incompatibleProperties.length - ? `\nincompatible properties: - - ${incompatibleProperties - .map((property) => { - return `\t${property}: expected ${existingProperties[property].type}, but got ${nextProperties[property].type}`; - }) - .join('\n')}` - : ''; - - super(`Incompatible schema update: ${missingErrorMessage} ${incompatibleErrorMessage}`); - } -} diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/index.test.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/index.test.ts new file mode 100644 index 0000000000000..bcdf2795d7515 --- /dev/null +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/index.test.ts @@ -0,0 +1,596 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { errors } from '@elastic/elasticsearch'; +import { + BulkDeleteOperation, + BulkIndexOperation, + BulkRequest, + BulkResponseItem, + IndexRequest, + IndicesGetAliasIndexAliases, + IndicesIndexState, + IndicesPutIndexTemplateRequest, + IndicesSimulateIndexTemplateResponse, + SearchRequest, + SearchResponse, +} from '@elastic/elasticsearch/lib/api/types'; +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { castArray, merge, remove } from 'lodash'; +import { Required } from 'utility-types'; +import { v4 } from 'uuid'; +import { StorageIndexAdapter } from '.'; +import { StorageSettings } from '..'; +import * as getSchemaVersionModule from '../get_schema_version'; + +type MockedElasticsearchClient = jest.Mocked & { + indices: jest.Mocked; +}; + +// Mock implementations for the ES Client and Logger +const createEsClientMock = (): MockedElasticsearchClient => { + return { + indices: { + putIndexTemplate: jest.fn(), + getIndexTemplate: jest.fn(), + create: jest.fn(), + getAlias: jest.fn(), + putAlias: jest.fn(), + existsIndexTemplate: jest.fn(), + existsAlias: jest.fn(), + exists: jest.fn(), + get: jest.fn(), + simulateIndexTemplate: jest.fn(), + putMapping: jest.fn(), + putSettings: jest.fn(), + }, + search: jest.fn(), + bulk: jest.fn(), + index: jest.fn(), + delete: jest.fn(), + } as unknown as MockedElasticsearchClient; +}; + +const createLoggerMock = (): jest.Mocked => { + const logger = { + debug: jest.fn(), + info: jest.fn(), + error: jest.fn(), + get: jest.fn(), + } as unknown as jest.Mocked; + + logger.get.mockReturnValue(logger); + + return logger; +}; + +const TEST_INDEX_NAME = 'test_index'; + +function getIndexName(counter: number) { + return `${TEST_INDEX_NAME}-00000${counter.toString()}`; +} + +describe('StorageIndexAdapter', () => { + let esClientMock: MockedElasticsearchClient; + let loggerMock: jest.Mocked; + let adapter: StorageIndexAdapter; // or a more specific type + + const storageSettings = { + name: TEST_INDEX_NAME, + schema: { + properties: { + foo: { + type: 'keyword', + required: true, + }, + }, + }, + } satisfies StorageSettings; + + beforeEach(() => { + esClientMock = createEsClientMock(); + loggerMock = createLoggerMock(); + + adapter = new StorageIndexAdapter(esClientMock, loggerMock, storageSettings); + + mockEmptyState(); + + mockCreateAPIs(); + + jest.spyOn(getSchemaVersionModule, 'getSchemaVersion').mockReturnValue('current_version'); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('creates a named logger', () => { + expect(loggerMock.get).toHaveBeenCalledWith('storage'); + expect(loggerMock.get).toHaveBeenCalledWith('test_index'); + }); + + it('does not install index templates or backing indices initially', () => { + expect(esClientMock.indices.putIndexTemplate).not.toHaveBeenCalled(); + expect(esClientMock.indices.create).not.toHaveBeenCalled(); + }); + + it('does not install index templates or backing indices after searching', async () => { + // Mock an ES search response + const mockSearchResponse = { + hits: { + hits: [{ _id: 'doc1', _source: { foo: 'bar' } }], + }, + } as unknown as SearchResponse<{ foo: 'bar' }>; + + esClientMock.search.mockResolvedValueOnce(mockSearchResponse); + + await adapter.search({ query: { match_all: {} } }); + + expect(esClientMock.indices.putIndexTemplate).not.toHaveBeenCalled(); + expect(esClientMock.indices.create).not.toHaveBeenCalled(); + }); + + it('does not fail a search when an index does not exist', async () => { + const mockSearchResponse = { + hits: { + hits: [], + }, + } as unknown as SearchResponse; + + esClientMock.search.mockResolvedValueOnce(mockSearchResponse); + + await adapter.search({ query: { match_all: {} } }); + + expect(esClientMock.search).toHaveBeenCalledWith( + expect.objectContaining({ + allow_no_indices: true, + }) + ); + }); + + describe('when writing/bootstrapping without an existing index', () => { + function verifyResources() { + // We expect that the adapter vaidates the components before writing + expect(esClientMock.indices.putIndexTemplate).toHaveBeenCalled(); + expect(esClientMock.indices.create).toHaveBeenCalledWith( + expect.objectContaining({ + index: 'test_index-000001', + }) + ); + } + + describe('when using index', () => { + it('creates the resources', async () => { + await adapter.index({ id: 'doc1', document: { foo: 'bar' } }); + + verifyResources(); + + expect(esClientMock.index).toHaveBeenCalledTimes(1); + }); + }); + + describe('when using bulk', () => { + it('creates the resources', async () => { + await adapter.bulk({ + operations: [{ index: { _id: 'foo' } }, { foo: 'bar' }], + }); + + verifyResources(); + + expect(esClientMock.bulk).toHaveBeenCalledTimes(1); + }); + }); + }); + + describe('when writing/bootstrapping with an existing, compatible index', () => { + beforeEach(async () => { + await esClientMock.indices.putIndexTemplate({ + name: TEST_INDEX_NAME, + _meta: { + version: 'current_version', + }, + template: { + mappings: { + _meta: { + version: 'current_version', + }, + properties: { + foo: { type: 'keyword', meta: { required: 'true' } }, + }, + }, + }, + }); + + await esClientMock.indices.create({ + index: getIndexName(1), + }); + + esClientMock.indices.putIndexTemplate.mockClear(); + esClientMock.indices.create.mockClear(); + }); + + it('does not recreate or update index template', async () => { + await adapter.index({ id: 'doc2', document: { foo: 'bar' } }); + + // confirm we did not create or update the template + expect(esClientMock.indices.putIndexTemplate).not.toHaveBeenCalled(); + expect(esClientMock.indices.create).not.toHaveBeenCalled(); + + // confirm we did index + expect(esClientMock.index).toHaveBeenCalledWith( + expect.objectContaining({ + index: 'test_index', + id: 'doc2', + }) + ); + }); + }); + + describe('when writing/bootstrapping with an existing, outdated index', () => { + beforeEach(async () => { + await esClientMock.indices.putIndexTemplate({ + name: TEST_INDEX_NAME, + _meta: { + version: 'first_version', + }, + template: { + mappings: { + _meta: { + version: 'first_version', + }, + properties: {}, + }, + }, + }); + + await esClientMock.indices.create({ + index: getIndexName(1), + }); + + esClientMock.indices.putIndexTemplate.mockClear(); + esClientMock.indices.create.mockClear(); + esClientMock.indices.simulateIndexTemplate.mockClear(); + }); + + it('updates index mappings on write', async () => { + await adapter.index({ id: 'docY', document: { foo: 'bar' } }); + + expect(esClientMock.indices.putIndexTemplate).toHaveBeenCalled(); + expect(esClientMock.indices.simulateIndexTemplate).toHaveBeenCalled(); + + expect(esClientMock.indices.putMapping).toHaveBeenCalledWith( + expect.objectContaining({ + properties: { + foo: { + type: 'keyword', + meta: { + multi_value: 'false', + required: 'true', + }, + }, + }, + }) + ); + }); + }); + + describe('when indexing', () => { + describe('a new document', () => { + it('indexes the document via alias with require_alias=true', async () => { + const res = await adapter.index({ id: 'doc_1', document: { foo: 'bar' } }); + + expect(esClientMock.index).toHaveBeenCalledWith( + expect.objectContaining({ + index: 'test_index', + require_alias: true, + id: 'doc_1', + document: { foo: 'bar' }, + }) + ); + + expect(res._index).toBe('test_index-000001'); + expect(res._id).toBe('doc_1'); + }); + }); + + describe('an existing document in any non-write index', () => { + beforeEach(async () => { + await adapter.index({ id: 'doc_1', document: { foo: 'bar' } }); + + await esClientMock.indices.create({ + index: getIndexName(2), + }); + }); + + it('deletes the dangling item from non-write indices', async () => { + await adapter.index({ id: 'doc_1', document: { foo: 'bar' } }); + + // Doc in prev index is deleted + expect(esClientMock.delete).toHaveBeenCalledWith( + expect.objectContaining({ + index: 'test_index-000001', + id: 'doc_1', + }) + ); + + // Doc is in write index now + expect(esClientMock.index).toHaveBeenCalledWith( + expect.objectContaining({ + index: 'test_index', + id: 'doc_1', + }) + ); + }); + }); + }); + + describe('when bulk indexing', () => { + describe('an existing document in any non-write index', () => { + beforeEach(async () => { + await adapter.bulk({ + operations: [ + { + index: { + _id: 'doc_1', + }, + }, + { + foo: 'bar', + }, + ], + }); + + await esClientMock.indices.create({ + index: getIndexName(2), + }); + }); + + it('deletes the dangling item from non-write indices', async () => { + await adapter.bulk({ + operations: [ + { + index: { + _id: 'doc_1', + }, + }, + { + foo: 'bar', + }, + ], + }); + + // delete operation is inserted + expect(esClientMock.bulk).toHaveBeenLastCalledWith( + expect.objectContaining({ + index: 'test_index', + operations: [ + { + index: { + _id: 'doc_1', + }, + }, + { + foo: 'bar', + }, + { + delete: { + _index: getIndexName(1), + _id: 'doc_1', + }, + }, + ], + }) + ); + }); + }); + }); + + function mockEmptyState() { + esClientMock.indices.getIndexTemplate.mockResolvedValue({ + index_templates: [], + }); + esClientMock.indices.existsIndexTemplate.mockResolvedValue(false); + + esClientMock.indices.simulateIndexTemplate.mockImplementation(async () => { + throw new errors.ResponseError({ statusCode: 404, warnings: [], meta: {} as any }); + }); + + esClientMock.indices.existsAlias.mockResolvedValue(false); + esClientMock.indices.exists.mockResolvedValue(false); + esClientMock.indices.getAlias.mockResolvedValue({}); + + esClientMock.index.mockImplementation(async () => { + throw new errors.ResponseError({ statusCode: 404, warnings: [], meta: {} as any }); + }); + + esClientMock.bulk.mockImplementation(async () => { + throw new errors.ResponseError({ statusCode: 404, warnings: [], meta: {} as any }); + }); + } + + function mockCreateAPIs() { + const indices: Map< + string, + Pick + > = new Map(); + + const docs: Array<{ _id: string; _source: Record; _index: string }> = []; + + function getCurrentWriteIndex() { + return Array.from(indices.entries()).find( + ([indexName, indexState]) => indexState.aliases![TEST_INDEX_NAME].is_write_index + )?.[0]; + } + + esClientMock.indices.putIndexTemplate.mockImplementation(async (_templateRequest) => { + const templateRequest = _templateRequest as Required< + IndicesPutIndexTemplateRequest, + 'template' + >; + + esClientMock.indices.existsIndexTemplate.mockResolvedValue(true); + esClientMock.indices.getIndexTemplate.mockResolvedValue({ + index_templates: [ + { + name: templateRequest.name, + index_template: { + _meta: templateRequest._meta, + template: templateRequest.template, + index_patterns: `${TEST_INDEX_NAME}*`, + composed_of: [], + }, + }, + ], + }); + + esClientMock.indices.simulateIndexTemplate.mockImplementation( + async (mockSimulateRequest): Promise => { + return { + template: { + aliases: templateRequest.template?.aliases ?? {}, + mappings: templateRequest.template?.mappings ?? {}, + settings: templateRequest.template?.settings ?? {}, + }, + }; + } + ); + + esClientMock.indices.create.mockImplementation(async (createIndexRequest) => { + const indexName = createIndexRequest.index; + + const prevIndices = Array.from(indices.entries()); + + prevIndices.forEach(([currentIndexName, indexState]) => { + indexState.aliases![TEST_INDEX_NAME] = { + is_write_index: false, + }; + }); + + indices.set(indexName, { + aliases: merge({}, templateRequest.template.aliases ?? {}, { + [TEST_INDEX_NAME]: { is_write_index: true }, + }), + mappings: templateRequest.template.mappings ?? {}, + settings: templateRequest.template.settings ?? {}, + }); + + esClientMock.indices.getAlias.mockImplementation(async (aliasRequest) => { + return Object.fromEntries( + Array.from(indices.entries()).map(([currentIndexName, indexState]) => { + return [ + currentIndexName, + { aliases: indexState.aliases ?? {} } satisfies IndicesGetAliasIndexAliases, + ]; + }) + ); + }); + + esClientMock.indices.get.mockImplementation(async () => { + return Object.fromEntries(indices.entries()); + }); + + esClientMock.index.mockImplementation(async (_indexRequest) => { + const indexRequest = _indexRequest as IndexRequest; + const id = indexRequest.id ?? v4(); + const index = getCurrentWriteIndex()!; + + docs.push({ + _id: id, + _index: index, + _source: indexRequest.document!, + }); + + return { + _id: id, + _index: index, + _shards: { + failed: 0, + successful: 1, + total: 1, + }, + _version: 1, + result: 'created', + }; + }); + + esClientMock.search.mockImplementation(async (_searchRequest) => { + const searchRequest = _searchRequest as SearchRequest; + + const ids = castArray(searchRequest.query?.bool?.filter ?? [])?.[0]?.terms + ?._id as string[]; + + const excludeIndex = castArray(searchRequest.query?.bool?.must_not)?.[0]?.term + ?._index as string; + + const matches = docs.filter((doc) => { + return ids.includes(doc._id) && doc._index !== excludeIndex; + }); + + return { + took: 0, + timed_out: false, + _shards: { + successful: 1, + failed: 0, + total: 1, + }, + hits: { + hits: matches, + total: { + value: matches.length, + relation: 'eq', + }, + }, + }; + }); + + esClientMock.bulk.mockImplementation(async (_bulkRequest) => { + const bulkRequest = _bulkRequest as BulkRequest>; + + const items: Array>> = []; + + bulkRequest.operations?.forEach((operation, index, operations) => { + if ('index' in operation) { + const indexOperation = operation.index as BulkIndexOperation; + const document = { + _id: indexOperation._id ?? v4(), + _index: indexOperation._index ?? getCurrentWriteIndex()!, + _source: operations[index + 1], + }; + docs.push(document); + + items.push({ index: { _id: document._id, _index: document._index, status: 200 } }); + } else if ('delete' in operation) { + const deleteOperation = operation.delete as BulkDeleteOperation; + remove(docs, (doc) => { + return doc._id === deleteOperation._id && doc._index === deleteOperation._index; + }); + + items.push({ + delete: { + _id: deleteOperation._id!, + _index: deleteOperation._index!, + status: 200, + }, + }); + } + }); + + return { + errors: false, + took: 0, + items, + }; + }); + + return { acknowledged: true, index: createIndexRequest.index, shards_acknowledged: true }; + }); + + return { acknowledged: true }; + }); + } +}); diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/index.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/index.ts index cb2542c605cd3..745bf96778c8a 100644 --- a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/index.ts +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/index.ts @@ -5,20 +5,18 @@ * 2.0. */ -import { - BulkResponseItem, +import type { IndexResponse, + IndicesIndexState, + IndicesIndexTemplate, IndicesPutIndexTemplateIndexTemplateMapping, MappingProperty, - Refresh, SearchRequest, } from '@elastic/elasticsearch/lib/api/types'; import { ElasticsearchClient, Logger } from '@kbn/core/server'; import { isResponseError } from '@kbn/es-errors'; import { InferSearchResponseOf } from '@kbn/es-types'; -import stringify from 'json-stable-stringify'; -import { last, mapValues, orderBy, padStart } from 'lodash'; -import objectHash from 'object-hash'; +import { last, mapValues, padStart } from 'lodash'; import { IStorageAdapter, IndexStorageSettings, @@ -30,11 +28,10 @@ import { StorageAdapterIndexResponse, StorageAdapterSearchRequest, StorageAdapterSearchResponse, - StorageSchema, } from '..'; +import { getSchemaVersion } from '../get_schema_version'; import { StorageClient } from '../storage_client'; import { StorageMappingProperty } from '../types'; -import { IncompatibleSchemaUpdateError } from './errors'; function getAliasName(name: string) { return name; @@ -53,42 +50,6 @@ function getIndexTemplateName(name: string) { return `${name}`; } -function getSchemaVersion(storage: IndexStorageSettings): string { - const version = objectHash(stringify(storage.schema.properties)); - return version; -} - -function isCompatibleOrThrow( - existingProperties: Record, - nextProperties: StorageSchema['properties'] -): void { - const missingProperties: string[] = []; - const incompatibleProperties: string[] = []; - Object.entries(existingProperties).forEach(([key, propertyInExisting]) => { - const propertyInNext = toElasticsearchMappingProperty(nextProperties[key]); - if (!propertyInNext) { - missingProperties.push(key); - } else if ( - propertyInNext.type !== propertyInExisting.type || - propertyInExisting.meta?.required !== propertyInNext.meta?.required || - propertyInExisting.meta?.multi_value !== propertyInNext.meta?.multi_value - ) { - incompatibleProperties.push(key); - } - }); - - const totalErrors = missingProperties.length + incompatibleProperties.length; - - if (totalErrors > 0) { - throw new IncompatibleSchemaUpdateError({ - existingProperties, - nextProperties, - missingProperties, - incompatibleProperties, - }); - } -} - function toElasticsearchMappingProperty(property: StorageMappingProperty): MappingProperty { const { required, multi_value: multiValue, enum: enums, ...rest } = property; @@ -103,14 +64,29 @@ function toElasticsearchMappingProperty(property: StorageMappingProperty): Mappi }; } +function catchConflictError(error: Error) { + if (isResponseError(error) && error.statusCode === 409) { + return; + } + throw error; +} + +/** + * Adapter for writing and reading documents to/from Elasticsearch, + * using plain indices. + * + */ export class StorageIndexAdapter implements IStorageAdapter { + private readonly logger: Logger; constructor( private readonly esClient: ElasticsearchClient, - private readonly logger: Logger, + logger: Logger, private readonly storage: TStorageSettings - ) {} + ) { + this.logger = logger.get('storage').get(this.storage.name); + } private getSearchIndexPattern(): string { return `${getAliasName(this.storage.name)}*`; @@ -120,9 +96,7 @@ export class StorageIndexAdapter return getAliasName(this.storage.name); } - private async createIndexTemplate(create: boolean = true): Promise { - this.logger.debug(`Creating index template (create = ${create})`); - + private async createOrUpdateIndexTemplate(): Promise { const version = getSchemaVersion(this.storage); const template: IndicesPutIndexTemplateIndexTemplateMapping = { @@ -132,46 +106,51 @@ export class StorageIndexAdapter }, properties: mapValues(this.storage.schema.properties, toElasticsearchMappingProperty), }, + aliases: { + [getAliasName(this.storage.name)]: { + is_write_index: true, + }, + }, }; - await this.esClient.indices.putIndexTemplate({ - name: getIndexTemplateName(this.storage.name), - create, - allow_auto_create: false, - index_patterns: getBackingIndexPattern(this.storage.name), - _meta: { - version, - }, - template, - }); + await this.esClient.indices + .putIndexTemplate({ + name: getIndexTemplateName(this.storage.name), + create: false, + allow_auto_create: false, + index_patterns: getBackingIndexPattern(this.storage.name), + _meta: { + version, + }, + template, + }) + .catch(catchConflictError); } - private async updateIndexTemplateIfNeeded(): Promise { - const version = getSchemaVersion(this.storage); - const indexTemplate = await this.esClient.indices + private async getExistingIndexTemplate(): Promise { + return await this.esClient.indices .getIndexTemplate({ name: getIndexTemplateName(this.storage.name), }) - .then((templates) => templates.index_templates[0].index_template); - - const currentVersion = indexTemplate._meta?.version; - - this.logger.debug( - `updateIndexTemplateIfNeeded: Current version = ${currentVersion}, next version = ${version}` - ); - - if (currentVersion === version) { - return; - } + .then((templates) => templates.index_templates[0]?.index_template); + } - isCompatibleOrThrow( - indexTemplate.template!.mappings!.properties!, - this.storage.schema.properties - ); + private async getCurrentWriteIndex(): Promise< + { name: string; state: IndicesIndexState } | undefined + > { + const [writeIndex, indices] = await Promise.all([ + this.getCurrentWriteIndexName(), + this.getExistingIndices(), + ]); - this.logger.debug(`Updating index template due to version mismatch`); + return writeIndex ? { name: writeIndex, state: indices[writeIndex] } : undefined; + } - await this.createIndexTemplate(false); + private async getExistingIndices() { + return this.esClient.indices.get({ + index: getBackingIndexPattern(this.storage.name), + allow_no_indices: true, + }); } private async getCurrentWriteIndexName(): Promise { @@ -205,125 +184,72 @@ export class StorageIndexAdapter private async createNextBackingIndex(): Promise { const writeIndex = await this.getCurrentWriteIndexName(); - this.logger.debug(`Creating next backing index, current write index = ${writeIndex}`); - const nextIndexName = getBackingIndexName( this.storage.name, writeIndex ? parseInt(last(writeIndex.split('-'))!, 10) : 1 ); - await this.esClient.indices.create({ - index: nextIndexName, - }); + await this.esClient.indices + .create({ + index: nextIndexName, + }) + .catch(catchConflictError); } - private async createAlias(): Promise { - const aliasName = getAliasName(this.storage.name); - - let indexName = await this.getCurrentWriteIndexName(); - - if (!indexName) { - const indices = await this.esClient.indices.get({ - index: getBackingIndexPattern(this.storage.name), - }); - - indexName = orderBy(Object.keys(indices), 'desc')[0]; - } - - if (!indexName) { - throw new Error(`Could not find backing index for ${aliasName}`); - } - - await this.esClient.indices.putAlias({ - index: indexName, - name: aliasName, - is_write_index: true, + private async updateMappingsOfExistingIndex({ name }: { name: string }) { + const simulateIndexTemplateResponse = await this.esClient.indices.simulateIndexTemplate({ + name: getIndexTemplateName(this.storage.name), }); - } - - private async rolloverIfNeeded() { - const [writeIndexName, indexTemplate, indices] = await Promise.all([ - this.getCurrentWriteIndexName(), - this.esClient.indices - .getIndexTemplate({ - name: getIndexTemplateName(this.storage.name), - }) - .then((templates) => templates.index_templates[0].index_template), - this.esClient.indices.get({ - index: getBackingIndexPattern(this.storage.name), - }), - ]); - - if (!writeIndexName) { - throw new Error(`No write index found for ${getAliasName(this.storage.name)}`); - } - if (!indexTemplate) { - throw new Error(`No index template found for ${getIndexTemplateName(this.storage.name)}`); + if (simulateIndexTemplateResponse.template.settings) { + await this.esClient.indices.putSettings({ + index: name, + settings: simulateIndexTemplateResponse.template.settings, + }); } - const writeIndex = indices[writeIndexName]; - - const isSameVersion = writeIndex.mappings?._meta?.version === indexTemplate._meta?.version; - - if (!isSameVersion) { - await this.createNextBackingIndex(); + if (simulateIndexTemplateResponse.template.mappings) { + await this.esClient.indices.putMapping({ + index: name, + ...simulateIndexTemplateResponse.template.mappings, + }); } } - private async bootstrap() { - const { name } = this.storage; - - this.logger.debug('Retrieving existing Elasticsearch components'); - - const [indexTemplateExists, aliasExists, backingIndexExists] = await Promise.all([ - this.esClient.indices.existsIndexTemplate({ - name: getIndexTemplateName(name), - }), - this.esClient.indices.existsAlias({ - name: getAliasName(name), - }), - this.esClient.indices.exists({ - index: getBackingIndexPattern(name), - allow_no_indices: false, - }), + /** + * Validates whether: + * - an index template exists + * - the index template has the right version (if not, update it) + * - a write index exists (if it doesn't, create it) + * - the write index has the right version (if not, update it) + */ + private async validateComponentsBeforeWriting(cb: () => Promise): Promise { + const [writeIndex, existingIndexTemplate] = await Promise.all([ + this.getCurrentWriteIndex(), + this.getExistingIndexTemplate(), ]); - this.logger.debug( - () => - `Existing components: ${JSON.stringify({ - indexTemplateExists, - aliasExists, - backingIndexExists, - })}` - ); + const expectedSchemaVersion = getSchemaVersion(this.storage); - if (!indexTemplateExists) { - await this.createIndexTemplate(); - } else { - await this.updateIndexTemplateIfNeeded(); + if (!existingIndexTemplate) { + this.logger.info(`Creating index template as it does not exist`); + await this.createOrUpdateIndexTemplate(); + } else if (existingIndexTemplate?._meta?.version !== expectedSchemaVersion) { + this.logger.info(`Updating existing index template`); + await this.createOrUpdateIndexTemplate(); } - if (!backingIndexExists) { + if (!writeIndex) { + this.logger.info(`Creating first backing index`); await this.createNextBackingIndex(); + } else if (writeIndex?.state.mappings?._meta?.version !== expectedSchemaVersion) { + this.logger.info(`Updating mappings of existing write index due to schema version mismatch`); + await this.updateMappingsOfExistingIndex({ + name: writeIndex.name, + }); } - if (!aliasExists) { - await this.createAlias(); - } - - await this.rolloverIfNeeded(); - } - - private async retryAfterBootstrap(cb: () => Promise): Promise { - return cb().catch(async (error) => { - if (isResponseError(error) && error.statusCode === 404) { - this.logger.info(`Write target for ${this.storage.name} not found, bootstrapping`); - await this.bootstrap(); - return cb(); - } - throw error; - }); + return await cb(); } async search>( @@ -336,13 +262,12 @@ export class StorageIndexAdapter }) as unknown as Promise>; } - private async removeDanglingItems({ ids, refresh }: { ids: string[]; refresh?: Refresh }) { + /** + * Get items from all non-write indices for the specified ids. + */ + private async getDanglingItems({ ids }: { ids: string[] }) { const writeIndex = await this.getCurrentWriteIndexName(); - this.logger.debug( - () => `Removing dangling items for ${ids.join(', ')}, write index is ${writeIndex}` - ); - if (writeIndex && ids.length) { const danglingItemsResponse = await this.search({ query: { @@ -360,91 +285,78 @@ export class StorageIndexAdapter size: 10_000, }); - const danglingItemsToDelete = danglingItemsResponse.hits.hits.map((hit) => ({ + return danglingItemsResponse.hits.hits.map((hit) => ({ id: hit._id!, index: hit._index, })); - - if (danglingItemsToDelete.length > 0) { - const shouldRefresh = refresh === true || refresh === 'true' || refresh === 'wait_for'; - - this.logger.debug(() => `Deleting ${danglingItemsToDelete.length} dangling items`); - - await this.esClient.deleteByQuery({ - index: this.getSearchIndexPattern(), - refresh: shouldRefresh, - wait_for_completion: shouldRefresh, - query: { - bool: { - should: danglingItemsToDelete.map((item) => { - return { - bool: { - filter: [ - { - term: { - _index: item.index, - }, - }, - { - term: { - _id: item.id, - }, - }, - ], - }, - }; - }), - minimum_should_match: 1, - }, - }, - }); - } } + return []; } async index(request: StorageAdapterIndexRequest): Promise { - const attemptIndex = (): Promise => { + const attemptIndex = async (): Promise => { + const [danglingItem] = request.id + ? await this.getDanglingItems({ ids: [request.id] }) + : [undefined]; + + if (danglingItem) { + await this.esClient.delete({ + id: danglingItem.id, + index: danglingItem.index, + refresh: false, + }); + } + return this.esClient.index({ ...request, + refresh: request.refresh, index: this.getWriteTarget(), require_alias: true, }); }; - return this.retryAfterBootstrap(attemptIndex).then(async (response) => { + return this.validateComponentsBeforeWriting(attemptIndex).then(async (response) => { this.logger.debug(() => `Indexed document ${request.id} into ${response._index}`); - if (request.id) { - await this.removeDanglingItems({ - ids: [request.id], - refresh: request.refresh, - }); - } return response; }); } - async bulk(request: StorageAdapterBulkRequest): Promise { - const attemptBulk = () => { + async bulk>( + request: StorageAdapterBulkRequest + ): Promise { + const attemptBulk = async () => { + const indexedIds = + request.operations?.flatMap((operation) => { + if ( + 'index' in operation && + operation.index && + typeof operation.index === 'object' && + '_id' in operation.index && + typeof operation.index._id === 'string' + ) { + return operation.index._id ?? []; + } + return []; + }) ?? []; + + const danglingItems = await this.getDanglingItems({ ids: indexedIds }); + + if (danglingItems.length) { + this.logger.debug(`Deleting ${danglingItems.length} dangling items`); + } + return this.esClient.bulk({ ...request, + operations: (request.operations || []).concat( + danglingItems.map((item) => ({ delete: { _index: item.index, _id: item.id } })) + ), index: this.getWriteTarget(), require_alias: true, }); }; - return this.retryAfterBootstrap(attemptBulk).then(async (response) => { - const ids = response.items - .filter( - (item): item is { index: BulkResponseItem & { _id: string } } => - !!item.index && !!item.index._id && !item.index.error - ) - .map((item) => item.index._id); - - if (ids.length) { - await this.removeDanglingItems({ ids, refresh: request.refresh }); - } - + return this.validateComponentsBeforeWriting(attemptBulk).then(async (response) => { return response; }); } diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/storage_client.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/storage_client.ts index eb7aecdbb6100..6bd2b211d1083 100644 --- a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/storage_client.ts +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/storage_client.ts @@ -5,11 +5,15 @@ * 2.0. */ -import { BulkRequest } from '@elastic/elasticsearch/lib/api/types'; import { withSpan } from '@kbn/apm-utils'; import { Logger } from '@kbn/core/server'; import { compact } from 'lodash'; -import { IStorageAdapter, StorageDocumentOf, StorageSettings } from '.'; +import { + IStorageAdapter, + StorageAdapterBulkOperation, + StorageDocumentOf, + StorageSettings, +} from '.'; import { ObservabilityESSearchRequest } from '../client/create_observability_es_client'; type StorageBulkOperation = @@ -76,7 +80,7 @@ export class StorageClient { async bulk(operations: Array>>) { const result = await this.storage.bulk({ refresh: 'wait_for', - operations: operations.flatMap((operation): BulkRequest['operations'] => { + operations: operations.flatMap((operation): StorageAdapterBulkOperation[] => { if ('index' in operation) { return [ { From 0df97c1fa37257e41e6c7c8fcbb18bab497638d0 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Fri, 27 Dec 2024 13:16:01 +0100 Subject: [PATCH 87/95] Fix tests --- .../es/storage/index_adapter/index.ts | 13 +++++++++++-- .../apis/streams/assets/dashboard.ts | 15 ++++----------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/index.ts b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/index.ts index 745bf96778c8a..258df256ed804 100644 --- a/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/index.ts +++ b/x-pack/packages/observability/observability_utils/observability_utils_server/es/storage/index_adapter/index.ts @@ -75,6 +75,9 @@ function catchConflictError(error: Error) { * Adapter for writing and reading documents to/from Elasticsearch, * using plain indices. * + * TODO: + * - Index Lifecycle Management + * - Schema upgrades w/ fallbacks */ export class StorageIndexAdapter implements IStorageAdapter @@ -132,7 +135,13 @@ export class StorageIndexAdapter .getIndexTemplate({ name: getIndexTemplateName(this.storage.name), }) - .then((templates) => templates.index_templates[0]?.index_template); + .then((templates) => templates.index_templates[0]?.index_template) + .catch((error) => { + if (isResponseError(error) && error.statusCode === 404) { + return undefined; + } + throw error; + }); } private async getCurrentWriteIndex(): Promise< @@ -234,7 +243,7 @@ export class StorageIndexAdapter if (!existingIndexTemplate) { this.logger.info(`Creating index template as it does not exist`); await this.createOrUpdateIndexTemplate(); - } else if (existingIndexTemplate?._meta?.version !== expectedSchemaVersion) { + } else if (existingIndexTemplate._meta?.version !== expectedSchemaVersion) { this.logger.info(`Updating existing index template`); await this.createOrUpdateIndexTemplate(); } diff --git a/x-pack/test/api_integration/apis/streams/assets/dashboard.ts b/x-pack/test/api_integration/apis/streams/assets/dashboard.ts index cffd6a6a60ba6..d6f379559d3aa 100644 --- a/x-pack/test/api_integration/apis/streams/assets/dashboard.ts +++ b/x-pack/test/api_integration/apis/streams/assets/dashboard.ts @@ -186,10 +186,6 @@ export default function ({ getService }: FtrProviderContext) { describe('after manually rolling over the index and relinking the dashboard', () => { before(async () => { - await esClient.indices.create({ - index: `.kibana_streams_assets-000002`, - }); - await esClient.indices.updateAliases({ actions: [ { @@ -199,16 +195,13 @@ export default function ({ getService }: FtrProviderContext) { is_write_index: false, }, }, - { - add: { - index: `.kibana_streams_assets-000002`, - alias: `.kibana_streams_assets`, - is_write_index: true, - }, - }, ], }); + await esClient.indices.create({ + index: `.kibana_streams_assets-000002`, + }); + await unlinkDashboard(SEARCH_DASHBOARD_ID); await linkDashboard(SEARCH_DASHBOARD_ID); }); From ec11424386fb79cf765bdb2f268756d16e819b64 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Fri, 27 Dec 2024 15:01:20 +0100 Subject: [PATCH 88/95] Clean up types & comments --- .../fleet/cypress.config.space_awareness.d.ts | 3 ++ .../fleet/cypress.config.space_awareness.js | 42 +++++++++++++++++++ .../es/storage/index_adapter/index.test.ts | 10 +---- .../stream_detail_dashboards_view/index.tsx | 8 ++-- .../public/hooks/use_dashboards_api.ts | 2 +- .../apis/streams/helpers/repository_client.ts | 9 +++- ...reate_supertest_service_from_repository.ts | 2 +- 7 files changed, 59 insertions(+), 17 deletions(-) create mode 100644 x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.d.ts create mode 100644 x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.js diff --git a/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.d.ts b/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.d.ts new file mode 100644 index 0000000000000..42cd75f66e3c9 --- /dev/null +++ b/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.d.ts @@ -0,0 +1,3 @@ +/// +declare const _default: Cypress.ConfigOptions; +export default _default; diff --git a/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.js b/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.js new file mode 100644 index 0000000000000..d680a87809d2e --- /dev/null +++ b/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.js @@ -0,0 +1,42 @@ +"use strict"; +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +Object.defineProperty(exports, "__esModule", { value: true }); +const cypress_config_1 = require("@kbn/cypress-config"); +// eslint-disable-next-line import/no-default-export +exports.default = (0, cypress_config_1.defineCypressConfig)({ + defaultCommandTimeout: 60000, + requestTimeout: 60000, + responseTimeout: 60000, + execTimeout: 120000, + pageLoadTimeout: 120000, + retries: { + runMode: 2, + }, + env: { + grepFilterSpecs: false, + }, + screenshotsFolder: '../../../../../target/kibana-fleet/cypress/screenshots', + trashAssetsBeforeRuns: false, + video: false, + videosFolder: '../../../../../target/kibana-fleet/cypress/videos', + viewportHeight: 900, + viewportWidth: 1440, + screenshotOnRunFailure: true, + e2e: { + baseUrl: 'http://localhost:5601', + experimentalRunAllSpecs: true, + experimentalMemoryManagement: true, + numTestsKeptInMemory: 3, + specPattern: './cypress/e2e/space_awareness/**/*.cy.ts', + supportFile: './cypress/support/e2e.ts', + setupNodeEvents(on, config) { + // eslint-disable-next-line @typescript-eslint/no-var-requires, @kbn/imports/no_boundary_crossing + return require('./cypress/plugins')(on, config); + }, + }, +}); diff --git a/x-pack/solutions/observability/packages/utils_server/es/storage/index_adapter/index.test.ts b/x-pack/solutions/observability/packages/utils_server/es/storage/index_adapter/index.test.ts index bcdf2795d7515..061ccfdaac219 100644 --- a/x-pack/solutions/observability/packages/utils_server/es/storage/index_adapter/index.test.ts +++ b/x-pack/solutions/observability/packages/utils_server/es/storage/index_adapter/index.test.ts @@ -31,7 +31,6 @@ type MockedElasticsearchClient = jest.Mocked & { indices: jest.Mocked; }; -// Mock implementations for the ES Client and Logger const createEsClientMock = (): MockedElasticsearchClient => { return { indices: { @@ -77,7 +76,7 @@ function getIndexName(counter: number) { describe('StorageIndexAdapter', () => { let esClientMock: MockedElasticsearchClient; let loggerMock: jest.Mocked; - let adapter: StorageIndexAdapter; // or a more specific type + let adapter: StorageIndexAdapter; const storageSettings = { name: TEST_INDEX_NAME, @@ -119,7 +118,6 @@ describe('StorageIndexAdapter', () => { }); it('does not install index templates or backing indices after searching', async () => { - // Mock an ES search response const mockSearchResponse = { hits: { hits: [{ _id: 'doc1', _source: { foo: 'bar' } }], @@ -154,7 +152,6 @@ describe('StorageIndexAdapter', () => { describe('when writing/bootstrapping without an existing index', () => { function verifyResources() { - // We expect that the adapter vaidates the components before writing expect(esClientMock.indices.putIndexTemplate).toHaveBeenCalled(); expect(esClientMock.indices.create).toHaveBeenCalledWith( expect.objectContaining({ @@ -216,11 +213,9 @@ describe('StorageIndexAdapter', () => { it('does not recreate or update index template', async () => { await adapter.index({ id: 'doc2', document: { foo: 'bar' } }); - // confirm we did not create or update the template expect(esClientMock.indices.putIndexTemplate).not.toHaveBeenCalled(); expect(esClientMock.indices.create).not.toHaveBeenCalled(); - // confirm we did index expect(esClientMock.index).toHaveBeenCalledWith( expect.objectContaining({ index: 'test_index', @@ -309,7 +304,6 @@ describe('StorageIndexAdapter', () => { it('deletes the dangling item from non-write indices', async () => { await adapter.index({ id: 'doc_1', document: { foo: 'bar' } }); - // Doc in prev index is deleted expect(esClientMock.delete).toHaveBeenCalledWith( expect.objectContaining({ index: 'test_index-000001', @@ -317,7 +311,6 @@ describe('StorageIndexAdapter', () => { }) ); - // Doc is in write index now expect(esClientMock.index).toHaveBeenCalledWith( expect.objectContaining({ index: 'test_index', @@ -363,7 +356,6 @@ describe('StorageIndexAdapter', () => { ], }); - // delete operation is inserted expect(esClientMock.bulk).toHaveBeenLastCalledWith( expect.objectContaining({ index: 'test_index', diff --git a/x-pack/solutions/observability/plugins/streams_app/public/components/stream_detail_dashboards_view/index.tsx b/x-pack/solutions/observability/plugins/streams_app/public/components/stream_detail_dashboards_view/index.tsx index 1a80e82cf0c5f..fbb98b877e71a 100644 --- a/x-pack/solutions/observability/plugins/streams_app/public/components/stream_detail_dashboards_view/index.tsx +++ b/x-pack/solutions/observability/plugins/streams_app/public/components/stream_detail_dashboards_view/index.tsx @@ -6,7 +6,7 @@ */ import { EuiButton, EuiFlexGroup, EuiFlexItem, EuiSearchBar } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import { StreamDefinition } from '@kbn/streams-plugin/common'; +import { StreamDefinition } from '@kbn/streams-schema'; import React, { useMemo, useState } from 'react'; import type { SanitizedDashboardAsset } from '@kbn/streams-plugin/server/routes/dashboards/route'; import { AddDashboardFlyout } from './add_dashboard_flyout'; @@ -19,8 +19,8 @@ export function StreamDetailDashboardsView({ definition }: { definition?: Stream const [isAddDashboardFlyoutOpen, setIsAddDashboardFlyoutOpen] = useState(false); - const dashboardsFetch = useDashboardsFetch(definition?.id); - const { addDashboards, removeDashboards } = useDashboardsApi(definition?.id); + const dashboardsFetch = useDashboardsFetch(definition?.name); + const { addDashboards, removeDashboards } = useDashboardsApi(definition?.name); const [isUnlinkLoading, setIsUnlinkLoading] = useState(false); const linkedDashboards = useMemo(() => { @@ -95,7 +95,7 @@ export function StreamDetailDashboardsView({ definition }: { definition?: Stream {definition && isAddDashboardFlyoutOpen ? ( { await addDashboards(dashboards); await dashboardsFetch.refresh(); diff --git a/x-pack/solutions/observability/plugins/streams_app/public/hooks/use_dashboards_api.ts b/x-pack/solutions/observability/plugins/streams_app/public/hooks/use_dashboards_api.ts index 43029d9cf97fe..57f40a36cef68 100644 --- a/x-pack/solutions/observability/plugins/streams_app/public/hooks/use_dashboards_api.ts +++ b/x-pack/solutions/observability/plugins/streams_app/public/hooks/use_dashboards_api.ts @@ -33,7 +33,7 @@ export const useDashboardsApi = (id?: string) => { }, body: { operations: dashboards.map((dashboard) => { - return { create: { id: dashboard.id } }; + return { index: { id: dashboard.id } }; }), }, }, diff --git a/x-pack/test/api_integration/apis/streams/helpers/repository_client.ts b/x-pack/test/api_integration/apis/streams/helpers/repository_client.ts index a8de14234b6cf..581243abbba11 100644 --- a/x-pack/test/api_integration/apis/streams/helpers/repository_client.ts +++ b/x-pack/test/api_integration/apis/streams/helpers/repository_client.ts @@ -6,8 +6,13 @@ */ import type { StreamsRouteRepository } from '@kbn/streams-plugin/server'; import supertest from 'supertest'; -import { getApiClientFromSupertest } from '../../../../common/utils/server_route_repository/create_supertest_service_from_repository'; +import { + RepositorySupertestClient, + getApiClientFromSupertest, +} from '../../../../common/utils/server_route_repository/create_supertest_service_from_repository'; -export function createStreamsRepositorySupertestClient(st: supertest.Agent) { +export function createStreamsRepositorySupertestClient( + st: supertest.Agent +): RepositorySupertestClient { return getApiClientFromSupertest(st); } diff --git a/x-pack/test/common/utils/server_route_repository/create_supertest_service_from_repository.ts b/x-pack/test/common/utils/server_route_repository/create_supertest_service_from_repository.ts index 4272ac2626a26..4c0b3da777a1e 100644 --- a/x-pack/test/common/utils/server_route_repository/create_supertest_service_from_repository.ts +++ b/x-pack/test/common/utils/server_route_repository/create_supertest_service_from_repository.ts @@ -21,7 +21,7 @@ type MaybeOptional> = RequiredKeys exte ? [TArgs] | [] : [TArgs]; -interface RepositorySupertestClient { +export interface RepositorySupertestClient { fetch: >( endpoint: TEndpoint, ...options: MaybeOptional< From 51beb26cab219df77fb60a99b9120e4ddabe5a38 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 27 Dec 2024 14:19:17 +0000 Subject: [PATCH 89/95] [CI] Auto-commit changed files from 'node scripts/telemetry_check' --- x-pack/test/tsconfig.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/test/tsconfig.json b/x-pack/test/tsconfig.json index 2dd261262a697..160d5ca4ac487 100644 --- a/x-pack/test/tsconfig.json +++ b/x-pack/test/tsconfig.json @@ -190,6 +190,8 @@ "@kbn/gen-ai-functional-testing", "@kbn/integration-assistant-plugin", "@kbn/core-elasticsearch-server", - "@kbn/streams-schema" + "@kbn/streams-schema", + "@kbn/server-route-repository-utils", + "@kbn/streams-plugin" ] } From f9b8345ede1b855fc56898d0dba4f434fc7ed749 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 27 Dec 2024 14:40:23 +0000 Subject: [PATCH 90/95] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../fleet/cypress.config.space_awareness.d.ts | 9 ++- .../fleet/cypress.config.space_awareness.js | 63 +++++++++---------- 2 files changed, 39 insertions(+), 33 deletions(-) diff --git a/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.d.ts b/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.d.ts index 42cd75f66e3c9..1167766e6ab8c 100644 --- a/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.d.ts +++ b/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.d.ts @@ -1,3 +1,10 @@ -/// +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// / declare const _default: Cypress.ConfigOptions; export default _default; diff --git a/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.js b/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.js index d680a87809d2e..289fff319e5a2 100644 --- a/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.js +++ b/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.js @@ -1,42 +1,41 @@ -"use strict"; /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -Object.defineProperty(exports, "__esModule", { value: true }); -const cypress_config_1 = require("@kbn/cypress-config"); +Object.defineProperty(exports, '__esModule', { value: true }); +const cypress_config_1 = require('@kbn/cypress-config'); // eslint-disable-next-line import/no-default-export exports.default = (0, cypress_config_1.defineCypressConfig)({ - defaultCommandTimeout: 60000, - requestTimeout: 60000, - responseTimeout: 60000, - execTimeout: 120000, - pageLoadTimeout: 120000, - retries: { - runMode: 2, - }, - env: { - grepFilterSpecs: false, - }, - screenshotsFolder: '../../../../../target/kibana-fleet/cypress/screenshots', - trashAssetsBeforeRuns: false, - video: false, - videosFolder: '../../../../../target/kibana-fleet/cypress/videos', - viewportHeight: 900, - viewportWidth: 1440, - screenshotOnRunFailure: true, - e2e: { - baseUrl: 'http://localhost:5601', - experimentalRunAllSpecs: true, - experimentalMemoryManagement: true, - numTestsKeptInMemory: 3, - specPattern: './cypress/e2e/space_awareness/**/*.cy.ts', - supportFile: './cypress/support/e2e.ts', - setupNodeEvents(on, config) { - // eslint-disable-next-line @typescript-eslint/no-var-requires, @kbn/imports/no_boundary_crossing - return require('./cypress/plugins')(on, config); - }, + defaultCommandTimeout: 60000, + requestTimeout: 60000, + responseTimeout: 60000, + execTimeout: 120000, + pageLoadTimeout: 120000, + retries: { + runMode: 2, + }, + env: { + grepFilterSpecs: false, + }, + screenshotsFolder: '../../../../../target/kibana-fleet/cypress/screenshots', + trashAssetsBeforeRuns: false, + video: false, + videosFolder: '../../../../../target/kibana-fleet/cypress/videos', + viewportHeight: 900, + viewportWidth: 1440, + screenshotOnRunFailure: true, + e2e: { + baseUrl: 'http://localhost:5601', + experimentalRunAllSpecs: true, + experimentalMemoryManagement: true, + numTestsKeptInMemory: 3, + specPattern: './cypress/e2e/space_awareness/**/*.cy.ts', + supportFile: './cypress/support/e2e.ts', + setupNodeEvents(on, config) { + // eslint-disable-next-line @typescript-eslint/no-var-requires, @kbn/imports/no_boundary_crossing + return require('./cypress/plugins')(on, config); }, + }, }); From 90254e3a193ed4137619939cbe097cb648b50737 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Fri, 27 Dec 2024 16:47:46 +0100 Subject: [PATCH 91/95] Fix quick checks --- .github/CODEOWNERS | 1 + .../fleet/cypress.config.space_awareness.d.ts | 3 -- .../fleet/cypress.config.space_awareness.js | 42 ---------------- .../fleet/cypress.config.space_awareness.ts | 49 ------------------- x-pack/test/tsconfig.json | 4 +- 5 files changed, 4 insertions(+), 95 deletions(-) delete mode 100644 x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.d.ts delete mode 100644 x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.js delete mode 100644 x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.ts diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ca9c1b0f90312..15cee0871b4fc 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1331,6 +1331,7 @@ packages/kbn-monaco/src/esql @elastic/kibana-esql /x-pack/solutions/observability/plugins/infra/server/routes/log_analysis @elastic/obs-ux-logs-team /x-pack/solutions/observability/plugins/infra/server/services/rules @elastic/obs-ux-infra_services-team @elastic/obs-ux-logs-team /x-pack/test/common/utils/synthtrace @elastic/obs-ux-infra_services-team @elastic/obs-ux-logs-team # Assigned per https://github.com/elastic/kibana/blob/main/packages/kbn-apm-synthtrace/kibana.jsonc#L5 +/x-pack/test/common/utils/server_route_repository @elastic/obs-knowledge-team # Infra Monitoring tests /x-pack/test/common/services/infra_synthtrace_kibana_client.ts @elastic/obs-ux-infra_services-team diff --git a/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.d.ts b/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.d.ts deleted file mode 100644 index 42cd75f66e3c9..0000000000000 --- a/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.d.ts +++ /dev/null @@ -1,3 +0,0 @@ -/// -declare const _default: Cypress.ConfigOptions; -export default _default; diff --git a/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.js b/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.js deleted file mode 100644 index d680a87809d2e..0000000000000 --- a/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.js +++ /dev/null @@ -1,42 +0,0 @@ -"use strict"; -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -Object.defineProperty(exports, "__esModule", { value: true }); -const cypress_config_1 = require("@kbn/cypress-config"); -// eslint-disable-next-line import/no-default-export -exports.default = (0, cypress_config_1.defineCypressConfig)({ - defaultCommandTimeout: 60000, - requestTimeout: 60000, - responseTimeout: 60000, - execTimeout: 120000, - pageLoadTimeout: 120000, - retries: { - runMode: 2, - }, - env: { - grepFilterSpecs: false, - }, - screenshotsFolder: '../../../../../target/kibana-fleet/cypress/screenshots', - trashAssetsBeforeRuns: false, - video: false, - videosFolder: '../../../../../target/kibana-fleet/cypress/videos', - viewportHeight: 900, - viewportWidth: 1440, - screenshotOnRunFailure: true, - e2e: { - baseUrl: 'http://localhost:5601', - experimentalRunAllSpecs: true, - experimentalMemoryManagement: true, - numTestsKeptInMemory: 3, - specPattern: './cypress/e2e/space_awareness/**/*.cy.ts', - supportFile: './cypress/support/e2e.ts', - setupNodeEvents(on, config) { - // eslint-disable-next-line @typescript-eslint/no-var-requires, @kbn/imports/no_boundary_crossing - return require('./cypress/plugins')(on, config); - }, - }, -}); diff --git a/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.ts b/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.ts deleted file mode 100644 index 236f17c3b5cc7..0000000000000 --- a/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.ts +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { defineCypressConfig } from '@kbn/cypress-config'; - -// eslint-disable-next-line import/no-default-export -export default defineCypressConfig({ - defaultCommandTimeout: 60000, - requestTimeout: 60000, - responseTimeout: 60000, - execTimeout: 120000, - pageLoadTimeout: 120000, - - retries: { - runMode: 2, - }, - - env: { - grepFilterSpecs: false, - }, - - screenshotsFolder: '../../../../../target/kibana-fleet/cypress/screenshots', - trashAssetsBeforeRuns: false, - video: false, - videosFolder: '../../../../../target/kibana-fleet/cypress/videos', - viewportHeight: 900, - viewportWidth: 1440, - screenshotOnRunFailure: true, - - e2e: { - baseUrl: 'http://localhost:5601', - - experimentalRunAllSpecs: true, - experimentalMemoryManagement: true, - numTestsKeptInMemory: 3, - - specPattern: './cypress/e2e/space_awareness/**/*.cy.ts', - supportFile: './cypress/support/e2e.ts', - - setupNodeEvents(on, config) { - // eslint-disable-next-line @typescript-eslint/no-var-requires, @kbn/imports/no_boundary_crossing - return require('./cypress/plugins')(on, config); - }, - }, -}); diff --git a/x-pack/test/tsconfig.json b/x-pack/test/tsconfig.json index 2dd261262a697..160d5ca4ac487 100644 --- a/x-pack/test/tsconfig.json +++ b/x-pack/test/tsconfig.json @@ -190,6 +190,8 @@ "@kbn/gen-ai-functional-testing", "@kbn/integration-assistant-plugin", "@kbn/core-elasticsearch-server", - "@kbn/streams-schema" + "@kbn/streams-schema", + "@kbn/server-route-repository-utils", + "@kbn/streams-plugin" ] } From 061338548f00dc836ab471f01b3bfa0dad32f9aa Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Sat, 28 Dec 2024 11:50:35 +0100 Subject: [PATCH 92/95] Revert changes to Fleet files --- .../fleet/cypress.config.space_awareness.d.ts | 10 ---------- ...reness.js => cypress.config.space_awareness.ts} | 14 +++++++++++--- 2 files changed, 11 insertions(+), 13 deletions(-) delete mode 100644 x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.d.ts rename x-pack/platform/plugins/shared/fleet/{cypress.config.space_awareness.js => cypress.config.space_awareness.ts} (87%) diff --git a/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.d.ts b/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.d.ts deleted file mode 100644 index 1167766e6ab8c..0000000000000 --- a/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.d.ts +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -// / -declare const _default: Cypress.ConfigOptions; -export default _default; diff --git a/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.js b/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.ts similarity index 87% rename from x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.js rename to x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.ts index 289fff319e5a2..236f17c3b5cc7 100644 --- a/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.js +++ b/x-pack/platform/plugins/shared/fleet/cypress.config.space_awareness.ts @@ -4,21 +4,25 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -Object.defineProperty(exports, '__esModule', { value: true }); -const cypress_config_1 = require('@kbn/cypress-config'); + +import { defineCypressConfig } from '@kbn/cypress-config'; + // eslint-disable-next-line import/no-default-export -exports.default = (0, cypress_config_1.defineCypressConfig)({ +export default defineCypressConfig({ defaultCommandTimeout: 60000, requestTimeout: 60000, responseTimeout: 60000, execTimeout: 120000, pageLoadTimeout: 120000, + retries: { runMode: 2, }, + env: { grepFilterSpecs: false, }, + screenshotsFolder: '../../../../../target/kibana-fleet/cypress/screenshots', trashAssetsBeforeRuns: false, video: false, @@ -26,13 +30,17 @@ exports.default = (0, cypress_config_1.defineCypressConfig)({ viewportHeight: 900, viewportWidth: 1440, screenshotOnRunFailure: true, + e2e: { baseUrl: 'http://localhost:5601', + experimentalRunAllSpecs: true, experimentalMemoryManagement: true, numTestsKeptInMemory: 3, + specPattern: './cypress/e2e/space_awareness/**/*.cy.ts', supportFile: './cypress/support/e2e.ts', + setupNodeEvents(on, config) { // eslint-disable-next-line @typescript-eslint/no-var-requires, @kbn/imports/no_boundary_crossing return require('./cypress/plugins')(on, config); From 57699d90c6d6e86ea37890a3f0ec838372b9c7ba Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Tue, 7 Jan 2025 09:58:36 +0100 Subject: [PATCH 93/95] Fix issue w/ deleted dashboard, usage example for storage index adapter --- .../utils_server/es/storage/README.md | 33 +++++++++++++++++++ .../server/lib/streams/assets/asset_client.ts | 4 +-- .../apis/streams/assets/dashboard.ts | 16 +++++++++ 3 files changed, 51 insertions(+), 2 deletions(-) diff --git a/x-pack/solutions/observability/packages/utils_server/es/storage/README.md b/x-pack/solutions/observability/packages/utils_server/es/storage/README.md index 2b83a7e3ea1b8..bc6a3c04b6ef8 100644 --- a/x-pack/solutions/observability/packages/utils_server/es/storage/README.md +++ b/x-pack/solutions/observability/packages/utils_server/es/storage/README.md @@ -41,3 +41,36 @@ Currently, we only have the StorageIndexAdapter which writes to plain indices. I - Data/Index Lifecycle Management - Migration scripts - Runtime mappings for older versions + +## Usage + +### Storage index adapter + +To use the storage index adapter, instantiate it with an authenticated Elasticsearch client: + +```ts + const storageSettings = { + name: '.kibana_streams_assets', + schema: { + properties: { + [ASSET_ASSET_ID]: types.keyword({ required: true }), + [ASSET_TYPE]: types.enum(Object.values(ASSET_TYPES), { required: true }), + }, + }, + } satisfies IndexStorageSettings; + + // create and configure the adapter + const adapter = new StorageIndexAdapter( + esClient: coreStart.elasticsearch.client.asInternalUser, + this.logger.get('assets'), + storageSettings + ); + + // get the client (which is shared across all adapters) + const client = adapter.getClient(); + + const response = await client.search('operation_name', { + track_total_hits: true + }); + +``` diff --git a/x-pack/solutions/observability/plugins/streams/server/lib/streams/assets/asset_client.ts b/x-pack/solutions/observability/plugins/streams/server/lib/streams/assets/asset_client.ts index 7a1f8434e8d5c..22e50af643bd2 100644 --- a/x-pack/solutions/observability/plugins/streams/server/lib/streams/assets/asset_client.ts +++ b/x-pack/solutions/observability/plugins/streams/server/lib/streams/assets/asset_client.ts @@ -282,7 +282,7 @@ export class AssetClient { return idsByType.dashboard.flatMap((dashboardId): Asset[] => { const dashboard = dashboardsById[dashboardId]; - if (dashboard) { + if (dashboard && !dashboard.error) { return [dashboardSavedObjectToAsset(dashboardId, dashboard)]; } return []; @@ -312,7 +312,7 @@ export class AssetClient { return idsByType.slo.flatMap((sloId): Asset[] => { const sloDefinition = sloDefinitionsById[sloId]; - if (sloDefinition) { + if (sloDefinition && !sloDefinition.error) { return [sloSavedObjectToAsset(sloId, sloDefinition)]; } return []; diff --git a/x-pack/test/api_integration/apis/streams/assets/dashboard.ts b/x-pack/test/api_integration/apis/streams/assets/dashboard.ts index d6f379559d3aa..7886a52da6165 100644 --- a/x-pack/test/api_integration/apis/streams/assets/dashboard.ts +++ b/x-pack/test/api_integration/apis/streams/assets/dashboard.ts @@ -241,6 +241,22 @@ export default function ({ getService }: FtrProviderContext) { expect(response.body.dashboards.length).to.eql(1); }); }); + + describe('after deleting the dashboards', () => { + before(async () => { + await unloadDashboards(); + }); + + it('no longer lists the dashboard as a linked asset', async () => { + const response = await apiClient.fetch('GET /api/streams/{id}/dashboards', { + params: { path: { id: 'logs' } }, + }); + + expect(response.status).to.eql(200); + + expect(response.body.dashboards.length).to.eql(0); + }); + }); }); describe('after using the bulk API', () => { From 7680759aae15839bb9d96f985894e45945859ad1 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Tue, 7 Jan 2025 10:30:30 +0100 Subject: [PATCH 94/95] Use signal from fetch hookC --- .../public/hooks/use_dashboards_fetch.ts | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/x-pack/solutions/observability/plugins/streams_app/public/hooks/use_dashboards_fetch.ts b/x-pack/solutions/observability/plugins/streams_app/public/hooks/use_dashboards_fetch.ts index 8dbca5f64c29c..c03e0cb2ea46f 100644 --- a/x-pack/solutions/observability/plugins/streams_app/public/hooks/use_dashboards_fetch.ts +++ b/x-pack/solutions/observability/plugins/streams_app/public/hooks/use_dashboards_fetch.ts @@ -4,12 +4,10 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { useAbortController } from '@kbn/observability-utils-browser/hooks/use_abort_controller'; import { useKibana } from './use_kibana'; import { useStreamsAppFetch } from './use_streams_app_fetch'; export const useDashboardsFetch = (id?: string) => { - const { signal } = useAbortController(); const { dependencies: { start: { @@ -18,19 +16,22 @@ export const useDashboardsFetch = (id?: string) => { }, } = useKibana(); - const dashboardsFetch = useStreamsAppFetch(() => { - if (!id) { - return Promise.resolve(undefined); - } - return streamsRepositoryClient.fetch('GET /api/streams/{id}/dashboards', { - signal, - params: { - path: { - id, + const dashboardsFetch = useStreamsAppFetch( + ({ signal }) => { + if (!id) { + return Promise.resolve(undefined); + } + return streamsRepositoryClient.fetch('GET /api/streams/{id}/dashboards', { + signal, + params: { + path: { + id, + }, }, - }, - }); - }, [id, signal, streamsRepositoryClient]); + }); + }, + [id, streamsRepositoryClient] + ); return dashboardsFetch; }; From 1266ea087639cb5f177540cc929ebb1ef6d3bce9 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Tue, 7 Jan 2025 16:57:54 +0100 Subject: [PATCH 95/95] Clarify comment --- .../observability/packages/utils_server/es/storage/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/solutions/observability/packages/utils_server/es/storage/README.md b/x-pack/solutions/observability/packages/utils_server/es/storage/README.md index bc6a3c04b6ef8..45f9015cad9fb 100644 --- a/x-pack/solutions/observability/packages/utils_server/es/storage/README.md +++ b/x-pack/solutions/observability/packages/utils_server/es/storage/README.md @@ -66,7 +66,7 @@ To use the storage index adapter, instantiate it with an authenticated Elasticse storageSettings ); - // get the client (which is shared across all adapters) + // get the client (its interface is shared across all adapters) const client = adapter.getClient(); const response = await client.search('operation_name', {