From 7256c6c64051ad65622df4abb5a0e095a2706160 Mon Sep 17 00:00:00 2001 From: Miriam <31922082+MiriamAparicio@users.noreply.github.com> Date: Thu, 12 Sep 2024 11:02:07 +0100 Subject: [PATCH] [ObsUX][Infra] Move getServices to apm_data_access plugin (#192565) Closes https://github.com/elastic/kibana/issues/190338 ### What was done - Creates a service in `apm_data_access` that returns services for a specific host - Remove all getServices() from infra plugin #### How to test In Infrastructure/Hosts, filter by `service.name: *`, when clicking on one of the hosts we expect to see in the host details, and also the flyout, a list of services with their proper agent icon image --- .../services/get_host_services/index.ts | 185 ++++++++++++++++++ .../server/services/get_services.ts | 2 + .../host_details/get_infra_services.ts | 10 +- .../server/lib/host_details/get_services.ts | 167 ---------------- .../infra/server/routes/services/index.ts | 39 ++-- 5 files changed, 218 insertions(+), 185 deletions(-) create mode 100644 x-pack/plugins/observability_solution/apm_data_access/server/services/get_host_services/index.ts delete mode 100644 x-pack/plugins/observability_solution/infra/server/lib/host_details/get_services.ts diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/services/get_host_services/index.ts b/x-pack/plugins/observability_solution/apm_data_access/server/services/get_host_services/index.ts new file mode 100644 index 0000000000000..b83320d162f3c --- /dev/null +++ b/x-pack/plugins/observability_solution/apm_data_access/server/services/get_host_services/index.ts @@ -0,0 +1,185 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { rangeQuery, termQuery } from '@kbn/observability-plugin/server'; +import { + AGENT_NAME, + HOST_HOSTNAME, + HOST_NAME, + METRICSET_NAME, + SERVICE_NAME, +} from '@kbn/apm-types/es_fields'; +import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { + RollupInterval, + TimeRangeMetadata, + getBucketSize, + getPreferredBucketSizeAndDataSource, +} from '../../../common'; +import { ApmDocumentType } from '../../../common/document_type'; +import type { ApmDataAccessServicesParams } from '../get_services'; + +const MAX_SIZE = 1000; + +export interface HostServicesRequest { + filters: Record; + start: number; + end: number; + size?: number; + documentSources: TimeRangeMetadata['sources']; +} + +const suitableTypes = [ApmDocumentType.TransactionMetric, ApmDocumentType.ErrorEvent]; + +export function createGetHostServices({ apmEventClient }: ApmDataAccessServicesParams) { + return async ({ start, end, size = MAX_SIZE, filters, documentSources }: HostServicesRequest) => { + const sourcesToUse = getPreferredBucketSizeAndDataSource({ + sources: documentSources.filter((s) => suitableTypes.includes(s.documentType)), + bucketSizeInSeconds: getBucketSize({ start, end, numBuckets: 50 }).bucketSize, + }); + + const commonFiltersList: QueryDslQueryContainer[] = [ + ...rangeQuery(start, end), + { + exists: { + field: SERVICE_NAME, + }, + }, + ]; + + if (filters[HOST_NAME]) { + commonFiltersList.push({ + bool: { + should: [ + ...termQuery(HOST_NAME, filters[HOST_NAME]), + ...termQuery(HOST_HOSTNAME, filters[HOST_HOSTNAME]), + ], + minimum_should_match: 1, + }, + }); + } + // get services from transaction metrics + const metricsQuery = await apmEventClient.search('get_apm_host_services_from_metrics', { + apm: { + sources: [ + { + documentType: ApmDocumentType.TransactionMetric, + rollupInterval: RollupInterval.OneMinute, + }, + ], + }, + body: { + track_total_hits: false, + size: 0, + query: { + bool: { + filter: [ + { + bool: { + should: [ + ...termQuery(METRICSET_NAME, 'app'), + { + bool: { + must: [...termQuery(METRICSET_NAME, 'transaction')], + }, + }, + ], + minimum_should_match: 1, + }, + }, + ...commonFiltersList, + ], + }, + }, + aggs: { + services: { + terms: { + field: SERVICE_NAME, + size, + }, + aggs: { + latestAgent: { + top_metrics: { + metrics: [{ field: AGENT_NAME }], + sort: { + '@timestamp': 'desc', + }, + size: 1, + }, + }, + }, + }, + }, + }, + }); + + // get services from logs + const logsQuery = await apmEventClient.search('get_apm_host_services_from_logs', { + apm: { + sources: [ + { + documentType: ApmDocumentType.ErrorEvent, + rollupInterval: sourcesToUse.source.rollupInterval, + }, + ], + }, + body: { + track_total_hits: false, + size: 0, + query: { + bool: { + filter: commonFiltersList, + }, + }, + aggs: { + services: { + terms: { + field: SERVICE_NAME, + size, + }, + aggs: { + latestAgent: { + top_metrics: { + metrics: [{ field: AGENT_NAME }], + sort: { + '@timestamp': 'desc', + }, + size: 1, + }, + }, + }, + }, + }, + }, + }); + + const servicesListBucketsFromMetrics = metricsQuery.aggregations?.services.buckets || []; + const servicesListBucketsFromLogs = logsQuery.aggregations?.services.buckets || []; + const serviceMap = [...servicesListBucketsFromMetrics, ...servicesListBucketsFromLogs].reduce( + (acc, bucket) => { + const serviceName = bucket.key as string; + const latestAgentEntry = bucket.latestAgent.top[0]; + const latestTimestamp = latestAgentEntry.sort[0] as string; + const agentName = latestAgentEntry.metrics[AGENT_NAME] as string | null; + // dedup and get the latest timestamp + const existingService = acc.get(serviceName); + if (!existingService || existingService.latestTimestamp < latestTimestamp) { + acc.set(serviceName, { latestTimestamp, agentName }); + } + return acc; + }, + new Map() + ); + const services = Array.from(serviceMap) + .slice(0, size) + .map(([serviceName, { agentName }]) => ({ + serviceName, + agentName, + })); + return { services }; + }; +} diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/services/get_services.ts b/x-pack/plugins/observability_solution/apm_data_access/server/services/get_services.ts index 03a31a1cc4534..71ae961b256b6 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/services/get_services.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/services/get_services.ts @@ -9,6 +9,7 @@ import { APMEventClient } from '../lib/helpers/create_es_client/create_apm_event import { createGetDocumentSources } from './get_document_sources'; import { getDocumentTypeConfig } from './get_document_type_config'; import { createGetHostNames } from './get_host_names'; +import { createGetHostServices } from './get_host_services'; export interface ApmDataAccessServicesParams { apmEventClient: APMEventClient; @@ -19,5 +20,6 @@ export function getServices(params: ApmDataAccessServicesParams) { getDocumentSources: createGetDocumentSources(params), getHostNames: createGetHostNames(params), getDocumentTypeConfig, + getHostServices: createGetHostServices(params), }; } diff --git a/x-pack/plugins/observability_solution/infra/common/http_api/host_details/get_infra_services.ts b/x-pack/plugins/observability_solution/infra/common/http_api/host_details/get_infra_services.ts index 718513416dad7..9f330567337eb 100644 --- a/x-pack/plugins/observability_solution/infra/common/http_api/host_details/get_infra_services.ts +++ b/x-pack/plugins/observability_solution/infra/common/http_api/host_details/get_infra_services.ts @@ -8,8 +8,7 @@ import { createLiteralValueFromUndefinedRT, inRangeFromStringRt, - dateRt, - datemathStringRt, + isoToEpochRt, } from '@kbn/io-ts-utils'; import * as rt from 'io-ts'; @@ -17,7 +16,6 @@ export const sizeRT = rt.union([ inRangeFromStringRt(1, 100), createLiteralValueFromUndefinedRT(10), ]); -export const assetDateRT = rt.union([dateRt, datemathStringRt]); export const servicesFiltersRT = rt.strict({ ['host.name']: rt.string, @@ -26,7 +24,7 @@ export const servicesFiltersRT = rt.strict({ export type ServicesFilter = rt.TypeOf; export const GetServicesRequestQueryRT = rt.intersection([ - rt.strict({ from: assetDateRT, to: assetDateRT, filters: rt.string }), + rt.strict({ from: isoToEpochRt, to: isoToEpochRt, filters: rt.string }), rt.partial({ size: sizeRT, validatedFilters: servicesFiltersRT, @@ -37,8 +35,8 @@ export type GetServicesRequestQuery = rt.TypeOf { - const { error, metric } = apmIndices; - const { filters, size = 10, from, to } = options; - const commonFiltersList: QueryDslQueryContainer[] = [ - { - range: { - '@timestamp': { - gte: from, - lte: to, - }, - }, - }, - { - exists: { - field: 'service.name', - }, - }, - ]; - - if (filters['host.name']) { - // also query for host.hostname field along with host.name, as some services may use this field - const HOST_HOSTNAME_FIELD = 'host.hostname'; - commonFiltersList.push({ - bool: { - should: [ - ...termQuery(HOST_NAME_FIELD, filters[HOST_NAME_FIELD]), - ...termQuery(HOST_HOSTNAME_FIELD, filters[HOST_NAME_FIELD]), - ], - minimum_should_match: 1, - }, - }); - } - const aggs = { - services: { - terms: { - field: 'service.name', - size, - }, - aggs: { - latestAgent: { - top_metrics: { - metrics: [{ field: 'agent.name' }], - sort: { - '@timestamp': 'desc', - }, - size: 1, - }, - }, - }, - }, - }; - // get services from transaction metrics - const metricsQuery = { - size: 0, - _source: false, - query: { - bool: { - filter: [ - { - term: { - [PROCESSOR_EVENT]: 'metric', - }, - }, - { - bool: { - should: [ - { - term: { - 'metricset.name': 'app', - }, - }, - { - bool: { - must: [ - { - term: { - 'metricset.name': 'transaction', - }, - }, - { - term: { - 'metricset.interval': '1m', // make this dynamic if we start returning time series data - }, - }, - ], - }, - }, - ], - minimum_should_match: 1, - }, - }, - ...commonFiltersList, - ], - }, - }, - aggs, - }; - // get services from logs - const logsQuery = { - size: 0, - _source: false, - query: { - bool: { - filter: commonFiltersList, - }, - }, - aggs, - }; - - const resultMetrics = await client<{}, ServicesAPIQueryAggregation>({ - body: metricsQuery, - index: [metric], - }); - const resultLogs = await client<{}, ServicesAPIQueryAggregation>({ - body: logsQuery, - index: [error], - }); - - const servicesListBucketsFromMetrics = resultMetrics.aggregations?.services?.buckets || []; - const servicesListBucketsFromLogs = resultLogs.aggregations?.services?.buckets || []; - const serviceMap = [...servicesListBucketsFromMetrics, ...servicesListBucketsFromLogs].reduce( - (acc, bucket) => { - const serviceName = bucket.key; - const latestAgentEntry = bucket.latestAgent.top[0]; - const latestTimestamp = latestAgentEntry.sort[0]; - const agentName = latestAgentEntry.metrics['agent.name']; - // dedup and get the latest timestamp - const existingService = acc.get(serviceName); - if (!existingService || existingService.latestTimestamp < latestTimestamp) { - acc.set(serviceName, { latestTimestamp, agentName }); - } - - return acc; - }, - new Map() - ); - - const services = Array.from(serviceMap) - .slice(0, size) - .map(([serviceName, { agentName }]) => ({ - serviceName, - agentName, - })); - return { services }; -}; diff --git a/x-pack/plugins/observability_solution/infra/server/routes/services/index.ts b/x-pack/plugins/observability_solution/infra/server/routes/services/index.ts index 86af345d5175e..9673b31788487 100644 --- a/x-pack/plugins/observability_solution/infra/server/routes/services/index.ts +++ b/x-pack/plugins/observability_solution/infra/server/routes/services/index.ts @@ -6,15 +6,14 @@ */ import { - GetServicesRequestQueryRT, GetServicesRequestQuery, + GetServicesRequestQueryRT, ServicesAPIResponseRT, } from '../../../common/http_api/host_details'; import { InfraBackendLibs } from '../../lib/infra_types'; -import { getServices } from '../../lib/host_details/get_services'; import { validateStringAssetFilters } from './lib/utils'; -import { createSearchClient } from '../../lib/create_search_client'; import { buildRouteValidationWithExcess } from '../../utils/route_validation'; +import { getApmDataAccessClient } from '../../lib/helpers/get_apm_data_access_client'; export const initServicesRoute = (libs: InfraBackendLibs) => { const { framework } = libs; @@ -33,18 +32,34 @@ export const initServicesRoute = (libs: InfraBackendLibs) => { }, }, }, - async (requestContext, request, response) => { - const [{ savedObjects }] = await libs.getStartServices(); + async (context, request, response) => { const { from, to, size = 10, validatedFilters } = request.query; - const client = createSearchClient(requestContext, framework, request); - const soClient = savedObjects.getScopedClient(request); - const apmIndices = await libs.plugins.apmDataAccess.setup.getApmIndices(soClient); - const services = await getServices(client, apmIndices, { - from, - to, - size, + const apmDataAccessClient = getApmDataAccessClient({ request, libs, context }); + const hasApmPrivileges = await apmDataAccessClient.hasPrivileges(); + + if (!hasApmPrivileges) { + return response.customError({ + statusCode: 403, + body: { + message: 'APM data access service is not available', + }, + }); + } + + const apmDataAccessServices = await apmDataAccessClient.getServices(); + + const apmDocumentSources = await apmDataAccessServices.getDocumentSources({ + start: from, + end: to, + }); + + const services = await apmDataAccessServices?.getHostServices({ + documentSources: apmDocumentSources, + start: from, + end: to, filters: validatedFilters!, + size, }); return response.ok({ body: ServicesAPIResponseRT.encode(services),