diff --git a/x-pack/plugins/data_quality/public/routes/dataset_quality_details/context.tsx b/x-pack/plugins/data_quality/public/routes/dataset_quality_details/context.tsx index 462cbbbd9288b..5f710c445471e 100644 --- a/x-pack/plugins/data_quality/public/routes/dataset_quality_details/context.tsx +++ b/x-pack/plugins/data_quality/public/routes/dataset_quality_details/context.tsx @@ -74,7 +74,10 @@ export function DatasetQualityDetailsContextProvider({ urlStateStorageContainer, datasetQualityDetailsState: state, }); - const breadcrumbValue = getBreadcrumbValue(state.dataStream, state.integration); + const breadcrumbValue = getBreadcrumbValue( + state.dataStream, + state.integration?.integration + ); setBreadcrumbs([{ text: breadcrumbValue }]); } ); diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.mock.ts b/x-pack/plugins/fleet/server/services/epm/package_service.mock.ts index eeaa80b0c9449..e72b218ab1fcc 100644 --- a/x-pack/plugins/fleet/server/services/epm/package_service.mock.ts +++ b/x-pack/plugins/fleet/server/services/epm/package_service.mock.ts @@ -17,6 +17,7 @@ const createClientMock = (): jest.Mocked => ({ fetchFindLatestPackage: jest.fn(), readBundledPackage: jest.fn(), getAgentPolicyConfigYAML: jest.fn(), + getLatestPackageInfo: jest.fn(), getPackage: jest.fn(), getPackageFieldsMetadata: jest.fn(), getPackages: jest.fn(), diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.test.ts b/x-pack/plugins/fleet/server/services/epm/package_service.test.ts index 479d355c00e68..ea7586b9ebd78 100644 --- a/x-pack/plugins/fleet/server/services/epm/package_service.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/package_service.test.ts @@ -43,6 +43,7 @@ const testKeys = [ 'getInstallation', 'ensureInstalledPackage', 'fetchFindLatestPackage', + 'getLatestPackageInfo', 'getPackage', 'getPackageFieldsMetadata', 'reinstallEsAssets', @@ -112,6 +113,23 @@ function getTest( }; break; case testKeys[3]: + test = { + method: mocks.packageClient.getLatestPackageInfo.bind(mocks.packageClient), + args: ['package name'], + spy: jest.spyOn(epmPackagesGet, 'getPackageInfo'), + spyArgs: [ + { + pkgName: 'package name', + pkgVersion: '', + savedObjectsClient: mocks.soClient, + prerelease: undefined, + }, + ], + spyResponse: { name: 'getLatestPackageInfo test' }, + expectedReturnValue: { name: 'getLatestPackageInfo test' }, + }; + break; + case testKeys[4]: test = { method: mocks.packageClient.getPackage.bind(mocks.packageClient), args: ['package name', '8.0.0'], @@ -127,7 +145,7 @@ function getTest( }, }; break; - case testKeys[4]: + case testKeys[5]: test = { method: mocks.packageClient.getPackageFieldsMetadata.bind(mocks.packageClient), args: [{ packageName: 'package_name', datasetName: 'dataset_name' }], @@ -141,7 +159,7 @@ function getTest( }, }; break; - case testKeys[5]: + case testKeys[6]: const pkg: InstallablePackage = { format_version: '1.0.0', name: 'package name', @@ -187,7 +205,7 @@ function getTest( ], }; break; - case testKeys[6]: + case testKeys[7]: const bundledPackage = { name: 'package name', version: '8.0.0', diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.ts b/x-pack/plugins/fleet/server/services/epm/package_service.ts index a097db584b460..782aeffb6a85c 100644 --- a/x-pack/plugins/fleet/server/services/epm/package_service.ts +++ b/x-pack/plugins/fleet/server/services/epm/package_service.ts @@ -56,6 +56,7 @@ import { getPackages, installPackage, getTemplateInputs, + getPackageInfo, } from './packages'; import { generatePackageInfoFromArchiveBuffer } from './archive'; import { getEsPackage } from './archive/storage'; @@ -113,6 +114,11 @@ export interface PackageClient { options?: Parameters['1'] ): ReturnType; + getLatestPackageInfo( + packageName: string, + prerelease?: boolean + ): ReturnType; + getPackages(params?: { excludeInstallStatus?: false; category?: CategoryId; @@ -328,6 +334,16 @@ class PackageClientImpl implements PackageClient { return getPackageFieldsMetadata(params, options); } + public async getLatestPackageInfo(packageName: string, prerelease?: boolean) { + await this.#runPreflight(READ_PACKAGE_INFO_AUTHZ); + return getPackageInfo({ + savedObjectsClient: this.internalSoClient, + pkgName: packageName, + pkgVersion: '', + prerelease, + }); + } + public async getPackages(params?: { excludeInstallStatus?: false; category?: CategoryId; diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts index 51a1421aec918..7220f9014b477 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts @@ -95,13 +95,24 @@ export const integrationRt = rt.intersection([ export type IntegrationType = rt.TypeOf; +export const checkAndLoadIntegrationResponseRt = rt.union([ + rt.type({ isIntegration: rt.literal(false), areAssetsAvailable: rt.boolean }), + rt.type({ + isIntegration: rt.literal(true), + areAssetsAvailable: rt.literal(true), + integration: integrationRt, + }), +]); + +export type CheckAndLoadIntegrationResponse = rt.TypeOf; + export const getIntegrationsResponseRt = rt.exact( rt.type({ integrations: rt.array(integrationRt), }) ); -export type IntegrationResponse = rt.TypeOf; +export type IntegrationsResponse = rt.TypeOf; export const degradedFieldRt = rt.type({ name: rt.string, diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts index ce74552b581b9..f6372e1dc61a2 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts @@ -5,9 +5,7 @@ * 2.0. */ -export interface GetDataStreamIntegrationParams { - integrationName: string; -} +import { Integration } from '../data_streams_stats/integration'; export interface AnalyzeDegradedFieldsParams { dataStream: string; @@ -19,3 +17,13 @@ export interface UpdateFieldLimitParams { dataStream: string; newFieldLimit: number; } + +export interface CheckAndLoadIntegrationParams { + dataStream: string; +} + +export interface IntegrationType { + isIntegration: boolean; + areAssetsAvailable: boolean; + integration?: Integration; +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx index 1056713ac2070..5dc8d6d367010 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx @@ -29,7 +29,11 @@ import { IncreaseFieldMappingLimit } from './increase_field_mapping_limit'; import { FieldLimitDocLink } from './field_limit_documentation_link'; import { MessageCallout } from './message_callout'; -export function FieldMappingLimit({ isIntegration }: { isIntegration: boolean }) { +export function FieldMappingLimit({ + areIntegrationAssetsAvailable, +}: { + areIntegrationAssetsAvailable: boolean; +}) { const accordionId = useGeneratedHtmlId({ prefix: increaseFieldMappingLimitTitle, }); @@ -66,7 +70,7 @@ export function FieldMappingLimit({ isIntegration }: { isIntegration: boolean }) - {isIntegration && ( + {areIntegrationAssetsAvailable && ( <> updateNewFieldLimit(newFieldLimit)} isLoading={isMitigationInProgress} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/index.tsx index 34f39f25a67ec..d9ca2128521f2 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/index.tsx @@ -15,7 +15,7 @@ import { PossibleMitigationTitle } from './title'; export function PossibleMitigations() { const { degradedFieldAnalysis, isAnalysisInProgress } = useDegradedFields(); const { integrationDetails } = useDatasetQualityDetailsState(); - const isIntegration = Boolean(integrationDetails?.integration); + const areIntegrationAssetsAvailable = !!integrationDetails?.integration?.areAssetsAvailable; return ( !isAnalysisInProgress && ( @@ -24,7 +24,7 @@ export function PossibleMitigations() { {degradedFieldAnalysis?.isFieldLimitIssue && ( <> - + )} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/component_template_link.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/component_template_link.tsx index 54bbe91f2f2e1..e8768d34cdcca 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/component_template_link.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/component_template_link.tsx @@ -13,7 +13,11 @@ import { useDatasetQualityDetailsState } from '../../../../../hooks'; import { getComponentTemplatePrefixFromIndexTemplate } from '../../../../../../common/utils/component_template_name'; import { otherMitigationsCustomComponentTemplate } from '../../../../../../common/translations'; -export function CreateEditComponentTemplateLink({ isIntegration }: { isIntegration: boolean }) { +export function CreateEditComponentTemplateLink({ + areIntegrationAssetsAvailable, +}: { + areIntegrationAssetsAvailable: boolean; +}) { const { services: { application, @@ -54,7 +58,7 @@ export function CreateEditComponentTemplateLink({ isIntegration }: { isIntegrati name, ]); - const templateUrl = isIntegration ? componentTemplatePath : indexTemplatePath; + const templateUrl = areIntegrationAssetsAvailable ? componentTemplatePath : indexTemplatePath; const onClickHandler = useCallback(async () => { const options = { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx index f931f3461fb57..34cdbe4b9b838 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx @@ -13,27 +13,27 @@ import { CreateEditPipelineLink } from './pipeline_link'; import { otherMitigationsLoadingAriaText } from '../../../../../../common/translations'; export function ManualMitigations() { - const { integrationDetails, loadingState, dataStreamSettings } = useDatasetQualityDetailsState(); - const isIntegrationPresentInSettings = dataStreamSettings?.integration; - const isIntegration = !!integrationDetails?.integration; - const { dataStreamSettingsLoading, integrationDetailsLoadings } = loadingState; - - const hasIntegrationCheckCompleted = - !dataStreamSettingsLoading && - ((isIntegrationPresentInSettings && !integrationDetailsLoadings) || - !isIntegrationPresentInSettings); + const { + integrationDetails, + loadingState: { integrationDetailsLoaded }, + } = useDatasetQualityDetailsState(); + const areIntegrationAssetsAvailable = !!integrationDetails?.integration?.areAssetsAvailable; return ( - + - + ); } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/pipeline_link.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/pipeline_link.tsx index 6179a3ed0736c..f4084c52e2efd 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/pipeline_link.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/pipeline_link.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useCallback, useMemo } from 'react'; +import React, { useCallback } from 'react'; import { FormattedMessage } from '@kbn/i18n-react'; import { i18n } from '@kbn/i18n'; import { @@ -34,7 +34,11 @@ const AccordionTitle = () => ( ); -export function CreateEditPipelineLink({ isIntegration }: { isIntegration: boolean }) { +export function CreateEditPipelineLink({ + areIntegrationAssetsAvailable, +}: { + areIntegrationAssetsAvailable: boolean; +}) { const { services: { share: { @@ -50,10 +54,7 @@ export function CreateEditPipelineLink({ isIntegration }: { isIntegration: boole const { datasetDetails } = useDatasetQualityDetailsState(); const { type, name } = datasetDetails; - const pipelineName = useMemo( - () => (isIntegration ? `${type}-${name}@custom` : `${type}@custom`), - [isIntegration, type, name] - ); + const pipelineName = areIntegrationAssetsAvailable ? `${type}-${name}@custom` : `${type}@custom`; const ingestPipelineLocator = locators.get('INGEST_PIPELINES_APP_LOCATOR'); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/details/dataset_summary.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/details/dataset_summary.tsx index 17032d7866d05..8bbc9aa085420 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/details/dataset_summary.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/details/dataset_summary.tsx @@ -7,7 +7,7 @@ import React, { Fragment } from 'react'; import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/field-types'; -import { EuiBadge, EuiFlexGroup, EuiPanel, EuiText } from '@elastic/eui'; +import { EuiBadge, EuiFlexGroup, EuiPanel, EuiSkeletonRectangle, EuiText } from '@elastic/eui'; import { css } from '@emotion/react'; import { IntegrationActionsMenu } from './integration_actions_menu'; import { @@ -29,7 +29,8 @@ export function DatasetSummary() { const { dataStreamDetailsLoading, dataStreamSettingsLoading, - integrationDetailsLoadings, + integrationDetailsLoading, + integrationDetailsLoaded, integrationDashboardsLoading, } = loadingState; const formattedLastActivity = dataStreamDetails?.lastActivity @@ -39,12 +40,19 @@ export function DatasetSummary() { ? dataFormatter.convert(dataStreamSettings.createdOn) : '-'; - return ( + return !integrationDetailsLoaded ? ( + + ) : ( - - {integrationDetails.integration?.name} + + + {integrationDetails.integration.integration?.name} + ), actionsMenu: ( ), - isLoading: integrationDetailsLoadings, + isLoading: integrationDetailsLoading, }, { fieldTitle: integrationVersionText, - fieldValue: integrationDetails.integration?.version, - isLoading: integrationDetailsLoadings, + fieldValue: integrationDetails.integration.integration?.version, + isLoading: integrationDetailsLoading, }, ] : []), diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/header.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/header.tsx index e45253b6405c8..60a26d4285732 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/header.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/header.tsx @@ -44,7 +44,8 @@ export function Header() { sendTelemetry, }); - const pageTitle = integrationDetails?.integration?.datasets?.[datasetDetails.name] ?? title; + const pageTitle = + integrationDetails?.integration?.integration?.datasets?.[datasetDetails.name] ?? title; return !loadingState.integrationDetailsLoaded ? ( - +

diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_details_telemetry.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_details_telemetry.ts index f613d3af7fdc4..4d2d838419271 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_details_telemetry.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_details_telemetry.ts @@ -42,7 +42,7 @@ export function useDatasetDetailsTelemetry() { breakdownField, isNonAggregatable, isBreakdownFieldEcs, - integration: integrationDetails.integration, + integration: integrationDetails.integration?.integration, }); } @@ -57,7 +57,7 @@ export function useDatasetDetailsTelemetry() { breakdownField, isNonAggregatable, isBreakdownFieldEcs, - integrationDetails.integration, + integrationDetails.integration?.integration, ]); const startTracking = useCallback(() => { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_details_state.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_details_state.ts index edd16652374a1..cfea77ff4fa39 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_details_state.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_details_state.ts @@ -51,7 +51,9 @@ export const useDatasetQualityDetailsState = () => { ); const dataStreamSettings = useSelector(service, (state) => - state.matches('initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields') + state.matches('initializing.dataStreamSettings.fetchingDataStreamDegradedFields') || + state.matches('initializing.dataStreamSettings.doneFetchingDegradedFields') || + state.matches('initializing.dataStreamSettings.errorFetchingDegradedFields') ? state.context.dataStreamSettings : undefined ); @@ -59,15 +61,17 @@ export const useDatasetQualityDetailsState = () => { const integrationDetails = { integration: useSelector(service, (state) => state.matches( - 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDetails.done' - ) + 'initializing.checkAndLoadIntegrationAndDashboards.loadingIntegrationDashboards' + ) || + state.matches( + 'initializing.checkAndLoadIntegrationAndDashboards.unauthorizedToLoadDashboards' + ) || + state.matches('initializing.checkAndLoadIntegrationAndDashboards.done') ? state.context.integration : undefined ), dashboard: useSelector(service, (state) => - state.matches( - 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.done' - ) + state.matches('initializing.checkAndLoadIntegrationAndDashboards.done') ? state.context.integrationDashboards : undefined ), @@ -77,7 +81,7 @@ export const useDatasetQualityDetailsState = () => { service, (state) => !state.matches( - 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.unauthorized' + 'initializing.checkAndLoadIntegrationAndDashboards.unauthorizedToLoadDashboards' ) ); @@ -106,14 +110,19 @@ export const useDatasetQualityDetailsState = () => { dataStreamSettingsLoading: state.matches( 'initializing.dataStreamSettings.fetchingDataStreamSettings' ), - integrationDetailsLoadings: state.matches( - 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDetails.fetching' - ), - integrationDetailsLoaded: state.matches( - 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDetails.done' + integrationDetailsLoading: state.matches( + 'initializing.checkAndLoadIntegrationAndDashboards.checkingAndLoadingIntegration' ), + integrationDetailsLoaded: + state.matches( + 'initializing.checkAndLoadIntegrationAndDashboards.loadingIntegrationDashboards' + ) || + state.matches( + 'initializing.checkAndLoadIntegrationAndDashboards.unauthorizedToLoadDashboards' + ) || + state.matches('initializing.checkAndLoadIntegrationAndDashboards.done'), integrationDashboardsLoading: state.matches( - 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.fetching' + 'initializing.checkAndLoadIntegrationAndDashboards.loadingIntegrationDashboards' ), })); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_docs_chart.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_docs_chart.ts index fb57e6e87a74f..d499535df1662 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_docs_chart.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_docs_chart.ts @@ -81,7 +81,8 @@ export const useDegradedDocsChart = () => { useEffect(() => { const dataStreamName = dataStream ?? DEFAULT_LOGS_DATA_VIEW; const datasetTitle = - integrationDetails?.integration?.datasets?.[datasetDetails.name] ?? datasetDetails.name; + integrationDetails?.integration?.integration?.datasets?.[datasetDetails.name] ?? + datasetDetails.name; const lensAttributes = getLensAttributes({ color: euiTheme.colors.danger, @@ -95,7 +96,7 @@ export const useDegradedDocsChart = () => { euiTheme.colors.danger, setAttributes, dataStream, - integrationDetails?.integration?.datasets, + integrationDetails?.integration?.integration?.datasets, datasetDetails.name, ]); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.ts index 49ceb50abc3cd..47b03074fe17e 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.ts @@ -77,10 +77,11 @@ export function useDegradedFields() { return renderedItems.find((item) => item.name === expandedDegradedField); }, [expandedDegradedField, renderedItems]); - const isDegradedFieldsLoading = useSelector(service, (state) => - state.matches( - 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.dataStreamDegradedFields.fetching' - ) + const isDegradedFieldsLoading = useSelector( + service, + (state) => + state.matches('initializing.dataStreamSettings.fetchingDataStreamSettings') || + state.matches('initializing.dataStreamSettings.fetchingDataStreamDegradedFields') ); const closeDegradedFieldFlyout = useCallback( diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts index 827cd4b0a1e49..7279bed2e1859 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts @@ -8,6 +8,8 @@ import { HttpStart } from '@kbn/core/public'; import { decodeOrThrow } from '@kbn/io-ts-utils'; import { + CheckAndLoadIntegrationResponse, + checkAndLoadIntegrationResponseRt, DataStreamRolloverResponse, dataStreamRolloverResponseRt, DegradedFieldAnalysis, @@ -17,10 +19,8 @@ import { getDataStreamDegradedFieldsResponseRt, getDataStreamsDetailsResponseRt, getDataStreamsSettingsResponseRt, - getIntegrationsResponseRt, IntegrationDashboardsResponse, integrationDashboardsRT, - IntegrationResponse, UpdateFieldLimitResponse, updateFieldLimitResponseRt, } from '../../../common/api_types'; @@ -40,7 +40,7 @@ import { IDataStreamDetailsClient } from './types'; import { Integration } from '../../../common/data_streams_stats/integration'; import { AnalyzeDegradedFieldsParams, - GetDataStreamIntegrationParams, + CheckAndLoadIntegrationParams, UpdateFieldLimitParams, } from '../../../common/data_stream_details/types'; import { DatasetQualityError } from '../../../common/errors'; @@ -139,43 +139,48 @@ export class DataStreamDetailsClient implements IDataStreamDetailsClient { )(response); } - public async getIntegrationDashboards({ integration }: GetIntegrationDashboardsParams) { + public async checkAndLoadIntegration({ dataStream }: CheckAndLoadIntegrationParams) { const response = await this.http - .get( - `/internal/dataset_quality/integrations/${integration}/dashboards` + .get( + `/internal/dataset_quality/data_streams/${dataStream}/integration/check` ) .catch((error) => { - throw new DatasetQualityError(`Failed to fetch integration dashboards": ${error}`, error); + throw new DatasetQualityError( + `Failed to check if data stream belongs to an integration": ${error}`, + error + ); }); - const { dashboards } = decodeOrThrow( - integrationDashboardsRT, + const decodedResponse = decodeOrThrow( + checkAndLoadIntegrationResponseRt, (message: string) => - new DatasetQualityError(`Failed to decode integration dashboards response: ${message}"`) + new DatasetQualityError(`Failed to decode integration check response: ${message}"`) )(response); - return dashboards; + return { + ...decodedResponse, + integration: decodedResponse.isIntegration + ? Integration.create(decodedResponse.integration) + : undefined, + }; } - public async getDataStreamIntegration( - params: GetDataStreamIntegrationParams - ): Promise { - const { integrationName } = params; + public async getIntegrationDashboards({ integration }: GetIntegrationDashboardsParams) { const response = await this.http - .get('/internal/dataset_quality/integrations') + .get( + `/internal/dataset_quality/integrations/${integration}/dashboards` + ) .catch((error) => { - throw new DatasetQualityError(`Failed to fetch integrations: ${error}`, error); + throw new DatasetQualityError(`Failed to fetch integration dashboards": ${error}`, error); }); - const { integrations } = decodeOrThrow( - getIntegrationsResponseRt, + const { dashboards } = decodeOrThrow( + integrationDashboardsRT, (message: string) => - new DatasetQualityError(`Failed to decode integrations response: ${message}`) + new DatasetQualityError(`Failed to decode integration dashboards response: ${message}"`) )(response); - const integration = integrations.find((i) => i.name === integrationName); - - if (integration) return Integration.create(integration); + return dashboards; } public async analyzeDegradedField({ diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts index 6eac8bd732840..f38c68a68bff5 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts @@ -6,7 +6,6 @@ */ import { HttpStart } from '@kbn/core/public'; -import { Integration } from '../../../common/data_streams_stats/integration'; import { GetDataStreamSettingsParams, DataStreamSettings, @@ -19,7 +18,8 @@ import { } from '../../../common/data_streams_stats'; import { AnalyzeDegradedFieldsParams, - GetDataStreamIntegrationParams, + IntegrationType, + CheckAndLoadIntegrationParams, UpdateFieldLimitParams, } from '../../../common/data_stream_details/types'; import { @@ -49,10 +49,8 @@ export interface IDataStreamDetailsClient { getDataStreamDegradedFieldValues( params: GetDataStreamDegradedFieldValuesPathParams ): Promise; + checkAndLoadIntegration(params: CheckAndLoadIntegrationParams): Promise; getIntegrationDashboards(params: GetIntegrationDashboardsParams): Promise; - getDataStreamIntegration( - params: GetDataStreamIntegrationParams - ): Promise; analyzeDegradedField(params: AnalyzeDegradedFieldsParams): Promise; setNewFieldLimit(params: UpdateFieldLimitParams): Promise; rolloverDataStream(params: { dataStream: string }): Promise; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts index 8e218819315b2..5e4790309a07f 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts @@ -17,7 +17,7 @@ import { getDataStreamTotalDocsResponseRt, getIntegrationsResponseRt, getNonAggregatableDatasetsRt, - IntegrationResponse, + IntegrationsResponse, NonAggregatableDatasets, } from '../../../common/api_types'; import { @@ -132,7 +132,7 @@ export class DataStreamsStatsClient implements IDataStreamsStatsClient { public async getIntegrations(): Promise { const response = await this.http - .get('/internal/dataset_quality/integrations') + .get('/internal/dataset_quality/integrations') .catch((error) => { throw new DatasetQualityError(`Failed to fetch integrations: ${error}`, error); }); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/notifications.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/notifications.ts index f5fdd063492a3..fbd2dd1dc1913 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/notifications.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/notifications.ts @@ -44,17 +44,10 @@ export const fetchIntegrationDashboardsFailedNotifier = (toasts: IToasts, error: }); }; -export const fetchDataStreamIntegrationFailedNotifier = ( - toasts: IToasts, - error: Error, - integrationName?: string -) => { +export const fetchDataStreamIntegrationFailedNotifier = (toasts: IToasts, error: Error) => { toasts.addDanger({ title: i18n.translate('xpack.datasetQuality.details.fetchIntegrationsFailed', { - defaultMessage: "We couldn't get {integrationName} integration info.", - values: { - integrationName, - }, + defaultMessage: "We couldn't get integration info.", }), text: error.message, }); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts index 8ac65a7dca4a7..363919b35f65e 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts @@ -28,6 +28,8 @@ import { UpdateFieldLimitResponse, } from '../../../common/api_types'; import { fetchNonAggregatableDatasetsFailedNotifier } from '../common/notifications'; + +import { IntegrationType } from '../../../common/data_stream_details'; import { fetchDataStreamDetailsFailedNotifier, assertBreakdownFieldEcsFailedNotifier, @@ -37,7 +39,6 @@ import { updateFieldLimitFailedNotifier, rolloverDataStreamFailedNotifier, } from './notifications'; -import { Integration } from '../../../common/data_streams_stats/integration'; export const createPureDatasetQualityDetailsControllerStateMachine = ( initialContext: DatasetQualityDetailsControllerContext @@ -151,7 +152,7 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( invoke: { src: 'loadDataStreamSettings', onDone: { - target: 'loadingIntegrationsAndDegradedFields', + target: 'fetchingDataStreamDegradedFields', actions: ['storeDataStreamSettings'], }, onError: [ @@ -160,111 +161,49 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( cond: 'isIndexNotFoundError', }, { - target: 'done', + target: 'errorFetchingDataStreamSettings', actions: ['notifyFetchDataStreamSettingsFailed'], }, ], }, }, - loadingIntegrationsAndDegradedFields: { - type: 'parallel', - states: { - dataStreamDegradedFields: { - initial: 'fetching', - states: { - fetching: { - invoke: { - src: 'loadDegradedFields', - onDone: { - target: 'done', - actions: ['storeDegradedFields', 'raiseDegradedFieldsLoaded'], - }, - onError: [ - { - target: '#DatasetQualityDetailsController.indexNotFound', - cond: 'isIndexNotFoundError', - }, - { - target: 'done', - }, - ], - }, - }, - done: { - on: { - UPDATE_DEGRADED_FIELDS_TABLE_CRITERIA: { - target: 'done', - actions: ['storeDegradedFieldTableOptions'], - }, - OPEN_DEGRADED_FIELD_FLYOUT: { - target: - '#DatasetQualityDetailsController.initializing.degradedFieldFlyout.open', - actions: [ - 'storeExpandedDegradedField', - 'resetFieldLimitServerResponse', - ], - }, - TOGGLE_CURRENT_QUALITY_ISSUES: { - target: 'fetching', - actions: ['toggleCurrentQualityIssues'], - }, - }, - }, - }, + errorFetchingDataStreamSettings: {}, + fetchingDataStreamDegradedFields: { + invoke: { + src: 'loadDegradedFields', + onDone: { + target: 'doneFetchingDegradedFields', + actions: ['storeDegradedFields', 'raiseDegradedFieldsLoaded'], }, - integrationDetails: { - initial: 'fetching', - states: { - fetching: { - invoke: { - src: 'loadDataStreamIntegration', - onDone: { - target: 'done', - actions: ['storeDataStreamIntegration'], - }, - onError: { - target: 'done', - actions: ['notifyFetchDatasetIntegrationsFailed'], - }, - }, - }, - done: {}, + onError: [ + { + target: '#DatasetQualityDetailsController.indexNotFound', + cond: 'isIndexNotFoundError', }, - }, - integrationDashboards: { - initial: 'fetching', - states: { - fetching: { - invoke: { - src: 'loadIntegrationDashboards', - onDone: { - target: 'done', - actions: ['storeIntegrationDashboards'], - }, - onError: [ - { - target: 'unauthorized', - cond: 'checkIfActionForbidden', - }, - { - target: 'done', - actions: ['notifyFetchIntegrationDashboardsFailed'], - }, - ], - }, - }, - done: {}, - unauthorized: { - type: 'final', - }, + { + target: 'errorFetchingDegradedFields', }, - }, + ], }, - onDone: { - target: 'done', + }, + doneFetchingDegradedFields: { + on: { + UPDATE_DEGRADED_FIELDS_TABLE_CRITERIA: { + target: 'doneFetchingDegradedFields', + actions: ['storeDegradedFieldTableOptions'], + }, + OPEN_DEGRADED_FIELD_FLYOUT: { + target: + '#DatasetQualityDetailsController.initializing.degradedFieldFlyout.open', + actions: ['storeExpandedDegradedField', 'resetFieldLimitServerResponse'], + }, + TOGGLE_CURRENT_QUALITY_ISSUES: { + target: 'fetchingDataStreamDegradedFields', + actions: ['toggleCurrentQualityIssues'], + }, }, }, - done: {}, + errorFetchingDegradedFields: {}, }, on: { UPDATE_TIME_RANGE: { @@ -272,6 +211,54 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( }, }, }, + checkAndLoadIntegrationAndDashboards: { + initial: 'checkingAndLoadingIntegration', + states: { + checkingAndLoadingIntegration: { + invoke: { + src: 'checkAndLoadIntegration', + onDone: [ + { + target: 'loadingIntegrationDashboards', + actions: 'storeDataStreamIntegration', + cond: 'isDataStreamIsPartOfIntegration', + }, + { + actions: 'storeDataStreamIntegration', + target: 'done', + }, + ], + onError: { + target: 'done', + actions: ['notifyFetchDatasetIntegrationsFailed'], + }, + }, + }, + loadingIntegrationDashboards: { + invoke: { + src: 'loadIntegrationDashboards', + onDone: { + target: 'done', + actions: ['storeIntegrationDashboards'], + }, + onError: [ + { + target: 'unauthorizedToLoadDashboards', + cond: 'checkIfActionForbidden', + }, + { + target: 'done', + actions: ['notifyFetchIntegrationDashboardsFailed'], + }, + ], + }, + }, + unauthorizedToLoadDashboards: { + type: 'final', + }, + done: {}, + }, + }, degradedFieldFlyout: { initial: 'pending', states: { @@ -522,7 +509,7 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( } : {}; }), - storeDataStreamIntegration: assign((context, event: DoneInvokeEvent) => { + storeDataStreamIntegration: assign((context, event: DoneInvokeEvent) => { return 'data' in event ? { integration: event.data, @@ -602,6 +589,14 @@ export const createPureDatasetQualityDetailsControllerStateMachine = ( !event.data.isLatestBackingIndexUpdated ); }, + isDataStreamIsPartOfIntegration: (_, event) => { + return ( + 'data' in event && + typeof event.data === 'object' && + 'isIntegration' in event.data && + event.data.isIntegration + ); + }, }, } ); @@ -634,9 +629,7 @@ export const createDatasetQualityDetailsControllerStateMachine = ({ notifyFetchIntegrationDashboardsFailed: (_context, event: DoneInvokeEvent) => fetchIntegrationDashboardsFailedNotifier(toasts, event.data), notifyFetchDatasetIntegrationsFailed: (context, event: DoneInvokeEvent) => { - const integrationName = - 'dataStreamSettings' in context ? context.dataStreamSettings?.integration : undefined; - return fetchDataStreamIntegrationFailedNotifier(toasts, event.data, integrationName); + return fetchDataStreamIntegrationFailedNotifier(toasts, event.data); }, notifySaveNewFieldLimitError: (_context, event: DoneInvokeEvent) => updateFieldLimitFailedNotifier(toasts, event.data), @@ -735,18 +728,15 @@ export const createDatasetQualityDetailsControllerStateMachine = ({ dataStream: context.dataStream, }); }, - loadDataStreamIntegration: (context) => { - if ('dataStreamSettings' in context && context.dataStreamSettings?.integration) { - return dataStreamDetailsClient.getDataStreamIntegration({ - integrationName: context.dataStreamSettings.integration, - }); - } - return Promise.resolve(); + checkAndLoadIntegration: (context) => { + return dataStreamDetailsClient.checkAndLoadIntegration({ + dataStream: context.dataStream, + }); }, loadIntegrationDashboards: (context) => { - if ('dataStreamSettings' in context && context.dataStreamSettings?.integration) { + if ('integration' in context && context.integration && context.integration.integration) { return dataStreamDetailsClient.getIntegrationDashboards({ - integration: context.dataStreamSettings.integration, + integration: context.integration.integration.name, }); } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts index cdebcfbe53d86..d33e5c4f0415e 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts @@ -20,7 +20,7 @@ import { UpdateFieldLimitResponse, } from '../../../common/api_types'; import { TableCriteria, TimeRangeConfig } from '../../../common/types'; -import { Integration } from '../../../common/data_streams_stats/integration'; +import { IntegrationType } from '../../../common/data_stream_details'; export interface DataStream { name: string; @@ -53,7 +53,7 @@ export interface WithDefaultControllerState { breakdownField?: string; isBreakdownFieldEcs?: boolean; isIndexNotFoundError?: boolean; - integration?: Integration; + integration?: IntegrationType; expandedDegradedField?: string; isNonAggregatable?: boolean; fieldLimit?: FieldLimit; @@ -84,8 +84,11 @@ export interface WithDataStreamSettings { } export interface WithIntegration { - integration: Integration; - integrationDashboards?: Dashboard[]; + integration: IntegrationType; +} + +export interface WithIntegrationDashboards { + integrationDashboards: Dashboard[]; } export interface WithDegradedFieldValues { @@ -116,15 +119,14 @@ export type DatasetQualityDetailsControllerTypeState = value: | 'initializing' | 'initializing.nonAggregatableDataset.fetching' - | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.dataStreamDegradedFields.fetching' + | 'initializing.dataStreamDetails.fetching' | 'initializing.dataStreamSettings.fetchingDataStreamSettings' - | 'initializing.dataStreamDetails.fetching'; + | 'initializing.dataStreamSettings.errorFetchingDataStreamSettings' + | 'initializing.checkAndLoadIntegrationAndDashboards.checkingAndLoadingIntegration'; context: WithDefaultControllerState; } | { - value: - | 'initializing.nonAggregatableDataset.done' - | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.dataStreamDegradedFields.fetching'; + value: 'initializing.nonAggregatableDataset.done'; context: WithDefaultControllerState & WithNonAggregatableDatasetStatus; } | { @@ -139,25 +141,25 @@ export type DatasetQualityDetailsControllerTypeState = value: 'initializing.checkBreakdownFieldIsEcs.done'; context: WithDefaultControllerState & WithBreakdownInEcsCheck; } - | { - value: 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.dataStreamDegradedFields.done'; - context: WithDefaultControllerState & - WithNonAggregatableDatasetStatus & - WithDegradedFieldsData; - } | { value: - | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields' - | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDetails.fetching' - | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.fetching' - | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.unauthorized'; + | 'initializing.dataStreamSettings.fetchingDataStreamDegradedFields' + | 'initializing.dataStreamSettings.errorFetchingDegradedFields'; context: WithDefaultControllerState & WithDataStreamSettings; } + | { + value: 'initializing.dataStreamSettings.doneFetchingDegradedFields'; + context: WithDefaultControllerState & WithDataStreamSettings & WithDegradedFieldsData; + } | { value: - | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDetails.done' - | 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.done'; - context: WithDefaultControllerState & WithDataStreamSettings & WithIntegration; + | 'initializing.checkAndLoadIntegrationAndDashboards.loadingIntegrationDashboards' + | 'initializing.checkAndLoadIntegrationAndDashboards.unauthorizedToLoadDashboards'; + context: WithDefaultControllerState & WithIntegration; + } + | { + value: 'initializing.checkAndLoadIntegrationAndDashboards.done'; + context: WithDefaultControllerState & WithIntegration & WithIntegrationDashboards; } | { value: 'initializing.degradedFieldFlyout.open'; @@ -233,8 +235,8 @@ export type DatasetQualityDetailsControllerEvent = | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent - | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent - | DoneInvokeEvent; + | DoneInvokeEvent + | DoneInvokeEvent; diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/check_and_load_integration/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/check_and_load_integration/index.ts new file mode 100644 index 0000000000000..a6606028b6cfc --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/check_and_load_integration/index.ts @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { PackageClient } from '@kbn/fleet-plugin/server'; +import { Logger } from '@kbn/logging'; +import { validateCustomComponentTemplate } from './validate_custom_component_template'; +import { getIntegration, getIntegrations } from '../../integrations/get_integrations'; +import { getComponentTemplatePrefixFromIndexTemplate } from '../../../../common/utils/component_template_name'; +import { CheckAndLoadIntegrationResponse } from '../../../../common/api_types'; +import { dataStreamService } from '../../../services'; + +// The function works on 2 conditions: +// 1. It checks if integration name is present in meta field response of the datastream. +// If yes, it considers it to be an integration. No further checks +// 2. If not, then it does the various checks +export async function checkAndLoadIntegration({ + esClient, + packageClient, + logger, + dataStream, +}: { + esClient: ElasticsearchClient; + packageClient: PackageClient; + logger: Logger; + dataStream: string; +}): Promise { + const [dataStreamInfo] = await dataStreamService.getMatchingDataStreams(esClient, dataStream); + + const indexTemplate = dataStreamInfo?.template; + const isManaged = dataStreamInfo?._meta?.managed; + + const integrationNameFromDataStream = dataStreamInfo?._meta?.package?.name; + + // Index template must be present and isManaged should be true or + // integration name should be present + // Else it's not an integration + if ((!indexTemplate || !isManaged) && !integrationNameFromDataStream) { + return { isIntegration: false, areAssetsAvailable: false }; + } + + // If integration name is present, then we find and return the integration + if (integrationNameFromDataStream) { + try { + const integrationDetailsMatchingDataStream = await getIntegration({ + packageClient, + logger, + packageName: integrationNameFromDataStream, + }); + + if (integrationDetailsMatchingDataStream) { + return { + isIntegration: true, + integration: integrationDetailsMatchingDataStream, + areAssetsAvailable: true, + }; + } + } catch (e) { + // This should ideally not happen. As integration name is present in Data stream + // meta response but the integration itself is not found + // Worst case i could think of is, may be the integration is deleted from the + // system at a later point of time + return { isIntegration: false, areAssetsAvailable: false }; + } + } + + // cleaning the index template name as some have @template suffix + const indexTemplateNameWithoutSuffix = getComponentTemplatePrefixFromIndexTemplate(indexTemplate); + + // Check if index template name has both type and dataset part + const isDedicatedComponentTemplate = indexTemplateNameWithoutSuffix.split('-').length === 2; + + // If only 1 part exists, then it's not a dedicated index template + // Data stream name must starts with this index template, then it's a dedicated index template else not + if (!isDedicatedComponentTemplate || !dataStream.startsWith(indexTemplateNameWithoutSuffix)) { + return { isIntegration: false, areAssetsAvailable: false }; + } + + const isValidCustomComponentTemplate = await validateCustomComponentTemplate({ + esClient, + indexTemplateName: indexTemplate, + }); + + if (!isValidCustomComponentTemplate) { + return { isIntegration: false, areAssetsAvailable: false }; + } + + const datasetName = indexTemplateNameWithoutSuffix.split('-')[1]; + + const allIntegrations = await getIntegrations({ packageClient, logger }); + const integrationFromDataset = allIntegrations.find( + (integration) => datasetName in (integration?.datasets ?? {}) + ); + + if (integrationFromDataset) { + return { isIntegration: true, integration: integrationFromDataset, areAssetsAvailable: true }; + } + + // Since the logic reached the last statement, it means it passed all checks for assets being available + return { isIntegration: false, areAssetsAvailable: true }; +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/check_and_load_integration/validate_custom_component_template.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/check_and_load_integration/validate_custom_component_template.ts new file mode 100644 index 0000000000000..13213814935f1 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/check_and_load_integration/validate_custom_component_template.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { createDatasetQualityESClient } from '../../../utils'; +import { getComponentTemplatePrefixFromIndexTemplate } from '../../../../common/utils/component_template_name'; + +export async function validateCustomComponentTemplate({ + esClient, + indexTemplateName, +}: { + esClient: ElasticsearchClient; + indexTemplateName: string; +}): Promise { + const datasetQualityESClient = createDatasetQualityESClient(esClient); + // cleaning the index template name as some have @template suffix + const componentTemplateName = getComponentTemplatePrefixFromIndexTemplate(indexTemplateName); + + try { + const { index_templates: indexTemplates } = await datasetQualityESClient.indexTemplates({ + name: indexTemplateName, + }); + + return indexTemplates.some((template) => + template.index_template.composed_of.includes(componentTemplateName + '@custom') + ); + } catch (error) { + return false; + } +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts index 288eff11b92a8..0ca2ae94214e8 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts @@ -16,38 +16,12 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { MAX_HOSTS_METRIC_VALUE } from '../../../../common/constants'; import { _IGNORED } from '../../../../common/es_fields'; -import { DataStreamDetails, DataStreamSettings } from '../../../../common/api_types'; +import { DataStreamDetails } from '../../../../common/api_types'; import { createDatasetQualityESClient } from '../../../utils'; -import { dataStreamService, datasetQualityPrivileges } from '../../../services'; +import { datasetQualityPrivileges } from '../../../services'; import { getDataStreams } from '../get_data_streams'; import { getDataStreamsMeteringStats } from '../get_data_streams_metering_stats'; -export async function getDataStreamSettings({ - esClient, - dataStream, -}: { - esClient: ElasticsearchClient; - dataStream: string; -}): Promise { - const [createdOn, [dataStreamInfo], datasetUserPrivileges] = await Promise.all([ - getDataStreamCreatedOn(esClient, dataStream), - dataStreamService.getMatchingDataStreams(esClient, dataStream), - datasetQualityPrivileges.getDatasetPrivileges(esClient, dataStream), - ]); - - const integration = dataStreamInfo?._meta?.package?.name; - const lastBackingIndex = dataStreamInfo?.indices?.slice(-1)[0]; - const indexTemplate = dataStreamInfo?.template; - - return { - createdOn, - integration, - datasetUserPrivileges, - lastBackingIndexName: lastBackingIndex?.index_name, - indexTemplate, - }; -} - export async function getDataStreamDetails({ esClient, dataStream, @@ -118,16 +92,6 @@ export async function getDataStreamDetails({ } } -async function getDataStreamCreatedOn(esClient: ElasticsearchClient, dataStream: string) { - const indexSettings = await dataStreamService.getDataStreamIndexSettings(esClient, dataStream); - - const indexesList = Object.values(indexSettings); - - return indexesList - .map((index) => Number(index.settings?.index?.creation_date)) - .sort((a, b) => a - b)[0]; -} - type TermAggregation = Record; const MAX_HOSTS = MAX_HOSTS_METRIC_VALUE + 1; // Adding 1 so that we can show e.g. '50+' diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_datastream_settings/get_datastream_created_on.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_datastream_settings/get_datastream_created_on.ts new file mode 100644 index 0000000000000..d9d22d326bffb --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_datastream_settings/get_datastream_created_on.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { dataStreamService } from '../../../services'; + +export async function getDataStreamCreatedOn(esClient: ElasticsearchClient, dataStream: string) { + const indexSettings = await dataStreamService.getDataStreamIndexSettings(esClient, dataStream); + + const indexesList = Object.values(indexSettings); + + return indexesList + .map((index) => Number(index.settings?.index?.creation_date)) + .sort((a, b) => a - b)[0]; +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_datastream_settings/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_datastream_settings/index.ts new file mode 100644 index 0000000000000..8044c8a5abf6a --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_datastream_settings/index.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { datasetQualityPrivileges, dataStreamService } from '../../../services'; +import { DataStreamSettings } from '../../../../common/api_types'; +import { getDataStreamCreatedOn } from './get_datastream_created_on'; + +export async function getDataStreamSettings({ + esClient, + dataStream, +}: { + esClient: ElasticsearchClient; + dataStream: string; +}): Promise { + const [createdOn, [dataStreamInfo], datasetUserPrivileges] = await Promise.all([ + getDataStreamCreatedOn(esClient, dataStream), + dataStreamService.getMatchingDataStreams(esClient, dataStream), + datasetQualityPrivileges.getDatasetPrivileges(esClient, dataStream), + ]); + + const integration = dataStreamInfo?._meta?.package?.name; + const lastBackingIndex = dataStreamInfo?.indices?.at(-1); + const indexTemplate = dataStreamInfo?.template; + + return { + createdOn, + integration, + datasetUserPrivileges, + lastBackingIndexName: lastBackingIndex?.index_name, + indexTemplate, + }; +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts index 3a60f0b9a8ef3..c8ed3296908e3 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts @@ -18,11 +18,12 @@ import { DataStreamDocsStat, UpdateFieldLimitResponse, DataStreamRolloverResponse, + CheckAndLoadIntegrationResponse, } from '../../../common/api_types'; import { rangeRt, typeRt, typesRt } from '../../types/default_api_types'; import { createDatasetQualityServerRoute } from '../create_datasets_quality_server_route'; import { datasetQualityPrivileges } from '../../services'; -import { getDataStreamDetails, getDataStreamSettings } from './get_data_stream_details'; +import { getDataStreamDetails } from './get_data_stream_details'; import { getDataStreams } from './get_data_streams'; import { getDataStreamsStats } from './get_data_streams_stats'; import { getDegradedDocsPaginated } from './get_degraded_docs'; @@ -34,6 +35,8 @@ import { getDataStreamsMeteringStats } from './get_data_streams_metering_stats'; import { getAggregatedDatasetPaginatedResults } from './get_dataset_aggregated_paginated_results'; import { updateFieldLimit } from './update_field_limit'; import { createDatasetQualityESClient } from '../../utils'; +import { getDataStreamSettings } from './get_datastream_settings'; +import { checkAndLoadIntegration } from './check_and_load_integration'; const statsRoute = createDatasetQualityServerRoute({ endpoint: 'GET /internal/dataset_quality/data_streams/stats', @@ -293,6 +296,38 @@ const dataStreamSettingsRoute = createDatasetQualityServerRoute({ }, }); +const checkAndLoadIntegrationRoute = createDatasetQualityServerRoute({ + endpoint: 'GET /internal/dataset_quality/data_streams/{dataStream}/integration/check', + params: t.type({ + path: t.type({ + dataStream: t.string, + }), + }), + options: { + tags: [], + }, + async handler(resources): Promise { + const { context, params, plugins, logger } = resources; + const { dataStream } = params.path; + const coreContext = await context.core; + + // Query dataStreams as the current user as the Kibana internal user may not have all the required permissions + const esClient = coreContext.elasticsearch.client.asCurrentUser; + + const fleetPluginStart = await plugins.fleet.start(); + const packageClient = fleetPluginStart.packageService.asInternalUser; + + const integration = await checkAndLoadIntegration({ + esClient, + packageClient, + logger, + dataStream, + }); + + return integration; + }, +}); + const dataStreamDetailsRoute = createDatasetQualityServerRoute({ endpoint: 'GET /internal/dataset_quality/data_streams/{dataStream}/details', params: t.type({ @@ -418,6 +453,7 @@ export const dataStreamsRouteRepository = { ...degradedFieldValuesRoute, ...dataStreamDetailsRoute, ...dataStreamSettingsRoute, + ...checkAndLoadIntegrationRoute, ...analyzeDegradedFieldRoute, ...updateFieldLimitRoute, ...rolloverDataStream, diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/index.ts index f377ea1e2642c..d00831a58aa7f 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/index.ts @@ -11,7 +11,7 @@ import { createDatasetQualityESClient } from '../../../utils'; import { updateComponentTemplate } from './update_component_template'; import { updateLastBackingIndexSettings } from './update_settings_last_backing_index'; import { UpdateFieldLimitResponse } from '../../../../common/api_types'; -import { getDataStreamSettings } from '../get_data_stream_details'; +import { getDataStreamSettings } from '../get_datastream_settings'; export async function updateFieldLimit({ esClient, diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/integrations/get_integrations.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/integrations/get_integrations.ts index 10e89db800a20..b8bf8719783ce 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/integrations/get_integrations.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/integrations/get_integrations.ts @@ -8,9 +8,29 @@ import { Logger } from '@kbn/core/server'; import { PackageClient } from '@kbn/fleet-plugin/server'; import { PackageNotFoundError } from '@kbn/fleet-plugin/server/errors'; -import { PackageListItem, RegistryDataStream } from '@kbn/fleet-plugin/common'; +import { PackageInfo, RegistryDataStream } from '@kbn/fleet-plugin/common'; import { IntegrationType } from '../../../common/api_types'; +export async function getIntegration({ + packageClient, + logger, + packageName, +}: { + packageClient: PackageClient; + logger: Logger; + packageName: string; +}): Promise { + const latestPackage = await packageClient.getLatestPackageInfo(packageName); + + return { + name: latestPackage.name, + title: latestPackage.title, + version: latestPackage.version, + icons: latestPackage.icons, + datasets: await getDatasets({ packageClient, logger, pkg: latestPackage }), + }; +} + export async function getIntegrations(options: { packageClient: PackageClient; logger: Logger; @@ -36,7 +56,7 @@ export async function getIntegrations(options: { const getDatasets = async (options: { packageClient: PackageClient; logger: Logger; - pkg: PackageListItem; + pkg: Pick; }) => { const { packageClient, logger, pkg } = options; diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts b/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts index 8a78b4163da95..cb6c963435f1e 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts @@ -13,6 +13,8 @@ import { FieldCapsRequest, FieldCapsResponse, Indices, + IndicesGetIndexTemplateRequest, + IndicesGetIndexTemplateResponse, IndicesGetMappingResponse, IndicesGetSettingsResponse, IndicesPutSettingsRequest, @@ -63,5 +65,10 @@ export function createDatasetQualityESClient(esClient: ElasticsearchClient) { rollover(params: { alias: string }): Promise { return esClient.indices.rollover(params); }, + indexTemplates( + params: IndicesGetIndexTemplateRequest + ): Promise { + return esClient.indices.getIndexTemplate(params); + }, }; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/tsconfig.json b/x-pack/plugins/observability_solution/dataset_quality/tsconfig.json index 8576b08bd9479..57b159cdbd295 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/tsconfig.json +++ b/x-pack/plugins/observability_solution/dataset_quality/tsconfig.json @@ -61,7 +61,8 @@ "@kbn/rison", "@kbn/task-manager-plugin", "@kbn/core-application-browser", - "@kbn/field-utils" + "@kbn/field-utils", + "@kbn/logging" ], "exclude": [ "target/**/*" diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index 95578949aad18..42be46e7ef534 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -15310,7 +15310,6 @@ "xpack.datasetQuality.details.fetchDataStreamDetailsFailed": "Nous n'avons pas pu obtenir les détails de votre flux de données.", "xpack.datasetQuality.details.fetchDataStreamSettingsFailed": "Les paramètres du flux de données n'ont pas pu être chargés.", "xpack.datasetQuality.details.fetchIntegrationDashboardsFailed": "Nous n'avons pas pu obtenir vos tableaux de bord d'intégration.", - "xpack.datasetQuality.details.fetchIntegrationsFailed": "Nous n'avons pas pu obtenir les informations relatives à l'intégration de {integrationName}.", "xpack.datasetQuality.details.indexTemplateActionText": "Modèle d'index", "xpack.datasetQuality.details.integrationActionsText": "Actions d'intégration", "xpack.datasetQuality.details.integrationnameText": "Intégration", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 4d6d670fb2a32..c64fa755a06c7 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -15289,7 +15289,6 @@ "xpack.datasetQuality.details.fetchDataStreamDetailsFailed": "データストリーム詳細を取得できませんでした。", "xpack.datasetQuality.details.fetchDataStreamSettingsFailed": "データストリーム設定を読み込めませんでした。", "xpack.datasetQuality.details.fetchIntegrationDashboardsFailed": "統合ダッシュボードを取得できませんでした。", - "xpack.datasetQuality.details.fetchIntegrationsFailed": "{integrationName}統合情報を取得できませんでした。", "xpack.datasetQuality.details.indexTemplateActionText": "インデックステンプレート", "xpack.datasetQuality.details.integrationActionsText": "統合アクション", "xpack.datasetQuality.details.integrationnameText": "統合", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 21db66f335af5..fb866121eb897 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -14983,7 +14983,6 @@ "xpack.datasetQuality.details.fetchDataStreamDetailsFailed": "无法获取数据流详情。", "xpack.datasetQuality.details.fetchDataStreamSettingsFailed": "无法加载数据流设置。", "xpack.datasetQuality.details.fetchIntegrationDashboardsFailed": "无法获取集成仪表板。", - "xpack.datasetQuality.details.fetchIntegrationsFailed": "无法获取 {integrationName} 集成信息。", "xpack.datasetQuality.details.indexTemplateActionText": "索引模板", "xpack.datasetQuality.details.integrationActionsText": "集成操作", "xpack.datasetQuality.details.integrationnameText": "集成", diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/check_and_load_integration.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/check_and_load_integration.ts new file mode 100644 index 0000000000000..801b5cde58833 --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/check_and_load_integration.ts @@ -0,0 +1,175 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { log, timerange } from '@kbn/apm-synthtrace-client'; +import expect from '@kbn/expect'; + +import { LogsSynthtraceEsClient } from '@kbn/apm-synthtrace'; +import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context'; +import { RoleCredentials, SupertestWithRoleScopeType } from '../../../services'; + +export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { + const samlAuth = getService('samlAuth'); + const roleScopedSupertest = getService('roleScopedSupertest'); + const synthtrace = getService('synthtrace'); + const packageApi = getService('packageApi'); + const start = '2024-11-04T11:00:00.000Z'; + const end = '2024-11-04T11:01:00.000Z'; + const type = 'logs'; + const dataset = 'synth'; + const nginxDataset = 'nginx.access'; + const apmDataset = 'apm.app.test'; + const namespace = 'default'; + const serviceName = 'my-service'; + const hostName = 'synth-host'; + const dataStreamName = `${type}-${dataset}-${namespace}`; + const nginxDataStreamName = `${type}-${nginxDataset}-${namespace}`; + const apmAppDataStreamName = `${type}-${apmDataset}-${namespace}`; + + const pkg = 'nginx'; + + async function callApiAs({ + roleScopedSupertestWithCookieCredentials, + apiParams: { dataStream }, + }: { + roleScopedSupertestWithCookieCredentials: SupertestWithRoleScopeType; + apiParams: { + dataStream: string; + }; + }) { + return roleScopedSupertestWithCookieCredentials.get( + `/internal/dataset_quality/data_streams/${dataStream}/integration/check` + ); + } + + describe('Check and load integrations', function () { + let adminRoleAuthc: RoleCredentials; + let supertestAdminWithCookieCredentials: SupertestWithRoleScopeType; + let synthtraceLogsEsClient: LogsSynthtraceEsClient; + + before(async () => { + synthtraceLogsEsClient = await synthtrace.createLogsSynthtraceEsClient(); + adminRoleAuthc = await samlAuth.createM2mApiKeyWithRoleScope('admin'); + supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope( + 'admin', + { + useCookieHeader: true, + withInternalHeaders: true, + } + ); + // Install nginx package + await packageApi.installPackage({ + roleAuthc: adminRoleAuthc, + pkg, + }); + + await synthtraceLogsEsClient.index([ + // Ingest degraded data in Nginx data stream + timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(nginxDataset) + .namespace(namespace) + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, + }) + ), + // ingest data in apm app data stream + timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(apmDataset) + .namespace(namespace) + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, + }) + ), + + // Ingest data in regular datastream which is not an integration + timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(dataset) + .namespace(namespace) + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, + }) + ), + ]); + }); + + after(async () => { + await synthtraceLogsEsClient.clean(); + await packageApi.uninstallPackage({ + roleAuthc: adminRoleAuthc, + pkg, + }); + await samlAuth.invalidateM2mApiKeyWithRoleScope(adminRoleAuthc); + }); + + describe('returns integration status', () => { + it('returns integration as false for regular data stream', async () => { + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: dataStreamName, + }, + }); + + expect(resp.body.isIntegration).to.be(false); + expect(resp.body.areAssetsAvailable).to.be(false); + }); + + it('returns integration as true for nginx data stream as we installed the integration', async () => { + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: nginxDataStreamName, + }, + }); + + expect(resp.body.isIntegration).to.be(true); + expect(resp.body.areAssetsAvailable).to.be(true); + expect(resp.body.integration.name).to.be(pkg); + expect(resp.body.integration.datasets[nginxDataset]).to.be.a('string'); + }); + + it('returns integration as false but assets are available for apm.app data stream as its preinstalled', async () => { + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + dataStream: apmAppDataStreamName, + }, + }); + + expect(resp.body.isIntegration).to.be(false); + expect(resp.body.areAssetsAvailable).to.be(true); + }); + }); + }); +} diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts index 0481e882aee6e..17ce93321b511 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts @@ -14,6 +14,7 @@ export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) loadTestFile(require.resolve('./data_stream_settings')); loadTestFile(require.resolve('./data_stream_rollover')); loadTestFile(require.resolve('./update_field_limit')); + loadTestFile(require.resolve('./check_and_load_integration')); loadTestFile(require.resolve('./data_stream_total_docs')); loadTestFile(require.resolve('./degraded_docs')); loadTestFile(require.resolve('./degraded_fields')); diff --git a/x-pack/test/functional/apps/dataset_quality/dataset_quality_details.ts b/x-pack/test/functional/apps/dataset_quality/dataset_quality_details.ts index 01dbbe1a0fc80..28a7cc9a9963d 100644 --- a/x-pack/test/functional/apps/dataset_quality/dataset_quality_details.ts +++ b/x-pack/test/functional/apps/dataset_quality/dataset_quality_details.ts @@ -250,7 +250,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid ); }); - it('Should navigate to integration overview page on clicking integration overview action', async () => { + it('should navigate to integration overview page on clicking integration overview action', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: bitbucketAuditDataStreamName, }); @@ -359,7 +359,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid ); }); - it('should show the degraded fields table with data when present', async () => { + it('should show the degraded fields table with data and spark plots when present', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDataStreamName, }); @@ -372,15 +372,6 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); expect(rows.length).to.eql(3); - }); - - it('should display Spark Plot for every row of degraded fields', async () => { - await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDataStreamName, - }); - - const rows = - await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); const sparkPlots = await testSubjects.findAll( PageObjects.datasetQuality.testSubjectSelectors.datasetQualitySparkPlot diff --git a/x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts b/x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts index f5d695b8f165c..19a2001603925 100644 --- a/x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts +++ b/x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts @@ -42,13 +42,16 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid const customComponentTemplateName = 'logs-synth@mappings'; const nginxAccessDatasetName = 'nginx.access'; - const customComponentTemplateNameNginx = 'logs-nginx.access@custom'; + const customComponentTemplateNameNginx = `logs-${nginxAccessDatasetName}@custom`; const nginxAccessDataStreamName = `${type}-${nginxAccessDatasetName}-${defaultNamespace}`; const nginxPkg = { name: 'nginx', version: '1.23.0', }; + const apmAppDatasetName = 'apm.app.tug'; + const apmAppDataStreamName = `${type}-${apmAppDatasetName}-${defaultNamespace}`; + describe('Degraded fields flyout', () => { describe('degraded field flyout open-close', () => { before(async () => { @@ -131,7 +134,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid // Install Nginx Integration and ingest logs for it await PageObjects.observabilityLogsExplorer.installPackage(nginxPkg); - // Create custom component template to avoid issues with LogsDB + // Create custom component template for Nginx to avoid issues with LogsDB await synthtrace.createComponentTemplate( customComponentTemplateNameNginx, logsNginxMappings(nginxAccessDatasetName) @@ -184,6 +187,29 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid .timestamp(timestamp) ); }), + // Ingest Degraded Logs with 26 fields in Apm DataSet + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(apmAppDatasetName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], + }) + .timestamp(timestamp) + ); + }), ]); // Set Limit of 25 @@ -199,6 +225,11 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid 'mapping.total_fields.limit': 42, }); + // Set Limit of 26 + await PageObjects.datasetQuality.setDataStreamSettings(apmAppDataStreamName, { + 'mapping.total_fields.limit': 25, + }); + await synthtrace.index([ // Ingest Degraded Logs with 26 field timerange(moment(to).subtract(count, 'minute'), moment(to)) @@ -248,11 +279,36 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid .timestamp(timestamp) ); }), + // Ingest Degraded Logs with 27 fields in Apm APP DataSet + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(apmAppDatasetName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], + 'cloud.project.id': generateShortId(), + }) + .timestamp(timestamp) + ); + }), ]); // Rollover Datastream to reset the limit to default which is 1000 await PageObjects.datasetQuality.rolloverDataStream(degradedDatasetWithLimitDataStreamName); await PageObjects.datasetQuality.rolloverDataStream(nginxAccessDataStreamName); + await PageObjects.datasetQuality.rolloverDataStream(apmAppDataStreamName); // Set Limit of 26 await PageObjects.datasetQuality.setDataStreamSettings( @@ -274,6 +330,16 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid } ); + // Set Limit of 27 + await PageObjects.datasetQuality.setDataStreamSettings( + PageObjects.datasetQuality.generateBackingIndexNameWithoutVersion({ + dataset: apmAppDatasetName, + }) + '-000002', + { + 'mapping.total_fields.limit': 27, + } + ); + await synthtrace.index([ // Ingest Degraded Logs with 26 field timerange(moment(to).subtract(count, 'minute'), moment(to)) @@ -323,6 +389,30 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid .timestamp(timestamp) ); }), + // Ingest Degraded Logs with 27 fields in Apm APP DataSet + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(apmAppDatasetName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], + 'cloud.project.id': generateShortId(), + }) + .timestamp(timestamp) + ); + }), ]); }); @@ -568,6 +658,8 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid expandedDegradedField: 'test_field', }); + await PageObjects.datasetQuality.waitUntilPossibleMitigationsLoaded(); + // Possible Mitigation Section should exist await testSubjects.existOrFail( 'datasetQualityDetailsDegradedFieldFlyoutPossibleMitigationTitle' @@ -703,6 +795,36 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid expect(linkURL?.endsWith('mapping-settings-limit.html')).to.be(true); }); + it('should display increase field limit as a possible mitigation for special packages like apm app', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: apmAppDataStreamName, + expandedDegradedField: 'cloud.project', + }); + + // Field Limit Mitigation Section should exist + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutFieldLimitMitigationAccordion' + ); + + // Should display the panel to increase field limit + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutIncreaseFieldLimitPanel' + ); + + // Should display official online documentation link + await testSubjects.existOrFail( + 'datasetQualityManualMitigationsPipelineOfficialDocumentationLink' + ); + + const linkButton = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineOfficialDocumentationLink' + ); + + const linkURL = await linkButton.getAttribute('href'); + + expect(linkURL?.endsWith('mapping-settings-limit.html')).to.be(true); + }); + it('should display increase field limit as a possible mitigation for non integration', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, @@ -764,10 +886,10 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid expect(newFieldLimit).to.be(newLimit); // Should display the apply button - await testSubjects.existOrFail('datasetQualityIncreaseFieldMappingLimitButtonButton'); + await testSubjects.existOrFail('datasetQualityIncreaseFieldMappingLimitButton'); const applyButton = await testSubjects.find( - 'datasetQualityIncreaseFieldMappingLimitButtonButton' + 'datasetQualityIncreaseFieldMappingLimitButton' ); const applyButtonDisabledStatus = await applyButton.getAttribute('disabled'); @@ -792,7 +914,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid ); const applyButton = await testSubjects.find( - 'datasetQualityIncreaseFieldMappingLimitButtonButton' + 'datasetQualityIncreaseFieldMappingLimitButton' ); const applyButtonDisabledStatus = await applyButton.getAttribute('disabled'); @@ -814,7 +936,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid }); const applyButton = await testSubjects.find( - 'datasetQualityIncreaseFieldMappingLimitButtonButton' + 'datasetQualityIncreaseFieldMappingLimitButton' ); await applyButton.click(); diff --git a/x-pack/test/functional/page_objects/dataset_quality.ts b/x-pack/test/functional/page_objects/dataset_quality.ts index cef881fe0797c..47a9261c906f7 100644 --- a/x-pack/test/functional/page_objects/dataset_quality.ts +++ b/x-pack/test/functional/page_objects/dataset_quality.ts @@ -205,7 +205,10 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv }, async waitUntilPossibleMitigationsLoaded() { - await find.waitForDeletedByCssSelector('.euiFlyoutBody .euiSkeletonRectangle', 20 * 1000); + await find.waitForDeletedByCssSelector( + '.euiFlyoutBody .datasetQualityDetailsFlyoutManualMitigationsLoading', + 20 * 1000 + ); }, async waitUntilDegradedFieldFlyoutLoaded() { diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_details.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_details.ts index 52d114802b524..f4f0c8172de49 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_details.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_details.ts @@ -365,7 +365,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { ); }); - it('should show the degraded fields table with data when present', async () => { + it('should show the degraded fields table with data and spark plots when present', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDataStreamName, }); @@ -378,17 +378,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); expect(rows.length).to.eql(3); - }); - - it('should display Spark Plot for every row of degraded fields', async () => { - await PageObjects.datasetQuality.navigateToDetails({ - dataStream: degradedDataStreamName, - }); - - await PageObjects.datasetQuality.waitUntilTableLoaded(); - - const rows = - await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows(); const sparkPlots = await testSubjects.findAll( PageObjects.datasetQuality.testSubjectSelectors.datasetQualitySparkPlot diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts index 154bd43b3ef7b..4b28ba5e897dd 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts @@ -44,13 +44,16 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const customComponentTemplateName = 'logs-synth@mappings'; const nginxAccessDatasetName = 'nginx.access'; - const customComponentTemplateNameNginx = 'logs-nginx.access@custom'; + const customComponentTemplateNameNginx = `logs-${nginxAccessDatasetName}@custom`; const nginxAccessDataStreamName = `${type}-${nginxAccessDatasetName}-${defaultNamespace}`; const nginxPkg = { name: 'nginx', version: '1.23.0', }; + const apmAppDatasetName = 'apm.app.tug'; + const apmAppDataStreamName = `${type}-${apmAppDatasetName}-${defaultNamespace}`; + describe('Degraded fields flyout', () => { describe('degraded field flyout open-close', () => { before(async () => { @@ -183,6 +186,29 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { .timestamp(timestamp) ); }), + // Ingest Degraded Logs with 26 fields in Apm DataSet + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(apmAppDatasetName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], + }) + .timestamp(timestamp) + ); + }), ]); // Set Limit of 25 @@ -198,6 +224,11 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { 'mapping.total_fields.limit': 42, }); + // Set Limit of 26 + await PageObjects.datasetQuality.setDataStreamSettings(apmAppDataStreamName, { + 'mapping.total_fields.limit': 25, + }); + await synthtrace.index([ // Ingest Degraded Logs with 26 field timerange(moment(to).subtract(count, 'minute'), moment(to)) @@ -247,11 +278,36 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { .timestamp(timestamp) ); }), + // Ingest Degraded Logs with 27 fields in Apm APP DataSet + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(apmAppDatasetName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], + 'cloud.project.id': generateShortId(), + }) + .timestamp(timestamp) + ); + }), ]); // Rollover Datastream to reset the limit to default which is 1000 await PageObjects.datasetQuality.rolloverDataStream(degradedDatasetWithLimitDataStreamName); await PageObjects.datasetQuality.rolloverDataStream(nginxAccessDataStreamName); + await PageObjects.datasetQuality.rolloverDataStream(apmAppDataStreamName); // Set Limit of 26 await PageObjects.datasetQuality.setDataStreamSettings( @@ -273,6 +329,16 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { } ); + // Set Limit of 27 + await PageObjects.datasetQuality.setDataStreamSettings( + PageObjects.datasetQuality.generateBackingIndexNameWithoutVersion({ + dataset: apmAppDatasetName, + }) + '-000002', + { + 'mapping.total_fields.limit': 27, + } + ); + await synthtrace.index([ // Ingest Degraded Logs with 26 field timerange(moment(to).subtract(count, 'minute'), moment(to)) @@ -322,6 +388,30 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { .timestamp(timestamp) ); }), + // Ingest Degraded Logs with 27 fields in Apm APP DataSet + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(1) + .fill(0) + .flatMap(() => + log + .create() + .dataset(apmAppDatasetName) + .message('a log message') + .logLevel(MORE_THAN_1024_CHARS) + .service(serviceName) + .namespace(defaultNamespace) + .defaults({ + 'service.name': serviceName, + 'trace.id': generateShortId(), + test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], + 'cloud.project.id': generateShortId(), + }) + .timestamp(timestamp) + ); + }), ]); await PageObjects.svlCommonPage.loginAsAdmin(); }); @@ -722,6 +812,36 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(linkURL?.endsWith('mapping-settings-limit.html')).to.be(true); }); + it('should display increase field limit as a possible mitigation for special packages like apm app', async () => { + await PageObjects.datasetQuality.navigateToDetails({ + dataStream: apmAppDataStreamName, + expandedDegradedField: 'cloud.project', + }); + + // Field Limit Mitigation Section should exist + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutFieldLimitMitigationAccordion' + ); + + // Should display the panel to increase field limit + await testSubjects.existOrFail( + 'datasetQualityDetailsDegradedFieldFlyoutIncreaseFieldLimitPanel' + ); + + // Should display official online documentation link + await testSubjects.existOrFail( + 'datasetQualityManualMitigationsPipelineOfficialDocumentationLink' + ); + + const linkButton = await testSubjects.find( + 'datasetQualityManualMitigationsPipelineOfficialDocumentationLink' + ); + + const linkURL = await linkButton.getAttribute('href'); + + expect(linkURL?.endsWith('mapping-settings-limit.html')).to.be(true); + }); + it('should display increase field limit as a possible mitigation for non integration', async () => { await PageObjects.datasetQuality.navigateToDetails({ dataStream: degradedDatasetWithLimitDataStreamName, @@ -787,10 +907,10 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(newFieldLimit).to.be(newLimit); // Should display the apply button - await testSubjects.existOrFail('datasetQualityIncreaseFieldMappingLimitButtonButton'); + await testSubjects.existOrFail('datasetQualityIncreaseFieldMappingLimitButton'); const applyButton = await testSubjects.find( - 'datasetQualityIncreaseFieldMappingLimitButtonButton' + 'datasetQualityIncreaseFieldMappingLimitButton' ); const applyButtonDisabledStatus = await applyButton.getAttribute('disabled'); @@ -817,7 +937,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { ); const applyButton = await testSubjects.find( - 'datasetQualityIncreaseFieldMappingLimitButtonButton' + 'datasetQualityIncreaseFieldMappingLimitButton' ); const applyButtonDisabledStatus = await applyButton.getAttribute('disabled'); @@ -844,7 +964,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await retry.tryForTime(5000, async () => { const applyButton = await testSubjects.find( - 'datasetQualityIncreaseFieldMappingLimitButtonButton' + 'datasetQualityIncreaseFieldMappingLimitButton' ); await applyButton.click(); @@ -864,7 +984,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.header.waitUntilLoadingHasFinished(); const applyButton = await testSubjects.find( - 'datasetQualityIncreaseFieldMappingLimitButtonButton' + 'datasetQualityIncreaseFieldMappingLimitButton' ); await applyButton.click();