From 529a4e3b196309c5b181703a9f0bdd94c7baf758 Mon Sep 17 00:00:00 2001 From: Ido Cohen <90558359+CohenIdo@users.noreply.github.com> Date: Wed, 18 Dec 2024 17:20:53 +0200 Subject: [PATCH] Deprecate Cloud Defend billing logic --- config/serverless.security.yml | 2 +- .../cloud_security/cloud_security_metering.ts | 20 +- .../cloud_security_metering_task.test.ts | 180 +----------------- .../server/cloud_security/constants.ts | 7 - .../defend_for_containers_metering.ts | 148 -------------- .../server/cloud_security/types.ts | 4 +- .../tsconfig.json | 1 - .../cloud_security_metering.ts | 76 +------- 8 files changed, 10 insertions(+), 428 deletions(-) delete mode 100644 x-pack/solutions/security/plugins/security_solution_serverless/server/cloud_security/defend_for_containers_metering.ts diff --git a/config/serverless.security.yml b/config/serverless.security.yml index 47a67c293565a..0e3a0c92f8546 100644 --- a/config/serverless.security.yml +++ b/config/serverless.security.yml @@ -106,7 +106,7 @@ xpack.fleet.internal.registry.excludePackages: [ 'dga', # Unsupported in serverless - 'cloud-defend', + 'cloud_defend', ] # fleet_server package installed to publish agent metrics xpack.fleet.packages: diff --git a/x-pack/solutions/security/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering.ts b/x-pack/solutions/security/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering.ts index 1a5fb8e64a265..08d5b0d088aaa 100644 --- a/x-pack/solutions/security/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering.ts +++ b/x-pack/solutions/security/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering.ts @@ -7,11 +7,10 @@ import type { Logger } from '@kbn/core/server'; import { ProductLine } from '../../common/product'; import { getCloudSecurityUsageRecord } from './cloud_security_metering_task'; -import { CLOUD_DEFEND, CNVM, CSPM, KSPM } from './constants'; +import { CNVM, CSPM, KSPM } from './constants'; import type { CloudSecuritySolutions } from './types'; import type { MeteringCallBackResponse, MeteringCallbackInput, Tier, UsageRecord } from '../types'; import type { ServerlessSecurityConfig } from '../config'; -import { getCloudDefendUsageRecords } from './defend_for_containers_metering'; export const cloudSecurityMetringCallback = async ({ esClient, @@ -35,24 +34,11 @@ export const cloudSecurityMetringCallback = async ({ const tier: Tier = getCloudProductTier(config, logger); try { - const cloudSecuritySolutions: CloudSecuritySolutions[] = [CSPM, KSPM, CNVM, CLOUD_DEFEND]; + const cloudSecuritySolutions: CloudSecuritySolutions[] = [CSPM, KSPM, CNVM]; const promiseResults = await Promise.allSettled( cloudSecuritySolutions.map((cloudSecuritySolution) => { - if (cloudSecuritySolution !== CLOUD_DEFEND) { - return getCloudSecurityUsageRecord({ - esClient, - projectId, - logger, - taskId, - lastSuccessfulReport, - cloudSecuritySolution, - tier, - }); - } - - // since lastSuccessfulReport is not used by getCloudSecurityUsageRecord, we want to verify if it is used by getCloudDefendUsageRecords before getCloudSecurityUsageRecord. - return getCloudDefendUsageRecords({ + return getCloudSecurityUsageRecord({ esClient, projectId, logger, diff --git a/x-pack/solutions/security/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task.test.ts b/x-pack/solutions/security/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task.test.ts index 6d55e52469283..423b26b5f7e98 100644 --- a/x-pack/solutions/security/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task.test.ts +++ b/x-pack/solutions/security/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task.test.ts @@ -16,15 +16,7 @@ import { import type { ServerlessSecurityConfig } from '../config'; import type { ProductTier } from '../../common/product'; -import { - CLOUD_SECURITY_TASK_TYPE, - CSPM, - KSPM, - CNVM, - CLOUD_DEFEND, - BILLABLE_ASSETS_CONFIG, -} from './constants'; -import { getCloudDefendUsageRecords } from './defend_for_containers_metering'; +import { CLOUD_SECURITY_TASK_TYPE, CSPM, KSPM, CNVM, BILLABLE_ASSETS_CONFIG } from './constants'; const mockEsClient = elasticsearchServiceMock.createStart().client.asInternalUser; const logger: ReturnType = loggingSystemMock.createLogger(); @@ -336,173 +328,3 @@ describe('should return the relevant product tier', () => { expect(tier).toBe('none'); }); }); - -describe('cloud defend metering', () => { - it('should return usageRecords with correct values', async () => { - const cloudSecuritySolution = 'cloud_defend'; - const agentId1 = chance.guid(); - const eventIngestedStr = '2024-05-28T12:10:51Z'; - const eventIngestedTimestamp = new Date(eventIngestedStr); - - // @ts-ignore - mockEsClient.search.mockResolvedValueOnce({ - hits: { - hits: [ - { - _id: 'someRecord', - _index: 'mockIndex', - _source: { - 'cloud_defend.block_action_enabled': true, - 'agent.id': agentId1, - event: { - ingested: eventIngestedStr, - }, - }, - }, - ], - }, - }); - - const projectId = chance.guid(); - const taskId = chance.guid(); - - const tier = 'essentials' as ProductTier; - - const result = await getCloudDefendUsageRecords({ - esClient: mockEsClient, - projectId, - taskId, - lastSuccessfulReport: new Date(), - cloudSecuritySolution, - logger, - tier, - }); - - const roundedIngestedTimestamp = eventIngestedTimestamp; - roundedIngestedTimestamp.setMinutes(0); - roundedIngestedTimestamp.setSeconds(0); - roundedIngestedTimestamp.setMilliseconds(0); - - expect(result).toEqual([ - { - id: expect.stringContaining( - `${projectId}_${agentId1}_${roundedIngestedTimestamp.toISOString()}` - ), - usage_timestamp: eventIngestedStr, - creation_timestamp: expect.any(String), - usage: { - type: CLOUD_SECURITY_TASK_TYPE, - sub_type: CLOUD_DEFEND, - quantity: 1, - period_seconds: 3600, - }, - source: { - id: taskId, - instance_group_id: projectId, - metadata: { - tier: 'essentials', - }, - }, - }, - ]); - }); - - it('should return an empty array when Elasticsearch returns an empty response', async () => { - // @ts-ignore - mockEsClient.search.mockResolvedValueOnce({ - hits: { - hits: [], - }, - }); - const tier = 'essentials' as ProductTier; - // Call the function with mock parameters - const result = await getCloudDefendUsageRecords({ - esClient: mockEsClient, - projectId: chance.guid(), - taskId: chance.guid(), - lastSuccessfulReport: new Date(), - cloudSecuritySolution: 'cloud_defend', - logger, - tier, - }); - - // Assert that the result is an empty array - expect(result).toEqual([]); - }); - - it('should handle errors from Elasticsearch', async () => { - // Mock Elasticsearch client's search method to throw an error - mockEsClient.search.mockRejectedValueOnce(new Error('Elasticsearch query failed')); - - const tier = 'essentials' as ProductTier; - - // Call the function with mock parameters - await getCloudDefendUsageRecords({ - esClient: mockEsClient, - projectId: chance.guid(), - taskId: chance.guid(), - lastSuccessfulReport: new Date(), - cloudSecuritySolution: 'cloud_defend', - logger, - tier, - }); - - // Assert that the logger's error method was called with the correct error message - expect(logger.error).toHaveBeenCalledWith( - 'Failed to fetch cloud_defend metering data Error: Elasticsearch query failed' - ); - }); - - it('should return usageRecords when Elasticsearch returns multiple records', async () => { - // Mock Elasticsearch response with multiple records - const agentId1 = chance.guid(); - const agentId2 = chance.guid(); - const eventIngestedStr1 = '2024-05-28T12:10:51Z'; - const eventIngestedStr2 = '2024-05-28T13:10:51Z'; - - // @ts-ignore - mockEsClient.search.mockResolvedValueOnce({ - hits: { - hits: [ - { - _id: 'record1', - _index: 'mockIndex', - _source: { - 'cloud_defend.block_action_enabled': true, - 'agent.id': agentId1, - event: { - ingested: eventIngestedStr1, - }, - }, - }, - { - _id: 'record2', - _index: 'mockIndex', - _source: { - 'cloud_defend.block_action_enabled': true, - 'agent.id': agentId2, - event: { - ingested: eventIngestedStr2, - }, - }, - }, - ], - }, - }); - const tier = 'essentials' as ProductTier; - - // Call the function with mock parameters - const result = await getCloudDefendUsageRecords({ - esClient: mockEsClient, - projectId: chance.guid(), - taskId: chance.guid(), - lastSuccessfulReport: new Date(), - cloudSecuritySolution: 'cloud_defend', - logger, - tier, - }); - - // Assert that the result contains usage records for both records - expect(result).toHaveLength(2); - }); -}); diff --git a/x-pack/solutions/security/plugins/security_solution_serverless/server/cloud_security/constants.ts b/x-pack/solutions/security/plugins/security_solution_serverless/server/cloud_security/constants.ts index fa98707e5ebeb..774f7e7fd2ae7 100644 --- a/x-pack/solutions/security/plugins/security_solution_serverless/server/cloud_security/constants.ts +++ b/x-pack/solutions/security/plugins/security_solution_serverless/server/cloud_security/constants.ts @@ -12,9 +12,7 @@ import { CDR_LATEST_NATIVE_VULNERABILITIES_INDEX_PATTERN, } from '@kbn/cloud-security-posture-common'; import { CNVM_POLICY_TEMPLATE } from '@kbn/cloud-security-posture-plugin/common/constants'; -import { INTEGRATION_PACKAGE_NAME } from '@kbn/cloud-defend-plugin/common/constants'; -export const CLOUD_DEFEND_HEARTBEAT_INDEX = 'metrics-cloud_defend.heartbeat-*'; export const CLOUD_SECURITY_TASK_TYPE = 'cloud_security'; export const AGGREGATION_PRECISION_THRESHOLD = 40000; export const ASSETS_SAMPLE_GRANULARITY = '24h'; @@ -23,7 +21,6 @@ export const THRESHOLD_MINUTES = 30; export const CSPM = CSPM_POLICY_TEMPLATE; export const KSPM = KSPM_POLICY_TEMPLATE; export const CNVM = CNVM_POLICY_TEMPLATE; -export const CLOUD_DEFEND = INTEGRATION_PACKAGE_NAME; export const METERING_CONFIGS = { [CSPM]: { @@ -38,10 +35,6 @@ export const METERING_CONFIGS = { index: CDR_LATEST_NATIVE_VULNERABILITIES_INDEX_PATTERN, assets_identifier: 'cloud.instance.id', }, - [CLOUD_DEFEND]: { - index: CLOUD_DEFEND_HEARTBEAT_INDEX, - assets_identifier: 'agent.id', - }, }; // see https://github.com/elastic/security-team/issues/8970 for billable asset definition diff --git a/x-pack/solutions/security/plugins/security_solution_serverless/server/cloud_security/defend_for_containers_metering.ts b/x-pack/solutions/security/plugins/security_solution_serverless/server/cloud_security/defend_for_containers_metering.ts deleted file mode 100644 index 61240df94f9b2..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution_serverless/server/cloud_security/defend_for_containers_metering.ts +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; -import type { - AggregationsAggregate, - SearchResponse, - SortResults, -} from '@elastic/elasticsearch/lib/api/types'; -import type { Tier, UsageRecord } from '../types'; -import type { CloudSecurityMeteringCallbackInput } from './types'; -import { CLOUD_DEFEND, CLOUD_SECURITY_TASK_TYPE, CLOUD_DEFEND_HEARTBEAT_INDEX } from './constants'; - -const BATCH_SIZE = 1000; -const SAMPLE_WEIGHT_SECONDS = 3600; // 1 Hour - -export interface CloudDefendHeartbeat { - '@timestamp': string; - 'agent.id': string; - event: { - ingested: string; - }; -} - -const buildMeteringRecord = ( - agentId: string, - timestampStr: string, - taskId: string, - tier: Tier, - projectId: string -): UsageRecord => { - const timestamp = new Date(timestampStr); - timestamp.setMinutes(0); - timestamp.setSeconds(0); - timestamp.setMilliseconds(0); - const creationTimestamp = new Date(); - const usageRecord = { - id: `${projectId}_${agentId}_${timestamp.toISOString()}`, - usage_timestamp: timestampStr, - creation_timestamp: creationTimestamp.toISOString(), - usage: { - type: CLOUD_SECURITY_TASK_TYPE, - sub_type: CLOUD_DEFEND, - period_seconds: SAMPLE_WEIGHT_SECONDS, - quantity: 1, - }, - source: { - id: taskId, - instance_group_id: projectId, - metadata: { - tier, - }, - }, - }; - - return usageRecord; -}; -export const getUsageRecords = async ( - esClient: ElasticsearchClient, - searchFrom: Date, - searchAfter?: SortResults -): Promise>> => { - return esClient.search( - { - index: CLOUD_DEFEND_HEARTBEAT_INDEX, - size: BATCH_SIZE, - sort: [{ 'event.ingested': 'asc' }, { 'agent.id': 'asc' }], - search_after: searchAfter, - query: { - bool: { - must: [ - { - range: { - 'event.ingested': { - // gt: searchFrom.toISOString(), Tech debt: https://github.com/elastic/security-team/issues/9895 - gte: `now-30m`, - }, - }, - }, - { - term: { - 'cloud_defend.block_action_enabled': true, - }, - }, - ], - }, - }, - }, - { ignore: [404] } - ); -}; - -export const getCloudDefendUsageRecords = async ({ - esClient, - projectId, - taskId, - lastSuccessfulReport, - cloudSecuritySolution, - tier, - logger, -}: CloudSecurityMeteringCallbackInput): Promise => { - try { - let allRecords: UsageRecord[] = []; - let searchAfter: SortResults | undefined; - let fetchMore = true; - - while (fetchMore) { - const usageRecords = await getUsageRecords(esClient, lastSuccessfulReport, searchAfter); - - if (!usageRecords?.hits?.hits?.length) { - break; - } - - const records = usageRecords.hits.hits.reduce((acc, { _source }) => { - if (!_source) { - return acc; - } - - const { event } = _source; - const record = buildMeteringRecord( - _source['agent.id'], - event.ingested, - taskId, - tier, - projectId - ); - - return [...acc, record]; - }, [] as UsageRecord[]); - - allRecords = [...allRecords, ...records]; - - if (usageRecords.hits.hits.length < BATCH_SIZE) { - fetchMore = false; - } else { - searchAfter = usageRecords.hits.hits[usageRecords.hits.hits.length - 1].sort; - } - } - - return allRecords; - } catch (err) { - logger.error(`Failed to fetch ${cloudSecuritySolution} metering data ${err}`); - } -}; diff --git a/x-pack/solutions/security/plugins/security_solution_serverless/server/cloud_security/types.ts b/x-pack/solutions/security/plugins/security_solution_serverless/server/cloud_security/types.ts index 0ca9a7b5b943a..c8149e75b720a 100644 --- a/x-pack/solutions/security/plugins/security_solution_serverless/server/cloud_security/types.ts +++ b/x-pack/solutions/security/plugins/security_solution_serverless/server/cloud_security/types.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { CSPM, KSPM, CNVM, CLOUD_DEFEND } from './constants'; +import type { CSPM, KSPM, CNVM } from './constants'; import type { MeteringCallbackInput, Tier } from '../types'; export interface CloudDefendAssetCountAggregation { @@ -39,7 +39,7 @@ export interface MinTimestamp { value_as_string: string; } -export type CloudSecuritySolutions = typeof CSPM | typeof KSPM | typeof CNVM | typeof CLOUD_DEFEND; +export type CloudSecuritySolutions = typeof CSPM | typeof KSPM | typeof CNVM; export interface CloudSecurityMeteringCallbackInput extends Omit { diff --git a/x-pack/solutions/security/plugins/security_solution_serverless/tsconfig.json b/x-pack/solutions/security/plugins/security_solution_serverless/tsconfig.json index 7b9a77199aeec..6b082dea68d86 100644 --- a/x-pack/solutions/security/plugins/security_solution_serverless/tsconfig.json +++ b/x-pack/solutions/security/plugins/security_solution_serverless/tsconfig.json @@ -38,7 +38,6 @@ "@kbn/fleet-plugin", "@kbn/serverless-security-settings", "@kbn/core-elasticsearch-server", - "@kbn/cloud-defend-plugin", "@kbn/core-logging-server-mocks", "@kbn/stack-connectors-plugin", "@kbn/actions-plugin", diff --git a/x-pack/test_serverless/api_integration/test_suites/security/cloud_security_posture/serverless_metering/cloud_security_metering.ts b/x-pack/test_serverless/api_integration/test_suites/security/cloud_security_posture/serverless_metering/cloud_security_metering.ts index b57dace68c4da..13ac5f96347c0 100644 --- a/x-pack/test_serverless/api_integration/test_suites/security/cloud_security_posture/serverless_metering/cloud_security_metering.ts +++ b/x-pack/test_serverless/api_integration/test_suites/security/cloud_security_posture/serverless_metering/cloud_security_metering.ts @@ -9,18 +9,13 @@ import expect from '@kbn/expect'; import { CDR_LATEST_NATIVE_VULNERABILITIES_INDEX_PATTERN } from '@kbn/cloud-security-posture-common'; import { LATEST_FINDINGS_INDEX_DEFAULT_NS } from '@kbn/cloud-security-posture-plugin/common/constants'; import * as http from 'http'; -import { - createPackagePolicy, - createCloudDefendPackagePolicy, -} from '@kbn/test-suites-xpack/api_integration/apis/cloud_security_posture/helper'; +import { createPackagePolicy } from '@kbn/test-suites-xpack/api_integration/apis/cloud_security_posture/helper'; import { EsIndexDataProvider } from '@kbn/test-suites-xpack/cloud_security_posture_api/utils'; import { RoleCredentials } from '../../../../../shared/services'; -import { getMockFindings, getMockDefendForContainersHeartbeats } from './mock_data'; +import { getMockFindings } from './mock_data'; import type { FtrProviderContext } from '../../../../ftr_provider_context'; import { UsageRecord, getInterceptedRequestPayload, setupMockServer } from './mock_usage_server'; -const CLOUD_DEFEND_HEARTBEAT_INDEX_DEFAULT_NS = 'metrics-cloud_defend.heartbeat-default'; - export default function (providerContext: FtrProviderContext) { const mockUsageApiApp = setupMockServer(); const { getService } = providerContext; @@ -32,7 +27,6 @@ export default function (providerContext: FtrProviderContext) { const svlUserManager = getService('svlUserManager'); const supertestWithoutAuth = getService('supertestWithoutAuth'); const findingsIndex = new EsIndexDataProvider(es, LATEST_FINDINGS_INDEX_DEFAULT_NS); - const cloudDefinedIndex = new EsIndexDataProvider(es, CLOUD_DEFEND_HEARTBEAT_INDEX_DEFAULT_NS); const vulnerabilitiesIndex = new EsIndexDataProvider( es, CDR_LATEST_NATIVE_VULNERABILITIES_INDEX_PATTERN @@ -74,7 +68,6 @@ export default function (providerContext: FtrProviderContext) { await findingsIndex.deleteAll(); await vulnerabilitiesIndex.deleteAll(); - await cloudDefinedIndex.deleteAll(); }); afterEach(async () => { @@ -82,7 +75,6 @@ export default function (providerContext: FtrProviderContext) { await esArchiver.unload('x-pack/test/functional/es_archives/fleet/empty_fleet_server'); await findingsIndex.deleteAll(); await vulnerabilitiesIndex.deleteAll(); - await cloudDefinedIndex.deleteAll(); }); after(async () => { await svlUserManager.invalidateM2mApiKeyWithRoleScope(roleAuthc); @@ -202,43 +194,6 @@ export default function (providerContext: FtrProviderContext) { }); }); - it('Should intercept usage API request for Defend for Containers', async () => { - await createCloudDefendPackagePolicy( - supertestWithoutAuth, - agentPolicyId, - roleAuthc, - internalRequestHeader - ); - - const blockActionEnabledHeartbeats = getMockDefendForContainersHeartbeats({ - isBlockActionEnables: true, - numberOfHearbeats: 2, - }); - - const blockActionDisabledHeartbeats = getMockDefendForContainersHeartbeats({ - isBlockActionEnables: false, - numberOfHearbeats: 2, - }); - - await cloudDefinedIndex.addBulk([ - ...blockActionEnabledHeartbeats, - ...blockActionDisabledHeartbeats, - ]); - - let interceptedRequestBody: UsageRecord[] = []; - - await retry.try(async () => { - interceptedRequestBody = getInterceptedRequestPayload(); - expect(interceptedRequestBody.length).to.greaterThan(0); - if (interceptedRequestBody.length > 0) { - const usageSubTypes = interceptedRequestBody.map((record) => record.usage.sub_type); - expect(usageSubTypes).to.contain('cloud_defend'); - expect(interceptedRequestBody.length).to.be(blockActionEnabledHeartbeats.length); - expect(interceptedRequestBody[0].usage.type).to.be('cloud_security'); - } - }); - }); - it('Should intercept usage API request with all integrations usage records', async () => { // Create one package policy - it takes care forCSPM, KSMP and CNVM await createPackagePolicy( @@ -253,13 +208,6 @@ export default function (providerContext: FtrProviderContext) { internalRequestHeader ); - // Create Defend for Containers package policy - await createCloudDefendPackagePolicy( - supertestWithoutAuth, - agentPolicyId, - roleAuthc, - internalRequestHeader - ); const billableFindingsCSPM = getMockFindings({ postureType: 'cspm', isBillableAsset: true, @@ -289,16 +237,6 @@ export default function (providerContext: FtrProviderContext) { numberOfFindings: 11, }); - const blockActionEnabledHeartbeats = getMockDefendForContainersHeartbeats({ - isBlockActionEnables: true, - numberOfHearbeats: 2, - }); - - const blockActionDisabledHeartbeats = getMockDefendForContainersHeartbeats({ - isBlockActionEnables: false, - numberOfHearbeats: 2, - }); - await Promise.all([ findingsIndex.addBulk([ ...billableFindingsCSPM, @@ -307,10 +245,6 @@ export default function (providerContext: FtrProviderContext) { ...notBillableFindingsKSPM, ]), vulnerabilitiesIndex.addBulk([...billableFindingsCNVM]), - cloudDefinedIndex.addBulk([ - ...blockActionEnabledHeartbeats, - ...blockActionDisabledHeartbeats, - ]), ]); // Intercept and verify usage API request @@ -323,16 +257,12 @@ export default function (providerContext: FtrProviderContext) { expect(usageSubTypes).to.contain('cspm'); expect(usageSubTypes).to.contain('kspm'); expect(usageSubTypes).to.contain('cnvm'); - expect(usageSubTypes).to.contain('cloud_defend'); const totalUsageQuantity = interceptedRequestBody.reduce( (acc, record) => acc + record.usage.quantity, 0 ); expect(totalUsageQuantity).to.be( - billableFindingsCSPM.length + - billableFindingsKSPM.length + - billableFindingsCNVM.length + - blockActionEnabledHeartbeats.length + billableFindingsCSPM.length + billableFindingsKSPM.length + billableFindingsCNVM.length ); }); });