[APM] Migrate service_groups API tests to be deployment-agnostic (ela…

…stic#199988)

### How to test
Closes elastic#198982
Part of elastic#193245

This PR contains the changes to migrate `service_groups` test folder to
deployment-agnostic testing strategy.

### How to test


- Serverless

```
node scripts/functional_tests_server --config x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts
node scripts/functional_test_runner --config x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts --grep="APM"
```

It's recommended to be run against
[MKI](https://github.com/crespocarlos/kibana/blob/main/x-pack/test_serverless/README.md#run-tests-on-mki)

- Stateful
```
node scripts/functional_tests_server --config x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts
node scripts/functional_test_runner --config x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts --grep="APM"
```

## Checks

- [ ] (OPTIONAL, only if a test has been unskipped) Run flaky test suite
- [x] local run for serverless
- [x] local run for stateful
- [x] MKI run for serverless

x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/index.ts

@@ -26,5 +26,6 @@ export default function apmApiIntegrationTests({
     loadTestFile(require.resolve('./observability_overview'));
     loadTestFile(require.resolve('./latency'));
     loadTestFile(require.resolve('./infrastructure'));
+    loadTestFile(require.resolve('./service_groups'));
   });
 }

x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_groups/index.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context';
+
+export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) {
+  describe('service_groups', () => {
+    loadTestFile(require.resolve('./save_service_group.spec.ts'));
+    loadTestFile(
+      require.resolve('./service_group_with_overflow/service_group_with_overflow.spec.ts')
+    );
+    loadTestFile(require.resolve('./service_group_count/service_group_count.spec.ts'));
+  });
+}

x-pack/test/apm_api_integration/tests/service_groups/save_service_group.spec.ts → x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_groups/save_service_group.spec.ts

@@ -5,20 +5,19 @@
  * 2.0.
  */
 import expect from '@kbn/expect';
-import { ApmApiError } from '../../common/apm_api_supertest';
-import { FtrProviderContext } from '../../common/ftr_provider_context';
-import { expectToReject } from '../../common/utils/expect_to_reject';
+import { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context';
+import { ApmApiError } from '../../../../../../apm_api_integration/common/apm_api_supertest';
+import { expectToReject } from '../../../../../../apm_api_integration/common/utils/expect_to_reject';
 import {
   createServiceGroupApi,
   deleteAllServiceGroups,
   getServiceGroupsApi,
 } from './service_groups_api_methods';
 
-export default function ApiTest({ getService }: FtrProviderContext) {
-  const registry = getService('registry');
-  const apmApiClient = getService('apmApiClient');
+export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) {
+  const apmApiClient = getService('apmApi');
 
-  registry.when('Service group create', { config: 'basic', archives: [] }, () => {
+  describe('Service group create', () => {
     afterEach(async () => {
       await deleteAllServiceGroups(apmApiClient);
     });

x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_groups/service_group_count/generate_data.ts

@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { apm, timerange } from '@kbn/apm-synthtrace-client';
+import type { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace';
+
+export async function generateData({
+  apmSynthtraceEsClient,
+  start,
+  end,
+}: {
+  apmSynthtraceEsClient: ApmSynthtraceEsClient;
+  start: number;
+  end: number;
+}) {
+  const synthServices = [
+    apm
+      .service({ name: 'synth-go', environment: 'testing', agentName: 'go' })
+      .instance('instance-1'),
+    apm
+      .service({ name: 'synth-java', environment: 'testing', agentName: 'java' })
+      .instance('instance-2'),
+    apm
+      .service({ name: 'opbeans-node', environment: 'testing', agentName: 'nodejs' })
+      .instance('instance-3'),
+  ];
+
+  await apmSynthtraceEsClient.index(
+    synthServices.map((service) =>
+      timerange(start, end)
+        .interval('5m')
+        .rate(1)
+        .generator((timestamp) =>
+          service
+            .transaction({
+              transactionName: 'GET /api/product/list',
+              transactionType: 'request',
+            })
+            .duration(2000)
+            .timestamp(timestamp)
+            .children(
+              service
+                .span({
+                  spanName: '/_search',
+                  spanType: 'db',
+                  spanSubtype: 'elasticsearch',
+                })
+                .destination('elasticsearch')
+                .duration(100)
+                .success()
+                .timestamp(timestamp),
+              service
+                .span({
+                  spanName: '/_search',
+                  spanType: 'db',
+                  spanSubtype: 'elasticsearch',
+                })
+                .destination('elasticsearch')
+                .duration(300)
+                .success()
+                .timestamp(timestamp)
+            )
+            .errors(
+              service.error({ message: 'error 1', type: 'foo' }).timestamp(timestamp),
+              service.error({ message: 'error 2', type: 'foo' }).timestamp(timestamp),
+              service.error({ message: 'error 3', type: 'bar' }).timestamp(timestamp)
+            )
+        )
+    )
+  );
+}

x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_groups/service_group_count/service_group_count.spec.ts

@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import expect from '@kbn/expect';
+import { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace';
+import { DeploymentAgnosticFtrProviderContext } from '../../../../../ftr_provider_context';
+import {
+  createServiceGroupApi,
+  deleteAllServiceGroups,
+  getServiceGroupCounts,
+} from '../service_groups_api_methods';
+import { generateData } from './generate_data';
+
+export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) {
+  const synthtrace = getService('synthtrace');
+  const apmApiClient = getService('apmApi');
+  const start = Date.now() - 24 * 60 * 60 * 1000;
+  const end = Date.now();
+
+  describe('Service group counts', () => {
+    let synthbeansServiceGroupId: string;
+    let opbeansServiceGroupId: string;
+    let apmSynthtraceEsClient: ApmSynthtraceEsClient;
+
+    before(async () => {
+      apmSynthtraceEsClient = await synthtrace.createApmSynthtraceEsClient();
+
+      const [, { body: synthbeansServiceGroup }, { body: opbeansServiceGroup }] = await Promise.all(
+        [
+          generateData({ start, end, apmSynthtraceEsClient }),
+          createServiceGroupApi({
+            apmApiClient,
+            groupName: 'synthbeans',
+            kuery: 'service.name: synth*',
+          }),
+          createServiceGroupApi({
+            apmApiClient,
+            groupName: 'opbeans',
+            kuery: 'service.name: opbeans*',
+          }),
+        ]
+      );
+      synthbeansServiceGroupId = synthbeansServiceGroup.id;
+      opbeansServiceGroupId = opbeansServiceGroup.id;
+    });
+
+    after(async () => {
+      await deleteAllServiceGroups(apmApiClient);
+      await apmSynthtraceEsClient.clean();
+    });
+
+    it('returns the correct number of services', async () => {
+      const response = await getServiceGroupCounts(apmApiClient);
+      expect(response.status).to.be(200);
+      expect(Object.keys(response.body).length).to.be(2);
+      expect(response.body[synthbeansServiceGroupId]).to.have.property('services', 2);
+      expect(response.body[opbeansServiceGroupId]).to.have.property('services', 1);
+    });
+  });
+}

x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_groups/service_group_with_overflow/service_group_with_overflow.spec.ts

@@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import expect from '@kbn/expect';
+import { ValuesType } from 'utility-types';
+import { ENVIRONMENT_ALL } from '@kbn/apm-plugin/common/environment_filter_values';
+import { ApmDocumentType } from '@kbn/apm-plugin/common/document_type';
+import { RollupInterval } from '@kbn/apm-plugin/common/rollup';
+import { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace';
+import { DeploymentAgnosticFtrProviderContext } from '../../../../../ftr_provider_context';
+import { createServiceGroupApi, deleteAllServiceGroups } from '../service_groups_api_methods';
+import { createServiceTransactionMetricsDocs } from './es_utils';
+import { generateData } from './generate_data';
+
+export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) {
+  const apmApiClient = getService('apmApi');
+  const es = getService('es');
+  const synthtrace = getService('synthtrace');
+
+  describe('Display overflow bucket in Service Groups', () => {
+    const indexName = 'metrics-apm.service_transaction.1m-default';
+    const start = '2023-06-21T06:50:15.910Z';
+    const end = '2023-06-21T06:59:15.910Z';
+    const startTime = new Date(start).getTime() + 1000;
+    const OVERFLOW_SERVICE_NAME = '_other';
+    let serviceGroupId: string;
+    let apmSynthtraceEsClient: ApmSynthtraceEsClient;
+
+    after(async () => {
+      await deleteAllServiceGroups(apmApiClient);
+      await apmSynthtraceEsClient.clean();
+    });
+
+    before(async () => {
+      apmSynthtraceEsClient = await synthtrace.createApmSynthtraceEsClient();
+
+      await generateData({ start, end, apmSynthtraceEsClient });
+
+      const docs = [
+        createServiceTransactionMetricsDocs({
+          time: startTime,
+          service: {
+            name: OVERFLOW_SERVICE_NAME,
+          },
+          overflowCount: 13,
+        }),
+      ];
+
+      const bulkActions = docs.reduce(
+        (prev, doc) => {
+          return [...prev, { create: { _index: indexName } }, doc];
+        },
+        [] as Array<
+          | {
+              create: {
+                _index: string;
+              };
+            }
+          | ValuesType<typeof docs>
+        >
+      );
+
+      await es.bulk({
+        body: bulkActions,
+        refresh: 'wait_for',
+      });
+
+      const serviceGroup = {
+        groupName: 'overflowGroup',
+        kuery: 'service.name: synth-go or service.name: synth-java',
+      };
+      const createResponse = await createServiceGroupApi({ apmApiClient, ...serviceGroup });
+      expect(createResponse.status).to.be(200);
+      serviceGroupId = createResponse.body.id;
+    });
+
+    it('get the overflow bucket even though its not added explicitly in the Service Group', async () => {
+      const response = await apmApiClient.readUser({
+        endpoint: `GET /internal/apm/services`,
+        params: {
+          query: {
+            start,
+            end,
+            environment: ENVIRONMENT_ALL.value,
+            kuery: '',
+            serviceGroup: serviceGroupId,
+            probability: 1,
+            documentType: ApmDocumentType.ServiceTransactionMetric,
+            rollupInterval: RollupInterval.OneMinute,
+            useDurationSummary: true,
+          },
+        },
+      });
+
+      const overflowBucket = response.body.items.find(
+        (service) => service.serviceName === OVERFLOW_SERVICE_NAME
+      );
+      expect(overflowBucket?.serviceName).to.equal(OVERFLOW_SERVICE_NAME);
+    });
+  });
+}

x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/service_groups/service_groups_api_methods.ts

@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ApmApiClient } from '../../../../services/apm_api';
+
+export async function getServiceGroupsApi(apmApiClient: ApmApiClient) {
+  return apmApiClient.writeUser({
+    endpoint: 'GET /internal/apm/service-groups',
+  });
+}
+
+export async function createServiceGroupApi({
+  apmApiClient,
+  serviceGroupId,
+  groupName,
+  kuery,
+  description,
+  color,
+}: {
+  apmApiClient: ApmApiClient;
+  serviceGroupId?: string;
+  groupName: string;
+  kuery: string;
+  description?: string;
+  color?: string;
+}) {
+  const response = await apmApiClient.writeUser({
+    endpoint: 'POST /internal/apm/service-group',
+    params: {
+      query: {
+        serviceGroupId,
+      },
+      body: {
+        groupName,
+        kuery,
+        description,
+        color,
+      },
+    },
+  });
+  return response;
+}
+
+export async function getServiceGroupCounts(apmApiClient: ApmApiClient) {
+  return apmApiClient.readUser({
+    endpoint: 'GET /internal/apm/service-group/counts',
+  });
+}
+
+export async function deleteAllServiceGroups(apmApiClient: ApmApiClient) {
+  return await getServiceGroupsApi(apmApiClient).then((response) => {
+    const promises = response.body.serviceGroups.map((item) => {
+      if (item.id) {
+        return apmApiClient.writeUser({
+          endpoint: 'DELETE /internal/apm/service-group',
+          params: { query: { serviceGroupId: item.id } },
+        });
+      }
+    });
+    return Promise.all(promises);
+  });
+}

x-pack/test/apm_api_integration/tests/service_groups/service_group_count/service_group_count.spec.ts

@@ -74,14 +74,6 @@ export default function ApiTest({ getService }: FtrProviderContext) {
       await apmSynthtraceEsClient.clean();
     });
 
-    it('returns the correct number of services', async () => {
-      const response = await getServiceGroupCounts(apmApiClient);
-      expect(response.status).to.be(200);
-      expect(Object.keys(response.body).length).to.be(2);
-      expect(response.body[synthbeansServiceGroupId]).to.have.property('services', 2);
-      expect(response.body[opbeansServiceGroupId]).to.have.property('services', 1);
-    });
-
     describe('with alerts', () => {
       let ruleId: string;
       before(async () => {