🌊 Refactor APIs to follow Elasticsearch conventions (#204671)

## Summary

This PR refactors the API by creating a new packaged called
`@kbn/streams-schema` where you can find all the Zod types along with
some type guards. I've also updated all the API's and calls to use the
new schemas.

---------

Co-authored-by: kibanamachine <[email protected]>
Co-authored-by: Joe Reuter <[email protected]>

.github/CODEOWNERS

@@ -763,6 +763,7 @@ x-pack/packages/kbn-ai-assistant @elastic/search-kibana
 x-pack/packages/kbn-alerting-comparators @elastic/response-ops
 x-pack/packages/kbn-alerting-state-types @elastic/response-ops
 x-pack/packages/kbn-random-sampling @elastic/kibana-visualizations
+x-pack/packages/kbn-streams-schema @elastic/streams-program-team
 x-pack/packages/kbn-synthetics-private-location @elastic/obs-ux-management-team
 x-pack/packages/maps/vector_tile_utils @elastic/kibana-presentation
 x-pack/packages/observability/observability_utils/observability_utils_browser @elastic/observability-ui

package.json

@@ -945,6 +945,7 @@
     "@kbn/std": "link:packages/kbn-std",
     "@kbn/streams-app-plugin": "link:x-pack/solutions/observability/plugins/streams_app",
     "@kbn/streams-plugin": "link:x-pack/solutions/observability/plugins/streams",
+    "@kbn/streams-schema": "link:x-pack/packages/kbn-streams-schema",
     "@kbn/synthetics-plugin": "link:x-pack/solutions/observability/plugins/synthetics",
     "@kbn/synthetics-private-location": "link:x-pack/packages/kbn-synthetics-private-location",
     "@kbn/task-manager-fixture-plugin": "link:x-pack/test/alerting_api_integration/common/plugins/task_manager_fixture",

tsconfig.base.json

@@ -1878,6 +1878,8 @@
       "@kbn/streams-app-plugin/*": ["x-pack/solutions/observability/plugins/streams_app/*"],
       "@kbn/streams-plugin": ["x-pack/solutions/observability/plugins/streams"],
       "@kbn/streams-plugin/*": ["x-pack/solutions/observability/plugins/streams/*"],
+      "@kbn/streams-schema": ["x-pack/packages/kbn-streams-schema"],
+      "@kbn/streams-schema/*": ["x-pack/packages/kbn-streams-schema/*"],
       "@kbn/synthetics-e2e": ["x-pack/solutions/observability/plugins/synthetics/e2e"],
       "@kbn/synthetics-e2e/*": ["x-pack/solutions/observability/plugins/synthetics/e2e/*"],
       "@kbn/synthetics-plugin": ["x-pack/solutions/observability/plugins/synthetics"],

x-pack/packages/kbn-streams-schema/README.md

@@ -0,0 +1,3 @@
+# @kbn/streams-schema
+
+This shared package contains the Zod schema definition for the Streams project.

x-pack/packages/kbn-streams-schema/index.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './src/apis';
+export * from './src/models';
+export * from './src/helpers';

x-pack/packages/kbn-streams-schema/jest.config.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../..',
+  roots: ['<rootDir>/x-pack/packages/kbn-streams-schema'],
+};

x-pack/packages/kbn-streams-schema/kibana.jsonc

@@ -0,0 +1,8 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/streams-schema",
+  "owner": "@elastic/streams-program-team",
+  "group": "observability",
+  "visibility": "shared"
+}
+

x-pack/packages/kbn-streams-schema/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "@kbn/streams-schema",
+  "description": "Streams Zod schema definition and common models shared between public and server.",
+  "private": true,
+  "version": "1.0.0",
+  "license": "Elastic License 2.0"
+}

x-pack/packages/kbn-streams-schema/src/apis/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './read_streams_response';
+export * from './list_streams_response';

x-pack/packages/kbn-streams-schema/src/apis/list_streams_response.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { z } from '@kbn/zod';
+import { streamDefintionSchema } from '../models';
+
+export const listStreamsResponseSchema = z.object({
+  streams: z.array(streamDefintionSchema),
+});
+
+export type ListStreamsResponse = z.infer<typeof listStreamsResponseSchema>;

x-pack/packages/kbn-streams-schema/src/apis/read_streams_response.test.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { readStreamResponse } from '../fixtures/read_streams_response';
+import { readStreamResponseSchema } from './read_streams_response';
+
+describe('ReadStreamResponse', () => {
+  it('should successfully parse', () => {
+    expect(readStreamResponseSchema.parse(readStreamResponse)).toMatchSnapshot();
+  });
+});

x-pack/packages/kbn-streams-schema/src/apis/read_streams_response.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { z } from '@kbn/zod';
+import { readStreamDefinitonSchema } from '../models';
+
+export const readStreamResponseSchema = z.object({
+  streams: z.array(readStreamDefinitonSchema),
+});
+
+export type ReadStreamResponse = z.infer<typeof readStreamResponseSchema>;

x-pack/packages/kbn-streams-schema/src/fixtures/ingest_read_stream.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ingestStream } from './ingest_stream';
+
+export const ingestReadStream = {
+  ...ingestStream,
+  inherited_fields: {
+    '@timestamp': {
+      type: 'date',
+      from: 'logs',
+    },
+    message: {
+      type: 'match_only_text',
+      from: 'logs',
+    },
+  },
+};

x-pack/packages/kbn-streams-schema/src/fixtures/ingest_stream.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ingestStreamConfig } from './ingest_stream_config';
+
+export const ingestStream = {
+  name: 'logs.nginx',
+  elasticsearch_assets: [],
+  stream: ingestStreamConfig,
+};

x-pack/packages/kbn-streams-schema/src/fixtures/ingest_stream_config.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const ingestStreamConfig = {
+  ingest: {
+    processing: [
+      {
+        config: {
+          grok: {
+            field: 'message',
+            patterns: ['%{TIMESTAMP_ISO8601:event.timestamp} %{GREEDY:rest}'],
+          },
+        },
+        condition: {
+          field: 'log.level',
+          operator: 'eq',
+          value: 'error',
+        },
+      },
+    ],
+    routing: [
+      {
+        name: 'logs.errors',
+        condition: {
+          field: 'log.level',
+          operator: 'eq',
+          value: 'error',
+        },
+      },
+    ],
+  },
+};

x-pack/packages/kbn-streams-schema/src/fixtures/read_streams_response.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ingestReadStream } from './ingest_read_stream';
+import { wiredReadStream } from './wired_read_stream';
+
+export const readStreamResponse = {
+  streams: [wiredReadStream, ingestReadStream],
+};

x-pack/packages/kbn-streams-schema/src/fixtures/wired_read_stream.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { wiredStream } from './wired_stream';
+
+export const wiredReadStream = {
+  ...wiredStream,
+  inherited_fields: {
+    '@timestamp': {
+      type: 'date',
+      from: 'logs',
+    },
+    message: {
+      type: 'match_only_text',
+      from: 'logs',
+    },
+  },
+};

x-pack/packages/kbn-streams-schema/src/fixtures/wired_stream.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { wiredStreamConfig } from './wired_stream_config';
+
+export const wiredStream = {
+  name: 'logs.nginx',
+  elasticsearch_assets: [],
+  stream: wiredStreamConfig,
+};