[Discover] Add a default "All logs" temporary data view in the Observ…

…ability Solution view (#205991)

## Summary

This PR adds an "All logs" ad hoc (temporary) data view to the Discover
Observability root profile based on the central log sources setting,
allowing quick access to logs (with the most up to date log sources)
without needing to first manually create a data view:
![CleanShot 2025-01-22 at 17 47
19@2x](https://github.com/user-attachments/assets/2c03ec79-0cf9-414e-8883-130599989c25)

To support this, a new `getDefaultAdHocDataViews` extension point has
been added to Discover, allowing profiles to specify an array of ad hoc
data view specs would should be created by default when the profile is
resolved, and automatically cleaned up when the profile changes or the
user leaves Discover.

Resolves #201669.
Resolves #189166.

### Notes

- The "All logs" ad hoc data view should only appear when using the
Observability Solution view (in any deployment type).
- Data view specs returned from `getDefaultAdHocDataViews` must include
consistent IDs across resolutions in order for Discover to manage them
correctly (e.g. to find and reload the data view after a page refresh).
Situations where we'd expect a change in ID (e.g. when saving to a
Discover session) are handled internally by Discover.
- To avoid a breaking change, the returned ad hoc data views have no
impact on the default data view shown when navigating to Discover. If
any persisted data views exist, one of them will be used as the default.
If no persisted data views exist, the first entry of the array returned
by `getDefaultAdHocDataViews` will be used as the default.
- We still want to notify users in Discover when they have no ES data at
all, and prompt them to install integrations. For this reason, the "no
data" page is still shown in Discover even if there are default profile
ad hoc data views (unlike if there are persisted data views, in which
case we use the default and hide the "no data" page).
- When saving a Discover session that uses a default profile ad hoc data
view, the data view will be copied on save as `{DATA_VIEW_NAME} (copy)`.
This allows us to assign a unique ID to the version that gets saved with
the Discover session, and avoids having to choose between the profile
data view or the embedded data view when reopening the session, which
has drawbacks:
- If choosing the profile data view, the Discover session may display
incorrectly if the log sources setting changed since it was saved, and
the user would no longer be able to view the session as it was intended
without first modifying the setting to the expected value.
- If choosing the embedded data view, the replacement shown after
opening the Discover session may not reflect the latest log sources
setting until a new session is started, and there would be no way for
the user to migrate the session to use the latest version of the profile
data view.

### Checklist

- [x] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/src/platform/packages/shared/kbn-i18n/README.md)
- [ ]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [ ] If a plugin configuration key changed, check if it needs to be
allowlisted in the cloud and added to the [docker
list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker)
- [ ] This was checked for breaking HTTP API changes, and any breaking
changes have been approved by the breaking-change committee. The
`release_note:breaking` label should be applied in these situations.
- [x] [Flaky Test
Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was
used on any tests changed
- [x] The PR description includes the appropriate Release Notes section,
and the correct `release_note:*` label is applied per the
[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)

src/platform/packages/shared/kbn-discover-utils/src/__mocks__/logs_context_service.ts

@@ -10,5 +10,8 @@
 import { DEFAULT_ALLOWED_LOGS_BASE_PATTERNS_REGEXP, getLogsContextService } from '../data_types';
 
 export const createLogsContextServiceMock = () => {
-  return getLogsContextService([DEFAULT_ALLOWED_LOGS_BASE_PATTERNS_REGEXP]);
+  return getLogsContextService({
+    allLogsIndexPattern: 'logs-*',
+    allowedDataSources: [DEFAULT_ALLOWED_LOGS_BASE_PATTERNS_REGEXP],
+  });
 };

src/platform/packages/shared/kbn-discover-utils/src/data_types/logs/logs_context_service.ts

@@ -11,6 +11,7 @@ import { createRegExpPatternFrom, testPatternAgainstAllowedList } from '@kbn/dat
 import type { LogsDataAccessPluginStart } from '@kbn/logs-data-access-plugin/public';
 
 export interface LogsContextService {
+  getAllLogsIndexPattern(): string | undefined;
   isLogsIndexPattern(indexPattern: unknown): boolean;
 }
 
@@ -32,34 +33,45 @@ export const DEFAULT_ALLOWED_LOGS_BASE_PATTERNS_REGEXP = createRegExpPatternFrom
   'data'
 );
 
-export const createLogsContextService = async ({ logsDataAccess }: LogsContextServiceDeps) => {
+export const createLogsContextService = async ({
+  logsDataAccess,
+}: LogsContextServiceDeps): Promise<LogsContextService> => {
+  let allLogsIndexPattern: string | undefined;
   let logSources: string[] | undefined;
 
   if (logsDataAccess) {
     const logSourcesService = logsDataAccess.services.logSourcesService;
-    logSources = (await logSourcesService.getLogSources())
+    allLogsIndexPattern = (await logSourcesService.getLogSources())
       .map((logSource) => logSource.indexPattern)
-      .join(',') // TODO: Will be replaced by helper in: https://github.com/elastic/kibana/pull/192003
-      .split(',');
+      .join(','); // TODO: Will be replaced by helper in: https://github.com/elastic/kibana/pull/192003
+    logSources = allLogsIndexPattern.split(',');
   }
 
   const ALLOWED_LOGS_DATA_SOURCES = [
     DEFAULT_ALLOWED_LOGS_BASE_PATTERNS_REGEXP,
     ...(logSources ? logSources : []),
   ];
 
-  return getLogsContextService(ALLOWED_LOGS_DATA_SOURCES);
+  return getLogsContextService({
+    allLogsIndexPattern,
+    allowedDataSources: ALLOWED_LOGS_DATA_SOURCES,
+  });
 };
 
-export const getLogsContextService = (allowedDataSources: Array<string | RegExp>) => {
+export const getLogsContextService = ({
+  allLogsIndexPattern,
+  allowedDataSources,
+}: {
+  allLogsIndexPattern: string | undefined;
+  allowedDataSources: Array<string | RegExp>;
+}): LogsContextService => {
+  const getAllLogsIndexPattern = () => allLogsIndexPattern;
   const isLogsIndexPattern = (indexPattern: unknown) => {
     return (
       typeof indexPattern === 'string' &&
       testPatternAgainstAllowedList(allowedDataSources)(indexPattern)
     );
   };
 
-  return {
-    isLogsIndexPattern,
-  };
+  return { getAllLogsIndexPattern, isLogsIndexPattern };
 };

src/platform/packages/shared/kbn-es-query/index.ts

@@ -104,6 +104,7 @@ export {
   uniqFilters,
   unpinFilter,
   updateFilter,
+  updateFilterReferences,
   extractTimeFilter,
   extractTimeRange,
   convertRangeFilterToTimeRange,

src/platform/packages/shared/kbn-es-query/src/filters/helpers/index.ts

@@ -15,4 +15,5 @@ export * from './meta_filter';
 export * from './only_disabled';
 export * from './extract_time_filter';
 export * from './convert_range_filter';
+export * from './update_filter_references';
 export * from './types';

src/platform/packages/shared/kbn-es-query/src/filters/helpers/update_filter_references.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { Filter } from '..';
+
+export function updateFilterReferences(
+  filters: Filter[],
+  fromDataView: string,
+  toDataView: string | undefined
+) {
+  return (filters || []).map((filter) => {
+    if (filter.meta.index === fromDataView) {
+      return {
+        ...filter,
+        meta: {
+          ...filter.meta,
+          index: toDataView,
+        },
+      };
+    } else {
+      return filter;
+    }
+  });
+}

src/platform/packages/shared/kbn-es-query/src/filters/index.ts

@@ -31,6 +31,7 @@ export {
   extractTimeFilter,
   extractTimeRange,
   convertRangeFilterToTimeRange,
+  updateFilterReferences,
 } from './helpers';
 
 export {

src/platform/plugins/shared/discover/public/__mocks__/saved_search.ts

@@ -14,13 +14,16 @@ import { dataViewWithTimefieldMock } from './data_view_with_timefield';
 import { dataViewAdHoc } from './data_view_complex';
 import { dataViewEsql } from './data_view_esql';
 
-export const savedSearchMock = {
-  id: 'the-saved-search-id',
-  title: 'A saved search',
-  searchSource: createSearchSourceMock({ index: dataViewMock }),
-  columns: ['default_column'],
-  sort: [],
-} as unknown as SavedSearch;
+export const createSavedSearchMock = () =>
+  ({
+    id: 'the-saved-search-id',
+    title: 'A saved search',
+    searchSource: createSearchSourceMock({ index: dataViewMock }),
+    columns: ['default_column'],
+    sort: [],
+  } as unknown as SavedSearch);
+
+export const savedSearchMock = createSavedSearchMock();
 
 export const savedSearchMockWithTimeField = {
   id: 'the-saved-search-id-with-timefield',
@@ -40,7 +43,10 @@ export const savedSearchMockWithESQL = {
   isTextBasedQuery: true,
 } as unknown as SavedSearch;
 
-export const savedSearchAdHoc = {
-  id: 'the-saved-search-with-ad-hoc',
-  searchSource: createSearchSourceMock({ index: dataViewAdHoc }),
-} as unknown as SavedSearch;
+export const createSavedSearchAdHocMock = () =>
+  ({
+    id: 'the-saved-search-with-ad-hoc',
+    searchSource: createSearchSourceMock({ index: dataViewAdHoc }),
+  } as unknown as SavedSearch);
+
+export const savedSearchAdHoc = createSavedSearchAdHocMock();

src/platform/plugins/shared/discover/public/__mocks__/services.ts

@@ -43,11 +43,11 @@ import { LocalStorageMock } from './local_storage_mock';
 import { createDiscoverDataViewsMock } from './data_views';
 import { SearchSourceDependencies } from '@kbn/data-plugin/common';
 import { SearchResponse } from '@elastic/elasticsearch/lib/api/types';
-import { urlTrackerMock } from './url_tracker.mock';
 import { createElement } from 'react';
 import { createContextAwarenessMocks } from '../context_awareness/__mocks__';
 import { DiscoverEBTManager } from '../services/discover_ebt_manager';
 import { discoverSharedPluginMock } from '@kbn/discover-shared-plugin/public/mocks';
+import { createUrlTrackerMock } from './url_tracker.mock';
 
 export function createDiscoverServicesMock(): DiscoverServices {
   const dataPlugin = dataPluginMock.createStartContract();
@@ -247,7 +247,7 @@ export function createDiscoverServicesMock(): DiscoverServices {
     },
     contextLocator: { getRedirectUrl: jest.fn(() => '') },
     singleDocLocator: { getRedirectUrl: jest.fn(() => '') },
-    urlTracker: urlTrackerMock,
+    urlTracker: createUrlTrackerMock(),
     profilesManager: profilesManagerMock,
     ebtManager: new DiscoverEBTManager(),
     setHeaderActionMenu: jest.fn(),

src/platform/plugins/shared/discover/public/__mocks__/url_tracker.mock.ts

@@ -9,8 +9,9 @@
 
 import { UrlTracker } from '../build_services';
 
-export const urlTrackerMock = {
-  setTrackedUrl: jest.fn(),
-  restorePreviousUrl: jest.fn(),
-  setTrackingEnabled: jest.fn(),
-} as UrlTracker;
+export const createUrlTrackerMock = () =>
+  ({
+    setTrackedUrl: jest.fn(),
+    restorePreviousUrl: jest.fn(),
+    setTrackingEnabled: jest.fn(),
+  } as UrlTracker);

src/platform/plugins/shared/discover/public/application/main/components/sidebar/discover_sidebar_responsive.test.tsx

@@ -34,6 +34,7 @@ import { buildDataTableRecord } from '@kbn/discover-utils';
 import type { DataTableRecord } from '@kbn/discover-utils/types';
 import type { DiscoverCustomizationId } from '../../../../customizations/customization_service';
 import { FieldListCustomization, SearchBarCustomization } from '../../../../customizations';
+import { InternalStateProvider } from '../../state_management/discover_internal_state_container';
 
 const mockSearchBarCustomization: SearchBarCustomization = {
   id: 'search_bar',
@@ -177,13 +178,13 @@ function getCompProps(options?: { hits?: DataTableRecord[] }): DiscoverSidebarRe
   };
 }
 
-function getAppStateContainer({ query }: { query?: Query | AggregateQuery }) {
-  const appStateContainer = getDiscoverStateMock({ isTimeBased: true }).appState;
-  appStateContainer.set({
+function getStateContainer({ query }: { query?: Query | AggregateQuery }) {
+  const stateContainer = getDiscoverStateMock({ isTimeBased: true });
+  stateContainer.appState.set({
     query: query ?? { query: '', language: 'lucene' },
     filters: [],
   });
-  return appStateContainer;
+  return stateContainer;
 }
 
 async function mountComponent(
@@ -192,7 +193,7 @@ async function mountComponent(
   services?: DiscoverServices
 ): Promise<ReactWrapper<DiscoverSidebarResponsiveProps>> {
   let comp: ReactWrapper<DiscoverSidebarResponsiveProps>;
-  const appState = getAppStateContainer(appStateParams);
+  const { appState, internalState } = getStateContainer(appStateParams);
   const mockedServices = services ?? createMockServices();
   mockedServices.data.dataViews.getIdsWithTitle = jest.fn(async () =>
     props.selectedDataView
@@ -208,7 +209,9 @@ async function mountComponent(
     comp = mountWithIntl(
       <KibanaContextProvider services={mockedServices}>
         <DiscoverAppStateProvider value={appState}>
-          <DiscoverSidebarResponsive {...props} />
+          <InternalStateProvider value={internalState}>
+            <DiscoverSidebarResponsive {...props} />
+          </InternalStateProvider>
         </DiscoverAppStateProvider>
       </KibanaContextProvider>
     );

src/platform/plugins/shared/discover/public/application/main/components/sidebar/discover_sidebar_responsive.tsx

@@ -37,6 +37,10 @@ import {
 import { useDiscoverCustomization } from '../../../../customizations';
 import { useAdditionalFieldGroups } from '../../hooks/sidebar/use_additional_field_groups';
 import { useIsEsqlMode } from '../../hooks/use_is_esql_mode';
+import {
+  selectDataViewsForPicker,
+  useInternalStateSelector,
+} from '../../state_management/discover_internal_state_container';
 
 const EMPTY_FIELD_COUNTS = {};
 
@@ -172,6 +176,8 @@ export function DiscoverSidebarResponsive(props: DiscoverSidebarResponsiveProps)
   );
   const selectedDataViewRef = useRef<DataView | null | undefined>(selectedDataView);
   const showFieldList = sidebarState.status !== DiscoverSidebarReducerStatus.INITIAL;
+  const { savedDataViews, managedDataViews, adHocDataViews } =
+    useInternalStateSelector(selectDataViewsForPicker);
 
   useEffect(() => {
     const subscription = props.documents$.subscribe((documentState) => {
@@ -326,6 +332,9 @@ export function DiscoverSidebarResponsive(props: DiscoverSidebarResponsiveProps)
       ) : (
         <DataViewPicker
           currentDataViewId={selectedDataView.id}
+          adHocDataViews={adHocDataViews}
+          managedDataViews={managedDataViews}
+          savedDataViews={savedDataViews}
           onChangeDataView={onChangeDataView}
           onAddField={createField}
           onDataViewCreated={createNewDataView}
@@ -338,7 +347,16 @@ export function DiscoverSidebarResponsive(props: DiscoverSidebarResponsiveProps)
         />
       )
     ) : null;
-  }, [selectedDataView, createNewDataView, onChangeDataView, createField, CustomDataViewPicker]);
+  }, [
+    selectedDataView,
+    CustomDataViewPicker,
+    adHocDataViews,
+    managedDataViews,
+    savedDataViews,
+    onChangeDataView,
+    createField,
+    createNewDataView,
+  ]);
 
   const onAddFieldToWorkspace = useCallback(
     (field: DataViewField) => {

src/platform/plugins/shared/discover/public/application/main/components/top_nav/discover_topnav.tsx

@@ -15,7 +15,10 @@ import { TextBasedLanguages } from '@kbn/esql-utils';
 import { DiscoverFlyouts, dismissAllFlyoutsExceptFor } from '@kbn/discover-utils';
 import { useSavedSearchInitial } from '../../state_management/discover_state_provider';
 import { ESQL_TRANSITION_MODAL_KEY } from '../../../../../common/constants';
-import { useInternalStateSelector } from '../../state_management/discover_internal_state_container';
+import {
+  selectDataViewsForPicker,
+  useInternalStateSelector,
+} from '../../state_management/discover_internal_state_container';
 import { useDiscoverServices } from '../../../../hooks/use_discover_services';
 import type { DiscoverStateContainer } from '../../state_management/discover_state';
 import { onSaveSearch } from './on_save_search';
@@ -49,9 +52,9 @@ export const DiscoverTopNav = ({
   const { dataViewEditor, navigation, dataViewFieldEditor, data, uiSettings, setHeaderActionMenu } =
     services;
   const query = useAppStateSelector((state) => state.query);
-  const adHocDataViews = useInternalStateSelector((state) => state.adHocDataViews);
+  const { savedDataViews, managedDataViews, adHocDataViews } =
+    useInternalStateSelector(selectDataViewsForPicker);
   const dataView = useInternalStateSelector((state) => state.dataView!);
-  const savedDataViews = useInternalStateSelector((state) => state.savedDataViews);
   const isESQLToDataViewTransitionModalVisible = useInternalStateSelector(
     (state) => state.isESQLToDataViewTransitionModalVisible
   );
@@ -172,9 +175,7 @@ export const DiscoverTopNav = ({
 
   const dataViewPickerProps: DataViewPickerProps = useMemo(() => {
     const isESQLModeEnabled = uiSettings.get(ENABLE_ESQL);
-    const supportedTextBasedLanguages: DataViewPickerProps['textBasedLanguages'] = isESQLModeEnabled
-      ? [TextBasedLanguages.ESQL]
-      : [];
+    const supportedTextBasedLanguages = isESQLModeEnabled ? [TextBasedLanguages.ESQL] : [];
 
     return {
       trigger: {
@@ -189,6 +190,7 @@ export const DiscoverTopNav = ({
       onChangeDataView: stateContainer.actions.onChangeDataView,
       textBasedLanguages: supportedTextBasedLanguages,
       adHocDataViews,
+      managedDataViews,
       savedDataViews,
       onEditDataView: stateContainer.actions.onDataViewEdited,
     };
@@ -197,6 +199,7 @@ export const DiscoverTopNav = ({
     addField,
     createNewDataView,
     dataView,
+    managedDataViews,
     savedDataViews,
     stateContainer,
     uiSettings,

src/platform/plugins/shared/discover/public/application/main/data_fetching/fetch_all.test.ts

@@ -72,6 +72,7 @@ describe('test fetchAll', () => {
         isDataViewLoading: false,
         savedDataViews: [],
         adHocDataViews: [],
+        defaultProfileAdHocDataViewIds: [],
         expandedDoc: undefined,
         customFilters: [],
         overriddenVisContextAfterInvalidation: undefined,
@@ -265,6 +266,7 @@ describe('test fetchAll', () => {
         isDataViewLoading: false,
         savedDataViews: [],
         adHocDataViews: [],
+        defaultProfileAdHocDataViewIds: [],
         expandedDoc: undefined,
         customFilters: [],
         overriddenVisContextAfterInvalidation: undefined,
@@ -389,6 +391,7 @@ describe('test fetchAll', () => {
           isDataViewLoading: false,
           savedDataViews: [],
           adHocDataViews: [],
+          defaultProfileAdHocDataViewIds: [],
           expandedDoc: undefined,
           customFilters: [],
           overriddenVisContextAfterInvalidation: undefined,

src/platform/plugins/shared/discover/public/application/main/discover_main_app.tsx

@@ -36,7 +36,7 @@ export function DiscoverMainApp(props: DiscoverMainProps) {
   const services = useDiscoverServices();
   const { chrome, docLinks, data, spaces, history } = services;
 
-  useUrlTracking(stateContainer.savedSearchState);
+  useUrlTracking(stateContainer);
 
   /**
    * Adhoc data views functionality