From 581a3d1e35be38af8d081256601222437d9e62e7 Mon Sep 17 00:00:00 2001 From: Sebastien Flory Date: Wed, 14 Aug 2024 13:41:41 +0200 Subject: [PATCH] Refactor Assistant builder: use Vaults, use Context --- .husky/pre-commit | 12 + front/components/ConnectorPermissionsTree.tsx | 8 +- .../DataSourceResourceSelectorTree.tsx | 50 +-- .../ManagedDataSourceDocumentModal.tsx | 6 +- .../components/assistant/AssistantDetails.tsx | 12 +- .../assistant_builder/AssistantBuilder.tsx | 12 +- .../AssistantBuilderContext.tsx | 11 +- .../AssistantBuilderDataSourceModal.tsx | 364 +++++++++--------- .../AssistantBuilderPreviewDrawer.tsx | 2 +- .../AssistantBuilderTablesModal.tsx | 206 +++++----- .../DataSourceSelectionSection.tsx | 48 ++- ...tsx => DataSourceViewResourceSelector.tsx} | 67 ++-- .../FolderOrWebsiteResourceSelector.tsx | 40 +- .../assistant_builder/FolderOrWebsiteTree.tsx | 22 +- .../assistant_builder/PickDataSource.tsx | 61 +-- .../PickDataSourceForTable.tsx | 51 ++- .../assistant_builder/PickTableInFolder.tsx | 24 +- .../assistant_builder/PickTablesManaged.tsx | 38 +- .../Sharing.tsx | 10 +- .../SlackIntegration.tsx | 7 +- .../TablesSelectionSection.tsx | 4 +- .../TryAssistant.tsx | 0 .../actions/ProcessAction.tsx | 4 +- .../actions/TablesQueryAction.tsx | 9 +- .../server_side_props_helpers.ts | 67 +++- front/components/assistant_builder/shared.ts | 54 ++- .../submitAssistantBuilderForm.ts | 62 ++- front/components/assistant_builder/types.ts | 8 +- .../assistant_builder/useSlackChannels.tsx | 14 +- .../EditVaultManagedDatasourcesViews.tsx | 6 +- .../vaults/VaultDataSourceViewContentList.tsx | 13 +- .../VaultManagedDatasourcesViewsModal.tsx | 43 ++- .../components/vaults/VaultResourcesList.tsx | 11 +- front/components/vaults/VaultSideBarMenu.tsx | 10 +- front/hooks/useParentResourcesById.ts | 12 +- .../api/assistant/configuration/process.ts | 31 +- .../api/assistant/configuration/retrieval.ts | 31 +- front/lib/api/assistant/global_agents.ts | 140 +++---- front/lib/api/vaults.ts | 4 + front/lib/assistant.ts | 86 +++-- front/lib/connector_providers.ts | 29 +- front/lib/resources/data_source_resource.ts | 6 +- .../resources/data_source_view_resource.ts | 17 +- front/lib/swr.ts | 57 ++- .../assistant/labs/transcripts/index.tsx | 15 +- .../[wId]/builder/assistants/[aId]/index.tsx | 36 +- .../pages/w/[wId]/builder/assistants/dust.tsx | 12 +- .../w/[wId]/builder/assistants/index.tsx | 2 +- .../pages/w/[wId]/builder/assistants/new.tsx | 36 +- .../builder/data-sources/[name]/index.tsx | 4 +- .../builder/data-sources/public-urls.tsx | 50 ++- front/scripts/helpers.ts | 7 + types/src/connectors/content_nodes.ts | 35 -- .../internal/agent_configuration.ts | 4 +- types/src/front/api_handlers/public/vaults.ts | 2 + .../src/front/assistant/actions/retrieval.ts | 2 +- types/src/front/data_source.ts | 8 + types/src/front/data_source_view.ts | 7 +- 58 files changed, 1024 insertions(+), 965 deletions(-) rename front/components/assistant_builder/{DataSourceResourceSelector.tsx => DataSourceViewResourceSelector.tsx} (63%) rename front/components/{assistant => assistant_builder}/Sharing.tsx (98%) rename front/components/{assistant => assistant_builder}/SlackIntegration.tsx (94%) rename front/components/{assistant => assistant_builder}/TryAssistant.tsx (100%) diff --git a/.husky/pre-commit b/.husky/pre-commit index eb145962ad36..00df19a3ec4d 100755 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -30,6 +30,18 @@ for directory in "${directories[@]}"; do echo "Linting failed. Please fix the issues before committing." exit 1 fi + if ! npm run tsc --prefix $root_path/$directory; then + echo "Type checking failed. Please fix the issues before committing." + exit 1 + fi + if ! npm run format:check --prefix $root_path/$directory; then + echo "Formatting check failed. Please fix the issues before committing." + exit 1 + fi + if ! npm run docs:check --prefix $root_path/$directory; then + echo "Documentation check failed. Please fix the issues before committing." + exit 1 + fi break fi done diff --git a/front/components/ConnectorPermissionsTree.tsx b/front/components/ConnectorPermissionsTree.tsx index 146ccf419460..57190f9e082e 100644 --- a/front/components/ConnectorPermissionsTree.tsx +++ b/front/components/ConnectorPermissionsTree.tsx @@ -11,7 +11,7 @@ import { import type { ConnectorProvider, DataSourceType, - WorkspaceType, + LightWorkspaceType, } from "@dust-tt/types"; import type { ConnectorPermission } from "@dust-tt/types"; import { useState } from "react"; @@ -62,7 +62,7 @@ export function PermissionTreeChildren({ useConnectorPermissionsHook, isSearchEnabled, }: { - owner: WorkspaceType; + owner: LightWorkspaceType; dataSource: DataSourceType; parentId: string | null; permissionFilter?: ConnectorPermission; @@ -89,7 +89,7 @@ export function PermissionTreeChildren({ const { resources, isResourcesLoading, isResourcesError } = useConnectorPermissionsHook({ owner, - dataSourceOrView: dataSource, + dataSource, parentId, filterPermission: permissionFilter || null, }); @@ -290,7 +290,7 @@ export function PermissionTree({ showExpand, isSearchEnabled, }: { - owner: WorkspaceType; + owner: LightWorkspaceType; dataSource: DataSourceType; permissionFilter?: ConnectorPermission; canUpdatePermissions?: boolean; diff --git a/front/components/DataSourceResourceSelectorTree.tsx b/front/components/DataSourceResourceSelectorTree.tsx index 8eacbb42ac2d..2f265de5f5e2 100644 --- a/front/components/DataSourceResourceSelectorTree.tsx +++ b/front/components/DataSourceResourceSelectorTree.tsx @@ -4,21 +4,20 @@ import { Tree, } from "@dust-tt/sparkle"; import type { - ContentNode, ContentNodesViewType, - DataSourceType, DataSourceViewType, - WorkspaceType, + LightContentNode, + LightWorkspaceType, } from "@dust-tt/types"; import type { ConnectorPermission, ContentNodeType } from "@dust-tt/types"; import { CircleStackIcon, FolderIcon } from "@heroicons/react/20/solid"; import { useEffect } from "react"; -import { useConnectorPermissions } from "@app/lib/swr"; +import { useVaultDataSourceViewContent } from "@app/lib/swr"; export default function DataSourceResourceSelectorTree({ owner, - dataSourceOrView, + dataSourceView, showExpand, //if not, it's flat parentIsSelected, selectedParents = [], @@ -27,14 +26,14 @@ export default function DataSourceResourceSelectorTree({ filterPermission = "read", viewType = "documents", }: { - owner: WorkspaceType; - dataSourceOrView: DataSourceType | DataSourceViewType; + owner: LightWorkspaceType; + dataSourceView: DataSourceViewType; showExpand: boolean; parentIsSelected?: boolean; selectedParents?: string[]; selectedResourceIds: string[]; onSelectChange: ( - resource: ContentNode, + resource: LightContentNode, parents: string[], selected: boolean ) => void; @@ -45,7 +44,7 @@ export default function DataSourceResourceSelectorTree({
void; filterPermission: ConnectorPermission; viewType: ContentNodesViewType; }) { - const { resources, isResourcesLoading, isResourcesError } = - useConnectorPermissions({ - owner: owner, - dataSourceOrView, + const { vaultContent, isVaultContentLoading, isVaultContentError } = + useVaultDataSourceViewContent({ + workspaceId: owner.sId, + vaultId: dataSourceView.vaultId, + dataSourceViewId: dataSourceView.sId, + viewType, parentId, filterPermission, - disabled: dataSourceOrView.connectorId === null, - viewType, + disabled: dataSourceView.dataSource.connectorId === null, }); useEffect(() => { if (parentIsSelected) { // Unselected previously selected children - resources + vaultContent .filter((r) => selectedResourceIds.includes(r.internalId)) .forEach((r) => { onSelectChange(r, parents, false); }); } }, [ - resources, + vaultContent, parentIsSelected, selectedResourceIds, onSelectChange, parents, ]); - if (isResourcesError) { + if (isVaultContentError) { return (
Failed to retrieve resources likely due to a revoked authorization. @@ -151,8 +151,8 @@ function DataSourceResourceSelectorChildren({ const isTablesView = viewType === "tables"; return ( - - {resources.map((r) => { + + {vaultContent.map((r) => { const isSelected = selectedResourceIds.includes(r.internalId); const partiallyChecked = !isSelected && @@ -186,7 +186,7 @@ function DataSourceResourceSelectorChildren({ renderTreeItems={() => ( { - if (documentId && dataSource?.name) { + if (documentId && dataSource) { setDownloading(true); fetch( `/api/w/${owner.sId}/data_sources/${encodeURIComponent( diff --git a/front/components/assistant/AssistantDetails.tsx b/front/components/assistant/AssistantDetails.tsx index 01fffbd13a35..36ce0f321078 100644 --- a/front/components/assistant/AssistantDetails.tsx +++ b/front/components/assistant/AssistantDetails.tsx @@ -17,11 +17,11 @@ import type { AgentActionConfigurationType, AgentConfigurationScope, AgentConfigurationType, - ContentNode, CoreAPITable, DataSourceConfiguration, DataSourceType, DustAppRunConfigurationType, + LightContentNode, RetrievalConfigurationType, TablesQueryConfigurationType, WorkspaceType, @@ -41,13 +41,13 @@ import type { KeyedMutator } from "swr"; import { AssistantDetailsDropdownMenu } from "@app/components/assistant/AssistantDetailsDropdownMenu"; import AssistantListActions from "@app/components/assistant/AssistantListActions"; import { ReadOnlyTextArea } from "@app/components/assistant/ReadOnlyTextArea"; -import { SharingDropdown } from "@app/components/assistant/Sharing"; import { assistantUsageMessage } from "@app/components/assistant/Usage"; +import { SharingDropdown } from "@app/components/assistant_builder/Sharing"; import { PermissionTreeChildren } from "@app/components/ConnectorPermissionsTree"; import ManagedDataSourceDocumentModal from "@app/components/ManagedDataSourceDocumentModal"; import { SendNotificationsContext } from "@app/components/sparkle/Notification"; import { updateAgentScope } from "@app/lib/client/dust_api"; -import { CONNECTOR_CONFIGURATIONS } from "@app/lib/connector_providers"; +import { getConnectorProviderLogo } from "@app/lib/connector_providers"; import { getDisplayNameForDataSource } from "@app/lib/data_sources"; import { useAgentConfiguration, @@ -376,9 +376,7 @@ function DataSourcesSection({ let dataSourceName = dsConfig.dataSourceId; if (ds) { - DsLogo = ds.connectorProvider - ? CONNECTOR_CONFIGURATIONS[ds.connectorProvider].logoComponent - : null; + DsLogo = getConnectorProviderLogo(ds.connectorProvider); dataSourceName = getDisplayNameForDataSource(ds); } @@ -446,7 +444,7 @@ function DataSourceSelectedNodes({ return ( <> - {dataSourceSelectedNodes.nodes.map((node: ContentNode) => ( + {dataSourceSelectedNodes.nodes.map((node: LightContentNode) => ( ds.connectorProvider === "slack" + const slackDataSource = dataSourceViews.find( + (dsv) => dsv.dataSource.connectorProvider === "slack" ); const defaultScope = flow === "workspace_assistants" ? "workspace" : "private"; @@ -149,7 +149,7 @@ export default function AssistantBuilder({ slackChannelsLinkedWithAgent, setSelectedSlackChannels, } = useSlackChannel({ - dataSources, + dataSourceViews, initialChannels: [], workspaceId: owner.sId, isPrivateAssistant: builderState.scope === "private", @@ -322,7 +322,7 @@ export default function AssistantBuilder({ }); } else { await mutate( - `/api/w/${owner.sId}/data_sources/${slackDataSource?.name}/managed/slack/channels_linked_with_agent` + `/api/w/${owner.sId}/data_sources/${slackDataSource?.dataSource.name}/managed/slack/channels_linked_with_agent` ); // Redirect to the assistant list once saved. diff --git a/front/components/assistant_builder/AssistantBuilderContext.tsx b/front/components/assistant_builder/AssistantBuilderContext.tsx index e2623d7f2b7f..8df2ab09bedd 100644 --- a/front/components/assistant_builder/AssistantBuilderContext.tsx +++ b/front/components/assistant_builder/AssistantBuilderContext.tsx @@ -1,28 +1,20 @@ -import type { - AppType, - DataSourceType, - DataSourceViewType, -} from "@dust-tt/types"; +import type { AppType, DataSourceViewType } from "@dust-tt/types"; import { createContext } from "react"; export const AssistantBuilderContext = createContext<{ dustApps: AppType[]; - dataSources: DataSourceType[]; dataSourceViews: DataSourceViewType[]; }>({ dustApps: [], - dataSources: [], dataSourceViews: [], }); export function AssistantBuilderProvider({ dustApps, - dataSources, dataSourceViews, children, }: { dustApps: AppType[]; - dataSources: DataSourceType[]; dataSourceViews: DataSourceViewType[]; children: React.ReactNode; }) { @@ -30,7 +22,6 @@ export function AssistantBuilderProvider({ diff --git a/front/components/assistant_builder/AssistantBuilderDataSourceModal.tsx b/front/components/assistant_builder/AssistantBuilderDataSourceModal.tsx index 52c5ec8bd795..60c5f9daa564 100644 --- a/front/components/assistant_builder/AssistantBuilderDataSourceModal.tsx +++ b/front/components/assistant_builder/AssistantBuilderDataSourceModal.tsx @@ -1,13 +1,13 @@ import { Modal } from "@dust-tt/sparkle"; import type { - ContentNode, - DataSourceType, + DataSourceViewType, + LightContentNode, WorkspaceType, } from "@dust-tt/types"; -import { useContext, useEffect, useState } from "react"; +import { isFolder, isWebsite } from "@dust-tt/types"; +import { useEffect, useMemo, useState } from "react"; -import { AssistantBuilderContext } from "@app/components/assistant_builder/AssistantBuilderContext"; -import DataSourceResourceSelector from "@app/components/assistant_builder/DataSourceResourceSelector"; +import DataSourceViewResourceSelector from "@app/components/assistant_builder/DataSourceViewResourceSelector"; import FolderOrWebsiteResourceSelector from "@app/components/assistant_builder/FolderOrWebsiteResourceSelector"; import PickDataSource from "@app/components/assistant_builder/PickDataSource"; import type { @@ -16,21 +16,36 @@ import type { } from "@app/components/assistant_builder/types"; import { useNavigationLock } from "@app/components/assistant_builder/useNavigationLock"; +type DisplayMode = + | "PickKind" // Choose type of source + | "SelectFromManaged" // Once a source is chosen, select resources from it, here is from managed + | "SelectFromFolder" // Select folders + | "SelectFromWebsite"; // Select websites + +function getConfiguration( + configurations: AssistantBuilderDataSourceConfigurations, + dataSourceView: DataSourceViewType +): AssistantBuilderDataSourceConfiguration { + return configurations[dataSourceView.sId]; +} + function getUpdatedConfigurations( currentConfigurations: AssistantBuilderDataSourceConfigurations, - dataSource: DataSourceType, + dataSourceView: DataSourceViewType, selected: boolean, - node: ContentNode + node: LightContentNode ) { - const oldConfiguration = currentConfigurations[dataSource.name] || { - dataSource: dataSource, - selectedResources: [], - isSelectAll: false, - }; + const oldConfiguration = + currentConfigurations[dataSourceView.sId] || + ({ + dataSourceView, + selectedResources: [], + isSelectAll: false, + } as AssistantBuilderDataSourceConfiguration); const newConfiguration = { ...oldConfiguration, - } satisfies AssistantBuilderDataSourceConfiguration; + }; if (selected) { newConfiguration.selectedResources = [ @@ -50,9 +65,9 @@ function getUpdatedConfigurations( newConfiguration.isSelectAll || newConfiguration.selectedResources.length > 0 ) { - newConfigurations[dataSource.name] = newConfiguration; + newConfigurations[dataSourceView.sId] = newConfiguration; } else { - delete newConfigurations[dataSource.name]; + delete newConfigurations[dataSourceView.sId]; } return newConfigurations; @@ -71,22 +86,123 @@ export default function AssistantBuilderDataSourceModal({ onSave: (dsConfigs: AssistantBuilderDataSourceConfigurations) => void; initialDataSourceConfigurations: AssistantBuilderDataSourceConfigurations; }) { - const { dataSources } = useContext(AssistantBuilderContext); - // Local modal state, that replaces the action's datasourceConfigurations state in the global // assistant builder state when the modal is saved. const [dataSourceConfigurations, setDataSourceConfigurations] = useState(null); // Navigation state - const [selectedDataSource, setSelectedDataSource] = - useState(null); - // Hack to filter out Folders from the list of data sources - const [shouldDisplayFoldersScreen, setShouldDisplayFoldersScreen] = - useState(false); - // Hack to filter out Websites from the list of data sources - const [shouldDisplayWebsitesScreen, setShouldDisplayWebsitesScreen] = - useState(false); + const [selectedDataSourceView, setSelectedDataSourceView] = + useState(null); + + const [displayMode, setDisplayMode] = useState("PickKind"); + + const onSelectChangeManaged = useMemo( + () => + ( + dsView: DataSourceViewType, + node: LightContentNode, + selected: boolean + ) => { + setDataSourceConfigurations((currentConfigurations) => { + if (currentConfigurations === null) { + // Unreachable + return null; + } + + return getUpdatedConfigurations( + currentConfigurations, + dsView, + selected, + node + ); + }); + }, + [] + ); + + const onSelectChangeFolderOrWebsite = useMemo( + () => + ( + dsView: DataSourceViewType, + selected: boolean, + contentNode?: LightContentNode + ) => { + setDataSourceConfigurations((currentConfigurations) => { + if (currentConfigurations === null) { + // Unreachable + return null; + } + + if (contentNode === undefined) { + if (selected) { + return { + ...currentConfigurations, + [dsView.sId]: { + dataSourceView: dsView, + selectedResources: [], + isSelectAll: true, + }, + }; + } else { + const newConfigurations = { ...currentConfigurations }; + delete newConfigurations[dsView.sId]; + return newConfigurations; + } + } + + return getUpdatedConfigurations( + currentConfigurations, + dsView, + selected, + contentNode + ); + }); + }, + [] + ); + + const onToggleSelectAll = useMemo( + () => (dsView: DataSourceViewType) => { + setDataSourceConfigurations((currentConfigurations) => { + if (currentConfigurations === null) { + // Unreachable + return null; + } + + const oldConfiguration = + currentConfigurations[dsView.sId] || + ({ + dataSourceView: dsView, + selectedResources: [], + isSelectAll: false, + } as AssistantBuilderDataSourceConfiguration); + + const newConfiguration = { + ...oldConfiguration, + isSelectAll: !oldConfiguration.isSelectAll, + selectedResources: [], + }; + + const newConfigurations = { ...currentConfigurations }; + + if ( + newConfiguration.isSelectAll || + newConfiguration.selectedResources.length > 0 + ) { + newConfigurations[dsView.sId] = newConfiguration; + } else { + delete newConfigurations[dsView.sId]; + } + + return { + ...currentConfigurations, + [dsView.sId]: newConfiguration, + }; + }); + }, + [] + ); useNavigationLock(true, { title: "Warning", @@ -99,10 +215,8 @@ export default function AssistantBuilderDataSourceModal({ if (!dataSourceConfigurations && isOpen) { setDataSourceConfigurations(initialDataSourceConfigurations); } else if (!isOpen) { - setDataSourceConfigurations(null); - setSelectedDataSource(null); - setShouldDisplayFoldersScreen(false); - setShouldDisplayWebsitesScreen(false); + setSelectedDataSourceView(null); + setDisplayMode("PickKind"); } }, [dataSourceConfigurations, initialDataSourceConfigurations, isOpen]); @@ -110,176 +224,80 @@ export default function AssistantBuilderDataSourceModal({ return null; } - const allFolders = dataSources.filter((ds) => !ds.connectorProvider); const alreadySelectedFolders = Object.values(dataSourceConfigurations).filter( - (ds) => !ds.dataSource.connectorProvider - ); - - const allWebsites = dataSources.filter( - (ds) => ds.connectorProvider === "webcrawler" + (ds) => isFolder(ds.dataSourceView.dataSource) ); const alreadySelectedWebsites = Object.values( dataSourceConfigurations - ).filter((ds) => ds.dataSource.connectorProvider === "webcrawler"); + ).filter((ds) => isWebsite(ds.dataSourceView.dataSource)); return ( { - if (shouldDisplayFoldersScreen) { - setShouldDisplayFoldersScreen(false); - } else if (shouldDisplayWebsitesScreen) { - setShouldDisplayWebsitesScreen(false); - } else if (selectedDataSource !== null) { - setSelectedDataSource(null); - } else { + if (displayMode === "PickKind") { setOpen(false); + } else { + setDisplayMode("PickKind"); } }} onSave={() => { onSave(dataSourceConfigurations); setOpen(false); }} - hasChanged={ - selectedDataSource !== null || - shouldDisplayFoldersScreen || - shouldDisplayWebsitesScreen - } + hasChanged={displayMode !== "PickKind"} variant="full-screen" title="Manage data sources selection" >
- {!selectedDataSource && - !shouldDisplayFoldersScreen && - !shouldDisplayWebsitesScreen && ( - { - setSelectedDataSource(ds); - }} - onPickFolders={() => { - setShouldDisplayFoldersScreen(true); - }} - onPickWebsites={() => { - setShouldDisplayWebsitesScreen(true); - }} - /> - )} - {!selectedDataSource && - (shouldDisplayFoldersScreen || shouldDisplayWebsitesScreen) && ( - { - setDataSourceConfigurations((currentConfigurations) => { - if (currentConfigurations === null) { - // Unreachable - return null; - } - - if (contentNode === undefined) { - if (selected) { - return { - ...currentConfigurations, - [ds.name]: { - dataSource: ds, - // TODO(GROUPS_INFRA) Replace with DataSourceViewType once the UI has it. - dataSourceView: null, - selectedResources: [], - isSelectAll: true, - }, - }; - } else { - const newConfigurations = { ...currentConfigurations }; - delete newConfigurations[ds.name]; - return newConfigurations; - } - } - - return getUpdatedConfigurations( - currentConfigurations, - ds, - selected, - contentNode - ); - }); - }} - /> - )} - {selectedDataSource && ( - { + setSelectedDataSourceView(dsView); + setDisplayMode("SelectFromManaged"); + }} + onPickFolders={() => { + setDisplayMode("SelectFromFolder"); + }} + onPickWebsites={() => { + setDisplayMode("SelectFromWebsite"); + }} + /> + )} + + {displayMode === "SelectFromFolder" && ( + + )} + + {displayMode === "SelectFromWebsite" && ( + + )} + + {displayMode === "SelectFromManaged" && selectedDataSourceView && ( + { - setDataSourceConfigurations((currentConfigurations) => { - if (currentConfigurations === null) { - // Unreachable - return null; - } - - return getUpdatedConfigurations( - currentConfigurations, - selectedDataSource, - selected, - node - ); - }); - }} - toggleSelectAll={() => { - setDataSourceConfigurations((currentConfigurations) => { - if (currentConfigurations === null) { - // Unreachable - return null; - } - const oldConfiguration = currentConfigurations[ - selectedDataSource.name - ] || { - dataSource: selectedDataSource, - selectedResources: [], - isSelectAll: false, - }; - - const newConfiguration = { - ...oldConfiguration, - isSelectAll: !oldConfiguration.isSelectAll, - selectedResources: [], - } satisfies AssistantBuilderDataSourceConfiguration; - - const newConfigurations = { ...currentConfigurations }; - - if ( - newConfiguration.isSelectAll || - newConfiguration.selectedResources.length > 0 - ) { - newConfigurations[selectedDataSource.name] = newConfiguration; - } else { - delete newConfigurations[selectedDataSource.name]; - } - - return { - ...currentConfigurations, - [selectedDataSource.name]: newConfiguration, - }; - }); - }} + onSelectChange={onSelectChangeManaged} + toggleSelectAll={onToggleSelectAll} /> )}
diff --git a/front/components/assistant_builder/AssistantBuilderPreviewDrawer.tsx b/front/components/assistant_builder/AssistantBuilderPreviewDrawer.tsx index fac1f83eff13..52a117b1a44a 100644 --- a/front/components/assistant_builder/AssistantBuilderPreviewDrawer.tsx +++ b/front/components/assistant_builder/AssistantBuilderPreviewDrawer.tsx @@ -25,7 +25,7 @@ import { AssistantInputBar } from "@app/components/assistant/conversation/input_ import { usePreviewAssistant, useTryAssistantCore, -} from "@app/components/assistant/TryAssistant"; +} from "@app/components/assistant_builder/TryAssistant"; import type { AssistantBuilderSetActionType, AssistantBuilderState, diff --git a/front/components/assistant_builder/AssistantBuilderTablesModal.tsx b/front/components/assistant_builder/AssistantBuilderTablesModal.tsx index c82e8b3c9e09..d7b13aed61e3 100644 --- a/front/components/assistant_builder/AssistantBuilderTablesModal.tsx +++ b/front/components/assistant_builder/AssistantBuilderTablesModal.tsx @@ -1,27 +1,23 @@ import { Modal, Spinner } from "@dust-tt/sparkle"; import type { - ContentNode, CoreAPITable, - DataSourceType, + DataSourceViewType, + LightContentNode, WorkspaceType, } from "@dust-tt/types"; -import { - canContainStructuredData, - getMicrosoftSheetContentNodeInternalIdFromTableId, - isFolder, -} from "@dust-tt/types"; import { getGoogleSheetContentNodeInternalIdFromTableId, + getMicrosoftSheetContentNodeInternalIdFromTableId, getNotionDatabaseContentNodeInternalIdFromTableId, - getTableIdForContentNode, + isFolder, } from "@dust-tt/types"; import * as React from "react"; import { useEffect, useMemo, useState } from "react"; -import { AssistantBuilderContext } from "@app/components/assistant_builder/AssistantBuilderContext"; import PickDataSourceForTable from "@app/components/assistant_builder/PickDataSourceForTable"; import { PickTableInFolder } from "@app/components/assistant_builder/PickTableInFolder"; import { PickTablesManaged } from "@app/components/assistant_builder/PickTablesManaged"; +import { getTableIdForContentNode } from "@app/components/assistant_builder/shared"; import type { AssistantBuilderTableConfiguration } from "@app/components/assistant_builder/types"; import { useDataSourceNodes } from "@app/lib/swr"; @@ -36,33 +32,27 @@ export default function AssistantBuilderTablesModal({ setOpen: (isOpen: boolean) => void; onSave: ( params: AssistantBuilderTableConfiguration[], - dataSource: DataSourceType + dataSourceView: DataSourceViewType ) => void; owner: WorkspaceType; tablesQueryConfiguration: Record; }) { - const { dataSources } = React.useContext(AssistantBuilderContext); + const [selectedDataSourceView, setSelectedDataSourceOrView] = + useState(null); - const supportedDataSources = useMemo( - () => dataSources.filter(canContainStructuredData), - [dataSources] - ); - - const [selectedDataSource, setSelectedDataSource] = - useState(null); const [internalIds, setInternalIds] = useState([]); const [isSaving, setIsSaving] = useState(false); const loadedInternalIds = useMemo(() => { - if (!selectedDataSource || !selectedDataSource?.connectorId) { + if (!selectedDataSourceView?.dataSource.connectorId) { return []; } const tableConfigs = Object.values(tablesQueryConfiguration).filter( - (c) => c.dataSourceId === selectedDataSource.name + (c) => c.dataSourceId === selectedDataSourceView.dataSource.name ); return tableConfigs.map((c) => { - switch (selectedDataSource.connectorProvider) { + switch (selectedDataSourceView.dataSource.connectorProvider) { case "google_drive": return getGoogleSheetContentNodeInternalIdFromTableId(c.tableId); case "notion": @@ -71,20 +61,20 @@ export default function AssistantBuilderTablesModal({ return getMicrosoftSheetContentNodeInternalIdFromTableId(c.tableId); default: throw new Error( - `Unsupported connector provider: ${selectedDataSource.connectorProvider}` + `Unsupported connector provider: ${selectedDataSourceView.dataSource.connectorProvider}` ); } }); - }, [selectedDataSource, tablesQueryConfiguration]); + }, [selectedDataSourceView, tablesQueryConfiguration]); useEffect(() => { setInternalIds(loadedInternalIds); }, [loadedInternalIds]); - const key = selectedDataSource + const key = selectedDataSourceView ? { workspaceId: owner.sId, - dataSourceName: selectedDataSource.name, + dataSourceName: selectedDataSourceView.dataSource.name, internalIds, } : { @@ -104,22 +94,77 @@ export default function AssistantBuilderTablesModal({ const selectedManagedTables = nodes.contentNodes; const parentsById = nodes.parentsById; + const onManagedSelectionChange = useMemo( + () => + ( + dataSourceView: DataSourceViewType, + previousNodes: LightContentNode[], + node: LightContentNode, + parents: string[], + selected: boolean + ) => { + setInternalIds((internalIds) => { + const newIds = internalIds.filter((id) => id !== node.internalId); + const newNodes = previousNodes.filter( + (n) => n.internalId !== node.internalId + ); + const newParentsById = Object.entries( + parentsById as Record> + ).reduce( + (acc, [key, value]) => + key === node.internalId + ? acc + : { + ...acc, + [key]: value, + }, + {} as Record> + ); + + if (selected) { + newIds.push(node.internalId); + newNodes.push(node); + newParentsById[node.internalId] = new Set( + // This is to get the same structure/order in the fallback as the endpoint return, from leaf to root, including leaf. + [...parents, node.internalId].reverse() + ); + } + // Optimistic update + const key = serializeUseDataSourceKey({ + workspaceId: owner.sId, + dataSourceName: dataSourceView.dataSource.name, + internalIds: newIds, + }); + setFallback((prev) => ({ + ...prev, + [key]: { + contentNodes: newNodes, + parentsById: newParentsById, + }, + })); + return newIds; + }); + }, + [owner.sId, parentsById, serializeUseDataSourceKey] + ); + async function save() { - if (!selectedDataSource || !selectedManagedTables) { + if (!selectedDataSourceView || !selectedManagedTables) { return; } setIsSaving(true); try { const tableIds = selectedManagedTables.map((n) => - getTableIdForContentNode(n) + getTableIdForContentNode(selectedDataSourceView.dataSource, n) ); const tables = await Promise.all( tableIds.map(async (id) => { const tableRes = await fetch( - `/api/w/${owner.sId}/data_sources/${selectedDataSource.name}/tables/${id}` + `/api/w/${owner.sId}/data_sources/${selectedDataSourceView.dataSource.name}/tables/${id}` ); + // TODO(GROUPS_INFRA): Move to data_source_views endpoint const { table } = (await tableRes.json()) as { table: CoreAPITable }; return table; }) @@ -131,18 +176,22 @@ export default function AssistantBuilderTablesModal({ tableName: table.name, })); - onSave(configs, selectedDataSource); + onSave(configs, selectedDataSourceView); } finally { setIsSaving(false); } } - const onClose = () => { - setOpen(false); - setTimeout(() => { - setSelectedDataSource(null); - setIsSaving(false); - }, 200); + const onClose = (e?: Event, forceClose: boolean = false) => { + if (selectedDataSourceView !== null && !forceClose) { + setSelectedDataSourceOrView(null); + } else { + setOpen(false); + setTimeout(() => { + setSelectedDataSourceOrView(null); + setIsSaving(false); + }, 200); + } }; return ( @@ -151,7 +200,7 @@ export default function AssistantBuilderTablesModal({ onClose={onClose} isSaving={isSaving} onSave={() => { - void save().then(onClose); + void save().then(() => onClose(undefined, true)); }} hasChanged={loadedInternalIds !== internalIds} variant="full-screen" @@ -160,17 +209,16 @@ export default function AssistantBuilderTablesModal({
{!selectedManagedTables ? ( - ) : !selectedDataSource ? ( + ) : !selectedDataSourceView ? ( { - setSelectedDataSource(ds); + onPick={(dsView: DataSourceViewType) => { + setSelectedDataSourceOrView(dsView); }} /> - ) : isFolder(selectedDataSource) ? ( + ) : isFolder(selectedDataSourceView.dataSource) ? ( { const config = { workspaceId: owner.sId, @@ -178,73 +226,27 @@ export default function AssistantBuilderTablesModal({ tableId: table.table_id, tableName: table.name, }; - onSave([config], selectedDataSource); + onSave([config], selectedDataSourceView); onClose(); }} onBack={() => { - setSelectedDataSource(null); + setSelectedDataSourceOrView(null); }} tablesQueryConfiguration={tablesQueryConfiguration} /> ) : ( - { - setInternalIds((internalIds) => { - const newIds = internalIds.filter( - (id) => id !== node.internalId - ); - const newNodes = selectedManagedTables.filter( - (n) => n.internalId !== node.internalId - ); - const newParentsById = Object.entries( - parentsById as Record> - ).reduce( - (acc, [key, value]) => - key === node.internalId - ? acc - : { - ...acc, - [key]: value, - }, - {} as Record> - ); - - if (selected) { - newIds.push(node.internalId); - newNodes.push(node); - newParentsById[node.internalId] = new Set( - // This is to get the same structure/order in the fallback as the endpoint return, from leaf to root, including leaf. - [...parents, node.internalId].reverse() - ); - } - // Optimistic update - const key = serializeUseDataSourceKey({ - workspaceId: owner.sId, - dataSourceName: selectedDataSource.name, - internalIds: newIds, - }); - setFallback((prev) => ({ - ...prev, - [key]: { - contentNodes: newNodes, - parentsById: newParentsById, - }, - })); - return newIds; - }); - }} - selectedNodes={selectedManagedTables} - onBack={() => { - setSelectedDataSource(null); - }} - parentsById={parentsById || {}} - /> + selectedDataSourceView && ( + { + setSelectedDataSourceOrView(null); + }} + parentsById={parentsById} + /> + ) )}
diff --git a/front/components/assistant_builder/DataSourceSelectionSection.tsx b/front/components/assistant_builder/DataSourceSelectionSection.tsx index 35214077719c..d8c6c3b1038a 100644 --- a/front/components/assistant_builder/DataSourceSelectionSection.tsx +++ b/front/components/assistant_builder/DataSourceSelectionSection.tsx @@ -5,7 +5,7 @@ import { IconButton, Tree, } from "@dust-tt/sparkle"; -import type { DataSourceType, WorkspaceType } from "@dust-tt/types"; +import type { DataSourceViewType, WorkspaceType } from "@dust-tt/types"; import { useContext, useState } from "react"; import { AssistantBuilderContext } from "@app/components/assistant_builder/AssistantBuilderContext"; @@ -13,7 +13,7 @@ import type { AssistantBuilderDataSourceConfiguration } from "@app/components/as import { PermissionTreeChildren } from "@app/components/ConnectorPermissionsTree"; import { EmptyCallToAction } from "@app/components/EmptyCallToAction"; import ManagedDataSourceDocumentModal from "@app/components/ManagedDataSourceDocumentModal"; -import { CONNECTOR_CONFIGURATIONS } from "@app/lib/connector_providers"; +import { getConnectorProviderLogo } from "@app/lib/connector_providers"; import { getDisplayNameForDataSource } from "@app/lib/data_sources"; import { useConnectorPermissions } from "@app/lib/swr"; import { classNames } from "@app/lib/utils"; @@ -31,20 +31,20 @@ export default function DataSourceSelectionSection({ openDataSourceModal: () => void; onDelete?: (name: string) => void; }) { - const { dataSources } = useContext(AssistantBuilderContext); + const { dataSourceViews } = useContext(AssistantBuilderContext); const [documentToDisplay, setDocumentToDisplay] = useState( null ); - const [dataSourceToDisplay, setDataSourceToDisplay] = - useState(null); + const [dataSourceViewToDisplay, setDataSourceViewToDisplay] = + useState(null); - const canAddDataSource = dataSources.length > 0; + const canAddDataSource = dataSourceViews.length > 0; return ( <> { @@ -82,16 +82,20 @@ export default function DataSourceSelectionSection({ ) : ( {Object.values(dataSourceConfigurations).map((dsConfig) => { - const LogoComponent = dsConfig.dataSource?.connectorProvider - ? CONNECTOR_CONFIGURATIONS[ - dsConfig.dataSource.connectorProvider - ].logoComponent - : null; + const LogoComponent = getConnectorProviderLogo( + dsConfig.dataSourceView.dataSource.connectorProvider + ); return ( @@ -103,12 +107,12 @@ export default function DataSourceSelectionSection({ {dsConfig.isSelectAll && ( { - setDataSourceToDisplay(dsConfig.dataSource); + setDataSourceViewToDisplay(dsConfig.dataSourceView); setDocumentToDisplay(documentId); }} useConnectorPermissionsHook={useConnectorPermissions} @@ -118,7 +122,7 @@ export default function DataSourceSelectionSection({ {dsConfig.selectedResources.map((node) => { return ( { if (node.dustDocumentId) { - setDataSourceToDisplay(dsConfig.dataSource); + setDataSourceViewToDisplay( + dsConfig.dataSourceView + ); setDocumentToDisplay(node.dustDocumentId); } }} @@ -163,12 +169,12 @@ export default function DataSourceSelectionSection({ > { - setDataSourceToDisplay(dsConfig.dataSource); + setDataSourceViewToDisplay(dsConfig.dataSourceView); setDocumentToDisplay(documentId); }} useConnectorPermissionsHook={useConnectorPermissions} diff --git a/front/components/assistant_builder/DataSourceResourceSelector.tsx b/front/components/assistant_builder/DataSourceViewResourceSelector.tsx similarity index 63% rename from front/components/assistant_builder/DataSourceResourceSelector.tsx rename to front/components/assistant_builder/DataSourceViewResourceSelector.tsx index a841da255266..6bc32432f903 100644 --- a/front/components/assistant_builder/DataSourceResourceSelector.tsx +++ b/front/components/assistant_builder/DataSourceViewResourceSelector.tsx @@ -1,36 +1,43 @@ import { Page, SliderToggle } from "@dust-tt/sparkle"; import type { ConnectorProvider, - ContentNode, - DataSourceType, - WorkspaceType, + DataSourceViewType, + LightContentNode, + LightWorkspaceType, } from "@dust-tt/types"; import { Transition } from "@headlessui/react"; -import { CONNECTOR_PROVIDER_TO_RESOURCE_NAME } from "@app/components/assistant_builder/shared"; +import { getConnectorProviderResourceName } from "@app/components/assistant_builder/shared"; import DataSourceResourceSelectorTree from "@app/components/DataSourceResourceSelectorTree"; import { useParentResourcesById } from "@app/hooks/useParentResourcesById"; -import { CONNECTOR_CONFIGURATIONS } from "@app/lib/connector_providers"; +import { + CONNECTOR_CONFIGURATIONS, + getConnectorProviderLogo, +} from "@app/lib/connector_providers"; import { getDisplayNameForDataSource } from "@app/lib/data_sources"; -export default function DataSourceResourceSelector({ - dataSource, +export default function DataSourceViewResourceSelector({ + dataSourceView, owner, selectedResources, isSelectAll, onSelectChange, toggleSelectAll, }: { - dataSource: DataSourceType | null; - owner: WorkspaceType; - selectedResources: ContentNode[]; + dataSourceView: DataSourceViewType; + owner: LightWorkspaceType; + selectedResources: LightContentNode[]; isSelectAll: boolean; - onSelectChange: (resource: ContentNode, selected: boolean) => void; - toggleSelectAll: () => void; + onSelectChange: ( + dsView: DataSourceViewType, + resource: LightContentNode, + selected: boolean + ) => void; + toggleSelectAll: (dsView: DataSourceViewType) => void; }) { const { parentsById, setParentsById } = useParentResourcesById({ owner, - dataSource, + dataSource: dataSourceView.dataSource, internalIds: selectedResources.map((r) => r.internalId), }); @@ -39,20 +46,20 @@ export default function DataSourceResourceSelector({ ]; return ( - + - {dataSource && ( + {dataSourceView && (
@@ -60,7 +67,7 @@ export default function DataSourceResourceSelector({ { - toggleSelectAll(); + toggleSelectAll(dataSourceView); setParentsById({}); }} size="xs" @@ -69,18 +76,20 @@ export default function DataSourceResourceSelector({
Select from available{" "} - {CONNECTOR_PROVIDER_TO_RESOURCE_NAME[ - dataSource.connectorProvider as ConnectorProvider - ]?.plural ?? "resources"} - : + {getConnectorProviderResourceName( + dataSourceView.dataSource + .connectorProvider as ConnectorProvider, + true + )}
r.internalId)} @@ -95,7 +104,7 @@ export default function DataSourceResourceSelector({ } return newParentsById; }); - onSelectChange(node, selected); + onSelectChange(dataSourceView, node, selected); }} parentIsSelected={isSelectAll} /> diff --git a/front/components/assistant_builder/FolderOrWebsiteResourceSelector.tsx b/front/components/assistant_builder/FolderOrWebsiteResourceSelector.tsx index 81e22d578a61..d483df178a82 100644 --- a/front/components/assistant_builder/FolderOrWebsiteResourceSelector.tsx +++ b/front/components/assistant_builder/FolderOrWebsiteResourceSelector.tsx @@ -6,13 +6,15 @@ import { Tree, } from "@dust-tt/sparkle"; import type { - ContentNode, - DataSourceType, + DataSourceViewType, + LightContentNode, WorkspaceType, } from "@dust-tt/types"; +import { isFolder, isWebsite } from "@dust-tt/types"; import { Transition } from "@headlessui/react"; -import { useState } from "react"; +import { useContext, useState } from "react"; +import { AssistantBuilderContext } from "@app/components/assistant_builder/AssistantBuilderContext"; import FolderOrWebsiteTree from "@app/components/assistant_builder/FolderOrWebsiteTree"; import type { AssistantBuilderDataSourceConfiguration } from "@app/components/assistant_builder/types"; import { subFilter } from "@app/lib/utils"; @@ -20,25 +22,30 @@ import { subFilter } from "@app/lib/utils"; export default function FolderOrWebsiteResourceSelector({ owner, type, - dataSources, selectedNodes, onSelectChange, }: { type: "folder" | "website"; owner: WorkspaceType; - dataSources: DataSourceType[]; selectedNodes: AssistantBuilderDataSourceConfiguration[]; onSelectChange: ( - ds: DataSourceType, + dsView: DataSourceViewType, selected: boolean, - resource?: ContentNode + resource?: LightContentNode ) => void; }) { const [query, setQuery] = useState(""); + const { dataSourceViews } = useContext(AssistantBuilderContext); - const filteredDataSources = dataSources.filter((ds) => { - return subFilter(query.toLowerCase(), ds.name.toLowerCase()); - }); + const filteredDsViews = dataSourceViews + .filter((dsView) => + type === "folder" + ? isFolder(dsView.dataSource) + : isWebsite(dsView.dataSource) + ) + .filter((ds) => { + return subFilter(query.toLowerCase(), ds.dataSource.name.toLowerCase()); + }); return ( @@ -59,21 +66,24 @@ export default function FolderOrWebsiteResourceSelector({
-
Select from available folders:
+
+ Select from available{" "} + {type === "folder" ? "folders" : "websites"}: +
- {filteredDataSources.map((dataSource) => { + {filteredDsViews.map((dsView) => { const currentConfig = selectedNodes.find( - (selectedNode) => selectedNode.dataSource.id === dataSource.id + (selectedNode) => selectedNode.dataSourceView.id === dsView.id ); return ( void; }) { const selectedResources = currentConfig?.selectedResources ?? []; const { parentsById, setParentsById } = useParentResourcesById({ owner, - dataSource, + dataSource: dataSourceView.dataSource, internalIds: selectedResources.map((r) => r.internalId), }); @@ -41,7 +41,7 @@ export default function FolderOrWebsiteTree({ return ( : } className="whitespace-nowrap" @@ -53,7 +53,7 @@ export default function FolderOrWebsiteTree({ : false, onChange: (checked) => { setParentsById({}); - onSelectChange(dataSource, checked); + onSelectChange(dataSourceView, checked); }, }} > @@ -61,7 +61,7 @@ export default function FolderOrWebsiteTree({ { @@ -74,7 +74,7 @@ export default function FolderOrWebsiteTree({ } return newParentsById; }); - onSelectChange(dataSource, selected, resource); + onSelectChange(dataSourceView, selected, resource); }} selectedResourceIds={selectedResources.map((r) => r.internalId)} /> diff --git a/front/components/assistant_builder/PickDataSource.tsx b/front/components/assistant_builder/PickDataSource.tsx index 9b6f36e59cf9..df8936d26dc4 100644 --- a/front/components/assistant_builder/PickDataSource.tsx +++ b/front/components/assistant_builder/PickDataSource.tsx @@ -6,60 +6,63 @@ import { Item, Page, } from "@dust-tt/sparkle"; -import type { DataSourceType } from "@dust-tt/types"; +import type { DataSourceViewType } from "@dust-tt/types"; +import { isFolder, isManaged, isWebsite } from "@dust-tt/types"; import { Transition } from "@headlessui/react"; +import { useContext } from "react"; -import { orderDatasourceByImportance } from "@app/lib/assistant"; -import { CONNECTOR_CONFIGURATIONS } from "@app/lib/connector_providers"; +import { AssistantBuilderContext } from "@app/components/assistant_builder/AssistantBuilderContext"; +import { orderDatasourceViewByImportance } from "@app/lib/assistant"; +import { getConnectorProviderLogoWithFallback } from "@app/lib/connector_providers"; import { getDisplayNameForDataSource } from "@app/lib/data_sources"; export default function PickDataSource({ - dataSources, - show, onPick, onPickFolders, onPickWebsites, }: { - dataSources: DataSourceType[]; - show: boolean; - onPick: (dataSource: DataSourceType) => void; + onPick: (dataSourceView: DataSourceViewType) => void; onPickFolders: () => void; onPickWebsites: () => void; }) { - const managedDataSources = dataSources.filter( - (ds) => ds.connectorProvider && ds.connectorProvider !== "webcrawler" + const { dataSourceViews } = useContext(AssistantBuilderContext); + + // We'll use dataSourceViews to get the data sources that are managed + const managedDataSourceViews = dataSourceViews.filter((dsView) => + isManaged(dsView.dataSource) ); // We want to display the folders & websites as a single parent entry // so we take them out of the list of data sources - const shouldDisplayFolderEntry = dataSources.some( - (ds) => !ds.connectorProvider + const shouldDisplayFolderEntry = dataSourceViews.some((dsView) => + isFolder(dsView.dataSource) ); - const shouldDisplayWebsiteEntry = dataSources.some( - (ds) => ds.connectorProvider === "webcrawler" + const shouldDisplayWebsiteEntry = dataSourceViews.some((dsView) => + isWebsite(dsView.dataSource) ); return ( - + - {orderDatasourceByImportance(managedDataSources).map((ds) => ( - { - onPick(ds); - }} - /> - ))} + {orderDatasourceViewByImportance(managedDataSourceViews).map( + (dsView) => ( + { + onPick(dsView); + }} + /> + ) + )} {shouldDisplayFolderEntry && ( void; + onPick: (source: DataSourceViewType) => void; }) { + const { dataSourceViews } = useContext(AssistantBuilderContext); + + const supportedDataSourceViews = useMemo( + () => + dataSourceViews.filter((dsView) => + canContainStructuredData(dsView.dataSource) + ), + [dataSourceViews] + ); + const [query, setQuery] = useState(""); - const filtered = dataSources.filter((ds) => { - return subFilter(query.toLowerCase(), ds.name.toLowerCase()); - }); + const nameFilter = (dsv: DataSourceViewType) => { + return subFilter(query.toLowerCase(), dsv.dataSource.name.toLowerCase()); + }; return ( - + - {orderDatasourceByImportance(filtered).map((ds) => ( + {orderDatasourceViewByImportance( + supportedDataSourceViews.filter((dsView) => nameFilter(dsView)) + ).map((dsView) => ( { - onPick(ds); + onPick(dsView); }} /> ))} diff --git a/front/components/assistant_builder/PickTableInFolder.tsx b/front/components/assistant_builder/PickTableInFolder.tsx index bc764789689e..43eeac39b586 100644 --- a/front/components/assistant_builder/PickTableInFolder.tsx +++ b/front/components/assistant_builder/PickTableInFolder.tsx @@ -1,7 +1,7 @@ import { Button, Item, Page, Searchbar, ServerIcon } from "@dust-tt/sparkle"; import type { CoreAPITable, - DataSourceType, + DataSourceViewType, WorkspaceType, } from "@dust-tt/types"; import { Transition } from "@headlessui/react"; @@ -9,33 +9,33 @@ import * as React from "react"; import { useMemo, useState } from "react"; import type { AssistantBuilderTableConfiguration } from "@app/components/assistant_builder/types"; -import { CONNECTOR_CONFIGURATIONS } from "@app/lib/connector_providers"; +import { getConnectorProviderLogoWithFallback } from "@app/lib/connector_providers"; import { useTables } from "@app/lib/swr"; import { compareForFuzzySort, subFilter } from "@app/lib/utils"; export const PickTableInFolder = ({ owner, - dataSource, + dataSourceView, onPick, onBack, tablesQueryConfiguration, }: { owner: WorkspaceType; - dataSource: DataSourceType; + dataSourceView: DataSourceViewType; onPick: (table: CoreAPITable) => void; onBack?: () => void; tablesQueryConfiguration: Record; }) => { const { tables } = useTables({ workspaceId: owner.sId, - dataSourceName: dataSource.name, + dataSourceName: dataSourceView.dataSource.name, }); const [query, setQuery] = useState(""); const tablesToDisplay = tables.filter( (t) => !tablesQueryConfiguration?.[ - `${owner.sId}/${dataSource.name}/${t.table_id}` + `${owner.sId}/${dataSourceView.dataSource.name}/${t.table_id}` ] ); const filtered = useMemo( @@ -49,7 +49,7 @@ export const PickTableInFolder = ({ const isAllSelected = !!tables.length && !tablesToDisplay.length; return ( - + {isAllSelected && ( @@ -82,12 +82,10 @@ export const PickTableInFolder = ({ return ( { onPick(table); diff --git a/front/components/assistant_builder/PickTablesManaged.tsx b/front/components/assistant_builder/PickTablesManaged.tsx index b4e7f347e981..767cb059d28f 100644 --- a/front/components/assistant_builder/PickTablesManaged.tsx +++ b/front/components/assistant_builder/PickTablesManaged.tsx @@ -1,7 +1,7 @@ import { Page, ServerIcon } from "@dust-tt/sparkle"; import type { - ContentNode, - DataSourceType, + DataSourceViewType, + LightContentNode, WorkspaceType, } from "@dust-tt/types"; import { Transition } from "@headlessui/react"; @@ -11,29 +11,31 @@ import DataSourceResourceSelectorTree from "@app/components/DataSourceResourceSe export const PickTablesManaged = ({ owner, - dataSource, + dataSourceView, onSelectionChange, selectedNodes, parentsById, }: { owner: WorkspaceType; - dataSource: DataSourceType; + dataSourceView: DataSourceViewType; onSelectionChange: ( - resource: ContentNode, + dsView: DataSourceViewType, + nodes: LightContentNode[], + resource: LightContentNode, parents: string[], selected: boolean ) => void; onBack?: () => void; - selectedNodes: ContentNode[]; - parentsById: Record>; + selectedNodes: LightContentNode[]; + parentsById: Record> | undefined; }) => { return ( - + [...c])), + ...new Set(Object.values(parentsById || {}).flatMap((c) => [...c])), ]} filterPermission="read" - viewType={"tables"} - onSelectChange={onSelectionChange} + viewType="tables" + onSelectChange={( + resource: LightContentNode, + parents: string[], + selected: boolean + ) => + onSelectionChange( + dataSourceView, + selectedNodes, + resource, + parents, + selected + ) + } /> diff --git a/front/components/assistant/Sharing.tsx b/front/components/assistant_builder/Sharing.tsx similarity index 98% rename from front/components/assistant/Sharing.tsx rename to front/components/assistant_builder/Sharing.tsx index 7273c7c5e142..eb3d04054301 100644 --- a/front/components/assistant/Sharing.tsx +++ b/front/components/assistant_builder/Sharing.tsx @@ -18,15 +18,15 @@ import { import type { AgentConfigurationScope, AgentConfigurationType, - DataSourceType, + DataSourceViewType, WorkspaceType, } from "@dust-tt/types"; import { isBuilder } from "@dust-tt/types"; import { useState } from "react"; -import type { SlackChannel } from "@app/components/assistant/SlackIntegration"; -import { SlackIntegration } from "@app/components/assistant/SlackIntegration"; import { assistantUsageMessage } from "@app/components/assistant/Usage"; +import type { SlackChannel } from "@app/components/assistant_builder/SlackIntegration"; +import { SlackIntegration } from "@app/components/assistant_builder/SlackIntegration"; import { useAgentConfiguration, useAgentUsage } from "@app/lib/swr"; type ConfirmationModalDataType = { @@ -119,7 +119,7 @@ export function SharingButton({ setNewScope: (scope: NonGlobalScope) => void; baseUrl: string; showSlackIntegration: boolean; - slackDataSource: DataSourceType | null; + slackDataSource: DataSourceViewType | null; slackChannelSelected: SlackChannel[]; setNewLinkedSlackChannels: (channels: SlackChannel[]) => void; }) { @@ -464,7 +464,7 @@ function SlackIntegrationDrawer({ }: { show: boolean; onClose: () => void; - slackDataSource: DataSourceType; + slackDataSource: DataSourceViewType; owner: WorkspaceType; existingSelection: SlackChannel[]; onSave: (channels: SlackChannel[]) => void; diff --git a/front/components/assistant/SlackIntegration.tsx b/front/components/assistant_builder/SlackIntegration.tsx similarity index 94% rename from front/components/assistant/SlackIntegration.tsx rename to front/components/assistant_builder/SlackIntegration.tsx index 52d1254d4b3a..ff5089ed2ce6 100644 --- a/front/components/assistant/SlackIntegration.tsx +++ b/front/components/assistant_builder/SlackIntegration.tsx @@ -1,8 +1,7 @@ import "react-image-crop/dist/ReactCrop.css"; import { Modal, Page } from "@dust-tt/sparkle"; -import type { WorkspaceType } from "@dust-tt/types"; -import type { DataSourceType } from "@dust-tt/types"; +import type { DataSourceViewType, WorkspaceType } from "@dust-tt/types"; import { useEffect, useState } from "react"; import React from "react"; @@ -20,7 +19,7 @@ export function SlackIntegration({ onSave, onClose, }: { - slackDataSource: DataSourceType; + slackDataSource: DataSourceViewType; owner: WorkspaceType; existingSelection: SlackChannel[]; show: boolean; @@ -70,7 +69,7 @@ export function SlackIntegration({ /> { setHasChanged(true); diff --git a/front/components/assistant_builder/TablesSelectionSection.tsx b/front/components/assistant_builder/TablesSelectionSection.tsx index 01bad36cb461..471b566c0dd1 100644 --- a/front/components/assistant_builder/TablesSelectionSection.tsx +++ b/front/components/assistant_builder/TablesSelectionSection.tsx @@ -24,8 +24,8 @@ export default function TablesSelectionSection({ openTableModal: () => void; onDelete?: (sId: string) => void; }) { - const { dataSources } = useContext(AssistantBuilderContext); - const canSelectTable = dataSources.length > 0; + const { dataSourceViews } = useContext(AssistantBuilderContext); + const canSelectTable = dataSourceViews.length > 0; return ( - actionConfiguration.dataSourceConfigurations[k].dataSource - .connectorProvider === null + actionConfiguration.dataSourceConfigurations[k].dataSourceView + .dataSource.connectorProvider === null ) && Object.keys(actionConfiguration.dataSourceConfigurations).length > 0; const generateSchemaFromInstructions = async () => { diff --git a/front/components/assistant_builder/actions/TablesQueryAction.tsx b/front/components/assistant_builder/actions/TablesQueryAction.tsx index 46af27e82e3f..0c41fa916fe0 100644 --- a/front/components/assistant_builder/actions/TablesQueryAction.tsx +++ b/front/components/assistant_builder/actions/TablesQueryAction.tsx @@ -46,13 +46,16 @@ export function ActionTablesQuery({ isOpen={showTableModal} setOpen={(isOpen) => setShowTableModal(isOpen)} owner={owner} - onSave={(tables, dataSource) => { + onSave={(tables, dataSourceView) => { setEdited(true); updateAction((previousAction) => { const newTables = { ...previousAction }; - if (dataSource.connectorId) { + if (dataSourceView.dataSource.connectorId) { Object.keys(newTables) - .filter((k) => newTables[k].dataSourceId === dataSource.name) + .filter( + (k) => + newTables[k].dataSourceId === dataSourceView.dataSource.name + ) .forEach((k) => delete newTables[k]); } for (const t of tables) { diff --git a/front/components/assistant_builder/server_side_props_helpers.ts b/front/components/assistant_builder/server_side_props_helpers.ts index 91e6dc53c01d..fee83a5d2e9e 100644 --- a/front/components/assistant_builder/server_side_props_helpers.ts +++ b/front/components/assistant_builder/server_side_props_helpers.ts @@ -2,6 +2,7 @@ import type { AgentConfigurationType, AppType, CoreAPITable, + DataSourceViewType, ProcessConfigurationType, RetrievalConfigurationType, TemplateAgentConfigurationType, @@ -32,17 +33,36 @@ import { getDefaultTablesQueryActionConfiguration, getDefaultWebsearchActionConfiguration, } from "@app/components/assistant_builder/types"; +import { getApps } from "@app/lib/api/app"; import config from "@app/lib/api/config"; +import type { Authenticator } from "@app/lib/auth"; import { tableKey } from "@app/lib/client/tables_query"; -import type { DataSourceResource } from "@app/lib/resources/data_source_resource"; +import { DataSourceViewResource } from "@app/lib/resources/data_source_view_resource"; +import { VaultResource } from "@app/lib/resources/vault_resource"; import logger from "@app/logger/logger"; +export const getAccessibleSourcesAndApps = async (auth: Authenticator) => { + const accessibleVaults = [ + await VaultResource.fetchWorkspaceGlobalVault(auth), + ]; + + const [dsViews, allDustApps] = await Promise.all([ + DataSourceViewResource.listByVaults(auth, accessibleVaults), + getApps(auth), + ]); + + return { + dataSourceViews: dsViews.map((dsView) => dsView.toJSON()), + dustApps: allDustApps, + }; +}; + export async function buildInitialActions({ - dataSourcesByName, + dataSourceViews, dustApps, configuration, }: { - dataSourcesByName: Record; + dataSourceViews: DataSourceViewType[]; dustApps: AppType[]; configuration: AgentConfigurationType | TemplateAgentConfigurationType; }): Promise { @@ -53,15 +73,13 @@ export async function buildInitialActions({ action: RetrievalConfigurationType | ProcessConfigurationType ) => { const selectedResources: { - dataSourceId: string; - dataSourceViewId: string | null; + dataSourceViewId: string; resources: string[] | null; isSelectAll: boolean; }[] = []; for (const ds of action.dataSources) { selectedResources.push({ - dataSourceId: ds.dataSourceId, dataSourceViewId: ds.dataSourceViewId, resources: ds.filter.parents?.in ?? null, isSelectAll: !ds.filter.parents, @@ -72,12 +90,19 @@ export async function buildInitialActions({ await Promise.all( selectedResources.map( async (sr): Promise => { - const dataSourceResource = dataSourcesByName[sr.dataSourceId]; + const dataSourceView = dataSourceViews.find( + (dsv) => dsv.sId === sr.dataSourceViewId + ); - if (!dataSourceResource.connectorId || !sr.resources) { + if (!dataSourceView) { + throw new Error( + `Could not find DataSourceView with id ${sr.dataSourceViewId}` + ); + } + + if (!dataSourceView.dataSource.connectorId || !sr.resources) { return { - dataSource: dataSourceResource.toJSON(), - dataSourceView: null, + dataSourceView, selectedResources: [], isSelectAll: sr.isSelectAll, }; @@ -87,7 +112,7 @@ export async function buildInitialActions({ logger ); const response = await connectorsAPI.getContentNodes({ - connectorId: dataSourceResource.connectorId, + connectorId: dataSourceView.dataSource.connectorId, internalIds: sr.resources, }); @@ -96,8 +121,7 @@ export async function buildInitialActions({ } return { - dataSource: dataSourceResource.toJSON(), - dataSourceView: null, + dataSourceView, selectedResources: response.value.nodes, isSelectAll: sr.isSelectAll, }; @@ -107,7 +131,7 @@ export async function buildInitialActions({ // key: dataSourceName, value: DataSourceConfig const dataSourceConfigurations = dataSourceConfigurationsArray.reduce( - (acc, curr) => ({ ...acc, [curr.dataSource.name]: curr }), + (acc, curr) => ({ ...acc, [curr.dataSourceView.dataSource.name]: curr }), {} as Record ); @@ -159,10 +183,19 @@ export async function buildInitialActions({ const coreAPITables: CoreAPITable[] = await Promise.all( action.tables.map(async (t) => { - const dataSource = dataSourcesByName[t.dataSourceId]; + const dataSourceView = dataSourceViews.find( + (dsv) => dsv.dataSource.sId === t.dataSourceId + ); + + if (!dataSourceView) { + throw new Error( + `Could not find DataSourceView with id ${t.dataSourceId}` + ); + } + const coreAPITable = await coreAPI.getTable({ - projectId: dataSource.dustAPIProjectId, - dataSourceName: dataSource.name, + projectId: dataSourceView.dataSource.dustAPIProjectId, + dataSourceName: dataSourceView.dataSource.name, tableId: t.tableId, }); diff --git a/front/components/assistant_builder/shared.ts b/front/components/assistant_builder/shared.ts index a9930953f5cb..0a4d1f6ed91c 100644 --- a/front/components/assistant_builder/shared.ts +++ b/front/components/assistant_builder/shared.ts @@ -1,5 +1,15 @@ +import type { + DataSourceType, + LightContentNode, + TimeframeUnit, +} from "@dust-tt/types"; import type { ConnectorProvider } from "@dust-tt/types"; -import type { TimeframeUnit } from "@dust-tt/types"; +import { + getGoogleSheetTableIdFromContentNodeInternalId, + getMicrosoftSheetContentNodeInternalIdFromTableId, + getNotionDatabaseTableIdFromContentNodeInternalId, + isGoogleSheetContentNodeInternalId, +} from "@dust-tt/types"; export const FILTERING_MODES = ["SEARCH", "TIMEFRAME"] as const; export type FilteringMode = (typeof FILTERING_MODES)[number]; @@ -15,7 +25,7 @@ export const TIME_FRAME_UNIT_TO_LABEL: Record = { year: "year(s)", }; -export const CONNECTOR_PROVIDER_TO_RESOURCE_NAME: Record< +const CONNECTOR_PROVIDER_TO_RESOURCE_NAME: Record< ConnectorProvider, { singular: string; @@ -32,6 +42,14 @@ export const CONNECTOR_PROVIDER_TO_RESOURCE_NAME: Record< webcrawler: { singular: "page", plural: "pages" }, }; +export const getConnectorProviderResourceName = ( + connectorProvider: ConnectorProvider, + plural: boolean +) => + CONNECTOR_PROVIDER_TO_RESOURCE_NAME[connectorProvider][ + plural ? "plural" : "singular" + ]; + export const DROID_AVATARS_BASE_PATH = "/static/droidavatar/"; export const DROID_AVATAR_FILES = [ @@ -255,3 +273,35 @@ export const EMOJI_AVATAR_BASE_URL = buildAvatarUrl( EMOJI_AVATARS_BASE_PATH, "" ); + +export function getTableIdForContentNode( + dataSource: DataSourceType, + contentNode: LightContentNode +): string { + if (contentNode.type !== "database") { + throw new Error(`ContentNode type ${contentNode.type} is not supported`); + } + switch (dataSource.connectorProvider) { + case "notion": + return getNotionDatabaseTableIdFromContentNodeInternalId( + contentNode.internalId + ); + case "google_drive": + if (!isGoogleSheetContentNodeInternalId(contentNode.internalId)) { + throw new Error( + `Googgle Drive ContentNode internalId ${contentNode.internalId} is not a Google Sheet internal ID` + ); + } + return getGoogleSheetTableIdFromContentNodeInternalId( + contentNode.internalId + ); + case "microsoft": + return getMicrosoftSheetContentNodeInternalIdFromTableId( + contentNode.internalId + ); + default: + throw new Error( + `Provider ${dataSource.connectorProvider} is not supported` + ); + } +} diff --git a/front/components/assistant_builder/submitAssistantBuilderForm.ts b/front/components/assistant_builder/submitAssistantBuilderForm.ts index a4a266cd1ec8..da187f786cc2 100644 --- a/front/components/assistant_builder/submitAssistantBuilderForm.ts +++ b/front/components/assistant_builder/submitAssistantBuilderForm.ts @@ -1,16 +1,15 @@ import type { AgentConfigurationType, LightAgentConfigurationType, - PostOrPatchAgentConfigurationRequestBodySchema, + PostOrPatchAgentConfigurationRequestBody, Result, WorkspaceType, } from "@dust-tt/types"; import { assertNever, Err, Ok } from "@dust-tt/types"; -import type * as t from "io-ts"; -import type { SlackChannel } from "@app/components/assistant/SlackIntegration"; import { isLegacyAssistantBuilderConfiguration } from "@app/components/assistant_builder/legacy_agent"; import { removeLeadingAt } from "@app/components/assistant_builder/NamingScreen"; +import type { SlackChannel } from "@app/components/assistant_builder/SlackIntegration"; import type { AssistantBuilderActionConfiguration, AssistantBuilderState, @@ -56,12 +55,10 @@ export async function submitAssistantBuilderForm({ } } - type BodyType = t.TypeOf< - typeof PostOrPatchAgentConfigurationRequestBodySchema + type ActionsType = NonNullable< + PostOrPatchAgentConfigurationRequestBody["assistant"]["actions"] >; - type ActionsType = NonNullable; - const map: (a: AssistantBuilderActionConfiguration) => ActionsType = (a) => { switch (a.type) { case "RETRIEVAL_SEARCH": @@ -82,10 +79,9 @@ export async function submitAssistantBuilderForm({ topK: "auto", dataSources: Object.values( a.configuration.dataSourceConfigurations - ).map(({ dataSource, selectedResources, isSelectAll }) => ({ - dataSourceId: dataSource.name, - // TODO(GROUPS_INFRA) Replace with DataSourceViewType once the UI has it. - dataSourceViewId: null, + ).map(({ dataSourceView, selectedResources, isSelectAll }) => ({ + dataSourceId: dataSourceView.dataSource.name, + dataSourceViewId: dataSourceView.sId, workspaceId: owner.sId, filter: { parents: !isSelectAll @@ -150,10 +146,9 @@ export async function submitAssistantBuilderForm({ description: a.description, dataSources: Object.values( a.configuration.dataSourceConfigurations - ).map(({ dataSource, selectedResources, isSelectAll }) => ({ - dataSourceId: dataSource.name, - // TODO(GROUPS_INFRA) Replace with DataSourceViewType once the UI has it. - dataSourceViewId: null, + ).map(({ dataSourceView, selectedResources, isSelectAll }) => ({ + dataSourceId: dataSourceView.dataSource.name, + dataSourceViewId: dataSourceView.sId, workspaceId: owner.sId, filter: { parents: !isSelectAll @@ -188,26 +183,25 @@ export async function submitAssistantBuilderForm({ ? undefined : builderState.maxStepsPerRun ?? undefined; - const body: t.TypeOf = - { - assistant: { - name: removeLeadingAt(handle), - pictureUrl: avatarUrl, - description: description, - instructions: instructions.trim(), - status: isDraft ? "draft" : "active", - scope: builderState.scope, - actions: actionParams, - model: { - modelId: builderState.generationSettings.modelSettings.modelId, - providerId: builderState.generationSettings.modelSettings.providerId, - temperature: builderState.generationSettings.temperature, - }, - maxStepsPerRun, - visualizationEnabled: builderState.visualizationEnabled, - templateId: builderState.templateId, + const body: PostOrPatchAgentConfigurationRequestBody = { + assistant: { + name: removeLeadingAt(handle), + pictureUrl: avatarUrl, + description: description, + instructions: instructions.trim(), + status: isDraft ? "draft" : "active", + scope: builderState.scope, + actions: actionParams, + model: { + modelId: builderState.generationSettings.modelSettings.modelId, + providerId: builderState.generationSettings.modelSettings.providerId, + temperature: builderState.generationSettings.temperature, }, - }; + maxStepsPerRun, + visualizationEnabled: builderState.visualizationEnabled, + templateId: builderState.templateId, + }, + }; const res = await fetch( !agentConfigurationId diff --git a/front/components/assistant_builder/types.ts b/front/components/assistant_builder/types.ts index 49eff1e14a8f..072dbad88a4c 100644 --- a/front/components/assistant_builder/types.ts +++ b/front/components/assistant_builder/types.ts @@ -2,9 +2,8 @@ import { CircleIcon, SquareIcon, TriangleIcon } from "@dust-tt/sparkle"; import type { AgentConfigurationScope, AppType, - ContentNode, - DataSourceType, DataSourceViewType, + LightContentNode, PlanType, ProcessSchemaPropertyType, SubscriptionType, @@ -38,9 +37,8 @@ export const ACTION_MODES = [ // Retrieval configuration export type AssistantBuilderDataSourceConfiguration = { - dataSource: DataSourceType; - dataSourceView: DataSourceViewType | null; - selectedResources: ContentNode[]; + dataSourceView: DataSourceViewType; + selectedResources: LightContentNode[]; isSelectAll: boolean; }; diff --git a/front/components/assistant_builder/useSlackChannels.tsx b/front/components/assistant_builder/useSlackChannels.tsx index 82548279b0a1..9d7256908c90 100644 --- a/front/components/assistant_builder/useSlackChannels.tsx +++ b/front/components/assistant_builder/useSlackChannels.tsx @@ -1,11 +1,11 @@ -import type { DataSourceType } from "@dust-tt/types"; +import type { DataSourceViewType } from "@dust-tt/types"; import { useEffect, useState } from "react"; -import type { SlackChannel } from "@app/components/assistant/SlackIntegration"; +import type { SlackChannel } from "@app/components/assistant_builder/SlackIntegration"; import { useSlackChannelsLinkedWithAgent } from "@app/lib/swr"; export function useSlackChannel({ - dataSources, + dataSourceViews, initialChannels, workspaceId, isPrivateAssistant, @@ -13,7 +13,7 @@ export function useSlackChannel({ isEdited, agentConfigurationId, }: { - dataSources: DataSourceType[]; + dataSourceViews: DataSourceViewType[]; initialChannels: SlackChannel[]; workspaceId: string; isPrivateAssistant: boolean; @@ -27,15 +27,15 @@ export function useSlackChannel({ const [slackChannelsInitialized, setSlackChannelsInitialized] = useState(false); - const slackDataSource = dataSources.find( - (ds) => ds.connectorProvider === "slack" + const slackDataSource = dataSourceViews.find( + (ds) => ds.dataSource.connectorProvider === "slack" ); // Retrieve all the slack channels that are linked with an agent. const { slackChannels: slackChannelsLinkedWithAgent } = useSlackChannelsLinkedWithAgent({ workspaceId, - dataSourceName: slackDataSource?.name ?? undefined, + dataSourceName: slackDataSource?.dataSource.name ?? undefined, disabled: !isBuilder, }); diff --git a/front/components/vaults/EditVaultManagedDatasourcesViews.tsx b/front/components/vaults/EditVaultManagedDatasourcesViews.tsx index e7a4b8603717..d0c7a334b09d 100644 --- a/front/components/vaults/EditVaultManagedDatasourcesViews.tsx +++ b/front/components/vaults/EditVaultManagedDatasourcesViews.tsx @@ -52,7 +52,7 @@ export function EditVaultManagedDataSourcesViews({ const promisesErrors = await Promise.all( selectedNodes.map(async (sDs) => { const existingViewForDs = vaultDataSourceViews.find( - (d) => d.name === sDs.name + (d) => d.dataSource.name === sDs.name ); const body = { @@ -143,7 +143,9 @@ export function EditVaultManagedDataSourcesViews({ }} owner={owner} systemVaultDataSourceViews={systemVaultDataSourceViews.filter( - (ds) => ds.connectorProvider && ds.connectorProvider !== "webcrawler" + (dsv) => + dsv.dataSource.connectorProvider && + dsv.dataSource.connectorProvider !== "webcrawler" )} onSave={async (selectedDataSources) => { await updateVaultDataSourceViews(selectedDataSources); diff --git a/front/components/vaults/VaultDataSourceViewContentList.tsx b/front/components/vaults/VaultDataSourceViewContentList.tsx index a506a0ba64b9..c7d24a85ab8b 100644 --- a/front/components/vaults/VaultDataSourceViewContentList.tsx +++ b/front/components/vaults/VaultDataSourceViewContentList.tsx @@ -69,13 +69,12 @@ export const VaultDataSourceViewContentList = ({ } ); - const rows: RowData[] = - vaultContent?.map((v) => ({ - ...v, - count: 0, - usage: 0, - onClick: () => onSelect(v.internalId), - })) || []; + const rows: RowData[] = vaultContent.map((v) => ({ + ...v, + count: 0, + usage: 0, + onClick: () => onSelect(v.internalId), + })); if (isVaultContentLoading) { return ( diff --git a/front/components/vaults/VaultManagedDatasourcesViewsModal.tsx b/front/components/vaults/VaultManagedDatasourcesViewsModal.tsx index 860b2b0f311f..730b89bbf0fa 100644 --- a/front/components/vaults/VaultManagedDatasourcesViewsModal.tsx +++ b/front/components/vaults/VaultManagedDatasourcesViewsModal.tsx @@ -9,7 +9,10 @@ import { useState } from "react"; import DataSourceResourceSelectorTree from "@app/components/DataSourceResourceSelectorTree"; import { useParentResourcesById } from "@app/hooks/useParentResourcesById"; -import { CONNECTOR_CONFIGURATIONS } from "@app/lib/connector_providers"; +import { + CONNECTOR_CONFIGURATIONS, + getConnectorProviderLogo, +} from "@app/lib/connector_providers"; export default function VaultManagedDataSourcesViewsModal({ isOpen, @@ -29,7 +32,7 @@ export default function VaultManagedDataSourcesViewsModal({ const [selectedNodes, setSelectedNodes] = useState( initialSelectedDataSources.map((ds) => ({ - name: ds.name, + name: ds.dataSource.name, parentsIn: ds.parentsIn ?? null, })) ); @@ -55,7 +58,7 @@ export default function VaultManagedDataSourcesViewsModal({ {systemVaultDataSourceViews.map((dataSourceView) => { return ( ds.name === dataSourceView.name + (ds) => ds.name === dataSourceView.dataSource.name ); // TODO(GROUPS_INFRA): useParentResourcesById should use views not data sources. const { parentsById, setParentsById } = useParentResourcesById({ owner, - dataSource: dataSourceView, + dataSource: dataSourceView.dataSource, internalIds: selectedNodesInDataSourceView?.parentsIn ?? [], }); @@ -119,7 +124,7 @@ function VaultManagedDataSourceViewsTree({ return ( : null} variant="folder" @@ -138,7 +143,7 @@ function VaultManagedDataSourceViewsTree({ // Setting selectedResources setSelectedNodes((prevState) => { const existingDs = prevState.find( - (ds) => ds.name === dataSourceView.name + (ds) => ds.name === dataSourceView.dataSource.name ); if (checked) { @@ -146,7 +151,7 @@ function VaultManagedDataSourceViewsTree({ existingDs.parentsIn = null; // null means select all. } else { prevState.push({ - name: dataSourceView.name, + name: dataSourceView.dataSource.name, parentsIn: null, // null means select all. }); } @@ -163,12 +168,8 @@ function VaultManagedDataSourceViewsTree({ > { @@ -190,7 +191,9 @@ function VaultManagedDataSourceViewsTree({ // Setting selectedResources setSelectedNodes((prevState: ManagedDataSourceViewsSelectedNodes) => { if (selected) { - const dsv = prevState.find((v) => v.name === dataSourceView.name); + const dsv = prevState.find( + (v) => v.name === dataSourceView.dataSource.name + ); if (dsv) { if (dsv.parentsIn === null) { dsv.parentsIn = [node.internalId]; @@ -199,14 +202,16 @@ function VaultManagedDataSourceViewsTree({ } } else { prevState.push({ - name: dataSourceView.name, + name: dataSourceView.dataSource.name, parentsIn: [node.internalId], }); } return prevState; } - const dsv = prevState.find((v) => v.name === dataSourceView.name); + const dsv = prevState.find( + (v) => v.name === dataSourceView.dataSource.name + ); if (dsv && dsv.parentsIn) { dsv.parentsIn = dsv.parentsIn.filter( (id) => id !== node.internalId diff --git a/front/components/vaults/VaultResourcesList.tsx b/front/components/vaults/VaultResourcesList.tsx index d6fbff045b31..abd91597bb59 100644 --- a/front/components/vaults/VaultResourcesList.tsx +++ b/front/components/vaults/VaultResourcesList.tsx @@ -29,7 +29,7 @@ import VaultCreateFolderModal from "@app/components/vaults/VaultCreateFolderModa import VaultCreateWebsiteModal from "@app/components/vaults/VaultCreateWebsiteModal"; import { useSubmitFunction } from "@app/lib/client/utils"; import { - CONNECTOR_CONFIGURATIONS, + getConnectorProviderLogoWithFallback, getDataSourceNameFromView, } from "@app/lib/connector_providers"; import { useDataSources, useVaultDataSourceViews } from "@app/lib/swr"; @@ -166,14 +166,15 @@ export const VaultResourcesList = ({ sId: r.sId, category: r.category, label: getDataSourceNameFromView(r), - icon: r.connectorProvider - ? CONNECTOR_CONFIGURATIONS[r.connectorProvider].logoComponent - : FolderIcon, + icon: getConnectorProviderLogoWithFallback( + r.dataSource.connectorProvider, + FolderIcon + ), usage: r.usage, count: 0, editedByUser: r.editedByUser, workspaceId: owner.sId, - dataSourceName: r.name, + dataSourceName: r.dataSource.name, onClick: () => onSelect(r.sId), })) || []; diff --git a/front/components/vaults/VaultSideBarMenu.tsx b/front/components/vaults/VaultSideBarMenu.tsx index 6a54f611579c..ff6466dcf3d6 100644 --- a/front/components/vaults/VaultSideBarMenu.tsx +++ b/front/components/vaults/VaultSideBarMenu.tsx @@ -23,7 +23,7 @@ import type { ReactElement } from "react"; import { Fragment, useEffect, useState } from "react"; import { - CONNECTOR_CONFIGURATIONS, + getConnectorProviderLogoWithFallback, getDataSourceNameFromView, } from "@app/lib/connector_providers"; import { useVaultDataSourceViews, useVaultInfo, useVaults } from "@app/lib/swr"; @@ -314,11 +314,11 @@ const VaultDataSourceViewItem = ({ vault: VaultType; }): ReactElement => { const router = useRouter(); - const configuration = item.connectorProvider - ? CONNECTOR_CONFIGURATIONS[item.connectorProvider] - : null; - const LogoComponent = configuration?.logoComponent ?? FolderIcon; + const LogoComponent = getConnectorProviderLogoWithFallback( + item.dataSource.connectorProvider, + FolderIcon + ); const dataSourceViewPath = `/w/${owner.sId}/data-sources/vaults/${vault.sId}/categories/${item.category}/data_source_views/${item.sId}`; return ( diff --git a/front/hooks/useParentResourcesById.ts b/front/hooks/useParentResourcesById.ts index 266b8ba118d4..ce705da8321b 100644 --- a/front/hooks/useParentResourcesById.ts +++ b/front/hooks/useParentResourcesById.ts @@ -1,8 +1,4 @@ -import type { - DataSourceType, - DataSourceViewType, - WorkspaceType, -} from "@dust-tt/types"; +import type { DataSourceType, LightWorkspaceType } from "@dust-tt/types"; import { useCallback, useEffect, useState } from "react"; import type { GetContentNodeParentsResponseBody } from "@app/pages/api/w/[wId]/data_sources/[name]/managed/parents"; @@ -12,8 +8,8 @@ export function useParentResourcesById({ dataSource, internalIds, }: { - owner: WorkspaceType; - dataSource: DataSourceType | DataSourceViewType | null; + owner: LightWorkspaceType; + dataSource: DataSourceType | null; internalIds: string[]; }) { const [parentsById, setParentsById] = useState>>( @@ -27,7 +23,7 @@ export function useParentResourcesById({ try { const res = await fetch( `/api/w/${owner.sId}/data_sources/${encodeURIComponent( - dataSource?.name || "" + dataSource?.name ?? "" )}/managed/parents`, { method: "POST", diff --git a/front/lib/api/assistant/configuration/process.ts b/front/lib/api/assistant/configuration/process.ts index cdf7eac95fd0..7a89dbd68cc8 100644 --- a/front/lib/api/assistant/configuration/process.ts +++ b/front/lib/api/assistant/configuration/process.ts @@ -118,32 +118,15 @@ export async function fetchAgentProcessActionConfigurations({ function getDataSource( dataSourceConfig: AgentDataSourceConfiguration ): DataSourceConfiguration { - const { dataSourceView, dataSource } = dataSourceConfig; - - if (dataSourceView) { - return { - dataSourceViewId: DataSourceViewResource.modelIdToSId({ - id: dataSourceView.id, - workspaceId: dataSourceView.workspaceId, - }), - dataSourceId: dataSourceView.dataSourceForView.name, - workspaceId: dataSourceView.workspace.sId, - filter: { - parents: - dataSourceConfig.parentsIn && dataSourceConfig.parentsNotIn - ? { - in: dataSourceConfig.parentsIn, - not: dataSourceConfig.parentsNotIn, - } - : null, - }, - }; - } + const { dataSourceView } = dataSourceConfig; return { - dataSourceId: dataSource.name, - dataSourceViewId: null, - workspaceId: dataSource.workspace.sId, + dataSourceViewId: DataSourceViewResource.modelIdToSId({ + id: dataSourceView.id, + workspaceId: dataSourceView.workspaceId, + }), + dataSourceId: dataSourceView.dataSourceForView.name, + workspaceId: dataSourceView.workspace.sId, filter: { parents: dataSourceConfig.parentsIn && dataSourceConfig.parentsNotIn diff --git a/front/lib/api/assistant/configuration/retrieval.ts b/front/lib/api/assistant/configuration/retrieval.ts index 5ef543208ac2..dd722cbe96f2 100644 --- a/front/lib/api/assistant/configuration/retrieval.ts +++ b/front/lib/api/assistant/configuration/retrieval.ts @@ -125,32 +125,15 @@ export async function fetchAgentRetrievalActionConfigurations({ function getDataSource( dataSourceConfig: AgentDataSourceConfiguration ): DataSourceConfiguration { - const { dataSourceView, dataSource } = dataSourceConfig; - - if (dataSourceView) { - return { - dataSourceViewId: DataSourceViewResource.modelIdToSId({ - id: dataSourceView.id, - workspaceId: dataSourceView.workspaceId, - }), - dataSourceId: dataSourceView.dataSourceForView.name, - workspaceId: dataSourceView.workspace.sId, - filter: { - parents: - dataSourceConfig.parentsIn && dataSourceConfig.parentsNotIn - ? { - in: dataSourceConfig.parentsIn, - not: dataSourceConfig.parentsNotIn, - } - : null, - }, - }; - } + const { dataSourceView } = dataSourceConfig; return { - dataSourceId: dataSource.name, - dataSourceViewId: null, - workspaceId: dataSource.workspace.sId, + dataSourceViewId: DataSourceViewResource.modelIdToSId({ + id: dataSourceView.id, + workspaceId: dataSourceView.workspaceId, + }), + dataSourceId: dataSourceView.dataSourceForView.name, + workspaceId: dataSourceView.workspace.sId, filter: { parents: dataSourceConfig.parentsIn && dataSourceConfig.parentsNotIn diff --git a/front/lib/api/assistant/global_agents.ts b/front/lib/api/assistant/global_agents.ts index 9d42c068ad1b..a085bb490db4 100644 --- a/front/lib/api/assistant/global_agents.ts +++ b/front/lib/api/assistant/global_agents.ts @@ -10,7 +10,7 @@ import type { AgentConfigurationType, AgentModelConfigurationType, ConnectorProvider, - DataSourceType, + DataSourceViewType, GlobalAgentStatus, } from "@dust-tt/types"; import { @@ -24,28 +24,24 @@ import { getSmallWhitelistedModel, GPT_3_5_TURBO_MODEL_CONFIG, GPT_4O_MODEL_CONFIG, - isDevelopment, isProviderWhitelisted, MISTRAL_LARGE_MODEL_CONFIG, MISTRAL_MEDIUM_MODEL_CONFIG, MISTRAL_SMALL_MODEL_CONFIG, } from "@dust-tt/types"; -import { DustAPI } from "@dust-tt/types"; import { DEFAULT_BROWSE_ACTION_NAME, DEFAULT_RETRIEVAL_ACTION_NAME, DEFAULT_WEBSEARCH_ACTION_NAME, } from "@app/lib/api/assistant/actions/names"; -import config from "@app/lib/api/config"; import { GLOBAL_AGENTS_SID } from "@app/lib/assistant"; import type { Authenticator } from "@app/lib/auth"; -import { prodAPICredentialsForOwner } from "@app/lib/auth"; import { GlobalAgentSettings } from "@app/lib/models/assistant/agent"; +import { DataSourceViewResource } from "@app/lib/resources/data_source_view_resource"; +import { VaultResource } from "@app/lib/resources/vault_resource"; import logger from "@app/logger/logger"; -import { getDataSources } from "../data_sources"; - // Used when returning an agent with status 'disabled_by_admin' const dummyModelConfiguration = { providerId: GPT_4O_MODEL_CONFIG.providerId, @@ -53,6 +49,11 @@ const dummyModelConfiguration = { temperature: 0, }; +type PrefetchedDataSourcesType = { + dataSourceViews: DataSourceViewType[]; + workspaceId: string; +}; + class HelperAssistantPrompt { private static instance: HelperAssistantPrompt; private staticPrompt: string | null; @@ -86,32 +87,26 @@ class HelperAssistantPrompt { async function getDataSourcesAndWorkspaceIdForGlobalAgents( auth: Authenticator ): Promise<{ - dataSources: DataSourceType[]; + dataSourceViews: DataSourceViewType[]; workspaceId: string; }> { const owner = auth.getNonNullableWorkspace(); - if (isDevelopment()) { - const prodCredentials = await prodAPICredentialsForOwner(owner); - const api = new DustAPI(config.getDustAPIConfig(), prodCredentials, logger); + const defaultVaults = [await VaultResource.fetchWorkspaceGlobalVault(auth)]; + const dataSourceViews = await DataSourceViewResource.listByVaults( + auth, + defaultVaults + ); - const dsRes = await api.getDataSources(prodCredentials.workspaceId); - if (dsRes.isErr()) { - throw new Error("Failed to retrieve data sources."); - } - return { - dataSources: dsRes.value, - // We use prodCredentials to make sure we are using the right workspaceId. In development - // this is the production Dust use case, in production we use the current workspace. - workspaceId: prodCredentials.workspaceId, - }; - } else { - const dataSources = await getDataSources(auth); - return { - dataSources, - workspaceId: owner.sId, - }; - } + return { + dataSourceViews: dataSourceViews.map((dsv) => { + return { + ...dsv.toJSON(), + assistantDefaultSelected: dsv.dataSource.assistantDefaultSelected, + }; + }), + workspaceId: owner.sId, + }; } /** @@ -595,10 +590,7 @@ function _getManagedDataSourceAgent( description: string; instructions: string | null; pictureUrl: string; - preFetchedDataSources: { - dataSources: DataSourceType[]; - workspaceId: string; - }; + preFetchedDataSources: PrefetchedDataSourcesType; } ): AgentConfigurationType | null { const owner = auth.getNonNullableWorkspace(); @@ -641,11 +633,11 @@ function _getManagedDataSourceAgent( }; } - // Check if there's a data source for this agent - const filteredDataSources = preFetchedDataSources.dataSources.filter( - (d) => d.connectorProvider === connectorProvider + // Check if there's a data source view for this agent + const filteredDataSourceViews = preFetchedDataSources.dataSourceViews.filter( + (dsView) => dsView.dataSource.connectorProvider === connectorProvider ); - if (filteredDataSources.length === 0) { + if (filteredDataSourceViews.length === 0) { return { id: -1, sId: agentId, @@ -689,10 +681,9 @@ function _getManagedDataSourceAgent( query: "auto", relativeTimeFrame: "auto", topK: "auto", - // TODO(GROUPS_INFRA) Prefetch data source views for managed data sources. - dataSources: filteredDataSources.map((ds) => ({ - dataSourceId: ds.name, - dataSourceViewId: null, + dataSources: filteredDataSourceViews.map((dsView) => ({ + dataSourceId: dsView.dataSource.name, + dataSourceViewId: dsView.sId, workspaceId: preFetchedDataSources.workspaceId, filter: { tags: null, parents: null }, })), @@ -713,10 +704,7 @@ function _getGoogleDriveGlobalAgent( preFetchedDataSources, }: { settings: GlobalAgentSettings | null; - preFetchedDataSources: { - dataSources: DataSourceType[]; - workspaceId: string; - }; + preFetchedDataSources: PrefetchedDataSourcesType; } ): AgentConfigurationType | null { return _getManagedDataSourceAgent(auth, { @@ -740,10 +728,7 @@ function _getSlackGlobalAgent( preFetchedDataSources, }: { settings: GlobalAgentSettings | null; - preFetchedDataSources: { - dataSources: DataSourceType[]; - workspaceId: string; - }; + preFetchedDataSources: PrefetchedDataSourcesType; } ) { return _getManagedDataSourceAgent(auth, { @@ -767,10 +752,7 @@ function _getGithubGlobalAgent( preFetchedDataSources, }: { settings: GlobalAgentSettings | null; - preFetchedDataSources: { - dataSources: DataSourceType[]; - workspaceId: string; - }; + preFetchedDataSources: PrefetchedDataSourcesType; } ) { return _getManagedDataSourceAgent(auth, { @@ -795,10 +777,7 @@ function _getNotionGlobalAgent( preFetchedDataSources, }: { settings: GlobalAgentSettings | null; - preFetchedDataSources: { - dataSources: DataSourceType[]; - workspaceId: string; - }; + preFetchedDataSources: PrefetchedDataSourcesType; } ) { return _getManagedDataSourceAgent(auth, { @@ -822,10 +801,7 @@ function _getIntercomGlobalAgent( preFetchedDataSources, }: { settings: GlobalAgentSettings | null; - preFetchedDataSources: { - dataSources: DataSourceType[]; - workspaceId: string; - }; + preFetchedDataSources: PrefetchedDataSourcesType; } ) { return _getManagedDataSourceAgent(auth, { @@ -849,10 +825,7 @@ function _getDustGlobalAgent( preFetchedDataSources, }: { settings: GlobalAgentSettings | null; - preFetchedDataSources: { - dataSources: DataSourceType[]; - workspaceId: string; - }; + preFetchedDataSources: PrefetchedDataSourcesType; } ): AgentConfigurationType | null { const owner = auth.getNonNullableWorkspace(); @@ -904,11 +877,11 @@ function _getDustGlobalAgent( }; } - const dataSources = preFetchedDataSources.dataSources.filter( - (d) => d.assistantDefaultSelected === true + const dataSourceViews = preFetchedDataSources.dataSourceViews.filter( + (dsView) => dsView.dataSource.assistantDefaultSelected === true ); - if (dataSources.length === 0) { + if (dataSourceViews.length === 0) { return { id: -1, sId: GLOBAL_AGENTS_SID.DUST, @@ -946,10 +919,9 @@ The assistant always respects the mardown format and generates spaces to nest co query: "auto", relativeTimeFrame: "auto", topK: "auto", - // TODO(GROUPS_INFRA) Prefetch data source views for managed data sources. - dataSources: dataSources.map((ds) => ({ - dataSourceId: ds.name, - dataSourceViewId: null, + dataSources: dataSourceViews.map((dsView) => ({ + dataSourceId: dsView.dataSource.name, + dataSourceViewId: dsView.sId, workspaceId: preFetchedDataSources.workspaceId, filter: { parents: null }, })), @@ -969,26 +941,31 @@ The assistant always respects the mardown format and generates spaces to nest co 4. If the user's query require neither internal company data or recent public knowledge, the assistant is allowed to answer without using any tool. The assistant always respects the mardown format and generates spaces to nest content.`; - dataSources.forEach((ds) => { - if (ds.connectorProvider && ds.connectorProvider !== "webcrawler") { + dataSourceViews.forEach((dsView) => { + if ( + dsView.dataSource.connectorProvider && + dsView.dataSource.connectorProvider !== "webcrawler" + ) { actions.push({ id: -1, - sId: GLOBAL_AGENTS_SID.DUST + "-datasource-action-" + ds.name, + sId: + GLOBAL_AGENTS_SID.DUST + + "-datasource-action-" + + dsView.dataSource.name, type: "retrieval_configuration", query: "auto", relativeTimeFrame: "auto", topK: "auto", dataSources: [ { - // TODO(GROUPS_INFRA) Prefetch data source views for managed data sources. - dataSourceId: ds.name, - dataSourceViewId: null, + dataSourceId: dsView.dataSource.name, + dataSourceViewId: dsView.sId, workspaceId: preFetchedDataSources.workspaceId, filter: { parents: null }, }, ], - name: "search_" + ds.name, - description: `The user's ${ds.connectorProvider} data source.`, + name: "search_" + dsView.dataSource.name, + description: `The user's ${dsView.dataSource.connectorProvider} data source.`, }); } }); @@ -1038,10 +1015,7 @@ The assistant always respects the mardown format and generates spaces to nest co function getGlobalAgent( auth: Authenticator, sId: string | number, - preFetchedDataSources: { - dataSources: DataSourceType[]; - workspaceId: string; - }, + preFetchedDataSources: PrefetchedDataSourcesType, helperPromptInstance: HelperAssistantPrompt, globaAgentSettings: GlobalAgentSettings[] ): AgentConfigurationType | null { diff --git a/front/lib/api/vaults.ts b/front/lib/api/vaults.ts index 32f90cdb410f..dfe6dfa6ce58 100644 --- a/front/lib/api/vaults.ts +++ b/front/lib/api/vaults.ts @@ -129,6 +129,8 @@ export const getManagedDataSourceContent = async ( preventSelection: r.preventSelection, dustDocumentId: r.dustDocumentId, lastUpdatedAt: r.lastUpdatedAt, + titleWithParentsContext: r.titleWithParentsContext, + sourceUrl: r.sourceUrl, })); return new Ok(results); @@ -162,6 +164,7 @@ export const getUnmanagedDataSourceContent = async ( preventSelection: false, dustDocumentId: doc.document_id, lastUpdatedAt: doc.timestamp, + sourceUrl: doc.source_url || null, })); return new Ok(documentsAsContentNodes); } else { @@ -183,6 +186,7 @@ export const getUnmanagedDataSourceContent = async ( preventSelection: false, dustDocumentId: table.table_id, lastUpdatedAt: table.timestamp, + sourceUrl: null, })); return new Ok(tablesAsContentNodes); diff --git a/front/lib/assistant.ts b/front/lib/assistant.ts index 6ad97f06e1dc..815bcc980afb 100644 --- a/front/lib/assistant.ts +++ b/front/lib/assistant.ts @@ -1,6 +1,6 @@ import type { AgentModelConfigurationType, - DataSourceType, + ConnectorProvider, LightAgentConfigurationType, } from "@dust-tt/types"; import type { SupportedModel } from "@dust-tt/types"; @@ -150,40 +150,56 @@ export function compareAgentsForSort( return 0; // Default: keep the original order } +type ComparableByProvider = { connectorProvider: ConnectorProvider | null }; +function compareByImportance( + a: ComparableByProvider, + b: ComparableByProvider +): number { + const aConnector = a.connectorProvider; + const bConnector = b.connectorProvider; + + const order = Object.keys(CONNECTOR_CONFIGURATIONS) + .filter( + (key) => + CONNECTOR_CONFIGURATIONS[key as keyof typeof CONNECTOR_CONFIGURATIONS] + .connectorProvider !== + CONNECTOR_CONFIGURATIONS.webcrawler.connectorProvider + ) + .map( + (key) => + CONNECTOR_CONFIGURATIONS[key as keyof typeof CONNECTOR_CONFIGURATIONS] + .connectorProvider + ); + + if (aConnector === CONNECTOR_CONFIGURATIONS.webcrawler.connectorProvider) { + return 1; + } + + if (bConnector === CONNECTOR_CONFIGURATIONS.webcrawler.connectorProvider) { + return -1; + } + + const indexA = aConnector ? order.indexOf(aConnector) : order.length; + const indexB = bConnector ? order.indexOf(bConnector) : order.length; + + if (indexA === -1 && indexB === -1) { + return 0; + } + + return indexA - indexB; +} + // Order in the following format : connectorProvider > empty > webcrawler -export function orderDatasourceByImportance(dataSources: DataSourceType[]) { - return dataSources.sort((a, b) => { - const aConnector = a.connectorProvider; - const bConnector = b.connectorProvider; - - const order = Object.keys(CONNECTOR_CONFIGURATIONS) - .filter( - (key) => - CONNECTOR_CONFIGURATIONS[key as keyof typeof CONNECTOR_CONFIGURATIONS] - .connectorProvider !== - CONNECTOR_CONFIGURATIONS.webcrawler.connectorProvider - ) - .map( - (key) => - CONNECTOR_CONFIGURATIONS[key as keyof typeof CONNECTOR_CONFIGURATIONS] - .connectorProvider - ); - - if (aConnector === CONNECTOR_CONFIGURATIONS.webcrawler.connectorProvider) { - return 1; - } - - if (bConnector === CONNECTOR_CONFIGURATIONS.webcrawler.connectorProvider) { - return -1; - } - - const indexA = aConnector ? order.indexOf(aConnector) : order.length; - const indexB = bConnector ? order.indexOf(bConnector) : order.length; - - if (indexA === -1 && indexB === -1) { - return 0; - } - - return indexA - indexB; +export function orderDatasourceByImportance( + dataSources: Type[] +) { + return dataSources.sort(compareByImportance); +} + +export function orderDatasourceViewByImportance< + Type extends { dataSource: ComparableByProvider }, +>(dataSourceViews: Type[]) { + return dataSourceViews.sort((a, b) => { + return compareByImportance(a.dataSource, b.dataSource); }); } diff --git a/front/lib/connector_providers.ts b/front/lib/connector_providers.ts index 75f3b8fadb09..d74588460270 100644 --- a/front/lib/connector_providers.ts +++ b/front/lib/connector_providers.ts @@ -13,6 +13,8 @@ import type { DataSourceViewType, WhitelistableFeature, } from "@dust-tt/types"; +import type { LucideIcon } from "lucide-react"; +import type { SVGProps } from "react"; export const CONNECTOR_CONFIGURATIONS: Record< ConnectorProvider, @@ -142,9 +144,30 @@ export const CONNECTOR_CONFIGURATIONS: Record< }; export function getDataSourceNameFromView(dsv: DataSourceViewType): string { - if (dsv.category === "managed" && dsv.connectorProvider) { - return CONNECTOR_CONFIGURATIONS[dsv.connectorProvider].name; + if (dsv.category === "managed" && dsv.dataSource.connectorProvider) { + return CONNECTOR_CONFIGURATIONS[dsv.dataSource.connectorProvider].name; } - return dsv.name; + return dsv.dataSource.name; +} + +type LogoType = ((props: SVGProps) => JSX.Element) | LucideIcon; + +export function getConnectorProviderLogo( + provider: ConnectorProvider | null +): LogoType | null { + if (!provider) { + return null; + } + return CONNECTOR_CONFIGURATIONS[provider].logoComponent; +} + +export function getConnectorProviderLogoWithFallback( + provider: ConnectorProvider | null, + fallback: LogoType +): LogoType { + if (!provider) { + return fallback; + } + return CONNECTOR_CONFIGURATIONS[provider].logoComponent; } diff --git a/front/lib/resources/data_source_resource.ts b/front/lib/resources/data_source_resource.ts index 96e6b3bef606..b2afc4f1010a 100644 --- a/front/lib/resources/data_source_resource.ts +++ b/front/lib/resources/data_source_resource.ts @@ -159,11 +159,7 @@ export class DataSourceResource extends ResourceWithVault { } static async listByVault(auth: Authenticator, vault: VaultResource) { - return this.baseFetchWithAuthorization(auth, { - where: { - vaultId: vault.id, - }, - }); + return this.listByVaults(auth, [vault]); } static async listByVaults(auth: Authenticator, vaults: VaultResource[]) { diff --git a/front/lib/resources/data_source_view_resource.ts b/front/lib/resources/data_source_view_resource.ts index b385ad35dd0c..aa6aeefee90f 100644 --- a/front/lib/resources/data_source_view_resource.ts +++ b/front/lib/resources/data_source_view_resource.ts @@ -134,11 +134,7 @@ export class DataSourceViewResource extends ResourceWithVault = fetcher; @@ -202,7 +200,7 @@ export function useProviders({ } export function useSavedRunStatus( - owner: WorkspaceType, + owner: LightWorkspaceType, app: AppType, refresh: (data: GetRunStatusResponseBody | undefined) => number ) { @@ -223,7 +221,7 @@ export function useSavedRunStatus( } export function useRunBlock( - owner: WorkspaceType, + owner: LightWorkspaceType, app: AppType, runId: string, type: string, @@ -246,7 +244,7 @@ export function useRunBlock( }; } -export function useDustAppSecrets(owner: WorkspaceType) { +export function useDustAppSecrets(owner: LightWorkspaceType) { const keysFetcher: Fetcher = fetcher; const { data, error } = useSWRWithDefaults( `/api/w/${owner.sId}/dust_app_secrets`, @@ -260,7 +258,7 @@ export function useDustAppSecrets(owner: WorkspaceType) { }; } -export function useKeys(owner: WorkspaceType) { +export function useKeys(owner: LightWorkspaceType) { const keysFetcher: Fetcher = fetcher; const { data, error } = useSWRWithDefaults( `/api/w/${owner.sId}/keys`, @@ -275,7 +273,7 @@ export function useKeys(owner: WorkspaceType) { } export function useRuns( - owner: WorkspaceType, + owner: LightWorkspaceType, app: AppType, limit: number, offset: number, @@ -298,7 +296,7 @@ export function useRuns( } export function useDocuments( - owner: WorkspaceType, + owner: LightWorkspaceType, dataSource: { name: string }, limit: number, offset: number, @@ -324,7 +322,7 @@ export function useDocuments( } export function useDataSources( - owner: WorkspaceType, + owner: LightWorkspaceType, options = { disabled: false } ) { const { disabled } = options; @@ -357,7 +355,7 @@ export function useMembers(owner: LightWorkspaceType) { }; } -export function useAdmins(owner: WorkspaceType) { +export function useAdmins(owner: LightWorkspaceType) { const membersFetcher: Fetcher = fetcher; const { data, error, mutate } = useSWRWithDefaults( `/api/w/${owner.sId}/members?role=admin`, @@ -372,7 +370,7 @@ export function useAdmins(owner: WorkspaceType) { }; } -export function useWorkspaceInvitations(owner: WorkspaceType) { +export function useWorkspaceInvitations(owner: LightWorkspaceType) { const workspaceInvitationsFetcher: Fetcher = fetcher; const { data, error, mutate } = useSWRWithDefaults( @@ -420,7 +418,7 @@ export function useDataSourceContentNodes({ dataSource, internalIds, }: { - owner: WorkspaceType; + owner: LightWorkspaceType; dataSource: DataSourceType; internalIds: string[]; }): { @@ -455,14 +453,14 @@ export function useDataSourceContentNodes({ export function useConnectorPermissions({ owner, - dataSourceOrView, + dataSource, parentId, filterPermission, disabled, viewType = "documents", }: { - owner: WorkspaceType; - dataSourceOrView: DataSourceType | DataSourceViewType; + owner: LightWorkspaceType; + dataSource: DataSourceType; parentId: string | null; filterPermission: ConnectorPermission | null; disabled?: boolean; @@ -472,7 +470,7 @@ export function useConnectorPermissions({ fetcher; let url = `/api/w/${owner.sId}/data_sources/${encodeURIComponent( - dataSourceOrView.name + dataSource.name )}/managed/permissions?viewType=${viewType}`; if (parentId) { url += `&parentId=${parentId}`; @@ -495,13 +493,13 @@ export function useConnectorPermissions({ export function usePokeConnectorPermissions({ owner, - dataSourceOrView, + dataSource, parentId, filterPermission, disabled, }: { - owner: WorkspaceType; - dataSourceOrView: DataSourceType | DataSourceViewType; + owner: LightWorkspaceType; + dataSource: DataSourceType; parentId: string | null; filterPermission: ConnectorPermission | null; disabled?: boolean; @@ -509,7 +507,7 @@ export function usePokeConnectorPermissions({ const permissionsFetcher: Fetcher = fetcher; - let url = `/api/poke/workspaces/${owner.sId}/data_sources/${dataSourceOrView.name}/managed/permissions?viewType=documents`; + let url = `/api/poke/workspaces/${owner.sId}/data_sources/${encodeURIComponent(dataSource.name)}/managed/permissions?viewType=documents`; if (parentId) { url += `&parentId=${parentId}`; } @@ -523,7 +521,7 @@ export function usePokeConnectorPermissions({ ); return { - resources: data ? data.resources : [], + resources: useMemo(() => (data ? data.resources : []), [data]), isResourcesLoading: !error && !data, isResourcesError: error, }; @@ -534,7 +532,7 @@ export function useConnectorConfig({ dataSource, configKey, }: { - owner: WorkspaceType; + owner: LightWorkspaceType; dataSource: DataSourceType; configKey: string; }) { @@ -1185,7 +1183,6 @@ export function useWorkspaceEnterpriseConnection({ mutateEnterpriseConnection: mutate, }; } - interface UseDataSourceKey { workspaceId: string; dataSourceName: string; @@ -1193,14 +1190,14 @@ interface UseDataSourceKey { } interface UseDataSourceResult { - contentNodes: ContentNode[]; + contentNodes: LightContentNode[]; parentsById: Record>; } export function useDataSourceNodes( key: UseDataSourceKey, options?: SWRConfiguration<{ - contentNodes: ContentNode[]; + contentNodes: LightContentNode[]; parentsById: Record>; }> ) { @@ -1389,7 +1386,7 @@ export function useVaultDataSourceViewContent({ ); return { - vaultContent: data?.nodes, + vaultContent: useMemo(() => data?.nodes ?? [], [data]), isVaultContentLoading: !error && !data, isVaultContentError: error, }; diff --git a/front/pages/w/[wId]/assistant/labs/transcripts/index.tsx b/front/pages/w/[wId]/assistant/labs/transcripts/index.tsx index 88d23b8b3c83..be8b4c1a19bb 100644 --- a/front/pages/w/[wId]/assistant/labs/transcripts/index.tsx +++ b/front/pages/w/[wId]/assistant/labs/transcripts/index.tsx @@ -63,8 +63,8 @@ export const getServerSideProps = withDefaultUserAuthRequirements<{ const dataSourcesViews = globalDataSourceViews .map((dsv) => dsv.toJSON()) - .filter((dsv) => !dsv.connectorId) - .sort((a, b) => a.name.localeCompare(b.name)); + .filter((dsv) => !dsv.dataSource.connectorId) + .sort((a, b) => a.dataSource.name.localeCompare(b.dataSource.name)); if ( !owner || @@ -268,12 +268,13 @@ export default function LabsTranscriptsIndex({ if (dataSource) { successMessage = - "The transcripts will be stored in the folder " + dataSource.name; + "The transcripts will be stored in the folder " + + dataSource.dataSource.name; } await makePatchRequest( transcriptConfigurationId, { - dataSourceId: dataSource ? dataSource.name : null, + dataSourceId: dataSource ? dataSource.dataSource.name : null, }, successMessage ); @@ -639,8 +640,8 @@ export default function LabsTranscriptsIndex({ className="flex w-full items-center justify-between rounded-md border border-gray-300 bg-white px-4 py-2 text-left text-sm font-medium shadow-sm hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-indigo-500 focus:ring-offset-2" disabled={!transcriptsConfigurationState.isActive} > - {transcriptsConfigurationState?.dataSource?.name || - "Do not store transcripts"} + {transcriptsConfigurationState?.dataSource?.dataSource + .name || "Do not store transcripts"}