Skip to content

Commit b80f4aa

Browse files
fishwww-wwc121914yu
fishwww-ww
and
c121914yu
authored
feat: improve global variables(time, file, dataset) (#5804)
* feat: improve global variables(time, file, dataset) * feat: optimize code * perf: time variables code * fix: model, file * fix: hide file upload * fix: ts * hide dataset select --------- Co-authored-by: archer <[email protected]>
1 parent e2ddc05 commit b80f4aa

File tree

34 files changed

+1287
-562
lines changed

34 files changed

+1287
-562
lines changed

‎packages/global/common/file/tools.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { detect } from 'jschardet';
22
import { documentFileType } from './constants';
33
import { ChatFileTypeEnum } from '../../core/chat/constants';
4-
import { type UserChatItemValueItemType } from '../../core/chat/type';
4+
import { type UserChatItemFileItemType } from '../../core/chat/type';
55
import * as fs from 'fs';
66

77
export const formatFileSize = (bytes: number): string => {
@@ -36,7 +36,7 @@ export const detectFileEncodingByPath = async (path: string) => {
3636
};
3737

3838
// Url => user upload file type
39-
export const parseUrlToFileType = (url: string): UserChatItemValueItemType['file'] | undefined => {
39+
export const parseUrlToFileType = (url: string): UserChatItemFileItemType | undefined => {
4040
if (typeof url !== 'string') return;
4141

4242
// Handle base64 image
@@ -74,13 +74,13 @@ export const parseUrlToFileType = (url: string): UserChatItemValueItemType['file
7474
// Default to image type for non-document files
7575
return {
7676
type: ChatFileTypeEnum.image,
77-
name: filename || 'null.png',
77+
name: filename || 'null',
7878
url
7979
};
8080
} catch (error) {
8181
return {
82-
type: ChatFileTypeEnum.image,
83-
name: 'invalid.png',
82+
type: ChatFileTypeEnum.file,
83+
name: url,
8484
url
8585
};
8686
}

‎packages/global/core/app/constants.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,3 +77,36 @@ export const defaultFileExtensionTypes = {
7777
canSelectCustomFileExtension: []
7878
};
7979
export type FileExtensionKeyType = keyof typeof defaultFileExtensionTypes;
80+
export const getUploadFileType = ({
81+
canSelectFile,
82+
canSelectImg,
83+
canSelectVideo,
84+
canSelectAudio,
85+
canSelectCustomFileExtension,
86+
customFileExtensionList
87+
}: {
88+
canSelectFile?: boolean;
89+
canSelectImg?: boolean;
90+
canSelectVideo?: boolean;
91+
canSelectAudio?: boolean;
92+
canSelectCustomFileExtension?: boolean;
93+
customFileExtensionList?: string[];
94+
}) => {
95+
const types: string[] = [];
96+
if (canSelectFile) {
97+
types.push(...defaultFileExtensionTypes.canSelectFile);
98+
}
99+
if (canSelectImg) {
100+
types.push(...defaultFileExtensionTypes.canSelectImg);
101+
}
102+
if (canSelectVideo) {
103+
types.push(...defaultFileExtensionTypes.canSelectVideo);
104+
}
105+
if (canSelectAudio) {
106+
types.push(...defaultFileExtensionTypes.canSelectAudio);
107+
}
108+
if (canSelectCustomFileExtension && customFileExtensionList) {
109+
types.push(...customFileExtensionList);
110+
}
111+
return types.join(', ');
112+
};

‎packages/global/core/app/type.d.ts‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,7 @@ export type SettingAIDataType = {
170170
};
171171

172172
// variable
173-
export type VariableItemType = {
173+
export type VariableItemType = AppFileSelectConfigType & {
174174
// id: string;
175175
key: string;
176176
label: string;
@@ -189,13 +189,8 @@ export type VariableItemType = {
189189
min?: number;
190190
// select
191191
list?: { label: string; value: string }[];
192-
// file
193-
canSelectFile?: boolean;
194-
canSelectImg?: boolean;
195-
maxFiles?: number;
196192
// timeSelect
197193
timeGranularity?: 'second' | 'minute' | 'hour' | 'day';
198-
timeType?: 'point' | 'range';
199194
timeRangeStart?: string;
200195
timeRangeEnd?: string;
201196

@@ -241,7 +236,7 @@ export type AppAutoExecuteConfigType = {
241236
};
242237
// File
243238
export type AppFileSelectConfigType = {
244-
maxFiles: number;
239+
maxFiles?: number;
245240
canSelectFile?: boolean;
246241
customPdfParse?: boolean;
247242
canSelectImg?: boolean;

‎packages/global/core/chat/type.d.ts‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -51,17 +51,18 @@ export type ChatWithAppSchema = Omit<ChatSchemaType, 'appId'> & {
5151
};
5252

5353
/* --------- chat item ---------- */
54+
export type UserChatItemFileItemType = {
55+
type: `${ChatFileTypeEnum}`;
56+
name?: string;
57+
key?: string;
58+
url: string;
59+
};
5460
export type UserChatItemValueItemType = {
5561
type: ChatItemValueTypeEnum.text | ChatItemValueTypeEnum.file;
5662
text?: {
5763
content: string;
5864
};
59-
file?: {
60-
type: `${ChatFileTypeEnum}`;
61-
name?: string;
62-
key?: string;
63-
url: string;
64-
};
65+
file?: UserChatItemFileItemType;
6566
};
6667
export type UserChatItemType = {
6768
obj: ChatRoleEnum.Human;

‎packages/global/core/workflow/constants.ts‎

Lines changed: 21 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -328,7 +328,7 @@ export enum VariableInputEnum {
328328
password = 'password',
329329
file = 'file',
330330

331-
modelSelect = 'modelSelect',
331+
llmSelect = 'llmSelect',
332332
datasetSelect = 'datasetSelect',
333333

334334
custom = 'custom',
@@ -386,20 +386,26 @@ export const variableConfigs: VariableConfigType[][] = [
386386
label: i18nT('common:core.workflow.inputType.switch'),
387387
value: VariableInputEnum.switch,
388388
defaultValueType: WorkflowIOValueTypeEnum.boolean
389+
},
390+
{
391+
icon: 'core/workflow/inputType/timePointSelect',
392+
label: i18nT('common:core.workflow.inputType.timePointSelect'),
393+
value: VariableInputEnum.timePointSelect,
394+
defaultValueType: WorkflowIOValueTypeEnum.string
395+
},
396+
{
397+
icon: 'core/workflow/inputType/timeRangeSelect',
398+
label: i18nT('common:core.workflow.inputType.timeRangeSelect'),
399+
value: VariableInputEnum.timeRangeSelect,
400+
defaultValueType: WorkflowIOValueTypeEnum.arrayString
401+
},
402+
{
403+
icon: 'core/workflow/inputType/model',
404+
label: i18nT('common:core.workflow.inputType.modelSelect'),
405+
value: VariableInputEnum.llmSelect,
406+
defaultValueType: WorkflowIOValueTypeEnum.string
389407
}
390408
// {
391-
// icon: 'core/workflow/inputType/timePointSelect',
392-
// label: i18nT('common:core.workflow.inputType.timePointSelect'),
393-
// value: VariableInputEnum.timePointSelect,
394-
// defaultValueType: WorkflowIOValueTypeEnum.string
395-
// },
396-
// {
397-
// icon: 'core/workflow/inputType/timeRangeSelect',
398-
// label: i18nT('common:core.workflow.inputType.timeRangeSelect'),
399-
// value: VariableInputEnum.timeRangeSelect,
400-
// defaultValueType: WorkflowIOValueTypeEnum.arrayString
401-
// }
402-
// {
403409
// icon: 'core/workflow/inputType/file',
404410
// label: i18nT('common:core.workflow.inputType.file'),
405411
// value: VariableInputEnum.file,
@@ -410,14 +416,14 @@ export const variableConfigs: VariableConfigType[][] = [
410416
// {
411417
// icon: 'core/workflow/inputType/model',
412418
// label: i18nT('common:core.workflow.inputType.modelSelect'),
413-
// value: VariableInputEnum.modelSelect,
419+
// value: VariableInputEnum.llmSelect,
414420
// defaultValueType: WorkflowIOValueTypeEnum.string
415421
// },
416422
// {
417423
// icon: 'core/workflow/inputType/dataset',
418424
// label: i18nT('common:core.workflow.inputType.datasetSelect'),
419425
// value: VariableInputEnum.datasetSelect,
420-
// defaultValueType: WorkflowIOValueTypeEnum.arrayString
426+
// defaultValueType: WorkflowIOValueTypeEnum.selectDataset
421427
// }
422428
// ],
423429
[

‎packages/global/core/workflow/runtime/utils.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -367,7 +367,7 @@ export const checkNodeRunStatus = ({
367367

368368
// Classify edges
369369
const { commonEdges, recursiveEdgeGroups } = splitNodeEdges(node);
370-
console.log(JSON.stringify({ commonEdges, recursiveEdgeGroups }, null, 2));
370+
371371
// Entry
372372
if (commonEdges.length === 0 && recursiveEdgeGroups.length === 0) {
373373
return 'run';

‎packages/global/core/workflow/utils.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -262,7 +262,7 @@ export const appData2FlowNodeIO = ({
262262
[VariableInputEnum.switch]: [FlowNodeInputTypeEnum.switch],
263263
[VariableInputEnum.password]: [FlowNodeInputTypeEnum.password],
264264
[VariableInputEnum.file]: [FlowNodeInputTypeEnum.fileSelect],
265-
[VariableInputEnum.modelSelect]: [FlowNodeInputTypeEnum.selectLLMModel],
265+
[VariableInputEnum.llmSelect]: [FlowNodeInputTypeEnum.selectLLMModel],
266266
[VariableInputEnum.datasetSelect]: [FlowNodeInputTypeEnum.selectDataset],
267267
[VariableInputEnum.internal]: [FlowNodeInputTypeEnum.hidden],
268268
[VariableInputEnum.custom]: [FlowNodeInputTypeEnum.input, FlowNodeInputTypeEnum.reference]

‎packages/service/core/workflow/dispatch/index.ts‎

Lines changed: 32 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ import type { ChatNodeUsageType } from '@fastgpt/global/support/wallet/bill/type
4343
import { addLog } from '../../../common/system/log';
4444
import { surrenderProcess } from '../../../common/system/tools';
4545
import type { DispatchFlowResponse, WorkflowDebugResponse } from './type';
46-
import { rewriteRuntimeWorkFlow, removeSystemVariable } from './utils';
46+
import { rewriteRuntimeWorkFlow, runtimeSystemVar2StoreType } from './utils';
4747
import { getHandleId } from '@fastgpt/global/core/workflow/utils';
4848
import { callbackMap } from './constants';
4949
import { anyValueDecrypt } from '../../../common/secret/utils';
@@ -79,7 +79,16 @@ export async function dispatchWorkFlow({
7979
concatUsage,
8080
...data
8181
}: Props & WorkflowUsageProps): Promise<DispatchFlowResponse> {
82-
const { res, stream, runningUserInfo, runningAppInfo, lastInteractive, histories, query } = data;
82+
const {
83+
res,
84+
stream,
85+
runningUserInfo,
86+
runningAppInfo,
87+
lastInteractive,
88+
histories,
89+
query,
90+
chatConfig
91+
} = data;
8392

8493
await checkTeamAIPoints(runningUserInfo.teamId);
8594
const [{ timezone, externalProvider }, newUsageId] = await Promise.all([
@@ -141,14 +150,15 @@ export async function dispatchWorkFlow({
141150
}
142151

143152
// Get default variables
153+
144154
const defaultVariables = {
145155
...externalProvider.externalWorkflowVariables,
146-
...getSystemVariables({
156+
...(await getSystemVariables({
147157
...data,
148158
query,
149159
histories,
150160
timezone
151-
})
161+
}))
152162
};
153163

154164
// Init some props
@@ -211,11 +221,11 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
211221
[DispatchNodeResponseKeyEnum.runTimes]: 1,
212222
[DispatchNodeResponseKeyEnum.assistantResponses]: [],
213223
[DispatchNodeResponseKeyEnum.toolResponses]: null,
214-
newVariables: removeSystemVariable(
224+
[DispatchNodeResponseKeyEnum.newVariables]: runtimeSystemVar2StoreType({
215225
variables,
216-
externalProvider.externalWorkflowVariables,
217-
data.chatConfig?.variables
218-
),
226+
removeObj: externalProvider.externalWorkflowVariables,
227+
userVariablesConfigs: data.chatConfig?.variables
228+
}),
219229
durationSeconds: 0
220230
};
221231
}
@@ -983,12 +993,6 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
983993
});
984994
}
985995

986-
const encryptedNewVariables = removeSystemVariable(
987-
variables,
988-
externalProvider.externalWorkflowVariables,
989-
data.chatConfig?.variables
990-
);
991-
992996
return {
993997
flowResponses: workflowQueue.chatResponses,
994998
flowUsages: workflowQueue.chatNodeUsages,
@@ -999,7 +1003,11 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
9991003
workflowQueue.chatAssistantResponse
10001004
),
10011005
[DispatchNodeResponseKeyEnum.toolResponses]: workflowQueue.toolRunResponse,
1002-
[DispatchNodeResponseKeyEnum.newVariables]: encryptedNewVariables,
1006+
[DispatchNodeResponseKeyEnum.newVariables]: runtimeSystemVar2StoreType({
1007+
variables,
1008+
removeObj: externalProvider.externalWorkflowVariables,
1009+
userVariablesConfigs: data.chatConfig?.variables
1010+
}),
10031011
[DispatchNodeResponseKeyEnum.memories]:
10041012
Object.keys(workflowQueue.system_memories).length > 0
10051013
? workflowQueue.system_memories
@@ -1009,7 +1017,7 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
10091017
};
10101018

10111019
/* get system variable */
1012-
const getSystemVariables = ({
1020+
const getSystemVariables = async ({
10131021
timezone,
10141022
runningAppInfo,
10151023
chatId,
@@ -1020,30 +1028,29 @@ const getSystemVariables = ({
10201028
variables
10211029
}: Props & {
10221030
timezone: string;
1023-
}): SystemVariablesType => {
1031+
}): Promise<SystemVariablesType> => {
10241032
// Get global variables(Label -> key; Key -> key)
10251033
const variablesConfig = chatConfig?.variables || [];
10261034

1027-
const variablesMap = variablesConfig.reduce<Record<string, any>>((acc, item) => {
1035+
const variablesMap: Record<string, any> = {};
1036+
for await (const item of variablesConfig) {
10281037
// For internal variables, ignore external input and use default value
10291038
if (item.type === VariableInputEnum.password) {
10301039
const val = variables[item.label] || variables[item.key] || item.defaultValue;
10311040
const actualValue = anyValueDecrypt(val);
1032-
acc[item.key] = valueTypeFormat(actualValue, item.valueType);
1041+
variablesMap[item.key] = valueTypeFormat(actualValue, item.valueType);
10331042
}
1034-
10351043
// API
10361044
else if (variables[item.label] !== undefined) {
1037-
acc[item.key] = valueTypeFormat(variables[item.label], item.valueType);
1045+
variablesMap[item.key] = valueTypeFormat(variables[item.label], item.valueType);
10381046
}
10391047
// Web
10401048
else if (variables[item.key] !== undefined) {
1041-
acc[item.key] = valueTypeFormat(variables[item.key], item.valueType);
1049+
variablesMap[item.key] = valueTypeFormat(variables[item.key], item.valueType);
10421050
} else {
1043-
acc[item.key] = valueTypeFormat(item.defaultValue, item.valueType);
1051+
variablesMap[item.key] = valueTypeFormat(item.defaultValue, item.valueType);
10441052
}
1045-
return acc;
1046-
}, {});
1053+
}
10471054

10481055
return {
10491056
...variablesMap,

‎packages/service/core/workflow/dispatch/tools/runUpdateVar.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import {
1111
} from '@fastgpt/global/core/workflow/runtime/utils';
1212
import { type TUpdateListItem } from '@fastgpt/global/core/workflow/template/system/variableUpdate/type';
1313
import { type ModuleDispatchProps } from '@fastgpt/global/core/workflow/runtime/type';
14-
import { removeSystemVariable } from '../utils';
14+
import { runtimeSystemVar2StoreType } from '../utils';
1515
import { isValidReferenceValue } from '@fastgpt/global/core/workflow/utils';
1616
import { valueTypeFormat } from '@fastgpt/global/core/workflow/runtime/utils';
1717

@@ -92,11 +92,11 @@ export const dispatchUpdateVariable = async (props: Props): Promise<Response> =>
9292
if (!runningAppInfo.isChildApp) {
9393
workflowStreamResponse?.({
9494
event: SseResponseEventEnum.updateVariables,
95-
data: removeSystemVariable(
95+
data: runtimeSystemVar2StoreType({
9696
variables,
97-
externalProvider.externalWorkflowVariables,
98-
chatConfig?.variables
99-
)
97+
removeObj: externalProvider.externalWorkflowVariables,
98+
userVariablesConfigs: chatConfig?.variables
99+
})
100100
});
101101
}
102102

0 commit comments

Comments
 (0)