Skip to content

Commit b4dda6a

Browse files
c121914yuc121914yu
authored
fix: Check the url to avoid ssrf attacks (#3965)
* fix: Check the url to avoid ssrf attacks * Delete docSite/content/zh-cn/docs/development/upgrading/490.md
1 parent e860c56 commit b4dda6a

File tree

4 files changed

+208
-53
lines changed

4 files changed

+208
-53
lines changed

‎packages/plugins/src/fetchUrl/template.json‎

Lines changed: 117 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
"nodeId": "lmpb9v2lo2lk",
1717
"name": "插件开始",
1818
"intro": "自定义配置外部输入，使用插件时，仅暴露自定义配置的输入",
19-
"avatar": "/imgs/workflow/input.png",
19+
"avatar": "core/workflow/template/workflowStart",
2020
"flowNodeType": "pluginInput",
2121
"showStatus": false,
2222
"position": {
@@ -26,14 +26,16 @@
2626
"version": "481",
2727
"inputs": [
2828
{
29-
"renderTypeList": ["reference"],
29+
"renderTypeList": ["input", "reference"],
3030
"selectedTypeIndex": 0,
3131
"valueType": "string",
3232
"key": "url",
3333
"label": "url",
3434
"description": "需要读取的网页链接",
3535
"required": true,
36-
"toolDescription": "需要读取的网页链接"
36+
"toolDescription": "需要读取的网页链接",
37+
"list": [],
38+
"defaultValue": ""
3739
}
3840
],
3941
"outputs": [
@@ -50,12 +52,12 @@
5052
"nodeId": "i7uow4wj2wdp",
5153
"name": "插件输出",
5254
"intro": "自定义配置外部输出，使用插件时，仅暴露自定义配置的输出",
53-
"avatar": "/imgs/workflow/output.png",
55+
"avatar": "core/workflow/template/pluginOutput",
5456
"flowNodeType": "pluginOutput",
5557
"showStatus": false,
5658
"position": {
57-
"x": 1607.7142331269129,
58-
"y": -150.8808596935447
59+
"x": 1853.935047606551,
60+
"y": -154.13661665265613
5961
},
6062
"version": "481",
6163
"inputs": [
@@ -81,12 +83,12 @@
8183
"nodeId": "ebLCxU43hHuZ",
8284
"name": "HTTP 请求",
8385
"intro": "可以发出一个 HTTP 请求，实现更为复杂的操作（联网搜索、数据库查询等）",
84-
"avatar": "/imgs/workflow/http.png",
86+
"avatar": "core/workflow/template/httpRequest",
8587
"flowNodeType": "httpRequest468",
8688
"showStatus": true,
8789
"position": {
88-
"x": 1050.9890727421412,
89-
"y": -415.2085119990912
90+
"x": 1054.2940501177068,
91+
"y": -503.13661665265613
9092
},
9193
"version": "481",
9294
"inputs": [
@@ -96,7 +98,7 @@
9698
"valueType": "dynamic",
9799
"label": "",
98100
"required": false,
99-
"description": "core.module.input.description.HTTP Dynamic Input",
101+
"description": "common:core.module.input.description.HTTP Dynamic Input",
100102
"customInputConfig": {
101103
"selectValueTypeList": [
102104
"string",
@@ -107,60 +109,105 @@
107109
"arrayNumber",
108110
"arrayBoolean",
109111
"arrayObject",
112+
"arrayAny",
110113
"any",
111114
"chatHistory",
112115
"datasetQuote",
113116
"dynamic",
114-
"selectApp",
115-
"selectDataset"
117+
"selectDataset",
118+
"selectApp"
116119
],
117120
"showDescription": false,
118121
"showDefaultValue": true
119-
}
122+
},
123+
"debugLabel": "",
124+
"toolDescription": ""
120125
},
121126
{
122127
"key": "system_httpMethod",
123128
"renderTypeList": ["custom"],
124129
"valueType": "string",
125130
"label": "",
126131
"value": "POST",
127-
"required": true
132+
"required": true,
133+
"debugLabel": "",
134+
"toolDescription": ""
135+
},
136+
{
137+
"key": "system_httpTimeout",
138+
"renderTypeList": ["custom"],
139+
"valueType": "number",
140+
"label": "",
141+
"value": 30,
142+
"min": 5,
143+
"max": 600,
144+
"required": true,
145+
"debugLabel": "",
146+
"toolDescription": ""
128147
},
129148
{
130149
"key": "system_httpReqUrl",
131150
"renderTypeList": ["hidden"],
132151
"valueType": "string",
133152
"label": "",
134-
"description": "core.module.input.description.Http Request Url",
153+
"description": "common:core.module.input.description.Http Request Url",
135154
"placeholder": "https://api.ai.com/getInventory",
136155
"required": false,
137-
"value": "fetchUrl"
156+
"value": "fetchUrl",
157+
"debugLabel": "",
158+
"toolDescription": ""
138159
},
139160
{
140161
"key": "system_httpHeader",
141162
"renderTypeList": ["custom"],
142163
"valueType": "any",
143164
"value": [],
144165
"label": "",
145-
"description": "core.module.input.description.Http Request Header",
146-
"placeholder": "core.module.input.description.Http Request Header",
147-
"required": false
166+
"description": "common:core.module.input.description.Http Request Header",
167+
"placeholder": "common:core.module.input.description.Http Request Header",
168+
"required": false,
169+
"debugLabel": "",
170+
"toolDescription": ""
148171
},
149172
{
150173
"key": "system_httpParams",
151174
"renderTypeList": ["hidden"],
152175
"valueType": "any",
153176
"value": [],
154177
"label": "",
155-
"required": false
178+
"required": false,
179+
"debugLabel": "",
180+
"toolDescription": ""
156181
},
157182
{
158183
"key": "system_httpJsonBody",
159184
"renderTypeList": ["hidden"],
160185
"valueType": "any",
161186
"value": "{\n \"url\": \"{{url}}\"\n}",
162187
"label": "",
163-
"required": false
188+
"required": false,
189+
"debugLabel": "",
190+
"toolDescription": ""
191+
},
192+
{
193+
"key": "system_httpFormBody",
194+
"renderTypeList": ["hidden"],
195+
"valueType": "any",
196+
"value": [],
197+
"label": "",
198+
"required": false,
199+
"debugLabel": "",
200+
"toolDescription": ""
201+
},
202+
{
203+
"key": "system_httpContentType",
204+
"renderTypeList": ["hidden"],
205+
"valueType": "string",
206+
"value": "json",
207+
"label": "",
208+
"required": false,
209+
"debugLabel": "",
210+
"toolDescription": ""
164211
},
165212
{
166213
"renderTypeList": ["reference"],
@@ -178,12 +225,13 @@
178225
"arrayNumber",
179226
"arrayBoolean",
180227
"arrayObject",
228+
"arrayAny",
181229
"any",
182230
"chatHistory",
183231
"datasetQuote",
184232
"dynamic",
185-
"selectApp",
186-
"selectDataset"
233+
"selectDataset",
234+
"selectApp"
187235
],
188236
"showDescription": false,
189237
"showDefaultValue": true
@@ -193,6 +241,23 @@
193241
}
194242
],
195243
"outputs": [
244+
{
245+
"id": "error",
246+
"key": "error",
247+
"label": "workflow:request_error",
248+
"description": "HTTP请求错误信息，成功时返回空",
249+
"valueType": "object",
250+
"type": "static"
251+
},
252+
{
253+
"id": "httpRawResponse",
254+
"key": "httpRawResponse",
255+
"required": true,
256+
"label": "workflow:raw_response",
257+
"description": "HTTP请求的原始响应。只能接受字符串或JSON类型响应数据。",
258+
"valueType": "any",
259+
"type": "static"
260+
},
196261
{
197262
"id": "system_addOutputParam",
198263
"key": "system_addOutputParam",
@@ -220,23 +285,6 @@
220285
"showDefaultValue": true
221286
}
222287
},
223-
{
224-
"id": "error",
225-
"key": "error",
226-
"label": "请求错误",
227-
"description": "HTTP请求错误信息，成功时返回空",
228-
"valueType": "object",
229-
"type": "static"
230-
},
231-
{
232-
"id": "httpRawResponse",
233-
"key": "httpRawResponse",
234-
"label": "原始响应",
235-
"required": true,
236-
"description": "HTTP请求的原始响应。只能接受字符串或JSON类型响应数据。",
237-
"valueType": "any",
238-
"type": "static"
239-
},
240288
{
241289
"id": "rH4tMV02robs",
242290
"valueType": "string",
@@ -260,6 +308,34 @@
260308
"sourceHandle": "ebLCxU43hHuZ-source-right",
261309
"targetHandle": "i7uow4wj2wdp-target-left"
262310
}
263-
]
311+
],
312+
"chatConfig": {
313+
"welcomeText": "",
314+
"variables": [],
315+
"questionGuide": {
316+
"open": false,
317+
"model": "gpt-4o-mini",
318+
"customPrompt": "You are an AI assistant tasked with predicting the user's next question based on the conversation history. Your goal is to generate 3 potential questions that will guide the user to continue the conversation. When generating these questions, adhere to the following rules:\n\n1. Use the same language as the user's last question in the conversation history.\n2. Keep each question under 20 characters in length.\n\nAnalyze the conversation history provided to you and use it as context to generate relevant and engaging follow-up questions. Your predictions should be logical extensions of the current topic or related areas that the user might be interested in exploring further.\n\nRemember to maintain consistency in tone and style with the existing conversation while providing diverse options for the user to choose from. Your goal is to keep the conversation flowing naturally and help the user delve deeper into the subject matter or explore related topics."
319+
},
320+
"ttsConfig": {
321+
"type": "web"
322+
},
323+
"whisperConfig": {
324+
"open": false,
325+
"autoSend": false,
326+
"autoTTSResponse": false
327+
},
328+
"chatInputGuide": {
329+
"open": false,
330+
"textList": [],
331+
"customUrl": ""
332+
},
333+
"instruction": "",
334+
"autoExecute": {
335+
"open": false,
336+
"defaultPrompt": ""
337+
},
338+
"_id": "677b59849d672185a5671b45"
339+
}
264340
}
265341
}

‎packages/service/common/string/cheerio.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { UrlFetchParams, UrlFetchResponse } from '@fastgpt/global/common/file/ap
22
import * as cheerio from 'cheerio';
33
import axios from 'axios';
44
import { htmlToMarkdown } from './utils';
5+
import { isInternalAddress } from '../system/utils';
56

67
export const cheerioToHtml = ({
78
fetchUrl,
@@ -75,6 +76,16 @@ export const urlsFetch = async ({
7576

7677
const response = await Promise.all(
7778
urlList.map(async (url) => {
79+
const isInternal = isInternalAddress(url);
80+
if (isInternal) {
81+
return {
82+
url,
83+
title: '',
84+
content: 'Cannot fetch internal url',
85+
selector: ''
86+
};
87+
}
88+
7889
try {
7990
const fetchRes = await axios.get(url, {
8091
timeout: 30000

‎packages/service/common/system/utils.ts‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
import { SERVICE_LOCAL_HOST } from './tools';
2+
3+
export const isInternalAddress = (url: string): boolean => {
4+
try {
5+
const parsedUrl = new URL(url);
6+
const hostname = parsedUrl.hostname;
7+
const fullUrl = parsedUrl.toString();
8+
9+
// Check for localhost and common internal domains
10+
if (hostname === SERVICE_LOCAL_HOST) {
11+
return true;
12+
}
13+
14+
// Metadata endpoints whitelist
15+
const metadataEndpoints = [
16+
// AWS
17+
'http://169.254.169.254/latest/meta-data/',
18+
// Azure
19+
'http://169.254.169.254/metadata/instance?api-version=2021-02-01',
20+
// GCP
21+
'http://metadata.google.internal/computeMetadata/v1/',
22+
// Alibaba Cloud
23+
'http://100.100.100.200/latest/meta-data/',
24+
// Tencent Cloud
25+
'http://metadata.tencentyun.com/latest/meta-data/',
26+
// Huawei Cloud
27+
'http://169.254.169.254/latest/meta-data/'
28+
];
29+
if (metadataEndpoints.some((endpoint) => fullUrl.startsWith(endpoint))) {
30+
return true;
31+
}
32+
33+
// For non-metadata URLs, check if it's a domain name
34+
const ipv4Pattern = /^(\d{1,3}\.){3}\d{1,3}$/;
35+
if (!ipv4Pattern.test(hostname)) {
36+
return true;
37+
}
38+
39+
// ... existing IP validation code ...
40+
const parts = hostname.split('.').map(Number);
41+
42+
if (parts.length !== 4 || parts.some((part) => part < 0 || part > 255)) {
43+
return false;
44+
}
45+
46+
// Only allow public IP ranges
47+
return (
48+
parts[0] !== 0 &&
49+
parts[0] !== 10 &&
50+
parts[0] !== 127 &&
51+
!(parts[0] === 169 && parts[1] === 254) &&
52+
!(parts[0] === 172 && parts[1] >= 16 && parts[1] <= 31) &&
53+
!(parts[0] === 192 && parts[1] === 168) &&
54+
!(parts[0] >= 224 && parts[0] <= 239) &&
55+
!(parts[0] >= 240 && parts[0] <= 255) &&
56+
!(parts[0] === 100 && parts[1] >= 64 && parts[1] <= 127) &&
57+
!(parts[0] === 9 && parts[1] === 0) &&
58+
!(parts[0] === 11 && parts[1] === 0)
59+
);
60+
} catch {
61+
return false; // If URL parsing fails, reject it as potentially unsafe
62+
}
63+
};

0 commit comments

Comments
 (0)