diff --git a/src/app/(main)/profile/[[...slugs]]/Client.tsx b/src/app/(main)/profile/[[...slugs]]/Client.tsx index 71672acfcd77..616b2d18ebf9 100644 --- a/src/app/(main)/profile/[[...slugs]]/Client.tsx +++ b/src/app/(main)/profile/[[...slugs]]/Client.tsx @@ -56,6 +56,7 @@ export const useStyles = createStyles( border-radius: unset; `, }) as Partial<{ + // eslint-disable-next-line unused-imports/no-unused-vars [k in keyof ElementsConfig]: any; }>, ); diff --git a/src/app/webapi/tokenizer/index.test.ts b/src/app/webapi/tokenizer/index.test.ts new file mode 100644 index 000000000000..15429268c907 --- /dev/null +++ b/src/app/webapi/tokenizer/index.test.ts @@ -0,0 +1,32 @@ +// @vitest-environment edge-runtime +import { describe, expect, it } from 'vitest'; + +import { POST } from './route'; + +describe('tokenizer Route', () => { + it('count hello world', async () => { + const txt = 'Hello, world!'; + const request = new Request('https://test.com', { + method: 'POST', + body: txt, + }); + + const response = await POST(request); + + const data = await response.json(); + expect(data.count).toEqual(4); + }); + + it('count Chinese', async () => { + const txt = '今天天气真好'; + const request = new Request('https://test.com', { + method: 'POST', + body: txt, + }); + + const response = await POST(request); + + const data = await response.json(); + expect(data.count).toEqual(5); + }); +}); diff --git a/src/app/webapi/tokenizer/route.ts b/src/app/webapi/tokenizer/route.ts new file mode 100644 index 000000000000..1c3220e4da1d --- /dev/null +++ b/src/app/webapi/tokenizer/route.ts @@ -0,0 +1,8 @@ +import { encode } from 'gpt-tokenizer/encoding/o200k_base'; +import { NextResponse } from 'next/server'; + +export const POST = async (req: Request) => { + const str = await req.text(); + + return NextResponse.json({ count: encode(str).length }); +}; diff --git a/src/database/server/models/session.ts b/src/database/server/models/session.ts index cca3f73f01ae..4fb43b315a24 100644 --- a/src/database/server/models/session.ts +++ b/src/database/server/models/session.ts @@ -170,7 +170,7 @@ export class SessionModel { if (!result) return; - // eslint-disable-next-line @typescript-eslint/no-unused-vars + // eslint-disable-next-line @typescript-eslint/no-unused-vars,unused-imports/no-unused-vars const { agent, clientId, ...session } = result; const sessionId = this.genId(); diff --git a/src/hooks/useTokenCount.ts b/src/hooks/useTokenCount.ts index 52df580a5d91..43ec9901e9d4 100644 --- a/src/hooks/useTokenCount.ts +++ b/src/hooks/useTokenCount.ts @@ -1,20 +1,32 @@ -import { startTransition, useEffect, useState } from 'react'; +import { debounce } from 'lodash-es'; +import { startTransition, useCallback, useEffect, useState } from 'react'; import { encodeAsync } from '@/utils/tokenizer'; export const useTokenCount = (input: string = '') => { const [value, setNum] = useState(0); - useEffect(() => { - startTransition(() => { - encodeAsync(input || '') + const debouncedEncode = useCallback( + debounce((text: string) => { + encodeAsync(text) .then(setNum) .catch(() => { - // 兜底采用字符数 - setNum(input.length); + setNum(text.length); }); + }, 300), + [], + ); + + useEffect(() => { + startTransition(() => { + debouncedEncode(input || ''); }); - }, [input]); + + // 清理函数 + return () => { + debouncedEncode.cancel(); + }; + }, [input, debouncedEncode]); return value; }; diff --git a/src/layout/AuthProvider/Clerk/useAppearance.ts b/src/layout/AuthProvider/Clerk/useAppearance.ts index f0b50c1d6398..7832ab02ec21 100644 --- a/src/layout/AuthProvider/Clerk/useAppearance.ts +++ b/src/layout/AuthProvider/Clerk/useAppearance.ts @@ -89,6 +89,7 @@ export const useStyles = createStyles( order: -1; `, }) as Partial<{ + // eslint-disable-next-line unused-imports/no-unused-vars [k in keyof ElementsConfig]: any; }>, ); diff --git a/src/server/context.ts b/src/server/context.ts index 0590c04488a5..eee5620add34 100644 --- a/src/server/context.ts +++ b/src/server/context.ts @@ -62,7 +62,9 @@ export const createContext = async (request: NextRequest): Promise => { userId = session.user.id; } return createContextInner({ authorizationHeader: authorization, nextAuth: auth, userId }); - } catch {} + } catch (e) { + console.error('next auth err', e); + } } return createContextInner({ authorizationHeader: authorization, userId }); diff --git a/src/server/routers/edge/index.ts b/src/server/routers/edge/index.ts index dcc243a8f7d9..c76200edb910 100644 --- a/src/server/routers/edge/index.ts +++ b/src/server/routers/edge/index.ts @@ -1,5 +1,5 @@ /** - * This file contains the root router of Lobe Chat tRPC-backend + * This file contains the edge router of Lobe Chat tRPC-backend */ import { publicProcedure, router } from '@/libs/trpc'; diff --git a/src/services/user/client.ts b/src/services/user/client.ts index c61ffd9c21df..f7ac9c7b11da 100644 --- a/src/services/user/client.ts +++ b/src/services/user/client.ts @@ -50,7 +50,7 @@ export class ClientService implements IUserService { await this.preferenceStorage.saveToLocalStorage(preference); } - // eslint-disable-next-line @typescript-eslint/no-unused-vars + // eslint-disable-next-line @typescript-eslint/no-unused-vars,unused-imports/no-unused-vars async updateGuide(guide: Partial) { throw new Error('Method not implemented.'); } diff --git a/src/types/worker.d.ts b/src/types/worker.d.ts new file mode 100644 index 000000000000..c260e9c9436d --- /dev/null +++ b/src/types/worker.d.ts @@ -0,0 +1,7 @@ +declare module '*.worker.ts' { + class WebpackWorker extends Worker { + constructor(); + } + + export default WebpackWorker; +} diff --git a/src/utils/fetch/__tests__/fetchSSE.test.ts b/src/utils/fetch/__tests__/fetchSSE.test.ts index a12b2b38ffc8..a9efd67eae73 100644 --- a/src/utils/fetch/__tests__/fetchSSE.test.ts +++ b/src/utils/fetch/__tests__/fetchSSE.test.ts @@ -139,8 +139,8 @@ describe('fetchSSE', () => { onFinish: mockOnFinish, }); - expect(mockOnMessageHandle).toHaveBeenNthCalledWith(1, { text: 'He', type: 'text' }); - expect(mockOnMessageHandle).toHaveBeenNthCalledWith(2, { text: 'llo World', type: 'text' }); + expect(mockOnMessageHandle).toHaveBeenNthCalledWith(1, { text: 'Hell', type: 'text' }); + expect(mockOnMessageHandle).toHaveBeenNthCalledWith(2, { text: 'o World', type: 'text' }); // more assertions for each character... expect(mockOnFinish).toHaveBeenCalledWith('Hello World', { observationId: null, @@ -232,8 +232,8 @@ describe('fetchSSE', () => { signal: abortController.signal, }); - expect(mockOnMessageHandle).toHaveBeenNthCalledWith(1, { text: 'He', type: 'text' }); - expect(mockOnMessageHandle).toHaveBeenNthCalledWith(2, { text: 'llo World', type: 'text' }); + expect(mockOnMessageHandle).toHaveBeenNthCalledWith(1, { text: 'Hell', type: 'text' }); + expect(mockOnMessageHandle).toHaveBeenNthCalledWith(2, { text: 'o World', type: 'text' }); expect(mockOnFinish).toHaveBeenCalledWith('Hello World', { type: 'done', diff --git a/src/utils/fetch/fetchSSE.ts b/src/utils/fetch/fetchSSE.ts index 7e6b480de15f..42457238c2c2 100644 --- a/src/utils/fetch/fetchSSE.ts +++ b/src/utils/fetch/fetchSSE.ts @@ -44,6 +44,10 @@ export interface FetchSSEOptions { smoothing?: boolean; } +const START_ANIMATION_SPEED = 4; + +const END_ANIMATION_SPEED = 15; + const createSmoothMessage = (params: { onTextUpdate: (delta: string, text: string) => void }) => { let buffer = ''; // why use queue: https://shareg.pt/GLBrjpK @@ -64,7 +68,7 @@ const createSmoothMessage = (params: { onTextUpdate: (delta: string, text: strin // define startAnimation function to display the text in buffer smooth // when you need to start the animation, call this function - const startAnimation = (speed = 2) => + const startAnimation = (speed = START_ANIMATION_SPEED) => new Promise((resolve) => { if (isAnimationActive) { resolve(); @@ -137,7 +141,7 @@ const createSmoothToolCalls = (params: { } }; - const startAnimation = (index: number, speed = 2) => + const startAnimation = (index: number, speed = START_ANIMATION_SPEED) => new Promise((resolve) => { if (isAnimationActives[index]) { resolve(); @@ -191,7 +195,7 @@ const createSmoothToolCalls = (params: { }); }; - const startAnimations = async (speed = 2) => { + const startAnimations = async (speed = START_ANIMATION_SPEED) => { const pools = toolCallsBuffer.map(async (_, index) => { if (outputQueues[index].length > 0 && !isAnimationActives[index]) { await startAnimation(index, speed); @@ -365,11 +369,11 @@ export const fetchSSE = async (url: string, options: RequestInit & FetchSSEOptio const observationId = response.headers.get(LOBE_CHAT_OBSERVATION_ID); if (textController.isTokenRemain()) { - await textController.startAnimation(15); + await textController.startAnimation(END_ANIMATION_SPEED); } if (toolCallsController.isTokenRemain()) { - await toolCallsController.startAnimations(15); + await toolCallsController.startAnimations(END_ANIMATION_SPEED); } await options?.onFinish?.(output, { observationId, toolCalls, traceId, type: finishedType }); diff --git a/src/utils/tokenizer.ts b/src/utils/tokenizer.ts deleted file mode 100644 index ecd43c24a941..000000000000 --- a/src/utils/tokenizer.ts +++ /dev/null @@ -1,5 +0,0 @@ -export const encodeAsync = async (str: string) => { - const { encode } = await import('gpt-tokenizer'); - - return encode(str).length; -}; diff --git a/src/utils/tokenizer/client.ts b/src/utils/tokenizer/client.ts new file mode 100644 index 000000000000..5493e8bd4b02 --- /dev/null +++ b/src/utils/tokenizer/client.ts @@ -0,0 +1,35 @@ +let worker: Worker | null = null; + +const getWorker = () => { + if (!worker && typeof Worker !== 'undefined') { + worker = new Worker(new URL('tokenizer.worker.ts', import.meta.url)); + } + return worker; +}; + +export const clientEncodeAsync = (str: string): Promise => + new Promise((resolve, reject) => { + const worker = getWorker(); + + if (!worker) { + // 如果 WebWorker 不可用，回退到字符串计算 + resolve(str.length); + return; + } + + const id = Date.now().toString(); + + const handleMessage = (event: MessageEvent) => { + if (event.data.id === id) { + worker.removeEventListener('message', handleMessage); + if (event.data.error) { + reject(new Error(event.data.error)); + } else { + resolve(event.data.result); + } + } + }; + + worker.addEventListener('message', handleMessage); + worker.postMessage({ id, str }); + }); diff --git a/src/utils/tokenizer/index.ts b/src/utils/tokenizer/index.ts new file mode 100644 index 000000000000..1c6071a6c116 --- /dev/null +++ b/src/utils/tokenizer/index.ts @@ -0,0 +1,15 @@ +export const encodeAsync = async (str: string): Promise => { + if (str.length === 0) return 0; + + // 50_000 is the limit of the client + // if the string is longer than 100_000, we will use the server + if (str.length <= 50_000) { + const { clientEncodeAsync } = await import('./client'); + + return await clientEncodeAsync(str); + } else { + const { serverEncodeAsync } = await import('./server'); + + return await serverEncodeAsync(str); + } +}; diff --git a/src/utils/tokenizer/server.ts b/src/utils/tokenizer/server.ts new file mode 100644 index 000000000000..c7631ff35bde --- /dev/null +++ b/src/utils/tokenizer/server.ts @@ -0,0 +1,11 @@ +export const serverEncodeAsync = async (str: string): Promise => { + try { + const res = await fetch('/webapi/tokenizer', { body: str, method: 'POST' }); + const data = await res.json(); + + return data.count; + } catch (e) { + console.error('serverEncodeAsync:', e); + return str.length; + } +}; diff --git a/src/utils/tokenizer/tokenizer.worker.ts b/src/utils/tokenizer/tokenizer.worker.ts new file mode 100644 index 000000000000..70cf6113ac2e --- /dev/null +++ b/src/utils/tokenizer/tokenizer.worker.ts @@ -0,0 +1,14 @@ +addEventListener('message', async (event) => { + const { id, str } = event.data; + try { + const { encode } = await import('gpt-tokenizer'); + + console.time('client tokenizer'); + const tokenCount = encode(str).length; + console.timeEnd('client tokenizer'); + + postMessage({ id, result: tokenCount }); + } catch (error) { + postMessage({ error: (error as Error).message, id }); + } +});