diff --git a/.gitignore b/.gitignore index ce28e0b9fc80..af5abc9fad6a 100644 --- a/.gitignore +++ b/.gitignore @@ -55,6 +55,8 @@ next-env.d.ts .next .env public/*.js +public/sitemap.xml +public/sitemap-index.xml bun.lockb sitemap*.xml robots.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index d308537fec02..82057eceb539 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,74 @@ # Changelog +### [Version 1.19.5](https://github.com/lobehub/lobe-chat/compare/v1.19.4...v1.19.5) + +Released on **2024-09-19** + +#### 💄 Styles + +- **misc**: Enable functioncall for stepfun models, Update qwen models. + +
+ +
+Improvements and Fixes + +#### Styles + +- **misc**: Enable functioncall for stepfun models, closes [#4022](https://github.com/lobehub/lobe-chat/issues/4022) ([afb3509](https://github.com/lobehub/lobe-chat/commit/afb3509)) +- **misc**: Update qwen models, closes [#4026](https://github.com/lobehub/lobe-chat/issues/4026) ([6169e8f](https://github.com/lobehub/lobe-chat/commit/6169e8f)) + +
+ +
+ +[![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top) + +
+ +### [Version 1.19.4](https://github.com/lobehub/lobe-chat/compare/v1.19.3...v1.19.4) + +Released on **2024-09-19** + +#### ♻ Code Refactoring + +- **misc**: Refactor the sitemap implement. + +
+ +
+Improvements and Fixes + +#### Code refactoring + +- **misc**: Refactor the sitemap implement, closes [#4012](https://github.com/lobehub/lobe-chat/issues/4012) ([d93a161](https://github.com/lobehub/lobe-chat/commit/d93a161)) + +
+ +
+ +[![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top) + +
+ +### [Version 1.19.3](https://github.com/lobehub/lobe-chat/compare/v1.19.2...v1.19.3) + +Released on **2024-09-19** + +
+ +
+Improvements and Fixes + +
+ +
+ +[![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top) + +
+ ### [Version 1.19.2](https://github.com/lobehub/lobe-chat/compare/v1.19.1...v1.19.2) Released on **2024-09-19** diff --git a/Dockerfile b/Dockerfile index 9611ae4a175a..ae9603a571b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,23 +1,30 @@ ## Base image for all the stages -FROM node:20-alpine AS base +FROM node:20-slim AS base ARG USE_CN_MIRROR +ENV DEBIAN_FRONTEND="noninteractive" + RUN \ # If you want to build docker in China, build with --build-arg USE_CN_MIRROR=true if [ "${USE_CN_MIRROR:-false}" = "true" ]; then \ - sed -i "s/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g" "/etc/apk/repositories"; \ + sed -i "s/deb.debian.org/mirrors.ustc.edu.cn/g" "/etc/apt/sources.list.d/debian.sources"; \ fi \ # Add required package & update base package - && apk update \ - && apk add --no-cache bind-tools proxychains-ng sudo \ - && apk upgrade --no-cache \ - # Add user nextjs to run the app + && apt update \ + && apt install busybox proxychains-ng -qy \ + && apt full-upgrade -qy \ + && apt autoremove -qy --purge \ + && apt clean -qy \ + # Configure BusyBox + && busybox --install -s \ + # Add nextjs:nodejs to run the app && addgroup --system --gid 1001 nodejs \ - && adduser --system --uid 1001 nextjs \ - && chown -R nextjs:nodejs "/etc/proxychains" \ - && echo "nextjs ALL=(ALL) NOPASSWD: /bin/chmod * /etc/resolv.conf" >> /etc/sudoers \ - && rm -rf /tmp/* /var/cache/apk/* + && adduser --system --home "/app" --gid 1001 -uid 1001 nextjs \ + # Set permission for nextjs:nodejs + && chown -R nextjs:nodejs "/etc/proxychains4.conf" \ + # Cleanup temp files + && rm -rf /tmp/* /var/lib/apt/lists/* /var/tmp/* ## Builder image, install all the dependencies and build the app FROM base AS builder @@ -89,7 +96,8 @@ FROM base # Copy all the files from app, set the correct permission for prerender cache COPY --from=app --chown=nextjs:nodejs /app /app -ENV NODE_ENV="production" +ENV NODE_ENV="production" \ + NODE_TLS_REJECT_UNAUTHORIZED="" # set hostname to localhost ENV HOSTNAME="0.0.0.0" \ @@ -121,6 +129,8 @@ ENV \ DEEPSEEK_API_KEY="" \ # Fireworks AI FIREWORKSAI_API_KEY="" FIREWORKSAI_MODEL_LIST="" \ + # GitHub + GITHUB_TOKEN="" GITHUB_MODEL_LIST="" \ # Google GOOGLE_API_KEY="" GOOGLE_PROXY_URL="" \ # Groq @@ -193,15 +203,7 @@ CMD \ 'tcp_read_time_out 15000' \ '[ProxyList]' \ "$protocol $host $port" \ - > "/etc/proxychains/proxychains.conf"; \ - fi; \ - # Fix DNS resolving issue in Docker Compose, ref https://github.com/lobehub/lobe-chat/pull/3837 - if [ -f "/etc/resolv.conf" ]; then \ - sudo chmod 666 "/etc/resolv.conf"; \ - resolv_conf=$(grep '^nameserver' "/etc/resolv.conf" | awk '{print "nameserver " $2}'); \ - printf "%s\n" \ - "$resolv_conf" \ - > "/etc/resolv.conf"; \ + > "/etc/proxychains4.conf"; \ fi; \ # Run the server ${PROXYCHAINS} node "/app/server.js"; diff --git a/Dockerfile.database b/Dockerfile.database index 1df23248e9c3..3f1138140b03 100644 --- a/Dockerfile.database +++ b/Dockerfile.database @@ -1,23 +1,30 @@ ## Base image for all the stages -FROM node:20-alpine AS base +FROM node:20-slim AS base ARG USE_CN_MIRROR +ENV DEBIAN_FRONTEND="noninteractive" + RUN \ # If you want to build docker in China, build with --build-arg USE_CN_MIRROR=true if [ "${USE_CN_MIRROR:-false}" = "true" ]; then \ - sed -i "s/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g" "/etc/apk/repositories"; \ + sed -i "s/deb.debian.org/mirrors.ustc.edu.cn/g" "/etc/apt/sources.list.d/debian.sources"; \ fi \ # Add required package & update base package - && apk update \ - && apk add --no-cache bind-tools proxychains-ng sudo \ - && apk upgrade --no-cache \ - # Add user nextjs to run the app + && apt update \ + && apt install busybox proxychains-ng -qy \ + && apt full-upgrade -qy \ + && apt autoremove -qy --purge \ + && apt clean -qy \ + # Configure BusyBox + && busybox --install -s \ + # Add nextjs:nodejs to run the app && addgroup --system --gid 1001 nodejs \ - && adduser --system --uid 1001 nextjs \ - && chown -R nextjs:nodejs "/etc/proxychains" \ - && echo "nextjs ALL=(ALL) NOPASSWD: /bin/chmod * /etc/resolv.conf" >> /etc/sudoers \ - && rm -rf /tmp/* /var/cache/apk/* + && adduser --system --home "/app" --gid 1001 -uid 1001 nextjs \ + # Set permission for nextjs:nodejs + && chown -R nextjs:nodejs "/etc/proxychains4.conf" \ + # Cleanup temp files + && rm -rf /tmp/* /var/lib/apt/lists/* /var/tmp/* ## Builder image, install all the dependencies and build the app FROM base AS builder @@ -102,7 +109,8 @@ FROM base # Copy all the files from app, set the correct permission for prerender cache COPY --from=app --chown=nextjs:nodejs /app /app -ENV NODE_ENV="production" +ENV NODE_ENV="production" \ + NODE_TLS_REJECT_UNAUTHORIZED="" # set hostname to localhost ENV HOSTNAME="0.0.0.0" \ @@ -153,6 +161,8 @@ ENV \ DEEPSEEK_API_KEY="" \ # Fireworks AI FIREWORKSAI_API_KEY="" FIREWORKSAI_MODEL_LIST="" \ + # GitHub + GITHUB_TOKEN="" GITHUB_MODEL_LIST="" \ # Google GOOGLE_API_KEY="" GOOGLE_PROXY_URL="" \ # Groq @@ -225,15 +235,7 @@ CMD \ 'tcp_read_time_out 15000' \ '[ProxyList]' \ "$protocol $host $port" \ - > "/etc/proxychains/proxychains.conf"; \ - fi; \ - # Fix DNS resolving issue in Docker Compose, ref https://github.com/lobehub/lobe-chat/pull/3837 - if [ -f "/etc/resolv.conf" ]; then \ - sudo chmod 666 "/etc/resolv.conf"; \ - resolv_conf=$(grep '^nameserver' "/etc/resolv.conf" | awk '{print "nameserver " $2}'); \ - printf "%s\n" \ - "$resolv_conf" \ - > "/etc/resolv.conf"; \ + > "/etc/proxychains4.conf"; \ fi; \ # Run migration node "/app/docker.cjs"; \ diff --git a/next-sitemap.config.mjs b/next-sitemap.config.mjs deleted file mode 100644 index 66295e4c9ab1..000000000000 --- a/next-sitemap.config.mjs +++ /dev/null @@ -1,53 +0,0 @@ -import { glob } from 'glob'; - -const isVercelPreview = process.env.VERCEL === '1' && process.env.VERCEL_ENV !== 'production'; - -const vercelPreviewUrl = `https://${process.env.VERCEL_URL}`; - -const siteUrl = isVercelPreview ? vercelPreviewUrl : 'https://lobechat.com'; - -/** @type {import('next-sitemap').IConfig} */ -const config = { - // next-sitemap does not work with app dir inside the /src dir (and have other problems e.g. with route groups) - // https://github.com/iamvishnusankar/next-sitemap/issues/700#issuecomment-1759458127 - // https://github.com/iamvishnusankar/next-sitemap/issues/701 - // additionalPaths is a workaround for this (once the issues are fixed, we can remove it) - additionalPaths: async () => { - const routes = await glob('src/app/**/page.{md,mdx,ts,tsx}', { - cwd: new URL('.', import.meta.url).pathname, - }); - - // https://nextjs.org/docs/app/building-your-application/routing/colocation#private-folders - const publicRoutes = routes.filter( - (page) => !page.split('/').some((folder) => folder.startsWith('_')), - ); - - // https://nextjs.org/docs/app/building-your-application/routing/colocation#route-groups - const publicRoutesWithoutRouteGroups = publicRoutes.map((page) => - page - .split('/') - .filter((folder) => !folder.startsWith('(') && !folder.endsWith(')')) - .join('/'), - ); - - const locs = publicRoutesWithoutRouteGroups.map((route) => { - const path = route.replace(/^src\/app/, '').replace(/\/[^/]+$/, ''); - const loc = path === '' ? siteUrl : `${siteUrl}/${path}`; - - return loc; - }); - - const paths = locs.map((loc) => ({ - changefreq: 'daily', - lastmod: new Date().toISOString(), - loc, - priority: 0.7, - })); - - return paths; - }, - generateRobotsTxt: true, - siteUrl, -}; - -export default config; diff --git a/next.config.mjs b/next.config.mjs index 2803fa6f6668..4ba5ed23ed34 100644 --- a/next.config.mjs +++ b/next.config.mjs @@ -107,6 +107,16 @@ const nextConfig = { output: buildWithDocker ? 'standalone' : undefined, reactStrictMode: true, redirects: async () => [ + { + destination: '/sitemap-index.xml', + permanent: true, + source: '/sitemap.xml', + }, + { + destination: '/discover', + permanent: true, + source: '/market', + }, { destination: '/settings/common', permanent: true, diff --git a/package.json b/package.json index 96e00983a2d6..538aa6e07302 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@lobehub/chat", - "version": "1.19.2", + "version": "1.19.5", "description": "Lobe Chat - an open-source, high-performance chatbot framework that supports speech synthesis, multimodal, and extensible Function Call plugin system. Supports one-click free deployment of your private ChatGPT/LLM web application.", "keywords": [ "framework", @@ -29,11 +29,11 @@ "build": "next build", "postbuild": "npm run build-sitemap && npm run build-migrate-db", "build-migrate-db": "bun run db:migrate", - "build-sitemap": "next-sitemap --config next-sitemap.config.mjs", + "build-sitemap": "tsx ./scripts/buildSitemapIndex/index.ts", "build:analyze": "ANALYZE=true next build", "build:docker": "DOCKER=true next build && npm run build-sitemap", "db:generate": "drizzle-kit generate", - "db:migrate": "MIGRATION_DB=1 tsx scripts/migrateServerDB/index.ts", + "db:migrate": "MIGRATION_DB=1 tsx ./scripts/migrateServerDB/index.ts", "db:push": "drizzle-kit push", "db:push-test": "NODE_ENV=test drizzle-kit push", "db:studio": "drizzle-kit studio", @@ -65,11 +65,11 @@ "test:update": "vitest -u", "type-check": "tsc --noEmit", "webhook:ngrok": "ngrok http http://localhost:3011", - "workflow:docs": "tsx scripts/docsWorkflow/index.ts", - "workflow:i18n": "tsx scripts/i18nWorkflow/index.ts", + "workflow:docs": "tsx ./scripts/docsWorkflow/index.ts", + "workflow:i18n": "tsx ./scripts/i18nWorkflow/index.ts", "workflow:mdx": "tsx ./scripts/mdxWorkflow/index.ts", "workflow:mdx-with-lint": "tsx ./scripts/mdxWorkflow/index.ts && eslint \"docs/**/*.mdx\" --quiet --fix", - "workflow:readme": "tsx scripts/readmeWorkflow/index.ts" + "workflow:readme": "tsx ./scripts/readmeWorkflow/index.ts" }, "lint-staged": { "*.md": [ @@ -172,7 +172,6 @@ "next": "14.2.8", "next-auth": "beta", "next-mdx-remote": "^4.4.1", - "next-sitemap": "^4.2.3", "nextjs-toploader": "^3.6.15", "numeral": "^2.0.6", "nuqs": "^1.17.8", diff --git a/scripts/buildSitemapIndex/index.ts b/scripts/buildSitemapIndex/index.ts new file mode 100644 index 000000000000..15a7fe83bfda --- /dev/null +++ b/scripts/buildSitemapIndex/index.ts @@ -0,0 +1,12 @@ +import { writeFileSync } from 'node:fs'; +import { resolve } from 'node:path'; + +import { sitemapModule } from '@/server/sitemap'; + +const genSitemap = () => { + const sitemapIndexXML = sitemapModule.getIndex(); + const filename = resolve(__dirname, '../../', 'public', 'sitemap-index.xml'); + writeFileSync(filename, sitemapIndexXML); +}; + +genSitemap(); diff --git a/src/app/(main)/discover/(detail)/assistant/[slug]/page.tsx b/src/app/(main)/discover/(detail)/assistant/[slug]/page.tsx index 2ff4082ee725..112ac455af40 100644 --- a/src/app/(main)/discover/(detail)/assistant/[slug]/page.tsx +++ b/src/app/(main)/discover/(detail)/assistant/[slug]/page.tsx @@ -105,6 +105,7 @@ const Page = async ({ params, searchParams }: Props) => { /> } /* ↓ cloud slot ↓ */ + /* ↑ cloud slot ↑ */ > diff --git a/src/app/(main)/discover/(detail)/model/[...slugs]/features/ProviderList/ProviderItem.tsx b/src/app/(main)/discover/(detail)/model/[...slugs]/features/ProviderList/ProviderItem.tsx index a3252d0ebb2e..4793d40feae3 100644 --- a/src/app/(main)/discover/(detail)/model/[...slugs]/features/ProviderList/ProviderItem.tsx +++ b/src/app/(main)/discover/(detail)/model/[...slugs]/features/ProviderList/ProviderItem.tsx @@ -67,6 +67,7 @@ const ProviderItem = memo(({ mobile, modelId, identifier }) = : '--', }, /* ↓ cloud slot ↓ */ + /* ↑ cloud slot ↑ */ ]; diff --git a/src/app/(main)/discover/(detail)/model/[...slugs]/page.tsx b/src/app/(main)/discover/(detail)/model/[...slugs]/page.tsx index a37aed167acf..4972c4feeff4 100644 --- a/src/app/(main)/discover/(detail)/model/[...slugs]/page.tsx +++ b/src/app/(main)/discover/(detail)/model/[...slugs]/page.tsx @@ -100,6 +100,7 @@ const Page = async ({ params, searchParams }: Props) => { mobile={mobile} sidebar={} /* ↓ cloud slot ↓ */ + /* ↑ cloud slot ↑ */ > diff --git a/src/app/(main)/discover/(detail)/plugin/[slug]/page.tsx b/src/app/(main)/discover/(detail)/plugin/[slug]/page.tsx index c0b6d6d39125..0f698ee339f9 100644 --- a/src/app/(main)/discover/(detail)/plugin/[slug]/page.tsx +++ b/src/app/(main)/discover/(detail)/plugin/[slug]/page.tsx @@ -89,6 +89,7 @@ const Page = async ({ params, searchParams }: Props) => { mobile={mobile} sidebar={} /* ↓ cloud slot ↓ */ + /* ↑ cloud slot ↑ */ > diff --git a/src/app/(main)/discover/(detail)/provider/[slug]/features/ModelList/ModelItem.tsx b/src/app/(main)/discover/(detail)/provider/[slug]/features/ModelList/ModelItem.tsx index 50cb2e257e90..4b4d04074351 100644 --- a/src/app/(main)/discover/(detail)/provider/[slug]/features/ModelList/ModelItem.tsx +++ b/src/app/(main)/discover/(detail)/provider/[slug]/features/ModelList/ModelItem.tsx @@ -79,6 +79,7 @@ const ModelItem = memo(({ mobile, meta, identifier }) => { : '--', }, /* ↓ cloud slot ↓ */ + /* ↑ cloud slot ↑ */ ]; diff --git a/src/app/(main)/discover/(detail)/provider/[slug]/page.tsx b/src/app/(main)/discover/(detail)/provider/[slug]/page.tsx index fdbf14eb04de..b9b33224d2bc 100644 --- a/src/app/(main)/discover/(detail)/provider/[slug]/page.tsx +++ b/src/app/(main)/discover/(detail)/provider/[slug]/page.tsx @@ -102,6 +102,7 @@ const Page = async ({ params, searchParams }: Props) => { mobile={mobile} sidebar={} /* ↓ cloud slot ↓ */ + /* ↑ cloud slot ↑ */ > diff --git a/src/app/(main)/discover/(list)/_layout/Desktop/Nav.tsx b/src/app/(main)/discover/(list)/_layout/Desktop/Nav.tsx index 0661b7e8a240..7053828d197a 100644 --- a/src/app/(main)/discover/(list)/_layout/Desktop/Nav.tsx +++ b/src/app/(main)/discover/(list)/_layout/Desktop/Nav.tsx @@ -108,6 +108,7 @@ const Nav = memo(() => { {!isHome && !isProviders && ( {/* ↓ cloud slot ↓ */} + {/* ↑ cloud slot ↑ */} )} diff --git a/src/app/(main)/discover/_layout/Desktop/index.tsx b/src/app/(main)/discover/_layout/Desktop/index.tsx index 98d5f569e1cd..9c6bd0d50566 100644 --- a/src/app/(main)/discover/_layout/Desktop/index.tsx +++ b/src/app/(main)/discover/_layout/Desktop/index.tsx @@ -14,6 +14,7 @@ const Layout = ({ children }: PropsWithChildren) => { {children} {/* ↓ cloud slot ↓ */} + {/* ↑ cloud slot ↑ */} ); diff --git a/src/app/page.tsx b/src/app/page.tsx index 6f0ec82c6c58..83a80a0a9b91 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -1,6 +1,6 @@ import { Metadata } from 'next'; -import { getCanonicalUrl } from '@/const/url'; +import { getCanonicalUrl } from '@/server/utils/url'; import Client from './(loading)/Client'; import Redirect from './(loading)/Redirect'; diff --git a/src/app/robots.tsx b/src/app/robots.tsx new file mode 100644 index 000000000000..fc257fdcf8f8 --- /dev/null +++ b/src/app/robots.tsx @@ -0,0 +1,16 @@ +import { MetadataRoute } from 'next'; + +import { sitemapModule } from '@/server/sitemap'; +import { getCanonicalUrl } from '@/server/utils/url'; + +export default function robots(): MetadataRoute.Robots { + return { + host: getCanonicalUrl(), + rules: { + allow: ['/'], + disallow: ['/api/*'], + userAgent: '*', + }, + sitemap: sitemapModule.getRobots(), + }; +} diff --git a/src/app/sitemap.tsx b/src/app/sitemap.tsx new file mode 100644 index 000000000000..cc42ece4d997 --- /dev/null +++ b/src/app/sitemap.tsx @@ -0,0 +1,30 @@ +import { MetadataRoute } from 'next'; + +import { SitemapType, sitemapModule } from '@/server/sitemap'; + +export const generateSitemaps = async () => { + // Fetch the total number of products and calculate the number of sitemaps needed + return sitemapModule.sitemapIndexs; +}; + +const Sitemap = async ({ id }: { id: SitemapType }): Promise => { + switch (id) { + case SitemapType.Pages: { + return sitemapModule.getPage(); + } + case SitemapType.Assistants: { + return sitemapModule.getAssistants(); + } + case SitemapType.Plugins: { + return sitemapModule.getPlugins(); + } + case SitemapType.Models: { + return sitemapModule.getModels(); + } + case SitemapType.Providers: { + return sitemapModule.getProviders(); + } + } +}; + +export default Sitemap; diff --git a/src/config/modelProviders/qwen.ts b/src/config/modelProviders/qwen.ts index d4849c248b03..7c13ebb5de88 100644 --- a/src/config/modelProviders/qwen.ts +++ b/src/config/modelProviders/qwen.ts @@ -1,22 +1,10 @@ import { ModelProviderCard } from '@/types/llm'; -// ref :https://help.aliyun.com/zh/dashscope/developer-reference/api-details +// ref: https://help.aliyun.com/zh/model-studio/getting-started/models const Qwen: ModelProviderCard = { chatModels: [ { - description: - '通义千问超大规模语言模型，支持长文本上下文，以及基于长文档、多文档等多个场景的对话功能。', - displayName: 'Qwen Long', - id: 'qwen-long', - pricing: { - currency: 'CNY', - input: 0.5, - output: 2, - }, - tokens: 1_000_000, // https://help.aliyun.com/zh/dashscope/developer-reference/model-introduction - }, - { - description: '通义千问超大规模语言模型，支持中文、英文等不同语言输入', + description: '通义千问超大规模语言模型，支持中文、英文等不同语言输入。', displayName: 'Qwen Turbo', enabled: true, functionCall: true, @@ -26,10 +14,10 @@ const Qwen: ModelProviderCard = { input: 0.3, output: 0.6, }, - tokens: 131_072, // https://help.aliyun.com/zh/dashscope/developer-reference/model-introduction + tokens: 131_072, }, { - description: '通义千问超大规模语言模型增强版，支持中文、英文等不同语言输入', + description: '通义千问超大规模语言模型增强版，支持中文、英文等不同语言输入。', displayName: 'Qwen Plus', enabled: true, functionCall: true, @@ -39,11 +27,11 @@ const Qwen: ModelProviderCard = { input: 0.8, output: 2, }, - tokens: 131_072, // https://help.aliyun.com/zh/dashscope/developer-reference/model-introduction + tokens: 131_072, }, { description: - '通义千问千亿级别超大规模语言模型，支持中文、英文等不同语言输入，当前通义千问2.5产品版本背后的API模型', + '通义千问千亿级别超大规模语言模型，支持中文、英文等不同语言输入，当前通义千问2.5产品版本背后的API模型。', displayName: 'Qwen Max', enabled: true, functionCall: true, @@ -53,7 +41,30 @@ const Qwen: ModelProviderCard = { input: 20, output: 60, }, - tokens: 32_768, // https://help.aliyun.com/zh/dashscope/developer-reference/model-introduction + tokens: 32_768, + }, + { + description: '通义千问代码模型。', + displayName: 'Qwen Coder', + id: 'qwen-coder-turbo-latest', + pricing: { + currency: 'CNY', + input: 2, + output: 6, + }, + tokens: 131_072, + }, + { + description: + '通义千问超大规模语言模型，支持长文本上下文，以及基于长文档、多文档等多个场景的对话功能。', + displayName: 'Qwen Long', + id: 'qwen-long', + pricing: { + currency: 'CNY', + input: 0.5, + output: 2, + }, + tokens: 1_000_000, }, { description: @@ -66,8 +77,8 @@ const Qwen: ModelProviderCard = { input: 8, output: 8, }, - tokens: 8192, - vision: true, // https://help.aliyun.com/zh/dashscope/developer-reference/tongyi-qianwen-vl-plus-api + tokens: 8000, + vision: true, }, { description: @@ -80,71 +91,120 @@ const Qwen: ModelProviderCard = { input: 20, output: 20, }, - tokens: 32_768, - vision: true, // https://help.aliyun.com/zh/dashscope/developer-reference/tongyi-qianwen-vl-plus-api + tokens: 32_000, + vision: true, }, - // ref :https://help.aliyun.com/zh/dashscope/developer-reference/tongyi-qianwen-7b-14b-72b-api-detailes { - description: '通义千问2.5对外开源的7B规模的模型', + description: + '通义千问数学模型是专门用于数学解题的语言模型。', + displayName: 'Qwen Math Turbo', + id: 'qwen-math-turbo-latest', + pricing: { + currency: 'CNY', + input: 2, + output: 6, + }, + tokens: 4096, + }, + { + description: + '通义千问数学模型是专门用于数学解题的语言模型。', + displayName: 'Qwen Math Plus', + id: 'qwen-math-plus-latest', + pricing: { + currency: 'CNY', + input: 4, + output: 12, + }, + tokens: 4096, + }, + { + description: '通义千问2.5对外开源的7B规模的模型。', displayName: 'Qwen2.5 7B', functionCall: true, id: 'qwen2.5-7b-instruct', - tokens: 131_072, // https://huggingface.co/Qwen/Qwen2.5-7B-Instruct + tokens: 131_072, + }, + { + description: '通义千问2.5对外开源的14B规模的模型。', + displayName: 'Qwen2.5 14B', + functionCall: true, + id: 'qwen2.5-14b-instruct', + tokens: 131_072, }, { - description: '通义千问2.5对外开源的32B规模的模型', + description: '通义千问2.5对外开源的32B规模的模型。', displayName: 'Qwen2.5 32B', functionCall: true, id: 'qwen2.5-32b-instruct', - tokens: 131_072, // https://huggingface.co/Qwen/Qwen2.5-32B-Instruct + tokens: 131_072, }, { - description: '通义千问2.5对外开源的72B规模的模型', + description: '通义千问2.5对外开源的72B规模的模型。', displayName: 'Qwen2.5 72B', functionCall: true, id: 'qwen2.5-72b-instruct', - tokens: 131_072, // https://huggingface.co/Qwen/Qwen2.5-72B-Instruct + tokens: 131_072, }, { - description: '通义千问2对外开源的7B规模的模型', - displayName: 'Qwen2 7B', - functionCall: true, - id: 'qwen2-7b-instruct', - tokens: 131_072, // https://huggingface.co/Qwen/Qwen2-7B-Instruct - }, - { - description: '通义千问2对外开源的57B规模14B激活参数的MOE模型', + description: '通义千问2对外开源的57B规模14B激活参数的MOE模型。', displayName: 'Qwen2 57B A14B MoE', functionCall: true, id: 'qwen2-57b-a14b-instruct', - tokens: 65_536, // https://huggingface.co/Qwen/Qwen2-57B-A14B-Instruct + tokens: 65_536, }, { - description: '通义千问2对外开源的72B规模的模型', - displayName: 'Qwen2 72B', - functionCall: true, - id: 'qwen2-72b-instruct', - tokens: 131_072, // https://huggingface.co/Qwen/Qwen2-72B-Instruct + description: 'Qwen-Math 模型具有强大的数学解题能力。', + displayName: 'Qwen2.5 Math 1.5B', + id: 'qwen2.5-math-1.5b-instruct', + pricing: { + currency: 'CNY', + input: 0, + output: 0, + }, + tokens: 4096, }, { - description: 'Qwen2-Math 模型具有强大的数学解题能力', - displayName: 'Qwen2 Math 72B', - functionCall: true, - id: 'qwen2-math-72b-instruct', - tokens: 4096, // https://help.aliyun.com/zh/dashscope/developer-reference/use-qwen2-math-by-calling-api + description: 'Qwen-Math 模型具有强大的数学解题能力。', + displayName: 'Qwen2.5 Math 7B', + id: 'qwen2.5-math-7b-instruct', + tokens: 4096, + }, + { + description: 'Qwen-Math 模型具有强大的数学解题能力。', + displayName: 'Qwen2.5 Math 72B', + id: 'qwen2.5-math-72b-instruct', + tokens: 4096, + }, + { + description: '通义千问代码模型开源版。', + displayName: 'Qwen2.5 Coder 1.5B', + id: 'qwen2.5-coder-1.5b-instruct', + pricing: { + currency: 'CNY', + input: 0, + output: 0, + }, + tokens: 131_072, + }, + { + description: '通义千问代码模型开源版。', + displayName: 'Qwen2.5 Coder 7B', + id: 'qwen2.5-coder-7b-instruct', + tokens: 131_072, }, { description: '以 Qwen-7B 语言模型初始化，添加图像模型，图像输入分辨率为448的预训练模型。', displayName: 'Qwen VL', id: 'qwen-vl-v1', - tokens: 8192, // https://huggingface.co/Qwen/Qwen-VL/blob/main/config.json + tokens: 8000, vision: true, }, { description: '通义千问VL支持灵活的交互方式，包括多图、多轮问答、创作等能力的模型。', displayName: 'Qwen VL Chat', id: 'qwen-vl-chat-v1', - tokens: 8192, // https://huggingface.co/Qwen/Qwen-VL-Chat/blob/main/config.json + tokens: 8000, vision: true, }, ], @@ -160,7 +220,7 @@ const Qwen: ModelProviderCard = { speed: 2, text: true, }, - url: 'https://tongyi.aliyun.com', + url: 'https://www.aliyun.com/product/bailian', }; export default Qwen; diff --git a/src/config/modelProviders/stepfun.ts b/src/config/modelProviders/stepfun.ts index b37da210f792..312eab2162fc 100644 --- a/src/config/modelProviders/stepfun.ts +++ b/src/config/modelProviders/stepfun.ts @@ -7,6 +7,7 @@ const Stepfun: ModelProviderCard = { { description: '支持大规模上下文交互，适合复杂对话场景。', displayName: 'Step 2 16K', + functionCall: true, enabled: true, id: 'step-2-16k', tokens: 16_000, @@ -14,12 +15,14 @@ const Stepfun: ModelProviderCard = { { description: '具备超长上下文处理能力，尤其适合长文档分析。', displayName: 'Step 1 256K', + functionCall: true, id: 'step-1-256k', tokens: 256_000, }, { description: '平衡性能与成本，适合一般场景。', displayName: 'Step 1 128K', + functionCall: true, enabled: true, id: 'step-1-128k', tokens: 128_000, @@ -27,6 +30,7 @@ const Stepfun: ModelProviderCard = { { description: '支持中等长度的对话，适用于多种应用场景。', displayName: 'Step 1 32K', + functionCall: true, enabled: true, id: 'step-1-32k', tokens: 32_000, @@ -34,6 +38,7 @@ const Stepfun: ModelProviderCard = { { description: '小型模型，适合轻量级任务。', displayName: 'Step 1 8K', + functionCall: true, enabled: true, id: 'step-1-8k', tokens: 8000, @@ -41,6 +46,7 @@ const Stepfun: ModelProviderCard = { { description: '高速模型，适合实时对话。', displayName: 'Step 1 Flash', + functionCall: true, enabled: true, id: 'step-1-flash', tokens: 8000, @@ -48,6 +54,7 @@ const Stepfun: ModelProviderCard = { { description: '支持视觉输入，增强多模态交互体验。', displayName: 'Step 1V 32K', + functionCall: true, enabled: true, id: 'step-1v-32k', tokens: 32_000, @@ -56,6 +63,7 @@ const Stepfun: ModelProviderCard = { { description: '小型视觉模型，适合基本的图文任务。', displayName: 'Step 1V 8K', + functionCall: true, enabled: true, id: 'step-1v-8k', tokens: 8000, diff --git a/src/const/url.ts b/src/const/url.ts index 42860ffad8b8..ef4f73f01cc2 100644 --- a/src/const/url.ts +++ b/src/const/url.ts @@ -2,6 +2,7 @@ import qs from 'query-string'; import urlJoin from 'url-join'; import { withBasePath } from '@/utils/basePath'; +import { isDev } from '@/utils/env'; import pkg from '../../package.json'; import { INBOX_SESSION_ID } from './session'; @@ -12,8 +13,6 @@ export const OFFICIAL_URL = 'https://lobechat.com/'; export const OFFICIAL_PREVIEW_URL = 'https://chat-preview.lobehub.com/'; export const OFFICIAL_SITE = 'https://lobehub.com/'; -export const getCanonicalUrl = (path: string) => urlJoin(OFFICIAL_URL, path); - export const OG_URL = '/og/cover.png?v=1'; export const GITHUB = pkg.homepage; @@ -73,3 +72,4 @@ export const mailTo = (email: string) => `mailto:${email}`; export const AES_GCM_URL = 'https://datatracker.ietf.org/doc/html/draft-ietf-avt-srtp-aes-gcm-01'; export const BASE_PROVIDER_DOC_URL = 'https://lobehub.com/docs/usage/providers'; +export const SITEMAP_BASE_URL = isDev ? '/sitemap.xml/' : 'sitemap'; diff --git a/src/server/ld.test.ts b/src/server/ld.test.ts new file mode 100644 index 000000000000..454844f1faad --- /dev/null +++ b/src/server/ld.test.ts @@ -0,0 +1,102 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest'; + +import { DEFAULT_LANG } from '@/const/locale'; + +import { AUTHOR_LIST, Ld } from './ld'; + +describe('Ld', () => { + const ld = new Ld(); + + describe('generate', () => { + it('should generate correct LD+JSON structure', () => { + const result = ld.generate({ + title: 'Test Title', + description: 'Test Description', + url: 'https://example.com/test', + locale: DEFAULT_LANG, + }); + + expect(result['@context']).toBe('https://schema.org'); + expect(Array.isArray(result['@graph'])).toBe(true); + expect(result['@graph'].length).toBeGreaterThan(0); + }); + }); + + describe('genOrganization', () => { + it('should generate correct organization structure', () => { + const org = ld.genOrganization(); + + expect(org['@type']).toBe('Organization'); + expect(org.name).toBe('LobeHub'); + expect(org.url).toBe('https://lobehub.com/'); + }); + }); + + describe('getAuthors', () => { + it('should return default author when no ids provided', () => { + const author = ld.getAuthors(); + expect(author['@type']).toBe('Organization'); + }); + + it('should return person when valid id provided', () => { + const author = ld.getAuthors(['arvinxx']); + expect(author['@type']).toBe('Person'); + // @ts-ignore + expect(author.name).toBe(AUTHOR_LIST.arvinxx.name); + }); + }); + + describe('genWebPage', () => { + it('should generate correct webpage structure', () => { + const webpage = ld.genWebPage({ + title: 'Test Page', + description: 'Test Description', + url: 'https://example.com/test', + locale: DEFAULT_LANG, + }); + + expect(webpage['@type']).toBe('WebPage'); + expect(webpage.name).toBe('Test Page · LobeChat'); + expect(webpage.description).toBe('Test Description'); + }); + }); + + describe('genImageObject', () => { + it('should generate correct image object', () => { + const image = ld.genImageObject({ + image: 'https://example.com/image.jpg', + url: 'https://example.com/test', + }); + + expect(image['@type']).toBe('ImageObject'); + expect(image.url).toBe('https://example.com/image.jpg'); + }); + }); + + describe('genWebSite', () => { + it('should generate correct website structure', () => { + const website = ld.genWebSite(); + + expect(website['@type']).toBe('WebSite'); + expect(website.name).toBe('LobeChat'); + }); + }); + + describe('genArticle', () => { + it('should generate correct article structure', () => { + const article = ld.genArticle({ + title: 'Test Article', + description: 'Test Description', + url: 'https://example.com/test', + author: ['arvinxx'], + identifier: 'test-id', + locale: DEFAULT_LANG, + }); + + expect(article['@type']).toBe('Article'); + expect(article.headline).toBe('Test Article · LobeChat'); + expect(article.author['@type']).toBe('Person'); + }); + }); +}); diff --git a/src/server/ld.ts b/src/server/ld.ts index 5d0121f990ff..945fde5163f6 100644 --- a/src/server/ld.ts +++ b/src/server/ld.ts @@ -3,15 +3,9 @@ import urlJoin from 'url-join'; import { BRANDING_NAME } from '@/const/branding'; import { DEFAULT_LANG } from '@/const/locale'; -import { - EMAIL_BUSINESS, - EMAIL_SUPPORT, - OFFICIAL_SITE, - OFFICIAL_URL, - X, - getCanonicalUrl, -} from '@/const/url'; +import { EMAIL_BUSINESS, EMAIL_SUPPORT, OFFICIAL_SITE, OFFICIAL_URL, X } from '@/const/url'; import { Locales } from '@/locales/resources'; +import { getCanonicalUrl } from '@/server/utils/url'; import pkg from '../../package.json'; @@ -37,7 +31,7 @@ export const AUTHOR_LIST = { }, }; -class Ld { +export class Ld { generate({ image = '/og/cover.png', article, diff --git a/src/server/metadata.test.ts b/src/server/metadata.test.ts new file mode 100644 index 000000000000..dcef1f47e516 --- /dev/null +++ b/src/server/metadata.test.ts @@ -0,0 +1,138 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest'; + +import { BRANDING_NAME } from '@/const/branding'; +import { OG_URL } from '@/const/url'; + +import { Meta } from './metadata'; + +describe('Metadata', () => { + const meta = new Meta(); + + describe('generate', () => { + it('should generate metadata with default values', () => { + const result = meta.generate({ + title: 'Test Title', + url: 'https://example.com', + }); + + expect(result).toMatchObject({ + title: 'Test Title', + description: expect.any(String), + openGraph: expect.objectContaining({ + title: `Test Title · ${BRANDING_NAME}`, + description: expect.any(String), + images: [{ url: OG_URL, alt: `Test Title · ${BRANDING_NAME}` }], + }), + twitter: expect.objectContaining({ + title: `Test Title · ${BRANDING_NAME}`, + description: expect.any(String), + images: [OG_URL], + }), + }); + }); + + it('should generate metadata with custom values', () => { + const result = meta.generate({ + title: 'Custom Title', + description: 'Custom description', + image: 'https://custom-image.com', + url: 'https://example.com/custom', + type: 'article', + tags: ['tag1', 'tag2'], + locale: 'fr-FR', + alternate: true, + }); + + expect(result).toMatchObject({ + title: 'Custom Title', + description: expect.stringContaining('Custom description'), + openGraph: expect.objectContaining({ + title: `Custom Title · ${BRANDING_NAME}`, + description: 'Custom description', + images: [{ url: 'https://custom-image.com', alt: `Custom Title · ${BRANDING_NAME}` }], + type: 'article', + locale: 'fr-FR', + }), + twitter: expect.objectContaining({ + title: `Custom Title · ${BRANDING_NAME}`, + description: 'Custom description', + images: ['https://custom-image.com'], + }), + alternates: expect.objectContaining({ + languages: expect.any(Object), + }), + }); + }); + }); + + describe('genAlternateLocales', () => { + it('should generate alternate locales correctly', () => { + const result = (meta as any).genAlternateLocales('en', '/test'); + + expect(result).toHaveProperty('x-default', expect.stringContaining('/test')); + expect(result).toHaveProperty('zh-CN', expect.stringContaining('hl=zh-CN')); + expect(result).not.toHaveProperty('en'); + }); + }); + + describe('genTwitter', () => { + it('should generate Twitter metadata correctly', () => { + const result = (meta as any).genTwitter({ + title: 'Twitter Title', + description: 'Twitter description', + image: 'https://twitter-image.com', + url: 'https://example.com/twitter', + }); + + expect(result).toEqual({ + card: 'summary_large_image', + title: 'Twitter Title', + description: 'Twitter description', + images: ['https://twitter-image.com'], + site: '@lobehub', + url: 'https://example.com/twitter', + }); + }); + }); + + describe('genOpenGraph', () => { + it('should generate OpenGraph metadata correctly', () => { + const result = (meta as any).genOpenGraph({ + title: 'OG Title', + description: 'OG description', + image: 'https://og-image.com', + url: 'https://example.com/og', + locale: 'es-ES', + type: 'article', + alternate: true, + }); + + expect(result).toMatchObject({ + title: 'OG Title', + description: 'OG description', + images: [{ url: 'https://og-image.com', alt: 'OG Title' }], + locale: 'es-ES', + type: 'article', + url: 'https://example.com/og', + siteName: 'LobeChat', + alternateLocale: expect.arrayContaining([ + 'ar', + 'bg-BG', + 'de-DE', + 'en-US', + 'es-ES', + 'fr-FR', + 'ja-JP', + 'ko-KR', + 'pt-BR', + 'ru-RU', + 'tr-TR', + 'zh-CN', + 'zh-TW', + 'vi-VN', + ]), + }); + }); + }); +}); diff --git a/src/server/metadata.ts b/src/server/metadata.ts index 53ba3ae98ae8..71dc1aea088b 100644 --- a/src/server/metadata.ts +++ b/src/server/metadata.ts @@ -3,8 +3,9 @@ import qs from 'query-string'; import { BRANDING_NAME } from '@/const/branding'; import { DEFAULT_LANG } from '@/const/locale'; -import { OG_URL, getCanonicalUrl } from '@/const/url'; +import { OG_URL } from '@/const/url'; import { Locales, locales } from '@/locales/resources'; +import { getCanonicalUrl } from '@/server/utils/url'; import { formatDescLength, formatTitleLength } from '@/utils/genOG'; export class Meta { @@ -59,7 +60,6 @@ export class Meta { let links: any = {}; const defaultLink = getCanonicalUrl(path); for (const alterLocales of locales) { - if (locale === alterLocales) continue; links[alterLocales] = qs.stringifyUrl({ query: { hl: alterLocales }, url: defaultLink, @@ -125,7 +125,7 @@ export class Meta { }; if (alternate) { - data['alternateLocale'] = locales.filter((l) => l !== locale); + data['alternateLocale'] = locales; } return data; diff --git a/src/server/modules/AssistantStore/index.test.ts b/src/server/modules/AssistantStore/index.test.ts index 4631beb02354..975bb67190f8 100644 --- a/src/server/modules/AssistantStore/index.test.ts +++ b/src/server/modules/AssistantStore/index.test.ts @@ -13,7 +13,7 @@ describe('AssistantStore', () => { it('should return the index URL for a not supported language', () => { const agentMarket = new AssistantStore(); - const url = agentMarket.getAgentIndexUrl('ko-KR'); + const url = agentMarket.getAgentIndexUrl('xxx' as any); expect(url).toBe('https://chat-agents.lobehub.com'); }); diff --git a/src/server/modules/AssistantStore/index.ts b/src/server/modules/AssistantStore/index.ts index 3c36224107af..313dcece6f13 100644 --- a/src/server/modules/AssistantStore/index.ts +++ b/src/server/modules/AssistantStore/index.ts @@ -4,10 +4,6 @@ import { appEnv } from '@/config/app'; import { DEFAULT_LANG, isLocaleNotSupport } from '@/const/locale'; import { Locales, normalizeLocale } from '@/locales/resources'; -const checkSupportLocale = (lang: Locales) => { - return isLocaleNotSupport(lang) || normalizeLocale(lang) !== 'zh-CN'; -}; - export class AssistantStore { private readonly baseUrl: string; @@ -16,13 +12,13 @@ export class AssistantStore { } getAgentIndexUrl = (lang: Locales = DEFAULT_LANG) => { - if (checkSupportLocale(lang)) return this.baseUrl; + if (isLocaleNotSupport(lang)) return this.baseUrl; return urlJoin(this.baseUrl, `index.${normalizeLocale(lang)}.json`); }; getAgentUrl = (identifier: string, lang: Locales = DEFAULT_LANG) => { - if (checkSupportLocale(lang)) return urlJoin(this.baseUrl, `${identifier}.json`); + if (isLocaleNotSupport(lang)) return urlJoin(this.baseUrl, `${identifier}.json`); return urlJoin(this.baseUrl, `${identifier}.${normalizeLocale(lang)}.json`); }; diff --git a/src/server/sitemap.test.ts b/src/server/sitemap.test.ts new file mode 100644 index 000000000000..31396dff4eb5 --- /dev/null +++ b/src/server/sitemap.test.ts @@ -0,0 +1,179 @@ +// @vitest-environment node +import { describe, expect, it, vi } from 'vitest'; + +import { getCanonicalUrl } from '@/server/utils/url'; +import { AssistantCategory, PluginCategory } from '@/types/discover'; + +import { LAST_MODIFIED, Sitemap, SitemapType } from './sitemap'; + +describe('Sitemap', () => { + const sitemap = new Sitemap(); + + describe('getIndex', () => { + it('should return a valid sitemap index', () => { + const index = sitemap.getIndex(); + expect(index).toContain(''); + expect(index).toContain(''); + [ + SitemapType.Pages, + SitemapType.Assistants, + SitemapType.Plugins, + SitemapType.Models, + SitemapType.Providers, + ].forEach((type) => { + expect(index).toContain(`${getCanonicalUrl(`/sitemap/${type}.xml`)}`); + }); + expect(index).toContain(`${LAST_MODIFIED}`); + }); + }); + + describe('getPage', () => { + it('should return a valid page sitemap', async () => { + const pageSitemap = await sitemap.getPage(); + expect(pageSitemap).toContainEqual( + expect.objectContaining({ + url: getCanonicalUrl('/'), + changeFrequency: 'monthly', + priority: 0.4, + }), + ); + expect(pageSitemap).toContainEqual( + expect.objectContaining({ + url: getCanonicalUrl('/discover'), + changeFrequency: 'daily', + priority: 0.7, + }), + ); + Object.values(AssistantCategory).forEach((category) => { + expect(pageSitemap).toContainEqual( + expect.objectContaining({ + url: getCanonicalUrl(`/discover/assistants/${category}`), + changeFrequency: 'daily', + priority: 0.7, + }), + ); + }); + Object.values(PluginCategory).forEach((category) => { + expect(pageSitemap).toContainEqual( + expect.objectContaining({ + url: getCanonicalUrl(`/discover/plugins/${category}`), + changeFrequency: 'daily', + priority: 0.7, + }), + ); + }); + }); + }); + + describe('getAssistants', () => { + it('should return a valid assistants sitemap', async () => { + vi.spyOn(sitemap['discoverService'], 'getAssistantList').mockResolvedValue([ + // @ts-ignore + { identifier: 'test-assistant', createdAt: '2023-01-01' }, + ]); + + const assistantsSitemap = await sitemap.getAssistants(); + expect(assistantsSitemap.length).toBe(14); + expect(assistantsSitemap).toContainEqual( + expect.objectContaining({ + url: getCanonicalUrl('/discover/assistant/test-assistant'), + lastModified: '2023-01-01T00:00:00.000Z', + }), + ); + expect(assistantsSitemap).toContainEqual( + expect.objectContaining({ + url: getCanonicalUrl('/discover/assistant/test-assistant?hl=zh-CN'), + lastModified: '2023-01-01T00:00:00.000Z', + }), + ); + }); + }); + + describe('getPlugins', () => { + it('should return a valid plugins sitemap', async () => { + vi.spyOn(sitemap['discoverService'], 'getPluginList').mockResolvedValue([ + // @ts-ignore + { identifier: 'test-plugin', createdAt: '2023-01-01' }, + ]); + + const pluginsSitemap = await sitemap.getPlugins(); + expect(pluginsSitemap.length).toBe(14); + expect(pluginsSitemap).toContainEqual( + expect.objectContaining({ + url: getCanonicalUrl('/discover/plugin/test-plugin'), + lastModified: '2023-01-01T00:00:00.000Z', + }), + ); + expect(pluginsSitemap).toContainEqual( + expect.objectContaining({ + url: getCanonicalUrl('/discover/plugin/test-plugin?hl=ja-JP'), + lastModified: '2023-01-01T00:00:00.000Z', + }), + ); + }); + }); + + describe('getModels', () => { + it('should return a valid models sitemap', async () => { + vi.spyOn(sitemap['discoverService'], 'getModelList').mockResolvedValue([ + // @ts-ignore + { identifier: 'test:model', createdAt: '2023-01-01' }, + ]); + + const modelsSitemap = await sitemap.getModels(); + expect(modelsSitemap.length).toBe(14); + expect(modelsSitemap).toContainEqual( + expect.objectContaining({ + url: getCanonicalUrl('/discover/model/test:model'), + lastModified: '2023-01-01T00:00:00.000Z', + }), + ); + expect(modelsSitemap).toContainEqual( + expect.objectContaining({ + url: getCanonicalUrl('/discover/model/test:model?hl=ko-KR'), + lastModified: '2023-01-01T00:00:00.000Z', + }), + ); + }); + }); + + describe('getProviders', () => { + it('should return a valid providers sitemap', async () => { + vi.spyOn(sitemap['discoverService'], 'getProviderList').mockResolvedValue([ + // @ts-ignore + { identifier: 'test-provider', createdAt: '2023-01-01' }, + ]); + + const providersSitemap = await sitemap.getProviders(); + expect(providersSitemap.length).toBe(14); + expect(providersSitemap).toContainEqual( + expect.objectContaining({ + url: getCanonicalUrl('/discover/provider/test-provider'), + lastModified: '2023-01-01T00:00:00.000Z', + }), + ); + expect(providersSitemap).toContainEqual( + expect.objectContaining({ + url: getCanonicalUrl('/discover/provider/test-provider?hl=ar'), + lastModified: '2023-01-01T00:00:00.000Z', + }), + ); + }); + }); + + describe('getRobots', () => { + it('should return correct robots.txt entries', () => { + const robots = sitemap.getRobots(); + expect(robots).toContain(getCanonicalUrl('/sitemap-index.xml')); + [ + SitemapType.Pages, + SitemapType.Assistants, + SitemapType.Plugins, + SitemapType.Models, + SitemapType.Providers, + ].forEach((type) => { + expect(robots).toContain(getCanonicalUrl(`/sitemap/${type}.xml`)); + }); + }); + }); +}); diff --git a/src/server/sitemap.ts b/src/server/sitemap.ts new file mode 100644 index 000000000000..e0aa38cdd9d1 --- /dev/null +++ b/src/server/sitemap.ts @@ -0,0 +1,243 @@ +import { flatten } from 'lodash-es'; +import { MetadataRoute } from 'next'; +import qs from 'query-string'; +import urlJoin from 'url-join'; + +import { DEFAULT_LANG } from '@/const/locale'; +import { SITEMAP_BASE_URL } from '@/const/url'; +import { Locales, locales as allLocales } from '@/locales/resources'; +import { DiscoverService } from '@/server/services/discover'; +import { getCanonicalUrl } from '@/server/utils/url'; +import { AssistantCategory, PluginCategory } from '@/types/discover'; +import { isDev } from '@/utils/env'; + +export interface SitemapItem { + alternates?: { + languages?: string; + }; + changeFrequency?: 'always' | 'hourly' | 'daily' | 'weekly' | 'monthly' | 'yearly' | 'never'; + lastModified?: string | Date; + priority?: number; + url: string; +} + +export enum SitemapType { + Assistants = 'assistants', + Models = 'models', + Pages = 'pages', + Plugins = 'plugins', + Providers = 'providers', +} + +export const LAST_MODIFIED = new Date().toISOString(); + +export class Sitemap { + sitemapIndexs = [ + { id: SitemapType.Pages }, + { id: SitemapType.Assistants }, + { id: SitemapType.Plugins }, + { id: SitemapType.Models }, + { id: SitemapType.Providers }, + ]; + + private discoverService = new DiscoverService(); + + private _generateSitemapLink(url: string) { + return [ + '', + `${url}`, + `${LAST_MODIFIED}`, + '', + ].join('\n'); + } + + private _formatTime(time?: string) { + try { + if (!time) return LAST_MODIFIED; + return new Date(time).toISOString() || LAST_MODIFIED; + } catch { + return LAST_MODIFIED; + } + } + + private _genSitemapItem = ( + lang: Locales, + url: string, + { + lastModified, + changeFrequency = 'monthly', + priority = 0.4, + noLocales, + locales = allLocales, + }: { + changeFrequency?: SitemapItem['changeFrequency']; + lastModified?: string; + locales?: typeof allLocales; + noLocales?: boolean; + priority?: number; + } = {}, + ) => { + const sitemap = { + changeFrequency, + lastModified: this._formatTime(lastModified), + priority, + url: + lang === DEFAULT_LANG + ? getCanonicalUrl(url) + : qs.stringifyUrl({ query: { hl: lang }, url: getCanonicalUrl(url) }), + }; + if (noLocales) return sitemap; + + const languages: any = {}; + for (const locale of locales) { + if (locale === lang) continue; + languages[locale] = qs.stringifyUrl({ + query: { hl: locale }, + url: getCanonicalUrl(url), + }); + } + return { + alternates: { + languages, + }, + ...sitemap, + }; + }; + + private _genSitemap( + url: string, + { + lastModified, + changeFrequency = 'monthly', + priority = 0.4, + noLocales, + locales = allLocales, + }: { + changeFrequency?: SitemapItem['changeFrequency']; + lastModified?: string; + locales?: typeof allLocales; + noLocales?: boolean; + priority?: number; + } = {}, + ) { + if (noLocales) + return [ + this._genSitemapItem(DEFAULT_LANG, url, { + changeFrequency, + lastModified, + locales, + noLocales, + priority, + }), + ]; + return locales.map((lang) => + this._genSitemapItem(lang, url, { + changeFrequency, + lastModified, + locales, + noLocales, + priority, + }), + ); + } + + getIndex(): string { + return [ + '', + '', + ...this.sitemapIndexs.map((item) => + this._generateSitemapLink( + getCanonicalUrl(SITEMAP_BASE_URL, isDev ? item.id : `${item.id}.xml`), + ), + ), + '', + ].join('\n'); + } + + async getAssistants(): Promise { + const list = await this.discoverService.getAssistantList(DEFAULT_LANG); + const sitmap = list.map((item) => + this._genSitemap(urlJoin('/discover/assistant', item.identifier), { + lastModified: item?.createdAt || LAST_MODIFIED, + }), + ); + return flatten(sitmap); + } + + async getPlugins(): Promise { + const list = await this.discoverService.getPluginList(DEFAULT_LANG); + const sitmap = list.map((item) => + this._genSitemap(urlJoin('/discover/plugin', item.identifier), { + lastModified: item?.createdAt || LAST_MODIFIED, + }), + ); + return flatten(sitmap); + } + + async getModels(): Promise { + const list = await this.discoverService.getModelList(DEFAULT_LANG); + const sitmap = list.map((item) => + this._genSitemap(urlJoin('/discover/model', item.identifier), { + lastModified: item?.createdAt || LAST_MODIFIED, + }), + ); + return flatten(sitmap); + } + + async getProviders(): Promise { + const list = await this.discoverService.getProviderList(DEFAULT_LANG); + const sitmap = list.map((item) => + this._genSitemap(urlJoin('/discover/provider', item.identifier), { + lastModified: item?.createdAt || LAST_MODIFIED, + }), + ); + return flatten(sitmap); + } + + async getPage(): Promise { + const assistantsCategory = Object.values(AssistantCategory); + const pluginCategory = Object.values(PluginCategory); + const modelCategory = await this.discoverService.getProviderList(DEFAULT_LANG); + return [ + ...this._genSitemap('/', { noLocales: true }), + ...this._genSitemap('/chat', { noLocales: true }), + ...this._genSitemap('/welcome', { noLocales: true }), + /* ↓ cloud slot ↓ */ + + /* ↑ cloud slot ↑ */ + ...this._genSitemap('/discover', { changeFrequency: 'daily', priority: 0.7 }), + ...this._genSitemap('/discover/assistants', { changeFrequency: 'daily', priority: 0.7 }), + ...assistantsCategory.flatMap((slug) => + this._genSitemap(`/discover/assistants/${slug}`, { + changeFrequency: 'daily', + priority: 0.7, + }), + ), + ...this._genSitemap('/discover/plugins', { changeFrequency: 'daily', priority: 0.7 }), + ...pluginCategory.flatMap((slug) => + this._genSitemap(`/discover/plugins/${slug}`, { + changeFrequency: 'daily', + priority: 0.7, + }), + ), + ...this._genSitemap('/discover/models', { changeFrequency: 'daily', priority: 0.7 }), + ...modelCategory.flatMap((slug) => + this._genSitemap(`/discover/models/${slug}`, { + changeFrequency: 'daily', + priority: 0.7, + }), + ), + ...this._genSitemap('/discover/providers', { changeFrequency: 'daily', priority: 0.7 }), + ]; + } + getRobots() { + return [ + getCanonicalUrl('/sitemap-index.xml'), + ...this.sitemapIndexs.map((index) => + getCanonicalUrl(SITEMAP_BASE_URL, isDev ? index.id : `${index.id}.xml`), + ), + ]; + } +} + +export const sitemapModule = new Sitemap(); diff --git a/src/server/translation.test.ts b/src/server/translation.test.ts new file mode 100644 index 000000000000..c7de88295f8b --- /dev/null +++ b/src/server/translation.test.ts @@ -0,0 +1,137 @@ +// @vitest-environment node +import { cookies } from 'next/headers'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { DEFAULT_LANG, LOBE_LOCALE_COOKIE } from '@/const/locale'; +import { normalizeLocale } from '@/locales/resources'; +import * as env from '@/utils/env'; + +import { getLocale, translation } from './translation'; + +// Mock external dependencies +vi.mock('next/headers', () => ({ + cookies: vi.fn(), +})); + +vi.mock('node:fs', () => ({ + existsSync: vi.fn(), + readFileSync: vi.fn(), +})); + +vi.mock('node:path', () => ({ + join: vi.fn(), +})); + +vi.mock('@/const/locale', () => ({ + DEFAULT_LANG: 'en-US', + LOBE_LOCALE_COOKIE: 'LOBE_LOCALE', +})); + +vi.mock('@/locales/resources', () => ({ + normalizeLocale: vi.fn((locale) => locale), +})); + +vi.mock('@/utils/env', () => ({ + isDev: false, +})); + +describe('getLocale', () => { + const mockCookieStore = { + get: vi.fn(), + }; + + beforeEach(() => { + vi.clearAllMocks(); + (cookies as any).mockReturnValue(mockCookieStore); + }); + + it('should return the provided locale if hl is specified', async () => { + const result = await getLocale('fr-FR'); + expect(result).toBe('fr-FR'); + expect(normalizeLocale).toHaveBeenCalledWith('fr-FR'); + }); + + it('should return the locale from cookie if available', async () => { + mockCookieStore.get.mockReturnValue({ value: 'de-DE' }); + const result = await getLocale(); + expect(result).toBe('de-DE'); + expect(mockCookieStore.get).toHaveBeenCalledWith(LOBE_LOCALE_COOKIE); + }); + + it('should return DEFAULT_LANG if no cookie is set', async () => { + mockCookieStore.get.mockReturnValue(undefined); + const result = await getLocale(); + expect(result).toBe(DEFAULT_LANG); + }); +}); + +describe('translation', () => { + const mockTranslations = { + key1: 'Value 1', + key2: 'Value 2 with {{param}}', + nested: { key: 'Nested value' }, + }; + + beforeEach(() => { + vi.clearAllMocks(); + (fs.existsSync as any).mockReturnValue(true); + (fs.readFileSync as any).mockReturnValue(JSON.stringify(mockTranslations)); + (path.join as any).mockImplementation((...args: any) => args.join('/')); + }); + + it('should return correct translation object', async () => { + const result = await translation('common', 'en-US'); + expect(result).toHaveProperty('locale', 'en-US'); + expect(result).toHaveProperty('t'); + expect(typeof result.t).toBe('function'); + }); + + it('should translate keys correctly', async () => { + const { t } = await translation('common', 'en-US'); + expect(t('key1')).toBe('Value 1'); + expect(t('key2', { param: 'test' })).toBe('Value 2 with test'); + expect(t('nested.key')).toBe('Nested value'); + }); + + it('should return key if translation is not found', async () => { + const { t } = await translation('common', 'en-US'); + expect(t('nonexistent.key')).toBe('nonexistent.key'); + }); + + it('should use fallback language if specified locale file does not exist', async () => { + (fs.existsSync as any).mockReturnValueOnce(false); + await translation('common', 'nonexistent-LANG'); + expect(fs.readFileSync).toHaveBeenCalledWith( + expect.stringContaining(`/${DEFAULT_LANG}/common.json`), + 'utf8', + ); + }); + + it('should use zh-CN in dev mode when fallback is needed', async () => { + (fs.existsSync as any).mockReturnValueOnce(false); + (env.isDev as unknown as boolean) = true; + await translation('common', 'nonexistent-LANG'); + expect(fs.readFileSync).toHaveBeenCalledWith( + expect.stringContaining('/zh-CN/common.json'), + 'utf8', + ); + }); + + it('should handle file reading errors', async () => { + const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + (fs.readFileSync as any).mockImplementation(() => { + throw new Error('File read error'); + }); + + const result = await translation('common', 'en-US'); + expect(result.t('any.key')).toBe('any.key'); + expect(consoleErrorSpy).toHaveBeenCalledWith( + 'Error while reading translation file', + expect.any(Error), + ); + + consoleErrorSpy.mockRestore(); + }); +}); diff --git a/src/server/utils/url.test.ts b/src/server/utils/url.test.ts new file mode 100644 index 000000000000..945d83f8f414 --- /dev/null +++ b/src/server/utils/url.test.ts @@ -0,0 +1,61 @@ +// @vitest-environment node +import urlJoin from 'url-join'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +// 模拟 urlJoin 函数 +vi.mock('url-join', () => ({ + default: vi.fn((...args) => args.join('/')), +})); + +describe('getCanonicalUrl', () => { + const originalEnv = process.env; + + beforeEach(() => { + // 在每个测试前重置 process.env + vi.resetModules(); + process.env = { ...originalEnv }; + }); + + afterEach(() => { + // 在每个测试后恢复原始的 process.env + process.env = originalEnv; + }); + + it('should return correct URL for production environment', async () => { + process.env.VERCEL = undefined; + process.env.VERCEL_ENV = undefined; + + const { getCanonicalUrl } = await import('./url'); // 动态导入以获取最新的环境变量状态 + const result = getCanonicalUrl('path', 'to', 'page'); + expect(result).toBe('https://lobechat.com/path/to/page'); + expect(urlJoin).toHaveBeenCalledWith('https://lobechat.com', 'path', 'to', 'page'); + }); + + it('should return correct URL for Vercel preview environment', async () => { + process.env.VERCEL = '1'; + process.env.VERCEL_ENV = 'preview'; + process.env.VERCEL_URL = 'preview-url.vercel.app'; + + const { getCanonicalUrl } = await import('./url'); // 动态导入 + const result = getCanonicalUrl('path', 'to', 'page'); + expect(result).toBe('https://preview-url.vercel.app/path/to/page'); + expect(urlJoin).toHaveBeenCalledWith('https://preview-url.vercel.app', 'path', 'to', 'page'); + }); + + it('should return production URL when VERCEL is set but VERCEL_ENV is production', async () => { + process.env.VERCEL = '1'; + process.env.VERCEL_ENV = 'production'; + + const { getCanonicalUrl } = await import('./url'); // 动态导入 + const result = getCanonicalUrl('path', 'to', 'page'); + expect(result).toBe('https://lobechat.com/path/to/page'); + expect(urlJoin).toHaveBeenCalledWith('https://lobechat.com', 'path', 'to', 'page'); + }); + + it('should work correctly without additional path arguments', async () => { + const { getCanonicalUrl } = await import('./url'); // 动态导入 + const result = getCanonicalUrl(); + expect(result).toBe('https://lobechat.com'); + expect(urlJoin).toHaveBeenCalledWith('https://lobechat.com'); + }); +}); diff --git a/src/server/utils/url.ts b/src/server/utils/url.ts new file mode 100644 index 000000000000..44106f4ff512 --- /dev/null +++ b/src/server/utils/url.ts @@ -0,0 +1,9 @@ +import urlJoin from 'url-join'; + +const isVercelPreview = process.env.VERCEL === '1' && process.env.VERCEL_ENV !== 'production'; + +const vercelPreviewUrl = `https://${process.env.VERCEL_URL}`; + +const siteUrl = isVercelPreview ? vercelPreviewUrl : 'https://lobechat.com'; + +export const getCanonicalUrl = (...paths: string[]) => urlJoin(siteUrl, ...paths);