♻️ refactor: refactor the sitemap implement (#4012)

* ✨ feat: Add new sitemap

* 🐛 fix: Fix url

* ✅ test: Add test

* 🐛 fix: Fix host

* ✅ test: Fix test

* ✅ test: Fix test

* 🐛 fix: Fix alternative

* 🐛 fix: Try to fix

* 🐛 fix: Fix build

* 🐛 fix: Fix build

* 🔧 chore: Update git ignore

* 🐛 fix: Fix review problem

.gitignore

@@ -55,6 +55,8 @@ next-env.d.ts
 .next
 .env
 public/*.js
+public/sitemap.xml
+public/sitemap-index.xml
 bun.lockb
 sitemap*.xml
 robots.txt

next.config.mjs

@@ -107,6 +107,16 @@ const nextConfig = {
   output: buildWithDocker ? 'standalone' : undefined,
   reactStrictMode: true,
   redirects: async () => [
+    {
+      destination: '/sitemap-index.xml',
+      permanent: true,
+      source: '/sitemap.xml',
+    },
+    {
+      destination: '/discover',
+      permanent: true,
+      source: '/market',
+    },
     {
       destination: '/settings/common',
       permanent: true,

package.json

@@ -29,11 +29,11 @@
     "build": "next build",
     "postbuild": "npm run build-sitemap && npm run build-migrate-db",
     "build-migrate-db": "bun run db:migrate",
-    "build-sitemap": "next-sitemap --config next-sitemap.config.mjs",
+    "build-sitemap": "tsx ./scripts/buildSitemapIndex/index.ts",
     "build:analyze": "ANALYZE=true next build",
     "build:docker": "DOCKER=true next build && npm run build-sitemap",
     "db:generate": "drizzle-kit generate",
-    "db:migrate": "MIGRATION_DB=1 tsx scripts/migrateServerDB/index.ts",
+    "db:migrate": "MIGRATION_DB=1 tsx ./scripts/migrateServerDB/index.ts",
     "db:push": "drizzle-kit push",
     "db:push-test": "NODE_ENV=test drizzle-kit push",
     "db:studio": "drizzle-kit studio",
@@ -65,11 +65,11 @@
     "test:update": "vitest -u",
     "type-check": "tsc --noEmit",
     "webhook:ngrok": "ngrok http http://localhost:3011",
-    "workflow:docs": "tsx scripts/docsWorkflow/index.ts",
-    "workflow:i18n": "tsx scripts/i18nWorkflow/index.ts",
+    "workflow:docs": "tsx ./scripts/docsWorkflow/index.ts",
+    "workflow:i18n": "tsx ./scripts/i18nWorkflow/index.ts",
     "workflow:mdx": "tsx ./scripts/mdxWorkflow/index.ts",
     "workflow:mdx-with-lint": "tsx ./scripts/mdxWorkflow/index.ts && eslint \"docs/**/*.mdx\" --quiet --fix",
-    "workflow:readme": "tsx scripts/readmeWorkflow/index.ts"
+    "workflow:readme": "tsx ./scripts/readmeWorkflow/index.ts"
   },
   "lint-staged": {
     "*.md": [
@@ -172,7 +172,6 @@
     "next": "14.2.8",
     "next-auth": "beta",
     "next-mdx-remote": "^4.4.1",
-    "next-sitemap": "^4.2.3",
     "nextjs-toploader": "^3.6.15",
     "numeral": "^2.0.6",
     "nuqs": "^1.17.8",

scripts/buildSitemapIndex/index.ts

@@ -0,0 +1,12 @@
+import { writeFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+
+import { sitemapModule } from '@/server/sitemap';
+
+const genSitemap = () => {
+  const sitemapIndexXML = sitemapModule.getIndex();
+  const filename = resolve(__dirname, '../../', 'public', 'sitemap-index.xml');
+  writeFileSync(filename, sitemapIndexXML);
+};
+
+genSitemap();

src/app/(main)/discover/(detail)/assistant/[slug]/page.tsx

@@ -105,6 +105,7 @@ const Page = async ({ params, searchParams }: Props) => {
           />
         }
         /* ↓ cloud slot ↓ */
+
         /* ↑ cloud slot ↑ */
       >
         <Temp data={data} identifier={identifier} />

src/app/(main)/discover/(detail)/model/[...slugs]/features/ProviderList/ProviderItem.tsx

@@ -67,6 +67,7 @@ const ProviderItem = memo<ProviderItemProps>(({ mobile, modelId, identifier }) =
         : '--',
     },
     /* ↓ cloud slot ↓ */
+
     /* ↑ cloud slot ↑ */
   ];
 

src/app/(main)/discover/(detail)/model/[...slugs]/page.tsx

@@ -100,6 +100,7 @@ const Page = async ({ params, searchParams }: Props) => {
         mobile={mobile}
         sidebar={<InfoSidebar data={data} identifier={identifier} mobile={mobile} />}
         /* ↓ cloud slot ↓ */
+
         /* ↑ cloud slot ↑ */
       >
         <ProviderList data={providerData} identifier={identifier} mobile={mobile} />

src/app/(main)/discover/(detail)/plugin/[slug]/page.tsx

@@ -89,6 +89,7 @@ const Page = async ({ params, searchParams }: Props) => {
         mobile={mobile}
         sidebar={<InfoSidebar data={data} identifier={identifier} mobile={mobile} />}
         /* ↓ cloud slot ↓ */
+
         /* ↑ cloud slot ↑ */
       >
         <ParameterList data={data} />

src/app/(main)/discover/(detail)/provider/[slug]/features/ModelList/ModelItem.tsx

@@ -79,6 +79,7 @@ const ModelItem = memo<SuggestionItemProps>(({ mobile, meta, identifier }) => {
         : '--',
     },
     /* ↓ cloud slot ↓ */
+
     /* ↑ cloud slot ↑ */
   ];
 

src/app/(main)/discover/(detail)/provider/[slug]/page.tsx

@@ -102,6 +102,7 @@ const Page = async ({ params, searchParams }: Props) => {
         mobile={mobile}
         sidebar={<InfoSidebar data={data} identifier={identifier} />}
         /* ↓ cloud slot ↓ */
+
         /* ↑ cloud slot ↑ */
       >
         <ModelList identifier={identifier} mobile={mobile} modelData={modelData} />

src/app/(main)/discover/(list)/_layout/Desktop/Nav.tsx

@@ -108,6 +108,7 @@ const Nav = memo(() => {
       {!isHome && !isProviders && (
         <Flexbox align={'center'} gap={4} horizontal>
           {/* ↓ cloud slot ↓ */}
+
           {/* ↑ cloud slot ↑ */}
         </Flexbox>
       )}

src/app/(main)/discover/_layout/Desktop/index.tsx

@@ -14,6 +14,7 @@ const Layout = ({ children }: PropsWithChildren) => {
         {children}
       </Flexbox>
       {/* ↓ cloud slot ↓ */}
+
       {/* ↑ cloud slot ↑ */}
     </>
   );

src/app/page.tsx

@@ -1,6 +1,6 @@
 import { Metadata } from 'next';
 
-import { getCanonicalUrl } from '@/const/url';
+import { getCanonicalUrl } from '@/server/utils/url';
 
 import Client from './(loading)/Client';
 import Redirect from './(loading)/Redirect';

src/app/robots.tsx

@@ -0,0 +1,16 @@
+import { MetadataRoute } from 'next';
+
+import { sitemapModule } from '@/server/sitemap';
+import { getCanonicalUrl } from '@/server/utils/url';
+
+export default function robots(): MetadataRoute.Robots {
+  return {
+    host: getCanonicalUrl(),
+    rules: {
+      allow: ['/'],
+      disallow: ['/api/*'],
+      userAgent: '*',
+    },
+    sitemap: sitemapModule.getRobots(),
+  };
+}

src/app/sitemap.tsx

@@ -0,0 +1,30 @@
+import { MetadataRoute } from 'next';
+
+import { SitemapType, sitemapModule } from '@/server/sitemap';
+
+export const generateSitemaps = async () => {
+  // Fetch the total number of products and calculate the number of sitemaps needed
+  return sitemapModule.sitemapIndexs;
+};
+
+const Sitemap = async ({ id }: { id: SitemapType }): Promise<MetadataRoute.Sitemap> => {
+  switch (id) {
+    case SitemapType.Pages: {
+      return sitemapModule.getPage();
+    }
+    case SitemapType.Assistants: {
+      return sitemapModule.getAssistants();
+    }
+    case SitemapType.Plugins: {
+      return sitemapModule.getPlugins();
+    }
+    case SitemapType.Models: {
+      return sitemapModule.getModels();
+    }
+    case SitemapType.Providers: {
+      return sitemapModule.getProviders();
+    }
+  }
+};
+
+export default Sitemap;

src/const/url.ts

@@ -2,6 +2,7 @@ import qs from 'query-string';
 import urlJoin from 'url-join';
 
 import { withBasePath } from '@/utils/basePath';
+import { isDev } from '@/utils/env';
 
 import pkg from '../../package.json';
 import { INBOX_SESSION_ID } from './session';
@@ -12,8 +13,6 @@ export const OFFICIAL_URL = 'https://lobechat.com/';
 export const OFFICIAL_PREVIEW_URL = 'https://chat-preview.lobehub.com/';
 export const OFFICIAL_SITE = 'https://lobehub.com/';
 
-export const getCanonicalUrl = (path: string) => urlJoin(OFFICIAL_URL, path);
-
 export const OG_URL = '/og/cover.png?v=1';
 
 export const GITHUB = pkg.homepage;
@@ -73,3 +72,4 @@ export const mailTo = (email: string) => `mailto:${email}`;
 
 export const AES_GCM_URL = 'https://datatracker.ietf.org/doc/html/draft-ietf-avt-srtp-aes-gcm-01';
 export const BASE_PROVIDER_DOC_URL = 'https://lobehub.com/docs/usage/providers';
+export const SITEMAP_BASE_URL = isDev ? '/sitemap.xml/' : 'sitemap';

src/server/ld.test.ts

@@ -0,0 +1,102 @@
+// @vitest-environment node
+import { describe, expect, it } from 'vitest';
+
+import { DEFAULT_LANG } from '@/const/locale';
+
+import { AUTHOR_LIST, Ld } from './ld';
+
+describe('Ld', () => {
+  const ld = new Ld();
+
+  describe('generate', () => {
+    it('should generate correct LD+JSON structure', () => {
+      const result = ld.generate({
+        title: 'Test Title',
+        description: 'Test Description',
+        url: 'https://example.com/test',
+        locale: DEFAULT_LANG,
+      });
+
+      expect(result['@context']).toBe('https://schema.org');
+      expect(Array.isArray(result['@graph'])).toBe(true);
+      expect(result['@graph'].length).toBeGreaterThan(0);
+    });
+  });
+
+  describe('genOrganization', () => {
+    it('should generate correct organization structure', () => {
+      const org = ld.genOrganization();
+
+      expect(org['@type']).toBe('Organization');
+      expect(org.name).toBe('LobeHub');
+      expect(org.url).toBe('https://lobehub.com/');
+    });
+  });
+
+  describe('getAuthors', () => {
+    it('should return default author when no ids provided', () => {
+      const author = ld.getAuthors();
+      expect(author['@type']).toBe('Organization');
+    });
+
+    it('should return person when valid id provided', () => {
+      const author = ld.getAuthors(['arvinxx']);
+      expect(author['@type']).toBe('Person');
+      // @ts-ignore
+      expect(author.name).toBe(AUTHOR_LIST.arvinxx.name);
+    });
+  });
+
+  describe('genWebPage', () => {
+    it('should generate correct webpage structure', () => {
+      const webpage = ld.genWebPage({
+        title: 'Test Page',
+        description: 'Test Description',
+        url: 'https://example.com/test',
+        locale: DEFAULT_LANG,
+      });
+
+      expect(webpage['@type']).toBe('WebPage');
+      expect(webpage.name).toBe('Test Page · LobeChat');
+      expect(webpage.description).toBe('Test Description');
+    });
+  });
+
+  describe('genImageObject', () => {
+    it('should generate correct image object', () => {
+      const image = ld.genImageObject({
+        image: 'https://example.com/image.jpg',
+        url: 'https://example.com/test',
+      });
+
+      expect(image['@type']).toBe('ImageObject');
+      expect(image.url).toBe('https://example.com/image.jpg');
+    });
+  });
+
+  describe('genWebSite', () => {
+    it('should generate correct website structure', () => {
+      const website = ld.genWebSite();
+
+      expect(website['@type']).toBe('WebSite');
+      expect(website.name).toBe('LobeChat');
+    });
+  });
+
+  describe('genArticle', () => {
+    it('should generate correct article structure', () => {
+      const article = ld.genArticle({
+        title: 'Test Article',
+        description: 'Test Description',
+        url: 'https://example.com/test',
+        author: ['arvinxx'],
+        identifier: 'test-id',
+        locale: DEFAULT_LANG,
+      });
+
+      expect(article['@type']).toBe('Article');
+      expect(article.headline).toBe('Test Article · LobeChat');
+      expect(article.author['@type']).toBe('Person');
+    });
+  });
+});