Merge pull request #395 from ieedan/docs-fixes

sites/docs/package.json

@@ -14,14 +14,12 @@
 	"devDependencies": {
 		"@fontsource-variable/inter": "^5.1.1",
 		"@fontsource-variable/jetbrains-mono": "^5.1.2",
-		"@shikijs/markdown-it": "^2.1.0",
 		"@sveltejs/adapter-vercel": "^5.5.3",
 		"@sveltejs/kit": "^2.16.1",
 		"@sveltejs/vite-plugin-svelte": "^5.0.3",
 		"@tailwindcss/typography": "^0.5.16",
 		"@types/dompurify": "^3.2.0",
 		"@types/eslint": "^9.6.1",
-		"@types/js-cookie": "^3.0.6",
 		"@types/markdown-it": "^14.1.2",
 		"@types/npm-package-arg": "^6.1.4",
 		"@upstash/redis": "^1.34.3",
@@ -33,7 +31,6 @@
 		"eslint-plugin-svelte": "^2.46.1",
 		"globals": "^15.14.0",
 		"isomorphic-dompurify": "^2.20.0",
-		"js-cookie": "^3.0.5",
 		"jsrepo": "^1.29.1",
 		"lucide-svelte": "^0.473.0",
 		"markdown-it": "^14.1.0",

sites/docs/src/hooks.server.ts

@@ -3,5 +3,7 @@ import { redirect } from '@sveltejs/kit';
 export const handle = async ({ event, resolve }) => {
 	if (event.url.pathname === '/about') throw redirect(303, '/docs/about');
 
-	return resolve(event);
+	const response = await resolve(event);
+
+	return response;
 };

sites/docs/src/lib/ts/markdown.ts

@@ -1,9 +1,12 @@
 import MarkdownIt from 'markdown-it';
-import Shiki from '@shikijs/markdown-it';
 import { markdownItTable } from 'markdown-it-table';
 
+let md: MarkdownIt | null = null;
+
 const markdownIt = async () => {
-	const md = MarkdownIt();
+	if (md != null) return md;
+
+	const newMd = MarkdownIt({ html: true });
 
 	const stripComments = (md: MarkdownIt) => {
 		md.core.ruler.before('normalize', 'strip_comments', function (state) {
@@ -12,16 +15,10 @@ const markdownIt = async () => {
 	};
 
 	// plugins
-	md.use(stripComments);
-	md.use(markdownItTable);
-	md.use(
-		await Shiki({
-			themes: {
-				light: 'github-light-default',
-				dark: 'github-dark-default'
-			}
-		})
-	);
+	newMd.use(stripComments);
+	newMd.use(markdownItTable);
+
+	md = newMd;
 
 	return md;
 };

sites/docs/src/lib/ts/redis-client.ts

@@ -1,7 +1,6 @@
 import { UPSTASH_REDIS_TOKEN, UPSTASH_REDIS_URL } from '$env/static/private';
 import { Redis } from '@upstash/redis';
 
-export const REGISTRY_CACHE_PREFIX = 'registry';
 export const VIEW_PREFIX = 'view';
 
 const redis = new Redis({

sites/docs/src/lib/ts/registry/index.ts

@@ -1,13 +1,17 @@
 import {
 	fetchManifest,
 	github,
+	gitlab,
 	type Manifest,
 	type RegistryProvider,
 	type RegistryProviderState
 } from 'jsrepo';
 import { markdownIt } from '../markdown';
 import DOMPurify from 'isomorphic-dompurify';
-import { GITHUB_TOKEN } from '$env/static/private';
+import { GITHUB_TOKEN, GITLAB_TOKEN } from '$env/static/private';
+import { redis } from '../redis-client';
+
+const REGISTRY_STATE_CACHE_PREFIX = 'registry:state';
 
 export type RegistryPageData = {
 	registryUrl: string;
@@ -17,14 +21,62 @@ export type RegistryPageData = {
 
 export type RegistryInfo = Omit<RegistryPageData, 'registryUrl'>;
 
-export const getRegistryData = async (
+/** Gets the provider state either locally or from the cache then caches the state
+ *
+ * @param registryUrl
+ * @param provider
+ * @param param2
+ * @returns
+ */
+export const getProviderState = async (
+	registryUrl: string,
 	provider: RegistryProvider,
-	registryUrl: string
-): Promise<RegistryInfo | undefined> => {
-	const providerState = await provider.state(registryUrl, { token: getProviderToken(provider) });
+	{ cache = true }: { cache?: boolean } = {}
+): Promise<RegistryProviderState> => {
+	const stateKey = `${REGISTRY_STATE_CACHE_PREFIX}:${registryUrl}`;
+
+	let state: RegistryProviderState | undefined = undefined;
+
+	// http never needs time to get the state
+	if (cache && provider.name !== 'http') {
+		const getCached = async (): Promise<RegistryProviderState | undefined> => {
+			const s = await redis.get(stateKey);
 
+			if (!s) return undefined;
+
+			// s has everything except for the provider
+			return {
+				...s,
+				provider
+			} as RegistryProviderState;
+		};
+
+		const getLocal = async () =>
+			await provider.state(registryUrl, { token: getProviderToken(provider) });
+
+		// whichever is faster we use they should say the same thing
+		const result = await Promise.race([getLocal(), getCached()]);
+
+		state = result;
+	}
+
+	if (!state) {
+		state = await provider.state(registryUrl, { token: getProviderToken(provider) });
+	}
+
+	// never cache http
+	if (provider.name !== 'http') {
+		await redis.set(stateKey, state);
+	}
+
+	return state;
+};
+
+export const getRegistryData = async (
+	providerState: RegistryProviderState
+): Promise<RegistryInfo | undefined> => {
 	const [manifestResult, readmeResult] = await Promise.all([
-		fetchManifest(providerState, { token: getProviderToken(provider) }),
+		fetchManifest(providerState, { token: getProviderToken(providerState.provider) }),
 		fetchReadme(providerState)
 	]);
 
@@ -82,6 +134,8 @@ const getProviderToken = (provider: RegistryProvider): string | undefined => {
 	switch (provider.name) {
 		case github.name:
 			return GITHUB_TOKEN;
+		case gitlab.name:
+			return GITLAB_TOKEN;
 		// add the rest of the tokens here
 	}
 

sites/docs/src/lib/ts/server-actions/search-registries/server.ts

@@ -1,5 +1,4 @@
-import { getRegistryData } from '$lib/ts/registry';
-import { checkCache, updateCache } from '$lib/ts/registry/cache';
+import { getProviderState } from '$lib/ts/registry';
 import { fail, redirect, type RequestEvent } from '@sveltejs/kit';
 import { selectProvider } from 'jsrepo';
 import { message, setError, superValidate } from 'sveltekit-superforms';
@@ -27,19 +26,8 @@ export const action = async (event: RequestEvent) => {
 		return message(form, 'You are already there!');
 	}
 
-	const cache = await checkCache(registryUrl);
-
-	if (cache) {
-		throw redirect(303, `/registry?url=${registryUrl}`);
-	}
-
-	const pageData = await getRegistryData(provider, registryUrl);
-
-	if (!pageData) {
-		return setError(form, 'search', 'Invalid registry url');
-	}
-
-	updateCache(registryUrl, pageData);
+	// just gets the state and sets the cache
+	await getProviderState(registryUrl, provider, { cache: true });
 
 	throw redirect(303, `/registry?url=${registryUrl}`);
 };