[experiment] proto use of experimental content monorepo api

.env

@@ -9,7 +9,31 @@ NEXT_PUBLIC_ALGOLIA_SEARCH_ONLY_API_KEY=2cd9898a6c253bfa3965d2b62a4f7f3d
 # which takes precedence over the NEXT_PUBLIC_ALGOLIA_INDEX env var.
 # NEXT_PUBLIC_ALGOLIA_INDEX=product_WAYPOINT
 
-MKTG_CONTENT_API="https://content.hashicorp.com"
+# From the dev-portal repo perspective, the marketing content API is _mostly_
+# used to fetch docs metadata, as well as docs content in the form of MDX files.
+# We use this environment variable for the purpose of docs metadata and content.
+# We hope to transition to a revised version of this API, backed by a single
+# content repository, at some point in the near future.
+#
+# NOTE: currently this points to a branch deployment off of
+# PR https://github.com/hashicorp/web-presence-experimental-docs/pull/18.
+# There's a chain of work that's still in progress, this is intended
+# as a prototype-ish demo of the new content API.
+MKTG_CONTENT_API="https://web-presence-experimental-docs.vercel.app"
+
+# To run things locally, check out the `zs.add-content-versions-route` of
+# `web-presence-experimental-docs`, and run `npm run dev` such that the dev
+# server for that app is running on port 3000. Then, you can run `npm run start`
+# in this `dev-portal` repo, which should start up on port 3001.
+# MKTG_CONTENT_API="http://localhost:3000"
+
+# The marketing content API serves a bunch of other purposes too... and the
+# dev-portal repo accesses some of those other functions directly. For example,
+# we determine which static paths to render. For these purposes, we do not have
+# a replacement in mind in the short-term. Given we do want to migrate other
+# functionality, we have a new environment variable here for each specific
+# purpose.
+STATIC_PATHS_API_URL="https://content.hashicorp.com"
 
 # Note: check .env.local.example for additional required env vars
 

build-libs/redirects.js

@@ -80,7 +80,23 @@ async function getRedirectsFromContentRepo(
 	 */
 	/** @type {string} */
 	let redirectsFileString
-	if (isDeveloperBuild) {
+	/**
+	 * TODO: replace this with a feature flag or something,
+	 * for now intent is to messily prototype and "make it work", so hard-coding
+	 * to test with all products makes sense at this stage, I think.
+	 */
+	const IS_CONTENT_API_PROTOTYPE = true
+	if (IS_CONTENT_API_PROTOTYPE) {
+		/**
+		 * TODO: fetch redirects from the new content API, which lives in
+		 * hashicorp/web-presence-experimental-docs.
+		 *
+		 * For now, ignoring authored redirects, as loading them from GitHub results
+		 * in mismatched versioning (since web-presence-experimental-docs has a
+		 * slightly old snapshot of all docs content).'
+		 */
+		return []
+	} else if (isDeveloperBuild) {
 		// For `hashicorp/dev-portal` builds, load redirects remotely
 		const latestContentRef = await getLatestContentRefForProduct(repoName)
 		redirectsFileString = await fetchGithubFile({

config/base.json

@@ -19,7 +19,7 @@
 			"session_max_age": 3600
 		},
 		"canonical_base_url": "https://developer.hashicorp.com",
-		"max_static_paths": 10,
+		"max_static_paths": 1,
 		"revalidate": 360,
 		"non_themed_paths": ["/sign-up"],
 		"datadog_config": {

next.config.js

@@ -104,6 +104,7 @@ module.exports = withHashicorp({
 			'www.datocms-assets.com',
 			'mktg-content-api-hashicorp.vercel.app',
 			'content.hashicorp.com',
+			'localhost',
 		],
 		dangerouslyAllowSVG: true,
 		contentSecurityPolicy: "default-src 'self'; script-src 'none'; sandbox;",

src/lib/__tests__/get-static-paths-from-analytics.test.ts

@@ -9,11 +9,11 @@ import { getStaticPathsFromAnalytics } from 'lib/get-static-paths-from-analytics
 
 import staticPathsResultFixture from './__fixtures__/static_paths_waypoint_docs.json'
 
-process.env.MKTG_CONTENT_API = 'https://content.hashicorp.com'
+process.env.STATIC_PATHS_API_URL = 'https://content.hashicorp.com'
 
 describe('getStaticPathsFromAnalytics', () => {
 	test('fetches static paths from the analytics endpoint - no valid paths', async () => {
-		nock(process.env.MKTG_CONTENT_API)
+		nock(process.env.STATIC_PATHS_API_URL)
 			.get('/api/static_paths')
 			.query({
 				product: 'developer',
@@ -33,7 +33,7 @@ describe('getStaticPathsFromAnalytics', () => {
 	})
 
 	test('fetches static paths from the analytics endpoint - filters with valid paths', async () => {
-		nock(process.env.MKTG_CONTENT_API)
+		nock(process.env.STATIC_PATHS_API_URL)
 			.get('/api/static_paths')
 			.query({
 				product: 'developer',

src/lib/get-static-paths-from-analytics.ts

@@ -46,7 +46,7 @@ export async function getStaticPathsFromAnalytics<Params = BaseParams>({
 }: GetStaticPathsFromAnalyticsOptions<Params>): Promise<StaticPaths<Params>> {
 	const endpoint = new URL(
 		`/api/static_paths?product=developer&param=${param}&limit=${limit}&path_prefix=${pathPrefix}`,
-		process.env.MKTG_CONTENT_API
+		process.env.STATIC_PATHS_API_URL
 	)
 
 	const { result } = await fetch(endpoint.toString()).then((res) => res.json())

src/lib/remark-plugins/remark-rewrite-image-urls.ts

@@ -0,0 +1,27 @@
+/**
+ * Copyright (c) HashiCorp, Inc.
+ * SPDX-License-Identifier: MPL-2.0
+ */
+
+import { visit } from 'unist-util-visit'
+import type { Plugin } from 'unified'
+import type { Image } from 'mdast'
+
+/**
+ * This is a generator function that returns a remark plugin
+ * to rewrite asset urls in markdown files.
+ */
+export function remarkRewriteImageUrls(args: {
+	urlRewriteFn?: (url: string) => string
+}): Plugin {
+	const { urlRewriteFn = (url) => url } = args
+
+	return function plugin() {
+		return function transform(tree) {
+			// @ts-expect-error Types Should be correct here
+			visit<Image>(tree, 'image', (node) => {
+				node.url = urlRewriteFn(node.url)
+			})
+		}
+	}
+}

src/lib/sitemap/docs-content-fields.ts

@@ -6,6 +6,10 @@
 import { makeSitemapField } from './helpers'
 
 export async function allDocsFields() {
+	/**
+	 * TODO: Update experimental content monorepo backed API to implement
+	 * an `all-docs-paths` endpoint.
+	 */
 	const getDocsPaths = await fetch(
 		`${process.env.MKTG_CONTENT_API}/api/all-docs-paths`
 	)

src/views/docs-view/loaders/content-api/index.ts

@@ -35,6 +35,7 @@ export async function fetchNavData(
 
 	const fullPath = `nav-data/${version}/${basePath}`
 	const url = `${MKTG_CONTENT_API}/api/content/${product}/${fullPath}`
+	console.log(`Fetching from MKTG_CONTENT_API "${url}"...`)
 
 	const response = await fetch(url)
 
@@ -53,6 +54,7 @@ export async function fetchDocument(
 	checkEnvVarsInDev()
 
 	const url = `${MKTG_CONTENT_API}/api/content/${product}/${fullPath}`
+	console.log(`Fetching from MKTG_CONTENT_API "${url}"...`)
 	const response = await fetch(url)
 
 	if (response.status !== 200) {
@@ -67,6 +69,7 @@ export async function fetchVersionMetadataList(product: string) {
 	checkEnvVarsInDev()
 
 	const url = `${MKTG_CONTENT_API}/api/content/${product}/version-metadata?partial=true`
+	console.log(`Fetching from MKTG_CONTENT_API "${url}"...`)
 	const response = await fetch(url)
 
 	if (response.status !== 200) {

src/views/docs-view/loaders/remote-content.ts

@@ -83,7 +83,7 @@ interface LoadStaticPropsReturn {
 
 const moizeOpts: Options = { isPromise: true, maxSize: Infinity }
 const cachedFetchNavData = moize(fetchNavData, moizeOpts)
-const cachedFetchVersionMetadataList = moize(
+export const cachedFetchVersionMetadataList = moize(
 	fetchVersionMetadataList,
 	moizeOpts
 )

src/views/docs-view/server.ts

@@ -10,7 +10,9 @@ import { Pluggable } from 'unified'
 import slugify from 'slugify'
 
 // HashiCorp Imports
-import RemoteContentLoader from './loaders/remote-content'
+import RemoteContentLoader, {
+	cachedFetchVersionMetadataList,
+} from './loaders/remote-content'
 import { anchorLinks } from '@hashicorp/remark-plugins'
 
 // Global imports
@@ -44,6 +46,9 @@ import { getCustomLayout } from './utils/get-custom-layout'
 import type { DocsViewPropOptions } from './utils/get-root-docs-path-generation-functions'
 import { DocsViewProps } from './types'
 import { isReleaseNotesPage } from 'lib/docs/is-release-notes-page'
+import { stripVersionFromPathParams } from './loaders/utils'
+import { rewriteImageUrlsForExperimentalContentApi } from './utils/rewrite-image-urls-for-experimental-content-api'
+import { remarkRewriteImageUrls } from 'lib/remark-plugins/remark-rewrite-image-urls'
 
 /**
  * Returns static generation functions which can be exported from a page to fetch docs data
@@ -172,10 +177,41 @@ export function getStaticGenerationFunctions<
 			)}`
 			const headings: AnchorLinksPluginHeading[] = [] // populated by anchorLinks plugin below
 
+			const [versionFromPath, paramsNoVersion] =
+				stripVersionFromPathParams(pathParts)
+
+			let versionForAssetLoader = versionFromPath
+			if (versionForAssetLoader === 'latest') {
+				const versionMetadata = await cachedFetchVersionMetadataList(
+					productSlugForLoader
+				)
+				versionForAssetLoader = versionMetadata.find((e) => e.isLatest)?.version
+			}
+
 			const loader = getLoader({
 				mainBranch,
 				remarkPlugins: [
 					...additionalRemarkPlugins,
+					/**
+					 * TODO: should only add asset rewriting for new content monorepo API
+					 * if the product has opted-in... so need a feature flag or something here...
+					 * but for prototyping purposes, trying this out for all products.
+					 *
+					 * TODO: maybe this URL rewriting should be done during content
+					 * migration? This would remove the need for changes to the dev-dot
+					 * front-end. It seems somewhat reasonable to have the content API
+					 * return an MDX document with image URLs that are already correct.
+					 */
+					remarkRewriteImageUrls({
+						urlRewriteFn: (url) =>
+							rewriteImageUrlsForExperimentalContentApi(
+								url,
+								paramsNoVersion.join('/'),
+								versionForAssetLoader,
+								productSlugForLoader,
+								basePathForLoader
+							),
+					}),
 					/**
 					 * Note on remark plugins for local vs remote loading:
 					 * includeMarkdown and paragraphCustomAlerts are already

src/views/docs-view/utils/get-root-docs-path-generation-functions.ts

@@ -46,11 +46,11 @@ export function getRootDocsPathGenerationFunctions(
 		product: productData,
 		productSlugForLoader: rootDocsPath.productSlugForLoader,
 		basePathForLoader: rootDocsPath.basePathForLoader,
-		mainBranch: rootDocsPath.mainBranch,
 		additionalRemarkPlugins: getAdditionalRemarkPlugins(
 			productData,
 			rootDocsPath
 		),
+		mainBranch: rootDocsPath.mainBranch,
 		getScope: generateGetScope(productData, rootDocsPath),
 		options,
 	}
@@ -90,11 +90,14 @@ function getAdditionalRemarkPlugins(
 	productData: ProductData,
 	rootDocsPath: RootDocsPath
 ): Pluggable[] {
+	//
+	let additionalRemarkPlugins = []
+	//
 	if (productData.slug == 'sentinel' && rootDocsPath.path == 'docs') {
-		return [remarkSentinel]
-	} else {
-		return []
+		additionalRemarkPlugins.push(remarkSentinel)
 	}
+	//
+	return additionalRemarkPlugins
 }
 
 /**

src/views/docs-view/utils/get-valid-versions.ts

@@ -49,6 +49,9 @@ export async function getValidVersions(
 		const validVersionsUrl = new URL(VERSIONS_ENDPOINT, CONTENT_API_URL)
 		validVersionsUrl.searchParams.set('product', productSlugForLoader)
 		validVersionsUrl.searchParams.set('fullPath', fullPath)
+		console.log(
+			`Fetching from MKTG_CONTENT_API "${validVersionsUrl.toString()}"...`
+		)
 		// Fetch known versions of this document
 		const response = await fetch(validVersionsUrl.toString())
 		const { versions: knownVersions } = await response.json()

src/views/docs-view/utils/rewrite-image-urls-for-experimental-content-api.ts

@@ -0,0 +1,68 @@
+const REPO_CONFIG_CONTENT_DIR: Record<string, string> = {
+	boundary: 'content',
+	consul: 'content',
+	'hcp-docs': 'content',
+	nomad: 'content',
+	packer: 'content',
+	'ptfe-releases': 'docs',
+	sentinel: 'content',
+	terraform: 'docs',
+	'terraform-cdk': 'docs',
+	'terraform-docs-agents': 'docs',
+	'terraform-docs-common': 'docs',
+	'terraform-plugin-framework': 'docs',
+	'terraform-plugin-log': 'docs',
+	'terraform-plugin-mux': 'docs',
+	'terraform-plugin-sdk': 'docs',
+	'terraform-plugin-testing': 'docs',
+	vagrant: 'content',
+	vault: 'content',
+	waypoint: 'content',
+}
+
+export function rewriteImageUrlsForExperimentalContentApi(
+	url,
+	currentPath,
+	_currentVersion,
+	productSlugForLoader,
+	docsBasePath
+) {
+	/**
+	 * Rewrite all URLs to use the content API for assets
+	 *
+	 * TODO: versioned assets are a work in progress.
+	 *
+	 * One option might be handle `currentVersion` here... but so far, in the
+	 * unified docs repo `web-presence-experimental-docs`, we don't have a clear
+	 * approach to versioned assets. Instead, all assets are currently in
+	 * a single not-versioned directory.
+	 *
+	 * No matter what URL adjustment we do, we'll probably _not_ want to do it
+	 * here, and instead we'll probably want to make `.mdx` image URL adjustments
+	 * at *build time*, as part of our build time MDX transforms, over in
+	 * `web-presence-experimental-docs`.
+	 */
+	let assetPrefix = `${process.env.MKTG_CONTENT_API}/assets/${productSlugForLoader}`
+	/**
+	 * TODO: we have some messy shims to handle asset organization.
+	 * Probably ideal to standardize this in the content monorepo instead,
+	 * maybe as part of a migration script in the content monorepo?
+	 */
+	if (url.startsWith('/')) {
+		// Rewrite absolute URLs
+		// TODO: maybe this should be done during content migration?
+		return `${assetPrefix}${url}`
+	} else if (url.startsWith('./')) {
+		// Rewrite relative URLs
+		// TODO: maybe this should be done during content migration?
+		const currentPathRelative = currentPath.split('/').slice(0, -1).join('/')
+		const contentDir = REPO_CONFIG_CONTENT_DIR[productSlugForLoader]
+		const contentDirPrefix = `/img/${contentDir}/${docsBasePath}` // rootDocsPath.path
+		const finalPath = `${assetPrefix}${contentDirPrefix}/${currentPathRelative}/${url.substring(
+			2
+		)}`
+		return finalPath
+	} else {
+		return url
+	}
+}