diff --git a/.env b/.env index 00e0355545..db1c11edf2 100644 --- a/.env +++ b/.env @@ -9,7 +9,31 @@ NEXT_PUBLIC_ALGOLIA_SEARCH_ONLY_API_KEY=2cd9898a6c253bfa3965d2b62a4f7f3d # which takes precedence over the NEXT_PUBLIC_ALGOLIA_INDEX env var. # NEXT_PUBLIC_ALGOLIA_INDEX=product_WAYPOINT -MKTG_CONTENT_API="https://content.hashicorp.com" +# From the dev-portal repo perspective, the marketing content API is _mostly_ +# used to fetch docs metadata, as well as docs content in the form of MDX files. +# We use this environment variable for the purpose of docs metadata and content. +# We hope to transition to a revised version of this API, backed by a single +# content repository, at some point in the near future. +# +# NOTE: currently this points to a branch deployment off of +# PR https://github.com/hashicorp/web-presence-experimental-docs/pull/18. +# There's a chain of work that's still in progress, this is intended +# as a prototype-ish demo of the new content API. +MKTG_CONTENT_API="https://web-presence-experimental-docs.vercel.app" + +# To run things locally, check out the `zs.add-content-versions-route` of +# `web-presence-experimental-docs`, and run `npm run dev` such that the dev +# server for that app is running on port 3000. Then, you can run `npm run start` +# in this `dev-portal` repo, which should start up on port 3001. +# MKTG_CONTENT_API="http://localhost:3000" + +# The marketing content API serves a bunch of other purposes too... and the +# dev-portal repo accesses some of those other functions directly. For example, +# we determine which static paths to render. For these purposes, we do not have +# a replacement in mind in the short-term. Given we do want to migrate other +# functionality, we have a new environment variable here for each specific +# purpose. +STATIC_PATHS_API_URL="https://content.hashicorp.com" # Note: check .env.local.example for additional required env vars diff --git a/build-libs/redirects.js b/build-libs/redirects.js index 9f273bc68b..f7853edbdb 100644 --- a/build-libs/redirects.js +++ b/build-libs/redirects.js @@ -80,7 +80,23 @@ async function getRedirectsFromContentRepo( */ /** @type {string} */ let redirectsFileString - if (isDeveloperBuild) { + /** + * TODO: replace this with a feature flag or something, + * for now intent is to messily prototype and "make it work", so hard-coding + * to test with all products makes sense at this stage, I think. + */ + const IS_CONTENT_API_PROTOTYPE = true + if (IS_CONTENT_API_PROTOTYPE) { + /** + * TODO: fetch redirects from the new content API, which lives in + * hashicorp/web-presence-experimental-docs. + * + * For now, ignoring authored redirects, as loading them from GitHub results + * in mismatched versioning (since web-presence-experimental-docs has a + * slightly old snapshot of all docs content).' + */ + return [] + } else if (isDeveloperBuild) { // For `hashicorp/dev-portal` builds, load redirects remotely const latestContentRef = await getLatestContentRefForProduct(repoName) redirectsFileString = await fetchGithubFile({ diff --git a/config/base.json b/config/base.json index 4e4321a3a5..68e76eb69f 100644 --- a/config/base.json +++ b/config/base.json @@ -19,7 +19,7 @@ "session_max_age": 3600 }, "canonical_base_url": "https://developer.hashicorp.com", - "max_static_paths": 10, + "max_static_paths": 1, "revalidate": 360, "non_themed_paths": ["/sign-up"], "datadog_config": { diff --git a/next.config.js b/next.config.js index c60be5d862..d78ab8f54a 100644 --- a/next.config.js +++ b/next.config.js @@ -104,6 +104,7 @@ module.exports = withHashicorp({ 'www.datocms-assets.com', 'mktg-content-api-hashicorp.vercel.app', 'content.hashicorp.com', + 'localhost', ], dangerouslyAllowSVG: true, contentSecurityPolicy: "default-src 'self'; script-src 'none'; sandbox;", diff --git a/src/lib/__tests__/get-static-paths-from-analytics.test.ts b/src/lib/__tests__/get-static-paths-from-analytics.test.ts index 3527cbaf23..7f97b74bca 100644 --- a/src/lib/__tests__/get-static-paths-from-analytics.test.ts +++ b/src/lib/__tests__/get-static-paths-from-analytics.test.ts @@ -9,11 +9,11 @@ import { getStaticPathsFromAnalytics } from 'lib/get-static-paths-from-analytics import staticPathsResultFixture from './__fixtures__/static_paths_waypoint_docs.json' -process.env.MKTG_CONTENT_API = 'https://content.hashicorp.com' +process.env.STATIC_PATHS_API_URL = 'https://content.hashicorp.com' describe('getStaticPathsFromAnalytics', () => { test('fetches static paths from the analytics endpoint - no valid paths', async () => { - nock(process.env.MKTG_CONTENT_API) + nock(process.env.STATIC_PATHS_API_URL) .get('/api/static_paths') .query({ product: 'developer', @@ -33,7 +33,7 @@ describe('getStaticPathsFromAnalytics', () => { }) test('fetches static paths from the analytics endpoint - filters with valid paths', async () => { - nock(process.env.MKTG_CONTENT_API) + nock(process.env.STATIC_PATHS_API_URL) .get('/api/static_paths') .query({ product: 'developer', diff --git a/src/lib/get-static-paths-from-analytics.ts b/src/lib/get-static-paths-from-analytics.ts index 73a2957ef9..357b884317 100644 --- a/src/lib/get-static-paths-from-analytics.ts +++ b/src/lib/get-static-paths-from-analytics.ts @@ -46,7 +46,7 @@ export async function getStaticPathsFromAnalytics({ }: GetStaticPathsFromAnalyticsOptions): Promise> { const endpoint = new URL( `/api/static_paths?product=developer¶m=${param}&limit=${limit}&path_prefix=${pathPrefix}`, - process.env.MKTG_CONTENT_API + process.env.STATIC_PATHS_API_URL ) const { result } = await fetch(endpoint.toString()).then((res) => res.json()) diff --git a/src/lib/remark-plugins/remark-rewrite-image-urls.ts b/src/lib/remark-plugins/remark-rewrite-image-urls.ts new file mode 100644 index 0000000000..4350db02f2 --- /dev/null +++ b/src/lib/remark-plugins/remark-rewrite-image-urls.ts @@ -0,0 +1,27 @@ +/** + * Copyright (c) HashiCorp, Inc. + * SPDX-License-Identifier: MPL-2.0 + */ + +import { visit } from 'unist-util-visit' +import type { Plugin } from 'unified' +import type { Image } from 'mdast' + +/** + * This is a generator function that returns a remark plugin + * to rewrite asset urls in markdown files. + */ +export function remarkRewriteImageUrls(args: { + urlRewriteFn?: (url: string) => string +}): Plugin { + const { urlRewriteFn = (url) => url } = args + + return function plugin() { + return function transform(tree) { + // @ts-expect-error Types Should be correct here + visit(tree, 'image', (node) => { + node.url = urlRewriteFn(node.url) + }) + } + } +} diff --git a/src/lib/sitemap/docs-content-fields.ts b/src/lib/sitemap/docs-content-fields.ts index a5a90946c1..25aff668f9 100644 --- a/src/lib/sitemap/docs-content-fields.ts +++ b/src/lib/sitemap/docs-content-fields.ts @@ -6,6 +6,10 @@ import { makeSitemapField } from './helpers' export async function allDocsFields() { + /** + * TODO: Update experimental content monorepo backed API to implement + * an `all-docs-paths` endpoint. + */ const getDocsPaths = await fetch( `${process.env.MKTG_CONTENT_API}/api/all-docs-paths` ) diff --git a/src/views/docs-view/loaders/content-api/index.ts b/src/views/docs-view/loaders/content-api/index.ts index d2c3b84af6..a5cd21b27c 100644 --- a/src/views/docs-view/loaders/content-api/index.ts +++ b/src/views/docs-view/loaders/content-api/index.ts @@ -35,6 +35,7 @@ export async function fetchNavData( const fullPath = `nav-data/${version}/${basePath}` const url = `${MKTG_CONTENT_API}/api/content/${product}/${fullPath}` + console.log(`Fetching from MKTG_CONTENT_API "${url}"...`) const response = await fetch(url) @@ -53,6 +54,7 @@ export async function fetchDocument( checkEnvVarsInDev() const url = `${MKTG_CONTENT_API}/api/content/${product}/${fullPath}` + console.log(`Fetching from MKTG_CONTENT_API "${url}"...`) const response = await fetch(url) if (response.status !== 200) { @@ -67,6 +69,7 @@ export async function fetchVersionMetadataList(product: string) { checkEnvVarsInDev() const url = `${MKTG_CONTENT_API}/api/content/${product}/version-metadata?partial=true` + console.log(`Fetching from MKTG_CONTENT_API "${url}"...`) const response = await fetch(url) if (response.status !== 200) { diff --git a/src/views/docs-view/loaders/remote-content.ts b/src/views/docs-view/loaders/remote-content.ts index 36b55e43e2..24e1b51a42 100644 --- a/src/views/docs-view/loaders/remote-content.ts +++ b/src/views/docs-view/loaders/remote-content.ts @@ -83,7 +83,7 @@ interface LoadStaticPropsReturn { const moizeOpts: Options = { isPromise: true, maxSize: Infinity } const cachedFetchNavData = moize(fetchNavData, moizeOpts) -const cachedFetchVersionMetadataList = moize( +export const cachedFetchVersionMetadataList = moize( fetchVersionMetadataList, moizeOpts ) diff --git a/src/views/docs-view/server.ts b/src/views/docs-view/server.ts index 84424c41ed..9c2a230657 100644 --- a/src/views/docs-view/server.ts +++ b/src/views/docs-view/server.ts @@ -10,7 +10,9 @@ import { Pluggable } from 'unified' import slugify from 'slugify' // HashiCorp Imports -import RemoteContentLoader from './loaders/remote-content' +import RemoteContentLoader, { + cachedFetchVersionMetadataList, +} from './loaders/remote-content' import { anchorLinks } from '@hashicorp/remark-plugins' // Global imports @@ -44,6 +46,9 @@ import { getCustomLayout } from './utils/get-custom-layout' import type { DocsViewPropOptions } from './utils/get-root-docs-path-generation-functions' import { DocsViewProps } from './types' import { isReleaseNotesPage } from 'lib/docs/is-release-notes-page' +import { stripVersionFromPathParams } from './loaders/utils' +import { rewriteImageUrlsForExperimentalContentApi } from './utils/rewrite-image-urls-for-experimental-content-api' +import { remarkRewriteImageUrls } from 'lib/remark-plugins/remark-rewrite-image-urls' /** * Returns static generation functions which can be exported from a page to fetch docs data @@ -172,10 +177,41 @@ export function getStaticGenerationFunctions< )}` const headings: AnchorLinksPluginHeading[] = [] // populated by anchorLinks plugin below + const [versionFromPath, paramsNoVersion] = + stripVersionFromPathParams(pathParts) + + let versionForAssetLoader = versionFromPath + if (versionForAssetLoader === 'latest') { + const versionMetadata = await cachedFetchVersionMetadataList( + productSlugForLoader + ) + versionForAssetLoader = versionMetadata.find((e) => e.isLatest)?.version + } + const loader = getLoader({ mainBranch, remarkPlugins: [ ...additionalRemarkPlugins, + /** + * TODO: should only add asset rewriting for new content monorepo API + * if the product has opted-in... so need a feature flag or something here... + * but for prototyping purposes, trying this out for all products. + * + * TODO: maybe this URL rewriting should be done during content + * migration? This would remove the need for changes to the dev-dot + * front-end. It seems somewhat reasonable to have the content API + * return an MDX document with image URLs that are already correct. + */ + remarkRewriteImageUrls({ + urlRewriteFn: (url) => + rewriteImageUrlsForExperimentalContentApi( + url, + paramsNoVersion.join('/'), + versionForAssetLoader, + productSlugForLoader, + basePathForLoader + ), + }), /** * Note on remark plugins for local vs remote loading: * includeMarkdown and paragraphCustomAlerts are already diff --git a/src/views/docs-view/utils/get-root-docs-path-generation-functions.ts b/src/views/docs-view/utils/get-root-docs-path-generation-functions.ts index dad0a9396c..eeadb97fae 100644 --- a/src/views/docs-view/utils/get-root-docs-path-generation-functions.ts +++ b/src/views/docs-view/utils/get-root-docs-path-generation-functions.ts @@ -46,11 +46,11 @@ export function getRootDocsPathGenerationFunctions( product: productData, productSlugForLoader: rootDocsPath.productSlugForLoader, basePathForLoader: rootDocsPath.basePathForLoader, - mainBranch: rootDocsPath.mainBranch, additionalRemarkPlugins: getAdditionalRemarkPlugins( productData, rootDocsPath ), + mainBranch: rootDocsPath.mainBranch, getScope: generateGetScope(productData, rootDocsPath), options, } @@ -90,11 +90,14 @@ function getAdditionalRemarkPlugins( productData: ProductData, rootDocsPath: RootDocsPath ): Pluggable[] { + // + let additionalRemarkPlugins = [] + // if (productData.slug == 'sentinel' && rootDocsPath.path == 'docs') { - return [remarkSentinel] - } else { - return [] + additionalRemarkPlugins.push(remarkSentinel) } + // + return additionalRemarkPlugins } /** diff --git a/src/views/docs-view/utils/get-valid-versions.ts b/src/views/docs-view/utils/get-valid-versions.ts index 829c93aa4a..25e416d9e2 100644 --- a/src/views/docs-view/utils/get-valid-versions.ts +++ b/src/views/docs-view/utils/get-valid-versions.ts @@ -49,6 +49,9 @@ export async function getValidVersions( const validVersionsUrl = new URL(VERSIONS_ENDPOINT, CONTENT_API_URL) validVersionsUrl.searchParams.set('product', productSlugForLoader) validVersionsUrl.searchParams.set('fullPath', fullPath) + console.log( + `Fetching from MKTG_CONTENT_API "${validVersionsUrl.toString()}"...` + ) // Fetch known versions of this document const response = await fetch(validVersionsUrl.toString()) const { versions: knownVersions } = await response.json() diff --git a/src/views/docs-view/utils/rewrite-image-urls-for-experimental-content-api.ts b/src/views/docs-view/utils/rewrite-image-urls-for-experimental-content-api.ts new file mode 100644 index 0000000000..485a3043f5 --- /dev/null +++ b/src/views/docs-view/utils/rewrite-image-urls-for-experimental-content-api.ts @@ -0,0 +1,68 @@ +const REPO_CONFIG_CONTENT_DIR: Record = { + boundary: 'content', + consul: 'content', + 'hcp-docs': 'content', + nomad: 'content', + packer: 'content', + 'ptfe-releases': 'docs', + sentinel: 'content', + terraform: 'docs', + 'terraform-cdk': 'docs', + 'terraform-docs-agents': 'docs', + 'terraform-docs-common': 'docs', + 'terraform-plugin-framework': 'docs', + 'terraform-plugin-log': 'docs', + 'terraform-plugin-mux': 'docs', + 'terraform-plugin-sdk': 'docs', + 'terraform-plugin-testing': 'docs', + vagrant: 'content', + vault: 'content', + waypoint: 'content', +} + +export function rewriteImageUrlsForExperimentalContentApi( + url, + currentPath, + _currentVersion, + productSlugForLoader, + docsBasePath +) { + /** + * Rewrite all URLs to use the content API for assets + * + * TODO: versioned assets are a work in progress. + * + * One option might be handle `currentVersion` here... but so far, in the + * unified docs repo `web-presence-experimental-docs`, we don't have a clear + * approach to versioned assets. Instead, all assets are currently in + * a single not-versioned directory. + * + * No matter what URL adjustment we do, we'll probably _not_ want to do it + * here, and instead we'll probably want to make `.mdx` image URL adjustments + * at *build time*, as part of our build time MDX transforms, over in + * `web-presence-experimental-docs`. + */ + let assetPrefix = `${process.env.MKTG_CONTENT_API}/assets/${productSlugForLoader}` + /** + * TODO: we have some messy shims to handle asset organization. + * Probably ideal to standardize this in the content monorepo instead, + * maybe as part of a migration script in the content monorepo? + */ + if (url.startsWith('/')) { + // Rewrite absolute URLs + // TODO: maybe this should be done during content migration? + return `${assetPrefix}${url}` + } else if (url.startsWith('./')) { + // Rewrite relative URLs + // TODO: maybe this should be done during content migration? + const currentPathRelative = currentPath.split('/').slice(0, -1).join('/') + const contentDir = REPO_CONFIG_CONTENT_DIR[productSlugForLoader] + const contentDirPrefix = `/img/${contentDir}/${docsBasePath}` // rootDocsPath.path + const finalPath = `${assetPrefix}${contentDirPrefix}/${currentPathRelative}/${url.substring( + 2 + )}` + return finalPath + } else { + return url + } +}