feat(cli): add Readme importer changelog, fix missing imported slugs …

…& callout components. (#5702) * Fix missing slugs and malformed callouts. * chore: update changelog * Improve versions changelog. * chore: update changelog --------- Co-authored-by: fern-support <[email protected]>
fern-api · Jan 22, 2025 · 1dc859f · 1dc859f
1 parent 89b06f2
commit 1dc859f
Show file tree

Hide file tree

Showing 9 changed files with 62 additions and 73 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -23,5 +23,6 @@ generators/openapi/* @dsinghvi
 # Specific rules for packages
 packages/cli/* @amckinney
 packages/cli/api-importers/* @eyw520
+packages/cli/docs-importers/* @eyw520
 packages/generators/cli/* @dsinghvi
 packages/generators/docker/* @dsinghvi
diff --git a/packages/cli/cli/versions.yml b/packages/cli/cli/versions.yml
@@ -1,3 +1,16 @@
+- changelogEntry:
+    - summary: |
+        The CLI now supports a --readme flag pointing to the URL of a Readme generated docs site and
+        migrates existing documentation to a fern-compatible repository.
+
+        To use this feature:
+        ```bash
+        fern init --readme https://url-to-readme-docs.com
+        ```
+      type: feat
+  irVersion: 55
+  version: 0.51.0
+
 - changelogEntry:
     - summary: |
         Improve performance of `fern docs dev` by only reloading the markdown content when 
@@ -7,6 +20,7 @@
   irVersion: 55
   version: 0.50.17
 
+
 - changelogEntry:
     - summary: |
         Improve performance of `fern docs dev` by debouncing across edits to multiple files, 

diff --git a/packages/cli/docs-importers/readme/src/ReadmeImporter.ts b/packages/cli/docs-importers/readme/src/ReadmeImporter.ts
@@ -196,20 +196,18 @@ export class ReadmeImporter extends DocsImporter<ReadmeImporter.Args> {
 
         const urlObj = new URL(url);
         const origin = urlObj.origin;
-
+        if (origin === "") {
+            return { success: false, message: `Invalid URL: ${url}` };
+        }
         const sidebar = retrieveRootNavElement(siteHast);
         if (!sidebar) {
             return { success: false, message: `${url.toString()}: Failed to find sidebar element` };
         }
         const navItems = parseSidebar(sidebar);
-        if (origin === "") {
-            return { success: false, message: `invalid URL provided to scrape site: ${url}` };
-        }
-
         const flatNavItems = navItems.flatMap((section) => section.pages);
         const listOfLinks = iterateOverNavItems(flatNavItems, origin);
         if (listOfLinks.length === 0) {
-            return { success: false, message: `no navigation links were able to be found: ${url}` };
+            return { success: false, message: `No navigation items found for URL: ${url}` };
         }
 
         const externalLinks = listOfLinks.filter((url: URL) => url.origin !== origin);
@@ -253,27 +251,33 @@ export class ReadmeImporter extends DocsImporter<ReadmeImporter.Args> {
                 return value;
             };
 
-            traverse(navItems).forEach(function (value) {
-                if (
-                    externalLinkReplaceMap.has(value) ||
-                    (Array.isArray(value) && value.some((item) => externalLinkReplaceMap.has(item)))
-                ) {
-                    this.update(replaceLinks(value, externalLinkReplaceMap));
-                } else if (
-                    rootPathReplaceMap.has(value) ||
-                    (Array.isArray(value) && value.some((item) => rootPathReplaceMap.has(item)))
-                ) {
-                    this.update(replaceLinks(value, rootPathReplaceMap));
-                }
-            });
+            for (const section of navItems) {
+                traverse(section.pages).forEach(function (value) {
+                    if (
+                        externalLinkReplaceMap.has(value) ||
+                        (Array.isArray(value) && value.some((item) => externalLinkReplaceMap.has(item)))
+                    ) {
+                        this.update(replaceLinks(value, externalLinkReplaceMap));
+                    } else if (
+                        rootPathReplaceMap.has(value) ||
+                        (Array.isArray(value) && value.some((item) => rootPathReplaceMap.has(item)))
+                    ) {
+                        this.update(replaceLinks(value, rootPathReplaceMap));
+                    }
+                });
+            }
 
-            traverse(navItems).forEach(function (value) {
-                if (typeof value === "string") {
-                    this.update(value.replace("/overview", ""));
-                } else if (Array.isArray(value)) {
-                    this.update(value.map((item) => (typeof item === "string" ? item.replace("/overview", "") : item)));
-                }
-            });
+            for (const section of navItems) {
+                traverse(section.pages).forEach(function (value) {
+                    if (typeof value === "string") {
+                        this.update(value.replace("/overview", ""));
+                    } else if (Array.isArray(value)) {
+                        this.update(
+                            value.map((item) => (typeof item === "string" ? item.replace("/overview", "") : item))
+                        );
+                    }
+                });
+            }
 
             const failedPaths = [...extResults, ...intResults, ...rootResults]
                 .filter((r) => !r.success)

diff --git a/packages/cli/docs-importers/readme/src/components/Callout.ts b/packages/cli/docs-importers/readme/src/components/Callout.ts
@@ -1,6 +1,7 @@
 import type { Element, ElementContent } from "hast";
 
 import { assertIsStringArray } from "../assert";
+import { convertHastChildrenToMdast } from "../customComponents/children";
 import type { HastNode, HastNodeIndex, HastNodeParent } from "../types/hastTypes";
 
 export function scrapeCallout(node: HastNode, _: HastNodeIndex, __: HastNodeParent): Element | undefined {
@@ -35,43 +36,13 @@ export function scrapeCallout(node: HastNode, _: HastNodeIndex, __: HastNodePare
             break;
     }
 
-    let icon: string | undefined;
-    for (const child of node.children) {
-        if (
-            child.type === "element" &&
-            child.tagName === "h2" &&
-            Array.isArray(child.properties?.className) &&
-            child.properties?.className.includes("callout-heading")
-        ) {
-            const iconSpan = child.children?.find(
-                (c) =>
-                    c.type === "element" &&
-                    c.tagName === "span" &&
-                    Array.isArray(c.properties?.className) &&
-                    c.properties?.className.includes("callout-icon")
-            );
-            if (iconSpan && iconSpan.type === "element" && iconSpan.children?.[0] && "value" in iconSpan.children[0]) {
-                icon = String(iconSpan.children[0].value);
-            }
-            break;
-        }
-    }
+    const textChildren = node.children.filter((child) => child.type === "element" && child.tagName === "p");
 
     const newNode: Element = {
         type: "element",
         tagName,
-        properties: {
-            ...(icon && { icon })
-        },
-        children: node.children.filter(
-            (child) =>
-                !(
-                    child.type === "element" &&
-                    child.tagName === "h2" &&
-                    Array.isArray(child.properties?.className) &&
-                    child.properties?.className.includes("callout-heading")
-                )
-        ) as Array<ElementContent>
+        properties: {},
+        children: convertHastChildrenToMdast(textChildren) as Array<ElementContent>
     };
 
     return newNode;

diff --git a/packages/cli/docs-importers/readme/src/components/Card.ts b/packages/cli/docs-importers/readme/src/components/Card.ts
@@ -2,6 +2,7 @@ import type { Element, ElementContent } from "hast";
 import { CONTINUE, EXIT, visit } from "unist-util-visit";
 
 import { assertIsDefined } from "../assert";
+import { convertHastChildrenToMdast } from "../customComponents/children";
 import { findTitle } from "../extract/title";
 import type { HastNode, HastNodeIndex, HastNodeParent } from "../types/hastTypes";
 
@@ -52,7 +53,7 @@ export function scrapeCard(node: HastNode, _: HastNodeIndex, parent: HastNodePar
             title,
             href
         },
-        children: node.children as Array<ElementContent>
+        children: convertHastChildrenToMdast(node.children as Array<Element>) as Array<ElementContent>
     };
 
     return newNode;

diff --git a/packages/cli/docs-importers/readme/src/parse/parsePage.ts b/packages/cli/docs-importers/readme/src/parse/parsePage.ts
@@ -5,6 +5,8 @@ import remarkMdx from "remark-mdx";
 import remarkStringify from "remark-stringify";
 import { unified } from "unified";
 
+import { TaskContext } from "@fern-api/task-context";
+
 import { unifiedRemoveBreaks } from "../cleaners/breaks";
 import { unifiedRemoveClassNames } from "../cleaners/className";
 import { remarkRemoveEmptyEmphases } from "../cleaners/emptyEmphasis";
@@ -41,6 +43,7 @@ import { htmlToHast } from "../utils/hast";
 import { normalizePath, removeTrailingSlash } from "../utils/strings";
 
 export async function parsePage(
+    context: TaskContext,
     html: string,
     url: string | URL,
     opts: {
@@ -115,14 +118,13 @@ export async function parsePage(
         }
 
         writePage(url, title, description, resultStr);
+        context.logger.debug(`Successfully parsed page ${urlStr}`);
         return {
             success: true,
             data: opts.rootPath ? [normalizePath(new URL(urlStr).pathname), opts.rootPath] : undefined
         };
     } catch (error) {
-        return {
-            success: false,
-            data: [urlStr, ""]
-        };
+        context.logger.debug(`Error parsing page ${urlStr}: ${error}`);
+        return { success: false, data: [urlStr, ""] };
     }
 }
diff --git a/packages/cli/docs-importers/readme/src/parse/parsePageGroup.ts b/packages/cli/docs-importers/readme/src/parse/parsePageGroup.ts
@@ -28,11 +28,11 @@ export async function parsePageGroup(
                 try {
                     if (opts.externalLinks) {
                         taskContext.logger.debug(`Scraping external link with URL: ${url}...`);
-                        return parsePage(`external-link-${index}`, url, { externalLink: true });
+                        return parsePage(taskContext, `external-link-${index}`, url, { externalLink: true });
                     } else {
                         const html = await fetchPageHtml({ url });
                         taskContext.logger.debug(`Scraping internal link with URL: ${url}...`);
-                        return parsePage(html, url, {
+                        return parsePage(taskContext, html, url, {
                             externalLink: false,
                             rootPath: opts.rootPaths ? opts.rootPaths[index] : undefined
                         });

diff --git a/packages/cli/docs-importers/readme/src/parse/parseSidebar.ts b/packages/cli/docs-importers/readme/src/parse/parseSidebar.ts
@@ -5,7 +5,6 @@ import { getFirstChild } from "../extract/firstChild";
 import { getText } from "../extract/text";
 import { findTitle } from "../extract/title";
 import { scrapedNavigationEntry, scrapedNavigationSection } from "../types/scrapedNavigation";
-import { removeTrailingSlash } from "../utils/strings";
 
 export function parseSidebar(rootNode: Element): Array<scrapedNavigationSection> {
     const result: Array<scrapedNavigationSection> = [];
@@ -154,10 +153,7 @@ export function parseListItem({
     }
 
     const childEntries = parseNavItems(childList);
-    const newLink =
-        linkHref !== "#" && childEntries.find((child) => typeof child === "string" && child.startsWith(linkHref))
-            ? removeTrailingSlash(linkHref) + "/overview"
-            : linkHref;
+    const newLink = linkHref;
 
     if (linkHref !== "#") {
         if (childEntries.includes(linkHref)) {

diff --git a/packages/cli/docs-importers/readme/src/utils/network.ts b/packages/cli/docs-importers/readme/src/utils/network.ts
@@ -92,7 +92,7 @@ export async function fetchPageHtml({ url, browser }: { url: string | URL; brows
         }
         throw new Error("An unknown error occured.");
     } catch (error) {
-        throw new Error("Error retrieving HTML for ${url.toString()}");
+        throw new Error(`Error retrieving HTML for ${url.toString()}`);
     }
 }
 
@@ -108,6 +108,6 @@ export async function fetchImage(url: string): Promise<NodeJS.TypedArray> {
 
         return imageData;
     } catch (error) {
-        throw new Error(`${url} - failed to retrieve image from source`);
+        throw new Error(`Failed to retrieve image from source url ${url}`);
     }
 }