Extract core pipeline stages into its own package

Add changeset

Move default pages back into the gatsby generator package

.changeset/tasty-bears-confess.md

@@ -0,0 +1,6 @@
+---
+'@brisk-docs/gatsby-generator': patch
+'@brisk-docs/pipeline-stages': patch
+---
+
+Extracted the core website pipeline generation stages into its own package.

.gitignore

@@ -24,6 +24,8 @@ scratchings.js
 /packages/gatsby-generator/static/favicon.ico
 /packages/gatsby-generator/typings
 
+/packages/pipeline-stages/data
+
 /packages/**/bundles
 /packages/gatsby-generator/public
 /packages/**/**/yarn.lock

package.json

@@ -37,7 +37,8 @@
     "packages": [
       "packages/file-viewer",
       "packages/react-changelogs",
-      "packages/gatsby-generator"
+      "packages/gatsby-generator",
+      "packages/pipeline-stages"
     ]
   },
   "dependencies": {

packages/file-viewer/package.json

@@ -10,7 +10,9 @@
   ],
   "author": "Peter Yu",
   "license": "MIT",
-  "files": ["dist"],
+  "files": [
+    "dist"
+  ],
   "peerDependencies": {
     "react": "16.10.2",
     "react-dom": "^16.8.4"

packages/gatsby-generator/package.json

@@ -35,7 +35,6 @@
   },
   "files": [
     "static",
-    "default-pages",
     "babel.config.js",
     "gatsby-node.js",
     "gatsby-config.js",
@@ -66,6 +65,7 @@
     "@babel/preset-react": "^7.0.0",
     "@babel/runtime": "^7.4.3",
     "@brisk-docs/file-viewer": "^0.2.5",
+    "@brisk-docs/pipeline-stages": "^0.1.0",
     "@brisk-docs/react-changelogs": "^0.1.11",
     "@emotion/core": "^10.0.9",
     "@emotion/styled": "^10.0.9",

packages/gatsby-generator/src/pipeline/buildPagesPipeline.ts

@@ -1,25 +1,34 @@
-import scanMetadata from './stages/scan-metadata';
-import generateWebsiteInfo from './stages/generate-website-info';
-import generatePages from './stages/generate-pages';
+import {
+  scanMetadataStage,
+  generateWebsiteInfoStage,
+  generatePagesStage,
+} from '@brisk-docs/pipeline-stages';
+
 import allPaths from './getAllPaths';
 
 const buildPipeline = async (configPath?: string) => {
-  const { rootPath, wrappersPath, pagesPath, pkgRoot, config } = await allPaths(
-    configPath,
-  );
+  const {
+    rootPath,
+    wrappersPath,
+    pagesPath,
+    pkgRoot,
+    config,
+    defaultPagesPath,
+  } = await allPaths(configPath);
 
-  return scanMetadata({
+  return scanMetadataStage({
     rootPath,
     packagePathPatterns: config.packagesPaths,
     customPackageFields: config.customPackageFields,
     docs: config.docs,
     showSubExamples: config.showSubExamples,
   })
-    .then(projectData => generateWebsiteInfo(projectData))
+    .then(projectData => generateWebsiteInfoStage(projectData))
     .then(websiteInfo =>
-      generatePages({
+      generatePagesStage({
         wrappersPath,
         pagesPath,
+        defaultPagesPath,
         packageRoot: pkgRoot,
         ...websiteInfo,
         ...config,

packages/gatsby-generator/src/pipeline/buildPipeline.ts

@@ -1,8 +1,10 @@
 import path from 'path';
 import fs from 'fs-extra';
-import scanMetadata from './stages/scan-metadata';
-import generateWebsiteInfo from './stages/generate-website-info';
-import generatePages from './stages/generate-pages';
+import {
+  scanMetadataStage,
+  generateWebsiteInfoStage,
+  generatePagesStage,
+} from '@brisk-docs/pipeline-stages';
 import runGatsby from './stages/run-gatsby';
 import allPaths from './getAllPaths';
 
@@ -11,22 +13,28 @@ const buildPipeline = async (
   configPath?: string,
   gatsbyOptions: string[] = [],
 ) => {
-  const { rootPath, wrappersPath, pagesPath, pkgRoot, config } = await allPaths(
-    configPath,
-  );
+  const {
+    rootPath,
+    wrappersPath,
+    pagesPath,
+    pkgRoot,
+    config,
+    defaultPagesPath,
+  } = await allPaths(configPath);
 
-  return scanMetadata({
+  return scanMetadataStage({
     rootPath,
     packagePathPatterns: config.packagesPaths,
     customPackageFields: config.customPackageFields,
     docs: config.docs,
     showSubExamples: config.showSubExamples,
   })
-    .then(projectData => generateWebsiteInfo(projectData))
+    .then(projectData => generateWebsiteInfoStage(projectData))
     .then(websiteInfo =>
-      generatePages({
+      generatePagesStage({
         wrappersPath,
         pagesPath,
+        defaultPagesPath,
         packageRoot: pkgRoot,
         ...websiteInfo,
         ...config,

packages/gatsby-generator/src/pipeline/devPipeline.ts

@@ -1,26 +1,34 @@
-import scanMetadata from './stages/scan-metadata';
-import generateWebsiteInfo from './stages/generate-website-info';
-import generatePages from './stages/generate-pages';
+import {
+  scanMetadataStage,
+  generateWebsiteInfoStage,
+  generatePagesStage,
+} from '@brisk-docs/pipeline-stages';
 import runGatsby from './stages/run-gatsby';
 import allPaths from './getAllPaths';
 
 const devPipeline = async (configPath?: string, gatsbyOptions?: string[]) => {
-  const { rootPath, wrappersPath, pagesPath, pkgRoot, config } = await allPaths(
-    configPath,
-  );
+  const {
+    rootPath,
+    wrappersPath,
+    pagesPath,
+    pkgRoot,
+    config,
+    defaultPagesPath,
+  } = await allPaths(configPath);
 
-  return scanMetadata({
+  return scanMetadataStage({
     rootPath,
     packagePathPatterns: config.packagesPaths,
     customPackageFields: config.customPackageFields,
     docs: config.docs,
     showSubExamples: config.showSubExamples,
   })
-    .then(projectData => generateWebsiteInfo(projectData))
+    .then(projectData => generateWebsiteInfoStage(projectData))
     .then(websiteInfo =>
-      generatePages({
+      generatePagesStage({
         wrappersPath,
         pagesPath,
+        defaultPagesPath,
         packageRoot: pkgRoot,
         ...websiteInfo,
         ...config,

packages/gatsby-generator/src/pipeline/getAllPaths.ts

@@ -13,13 +13,15 @@ const getPathData = async (configPath?: string) => {
     `./un-src/components/page-templates`,
   );
   const pagesPath = path.resolve(pkgRoot, `./pages`);
+  const defaultPagesPath = path.join(pkgRoot, 'default-pages');
 
   return {
     rootPath: cwd,
     wrappersPath,
     pagesPath,
     pkgRoot,
     config,
+    defaultPagesPath,
   };
 };
 export default getPathData;

packages/gatsby-generator/src/pipeline/stages/build-website/index.ts

@@ -1,4 +1,4 @@
-import createStage from '../make-pipeline-stage';
+import { makePipelineStage } from '@brisk-docs/pipeline-stages';
 // @ts-ignore
 import createNextServer from '../run-website/next-server';
 
@@ -10,7 +10,7 @@ interface StageInput {
   nextOptions?: string[];
 }
 
-export default createStage(
+export default makePipelineStage(
   'build-website',
   async (input: StageInput): Promise<void> => {
     return createNextServer(input);

packages/gatsby-generator/src/pipeline/stages/run-gatsby/index.ts

@@ -1,4 +1,4 @@
-import createStage from '../make-pipeline-stage';
+import { makePipelineStage } from '@brisk-docs/pipeline-stages';
 
 // @ts-ignore: Importing non-ts file with no definition
 import createGatsbyServer from './gatsby-server';
@@ -14,7 +14,7 @@ interface StageInput {
 // Boilerplate, uncomment when used
 // interface StageConfig {}
 
-export default createStage(
+export default makePipelineStage(
   'run-gatsby',
   async (input: StageInput): Promise<void> => createGatsbyServer(input),
 );

packages/gatsby-generator/src/pipeline/stages/run-website/index.ts

@@ -1,4 +1,4 @@
-import createStage from '../make-pipeline-stage';
+import { makePipelineStage } from '@brisk-docs/pipeline-stages';
 
 // @ts-ignore: Importing non-ts file with no definition
 import createNextServer from './next-server';
@@ -14,7 +14,7 @@ interface StageInput {
 // Boilerplate, uncomment when used
 // interface StageConfig {}
 
-export default createStage(
+export default makePipelineStage(
   'run-website',
   async (input: StageInput): Promise<void> => {
     createNextServer(input);

packages/pipeline-stages/README.md

@@ -0,0 +1,13 @@
+# Brisk Docs Core Pipeline
+
+This library contains the core modules that Brisk Docs uses to read docs from a monorepo and generate the code for
+pages in the docs website.
+
+Brisk's website generation functionality is broken down into a series of stages that can be run independently or in a
+complete pipeline. This package exposes the following stages:
+
+ - Scan metadata: Scans the source of a monorepo and extract information about packages and docs within
+ - Generate website info: Creates a plan for the structure of the docs website using provided metadata
+ - Generate pages: Writes files to disk for all the pages to be built into the running website
+
+For creating and running a docs website end to end please use `@brisk-docs/gatsby-generator`.

packages/gatsby-generator/docs/build-pipeline/README.md → packages/pipeline-stages/docs/README.md

@@ -7,5 +7,3 @@ Brisk docs uses a pipeline of build stages to take the docs in a codebase and tu
 - [Scan metadata stage](./1-scan-metadata-stage.md)
 - [Generate website info stage](./2-generate-website-info-stage.md)
 - [Generate pages stage](./3-generate-pages-stage.md)
-- [Build website stage](./4-build-website-stage.md)
-- [Start website stage](./5-start-website-stage.md)

packages/pipeline-stages/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "@brisk-docs/pipeline-stages",
+  "version": "0.1.0",
+  "main": "dist/pipeline-stages.cjs.js",
+  "files": [
+    "default-pages",
+    "dist"
+  ],
+  "dependencies": {
+    "@babel/runtime": "^7.4.3",
+    "filenamify": "^4.0.0",
+    "fs-extra": "^7.0.1",
+    "glob": "^7.1.3",
+    "js-yaml": "^3.13.1",
+    "lodash.flatten": "^4.4.0",
+    "outdent": "^0.7.0",
+    "pkg-dir": "^4.2.0",
+    "remark-frontmatter": "^1.3.2",
+    "remark-parse": "^6.0.3",
+    "title-case": "^2.1.1",
+    "unified": "^7.1.0"
+  },
+  "peerDependencies": {},
+  "devDependencies": {
+    "jest-fixtures": "^0.6.0",
+    "mock-fs": "^4.10.1"
+  }
+}

packages/gatsby-generator/src/pipeline/stages/common/configuration-options.ts → packages/pipeline-stages/src/common/configuration-options.ts

@@ -1,4 +1,4 @@
-import { TemplateSpecifier } from '../../../../types';
+import { TemplateSpecifier } from '../types';
 
 export interface ProjectDocsConfig {
   // absolute path to the docs in the filesystem

packages/gatsby-generator/src/pipeline/stages/generate-pages/index.ts → packages/pipeline-stages/src/generate-pages/index.ts

@@ -26,6 +26,8 @@ export type StageInput = {
   pagesPath: string;
   // The absolute path to the root of the package.
   packageRoot: string;
+  // The absolute path to the directory containing default pages
+  defaultPagesPath: string;
 } & WebsiteInfoSpec &
   BriskConfiguration;
 
@@ -47,9 +49,9 @@ const generateGenericPage = (
 export default createStage(
   'generate-pages',
   async (input: StageInput): Promise<StageOutput> => {
-    const { pagesPath, wrappersPath } = input;
+    const { pagesPath, wrappersPath, defaultPagesPath } = input;
     await cleanPages(pagesPath);
-    await addBasePages(input.packageRoot, pagesPath);
+    await addBasePages(defaultPagesPath, pagesPath);
 
     const generatorConfig = { pagesPath, wrappersPath };
 

packages/gatsby-generator/src/pipeline/stages/generate-pages/integration.test.ts → packages/pipeline-stages/src/generate-pages/integration.test.ts

@@ -1,14 +1,16 @@
-import { createTempDir } from 'jest-fixtures';
+import { createTempDir, copyFixtureIntoTempDir } from 'jest-fixtures';
 import path from 'path';
 import fs from 'fs-extra';
 import generatePagesStage, { StageInput } from './index';
 
 describe('Generate pages build stage integration', () => {
   let pagesPath: string;
   let wrappersPath: string;
+  let defaultPagesPath: string;
 
   beforeEach(async () => {
     const cwd = await createTempDir();
+    defaultPagesPath = await copyFixtureIntoTempDir(__dirname, 'default-pages');
     pagesPath = path.join(cwd, 'pages');
     wrappersPath = path.join(cwd, 'wrappers');
   });
@@ -18,6 +20,7 @@ describe('Generate pages build stage integration', () => {
     const input: StageInput = {
       pagesPath,
       wrappersPath,
+      defaultPagesPath,
       pages: {
         packageDocPages: [
           {
@@ -79,7 +82,7 @@ describe('Generate pages build stage integration', () => {
           },
         ],
       },
-      packageRoot: path.resolve(__dirname, '..', '..', '..', '..'),
+      packageRoot: path.resolve(__dirname, '..', '..'),
       sitemap: { packages: [], docs: { docs: [] } },
       readmePageData: [],
       packagesMeta: [],
@@ -123,5 +126,8 @@ describe('Generate pages build stage integration', () => {
       path.join(pagesPath, 'package-home-pages'),
     );
     expect(packageHomePages).toEqual(['home1.js']);
+
+    const rootPages = await fs.readdir(pagesPath);
+    expect(rootPages).toContain('default-page.js');
   });
 });