diff --git a/README.md b/README.md index f6a9c1c..73b7afb 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,6 @@ Options: -o, --output Specify the relative or absolute output directory -v, --version Specify the target version of Node.js, semver compliant (default: "v22.6.0") -c, --changelog Specify the path (file: or https://) to the CHANGELOG.md file (default: "https://raw.githubusercontent.com/nodejs/node/HEAD/CHANGELOG.md") - -t, --target [mode...] Set the processing target modes (choices: "json-simple", "legacy-html", "legacy-html-all", "man-page", "legacy-json", "legacy-json-all", "addon-verify") + -t, --target [mode...] Set the processing target modes (choices: "json-simple", "legacy-html", "legacy-html-all", "man-page", "legacy-json", "legacy-json-all", "addon-verify", "api-links") -h, --help display help for command ``` diff --git a/bin/cli.mjs b/bin/cli.mjs index 5bb332b..ba6f8ea 100755 --- a/bin/cli.mjs +++ b/bin/cli.mjs @@ -9,8 +9,8 @@ import { coerce } from 'semver'; import { DOC_NODE_CHANGELOG_URL, DOC_NODE_VERSION } from '../src/constants.mjs'; import createGenerator from '../src/generators.mjs'; import generators from '../src/generators/index.mjs'; -import createLoader from '../src/loader.mjs'; -import createParser from '../src/parser.mjs'; +import createMarkdownLoader from '../src/loaders/markdown.mjs'; +import createMarkdownParser from '../src/parsers/markdown.mjs'; import createNodeReleases from '../src/releases.mjs'; const availableGenerators = Object.keys(generators); @@ -68,8 +68,8 @@ program */ const { input, output, target = [], version, changelog } = program.opts(); -const { loadFiles } = createLoader(); -const { parseApiDocs } = createParser(); +const { loadFiles } = createMarkdownLoader(); +const { parseApiDocs } = createMarkdownParser(); const apiDocFiles = loadFiles(input); @@ -83,6 +83,8 @@ const { getAllMajors } = createNodeReleases(changelog); await runGenerators({ // A list of target modes for the API docs parser generators: target, + // Resolved `input` to be used + input: input, // Resolved `output` path to be used output: resolve(output), // Resolved SemVer of current Node.js version diff --git a/package-lock.json b/package-lock.json index 49d430f..777215e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,8 +6,10 @@ "": { "name": "@node-core/api-docs-tooling", "dependencies": { + "acorn": "^8.14.0", "commander": "^13.1.0", "dedent": "^1.5.3", + "estree-util-visit": "^2.0.0", "github-slugger": "^2.0.0", "glob": "^11.0.1", "hast-util-to-string": "^3.0.1", @@ -471,8 +473,16 @@ "node_modules/@types/estree": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz", - "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==", - "dev": true + "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==" + }, + "node_modules/@types/estree-jsx": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@types/estree-jsx/-/estree-jsx-1.0.5.tgz", + "integrity": "sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg==", + "license": "MIT", + "dependencies": { + "@types/estree": "*" + } }, "node_modules/@types/hast": { "version": "3.0.4", @@ -1259,6 +1269,20 @@ "node": ">=4.0" } }, + "node_modules/estree-util-visit": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/estree-util-visit/-/estree-util-visit-2.0.0.tgz", + "integrity": "sha512-m5KgiH85xAhhW8Wta0vShLcUvOsh3LLPI2YVwcbio1l7E09NTLL1EyMZFM1OyWowoH0skScNbhOPl4kcBgzTww==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/esutils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", diff --git a/package.json b/package.json index 049087a..e335033 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,9 @@ "prettier": "3.4.2" }, "dependencies": { + "acorn": "^8.14.0", "commander": "^13.1.0", + "estree-util-visit": "^2.0.0", "dedent": "^1.5.3", "github-slugger": "^2.0.0", "glob": "^11.0.1", diff --git a/src/generators.mjs b/src/generators.mjs index 1e1c345..931f583 100644 --- a/src/generators.mjs +++ b/src/generators.mjs @@ -1,10 +1,20 @@ 'use strict'; -import availableGenerators from './generators/index.mjs'; +import publicGenerators from './generators/index.mjs'; +import astJs from './generators/ast-js/index.mjs'; + +const availableGenerators = { + ...publicGenerators, + // This one is a little special since we don't want it to run unless we need + // it and we also don't want it to be publicly accessible through the CLI. + 'ast-js': astJs, +}; /** * @typedef {{ ast: import('./generators/types.d.ts').GeneratorMetadata}} AstGenerator The AST "generator" is a facade for the AST tree and it isn't really a generator * @typedef {import('./generators/types.d.ts').AvailableGenerators & AstGenerator} AllGenerators A complete set of the available generators, including the AST one + * @param markdownInput + * @param jsInput * * This method creates a system that allows you to register generators * and then execute them in a specific order, keeping track of the @@ -18,9 +28,10 @@ import availableGenerators from './generators/index.mjs'; * Generators can also write to files. These would usually be considered * the final generators in the chain. * - * @param {ApiDocMetadataEntry} input The parsed API doc metadata entries + * @param {ApiDocMetadataEntry} markdownInput The parsed API doc metadata entries + * @param {Array} parsedJsFiles */ -const createGenerator = input => { +const createGenerator = markdownInput => { /** * We store all the registered generators to be processed * within a Record, so we can access their results at any time whenever needed @@ -28,7 +39,9 @@ const createGenerator = input => { * * @type {{ [K in keyof AllGenerators]: ReturnType }} */ - const cachedGenerators = { ast: Promise.resolve(input) }; + const cachedGenerators = { + ast: Promise.resolve(markdownInput), + }; /** * Runs the Generator engine with the provided top-level input and the given generator options diff --git a/src/generators/api-links/constants.mjs b/src/generators/api-links/constants.mjs new file mode 100644 index 0000000..bff6072 --- /dev/null +++ b/src/generators/api-links/constants.mjs @@ -0,0 +1,4 @@ +'use strict'; + +// Checks if a string is a valid name for a constructor in JavaScript +export const CONSTRUCTOR_EXPRESSION = /^[A-Z]/; diff --git a/src/generators/api-links/index.mjs b/src/generators/api-links/index.mjs new file mode 100644 index 0000000..b0b0064 --- /dev/null +++ b/src/generators/api-links/index.mjs @@ -0,0 +1,105 @@ +'use strict'; + +import { basename, dirname, join } from 'node:path'; +import { writeFile } from 'node:fs/promises'; +import { + getBaseGitHubUrl, + getCurrentGitHash, +} from './utils/getBaseGitHubUrl.mjs'; +import { extractExports } from './utils/extractExports.mjs'; +import { findDefinitions } from './utils/findDefinitions.mjs'; +import { checkIndirectReferences } from './utils/checkIndirectReferences.mjs'; + +/** + * This generator is responsible for mapping publicly accessible functions in + * Node.js to their source locations in the Node.js repository. + * + * This is a top-level generator. It takes in the raw AST tree of the JavaScript + * source files. It outputs a `apilinks.json` file into the specified output + * directory. + * + * @typedef {Array} Input + * + * @type {import('../types.d.ts').GeneratorMetadata>} + */ +export default { + name: 'api-links', + + version: '1.0.0', + + description: + 'Creates a mapping of publicly accessible functions to their source locations in the Node.js repository.', + + // Unlike the rest of the generators, this utilizes Javascript sources being + // passed into the input field rather than Markdown. + dependsOn: 'ast-js', + + /** + * Generates the `apilinks.json` file. + * + * @param {Input} input + * @param {Partial} options + */ + async generate(input, { output }) { + /** + * @type Record + */ + const definitions = {}; + + /** + * @type {string} + */ + let baseGithubLink; + + if (input.length > 0) { + const repositoryDirectory = dirname(input[0].path); + + const repository = getBaseGitHubUrl(repositoryDirectory); + + const tag = getCurrentGitHash(repositoryDirectory); + + baseGithubLink = `${repository}/blob/${tag}`; + } + + input.forEach(program => { + /** + * Mapping of definitions to their line number + * @type {Record} + * @example { 'someclass.foo': 10 } + */ + const nameToLineNumberMap = {}; + + // `http.js` -> `http` + const programBasename = basename(program.path, '.js'); + + const exports = extractExports( + program, + programBasename, + nameToLineNumberMap + ); + + findDefinitions(program, programBasename, nameToLineNumberMap, exports); + + checkIndirectReferences(program, exports, nameToLineNumberMap); + + const githubLink = + `${baseGithubLink}/lib/${programBasename}.js`.replaceAll('\\', '/'); + + // Add the exports we found in this program to our output + Object.keys(nameToLineNumberMap).forEach(key => { + const lineNumber = nameToLineNumberMap[key]; + + definitions[key] = `${githubLink}#L${lineNumber}`; + }); + }); + + if (output) { + await writeFile( + join(output, 'apilinks.json'), + JSON.stringify(definitions) + ); + } + + return definitions; + }, +}; diff --git a/src/generators/api-links/types.d.ts b/src/generators/api-links/types.d.ts new file mode 100644 index 0000000..1fc8ea2 --- /dev/null +++ b/src/generators/api-links/types.d.ts @@ -0,0 +1,5 @@ +export interface ProgramExports { + ctors: Array; + identifiers: Array; + indirects: Record; +} diff --git a/src/generators/api-links/utils/checkIndirectReferences.mjs b/src/generators/api-links/utils/checkIndirectReferences.mjs new file mode 100644 index 0000000..982bb11 --- /dev/null +++ b/src/generators/api-links/utils/checkIndirectReferences.mjs @@ -0,0 +1,24 @@ +import { visit } from 'estree-util-visit'; + +/** + * @param {import('acorn').Program} program + * @param {import('../types.d.ts').ProgramExports} exports + * @param {Record} nameToLineNumberMap + */ +export function checkIndirectReferences(program, exports, nameToLineNumberMap) { + if (Object.keys(exports.indirects).length === 0) { + return; + } + + visit(program, node => { + if (!node.loc || node.type !== 'FunctionDeclaration') { + return; + } + + const name = node.id.name; + + if (name in exports.indirects) { + nameToLineNumberMap[exports.indirects[name]] = node.loc.start.line; + } + }); +} diff --git a/src/generators/api-links/utils/extractExports.mjs b/src/generators/api-links/utils/extractExports.mjs new file mode 100644 index 0000000..228eb5b --- /dev/null +++ b/src/generators/api-links/utils/extractExports.mjs @@ -0,0 +1,265 @@ +'use strict'; + +import { visit } from 'estree-util-visit'; +import { CONSTRUCTOR_EXPRESSION } from '../constants.mjs'; + +/** + * @see https://github.com/estree/estree/blob/master/es5.md#assignmentexpression + * + * @param {import('acorn').ExpressionStatement} node + * @param {string} basename + * @param {Record} nameToLineNumberMap + * @returns {import('../types').ProgramExports | undefined} + */ +function handleExpression(node, basename, nameToLineNumberMap) { + const { expression } = node; + + if (expression.type !== 'AssignmentExpression') { + return; + } + + // `a=b`, lhs=`a` and rhs=`b` + let { left: lhs, right: rhs, loc } = expression; + + if (lhs.type !== 'MemberExpression') { + return undefined; + } + + if (lhs.object.type === 'MemberExpression') { + lhs = lhs.object; + } + + /** + * @type {import('../types').ProgramExports} + */ + const exports = { + ctors: [], + identifiers: [], + indirects: {}, + }; + + if (lhs.object.name === 'exports') { + // This is an assignment to a property in `module.exports` or `exports` + // (i.e. `module.exports.asd = ...`) + + switch (rhs.type) { + /** @see https://github.com/estree/estree/blob/master/es5.md#functionexpression */ + case 'FunctionExpression': { + // module.exports.something = () => {} + nameToLineNumberMap[`${basename}.${lhs.property.name}`] = + loc.start.line; + + break; + } + /** @see https://github.com/estree/estree/blob/master/es5.md#identifier */ + case 'Identifier': { + // Save this for later in case it's referenced + // module.exports.asd = something + if (rhs.name === lhs.property.name) { + exports.indirects[lhs.property.name] = + `${basename}.${lhs.property.name}`; + } + + break; + } + default: { + if (lhs.property.name !== undefined) { + // Something else, let's save it for when we're searching for + // declarations + exports.identifiers.push(lhs.property.name); + } + + break; + } + } + } else if (lhs.object.name === 'module' && lhs.property.name === 'exports') { + // This is an assignment to `module.exports` as a whole + // (i.e. `module.exports = {}`) + + // We need to move right until we find the value of the assignment. + // (if `a=b`, we want `b`) + while (rhs.type === 'AssignmentExpression') { + rhs = rhs.right; + } + + switch (rhs.type) { + /** @see https://github.com/estree/estree/blob/master/es5.md#newexpression */ + case 'NewExpression': { + // module.exports = new Asd() + exports.ctors.push(rhs.callee.name); + break; + } + /** @see https://github.com/estree/estree/blob/master/es5.md#objectexpression */ + case 'ObjectExpression': { + // module.exports = {} + // we need to go through all of the properties and register them + rhs.properties.forEach(({ value }) => { + switch (value.type) { + case 'Identifier': { + exports.identifiers.push(value.name); + + if (CONSTRUCTOR_EXPRESSION.test(value.name[0])) { + exports.ctors.push(value.name); + } + + break; + } + case 'CallExpression': { + if (value.callee.name !== 'deprecate') { + break; + } + + // Handle exports wrapped in the `deprecate` function + // Ex/ https://github.com/nodejs/node/blob/e96072ad57348ce423a8dd7639dcc3d1c34e847d/lib/buffer.js#L1334 + + exports.identifiers.push(value.arguments[0].name); + + break; + } + default: { + // Not relevant + } + } + }); + + break; + } + /** @see https://github.com/estree/estree/blob/master/es5.md#identifier */ + case 'Identifier': { + // Something else, let's save it for when we're searching for + // declarations + + if (rhs.name !== undefined) { + exports.identifiers.push(rhs.name); + } + + break; + } + default: { + // Not relevant + break; + } + } + } + + return exports; +} + +/** + * @see https://github.com/estree/estree/blob/master/es5.md#variabledeclaration + * + * @param {import('acorn').VariableDeclaration} node + * @param {string} basename + * @param {Record} nameToLineNumberMap + * @returns {import('../types').ProgramExports | undefined} + */ +function handleVariableDeclaration(node, basename, nameToLineNumberMap) { + /** + * @type {import('../types').ProgramExports} + */ + const exports = { + ctors: [], + identifiers: [], + indirects: {}, + }; + + node.declarations.forEach(({ init: lhs, id }) => { + while (lhs && lhs.type === 'AssignmentExpression') { + // Move left until we get to what we're assigning to + // (if `a=b`, we want `a`) + lhs = lhs.left; + } + + if (!lhs || lhs.type !== 'MemberExpression') { + // Doesn't exist or we're not writing to an object + // (aka it's just a regular variable like `const a = 123`) + return; + } + + switch (lhs.object.name) { + case 'exports': { + nameToLineNumberMap[`${basename}.${lhs.property.name}`] = + node.start.line; + + break; + } + case 'module': { + if (lhs.property.name !== 'exports') { + break; + } + + exports.ctors.push(id.name); + nameToLineNumberMap[id.name] = node.loc.start.line; + + break; + } + default: { + // Not relevant to us + break; + } + } + }); + + return exports; +} + +/** + * We need to find what a source file exports so we know what to include in + * the final result. We can do this by going through every statement in the + * program looking for assignments to `module.exports`. + * + * Noteworthy that exports can happen throughout the program so we need to + * go through the entire thing. + * + * @param {import('acorn').Program} program + * @param {string} basename + * @param {Record} nameToLineNumberMap + * @returns {import('../types').ProgramExports} + */ +export function extractExports(program, basename, nameToLineNumberMap) { + /** + * @type {import('../types').ProgramExports} + */ + const exports = { + ctors: [], + identifiers: [], + indirects: {}, + }; + + const TYPE_TO_HANDLER_MAP = { + /** + * @param {import('acorn').Node} node + */ + ExpressionStatement: node => + handleExpression(node, basename, nameToLineNumberMap), + + /** + * @param {import('acorn').Node} node + */ + VariableDeclaration: node => + handleVariableDeclaration(node, basename, nameToLineNumberMap), + }; + + visit(program, node => { + if (!node.loc) { + return; + } + + if (node.type in TYPE_TO_HANDLER_MAP) { + const handler = TYPE_TO_HANDLER_MAP[node.type]; + + const output = handler(node); + + if (output) { + exports.ctors.push(...output.ctors); + exports.identifiers.push(...output.identifiers); + + Object.keys(output.indirects).forEach(key => { + exports.indirects[key] = output.indirects[key]; + }); + } + } + }); + + return exports; +} diff --git a/src/generators/api-links/utils/findDefinitions.mjs b/src/generators/api-links/utils/findDefinitions.mjs new file mode 100644 index 0000000..a161e3e --- /dev/null +++ b/src/generators/api-links/utils/findDefinitions.mjs @@ -0,0 +1,184 @@ +'use strict'; + +import { visit } from 'estree-util-visit'; + +/** + * @see https://github.com/estree/estree/blob/master/es5.md#expressionstatement + * + * @param {import('acorn').ExpressionStatement} node + * @param {Record} nameToLineNumberMap + * @param {import('../types').ProgramExports} exports + */ +function handleAssignmentExpression(node, nameToLineNumberMap, exports) { + const { expression } = node; + + if (expression.type !== 'AssignmentExpression') { + return; + } + + const { left: lhs, right: rhs } = expression; + + if (lhs.type !== 'MemberExpression') { + // Not an assignment to a member, not relevant to us + return; + } + + /** + * The property that's being written to + */ + let object; + + /** + * The lowercase name of the object that's being written to + */ + let objectName; + + switch (lhs.object.type) { + /** @see https://github.com/estree/estree/blob/master/es5.md#memberexpression */ + case 'MemberExpression': { + if (lhs.object.property.name !== 'prototype') { + return; + } + + // Something like `ClassName.prototype.asd = 123` + object = lhs.object.object; + + objectName = object.name ? object.name : object.object.name; + objectName = objectName.toLowerCase(); + + // Special case for buffer since some of the docs refer to it as `buf` + // https://github.com/nodejs/node/pull/22405#issuecomment-414452461 + if (objectName === 'buffer') { + objectName = 'buf'; + } + + break; + } + /** @see https://github.com/estree/estree/blob/master/es5.md#identifier */ + case 'Identifier': { + object = lhs.object; + objectName = object.name; + + break; + } + default: { + // Not relevant to us + return; + } + } + + if (!exports.ctors.includes(object.name)) { + // The object being written to isn't exported, not relevant to us + return; + } + + /** + * Name/key for this exported object that we're putting in the output + * @example `clientrequest._finish` + */ + const name = `${objectName}${lhs.computed ? `[${lhs.property.name}]` : `.${lhs.property.name}`}`; + + nameToLineNumberMap[name] = node.loc.start.line; + + if (lhs.property.name === rhs.name) { + exports.indirects[rhs.name] = name; + } +} + +/** + * @param {import('acorn').FunctionDeclaration} node + * @param {string} basename + * @param {Record} nameToLineNumberMap + * @param {import('../types').ProgramExports} exports + */ +function handleFunctionDeclaration( + node, + basename, + nameToLineNumberMap, + exports +) { + if (!exports.identifiers.includes(node.id.name)) { + // Function isn't exported, not relevant to us + return; + } + + if (basename.startsWith('_')) { + // Internal function, don't include it in the docs + return; + } + + nameToLineNumberMap[`${basename}.${node.id.name}`] = node.loc.start.line; +} + +/** + * @param {import('acorn').ClassDeclaration} node + * @param {Record} nameToLineNumberMap + * @param {import('../types').ProgramExports} exports + */ +function handleClassDeclaration(node, nameToLineNumberMap, exports) { + if (!exports.ctors.includes(node.id.name)) { + // Class isn't exported, not relevant to us + return; + } + + // WASI -> wASI, Agent -> agent + const name = node.id.name[0].toLowerCase() + node.id.name.substring(1); + + nameToLineNumberMap[node.id.name] = node.loc.start.line; + + node.body.body.forEach(({ key, type, kind, loc }) => { + if (!loc || type !== 'MethodDefinition') { + return; + } + + const outputKey = + kind === 'constructor' ? `new ${node.id.name}` : `${name}.${key.name}`; + + nameToLineNumberMap[outputKey] = loc.start.line; + }); +} + +/** + * @param {import('acorn').Program} program + * @param {string} basename + * @param {Record} nameToLineNumberMap + * @param {import('../types').ProgramExports} exports + */ +export function findDefinitions( + program, + basename, + nameToLineNumberMap, + exports +) { + const TYPE_TO_HANDLER_MAP = { + /** + * @param {import('acorn').Node} node + */ + ExpressionStatement: node => + handleAssignmentExpression(node, nameToLineNumberMap, exports), + + /** + * @param {import('acorn').Node} node + */ + FunctionDeclaration: node => + handleFunctionDeclaration(node, basename, nameToLineNumberMap, exports), + + /** + * @param {import('acorn').Node} node + */ + ClassDeclaration: node => + handleClassDeclaration(node, nameToLineNumberMap, exports), + }; + + visit(program, node => { + if (!node.loc) { + return; + } + + if (node.type in TYPE_TO_HANDLER_MAP) { + const handler = TYPE_TO_HANDLER_MAP[node.type]; + + handler(node); + } + }); +} diff --git a/src/generators/api-links/utils/getBaseGitHubUrl.mjs b/src/generators/api-links/utils/getBaseGitHubUrl.mjs new file mode 100644 index 0000000..8a4088a --- /dev/null +++ b/src/generators/api-links/utils/getBaseGitHubUrl.mjs @@ -0,0 +1,35 @@ +'use strict'; + +import { execSync } from 'node:child_process'; + +/** + * @param {string} cwd + */ +export function getBaseGitHubUrl(cwd) { + let url = execSync('git remote get-url origin', { cwd }).toString().trim(); + + if (url.startsWith('git@')) { + // It's an ssh url, we need to transform it to be https + // Ex/ git@github.com:nodejs/node.git -> https://github.com/nodejs/node.git + let [, repository] = url.split(':'); + + url = `https://github.com/${repository}`; + } + + // https://github.com/nodejs/node.git -> https://github.com/nodejs/node + if (url.endsWith('.git')) { + url = url.substring(0, url.length - 4); + } + + return url; +} + +/** + * + * @param cwd + */ +export function getCurrentGitHash(cwd) { + const hash = execSync('git rev-parse HEAD', { cwd }).toString().trim(); + + return hash; +} diff --git a/src/generators/ast-js/index.mjs b/src/generators/ast-js/index.mjs new file mode 100644 index 0000000..440365a --- /dev/null +++ b/src/generators/ast-js/index.mjs @@ -0,0 +1,43 @@ +import createJsLoader from '../../loaders/javascript.mjs'; +import createJsParser from '../../parsers/javascript.mjs'; + +/** + * This generator parses Javascript sources passed into the generator's input + * field. This is separate from the Markdown parsing step since it's not as + * commonly used and can take up a significant amount of memory. + * + * Putting this with the rest of the generators allows it to be lazily loaded + * so we're only parsing the Javascript sources when we need to. + * + * @typedef {unknown} Input + * + * @type {import('../types.d.ts').GeneratorMetadata>} + */ +export default { + name: 'ast-js', + + version: '1.0.0', + + description: 'Parses Javascript source files passed into the input.', + + dependsOn: 'ast', + + /** + * @param {Input} _ + * @param {Partial} options + */ + async generate(_, options) { + const { loadFiles } = createJsLoader(); + + // Load all of the Javascript sources into memory + const sourceFiles = loadFiles(options.input ?? []); + + const { parseJsSources } = createJsParser(); + + // Parse the Javascript sources into ASTs + const parsedJsFiles = await parseJsSources(sourceFiles); + + // Return the ASTs so they can be used in another generator + return parsedJsFiles; + }, +}; diff --git a/src/generators/index.mjs b/src/generators/index.mjs index 45a0f54..512271c 100644 --- a/src/generators/index.mjs +++ b/src/generators/index.mjs @@ -7,6 +7,7 @@ import manPage from './man-page/index.mjs'; import legacyJson from './legacy-json/index.mjs'; import legacyJsonAll from './legacy-json-all/index.mjs'; import addonVerify from './addon-verify/index.mjs'; +import apiLinks from './api-links/index.mjs'; export default { 'json-simple': jsonSimple, @@ -16,4 +17,5 @@ export default { 'legacy-json': legacyJson, 'legacy-json-all': legacyJsonAll, 'addon-verify': addonVerify, + 'api-links': apiLinks, }; diff --git a/src/generators/legacy-html/assets/style.css b/src/generators/legacy-html/assets/style.css index 5f7fc50..9086b6b 100644 --- a/src/generators/legacy-html/assets/style.css +++ b/src/generators/legacy-html/assets/style.css @@ -137,8 +137,8 @@ code, .pre, span.type, a.type { - font-family: SFMono-Regular, Menlo, Consolas, 'Liberation Mono', 'Courier New', - monospace; + font-family: SFMono-Regular, Menlo, Consolas, 'Liberation Mono', + 'Courier New', monospace; font-size: 0.9em; } diff --git a/src/generators/types.d.ts b/src/generators/types.d.ts index 348ceed..110b5dc 100644 --- a/src/generators/types.d.ts +++ b/src/generators/types.d.ts @@ -9,6 +9,10 @@ declare global { // This is the runtime config passed to the API doc generators export interface GeneratorOptions { + // The path to the input source files. This parameter accepts globs and can + // be a glob when passed to a generator. + input: string | string[]; + // The path used to output generated files, this is to be considered // the base path that any generator will use for generating files // This parameter accepts globs but when passed to generators will contain @@ -55,8 +59,12 @@ declare global { * * The 'ast' generator is the top-level parser, and if 'ast' is passed to `dependsOn`, then the generator * will be marked as a top-level generator. + * + * The `ast-js` generator is the top-level parser for JavaScript files. It + * passes the ASTs for any JavaScript files given in the input. Like `ast`, + * any generator depending on it is marked as a top-level generator. */ - dependsOn: keyof AvailableGenerators | 'ast'; + dependsOn: keyof AvailableGenerators | 'ast' | 'ast-js'; /** * Generators are abstract and the different generators have different sort of inputs and outputs. diff --git a/src/index.mjs b/src/index.mjs index af51f3d..da9a5fa 100644 --- a/src/index.mjs +++ b/src/index.mjs @@ -1,8 +1,10 @@ export * as constants from './constants.mjs'; export { default as generators } from './generators/index.mjs'; export { default as createGenerator } from './generators.mjs'; -export { default as createLoader } from './loader.mjs'; +export * from './loaders/markdown.mjs'; +export * from './loaders/javascript.mjs'; export { default as createMetadata } from './metadata.mjs'; -export { default as createParser } from './parser.mjs'; +export * from './parsers/markdown.mjs'; +export * from './parsers/javascript.mjs'; export { default as createQueries } from './queries.mjs'; export { default as createNodeReleases } from './releases.mjs'; diff --git a/src/loaders/javascript.mjs b/src/loaders/javascript.mjs new file mode 100644 index 0000000..78fd2c4 --- /dev/null +++ b/src/loaders/javascript.mjs @@ -0,0 +1,33 @@ +'use strict'; + +import { readFile } from 'node:fs/promises'; +import { extname } from 'node:path'; + +import { globSync } from 'glob'; +import { VFile } from 'vfile'; + +/** + * This creates a "loader" for loading Javascript source files into VFiles. + */ +const createLoader = () => { + /** + * Loads the JavaScript source files and transforms them into VFiles + * + * @param {string | Array} searchPath + */ + const loadFiles = searchPath => { + const resolvedFiles = globSync(searchPath).filter( + filePath => extname(filePath) === '.js' + ); + + return resolvedFiles.map(async filePath => { + const fileContents = await readFile(filePath, 'utf-8'); + + return new VFile({ path: filePath, value: fileContents }); + }); + }; + + return { loadFiles }; +}; + +export default createLoader; diff --git a/src/loader.mjs b/src/loaders/markdown.mjs similarity index 100% rename from src/loader.mjs rename to src/loaders/markdown.mjs diff --git a/src/parsers/javascript.mjs b/src/parsers/javascript.mjs new file mode 100644 index 0000000..e68e9d9 --- /dev/null +++ b/src/parsers/javascript.mjs @@ -0,0 +1,55 @@ +'use strict'; + +import * as acorn from 'acorn'; + +/** + * Creates a Javascript source parser for a given source file + */ +const createParser = () => { + /** + * Parses a given JavaScript file into an ESTree AST representation of it + * + * @param {import('vfile').VFile | Promise} sourceFile + * @returns {Promise} + */ + const parseJsSource = async sourceFile => { + // We allow the API doc VFile to be a Promise of a VFile also, + // hence we want to ensure that it first resolves before we pass it to the parser + const resolvedSourceFile = await Promise.resolve(sourceFile); + + if (typeof resolvedSourceFile.value !== 'string') { + throw new TypeError( + `expected resolvedSourceFile.value to be string but got ${typeof resolvedSourceFile.value}` + ); + } + + const res = acorn.parse(resolvedSourceFile.value, { + allowReturnOutsideFunction: true, + ecmaVersion: 'latest', + locations: true, + }); + + return { + ...res, + path: resolvedSourceFile.path, + }; + }; + + /** + * Parses multiple JavaScript files into ESTree ASTs by wrapping parseJsSource + * + * @param {Array>} apiDocs List of API doc files to be parsed + * @returns {Promise>} + */ + const parseJsSources = async apiDocs => { + // We do a Promise.all, to ensure that each API doc is resolved asynchronously + // but all need to be resolved first before we return the result to the caller + const resolvedApiDocEntries = await Promise.all(apiDocs.map(parseJsSource)); + + return resolvedApiDocEntries; + }; + + return { parseJsSource, parseJsSources }; +}; + +export default createParser; diff --git a/src/parser.mjs b/src/parsers/markdown.mjs similarity index 97% rename from src/parser.mjs rename to src/parsers/markdown.mjs index c4c3434..65707e3 100644 --- a/src/parser.mjs +++ b/src/parsers/markdown.mjs @@ -6,11 +6,11 @@ import { remove } from 'unist-util-remove'; import { selectAll } from 'unist-util-select'; import { SKIP, visit } from 'unist-util-visit'; -import createMetadata from './metadata.mjs'; -import createQueries from './queries.mjs'; +import createMetadata from '../metadata.mjs'; +import createQueries from '../queries.mjs'; -import { getRemark } from './utils/remark.mjs'; -import { createNodeSlugger } from './utils/slugger.mjs'; +import { getRemark } from '../utils/remark.mjs'; +import { createNodeSlugger } from '../utils/slugger.mjs'; /** * Creates an API doc parser for a given Markdown API doc file diff --git a/src/types.d.ts b/src/types.d.ts index a0be7da..ebae255 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -1,4 +1,5 @@ import type { Heading, Root } from '@types/mdast'; +import type { Program } from 'acorn'; import type { SemVer } from 'semver'; import type { Data, Node, Parent } from 'unist'; @@ -100,6 +101,11 @@ declare global { tags: Array; } + export interface JsProgram extends Program { + // Path to the program's source (i.e. `../node/lib/zlib.js`) + path: string; + } + export interface ApiDocReleaseEntry { version: SemVer; isLts: boolean;