diff --git a/src/full.ts b/src/full.ts index d1d744fd..3b065d4f 100644 --- a/src/full.ts +++ b/src/full.ts @@ -1,18 +1,18 @@ import * as yargs from "yargs"; import appInsights = require("applicationinsights"); -import { Fetcher } from "./util/io"; import calculateVersions from "./calculate-versions"; import clean from "./clean"; import createSearchIndex from "./create-search-index"; import generatePackages from "./generate-packages"; -import { getDefinitelyTyped } from "./get-definitely-typed"; +import { getDefinitelyTyped, resolveDefinitelyTypedMaster } from "./get-definitely-typed"; import { Options } from "./lib/common"; import { UncachedNpmInfoClient } from "./lib/npm-client"; import parseDefinitions from "./parse-definitions"; import publishPackages from "./publish-packages"; import publishRegistry from "./publish-registry"; import uploadBlobsAndUpdateIssue from "./upload-blobs"; +import { Fetcher } from "./util/io"; import { assertDefined, currentTimeStamp, logUncaughtErrors, numberOfOsProcesses } from "./util/util"; import validate from "./validate"; @@ -26,12 +26,13 @@ if (!module.parent) { export default async function full(dry: boolean, timeStamp: string, githubAccessToken: string, fetcher: Fetcher, options: Options): Promise { const infoClient = new UncachedNpmInfoClient(); await clean(); - const dt = await getDefinitelyTyped(options); + const commit = await resolveDefinitelyTypedMaster(githubAccessToken, fetcher); + const dt = await getDefinitelyTyped(options, commit); const allPackages = await parseDefinitions(dt, options.parseInParallel ? { nProcesses: numberOfOsProcesses, definitelyTypedPath: assertDefined(options.definitelyTypedPath) } : undefined); const changedPackages = await calculateVersions(dt, infoClient); - await generatePackages(dt, allPackages, changedPackages); + await generatePackages(dt, commit, allPackages, changedPackages); await createSearchIndex(allPackages, infoClient); await publishPackages(changedPackages, dry, githubAccessToken, fetcher); await publishRegistry(dt, allPackages, dry, infoClient); diff --git a/src/generate-packages.ts b/src/generate-packages.ts index da7f8785..482f9f10 100644 --- a/src/generate-packages.ts +++ b/src/generate-packages.ts @@ -1,12 +1,13 @@ import { emptyDir } from "fs-extra"; import * as yargs from "yargs"; -import { FS, getDefinitelyTyped } from "./get-definitely-typed"; +import { FS, getDefinitelyTyped, resolveDefinitelyTypedMaster } from "./get-definitely-typed"; import { Options } from "./lib/common"; import { generateNotNeededPackage, generateTypingPackage } from "./lib/package-generator"; import { AllPackages } from "./lib/packages"; import { outputDirPath } from "./lib/settings"; import { ChangedPackages, readChangedPackages } from "./lib/versions"; +import { Fetcher } from "./util/io"; import { logger, writeLog } from "./util/logging"; import { writeTgz } from "./util/tgz"; import { logUncaughtErrors } from "./util/util"; @@ -14,20 +15,23 @@ import { logUncaughtErrors } from "./util/util"; if (!module.parent) { const tgz = !!yargs.argv.tgz; logUncaughtErrors(async () => { - const dt = await getDefinitelyTyped(Options.defaults); + const githubAccessToken = process.env["GH_API_TOKEN"] || ""; + const fetcher = new Fetcher(); + const commit = await resolveDefinitelyTypedMaster(githubAccessToken, fetcher); + const dt = await getDefinitelyTyped(Options.defaults, commit); const allPackages = await AllPackages.read(dt); - await generatePackages(dt, allPackages, await readChangedPackages(allPackages), tgz); + await generatePackages(dt, commit, allPackages, await readChangedPackages(allPackages), tgz); }); } -export default async function generatePackages(dt: FS, allPackages: AllPackages, changedPackages: ChangedPackages, tgz = false): Promise { +export default async function generatePackages(dt: FS, gitHead: string, allPackages: AllPackages, changedPackages: ChangedPackages, tgz = false): Promise { const [log, logResult] = logger(); log("\n## Generating packages\n"); await emptyDir(outputDirPath); for (const { pkg, version } of changedPackages.changedTypings) { - await generateTypingPackage(pkg, allPackages, version, dt); + await generateTypingPackage(pkg, allPackages, version, dt, gitHead); if (tgz) { await writeTgz(pkg.outputDirectory, `${pkg.outputDirectory}.tgz`); } diff --git a/src/get-definitely-typed.ts b/src/get-definitely-typed.ts index 9daf8fc7..392eb9ef 100644 --- a/src/get-definitely-typed.ts +++ b/src/get-definitely-typed.ts @@ -8,8 +8,9 @@ import * as zlib from "zlib"; import { Options } from "./lib/common"; import { dataDirPath, definitelyTypedZipUrl } from "./lib/settings"; -import { readFile, readJson, stringOfStream } from "./util/io"; -import { assertDefined, assertSorted, Awaitable, exec, joinPaths, withoutStart, logUncaughtErrors } from "./util/util"; +import { Fetcher, readFile, readJson, stringOfStream } from "./util/io"; +import { assertDefined, assertSorted, Awaitable, exec, joinPaths, logUncaughtErrors, withoutStart } from "./util/util"; +import { queryGithub } from "./util/github"; /** * Readonly filesystem. @@ -43,10 +44,18 @@ if (!module.parent) { }); } -export async function getDefinitelyTyped(options: Options): Promise { +/** + * Return the commit SHA1 of the master tip of DefinitelyTyped + */ +export async function resolveDefinitelyTypedMaster(githubAccessToken: string, fetcher: Fetcher): Promise { + const masterRef = await queryGithub("repos/DefinitelyTyped/DefinitelyTyped/git/refs/heads/master", githubAccessToken, fetcher) as { object: { sha: string } }; + return masterRef.object.sha +} + +export async function getDefinitelyTyped(options: Options, revision: string = 'master'): Promise { if (options.definitelyTypedPath === undefined) { await ensureDir(dataDirPath); - return downloadAndExtractFile(definitelyTypedZipUrl); + return downloadAndExtractFile(definitelyTypedZipUrl + "/" + revision); } else { const { error, stderr, stdout } = await exec("git diff --name-only", options.definitelyTypedPath); if (error) { throw error; } diff --git a/src/lib/package-generator.ts b/src/lib/package-generator.ts index 0a224d95..dca0e294 100644 --- a/src/lib/package-generator.ts +++ b/src/lib/package-generator.ts @@ -13,11 +13,11 @@ import { sourceBranch } from "./settings"; const mitLicense = readFileSync(joinPaths(__dirname, "..", "..", "LICENSE"), "utf-8"); -export async function generateTypingPackage(typing: TypingsData, packages: AllPackages, version: string, dt: FS): Promise { +export async function generateTypingPackage(typing: TypingsData, packages: AllPackages, version: string, dt: FS, gitHead: string): Promise { const typesDirectory = dt.subDir("types").subDir(typing.name); const packageFS = typing.isLatest ? typesDirectory : typesDirectory.subDir(`v${typing.major}`); - const packageJson = await createPackageJSON(typing, version, packages); + const packageJson = await createPackageJSON(typing, version, gitHead, packages); await writeCommonOutputs(typing, packageJson, createReadme(typing)); await Promise.all(typing.files.map(async file => writeFile(await outputFilePath(typing, file), await packageFS.readFile(file)))); } @@ -52,11 +52,12 @@ async function outputFilePath(pkg: AnyPackage, filename: string): Promise { +async function createPackageJSON(typing: TypingsData, version: string, gitHead: string, packages: AllPackages): Promise { // Use the ordering of fields from https://docs.npmjs.com/files/package.json const out: {} = { name: typing.fullNpmName, version, + gitHead, description: `TypeScript definitions for ${typing.libraryName}`, // keywords, // homepage, diff --git a/src/lib/settings.ts b/src/lib/settings.ts index fe4299e1..f311cf28 100644 --- a/src/lib/settings.ts +++ b/src/lib/settings.ts @@ -13,8 +13,8 @@ export const outputDirPath = joinPaths(root, "output"); export const validateOutputPath = joinPaths(root, "validateOutput"); export const logDir = joinPaths(root, "logs"); -/** URL to download the repository from. */ -export const definitelyTypedZipUrl = "https://codeload.github.com/DefinitelyTyped/DefinitelyTyped/tar.gz/master"; +/** URL to download the repository from. Needs to be appended with a revision. */ +export const definitelyTypedZipUrl = "https://codeload.github.com/DefinitelyTyped/DefinitelyTyped/tar.gz"; /** The branch that DefinitelyTyped is sourced from. */ export const sourceBranch = "master"; diff --git a/src/publish-packages.ts b/src/publish-packages.ts index 35db5891..7507f36e 100644 --- a/src/publish-packages.ts +++ b/src/publish-packages.ts @@ -1,13 +1,14 @@ import * as yargs from "yargs"; import appInsights = require("applicationinsights"); -import { Fetcher } from "./util/io"; import { getDefinitelyTyped } from "./get-definitely-typed"; import { Options } from "./lib/common"; import { NpmPublishClient } from "./lib/npm-client"; import { deprecateNotNeededPackage, publishNotNeededPackage, publishTypingsPackage } from "./lib/package-publisher"; import { AllPackages } from "./lib/packages"; import { ChangedPackages, readChangedPackages } from "./lib/versions"; +import { queryGithub } from "./util/github"; +import { Fetcher } from "./util/io"; import { logger, writeLog } from "./util/logging"; import { logUncaughtErrors } from "./util/util"; @@ -86,17 +87,3 @@ export default async function publishPackages(changedPackages: ChangedPackages, await writeLog("publishing.md", logResult()); console.log("Done!"); } - -async function queryGithub(path: string, githubToken: string, fetcher: Fetcher) { - const [log] = logger(); - log("Requesting from github: " + path); - return await fetcher.fetchJson({ - hostname: "api.github.com", - path: path + "&access_token=" + githubToken, - method: "GET", - headers: { - // arbitrary string, but something must be provided - "User-Agent": "types-publisher", - }, - }); -} diff --git a/src/util/github.ts b/src/util/github.ts new file mode 100644 index 00000000..f946236f --- /dev/null +++ b/src/util/github.ts @@ -0,0 +1,16 @@ +import { Fetcher } from "./io"; +import { logger } from "./logging"; + +export async function queryGithub(path: string, githubToken: string, fetcher: Fetcher) { + const [log] = logger(); + log("Requesting from github: " + path); + return fetcher.fetchJson({ + hostname: "api.github.com", + path: path + "&access_token=" + githubToken, + method: "GET", + headers: { + // arbitrary string, but something must be provided + "User-Agent": "types-publisher", + }, + }); +}