From bf27c30fd3788c971b281ce2b8d8c111cf783de3 Mon Sep 17 00:00:00 2001 From: Jack Bates Date: Fri, 6 May 2022 09:37:21 -0700 Subject: [PATCH] Add dry runs to CI --- .github/workflows/ci.yml | 15 +++++++ .github/workflows/publish-packages.yml | 43 ++++++++++++++++++++ .github/workflows/publish-registry.yml | 40 ++++++++++++++++++ .github/workflows/retag.yml | 40 ++++++++++++++++++ .github/workflows/update-ts-version-tags.yml | 35 ---------------- package.json | 2 +- packages/publisher/src/calculate-versions.ts | 10 ++++- packages/publisher/src/generate-packages.ts | 5 ++- packages/publisher/src/parse-definitions.ts | 5 ++- packages/publisher/src/publish-packages.ts | 10 +++-- packages/publisher/src/publish-registry.ts | 10 +++-- packages/retag/package.json | 3 -- packages/retag/src/index.ts | 8 +++- 13 files changed, 173 insertions(+), 53 deletions(-) create mode 100644 .github/workflows/publish-packages.yml create mode 100644 .github/workflows/publish-registry.yml create mode 100644 .github/workflows/retag.yml delete mode 100644 .github/workflows/update-ts-version-tags.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7bdfe4fd9c..e61fca0ef4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,21 @@ jobs: run: yarn lint - name: test run: yarn test + publish-packages-dry: + needs: build_and_test + uses: ./.github/workflows/publish-packages.yml + with: + dry: true + publish-registry-dry: + needs: build_and_test + uses: ./.github/workflows/publish-registry.yml + with: + dry: true + retag-dry: + needs: build_and_test + uses: ./.github/workflows/retag.yml + with: + dry: true publish_alpha: name: publish alpha release runs-on: ubuntu-latest diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml new file mode 100644 index 0000000000..a3157b717c --- /dev/null +++ b/.github/workflows/publish-packages.yml @@ -0,0 +1,43 @@ +name: Publish packages +on: + workflow_call: + inputs: + dry: + description: Dry run + type: boolean + workflow_dispatch: + inputs: + dry: + description: Dry run + type: boolean +jobs: + publish-packages: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + cache: yarn + - run: yarn install --frozen-lockfile + - run: yarn build + - name: Parse declarations + run: yarn workspace @definitelytyped/publisher parse + - uses: actions/cache@v3 + with: + path: packages/utils/cache/ + key: cache-${{ github.run_id }} + restore-keys: cache- + - name: Calculate versions + run: yarn workspace @definitelytyped/publisher calculate-versions + - name: Generate packages + run: yarn workspace @definitelytyped/publisher generate + - name: Publish packages${{ (inputs || github.event.inputs).dry && ' dry run' || '' }} + run: yarn workspace @definitelytyped/publisher publish-packages${{ (inputs || github.event.inputs).dry && ' --dry' || '' }} + env: + GH_API_TOKEN: ${{ secrets.GH_API_TOKEN }} + NPM_TOKEN: ${{ secrets.NPM_RETAG_TOKEN }} + - if: always() + uses: actions/upload-artifact@v3 + with: + name: ${{ github.job }} + path: packages/definitions-parser/data/ diff --git a/.github/workflows/publish-registry.yml b/.github/workflows/publish-registry.yml new file mode 100644 index 0000000000..70954c32f9 --- /dev/null +++ b/.github/workflows/publish-registry.yml @@ -0,0 +1,40 @@ +name: Publish registry +on: + workflow_call: + inputs: + dry: + description: Dry run + type: boolean + workflow_dispatch: + inputs: + dry: + description: Dry run + type: boolean +jobs: + publish-registry: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + cache: yarn + - run: yarn install --frozen-lockfile + - run: yarn build + - name: Parse declarations + run: yarn workspace @definitelytyped/publisher parse + - uses: actions/cache@v3 + with: + path: packages/utils/cache/ + key: cache-${{ github.run_id }} + restore-keys: cache- + - name: Calculate versions + run: yarn workspace @definitelytyped/publisher calculate-versions + - name: Publish registry${{ (inputs || github.event.inputs).dry && ' dry run' || '' }} + run: yarn workspace @definitelytyped/publisher publish-registry${{ (inputs || github.event.inputs).dry && ' --dry' || '' }} + env: + NPM_TOKEN: ${{ secrets.NPM_RETAG_TOKEN }} + - if: always() + uses: actions/upload-artifact@v3 + with: + name: ${{ github.job }} + path: packages/definitions-parser/data/ diff --git a/.github/workflows/retag.yml b/.github/workflows/retag.yml new file mode 100644 index 0000000000..3093c91777 --- /dev/null +++ b/.github/workflows/retag.yml @@ -0,0 +1,40 @@ +name: Update npm tags +on: + schedule: + # https://crontab.guru/#0_0_*_*_0 + - cron: 0 0 * * 0 + workflow_call: + inputs: + dry: + description: Dry run + type: boolean + workflow_dispatch: + inputs: + dry: + description: Dry run + type: boolean +jobs: + retag: + if: github.event_name != 'schedule' || github.repository == 'microsoft/DefinitelyTyped-tools' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + cache: yarn + - run: yarn install --frozen-lockfile + - run: yarn build + - uses: actions/cache@v3 + with: + path: packages/utils/cache/ + key: cache-${{ github.run_id }} + restore-keys: cache- + - name: Update npm tags${{ (inputs || github.event.inputs).dry && ' dry run' || '' }} + run: yarn retag${{ (inputs || github.event.inputs).dry && ' --dry' || '' }} + env: + NPM_TOKEN: ${{ secrets.NPM_RETAG_TOKEN }} + - if: always() + uses: actions/upload-artifact@v3 + with: + name: ${{ github.job }} + path: packages/definitions-parser/data/ diff --git a/.github/workflows/update-ts-version-tags.yml b/.github/workflows/update-ts-version-tags.yml deleted file mode 100644 index 220c96e542..0000000000 --- a/.github/workflows/update-ts-version-tags.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Update ts* tags for ATA - -# For testing -# on: pull_request - -# For production -on: - workflow_dispatch: - inputs: - checkout: - description: ref to deploy - required: true - default: master - schedule: - # https://crontab.guru/#5_8_*_*_1 - - cron: "5 8 * * 1" - -jobs: - publish: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 - with: - node-version: '14' - - name: retag - run: | - git clone --depth 1 https://github.com/DefinitelyTyped/DefinitelyTyped ../DefinitelyTyped - yarn - yarn build - node packages/retag/dist/index.js - env: - NPM_TOKEN: ${{ secrets.NPM_RETAG_TOKEN }} - GH_API_TOKEN: ${{ secrets.GH_API_TOKEN }} - \ No newline at end of file diff --git a/package.json b/package.json index c2035515c5..b717b715a8 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "format": "prettier --write 'packages/**/*.ts'", "test": "jest", "build": "tsc -b .", - "retag": "node packages/retag/dist/retag.js" + "retag": "node --require source-map-support/register packages/retag/" }, "devDependencies": { "@types/jest": "^25.1.3", diff --git a/packages/publisher/src/calculate-versions.ts b/packages/publisher/src/calculate-versions.ts index cb0bb7a006..2fd36b8494 100644 --- a/packages/publisher/src/calculate-versions.ts +++ b/packages/publisher/src/calculate-versions.ts @@ -1,4 +1,5 @@ -import { defaultLocalOptions } from "./lib/common"; +import process from "process"; +import { defaultLocalOptions, defaultRemoteOptions } from "./lib/common"; import { ChangedPackages, ChangedPackagesJson, ChangedTypingJson, versionsFilename } from "./lib/versions"; import { getDefinitelyTyped, AllPackages, NotNeededPackage, writeDataFile } from "@definitelytyped/definitions-parser"; import { logUncaughtErrors, loggerWithErrors, FS, LoggerWithErrors, defaultCacheDir } from "@definitelytyped/utils"; @@ -7,7 +8,12 @@ import * as pacote from "pacote"; if (!module.parent) { const log = loggerWithErrors()[0]; - logUncaughtErrors(async () => calculateVersions(await getDefinitelyTyped(defaultLocalOptions, log), log)); + logUncaughtErrors(async () => + calculateVersions( + await getDefinitelyTyped(process.env.GITHUB_ACTIONS ? defaultRemoteOptions : defaultLocalOptions, log), + log + ) + ); } export default async function calculateVersions(dt: FS, log: LoggerWithErrors): Promise { diff --git a/packages/publisher/src/generate-packages.ts b/packages/publisher/src/generate-packages.ts index 04d4ca3406..a24de6f68a 100644 --- a/packages/publisher/src/generate-packages.ts +++ b/packages/publisher/src/generate-packages.ts @@ -1,9 +1,10 @@ import { makeTypesVersionsForPackageJson } from "@definitelytyped/header-parser"; import { emptyDir, mkdir, mkdirp, readFileSync } from "fs-extra"; import path = require("path"); +import process from "process"; import yargs = require("yargs"); -import { defaultLocalOptions } from "./lib/common"; +import { defaultLocalOptions, defaultRemoteOptions } from "./lib/common"; import { outputDirPath, sourceBranch } from "./lib/settings"; import { assertNever, @@ -42,7 +43,7 @@ if (!module.parent) { const tgz = !!yargs.argv.tgz; logUncaughtErrors(async () => { const log = loggerWithErrors()[0]; - const dt = await getDefinitelyTyped(defaultLocalOptions, log); + const dt = await getDefinitelyTyped(process.env.GITHUB_ACTIONS ? defaultRemoteOptions : defaultLocalOptions, log); const allPackages = await AllPackages.read(dt); await generatePackages(dt, allPackages, await readChangedPackages(allPackages), tgz); }); diff --git a/packages/publisher/src/parse-definitions.ts b/packages/publisher/src/parse-definitions.ts index 1cd5085adb..de3e61af56 100644 --- a/packages/publisher/src/parse-definitions.ts +++ b/packages/publisher/src/parse-definitions.ts @@ -1,7 +1,8 @@ import os from "os"; +import process from "process"; import yargs from "yargs"; import { logUncaughtErrors, loggerWithErrors, assertDefined, FS } from "@definitelytyped/utils"; -import { defaultLocalOptions } from "./lib/common"; +import { defaultLocalOptions, defaultRemoteOptions } from "./lib/common"; import { getTypingInfo } from "@definitelytyped/definitions-parser/dist/lib/definition-parser"; import { getDefinitelyTyped, @@ -16,7 +17,7 @@ if (!module.parent) { nProcesses: { type: "number" }, }).argv; - const options = defaultLocalOptions; + const options = process.env.GITHUB_ACTIONS ? defaultRemoteOptions : defaultLocalOptions; logUncaughtErrors(async () => { const log = loggerWithErrors()[0]; const dt = await getDefinitelyTyped(options, log); diff --git a/packages/publisher/src/publish-packages.ts b/packages/publisher/src/publish-packages.ts index 1e4d3a9f49..873852a87c 100644 --- a/packages/publisher/src/publish-packages.ts +++ b/packages/publisher/src/publish-packages.ts @@ -1,7 +1,8 @@ +import process from "process"; import applicationinsights = require("applicationinsights"); import * as yargs from "yargs"; -import { defaultLocalOptions } from "./lib/common"; +import { defaultLocalOptions, defaultRemoteOptions } from "./lib/common"; import { deprecateNotNeededPackage, publishNotNeededPackage, publishTypingsPackage } from "./lib/package-publisher"; import { getDefinitelyTyped, AllPackages } from "@definitelytyped/definitions-parser"; import { @@ -20,7 +21,10 @@ if (!module.parent) { const dry = !!yargs.argv.dry; const deprecateName = yargs.argv.deprecate as string | undefined; logUncaughtErrors(async () => { - const dt = await getDefinitelyTyped(defaultLocalOptions, loggerWithErrors()[0]); + const dt = await getDefinitelyTyped( + process.env.GITHUB_ACTIONS ? defaultRemoteOptions : defaultLocalOptions, + loggerWithErrors()[0] + ); if (deprecateName !== undefined) { // A '--deprecate' command is available in case types-publisher got stuck *while* trying to deprecate a package. // Normally this should not be needed. @@ -56,7 +60,7 @@ export default async function publishPackages( log("=== Publishing packages ==="); } - const client = await NpmPublishClient.create(await getSecret(Secret.NPM_TOKEN), undefined); + const client = await NpmPublishClient.create(dry ? "" : await getSecret(Secret.NPM_TOKEN), undefined); for (const cp of changedPackages.changedTypings) { log(`Publishing ${cp.pkg.desc}...`); diff --git a/packages/publisher/src/publish-registry.ts b/packages/publisher/src/publish-registry.ts index b5df7f5c42..bf87e50d4c 100644 --- a/packages/publisher/src/publish-registry.ts +++ b/packages/publisher/src/publish-registry.ts @@ -1,8 +1,9 @@ import assert = require("assert"); +import process from "process"; import { emptyDir } from "fs-extra"; import * as yargs from "yargs"; -import { defaultLocalOptions } from "./lib/common"; +import { defaultLocalOptions, defaultRemoteOptions } from "./lib/common"; import { outputDirPath, validateOutputPath } from "./lib/settings"; import { getDefinitelyTyped, @@ -44,7 +45,10 @@ Generated by [types-publisher](${pkg.homepage}).`; if (!module.parent) { const dry = !!yargs.argv.dry; logUncaughtErrors(async () => { - const dt = await getDefinitelyTyped(defaultLocalOptions, loggerWithErrors()[0]); + const dt = await getDefinitelyTyped( + process.env.GITHUB_ACTIONS ? defaultRemoteOptions : defaultLocalOptions, + loggerWithErrors()[0] + ); await publishRegistry(dt, await AllPackages.read(dt), dry); }); } @@ -72,7 +76,7 @@ export default async function publishRegistry(dt: FS, allPackages: AllPackages, const packageJson = generatePackageJson(typesRegistry, newVersion, newContentHash); await generate(registry, packageJson); - const token = await getSecret(Secret.NPM_TOKEN); + const token = dry ? "" : await getSecret(Secret.NPM_TOKEN); const publishClient = () => NpmPublishClient.create(token, { defaultTag: "next" }); if (highestSemverVersion !== npmVersion) { diff --git a/packages/retag/package.json b/packages/retag/package.json index edfbfd93b9..59cc52aa77 100644 --- a/packages/retag/package.json +++ b/packages/retag/package.json @@ -31,9 +31,6 @@ "@types/yargs": "^15.0.5" }, "main": "dist/index.js", - "bin": { - "retag": "./dist/retag.js" - }, "publishConfig": { "access": "public" } diff --git a/packages/retag/src/index.ts b/packages/retag/src/index.ts index 47d762494d..92d9af5ff5 100644 --- a/packages/retag/src/index.ts +++ b/packages/retag/src/index.ts @@ -48,10 +48,14 @@ async function main() { */ async function tag(dry: boolean, nProcesses: number, name?: string) { const log = loggerWithErrors()[0]; - const options = { definitelyTypedPath: "../DefinitelyTyped", progress: true, parseInParallel: true }; + const options = process.env.GITHUB_ACTIONS + ? { definitelyTypedPath: undefined, progress: false, parseInParallel: false } + : { definitelyTypedPath: "../DefinitelyTyped", progress: true, parseInParallel: true }; await parseDefinitions( await getDefinitelyTyped(options, log), - { nProcesses: nProcesses || os.cpus().length, definitelyTypedPath: "../DefinitelyTyped" }, + options.parseInParallel + ? { nProcesses: nProcesses || os.cpus().length, definitelyTypedPath: "../DefinitelyTyped" } + : undefined, log );