Skip to content

Commit

Permalink
Use pacote second attempt (#483)
Browse files Browse the repository at this point in the history
* Use cached npm metadata

* Simply read the npm cache

* Use pacote
  • Loading branch information
jablko committed Jun 28, 2022
1 parent a14dcb6 commit 90eac4d
Show file tree
Hide file tree
Showing 23 changed files with 719 additions and 586 deletions.
5 changes: 1 addition & 4 deletions .github/workflows/publish-registry.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,11 @@ jobs:
- run: yarn build
- name: Parse declarations
run: yarn workspace @definitelytyped/publisher parse
# https://github.com/microsoft/types-publisher/commit/7fa7f9c3cbd0c8bd6131398b4fea1faf9baf9665#diff-c4063aea1f6d1c21c51407252387627e9a506c735f8dbd58fbf82ae821fae128R170
- uses: actions/cache@v3
with:
path: packages/publisher/cache/
path: packages/utils/cache/
key: cache-${{ github.run_id }}
restore-keys: cache-
- name: Calculate versions
run: yarn workspace @definitelytyped/publisher calculate-versions
- name: Publish registry
run: yarn workspace @definitelytyped/publisher publish-registry
env:
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,6 @@
"ts-jest": "^25.2.1",
"tslint": "^6.1.2",
"tslint-microsoft-contrib": "^6.2.0",
"typescript": "^4.5.5"
"typescript": "^4.7.4"
}
}
4 changes: 3 additions & 1 deletion packages/definitions-parser/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,12 @@
"@definitelytyped/utils": "^0.0.118",
"@types/node": "^14.14.35",
"fs-extra": "^9.1.0",
"pacote": "^13.6.1",
"semver": "^7.3.7"
},
"devDependencies": {
"@types/fs-extra": "^9.0.8"
"@types/fs-extra": "^9.0.8",
"@types/pacote": "^11.1.5"
},
"peerDependencies": {
"typescript": "*"
Expand Down
49 changes: 13 additions & 36 deletions packages/definitions-parser/src/check-parse-results.ts
Original file line number Diff line number Diff line change
@@ -1,37 +1,13 @@
import { ParseDefinitionsOptions } from "./get-definitely-typed";
import { TypingsData, AllPackages, formatTypingVersion } from "./packages";
import {
assertDefined,
mapDefined,
nAtATime,
FS,
logger,
writeLog,
Logger,
UncachedNpmInfoClient,
NpmInfoRawVersions,
NpmInfoVersion,
max,
min,
} from "@definitelytyped/utils";
import { mapDefined, nAtATime, FS, logger, writeLog, Logger, cacheDir, max, min } from "@definitelytyped/utils";
import * as pacote from "pacote";
import * as semver from "semver";

export async function checkParseResults(
includeNpmChecks: false,
dt: FS,
options: ParseDefinitionsOptions
): Promise<void>;
export async function checkParseResults(
includeNpmChecks: true,
dt: FS,
options: ParseDefinitionsOptions,
client: UncachedNpmInfoClient
): Promise<void>;
export async function checkParseResults(
includeNpmChecks: boolean,
dt: FS,
options: ParseDefinitionsOptions,
client?: UncachedNpmInfoClient
options: ParseDefinitionsOptions
): Promise<void> {
const allPackages = await AllPackages.read(dt);
const [log, logResult] = logger();
Expand All @@ -57,7 +33,7 @@ export async function checkParseResults(
await nAtATime(
10,
allPackages.allTypings(),
(pkg) => checkNpm(pkg, log, dependedOn, client!),
(pkg) => checkNpm(pkg, log, dependedOn),
options.progress
? {
name: "Checking for typed packages...",
Expand Down Expand Up @@ -120,14 +96,16 @@ Check the path mappings for [${Array.from(allPackages.allDependencyTypings(pkg))
async function checkNpm(
{ major, minor, name, libraryName, projectName, contributors }: TypingsData,
log: Logger,
dependedOn: ReadonlySet<string>,
client: UncachedNpmInfoClient
dependedOn: ReadonlySet<string>
): Promise<void> {
if (notNeededExceptions.has(name)) {
return;
}

const info = await client.fetchRawNpmInfo(name); // Gets info for the real package, not the @types package
const info = await pacote.packument(name, { cache: cacheDir, fullMetadata: true }).catch((reason) => {
if (reason.code !== "E404") throw reason;
return undefined;
}); // Gets info for the real package, not the @types package
if (!info) {
return;
}
Expand Down Expand Up @@ -169,21 +147,20 @@ async function checkNpm(
}
}

export async function packageHasTypes(packageName: string, client: UncachedNpmInfoClient): Promise<boolean> {
const info = assertDefined(await client.fetchRawNpmInfo(packageName));
return versionHasTypes(info.versions[info["dist-tags"].latest]);
export async function packageHasTypes(packageName: string): Promise<boolean> {
return versionHasTypes(await pacote.manifest(packageName, { cache: cacheDir, fullMetadata: true }));
}

function getRegularVersions(
versions: NpmInfoRawVersions
versions: pacote.Packument["versions"]
): readonly { readonly version: semver.SemVer; readonly hasTypes: boolean }[] {
return Object.entries(versions).map(([versionString, info]) => ({
version: new semver.SemVer(versionString),
hasTypes: versionHasTypes(info),
}));
}

function versionHasTypes(info: NpmInfoVersion): boolean {
function versionHasTypes(info: pacote.Manifest): boolean {
return "types" in info || "typings" in info;
}

Expand Down
51 changes: 24 additions & 27 deletions packages/definitions-parser/src/git.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@ import {
FS,
consoleLogger,
assertDefined,
UncachedNpmInfoClient,
NpmInfo,
cacheDir,
} from "@definitelytyped/utils";
import * as pacote from "pacote";
import * as semver from "semver";
import { getAffectedPackages } from "./get-affected-packages";

Expand Down Expand Up @@ -96,11 +96,8 @@ export async function getAffectedPackagesFromDiff(
const allPackages = await AllPackages.read(dt);
const diffs = await gitDiff(consoleLogger.info, definitelyTypedPath);
if (diffs.find((d) => d.file === "notNeededPackages.json")) {
const uncached = new UncachedNpmInfoClient();
for (const deleted of getNotNeededPackages(allPackages, diffs)) {
const source = await uncached.fetchNpmInfo(deleted.libraryName); // eg @babel/parser
const typings = await uncached.fetchNpmInfo(deleted.fullNpmName); // eg @types/babel__parser
checkNotNeededPackage(deleted, source, typings);
checkNotNeededPackage(deleted);
}
}

Expand Down Expand Up @@ -133,30 +130,30 @@ export async function getAffectedPackagesFromDiff(
* 2. asOfVersion must be newer than `@types/name@latest` on npm
* 3. `name@asOfVersion` must exist on npm
*/
export function checkNotNeededPackage(
unneeded: NotNeededPackage,
source: NpmInfo | undefined,
typings: NpmInfo | undefined
) {
source = assertDefined(
source,
`The entry for ${unneeded.fullNpmName} in notNeededPackages.json has
export async function checkNotNeededPackage(unneeded: NotNeededPackage) {
await pacote.manifest(`${unneeded.libraryName}@${unneeded.version}`, { cache: cacheDir }).catch((reason) => {
throw reason.code === "E404"
? new Error(
`The entry for ${unneeded.fullNpmName} in notNeededPackages.json has
"libraryName": "${unneeded.libraryName}", but there is no npm package with this name.
Unneeded packages have to be replaced with a package on npm.`
);
typings = assertDefined(typings, `Unexpected error: @types package not found for ${unneeded.fullNpmName}`);
const latestTypings = assertDefined(
typings.distTags.get("latest"),
`Unexpected error: ${unneeded.fullNpmName} is missing the "latest" tag.`
);
Unneeded packages have to be replaced with a package on npm.`,
{ cause: reason }
)
: reason.code === "ETARGET"
? new Error(`The specified version ${unneeded.version} of ${unneeded.libraryName} is not on npm.`, {
cause: reason,
})
: reason;
}); // eg @babel/parser
const typings = await pacote.manifest(unneeded.fullNpmName, { cache: cacheDir }).catch((reason) => {
throw reason.code === "E404"
? new Error(`Unexpected error: @types package not found for ${unneeded.fullNpmName}`, { cause: reason })
: reason;
}); // eg @types/babel__parser
assert(
semver.gt(unneeded.version, latestTypings),
semver.gt(unneeded.version, typings.version),
`The specified version ${unneeded.version} of ${unneeded.libraryName} must be newer than the version
it is supposed to replace, ${latestTypings} of ${unneeded.fullNpmName}.`
);
assert(
source.versions.has(String(unneeded.version)),
`The specified version ${unneeded.version} of ${unneeded.libraryName} is not on npm.`
it is supposed to replace, ${typings.version} of ${unneeded.fullNpmName}.`
);
}

Expand Down
119 changes: 44 additions & 75 deletions packages/definitions-parser/test/git.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { NpmInfo } from "@definitelytyped/utils";
import * as util from "util";
import * as pacote from "pacote";
import { createTypingsVersionRaw, testo } from "./utils";
import { GitDiff, getNotNeededPackages, checkNotNeededPackage } from "../src/git";
import { NotNeededPackage, TypesDataFile, AllPackages } from "../src/packages";
Expand Down Expand Up @@ -65,94 +66,62 @@ testo({
// TODO: Test with dependents, etc etc
});

const empty: NpmInfo = {
homepage: "",
distTags: new Map(),
versions: new Map(),
time: new Map(),
};
jest.mock("pacote", () => ({
async manifest(spec: string, opts: pacote.Options) {
switch (spec) {
case "[email protected]": // Older than the @types/jest package.
case "[email protected]": // The same version as the @types/jest package.
case "[email protected]": // Newer than the @types/jest package.
// These versions exist (don't throw).
return;
case "[email protected]": // A nonexistent version of the replacement package.
// eslint-disable-next-line no-throw-literal
throw { code: "ETARGET" };
case "@types/jest": // The @types/jest package.
return { version: "50.0.0" };
case "[email protected]": // A nonexistent replacement package.
case "@types/nonexistent": // A nonexistent @types package.
// eslint-disable-next-line no-throw-literal
throw { code: opts.offline ? "ENOTCACHED" : "E404" };
}
throw new Error(`Unexpected npm registry fetch: ${util.inspect(spec)}`);
},
}));

const newerReplacement = new NotNeededPackage("jest", "jest", "100.0.0");
const olderReplacement = new NotNeededPackage("jest", "jest", "4.0.0");
const sameVersion = new NotNeededPackage("jest", "jest", "50.0.0");
const nonexistentReplacementVersion = new NotNeededPackage("jest", "jest", "999.0.0");
const nonexistentReplacementPackage = new NotNeededPackage("jest", "nonexistent", "100.0.0");
const nonexistentTypesPackage = new NotNeededPackage("nonexistent", "jest", "100.0.0");

testo({
missingSource() {
expect(() => checkNotNeededPackage(jestNotNeeded[0], undefined, empty)).toThrow(
return expect(checkNotNeededPackage(nonexistentReplacementPackage)).rejects.toThrow(
"The entry for @types/jest in notNeededPackages.json"
);
},
missingTypings() {
expect(() => checkNotNeededPackage(jestNotNeeded[0], empty, undefined)).toThrow(
"@types package not found for @types/jest"
);
},
missingTypingsLatest() {
expect(() => checkNotNeededPackage(jestNotNeeded[0], empty, empty)).toThrow(
'@types/jest is missing the "latest" tag'
return expect(checkNotNeededPackage(nonexistentTypesPackage)).rejects.toThrow(
"@types package not found for @types/nonexistent"
);
},
deprecatedSameVersion() {
expect(() => {
checkNotNeededPackage(jestNotNeeded[0], empty, {
homepage: "jest.com",
distTags: new Map([["latest", "100.0.0"]]),
versions: new Map(),
time: new Map([["modified", ""]]),
});
}).toThrow(`The specified version 100.0.0 of jest must be newer than the version
it is supposed to replace, 100.0.0 of @types/jest.`);
return expect(checkNotNeededPackage(sameVersion)).rejects
.toThrow(`The specified version 50.0.0 of jest must be newer than the version
it is supposed to replace, 50.0.0 of @types/jest.`);
},
deprecatedOlderVersion() {
expect(() => {
checkNotNeededPackage(jestNotNeeded[0], empty, {
homepage: "jest.com",
distTags: new Map([["latest", "999.0.0"]]),
versions: new Map(),
time: new Map([["modified", ""]]),
});
}).toThrow(`The specified version 100.0.0 of jest must be newer than the version
it is supposed to replace, 999.0.0 of @types/jest.`);
return expect(checkNotNeededPackage(olderReplacement)).rejects
.toThrow(`The specified version 4.0.0 of jest must be newer than the version
it is supposed to replace, 50.0.0 of @types/jest.`);
},
missingNpmVersion() {
expect(() => {
checkNotNeededPackage(jestNotNeeded[0], empty, {
homepage: "jest.com",
distTags: new Map([["latest", "4.0.0"]]),
versions: new Map(),
time: new Map([["modified", ""]]),
});
}).toThrow("The specified version 100.0.0 of jest is not on npm.");
},
olderNpmVersion() {
expect(() =>
checkNotNeededPackage(
jestNotNeeded[0],
{
homepage: "jest.com",
distTags: new Map(),
versions: new Map([["50.0.0", {}]]),
time: new Map([["modified", ""]]),
},
{
homepage: "jest.com",
distTags: new Map([["latest", "4.0.0"]]),
versions: new Map(),
time: new Map([["modified", ""]]),
}
)
).toThrow("The specified version 100.0.0 of jest is not on npm.");
return expect(checkNotNeededPackage(nonexistentReplacementVersion)).rejects.toThrow(
"The specified version 999.0.0 of jest is not on npm."
);
},
ok() {
checkNotNeededPackage(
jestNotNeeded[0],
{
homepage: "jest.com",
distTags: new Map(),
versions: new Map([["100.0.0", {}]]),
time: new Map([["modified", ""]]),
},
{
homepage: "jest.com",
distTags: new Map([["latest", "4.0.0"]]),
versions: new Map(),
time: new Map([["modified", ""]]),
}
);
return checkNotNeededPackage(newerReplacement);
},
});
2 changes: 1 addition & 1 deletion packages/perf/src/measure/measurePerf.ts
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,7 @@ function getLatestTypesVersionForTypeScriptVersion(
): string | undefined {
const tsVersion = new semver.SemVer(typeScriptVersion);
for (let i = typesVersions.length - 1; i > 0; i--) {
if (semver.gte(tsVersion, `${typesVersions[i]}.0-`)) {
if (semver.gte(tsVersion, `${typesVersions[i]}.0-0`)) {
return typesVersions[i];
}
}
Expand Down
1 change: 0 additions & 1 deletion packages/publisher/.gitignore
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
bin/
cache/
data/
logs/
node_modules/
Expand Down
2 changes: 2 additions & 0 deletions packages/publisher/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
"hh-mm-ss": "^1.2.0",
"longjohn": "^0.2.11",
"oboe": "^2.1.3",
"pacote": "^13.6.1",
"semver": "^7.3.7",
"source-map-support": "^0.4.0",
"typescript": "^4.1.0",
Expand All @@ -31,6 +32,7 @@
"@types/hh-mm-ss": "^1.2.1",
"@types/mz": "^0.0.31",
"@types/oboe": "^2.0.28",
"@types/pacote": "^11.1.5",
"@types/source-map-support": "^0.4.0",
"@types/yargs": "^15.0.4"
},
Expand Down
Loading

0 comments on commit 90eac4d

Please sign in to comment.