Skip to content

Commit

Permalink
Run arethetypeswrong from in-memory tarball data (#943)
Browse files Browse the repository at this point in the history
* Run arethetypeswrong from in-memory tarball data

* Remove extra dependency

* Pin attw version

* Add changeset
  • Loading branch information
andrewbranch authored Feb 8, 2024
1 parent 9da3fc7 commit 31de5d3
Show file tree
Hide file tree
Showing 6 changed files with 60 additions and 116 deletions.
6 changes: 6 additions & 0 deletions .changeset/dull-pets-call.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
"@definitelytyped/dtslint": patch
"@definitelytyped/utils": patch
---

Run arethetypeswrong from in-memory tarball data
12 changes: 3 additions & 9 deletions packages/dtslint/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@
"test": "../../node_modules/.bin/jest --config ../../jest.config.js packages/dtslint"
},
"dependencies": {
"@arethetypeswrong/cli": "^0.13.6",
"@arethetypeswrong/core": "^0.13.6",
"@arethetypeswrong/cli": "0.13.8",
"@arethetypeswrong/core": "0.13.6",
"@definitelytyped/header-parser": "workspace:*",
"@definitelytyped/typescript-versions": "workspace:*",
"@definitelytyped/utils": "workspace:*",
Expand All @@ -35,10 +35,7 @@
"eslint": "^8.56.0",
"eslint-plugin-import": "^2.29.1",
"semver": "^7.5.4",
"strip-json-comments": "^3.1.1",
"tar": "^6.2.0",
"tmp": "^0.2.1",
"which": "^4.0.0"
"strip-json-comments": "^3.1.1"
},
"peerDependencies": {
"typescript": ">= 3.0.0-dev || >= 3.1.0-dev || >= 3.2.0-dev || >= 3.3.0-dev || >= 3.4.0-dev || >= 3.5.0-dev || >= 3.6.0-dev || >= 3.7.0-dev || >= 3.8.0-dev || >= 3.9.0-dev || >= 4.0.0-dev || >=5.0.0-dev"
Expand All @@ -47,9 +44,6 @@
"@types/eslint": "^8.56.2",
"@types/semver": "^7.5.5",
"@types/strip-json-comments": "^3.0.0",
"@types/tar": "^6.1.9",
"@types/tmp": "^0.2.6",
"@types/which": "^3.0.3",
"typescript": "^5.3.3"
},
"engines": {
Expand Down
99 changes: 34 additions & 65 deletions packages/dtslint/src/checks.ts
Original file line number Diff line number Diff line change
@@ -1,20 +1,13 @@
import type * as attw from "@arethetypeswrong/core" with { "resolution-mode": "import" };
import * as header from "@definitelytyped/header-parser";
import { AllTypeScriptVersion } from "@definitelytyped/typescript-versions";
import { assertDefined, assertNever, deepEquals } from "@definitelytyped/utils";
import { execFileSync } from "child_process";
import fs, { readdirSync } from "fs";
import { dirname, join as joinPaths } from "path";
import { assertNever, createTgz, deepEquals, streamToBuffer } from "@definitelytyped/utils";
import fs from "fs";
import { join as joinPaths } from "path";
import { satisfies } from "semver";
import { pipeline } from "stream/promises";
import { CompilerOptions } from "typescript";
import which from "which";
import { createGunzip } from "zlib";
import { packageNameFromPath, readJson } from "./util";
import tmp = require("tmp");
import tar = require("tar");

tmp.setGracefulCleanup();
const tmpDir = tmp.dirSync().name;
const npmVersionExemptions = new Set(
fs.readFileSync(joinPaths(__dirname, "../expectedNpmVersionFailures.txt"), "utf-8").split(/\r?\n/),
);
Expand Down Expand Up @@ -165,51 +158,45 @@ export function checkTsconfig(dirPath: string, config: Tsconfig): string[] {
return errors;
}

export function runAreTheTypesWrong(
export async function runAreTheTypesWrong(
dirName: string,
dirPath: string,
implementationTarballPath: string,
implementationPackage: attw.Package,
configPath: string,
expectError: boolean,
): {
): Promise<{
warnings?: string[];
errors?: string[];
} {
}> {
let warnings: string[] | undefined;
let errors: string[] | undefined;
let result: {
status: "pass" | "fail" | "error";
output: string;
};
const attwPackageJsonPath = require.resolve("@arethetypeswrong/cli/package.json");
const attwBinPath = joinPaths(dirname(attwPackageJsonPath), readJson(attwPackageJsonPath).bin.attw);
const npmPath = which.sync("pnpm", { nothrow: true }) || which.sync("npm");
execFileSync(npmPath, ["pack"], {
cwd: dirPath,
stdio: "ignore",
env: { ...process.env, COREPACK_ENABLE_STRICT: "0" },

const tgz = createTgz(dirPath, (err) => {
throw new Error(`Error creating tarball for ${dirName}: ${err.stack ?? err.message}`);
});
const tarballName = assertDefined(readdirSync(dirPath).find((name) => name.endsWith(".tgz")));

const [attw, render, { getExitCode }] = await Promise.all([
import("@arethetypeswrong/core"),
import("@arethetypeswrong/cli/internal/render"),
import("@arethetypeswrong/cli/internal/getExitCode"),
]);
const config = JSON.parse(fs.readFileSync(configPath, "utf-8"));
const pkg = implementationPackage.mergedWithTypes(attw.createPackageFromTarballData(await streamToBuffer(tgz)));

try {
const output = execFileSync(
attwBinPath,
[implementationTarballPath, "--definitely-typed", tarballName, "--config-path", configPath],
{
cwd: dirPath,
stdio: "pipe",
encoding: "utf8",
},
);
result = { status: "pass", output };
} catch (err) {
const status = err && typeof err === "object" && "status" in err && err.status === 1 ? "fail" : "error";
const stdout =
err && typeof err === "object" && "stdout" in err && typeof err.stdout === "string" ? err.stdout : undefined;
const stderr =
err && typeof err === "object" && "stderr" in err && typeof err.stderr === "string" ? err.stderr : undefined;
result = { status, output: [stdout, stderr].filter(Boolean).join("\n") };
} finally {
fs.unlinkSync(joinPaths(dirPath, tarballName));
const checkResult = await attw.checkPackage(pkg);
if (!checkResult.types) {
throw new Error("No types found in synthesized attw package");
}
const output = await render.typed(checkResult, { format: "auto", ignoreRules: config.ignoreRules });
const status = getExitCode(checkResult, { ignoreRules: config.ignoreRules }) === 0 ? "pass" : "fail";
result = { status, output };
} catch (err: any) {
result = { status: "error", output: err.stack ?? err.message };
}

const { status, output } = result;
Expand Down Expand Up @@ -245,32 +232,25 @@ export function runAreTheTypesWrong(
return { warnings, errors };
}

export interface ImplementationPackageInfo {
packageName: string;
packageVersion: string;
tarballPath: string;
unpackedPath: string;
}

export async function checkNpmVersionAndGetMatchingImplementationPackage(
packageJson: header.Header,
packageDirectoryNameWithVersion: string,
): Promise<{
warnings?: string[];
errors?: string[];
implementationPackage?: ImplementationPackageInfo;
implementationPackage?: attw.Package;
}> {
let warnings: string[] | undefined;
let errors: string[] | undefined;
let hasNpmVersionMismatch = false;
let implementationPackage;
const attw = await import("@arethetypeswrong/core");
const typesPackageVersion = `${packageJson.libraryMajorVersion}.${packageJson.libraryMinorVersion}`;
const pkg = await tryPromise(
const packageId = await tryPromise(
attw.resolveImplementationPackageForTypesPackage(packageJson.name, `${typesPackageVersion}.9999`),
);
if (pkg) {
const { packageName, packageVersion, tarballUrl } = pkg;
if (packageId) {
const { packageName, packageVersion, tarballUrl } = packageId;
if (packageJson.nonNpm === true) {
(errors ??= []).push(
`Package ${packageJson.name} is marked as non-npm, but ${packageName} exists on npm. ` +
Expand All @@ -295,18 +275,7 @@ export async function checkNpmVersionAndGetMatchingImplementationPackage(
`that does not conflict with an existing npm package.`,
);
} else {
const tarballPath = joinPaths(tmpDir, `${packageName.replace(/\//g, "-")}-${packageVersion}.tgz`);
const unpackedPath = joinPaths(tmpDir, `${packageName}-${packageVersion}`);
await pipeline((await fetch(tarballUrl)).body!, fs.createWriteStream(tarballPath));
await fs.promises.mkdir(unpackedPath, { recursive: true });
await pipeline(fs.createReadStream(tarballPath), createGunzip(), tar.extract({ cwd: unpackedPath }));
const containerDir = (await fs.promises.readdir(unpackedPath))[0];
implementationPackage = {
packageName,
packageVersion,
tarballPath,
unpackedPath: joinPaths(unpackedPath, containerDir),
};
implementationPackage = await attw.createPackageFromTarballUrl(tarballUrl);
}
}
} else if (packageJson.nonNpm === "conflict") {
Expand Down
2 changes: 1 addition & 1 deletion packages/dtslint/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ async function runTests(
const failingPackages = ((await readJson(attwJson)) as any).failingPackages;
const dirName = dirPath.slice(dtRoot.length + "/types/".length);
const expectError = !!failingPackages?.includes(dirName);
const attwResult = runAreTheTypesWrong(dirName, dirPath, implementationPackage.tarballPath, attwJson, expectError);
const attwResult = await runAreTheTypesWrong(dirName, dirPath, implementationPackage, attwJson, expectError);
(warnings ??= []).push(...(attwResult.warnings ?? []));
(errors ??= []).push(...(attwResult.errors ?? []));
}
Expand Down
11 changes: 11 additions & 0 deletions packages/utils/src/io.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import { parseJson, withoutStart, sleep, tryParseJson, isObject } from "./miscel
import { FS, Dir, InMemoryFS } from "./fs";
import { assertDefined } from "./assertions";
import { LoggerWithErrors } from "./logging";
import { pipeline } from "stream/promises";

export async function readFile(path: string): Promise<string> {
const res = await fs.promises.readFile(path, { encoding: "utf8" });
Expand Down Expand Up @@ -235,6 +236,16 @@ export function downloadAndExtractFile(url: string, log: LoggerWithErrors): Prom
});
}

export async function streamToBuffer(stream: NodeJS.ReadableStream): Promise<Buffer> {
const chunks: Buffer[] = [];
await pipeline(stream, async (source) => {
for await (const chunk of source) {
chunks.push(Buffer.from(chunk));
}
});
return Buffer.concat(chunks);
}

export function gzip(input: NodeJS.ReadableStream): NodeJS.ReadableStream {
return input.pipe(zlib.createGzip());
}
Expand Down
46 changes: 5 additions & 41 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 31de5d3

Please sign in to comment.