From 7617eed3ca90dc7d48e908fbc23cdcf6c531a6e7 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Thu, 4 Jan 2024 14:16:28 +0000 Subject: [PATCH] fetch_package initial commit --- fetch_package/CHANGELOG.md | 2 + fetch_package/LICENSE | 202 +++++++++++++++++++++++ fetch_package/README.md | 33 ++++ fetch_package/bin/fetch_package.dart | 230 +++++++++++++++++++++++++++ fetch_package/mono_pkg.yaml | 8 + fetch_package/pubspec.yaml | 20 +++ 6 files changed, 495 insertions(+) create mode 100644 fetch_package/CHANGELOG.md create mode 100644 fetch_package/LICENSE create mode 100644 fetch_package/README.md create mode 100644 fetch_package/bin/fetch_package.dart create mode 100644 fetch_package/mono_pkg.yaml create mode 100644 fetch_package/pubspec.yaml diff --git a/fetch_package/CHANGELOG.md b/fetch_package/CHANGELOG.md new file mode 100644 index 00000000..15a45ab4 --- /dev/null +++ b/fetch_package/CHANGELOG.md @@ -0,0 +1,2 @@ +## v0.1.0 + * Initial release. diff --git a/fetch_package/LICENSE b/fetch_package/LICENSE new file mode 100644 index 00000000..7a4a3ea2 --- /dev/null +++ b/fetch_package/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/fetch_package/README.md b/fetch_package/README.md new file mode 100644 index 00000000..c4583dae --- /dev/null +++ b/fetch_package/README.md @@ -0,0 +1,33 @@ +# fetch_package + +A utility for easily downloading and extracting a package archive from pub.dev +or other repository. + +Useful for reviewing the source-code of a pub package or for easily running +examples. + +## Usage + +This package is intended for global installation. Run this to install the latest version: + +``` +> dart pub global activate fetch_package # Only on first run +``` + +If you want to inspect the source code of the latest version of this package: + +``` +> dart pub global run fetch_package fetch_package +``` + +To inspect the source of a specific version: + +``` +> dart pub global run fetch_package fetch_package 0.1.0 +``` + +You can also download the archive of the package: + +``` +> dart pub global run fetch_package fetch_package --archive +``` \ No newline at end of file diff --git a/fetch_package/bin/fetch_package.dart b/fetch_package/bin/fetch_package.dart new file mode 100644 index 00000000..067be674 --- /dev/null +++ b/fetch_package/bin/fetch_package.dart @@ -0,0 +1,230 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'dart:convert'; +import 'dart:io'; + +import 'package:collection/collection.dart'; +import 'package:http/http.dart' as http; +import 'package:args/args.dart'; +import 'package:path/path.dart' as path; +import 'package:retry/retry.dart'; +import 'package:tar/tar.dart'; + +Future main(List args) async { + final client = http.Client(); + ArgParser argParser = ArgParser(); + argParser.addFlag('archive', + help: 'Download the package as a tar.gz archive', negatable: false); + argParser.addOption('destination', + help: 'Download the package in this dir', defaultsTo: '.'); + argParser.addOption('repository', + help: 'The package repository to download from', + defaultsTo: 'https://pub.dev'); + argParser.addFlag('help', help: 'Show this help text', negatable: false); + final usage = 'Usage: fetch_package [] []\n' + '${argParser.usage}'; + + final ArgResults argResults; + try { + argResults = argParser.parse(args); + } on FormatException catch (e) { + fail('${e.message}\n\n' + '$usage'); + } + if (argResults['help'] || + !(argResults.rest.length == 1 || argResults.rest.length == 2)) { + fail('$usage'); + } + final package = argResults.rest[0]; + var version = argResults.rest.length == 2 ? args[1] : null; + final destination = argResults['destination'] as String; + if (!Directory(destination).existsSync()) { + fail('Destination directory `$destination` doesn\'t exist'); + } + final versionListing = await retry(() async { + final versionListingText = await client.read( + Uri.parse(argResults['repository']).resolve('api/packages/$package')); + return jsonDecode(versionListingText) as Map; + }, retryIf: (e) => e is IOException || e is http.ClientException); + if (version == null) { + version = versionListing['latest']?['version']; + if (version == null) { + fail( + 'Version listing did not give a latest version. Please specify an exact version'); + } + print('Choosing the latest version: $version'); + } + + final archiveUrl = (versionListing['versions'] as List) + .firstWhereOrNull((v) => v['version'] == version)?['archive_url'] + as String?; + if (archiveUrl == null) { + final versions = versionListing['versions'].map((v) => v['version']); + fail('Version $version not found.\n\n' + 'Available versions: ${versions.join(', ')}.'); + } + final bytes = await retry(() async { + return client.readBytes(Uri.parse(archiveUrl)); + }, retryIf: (e) => e is IOException || e is http.ClientException); + if (argResults['archive']) { + final archivePath = path.join(destination, '$package-$version.tar.gz'); + if (File(archivePath).existsSync()) { + fail('$archivePath already exists'); + } + File(archivePath).writeAsBytesSync(bytes); + print('Downloaded package archive to $archivePath.'); + } else { + final dir = path.join(destination, package); + if (Directory(dir).existsSync()) { + fail('$dir already exists.'); + } + Directory(dir).createSync(recursive: true); + try { + extractTarGz(bytes, dir); + print('Extracted $package-$version to $dir.'); + } on Exception { + Directory(dir).deleteSync(); + } + } + client.close(); +} + +/// Extracts a `.tar.gz` file from [bytes] to [destination]. +Future extractTarGz(List bytes, String destination) async { + destination = path.absolute(destination); + final reader = TarReader(Stream.fromIterable([gzip.decode(bytes)])); + final paths = {}; + while (await reader.moveNext()) { + final entry = reader.current; + + final filePath = path.joinAll([ + destination, + // Tar file names always use forward slashes + ...path.posix.split(entry.name), + ]); + if (!paths.add(filePath)) { + // The tar file contained the same entry twice. Assume it is broken. + await reader.cancel(); + throw FormatException('Tar file contained duplicate path ${entry.name}'); + } + + if (!path.isWithin(destination, filePath)) { + // The tar contains entries that would be written outside of the + // destination. That doesn't happen by accident, assume that the tar file + // is malicious. + await reader.cancel(); + throw FormatException('Invalid tar entry: ${entry.name}'); + } + + final parentDirectory = path.dirname(filePath); + + bool checkValidTarget(String linkTarget) { + final isValid = path.isWithin(destination, linkTarget); + if (!isValid) { + print('Skipping ${entry.name}: Invalid link target'); + } + + return isValid; + } + + switch (entry.type) { + case TypeFlag.dir: + Directory(filePath).createSync(recursive: true); + break; + case TypeFlag.reg: + case TypeFlag.regA: + // Regular file + Directory(parentDirectory).createSync(recursive: true); + await createFileFromStream(entry.contents, filePath); + + if (Platform.isLinux || Platform.isMacOS) { + // Apply executable bits from tar header, but don't change r/w bits + // from the default + final mode = _defaultMode | (entry.header.mode & _executableMask); + + if (mode != _defaultMode) { + _chmod(mode, filePath); + } + } + break; + case TypeFlag.symlink: + // Link to another file in this tar, relative from this entry. + final resolvedTarget = path.joinAll( + [parentDirectory, ...path.posix.split(entry.header.linkName!)], + ); + if (!checkValidTarget(resolvedTarget)) { + // Don't allow links to files outside of this tar. + break; + } + + Directory(parentDirectory).createSync(recursive: true); + Link(filePath) + .createSync(path.relative(resolvedTarget, from: parentDirectory)); + break; + case TypeFlag.link: + // We generate hardlinks as symlinks too, but their linkName is relative + // to the root of the tar file (unlike symlink entries, whose linkName + // is relative to the entry itself). + final fromDestination = path.join(destination, entry.header.linkName); + if (!checkValidTarget(fromDestination)) { + break; // Link points outside of the tar file. + } + + final fromFile = path.relative(fromDestination, from: parentDirectory); + Directory(parentDirectory).createSync(recursive: true); + Link(filePath).createSync(fromFile); + break; + default: + // Only extract files + continue; + } + } +} + +/// Writes [stream] to a new file at path [file]. +/// +/// Replaces any file already at that path. Completes when the file is done +/// being written. +Future createFileFromStream(Stream> stream, String file) async { + deleteIfLink(file); + await stream.pipe(File(file).openWrite()); +} + +/// Deletes [file] if it's a symlink. +/// +/// The [File] class overwrites the symlink targets when writing to a file, +/// which is never what we want, so this delete the symlink first if necessary. +void deleteIfLink(String file) { + if (File(file).statSync().type != FileSystemEntityType.link) return; + Link(file).deleteSync(); +} + +/// The assumed default file mode on Linux and macOS +const _defaultMode = 420; // 644₈ + +/// Mask for executable bits in file modes. +const _executableMask = 0x49; // 001 001 001 + +void _chmod(int mode, String file) { + final result = Process.runSync('chmod', [mode.toRadixString(8), file]); + if (result.exitCode != 0) { + fail('Could not change mode of $file ${result.stdout} ${result.stderr}'); + } +} + +Never fail(String message) { + stderr.writeln(message); + exit(-1); +} diff --git a/fetch_package/mono_pkg.yaml b/fetch_package/mono_pkg.yaml new file mode 100644 index 00000000..7fd029b8 --- /dev/null +++ b/fetch_package/mono_pkg.yaml @@ -0,0 +1,8 @@ +sdk: + - stable +stages: + - analyze: + - analyze + - format + - tests: + - test diff --git a/fetch_package/pubspec.yaml b/fetch_package/pubspec.yaml new file mode 100644 index 00000000..e4b6c583 --- /dev/null +++ b/fetch_package/pubspec.yaml @@ -0,0 +1,20 @@ +name: fetch_package +version: 0.1.0-dev +environment: + sdk: ^3.0.0 +description: >- + Utility for downloading packages from pub.dev +homepage: https://github.com/google/dart-neats/tree/master/fetch_package +repository: https://github.com/google/dart-neats.git +issue_tracker: https://github.com/google/dart-neats/labels/pkg:retry +dependencies: + args: ^2.4.2 + collection: ^1.18.0 + http: ^1.1.2 + path: ^1.9.0 + retry: ^3.1.2 + tar: ^1.0.3 +dev_dependencies: + test: +topics: + - packages