From 9ee655960ada8b5027b16cc1b3e2c3cddad9e006 Mon Sep 17 00:00:00 2001 From: Ian2020 Date: Fri, 3 Nov 2023 11:23:53 +0000 Subject: [PATCH] Fuller test and support for git refs --- .../providers/__init__.py | 9 ++ node/flatpak_node_generator/providers/npm.py | 107 ++++++++---------- .../data/packages/git/package-lock.v1.json | 33 ++++++ .../data/packages/git/package-lock.v2.json | 78 +++++++++++++ .../data/packages/git/package-lock.v3.json | 49 ++++++++ node/tests/data/packages/git/package.json | 10 ++ node/tests/data/packages/git/yarn.lock | 37 ++++++ .../packages/minimal-git/package-lock.v3.json | 2 +- node/tests/test_providers.py | 21 +++- 9 files changed, 280 insertions(+), 66 deletions(-) create mode 100644 node/tests/data/packages/git/package-lock.v1.json create mode 100644 node/tests/data/packages/git/package-lock.v2.json create mode 100644 node/tests/data/packages/git/package-lock.v3.json create mode 100644 node/tests/data/packages/git/package.json create mode 100644 node/tests/data/packages/git/yarn.lock diff --git a/node/flatpak_node_generator/providers/__init__.py b/node/flatpak_node_generator/providers/__init__.py index ca6f5685..93ef6c4b 100644 --- a/node/flatpak_node_generator/providers/__init__.py +++ b/node/flatpak_node_generator/providers/__init__.py @@ -16,6 +16,7 @@ 'git': {}, 'git+http': {'scheme': 'http'}, 'git+https': {'scheme': 'https'}, + 'git+ssh': {'scheme': 'https'}, } @@ -33,6 +34,14 @@ def parse_git_source(self, version: str, from_: Optional[str] = None) -> GitSour if not new_url.netloc: path = new_url.path.split('/') new_url = new_url._replace(netloc=path[0], path='/'.join(path[1:])) + # Replace https://git@github.com:ianstormtaylor/to-camel-case.git + # wth https://git@github.com/ianstormtaylor/to-camel-case.git + # for git+ssh URLs + if ':' in new_url.netloc: + netloc_split = new_url.netloc.split(':') + new_url = new_url._replace( + netloc=netloc_split[0], path=f'/{netloc_split[1]}{new_url.path}' + ) return GitSource( original=original_url.geturl(), diff --git a/node/flatpak_node_generator/providers/npm.py b/node/flatpak_node_generator/providers/npm.py index a60cb08e..222d2e04 100644 --- a/node/flatpak_node_generator/providers/npm.py +++ b/node/flatpak_node_generator/providers/npm.py @@ -130,10 +130,7 @@ def _process_packages_v2( integrity=integrity, resolved=info['resolved'] ) elif resolved_url.scheme.startswith('git+'): - raise NotImplementedError( - 'Git sources in lockfile v2 format are not supported yet' - f' (package {install_path} in {lockfile})' - ) + source = self.parse_git_source(info['resolved']) else: raise NotImplementedError( f"Don't know how to handle package {install_path} in {lockfile}" @@ -425,72 +422,58 @@ def _finalize(self) -> None: ) if self.git_sources: - # Generate jq scripts to patch the package*.json files. - scripts = { - 'package.json': r""" - walk( - if type == "object" - then - to_entries | map( - if (.value | type == "string") and $data[.value] - then .value = "git+file:\($buildroot)/\($data[.value])" - else . - end - ) | from_entries - else . - end - ) - """, - 'package-lock.json': r""" - walk( - if type == "object" and (.version | type == "string") and $data[.version] - then - .version = "git+file:\($buildroot)/\($data[.version])" - else . - end - ) - """, - } + # Generate jq script to patch the package.json file + script = r""" + walk( + if type == "object" + then + to_entries | map( + if (.value | type == "string") and $data[.value] + then .value = "git+file:\($buildroot)/\($data[.value])" + else . + end + ) | from_entries + else . + end + ) + """ for lockfile, sources in self.git_sources.items(): prefix = self.relative_lockfile_dir(lockfile) - data: Dict[str, Dict[str, str]] = { - 'package.json': {}, - 'package-lock.json': {}, - } + data: Dict[str, str] = {} for path, source in sources.items(): GIT_URL_PREFIX = 'git+' new_version = f'{path}#{source.commit}' - assert source.from_ is not None - data['package.json'][source.from_] = new_version - data['package-lock.json'][source.original] = new_version - - if source.from_.startswith(GIT_URL_PREFIX): - data['package.json'][ - source.from_[len(GIT_URL_PREFIX) :] - ] = new_version - - if source.original.startswith(GIT_URL_PREFIX): - data['package-lock.json'][ - source.original[len(GIT_URL_PREFIX) :] - ] = new_version - - for filename, script in scripts.items(): - target = Path('$FLATPAK_BUILDER_BUILDDIR') / prefix / filename - script = ( - textwrap.dedent(script.lstrip('\n')).strip().replace('\n', '') - ) - json_data = json.dumps(data[filename]) - patch_commands[lockfile].append( - 'jq' - ' --arg buildroot "$FLATPAK_BUILDER_BUILDDIR"' - f' --argjson data {shlex.quote(json_data)}' - f' {shlex.quote(script)} {target}' - f' > {target}.new' - ) - patch_commands[lockfile].append(f'mv {target}{{.new,}}') + targets = [] + source_url = source.from_ or source.original + + if ( + source_url.startswith(GIT_URL_PREFIX + 'ssh') + and ':' not in urllib.parse.urlparse(source_url).netloc + ): + source_url = re.sub(r'(://[^/]+)/', r'\1:', source_url) + elif source_url.startswith(GIT_URL_PREFIX): + targets.append(source_url[len(GIT_URL_PREFIX) :]) + + targets.append(source_url) + for t in targets: + data[t] = new_version + data[t.replace('#' + source.commit, '')] = new_version + + filename = 'package.json' + target = Path('$FLATPAK_BUILDER_BUILDDIR') / prefix / filename + script = textwrap.dedent(script.lstrip('\n')).strip().replace('\n', '') + json_data = json.dumps(data) + patch_commands[lockfile].append( + 'jq' + ' --arg buildroot "$FLATPAK_BUILDER_BUILDDIR"' + f' --argjson data {shlex.quote(json_data)}' + f' {shlex.quote(script)} {target}' + f' > {target}.new' + ) + patch_commands[lockfile].append(f'mv {target}{{.new,}}') patch_all_commands: List[str] = [] for lockfile in self.all_lockfiles: diff --git a/node/tests/data/packages/git/package-lock.v1.json b/node/tests/data/packages/git/package-lock.v1.json new file mode 100644 index 00000000..4905b6d0 --- /dev/null +++ b/node/tests/data/packages/git/package-lock.v1.json @@ -0,0 +1,33 @@ +{ + "name": "@flatpak-node-generator-tests/git", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "to-camel-case": { + "version": "git+ssh://git@github.com/ianstormtaylor/to-camel-case.git#00a20429b600ddb6e4f8ff5b17c52914f40fe67d", + "from": "git+ssh://git@github.com/ianstormtaylor/to-camel-case.git", + "requires": { + "to-space-case": "^1.0.0" + } + }, + "to-capital-case": { + "version": "git+https://git@github.com/ianstormtaylor/to-capital-case.git#b82f61e00e099b01514e25177bb2d56d0f64b157", + "from": "git+https://git@github.com/ianstormtaylor/to-capital-case.git", + "requires": { + "to-space-case": "^1.0.0" + } + }, + "to-no-case": { + "version": "git+https://git@github.com/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90", + "from": "git+https://git@github.com/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90" + }, + "to-space-case": { + "version": "git+ssh://git@github.com/ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640", + "from": "git+ssh://git@github.com/ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640", + "requires": { + "to-no-case": "^1.0.0" + } + } + } +} diff --git a/node/tests/data/packages/git/package-lock.v2.json b/node/tests/data/packages/git/package-lock.v2.json new file mode 100644 index 00000000..b19bb2ec --- /dev/null +++ b/node/tests/data/packages/git/package-lock.v2.json @@ -0,0 +1,78 @@ +{ + "name": "@flatpak-node-generator-tests/git", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "@flatpak-node-generator-tests/git", + "version": "1.0.0", + "dependencies": { + "to-camel-case": "git+ssh://git@github.com:ianstormtaylor/to-camel-case.git", + "to-capital-case": "git+https://git@github.com/ianstormtaylor/to-capital-case.git", + "to-no-case": "git+https://git@github.com/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90", + "to-space-case": "git+ssh://git@github.com:ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640" + } + }, + "node_modules/to-camel-case": { + "version": "1.0.0", + "resolved": "git+ssh://git@github.com/ianstormtaylor/to-camel-case.git#00a20429b600ddb6e4f8ff5b17c52914f40fe67d", + "license": "MIT", + "dependencies": { + "to-space-case": "^1.0.0" + } + }, + "node_modules/to-capital-case": { + "version": "1.0.0", + "resolved": "git+https://git@github.com/ianstormtaylor/to-capital-case.git#b82f61e00e099b01514e25177bb2d56d0f64b157", + "license": "MIT", + "dependencies": { + "to-space-case": "^1.0.0" + } + }, + "node_modules/to-no-case": { + "version": "1.0.0", + "resolved": "git+https://git@github.com/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90", + "integrity": "sha512-E6uMYyFvmgQWszuxWtCz0NN+BgdVvQjVCU6Nnq0BSxi8q/QKRCBs14pGUWbPQghmpjJbdkKgjICDWihePrJGOQ==", + "license": "MIT" + }, + "node_modules/to-space-case": { + "version": "1.0.0", + "resolved": "git+ssh://git@github.com/ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640", + "integrity": "sha512-i7ZtWv1WzR6xrLpd5wpSU/RZxoEDaoD3uLU5pDgXEZfUZJh0QpPtjJdLf7i+nr24Z80ls8qYCfBeFpZk6RzXKQ==", + "license": "MIT", + "dependencies": { + "to-no-case": "^1.0.0" + } + } + }, + "dependencies": { + "to-camel-case": { + "version": "git+ssh://git@github.com/ianstormtaylor/to-camel-case.git#00a20429b600ddb6e4f8ff5b17c52914f40fe67d", + "from": "to-camel-case@git+ssh://git@github.com:ianstormtaylor/to-camel-case.git", + "requires": { + "to-space-case": "^1.0.0" + } + }, + "to-capital-case": { + "version": "git+https://git@github.com/ianstormtaylor/to-capital-case.git#b82f61e00e099b01514e25177bb2d56d0f64b157", + "from": "to-capital-case@git+https://git@github.com/ianstormtaylor/to-capital-case.git", + "requires": { + "to-space-case": "^1.0.0" + } + }, + "to-no-case": { + "version": "git+https://git@github.com/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90", + "integrity": "sha512-E6uMYyFvmgQWszuxWtCz0NN+BgdVvQjVCU6Nnq0BSxi8q/QKRCBs14pGUWbPQghmpjJbdkKgjICDWihePrJGOQ==", + "from": "to-no-case@git+https://git@github.com/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90" + }, + "to-space-case": { + "version": "git+ssh://git@github.com/ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640", + "integrity": "sha512-i7ZtWv1WzR6xrLpd5wpSU/RZxoEDaoD3uLU5pDgXEZfUZJh0QpPtjJdLf7i+nr24Z80ls8qYCfBeFpZk6RzXKQ==", + "from": "to-space-case@git+ssh://git@github.com:ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640", + "requires": { + "to-no-case": "^1.0.0" + } + } + } +} diff --git a/node/tests/data/packages/git/package-lock.v3.json b/node/tests/data/packages/git/package-lock.v3.json new file mode 100644 index 00000000..e6d406d1 --- /dev/null +++ b/node/tests/data/packages/git/package-lock.v3.json @@ -0,0 +1,49 @@ +{ + "name": "@flatpak-node-generator-tests/git", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@flatpak-node-generator-tests/git", + "version": "1.0.0", + "dependencies": { + "to-camel-case": "git+ssh://git@github.com:ianstormtaylor/to-camel-case.git", + "to-capital-case": "git+https://git@github.com/ianstormtaylor/to-capital-case.git", + "to-no-case": "git+https://git@github.com/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90", + "to-space-case": "git+ssh://git@github.com:ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640" + } + }, + "node_modules/to-camel-case": { + "version": "1.0.0", + "resolved": "git+ssh://git@github.com/ianstormtaylor/to-camel-case.git#00a20429b600ddb6e4f8ff5b17c52914f40fe67d", + "license": "MIT", + "dependencies": { + "to-space-case": "^1.0.0" + } + }, + "node_modules/to-capital-case": { + "version": "1.0.0", + "resolved": "git+https://git@github.com/ianstormtaylor/to-capital-case.git#b82f61e00e099b01514e25177bb2d56d0f64b157", + "license": "MIT", + "dependencies": { + "to-space-case": "^1.0.0" + } + }, + "node_modules/to-no-case": { + "version": "1.0.0", + "resolved": "git+https://git@github.com/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90", + "integrity": "sha512-E6uMYyFvmgQWszuxWtCz0NN+BgdVvQjVCU6Nnq0BSxi8q/QKRCBs14pGUWbPQghmpjJbdkKgjICDWihePrJGOQ==", + "license": "MIT" + }, + "node_modules/to-space-case": { + "version": "1.0.0", + "resolved": "git+ssh://git@github.com/ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640", + "integrity": "sha512-i7ZtWv1WzR6xrLpd5wpSU/RZxoEDaoD3uLU5pDgXEZfUZJh0QpPtjJdLf7i+nr24Z80ls8qYCfBeFpZk6RzXKQ==", + "license": "MIT", + "dependencies": { + "to-no-case": "^1.0.0" + } + } + } +} diff --git a/node/tests/data/packages/git/package.json b/node/tests/data/packages/git/package.json new file mode 100644 index 00000000..3ba96218 --- /dev/null +++ b/node/tests/data/packages/git/package.json @@ -0,0 +1,10 @@ +{ + "name": "@flatpak-node-generator-tests/git", + "version": "1.0.0", + "dependencies": { + "to-camel-case": "git+ssh://git@github.com:ianstormtaylor/to-camel-case.git", + "to-capital-case": "git+https://git@github.com/ianstormtaylor/to-capital-case.git", + "to-no-case": "git+https://git@github.com/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90", + "to-space-case": "git+ssh://git@github.com:ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640" + } +} diff --git a/node/tests/data/packages/git/yarn.lock b/node/tests/data/packages/git/yarn.lock new file mode 100644 index 00000000..b2640c97 --- /dev/null +++ b/node/tests/data/packages/git/yarn.lock @@ -0,0 +1,37 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"to-camel-case@git+ssh://git@github.com:ianstormtaylor/to-camel-case.git": + version "1.0.0" + resolved "git+ssh://git@github.com:ianstormtaylor/to-camel-case.git#00a20429b600ddb6e4f8ff5b17c52914f40fe67d" + dependencies: + to-space-case "^1.0.0" + +"to-capital-case@git+https://git@github.com/ianstormtaylor/to-capital-case.git": + version "1.0.0" + resolved "git+https://git@github.com/ianstormtaylor/to-capital-case.git#b82f61e00e099b01514e25177bb2d56d0f64b157" + dependencies: + to-space-case "^1.0.0" + +to-no-case@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/to-no-case/-/to-no-case-1.0.2.tgz#c722907164ef6b178132c8e69930212d1b4aa16a" + integrity sha512-Z3g735FxuZY8rodxV4gH7LxClE4H0hTIyHNIHdk+vpQxjLm0cwnKXq/OFVZ76SOQmto7txVcwSCwkU5kqp+FKg== + +"to-no-case@git+https://git@github.com/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90": + version "1.0.0" + resolved "git+https://git@github.com/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90" + +to-space-case@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/to-space-case/-/to-space-case-1.0.0.tgz#b052daafb1b2b29dc770cea0163e5ec0ebc9fc17" + integrity sha512-rLdvwXZ39VOn1IxGL3V6ZstoTbwLRckQmn/U8ZDLuWwIXNpuZDhQ3AiRUlhTbOXFVE9C+dR51wM0CBDhk31VcA== + dependencies: + to-no-case "^1.0.0" + +"to-space-case@git+ssh://git@github.com:ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640": + version "1.0.0" + resolved "git+ssh://git@github.com:ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640" + dependencies: + to-no-case "^1.0.0" diff --git a/node/tests/data/packages/minimal-git/package-lock.v3.json b/node/tests/data/packages/minimal-git/package-lock.v3.json index 6e06d6ff..eed800cb 100644 --- a/node/tests/data/packages/minimal-git/package-lock.v3.json +++ b/node/tests/data/packages/minimal-git/package-lock.v3.json @@ -13,7 +13,7 @@ }, "node_modules/nop": { "version": "1.0.0", - "resolved": "git+ssh://git@github.com/supershabam/nop.git#f110e75f62cfe3bf4468ac3b74e3dc72ab9ae4bf", + "resolved": "git+https://git@github.com/supershabam/nop.git#f110e75f62cfe3bf4468ac3b74e3dc72ab9ae4bf", "integrity": "sha512-xCwdA7C4QIORvTMytKHMlkEN6axJGimR0gv5vgjKpEKRvQrPOwhjJnrZEcd5g0LP+7IY38+TY7MP59HRY6gcwA==", "license": "MIT" } diff --git a/node/tests/test_providers.py b/node/tests/test_providers.py index 3d29c815..14efc514 100644 --- a/node/tests/test_providers.py +++ b/node/tests/test_providers.py @@ -13,9 +13,6 @@ async def test_minimal_git( provider_factory_spec: ProviderFactorySpec, node_version: int, ) -> None: - if node_version >= 18: - pytest.xfail(reason='Git sources not yet supported for lockfile v2 syntax') - with ManifestGenerator() as gen: await provider_factory_spec.generate_modules('minimal-git', gen, node_version) @@ -29,6 +26,24 @@ async def test_minimal_git( ) +async def test_git( + flatpak_builder: FlatpakBuilder, + provider_factory_spec: ProviderFactorySpec, + node_version: int, +) -> None: + with ManifestGenerator() as gen: + await provider_factory_spec.generate_modules('git', gen, node_version) + + flatpak_builder.build( + sources=itertools.chain(gen.ordered_sources()), + commands=[ + provider_factory_spec.install_command, + """node -e 'require("to-camel-case");require("to-capital-case");require("to-no-case");require("to-space-case");'""", + ], + use_node=node_version, + ) + + async def test_local( flatpak_builder: FlatpakBuilder, provider_factory_spec: ProviderFactorySpec,