Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[node] support git sources in lockfile v2 format #382

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
+385 −66
Open

[node] support git sources in lockfile v2 format #382

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9e4677e
node: support git urls for lockfile v2
Ian2020 Nov 3, 2023
f28defd
node: support shortcut git urls for lockfile v2
Ian2020 Nov 16, 2023
4365605
Fix typo in comment
Ian2020 Aug 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions node/flatpak_node_generator/providers/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
'git': {},
'git+http': {'scheme': 'http'},
'git+https': {'scheme': 'https'},
'git+ssh': {'scheme': 'https'},
}


Expand All @@ -33,6 +34,14 @@ def parse_git_source(self, version: str, from_: Optional[str] = None) -> GitSour
if not new_url.netloc:
path = new_url.path.split('/')
new_url = new_url._replace(netloc=path[0], path='/'.join(path[1:]))
# Replace https://[email protected]:ianstormtaylor/to-camel-case.git
# with https://[email protected]/ianstormtaylor/to-camel-case.git
# for git+ssh URLs
if ':' in new_url.netloc:
netloc_split = new_url.netloc.split(':')
new_url = new_url._replace(
netloc=netloc_split[0], path=f'/{netloc_split[1]}{new_url.path}'
)

return GitSource(
original=original_url.geturl(),
Expand Down
128 changes: 66 additions & 62 deletions node/flatpak_node_generator/providers/npm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,10 +130,7 @@ def _process_packages_v2(
integrity=integrity, resolved=info['resolved']
)
elif resolved_url.scheme.startswith('git+'):
raise NotImplementedError(
'Git sources in lockfile v2 format are not supported yet'
f' (package {install_path} in {lockfile})'
)
source = self.parse_git_source(info['resolved'])
else:
raise NotImplementedError(
f"Don't know how to handle package {install_path} in {lockfile}"
Expand Down Expand Up @@ -425,72 +422,79 @@ def _finalize(self) -> None:
)

if self.git_sources:
# Generate jq scripts to patch the package*.json files.
scripts = {
'package.json': r"""
walk(
if type == "object"
then
to_entries | map(
if (.value | type == "string") and $data[.value]
then .value = "git+file:\($buildroot)/\($data[.value])"
else .
end
) | from_entries
else .
end
)
""",
'package-lock.json': r"""
walk(
if type == "object" and (.version | type == "string") and $data[.version]
then
.version = "git+file:\($buildroot)/\($data[.version])"
else .
end
)
""",
}
# Generate jq script to patch the package.json file
script = r"""
walk(
if type == "object"
then
to_entries | map(
if (.value | type == "string") and $data[.value]
then .value = "git+file:\($buildroot)/\($data[.value])"
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Using this fork to update the dependencies for the Stretchly Flatpak resulted in the following error:

npm error code 128
npm error An unknown git error occurred
npm error command git --no-replace-objects ls-remote file:/var/home/jordan/Projects/net.hovancik.Stretchly/.flatpak-builder/build/stretchly-14/flatpak-node/git-packages/dbus-final-3e43f60d80bbcdf0dfa0b59b838097d6af4d17ba
npm error ssh: Could not resolve hostname file: Temporary failure in name resolution
npm error fatal: Could not read from remote repository.
npm error
npm error Please make sure you have the correct access rights
npm error and the repository exists.

I was able to fix this by using the syntax file:// instead of just file:.
This matches the documented syntax for Git URL's.

Suggested change
then .value = "git+file:\($buildroot)/\($data[.value])"
then .value = "git+file://\($buildroot)/\($data[.value])"

else .
end
) | from_entries
else .
end
)
"""

for lockfile, sources in self.git_sources.items():
prefix = self.relative_lockfile_dir(lockfile)
data: Dict[str, Dict[str, str]] = {
'package.json': {},
'package-lock.json': {},
}
data: Dict[str, str] = {}

for path, source in sources.items():
GIT_URL_PREFIX = 'git+'

new_version = f'{path}#{source.commit}'
assert source.from_ is not None
data['package.json'][source.from_] = new_version
data['package-lock.json'][source.original] = new_version

if source.from_.startswith(GIT_URL_PREFIX):
data['package.json'][
source.from_[len(GIT_URL_PREFIX) :]
] = new_version

if source.original.startswith(GIT_URL_PREFIX):
data['package-lock.json'][
source.original[len(GIT_URL_PREFIX) :]
] = new_version

for filename, script in scripts.items():
target = Path('$FLATPAK_BUILDER_BUILDDIR') / prefix / filename
script = (
textwrap.dedent(script.lstrip('\n')).strip().replace('\n', '')
)
json_data = json.dumps(data[filename])
patch_commands[lockfile].append(
'jq'
' --arg buildroot "$FLATPAK_BUILDER_BUILDDIR"'
f' --argjson data {shlex.quote(json_data)}'
f' {shlex.quote(script)} {target}'
f' > {target}.new'
)
patch_commands[lockfile].append(f'mv {target}{{.new,}}')
targets = []
source_url = urllib.parse.urlparse(source.from_ or source.original)

# https://github.com/npm/hosted-git-info
hosted_git = [
('@github.com', 'github'),
('@bitbucket.org', 'bitbucket'),
('@gitlab.com', 'gitlab'),
('@gist.github.com', 'gist'),
('@git.sr.ht', 'sourcehut'),
]
for domain, shortcut in hosted_git:
if domain in source_url.netloc.lower():
targets.append(
f"{shortcut}:{source_url.path[1:].replace('.git', '')}"
f'#{source_url.fragment}'
)
break

if (
source_url.scheme.startswith(GIT_URL_PREFIX + 'ssh')
and ':' not in source_url.netloc
):
path_match = re.compile(r'^/([^/]+)(.*)').match(source_url.path)
if path_match:
parent, child = path_match.groups()
source_url = source_url._replace(
netloc=f'{source_url.netloc}:{parent}', path=child
)
elif source_url.scheme.startswith(GIT_URL_PREFIX):
targets.append(source_url.geturl()[len(GIT_URL_PREFIX) :])

targets.append(source_url.geturl())
for t in targets:
data[t] = new_version
data[t.replace('#' + source.commit, '')] = new_version

filename = 'package.json'
target = Path('$FLATPAK_BUILDER_BUILDDIR') / prefix / filename
script = textwrap.dedent(script.lstrip('\n')).strip().replace('\n', '')
json_data = json.dumps(data)
patch_commands[lockfile].append(
'jq'
' --arg buildroot "$FLATPAK_BUILDER_BUILDDIR"'
f' --argjson data {shlex.quote(json_data)}'
f' {shlex.quote(script)} {target}'
f' > {target}.new'
)
patch_commands[lockfile].append(f'mv {target}{{.new,}}')

patch_all_commands: List[str] = []
for lockfile in self.all_lockfiles:
Expand Down
45 changes: 45 additions & 0 deletions node/tests/data/packages/git/package-lock.v1.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
{
"name": "@flatpak-node-generator-tests/git",
"version": "1.0.0",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
"is-empty-object": {
"version": "git+ssh://[email protected]/gummesson/is-empty-object.git#7b50c8eb4e14135631f7c94e01c0c8a36e5d75f8",
"from": "is-empty-object@github:gummesson/is-empty-object#7b50c8eb4e14135631f7c94e01c0c8a36e5d75f8"
},
"is-number": {
"version": "git+ssh://[email protected]/jonschlinkert/is-number.git#98e8ff1da1a89f93d1397a24d7413ed15421c139",
"from": "is-number@github:jonschlinkert/is-number"
},
"person-lib": {
"version": "git+ssh://[email protected]/volodymyrkr/person-lib.git#752fd1828b1eb3a9635bf725ae5e1704a375e524",
"from": "person-lib@gitlab:volodymyrkr/person-lib"
},
"to-camel-case": {
"version": "git+ssh://[email protected]/ianstormtaylor/to-camel-case.git#00a20429b600ddb6e4f8ff5b17c52914f40fe67d",
"from": "git+ssh://[email protected]/ianstormtaylor/to-camel-case.git",
"requires": {
"to-space-case": "^1.0.0"
}
},
"to-capital-case": {
"version": "git+https://[email protected]/ianstormtaylor/to-capital-case.git#b82f61e00e099b01514e25177bb2d56d0f64b157",
"from": "git+https://[email protected]/ianstormtaylor/to-capital-case.git",
"requires": {
"to-space-case": "^1.0.0"
}
},
"to-no-case": {
"version": "git+https://[email protected]/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90",
"from": "git+https://[email protected]/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90"
},
"to-space-case": {
"version": "git+ssh://[email protected]/ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640",
"from": "git+ssh://[email protected]/ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640",
"requires": {
"to-no-case": "^1.0.0"
}
}
}
}
113 changes: 113 additions & 0 deletions node/tests/data/packages/git/package-lock.v2.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
{
"name": "@flatpak-node-generator-tests/git",
"version": "1.0.0",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@flatpak-node-generator-tests/git",
"version": "1.0.0",
"dependencies": {
"is-empty-object": "github:gummesson/is-empty-object#7b50c8eb4e14135631f7c94e01c0c8a36e5d75f8",
"is-number": "github:jonschlinkert/is-number",
"person-lib": "gitlab:volodymyrkr/person-lib",
"to-camel-case": "git+ssh://[email protected]:ianstormtaylor/to-camel-case.git",
"to-capital-case": "git+https://[email protected]/ianstormtaylor/to-capital-case.git",
"to-no-case": "git+https://[email protected]/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90",
"to-space-case": "git+ssh://[email protected]:ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640"
}
},
"node_modules/is-empty-object": {
"version": "1.1.1",
"resolved": "git+ssh://[email protected]/gummesson/is-empty-object.git#7b50c8eb4e14135631f7c94e01c0c8a36e5d75f8",
"integrity": "sha512-OQNk2je1cKQ0Y0AYZ2X9hwapnDsOaKIa9wDdCjtmaU6JsnNHmbk27kPn1HNtgeIuLca3HvrBocDRt6y7+2aHJA==",
"license": "MIT"
},
"node_modules/is-number": {
"version": "7.0.0",
"resolved": "git+ssh://[email protected]/jonschlinkert/is-number.git#98e8ff1da1a89f93d1397a24d7413ed15421c139",
"license": "MIT",
"engines": {
"node": ">=0.12.0"
}
},
"node_modules/person-lib": {
"version": "1.0.1",
"resolved": "git+ssh://[email protected]/volodymyrkr/person-lib.git#752fd1828b1eb3a9635bf725ae5e1704a375e524",
"license": "ISC"
},
"node_modules/to-camel-case": {
"version": "1.0.0",
"resolved": "git+ssh://[email protected]/ianstormtaylor/to-camel-case.git#00a20429b600ddb6e4f8ff5b17c52914f40fe67d",
"license": "MIT",
"dependencies": {
"to-space-case": "^1.0.0"
}
},
"node_modules/to-capital-case": {
"version": "1.0.0",
"resolved": "git+https://[email protected]/ianstormtaylor/to-capital-case.git#b82f61e00e099b01514e25177bb2d56d0f64b157",
"license": "MIT",
"dependencies": {
"to-space-case": "^1.0.0"
}
},
"node_modules/to-no-case": {
"version": "1.0.0",
"resolved": "git+https://[email protected]/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90",
"integrity": "sha512-E6uMYyFvmgQWszuxWtCz0NN+BgdVvQjVCU6Nnq0BSxi8q/QKRCBs14pGUWbPQghmpjJbdkKgjICDWihePrJGOQ==",
"license": "MIT"
},
"node_modules/to-space-case": {
"version": "1.0.0",
"resolved": "git+ssh://[email protected]/ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640",
"integrity": "sha512-i7ZtWv1WzR6xrLpd5wpSU/RZxoEDaoD3uLU5pDgXEZfUZJh0QpPtjJdLf7i+nr24Z80ls8qYCfBeFpZk6RzXKQ==",
"license": "MIT",
"dependencies": {
"to-no-case": "^1.0.0"
}
}
},
"dependencies": {
"is-empty-object": {
"version": "git+ssh://[email protected]/gummesson/is-empty-object.git#7b50c8eb4e14135631f7c94e01c0c8a36e5d75f8",
"integrity": "sha512-OQNk2je1cKQ0Y0AYZ2X9hwapnDsOaKIa9wDdCjtmaU6JsnNHmbk27kPn1HNtgeIuLca3HvrBocDRt6y7+2aHJA==",
"from": "is-empty-object@github:gummesson/is-empty-object#7b50c8eb4e14135631f7c94e01c0c8a36e5d75f8"
},
"is-number": {
"version": "git+ssh://[email protected]/jonschlinkert/is-number.git#98e8ff1da1a89f93d1397a24d7413ed15421c139",
"from": "is-number@github:jonschlinkert/is-number"
},
"person-lib": {
"version": "git+ssh://[email protected]/volodymyrkr/person-lib.git#752fd1828b1eb3a9635bf725ae5e1704a375e524",
"from": "person-lib@gitlab:volodymyrkr/person-lib"
},
"to-camel-case": {
"version": "git+ssh://[email protected]/ianstormtaylor/to-camel-case.git#00a20429b600ddb6e4f8ff5b17c52914f40fe67d",
"from": "to-camel-case@git+ssh://[email protected]:ianstormtaylor/to-camel-case.git",
"requires": {
"to-space-case": "^1.0.0"
}
},
"to-capital-case": {
"version": "git+https://[email protected]/ianstormtaylor/to-capital-case.git#b82f61e00e099b01514e25177bb2d56d0f64b157",
"from": "to-capital-case@git+https://[email protected]/ianstormtaylor/to-capital-case.git",
"requires": {
"to-space-case": "^1.0.0"
}
},
"to-no-case": {
"version": "git+https://[email protected]/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90",
"integrity": "sha512-E6uMYyFvmgQWszuxWtCz0NN+BgdVvQjVCU6Nnq0BSxi8q/QKRCBs14pGUWbPQghmpjJbdkKgjICDWihePrJGOQ==",
"from": "to-no-case@git+https://[email protected]/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90"
},
"to-space-case": {
"version": "git+ssh://[email protected]/ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640",
"integrity": "sha512-i7ZtWv1WzR6xrLpd5wpSU/RZxoEDaoD3uLU5pDgXEZfUZJh0QpPtjJdLf7i+nr24Z80ls8qYCfBeFpZk6RzXKQ==",
"from": "to-space-case@git+ssh://[email protected]:ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640",
"requires": {
"to-no-case": "^1.0.0"
}
}
}
}
Loading