Fuller test and support for git refs

flatpak · Nov 16, 2023 · 9ee6559 · 9ee6559
1 parent 30db60f
commit 9ee6559
Show file tree

Hide file tree

Showing 9 changed files with 280 additions and 66 deletions.
diff --git a/node/flatpak_node_generator/providers/__init__.py b/node/flatpak_node_generator/providers/__init__.py
@@ -16,6 +16,7 @@
     'git': {},
     'git+http': {'scheme': 'http'},
     'git+https': {'scheme': 'https'},
+    'git+ssh': {'scheme': 'https'},
 }
 
 
@@ -33,6 +34,14 @@ def parse_git_source(self, version: str, from_: Optional[str] = None) -> GitSour
         if not new_url.netloc:
             path = new_url.path.split('/')
             new_url = new_url._replace(netloc=path[0], path='/'.join(path[1:]))
+        # Replace https://[email protected]:ianstormtaylor/to-camel-case.git
+        # wth     https://[email protected]/ianstormtaylor/to-camel-case.git
+        # for git+ssh URLs
+        if ':' in new_url.netloc:
+            netloc_split = new_url.netloc.split(':')
+            new_url = new_url._replace(
+                netloc=netloc_split[0], path=f'/{netloc_split[1]}{new_url.path}'
+            )
 
         return GitSource(
             original=original_url.geturl(),

diff --git a/node/flatpak_node_generator/providers/npm.py b/node/flatpak_node_generator/providers/npm.py
@@ -130,10 +130,7 @@ def _process_packages_v2(
                         integrity=integrity, resolved=info['resolved']
                     )
                 elif resolved_url.scheme.startswith('git+'):
-                    raise NotImplementedError(
-                        'Git sources in lockfile v2 format are not supported yet'
-                        f' (package {install_path} in {lockfile})'
-                    )
+                    source = self.parse_git_source(info['resolved'])
             else:
                 raise NotImplementedError(
                     f"Don't know how to handle package {install_path} in {lockfile}"
@@ -425,72 +422,58 @@ def _finalize(self) -> None:
         )
 
         if self.git_sources:
-            # Generate jq scripts to patch the package*.json files.
-            scripts = {
-                'package.json': r"""
-                    walk(
-                        if type == "object"
-                        then
-                            to_entries | map(
-                                if (.value | type == "string") and $data[.value]
-                                then .value = "git+file:\($buildroot)/\($data[.value])"
-                                else .
-                                end
-                            ) | from_entries
-                        else .
-                        end
-                    )
-                """,
-                'package-lock.json': r"""
-                    walk(
-                        if type == "object" and (.version | type == "string") and $data[.version]
-                        then
-                            .version = "git+file:\($buildroot)/\($data[.version])"
-                        else .
-                        end
-                    )
-                """,
-            }
+            # Generate jq script to patch the package.json file
+            script = r"""
+                        walk(
+                            if type == "object"
+                            then
+                                to_entries | map(
+                                    if (.value | type == "string") and $data[.value]
+                                    then .value = "git+file:\($buildroot)/\($data[.value])"
+                                    else .
+                                    end
+                                ) | from_entries
+                            else .
+                            end
+                        )
+                    """
 
             for lockfile, sources in self.git_sources.items():
                 prefix = self.relative_lockfile_dir(lockfile)
-                data: Dict[str, Dict[str, str]] = {
-                    'package.json': {},
-                    'package-lock.json': {},
-                }
+                data: Dict[str, str] = {}
 
                 for path, source in sources.items():
                     GIT_URL_PREFIX = 'git+'
 
                     new_version = f'{path}#{source.commit}'
-                    assert source.from_ is not None
-                    data['package.json'][source.from_] = new_version
-                    data['package-lock.json'][source.original] = new_version
-
-                    if source.from_.startswith(GIT_URL_PREFIX):
-                        data['package.json'][
-                            source.from_[len(GIT_URL_PREFIX) :]
-                        ] = new_version
-
-                    if source.original.startswith(GIT_URL_PREFIX):
-                        data['package-lock.json'][
-                            source.original[len(GIT_URL_PREFIX) :]
-                        ] = new_version
-
-                for filename, script in scripts.items():
-                    target = Path('$FLATPAK_BUILDER_BUILDDIR') / prefix / filename
-                    script = (
-                        textwrap.dedent(script.lstrip('\n')).strip().replace('\n', '')
-                    )
-                    json_data = json.dumps(data[filename])
-                    patch_commands[lockfile].append(
-                        'jq'
-                        ' --arg buildroot "$FLATPAK_BUILDER_BUILDDIR"'
-                        f' --argjson data {shlex.quote(json_data)}'
-                        f' {shlex.quote(script)} {target}'
-                        f' > {target}.new'
-                    )
-                    patch_commands[lockfile].append(f'mv {target}{{.new,}}')
+                    targets = []
+                    source_url = source.from_ or source.original
+
+                    if (
+                        source_url.startswith(GIT_URL_PREFIX + 'ssh')
+                        and ':' not in urllib.parse.urlparse(source_url).netloc
+                    ):
+                        source_url = re.sub(r'(://[^/]+)/', r'\1:', source_url)
+                    elif source_url.startswith(GIT_URL_PREFIX):
+                        targets.append(source_url[len(GIT_URL_PREFIX) :])
+
+                    targets.append(source_url)
+                    for t in targets:
+                        data[t] = new_version
+                        data[t.replace('#' + source.commit, '')] = new_version
+
+                filename = 'package.json'
+                target = Path('$FLATPAK_BUILDER_BUILDDIR') / prefix / filename
+                script = textwrap.dedent(script.lstrip('\n')).strip().replace('\n', '')
+                json_data = json.dumps(data)
+                patch_commands[lockfile].append(
+                    'jq'
+                    ' --arg buildroot "$FLATPAK_BUILDER_BUILDDIR"'
+                    f' --argjson data {shlex.quote(json_data)}'
+                    f' {shlex.quote(script)} {target}'
+                    f' > {target}.new'
+                )
+                patch_commands[lockfile].append(f'mv {target}{{.new,}}')
 
         patch_all_commands: List[str] = []
         for lockfile in self.all_lockfiles:

diff --git a/node/tests/data/packages/git/package-lock.v1.json b/node/tests/data/packages/git/package-lock.v1.json
@@ -0,0 +1,33 @@
+{
+  "name": "@flatpak-node-generator-tests/git",
+  "version": "1.0.0",
+  "lockfileVersion": 1,
+  "requires": true,
+  "dependencies": {
+    "to-camel-case": {
+      "version": "git+ssh://[email protected]/ianstormtaylor/to-camel-case.git#00a20429b600ddb6e4f8ff5b17c52914f40fe67d",
+      "from": "git+ssh://[email protected]/ianstormtaylor/to-camel-case.git",
+      "requires": {
+        "to-space-case": "^1.0.0"
+      }
+    },
+    "to-capital-case": {
+      "version": "git+https://[email protected]/ianstormtaylor/to-capital-case.git#b82f61e00e099b01514e25177bb2d56d0f64b157",
+      "from": "git+https://[email protected]/ianstormtaylor/to-capital-case.git",
+      "requires": {
+        "to-space-case": "^1.0.0"
+      }
+    },
+    "to-no-case": {
+      "version": "git+https://[email protected]/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90",
+      "from": "git+https://[email protected]/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90"
+    },
+    "to-space-case": {
+      "version": "git+ssh://[email protected]/ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640",
+      "from": "git+ssh://[email protected]/ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640",
+      "requires": {
+        "to-no-case": "^1.0.0"
+      }
+    }
+  }
+}
diff --git a/node/tests/data/packages/git/package-lock.v2.json b/node/tests/data/packages/git/package-lock.v2.json
@@ -0,0 +1,78 @@
+{
+  "name": "@flatpak-node-generator-tests/git",
+  "version": "1.0.0",
+  "lockfileVersion": 2,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "@flatpak-node-generator-tests/git",
+      "version": "1.0.0",
+      "dependencies": {
+        "to-camel-case": "git+ssh://[email protected]:ianstormtaylor/to-camel-case.git",
+        "to-capital-case": "git+https://[email protected]/ianstormtaylor/to-capital-case.git",
+        "to-no-case": "git+https://[email protected]/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90",
+        "to-space-case": "git+ssh://[email protected]:ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640"
+      }
+    },
+    "node_modules/to-camel-case": {
+      "version": "1.0.0",
+      "resolved": "git+ssh://[email protected]/ianstormtaylor/to-camel-case.git#00a20429b600ddb6e4f8ff5b17c52914f40fe67d",
+      "license": "MIT",
+      "dependencies": {
+        "to-space-case": "^1.0.0"
+      }
+    },
+    "node_modules/to-capital-case": {
+      "version": "1.0.0",
+      "resolved": "git+https://[email protected]/ianstormtaylor/to-capital-case.git#b82f61e00e099b01514e25177bb2d56d0f64b157",
+      "license": "MIT",
+      "dependencies": {
+        "to-space-case": "^1.0.0"
+      }
+    },
+    "node_modules/to-no-case": {
+      "version": "1.0.0",
+      "resolved": "git+https://[email protected]/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90",
+      "integrity": "sha512-E6uMYyFvmgQWszuxWtCz0NN+BgdVvQjVCU6Nnq0BSxi8q/QKRCBs14pGUWbPQghmpjJbdkKgjICDWihePrJGOQ==",
+      "license": "MIT"
+    },
+    "node_modules/to-space-case": {
+      "version": "1.0.0",
+      "resolved": "git+ssh://[email protected]/ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640",
+      "integrity": "sha512-i7ZtWv1WzR6xrLpd5wpSU/RZxoEDaoD3uLU5pDgXEZfUZJh0QpPtjJdLf7i+nr24Z80ls8qYCfBeFpZk6RzXKQ==",
+      "license": "MIT",
+      "dependencies": {
+        "to-no-case": "^1.0.0"
+      }
+    }
+  },
+  "dependencies": {
+    "to-camel-case": {
+      "version": "git+ssh://[email protected]/ianstormtaylor/to-camel-case.git#00a20429b600ddb6e4f8ff5b17c52914f40fe67d",
+      "from": "to-camel-case@git+ssh://[email protected]:ianstormtaylor/to-camel-case.git",
+      "requires": {
+        "to-space-case": "^1.0.0"
+      }
+    },
+    "to-capital-case": {
+      "version": "git+https://[email protected]/ianstormtaylor/to-capital-case.git#b82f61e00e099b01514e25177bb2d56d0f64b157",
+      "from": "to-capital-case@git+https://[email protected]/ianstormtaylor/to-capital-case.git",
+      "requires": {
+        "to-space-case": "^1.0.0"
+      }
+    },
+    "to-no-case": {
+      "version": "git+https://[email protected]/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90",
+      "integrity": "sha512-E6uMYyFvmgQWszuxWtCz0NN+BgdVvQjVCU6Nnq0BSxi8q/QKRCBs14pGUWbPQghmpjJbdkKgjICDWihePrJGOQ==",
+      "from": "to-no-case@git+https://[email protected]/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90"
+    },
+    "to-space-case": {
+      "version": "git+ssh://[email protected]/ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640",
+      "integrity": "sha512-i7ZtWv1WzR6xrLpd5wpSU/RZxoEDaoD3uLU5pDgXEZfUZJh0QpPtjJdLf7i+nr24Z80ls8qYCfBeFpZk6RzXKQ==",
+      "from": "to-space-case@git+ssh://[email protected]:ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640",
+      "requires": {
+        "to-no-case": "^1.0.0"
+      }
+    }
+  }
+}
diff --git a/node/tests/data/packages/git/package-lock.v3.json b/node/tests/data/packages/git/package-lock.v3.json
@@ -0,0 +1,49 @@
+{
+  "name": "@flatpak-node-generator-tests/git",
+  "version": "1.0.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "@flatpak-node-generator-tests/git",
+      "version": "1.0.0",
+      "dependencies": {
+        "to-camel-case": "git+ssh://[email protected]:ianstormtaylor/to-camel-case.git",
+        "to-capital-case": "git+https://[email protected]/ianstormtaylor/to-capital-case.git",
+        "to-no-case": "git+https://[email protected]/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90",
+        "to-space-case": "git+ssh://[email protected]:ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640"
+      }
+    },
+    "node_modules/to-camel-case": {
+      "version": "1.0.0",
+      "resolved": "git+ssh://[email protected]/ianstormtaylor/to-camel-case.git#00a20429b600ddb6e4f8ff5b17c52914f40fe67d",
+      "license": "MIT",
+      "dependencies": {
+        "to-space-case": "^1.0.0"
+      }
+    },
+    "node_modules/to-capital-case": {
+      "version": "1.0.0",
+      "resolved": "git+https://[email protected]/ianstormtaylor/to-capital-case.git#b82f61e00e099b01514e25177bb2d56d0f64b157",
+      "license": "MIT",
+      "dependencies": {
+        "to-space-case": "^1.0.0"
+      }
+    },
+    "node_modules/to-no-case": {
+      "version": "1.0.0",
+      "resolved": "git+https://[email protected]/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90",
+      "integrity": "sha512-E6uMYyFvmgQWszuxWtCz0NN+BgdVvQjVCU6Nnq0BSxi8q/QKRCBs14pGUWbPQghmpjJbdkKgjICDWihePrJGOQ==",
+      "license": "MIT"
+    },
+    "node_modules/to-space-case": {
+      "version": "1.0.0",
+      "resolved": "git+ssh://[email protected]/ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640",
+      "integrity": "sha512-i7ZtWv1WzR6xrLpd5wpSU/RZxoEDaoD3uLU5pDgXEZfUZJh0QpPtjJdLf7i+nr24Z80ls8qYCfBeFpZk6RzXKQ==",
+      "license": "MIT",
+      "dependencies": {
+        "to-no-case": "^1.0.0"
+      }
+    }
+  }
+}
diff --git a/node/tests/data/packages/git/package.json b/node/tests/data/packages/git/package.json
@@ -0,0 +1,10 @@
+{
+  "name": "@flatpak-node-generator-tests/git",
+  "version": "1.0.0",
+  "dependencies": {
+    "to-camel-case": "git+ssh://[email protected]:ianstormtaylor/to-camel-case.git",
+    "to-capital-case": "git+https://[email protected]/ianstormtaylor/to-capital-case.git",
+    "to-no-case": "git+https://[email protected]/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90",
+    "to-space-case": "git+ssh://[email protected]:ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640"
+  }
+}
diff --git a/node/tests/data/packages/git/yarn.lock b/node/tests/data/packages/git/yarn.lock
@@ -0,0 +1,37 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"to-camel-case@git+ssh://[email protected]:ianstormtaylor/to-camel-case.git":
+  version "1.0.0"
+  resolved "git+ssh://[email protected]:ianstormtaylor/to-camel-case.git#00a20429b600ddb6e4f8ff5b17c52914f40fe67d"
+  dependencies:
+    to-space-case "^1.0.0"
+
+"to-capital-case@git+https://[email protected]/ianstormtaylor/to-capital-case.git":
+  version "1.0.0"
+  resolved "git+https://[email protected]/ianstormtaylor/to-capital-case.git#b82f61e00e099b01514e25177bb2d56d0f64b157"
+  dependencies:
+    to-space-case "^1.0.0"
+
+to-no-case@^1.0.0:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/to-no-case/-/to-no-case-1.0.2.tgz#c722907164ef6b178132c8e69930212d1b4aa16a"
+  integrity sha512-Z3g735FxuZY8rodxV4gH7LxClE4H0hTIyHNIHdk+vpQxjLm0cwnKXq/OFVZ76SOQmto7txVcwSCwkU5kqp+FKg==
+
+"to-no-case@git+https://[email protected]/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90":
+  version "1.0.0"
+  resolved "git+https://[email protected]/ianstormtaylor/to-no-case.git#9078578dcf394e63f34fd7c6666772192e537b90"
+
+to-space-case@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/to-space-case/-/to-space-case-1.0.0.tgz#b052daafb1b2b29dc770cea0163e5ec0ebc9fc17"
+  integrity sha512-rLdvwXZ39VOn1IxGL3V6ZstoTbwLRckQmn/U8ZDLuWwIXNpuZDhQ3AiRUlhTbOXFVE9C+dR51wM0CBDhk31VcA==
+  dependencies:
+    to-no-case "^1.0.0"
+
+"to-space-case@git+ssh://[email protected]:ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640":
+  version "1.0.0"
+  resolved "git+ssh://[email protected]:ianstormtaylor/to-space-case.git#aa68213d1211745ce7c6c725ba072e6b13bef640"
+  dependencies:
+    to-no-case "^1.0.0"
diff --git a/node/tests/data/packages/minimal-git/package-lock.v3.json b/node/tests/data/packages/minimal-git/package-lock.v3.json
@@ -13,7 +13,7 @@
     },
     "node_modules/nop": {
       "version": "1.0.0",
-      "resolved": "git+ssh://[email protected]/supershabam/nop.git#f110e75f62cfe3bf4468ac3b74e3dc72ab9ae4bf",
+      "resolved": "git+https://[email protected]/supershabam/nop.git#f110e75f62cfe3bf4468ac3b74e3dc72ab9ae4bf",
       "integrity": "sha512-xCwdA7C4QIORvTMytKHMlkEN6axJGimR0gv5vgjKpEKRvQrPOwhjJnrZEcd5g0LP+7IY38+TY7MP59HRY6gcwA==",
       "license": "MIT"
     }