Refactor rc file loading / local registry support + add tests

.github/workflows/node.yaml

@@ -26,6 +26,11 @@ jobs:
         poetry-version:
           - '1.1.13'
     runs-on: ubuntu-22.04
+    services:
+      verdaccio:
+        image: verdaccio/verdaccio
+        ports:
+          - 4873:4873
     steps:
       - uses: actions/checkout@v3
       - name: Configure git
@@ -45,6 +50,8 @@ jobs:
           flatpak --user remote-add flathub https://flathub.org/repo/flathub.flatpakrepo
           flatpak --user install -y flathub \
             org.freedesktop.{Platform,Sdk{,.Extension.node{14,16,18}}}//22.08
+      - name: Setup the local npm registry
+        run: tools/setup-local-registry.sh
       - name: Install dependencies
         run: poetry install
       - name: Run checks

node/README.md

@@ -321,6 +321,19 @@ $ poetry run pytest -n auto
 Note that these tests can take up quite a bit of space in /tmp, so if you hit `No space
 left on device` errors, try expanding `/tmp` or changing `$TMPDIR`.
 
+### Local Registry
+
+Some of the tests require a local npm registry to work. For this purpose, you can use
+[Verdaccio](https://verdaccio.org/), preferably via Docker / podman:
+
+```bash
+$ docker run --rm -it -p 4873:4873 verdaccio/verdaccio
+```
+
+Then run `tools/setup-local-registry.sh` to set up this registry with a pre-published
+package. (Note that running it twice will result in an error, since it tries to publish
+the same package twice.)
+
 ### Utility Scripts
 
 A few utility scripts are included in the `tools` directory:
@@ -333,6 +346,8 @@ A few utility scripts are included in the `tools` directory:
   - `lockfile-utils.sh peek-cache PACKAGE-MANAGER PACKAGE` will install the dependencies
     from the corresponding lockfile and then extract the resulting package cache (npm)
     or mirror directory (yarn), for closer examination.
+- `setup-local-registry.sh` will set up a local npm registry as [described
+  above](#local-registry).
 - `b64-to-hex.sh` will convert a base64 hash value from npm into hex, e.g.:
   ```
   $ echo x+sXyT4RLLEIb6bY5R+wZnt5pfk= | tools/b64-to-hex.sh

node/flatpak_node_generator/main.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-from typing import Iterator, List, Set
+from typing import Dict, Iterator, List, Set
 
 import argparse
 import asyncio
@@ -12,7 +12,7 @@
 from .node_headers import NodeHeaders
 from .package import Package
 from .progress import GeneratorProgress
-from .providers import ProviderFactory
+from .providers import Config, ProviderFactory
 from .providers.npm import NpmLockfileProvider, NpmModuleProvider, NpmProviderFactory
 from .providers.special import SpecialSourceProvider
 from .providers.yarn import YarnProviderFactory
@@ -189,20 +189,20 @@ async def _async_main() -> None:
 
     print('Reading packages from lockfiles...')
     packages: Set[Package] = set()
-    rcfile_node_headers: Set[NodeHeaders] = set()
+    config_node_headers: Set[NodeHeaders] = set()
 
-    for lockfile in lockfiles:
-        lockfile_provider = provider_factory.create_lockfile_provider()
-        rcfile_providers = provider_factory.create_rcfile_providers()
+    lockfile_provider = provider_factory.create_lockfile_provider()
+    config_provider = provider_factory.create_config_provider()
+
+    lockfile_configs: Dict[Path, Config] = {}
 
+    for lockfile in lockfiles:
         packages.update(lockfile_provider.process_lockfile(lockfile))
+        lockfile_configs[lockfile] = config = config_provider.load_config(lockfile)
 
-        for rcfile_provider in rcfile_providers:
-            rcfile = lockfile.parent / rcfile_provider.RCFILE_NAME
-            if rcfile.is_file():
-                nh = rcfile_provider.get_node_headers(rcfile)
-                if nh is not None:
-                    rcfile_node_headers.add(nh)
+        nh = config.get_node_headers()
+        if nh is not None:
+            config_node_headers.add(nh)
 
     print(f'{len(packages)} packages read.')
 
@@ -220,14 +220,18 @@ async def _async_main() -> None:
         )
         special = SpecialSourceProvider(gen, options)
 
-        with provider_factory.create_module_provider(gen, special) as module_provider:
+        with provider_factory.create_module_provider(
+            gen,
+            special,
+            lockfile_configs,
+        ) as module_provider:
             with GeneratorProgress(
                 packages,
                 module_provider,
                 args.max_parallel,
             ) as progress:
                 await progress.run()
-        for headers in rcfile_node_headers:
+        for headers in config_node_headers:
             print(f'Generating headers {headers.runtime} @ {headers.target}')
             await special.generate_node_headers(headers)
 

node/flatpak_node_generator/providers/__init__.py

@@ -1,7 +1,8 @@
+from dataclasses import dataclass
 from pathlib import Path
-from typing import ContextManager, Dict, Iterator, List, Optional
+from typing import Any, ContextManager, Dict, Iterator, List, Optional, Tuple
 
-import re
+import dataclasses
 import urllib.parse
 
 from ..manifest import ManifestGenerator
@@ -45,32 +46,48 @@ def process_lockfile(self, lockfile: Path) -> Iterator[Package]:
         raise NotImplementedError()
 
 
-class RCFileProvider:
-    RCFILE_NAME: str
+@dataclass
+class Config:
+    data: Dict[str, Any] = dataclasses.field(default_factory=lambda: {})
 
-    def parse_rcfile(self, rcfile: Path) -> Dict[str, str]:
-        with open(rcfile, 'r') as r:
-            rcfile_text = r.read()
-        parser_re = re.compile(
-            r'^(?!#|;)(\S+)(?:\s+|\s*=\s*)(?:"(.+)"|(\S+))$', re.MULTILINE
-        )
-        result: Dict[str, str] = {}
-        for key, quoted_val, val in parser_re.findall(rcfile_text):
-            result[key] = quoted_val or val
-        return result
-
-    def get_node_headers(self, rcfile: Path) -> Optional[NodeHeaders]:
-        rc_data = self.parse_rcfile(rcfile)
-        if 'target' not in rc_data:
+    def merge_new_keys_only(self, other: Dict[str, Any]) -> None:
+        for key, value in other.items():
+            if key not in self.data:
+                self.data[key] = value
+
+    def get_node_headers(self) -> Optional[NodeHeaders]:
+        if 'target' not in self.data:
             return None
-        target = rc_data['target']
-        runtime = rc_data.get('runtime')
-        disturl = rc_data.get('disturl')
+        target = self.data['target']
+        runtime = self.data.get('runtime')
+        disturl = self.data.get('disturl')
 
         assert isinstance(runtime, str) and isinstance(disturl, str)
 
         return NodeHeaders.with_defaults(target, runtime, disturl)
 
+    def get_registry_for_scope(self, scope: str) -> Optional[str]:
+        return self.data.get(f'{scope}:registry')
+
+
+class ConfigProvider:
+    @property
+    def _filename(self) -> str:
+        raise NotImplementedError()
+
+    def parse_config(self, path: Path) -> Dict[str, Any]:
+        raise NotImplementedError()
+
+    def load_config(self, lockfile: Path) -> Config:
+        config = Config()
+
+        for parent in lockfile.parents:
+            path = parent / self._filename
+            if path.exists():
+                config.merge_new_keys_only(self.parse_config(path))
+
+        return config
+
 
 class ModuleProvider(ContextManager['ModuleProvider']):
     async def generate_package(self, package: Package) -> None:
@@ -81,10 +98,13 @@ class ProviderFactory:
     def create_lockfile_provider(self) -> LockfileProvider:
         raise NotImplementedError()
 
-    def create_rcfile_providers(self) -> List[RCFileProvider]:
+    def create_config_provider(self) -> ConfigProvider:
         raise NotImplementedError()
 
     def create_module_provider(
-        self, gen: ManifestGenerator, special: SpecialSourceProvider
+        self,
+        gen: ManifestGenerator,
+        special: SpecialSourceProvider,
+        lockfile_configs: Dict[Path, Config],
     ) -> ModuleProvider:
         raise NotImplementedError()