diff --git a/doc/languages-frameworks/vim.section.md b/doc/languages-frameworks/vim.section.md index c79fd3a55df99..4bb2b6e4f2550 100644 --- a/doc/languages-frameworks/vim.section.md +++ b/doc/languages-frameworks/vim.section.md @@ -264,10 +264,15 @@ nix-shell -p vimPluginsUpdater --run 'vim-plugins-updater --github-token=mytoken Alternatively, set the number of processes to a lower count to avoid rate-limiting. ```sh - nix-shell -p vimPluginsUpdater --run 'vim-plugins-updater --proc 1' ``` +If you want to update only certain plugins, you can specify them after the `update` command. Note that you must use the same plugin names as the `pkgs/applications/editors/vim/plugins/vim-plugin-names` file. + +```sh +nix-shell -p vimPluginsUpdater --run 'vim-plugins-updater update "nvim-treesitter" "LazyVim"' +``` + ## How to maintain an out-of-tree overlay of vim plugins ? {#vim-out-of-tree-overlays} You can use the updater script to generate basic packages out of a custom vim diff --git a/maintainers/scripts/pluginupdate-py/pluginupdate.py b/maintainers/scripts/pluginupdate-py/pluginupdate.py index e37e82fef3e7b..2f55c1359de72 100644 --- a/maintainers/scripts/pluginupdate-py/pluginupdate.py +++ b/maintainers/scripts/pluginupdate-py/pluginupdate.py @@ -32,7 +32,7 @@ from multiprocessing.dummy import Pool from pathlib import Path from tempfile import NamedTemporaryFile -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable from urllib.parse import urljoin, urlparse import git @@ -94,7 +94,7 @@ def make_request(url: str, token=None) -> urllib.request.Request: # a dictionary of plugins and their new repositories -Redirects = Dict["PluginDesc", "Repo"] +Redirects = dict["PluginDesc", "Repo"] class Repo: @@ -103,7 +103,7 @@ def __init__(self, uri: str, branch: str) -> None: """Url to the repo""" self._branch = branch # Redirect is the new Repo to use - self.redirect: Optional["Repo"] = None + self.redirect: "Repo | None" = None self.token = "dummy_token" @property @@ -125,14 +125,14 @@ def has_submodules(self) -> bool: return True @retry(urllib.error.URLError, tries=4, delay=3, backoff=2) - def latest_commit(self) -> Tuple[str, datetime]: + def latest_commit(self) -> tuple[str, datetime]: log.debug("Latest commit") loaded = self._prefetch(None) updated = datetime.strptime(loaded["date"], "%Y-%m-%dT%H:%M:%S%z") return loaded["rev"], updated - def _prefetch(self, ref: Optional[str]): + def _prefetch(self, ref: str | None): cmd = ["nix-prefetch-git", "--quiet", "--fetch-submodules", self.uri] if ref is not None: cmd.append(ref) @@ -141,7 +141,7 @@ def _prefetch(self, ref: Optional[str]): loaded = json.loads(data) return loaded - def prefetch(self, ref: Optional[str]) -> str: + def prefetch(self, ref: str | None) -> str: log.info("Prefetching %s", self.uri) loaded = self._prefetch(ref) return loaded["sha256"] @@ -186,7 +186,7 @@ def has_submodules(self) -> bool: return True @retry(urllib.error.URLError, tries=4, delay=3, backoff=2) - def latest_commit(self) -> Tuple[str, datetime]: + def latest_commit(self) -> tuple[str, datetime]: commit_url = self.url(f"commits/{self.branch}.atom") log.debug("Sending request to %s", commit_url) commit_req = make_request(commit_url, self.token) @@ -252,14 +252,14 @@ def as_nix(self, plugin: "Plugin") -> str: class PluginDesc: repo: Repo branch: str - alias: Optional[str] + alias: str | None @property def name(self): return self.alias or self.repo.name @staticmethod - def load_from_csv(config: FetchConfig, row: Dict[str, str]) -> "PluginDesc": + def load_from_csv(config: FetchConfig, row: dict[str, str]) -> "PluginDesc": log.debug("Loading row %s", row) branch = row["branch"] repo = make_repo(row["repo"], branch.strip()) @@ -292,7 +292,7 @@ class Plugin: commit: str has_submodules: bool sha256: str - date: Optional[datetime] = None + date: datetime | None = None @property def normalized_name(self) -> str: @@ -303,7 +303,7 @@ def version(self) -> str: assert self.date is not None return self.date.strftime("%Y-%m-%d") - def as_json(self) -> Dict[str, str]: + def as_json(self) -> dict[str, str]: copy = self.__dict__.copy() del copy["date"] return copy @@ -312,7 +312,7 @@ def as_json(self) -> Dict[str, str]: def load_plugins_from_csv( config: FetchConfig, input_file: Path, -) -> List[PluginDesc]: +) -> list[PluginDesc]: log.debug("Load plugins from csv %s", input_file) plugins = [] with open(input_file, newline="") as csvfile: @@ -359,10 +359,10 @@ def __init__( name: str, root: Path, get_plugins: str, - default_in: Optional[Path] = None, - default_out: Optional[Path] = None, - deprecated: Optional[Path] = None, - cache_file: Optional[str] = None, + default_in: Path | None = None, + default_out: Path | None = None, + deprecated: Path | None = None, + cache_file: str | None = None, ): log.debug("get_plugins:", get_plugins) self.name = name @@ -388,6 +388,19 @@ def add(self, args): fetch_config, args.input_file, editor.deprecated, append=append ) plugin, _ = prefetch_plugin(pdesc) + + if ( # lua updater doesn't support updating individual plugin + self.name != "lua" + ): + # update generated.nix + update = self.get_update( + args.input_file, + args.outfile, + fetch_config, + [plugin.normalized_name], + ) + update() + autocommit = not args.no_commit if autocommit: commit( @@ -404,16 +417,35 @@ def update(self, args): """CSV spec""" print("the update member function should be overridden in subclasses") - def get_current_plugins(self, nixpkgs: str) -> List[Plugin]: + def get_current_plugins( + self, config: FetchConfig, nixpkgs: str + ) -> list[tuple[PluginDesc, Plugin]]: """To fill the cache""" data = run_nix_expr(self.get_plugins, nixpkgs) plugins = [] for name, attr in data.items(): - p = Plugin(name, attr["rev"], attr["submodules"], attr["sha256"]) - plugins.append(p) + checksum = attr["checksum"] + + # https://github.com/NixOS/nixpkgs/blob/8a335419/pkgs/applications/editors/neovim/build-neovim-plugin.nix#L36 + # https://github.com/NixOS/nixpkgs/pull/344478#discussion_r1786646055 + version = re.search(r"\d\d\d\d-\d\d?-\d\d?", attr["version"]) + if version is None: + raise ValueError(f"Cannot parse version: {attr['version']}") + date = datetime.strptime(version.group(), "%Y-%m-%d") + + pdesc = PluginDesc.load_from_string(config, f'{attr["homePage"]} as {name}') + p = Plugin( + attr["pname"], + checksum["rev"], + checksum["submodules"], + checksum["sha256"], + date, + ) + + plugins.append((pdesc, p)) return plugins - def load_plugin_spec(self, config: FetchConfig, plugin_file) -> List[PluginDesc]: + def load_plugin_spec(self, config: FetchConfig, plugin_file) -> list[PluginDesc]: """CSV spec""" return load_plugins_from_csv(config, plugin_file) @@ -421,28 +453,115 @@ def generate_nix(self, _plugins, _outfile: str): """Returns nothing for now, writes directly to outfile""" raise NotImplementedError() - def get_update(self, input_file: str, outfile: str, config: FetchConfig): - cache: Cache = Cache(self.get_current_plugins(self.nixpkgs), self.cache_file) + def filter_plugins_to_update( + self, plugin: PluginDesc, to_update: list[str] + ) -> bool: + """Function for filtering out plugins, that user doesn't want to update. + + It is mainly used for updating only specific plugins, not all of them. + By default it filters out plugins not present in `to_update`, + assuming `to_update` is a list of plugin names (the same as in the + result expression). + + This function is never called if `to_update` is empty. + Feel free to override this function in derived classes. + + Note: + Known bug: you have to use a deprecated name, instead of new one. + This is because we resolve deprecations later and can't get new + plugin URL before we request info about it. + + Although, we could parse deprecated.json, but it's a whole bunch + of spaghetti code, which I don't want to write. + + Arguments: + plugin: Plugin on which you decide whether to ignore or not. + to_update: + List of strings passed to via the `--update` command line parameter. + By default, we assume it is a list of URIs identical to what + is in the input file. + + Returns: + True if we should update plugin and False if not. + """ + return plugin.name.replace(".", "-") in to_update + + def get_update( + self, + input_file: str, + output_file: str, + config: FetchConfig, + to_update: list[str] | None, + ): + if to_update is None: + to_update = [] + + current_plugins = self.get_current_plugins(config, self.nixpkgs) + current_plugin_specs = self.load_plugin_spec(config, input_file) + + cache: Cache = Cache( + [plugin for _description, plugin in current_plugins], self.cache_file + ) _prefetch = functools.partial(prefetch, cache=cache) - def update() -> dict: - plugins = self.load_plugin_spec(config, input_file) + plugins_to_update = ( + current_plugin_specs + if len(to_update) == 0 + else [ + description + for description in current_plugin_specs + if self.filter_plugins_to_update(description, to_update) + ] + ) + + def update() -> Redirects: + if len(plugins_to_update) == 0: + log.error( + "\n\n\n\nIt seems like you provided some arguments to `--update`:\n" + + ", ".join(to_update) + + "\nBut after filtering, the result list of plugins is empty\n" + "\n" + "Are you sure you provided the same URIs as in your input file?\n" + "(" + str(input_file) + ")\n\n" + ) + return {} try: pool = Pool(processes=config.proc) - results = pool.map(_prefetch, plugins) + results = pool.map(_prefetch, plugins_to_update) finally: cache.store() + print(f"{len(results)} of {len(current_plugins)} were checked") + # Do only partial update of out file + if len(results) != len(current_plugins): + results = self.merge_results(current_plugins, results) plugins, redirects = check_results(results) plugins = sorted(plugins, key=lambda v: v[1].normalized_name) - self.generate_nix(plugins, outfile) + self.generate_nix(plugins, output_file) return redirects return update + def merge_results( + self, + current: list[tuple[PluginDesc, Plugin]], + fetched: list[tuple[PluginDesc, Exception | Plugin, Repo | None]], + ) -> list[tuple[PluginDesc, Exception | Plugin, Repo | None]]: + # transforming this to dict, so lookup is O(1) instead of O(n) (n is len(current)) + result: dict[str, tuple[PluginDesc, Exception | Plugin, Repo | None]] = { + # also adding redirect (third item in the result tuple) + pl.normalized_name: (pdesc, pl, None) + for pdesc, pl in current + } + + for plugin_desc, plugin, redirect in fetched: + result[plugin.normalized_name] = (plugin_desc, plugin, redirect) + + return list(result.values()) + @property def attr_path(self): return self.name + "Plugins" @@ -544,6 +663,12 @@ def create_parser(self): description="Update all or a subset of existing plugins", add_help=False, ) + pupdate.add_argument( + "update_only", + default=None, + nargs="*", + help="Plugin URLs to update (must be the same as in the input file)", + ) pupdate.set_defaults(func=self.update) return main @@ -587,8 +712,8 @@ def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: def prefetch_plugin( p: PluginDesc, - cache: "Optional[Cache]" = None, -) -> Tuple[Plugin, Optional[Repo]]: + cache: "Cache | None" = None, +) -> tuple[Plugin, Repo | None]: commit = None log.info(f"Fetching last commit for plugin {p.name} from {p.repo.uri}@{p.branch}") commit, date = p.repo.latest_commit() @@ -621,10 +746,10 @@ def print_download_error(plugin: PluginDesc, ex: Exception): def check_results( - results: List[Tuple[PluginDesc, Union[Exception, Plugin], Optional[Repo]]], -) -> Tuple[List[Tuple[PluginDesc, Plugin]], Redirects]: + results: list[tuple[PluginDesc, Exception | Plugin, Repo | None]], +) -> tuple[list[tuple[PluginDesc, Plugin]], Redirects]: """ """ - failures: List[Tuple[PluginDesc, Exception]] = [] + failures: list[tuple[PluginDesc, Exception]] = [] plugins = [] redirects: Redirects = {} for pdesc, result, redirect in results: @@ -637,11 +762,10 @@ def check_results( new_pdesc = PluginDesc(redirect, pdesc.branch, pdesc.alias) plugins.append((new_pdesc, result)) - print(f"{len(results) - len(failures)} plugins were checked", end="") if len(failures) == 0: return plugins, redirects else: - log.error(f", {len(failures)} plugin(s) could not be downloaded:\n") + log.error(f"{len(failures)} plugin(s) could not be downloaded:\n") for plugin, exception in failures: print_download_error(plugin, exception) @@ -661,7 +785,7 @@ def make_repo(uri: str, branch) -> Repo: return repo -def get_cache_path(cache_file_name: str) -> Optional[Path]: +def get_cache_path(cache_file_name: str) -> Path | None: xdg_cache = os.environ.get("XDG_CACHE_HOME", None) if xdg_cache is None: home = os.environ.get("HOME", None) @@ -673,7 +797,7 @@ def get_cache_path(cache_file_name: str) -> Optional[Path]: class Cache: - def __init__(self, initial_plugins: List[Plugin], cache_file_name: str) -> None: + def __init__(self, initial_plugins: list[Plugin], cache_file_name: str) -> None: self.cache_file = get_cache_path(cache_file_name) downloads = {} @@ -682,11 +806,11 @@ def __init__(self, initial_plugins: List[Plugin], cache_file_name: str) -> None: downloads.update(self.load()) self.downloads = downloads - def load(self) -> Dict[str, Plugin]: + def load(self) -> dict[str, Plugin]: if self.cache_file is None or not self.cache_file.exists(): return {} - downloads: Dict[str, Plugin] = {} + downloads: dict[str, Plugin] = {} with open(self.cache_file) as f: data = json.load(f) for attr in data.values(): @@ -707,7 +831,7 @@ def store(self) -> None: data[name] = attr.as_json() json.dump(data, f, indent=4, sort_keys=True) - def __getitem__(self, key: str) -> Optional[Plugin]: + def __getitem__(self, key: str) -> Plugin | None: return self.downloads.get(key, None) def __setitem__(self, key: str, value: Plugin) -> None: @@ -716,7 +840,7 @@ def __setitem__(self, key: str, value: Plugin) -> None: def prefetch( pluginDesc: PluginDesc, cache: Cache -) -> Tuple[PluginDesc, Union[Exception, Plugin], Optional[Repo]]: +) -> tuple[PluginDesc, Exception | Plugin, Repo | None]: try: plugin, redirect = prefetch_plugin(pluginDesc, cache) cache[plugin.commit] = plugin @@ -731,7 +855,7 @@ def rewrite_input( deprecated: Path, # old pluginDesc and the new redirects: Redirects = {}, - append: List[PluginDesc] = [], + append: list[PluginDesc] = [], ): log.info("Rewriting input file %s", input_file) plugins = load_plugins_from_csv(config, input_file) @@ -779,7 +903,7 @@ def rewrite_input( writer.writerow(asdict(plugin)) -def commit(repo: git.Repo, message: str, files: List[Path]) -> None: +def commit(repo: git.Repo, message: str, files: list[Path]) -> None: repo.index.add([str(f.resolve()) for f in files]) if repo.index.diff("HEAD"): @@ -802,7 +926,14 @@ def update_plugins(editor: Editor, args): ) fetch_config = FetchConfig(args.proc, args.github_token) - update = editor.get_update(args.input_file, args.outfile, fetch_config) + update = editor.get_update( + input_file=args.input_file, + output_file=args.outfile, + config=fetch_config, + to_update=getattr( # if script was called without arguments + args, "update_only", None + ), + ) start_time = time.time() redirects = update() diff --git a/pkgs/applications/editors/kakoune/plugins/update.py b/pkgs/applications/editors/kakoune/plugins/update.py index 333c96807dd4c..9b06d0c06e9c8 100755 --- a/pkgs/applications/editors/kakoune/plugins/update.py +++ b/pkgs/applications/editors/kakoune/plugins/update.py @@ -12,7 +12,6 @@ import os import sys from pathlib import Path -from typing import List, Tuple # Import plugin update library from maintainers/scripts/pluginupdate.py ROOT = Path(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))) # type: ignore @@ -21,13 +20,11 @@ ) import pluginupdate -GET_PLUGINS = f"""( -with import {{ }}; +GET_PLUGINS = f"""with import {{ }}; let inherit (kakouneUtils.override {{ }}) buildKakounePluginFrom2Nix; - generated = callPackage {ROOT}/generated.nix {{ - inherit buildKakounePluginFrom2Nix; - }}; + generated = callPackage {ROOT}/generated.nix {{ inherit buildKakounePluginFrom2Nix; }}; + hasChecksum = value: lib.isAttrs value @@ -35,20 +32,23 @@ "src" "outputHash" ] value; - getChecksum = - name: value: - if hasChecksum value then - {{ - submodules = value.src.fetchSubmodules or false; - sha256 = value.src.outputHash; - rev = value.src.rev; - }} - else - null; - checksums = lib.mapAttrs getChecksum generated; + + parse = name: value: {{ + pname = value.pname; + version = value.version; + homePage = value.meta.homepage; + checksum = + if hasChecksum value then + {{ + submodules = value.src.fetchSubmodules or false; + sha256 = value.src.outputHash; + rev = value.src.rev; + }} + else + null; + }}; in -lib.filterAttrs (n: v: v != null) checksums -)""" +lib.mapAttrs parse generated""" HEADER = "# This file has been @generated by ./pkgs/applications/editors/kakoune/plugins/update.py. Do not edit!" @@ -56,7 +56,7 @@ class KakouneEditor(pluginupdate.Editor): def generate_nix( self, - plugins: List[Tuple[pluginupdate.PluginDesc, pluginupdate.Plugin]], + plugins: list[tuple[pluginupdate.PluginDesc, pluginupdate.Plugin]], outfile: str, ): with open(outfile, "w+") as f: diff --git a/pkgs/applications/editors/vim/plugins/get-plugins.nix b/pkgs/applications/editors/vim/plugins/get-plugins.nix index 6c587a94821f4..c32f3880d6f2e 100644 --- a/pkgs/applications/editors/vim/plugins/get-plugins.nix +++ b/pkgs/applications/editors/vim/plugins/get-plugins.nix @@ -6,6 +6,7 @@ let generated = callPackage { inherit buildNeovimPlugin buildVimPlugin; } { } { }; + hasChecksum = value: lib.isAttrs value @@ -13,16 +14,20 @@ let "src" "outputHash" ] value; - getChecksum = - name: value: - if hasChecksum value then - { - submodules = value.src.fetchSubmodules or false; - sha256 = value.src.outputHash; - rev = value.src.rev; - } - else - null; - checksums = lib.mapAttrs getChecksum generated; + + parse = name: value: { + pname = value.pname; + version = value.version; + homePage = value.meta.homepage; + checksum = + if hasChecksum value then + { + submodules = value.src.fetchSubmodules or false; + sha256 = value.src.outputHash; + rev = value.src.rev; + } + else + null; + }; in -lib.filterAttrs (n: v: v != null) checksums +lib.mapAttrs parse generated diff --git a/pkgs/by-name/lu/luarocks-packages-updater/updater.py b/pkgs/by-name/lu/luarocks-packages-updater/updater.py index 2471e46f5f427..49ac8f8379df6 100755 --- a/pkgs/by-name/lu/luarocks-packages-updater/updater.py +++ b/pkgs/by-name/lu/luarocks-packages-updater/updater.py @@ -17,7 +17,6 @@ from dataclasses import dataclass from multiprocessing.dummy import Pool from pathlib import Path -from typing import List, Optional, Tuple import pluginupdate from pluginupdate import FetchConfig, update_plugins @@ -49,18 +48,18 @@ class LuaPlugin: """Name of the plugin, as seen on luarocks.org""" rockspec: str """Full URI towards the rockspec""" - ref: Optional[str] + ref: str | None """git reference (branch name/tag)""" - version: Optional[str] + version: str | None """Set it to pin a package """ - server: Optional[str] + server: str | None """luarocks.org registers packages under different manifests. Its value can be 'http://luarocks.org/dev' """ - luaversion: Optional[str] + luaversion: str | None """lua version if a package is available only for a specific lua version""" - maintainers: Optional[str] - """ Optional string listing maintainers separated by spaces""" + maintainers: str | None + """Optional string listing maintainers separated by spaces""" @property def normalized_name(self) -> str: @@ -77,7 +76,7 @@ def create_parser(self): def get_current_plugins(self): return [] - def load_plugin_spec(self, input_file) -> List[LuaPlugin]: + def load_plugin_spec(self, input_file) -> list[LuaPlugin]: luaPackages = [] csvfilename = input_file log.info("Loading package descriptions from %s", csvfilename) @@ -95,7 +94,7 @@ def load_plugin_spec(self, input_file) -> List[LuaPlugin]: def update(self, args): update_plugins(self, args) - def generate_nix(self, results: List[Tuple[LuaPlugin, str]], outfilename: str): + def generate_nix(self, results: list[tuple[LuaPlugin, str]], outfilename: str): with tempfile.NamedTemporaryFile("w+") as f: f.write(HEADER) header2 = textwrap.dedent( @@ -121,7 +120,16 @@ def generate_nix(self, results: List[Tuple[LuaPlugin, str]], outfilename: str): def attr_path(self): return "luaPackages" - def get_update(self, input_file: str, outfile: str, config: FetchConfig): + def get_update( + self, + input_file: str, + outfile: str, + config: FetchConfig, + # TODO: implement support for adding/updating individual plugins + to_update: list[str] | None, + ): + if to_update is not None: + raise NotImplementedError("For now, lua updater doesn't support updating individual packages.") _prefetch = generate_pkg_nix def update() -> dict: