feat: <FINALLY> implement lookup by provides field (fixes #402)

pikaur/aur.py

@@ -8,6 +8,7 @@
 from .config import PikaurConfig
 from .core import DataType
 from .exceptions import AURError
+from .logging import create_logger
 from .progressbar import ThreadSafeProgressBar
 from .urllib_helper import get_gzip_from_url, get_json_from_url
 
@@ -20,6 +21,16 @@
 MAX_URL_LENGTH: "Final" = 8177  # default value in many web servers
 
 
+logger = create_logger("aur_module")
+
+
+class NotFound:
+    pass
+
+
+NOT_FOUND: "Final[NotFound]" = NotFound()
+
+
 class AurRPCErrors:
     ERROR_KEY: "Final" = "error"
     TOO_MANY_RESULTS: "Final" = "Too many package results."
@@ -137,12 +148,14 @@ def strip_aur_repo_name(pkg_name: str) -> str:
     return pkg_name
 
 
-def aur_rpc_search_name_desc(search_query: str) -> list[AURPackageInfo]:
+def aur_rpc_search(
+        search_query: str, search_by: str = "name-desc",
+) -> list[AURPackageInfo]:
     url = construct_aur_rpc_url_from_params({
         "v": 5,
         "type": "search",
         "arg": strip_aur_repo_name(search_query),
-        "by": "name-desc",
+        "by": search_by,
     })
     result_json = get_json_from_url(url)
     if AurRPCErrors.ERROR_KEY in result_json:
@@ -156,6 +169,10 @@ def aur_rpc_search_name_desc(search_query: str) -> list[AURPackageInfo]:
     ]
 
 
+def aur_rpc_search_provider(package_name: str) -> tuple[str, list[AURPackageInfo]]:
+    return package_name, aur_rpc_search(search_query=package_name, search_by="provides")
+
+
 def _get_aur_rpc_info_url(search_queries: list[str]) -> str:
     uri = parse.urlencode({
         "v": 5,
@@ -181,7 +198,8 @@ def aur_rpc_info(search_queries: list[str]) -> list[AURPackageInfo]:
 
 
 def aur_rpc_info_with_progress(
-        search_queries: list[str], *, progressbar_length: int, with_progressbar: bool,
+        search_queries: list[str],
+        *, progressbar_length: int, with_progressbar: bool,
 ) -> list[AURPackageInfo]:
     result = aur_rpc_info(search_queries)
     if with_progressbar:
@@ -193,6 +211,20 @@ def aur_rpc_info_with_progress(
     return result
 
 
+def aur_rpc_search_provider_with_progress(
+        package_name: str,
+        *, progressbar_length: int, with_progressbar: bool,
+) -> tuple[str, list[AURPackageInfo]]:
+    result = aur_rpc_search_provider(package_name=package_name)
+    if with_progressbar:
+        progressbar = ThreadSafeProgressBar.get(
+            progressbar_length=progressbar_length,
+            progressbar_id="aur_provides_search",
+        )
+        progressbar.update()
+    return result
+
+
 class AurPackageListCache:
 
     cache: ClassVar[list[str]] = []
@@ -206,17 +238,34 @@ def get(cls) -> list[str]:
 
 class AurPackageSearchCache:
 
-    cache: ClassVar[dict[str, AURPackageInfo]] = {}
+    cache: ClassVar[dict[str, AURPackageInfo | NotFound]] = {}
 
     @classmethod
     def put(cls, pkg: AURPackageInfo) -> None:
         cls.cache[pkg.name] = pkg
 
     @classmethod
-    def get(cls, pkg_name: str) -> AURPackageInfo | None:
+    def put_not_found(cls, pkg_name: str) -> None:
+        cls.cache[pkg_name] = NOT_FOUND
+
+    @classmethod
+    def get(cls, pkg_name: str) -> AURPackageInfo | NotFound | None:
         return cls.cache.get(pkg_name)
 
 
+class AurProvidedPackageSearchCache:
+
+    cache: ClassVar[dict[str, list[AURPackageInfo]]] = {}
+
+    @classmethod
+    def put(cls, provides: str, pkgs: list[AURPackageInfo]) -> None:
+        cls.cache[provides] = pkgs
+
+    @classmethod
+    def get(cls, provides: str) -> list[AURPackageInfo] | None:
+        return cls.cache.get(provides)
+
+
 def get_all_aur_names() -> list[str]:
     return AurPackageListCache.get()
 
@@ -243,12 +292,20 @@ def find_aur_packages(
     # @TODO: return only packages for the current architecture
     package_names = [strip_aur_repo_name(name) for name in package_names]
     num_packages = len(package_names)
-    json_results = []
+    json_results: list[AURPackageInfo] = []
+    cached_not_found_pkgs: list[str] = []
     for package_name in package_names[:]:
         aur_pkg = AurPackageSearchCache.get(package_name)
-        if aur_pkg:
+        if aur_pkg is NOT_FOUND:
+            package_names.remove(package_name)
+            cached_not_found_pkgs.append(package_name)
+            logger.debug("find_aur_packages: {} cached as not found", package_name)
+        elif isinstance(aur_pkg, AURPackageInfo):
             json_results.append(aur_pkg)
             package_names.remove(package_name)
+            logger.debug("find_aur_packages: {} cached", package_name)
+        else:
+            logger.debug("find_aur_packages: {} uncached", package_name)
 
     if package_names:
         with ThreadPool() as pool:
@@ -272,6 +329,7 @@ def find_aur_packages(
 
     found_aur_packages = [
         result.name for result in json_results
+        if isinstance(result, AURPackageInfo)
     ]
     not_found_packages: list[str] = (
         [] if num_packages == len(found_aur_packages)
@@ -280,9 +338,74 @@ def find_aur_packages(
             if package not in found_aur_packages
         ]
     )
+    for not_found_pkgname in not_found_packages:
+        AurPackageSearchCache.put_not_found(not_found_pkgname)
+    not_found_packages += cached_not_found_pkgs
     return json_results, not_found_packages
 
 
+def find_aur_provided_deps(
+        package_names: list[str], *, with_progressbar: bool = False,
+) -> tuple[list[AURPackageInfo], list[str]]:
+
+    # @TODO: return only packages for the current architecture
+    package_names = [strip_aur_repo_name(name) for name in package_names]
+    num_packages = len(package_names)
+    json_results = []
+    cached_not_found_pkgs: list[str] = []
+    for package_name in package_names[:]:
+        aur_pkgs = AurProvidedPackageSearchCache.get(package_name)
+        if aur_pkgs is None:
+            logger.debug("find_aur_provided_deps: {} not cached", package_name)
+        elif len(aur_pkgs) == 0:
+            package_names.remove(package_name)
+            cached_not_found_pkgs.append(package_name)
+            logger.debug("find_aur_provided_deps: {} cached as not found", package_name)
+        else:
+            # @TODO: dynamicly select package provider
+            json_results.append(aur_pkgs[0])
+            package_names.remove(package_name)
+            logger.debug("find_aur_provided_deps: {} cached", package_name)
+
+    if package_names:
+        with ThreadPool() as pool:
+            requests = [
+                pool.apply_async(aur_rpc_search_provider_with_progress, [], {
+                    "package_name": package_name,
+                    "progressbar_length": len(package_names),
+                    "with_progressbar": with_progressbar,
+                })
+                for package_name in package_names
+            ]
+            pool.close()
+            results = [request.get() for request in requests]
+            pool.join()
+            for provided_pkg_name, aur_pkgs in results:
+                if not aur_pkgs:
+                    continue
+                AurProvidedPackageSearchCache.put(pkgs=aur_pkgs, provides=provided_pkg_name)
+                for aur_pkg in aur_pkgs:
+                    if provided_pkg_name in package_names:
+                        # @TODO: dynamicly select package provider
+                        json_results += [aur_pkg]
+                        break
+
+    found_aur_packages = [
+        result.name for result in json_results
+    ]
+    not_found_packages: list[str] = (
+        [] if num_packages == len(found_aur_packages)
+        else [
+            package for package in package_names
+            if package not in found_aur_packages
+        ]
+    )
+    result_names = list({pkg.name for pkg in json_results})
+    full_pkg_infos, _ = find_aur_packages(result_names)
+    not_found_packages += cached_not_found_pkgs
+    return full_pkg_infos, not_found_packages
+
+
 def get_repo_url(package_base_name: str) -> str:
     return f"{AurBaseUrl.get()}/{package_base_name}.git"
 

pikaur/aur_deps.py

@@ -3,14 +3,15 @@
 from multiprocessing.pool import ThreadPool
 from typing import TYPE_CHECKING
 
-from .aur import find_aur_packages
+from .aur import find_aur_packages, find_aur_provided_deps
 from .core import PackageSource
 from .exceptions import (
     DependencyVersionMismatchError,
     PackagesNotFoundInAURError,
     PackagesNotFoundInRepoError,
 )
 from .i18n import translate
+from .logging import create_logger
 from .pacman import PackageDB
 from .pprint import print_error
 from .version import VersionMatcher
@@ -19,6 +20,9 @@
     from .aur import AURPackageInfo
 
 
+logger = create_logger("aur_deps")
+
+
 def check_deps_versions(
         deps_pkg_names: list[str],
         version_matchers: dict[str, VersionMatcher],
@@ -245,7 +249,11 @@ def find_missing_deps_for_aur_pkg(
     aur_deps_info, not_found_aur_deps = find_aur_packages(
         not_found_repo_pkgs,
     )
-    # @TODO: find packages Provided by AUR packages
+    provided_aur_deps_info, not_found_aur_deps = find_aur_provided_deps(
+        not_found_aur_deps,
+    )
+    aur_deps_info += provided_aur_deps_info
+
     handle_not_found_aur_pkgs(
         aur_pkg_name=aur_pkg_name,
         aur_pkgs_info=aur_pkgs_info,
@@ -254,17 +262,31 @@ def find_missing_deps_for_aur_pkg(
     )
 
     # check versions of found AUR packages:
+    logger.debug("version_matchers={}", version_matchers)
     for aur_dep_info in aur_deps_info:
         aur_dep_name = aur_dep_info.name
-        version_matcher = version_matchers[aur_dep_name]
-        if not version_matcher(aur_dep_info.version):
-            raise DependencyVersionMismatchError(
-                version_found=aur_dep_info.version,
-                dependency_line=version_matcher.line,
-                who_depends=aur_pkg_name,
-                depends_on=aur_dep_name,
-                location=PackageSource.AUR,
-            )
+        version_matcher = version_matchers.get(
+            aur_dep_name,
+        )
+        pkg_version_matchers: list[VersionMatcher] = []
+        if version_matcher:
+            pkg_version_matchers = [version_matcher]
+        else:
+            for provide in aur_dep_info.provides:
+                version_matcher = version_matchers.get(VersionMatcher(provide).pkg_name)
+                if version_matcher is not None:
+                    pkg_version_matchers.append(version_matcher)
+        logger.debug("{} pkg version_matchers={}", aur_dep_name, pkg_version_matchers)
+        for version_matcher in pkg_version_matchers:
+            if not version_matcher(aur_dep_info.version):
+                raise DependencyVersionMismatchError(
+                    version_found=aur_dep_info.version,
+                    dependency_line=version_matcher.line,
+                    who_depends=aur_pkg_name,
+                    depends_on=aur_dep_name,
+                    location=PackageSource.AUR,
+                )
+            # not_found_repo_pkgs.remove(version_matcher.pkg_name)
 
     return not_found_repo_pkgs
 
@@ -280,6 +302,7 @@ def find_aur_deps(  # pylint: disable=too-many-branches
         aur_pkg.name
         for aur_pkg in aur_pkgs_infos
     ]
+    logger.debug("find_aur_deps: package_names={}", package_names)
     result_aur_deps: dict[str, list[str]] = {}
 
     initial_pkg_infos = initial_pkg_infos2_todo = aur_pkgs_infos[:]  # @TODO: var name
@@ -292,7 +315,12 @@ def find_aur_deps(  # pylint: disable=too-many-branches
             initial_pkg_infos = []
         else:
             aur_pkgs_info, not_found_aur_pkgs = find_aur_packages(iter_package_names)
+            provided_aur_deps_info, not_found_aur_pkgs = find_aur_provided_deps(
+                not_found_aur_pkgs,
+            )
+            aur_pkgs_info += provided_aur_deps_info
         if not_found_aur_pkgs:
+            logger.debug("not_found_aur_pkgs={}", not_found_aur_pkgs)
             raise PackagesNotFoundInAURError(packages=not_found_aur_pkgs)
         for aur_pkg in aur_pkgs_info:
             aur_pkg_deps = get_aur_pkg_deps_and_version_matchers(
@@ -320,7 +348,11 @@ def find_aur_deps(  # pylint: disable=too-many-branches
             for aur_pkg_name, request in all_requests.items():
                 try:
                     results = request.get()
-                except Exception:
+                except Exception as exc:
+                    logger.debug(
+                        "exception during aur search: {}: {}",
+                        exc.__class__.__name__, exc,
+                    )
                     print_error(translate(
                         "Can't resolve dependencies for AUR package '{pkg}':",
                     ).format(pkg=aur_pkg_name))
@@ -336,6 +368,7 @@ def find_aur_deps(  # pylint: disable=too-many-branches
                 new_aur_deps.append(pkg_name)
                 iter_package_names.append(pkg_name)
 
+    logger.debug("find_aur_deps: result_aur_deps={}", result_aur_deps)
     return result_aur_deps