Merge pull request #838 from tsteven4/qtextensions

search for modules in extensions directory

aqt/archives.py

@@ -33,6 +33,12 @@
 from aqt.metadata import QtRepoProperty, Version
 
 
+@dataclass
+class UpdateXmls:
+    target_folder: str
+    xml_text: str
+
+
 @dataclass
 class TargetConfig:
     version: str
@@ -371,6 +377,7 @@ def _target_packages(self) -> ModuleToPackage:
             package_names = [
                 f"qt.qt{self.version.major}.{self._version_str()}.{suffix}",
                 f"qt.{self._version_str()}.{suffix}",
+                f"extensions.{module}.{self._version_str()}.{self.arch}",
             ]
             if not module.startswith("addons."):
                 package_names.append(f"qt.qt{self.version.major}.{self._version_str()}.addons.{suffix}")
@@ -416,7 +423,25 @@ def _get_archives_base(self, name, target_packages):
         )
         update_xml_url = posixpath.join(os_target_folder, "Updates.xml")
         update_xml_text = self._download_update_xml(update_xml_url)
-        self._parse_update_xml(os_target_folder, update_xml_text, target_packages)
+        update_xmls = [UpdateXmls(os_target_folder, update_xml_text)]
+
+        if self.version >= Version("6.8.0"):
+            arch = self.arch
+            if self.os_name == "windows":
+                arch = self.arch.replace("win64_", "", 1)
+            elif self.os_name == "linux":
+                arch = "x86_64"
+            elif self.os_name == "linux_arm64":
+                arch = "arm64"
+            for ext in ["qtwebengine", "qtpdf"]:
+                extensions_target_folder = posixpath.join(
+                    "online/qtsdkrepository", os_name, "extensions", ext, self._version_str(), arch
+                )
+                extensions_xml_url = posixpath.join(extensions_target_folder, "Updates.xml")
+                extensions_xml_text = self._download_update_xml(extensions_xml_url)
+                update_xmls.append(UpdateXmls(extensions_target_folder, extensions_xml_text))
+
+        self._parse_update_xmls(update_xmls, target_packages)
 
     def _download_update_xml(self, update_xml_path):
         """Hook for unit test."""
@@ -471,6 +496,12 @@ def _parse_update_xml(self, os_target_folder, update_xml_text, target_packages:
                         pkg_update_name=packageupdate.name,  # For testing purposes
                     )
                 )
+
+    def _parse_update_xmls(self, update_xmls, target_packages: Optional[ModuleToPackage]):
+        if not target_packages:
+            target_packages = ModuleToPackage({})
+        for update_xml in update_xmls:
+            self._parse_update_xml(update_xml.target_folder, update_xml.xml_text, target_packages)
         # if we have located every requested package, then target_packages will be empty
         if not self.all_extra and len(target_packages) > 0:
             message = f"The packages {target_packages} were not found while parsing XML of package information!"

aqt/metadata.py

@@ -231,14 +231,39 @@ def is_qt(self) -> bool:
     def is_tools(self) -> bool:
         return self.category == "tools"
 
-    def to_url(self) -> str:
-        return "online/qtsdkrepository/{os}{arch}/{target}/".format(
+    def to_os_arch(self) -> str:
+        return "{os}{arch}".format(
             os=self.host,
             arch=(
                 "_x86"
                 if self.host == "windows"
                 else ("" if self.host in ("linux_arm64", "all_os", "windows_arm64") else "_x64")
             ),
+        )
+
+    def to_extension_folder(self, module, version, arch) -> str:
+        extarch = arch
+        if self.host == "windows":
+            extarch = arch.replace("win64_", "", 1)
+        elif self.host == "linux":
+            extarch = "x86_64"
+        elif self.host == "linux_arm64":
+            extarch = "arm64"
+        return "online/qtsdkrepository/{osarch}/extensions/{ext}/{ver}/{extarch}/".format(
+            osarch=self.to_os_arch(),
+            ext=module,
+            ver=version,
+            extarch=extarch,
+        )
+
+    def to_extension_url(self) -> str:
+        return "online/qtsdkrepository/{osarch}/extensions/".format(
+            osarch=self.to_os_arch(),
+        )
+
+    def to_url(self) -> str:
+        return "online/qtsdkrepository/{osarch}/{target}/".format(
+            osarch=self.to_os_arch(),
             target=self.target,
         )
 
@@ -667,6 +692,10 @@ def filter_by(ver: Version, ext: str) -> bool:
     def fetch_latest_version(self, ext: str) -> Optional[Version]:
         return self.fetch_versions(ext).latest()
 
+    def fetch_extensions(self) -> List[str]:
+        html_doc = self.fetch_http(self.archive_id.to_extension_url(), False)
+        return list(self.iterate_folders(html_doc, self.base_url))
+
     def fetch_tools(self) -> List[str]:
         html_doc = self.fetch_http(self.archive_id.to_url(), False)
         return list(self.iterate_folders(html_doc, self.base_url, filter_category="tools"))
@@ -824,6 +853,14 @@ def _fetch_module_metadata(self, folder: str, predicate: Optional[Callable[[Elem
             predicate=predicate if predicate else MetadataFactory._has_nonempty_downloads,
         )
 
+    def _fetch_extension_metadata(self, url: str, predicate: Optional[Callable[[Element], bool]] = None):
+        rest_of_url = posixpath.join(url, "Updates.xml")
+        xml = self.fetch_http(rest_of_url) if not Settings.ignore_hash else self.fetch_http(rest_of_url, False)
+        return xml_to_modules(
+            xml,
+            predicate=predicate if predicate else MetadataFactory._has_nonempty_downloads,
+        )
+
     def fetch_modules(self, version: Version, arch: str) -> List[str]:
         """Returns list of modules"""
         extension = QtRepoProperty.extension_for_arch(arch, version >= Version("6.0.0"))
@@ -849,6 +886,9 @@ def to_module_arch(name: str) -> Tuple[Optional[str], Optional[str]]:
             module, _arch = to_module_arch(name)
             if _arch == arch:
                 modules.add(cast(str, module))
+        if version >= Version("6.8.0"):
+            for ext in self.fetch_extensions():
+                modules.add(ext)
         return sorted(modules)
 
     @staticmethod
@@ -863,6 +903,8 @@ def fetch_long_modules(self, version: Version, arch: str) -> ModuleData:
         extension = QtRepoProperty.extension_for_arch(arch, version >= Version("6.0.0"))
         qt_ver_str = self._get_qt_version_str(version)
         # Example: re.compile(r"^(preview\.)?qt\.(qt5\.)?590(\.addons)?\.(?P<module>[^.]+)\.gcc_64$")
+        #          qt.qt6.680.addons.qtwebsockets.win64_msvc2022_64
+        #          qt.qt6.680.debug_info.win64_msvc2022_64
         pattern = re.compile(
             r"^(preview\.)?qt\.(qt"
             + str(version.major)
@@ -885,6 +927,18 @@ def matches_arch(element: Element) -> bool:
                 if module is not None:
                     m[module] = value
 
+        # Examples: extensions.qtwebengine.680.debug_information
+        #           extensions.qtwebengine.680.win64_msvc2022_64
+        ext_pattern = re.compile(r"^extensions\." + r"(?P<module>[^.]+)\." + qt_ver_str + r"\." + arch + r"$")
+        if version >= Version("6.8.0"):
+            for ext in self.fetch_extensions():
+                ext_meta = self._fetch_extension_metadata(self.archive_id.to_extension_folder(ext, qt_ver_str, arch))
+                for key, value in ext_meta.items():
+                    ext_match = ext_pattern.match(key)
+                    if ext_match is not None:
+                        module = ext_match.group("module")
+                        if module is not None:
+                            m[module] = value
         return ModuleData(m)
 
     def fetch_modules_sde(self, cmd_type: str, version: Version) -> List[str]: