Improve toolset checkForUpdate logic.

Libraries/PyKotor/src/pykotor/resource/type.py

@@ -58,6 +58,7 @@ class ResourceTuple(NamedTuple):
     category: str
     contents: str
     is_invalid: bool = False
+    is_plaintext_gff: bool = False
 
     def __getitem__(self, key):
         return getattr(self, key)
@@ -204,31 +205,31 @@ class ResourceType(Enum):
     TLK_XML = ResourceTuple(50001, "tlk.xml", "Talk Tables", "plaintext")
     MDL_ASCII = ResourceTuple(50002, "mdl.ascii", "Models", "plaintext")
     TwoDA_CSV = ResourceTuple(50003, "2da.csv", "2D Arrays", "plaintext")
-    GFF_XML = ResourceTuple(50004, "gff.xml", "Other", "plaintext")
-    IFO_XML = ResourceTuple(50005, "ifo.xml", "Module Data", "plaintext")
-    GIT_XML = ResourceTuple(50006, "git.xml", "Module Data", "plaintext")
-    UTI_XML = ResourceTuple(50007, "uti.xml", "Items", "plaintext")
-    UTC_XML = ResourceTuple(50008, "utc.xml", "Creatures", "plaintext")
-    DLG_XML = ResourceTuple(50009, "dlg.xml", "Dialogs", "plaintext")
+    GFF_XML = ResourceTuple(50004, "gff.xml", "Other", "plaintext", is_plaintext_gff=True)
+    IFO_XML = ResourceTuple(50005, "ifo.xml", "Module Data", "plaintext", is_plaintext_gff=True)
+    GIT_XML = ResourceTuple(50006, "git.xml", "Module Data", "plaintext", is_plaintext_gff=True)
+    UTI_XML = ResourceTuple(50007, "uti.xml", "Items", "plaintext", is_plaintext_gff=True)
+    UTC_XML = ResourceTuple(50008, "utc.xml", "Creatures", "plaintext", is_plaintext_gff=True)
+    DLG_XML = ResourceTuple(50009, "dlg.xml", "Dialogs", "plaintext", is_plaintext_gff=True)
     ITP_XML = ResourceTuple(50010, "itp.xml", "Palettes", "plaintext")
-    UTT_XML = ResourceTuple(50011, "utt.xml", "Triggers", "plaintext")
-    UTS_XML = ResourceTuple(50012, "uts.xml", "Sounds", "plaintext")
-    FAC_XML = ResourceTuple(50013, "fac.xml", "Factions", "plaintext")
-    UTE_XML = ResourceTuple(50014, "ute.xml", "Encounters", "plaintext")
-    UTD_XML = ResourceTuple(50015, "utd.xml", "Doors", "plaintext")
-    UTP_XML = ResourceTuple(50016, "utp.xml", "Placeables", "plaintext")
-    GUI_XML = ResourceTuple(50017, "gui.xml", "GUIs", "plaintext")
-    UTM_XML = ResourceTuple(50018, "utm.xml", "Merchants", "plaintext")
-    JRL_XML = ResourceTuple(50019, "jrl.xml", "Journals", "plaintext")
-    UTW_XML = ResourceTuple(50020, "utw.xml", "Waypoints", "plaintext")
-    PTH_XML = ResourceTuple(50021, "pth.xml", "Paths", "plaintext")
+    UTT_XML = ResourceTuple(50011, "utt.xml", "Triggers", "plaintext", is_plaintext_gff=True)
+    UTS_XML = ResourceTuple(50012, "uts.xml", "Sounds", "plaintext", is_plaintext_gff=True)
+    FAC_XML = ResourceTuple(50013, "fac.xml", "Factions", "plaintext", is_plaintext_gff=True)
+    UTE_XML = ResourceTuple(50014, "ute.xml", "Encounters", "plaintext", is_plaintext_gff=True)
+    UTD_XML = ResourceTuple(50015, "utd.xml", "Doors", "plaintext", is_plaintext_gff=True)
+    UTP_XML = ResourceTuple(50016, "utp.xml", "Placeables", "plaintext", is_plaintext_gff=True)
+    GUI_XML = ResourceTuple(50017, "gui.xml", "GUIs", "plaintext", is_plaintext_gff=True)
+    UTM_XML = ResourceTuple(50018, "utm.xml", "Merchants", "plaintext", is_plaintext_gff=True)
+    JRL_XML = ResourceTuple(50019, "jrl.xml", "Journals", "plaintext", is_plaintext_gff=True)
+    UTW_XML = ResourceTuple(50020, "utw.xml", "Waypoints", "plaintext", is_plaintext_gff=True)
+    PTH_XML = ResourceTuple(50021, "pth.xml", "Paths", "plaintext", is_plaintext_gff=True)
     LIP_XML = ResourceTuple(50022, "lip.xml", "Lips", "plaintext")
     SSF_XML = ResourceTuple(50023, "ssf.xml", "Soundsets", "plaintext")
-    ARE_XML = ResourceTuple(50023, "are.xml", "Module Data", "plaintext")
+    ARE_XML = ResourceTuple(50023, "are.xml", "Module Data", "plaintext", is_plaintext_gff=True)
     TwoDA_JSON = ResourceTuple(50024, "2da.json", "2D Arrays", "plaintext")
     TLK_JSON = ResourceTuple(50025, "tlk.json", "Talk Tables", "plaintext")
     LIP_JSON = ResourceTuple(50026, "lip.json", "Lips", "plaintext")
-    RES_XML = ResourceTuple(50027, "res.xml", "Save Data", "plaintext")
+    RES_XML = ResourceTuple(50027, "res.xml", "Save Data", "plaintext", is_plaintext_gff=True)
 
     def __new__(cls, *args, **kwargs):
         obj: ResourceType = object.__new__(cls)  # type: ignore[annotation-unchecked]
@@ -246,12 +247,14 @@ def __init__(
         category: str,
         contents: str,
         is_invalid: bool = False,  # noqa: FBT001, FBT002
+        is_plaintext_gff: bool = False,  # noqa: FBT001, FBT002
     ):
         self.type_id: int = type_id  # type: ignore[misc]
         self.extension: str = extension.strip().lower()
         self.category: str = category
         self.contents: str = contents
         self.is_invalid: bool = is_invalid
+        self.is_plaintext_gff: bool = is_plaintext_gff
 
     def __bool__(self) -> bool:
         return not self.is_invalid

Tools/HolocronToolset/src/toolset/config.py

@@ -1,10 +1,25 @@
 from __future__ import annotations
 
-# config.py
+import base64
+import json
+import re
+
+from contextlib import suppress
+from typing import TYPE_CHECKING, Any
+
+import requests
+
+from PyQt5.QtWidgets import QMessageBox
+
+from utility.error_handling import universal_simplify_exception
+from utility.system.path import Path, PurePath
+
+if TYPE_CHECKING:
+    import os
 
 LOCAL_PROGRAM_INFO = \
 {  #<---JSON_START--->#{
-    "currentVersion": "2.2.1b18",
+    "currentVersion": "2.2.1",
     "toolsetLatestVersion": "2.1.2",
     "toolsetLatestBetaVersion": "2.2.1b18",
     "updateInfoLink": "https://api.github.com/repos/NickHugi/PyKotor/contents/Tools/HolocronToolset/src/toolset/config.py",
@@ -20,3 +35,142 @@
     },
     "help": {"version": 3}
 } #<---JSON_END--->#
+
+CURRENT_VERSION = LOCAL_PROGRAM_INFO["currentVersion"]
+
+def getRemoteToolsetUpdateInfo(*, useBetaChannel: bool = False, silent: bool = False) -> Exception | dict[str, Any]:
+    if useBetaChannel:
+        UPDATE_INFO_LINK = LOCAL_PROGRAM_INFO["updateBetaInfoLink"]
+    else:
+        UPDATE_INFO_LINK = LOCAL_PROGRAM_INFO["updateInfoLink"]
+
+    try:  # Download this same file config.py from the repo and only parse the json between the markers. This prevents remote execution security issues.
+        req = requests.get(UPDATE_INFO_LINK, timeout=15)
+        req.raise_for_status()
+        file_data = req.json()
+        base64_content = file_data["content"]
+        decoded_content = base64.b64decode(base64_content)  # Correctly decoding the base64 content
+        decoded_content_str = decoded_content.decode(encoding="utf-8")
+        # use for testing only:
+        #with open("config.py") as f:
+        #    decoded_content_str = f.read()
+        # Use regex to extract the JSON part between the markers
+        json_data_match = re.search(r"<---JSON_START--->\#(.*)\#<---JSON_END--->", decoded_content_str, flags=re.DOTALL)
+
+        if json_data_match:
+            json_str = json_data_match.group(1)
+            remoteInfo = json.loads(json_str)
+            if not isinstance(remoteInfo, dict):
+                raise TypeError(f"Expected remoteInfo to be a dict, instead got type {remoteInfo.__class__.__name__}")  # noqa: TRY301
+        else:
+            raise ValueError(f"JSON data not found or markers are incorrect: {json_data_match}")  # noqa: TRY301
+    except Exception as e:  # noqa: BLE001
+        errMsg = str(universal_simplify_exception(e))
+        result = silent or QMessageBox.question(
+            parent=None,
+            title="Error occurred fetching update information.",
+            text=(
+                "An error occurred while fetching the latest toolset information.<br><br>" +
+                errMsg.replace("\n", "<br>") +
+                "<br><br>" +
+                "Would you like to check against the local database instead?"
+            ),
+            buttons=QMessageBox.Yes | QMessageBox.No,
+            defaultButton=QMessageBox.Yes,
+        )
+        if result not in {QMessageBox.Yes, True}:
+            return e
+        remoteInfo = LOCAL_PROGRAM_INFO
+    return remoteInfo
+
+
+def remoteVersionNewer(localVersion: str, remoteVersion: str) -> bool | None:
+    version_check: bool | None = None
+    with suppress(Exception):
+        from packaging import version
+
+        version_check = version.parse(remoteVersion) > version.parse(localVersion)
+    if version_check is None:
+        with suppress(Exception):
+            from distutils.version import LooseVersion
+
+            version_check = LooseVersion(remoteVersion) > LooseVersion(localVersion)
+    return version_check
+
+
+def download_github_file(
+    url_or_repo: str,
+    local_path: os.PathLike | str,
+    repo_path: os.PathLike | str | None = None,
+):
+    local_path = Path(local_path)
+    local_path.parent.mkdir(parents=True, exist_ok=True)
+
+    if repo_path is not None:
+        # Construct the API URL for the file in the repository
+        owner, repo = PurePath(url_or_repo).parts[-2:]
+        api_url = f"https://api.github.com/repos/{owner}/{repo}/contents/{PurePath(repo_path).as_posix()}"
+
+        file_info: dict[str, str] = _request_api_data(api_url)
+        # Check if it's a file and get the download URL
+        if file_info["type"] == "file":
+            download_url = file_info["download_url"]
+        else:
+            msg = "The provided repo_path does not point to a file."
+            raise ValueError(msg)
+    else:
+        # Direct URL
+        download_url = url_or_repo
+
+    # Download the file
+    with requests.get(download_url, stream=True, timeout=15) as r:
+        r.raise_for_status()
+        with local_path.open("wb") as f:
+            for chunk in r.iter_content(chunk_size=8192):
+                f.write(chunk)
+
+
+def download_github_directory(
+    repo: os.PathLike | str,
+    local_dir: os.PathLike | str,
+    repo_path: os.PathLike | str,
+):
+    """This method should not be used due to github's api restrictions. Use download_file to get a .zip of the folder instead."""  # noqa: D404
+    repo = PurePath(repo)
+    repo_path = PurePath(repo_path)
+    api_url = f"https://api.github.com/repos/{repo.as_posix()}/contents/{repo_path.as_posix()}"
+    data = _request_api_data(api_url)
+    for item in data:
+        item_path = Path(item["path"])
+        local_path = item_path.relative_to("toolset")
+
+        if item["type"] == "file":
+            download_github_file(item["download_url"], Path(local_dir, local_path))
+        elif item["type"] == "dir":
+            download_github_directory(repo, item_path, local_path)
+
+
+def download_github_directory_fallback(
+    repo: os.PathLike | str,
+    local_dir: os.PathLike | str,
+    repo_path: os.PathLike | str,
+):
+    """There were two versions of this function and I can't remember which one worked."""
+    repo = PurePath.pathify(repo)
+    repo_path = PurePath.pathify(repo_path)
+    api_url = f"https://api.github.com/repos/{repo.as_posix()}/contents/{repo_path.as_posix()}"
+    data = _request_api_data(api_url)
+    for item in data:
+        item_path = Path(item["path"])
+        local_path = item_path.relative_to("toolset")
+
+        if item["type"] == "file":
+            download_github_file(item["download_url"], local_path)
+        elif item["type"] == "dir":
+            download_github_directory(repo, item_path, local_path)
+
+
+def _request_api_data(api_url: str) -> Any:
+    response: requests.Response = requests.get(api_url, timeout=15)
+    response.raise_for_status()
+    return response.json()

Tools/HolocronToolset/src/toolset/gui/widgets/settings/installations.py

@@ -259,6 +259,10 @@ def installations(self) -> dict[str, InstallationConfig]:
         "useBetaChannel",
         False,
     )
+    alsoCheckReleaseVersion = Settings._addSetting(
+        "alsoCheckReleaseVersion",
+        True,
+    )
     firstTime = Settings._addSetting(
         "firstTime",
         True,

Tools/HolocronToolset/src/toolset/gui/widgets/settings/misc.py

@@ -20,6 +20,7 @@ def __init__(self, parent: QWidget):
         self.setupValues()
 
     def setupValues(self):
+        self.ui.alsoCheckReleaseVersion.setChecked(self.settings.alsoCheckReleaseVersion)
         self.ui.useBetaChannel.setChecked(self.settings.useBetaChannel)
         self.ui.saveRimCheck.setChecked(not self.settings.disableRIMSaving)
         self.ui.mergeRimCheck.setChecked(self.settings.joinRIMsTogether)
@@ -34,6 +35,7 @@ def setupValues(self):
         self.ui.nssCompEdit.setText(self.settings.nssCompilerPath)
 
     def save(self):
+        self.settings.alsoCheckReleaseVersion = self.ui.alsoCheckReleaseVersion.isChecked()
         self.settings.useBetaChannel = not self.ui.useBetaChannel.isChecked()
         self.settings.disableRIMSaving = not self.ui.saveRimCheck.isChecked()
         self.settings.joinRIMsTogether = self.ui.mergeRimCheck.isChecked()