diff --git a/builder/main.py b/builder/main.py index 36ad3ae90..8536591aa 100644 --- a/builder/main.py +++ b/builder/main.py @@ -17,6 +17,13 @@ platform: PlatformBase = env.PioPlatform() board: PlatformBoardConfig = env.BoardConfig() +python_deps = { + "ltchiptool": ">=4.5.1,<5.0", +} +env.SConscript("python-venv.py", exports="env") +env.ConfigurePythonVenv() +env.InstallPythonDependencies(python_deps) + # Utilities env.SConscript("utils/config.py", exports="env") env.SConscript("utils/cores.py", exports="env") @@ -24,7 +31,7 @@ env.SConscript("utils/flash.py", exports="env") env.SConscript("utils/libs-external.py", exports="env") env.SConscript("utils/libs-queue.py", exports="env") -env.SConscript("utils/ltchiptool.py", exports="env") +env.SConscript("utils/ltchiptool-util.py", exports="env") # Firmware name if env.get("PROGNAME", "program") == "program": diff --git a/builder/python-venv.py b/builder/python-venv.py new file mode 100644 index 000000000..6869f32e3 --- /dev/null +++ b/builder/python-venv.py @@ -0,0 +1,122 @@ +# Copyright (c) Kuba Szczodrzyński 2023-09-07. + +import json +import site +import subprocess +import sys +from pathlib import Path + +import semantic_version +from platformio.compat import IS_WINDOWS +from platformio.package.version import pepver_to_semver +from platformio.platform.base import PlatformBase +from SCons.Script import DefaultEnvironment, Environment + +env: Environment = DefaultEnvironment() +platform: PlatformBase = env.PioPlatform() + +# code borrowed and modified from espressif32/builder/frameworks/espidf.py + + +def env_configure_python_venv(env: Environment): + venv_path = Path(env.subst("${PROJECT_CORE_DIR}"), "penv", ".libretiny") + + pip_path = venv_path.joinpath( + "Scripts" if IS_WINDOWS else "bin", + "pip" + (".exe" if IS_WINDOWS else ""), + ) + python_path = venv_path.joinpath( + "Scripts" if IS_WINDOWS else "bin", + "python" + (".exe" if IS_WINDOWS else ""), + ) + site_path = venv_path.joinpath( + "Lib" if IS_WINDOWS else "lib", + "." if IS_WINDOWS else f"python{sys.version_info[0]}.{sys.version_info[1]}", + "site-packages", + ) + + if not pip_path.is_file(): + # Use the built-in PlatformIO Python to create a standalone virtual env + result = env.Execute( + env.VerboseAction( + f'"$PYTHONEXE" -m venv --clear "{venv_path.absolute()}"', + "LibreTiny: Creating a virtual environment for Python dependencies", + ) + ) + if not python_path.is_file(): + # Creating the venv failed + raise RuntimeError( + f"Failed to create virtual environment. Error code {result}" + ) + if not pip_path.is_file(): + # Creating the venv succeeded but pip didn't get installed + # (i.e. Debian/Ubuntu without ensurepip) + print( + "LibreTiny: Failed to install pip, running get-pip.py", file=sys.stderr + ) + import requests + + with requests.get("https://bootstrap.pypa.io/get-pip.py") as r: + p = subprocess.Popen( + args=str(python_path.absolute()), + stdin=subprocess.PIPE, + ) + p.communicate(r.content) + p.wait() + + assert ( + pip_path.is_file() + ), f"Error: Missing the pip binary in virtual environment `{pip_path.absolute()}`" + assert ( + python_path.is_file() + ), f"Error: Missing Python executable file `{python_path.absolute()}`" + assert ( + site_path.is_dir() + ), f"Error: Missing site-packages directory `{site_path.absolute()}`" + + env.Replace(LTPYTHONEXE=python_path.absolute(), LTPYTHONENV=venv_path.absolute()) + site.addsitedir(str(site_path.absolute())) + + +def env_install_python_dependencies(env: Environment, dependencies: dict): + try: + pip_output = subprocess.check_output( + [ + env.subst("${LTPYTHONEXE}"), + "-m", + "pip", + "list", + "--format=json", + "--disable-pip-version-check", + ] + ) + pip_data = json.loads(pip_output) + packages = {p["name"]: pepver_to_semver(p["version"]) for p in pip_data} + except: + print( + "LibreTiny: Warning! Couldn't extract the list of installed Python packages" + ) + packages = {} + + to_install = [] + for name, spec in dependencies.items(): + install_spec = f'"{name}{dependencies[name]}"' + if name not in packages: + to_install.append(install_spec) + elif spec: + version_spec = semantic_version.Spec(spec) + if not version_spec.match(packages[name]): + to_install.append(install_spec) + + if to_install: + env.Execute( + env.VerboseAction( + '"${LTPYTHONEXE}" -m pip install --prefer-binary -U ' + + " ".join(to_install), + "LibreTiny: Installing Python dependencies", + ) + ) + + +env.AddMethod(env_configure_python_venv, "ConfigurePythonVenv") +env.AddMethod(env_install_python_dependencies, "InstallPythonDependencies") diff --git a/builder/utils/env.py b/builder/utils/env.py index 43fa2005b..1453664c3 100644 --- a/builder/utils/env.py +++ b/builder/utils/env.py @@ -8,6 +8,7 @@ from typing import Dict from ltchiptool import Family, get_version +from ltchiptool.util.lvm import LVM from ltchiptool.util.misc import sizeof from platformio.platform.base import PlatformBase from platformio.platform.board import PlatformBoardConfig @@ -77,7 +78,7 @@ def env_configure( # ltchiptool config: # -r output raw log messages # -i 1 indent log messages - LTCHIPTOOL='"${PYTHONEXE}" -m ltchiptool -r -i 1', + LTCHIPTOOL='"${LTPYTHONEXE}" -m ltchiptool -r -i 1 -L "${LT_DIR}"', # Fix for link2bin to get tmpfile name in argv LINKCOM="${LINK} ${LINKARGS}", LINKARGS="${TEMPFILE('-o $TARGET $LINKFLAGS $__RPATH $SOURCES $_LIBDIRFLAGS $_LIBFLAGS', '$LINKCOMSTR')}", @@ -87,6 +88,8 @@ def env_configure( ) # Store family parameters as environment variables env.Replace(**dict(family)) + # Set platform directory in ltchiptool (for use in this process only) + LVM.add_path(platform.get_dir()) return family diff --git a/builder/utils/ltchiptool.py b/builder/utils/ltchiptool-util.py similarity index 100% rename from builder/utils/ltchiptool.py rename to builder/utils/ltchiptool-util.py diff --git a/platform.py b/platform.py index b4f27ab16..93a717c57 100644 --- a/platform.py +++ b/platform.py @@ -1,12 +1,12 @@ # Copyright (c) Kuba Szczodrzyński 2022-04-20. -import importlib import json import os import platform +import site import sys from os.path import dirname -from subprocess import Popen +from pathlib import Path from typing import Dict, List import click @@ -15,73 +15,8 @@ from platformio.package.meta import PackageItem from platformio.platform.base import PlatformBase from platformio.platform.board import PlatformBoardConfig -from semantic_version import SimpleSpec, Version - -LTCHIPTOOL_VERSION = "^4.2.3" - - -# Install & import tools -def check_ltchiptool(install: bool): - if install: - # update ltchiptool to a supported version - print("Installing/updating ltchiptool") - p = Popen( - [ - sys.executable, - "-m", - "pip", - "install", - "-U", - "--force-reinstall", - f"ltchiptool >= {LTCHIPTOOL_VERSION[1:]}, < 5.0", - ], - ) - p.wait() - - # unload all modules from the old version - for name, module in list(sorted(sys.modules.items())): - if not name.startswith("ltchiptool"): - continue - del sys.modules[name] - del module - - # try to import it - ltchiptool = importlib.import_module("ltchiptool") - - # check if the version is known - version = Version.coerce(ltchiptool.get_version()).truncate() - if version in SimpleSpec(LTCHIPTOOL_VERSION): - return - if not install: - raise ImportError(f"Version incompatible: {version}") - - -def try_check_ltchiptool(): - install_modes = [False, True] - exception = None - for install in install_modes: - try: - check_ltchiptool(install) - return - except (ImportError, AttributeError) as ex: - exception = ex - print( - "!!! Installing ltchiptool failed, or version outdated. " - "Please install ltchiptool manually using pip. " - f"Cannot continue. {type(exception).name}: {exception}" - ) - raise exception - - -try_check_ltchiptool() -import ltchiptool - -# Remove current dir so it doesn't conflict with PIO -if dirname(__file__) in sys.path: - sys.path.remove(dirname(__file__)) -# Let ltchiptool know about LT's location -ltchiptool.lt_set_path(dirname(__file__)) +site.addsitedir(Path(__file__).absolute().parent.joinpath("tools")) def get_os_specifiers(): @@ -119,6 +54,12 @@ def __init__(self, manifest_path): super().__init__(manifest_path) self.custom_opts = {} self.versions = {} + self.verbose = ( + "-v" in sys.argv + or "--verbose" in sys.argv + or "PIOVERBOSE=1" in sys.argv + or os.environ.get("PIOVERBOSE", "0") == "1" + ) def print(self, *args, **kwargs): if not self.verbose: @@ -137,11 +78,8 @@ def get_package_spec(self, name, version=None): return spec def configure_default_packages(self, options: dict, targets: List[str]): - from ltchiptool.util.dict import RecursiveDict + from libretiny import RecursiveDict - self.verbose = ( - "-v" in sys.argv or "--verbose" in sys.argv or "PIOVERBOSE=1" in sys.argv - ) self.print(f"configure_default_packages(targets={targets})") pioframework = options.get("pioframework") or ["base"] @@ -298,19 +236,19 @@ def get_boards(self, id_=None): return result def update_board(self, board: PlatformBoardConfig): + from libretiny import Board, Family, merge_dicts + if "_base" in board: - board._manifest = ltchiptool.Board.get_data(board._manifest) + board._manifest = Board.get_data(board._manifest) board._manifest.pop("_base") if self.custom("board"): - from ltchiptool.util.dict import merge_dicts - with open(self.custom("board"), "r") as f: custom_board = json.load(f) board._manifest = merge_dicts(board._manifest, custom_board) family = board.get("build.family") - family = ltchiptool.Family.get(short_name=family) + family = Family.get(short_name=family) # add "frameworks" key with the default "base" board.manifest["frameworks"] = ["base"] # add "arduino" framework if supported diff --git a/tools/libretiny/__init__.py b/tools/libretiny/__init__.py new file mode 100644 index 000000000..fa3c06ec9 --- /dev/null +++ b/tools/libretiny/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) Kuba Szczodrzyński 2023-09-07. + +from .board import Board +from .dict import RecursiveDict, merge_dicts +from .family import Family + +# TODO refactor and remove all this from here + +__all__ = [ + "Board", + "Family", + "RecursiveDict", + "merge_dicts", +] diff --git a/tools/libretiny/board.py b/tools/libretiny/board.py new file mode 100644 index 000000000..a5b4a91dc --- /dev/null +++ b/tools/libretiny/board.py @@ -0,0 +1,34 @@ +# Copyright (c) Kuba Szczodrzyński 2022-07-29. + +from typing import Union + +from genericpath import isfile + +from .dict import merge_dicts +from .fileio import readjson +from .lvm import lvm_load_json + + +class Board: + @staticmethod + def get_data(board: Union[str, dict]) -> dict: + if not isinstance(board, dict): + if isfile(board): + board = readjson(board) + if not board: + raise FileNotFoundError(f"Board not found: {board}") + else: + source = board + board = lvm_load_json(f"boards/{board}.json") + board["source"] = source + if "_base" in board: + base = board["_base"] + if not isinstance(base, list): + base = [base] + result = {} + for base_name in base: + board_base = lvm_load_json(f"boards/_base/{base_name}.json") + merge_dicts(result, board_base) + merge_dicts(result, board) + board = result + return board diff --git a/tools/libretiny/dict.py b/tools/libretiny/dict.py new file mode 100644 index 000000000..f8ecebde1 --- /dev/null +++ b/tools/libretiny/dict.py @@ -0,0 +1,65 @@ +# Copyright (c) Kuba Szczodrzyński 2022-07-29. + +from .obj import get, has, pop, set_ + + +class RecursiveDict(dict): + def __init__(self, data: dict = None): + if data: + data = { + key: RecursiveDict(value) if isinstance(value, dict) else value + for key, value in data.items() + } + super().__init__(data) + else: + super().__init__() + + def __getitem__(self, key): + if "." not in key: + return super().get(key, None) + return get(self, key) + + def __setitem__(self, key, value): + if "." not in key: + return super().__setitem__(key, value) + set_(self, key, value, newtype=RecursiveDict) + + def __delitem__(self, key): + if "." not in key: + return super().pop(key, None) + return pop(self, key) + + def __contains__(self, key) -> bool: + if "." not in key: + return super().__contains__(key) + return has(self, key) + + def get(self, key, default=None): + if "." not in key: + return super().get(key, default) + return get(self, key) or default + + def pop(self, key, default=None): + if "." not in key: + return super().pop(key, default) + return pop(self, key, default) + + +def merge_dicts(d1, d2): + # force RecursiveDict instances to be treated as regular dicts + d1_type = dict if isinstance(d1, RecursiveDict) else type(d1) + d2_type = dict if isinstance(d2, RecursiveDict) else type(d2) + if d1 is not None and d1_type != d2_type: + raise TypeError(f"d1 and d2 are of different types: {type(d1)} vs {type(d2)}") + if isinstance(d2, list): + if d1 is None: + d1 = [] + d1.extend(merge_dicts(None, item) for item in d2) + elif isinstance(d2, dict): + if d1 is None: + d1 = {} + for key in d2: + d1[key] = merge_dicts(d1.get(key, None), d2[key]) + else: + d1 = d2 + return d1 diff --git a/tools/libretiny/family.py b/tools/libretiny/family.py new file mode 100644 index 000000000..8e94cabaa --- /dev/null +++ b/tools/libretiny/family.py @@ -0,0 +1,97 @@ +# Copyright (c) Kuba Szczodrzyński 2022-06-02. + +from dataclasses import dataclass, field +from typing import List, Optional, Union + +from .lvm import lvm_load_json, lvm_path + +LT_FAMILIES: List["Family"] = [] + + +@dataclass +class Family: + name: str + parent: Union["Family", None] + code: str + description: str + id: Optional[int] = None + short_name: Optional[str] = None + package: Optional[str] = None + mcus: List[str] = field(default_factory=lambda: []) + children: List["Family"] = field(default_factory=lambda: []) + + # noinspection PyTypeChecker + def __post_init__(self): + if self.id: + self.id = int(self.id, 16) + self.mcus = set(self.mcus) + + @classmethod + def get_all(cls) -> List["Family"]: + global LT_FAMILIES + if LT_FAMILIES: + return LT_FAMILIES + families = lvm_load_json("families.json") + LT_FAMILIES = [ + cls(name=k, **v) for k, v in families.items() if isinstance(v, dict) + ] + # attach parents and children to all families + for family in LT_FAMILIES: + if family.parent is None: + continue + try: + parent = next(f for f in LT_FAMILIES if f.name == family.parent) + except StopIteration: + raise ValueError( + f"Family parent '{family.parent}' of '{family.name}' doesn't exist" + ) + family.parent = parent + parent.children.append(family) + return LT_FAMILIES + + @classmethod + def get( + cls, + any: str = None, + id: Union[str, int] = None, + short_name: str = None, + name: str = None, + code: str = None, + description: str = None, + ) -> "Family": + if any: + id = any + short_name = any + name = any + code = any + description = any + if id and isinstance(id, str) and id.startswith("0x"): + id = int(id, 16) + for family in cls.get_all(): + if id and family.id == id: + return family + if short_name and family.short_name == short_name.upper(): + return family + if name and family.name == name.lower(): + return family + if code and family.code == code.lower(): + return family + if description and family.description == description: + return family + if any: + raise ValueError(f"Family not found - {any}") + items = [hex(id) if id else None, short_name, name, code, description] + text = ", ".join(filter(None, items)) + raise ValueError(f"Family not found - {text}") + + @property + def has_arduino_core(self) -> bool: + if lvm_path().joinpath("cores", self.name, "arduino").is_dir(): + return True + if self.parent: + return self.parent.has_arduino_core + return False + + @property + def target_package(self) -> Optional[str]: + return self.package or self.parent and self.parent.target_package diff --git a/tools/libretiny/fileio.py b/tools/libretiny/fileio.py new file mode 100644 index 000000000..bfba581ad --- /dev/null +++ b/tools/libretiny/fileio.py @@ -0,0 +1,17 @@ +# Copyright (c) Kuba Szczodrzyński 2022-06-10. + +import json +from json import JSONDecodeError +from os.path import isfile +from typing import Optional, Union + + +def readjson(file: str) -> Optional[Union[dict, list]]: + """Read a JSON file into a dict or list.""" + if not isfile(file): + return None + with open(file, "r", encoding="utf-8") as f: + try: + return json.load(f) + except JSONDecodeError: + return None diff --git a/tools/libretiny/lvm.py b/tools/libretiny/lvm.py new file mode 100644 index 000000000..4f224eb52 --- /dev/null +++ b/tools/libretiny/lvm.py @@ -0,0 +1,19 @@ +# Copyright (c) Kuba Szczodrzyński 2023-3-18. + +import json +from pathlib import Path +from typing import Dict, Union + +json_cache: Dict[str, Union[list, dict]] = {} +libretiny_path = Path(__file__).parents[2] + + +def lvm_load_json(path: str) -> Union[list, dict]: + if path not in json_cache: + with libretiny_path.joinpath(path).open("rb") as f: + json_cache[path] = json.load(f) + return json_cache[path] + + +def lvm_path() -> Path: + return libretiny_path diff --git a/tools/libretiny/obj.py b/tools/libretiny/obj.py new file mode 100644 index 000000000..6090a9d5d --- /dev/null +++ b/tools/libretiny/obj.py @@ -0,0 +1,62 @@ +# Copyright (c) Kuba Szczodrzyński 2022-06-02. + +from enum import Enum +from typing import Type + +# The following helpers force using base dict class' methods. +# Because RecursiveDict uses these helpers, this prevents it +# from running into endless nested loops. + + +def get(data: dict, path: str): + if not isinstance(data, dict) or not path: + return None + if dict.__contains__(data, path): + return dict.get(data, path, None) + key, _, path = path.partition(".") + return get(dict.get(data, key, None), path) + + +def pop(data: dict, path: str, default=None): + if not isinstance(data, dict) or not path: + return default + if dict.__contains__(data, path): + return dict.pop(data, path, default) + key, _, path = path.partition(".") + return pop(dict.get(data, key, None), path, default) + + +def has(data: dict, path: str) -> bool: + if not isinstance(data, dict) or not path: + return False + if dict.__contains__(data, path): + return True + key, _, path = path.partition(".") + return has(dict.get(data, key, None), path) + + +def set_(data: dict, path: str, value, newtype=dict): + if not isinstance(data, dict) or not path: + return + # can't use __contains__ here, as we're setting, + # so it's obvious 'data' doesn't have the item + if "." not in path: + dict.__setitem__(data, path, value) + else: + key, _, path = path.partition(".") + # allow creation of parent objects + if key in data: + sub_data = dict.__getitem__(data, key) + else: + sub_data = newtype() + dict.__setitem__(data, key, sub_data) + set_(sub_data, path, value) + + +def str2enum(cls: Type[Enum], key: str): + if not key: + return None + try: + return next(e for e in cls if e.name.lower() == key.lower()) + except StopIteration: + return None