feat: Update mpflash version to 0.8.0

The `pyproject.toml` file in the `mpflash` module has been updated to set the version to 0.8.0. This change reflects the new version of the tool and ensures consistency with the package metadata.

Resolve board IDs in `cli_flash.py`

The `resolve_board_ids` function in `cli_flash.py` has been updated to correctly resolve board IDs and update the `params.boards` list. This change ensures that the board IDs are properly resolved and used in the flash process.

Add `Board` dataclass to `board.py`

A new `Board` dataclass has been added to the `board.py` file. This dataclass represents MicroPython board definitions and includes attributes such as `port`, `board_id`, `board_name`, and `description`. This change improves the organization and readability of the code.

Update board information handling in `store.py`

The `write_boardinfo_json` function in `store.py` has been updated to write the board information to a zip file instead of a JSON file. This change improves the file organization and allows for easier distribution of the board information.

These changes address various updates and improvements in the `mpflash` module.

src/mpflash/mpflash/add_firmware.py

@@ -0,0 +1,98 @@
+import shutil
+from pathlib import Path
+from typing import Union
+
+import jsonlines
+import requests
+from loguru import logger as log
+
+# re-use logic from mpremote
+from mpremote.mip import _rewrite_url as rewrite_url  # type: ignore
+
+from mpflash.common import FWInfo
+from mpflash.config import config
+from mpflash.vendor.versions import get_preview_mp_version, get_stable_mp_version
+
+
+def add_firmware(
+    source: Union[Path, str],
+    new_fw: FWInfo,
+    *,
+    force: bool = False,
+    custom: bool = False,
+    description: str = "",
+) -> bool:
+    """Add a firmware to the firmware folder.
+
+    stored in the port folder, with the same filename as the source.
+
+    """
+    # Check minimal info needed
+    if not new_fw.port or not new_fw.board:
+        log.error("Port and board are required")
+        return False
+    if not isinstance(source, Path) and not source.startswith("http"):
+        log.error(f"Invalid source {source}")
+        return False
+
+    # use sensible defaults
+    source_2 = Path(source)
+    new_fw.ext = new_fw.ext or source_2.suffix
+    new_fw.variant = new_fw.variant or new_fw.board
+    new_fw.custom = new_fw.custom or custom
+    new_fw.description = new_fw.description or description
+    if not new_fw.version:
+        # TODO: Get version from filename
+        # or use the last preview version
+        new_fw.version = get_preview_mp_version() if new_fw.preview else get_stable_mp_version()
+
+    config.firmware_folder.mkdir(exist_ok=True)
+
+    fw_filename = config.firmware_folder / new_fw.port / source_2.name
+
+    new_fw.filename = str(fw_filename.relative_to(config.firmware_folder))
+    new_fw.firmware = source.as_uri() if isinstance(source, Path) else source
+
+    if not copy_firmware(source, fw_filename, force):
+        log.error(f"Failed to copy {source} to {fw_filename}")
+        return False
+    # add to inventory
+    with jsonlines.open(config.firmware_folder / "firmware.jsonl", "a") as writer:
+        log.info(f"Adding {new_fw.port} {new_fw.board}")
+        log.info(f"    to {fw_filename}")
+
+        writer.write(new_fw.to_dict())
+    return True
+
+
+def copy_firmware(source: Union[Path, str], fw_filename: Path, force: bool = False):
+    """Add a firmware to the firmware folder.
+    stored in the port folder, with the same filename as the source.
+    """
+    if fw_filename.exists() and not force:
+        log.error(f" {fw_filename} already exists. Use --force to overwrite")
+        return False
+    fw_filename.parent.mkdir(exist_ok=True)
+    if isinstance(source, Path):
+        if not source.exists():
+            log.error(f"File {source} does not exist")
+            return False
+        # file copy
+        log.debug(f"Copy {source} to {fw_filename}")
+        shutil.copy(source, fw_filename)
+        return True
+    # handle github urls
+    url = rewrite_url(source)
+    if str(source).startswith("http://") or str(source).startswith("https://"):
+        log.debug(f"Download {url} to {fw_filename}")
+        response = requests.get(url)
+
+        if response.status_code == 200:
+            with open(fw_filename, "wb") as file:
+                file.write(response.content)
+                log.info("File downloaded and saved successfully.")
+                return True
+        else:
+            print("Failed to download the file.")
+            return False
+    return False

src/mpflash/mpflash/cli_flash.py

@@ -205,8 +205,8 @@ def resolve_board_ids(params):
         if " " in board_id:
             try:
                 if info := find_known_board(board_id):
-                    log.info(f"Resolved board description: {info.board}")
+                    log.info(f"Resolved board description: {info.board_id}")
                     params.boards.remove(board_id)
-                    params.boards.append(info.board)
+                    params.boards.append(info.board_id)
             except Exception as e:
                 log.warning(f"Unable to resolve board description: {e}")

src/mpflash/mpflash/cli_main.py

@@ -2,6 +2,8 @@
 
 # import rich_click as click
 
+import os
+
 import click
 from loguru import logger as log
 
@@ -17,7 +19,8 @@ def mpflash():
     cli.add_command(cli_flash_board)
 
     # cli(auto_envvar_prefix="MPFLASH")
-    if 1:
+    if False and os.environ.get("COMPUTERNAME") == "JOSVERL-S4":
+        # intentional less error suppression on dev machine
         result = cli(standalone_mode=False)
     else:
         try:

src/mpflash/mpflash/common.py

@@ -36,19 +36,24 @@
 
 @dataclass
 class FWInfo:
+    """
+    Downloaded Firmware information
+    is somewhat related to the BOARD class in the mpboard_id module
+    """
+
     port: str  # MicroPython port
     board: str  # MicroPython board
-    filename: str = ""  # relative filename of the firmware image
-    firmware: str = ""  # url or path to original firmware image
-    variant: str = ""  # MicroPython variant
-    preview: bool = False  # True if the firmware is a preview version
-    version: str = ""  # MicroPython version
-    url: str = ""  # url to the firmware image download folder
-    build: str = "0"  # The build = number of commits since the last release
-    ext: str = ""  # The build = number of commits since the last release
-    family: str = "micropython"  # The family of the firmware
-    custom: bool = False  # True if the firmware is a custom build
-    description: str = ""  # Description used by this firmware (custom only)
+    filename: str = field(default="")  # relative filename of the firmware image
+    firmware: str = field(default="")  # url or path to original firmware image
+    variant: str = field(default="")  # MicroPython variant
+    preview: bool = field(default=False)  # True if the firmware is a preview version
+    version: str = field(default="")  # MicroPython version
+    url: str = field(default="")  # url to the firmware image download folder
+    build: str = field(default="0")  # The build = number of commits since the last release
+    ext: str = field(default="")  # the file extension of the firmware
+    family: str = field(default="micropython")  # The family of the firmware
+    custom: bool = field(default=False)  # True if the firmware is a custom build
+    description: str = field(default="")  # Description used by this firmware (custom only)
 
     def to_dict(self) -> dict:
         """Convert the object to a dictionary"""

src/mpflash/mpflash/download.py

@@ -52,10 +52,10 @@ def get_board_urls(page_url: str) -> List[Dict[str, str]]:
 
     Args:
         page_url (str): The url of the page to get the board urls from.
-    
+
     Returns:
         List[Dict[str, str]]: A list of dictionaries containing the board name and url.
-        
+
     """
     downloads_html = get_page(page_url)
     soup = BeautifulSoup(downloads_html, "html.parser")
@@ -124,10 +124,11 @@ def get_boards(ports: List[str], boards: List[str], clean: bool) -> List[FWInfo]
 
         for board in track(urls, description=f"Checking {port} download pages", transient=True, refresh_per_second=2):
             # add a board to the list for each firmware found
-            firmwares:List[str] = []
+            firmware_urls: List[str] = []
             for ext in PORT_FWTYPES[port]:
-                firmwares += board_firmware_urls(board["url"], MICROPYTHON_ORG_URL, ext)
-            for _url in firmwares:
+                firmware_urls += board_firmware_urls(board["url"], MICROPYTHON_ORG_URL, ext)
+            for _url in firmware_urls:
+                board["firmware"] = _url
                 fname = Path(board["firmware"]).name
                 if clean:
                     # remove date from firmware name
@@ -138,7 +139,7 @@ def get_boards(ports: List[str], boards: List[str], clean: bool) -> List[FWInfo]
                     filename=fname,
                     port=port,
                     board=board["board"],
-                    preview= "preview" in _url,
+                    preview="preview" in _url,
                     firmware=_url,
                     version="",
                 )
@@ -195,7 +196,7 @@ def download_firmwares(
 
     with jsonlines.open(firmware_folder / "firmware.jsonl", "a") as writer:
         for board in unique_boards:
-            filename = firmware_folder / board.port/ board.filename
+            filename = firmware_folder / board.port / board.filename
             filename.parent.mkdir(exist_ok=True)
             if filename.exists() and not force:
                 skipped += 1
@@ -207,7 +208,7 @@ def download_firmwares(
                 r = requests.get(board.firmware, allow_redirects=True)
                 with open(filename, "wb") as fw:
                     fw.write(r.content)
-                board.filename= str(filename.relative_to(firmware_folder))
+                board.filename = str(filename.relative_to(firmware_folder))
             except requests.RequestException as e:
                 log.exception(e)
                 continue

src/mpflash/mpflash/mpboard_id/__init__.py

@@ -4,61 +4,18 @@
 
 """
 
-import json
-from dataclasses import dataclass
 from functools import lru_cache
-from pathlib import Path
-from typing import List, Optional, Tuple, Union
+from typing import List, Optional, Tuple
 
-from mpflash.common import PORT_FWTYPES
 from mpflash.errors import MPFlashError
+from mpflash.mpboard_id.board import Board
+from mpflash.mpboard_id.store import read_known_boardinfo
 from mpflash.vendor.versions import clean_version
 
 # KNOWN ports and boards are sourced from the micropython repo,
 # this info is stored in the board_info.json file
 
 
-# Board  based on the dataclass Board but changed to TypedDict
-# - source : get_boardnames.py
-@dataclass
-class Board:
-    """
-    MicroPython Board definitions, parsed from the make and header files
-    """
-
-    port: str  # micropython port
-    board: str  # BOARD_ID (Foldername) as used in the make files
-    board_name: str  # Short board description
-    description: str  # Long board description
-    path: Union[Path, str]
-    version: str
-    mcu_name: str
-    cpu: str = ""
-    family: str = "micropython"
-
-    def __post_init__(self):
-        if not self.cpu:
-            if " with " in self.description:
-                self.cpu = self.description.split(" with ")[-1]
-            else:
-                self.cpu = self.port
-
-    @staticmethod
-    def from_dict(data: dict) -> "Board":
-        return Board(**data)
-
-    def to_dict(self) -> dict:
-        return self.__dict__
-
-
-@lru_cache(maxsize=None)
-def read_known_boardinfo() -> List[Board]:
-    """Reads the board_info.json file and returns the data as a list of Board objects"""
-    with open(Path(__file__).parent / "board_info.json", "r") as file:
-        data = json.load(file)
-    return [Board.from_dict(board) for board in data]
-
-
 def get_known_ports() -> List[str]:
     # TODO: Filter for Version
     mp_boards = read_known_boardinfo()
@@ -72,8 +29,13 @@ def get_known_boards_for_port(port: Optional[str] = "", versions: Optional[List[
     Returns a list of boards for the given port and version(s)
 
     port: The Micropython port to filter for
-    versions:  The Micropython versions to filter for (actual versions required)"""
+    versions:  Optional, The Micropython versions to filter for (actual versions required)
+    """
     mp_boards = read_known_boardinfo()
+    if versions:
+        preview_or_stable = "preview" in versions or "stable" in versions
+    else:
+        preview_or_stable = False
 
     # filter for 'preview' as they are not in the board_info.json
     # instead use stable version
@@ -86,6 +48,13 @@ def get_known_boards_for_port(port: Optional[str] = "", versions: Optional[List[
         versions = [clean_version(v) for v in versions]
         # filter for the version(s)
         mp_boards = [board for board in mp_boards if board.version in versions]
+        if not mp_boards and preview_or_stable:
+            # nothing found - perhaps there is a newer version for which we do not have the board info yet
+            # use the latest known version from the board info
+            mp_boards = read_known_boardinfo()
+            last_known_version = sorted({b.version for b in mp_boards})[-1]
+            mp_boards = [board for board in mp_boards if board.version == last_known_version]
+
     # filter for the port
     if port:
         mp_boards = [board for board in mp_boards if board.port == port]
@@ -101,7 +70,7 @@ def known_stored_boards(port: str, versions: Optional[List[str]] = None) -> List
     """
     mp_boards = get_known_boards_for_port(port, versions)
 
-    boards = set({(f"{board.version} {board.description}", board.board) for board in mp_boards})
+    boards = set({(f"{board.version} {board.description}", board.board_id) for board in mp_boards})
     return sorted(list(boards))
 
 
@@ -110,7 +79,7 @@ def find_known_board(board_id: str) -> Board:
     """Find the board for the given BOARD_ID or 'board description' and return the board info as a Board object"""
     info = read_known_boardinfo()
     for board_info in info:
-        if board_id in (board_info.board, board_info.description):
+        if board_id in (board_info.board_id, board_info.description):
             if not board_info.cpu:
                 if " with " in board_info.description:
                     board_info.cpu = board_info.description.split(" with ")[-1]