W5500/SPI Ethernet Support (T-ETH-Lite-S3)

pio-scripts/download_fs.py

@@ -0,0 +1,285 @@
+# Written by Maximilian Gerhardt <[email protected]>
+# 29th December 2020
+# and Christian Baars, Johann Obermeier
+# 2023 / 2024
+# License: Apache
+# Expanded from functionality provided by PlatformIO's espressif32 and espressif8266 platforms, credited below.
+# This script provides functions to download the filesystem (LittleFS) from a running ESP32 / ESP8266
+# over the serial bootloader using esptool.py, and mklittlefs for extracting.
+# run by either using the VSCode task "Custom" -> "Download Filesystem"
+# or by doing 'pio run -t downloadfs' (with optional '-e <environment>') from the commandline.
+# output will be saved, by default, in the "unpacked_fs" of the project.
+# this folder can be changed by writing 'custom_unpack_dir = some_other_dir' in the corresponding platformio.ini
+# environment.
+import re
+import sys
+from os.path import isfile, join
+from enum import Enum
+import os
+import subprocess
+import shutil
+
+Import("env")
+platform = env.PioPlatform()
+board = env.BoardConfig()
+mcu = board.get("build.mcu", "esp32")
+
+
+class FSType(Enum):
+    LITTLEFS="littlefs"
+    FATFS="fatfs"
+
+class FSInfo:
+    def __init__(self, fs_type, start, length, page_size, block_size):
+        self.fs_type = fs_type
+        self.start = start
+        self.length = length
+        self.page_size = page_size
+        self.block_size = block_size
+    def __repr__(self):
+        return f"FS type {self.fs_type} Start {hex(self.start)} Len {self.length} Page size {self.page_size} Block size {self.block_size}"
+    # extract command supposed to be implemented by subclasses
+    def get_extract_cmd(self, input_file, output_dir):
+        raise NotImplementedError()
+
+class FS_Info(FSInfo):
+    def __init__(self, start, length, page_size, block_size):
+        self.tool = env["MKFSTOOL"]
+        self.tool = join(platform.get_package_dir("tool-mklittlefs"), self.tool)
+        super().__init__(FSType.LITTLEFS, start, length, page_size, block_size)
+    def __repr__(self):
+        return f"{self.fs_type} Start {hex(self.start)} Len {hex(self.length)} Page size {hex(self.page_size)} Block size {hex(self.block_size)}"
+    def get_extract_cmd(self, input_file, output_dir):
+        return f'"{self.tool}" -b {self.block_size} -s {self.length} -p {self.page_size} --unpack "{output_dir}" "{input_file}"'
+
+# SPIFFS helpers copied from ESP32, https://github.com/platformio/platform-espressif32/blob/develop/builder/main.py
+# Copyright 2014-present PlatformIO <[email protected]>
+# Licensed under the Apache License, Version 2.0 (the "License");
+
+def _parse_size(value):
+    if isinstance(value, int):
+        return value
+    elif value.isdigit():
+        return int(value)
+    elif value.startswith("0x"):
+        return int(value, 16)
+    elif value[-1].upper() in ("K", "M"):
+        base = 1024 if value[-1].upper() == "K" else 1024 * 1024
+        return int(value[:-1]) * base
+    return value
+
+## FS helpers for ESP8266
+# copied from https://github.com/platformio/platform-espressif8266/blob/develop/builder/main.py
+# Copyright 2014-present PlatformIO <[email protected]>
+# Licensed under the Apache License, Version 2.0 (the "License");
+
+def _parse_ld_sizes(ldscript_path):
+    assert ldscript_path
+    result = {}
+    # get flash size from LD script path
+    match = re.search(r"\.flash\.(\d+[mk]).*\.ld", ldscript_path)
+    if match:
+        result['flash_size'] = _parse_size(match.group(1))
+
+    appsize_re = re.compile(
+        r"irom0_0_seg\s*:.+len\s*=\s*(0x[\da-f]+)", flags=re.I)
+    filesystem_re = re.compile(
+        r"PROVIDE\s*\(\s*_%s_(\w+)\s*=\s*(0x[\da-f]+)\s*\)" % "FS"
+        if "arduino" in env.subst("$PIOFRAMEWORK")
+        else "SPIFFS",
+        flags=re.I,
+    )
+    with open(ldscript_path) as fp:
+        for line in fp.readlines():
+            line = line.strip()
+            if not line or line.startswith("/*"):
+                continue
+            match = appsize_re.search(line)
+            if match:
+                result['app_size'] = _parse_size(match.group(1))
+                continue
+            match = filesystem_re.search(line)
+            if match:
+                result['fs_%s' % match.group(1)] = _parse_size(
+                    match.group(2))
+    return result
+
+def esp8266_fetch_fs_size(env):
+    ldsizes = _parse_ld_sizes(env.GetActualLDScript())
+    for key in ldsizes:
+        if key.startswith("fs_"):
+            env[key.upper()] = ldsizes[key]
+
+    assert all([
+        k in env
+        for k in ["FS_START", "FS_END", "FS_PAGE", "FS_BLOCK"]
+    ])
+
+    # esptool flash starts from 0
+    for k in ("FS_START", "FS_END"):
+        _value = 0
+        if env[k] < 0x40300000:
+            _value = env[k] & 0xFFFFF
+        elif env[k] < 0x411FB000:
+            _value = env[k] & 0xFFFFFF
+            _value -= 0x200000  # correction
+        else:
+            _value = env[k] & 0xFFFFFF
+            _value += 0xE00000  # correction
+
+        env[k] = _value
+
+## Script interface functions
+def parse_partition_table(content):
+    entries = [e for e in content.split(b'\xaaP') if len(e) > 0]
+    #print("Partition data:")
+    for entry in entries:
+        type = entry[1]
+        if type in [0x82,0x83]: # SPIFFS or LITTLEFS
+            offset = int.from_bytes(entry[2:5], byteorder='little', signed=False)
+            size = int.from_bytes(entry[6:9], byteorder='little', signed=False)
+            print("type:",hex(type))
+            print("address:",hex(offset))
+            print("size:",hex(size))
+            env["FS_START"] = offset
+            env["FS_SIZE"] = size
+            env["FS_PAGE"] = int("0x100", 16)
+            env["FS_BLOCK"] = int("0x1000", 16)
+
+def get_partition_table():
+    esptoolpy = join(platform.get_package_dir("tool-esptoolpy") or "", "esptool.py")
+    upload_port = join(env.get("UPLOAD_PORT", "none"))
+    download_speed = join(str(board.get("download.speed", "115200")))
+    if "none" in upload_port:
+        env.AutodetectUploadPort()
+        upload_port = join(env.get("UPLOAD_PORT", "none"))
+    fs_file = join(env["PROJECT_DIR"], "partition_table_from_flash.bin")
+    esptoolpy_flags = [
+            "--chip", mcu,
+            "--port", upload_port,
+            "--baud",  download_speed,
+            "--before", "default_reset",
+            "--after", "hard_reset",
+            "read_flash",
+            "0x8000",
+            "0x1000",
+            fs_file
+    ]
+    esptoolpy_cmd = [env["PYTHONEXE"], esptoolpy] + esptoolpy_flags
+    try:
+        returncode = subprocess.call(esptoolpy_cmd, shell=False)
+    except subprocess.CalledProcessError as exc:
+        print("Downloading failed with " + str(exc))
+    with open(fs_file, mode="rb") as file:
+        content = file.read()
+        parse_partition_table(content)
+
+def get_fs_type_start_and_length():
+    platform = env["PIOPLATFORM"]
+    if platform == "espressif32":
+        print(f"Retrieving filesystem info for {mcu}.")
+        get_partition_table()
+        print("FS_START: " + hex(env["FS_START"]))
+        print("FS_SIZE: " + hex(env["FS_SIZE"]))
+        print("FS_PAGE: " + hex(env["FS_PAGE"]))
+        print("FS_BLOCK: " + hex(env["FS_BLOCK"]))
+        return FS_Info(env["FS_START"], env["FS_SIZE"], env["FS_PAGE"], env["FS_BLOCK"])
+    elif platform == "espressif8266":
+        print("Retrieving filesystem info for ESP8266.")
+        filesystem = board.get("build.filesystem", "littlefs")
+        if filesystem not in ("littlefs"):
+            print("Unrecognized board_build.filesystem option '" + str(filesystem) + "'.")
+            env.Exit(1)
+        # fetching sizes is the same for all filesystems
+        esp8266_fetch_fs_size(env)
+        #print("FS_START: " + hex(env["FS_START"]))
+        #print("FS_SIZE: " + hex(env["FS_END"] - env["FS_START"]))
+        #print("FS_PAGE: " + hex(env["FS_PAGE"]))
+        #print("FS_BLOCK: " + hex(env["FS_BLOCK"]))
+        if filesystem == "littlefs":
+            print("Recognized LittleFS filesystem.")
+            return FS_Info(env["FS_START"], env["FS_END"] - env["FS_START"], env["FS_PAGE"], env["FS_BLOCK"])
+        else:
+            print("Unrecongized configuration.")
+    pass
+
+def download_fs(fs_info: FSInfo):
+    print(fs_info)
+    esptoolpy = join(platform.get_package_dir("tool-esptoolpy") or "", "esptool.py")
+    upload_port = join(env.get("UPLOAD_PORT", "none"))
+    download_speed = join(str(board.get("download.speed", "115200")))
+    if "none" in upload_port:
+        env.AutodetectUploadPort()
+        upload_port = join(env.get("UPLOAD_PORT", "none"))
+    fs_file = join(env.subst("$BUILD_DIR"), f"downloaded_fs_{hex(fs_info.start)}_{hex(fs_info.length)}.bin")
+    esptoolpy_flags = [
+            "--chip", mcu,
+            "--port", upload_port,
+            "--baud",  download_speed,
+            "--before", "default_reset",
+            "--after", "hard_reset",
+            "read_flash",
+            hex(fs_info.start),
+            hex(fs_info.length),
+            fs_file
+    ]
+    esptoolpy_cmd = [env["PYTHONEXE"], esptoolpy] + esptoolpy_flags
+    print(esptoolpy_cmd)
+    print("Download filesystem image")
+    try:
+        returncode = subprocess.call(esptoolpy_cmd, shell=False)
+        return (True, fs_file)
+    except subprocess.CalledProcessError as exc:
+        print("Downloading failed with " + str(exc))
+        return (False, "")
+
+def unpack_fs(fs_info: FSInfo, downloaded_file: str):
+    # by writing custom_unpack_dir = some_dir in the platformio.ini, one can
+    # control the unpack directory
+    unpack_dir = env.GetProjectOption("custom_unpack_dir", "unpacked_fs")
+    if not os.path.exists(downloaded_file):
+        print(f"ERROR: {downloaded_file} with filesystem not found, maybe download failed due to download_speed setting being too high.")
+        assert(0)
+    try:
+        if os.path.exists(unpack_dir):
+            shutil.rmtree(unpack_dir)
+    except Exception as exc:
+        print("Exception while attempting to remove the folder '" + str(unpack_dir) + "': " + str(exc))
+    if not os.path.exists(unpack_dir):
+        os.makedirs(unpack_dir)
+
+    cmd = fs_info.get_extract_cmd(downloaded_file, unpack_dir)
+    print("Unpack files from filesystem image")
+    try:
+        print(env["MKFSTOOL"])
+        print(platform.get_package_dir("tool-mklittlefs"))
+        print(cmd, unpack_dir);
+        returncode = subprocess.call(cmd, shell=True)
+        return (True, unpack_dir)
+    except subprocess.CalledProcessError as exc:
+        print("Unpacking filesystem failed with " + str(exc))
+        return (False, "")
+
+def display_fs(extracted_dir):
+    # extract command already nicely lists all extracted files.
+    # no need to display that ourselves. just display a summary
+    file_count = sum([len(files) for r, d, files in os.walk(extracted_dir)])
+    print("Extracted " + str(file_count) + " file(s) from filesystem.")
+
+def command_download_fs(*args, **kwargs):
+    info = get_fs_type_start_and_length()
+    download_ok, downloaded_file = download_fs(info)
+    unpack_ok, unpacked_dir = unpack_fs(info, downloaded_file)
+    if unpack_ok is True:
+        display_fs(unpacked_dir)
+
+
+env.AddCustomTarget(
+    name="downloadfs",
+    dependencies=None,
+    actions=[
+        command_download_fs
+    ],
+    title="Download Filesystem",
+    description="Downloads and displays files stored in the target ESP32/ESP8266"
+)

platformio.ini

@@ -341,7 +341,8 @@ build_flags = -g
   -DARDUINO_ARCH_ESP32
   #-DCONFIG_LITTLEFS_FOR_IDF_3_2
   -D CONFIG_ASYNC_TCP_USE_WDT=0
-  -D CONFIG_ASYNC_TCP_TASK_STACK_SIZE=9472                ;; WLEDMM increase stack by 1.25Kb, as audioreactive needs bigger SETTINGS_STACK_BUF_SIZE
+  ; -D CONFIG_ASYNC_TCP_TASK_STACK_SIZE=9472                ;; WLEDMM increase stack by 1.25Kb, as audioreactive needs bigger SETTINGS_STACK_BUF_SIZE
+  -D CONFIG_ASYNC_TCP_TASK_STACK_SIZE=10072 
   #use LITTLEFS library by lorol in ESP32 core 1.x.x instead of built-in in 2.x.x
   -D LOROL_LITTLEFS
   ; -D WLEDMM_TWOPATH    ;; use I2S1 as the second bus --> ~15% faster on "V3" builds - may flicker a bit more
@@ -389,7 +390,8 @@ build_flagsV4 = -g
   -DARDUINO_ARCH_ESP32 -DESP32
   -DCONFIG_LITTLEFS_FOR_IDF_3_2 -DLFS_THREADSAFE
   -D CONFIG_ASYNC_TCP_USE_WDT=0
-  -D CONFIG_ASYNC_TCP_TASK_STACK_SIZE=9472              ;; WLEDMM increase stack by 1.25Kb, as audioreactive needs bigger SETTINGS_STACK_BUF_SIZE
+  ; -D CONFIG_ASYNC_TCP_TASK_STACK_SIZE=9472              ;; WLEDMM increase stack by 1.25Kb, as audioreactive needs bigger SETTINGS_STACK_BUF_SIZE
+  -D CONFIG_ASYNC_TCP_TASK_STACK_SIZE=10072
   ; -D WLEDMM_TWOPATH  ;; use I2S1 as the second bus --> slightly faster on some setups
   ; -D WLEDMM_SLOWPATH ;; don't use I2S for LED bus
   ; -DARDUINO_USB_CDC_ON_BOOT=0 ;; mandatory for "classic ESP32" when building with arduino-esp32 >=2.0.3
@@ -419,7 +421,8 @@ build_flags = -g
   -DARDUINO_ARCH_ESP32 -DESP32
   #-DCONFIG_LITTLEFS_FOR_IDF_3_2
   -D CONFIG_ASYNC_TCP_USE_WDT=0
-  -D CONFIG_ASYNC_TCP_TASK_STACK_SIZE=9472              ;; WLEDMM increase stack by 1.25Kb, as audioreactive needs bigger SETTINGS_STACK_BUF_SIZE
+  ; -D CONFIG_ASYNC_TCP_TASK_STACK_SIZE=9472              ;; WLEDMM increase stack by 1.25Kb, as audioreactive needs bigger SETTINGS_STACK_BUF_SIZE
+  -D CONFIG_ASYNC_TCP_TASK_STACK_SIZE=10072
   -DARDUINO_USB_CDC_ON_BOOT=0 ;; this flag is mandatory for "classic ESP32" when building with arduino-esp32 >=2.0.3
   ; -D WLEDMM_TWOPATH  ;; use I2S1 as the second bus --> slightly faster on some setups
   ; -D WLEDMM_SLOWPATH ;; don't use I2S for LED bus
@@ -445,7 +448,8 @@ build_flags = -g
   -DCONFIG_IDF_TARGET_ESP32S2=1
   -DCONFIG_LITTLEFS_FOR_IDF_3_2 -DLFS_THREADSAFE   ;; WLEDMM
   -D CONFIG_ASYNC_TCP_USE_WDT=0
-  -D CONFIG_ASYNC_TCP_TASK_STACK_SIZE=8614         ;; WLEDMM increase stack by 1Kb, as audioreactive needs bigger SETTINGS_STACK_BUF_SIZE
+  ; -D CONFIG_ASYNC_TCP_TASK_STACK_SIZE=8614         ;; WLEDMM increase stack by 1Kb, as audioreactive needs bigger SETTINGS_STACK_BUF_SIZE
+  -D CONFIG_ASYNC_TCP_TASK_STACK_SIZE=10072
   -DARDUINO_USB_MSC_ON_BOOT=0 -DARDUINO_USB_DFU_ON_BOOT=0
   -DCO
   -DARDUINO_USB_MODE=0 ;; this flag is mandatory for ESP32-S2 !
@@ -468,7 +472,8 @@ build_flags = -g
   -DCONFIG_IDF_TARGET_ESP32C3=1
   -DCONFIG_LITTLEFS_FOR_IDF_3_2 -DLFS_THREADSAFE   ;; WLEDMM
   -D CONFIG_ASYNC_TCP_USE_WDT=0
-  -D CONFIG_ASYNC_TCP_TASK_STACK_SIZE=9472         ;; WLEDMM increase stack by 1.25Kb, as audioreactive needs bigger SETTINGS_STACK_BUF_SIZE
+  ; -D CONFIG_ASYNC_TCP_TASK_STACK_SIZE=9472         ;; WLEDMM increase stack by 1.25Kb, as audioreactive needs bigger SETTINGS_STACK_BUF_SIZE
+  -D CONFIG_ASYNC_TCP_TASK_STACK_SIZE=10072
   -DCO
   -DARDUINO_USB_MODE=1 ;; this flag is mandatory for ESP32-C3
   ;; please make sure that the following flags are properly set (to 0 or 1) by your board.json, or included in your custom platformio_override.ini entry:
@@ -491,7 +496,8 @@ build_flags = -g
   -DCONFIG_IDF_TARGET_ESP32S3=1
   -DCONFIG_LITTLEFS_FOR_IDF_3_2 -DLFS_THREADSAFE   ;; WLEDMM
   -D CONFIG_ASYNC_TCP_USE_WDT=0
-  -D CONFIG_ASYNC_TCP_TASK_STACK_SIZE=9472         ;; WLEDMM increase stack by 1.25Kb, as audioreactive needs bigger SETTINGS_STACK_BUF_SIZE
+  ; -D CONFIG_ASYNC_TCP_TASK_STACK_SIZE=9472         ;; WLEDMM increase stack by 1.25Kb, as audioreactive needs bigger SETTINGS_STACK_BUF_SIZE
+  -D CONFIG_ASYNC_TCP_TASK_STACK_SIZE=10072
   -DARDUINO_USB_MSC_ON_BOOT=0 -DARDUINO_DFU_ON_BOOT=0
   -DCO
   ;; please make sure that the following flags are properly set (to 0 or 1) by your board.json, or included in your custom platformio_override.ini entry:
@@ -502,7 +508,6 @@ lib_deps =
   ;; makuna/NeoPixelBus @ 2.7.9 ;; experimental
   ${env.lib_deps}
 
-
 # ------------------------------------------------------------------------------
 # WLED BUILDS
 # ------------------------------------------------------------------------------

tools/ESP32-Chip_info.hpp

@@ -566,15 +566,15 @@ void showRealSpeed() {
 
   Serial.print("FLASH CHIP FREQ (magic): "); Serial.print(ESP.getFlashChipSpeed()/1000000.0, 1); Serial.println(" MHz");
   Serial.print("FLASH SIZE (magic byte): "); Serial.print(ESP.getFlashChipSize() / (1024.0 * 1024), 2); Serial.println(" MB");
-  Serial.print("FLASH MODE (magic byte): "); Serial.print(ESP.getFlashChipMode()); Serial.println(" ;  0=QIO, 1=QOUT, 2=DIO, 3=DOUT or other\n");
+  // Serial.print("FLASH MODE (magic byte): "); Serial.print(ESP.getFlashChipMode()); Serial.println(" ;  0=QIO, 1=QOUT, 2=DIO, 3=DOUT or other\n");
 
   Serial.flush();
   Serial.print("FLASH CHIP ID:   0x"); Serial.println(my_ESP_getFlashChipId(), HEX);
 #if !defined(CONFIG_IDF_TARGET_ESP32S2) && !defined(CONFIG_IDF_TARGET_ESP32C3)
   //Serial.print("FLASH CHIP FREQ: "); Serial.print(my_ESP_getFlashChipSpeed() / 1000000.0, 1); Serial.println(" MHz"); // this seems to crash on -S2
 #endif
   Serial.print("FLASH REAL SIZE: "); Serial.print(my_ESP_getFlashChipRealSize() / (1024.0 * 1024), 2); Serial.println(" MB");
-  Serial.print("FLASH REAL MODE: "); Serial.println(my_ESP_getFlashChipMode());
+  // Serial.print("FLASH REAL MODE: "); Serial.println(my_ESP_getFlashChipMode());
 
   for(int aa=0; aa<42; aa++) Serial.print("-"); Serial.println();
   Serial.flush();