aiohttp client GET does not return expected amount of data when fetched slowly asynchronously

[cheops]

TLDR: actual_length != expected_length

I want to do a OTA download of firmware running asynchonously (display is still updating lvgl) and writing it to a flash partition.
Below a simple test program that does a download and simulates a slow write.

I tried several ways to read from the response, these give problems

• test_mode 1 uses ClientResponse.read()
• test_mode 2 uses ClientResponse.content = Stream.readinto()

this does not cause any problems (but needs lots of memory)

• firmware = await r.read()

The board is custom with ESP32-S3 WROOM 8MB PSRAM 16MB FLASH module.

import network
import asyncio
import aiohttp
import logging
import time

# test_mode 1 uses ClientResponse.read()
# test_mode 2 uses ClientResponse.content = Stream.readinto()
test_mode = 1

slow_write_sleep_ms = 500

SSID = "your-wifi-ssid"
KEY = "your-wifi-password"

# github gives me problems
url = "https://api.github.com/repos/cheops/fri3d-ota/git/blobs/6dc83d902d2df34479bf3e0d17b9f314640f6675"
expected_length = 1778528

# micropython server mostly works
# url = "https://micropython.org/resources/firmware/ESP32_GENERIC_S3-20240602-v1.23.0.bin"
# expected_length = 1628144

logging.basicConfig(level=logging.DEBUG, force=True)
logger = logging.getLogger('examples.ota_update.async_download')

wlan = network.WLAN(network.STA_IF)

def connect_wifi():
    wlan.active(True)
    wlan.connect(SSID, KEY)
    while wlan.status() != network.STAT_GOT_IP:
        time.sleep(1)
    logger.debug(wlan.ifconfig())


def disconnect_wifi():
    wlan.disconnect()
    wlan.active(False)


def write(b):
    # simulate slow write
    time.sleep_ms(slow_write_sleep_ms)
    n = len(b)
    logger.debug('writing %d bytes', n)
    return n


async def writer_readinto(f, blocksize):
    mv = memoryview(bytearray(blocksize))
    tot = 0
    while (n := await f.readinto(mv)) != 0:
        tot += write(mv[:n])
    return tot


async def writer_read(f, blocksize):
    tot = 0
    while True:
        b = await f.read(blocksize)
        if len(b) == 0:
            break
        mv = memoryview(b)
        tot += write(mv)
    return tot

async def main():

    headers = {"User-Agent": "micropython", "Accept": "application/vnd.github.raw+json"}

    connect_wifi()
    async with aiohttp.ClientSession(headers=headers) as session:
        async with session.get(url) as r:
            code: int = r.status
            logger.debug(r.headers)
            content_length = int(r.headers['Content-Length']) if 'Content-Length' in headers else 0

            if code != 200:
                r_text = await r.text()
                print(r_text)
                raise ValueError(f"HTTP Error: {code}")

            if test_mode == 1:
                actual_length = await writer_read(r, 4096)
            elif test_mode == 2:
                content = r.content  # this bypasses decoding, so do not use this with Content-Encoding gzip or deflate
                actual_length = await writer_readinto(content, 4096)

            # this works, but needs lots of memory
            # firmware = await r.read()


            diff = actual_length - expected_length
    
    disconnect_wifi()
    logger.debug("expected_length: %d, content_length: %d, actual_length: %d, diff %d", expected_length, content_length, actual_length, diff)

asyncio.run(main())

[tomvanbraeckel]

Which version(s) of ESP-IDF is this failing on? 4.4.8? 5.5.2? Thanks!

[cheops]

Tested with micropython 1.23 build with ESP-IDF 5.2.2