From 36eebb6e922faf4b033efa589f4d12869c8bf3af Mon Sep 17 00:00:00 2001 From: Mitch Bradley Date: Fri, 5 May 2023 19:11:43 -1000 Subject: [PATCH 01/11] build-release.py creates new release format --- FluidNC/include/Driver/sdspi.h | 2 +- FluidNC/src/ProcessSettings.cpp | 6 ++ FluidNC/src/Protocol.cpp | 12 ++- FluidNC/src/Protocol.h | 2 + build-release.py | 158 ++++++++++++++++++++++++++++++-- 5 files changed, 171 insertions(+), 9 deletions(-) diff --git a/FluidNC/include/Driver/sdspi.h b/FluidNC/include/Driver/sdspi.h index c60ecbfbb..be038b019 100644 --- a/FluidNC/include/Driver/sdspi.h +++ b/FluidNC/include/Driver/sdspi.h @@ -4,4 +4,4 @@ bool sd_init_slot(uint32_t freq_hz, int cs_pin, int cd_pin = -1, int wp_pin = -1 void sd_unmount(); void sd_deinit_slot(); -std::error_code sd_mount(int max_files = 5); +std::error_code sd_mount(int max_files = 1); diff --git a/FluidNC/src/ProcessSettings.cpp b/FluidNC/src/ProcessSettings.cpp index 5898e62c2..80793d7fb 100644 --- a/FluidNC/src/ProcessSettings.cpp +++ b/FluidNC/src/ProcessSettings.cpp @@ -749,6 +749,11 @@ static Error setReportInterval(const char* value, WebUI::AuthenticationLevel aut return Error::Ok; } +static Error showHeap(const char* value, WebUI::AuthenticationLevel auth_level, Channel& out) { + log_info("Heap free: " << xPortGetFreeHeapSize() << " min: " << heapLowWater); + return Error::Ok; +} + // Commands use the same syntax as Settings, but instead of setting or // displaying a persistent value, a command causes some action to occur. // That action could be anything, from displaying a run-time parameter @@ -798,6 +803,7 @@ void make_user_commands() { new UserCommand("N", "GCode/StartupLines", report_startup_lines, notIdleOrAlarm); new UserCommand("RST", "Settings/Restore", restore_settings, notIdleOrAlarm, WA); + new UserCommand("Heap", "Heap/Show", showHeap, anyState); new UserCommand("SS", "Startup/Show", showStartupLog, anyState); new UserCommand("RI", "Report/Interval", setReportInterval, anyState); diff --git a/FluidNC/src/Protocol.cpp b/FluidNC/src/Protocol.cpp index 3937c9c6d..96431ef49 100644 --- a/FluidNC/src/Protocol.cpp +++ b/FluidNC/src/Protocol.cpp @@ -261,7 +261,10 @@ static void check_startup_state() { } } -void protocol_main_loop() { +const uint32_t heapWarnThreshold = 15000; + +uint32_t heapLowWater = UINT_MAX; +void protocol_main_loop() { check_startup_state(); start_polling(); @@ -309,6 +312,13 @@ void protocol_main_loop() { idleEndTime = 0; // config->_axes->set_disable(true); } + uint32_t newHeapSize = xPortGetFreeHeapSize(); + if (newHeapSize < heapLowWater) { + heapLowWater = newHeapSize; + if (heapLowWater < heapWarnThreshold) { + log_warn("Low memory: " << heapLowWater << " bytes"); + } + } } return; /* Never reached */ } diff --git a/FluidNC/src/Protocol.h b/FluidNC/src/Protocol.h index 8234d2864..d28e16b37 100644 --- a/FluidNC/src/Protocol.h +++ b/FluidNC/src/Protocol.h @@ -120,3 +120,5 @@ void send_line(Channel& channel, const std::string* message); void send_line(Channel& channel, const std::string& message); void drain_messages(); + +extern uint32_t heapLowWater; diff --git a/build-release.py b/build-release.py index 97dd45959..e9768c800 100644 --- a/build-release.py +++ b/build-release.py @@ -6,6 +6,7 @@ from zipfile import ZipFile, ZipInfo import subprocess, os, sys, shutil import urllib.request +import io, hashlib verbose = '-v' in sys.argv @@ -24,7 +25,7 @@ def buildEnv(pioEnv, verbose=True, extraArgs=None): if verbose: app = subprocess.Popen(cmd, env=environ) else: - app = subprocess.Popen(cmd, env=environ, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1) + app = subprocess.Popen(cmd, env=environ, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) for line in app.stdout: line = line.decode('utf8') if "Took" in line or 'Uploading' in line or ("error" in line.lower() and "Compiling" not in line): @@ -41,7 +42,7 @@ def buildFs(pioEnv, verbose=verbose, extraArgs=None): if verbose: app = subprocess.Popen(cmd, env=environ) else: - app = subprocess.Popen(cmd, env=environ, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1) + app = subprocess.Popen(cmd, env=environ, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) for line in app.stdout: line = line.decode('utf8') if "Took" in line or 'Uploading' in line or ("error" in line.lower() and "Compiling" not in line): @@ -71,17 +72,160 @@ def copyToZip(zipObj, platform, fileName, destPath, mode=0o100755): if not os.path.exists(relPath): os.makedirs(relPath) +manifestRelPath = os.path.join(relPath, 'current') +if os.path.exists(manifestRelPath): + shutil.rmtree(manifestRelPath) + +os.makedirs(manifestRelPath) + +manifest = { + "name": "FluidNC", + "version": tag, + "source_url": "https://github.com/bdring/FluidNC/tree/" + tag, + "release_url": "https://github.com/bdring/FluidNC/releases/tag/" + tag, + "funding_url": "https://www.paypal.com/donate/?hosted_button_id=8DYLB6ZYYDG7Y", + "images": {}, + "installable": { + "name": "installable", + "description": "Things you can install", + "choice-name": "Processor type", + "choices": [] + }, +} + # We avoid doing this every time, instead checking in a new NoFile.h as necessary # if buildEmbeddedPage() != 0: # sys.exit(1) -if buildFs('wifi', verbose=verbose) != 0: - sys.exit(1) +# if buildFs('wifi', verbose=verbose) != 0: +# sys.exit(1) + +def addImage(name, offset, filename, srcpath, dstpath): + fulldstpath = os.path.join(manifestRelPath,os.path.normpath(dstpath)) + + os.makedirs(fulldstpath, exist_ok=True) + + fulldstfile = os.path.join(fulldstpath, filename) + + shutil.copy(os.path.join(srcpath, filename), fulldstfile) -for envName in ['wifi','bt']: - if buildEnv(envName, verbose=verbose) != 0: + print("image ", name) + + with open(fulldstfile, "rb") as f: + data = f.read() + image = { + # "name": name, + "size": os.path.getsize(fulldstfile), + "offset": offset, + "path": dstpath + '/' + filename, + "signature": { + "algorithm": "SHA2-256", + "value": hashlib.sha256(data).hexdigest() + } + } + if manifest['images'].get(name) != None: + print("Duplicate image name", name) sys.exit(1) - shutil.copy(os.path.join('.pio', 'build', envName, 'firmware.elf'), os.path.join(relPath, envName + '-' + 'firmware.elf')) + manifest['images'][name] = image + # manifest['images'].append(image) + +flashsize = "4m" + +mcu = "esp32" +for mcu in ['esp32']: + for envName in ['wifi','bt', 'noradio']: + if buildEnv(envName, verbose=verbose) != 0: + sys.exit(1) + buildDir = os.path.join('.pio', 'build', envName) + shutil.copy(os.path.join(buildDir, 'firmware.elf'), os.path.join(relPath, envName + '-' + 'firmware.elf')) + + addImage(mcu + '-' + envName + '-firmware', '0x10000', 'firmware.bin', buildDir, mcu + '/' + envName) + + if envName == 'wifi': + if buildFs('wifi', verbose=verbose) != 0: + sys.exit(1) + + # bootapp is a data partition that the bootloader and OTA use to determine which + # image to run. Its initial value is in a file "boot_app0.bin" in the platformio + # framework package. We copy it to the build directory so addImage can find it + bootappsrc = os.path.join(os.path.expanduser('~'),'.platformio','packages','framework-arduinoespressif32','tools','partitions', 'boot_app0.bin') + shutil.copy(bootappsrc, buildDir) + + addImage(mcu + '-' + envName + '-' + flashsize + '-filesystem', '0xe000', 'littlefs.bin', buildDir, mcu + '/' + envName + '/' + flashsize) + addImage(mcu + '-' + flashsize + '-partitions', '0x8000', 'partitions.bin', buildDir, mcu + '/' + flashsize) + addImage(mcu + '-bootloader', '0x1000', 'bootloader.bin', buildDir, mcu) + addImage(mcu + '-bootapp', '0xe000', 'boot_app0.bin', buildDir, mcu) + +def addSection(node, name, description, choice): + section = { + "name": name, + "description": description, + } + if choice != None: + section['choice-name'] = choice + section['choices'] = [] + node.append(section) + +def addMCU(name, description, choice=None): + addSection(manifest['installable']['choices'], name, description, choice) + +def addVariant(variant, description, choice=None): + node1 = manifest['installable']['choices'] + node1len = len(node1) + addSection(node1[node1len-1]['choices'], variant, description, choice) + +def addInstallable(install_type, erase, images): + for image in images: + if manifest['images'].get(image) == None: + # imagefiles = [obj for obj in manifest['images'] if obj['name'] == image] + # if len(imagefiles) == 0: + print("Missing image", image) + sys.exit(1) + # if len(imagefiles) > 1: + # print("Duplicate image", image) + # sys.exit(2) + + node1 = manifest['installable']['choices'] + node1len = len(node1) + node2 = node1[node1len-1]['choices'] + node2len = len(node2) + installable = { + "name": install_type["name"], + "description": install_type["description"], + "erase": erase, + "images": images + } + node2[node2len-1]['choices'].append(installable) + +fresh_install = { "name": "fresh-install", "description": "Complete FluidNC installation, erasing all previous data"} +firmware_update = { "name": "firmware-update", "description": "Update FluidNC to latest firmware version, preserving previous filesystem data."} +filesystem_update = { "name": "filesystem-update", "description": "Update FluidNC filesystem only, erasing previous filesystem data."} + +def makeManifest(): + addMCU("esp32", "ESP32-WROOM", "Firmware variant") + + addVariant("wifi", "Supports WiFi and WebUI", "Installation type") + addInstallable(fresh_install, True, ["esp32-4m-partitions", "esp32-bootloader", "esp32-bootapp", "esp32-wifi-firmware", "esp32-wifi-4m-filesystem"]) + addInstallable(firmware_update, False, ["esp32-wifi-firmware"]) + addInstallable(filesystem_update, False, ["esp32-wifi-4m-filesystem"]) + + addVariant("bt", "Supports Bluetooth serial", "Installation type") + addInstallable(fresh_install, True, ["esp32-4m-partitions", "esp32-bootloader", "esp32-bootapp", "esp32-bt-firmware"]) + addInstallable(firmware_update, False, ["esp32-bt-firmware"]) + + addVariant("noradio", "Supports neither WiFi nor Bluetooth", "Installation type") + addInstallable(fresh_install, True, ["esp32-4m-partitions", "esp32-bootloader", "esp32-bootapp", "esp32-noradio-firmware"]) + addInstallable(firmware_update, False, ["esp32-noradio-firmware"]) + +makeManifest() + +import json +def printManifest(): + print(json.dumps(manifest, indent=2)) + +with open(os.path.join(manifestRelPath, "manifest.json"), "w") as manifest_file: + json.dump(manifest, manifest_file, indent=2) + for platform in ['win64', 'posix']: print("Creating zip file for ", platform) From f583b298c085e8aeb64e6e5fe5626300a4b3f66f Mon Sep 17 00:00:00 2001 From: Mitch Bradley Date: Fri, 5 May 2023 19:13:35 -1000 Subject: [PATCH 02/11] Added deploy to github pages step to release action --- .github/workflows/manual.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/manual.yml b/.github/workflows/manual.yml index 47597204e..a210a4898 100644 --- a/.github/workflows/manual.yml +++ b/.github/workflows/manual.yml @@ -47,6 +47,14 @@ jobs: release/*.zip release/*.elf draft: True + - name: Deploy to GitHub Pages + uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: releases/current + repository-name: MitchBradley/MitchBradley.github.io + ssh-key: ${{ secrets.GHP_DEPLOY_KEY }} + target-folder: releases/${{ github.event.inputs.tag }} + # - name: Upload mac bundle # uses: actions/upload-artifact@v2 # with: From c6fc31415b2467d659a141010d7f212f83fb9371 Mon Sep 17 00:00:00 2001 From: Mitch Bradley Date: Fri, 5 May 2023 22:58:35 -1000 Subject: [PATCH 03/11] Fixed typos in manual.yaml deploy step --- .github/workflows/manual.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/manual.yml b/.github/workflows/manual.yml index a210a4898..182a4c14c 100644 --- a/.github/workflows/manual.yml +++ b/.github/workflows/manual.yml @@ -48,13 +48,12 @@ jobs: release/*.elf draft: True - name: Deploy to GitHub Pages - uses: JamesIves/github-pages-deploy-action@v4 - with: - folder: releases/current - repository-name: MitchBradley/MitchBradley.github.io - ssh-key: ${{ secrets.GHP_DEPLOY_KEY }} - target-folder: releases/${{ github.event.inputs.tag }} - + uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: release/current + repository-name: MitchBradley/MitchBradley.github.io + ssh-key: ${{ secrets.GHP_DEPLOY_KEY }} + target-folder: releases/${{ github.event.inputs.tag }} # - name: Upload mac bundle # uses: actions/upload-artifact@v2 # with: From 2da75e6a412f80c72f78f936bdab3b45ff40b80c Mon Sep 17 00:00:00 2001 From: Mitch Bradley Date: Mon, 29 May 2023 06:21:15 -1000 Subject: [PATCH 04/11] Wrong load address for the filesystem image --- build-release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-release.py b/build-release.py index e9768c800..6d2a18391 100644 --- a/build-release.py +++ b/build-release.py @@ -151,7 +151,7 @@ def addImage(name, offset, filename, srcpath, dstpath): bootappsrc = os.path.join(os.path.expanduser('~'),'.platformio','packages','framework-arduinoespressif32','tools','partitions', 'boot_app0.bin') shutil.copy(bootappsrc, buildDir) - addImage(mcu + '-' + envName + '-' + flashsize + '-filesystem', '0xe000', 'littlefs.bin', buildDir, mcu + '/' + envName + '/' + flashsize) + addImage(mcu + '-' + envName + '-' + flashsize + '-filesystem', '0x3d0000', 'littlefs.bin', buildDir, mcu + '/' + envName + '/' + flashsize) addImage(mcu + '-' + flashsize + '-partitions', '0x8000', 'partitions.bin', buildDir, mcu + '/' + flashsize) addImage(mcu + '-bootloader', '0x1000', 'bootloader.bin', buildDir, mcu) addImage(mcu + '-bootapp', '0xe000', 'boot_app0.bin', buildDir, mcu) From f573ad9a4568ad914df4dc31afc6c178c9974e01 Mon Sep 17 00:00:00 2001 From: Mitch Bradley Date: Fri, 9 Jun 2023 11:04:34 -1000 Subject: [PATCH 05/11] Deploy to bdring/fluidnc.github.io instead of MitchBradley/... --- .github/workflows/manual.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual.yml b/.github/workflows/manual.yml index 182a4c14c..6e47bbabb 100644 --- a/.github/workflows/manual.yml +++ b/.github/workflows/manual.yml @@ -51,7 +51,7 @@ jobs: uses: JamesIves/github-pages-deploy-action@v4 with: folder: release/current - repository-name: MitchBradley/MitchBradley.github.io + repository-name: bdring/fluidnc.github.io ssh-key: ${{ secrets.GHP_DEPLOY_KEY }} target-folder: releases/${{ github.event.inputs.tag }} # - name: Upload mac bundle From b0e07d7d2b8ccac1bac04c6c557e4df303ce2def Mon Sep 17 00:00:00 2001 From: Mitch Bradley Date: Sat, 10 Jun 2023 07:16:33 -1000 Subject: [PATCH 06/11] Test action --- .github/workflows/test.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..28440f653 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,10 @@ +name: Test + +on: + workflow_dispatch: + +jobs: + test: + steps: + - name: Show secret + - run: echo ${{ secrets.JUNK }} From e83fd2533f54cac143ab9b769177d0b30a69cecb Mon Sep 17 00:00:00 2001 From: Mitch Bradley Date: Sat, 10 Jun 2023 09:09:17 -1000 Subject: [PATCH 07/11] Deploy to releases branch, not gh-pages --- .github/workflows/manual.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/manual.yml b/.github/workflows/manual.yml index 6e47bbabb..f980ed92f 100644 --- a/.github/workflows/manual.yml +++ b/.github/workflows/manual.yml @@ -50,6 +50,7 @@ jobs: - name: Deploy to GitHub Pages uses: JamesIves/github-pages-deploy-action@v4 with: + branch: releases folder: release/current repository-name: bdring/fluidnc.github.io ssh-key: ${{ secrets.GHP_DEPLOY_KEY }} From 82bf5bcfedfde57e06c6682d392ed9eb88c90412 Mon Sep 17 00:00:00 2001 From: Mitch Bradley Date: Sat, 10 Jun 2023 09:12:03 -1000 Subject: [PATCH 08/11] Deleted junk file --- .github/workflows/test.yml | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 28440f653..000000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: Test - -on: - workflow_dispatch: - -jobs: - test: - steps: - - name: Show secret - - run: echo ${{ secrets.JUNK }} From db6f00fee996310a6d3a5ccac6093d101ac4320f Mon Sep 17 00:00:00 2001 From: Mitch Bradley Date: Tue, 13 Jun 2023 09:11:00 -1000 Subject: [PATCH 09/11] Removed bufsize in build-release.py to suppress warnings --- build-release.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-release.py b/build-release.py index 97dd45959..e1095603d 100644 --- a/build-release.py +++ b/build-release.py @@ -24,7 +24,7 @@ def buildEnv(pioEnv, verbose=True, extraArgs=None): if verbose: app = subprocess.Popen(cmd, env=environ) else: - app = subprocess.Popen(cmd, env=environ, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1) + app = subprocess.Popen(cmd, env=environ, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) for line in app.stdout: line = line.decode('utf8') if "Took" in line or 'Uploading' in line or ("error" in line.lower() and "Compiling" not in line): @@ -41,7 +41,7 @@ def buildFs(pioEnv, verbose=verbose, extraArgs=None): if verbose: app = subprocess.Popen(cmd, env=environ) else: - app = subprocess.Popen(cmd, env=environ, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1) + app = subprocess.Popen(cmd, env=environ, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) for line in app.stdout: line = line.decode('utf8') if "Took" in line or 'Uploading' in line or ("error" in line.lower() and "Compiling" not in line): From 55e4c961062679b4f69f9417b0be3ed698f3d668 Mon Sep 17 00:00:00 2001 From: Mitch Bradley Date: Tue, 13 Jun 2023 09:12:03 -1000 Subject: [PATCH 10/11] Fix #919 - cannot xmodem upload in alarm state --- FluidNC/src/ProcessSettings.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FluidNC/src/ProcessSettings.cpp b/FluidNC/src/ProcessSettings.cpp index 80793d7fb..08b27a2dd 100644 --- a/FluidNC/src/ProcessSettings.cpp +++ b/FluidNC/src/ProcessSettings.cpp @@ -764,7 +764,7 @@ void make_user_commands() { new UserCommand("CI", "Channel/Info", showChannelInfo, anyState); new UserCommand("XR", "Xmodem/Receive", xmodem_receive, notIdleOrAlarm); - new UserCommand("XS", "Xmodem/Send", xmodem_send, notIdleOrJog); + new UserCommand("XS", "Xmodem/Send", xmodem_send, notIdleOrAlarm); new UserCommand("CD", "Config/Dump", dump_config, anyState); new UserCommand("", "Help", show_help, anyState); new UserCommand("T", "State", showState, anyState); From 214a5e67afeed273f07d829a74a1f68f508e606b Mon Sep 17 00:00:00 2001 From: Mitch Bradley Date: Tue, 13 Jun 2023 10:43:05 -1000 Subject: [PATCH 11/11] Release artifacts to bdring/fluidnc-releases --- .github/workflows/manual.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/manual.yml b/.github/workflows/manual.yml index f980ed92f..2630c03b6 100644 --- a/.github/workflows/manual.yml +++ b/.github/workflows/manual.yml @@ -47,14 +47,18 @@ jobs: release/*.zip release/*.elf draft: True - - name: Deploy to GitHub Pages - uses: JamesIves/github-pages-deploy-action@v4 + - name: Deploy to fluidnc-releases + uses: datalbry/copy_folder_to_another_repo_action@1.0.0 + env: + API_TOKEN_GITHUB: ${{ secrets.RELEASE_COPY_TOKEN }} with: - branch: releases - folder: release/current - repository-name: bdring/fluidnc.github.io - ssh-key: ${{ secrets.GHP_DEPLOY_KEY }} - target-folder: releases/${{ github.event.inputs.tag }} + source_folder: 'release/current' + destination_repo: 'bdring/fluidnc-releases' + destination_branch: 'main' + destination_folder: releases/${{ github.event.inputs.tag }} + user_email: bdring@buildlog.net + user_name: 'Bart Dring' + commit_msg: Release ${{ github.event.inputs.tag }} # - name: Upload mac bundle # uses: actions/upload-artifact@v2 # with: