Support for external spi flash (#847)

* Support for SDCARD and NORFLASH and optionally NANDFLASH with a patch to esp-idf

* Merged master

* Merged master

* Increased toit stack size (for flash)

* Review comment updates

* Updated comments

* nand fixes

* Review (take 2)

lib/flash.toit

@@ -0,0 +1,147 @@
+// Copyright (C) 2022 Toitware ApS. All rights reserved.
+// Use of this source code is governed by an MIT-style license that can be
+// found in the lib/LICENSE file.
+
+import spi
+import reader show Reader
+import writer show Writer
+import gpio
+
+FLASH_5MHZ  ::= 0
+FLASH_10MHZ ::= 1
+FLASH_20MHZ ::= 2
+FLASH_26MHZ ::= 3
+FLASH_40MHZ ::= 4
+FLASH_80MHZ ::= 5
+
+class Flash:
+  flash_ := null
+  mount_point/string
+
+  /**
+  Mounts an SD-card as a FAT file system under $mount_point on the $spi_bus without formatting the flash.
+
+  The $cs is the chip select pin for the SD-card holder.
+  */
+  constructor.sdcard_unformatted
+      --.mount_point/string
+      --spi_bus/spi.Bus
+      --cs/gpio.Pin:
+    if not mount_point.starts_with "/": throw "INVALID_ARGUMENT"
+    flash_ = init_sdcard_ mount_point spi_bus.spi_ cs.num 0 0 0
+
+  /**
+  Mounts an SD-card as a FAT file system under $mount_point on the $spi_bus and formats the SD-card
+    with $max_files and $allocation_unit_size if it is not already formatted.
+
+  The $cs is the chip select pin for the SD-card holder.
+  */
+  constructor.sdcard
+      --.mount_point/string
+      --spi_bus/spi.Bus
+      --cs/gpio.Pin
+      --max_files/int=5
+      --allocation_unit_size/int=16384:
+    if not mount_point.starts_with "/": throw "INVALID_ARGUMENT"
+    flash_ = init_sdcard_ mount_point spi_bus.spi_ cs.num 1 max_files allocation_unit_size
+
+  /**
+  Mounts an external NOR flash chip on the $spi_bus without formatting the flash.
+
+  The $cs is the chip select pin for the chip on the $spi_bus and $frequency is the SPI frequency.
+    $frequency should be one of the FLASH_FREQ_ constants.
+  */
+  constructor.nor_unformatted
+      --.mount_point/string
+      --spi_bus/spi.Bus
+      --cs/gpio.Pin
+      --frequency/int=FLASH_40MHZ:
+    if not mount_point.starts_with "/": throw "INVALID_ARGUMENT"
+    flash_ = init_nor_flash_ mount_point spi_bus.spi_ cs.num frequency 0 0 0
+
+  /**
+  Mounts an external NOR flash chip on the $spi_bus and format the SD-card with $max_files and
+    $allocation_unit_size if it is not formatted.
+
+  The $cs is the chip select pin for the chip on the $spi_bus and $frequency is the SPI frequency.
+    $frequency should be one of the FLASH_FREQ_ constants.
+  */
+  constructor.nor
+      --.mount_point/string
+      --spi_bus/spi.Bus
+      --cs/gpio.Pin
+      --frequency/int=FLASH_40MHZ
+      --max_files/int=5
+      --allocation_unit_size/int=16384:
+    if not mount_point.starts_with "/": throw "INVALID_ARGUMENT"
+    flash_ = init_nor_flash_ mount_point spi_bus.spi_ cs.num frequency 1 max_files allocation_unit_size
+
+  /**
+  Mounts an external NAND flash chip on the $spi_bus without formatting the flash.
+
+  The $cs is the chip select pin for the chip on the $spi_bus and $frequency is the SPI frequency.
+    $frequency should be one of the FLASH_FREQ_ constants
+  */
+  constructor.nand_unformatted
+      --.mount_point/string
+      --spi_bus/spi.Bus
+      --cs/gpio.Pin
+      --frequency/int=FLASH_40MHZ:
+    if not mount_point.starts_with "/": throw "INVALID_ARGUMENT"
+
+    frequency_mhz := frequency_to_mhz_ frequency
+    flash_ = init_nand_flash_ mount_point spi_bus.spi_ cs.num frequency_mhz 0 0 0
+
+  /**
+  Mounts an external NAND flash chip on the $spi_bus and formats the flash with $max_files and
+     $allocation_unit_size if it is not already formatted.
+
+  The $cs is the chip select pin for the chip on the $spi_bus and $frequency is the SPI frequency.
+    $frequency should be one of the FLASH_FREQ_ constants
+  */
+  constructor.nand
+      --.mount_point/string
+      --spi_bus/spi.Bus
+      --cs/gpio.Pin
+      --frequency/int=FLASH_40MHZ
+      --max_files/int=5
+      --allocation_unit_size/int=2048:
+    if not mount_point.starts_with "/": throw "INVALID_ARGUMENT"
+
+    frequency_mhz := frequency_to_mhz_ frequency
+    flash_ = init_nand_flash_ mount_point spi_bus.spi_ cs.num frequency_mhz 1 max_files allocation_unit_size
+
+  /**
+  Unmounts and releases resources for the external storage.
+  */
+  close:
+    close_spi_flash_ flash_
+
+frequency_to_mhz_ frequency:
+  if frequency == FLASH_5MHZ:
+    return 5_000_000
+  else if frequency == FLASH_10MHZ:
+    return 10_000_000
+  else if frequency == FLASH_20MHZ:
+    return 20_000_000
+  else if frequency == FLASH_26MHZ:
+    return (80_000_000/3).to_int
+  else if frequency == FLASH_40MHZ:
+    return 40_000_000
+  else if frequency == FLASH_80MHZ:
+    return 80_000_000
+  else:
+    throw "INVALID_ARGUMENT"
+
+
+init_nor_flash_ mount_point spi_bus cs frequency format max_files allocation_unit_size -> any:
+  #primitive.spi_flash.init_nor_flash
+
+init_nand_flash_ mount_point spi_bus cs frequency format max_files allocation_unit_size -> any:
+  #primitive.spi_flash.init_nand_flash
+
+init_sdcard_ mount_point spi_bus cs format max_files allocation_unit_size -> any:
+  #primitive.spi_flash.init_sdcard
+
+close_spi_flash_ flash:
+  #primitive.spi_flash.close

src/primitive.h

@@ -58,6 +58,7 @@ namespace toit {
   M(touch,   MODULE_TOUCH)                   \
   M(programs_registry, MODULE_PROGRAMS_REGISTRY) \
   M(flash,   MODULE_FLASH_REGISTRY)          \
+  M(spi_flash, MODULE_SPI_FLASH)             \
   M(file,    MODULE_FILE)                    \
   M(pipe,    MODULE_PIPE)                    \
   M(zlib,    MODULE_ZLIB)                    \
@@ -577,6 +578,12 @@ namespace toit {
   PRIMITIVE(cancel_reservation, 1)           \
   PRIMITIVE(erase_flash_registry, 0)         \
 
+#define MODULE_SPI_FLASH(PRIMITIVE)          \
+  PRIMITIVE(init_sdcard, 6)                  \
+  PRIMITIVE(init_nor_flash, 7)               \
+  PRIMITIVE(init_nand_flash, 7)              \
+  PRIMITIVE(close, 1)                        \
+
 #define MODULE_FILE(PRIMITIVE)               \
   PRIMITIVE(open, 3)                         \
   PRIMITIVE(read, 1)                         \
@@ -902,6 +909,7 @@ namespace toit {
 #define _A_T_ResourceGroup(N, name)       MAKE_UNPACKING_MACRO(ResourceGroup, N, name)
 #define _A_T_SPIDevice(N, name)           MAKE_UNPACKING_MACRO(SPIDevice, N, name)
 #define _A_T_SPIResourceGroup(N, name)    MAKE_UNPACKING_MACRO(SPIResourceGroup, N, name)
+#define _A_T_SPIFlashResourceGroup(N, name)  MAKE_UNPACKING_MACRO(SPIFlashResourceGroup, N, name)
 #define _A_T_SignalResourceGroup(N, name) MAKE_UNPACKING_MACRO(SignalResourceGroup, N, name)
 #define _A_T_SocketResourceGroup(N, name) MAKE_UNPACKING_MACRO(SocketResourceGroup, N, name)
 #define _A_T_TCPResourceGroup(N, name)    MAKE_UNPACKING_MACRO(TCPResourceGroup, N, name)

src/primitive_file_posix.cc

@@ -20,7 +20,7 @@
 #include "process.h"
 #include "objects_inline.h"
 
-#ifdef TOIT_POSIX
+#if defined(TOIT_POSIX) || defined(TOIT_FREERTOS)
 
 #include <dirent.h>
 #include <errno.h>
@@ -30,13 +30,25 @@
 #include <sys/types.h>
 #include <unistd.h>
 
+#ifdef TOIT_FREERTOS
+// The ESP32 has no notion of a shell and no cwd, so assume all paths are absolute.
+#define FILE_OPEN_(dirfd, ...)                             open(__VA_ARGS__)
+#define FILE_UNLINK_(dirfd, pathname, flags)               unlink(pathname)
+#define FILE_RENAME_(olddirfd, oldpath, newdirfd, newpath) rename(oldpath, newpath)
+#define FILE_MKDIR_(dirfd, ...)                            mkdir(__VA_ARGS__)
+#else
 // Old C library version of stat.
 extern "C" {
 
 extern int __fxstat64(int ver, int fd, struct stat64* stat_buf);
 extern int __fxstatat64(int ver, int dirfd, const char* path, struct stat64* stat_buf, int flags);
 
 }
+#define FILE_OPEN_(...)     openat(__VA_ARGS__)
+#define FILE_UNLINK_(...)   unlinkat(__VA_ARGS__)
+#define FILE_RENAME_(...)   renameat(__VA_ARGS__)
+#define FILE_MKDIR_(...)    mkdirat(__VA_ARGS__)
+#endif // TOIT_FREERTOS
 
 #ifdef BUILD_64
 # define STAT_VERSION 1
@@ -118,7 +130,7 @@ PRIMITIVE(open) {
   if ((flags & FILE_CREAT) != 0) os_flags |= O_CREAT;
   if ((flags & FILE_TRUNC) != 0) os_flags |= O_TRUNC;
   bool is_dev_null = strcmp(pathname, "/dev/null") == 0;
-  int fd = openat(current_dir(process), pathname, os_flags, mode);
+  int fd = FILE_OPEN_(current_dir(process), pathname, os_flags, mode);
   AutoCloser closer(fd);
   if (fd < 0) return return_open_error(process, errno);
 #ifndef TOIT_LINUX
@@ -170,13 +182,20 @@ PRIMITIVE(opendir) {
   ByteArray* proxy = process->object_heap()->allocate_proxy();
   if (proxy == null) ALLOCATION_FAILED;
 
-  int fd = openat(current_dir(process), pathname, O_RDONLY | O_DIRECTORY);
+#ifndef TOIT_FREERTOS
+  int fd = FILE_OPEN_(current_dir(process), pathname, O_RDONLY | O_DIRECTORY);
   if (fd < 0) return return_open_error(process, errno);
   DIR* dir = fdopendir(fd);
   if (dir == null) {
     close(fd);
     return return_open_error(process, errno);
   }
+#else
+  DIR* dir = opendir(pathname);
+  if (dir == null) {
+    return return_open_error(process, errno);
+  }
+#endif
 
   LeakyDirectory* directory = _new LeakyDirectory(dir);
   if (directory == null) {
@@ -192,14 +211,20 @@ PRIMITIVE(opendir2) {
   ARGS(SimpleResourceGroup, group, cstring, pathname);
   ByteArray* proxy = process->object_heap()->allocate_proxy();
   if (proxy == null) ALLOCATION_FAILED;
-
-  int fd = openat(current_dir(process), pathname, O_RDONLY | O_DIRECTORY);
+#ifndef TOIT_FREERTOS
+  int fd = FILE_OPEN_(current_dir(process), pathname, O_RDONLY | O_DIRECTORY);
   if (fd < 0) return return_open_error(process, errno);
   DIR* dir = fdopendir(fd);
   if (dir == null) {
     close(fd);
     return return_open_error(process, errno);
   }
+#else
+  DIR* dir = opendir(pathname);
+  if (dir == null) {
+    return return_open_error(process, errno);
+  }
+#endif
 
   Directory* directory = _new Directory(group, dir);
   if (directory == null) {
@@ -341,7 +366,11 @@ Object* time_stamp(Process* process, struct timespec time) {
 // Otherwise returns an array with indices from the FILE_ST_xxx constants.
 PRIMITIVE(stat) {
   ARGS(cstring, pathname, bool, follow_links);
-#ifndef TOIT_LINUX
+#if defined(TOIT_FREERTOS)
+  USE(follow_links);
+  struct stat statbuf;
+  int result = stat(pathname, &statbuf); // FAT does not have symbolic links
+#elif !defined(TOIT_LINUX)
   struct stat statbuf;
   int result = fstatat(current_dir(process), pathname, &statbuf, follow_links ? 0 : AT_SYMLINK_NOFOLLOW);
 #else
@@ -372,7 +401,7 @@ PRIMITIVE(stat) {
   Object* size = Primitive::integer(statbuf.st_size, process);
   if (Primitive::is_error(size)) return size;
 
-#if defined(TOIT_LINUX)
+#if defined(TOIT_LINUX) || defined(TOIT_FREERTOS)
   Object* atime = time_stamp(process, statbuf.st_atim);
   if (Primitive::is_error(atime)) return atime;
 
@@ -408,38 +437,42 @@ PRIMITIVE(stat) {
 
 PRIMITIVE(unlink) {
   ARGS(cstring, pathname);
-  int result = unlinkat(current_dir(process), pathname, 0);
+  int result = FILE_UNLINK_(current_dir(process), pathname, 0);
   if (result < 0) return return_open_error(process, errno);
   return process->program()->null_object();
 }
 
 PRIMITIVE(rmdir) {
   ARGS(cstring, pathname);
-  int result = unlinkat(current_dir(process), pathname, AT_REMOVEDIR);
+  int result = FILE_UNLINK_(current_dir(process), pathname, AT_REMOVEDIR);
   if (result < 0) return return_open_error(process, errno);
   return process->program()->null_object();
 }
 
 PRIMITIVE(rename) {
   ARGS(cstring, old_name, cstring, new_name);
-  int result = renameat(current_dir(process), old_name, current_dir(process), new_name);
+  int result = FILE_RENAME_(current_dir(process), old_name, current_dir(process), new_name);
   if (result < 0) return return_open_error(process, errno);
   return process->program()->null_object();
 }
 
 PRIMITIVE(chdir) {
+#ifndef TOIT_FREERTOS
   ARGS(cstring, pathname);
   int old_dir = current_dir(process);
-  int new_dir = openat(old_dir, pathname, O_DIRECTORY | O_RDONLY);
+  int new_dir = FILE_OPEN_(old_dir, pathname, O_DIRECTORY | O_RDONLY);
   if (new_dir < 0) return return_open_error(process, errno);
   process->set_current_directory(new_dir);
   close(old_dir);
   return process->program()->null_object();
+#else
+  UNIMPLEMENTED_PRIMITIVE;
+#endif
 }
 
 PRIMITIVE(mkdir) {
   ARGS(cstring, pathname, int, mode);
-  int result = mkdirat(current_dir(process), pathname, mode);
+  int result = FILE_MKDIR_(current_dir(process), pathname, mode);
   return result < 0
     ? return_open_error(process, errno)
     : process->program()->null_object();
@@ -486,6 +519,13 @@ PRIMITIVE(is_open_file) {
 
 PRIMITIVE(realpath) {
   ARGS(cstring, filename);
+#ifdef TOIT_FREERTOS
+  String* result = process->allocate_string(filename);
+  if (result == null) {
+    ALLOCATION_FAILED;
+  }
+  return result;
+#else
   char* c_result = realpath(filename, null);
   if (c_result == null) {
     if (errno == ENOMEM) MALLOC_FAILED;
@@ -498,6 +538,7 @@ PRIMITIVE(realpath) {
     ALLOCATION_FAILED;
   }
   return result;
+#endif // TOIT_FREERTOS
 }
 
 PRIMITIVE(cwd) {

src/resources/spi_esp32.cc

@@ -51,28 +51,19 @@ ResourcePool<spi_host_device_t, kInvalidHostDevice> spi_host_devices(
 #endif
 );
 
-class SPIResourceGroup : public ResourceGroup {
- public:
-  TAG(SPIResourceGroup);
-  SPIResourceGroup(Process* process, EventSource* event_source, spi_host_device_t host_device, int dma_chan)
+SPIResourceGroup::SPIResourceGroup(Process* process, EventSource* event_source, spi_host_device_t host_device,
+                                   int dma_channel)
     : ResourceGroup(process, event_source)
     , host_device_(host_device)
-    , dma_chan_(dma_chan) {}
-
-  ~SPIResourceGroup() {
-    SystemEventSource::instance()->run([&]() -> void {
-      FATAL_IF_NOT_ESP_OK(spi_bus_free(host_device_));
-    });
-    spi_host_devices.put(host_device_);
-    dma_channels.put(dma_chan_);
-  }
-
-  spi_host_device_t host_device() { return host_device_; }
+    , dma_channel_(dma_channel) {}
 
- private:
-  spi_host_device_t host_device_;
-  int dma_chan_;
-};
+SPIResourceGroup::~SPIResourceGroup() {
+  SystemEventSource::instance()->run([&]() -> void {
+    FATAL_IF_NOT_ESP_OK(spi_bus_free(host_device_));
+  });
+  spi_host_devices.put(host_device_);
+  dma_channels.put(dma_channel_);
+}
 
 MODULE_IMPLEMENTATION(spi, MODULE_SPI);