Merge 1d2fff18d5 into 4dd73a211a

lib/ResourcesFS/ResourcesFS.cpp

@@ -0,0 +1,110 @@
+#include "ResourcesFS.h"
+
+#include <Arduino.h>
+#include <esp_partition.h>
+
+#include <vector>
+
+ResourcesFS ResourcesFS::instance;
+
+class ResourcesFS::Impl {
+ public:
+  const esp_partition_t* partition = nullptr;
+  const Header* header = nullptr;
+  const uint8_t* mmap_data = nullptr;
+};
+
+bool ResourcesFS::begin(bool remount) {
+  if (!remount) {
+    assert(impl == nullptr && "begin called multiple times");
+    impl = new Impl();
+  } else {
+    assert(impl != nullptr && "remount called before initial begin");
+  }
+
+  impl->partition = esp_partition_find_first(ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_SPIFFS, nullptr);
+  if (!impl->partition) {
+    Serial.printf("[%lu] [FSS] SPIFFS partition not found, skipping\n", millis());
+    impl->header = nullptr;
+    impl->mmap_data = nullptr;
+    return false;
+  }
+
+  spi_flash_mmap_handle_t map_handle;  // unused
+  size_t len = impl->partition->size;
+  if (len > MAX_ALLOC_SIZE) {
+    len = MAX_ALLOC_SIZE;
+  }
+
+  auto err =
+      esp_partition_mmap(impl->partition, 0, len, SPI_FLASH_MMAP_DATA, (const void**)&impl->mmap_data, &map_handle);
+  if (err != ESP_OK || impl->mmap_data == nullptr) {
+    Serial.printf("[%lu] [FSS] mmap failed, code: %d, skipping\n", millis(), err);
+    impl->header = nullptr;
+    impl->mmap_data = nullptr;
+    return false;
+  }
+
+  impl->header = (const Header*)impl->mmap_data;
+  if (impl->header->magic != MAGIC) {
+    Serial.printf("[%lu] [FSS] Invalid magic: 0x%08X, skipping\n", millis(), impl->header->magic);
+    impl->header = nullptr;
+    impl->mmap_data = nullptr;
+    return false;
+  }
+
+  Serial.printf("[%lu] [FSS] ResourcesFS initialized\n", millis());
+  return true;
+}
+
+const ResourcesFS::Header* ResourcesFS::getRoot() {
+  assert(impl != nullptr);
+  return impl->header;
+}
+
+const uint8_t* ResourcesFS::mmap(const ResourcesFS::FileEntry* entry) {
+  assert(impl != nullptr);
+  assert(impl->header != nullptr);
+  assert(impl->mmap_data != nullptr);
+
+  size_t offset = sizeof(Header);
+  for (size_t i = 0; i < MAX_FILES; i++) {
+    const FileEntry& e = impl->header->entries[i];
+    if (&e == entry) {
+      break;
+    }
+    offset += e.size;
+    offset += get_padding(e.size);
+  }
+  return impl->mmap_data + offset;
+}
+
+bool ResourcesFS::erase(size_t size) {
+  assert(impl != nullptr);
+  assert(impl->partition != nullptr);
+
+  // align size to sector size
+  static constexpr size_t sector_size = 4096;
+  size = (size + sector_size - 1) / sector_size * sector_size;
+
+  auto err = esp_partition_erase_range(impl->partition, 0, size);
+  if (err != ESP_OK) {
+    Serial.printf("[%lu] [FSS] erase failed, code %d\n", millis(), err);
+    return false;
+  }
+
+  return true;
+}
+
+bool ResourcesFS::write(uint32_t offset, const uint8_t* data, size_t len) {
+  assert(impl != nullptr);
+  assert(impl->partition != nullptr);
+  assert(offset + len <= impl->partition->size);
+
+  auto err = esp_partition_write(impl->partition, offset, data, len);
+  if (err != ESP_OK) {
+    Serial.printf("[%lu] [FSS] write failed, offset %u, len %u, code %d\n", millis(), offset, len, err);
+    return false;
+  }
+  return true;
+}

lib/ResourcesFS/ResourcesFS.h

@@ -0,0 +1,123 @@
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+
+// Simple implementation of read-only packed filesystem
+// Inspired by packed resources used by some STM32 smartwatch / smartband firmwares
+class ResourcesFS {
+ public:
+  ResourcesFS() = default;
+  ~ResourcesFS() = default;
+
+  // FIXME: true MAX_ALLOC_SIZE = ~3.5MB but not enough pages to do mmap
+  // See: spi_flash_mmap_get_free_pages() --> ~31 free pages on real device
+
+  static constexpr size_t MAX_FILES = 32;
+  static constexpr size_t MAX_FILE_NAME_LENGTH = 32;
+  static constexpr size_t MAX_ALLOC_SIZE = 28 * 64 * 1024;  // 28 pages == ~1.8 MB
+  static constexpr size_t ALIGNMENT = 4;                    // bytes
+  static constexpr uint32_t MAGIC = 0x46535631;             // "FSV1"
+
+  enum FileType {
+    FILETYPE_INVALID = 0,
+    FILETYPE_FONT_REGULAR = 1,
+  };
+
+  struct __attribute__((packed)) FileEntry {
+    uint32_t type;
+    uint32_t size;
+    char name[MAX_FILE_NAME_LENGTH];
+  };
+  static_assert(sizeof(FileEntry) == (4 + 4 + MAX_FILE_NAME_LENGTH));
+
+  struct __attribute__((packed)) Header {
+    uint32_t magic;
+    FileEntry entries[MAX_FILES];
+  };
+  static_assert(sizeof(Header) == 4 + MAX_FILES * sizeof(FileEntry));
+  static_assert(sizeof(Header) % ALIGNMENT == 0);
+
+  static size_t get_padding(size_t size) {
+    size_t remainder = size % ALIGNMENT;
+    return (remainder == 0) ? 0 : (ALIGNMENT - remainder);
+  }
+
+  // returns true if mounted successfully
+  // remount should only be used after write/erase operations
+  bool begin(bool remount = false);
+
+  // returns nullptr if not mounted
+  const Header* getRoot();
+
+  // always return a valid pointer; undefined behavior if entry is invalid
+  const uint8_t* mmap(const FileEntry* entry);
+
+  // flash writing
+  // (note: erase must be called before writing, otherwise write will result in corrupted data)
+  bool erase(size_t size);
+  bool write(uint32_t offset, const uint8_t* data, size_t len);
+
+#ifdef CREATE_RESOURCES
+  // to be used by host CLI tool that creates the packed filesystem
+  uint8_t writeData[MAX_ALLOC_SIZE];
+  size_t writeDataSize = 0;
+
+  void beginCreate() {
+    Header* header = (Header*)writeData;
+    header->magic = MAGIC;
+    for (size_t i = 0; i < MAX_FILES; i++) {
+      header->entries[i].type = FILETYPE_INVALID;
+      header->entries[i].size = 0;
+      header->entries[i].name[0] = '\0';
+    }
+    writeDataSize = sizeof(Header);
+  }
+
+  uint8_t* getWriteData() { return writeData; }
+
+  size_t getWriteSize() { return writeDataSize; }
+
+  // return error message or nullptr if successful
+  const char* addFileEntry(const FileEntry& entry, const uint8_t* data) {
+    Header* header = (Header*)writeData;
+    if (entry.size % ALIGNMENT != 0) {
+      return "File size must be multiple of alignment";
+    }
+    if (writeDataSize + entry.size > MAX_ALLOC_SIZE) {
+      return "Not enough space in ResourcesFS image";
+    }
+    if (entry.size == 0 || entry.name[0] == '\0') {
+      return "Invalid file entry";
+    }
+    // find empty slot
+    for (size_t i = 0; i < MAX_FILES; i++) {
+      if (header->entries[i].type == FILETYPE_INVALID) {
+        header->entries[i] = entry;
+        // copy data
+        size_t offset = writeDataSize;
+        for (size_t j = 0; j < entry.size; j++) {
+          writeData[offset + j] = data[j];
+        }
+        writeDataSize += entry.size;  // (no need padding here as size is already aligned)
+        return nullptr;               // success
+      } else if (strncmp(header->entries[i].name, entry.name, MAX_FILE_NAME_LENGTH) == 0) {
+        return "File with the same name already exists";
+      }
+    }
+    return "No empty slot available";
+  }
+#endif
+
+  static ResourcesFS& getInstance() { return instance; }
+
+ private:
+  // using pimpl to allow re-using this header on host system without Arduino dependencies
+  class Impl;
+  Impl* impl = nullptr;
+
+  static ResourcesFS instance;
+};
+
+#define Resources ResourcesFS::getInstance()

scripts/make_resources.py

@@ -0,0 +1,110 @@
+import argparse
+import struct
+import sys
+import os
+
+# TODO: sync with ResourcesFS.h
+
+MAX_FILES = 32
+MAX_FILE_NAME_LENGTH = 32
+ALIGNMENT = 4
+MAGIC = 0x46535631
+MAX_ALLOC_SIZE = 3 * 1024 * 1024
+
+filetype_map = {
+  'INVALID': 0,
+  'FONT_REGULAR': 1,
+}
+
+def main():
+  parser = argparse.ArgumentParser(description='Generate resources.bin')
+  parser.add_argument('-o', default='resources.bin', help='specify output binary file (default: resources.bin)')
+  parser.add_argument('inputs', nargs='*', help='file1:type1 file2:type2 ... (note: if file name contains extension, it will be stripped; example: my_font.bin:FONT_REGULAR will be stored as "my_font")')
+  args = parser.parse_args()
+
+  write_data = bytearray()
+  write_data += struct.pack('<I', MAGIC)
+  for _ in range(MAX_FILES):
+    write_data += struct.pack('<II32s', 0, 0, b'\x00' * 32)
+  current_size = len(write_data)
+
+  for inp in args.inputs:
+    try:
+      file_path, type_str = inp.split(':')
+    except ValueError:
+      print(f"Invalid input format: {inp}. Expected file:type")
+      sys.exit(1)
+
+    basename = os.path.basename(file_path)
+    name = os.path.splitext(basename)[0]
+    name_len = len(name.encode('ascii'))
+    if name_len > MAX_FILE_NAME_LENGTH - 1:
+      print(f"File name too long: {name} (max {MAX_FILE_NAME_LENGTH - 1} chars)")
+      sys.exit(1)
+    name_bytes = name.encode('ascii') + b'\x00' * (MAX_FILE_NAME_LENGTH - name_len)
+
+    if type_str.isdigit():
+      type_int = int(type_str)
+    else:
+      upper_type = type_str.upper()
+      if upper_type in filetype_map:
+        type_int = filetype_map[upper_type]
+      else:
+        print(f"Unknown file type: {type_str}")
+        sys.exit(1)
+
+    try:
+      with open(file_path, 'rb') as f:
+        data = f.read()
+    except IOError as e:
+      print(f"Error reading file {file_path}: {e}")
+      sys.exit(1)
+
+    size = len(data)
+    if size == 0:
+      print(f"Invalid file entry: empty file {file_path}")
+      sys.exit(1)
+    if size % ALIGNMENT != 0:
+      print(f"File size must be multiple of alignment ({ALIGNMENT}): {file_path} size={size}")
+      sys.exit(1)
+    if current_size + size > MAX_ALLOC_SIZE:
+      print(f"Not enough space in ResourcesFS image for {file_path}")
+      sys.exit(1)
+
+    # Find empty slot and check for duplicates
+    found = -1
+    for i in range(MAX_FILES):
+      offset = 4 + i * 40
+      t, s, n = struct.unpack_from('<II32s', write_data, offset)
+      name_existing = n.split(b'\x00', 1)[0].decode('ascii', errors='ignore')
+      if t == 0:
+        if found == -1:
+          found = i
+      elif name_existing == name:
+        print(f"File with the same name already exists: {name}")
+        sys.exit(1)
+
+    if found == -1:
+      print("No empty slot available")
+      sys.exit(1)
+
+    # Set entry
+    offset = 4 + found * 40
+    write_data[offset:offset + 8] = struct.pack('<II', type_int, size)
+    write_data[offset + 8:offset + 40] = name_bytes
+
+    # Append data
+    write_data += data
+    current_size += size
+
+    print(f"Added file: {name}, type={type_int}, size={size}")
+
+  try:
+    with open(args.o, 'wb') as f:
+      f.write(write_data)
+  except IOError as e:
+    print(f"Error writing output file {args.o}: {e}")
+    sys.exit(1)
+
+if __name__ == '__main__':
+  main()

src/main.cpp

@@ -5,6 +5,7 @@
 #include <HalGPIO.h>
 #include <SDCardManager.h>
 #include <SPI.h>
+#include <ResourcesFS.h>
 #include <builtinFonts/all.h>
 
 #include <cstring>
@@ -266,6 +267,63 @@ void setupDisplayAndFonts() {
   Serial.printf("[%lu] [   ] Fonts setup\n", millis());
 }
 
+void demoResourcesFS() {
+  static const char* RESOURCES_FILE = "/resources.bin";
+  FsFile file = SdMan.open(RESOURCES_FILE, O_RDONLY);
+  if (!file) {
+    Serial.printf("[%lu] [   ] No custom resources to flash\n", millis());
+    return;
+  }
+  const size_t fileSize = file.size();
+  Serial.printf("[%lu] [   ] Flashing custom resources (%u bytes)\n", millis(), fileSize);
+
+  if (!Resources.erase(fileSize)) {
+    return; // failed
+  }
+
+  static constexpr size_t CHUNK_SIZE = 4096;
+  uint8_t buffer[CHUNK_SIZE];
+  size_t bytesFlashed = 0;
+  while (bytesFlashed < fileSize) {
+    size_t toRead = std::min(CHUNK_SIZE, fileSize - bytesFlashed);
+    int bytesRead = file.read(buffer, toRead);
+    if (bytesRead <= 0) {
+      Serial.printf("[%lu] [   ] Error reading resources file\n", millis());
+      return;
+    }
+    auto ok = Resources.write(bytesFlashed, buffer, bytesRead);
+    if (!ok) {
+      return;
+    }
+    bytesFlashed += bytesRead;
+  }
+  Serial.printf("[%lu] [   ] Finished flashing custom resources\n", millis());
+  SdMan.remove(RESOURCES_FILE); // remove the file after flashing
+  file.close();
+
+  // attempt to remount
+  if (!Resources.begin(true)) {
+    Serial.printf("[%lu] [   ] Error mounting flashed resources\n", millis());
+    return;
+  }
+
+  // try to list all files
+  Serial.printf("[%lu] [   ] Listing flashed resources:\n", millis());
+  const auto* root = Resources.getRoot();
+  for (size_t i = 0; i < ResourcesFS::MAX_FILES; i++) {
+    const auto& entry = root->entries[i];
+    if (entry.type != ResourcesFS::FILETYPE_INVALID) {
+      Serial.printf("  - Name: %s, Type: %u, Size: %u bytes\n", entry.name, entry.type, entry.size);
+      Serial.printf("    First 8 bytes of the file: ");
+      const auto* data = Resources.mmap(&entry);
+      for (size_t j = 0; j < std::min((size_t)8, (size_t)entry.size); j++) {
+        Serial.printf("%02X ", data[j]);
+      }
+      Serial.printf("\n");
+    }
+  }
+}
+
 void setup() {
   t1 = millis();
 
@@ -291,6 +349,9 @@ void setup() {
     return;
   }
 
+  Resources.begin();
+  demoResourcesFS();
+
   SETTINGS.loadFromFile();
   KOREADER_STORE.loadFromFile();