Formatting

README.md

@@ -98,9 +98,9 @@ CrossPoint Reader is pretty aggressive about caching data down to the SD card to
 has ~380KB of usable RAM, so we have to be careful. A lot of the decisions made in the design of the firmware were based
 on this constraint.
 
-### EPUB caching
+### Data caching
 
-The first time chapters of an EPUB are loaded, they are cached to the SD card. Subsequent loads are served from the 
+The first time chapters of a book are loaded, they are cached to the SD card. Subsequent loads are served from the 
 cache. This cache directory exists at `.crosspoint` on the SD card. The structure is as follows:
 
 
@@ -108,25 +108,22 @@ cache. This cache directory exists at `.crosspoint` on the SD card. The structur
 .crosspoint/
 ├── epub_12471232/       # Each EPUB is cached to a subdirectory named `epub_<hash>`
 │   ├── progress.bin     # Stores reading progress (chapter, page, etc.)
-│   ├── 0/               # Each chapter is stored in a subdirectory named by its index (based on the spine order)
+│   ├── cover.bmp        # Book cover image (once generated)
-│   │   ├── section.bin  # Section metadata (page count)
+│   ├── book.bin         # Book metadata (title, author, spine, table of contents, etc.)
-│   │   ├── page_0.bin   # Each page is stored in a separate file, it
+│   └── sections/        # All chapter data is stored in the sections subdirectory
-│   │   ├── page_1.bin   #   contains the position (x, y) and text for each word
+│       ├── 0.bin        # Chapter data (screen count, all text layout info, etc.)
-│   │   └── ...
+│       ├── 1.bin        #     files are named by their index in the spine
-│   ├── 1/
-│   │   ├── section.bin
-│   │   ├── page_0.bin
-│   │   ├── page_1.bin
-│   │   └── ...
 │       └── ...
 │
 └── epub_189013891/
 ```
 
-Deleting the `.crosspoint` directory will clear the cache. 
+Deleting the `.crosspoint` directory will clear the entire cache. 
 
-Due the way it's currently implemented, the cache is not automatically cleared when the EPUB is deleted and moving an 
+Due the way it's currently implemented, the cache is not automatically cleared when a book is deleted and moving a book
-EPUB file will reset the reading progress.
+file will use a new cache directory, resetting the reading progress.
+
+For more details on the internal file structures, see the [file formats document](./docs/file-formats.md).
 
 ## Contributing
 

docs/file-formats.md

@@ -0,0 +1,9 @@
+# File Formats
+
+## `book.bin`
+
+![](./images/file-formats/book-bin.png)
+
+## `section.bin`
+
+![](./images/file-formats/section-bin.png)

lib/Epub/Epub/Page.cpp

@@ -7,12 +7,12 @@ void PageLine::render(GfxRenderer& renderer, const int fontId, const int xOffset
   block->render(renderer, fontId, xPos + xOffset, yPos + yOffset);
 }
 
-void PageLine::serialize(File& file) {
+bool PageLine::serialize(File& file) {
   serialization::writePod(file, xPos);
   serialization::writePod(file, yPos);
 
   // serialize TextBlock pointed to by PageLine
-  block->serialize(file);
+  return block->serialize(file);
 }
 
 std::unique_ptr<PageLine> PageLine::deserialize(File& file) {
@@ -31,17 +31,21 @@ void Page::render(GfxRenderer& renderer, const int fontId, const int xOffset, co
   }
 }
 
-void Page::serialize(File& file) const {
+bool Page::serialize(File& file) const {
   const uint32_t count = elements.size();
   serialization::writePod(file, count);
 
   for (const auto& el : elements) {
     // Only PageLine exists currently
     serialization::writePod(file, static_cast<uint8_t>(TAG_PageLine));
-    el->serialize(file);
+    if (!el->serialize(file)) {
+      return false;
     }
   }
 
+  return true;
+}
+
 std::unique_ptr<Page> Page::deserialize(File& file) {
   auto page = std::unique_ptr<Page>(new Page());
 

lib/Epub/Epub/Page.h

@@ -18,7 +18,7 @@ class PageElement {
   explicit PageElement(const int16_t xPos, const int16_t yPos) : xPos(xPos), yPos(yPos) {}
   virtual ~PageElement() = default;
   virtual void render(GfxRenderer& renderer, int fontId, int xOffset, int yOffset) = 0;
-  virtual void serialize(File& file) = 0;
+  virtual bool serialize(File& file) = 0;
 };
 
 // a line from a block element
@@ -29,7 +29,7 @@ class PageLine final : public PageElement {
   PageLine(std::shared_ptr<TextBlock> block, const int16_t xPos, const int16_t yPos)
       : PageElement(xPos, yPos), block(std::move(block)) {}
   void render(GfxRenderer& renderer, int fontId, int xOffset, int yOffset) override;
-  void serialize(File& file) override;
+  bool serialize(File& file) override;
   static std::unique_ptr<PageLine> deserialize(File& file);
 };
 
@@ -38,6 +38,6 @@ class Page {
   // the list of block index and line numbers on this page
   std::vector<std::shared_ptr<PageElement>> elements;
   void render(GfxRenderer& renderer, int fontId, int xOffset, int yOffset) const;
-  void serialize(File& file) const;
+  bool serialize(File& file) const;
   static std::unique_ptr<Page> deserialize(File& file);
 };

lib/Epub/Epub/Section.cpp

@@ -11,7 +11,6 @@ namespace {
 constexpr uint8_t SECTION_FILE_VERSION = 7;
 constexpr size_t HEADER_SIZE = sizeof(uint8_t) + sizeof(int) + sizeof(float) + sizeof(bool) + sizeof(int) +
                                sizeof(int) + sizeof(int) + sizeof(size_t);
-constexpr char sectionFileName[] = "/section.bin";
 }  // namespace
 
 size_t Section::onPageComplete(std::unique_ptr<Page> page) {
@@ -19,8 +18,12 @@ size_t Section::onPageComplete(std::unique_ptr<Page> page) {
     Serial.printf("[%lu] [SCT] File not open for writing page %d\n", millis(), pageCount);
     return 0;
   }
+
   const auto position = file.position();
-  page->serialize(file);
+  if (!page->serialize(file)) {
+    Serial.printf("[%lu] [SCT] Failed to serialize page %d\n", millis(), pageCount);
+    return 0;
+  }
   Serial.printf("[%lu] [SCT] Page %d processed\n", millis(), pageCount);
 
   pageCount++;
@@ -49,7 +52,7 @@ void Section::writeSectionFileHeader(const int fontId, const float lineCompressi
 
 bool Section::loadSectionFile(const int fontId, const float lineCompression, const bool extraParagraphSpacing,
                               const int viewportWidth, const int viewportHeight) {
-  if (!FsHelpers::openFileForRead("SCT", cachePath + sectionFileName, file)) {
+  if (!FsHelpers::openFileForRead("SCT", filePath, file)) {
     return false;
   }
 
@@ -89,19 +92,14 @@ bool Section::loadSectionFile(const int fontId, const float lineCompression, con
   return true;
 }
 
-void Section::setupCacheDir() const {
-  epub->setupCacheDir();
-  SD.mkdir(cachePath.c_str());
-}
-
 // Your updated class method (assuming you are using the 'SD' object, which is a wrapper for a specific filesystem)
 bool Section::clearCache() const {
-  if (!SD.exists(cachePath.c_str())) {
+  if (!SD.exists(filePath.c_str())) {
     Serial.printf("[%lu] [SCT] Cache does not exist, no action needed\n", millis());
     return true;
   }
 
-  if (!FsHelpers::removeDir(cachePath.c_str())) {
+  if (!SD.remove(filePath.c_str())) {
     Serial.printf("[%lu] [SCT] Failed to clear cache\n", millis());
     return false;
   }
@@ -159,7 +157,7 @@ bool Section::createSectionFile(const int fontId, const float lineCompression, c
     progressSetupFn();
   }
 
-  if (!FsHelpers::openFileForWrite("SCT", cachePath + sectionFileName, file)) {
+  if (!FsHelpers::openFileForWrite("SCT", filePath, file)) {
     return false;
   }
   writeSectionFileHeader(fontId, lineCompression, extraParagraphSpacing, viewportWidth, viewportHeight);
@@ -175,15 +173,28 @@ bool Section::createSectionFile(const int fontId, const float lineCompression, c
   if (!success) {
     Serial.printf("[%lu] [SCT] Failed to parse XML and build pages\n", millis());
     file.close();
-    SD.remove((cachePath + sectionFileName).c_str());
+    SD.remove(filePath.c_str());
     return false;
   }
 
   const auto lutOffset = file.position();
+  bool hasFailedLutRecords = false;
   // Write LUT
   for (const auto& pos : lut) {
+    if (pos == 0) {
+      hasFailedLutRecords = true;
+      break;
+    }
     serialization::writePod(file, pos);
   }
+
+  if (hasFailedLutRecords) {
+    Serial.printf("[%lu] [SCT] Failed to write LUT due to invalid page positions\n", millis());
+    file.close();
+    SD.remove(filePath.c_str());
+    return false;
+  }
+
   // Go back and write LUT offset
   file.seek(HEADER_SIZE - sizeof(size_t) - sizeof(pageCount));
   serialization::writePod(file, pageCount);
@@ -193,7 +204,7 @@ bool Section::createSectionFile(const int fontId, const float lineCompression, c
 }
 
 std::unique_ptr<Page> Section::loadPageFromSectionFile() {
-  if (!FsHelpers::openFileForRead("SCT", cachePath + sectionFileName, file)) {
+  if (!FsHelpers::openFileForRead("SCT", filePath, file)) {
     return nullptr;
   }
 

lib/Epub/Epub/Section.h

@@ -11,7 +11,7 @@ class Section {
   std::shared_ptr<Epub> epub;
   const int spineIndex;
   GfxRenderer& renderer;
-  std::string cachePath;
+  std::string filePath;
   File file;
 
   void writeSectionFileHeader(int fontId, float lineCompression, bool extraParagraphSpacing, int viewportWidth,
@@ -26,11 +26,10 @@ class Section {
       : epub(epub),
         spineIndex(spineIndex),
         renderer(renderer),
-        cachePath(epub->getCachePath() + "/" + std::to_string(spineIndex)) {}
+        filePath(epub->getCachePath() + "/sections/" + std::to_string(spineIndex) + ".bin") {}
   ~Section() = default;
   bool loadSectionFile(int fontId, float lineCompression, bool extraParagraphSpacing, int viewportWidth,
                        int viewportHeight);
-  void setupCacheDir() const;
   bool clearCache() const;
   bool createSectionFile(int fontId, float lineCompression, bool extraParagraphSpacing, int viewportWidth,
                          int viewportHeight, const std::function<void()>& progressSetupFn = nullptr,

lib/Epub/Epub/blocks/TextBlock.cpp

@@ -24,34 +24,33 @@ void TextBlock::render(const GfxRenderer& renderer, const int fontId, const int
   }
 }
 
-void TextBlock::serialize(File& file) const {
+bool TextBlock::serialize(File& file) const {
-  // words
+  if (words.size() != wordXpos.size() || words.size() != wordStyles.size()) {
-  const uint32_t wc = words.size();
+    Serial.printf("[%lu] [TXB] Serialization failed: size mismatch (words=%u, xpos=%u, styles=%u)\n", millis(),
-  serialization::writePod(file, wc);
+                  words.size(), wordXpos.size(), wordStyles.size());
+    return false;
+  }
+
+  // Word data
+  serialization::writePod(file, static_cast<uint32_t>(words.size()));
   for (const auto& w : words) serialization::writeString(file, w);
-
-  // wordXpos
-  const uint32_t xc = wordXpos.size();
-  serialization::writePod(file, xc);
   for (auto x : wordXpos) serialization::writePod(file, x);
-
-  // wordStyles
-  const uint32_t sc = wordStyles.size();
-  serialization::writePod(file, sc);
   for (auto s : wordStyles) serialization::writePod(file, s);
 
-  // style
+  // Block style
   serialization::writePod(file, style);
+
+  return true;
 }
 
 std::unique_ptr<TextBlock> TextBlock::deserialize(File& file) {
-  uint32_t wc, xc, sc;
+  uint32_t wc;
   std::list<std::string> words;
   std::list<uint16_t> wordXpos;
   std::list<EpdFontStyle> wordStyles;
   BLOCK_STYLE style;
 
-  // words
+  // Word count
   serialization::readPod(file, wc);
 
   // Sanity check: prevent allocation of unreasonably large lists (max 10000 words per block)
@@ -60,27 +59,15 @@ std::unique_ptr<TextBlock> TextBlock::deserialize(File& file) {
     return nullptr;
   }
 
+  // Word data
   words.resize(wc);
+  wordXpos.resize(wc);
+  wordStyles.resize(wc);
   for (auto& w : words) serialization::readString(file, w);
-
-  // wordXpos
-  serialization::readPod(file, xc);
-  wordXpos.resize(xc);
   for (auto& x : wordXpos) serialization::readPod(file, x);
-
-  // wordStyles
-  serialization::readPod(file, sc);
-  wordStyles.resize(sc);
   for (auto& s : wordStyles) serialization::readPod(file, s);
 
-  // Validate data consistency: all three lists must have the same size
+  // Block style
-  if (wc != xc || wc != sc) {
-    Serial.printf("[%lu] [TXB] Deserialization failed: size mismatch (words=%u, xpos=%u, styles=%u)\n", millis(), wc,
-                  xc, sc);
-    return nullptr;
-  }
-
-  // style
   serialization::readPod(file, style);
 
   return std::unique_ptr<TextBlock>(new TextBlock(std::move(words), std::move(wordXpos), std::move(wordStyles), style));

lib/Epub/Epub/blocks/TextBlock.h

@@ -36,6 +36,6 @@ class TextBlock final : public Block {
   // given a renderer works out where to break the words into lines
   void render(const GfxRenderer& renderer, int fontId, int x, int y) const;
   BlockType getType() override { return TEXT_BLOCK; }
-  void serialize(File& file) const;
+  bool serialize(File& file) const;
   static std::unique_ptr<TextBlock> deserialize(File& file);
 };

src/activities/reader/EpubReaderActivity.cpp

@@ -282,8 +282,6 @@ void EpubReaderActivity::renderScreen() {
         pagesUntilFullRefresh = 0;
       }
 
-      section->setupCacheDir();
-
       // Setup callback - only called for chapters >= 50KB, redraws with progress bar
       auto progressSetup = [this, boxXWithBar, boxWidthWithBar, boxHeightWithBar, barX, barY] {
         renderer.fillRect(boxXWithBar, boxY, boxWidthWithBar, boxHeightWithBar, false);