snark13/Xteink-X4-crosspoint-reader
1
0
Fork 0
mirror of https://github.com/daveallie/crosspoint-reader.git synced 2026-02-05 15:17:37 +03:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Consolidate chapter page data into single file

Browse Source 
Has ~80% speedup to page generation (indexing) time, and makes cache cleanup a lot faster
This commit is contained in:
Dave Allie 2025-12-29 12:26:38 +11:00
parent b1763821b5
commit ea3584a918
No known key found for this signature in database
GPG Key ID: F2FDDB3AD8D0276F
4 changed files with 92 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
lib/Epub/Epub/Page.cpp
View File

@ -3,10 +3,6 @@
#include <HardwareSerial.h> #include <HardwareSerial.h>
#include <Serialization.h> #include <Serialization.h>
namespace {
constexpr uint8_t PAGE_FILE_VERSION = 3;
}
void PageLine::render(GfxRenderer& renderer, const int fontId, const int xOffset, const int yOffset) { void PageLine::render(GfxRenderer& renderer, const int fontId, const int xOffset, const int yOffset) {
block->render(renderer, fontId, xPos + xOffset, yPos + yOffset); block->render(renderer, fontId, xPos + xOffset, yPos + yOffset);
} }
@ -36,8 +32,6 @@ void Page::render(GfxRenderer& renderer, const int fontId, const int xOffset, co
} }
void Page::serialize(File& file) const { void Page::serialize(File& file) const {
serialization::writePod(file, PAGE_FILE_VERSION);
const uint32_t count = elements.size(); const uint32_t count = elements.size();
serialization::writePod(file, count); serialization::writePod(file, count);
@ -49,13 +43,6 @@ void Page::serialize(File& file) const {
} }
std::unique_ptr<Page> Page::deserialize(File& file) { std::unique_ptr<Page> Page::deserialize(File& file) {
uint8_t version;
serialization::readPod(file, version);
if (version != PAGE_FILE_VERSION) {
Serial.printf("[%lu] [PGE] Deserialization failed: Unknown version %u\n", millis(), version);
return nullptr;
}
auto page = std::unique_ptr<Page>(new Page()); auto page = std::unique_ptr<Page>(new Page());
uint32_t count; uint32_t count;

128
lib/Epub/Epub/Section.cpp
View File

@ -8,54 +8,57 @@
#include "parsers/ChapterHtmlSlimParser.h" #include "parsers/ChapterHtmlSlimParser.h"
namespace { namespace {
constexpr uint8_t SECTION_FILE_VERSION = 6; constexpr uint8_t SECTION_FILE_VERSION = 7;
constexpr size_t HEADER_SIZE = sizeof(uint8_t) + sizeof(int) + sizeof(float) + sizeof(bool) + sizeof(int) +
sizeof(int) + sizeof(int) + sizeof(size_t);
constexpr char sectionFileName[] = "/section.bin";
} // namespace } // namespace
void Section::onPageComplete(std::unique_ptr<Page> page) { size_t Section::onPageComplete(std::unique_ptr<Page> page) {
const auto filePath = cachePath + "/page_" + std::to_string(pageCount) + ".bin"; if (!file) {
Serial.printf("[%lu] [SCT] File not open for writing page %d\n", millis(), pageCount);
File outputFile; return 0;
if (!FsHelpers::openFileForWrite("SCT", filePath, outputFile)) {
return;
} }
page->serialize(outputFile); const auto position = file.position();
outputFile.close(); page->serialize(file);
Serial.printf("[%lu] [SCT] Page %d processed\n", millis(), pageCount); Serial.printf("[%lu] [SCT] Page %d processed\n", millis(), pageCount);
pageCount++; pageCount++;
return position;
} }
void Section::writeCacheMetadata(const int fontId, const float lineCompression, const bool extraParagraphSpacing, void Section::writeSectionFileHeader(const int fontId, const float lineCompression, const bool extraParagraphSpacing,
const int viewportWidth, const int viewportHeight) const { const int viewportWidth, const int viewportHeight) {
File outputFile; if (!file) {
if (!FsHelpers::openFileForWrite("SCT", cachePath + "/section.bin", outputFile)) { Serial.printf("[%lu] [SCT] File not open for writing header\n", millis());
return; return;
} }
serialization::writePod(outputFile, SECTION_FILE_VERSION); static_assert(HEADER_SIZE == sizeof(SECTION_FILE_VERSION) + sizeof(fontId) + sizeof(lineCompression) +
serialization::writePod(outputFile, fontId); sizeof(extraParagraphSpacing) + sizeof(viewportWidth) + sizeof(viewportHeight) +
serialization::writePod(outputFile, lineCompression); sizeof(pageCount) + sizeof(size_t),
serialization::writePod(outputFile, extraParagraphSpacing); "Header size mismatch");
serialization::writePod(outputFile, viewportWidth); serialization::writePod(file, SECTION_FILE_VERSION);
serialization::writePod(outputFile, viewportHeight); serialization::writePod(file, fontId);
serialization::writePod(outputFile, pageCount); serialization::writePod(file, lineCompression);
outputFile.close(); serialization::writePod(file, extraParagraphSpacing);
serialization::writePod(file, viewportWidth);
serialization::writePod(file, viewportHeight);
serialization::writePod(file, pageCount); // Placeholder for page count (will be initially 0 when written)
serialization::writePod(file, static_cast<size_t>(0)); // Placeholder for LUT offset
} }
bool Section::loadCacheMetadata(const int fontId, const float lineCompression, const bool extraParagraphSpacing, bool Section::loadSectionFile(const int fontId, const float lineCompression, const bool extraParagraphSpacing,
const int viewportWidth, const int viewportHeight) { const int viewportWidth, const int viewportHeight) {
const auto sectionFilePath = cachePath + "/section.bin"; if (!FsHelpers::openFileForRead("SCT", cachePath + sectionFileName, file)) {
File inputFile;
if (!FsHelpers::openFileForRead("SCT", sectionFilePath, inputFile)) {
return false; return false;
} }
// Match parameters // Match parameters
{ {
uint8_t version; uint8_t version;
serialization::readPod(inputFile, version); serialization::readPod(file, version);
if (version != SECTION_FILE_VERSION) { if (version != SECTION_FILE_VERSION) {
inputFile.close(); file.close();
Serial.printf("[%lu] [SCT] Deserialization failed: Unknown version %u\n", millis(), version); Serial.printf("[%lu] [SCT] Deserialization failed: Unknown version %u\n", millis(), version);
clearCache(); clearCache();
return false; return false;
@ -64,24 +67,24 @@ bool Section::loadCacheMetadata(const int fontId, const float lineCompression, c
int fileFontId, fileViewportWidth, fileViewportHeight; int fileFontId, fileViewportWidth, fileViewportHeight;
float fileLineCompression; float fileLineCompression;
bool fileExtraParagraphSpacing; bool fileExtraParagraphSpacing;
serialization::readPod(inputFile, fileFontId); serialization::readPod(file, fileFontId);
serialization::readPod(inputFile, fileLineCompression); serialization::readPod(file, fileLineCompression);
serialization::readPod(inputFile, fileExtraParagraphSpacing); serialization::readPod(file, fileExtraParagraphSpacing);
serialization::readPod(inputFile, fileViewportWidth); serialization::readPod(file, fileViewportWidth);
serialization::readPod(inputFile, fileViewportHeight); serialization::readPod(file, fileViewportHeight);
if (fontId != fileFontId || lineCompression != fileLineCompression || if (fontId != fileFontId || lineCompression != fileLineCompression ||
extraParagraphSpacing != fileExtraParagraphSpacing || viewportWidth != fileViewportWidth || extraParagraphSpacing != fileExtraParagraphSpacing || viewportWidth != fileViewportWidth ||
viewportHeight != fileViewportHeight) { viewportHeight != fileViewportHeight) {
inputFile.close(); file.close();
Serial.printf("[%lu] [SCT] Deserialization failed: Parameters do not match\n", millis()); Serial.printf("[%lu] [SCT] Deserialization failed: Parameters do not match\n", millis());
clearCache(); clearCache();
return false; return false;
} }
} }
serialization::readPod(inputFile, pageCount); serialization::readPod(file, pageCount);
inputFile.close(); file.close();
Serial.printf("[%lu] [SCT] Deserialization succeeded: %d pages\n", millis(), pageCount); Serial.printf("[%lu] [SCT] Deserialization succeeded: %d pages\n", millis(), pageCount);
return true; return true;
} }
@ -107,10 +110,10 @@ bool Section::clearCache() const {
return true; return true;
} }
bool Section::persistPageDataToSD(const int fontId, const float lineCompression, const bool extraParagraphSpacing, bool Section::createSectionFile(const int fontId, const float lineCompression, const bool extraParagraphSpacing,
const int viewportWidth, const int viewportHeight, const int viewportWidth, const int viewportHeight,
const std::function<void()>& progressSetupFn, const std::function<void()>& progressSetupFn,
const std::function<void(int)>& progressFn) { const std::function<void(int)>& progressFn) {
constexpr size_t MIN_SIZE_FOR_PROGRESS = 50 * 1024; // 50KB constexpr size_t MIN_SIZE_FOR_PROGRESS = 50 * 1024; // 50KB
const auto localPath = epub->getSpineItem(spineIndex).href; const auto localPath = epub->getSpineItem(spineIndex).href;
const auto tmpHtmlPath = epub->getCachePath() + "/.tmp_" + std::to_string(spineIndex) + ".html"; const auto tmpHtmlPath = epub->getCachePath() + "/.tmp_" + std::to_string(spineIndex) + ".html";
@ -156,30 +159,53 @@ bool Section::persistPageDataToSD(const int fontId, const float lineCompression,
progressSetupFn(); progressSetupFn();
} }
if (!FsHelpers::openFileForWrite("SCT", cachePath + sectionFileName, file)) {
return false;
}
writeSectionFileHeader(fontId, lineCompression, extraParagraphSpacing, viewportWidth, viewportHeight);
std::vector<size_t> lut = {};
ChapterHtmlSlimParser visitor( ChapterHtmlSlimParser visitor(
tmpHtmlPath, renderer, fontId, lineCompression, extraParagraphSpacing, viewportWidth, viewportHeight, tmpHtmlPath, renderer, fontId, lineCompression, extraParagraphSpacing, viewportWidth, viewportHeight,
[this](std::unique_ptr<Page> page) { this->onPageComplete(std::move(page)); }, progressFn); [this, &lut](std::unique_ptr<Page> page) { lut.emplace_back(this->onPageComplete(std::move(page))); },
progressFn);
success = visitor.parseAndBuildPages(); success = visitor.parseAndBuildPages();
SD.remove(tmpHtmlPath.c_str()); SD.remove(tmpHtmlPath.c_str());
if (!success) { if (!success) {
Serial.printf("[%lu] [SCT] Failed to parse XML and build pages\n", millis()); Serial.printf("[%lu] [SCT] Failed to parse XML and build pages\n", millis());
file.close();
SD.remove((cachePath + sectionFileName).c_str());
return false; return false;
} }
writeCacheMetadata(fontId, lineCompression, extraParagraphSpacing, viewportWidth, viewportHeight); const auto lutOffset = file.position();
// Write LUT
for (const auto& pos : lut) {
serialization::writePod(file, pos);
}
// Go back and write LUT offset
file.seek(HEADER_SIZE - sizeof(size_t) - sizeof(pageCount));
serialization::writePod(file, pageCount);
serialization::writePod(file, lutOffset);
file.close();
return true; return true;
} }
std::unique_ptr<Page> Section::loadPageFromSD() const { std::unique_ptr<Page> Section::loadPageFromSectionFile() {
const auto filePath = cachePath + "/page_" + std::to_string(currentPage) + ".bin"; if (!FsHelpers::openFileForRead("SCT", cachePath + sectionFileName, file)) {
File inputFile;
if (!FsHelpers::openFileForRead("SCT", filePath, inputFile)) {
return nullptr; return nullptr;
} }
auto page = Page::deserialize(inputFile);
inputFile.close(); file.seek(HEADER_SIZE - sizeof(size_t));
size_t lutOffset;
serialization::readPod(file, lutOffset);
file.seek(lutOffset + sizeof(size_t) * currentPage);
size_t pagePos;
serialization::readPod(file, pagePos);
file.seek(pagePos);
auto page = Page::deserialize(file);
file.close();
return page; return page;
} }

19
lib/Epub/Epub/Section.h
View File

@ -12,10 +12,11 @@ class Section {
const int spineIndex; const int spineIndex;
GfxRenderer& renderer; GfxRenderer& renderer;
std::string cachePath; std::string cachePath;
File file;
void writeCacheMetadata(int fontId, float lineCompression, bool extraParagraphSpacing, int viewportWidth, void writeSectionFileHeader(int fontId, float lineCompression, bool extraParagraphSpacing, int viewportWidth,
int viewportHeight) const; int viewportHeight);
void onPageComplete(std::unique_ptr<Page> page); size_t onPageComplete(std::unique_ptr<Page> page);
public: public:
int pageCount = 0; int pageCount = 0;
@ -27,12 +28,12 @@ class Section {
renderer(renderer), renderer(renderer),
cachePath(epub->getCachePath() + "/" + std::to_string(spineIndex)) {} cachePath(epub->getCachePath() + "/" + std::to_string(spineIndex)) {}
~Section() = default; ~Section() = default;
bool loadCacheMetadata(int fontId, float lineCompression, bool extraParagraphSpacing, int viewportWidth, bool loadSectionFile(int fontId, float lineCompression, bool extraParagraphSpacing, int viewportWidth,
int viewportHeight); int viewportHeight);
void setupCacheDir() const; void setupCacheDir() const;
bool clearCache() const; bool clearCache() const;
bool persistPageDataToSD(int fontId, float lineCompression, bool extraParagraphSpacing, int viewportWidth, bool createSectionFile(int fontId, float lineCompression, bool extraParagraphSpacing, int viewportWidth,
int viewportHeight, const std::function<void()>& progressSetupFn = nullptr, int viewportHeight, const std::function<void()>& progressSetupFn = nullptr,
const std::function<void(int)>& progressFn = nullptr); const std::function<void(int)>& progressFn = nullptr);
std::unique_ptr<Page> loadPageFromSD() const; std::unique_ptr<Page> loadPageFromSectionFile();
}; };

10
src/activities/reader/EpubReaderActivity.cpp
View File

@ -254,8 +254,8 @@ void EpubReaderActivity::renderScreen() {
const auto viewportWidth = renderer.getScreenWidth() - orientedMarginLeft - orientedMarginRight; const auto viewportWidth = renderer.getScreenWidth() - orientedMarginLeft - orientedMarginRight;
const auto viewportHeight = renderer.getScreenHeight() - orientedMarginTop - orientedMarginBottom; const auto viewportHeight = renderer.getScreenHeight() - orientedMarginTop - orientedMarginBottom;
if (!section->loadCacheMetadata(READER_FONT_ID, lineCompression, SETTINGS.extraParagraphSpacing, viewportWidth, if (!section->loadSectionFile(READER_FONT_ID, lineCompression, SETTINGS.extraParagraphSpacing, viewportWidth,
viewportHeight)) { viewportHeight)) {
Serial.printf("[%lu] [ERS] Cache not found, building...\n", millis()); Serial.printf("[%lu] [ERS] Cache not found, building...\n", millis());
// Progress bar dimensions // Progress bar dimensions
@ -300,8 +300,8 @@ void EpubReaderActivity::renderScreen() {
renderer.displayBuffer(EInkDisplay::FAST_REFRESH); renderer.displayBuffer(EInkDisplay::FAST_REFRESH);
}; };
if (!section->persistPageDataToSD(READER_FONT_ID, lineCompression, SETTINGS.extraParagraphSpacing, viewportWidth, if (!section->createSectionFile(READER_FONT_ID, lineCompression, SETTINGS.extraParagraphSpacing, viewportWidth,
viewportHeight, progressSetup, progressCallback)) { viewportHeight, progressSetup, progressCallback)) {
Serial.printf("[%lu] [ERS] Failed to persist page data to SD\n", millis()); Serial.printf("[%lu] [ERS] Failed to persist page data to SD\n", millis());
section.reset(); section.reset();
return; return;
@ -336,7 +336,7 @@ void EpubReaderActivity::renderScreen() {
} }
{ {
auto p = section->loadPageFromSD(); auto p = section->loadPageFromSectionFile();
if (!p) { if (!p) {
Serial.printf("[%lu] [ERS] Failed to load page from SD - clearing section cache\n", millis()); Serial.printf("[%lu] [ERS] Failed to load page from SD - clearing section cache\n", millis());
section->clearCache(); section->clearCache();