Merge pull request #5 from swwilshub/claude/websocket-upload-GwiXb

Add WebSocket upload for faster file transfers

platformio.ini

@@ -45,6 +45,7 @@ lib_deps =
   SDCardManager=symlink://open-x4-sdk/libs/hardware/SDCardManager
   ArduinoJson @ 7.4.2
   QRCode @ 0.0.1
+  links2004/WebSockets @ ^2.4.1
 
 [env:default]
 extends = base

src/network/CrossPointWebServer.cpp

@@ -16,6 +16,18 @@ namespace {
 // Note: Items starting with "." are automatically hidden
 const char* HIDDEN_ITEMS[] = {"System Volume Information", "XTCache"};
 constexpr size_t HIDDEN_ITEMS_COUNT = sizeof(HIDDEN_ITEMS) / sizeof(HIDDEN_ITEMS[0]);
+
+// Static pointer for WebSocket callback (WebSocketsServer requires C-style callback)
+CrossPointWebServer* wsInstance = nullptr;
+
+// WebSocket upload state
+FsFile wsUploadFile;
+String wsUploadFileName;
+String wsUploadPath;
+size_t wsUploadSize = 0;
+size_t wsUploadReceived = 0;
+unsigned long wsUploadStartTime = 0;
+bool wsUploadInProgress = false;
 }  // namespace
 
 // File listing page template - now using generated headers:
@@ -87,12 +99,22 @@ void CrossPointWebServer::begin() {
   Serial.printf("[%lu] [WEB] [MEM] Free heap after route setup: %d bytes\n", millis(), ESP.getFreeHeap());
 
   server->begin();
+
+  // Start WebSocket server for fast binary uploads
+  Serial.printf("[%lu] [WEB] Starting WebSocket server on port %d...\n", millis(), wsPort);
+  wsServer.reset(new WebSocketsServer(wsPort));
+  wsInstance = const_cast<CrossPointWebServer*>(this);
+  wsServer->begin();
+  wsServer->onEvent(wsEventCallback);
+  Serial.printf("[%lu] [WEB] WebSocket server started\n", millis());
+
   running = true;
 
   Serial.printf("[%lu] [WEB] Web server started on port %d\n", millis(), port);
   // Show the correct IP based on network mode
   const String ipAddr = apMode ? WiFi.softAPIP().toString() : WiFi.localIP().toString();
   Serial.printf("[%lu] [WEB] Access at http://%s/\n", millis(), ipAddr.c_str());
+  Serial.printf("[%lu] [WEB] WebSocket at ws://%s:%d/\n", millis(), ipAddr.c_str(), wsPort);
   Serial.printf("[%lu] [WEB] [MEM] Free heap after server.begin(): %d bytes\n", millis(), ESP.getFreeHeap());
 }
 
@@ -108,6 +130,21 @@ void CrossPointWebServer::stop() {
 
   Serial.printf("[%lu] [WEB] [MEM] Free heap before stop: %d bytes\n", millis(), ESP.getFreeHeap());
 
+  // Close any in-progress WebSocket upload
+  if (wsUploadInProgress && wsUploadFile) {
+    wsUploadFile.close();
+    wsUploadInProgress = false;
+  }
+
+  // Stop WebSocket server
+  if (wsServer) {
+    Serial.printf("[%lu] [WEB] Stopping WebSocket server...\n", millis());
+    wsServer->close();
+    wsServer.reset();
+    wsInstance = nullptr;
+    Serial.printf("[%lu] [WEB] WebSocket server stopped\n", millis());
+  }
+
   // Add delay to allow any in-flight handleClient() calls to complete
   delay(100);
   Serial.printf("[%lu] [WEB] Waited 100ms for handleClient to finish\n", millis());
@@ -149,6 +186,11 @@ void CrossPointWebServer::handleClient() const {
   }
 
   server->handleClient();
+
+  // Handle WebSocket events
+  if (wsServer) {
+    wsServer->loop();
+  }
 }
 
 void CrossPointWebServer::handleRoot() const {
@@ -617,3 +659,143 @@ void CrossPointWebServer::handleDelete() const {
     server->send(500, "text/plain", "Failed to delete item");
   }
 }
+
+// WebSocket callback trampoline
+void CrossPointWebServer::wsEventCallback(uint8_t num, WStype_t type, uint8_t* payload, size_t length) {
+  if (wsInstance) {
+    wsInstance->onWebSocketEvent(num, type, payload, length);
+  }
+}
+
+// WebSocket event handler for fast binary uploads
+// Protocol:
+//   1. Client sends TEXT message: "START:<filename>:<size>:<path>"
+//   2. Client sends BINARY messages with file data chunks
+//   3. Server sends TEXT "PROGRESS:<received>:<total>" after each chunk
+//   4. Server sends TEXT "DONE" or "ERROR:<message>" when complete
+void CrossPointWebServer::onWebSocketEvent(uint8_t num, WStype_t type, uint8_t* payload, size_t length) {
+  switch (type) {
+    case WStype_DISCONNECTED:
+      Serial.printf("[%lu] [WS] Client %u disconnected\n", millis(), num);
+      // Clean up any in-progress upload
+      if (wsUploadInProgress && wsUploadFile) {
+        wsUploadFile.close();
+        // Delete incomplete file
+        String filePath = wsUploadPath;
+        if (!filePath.endsWith("/")) filePath += "/";
+        filePath += wsUploadFileName;
+        SdMan.remove(filePath.c_str());
+        Serial.printf("[%lu] [WS] Deleted incomplete upload: %s\n", millis(), filePath.c_str());
+      }
+      wsUploadInProgress = false;
+      break;
+
+    case WStype_CONNECTED: {
+      Serial.printf("[%lu] [WS] Client %u connected\n", millis(), num);
+      break;
+    }
+
+    case WStype_TEXT: {
+      // Parse control messages
+      String msg = String((char*)payload);
+      Serial.printf("[%lu] [WS] Text from client %u: %s\n", millis(), num, msg.c_str());
+
+      if (msg.startsWith("START:")) {
+        // Parse: START:<filename>:<size>:<path>
+        int firstColon = msg.indexOf(':', 6);
+        int secondColon = msg.indexOf(':', firstColon + 1);
+
+        if (firstColon > 0 && secondColon > 0) {
+          wsUploadFileName = msg.substring(6, firstColon);
+          wsUploadSize = msg.substring(firstColon + 1, secondColon).toInt();
+          wsUploadPath = msg.substring(secondColon + 1);
+          wsUploadReceived = 0;
+          wsUploadStartTime = millis();
+
+          // Ensure path is valid
+          if (!wsUploadPath.startsWith("/")) wsUploadPath = "/" + wsUploadPath;
+          if (wsUploadPath.length() > 1 && wsUploadPath.endsWith("/")) {
+            wsUploadPath = wsUploadPath.substring(0, wsUploadPath.length() - 1);
+          }
+
+          // Build file path
+          String filePath = wsUploadPath;
+          if (!filePath.endsWith("/")) filePath += "/";
+          filePath += wsUploadFileName;
+
+          Serial.printf("[%lu] [WS] Starting upload: %s (%d bytes) to %s\n", millis(), wsUploadFileName.c_str(),
+                        wsUploadSize, filePath.c_str());
+
+          // Check if file exists and remove it
+          esp_task_wdt_reset();
+          if (SdMan.exists(filePath.c_str())) {
+            SdMan.remove(filePath.c_str());
+          }
+
+          // Open file for writing
+          esp_task_wdt_reset();
+          if (!SdMan.openFileForWrite("WS", filePath, wsUploadFile)) {
+            wsServer->sendTXT(num, "ERROR:Failed to create file");
+            wsUploadInProgress = false;
+            return;
+          }
+          esp_task_wdt_reset();
+
+          wsUploadInProgress = true;
+          wsServer->sendTXT(num, "READY");
+        } else {
+          wsServer->sendTXT(num, "ERROR:Invalid START format");
+        }
+      }
+      break;
+    }
+
+    case WStype_BIN: {
+      if (!wsUploadInProgress || !wsUploadFile) {
+        wsServer->sendTXT(num, "ERROR:No upload in progress");
+        return;
+      }
+
+      // Write binary data directly to file
+      esp_task_wdt_reset();
+      size_t written = wsUploadFile.write(payload, length);
+      esp_task_wdt_reset();
+
+      if (written != length) {
+        wsUploadFile.close();
+        wsUploadInProgress = false;
+        wsServer->sendTXT(num, "ERROR:Write failed - disk full?");
+        return;
+      }
+
+      wsUploadReceived += written;
+
+      // Send progress update (every 64KB or at end)
+      static size_t lastProgressSent = 0;
+      if (wsUploadReceived - lastProgressSent >= 65536 || wsUploadReceived >= wsUploadSize) {
+        String progress = "PROGRESS:" + String(wsUploadReceived) + ":" + String(wsUploadSize);
+        wsServer->sendTXT(num, progress);
+        lastProgressSent = wsUploadReceived;
+      }
+
+      // Check if upload complete
+      if (wsUploadReceived >= wsUploadSize) {
+        wsUploadFile.close();
+        wsUploadInProgress = false;
+
+        unsigned long elapsed = millis() - wsUploadStartTime;
+        float kbps = (elapsed > 0) ? (wsUploadSize / 1024.0) / (elapsed / 1000.0) : 0;
+
+        Serial.printf("[%lu] [WS] Upload complete: %s (%d bytes in %lu ms, %.1f KB/s)\n", millis(),
+                      wsUploadFileName.c_str(), wsUploadSize, elapsed, kbps);
+
+        wsServer->sendTXT(num, "DONE");
+        lastProgressSent = 0;
+      }
+      break;
+    }
+
+    default:
+      break;
+  }
+}

src/network/CrossPointWebServer.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <WebServer.h>
+#include <WebSocketsServer.h>
 
 #include <vector>
 
@@ -34,9 +35,15 @@ class CrossPointWebServer {
 
  private:
   std::unique_ptr<WebServer> server = nullptr;
+  std::unique_ptr<WebSocketsServer> wsServer = nullptr;
   bool running = false;
   bool apMode = false;  // true when running in AP mode, false for STA mode
   uint16_t port = 80;
+  uint16_t wsPort = 81;  // WebSocket port
+
+  // WebSocket upload state
+  void onWebSocketEvent(uint8_t num, WStype_t type, uint8_t* payload, size_t length);
+  static void wsEventCallback(uint8_t num, WStype_t type, uint8_t* payload, size_t length);
 
   // File scanning
   void scanFiles(const char* path, const std::function<void(FileInfo)>& callback) const;

src/network/html/FilesPage.html

@@ -816,6 +816,124 @@
   }
 
 let failedUploadsGlobal = [];
+let wsConnection = null;
+const WS_PORT = 81;
+const WS_CHUNK_SIZE = 16384; // 16KB chunks for WebSocket - larger = faster
+
+// Get WebSocket URL based on current page location
+function getWsUrl() {
+  const host = window.location.hostname;
+  return `ws://${host}:${WS_PORT}/`;
+}
+
+// Upload file via WebSocket (faster, binary protocol)
+function uploadFileWebSocket(file, onProgress, onComplete, onError) {
+  return new Promise((resolve, reject) => {
+    const ws = new WebSocket(getWsUrl());
+    let uploadStarted = false;
+
+    ws.onopen = function() {
+      console.log('[WS] Connected, starting upload:', file.name);
+      // Send start message: START:<filename>:<size>:<path>
+      ws.send(`START:${file.name}:${file.size}:${currentPath}`);
+    };
+
+    ws.onmessage = function(event) {
+      const msg = event.data;
+      console.log('[WS] Message:', msg);
+
+      if (msg === 'READY') {
+        uploadStarted = true;
+        // Start sending binary data in chunks
+        sendFileChunks(ws, file, onProgress);
+      } else if (msg.startsWith('PROGRESS:')) {
+        const parts = msg.split(':');
+        const received = parseInt(parts[1]);
+        const total = parseInt(parts[2]);
+        if (onProgress) onProgress(received, total);
+      } else if (msg === 'DONE') {
+        ws.close();
+        if (onComplete) onComplete();
+        resolve();
+      } else if (msg.startsWith('ERROR:')) {
+        const error = msg.substring(6);
+        ws.close();
+        if (onError) onError(error);
+        reject(new Error(error));
+      }
+    };
+
+    ws.onerror = function(event) {
+      console.error('[WS] Error:', event);
+      if (!uploadStarted) {
+        // WebSocket connection failed, reject to trigger fallback
+        reject(new Error('WebSocket connection failed'));
+      }
+    };
+
+    ws.onclose = function() {
+      console.log('[WS] Connection closed');
+    };
+  });
+}
+
+// Send file in chunks via WebSocket
+async function sendFileChunks(ws, file, onProgress) {
+  const totalSize = file.size;
+  let offset = 0;
+
+  while (offset < totalSize) {
+    const chunk = file.slice(offset, offset + WS_CHUNK_SIZE);
+    const buffer = await chunk.arrayBuffer();
+
+    // Wait for buffer to clear if needed
+    while (ws.bufferedAmount > WS_CHUNK_SIZE * 4) {
+      await new Promise(r => setTimeout(r, 10));
+    }
+
+    ws.send(buffer);
+    offset += chunk.size;
+
+    // Update local progress (server will confirm)
+    if (onProgress) onProgress(offset, totalSize);
+  }
+}
+
+// Upload file via HTTP (fallback method)
+function uploadFileHTTP(file, onProgress, onComplete, onError) {
+  return new Promise((resolve, reject) => {
+    const formData = new FormData();
+    formData.append('file', file);
+
+    const xhr = new XMLHttpRequest();
+    xhr.open('POST', '/upload?path=' + encodeURIComponent(currentPath), true);
+
+    xhr.upload.onprogress = function(e) {
+      if (e.lengthComputable && onProgress) {
+        onProgress(e.loaded, e.total);
+      }
+    };
+
+    xhr.onload = function() {
+      if (xhr.status === 200) {
+        if (onComplete) onComplete();
+        resolve();
+      } else {
+        const error = xhr.responseText || 'Upload failed';
+        if (onError) onError(error);
+        reject(new Error(error));
+      }
+    };
+
+    xhr.onerror = function() {
+      const error = 'Network error';
+      if (onError) onError(error);
+      reject(new Error(error));
+    };
+
+    xhr.send(formData);
+  });
+}
 
 function uploadFile() {
   const fileInput = document.getElementById('fileInput');
@@ -836,8 +954,9 @@ function uploadFile() {
 
   let currentIndex = 0;
   const failedFiles = [];
+  let useWebSocket = true; // Try WebSocket first
 
-  function uploadNextFile() {
+  async function uploadNextFile() {
     if (currentIndex >= files.length) {
       // All files processed - show summary
       if (failedFiles.length === 0) {
@@ -845,67 +964,71 @@ function uploadFile() {
         progressText.textContent = 'All uploads complete!';
         setTimeout(() => {
           closeUploadModal();
-          hydrate(); // Refresh file list instead of reloading
+          hydrate();
         }, 1000);
       } else {
         progressFill.style.backgroundColor = '#e74c3c';
         const failedList = failedFiles.map(f => f.name).join(', ');
         progressText.textContent = `${files.length - failedFiles.length}/${files.length} uploaded. Failed: ${failedList}`;
-        
-        // Store failed files globally and show banner
         failedUploadsGlobal = failedFiles;
-        
         setTimeout(() => {
           closeUploadModal();
           showFailedUploadsBanner();
-          hydrate(); // Refresh file list to show successfully uploaded files
+          hydrate();
         }, 2000);
       }
       return;
     }
 
     const file = files[currentIndex];
-    const formData = new FormData();
-    formData.append('file', file);
-
-    const xhr = new XMLHttpRequest();
-    // Include path as query parameter since multipart form data doesn't make
-    // form fields available until after file upload completes
-    xhr.open('POST', '/upload?path=' + encodeURIComponent(currentPath), true);
-
     progressFill.style.width = '0%';
-    progressFill.style.backgroundColor = '#4caf50';
-    progressText.textContent = `Uploading ${file.name} (${currentIndex + 1}/${files.length})`;
+    progressFill.style.backgroundColor = '#27ae60';
+    const methodText = useWebSocket ? ' [WS]' : ' [HTTP]';
+    progressText.textContent = `Uploading ${file.name} (${currentIndex + 1}/${files.length})${methodText}`;
 
-    xhr.upload.onprogress = function (e) {
-      if (e.lengthComputable) {
-        const percent = Math.round((e.loaded / e.total) * 100);
-        progressFill.style.width = percent + '%';
-        progressText.textContent =
-          `Uploading ${file.name} (${currentIndex + 1}/${files.length}) — ${percent}%`;
-      }
+    const onProgress = (loaded, total) => {
+      const percent = Math.round((loaded / total) * 100);
+      progressFill.style.width = percent + '%';
+      const speed = ''; // Could calculate speed here
+      progressText.textContent = `Uploading ${file.name} (${currentIndex + 1}/${files.length})${methodText} — ${percent}%`;
     };
 
-    xhr.onload = function () {
-      if (xhr.status === 200) {
-        currentIndex++;
-        uploadNextFile(); // upload next file
-      } else {
-        // Track failure and continue with next file
-        failedFiles.push({ name: file.name, error: xhr.responseText, file: file });
-        currentIndex++;
-        uploadNextFile();
-      }
-    };
-
-    xhr.onerror = function () {
-      // Track network error and continue with next file
-      failedFiles.push({ name: file.name, error: 'network error', file: file });
+    const onComplete = () => {
       currentIndex++;
       uploadNextFile();
     };
 
-    xhr.send(formData);
+    const onError = (error) => {
+      failedFiles.push({ name: file.name, error: error, file: file });
+      currentIndex++;
+      uploadNextFile();
+    };
+
+    try {
+      if (useWebSocket) {
+        await uploadFileWebSocket(file, onProgress, null, null);
+        onComplete();
+      } else {
+        await uploadFileHTTP(file, onProgress, null, null);
+        onComplete();
+      }
+    } catch (error) {
+      console.error('Upload error:', error);
+      if (useWebSocket && error.message === 'WebSocket connection failed') {
+        // Fall back to HTTP for all subsequent uploads
+        console.log('WebSocket failed, falling back to HTTP');
+        useWebSocket = false;
+        // Retry this file with HTTP
+        try {
+          await uploadFileHTTP(file, onProgress, null, null);
+          onComplete();
+        } catch (httpError) {
+          onError(httpError.message);
+        }
+      } else {
+        onError(error.message);
+      }
+    }
   }
 
   uploadNextFile();