[ai] memory optimized web serving

scripts/minify_web.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 """
-Simple build tool to minify web assets in the `web/` folder.
+Simple build tool to minify web assets in the `web/` folder and generate gzipped versions.
 
 Usage:
   python3 scripts/minify_web.py
@@ -15,6 +15,11 @@ and write minified outputs to:
   - web/cleaned/style.css
   - web/cleaned/script.js
 
+Additionally, gzipped variants are produced to enable efficient on-device serving:
+  - web/cleaned/index.html.gz
+  - web/cleaned/style.css.gz
+  - web/cleaned/script.js.gz
+
 The minifiers are intentionally conservative (no external deps) and aim to be
 safe for typical static files used in this project. They remove comments,
 collapse unnecessary whitespace and do small syntax-preserving transformations.
@@ -29,6 +34,7 @@ from pathlib import Path
 import re
 import sys
 import os
+import gzip
 
 BASE = Path(__file__).resolve().parent.parent
 WEB = BASE / "web"
@@ -218,6 +224,15 @@ def write_file(path: Path, data: str):
     except Exception as e:
         print(f"ERROR writing {path}: {e}", file=sys.stderr)
 
+def write_gzip(path: Path, data: str):
+    """Write UTF-8 text as gzip with deterministic mtime for reproducible builds."""
+    try:
+        gz_bytes = gzip.compress(data.encode('utf-8'), mtime=0)
+        path.write_bytes(gz_bytes)
+        print(f"Wrote {path} ({len(gz_bytes)} bytes, gzip)")
+    except Exception as e:
+        print(f"ERROR writing {path}: {e}", file=sys.stderr)
+
 def minify_all():
     ensure_clean_dir()
 
@@ -228,6 +243,7 @@ def minify_all():
         s = read_file(index)
         out = minify_html(s)
         write_file(CLEAN / "index.html", out)
+        write_gzip(CLEAN / "index.html.gz", out)
     else:
         print("No index.html found in web/")
 
@@ -238,6 +254,7 @@ def minify_all():
         s = read_file(css)
         out = minify_css(s)
         write_file(CLEAN / "style.css", out)
+        write_gzip(CLEAN / "style.css.gz", out)
     else:
         print("No style.css found in web/")
 
@@ -248,6 +265,7 @@ def minify_all():
         s = read_file(js)
         out = minify_js(s)
         write_file(CLEAN / "script.js", out)
+        write_gzip(CLEAN / "script.js.gz", out)
     else:
         print("No script.js found in web/")
 

src/main.cpp

@@ -1109,35 +1109,59 @@ void init_webserver() {
 server.on("/", HTTP_GET, [](AsyncWebServerRequest *request)
               {
                 webreq_enter();
-                request->onDisconnect([](){ webreq_exit(); sd_lock_release();});
+                /* onDisconnect set later with cleanup */
                 deactivateRFID();
                 activateSD();
-                sd_lock_acquire();
                 static String htmlPath = "";
+                static String htmlPathGz = "";
                 if (htmlPath.isEmpty()) {
                   htmlPath = getSysDir(index_file);
+                  htmlPathGz = htmlPath + F(".gz");
                 }
 
+                // Prefer gz if present
+                bool useGz = SD.exists(htmlPathGz);
+                const String &sendPath = useGz ? htmlPathGz : htmlPath;
 
-                if (SD.exists(htmlPath))
+                if (SD.exists(sendPath))
                 {
-                  uint32_t fsize = 0;
-                  {
-                    File f = SD.open(htmlPath);
-                    if (f) { fsize = f.size(); f.close(); }
-                  }
 #ifdef DEBUG
-                  Serial.printf("Serving %s size=%u heap=%u webreq_cnt=%u\n", htmlPath.c_str(), (unsigned)fsize, (unsigned)xPortGetFreeHeapSize(), (unsigned)webreq_cnt);
+                  Serial.printf("Serving %s heap=%u webreq_cnt=%u\n", sendPath.c_str(), (unsigned)xPortGetFreeHeapSize(), (unsigned)webreq_cnt);
 #endif
-                  AsyncWebServerResponse *response = request->beginResponse(SD, htmlPath, txt_html_charset);
+                  // Chunked streaming with short SD lock per read
+                  struct FileCtx { File f; };
+                  FileCtx* ctx = new FileCtx();
+                  ctx->f = SD.open(sendPath);
+                  if (!ctx->f) { delete ctx; request->send(500, txt_plain, F("Open failed")); return; }
+                  auto response = request->beginChunkedResponse(String(txt_html_charset),
+                      [ctx](uint8_t *buffer, size_t maxLen, size_t index) -> size_t {
+                        size_t toRead = maxLen;
+                        if (toRead > 512) toRead = 512;
+                        sd_lock_acquire();
+                        size_t n = ctx->f.read(buffer, toRead);
+                        sd_lock_release();
+                        if (n == 0) { ctx->f.close(); }
+                        return n;
+                      });
                   response->addHeader(hdr_cache_control_key, hdr_cache_control_val);
                   response->addHeader(hdr_connection_key, hdr_connection_val);
+                  if (useGz) {
+                    response->addHeader(F("Content-Encoding"), F("gzip"));
+                  }
+                  // Ensure FileCtx cleanup even on aborted connections
+                  request->onDisconnect([ctx](){
+                    sd_lock_acquire();
+                    if (ctx->f) ctx->f.close();
+                    sd_lock_release();
+                    delete ctx;
+                    webreq_exit();
+                  });
                   request->send(response);
                 }
                 else
                 {
                   // Fallback: serve minimal error if file not found
-                  request->send(404, txt_plain, F("ERROR: /system/index.html not found!"));
+                  request->send(404, txt_plain, F("ERROR: /system/index.html(.gz) not found!"));
                 }
 
               });
@@ -1145,36 +1169,62 @@ server.on("/", HTTP_GET, [](AsyncWebServerRequest *request)
     server.on("/style.css", HTTP_GET, [](AsyncWebServerRequest *request)
               {
                 webreq_enter();
-                request->onDisconnect([](){ webreq_exit(); sd_lock_release();});
+                /* onDisconnect set later with cleanup */
                 deactivateRFID();
                 activateSD();
                 // Ensure SD is active and RFID is deactivated while serving files.
                 static String cssPath = "";
+                static String cssPathGz = "";
                 if (cssPath.isEmpty()) {
                   cssPath = getSysDir(style_file);
+                  cssPathGz = cssPath + F(".gz");
                 }
 
-                if (SD.exists(cssPath))
+                // Prefer gz if present
+                bool useGz = SD.exists(cssPathGz);
+                const String &sendPath = useGz ? cssPathGz : cssPath;
+
+                if (SD.exists(sendPath))
                 {
-                  uint32_t fsize = 0;
-                  {                    
-                    File f = SD.open(cssPath);
-                    if (f) { fsize = f.size(); f.close(); }
-                  }
 #ifdef DEBUG
-                  Serial.printf("Serving %s size=%u heap=%u webreq_cnt=%u\n", cssPath.c_str(), (unsigned)fsize, (unsigned)xPortGetFreeHeapSize(), (unsigned)webreq_cnt);
+                  Serial.printf("Serving %s heap=%u webreq_cnt=%u\n", sendPath.c_str(), (unsigned)xPortGetFreeHeapSize(), (unsigned)webreq_cnt);
 #endif
                   {
-                    AsyncWebServerResponse *resp = request->beginResponse(SD, cssPath, F("text/css"));
+                    // Chunked streaming with short SD lock per read
+                    struct FileCtx { File f; };
+                    FileCtx* ctx = new FileCtx();
+                    ctx->f = SD.open(sendPath);
+                    if (!ctx->f) { delete ctx; request->send(500, txt_plain, F("Open failed")); return; }
+                    auto resp = request->beginChunkedResponse(F("text/css"),
+                        [ctx](uint8_t *buffer, size_t maxLen, size_t index) -> size_t {
+                          size_t toRead = maxLen;
+                          if (toRead > 512) toRead = 512;
+                          sd_lock_acquire();
+                          size_t n = ctx->f.read(buffer, toRead);
+                          sd_lock_release();
+                          if (n == 0) { ctx->f.close(); }
+                          return n;
+                        });
                     resp->addHeader(hdr_cache_control_key, F("public, max-age=300"));
                     resp->addHeader(hdr_connection_key, hdr_connection_val);
+                    if (useGz) {
+                      resp->addHeader(F("Content-Encoding"), F("gzip"));
+                    }
+                    // Ensure FileCtx cleanup even on aborted connections
+                    request->onDisconnect([ctx](){
+                      sd_lock_acquire();
+                      if (ctx->f) ctx->f.close();
+                      sd_lock_release();
+                      delete ctx;
+                      webreq_exit();
+                    });
                     request->send(resp);
                   }
                 }
                 else
                 {
                   // Fallback: serve minimal CSS if file not found
-                  request->send(404, txt_plain, F("ERROR: /system/style.css not found!"));
+                  request->send(404, txt_plain, F("ERROR: /system/style.css(.gz) not found!"));
                 }
 
               });
@@ -1182,37 +1232,62 @@ server.on("/", HTTP_GET, [](AsyncWebServerRequest *request)
     server.on("/script.js", HTTP_GET, [](AsyncWebServerRequest *request)
               {
                 webreq_enter();
-                request->onDisconnect([](){ webreq_exit(); sd_lock_release();});
+                /* onDisconnect set later with cleanup */
                 deactivateRFID();
                 activateSD();
-                sd_lock_acquire();
 
                 static String jsPath = "";
+                static String jsPathGz = "";
                 if (jsPath.isEmpty()) {
                   jsPath = getSysDir(script_file);
+                  jsPathGz = jsPath + F(".gz");
                 }
                 
-                if (SD.exists(jsPath))
+                // Prefer gz if present
+                bool useGz = SD.exists(jsPathGz);
+                const String &sendPath = useGz ? jsPathGz : jsPath;
+
+                if (SD.exists(sendPath))
                 {
-                  uint32_t fsize = 0;
-                  {
-                    File f = SD.open(jsPath);
-                    if (f) { fsize = f.size(); f.close(); }
-                  }
 #ifdef DEBUG
-                  Serial.printf("Serving %s size=%u heap=%u webreq_cnt=%u\n", jsPath.c_str(), (unsigned)fsize, (unsigned)xPortGetFreeHeapSize(), (unsigned)webreq_cnt);
+                  Serial.printf("Serving %s heap=%u webreq_cnt=%u\n", sendPath.c_str(), (unsigned)xPortGetFreeHeapSize(), (unsigned)webreq_cnt);
 #endif
                   {
-                    AsyncWebServerResponse *resp = request->beginResponse(SD, jsPath, F("application/javascript"));
+                    // Chunked streaming with short SD lock per read
+                    struct FileCtx { File f; };
+                    FileCtx* ctx = new FileCtx();
+                    ctx->f = SD.open(sendPath);
+                    if (!ctx->f) { delete ctx; request->send(500, txt_plain, F("Open failed")); return; }
+                    auto resp = request->beginChunkedResponse(F("application/javascript"),
+                        [ctx](uint8_t *buffer, size_t maxLen, size_t index) -> size_t {
+                          size_t toRead = maxLen;
+                          if (toRead > 512) toRead = 512;
+                          sd_lock_acquire();
+                          size_t n = ctx->f.read(buffer, toRead);
+                          sd_lock_release();
+                          if (n == 0) { ctx->f.close(); }
+                          return n;
+                        });
                     resp->addHeader(hdr_cache_control_key, F("public, max-age=300"));
                     resp->addHeader(hdr_connection_key, hdr_connection_val);
+                    if (useGz) {
+                      resp->addHeader(F("Content-Encoding"), F("gzip"));
+                    }
+                    // Ensure FileCtx cleanup even on aborted connections
+                    request->onDisconnect([ctx](){
+                      sd_lock_acquire();
+                      if (ctx->f) ctx->f.close();
+                      sd_lock_release();
+                      delete ctx;
+                      webreq_exit();
+                    });
                     request->send(resp);
                   }
                 }
                 else
                 {
                   // Fallback: serve minimal JS if file not found
-                  request->send(404, txt_plain, F("ERROR: /system/script.js not found!"));
+                  request->send(404, txt_plain, F("ERROR: /system/script.js(.gz) not found!"));
                 }
 
               });
@@ -1221,10 +1296,8 @@ server.on("/", HTTP_GET, [](AsyncWebServerRequest *request)
     server.on("/directory", HTTP_GET, [](AsyncWebServerRequest *request)
               {
                 webreq_enter();
-                request->onDisconnect([](){ webreq_exit(); sd_lock_release(); });
+                request->onDisconnect([](){ webreq_exit(); });
-                // Acquire SD lock to prevent concurrent modifications to directory tree
+                // Stream the response directly from the directory tree to avoid large temporary Strings
-                // and to prevent audio playback from interrupting the stream
-                sd_lock_acquire();
                 // Stream the response directly from the directory tree to avoid large temporary Strings
                 AsyncResponseStream* stream = request->beginResponseStream(txt_html_charset);
 #ifdef DEBUG                
@@ -1240,8 +1313,7 @@ server.on("/", HTTP_GET, [](AsyncWebServerRequest *request)
     server.on("/mapping", HTTP_GET, [](AsyncWebServerRequest *request)
               {
                 webreq_enter();
-                request->onDisconnect([](){ webreq_exit(); sd_lock_release();});
+                request->onDisconnect([](){ webreq_exit();});
-                sd_lock_acquire();
                 // Stream mapping to avoid Content-Length mismatches and reduce heap spikes
                 AsyncResponseStream* stream = request->beginResponseStream(txt_html_charset);
 #ifdef DEBUG                
@@ -1254,6 +1326,7 @@ server.on("/", HTTP_GET, [](AsyncWebServerRequest *request)
                 request->send(stream);
               });
 
+
     server.on("/state", HTTP_GET, [](AsyncWebServerRequest *request)
               { 
               webreq_enter();
@@ -1316,7 +1389,16 @@ server.on("/", HTTP_GET, [](AsyncWebServerRequest *request)
               [](AsyncWebServerRequest *request)
               {
                 webreq_enter();
-                request->onDisconnect([](){ webreq_exit(); });
+                // Ensure any in-progress upload file is closed on client abort to free the FD
+                request->onDisconnect([request](){
+                  // Close temporary upload file if still open
+                  if (request->_tempFile) {
+                    sd_lock_acquire();
+                    request->_tempFile.close();
+                    sd_lock_release();
+                  }
+                  webreq_exit();
+                });
                 request->send(200);
               },
               handleUpload);