From c32eabf4648b87df50f5fa8c66c926b9116b9b2c Mon Sep 17 00:00:00 2001 From: Stefan Ostermann Date: Sun, 2 Nov 2025 00:04:31 +0100 Subject: [PATCH] [ai] memory optimized web serving --- scripts/minify_web.py | 20 ++++- src/main.cpp | 178 ++++++++++++++++++++++++++++++------------ 2 files changed, 149 insertions(+), 49 deletions(-) diff --git a/scripts/minify_web.py b/scripts/minify_web.py index 0b8658b..77c7d73 100644 --- a/scripts/minify_web.py +++ b/scripts/minify_web.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """ -Simple build tool to minify web assets in the `web/` folder. +Simple build tool to minify web assets in the `web/` folder and generate gzipped versions. Usage: python3 scripts/minify_web.py @@ -15,6 +15,11 @@ and write minified outputs to: - web/cleaned/style.css - web/cleaned/script.js +Additionally, gzipped variants are produced to enable efficient on-device serving: + - web/cleaned/index.html.gz + - web/cleaned/style.css.gz + - web/cleaned/script.js.gz + The minifiers are intentionally conservative (no external deps) and aim to be safe for typical static files used in this project. They remove comments, collapse unnecessary whitespace and do small syntax-preserving transformations. @@ -29,6 +34,7 @@ from pathlib import Path import re import sys import os +import gzip BASE = Path(__file__).resolve().parent.parent WEB = BASE / "web" @@ -218,6 +224,15 @@ def write_file(path: Path, data: str): except Exception as e: print(f"ERROR writing {path}: {e}", file=sys.stderr) +def write_gzip(path: Path, data: str): + """Write UTF-8 text as gzip with deterministic mtime for reproducible builds.""" + try: + gz_bytes = gzip.compress(data.encode('utf-8'), mtime=0) + path.write_bytes(gz_bytes) + print(f"Wrote {path} ({len(gz_bytes)} bytes, gzip)") + except Exception as e: + print(f"ERROR writing {path}: {e}", file=sys.stderr) + def minify_all(): ensure_clean_dir() @@ -228,6 +243,7 @@ def minify_all(): s = read_file(index) out = minify_html(s) write_file(CLEAN / "index.html", out) + write_gzip(CLEAN / "index.html.gz", out) else: print("No index.html found in web/") @@ -238,6 +254,7 @@ def minify_all(): s = read_file(css) out = minify_css(s) write_file(CLEAN / "style.css", out) + write_gzip(CLEAN / "style.css.gz", out) else: print("No style.css found in web/") @@ -248,6 +265,7 @@ def minify_all(): s = read_file(js) out = minify_js(s) write_file(CLEAN / "script.js", out) + write_gzip(CLEAN / "script.js.gz", out) else: print("No script.js found in web/") diff --git a/src/main.cpp b/src/main.cpp index 15a26a0..74e1d56 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1109,35 +1109,59 @@ void init_webserver() { server.on("/", HTTP_GET, [](AsyncWebServerRequest *request) { webreq_enter(); - request->onDisconnect([](){ webreq_exit(); sd_lock_release();}); + /* onDisconnect set later with cleanup */ deactivateRFID(); activateSD(); - sd_lock_acquire(); static String htmlPath = ""; + static String htmlPathGz = ""; if (htmlPath.isEmpty()) { htmlPath = getSysDir(index_file); + htmlPathGz = htmlPath + F(".gz"); } - - - if (SD.exists(htmlPath)) + + // Prefer gz if present + bool useGz = SD.exists(htmlPathGz); + const String &sendPath = useGz ? htmlPathGz : htmlPath; + + if (SD.exists(sendPath)) { - uint32_t fsize = 0; - { - File f = SD.open(htmlPath); - if (f) { fsize = f.size(); f.close(); } - } -#ifdef DEBUG - Serial.printf("Serving %s size=%u heap=%u webreq_cnt=%u\n", htmlPath.c_str(), (unsigned)fsize, (unsigned)xPortGetFreeHeapSize(), (unsigned)webreq_cnt); -#endif - AsyncWebServerResponse *response = request->beginResponse(SD, htmlPath, txt_html_charset); +#ifdef DEBUG + Serial.printf("Serving %s heap=%u webreq_cnt=%u\n", sendPath.c_str(), (unsigned)xPortGetFreeHeapSize(), (unsigned)webreq_cnt); +#endif + // Chunked streaming with short SD lock per read + struct FileCtx { File f; }; + FileCtx* ctx = new FileCtx(); + ctx->f = SD.open(sendPath); + if (!ctx->f) { delete ctx; request->send(500, txt_plain, F("Open failed")); return; } + auto response = request->beginChunkedResponse(String(txt_html_charset), + [ctx](uint8_t *buffer, size_t maxLen, size_t index) -> size_t { + size_t toRead = maxLen; + if (toRead > 512) toRead = 512; + sd_lock_acquire(); + size_t n = ctx->f.read(buffer, toRead); + sd_lock_release(); + if (n == 0) { ctx->f.close(); } + return n; + }); response->addHeader(hdr_cache_control_key, hdr_cache_control_val); response->addHeader(hdr_connection_key, hdr_connection_val); + if (useGz) { + response->addHeader(F("Content-Encoding"), F("gzip")); + } + // Ensure FileCtx cleanup even on aborted connections + request->onDisconnect([ctx](){ + sd_lock_acquire(); + if (ctx->f) ctx->f.close(); + sd_lock_release(); + delete ctx; + webreq_exit(); + }); request->send(response); } else { // Fallback: serve minimal error if file not found - request->send(404, txt_plain, F("ERROR: /system/index.html not found!")); + request->send(404, txt_plain, F("ERROR: /system/index.html(.gz) not found!")); } }); @@ -1145,36 +1169,62 @@ server.on("/", HTTP_GET, [](AsyncWebServerRequest *request) server.on("/style.css", HTTP_GET, [](AsyncWebServerRequest *request) { webreq_enter(); - request->onDisconnect([](){ webreq_exit(); sd_lock_release();}); + /* onDisconnect set later with cleanup */ deactivateRFID(); - activateSD(); + activateSD(); // Ensure SD is active and RFID is deactivated while serving files. static String cssPath = ""; + static String cssPathGz = ""; if (cssPath.isEmpty()) { cssPath = getSysDir(style_file); + cssPathGz = cssPath + F(".gz"); } - if (SD.exists(cssPath)) + // Prefer gz if present + bool useGz = SD.exists(cssPathGz); + const String &sendPath = useGz ? cssPathGz : cssPath; + + if (SD.exists(sendPath)) { - uint32_t fsize = 0; - { - File f = SD.open(cssPath); - if (f) { fsize = f.size(); f.close(); } - } -#ifdef DEBUG - Serial.printf("Serving %s size=%u heap=%u webreq_cnt=%u\n", cssPath.c_str(), (unsigned)fsize, (unsigned)xPortGetFreeHeapSize(), (unsigned)webreq_cnt); -#endif +#ifdef DEBUG + Serial.printf("Serving %s heap=%u webreq_cnt=%u\n", sendPath.c_str(), (unsigned)xPortGetFreeHeapSize(), (unsigned)webreq_cnt); +#endif { - AsyncWebServerResponse *resp = request->beginResponse(SD, cssPath, F("text/css")); + // Chunked streaming with short SD lock per read + struct FileCtx { File f; }; + FileCtx* ctx = new FileCtx(); + ctx->f = SD.open(sendPath); + if (!ctx->f) { delete ctx; request->send(500, txt_plain, F("Open failed")); return; } + auto resp = request->beginChunkedResponse(F("text/css"), + [ctx](uint8_t *buffer, size_t maxLen, size_t index) -> size_t { + size_t toRead = maxLen; + if (toRead > 512) toRead = 512; + sd_lock_acquire(); + size_t n = ctx->f.read(buffer, toRead); + sd_lock_release(); + if (n == 0) { ctx->f.close(); } + return n; + }); resp->addHeader(hdr_cache_control_key, F("public, max-age=300")); resp->addHeader(hdr_connection_key, hdr_connection_val); + if (useGz) { + resp->addHeader(F("Content-Encoding"), F("gzip")); + } + // Ensure FileCtx cleanup even on aborted connections + request->onDisconnect([ctx](){ + sd_lock_acquire(); + if (ctx->f) ctx->f.close(); + sd_lock_release(); + delete ctx; + webreq_exit(); + }); request->send(resp); } } else { // Fallback: serve minimal CSS if file not found - request->send(404, txt_plain, F("ERROR: /system/style.css not found!")); + request->send(404, txt_plain, F("ERROR: /system/style.css(.gz) not found!")); } }); @@ -1182,37 +1232,62 @@ server.on("/", HTTP_GET, [](AsyncWebServerRequest *request) server.on("/script.js", HTTP_GET, [](AsyncWebServerRequest *request) { webreq_enter(); - request->onDisconnect([](){ webreq_exit(); sd_lock_release();}); + /* onDisconnect set later with cleanup */ deactivateRFID(); activateSD(); - sd_lock_acquire(); static String jsPath = ""; + static String jsPathGz = ""; if (jsPath.isEmpty()) { jsPath = getSysDir(script_file); + jsPathGz = jsPath + F(".gz"); } - if (SD.exists(jsPath)) + // Prefer gz if present + bool useGz = SD.exists(jsPathGz); + const String &sendPath = useGz ? jsPathGz : jsPath; + + if (SD.exists(sendPath)) { - uint32_t fsize = 0; +#ifdef DEBUG + Serial.printf("Serving %s heap=%u webreq_cnt=%u\n", sendPath.c_str(), (unsigned)xPortGetFreeHeapSize(), (unsigned)webreq_cnt); +#endif { - File f = SD.open(jsPath); - if (f) { fsize = f.size(); f.close(); } - } -#ifdef DEBUG - Serial.printf("Serving %s size=%u heap=%u webreq_cnt=%u\n", jsPath.c_str(), (unsigned)fsize, (unsigned)xPortGetFreeHeapSize(), (unsigned)webreq_cnt); -#endif - { - AsyncWebServerResponse *resp = request->beginResponse(SD, jsPath, F("application/javascript")); + // Chunked streaming with short SD lock per read + struct FileCtx { File f; }; + FileCtx* ctx = new FileCtx(); + ctx->f = SD.open(sendPath); + if (!ctx->f) { delete ctx; request->send(500, txt_plain, F("Open failed")); return; } + auto resp = request->beginChunkedResponse(F("application/javascript"), + [ctx](uint8_t *buffer, size_t maxLen, size_t index) -> size_t { + size_t toRead = maxLen; + if (toRead > 512) toRead = 512; + sd_lock_acquire(); + size_t n = ctx->f.read(buffer, toRead); + sd_lock_release(); + if (n == 0) { ctx->f.close(); } + return n; + }); resp->addHeader(hdr_cache_control_key, F("public, max-age=300")); resp->addHeader(hdr_connection_key, hdr_connection_val); + if (useGz) { + resp->addHeader(F("Content-Encoding"), F("gzip")); + } + // Ensure FileCtx cleanup even on aborted connections + request->onDisconnect([ctx](){ + sd_lock_acquire(); + if (ctx->f) ctx->f.close(); + sd_lock_release(); + delete ctx; + webreq_exit(); + }); request->send(resp); } } else { // Fallback: serve minimal JS if file not found - request->send(404, txt_plain, F("ERROR: /system/script.js not found!")); + request->send(404, txt_plain, F("ERROR: /system/script.js(.gz) not found!")); } }); @@ -1221,10 +1296,8 @@ server.on("/", HTTP_GET, [](AsyncWebServerRequest *request) server.on("/directory", HTTP_GET, [](AsyncWebServerRequest *request) { webreq_enter(); - request->onDisconnect([](){ webreq_exit(); sd_lock_release(); }); - // Acquire SD lock to prevent concurrent modifications to directory tree - // and to prevent audio playback from interrupting the stream - sd_lock_acquire(); + request->onDisconnect([](){ webreq_exit(); }); + // Stream the response directly from the directory tree to avoid large temporary Strings // Stream the response directly from the directory tree to avoid large temporary Strings AsyncResponseStream* stream = request->beginResponseStream(txt_html_charset); #ifdef DEBUG @@ -1240,8 +1313,7 @@ server.on("/", HTTP_GET, [](AsyncWebServerRequest *request) server.on("/mapping", HTTP_GET, [](AsyncWebServerRequest *request) { webreq_enter(); - request->onDisconnect([](){ webreq_exit(); sd_lock_release();}); - sd_lock_acquire(); + request->onDisconnect([](){ webreq_exit();}); // Stream mapping to avoid Content-Length mismatches and reduce heap spikes AsyncResponseStream* stream = request->beginResponseStream(txt_html_charset); #ifdef DEBUG @@ -1254,6 +1326,7 @@ server.on("/", HTTP_GET, [](AsyncWebServerRequest *request) request->send(stream); }); + server.on("/state", HTTP_GET, [](AsyncWebServerRequest *request) { webreq_enter(); @@ -1316,7 +1389,16 @@ server.on("/", HTTP_GET, [](AsyncWebServerRequest *request) [](AsyncWebServerRequest *request) { webreq_enter(); - request->onDisconnect([](){ webreq_exit(); }); + // Ensure any in-progress upload file is closed on client abort to free the FD + request->onDisconnect([request](){ + // Close temporary upload file if still open + if (request->_tempFile) { + sd_lock_acquire(); + request->_tempFile.close(); + sd_lock_release(); + } + webreq_exit(); + }); request->send(200); }, handleUpload);