From 57e3fce95e580b5b823c7b9d585324d9cc8a8e47 Mon Sep 17 00:00:00 2001 From: eraiza0816 <10572974+eraiza0816@users.noreply.github.com> Date: Tue, 11 Aug 2026 03:04:19 +0900 Subject: [PATCH] feat(webui): minify and gzip-compress the embedded web assets Two build-time steps shrink the embedded Web UI (html/) from 113,015 to 34,784 bytes (30.8 %): - tools/minify.py (new): string-literal-safe minifier that strips comments and indentation; comments are pure overhead on the wire, so the transfer size drops already before gzip (113,015 -> 101,498). Binary files (.ico) are copied through unchanged. The output goes to output/html_min/, the new fileadder input. - fileadder -z: gzip-compresses every added file (zlib gzip format, Z_BEST_COMPRESSION); the generated f_data entries carry a gzip flag. tools/Makefile links the tool against -lz. - httpd: adds "Content-Encoding: gzip" for flagged entries; the 8051 only copies compressed bytes, decompression happens in the browser. The JSON API stays uncompressed. Fixes required to make the sizes deterministic: - fileadder terminated files at data_read + 1, so the strlen()-based size computation depended on uninitialised buffer content; the NUL now lands directly after the content. - version.h was .PHONY, so every build rewrote it while parallel builds could read a half-written copy. - the file list was unscoped and missed .css/.ico; it is now scoped to html/ and covers all embedded types. Measured with make MACHINE=KP_9000_6XHML_X2: data block end moves from 0x5b977 to 0x487e0 (~78 KB freed on a 512 KB image). Verified with MACHINE=PCB_K0402WS_V3 and on real hardware (PCB-K0402WS-V3). --- Makefile | 23 +++++++++++---- httpd/httpd.c | 5 +++- tools/Makefile | 2 +- tools/fileadder.c | 69 +++++++++++++++++++++++++++++++++++++++------ tools/minify.py | 72 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 156 insertions(+), 15 deletions(-) create mode 100644 tools/minify.py diff --git a/Makefile b/Makefile index 0cbc04f8..b2069549 100644 --- a/Makefile +++ b/Makefile @@ -75,10 +75,23 @@ SRCS += \ OBJS = ${SRCS:%.c=$(BUILDDIR)/%.rel} DEPS := ${SRCS:%.c=$(BUILDDIR)/%.d} -HTML := $(shell find $(html) -name '*.js' -or -name '*.html' -or -name '*.svg') +HTML := $(shell find html -name '*.js' -or -name '*.html' -or -name '*.svg' -or -name '*.css' -or -name '*.ico') -html_data.c html_data.h: $(HTML) tools/output/fileadder - tools/output/fileadder -a $(HTML_LOCATION) -s $(IMAGESIZE) -b BANK1 -d html -p html_data +# Minified copy of the web UI sources, used as the fileadder input. +# The raw html/ sources stay untouched for development; the minified +# copy is a build artifact under output/. +HTML_MIN := output/html_min +.PHONY: html_min +html_min: $(HTML) + rm -rf $(HTML_MIN) + mkdir -p $(HTML_MIN) + @for f in $(HTML); do python3 tools/minify.py $$f $(HTML_MIN)/$$(basename $$f) || exit 1; done + +html_data.c html_data.h: $(HTML) tools/output/fileadder html_min + tools/output/fileadder -a $(HTML_LOCATION) -s $(IMAGESIZE) -b BANK1 -z -d $(HTML_MIN) -p html_data +# httpd.c includes html_data.h; declare it explicitly so a parallel +# build (-j) does not compile httpd before the header is generated. +$(BUILDDIR)/httpd/httpd.rel: html_data.h $(VERSION_HEADER): @echo "#ifndef VERSION_H" > $(VERSION_HEADER) @@ -118,11 +131,11 @@ $(BUILDDIR)/rtlplayground-$(FILENAME_EXTENSION).bin: $(BUILDDIR)/rtlplayground.i tools/output/imagebuilder -i $^ $@ tools/output/fileadder -a $(DEFAULT_CONFIG_LOCATION) -s $(IMAGESIZE) -d config.txt $@ tools/output/fileadder -a $(CONFIG_LOCATION) -s $(IMAGESIZE) -d config.txt $@ - tools/output/fileadder -a $(HTML_LOCATION) -s $(IMAGESIZE) -d html -p html_data -b BANK1 $@ + tools/output/fileadder -a $(HTML_LOCATION) -s $(IMAGESIZE) -z -d $(HTML_MIN) -p html_data -b BANK1 $@ tools/output/crc_calculator -u $@ ln -sf $(MACHINE)/rtlplayground-$(FILENAME_EXTENSION).bin output/rtlplayground.bin -.PHONY: clean all $(SUBDIRS) $(VERSION_HEADER) +.PHONY: clean all $(SUBDIRS) .PHONY: machine_check: diff --git a/httpd/httpd.c b/httpd/httpd.c index 561bd936..622cacb2 100644 --- a/httpd/httpd.c +++ b/httpd/httpd.c @@ -719,7 +719,10 @@ void httpd_appcall(void) * response; without advertising it a browser reuses the socket * from its keep-alive pool and the next request hits the already * closed connection (a POST is then dropped without a retry). */ - slen += strtox(outbuf + slen, "; charset=UTF-8\r\nCache-Control: max-age=60, must-revalidate\r\nConnection: close\r\nAccess-Control-Allow-Origin: *\r\nContent-Security-Policy: default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; connect-src 'self'; form-action 'self'\r\n\r\n"); + slen += strtox(outbuf + slen, "; charset=UTF-8\r\n"); + if (f_data[entry].gzip) + slen += strtox(outbuf + slen, "Content-Encoding: gzip\r\n"); + slen += strtox(outbuf + slen, "Cache-Control: max-age=60, must-revalidate\r\nConnection: close\r\nAccess-Control-Allow-Origin: *\r\nContent-Security-Policy: default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; connect-src 'self'; form-action 'self'\r\n\r\n"); len_left = f_data[entry].len; if (len_left > (TCP_OUTBUF_SIZE - slen)) { diff --git a/tools/Makefile b/tools/Makefile index c7143c53..1b1c6685 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -15,7 +15,7 @@ $(BUILDDIR)/injector: injector.c gcc $^ $(CCFLAGS) $@ $(BUILDDIR)/fileadder: fileadder.c - gcc $^ $(CCFLAGS) $@ + gcc $^ $(CCFLAGS) $@ -lz $(BUILDDIR)/crc_calculator: crc_calculator.c gcc $^ $(CCFLAGS) $@ diff --git a/tools/fileadder.c b/tools/fileadder.c index 71d862b3..77c53253 100644 --- a/tools/fileadder.c +++ b/tools/fileadder.c @@ -11,6 +11,7 @@ #include #include #include +#include #define OFFSET 2 @@ -33,6 +34,7 @@ int fbuf_p; int xbuf_p; bool addsDir = false; int callNum = 0; +struct arguments arguments; const char *argp_program_version = "fileadder 0.1"; const char *argp_program_bug_address = ""; @@ -45,6 +47,7 @@ static struct argp_option options[] = { { "address", 'a', "SIZE", 0, "Address where data is placed, default is 0x1000000 if option is used, otherwise 0x1fd000"}, { "prefix", 'p', "FILE", 0, "Prefix for header and index file generation"}, { "bank", 'b', "BANKNAME", 0, "Generate #pragma with given bank-name"}, + { "gzip", 'z', 0, 0, "gzip-compress the added files"}, { 0 } }; @@ -57,6 +60,7 @@ struct arguments { char *bank; bool overwrite; bool add_zero; + bool gzip; }; @@ -86,6 +90,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) case 'b': arguments->bank = arg? arg : "BANK1"; break; + case 'z': + arguments->gzip = true; + break; default: return ARGP_ERR_UNKNOWN; } @@ -118,6 +125,47 @@ int addfile(const char *name, int addr) } +/* + * gzip-compresses the data at buffer[addr] in place (gzip format, as + * expected by the Content-Encoding: gzip header the httpd serves it + * with). Returns the compressed size. + */ +int gzipBuffer(int addr, int len) +{ + uLong bound = compressBound(len); + unsigned char *out = malloc(bound); + if (!out) { + perror("malloc"); + return 0; + } + + z_stream zs; + memset(&zs, 0, sizeof(zs)); + if (deflateInit2(&zs, Z_BEST_COMPRESSION, Z_DEFLATED, 15 + 16, 8, + Z_DEFAULT_STRATEGY) != Z_OK) { + fprintf(stderr, "gzip: deflateInit2 failed\n"); + free(out); + return 0; + } + zs.next_in = (Bytef *)&buffer[addr]; + zs.avail_in = len; + zs.next_out = out; + zs.avail_out = bound; + if (deflate(&zs, Z_FINISH) != Z_STREAM_END) { + fprintf(stderr, "gzip: deflate failed\n"); + deflateEnd(&zs); + free(out); + return 0; + } + int out_len = zs.total_out; + deflateEnd(&zs); + + memcpy(&buffer[addr], out, out_len); + free(out); + return out_len; +} + + int hasSuffix(const char *str, const char *suffix) { if (!str || !suffix) @@ -164,9 +212,9 @@ int addidx(const char *name, int addr, int len) defbuf_p += snprintf(&dbuf[defbuf_p], DEF_SIZE - defbuf_p, "#define FDATA_START_%s 0x%x\n", s, addr); defbuf_p += snprintf(&dbuf[defbuf_p], DEF_SIZE - defbuf_p, "#define FDATA_SIZE_%s %d\n", s, len); - ibuf_p += snprintf(&ibuf[ibuf_p], INDEX_SIZE - ibuf_p, " {\"/%s\", FDATA_START_%s, FDATA_SIZE_%s, %s},\n", name, s, s, getMime(name)); + ibuf_p += snprintf(&ibuf[ibuf_p], INDEX_SIZE - ibuf_p, " {\"/%s\", FDATA_START_%s, FDATA_SIZE_%s, %s, %d},\n", name, s, s, getMime(name), arguments.gzip? 1 : 0); if (!strcmp(name, "index.html")) - ibuf_p += snprintf(&ibuf[ibuf_p], INDEX_SIZE - ibuf_p, " {\"/\", FDATA_START_%s, FDATA_SIZE_%s, mime_HTML},\n", s, s); + ibuf_p += snprintf(&ibuf[ibuf_p], INDEX_SIZE - ibuf_p, " {\"/\", FDATA_START_%s, FDATA_SIZE_%s, mime_HTML, %d},\n", s, s, arguments.gzip? 1 : 0); return 0; } @@ -216,7 +264,6 @@ int replaceCalls(int pos) int main(int argc, char **argv) { - struct arguments arguments; int arg_index; char tmpfilename[] = "image_XXXXXX"; struct stat s; @@ -227,6 +274,7 @@ int main(int argc, char **argv) arguments.overwrite = true; arguments.prefix = 0; arguments.bank = NULL; + arguments.gzip = false; argp_parse(&argp, argc, argv, 0, &arg_index, &arguments); if (!arg_index) @@ -269,7 +317,7 @@ int main(int argc, char **argv) defbuf_p += snprintf(&dbuf[defbuf_p], DEF_SIZE - defbuf_p, "#include \n\n"); defbuf_p += snprintf(&dbuf[defbuf_p], DEF_SIZE - defbuf_p, "typedef enum mime_type_e {\n mime_HTML = 0,\n mime_SVG,\n mime_ICO,\n mime_PNG,\n mime_JS,\n mime_CSS,\n mime_TXT\n} mime_type_t;\n\n"); - defbuf_p += snprintf(&dbuf[defbuf_p], DEF_SIZE - defbuf_p, "struct f_data {\n __code char *file;\n uint32_t start;\n uint16_t len;\n mime_type_t mime;\n};\n\n"); + defbuf_p += snprintf(&dbuf[defbuf_p], DEF_SIZE - defbuf_p, "struct f_data {\n __code char *file;\n uint32_t start;\n uint16_t len;\n mime_type_t mime;\n uint8_t gzip;\n};\n\n"); // defbuf_p += snprintf(&dbuf[defbuf_p], DEF_SIZE - defbuf_p, "typedef uint16_t (* fcall_ptr)(void);\n\n"); ibuf_p += snprintf(&ibuf[ibuf_p], INDEX_SIZE - ibuf_p, "// This file is automatically generated, do not edit!\n\n"); @@ -309,16 +357,21 @@ int main(int argc, char **argv) size_t data_read = addfile(pathbuffer, addr); if (data_read) { - if (arguments.add_zero) { - buffer[addr + data_read + 1] = '\0'; - data_read++; - } + if (arguments.add_zero) { + /* NUL-terminate the data so the file table and the + * strlen()-based size computation are deterministic + * (the buffer past the file is otherwise uninitialised) */ + buffer[addr + data_read] = '\0'; + data_read++; + } printf("Data inserted from %s at 0x%x, size: %ld\n", pathbuffer, addr, data_read); } int old_len = data_read; data_read = replaceCalls(addr); if (old_len > data_read) memset(buffer + addr + data_read, 0, old_len - data_read); + if (arguments.gzip) + data_read = gzipBuffer(addr, data_read); addidx(in_file->d_name, addr, data_read); addr += data_read; } diff --git a/tools/minify.py b/tools/minify.py new file mode 100644 index 00000000..8c033044 --- /dev/null +++ b/tools/minify.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Safe minifier for the WebUI sources (HTML/JS). + +Removes comments and line-leading whitespace without touching string +literals, so the output is functionally identical to the input. Used +by the firmware Makefile before the files are embedded in the flash; +the raw sources in html/ stay untouched for development. + +Binary files (.ico) are copied through unchanged. + +Usage: minify.py +""" +import re +import shutil +import sys + + +def js_min(s): + out = [] + i = 0 + n = len(s) + while i < n: + c = s[i] + if c == '"' or c == "'": + j = i + 1 + while j < n and s[j] != c: + if s[j] == '\\': + j += 1 + j += 1 + out.append(s[i:j + 1]) + i = j + 1 + elif c == '/' and i + 1 < n and s[i + 1] == '/': + j = s.find('\n', i) + i = j if j >= 0 else n + elif c == '/' and i + 1 < n and s[i + 1] == '*': + j = s.find('*/', i + 2) + i = j + 2 if j >= 0 else n + else: + out.append(c) + i += 1 + lines = [l.strip() for l in ''.join(out).split('\n')] + return '\n'.join(l for l in lines if l) + '\n' + + +def html_min(s): + s = re.sub(r'', '', s, flags=re.S) + lines = [l.strip() for l in s.split('\n')] + return '\n'.join(l for l in lines if l) + '\n' + + +def main(): + if len(sys.argv) != 3: + print('usage: minify.py ', file=sys.stderr) + sys.exit(1) + src, dst = sys.argv[1], sys.argv[2] + if src.endswith('.ico'): + shutil.copyfile(src, dst) + print('%s: copied %d bytes' % (src.split('/')[-1], len(open(src, 'rb').read()))) + return + with open(src, 'r', encoding='utf-8', errors='replace') as f: + s = f.read() + if src.endswith('.js'): + out = js_min(s) + else: + out = html_min(s) + with open(dst, 'w', encoding='utf-8') as f: + f.write(out) + print('%s: %d -> %d bytes' % (src.split('/')[-1], len(s), len(out))) + + +if __name__ == '__main__': + main()