feat(webui): minify and gzip-compress the embedded web assets

Two build-time steps shrink the embedded Web UI (html/) from 113,015 to
34,784 bytes (30.8 %):

- tools/minify.py (new): string-literal-safe minifier that strips
  comments and indentation; comments are pure overhead on the wire, so
  the transfer size drops already before gzip (113,015 -> 101,498).
  Binary files (.ico) are copied through unchanged.  The output goes to
  output/html_min/, the new fileadder input.
- fileadder -z: gzip-compresses every added file (zlib gzip format,
  Z_BEST_COMPRESSION); the generated f_data entries carry a gzip flag.
  tools/Makefile links the tool against -lz.
- httpd: adds "Content-Encoding: gzip" for flagged entries; the 8051
  only copies compressed bytes, decompression happens in the browser.
  The JSON API stays uncompressed.

Fixes required to make the sizes deterministic:
- fileadder terminated files at data_read + 1, so the strlen()-based
  size computation depended on uninitialised buffer content; the NUL
  now lands directly after the content.
- version.h was .PHONY, so every build rewrote it while parallel builds
  could read a half-written copy.
- the file list was unscoped and missed .css/.ico; it is now scoped to
  html/ and covers all embedded types.

Measured with make MACHINE=KP_9000_6XHML_X2: data block end moves from
0x5b977 to 0x487e0 (~78 KB freed on a 512 KB image).  Verified with
MACHINE=PCB_K0402WS_V3 and on real hardware (PCB-K0402WS-V3).

Makefile

@@ -75,10 +75,23 @@ SRCS += \
 
 OBJS = ${SRCS:%.c=$(BUILDDIR)/%.rel}
 DEPS := ${SRCS:%.c=$(BUILDDIR)/%.d}
-HTML := $(shell find $(html) -name '*.js' -or -name '*.html' -or -name '*.svg')
+HTML := $(shell find html -name '*.js' -or -name '*.html' -or -name '*.svg' -or -name '*.css' -or -name '*.ico')
 
-html_data.c html_data.h: $(HTML) tools/output/fileadder
+# Minified copy of the web UI sources, used as the fileadder input.
-	tools/output/fileadder -a $(HTML_LOCATION) -s $(IMAGESIZE) -b BANK1 -d html -p html_data
+# The raw html/ sources stay untouched for development; the minified
+# copy is a build artifact under output/.
+HTML_MIN := output/html_min
+.PHONY: html_min
+html_min: $(HTML)
+	rm -rf $(HTML_MIN)
+	mkdir -p $(HTML_MIN)
+	@for f in $(HTML); do python3 tools/minify.py $$f $(HTML_MIN)/$$(basename $$f) || exit 1; done
+
+html_data.c html_data.h: $(HTML) tools/output/fileadder html_min
+	tools/output/fileadder -a $(HTML_LOCATION) -s $(IMAGESIZE) -b BANK1 -z -d $(HTML_MIN) -p html_data
+# httpd.c includes html_data.h; declare it explicitly so a parallel
+# build (-j) does not compile httpd before the header is generated.
+$(BUILDDIR)/httpd/httpd.rel: html_data.h
 
 $(VERSION_HEADER):
 	@echo "#ifndef VERSION_H" > $(VERSION_HEADER)
@@ -118,11 +131,11 @@ $(BUILDDIR)/rtlplayground-$(FILENAME_EXTENSION).bin: $(BUILDDIR)/rtlplayground.i
 	tools/output/imagebuilder -i $^ $@
 	tools/output/fileadder -a $(DEFAULT_CONFIG_LOCATION) -s $(IMAGESIZE) -d config.txt $@
 	tools/output/fileadder -a $(CONFIG_LOCATION) -s $(IMAGESIZE) -d config.txt $@
-	tools/output/fileadder -a $(HTML_LOCATION) -s $(IMAGESIZE) -d html -p html_data -b BANK1 $@
+	tools/output/fileadder -a $(HTML_LOCATION) -s $(IMAGESIZE) -z -d $(HTML_MIN) -p html_data -b BANK1 $@
 	tools/output/crc_calculator -u $@
 	ln -sf $(MACHINE)/rtlplayground-$(FILENAME_EXTENSION).bin output/rtlplayground.bin
 
-.PHONY: clean all $(SUBDIRS) $(VERSION_HEADER)
+.PHONY: clean all $(SUBDIRS)
 
 .PHONY:
 machine_check:

httpd/httpd.c

@@ -719,7 +719,10 @@ void httpd_appcall(void)
 			 * response; without advertising it a browser reuses the socket
 			 * from its keep-alive pool and the next request hits the already
 			 * closed connection (a POST is then dropped without a retry). */
-			slen += strtox(outbuf + slen, "; charset=UTF-8\r\nCache-Control: max-age=60, must-revalidate\r\nConnection: close\r\nAccess-Control-Allow-Origin: *\r\nContent-Security-Policy: default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; connect-src 'self'; form-action 'self'\r\n\r\n");
+			slen += strtox(outbuf + slen, "; charset=UTF-8\r\n");
+			if (f_data[entry].gzip)
+				slen += strtox(outbuf + slen, "Content-Encoding: gzip\r\n");
+			slen += strtox(outbuf + slen, "Cache-Control: max-age=60, must-revalidate\r\nConnection: close\r\nAccess-Control-Allow-Origin: *\r\nContent-Security-Policy: default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; connect-src 'self'; form-action 'self'\r\n\r\n");
 
 			len_left = f_data[entry].len;
 			if (len_left > (TCP_OUTBUF_SIZE - slen)) {

tools/Makefile

@@ -15,7 +15,7 @@ $(BUILDDIR)/injector: injector.c
 	gcc $^ $(CCFLAGS) $@
 
 $(BUILDDIR)/fileadder: fileadder.c
-	gcc $^ $(CCFLAGS) $@
+	gcc $^ $(CCFLAGS) $@ -lz
 
 $(BUILDDIR)/crc_calculator: crc_calculator.c
 	gcc $^ $(CCFLAGS) $@

tools/fileadder.c

@@ -11,6 +11,7 @@
 #include <string.h>
 #include <argp.h>
 #include <stdbool.h>
+#include <zlib.h>
 
 #define OFFSET 2
 
@@ -33,6 +34,7 @@ int fbuf_p;
 int xbuf_p;
 bool addsDir = false;
 int callNum = 0;
+struct arguments arguments;
 
 const char *argp_program_version = "fileadder 0.1";
 const char *argp_program_bug_address = "<git@logicog.de>";
@@ -45,6 +47,7 @@ static struct argp_option options[] = {
     { "address", 'a', "SIZE", 0, "Address where data is placed, default is 0x1000000 if option is used, otherwise 0x1fd000"},
     { "prefix", 'p', "FILE", 0, "Prefix for header and index file generation"},
     { "bank", 'b', "BANKNAME", 0, "Generate #pragma with given bank-name"},
+    { "gzip", 'z', 0, 0, "gzip-compress the added files"},
     { 0 }
 };
 
@@ -57,6 +60,7 @@ struct arguments {
 	char *bank;
 	bool overwrite;
 	bool add_zero;
+	bool gzip;
 };
 
 
@@ -86,6 +90,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state)
 	case 'b':
 		arguments->bank = arg? arg : "BANK1";
 		break;
+	case 'z':
+		arguments->gzip = true;
+		break;
 	default:
 		return ARGP_ERR_UNKNOWN;
     }
@@ -118,6 +125,47 @@ int addfile(const char *name, int addr)
 }
 
 
+/*
+ * gzip-compresses the data at buffer[addr] in place (gzip format, as
+ * expected by the Content-Encoding: gzip header the httpd serves it
+ * with).  Returns the compressed size.
+ */
+int gzipBuffer(int addr, int len)
+{
+	uLong bound = compressBound(len);
+	unsigned char *out = malloc(bound);
+	if (!out) {
+		perror("malloc");
+		return 0;
+	}
+
+	z_stream zs;
+	memset(&zs, 0, sizeof(zs));
+	if (deflateInit2(&zs, Z_BEST_COMPRESSION, Z_DEFLATED, 15 + 16, 8,
+			 Z_DEFAULT_STRATEGY) != Z_OK) {
+		fprintf(stderr, "gzip: deflateInit2 failed\n");
+		free(out);
+		return 0;
+	}
+	zs.next_in = (Bytef *)&buffer[addr];
+	zs.avail_in = len;
+	zs.next_out = out;
+	zs.avail_out = bound;
+	if (deflate(&zs, Z_FINISH) != Z_STREAM_END) {
+		fprintf(stderr, "gzip: deflate failed\n");
+		deflateEnd(&zs);
+		free(out);
+		return 0;
+	}
+	int out_len = zs.total_out;
+	deflateEnd(&zs);
+
+	memcpy(&buffer[addr], out, out_len);
+	free(out);
+	return out_len;
+}
+
+
 int hasSuffix(const char *str, const char *suffix)
 {
     if (!str || !suffix)
@@ -164,9 +212,9 @@ int addidx(const char *name, int addr, int len)
 
 	defbuf_p += snprintf(&dbuf[defbuf_p], DEF_SIZE - defbuf_p, "#define FDATA_START_%s 0x%x\n", s, addr);
 	defbuf_p += snprintf(&dbuf[defbuf_p], DEF_SIZE - defbuf_p, "#define FDATA_SIZE_%s %d\n", s, len);
-	ibuf_p += snprintf(&ibuf[ibuf_p], INDEX_SIZE - ibuf_p, "  {\"/%s\", FDATA_START_%s, FDATA_SIZE_%s, %s},\n", name, s, s, getMime(name));
+	ibuf_p += snprintf(&ibuf[ibuf_p], INDEX_SIZE - ibuf_p, "  {\"/%s\", FDATA_START_%s, FDATA_SIZE_%s, %s, %d},\n", name, s, s, getMime(name), arguments.gzip? 1 : 0);
 	if (!strcmp(name, "index.html"))
-		ibuf_p += snprintf(&ibuf[ibuf_p], INDEX_SIZE - ibuf_p, "  {\"/\", FDATA_START_%s, FDATA_SIZE_%s, mime_HTML},\n", s, s);
+		ibuf_p += snprintf(&ibuf[ibuf_p], INDEX_SIZE - ibuf_p, "  {\"/\", FDATA_START_%s, FDATA_SIZE_%s, mime_HTML, %d},\n", s, s, arguments.gzip? 1 : 0);
 	return 0;
 }
 
@@ -216,7 +264,6 @@ int replaceCalls(int pos)
 
 int main(int argc, char **argv)
 {
-	struct arguments arguments;
 	int arg_index;
 	char tmpfilename[] = "image_XXXXXX";
 	struct stat s;
@@ -227,6 +274,7 @@ int main(int argc, char **argv)
 	arguments.overwrite = true;
 	arguments.prefix = 0;
 	arguments.bank = NULL;
+	arguments.gzip = false;
 
 	argp_parse(&argp, argc, argv, 0, &arg_index, &arguments);
 	if (!arg_index)
@@ -269,7 +317,7 @@ int main(int argc, char **argv)
 	defbuf_p += snprintf(&dbuf[defbuf_p], DEF_SIZE - defbuf_p, "#include <stdint.h>\n\n");
 	defbuf_p += snprintf(&dbuf[defbuf_p], DEF_SIZE - defbuf_p,
 			     "typedef enum mime_type_e {\n  mime_HTML = 0,\n  mime_SVG,\n  mime_ICO,\n  mime_PNG,\n  mime_JS,\n  mime_CSS,\n  mime_TXT\n} mime_type_t;\n\n");
-	defbuf_p += snprintf(&dbuf[defbuf_p], DEF_SIZE - defbuf_p, "struct f_data {\n  __code char *file;\n  uint32_t start;\n  uint16_t len;\n  mime_type_t mime;\n};\n\n");
+	defbuf_p += snprintf(&dbuf[defbuf_p], DEF_SIZE - defbuf_p, "struct f_data {\n  __code char *file;\n  uint32_t start;\n  uint16_t len;\n  mime_type_t mime;\n  uint8_t gzip;\n};\n\n");
 	// defbuf_p += snprintf(&dbuf[defbuf_p], DEF_SIZE - defbuf_p, "typedef uint16_t (* fcall_ptr)(void);\n\n");
 
 	ibuf_p += snprintf(&ibuf[ibuf_p], INDEX_SIZE - ibuf_p, "// This file is automatically generated, do not edit!\n\n");
@@ -310,7 +358,10 @@ int main(int argc, char **argv)
 			size_t data_read = addfile(pathbuffer, addr);
 			if (data_read) {
 			if (arguments.add_zero) {
-					buffer[addr + data_read + 1] = '\0';
+				/* NUL-terminate the data so the file table and the
+				 * strlen()-based size computation are deterministic
+				 * (the buffer past the file is otherwise uninitialised) */
+				buffer[addr + data_read] = '\0';
 				data_read++;
 			}
 				printf("Data inserted from %s at 0x%x, size: %ld\n", pathbuffer, addr, data_read);
@@ -319,6 +370,8 @@ int main(int argc, char **argv)
 			data_read = replaceCalls(addr);
 			if (old_len > data_read)
 				memset(buffer + addr + data_read, 0, old_len - data_read);
+			if (arguments.gzip)
+				data_read = gzipBuffer(addr, data_read);
 			addidx(in_file->d_name, addr, data_read);
 			addr += data_read;
 		}

tools/minify.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+"""Safe minifier for the WebUI sources (HTML/JS).
+
+Removes comments and line-leading whitespace without touching string
+literals, so the output is functionally identical to the input.  Used
+by the firmware Makefile before the files are embedded in the flash;
+the raw sources in html/ stay untouched for development.
+
+Binary files (.ico) are copied through unchanged.
+
+Usage: minify.py <src> <dst>
+"""
+import re
+import shutil
+import sys
+
+
+def js_min(s):
+    out = []
+    i = 0
+    n = len(s)
+    while i < n:
+        c = s[i]
+        if c == '"' or c == "'":
+            j = i + 1
+            while j < n and s[j] != c:
+                if s[j] == '\\':
+                    j += 1
+                j += 1
+            out.append(s[i:j + 1])
+            i = j + 1
+        elif c == '/' and i + 1 < n and s[i + 1] == '/':
+            j = s.find('\n', i)
+            i = j if j >= 0 else n
+        elif c == '/' and i + 1 < n and s[i + 1] == '*':
+            j = s.find('*/', i + 2)
+            i = j + 2 if j >= 0 else n
+        else:
+            out.append(c)
+            i += 1
+    lines = [l.strip() for l in ''.join(out).split('\n')]
+    return '\n'.join(l for l in lines if l) + '\n'
+
+
+def html_min(s):
+    s = re.sub(r'<!--.*?-->', '', s, flags=re.S)
+    lines = [l.strip() for l in s.split('\n')]
+    return '\n'.join(l for l in lines if l) + '\n'
+
+
+def main():
+    if len(sys.argv) != 3:
+        print('usage: minify.py <src> <dst>', file=sys.stderr)
+        sys.exit(1)
+    src, dst = sys.argv[1], sys.argv[2]
+    if src.endswith('.ico'):
+        shutil.copyfile(src, dst)
+        print('%s: copied %d bytes' % (src.split('/')[-1], len(open(src, 'rb').read())))
+        return
+    with open(src, 'r', encoding='utf-8', errors='replace') as f:
+        s = f.read()
+    if src.endswith('.js'):
+        out = js_min(s)
+    else:
+        out = html_min(s)
+    with open(dst, 'w', encoding='utf-8') as f:
+        f.write(out)
+    print('%s: %d -> %d bytes' % (src.split('/')[-1], len(s), len(out)))
+
+
+if __name__ == '__main__':
+    main()