Update build scripts for WebAssembly version of gap-system for faster startup and on-demand loading

etc/emscripten/README.md

@@ -6,6 +6,8 @@ Files:
 
 - `web-template`: Uses 'xterm-pty' to create a "nice" interface to the Wasm GAP.
 
+- `build_startup_manifest.js`: Run it in the web root directory to build `startup_manifest.json` that contains resources to preload.
+
 See 'run-web-demo.sh' as an example on how to set up a working website.
 
 Note that this demo uses xterm-pty, a library which provides a terminal interface

etc/emscripten/build.sh

@@ -71,7 +71,7 @@ mkdir -p "$AUX_PREFIX"
 # -O2 : Some optimisation
 # EXEEXT=.html -- this is actually a GAP makefile option, it lets us make the
 # output 'gap.html', which makes emscripten output a html page we can load
-# --preload-file : The directories containing files GAP needs to run
+# --pre-js lazy_fs.js : Prepend lazy_fs.js that is generated for lazy loading
 
 # Run configure if we don't have a makefile, or someone configured this
 # GAP for standard building (emscripten builds will use 'emcc')
@@ -82,25 +82,22 @@ if [[ ! -f GNUmakefile ]] || ! grep '/emcc' GNUmakefile > /dev/null; then
     LDFLAGS="-s ASYNCIFY=1 -O2"
 fi;
 
-# Get basic required packages
-emmake make bootstrap-pkg-minimal
+# Get full required packages
+emmake make bootstrap-pkg-full
 
 # Copy in files from native_build
 cp native-build/build/c_*.c native-build/build/ffdata.* src/
 
+# Dynamically find and append ALL required files to the JS array
+# The flag -type f is safe because the only symbolic link is 'tst/mockpkg/Makefile.gappkg',
+# which is safe to ignore
+find pkg lib grp tst doc hpcgap dev benchmark -type f | python3 etc/emscripten/generate_mapping.py
+
+if [ $? -ne 0 ]; then
+    echo "Build aborted: generate_mapping.py failed."
+    exit 1
+fi
+
 # The EXEEXT is usually for windows, but here it lets us set GAP's extension,
-# which lets us produce a html page to run GAP in
-emmake make -j8 LDFLAGS="--preload-file pkg --preload-file lib --preload-file grp --preload-file tst -s ASYNCIFY=1 -sTOTAL_STACK=32mb -sASYNCIFY_STACK_SIZE=32000000 -sINITIAL_MEMORY=2048mb -O2" EXEEXT=".html"
-
-# SPLIT THE DATA FILE
-echo "Splitting gap.data..."
-split -b 75m gap.data "gap.data.part"
-
-# Rename to .part1, .part2...
-i=1
-for f in gap.data.part*; do
-    mv "$f" "gap.data.part$i"
-    echo "Created gap.data.part$i"
-    ((i++))
-done
-rm gap.data
+# which lets us produce a html page to run GAP in.
+emmake make -j8 LDFLAGS="-lidbfs.js --pre-js lazy_fs.js -s ASYNCIFY=1 -sTOTAL_STACK=32mb -sASYNCIFY_STACK_SIZE=32000000 -sINITIAL_MEMORY=2048mb -O2" EXEEXT=".html"

etc/emscripten/build_startup_manifest.js

@@ -0,0 +1,60 @@
+#!/usr/bin/env node
+"use strict";
+
+const http = require('http');
+const fs = require('fs');
+const path = require('path');
+
+const PORT = 9999;
+const BASE_DIR = process.cwd(); 
+const LOG_FILE = path.join(BASE_DIR, 'startup_manifest.json');
+
+const requestedFiles = new Set();
+fs.writeFileSync(LOG_FILE, '[\n]');
+
+const server = http.createServer((req, res) => {
+    let urlPath = req.url.split('?')[0];
+    if (urlPath === '/') {
+        urlPath = '/index.html';
+    }
+
+    const filePath = path.join(BASE_DIR, urlPath);
+
+    fs.readFile(filePath, (err, data) => {
+        if (err) {
+            console.error(`[404] Ignored missing file: ${urlPath}`);
+            res.writeHead(404);
+            res.end(`404: File not found`);
+            return;
+        }
+
+        if (urlPath !== '/favicon.ico' && !requestedFiles.has(urlPath)) {
+            requestedFiles.add(urlPath);
+
+            const manifest = Array.from(requestedFiles);
+            fs.writeFileSync(LOG_FILE, JSON.stringify(manifest, null, 4));
+
+            console.log(`[Loaded & Logged] ${urlPath} (Total: ${requestedFiles.size})`);
+        }
+
+        let contentType = 'application/octet-stream';
+        if (urlPath.endsWith('.html')) contentType = 'text/html';
+        else if (urlPath.endsWith('.js')) contentType = 'text/javascript';
+        else if (urlPath.endsWith('.wasm')) contentType = 'application/wasm';
+
+        res.writeHead(200, {
+            'Content-Type': contentType,
+            'Cross-Origin-Opener-Policy': 'same-origin',
+            'Cross-Origin-Embedder-Policy': 'require-corp',
+            'Access-Control-Allow-Origin': '*'
+        });
+
+        res.end(data);
+    });
+});
+
+server.listen(PORT, '0.0.0.0', () => {
+    console.log(`\n Tracker running at http://localhost:${PORT}/`);
+    console.log(`Serving files from: ${BASE_DIR}`);
+    console.log(`Logging valid loaded files to: ${LOG_FILE}\n`);
+});

etc/emscripten/copy_hashed_assets.py

@@ -0,0 +1,33 @@
+import sys
+import hashlib
+import os
+import shutil
+
+def main():
+    if len(sys.argv) < 2:
+        print("Error: Must provide destination directory.", file=sys.stderr)
+        sys.exit(1)
+
+    dest_dir = sys.argv[1]
+    os.makedirs(dest_dir, exist_ok=True)
+
+    copied_count = 0
+
+    for line in sys.stdin:
+        path = line.strip()
+        if not path:
+            continue
+
+        h = hashlib.md5(path.encode('utf-8')).hexdigest()
+        _, ext = os.path.splitext(path)
+        hashed_name = f"{h}{ext}"
+
+        dest_path = os.path.join(dest_dir, hashed_name)
+
+        shutil.copy2(path, dest_path)
+        copied_count += 1
+
+    print(f"Successfully copied {copied_count} files into {dest_dir}", file=sys.stderr)
+
+if __name__ == "__main__":
+    main()

etc/emscripten/generate_mapping.py

@@ -0,0 +1,118 @@
+import sys
+import hashlib
+import os
+
+def main():
+    seen_hashes = {}
+    mappings = []
+
+    for line in sys.stdin:
+        path = line.strip()
+        if not path:
+            continue
+
+        h = hashlib.md5(path.encode('utf-8')).hexdigest()
+        _, ext = os.path.splitext(path)
+        hashed_name = f"{h}{ext}"
+
+        if hashed_name in seen_hashes:
+            print("\nError: Hash collision detected!", file=sys.stderr)
+            print(f"File 1: {seen_hashes[hashed_name]}", file=sys.stderr)
+            print(f"File 2: {path}", file=sys.stderr)
+            sys.exit(1)
+
+        seen_hashes[hashed_name] = path
+        mappings.append((path, hashed_name))
+
+    try:
+        with open('lazy_fs.js', 'w', encoding='utf-8') as f:
+            f.write("Module.preRun = Module.preRun || [];\n")
+            f.write("Module.preRun.push(function() {\n")
+            f.write("    var fileMap = {\n")
+
+            for path, hashed_name in mappings:
+                safe_path = path.replace('"', '\\"')
+                f.write(f'        "{safe_path}": "{hashed_name}",\n')
+
+            f.write("    };\n\n")
+            f.write('    var physicalDir = "assets/";\n')
+
+            f.write("    var createdDirs = {};\n")
+            f.write("    Object.keys(fileMap).forEach(function(virtualPath) {\n")
+            f.write("        var parts = virtualPath.split('/');\n")
+            f.write("        parts.pop();\n")
+            f.write("        var parentDir = '/' + parts.join('/');\n")
+            f.write("        if (!createdDirs[parentDir]) {\n")
+            f.write("            try { FS.mkdirTree(parentDir); } catch(e) {}\n")
+            f.write("            createdDirs[parentDir] = true;\n")
+            f.write("        }\n")
+            f.write("    });\n\n")
+
+            f.write("    try { FS.mkdirTree('/gap_idb_cache'); } catch(e) {}\n")
+            f.write("    FS.mount(IDBFS, {}, '/gap_idb_cache');\n")
+            f.write("    addRunDependency('idbfs_sync');\n\n")
+
+            f.write("    FS.syncfs(true, async function(err) {\n")
+            f.write("        var needsSave = false;\n")
+            f.write("        var startupSet = new Set();\n")
+
+            f.write("        try {\n")
+            f.write("            const manifestRes = await fetch('startup_manifest.json');\n")
+            f.write("            if (manifestRes.ok) {\n")
+            f.write("                const manifest = await manifestRes.json();\n")
+            f.write("                manifest.forEach(p => {\n")
+            f.write("                    if (p.startsWith('/')) p = p.substring(1);\n")
+            f.write("                    startupSet.add(p);\n")
+            f.write("                });\n")
+            f.write("            }\n")
+            f.write("        } catch (e) {}\n\n")
+
+            f.write("        var fetchPromises = Object.keys(fileMap).map(async function(virtualPath) {\n")
+            f.write("            var physicalName = fileMap[virtualPath];\n")
+            f.write("            var physicalPath = physicalDir + physicalName;\n")
+            f.write("            var cachePath = '/gap_idb_cache/' + physicalName;\n")
+            f.write("            var finalPath = '/' + virtualPath;\n\n")
+
+            f.write("            if (startupSet.has(physicalPath)) {\n")
+            f.write("                try {\n")
+            f.write("                    FS.stat(cachePath);\n")
+            f.write("                    FS.writeFile(finalPath, FS.readFile(cachePath));\n")
+            f.write("                } catch (e) {\n")
+            f.write("                    try {\n")
+            f.write("                        const response = await fetch(physicalPath);\n")
+            f.write("                        if (response.ok) {\n")
+            f.write("                            const buffer = await response.arrayBuffer();\n")
+            f.write("                            const data = new Uint8Array(buffer);\n")
+            f.write("                            FS.writeFile(finalPath, data);\n")
+            f.write("                            FS.writeFile(cachePath, data);\n")
+            f.write("                            needsSave = true;\n")
+            f.write("                        }\n")
+            f.write("                    } catch (fetchErr) {}\n")
+            f.write("                }\n")
+            f.write("            } else {\n")
+            f.write("                var parts = virtualPath.split('/');\n")
+            f.write("                var fileName = parts.pop();\n")
+            f.write("                var parentDir = '/' + parts.join('/');\n")
+            f.write("                FS.createLazyFile(parentDir, fileName, physicalPath, true, false);\n")
+            f.write("            }\n")
+            f.write("        });\n\n")
+
+            f.write("        await Promise.all(fetchPromises);\n")
+            f.write("        if (needsSave) {\n")
+            f.write("            FS.syncfs(false, function(saveErr) {\n")
+            f.write("                removeRunDependency('idbfs_sync');\n")
+            f.write("            });\n")
+            f.write("        } else {\n")
+            f.write("            removeRunDependency('idbfs_sync');\n")
+            f.write("        }\n")
+            f.write("    });\n")
+            f.write("});\n")
+
+        print(f"Successfully mapped {len(mappings)} files into lazy_fs.js", file=sys.stderr)
+
+    except Exception as e:
+        print(f"Failed to write lazy_fs.js: {e}", file=sys.stderr)
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

etc/emscripten/run-web-demo.sh

@@ -4,8 +4,16 @@ set -eux
 
 echo This script assumes you have already run 'build.sh'
 
-mkdir -p web-example
-cp etc/emscripten/web-template/* web-example
-cp gap.js gap.wasm gap.data.part* web-example
+mkdir -p web-example/assets
+cp etc/emscripten/web-template/* web-example/
+cp gap.js gap.wasm gap.worker.js web-example/ 
+
+find pkg lib grp tst doc hpcgap dev benchmark -type f | python3 etc/emscripten/copy_hashed_assets.py web-example/assets
+
+if [ $? -ne 0 ]; then
+    echo "Copying failed."
+    exit 1
+fi
+
 cd web-example
 ../etc/emscripten/server.rb

etc/emscripten/web-template/gap-worker.js

@@ -1,69 +1,8 @@
 importScripts("https://cdn.jsdelivr.net/npm/xterm-pty@0.9.4/workerTools.js");
 
 onmessage = (msg) => {
-  // We wrap the initialization in an async function to handle the data fetching
-  async function loadAndStart() {
-    const buffers = [];
-    let i = 1;
-
-    // Download all split parts
-    // It will look for gap.data.part1, part2, etc. until it hits a 404.
-    while (true) {
-      const url = `gap.data.part${i}`;
-      try {
-        const response = await fetch(url);
-        if (!response.ok) break; // Stop when we hit 404
-
-        const buf = await response.arrayBuffer();
-        buffers.push(new Uint8Array(buf));
-        i++;
-      } catch (e) {
-        break;
-      }
-    }
-
-    // Prepare the Module object BEFORE importing gap.js
-    self.Module = self.Module || {};
-
-    // Merge data.
-    if (buffers.length > 0) {
-      const totalLength = buffers.reduce((acc, b) => acc + b.length, 0);
-      const mergedData = new Uint8Array(totalLength);
-      let offset = 0;
-      for (const buffer of buffers) {
-        mergedData.set(buffer, offset);
-        offset += buffer.length;
-      }
-
-      console.log(`Worker: Loaded ${buffers.length} parts. Total size: ${totalLength} bytes.`);
-
-      // Override the default downloader.
-      // When gap.js asks for 'gap.data', we give it our merged array immediately.
-      // This stops it from trying to fetch 'gap.data' via XHR.
-      self.Module.getPreloadedPackage = function(remotePackageName, remotePackageSize) {
-        if (remotePackageName === 'gap.data') {
-          return mergedData.buffer; 
-        }
-        return null; // Let other files download normally if any
-      };
-
-      // Just in case: also write it to FS in preRun.
-      self.Module.preRun = self.Module.preRun || [];
-      self.Module.preRun.push(() => {
-         try {
-
-             FS.writeFile('/gap.data', mergedData); 
-         } catch(e) { /* ignore if already handled by getPreloadedPackage */ }
-      });
-    } else {
-      console.warn("Worker: No gap.data parts found. The standard downloader will likely fail with 404.");
-    }
-
-    // Load GAP.
-    importScripts("gap.js");
-
-    emscriptenHack(new TtyClient(msg.data));
-  }
-
-  loadAndStart();
+  // Prepare the Module object BEFORE importing gap.js
+  self.Module = self.Module || {};
+  importScripts("gap.js");
+  emscriptenHack(new TtyClient(msg.data));
 };

etc/emscripten/web-template/index.html

@@ -1,6 +1,7 @@
 <html>
   <head>
     <title>gap-wasm</title>
+    <script src="coi-serviceworker.js"></script>
     <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/xterm@4.17.0/css/xterm.css">
   </head>
   <body>
@@ -17,9 +18,5 @@
       const worker = new Worker("gap-worker.js");
       new TtyServer(slave).start(worker);
     </script>
-
-    <!-- Hack to get around lack of SharedArrayBuffer on github pages -->
-    <script src="coi-serviceworker.js"></script>
   </body>
-</html>
-
+</html>