Update build scripts for WebAssembly version of gap-system for faster startup and on-demand loading

etc/emscripten/README.md

@@ -6,6 +6,8 @@ Files:
 
 - `web-template`: Uses 'xterm-pty' to create a "nice" interface to the Wasm GAP.
 
+- `build_startup_manifest.js`: Run it in the web root directory to build `startup_manifest.json` that contains resources to preload.
+
 See 'run-web-demo.sh' as an example on how to set up a working website.
 
 Note that this demo uses xterm-pty, a library which provides a terminal interface
@@ -21,4 +23,4 @@ For more details, see for [this article](https://developer.mozilla.org/en-US/doc
 
 The file "coi-serviceworker.js" works around this problem on Github pages. This won't
 work locally, so "server.rb" is a simple ruby script, which just starts a web-server
-which returns the required headers.
+which returns the required headers.

etc/emscripten/build.sh

@@ -71,7 +71,7 @@ mkdir -p "$AUX_PREFIX"
 # -O2 : Some optimisation
 # EXEEXT=.html -- this is actually a GAP makefile option, it lets us make the
 # output 'gap.html', which makes emscripten output a html page we can load
-# --preload-file : The directories containing files GAP needs to run
+# --pre-js lazy_fs.js : Prepend lazy_fs.js that is generated for lazy loading
 
 # Run configure if we don't have a makefile, or someone configured this
 # GAP for standard building (emscripten builds will use 'emcc')
@@ -82,25 +82,22 @@ if [[ ! -f GNUmakefile ]] || ! grep '/emcc' GNUmakefile > /dev/null; then
     LDFLAGS="-s ASYNCIFY=1 -O2"
 fi;
 
-# Get basic required packages
-emmake make bootstrap-pkg-minimal
+# Get full required packages
+emmake make bootstrap-pkg-full
 
 # Copy in files from native_build
 cp native-build/build/c_*.c native-build/build/ffdata.* src/
 
+# Dynamically find and append ALL required files to the JS array
+# The flag -type f is safe because the only symbolic link is 'tst/mockpkg/Makefile.gappkg',
+# which is safe to ignore
+find pkg lib grp tst doc hpcgap dev benchmark -type f | python3 etc/emscripten/generate_gap_fs_json.py
+
+if [ $? -ne 0 ]; then
+    echo "Build aborted: generate_gap_fs_json.py failed."
+    exit 1
+fi
+
 # The EXEEXT is usually for windows, but here it lets us set GAP's extension,
-# which lets us produce a html page to run GAP in
-emmake make -j8 LDFLAGS="--preload-file pkg --preload-file lib --preload-file grp --preload-file tst -s ASYNCIFY=1 -sTOTAL_STACK=32mb -sASYNCIFY_STACK_SIZE=32000000 -sINITIAL_MEMORY=2048mb -O2" EXEEXT=".html"
-
-# SPLIT THE DATA FILE
-echo "Splitting gap.data..."
-split -b 75m gap.data "gap.data.part"
-
-# Rename to .part1, .part2...
-i=1
-for f in gap.data.part*; do
-    mv "$f" "gap.data.part$i"
-    echo "Created gap.data.part$i"
-    ((i++))
-done
-rm gap.data
+# which lets us produce a html page to run GAP in.
+emmake make -j8 LDFLAGS="-lidbfs.js -s ASYNCIFY=1 -sTOTAL_STACK=32mb -sASYNCIFY_STACK_SIZE=32000000 -sINITIAL_MEMORY=2048mb -O2" EXEEXT=".html"

etc/emscripten/build_startup_manifest.js

@@ -0,0 +1,71 @@
+#!/usr/bin/env node
+"use strict";
+
+const http = require('http');
+const fs = require('fs');
+const path = require('path');
+
+const PORT = 9999;
+const BASE_DIR = process.cwd(); 
+const LOG_FILE = path.join(BASE_DIR, 'startup_manifest.json');
+
+const requestedFiles = new Set();
+fs.writeFileSync(LOG_FILE, '[\n]');
+
+const server = http.createServer((req, res) => {
+    let urlPath = req.url.split('?')[0];
+    if (urlPath === '/') {
+        urlPath = '/index.html';
+    }
+
+    let localDiskPath = urlPath;
+    try {
+        localDiskPath = decodeURIComponent(urlPath);
+    } catch (e) {}
+
+    const filePath = path.join(BASE_DIR, localDiskPath);
+
+    fs.readFile(filePath, (err, data) => {
+        if (err) {
+            console.error(`[404] Ignored missing file: ${localDiskPath}`);
+            res.writeHead(404);
+            res.end(`404: File not found`);
+            return;
+        }
+
+        const ignored = ['/favicon.ico', '/index.html', '/startup_manifest.json', 'coi-serviceworker.js', 'gap-worker.js', 'gap-fs.js', 'gap.js', 'gap.wasm', 'gap-fs.json'];
+        if (!ignored.includes(localDiskPath)) {
+            let manifestPath = localDiskPath.startsWith('/') ? localDiskPath.substring(1) : localDiskPath;
+
+            if (!requestedFiles.has(manifestPath)) {
+                requestedFiles.add(manifestPath);
+
+                const manifest = Array.from(requestedFiles);
+                fs.writeFileSync(LOG_FILE, JSON.stringify(manifest, null, 4));
+
+                console.log(`[Loaded & Logged] ${manifestPath} (Total: ${requestedFiles.size})`);
+            }
+        }
+
+        let contentType = 'application/octet-stream';
+        if (urlPath.endsWith('.html')) contentType = 'text/html';
+        else if (urlPath.endsWith('.js')) contentType = 'text/javascript';
+        else if (urlPath.endsWith('.wasm')) contentType = 'application/wasm';
+        else if (urlPath.endsWith('.css')) contentType = 'text/css';
+
+        res.writeHead(200, {
+            'Content-Type': contentType,
+            'Cross-Origin-Opener-Policy': 'same-origin',
+            'Cross-Origin-Embedder-Policy': 'require-corp',
+            'Access-Control-Allow-Origin': '*'
+        });
+
+        res.end(data);
+    });
+});
+
+server.listen(PORT, '0.0.0.0', () => {
+    console.log(`\n Tracker running at http://localhost:${PORT}/`);
+    console.log(`Serving files from: ${BASE_DIR}`);
+    console.log(`Logging valid loaded files to: ${LOG_FILE}\n`);
+});

etc/emscripten/generate_gap_fs_json.py

@@ -0,0 +1,18 @@
+import sys
+import json
+import os
+
+def main():
+    paths = [line.strip() for line in sys.stdin if line.strip()]
+
+    try:
+        with open('gap-fs.json', 'w', encoding='utf-8') as f:
+            json.dump(paths, f, separators=(',', ':'))
+
+        print(f"Successfully wrote {len(paths)} files to gap-fs.json", file=sys.stderr)
+    except Exception as e:
+        print(f"Failed to write gap-fs.json: {e}", file=sys.stderr)
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

etc/emscripten/run-web-demo.sh

@@ -5,7 +5,9 @@ set -eux
 echo This script assumes you have already run 'build.sh'
 
 mkdir -p web-example
-cp etc/emscripten/web-template/* web-example
-cp gap.js gap.wasm gap.data.part* web-example
+cp etc/emscripten/web-template/* web-example/
+cp gap.js gap.wasm gap.worker.js gap-fs.json web-example/ 
+
+cp -r pkg lib grp tst doc hpcgap dev benchmark web-example/
 cd web-example
 ../etc/emscripten/server.rb

etc/emscripten/web-template/gap-fs.js

@@ -0,0 +1,97 @@
+self.Module = self.Module || {};
+self.Module.preRun = self.Module.preRun || [];
+
+self.Module.preRun.push(function() {
+    addRunDependency('gap_fs_init');
+
+    async function initFS() {
+        try {
+            const mapRes = await fetch('gap-fs.json');
+            const fileList = await mapRes.json();
+            const physicalDir = "";
+
+            var createdDirs = {};
+            fileList.forEach(function(appPath) {
+                var parts = appPath.split('/');
+                parts.pop();
+                var parentDir = '/' + parts.join('/');
+                if (!createdDirs[parentDir]) {
+                    try { FS.mkdirTree(parentDir); } catch(e) {}
+                    createdDirs[parentDir] = true;
+                }
+            });
+
+            try { FS.mkdirTree('/gap_idb_cache'); } catch(e) {}
+            FS.mount(IDBFS, {}, '/gap_idb_cache');
+
+            FS.syncfs(true, async function(err) {
+                fileList.forEach(function(appPath) {
+                    var parts = appPath.split('/');
+                    parts.pop();
+                    var parentDir = '/' + parts.join('/');
+                    try { FS.mkdirTree('/gap_idb_cache' + parentDir); } catch(e) {}
+                });
+
+                var needsSave = false;
+                var startupSet = new Set();
+
+                try {
+                    const manifestRes = await fetch('startup_manifest.json');
+                    if (manifestRes.ok) {
+                        const manifest = await manifestRes.json();
+                        manifest.forEach(p => {
+                            if (p.startsWith('/')) p = p.substring(1);
+                            startupSet.add(p);
+                        });
+                    }
+                } catch (e) {}
+
+                var fetchPromises = fileList.map(async function(appPath) {
+                    var fetchRelativePath = appPath.split('/').map(encodeURIComponent).join('/');
+                    var fetchPath = physicalDir + fetchRelativePath;
+
+                    var idbfsPath = '/gap_idb_cache/' + appPath;
+                    var finalAppPath = '/' + appPath;
+
+                    if (startupSet.has(appPath)) { 
+                        try {
+                            FS.stat(idbfsPath);
+                            FS.writeFile(finalAppPath, FS.readFile(idbfsPath));
+                        } catch (e) {
+                            try {
+                                const response = await fetch(fetchPath);
+                                if (response.ok) {
+                                    const buffer = await response.arrayBuffer();
+                                    const data = new Uint8Array(buffer);
+                                    FS.writeFile(finalAppPath, data);
+                                    FS.writeFile(idbfsPath, data);
+                                    needsSave = true;
+                                }
+                            } catch (fetchErr) {}
+                        }
+                    } else {
+                        var parts = appPath.split('/');
+                        var fileName = parts.pop();
+                        var parentDir = '/' + parts.join('/');
+                        FS.createLazyFile(parentDir, fileName, fetchPath, true, false);
+                    }
+                });
+
+                await Promise.all(fetchPromises);
+
+                if (needsSave) {
+                    FS.syncfs(false, function(saveErr) {
+                        removeRunDependency('gap_fs_init');
+                    });
+                } else {
+                    removeRunDependency('gap_fs_init');
+                }
+            });
+        } catch(err) {
+            console.error("Failed to initialize GAP FS:", err);
+            removeRunDependency('gap_fs_init');
+        }
+    }
+
+    initFS();
+});

etc/emscripten/web-template/gap-worker.js

@@ -1,69 +1,9 @@
 importScripts("https://cdn.jsdelivr.net/npm/xterm-pty@0.9.4/workerTools.js");
 
 onmessage = (msg) => {
-  // We wrap the initialization in an async function to handle the data fetching
-  async function loadAndStart() {
-    const buffers = [];
-    let i = 1;
-
-    // Download all split parts
-    // It will look for gap.data.part1, part2, etc. until it hits a 404.
-    while (true) {
-      const url = `gap.data.part${i}`;
-      try {
-        const response = await fetch(url);
-        if (!response.ok) break; // Stop when we hit 404
-
-        const buf = await response.arrayBuffer();
-        buffers.push(new Uint8Array(buf));
-        i++;
-      } catch (e) {
-        break;
-      }
-    }
-
-    // Prepare the Module object BEFORE importing gap.js
-    self.Module = self.Module || {};
-
-    // Merge data.
-    if (buffers.length > 0) {
-      const totalLength = buffers.reduce((acc, b) => acc + b.length, 0);
-      const mergedData = new Uint8Array(totalLength);
-      let offset = 0;
-      for (const buffer of buffers) {
-        mergedData.set(buffer, offset);
-        offset += buffer.length;
-      }
-
-      console.log(`Worker: Loaded ${buffers.length} parts. Total size: ${totalLength} bytes.`);
-
-      // Override the default downloader.
-      // When gap.js asks for 'gap.data', we give it our merged array immediately.
-      // This stops it from trying to fetch 'gap.data' via XHR.
-      self.Module.getPreloadedPackage = function(remotePackageName, remotePackageSize) {
-        if (remotePackageName === 'gap.data') {
-          return mergedData.buffer; 
-        }
-        return null; // Let other files download normally if any
-      };
-
-      // Just in case: also write it to FS in preRun.
-      self.Module.preRun = self.Module.preRun || [];
-      self.Module.preRun.push(() => {
-         try {
-
-             FS.writeFile('/gap.data', mergedData); 
-         } catch(e) { /* ignore if already handled by getPreloadedPackage */ }
-      });
-    } else {
-      console.warn("Worker: No gap.data parts found. The standard downloader will likely fail with 404.");
-    }
-
-    // Load GAP.
-    importScripts("gap.js");
-
-    emscriptenHack(new TtyClient(msg.data));
-  }
-
-  loadAndStart();
+  // Prepare the Module object BEFORE importing gap.js
+  self.Module = self.Module || {};
+  importScripts("gap-fs.js");
+  importScripts("gap.js");
+  emscriptenHack(new TtyClient(msg.data));
 };

etc/emscripten/web-template/index.html

@@ -1,6 +1,7 @@
 <html>
   <head>
     <title>gap-wasm</title>
+    <script src="coi-serviceworker.js"></script>
     <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/xterm@4.17.0/css/xterm.css">
   </head>
   <body>
@@ -17,9 +18,5 @@
       const worker = new Worker("gap-worker.js");
       new TtyServer(slave).start(worker);
     </script>
-
-    <!-- Hack to get around lack of SharedArrayBuffer on github pages -->
-    <script src="coi-serviceworker.js"></script>
   </body>
-</html>
-
+</html>