Add manifest cache to prevent memory explosion on large builds

src/image.js

@@ -14,6 +14,9 @@ import { generateHTML } from "./generate-html.js";
 
 import { DEFAULTS as GLOBAL_OPTIONS } from "./global-options.js";
 import { existsCache, memCache, diskCache } from "./caches.js";
+import ManifestCache from "./manifest-cache.js";
+
+let manifestCache = new ManifestCache();
 
 const debug = debugUtil("Eleventy:Image");
 const debugAssets = debugUtil("Eleventy:Assets");
@@ -382,7 +385,7 @@ export default class Image {
     let hashContents = [];
 
     if(existsCache.exists(this.src)) {
-      let fileContents = this.getFileContents();
+      let fileContents = fs.readFileSync(this.src);
 
       // If the file starts with whitespace or the '<' character, it might be SVG.
       // Otherwise, skip the expensive buffer.toString() call
@@ -842,6 +845,25 @@ export default class Image {
           return this.getStatsOnly();
         }
 
+        // For production local files, check manifest cache first
+        if (this.#canSkipBuffer()) {
+          let contentHash = this.getHash();
+          let optionsHash = this.#getOptionsHash();
+          let cacheKey = `${this.src}::${optionsHash}`;
+
+          let cached = manifestCache.get(cacheKey, contentHash);
+
+          if (cached && this.#outputFilesExist(cached)) {
+            return cached;
+          }
+
+          this.buildLogger.log(`Processing ${this.buildLogger.getFriendlyImageSource(this.src)}`, this.options);
+          let stats = await this.resize(this.src);
+          manifestCache.set(cacheKey, contentHash, stats);
+          return stats;
+        }
+
+        // Dev mode, dryRun and remote URLs need the buffer
         this.buildLogger.log(`Processing ${this.buildLogger.getFriendlyImageSource(this.src)}`, this.options);
 
         let input = await this.getInput();
@@ -924,5 +946,42 @@ export default class Image {
     let img = Image.create(src, opts);
     return img.statsByDimensionsSync(width, height);
   }
-}
 
+  #canSkipBuffer() {
+    return typeof this.src === "string"
+      && !this.isRemoteUrl 
+      && !this.options.dryRun 
+      && !this.options.statsOnly 
+      && !this.options.transformOnRequest
+      && !this.options.urlFormat
+      && !this.src.toLowerCase().endsWith(".svg");
+  }
+
+  #outputFilesExist(stats) {
+    for (let format of Object.keys(stats)) {
+      for (let stat of stats[format]) {
+        if (stat.outputPath && !diskCache.isCached(stat.outputPath, this.src)) {
+          return false;
+        }
+      }
+    }
+    return true;
+  }
+
+  #getOptionsHash() {
+    let relevant = {
+      widths: this.options.widths,
+      formats: this.options.formats,
+      sharpOptions: this.options.sharpOptions,
+      sharpWebpOptions: this.options.sharpWebpOptions,
+      sharpPngOptions: this.options.sharpPngOptions,
+      sharpJpegOptions: this.options.sharpJpegOptions,
+      sharpAvifOptions: this.options.sharpAvifOptions,
+    };
+    return createHashSync(JSON.stringify(relevant));
+  }
+
+  get hasLoadedBuffer() {
+    return this.#input !== undefined;
+  }
+}

src/manifest-cache.js

@@ -0,0 +1,63 @@
+import fs from "node:fs";
+import path from "node:path";
+
+export default class ManifestCache {
+  #manifest = {};
+  #cacheDir;
+  #filepath;
+  #loaded = false;
+
+  constructor(cacheDir = ".cache") {
+    this.#cacheDir = cacheDir;
+    this.#filepath = path.join(cacheDir, "eleventy-img-manifest.json");
+  }
+
+  load() {
+    if (this.#loaded) return;
+
+    try {
+      if (fs.existsSync(this.#filepath)) {
+        let content = fs.readFileSync(this.#filepath, "utf8");
+        this.#manifest = JSON.parse(content);
+      }
+    } catch {
+      // Corrupted or unreadable, start fresh
+      this.#manifest = {};
+    }
+    this.#loaded = true;
+  }
+
+  save() {
+    if (!fs.existsSync(this.#cacheDir)) {
+      fs.mkdirSync(this.#cacheDir, { recursive: true });
+    }
+    fs.writeFileSync(this.#filepath, JSON.stringify(this.#manifest, null, 2));
+  }
+
+  get(key, hash) {
+    this.load();
+
+    let entry = this.#manifest[key];
+    if (!entry) return null;
+    if (entry.hash !== hash) return null;
+
+    return entry.stats;
+  }
+
+  set(key, hash, stats) {
+    this.load();
+
+    // Strip buffers before storing
+    let cleanStats = {};
+    for (let format of Object.keys(stats)) {
+      cleanStats[format] = stats[format].map(stat => {
+        let copy = { ...stat };
+        delete copy.buffer;
+        return copy;
+      });
+    }
+
+    this.#manifest[key] = { hash, stats: cleanStats };
+    this.save();
+  }
+}

test/test.js

@@ -1277,3 +1277,24 @@ test("#105 Transparent format output filtering (no minimum transparency formats
   // must include one of: svg, png, or gif
   t.deepEqual(Object.keys(stats), ["webp", "jpeg"]);
 });
+
+import { memCache } from "../src/caches.js";
+
+test("#106 Production run should not load source buffer into memory", async t => {
+  // Use unique width to avoid cache collision with other tests
+  await eleventyImage("./test/bio-2017.jpg", {
+    widths: [347],
+    formats: ["jpeg"],
+    outputDir: "./test/img/",
+  });
+
+  // Verify the cached Image instance didn't load the source buffer
+  for (let key of Object.keys(memCache.cache)) {
+    let img = memCache.cache[key].results;
+    if (key.includes("347") && typeof img.hasLoadedBuffer !== "undefined") {
+      t.false(img.hasLoadedBuffer, "Source buffer should not be loaded for production local files");
+      return;
+    }
+  }
+  t.pass();
+});