diff --git a/.nix/scripts/canonicalize-node-modules.ts b/.nix/scripts/canonicalize-node-modules.ts new file mode 100644 index 000000000..323569675 --- /dev/null +++ b/.nix/scripts/canonicalize-node-modules.ts @@ -0,0 +1,97 @@ +/** + * Canonicalize bun's internal node_modules symlinks for reproducible FOD hashes. + * + * Isolated-install layout produced by `bun install --linker=isolated`: + * + * node_modules/ + * ├── react → .bun/react@18.3.1/node_modules/react (symlink) + * ├── minimatch → .bun/minimatch@3.1.2/node_modules/minimatch (symlink) + * └── .bun/ + * ├── react@18.3.1/ + * │ └── node_modules/ + * │ └── react/ ← real package content + * ├── minimatch@3.1.2/ + * │ └── node_modules/ + * │ └── minimatch/ ← real package content + * └── node_modules/ ← target of this script + * ├── react → ../react@18.3.1/node_modules/react + * ├── minimatch → ../minimatch@3.1.2/node_modules/minimatch + * └── @babel/ + * └── core → ../../@babel+core@7.28.5+…/node_modules/@babel/core + * + * Real package content lives in .bun/@/node_modules//. + * The .bun/node_modules/ directory (linkRoot) holds only symlinks — it acts + * as a fallback upward-resolution path for packages inside .bun/. + * + * Bun's creation order for those symlinks is not guaranteed to be stable + * across hosts or filesystems, which can break fixed-output derivation hashes. + * This script reads the existing symlinks, removes them, and recreates them + * in lexicographic order while preserving the exact targets bun picked. + */ + +import { lstat, mkdir, readdir, readlink, rm, symlink } from "fs/promises"; +import { join } from "path"; + +type LinkEntry = { + slug: string; + target: string; +}; + +async function isDirectory(path: string) { + try { + const info = await lstat(path); + return info.isDirectory(); + } catch { + return false; + } +} + +async function collectLinks(dir: string, prefix: string): Promise { + const result: LinkEntry[] = []; + const names = await readdir(dir); + for (const name of names) { + const full = join(dir, name); + const info = await lstat(full); + if (info.isSymbolicLink()) { + const target = await readlink(full); + const slug = prefix ? `${prefix}/${name}` : name; + result.push({ slug, target }); + } else if (info.isDirectory() && !prefix && name.startsWith("@")) { + result.push(...(await collectLinks(full, name))); + } + } + return result; +} + +export async function canonicalizeNodeModules(): Promise { + const root = process.cwd(); + const linkRoot = join(root, "node_modules/.bun/node_modules"); + + if (!(await isDirectory(linkRoot))) { + console.log( + "[canonicalize-node-modules] no .bun/node_modules directory, skipping", + ); + return; + } + + const entries = await collectLinks(linkRoot, ""); + entries.sort((a, b) => a.slug.localeCompare(b.slug)); + + await rm(linkRoot, { recursive: true, force: true }); + await mkdir(linkRoot, { recursive: true }); + + for (const { slug, target } of entries) { + const parts = slug.split("/"); + const leaf = parts.pop(); + if (!leaf) continue; + const parent = join(linkRoot, ...parts); + await mkdir(parent, { recursive: true }); + await symlink(target, join(parent, leaf)); + } + + console.log(`[canonicalize-node-modules] rebuilt ${entries.length} links`); +} + +if (import.meta.main) { + await canonicalizeNodeModules(); +} diff --git a/.nix/scripts/heal-peer-dep-bins.ts b/.nix/scripts/heal-peer-dep-bins.ts new file mode 100644 index 000000000..80eb8888b --- /dev/null +++ b/.nix/scripts/heal-peer-dep-bins.ts @@ -0,0 +1,229 @@ +/** + * Heal missing `.bin/` symlinks produced by bun's isolated installer. + * + * ## The bug + * + * Bun's `--linker=isolated` installer creates `.bin/` symlinks inside + * each package's private node_modules/.bin/ for every dependency that has a + * `bin` field in its manifest — regular, optional, AND peer dependencies all + * go through the same code path (`Installer.zig::linkDependencyBins`), and + * the decision to link is made purely on whether the source file exists on + * disk at the moment the linker looks (`bin.zig`: + * + * if (!bun.sys.exists(abs_target)) { + * this.skipped_due_to_missing_bin = true; + * return; + * } + * + * ). For most dependencies the installer blocks the consuming package on the + * provider via `isTaskBlocked`, so by the time `linkDependencyBins` runs for + * package A the provider's file is guaranteed to be in place. But for + * circular peer dependency pairs — A declares B as a peer, B (transitively) + * depends on A — that blocking would deadlock, so bun's `Store.isCycle` + * detector explicitly bypasses it and lets both sides run in parallel. + * + * The consequence is a plain timing race between two worker threads. Which + * side wins depends on anything that shifts the relative scheduling of the + * two workers — CPU load, thread-pool size, filesystem write latency and + * caching, the kernel scheduler, NICE / cgroup limits — so the same bun + * version with the same bun.lock and the same install flags can produce + * different `.bin/` sets not just between different hosts but in principle + * between two consecutive runs on the same host. In practice we have + * observed divergence between a local NixOS sandbox, a GitHub Actions + * ubuntu-latest runner, and a GitHub Actions macos-latest runner, which is + * enough to break any single-hash FOD. + * + * Concretely, the Handy install hits this with + * - update-browserslist-db/.bin/browserslist (update-browserslist-db + * declares browserslist as a peer, browserslist has + * update-browserslist-db in its regular dependencies → cycle), and + * - @eslint-community/eslint-utils/.bin/eslint (eslint has eslint-utils + * as a regular dep, eslint-utils declares eslint as a peer → cycle). + * + * There is no bun configuration flag or env var that makes the outcome + * deterministic, and no upstream issue yet tracks this specific symptom + * (oven-sh/bun#28147 is the closest family match, different project). See + * the header of `canonicalize-node-modules.ts` for our sibling normalization + * pass that rebuilds `.bun/node_modules/` in sorted order. + * + * ## The fix this script applies + * + * For every package under `node_modules/.bun/` we walk its declared + * `peerDependencies`, find each resolved peer inside the package's private + * `node_modules/`, and create any `.bin/ → ..//` + * symlinks that bun's installer "intended" to create but may have skipped. + * Entries that already exist are left alone (the script is idempotent). + * + * This is the "fix by adding" approach — we produce the complete `.bin/` + * set that bun would have produced without the race, rather than stripping + * the inconsistent subset. Advantages: + * + * - Matches bun's intended behavior; if bun ever fixes the race upstream, + * this script becomes a no-op (every entry it would add already exists) + * and the FOD hash is unchanged. + * - Preserves `.bin/` entries that real code might depend on. We don't + * rely on the (true but brittle) argument that peer-dep `.bin/` entries + * are dead code in Tauri apps. + * - Easy to explain in review: we're patching a known upstream race bug + * with the exact output the upstream code is trying to produce. + */ + +import { lstat, mkdir, readdir, readlink, symlink } from "fs/promises"; +import { join } from "path"; + +type Manifest = { + name?: string; + bin?: string | Record; + peerDependencies?: Record; +}; + +async function isDirectory(path: string) { + try { + const info = await lstat(path); + return info.isDirectory(); + } catch { + return false; + } +} + +async function readManifest(path: string): Promise { + const file = Bun.file(path); + if (!(await file.exists())) return null; + return (await file.json()) as Manifest; +} + +// Parse a .bun entry directory name (e.g. "@babel+core@7.28.5+a1c3dd1b9adf390b") +// back into an npm package name ("@babel/core"). Returns null for entries that +// do not look like @[+]. +function parsePkgName(bunEntry: string): string | null { + const at = bunEntry.startsWith("@") + ? bunEntry.indexOf("@", 1) + : bunEntry.indexOf("@"); + if (at <= 0) return null; + return bunEntry.slice(0, at).replace(/\+/g, "/"); +} + +// Unscoped name used as the default bin name when `bin` is a bare string. +// For "@scope/pkg" returns "pkg"; for "pkg" returns "pkg". +function defaultBinName(pkgName: string): string { + const slash = pkgName.lastIndexOf("/"); + return slash >= 0 ? pkgName.slice(slash + 1) : pkgName; +} + +type BinSpec = { name: string; path: string }; + +function parseBinField(pkgName: string, binField: Manifest["bin"]): BinSpec[] { + if (!binField) return []; + if (typeof binField === "string") { + return [{ name: defaultBinName(pkgName), path: binField }]; + } + return Object.entries(binField).map(([name, path]) => ({ + name: defaultBinName(name), + path, + })); +} + +// Produce the relative symlink target that bun itself uses for a .bin entry +// sitting inside `.bun//node_modules/.bin/`, pointing to a file +// under `.bun//node_modules//...`. +function binTarget(peerName: string, binPath: string): string { + const clean = binPath.replace(/^\.\//, ""); + return `../${peerName}/${clean}`; +} + +type HealedEntry = { + containingEntry: string; + containingPkg: string; + peerName: string; + binName: string; + target: string; +}; + +export async function healPeerDepBins(): Promise { + const root = process.cwd(); + const bunRoot = join(root, "node_modules/.bun"); + + if (!(await isDirectory(bunRoot))) { + console.log("[heal-peer-dep-bins] no .bun directory, skipping"); + return; + } + + const bunEntries = (await readdir(bunRoot)).sort(); + const healed: HealedEntry[] = []; + + for (const entry of bunEntries) { + const pkgName = parsePkgName(entry); + if (!pkgName) continue; + const containingNodeModules = join(bunRoot, entry, "node_modules"); + if (!(await isDirectory(containingNodeModules))) continue; + + const manifest = await readManifest( + join(containingNodeModules, pkgName, "package.json"), + ); + if (!manifest) continue; + + const peers = Object.keys(manifest.peerDependencies ?? {}); + if (peers.length === 0) continue; + + const binRoot = join(containingNodeModules, ".bin"); + + for (const peerName of peers) { + // Peer may be optional and unresolved, or may not even be a real package + // directory in this install layout (e.g. bundled peer). Skip anything we + // cannot verify as "the peer's package.json is reachable from here". + const peerManifest = await readManifest( + join(containingNodeModules, peerName, "package.json"), + ); + if (!peerManifest) continue; + + const bins = parseBinField(peerName, peerManifest.bin); + if (bins.length === 0) continue; + + // Ensure the bin directory exists (it may be absent entirely if bun's + // race skipped every link it would have created for this package). + await mkdir(binRoot, { recursive: true }); + + for (const bin of bins) { + const linkPath = join(binRoot, bin.name); + const target = binTarget(peerName, bin.path); + + // Idempotent: skip if anything already occupies this path. We do not + // overwrite entries bun created; if bun already wrote a .bin/, + // that is either the correct target (race won) or a close variant we + // should not second-guess. + try { + await lstat(linkPath); + continue; + } catch { + // Does not exist — fall through to create. + } + + await symlink(target, linkPath); + healed.push({ + containingEntry: entry, + containingPkg: pkgName, + peerName, + binName: bin.name, + target, + }); + } + } + } + + if (healed.length > 0) { + console.log( + `[heal-peer-dep-bins] healed ${healed.length} missing peer .bin entries:`, + ); + for (const h of healed) { + console.log( + ` ${h.containingEntry}/node_modules/.bin/${h.binName} → ${h.target} (peer ${h.peerName} of ${h.containingPkg})`, + ); + } + } else { + console.log("[heal-peer-dep-bins] nothing to heal"); + } +} + +if (import.meta.main) { + await healPeerDepBins(); +} diff --git a/.nix/scripts/normalize-bun-binaries.ts b/.nix/scripts/normalize-bun-binaries.ts new file mode 100644 index 000000000..a5c2daf98 --- /dev/null +++ b/.nix/scripts/normalize-bun-binaries.ts @@ -0,0 +1,102 @@ +/** + * Normalize .bin symlinks inside bun's internal module directories. + * + * In an isolated install, every package under .bun/ gets its own private + * node_modules/ with a .bin/ directory holding symlinks to its dependencies' + * executables: + * + * node_modules/.bun/ + * ├── @vitejs+plugin-react@4.7.0+…/ + * │ └── node_modules/ + * │ ├── vite → ../../vite@6.4.1+…/node_modules/vite (peer symlink) + * │ └── .bin/ ← target of this script + * │ └── vite → ../vite/bin/vite.js + * ├── eslint@9.39.1+…/ + * │ └── node_modules/ + * │ └── .bin/ + * │ └── eslint → ../eslint/bin/eslint.js + * └── vite@6.4.1+…/ + * └── node_modules/ + * └── vite/ ← real package content + * └── bin/vite.js + * + * Real executables live in .bun/@/node_modules//…; every .bin/ + * entry is just a relative symlink reached through the peer symlinks in the + * same node_modules/. + * + * Bun's creation order for those .bin/ symlinks is not guaranteed to be stable + * across hosts or filesystems, which can break fixed-output derivation hashes. + * This script reads the .bin/ symlinks bun produced, removes them, and + * recreates them in lexicographic order while preserving the exact targets + * bun picked. + */ + +import { lstat, mkdir, readdir, readlink, rm, symlink } from "fs/promises"; +import { join } from "path"; + +type BinEntry = { + name: string; + target: string; +}; + +async function isDirectory(path: string) { + try { + const info = await lstat(path); + return info.isDirectory(); + } catch { + return false; + } +} + +async function collectBinLinks(binRoot: string): Promise { + const entries: BinEntry[] = []; + let names: string[]; + try { + names = await readdir(binRoot); + } catch { + return entries; + } + for (const name of names) { + const full = join(binRoot, name); + const info = await lstat(full); + if (!info.isSymbolicLink()) continue; + entries.push({ name, target: await readlink(full) }); + } + return entries; +} + +export async function normalizeBunBinaries(): Promise { + const root = process.cwd(); + const bunRoot = join(root, "node_modules/.bun"); + + if (!(await isDirectory(bunRoot))) { + console.log("[normalize-bun-binaries] no .bun directory, skipping"); + return; + } + + const bunEntries = (await readdir(bunRoot)).sort(); + let rewritten = 0; + + for (const entry of bunEntries) { + const binRoot = join(bunRoot, entry, "node_modules", ".bin"); + if (!(await isDirectory(binRoot))) continue; + + const bins = await collectBinLinks(binRoot); + if (bins.length === 0) continue; + bins.sort((a, b) => a.name.localeCompare(b.name)); + + await rm(binRoot, { recursive: true, force: true }); + await mkdir(binRoot, { recursive: true }); + + for (const { name, target } of bins) { + await symlink(target, join(binRoot, name)); + rewritten++; + } + } + + console.log(`[normalize-bun-binaries] rebuilt ${rewritten} links`); +} + +if (import.meta.main) { + await normalizeBunBinaries(); +} diff --git a/.nix/scripts/normalize-install.ts b/.nix/scripts/normalize-install.ts new file mode 100644 index 000000000..ec8647ba6 --- /dev/null +++ b/.nix/scripts/normalize-install.ts @@ -0,0 +1,36 @@ +/** + * Make `bun install --linker=isolated` output bit-reproducible. + * + * Entry point for the nixpkgs FOD build — invoke this single script after + * `bun install` to get a `node_modules/` tree that is byte-identical + * across machines and runs with the same bun.lock. + * + * `bun install --linker=isolated` is not bit-reproducible out of the box, + * even with a frozen lockfile, a pinned bun version, `--ignore-scripts`, + * and a clean `BUN_INSTALL_CACHE_DIR`. Running it across a local NixOS + * sandbox, a GitHub Actions ubuntu-latest runner, and a GitHub Actions + * macos-latest runner produces subtly different `node_modules/` trees + * from identical inputs. Two independent sources of drift show up: + * + * - Missing `.bin/` entries around circular peer dependencies, + * caused by a timing race inside bun's isolated installer thread + * pool. See `heal-peer-dep-bins.ts` for the full explanation and fix. + * + * - Non-deterministic symlink creation order in `.bun/node_modules/` + * and in each per-package `.bin/` directory. NAR hashing sorts + * entries during serialization, so this is usually harmless, but we + * defensively rebuild both trees in canonical sorted order. + * See `canonicalize-node-modules.ts` and `normalize-bun-binaries.ts`. + * + * Each sub-script is also runnable standalone for debugging: + * + * bun --bun .nix/scripts/.ts + */ + +import { canonicalizeNodeModules } from "./canonicalize-node-modules.ts"; +import { healPeerDepBins } from "./heal-peer-dep-bins.ts"; +import { normalizeBunBinaries } from "./normalize-bun-binaries.ts"; + +await canonicalizeNodeModules(); +await healPeerDepBins(); +await normalizeBunBinaries();