-
-
Notifications
You must be signed in to change notification settings - Fork 1.7k
chore(nix): add bun node_modules normalization scripts #1256
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,104 @@ | ||
| /** | ||
| * Canonicalize bun's internal node_modules symlinks for reproducible FOD hashes. | ||
| * | ||
| * Bun's isolated install strategy stores packages in node_modules/.bun/ with | ||
| * versioned directory names (e.g. "packagename@1.2.3"). It then creates symlinks | ||
| * in node_modules/.bun/node_modules/ pointing to these directories. The order of | ||
| * symlink creation is non-deterministic, which breaks fixed-output derivation hashes. | ||
| * | ||
| * This script rebuilds all symlinks deterministically: sorted by name, picking the | ||
| * highest semver version when multiple versions of the same package exist. | ||
| * | ||
| * Adapted from opencode (https://github.com/anomalyco/opencode), MIT license. | ||
| */ | ||
|
|
||
| import { lstat, mkdir, readdir, rm, symlink } from "fs/promises"; | ||
| import { join, relative } from "path"; | ||
|
|
||
| type Entry = { | ||
| dir: string; | ||
| version: string; | ||
| }; | ||
|
|
||
| async function isDirectory(path: string) { | ||
| try { | ||
| const info = await lstat(path); | ||
| return info.isDirectory(); | ||
| } catch { | ||
| return false; | ||
| } | ||
| } | ||
|
|
||
| const isValidSemver = (v: string) => Bun.semver.satisfies(v, "x.x.x"); | ||
|
|
||
| function parseEntry(label: string) { | ||
| const marker = label.startsWith("@") | ||
| ? label.indexOf("@", 1) | ||
| : label.indexOf("@"); | ||
| if (marker <= 0) return null; | ||
| const name = label.slice(0, marker).replace(/\+/g, "/"); | ||
| const version = label.slice(marker + 1); | ||
| if (!name || !version) return null; | ||
| return { name, version }; | ||
| } | ||
|
|
||
| const root = process.cwd(); | ||
| const bunRoot = join(root, "node_modules/.bun"); | ||
| const linkRoot = join(bunRoot, "node_modules"); | ||
|
|
||
| if (!(await isDirectory(bunRoot))) { | ||
| console.log("[canonicalize-node-modules] no .bun directory, skipping"); | ||
| process.exit(0); | ||
| } | ||
|
|
||
| const directories = (await readdir(bunRoot)).sort(); | ||
| const versions = new Map<string, Entry[]>(); | ||
|
|
||
| for (const entry of directories) { | ||
| const full = join(bunRoot, entry); | ||
| if (!(await isDirectory(full))) continue; | ||
| const parsed = parseEntry(entry); | ||
| if (!parsed) continue; | ||
| const list = versions.get(parsed.name) ?? []; | ||
| list.push({ dir: full, version: parsed.version }); | ||
| versions.set(parsed.name, list); | ||
| } | ||
|
|
||
| const selections = new Map<string, Entry>(); | ||
|
|
||
| for (const [slug, list] of versions) { | ||
| list.sort((a, b) => { | ||
| const aValid = isValidSemver(a.version); | ||
| const bValid = isValidSemver(b.version); | ||
| if (aValid && bValid) return -Bun.semver.order(a.version, b.version); | ||
| if (aValid) return -1; | ||
| if (bValid) return 1; | ||
| return b.version.localeCompare(a.version); | ||
| }); | ||
| const first = list[0]; | ||
| if (first) selections.set(slug, first); | ||
| } | ||
|
|
||
| await rm(linkRoot, { recursive: true, force: true }); | ||
| await mkdir(linkRoot, { recursive: true }); | ||
|
|
||
| let count = 0; | ||
| for (const [slug, entry] of Array.from(selections.entries()).sort((a, b) => | ||
| a[0].localeCompare(b[0]), | ||
| )) { | ||
| const parts = slug.split("/"); | ||
| const leaf = parts.pop(); | ||
| if (!leaf) continue; | ||
| const parent = join(linkRoot, ...parts); | ||
| await mkdir(parent, { recursive: true }); | ||
| const linkPath = join(parent, leaf); | ||
| const desired = join(entry.dir, "node_modules", slug); | ||
| if (!(await isDirectory(desired))) continue; | ||
| const relativeTarget = relative(parent, desired); | ||
| const resolved = relativeTarget.length === 0 ? "." : relativeTarget; | ||
| await rm(linkPath, { recursive: true, force: true }); | ||
| await symlink(resolved, linkPath); | ||
| count++; | ||
| } | ||
|
|
||
| console.log(`[canonicalize-node-modules] rebuilt ${count} links`); | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,133 @@ | ||
| /** | ||
| * Normalize .bin symlinks inside bun's internal module directories. | ||
| * | ||
| * Bun may create .bin/ symlinks in non-deterministic order within each | ||
| * node_modules/.bun/<pkg>@<ver>/node_modules/.bin/ directory. This script | ||
| * rebuilds them deterministically by reading package.json "bin" fields and | ||
| * recreating symlinks in sorted order. | ||
| * | ||
| * Adapted from opencode (https://github.com/anomalyco/opencode), MIT license. | ||
| */ | ||
|
|
||
| import { lstat, mkdir, readdir, rm, symlink } from "fs/promises"; | ||
| import { join, relative } from "path"; | ||
|
|
||
| type PackageManifest = { | ||
| name?: string; | ||
| bin?: string | Record<string, string>; | ||
| }; | ||
|
|
||
| async function exists(path: string) { | ||
| try { | ||
| await lstat(path); | ||
| return true; | ||
| } catch { | ||
| return false; | ||
| } | ||
| } | ||
|
|
||
| async function isDirectory(path: string) { | ||
| try { | ||
| const info = await lstat(path); | ||
| return info.isDirectory(); | ||
| } catch { | ||
| return false; | ||
| } | ||
| } | ||
|
|
||
| async function readManifest(dir: string) { | ||
| const file = Bun.file(join(dir, "package.json")); | ||
| if (!(await file.exists())) return null; | ||
| return (await file.json()) as PackageManifest; | ||
| } | ||
|
|
||
| async function collectPackages(modulesRoot: string) { | ||
| const found: string[] = []; | ||
| const topLevel = (await readdir(modulesRoot)).sort(); | ||
| for (const name of topLevel) { | ||
| if (name === ".bin" || name === ".bun") continue; | ||
| const full = join(modulesRoot, name); | ||
| if (!(await isDirectory(full))) continue; | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think this is skipping most packages in an isolated Bun install.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Confirmed — reproduced 86 → 27 on the current lockfile (losing |
||
| if (name.startsWith("@")) { | ||
| const scoped = (await readdir(full)).sort(); | ||
| for (const child of scoped) { | ||
| const scopedDir = join(full, child); | ||
| if (await isDirectory(scopedDir)) found.push(scopedDir); | ||
| } | ||
| continue; | ||
| } | ||
| found.push(full); | ||
| } | ||
| return found.sort(); | ||
| } | ||
|
|
||
| function normalizeBinName(name: string) { | ||
| const slash = name.lastIndexOf("/"); | ||
| return slash >= 0 ? name.slice(slash + 1) : name; | ||
| } | ||
|
|
||
| const root = process.cwd(); | ||
| const bunRoot = join(root, "node_modules/.bun"); | ||
|
|
||
| if (!(await isDirectory(bunRoot))) { | ||
| console.log("[normalize-bun-binaries] no .bun directory, skipping"); | ||
| process.exit(0); | ||
| } | ||
|
|
||
| const bunEntries = (await readdir(bunRoot)).sort(); | ||
| let rewritten = 0; | ||
|
|
||
| for (const entry of bunEntries) { | ||
| const modulesRoot = join(bunRoot, entry, "node_modules"); | ||
| if (!(await exists(modulesRoot))) continue; | ||
|
|
||
| const binRoot = join(modulesRoot, ".bin"); | ||
| await rm(binRoot, { recursive: true, force: true }); | ||
| await mkdir(binRoot, { recursive: true }); | ||
|
|
||
| const packageDirs = await collectPackages(modulesRoot); | ||
| for (const packageDir of packageDirs) { | ||
| const manifest = await readManifest(packageDir); | ||
| if (!manifest?.bin) continue; | ||
|
|
||
| const seen = new Set<string>(); | ||
| const binField = manifest.bin; | ||
|
|
||
| if (typeof binField === "string") { | ||
| const fallback = manifest.name ?? packageDir.split("/").pop(); | ||
| if (fallback) { | ||
| const normalizedName = normalizeBinName(fallback); | ||
| if (!seen.has(normalizedName)) { | ||
| const resolved = join(packageDir, binField); | ||
| if (await exists(resolved)) { | ||
| const destination = join(binRoot, normalizedName); | ||
| const relativeTarget = relative(binRoot, resolved) || "."; | ||
| await rm(destination, { force: true }); | ||
| await symlink(relativeTarget, destination); | ||
| seen.add(normalizedName); | ||
| rewritten++; | ||
| } | ||
| } | ||
| } | ||
| } else { | ||
| const entries = Object.entries(binField).sort((a, b) => | ||
| a[0].localeCompare(b[0]), | ||
| ); | ||
| for (const [name, target] of entries) { | ||
| if (!name || !target) continue; | ||
| const normalizedName = normalizeBinName(name); | ||
| if (seen.has(normalizedName)) continue; | ||
| const resolved = join(packageDir, target); | ||
| if (!(await exists(resolved))) continue; | ||
| const destination = join(binRoot, normalizedName); | ||
| const relativeTarget = relative(binRoot, resolved) || "."; | ||
| await rm(destination, { force: true }); | ||
| await symlink(relativeTarget, destination); | ||
| seen.add(normalizedName); | ||
| rewritten++; | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| console.log(`[normalize-bun-binaries] rebuilt ${rewritten} links`); | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I think this goes a bit further than normalization. This loop doesn't preserve the targets Bun picked; it re-selects them by highest semver / string order. When I tried it against Handy's current
bun.lock, it rewired entries undernode_modules/.bun/node_moduleslikeminimatch(3.1.2 -> 9.0.5) andbrace-expansion(1.1.12 -> 2.0.2). Since that directory is still on the upward resolution path, that can change what actually gets resolved at runtime. Could we make the link creation deterministic without changing which target each slug points to?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Confirmed — reproduces exactly on a fresh
bun install --linker=isolated(minimatch 3.1.2 → 9.0.5,brace-expansion 1.1.12 → 2.0.2). Fixed in 4236f7b: the script now reads the existing symlinks in.bun/node_modules/, removes them, and recreates them in sorted order with bun's exact targets — no re-selection.