Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
ddb3d47
feat(nix): replace monolithic pnpm.fetchDeps with per-package CAFS la…
lietblue Mar 24, 2026
3e5bb34
fix(nix): use rm -rf to clean temp dir in cafs-add to handle read-onl…
lietblue Mar 24, 2026
baa58ae
fix(nix): chmod -R u+rwX before rm to handle 0555 tarball directories
lietblue Mar 24, 2026
653a878
fix(nix): extract tarballs with --no-same-permissions to prevent mode…
lietblue Mar 24, 2026
f1d6842
fix(nix): chmod -R u+rwX after extraction to restore write bits strip…
lietblue Mar 24, 2026
f711ad7
fix(nix): two-pass extraction to handle tarballs with non-executable …
lietblue Mar 24, 2026
c0d20f4
perf(nix): stream-hash files in cafs-add to avoid loading entire file…
lietblue Mar 24, 2026
e75787c
fix(nix): loop tar+chmod until success to handle nested no-execute di…
lietblue Mar 24, 2026
efa2cc7
fix(nix): fix three merge-phase failures in pnpm store build
lietblue Apr 7, 2026
393f079
fix(nix): align CAFS output with pnpm v10 store format
lietblue Apr 7, 2026
886179a
fix(nix): add trailing newline to pkgStoresList to ensure last packag…
lietblue Apr 7, 2026
55f867b
docs(nix): document pnpm CAFS store merging and trailing newline issue
lietblue Apr 7, 2026
987b312
fix(nix): set execute permissions on CAFS -exec files in pnpmDeps
lietblue Apr 7, 2026
7e0306c
fix: move chmod to buildPhase to ensure -exec files get execute permi…
lietblue Apr 7, 2026
7e96b7a
chore: update nixpkgs
lietblue Apr 7, 2026
ad0f58d
fix(nix): align nix:gen pnpm-store with CAFS v10 merge layout
lietblue Apr 7, 2026
d7ffc42
chore(nix): remove nix.md
lietblue Apr 7, 2026
d209c32
refactor(nix): replace custom CAFS with @pnpm/store.cafs, output JSON
lietblue Apr 8, 2026
85a3f98
refactor(nix): use @pnpm/constants for STORE_VERSION, remove pnpm fro…
lietblue Apr 8, 2026
0f17442
refactor(nix): remove bundled .mjs, run .ts directly with node_modules
lietblue Apr 8, 2026
dd6b1c2
fix(nix): replace deprecated pnpm.configHook with pnpmConfigHook
lietblue Apr 8, 2026
f902008
fix(nix): include pnpm binary where pnpmConfigHook runs
lietblue Apr 8, 2026
856cfb8
refactor(nix): replace generated JSON with IFD from pnpm-lock.yaml
lietblue Apr 13, 2026
3364ab3
refactor(nix): derive cafsScript deps from lockfile graph, drop hardc…
lietblue Apr 15, 2026
64e8c62
[autofix.ci] apply automated fixes
autofix-ci[bot] Apr 24, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 0 additions & 45 deletions .github/workflows/update-nix-pnpm-deps-hash.yaml

This file was deleted.

6 changes: 3 additions & 3 deletions flake.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 6 additions & 6 deletions nix/common.nix
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@
lib,
stdenvNoCC,

callPackage,

pnpmConfigHook,
pnpm,

cacert,
Expand All @@ -24,11 +27,7 @@ stdenvNoCC.mkDerivation (final: {
!isEditorMetadataDirectory;
};

pnpmDeps = pnpm.fetchDeps {
inherit (final) pname version src;
fetcherVersion = 2;
hash = builtins.readFile ./pnpm-deps-hash.txt;
};
pnpmDeps = callPackage ./pnpm-store.nix { };

# Cache of assets downloaded during vite build
assets = stdenvNoCC.mkDerivation {
Expand All @@ -39,7 +38,8 @@ stdenvNoCC.mkDerivation (final: {
cacert # For network request
gitMinimal # For unplugin-info
nodejs
pnpm.configHook
pnpm
pnpmConfigHook
];

buildPhase = ''
Expand Down
4 changes: 3 additions & 1 deletion nix/package.nix
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
makeWrapper,
gitMinimal,
pnpm,
pnpmConfigHook,

asar,
electron,
Expand All @@ -20,7 +21,8 @@
makeWrapper
gitMinimal
nodejs
pnpm.configHook
pnpm
pnpmConfigHook
];

desktopItems = [
Expand Down
1 change: 0 additions & 1 deletion nix/pnpm-deps-hash.txt

This file was deleted.

207 changes: 207 additions & 0 deletions nix/pnpm-store.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,207 @@
# Hand-written Nix derivation for per-package pnpm CAFS store.
#
# Architecture:
# 1. IFD: convert pnpm-lock.yaml → JSON via yj (tiny, cached after first eval)
# 2. Pure Nix: extract package metadata (name, version, url, integrity)
# 3. cafsScript: bundles the .ts source with its node_modules (from lockfile graph)
# 4. makePkgStore: per-package derivation that runs the .ts script via Node
# 5. Merge derivation: combines all fragments with cp -rn
#
# All pnpm internal format knowledge is in pnpm-cafs-add.ts (via @pnpm/store.cafs
# and @pnpm/constants). This Nix file is completely format-agnostic.
#
# NOTICE: This file uses IFD (Import From Derivation) to parse pnpm-lock.yaml
# at evaluation time. nix build allows IFD by default; --pure-eval (used by
# nix flake check) disables it — override with --allow-import-from-derivation
# or allow-import-from-derivation = true in nix.conf.
{
stdenvNoCC,
lib,
fetchurl,
nodejs,
runCommand,
yj,
}:
Comment thread
Weathercold marked this conversation as resolved.
let
# ---------------------------------------------------------------------------
# IFD: parse pnpm-lock.yaml → JSON (sub-second, cached by Nix store)
# ---------------------------------------------------------------------------
lockfileJson = runCommand "pnpm-lock-json" {
nativeBuildInputs = [ yj ];
} ''
yj < ${../pnpm-lock.yaml} > $out
'';
lockfile = builtins.fromJSON (builtins.readFile lockfileJson);

# ---------------------------------------------------------------------------
# Pure Nix: extract package metadata from lockfile
# ---------------------------------------------------------------------------

# Parse "name@version" key — last @ is the separator (handles @scope/name@version)
parseKey = key:
let
# builtins.match returns null on no match, or a list of capture groups
# "@scope/name@1.0.0" → ["@scope/name" "1.0.0"]
# "lodash@4.17.21" → ["lodash" "4.17.21"]
m = builtins.match "(.+)@([^@]+)" key;
in {
name = builtins.elemAt m 0;
version = builtins.elemAt m 1;
};

# Compute npm registry tarball URL from package name and version.
# Scoped: @scope/name → registry.npmjs.org/@scope/name/-/name-version.tgz
# Unscoped: name → registry.npmjs.org/name/-/name-version.tgz
npmTarballUrl = name: version:
let
hasScope = lib.hasInfix "/" name;
unscopedName = if hasScope
then lib.last (lib.splitString "/" name)
else name;
in "https://registry.npmjs.org/${name}/-/${unscopedName}-${version}.tgz";

# Build package entry from lockfile key + entry, or null if skipped
mkPkgEntry = key: entry:
let
parsed = parseKey key;
integrity = entry.resolution.integrity or null;
url = entry.resolution.tarball or (npmTarballUrl parsed.name parsed.version);
in
if (entry.bundled or false) || integrity == null
then null
else {
inherit (parsed) name version;
inherit url integrity;
};

# Extract all non-null package entries from lockfile
rawEntries = lib.mapAttrs mkPkgEntry (lockfile.packages or {});
Comment thread
lietblue marked this conversation as resolved.
packages = lib.filterAttrs (_: v: v != null) rawEntries;

# ---------------------------------------------------------------------------
# Runtime dependencies of nix/scripts/pnpm-cafs-add.ts, derived automatically
# from the lockfile. The nix/scripts workspace declares @pnpm/constants and
# @pnpm/store.cafs as direct deps; this section walks the lockfile's snapshot
# graph to compute the full transitive closure. No manual version list needed.
# ---------------------------------------------------------------------------
snapshots = lockfile.snapshots or {};

# Read the nix/scripts workspace's resolved dependencies from the lockfile
scriptImporter = lockfile.importers."nix/scripts" or {};
scriptDirectDeps =
lib.mapAttrsToList (name: entry: "${name}@${entry.version}")
(scriptImporter.dependencies or {});

# Walk the snapshot dependency graph to collect the transitive closure.
# Each snapshot entry maps a package key to its runtime dependencies.
# Returns an attrset used as a visited set (keys = "name@version").
depClosure = visited: queue:
if queue == [] then visited
else
let
key = builtins.head queue;
rest = builtins.tail queue;
in
if visited ? ${key} then depClosure visited rest
else
let
snap = snapshots.${key} or {};
newDeps = lib.mapAttrsToList (n: v: "${n}@${v}") (snap.dependencies or {});
in
depClosure (visited // { ${key} = true; }) (rest ++ newDeps);

# Full transitive closure, filtered to packages that exist in the lockfile
# (skips @types/* and other declaration-only packages without tarballs).
cafsScriptDeps =
builtins.filter (key: packages ? ${key})
(builtins.attrNames (depClosure {} scriptDirectDeps));

# Fetch a package tarball by its key
fetchPkg = key:
let pkg = packages.${key};
in fetchurl { url = pkg.url; hash = pkg.integrity; };

# Build a node_modules directory with the CAFS script's runtime dependencies.
# Extracts each transitive dep into a flat node_modules tree.
cafsScript = stdenvNoCC.mkDerivation {
name = "pnpm-cafs-add";
dontUnpack = true;
dontConfigure = true;
dontInstall = true;
dontFixup = true;
buildPhase = ''
# Extract each runtime dependency into a flat node_modules
${lib.concatStringsSep "\n" (map (key:
let
pkg = packages.${key};
modPath = pkg.name;
parentDir =
if lib.hasPrefix "@" modPath
then "$out/node_modules/${lib.head (lib.splitString "/" modPath)}"
else "$out/node_modules";
in ''
mkdir -p "${parentDir}"
mkdir -p "$out/node_modules/${modPath}"
tar xzf ${fetchPkg key} --strip-components=1 -C "$out/node_modules/${modPath}"
''
) cafsScriptDeps)}
cp ${./scripts/pnpm-cafs-add.ts} $out/pnpm-cafs-add.ts
'';
};

# Each package gets its own derivation: fetch tarball → write CAFS fragment.
# Runs the .ts source directly via Node's built-in TypeScript stripping.
makePkgStore = key: pkg:
let
drv = fetchurl {
url = pkg.url;
hash = pkg.integrity;
};
in
stdenvNoCC.mkDerivation {
name = "airi-pnpm-pkg-${lib.replaceStrings [ "@" "/" ] [ "" "-" ] key}";
nativeBuildInputs = [ nodejs ];
Comment thread
Weathercold marked this conversation as resolved.
buildPhase = ''
node --experimental-strip-types \
${cafsScript}/pnpm-cafs-add.ts \
"${drv}" \
"$out" \
"${key}" \
"${pkg.integrity}"
'';
dontConfigure = true;
dontUnpack = true;
dontInstall = true;
dontFixup = true;
};

pkgStores = lib.mapAttrs makePkgStore packages;

in
# Merge all per-package CAFS fragments into one complete pnpm store.
# CAFS is content-addressed: cp -rn is safe (same hash = same file, no conflicts).
#
# NOTICE: pkgStoresList is passed via passAsFile instead of being interpolated
# directly into buildPhase. With ~3000 packages each path is ~50 chars, the
# concatenated string exceeds ARG_MAX (~2 MB) when bash is invoked with all
# environment variables, causing E2BIG. passAsFile writes the content to a
# temp file and sets pkgStoresListPath, keeping the env small. Nix still
# tracks the string context (i.e. dependencies on all pkgStores) correctly.
stdenvNoCC.mkDerivation {
name = "airi-pnpm-deps";
passAsFile = [ "pkgStoresList" ];
pkgStoresList = (lib.concatStringsSep "\n" (lib.attrValues pkgStores)) + "\n";
buildPhase = ''
while IFS= read -r store; do
[ -z "$store" ] && continue
cp -rn --no-preserve=mode "$store/." "$out/"
done < "$pkgStoresListPath"
# NOTICE: CAFS files with -exec suffix denote executable binaries (e.g. turbo, esbuild).
# Set their permissions to 555 (r-xr-xr-x) so pnpm can execute them during install.
find "$out" -type f -name "*-exec" -exec chmod 555 {} +
'';
dontConfigure = true;
dontUnpack = true;
dontInstall = true;
dontFixup = true;
}
9 changes: 9 additions & 0 deletions nix/scripts/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"name": "@proj-airi/nix-scripts",
"version": "0.0.0",
"private": true,
"dependencies": {
"@pnpm/constants": "catalog:",
"@pnpm/store.cafs": "1000.1.4"
}
}
Loading
Loading