diff --git a/change/change-39d2ed64-e3c5-4189-a32d-a0c18f5ea28a.json b/change/change-39d2ed64-e3c5-4189-a32d-a0c18f5ea28a.json new file mode 100644 index 000000000..a23f8b672 --- /dev/null +++ b/change/change-39d2ed64-e3c5-4189-a32d-a0c18f5ea28a.json @@ -0,0 +1,39 @@ +{ + "changes": [ + { + "packageName": "@lage-run/lockfile", + "type": "patch", + "dependentChangeType": "patch", + "comment": "New experimental package that analyzes a pnpm lockfile (`lockfileVersion 9.x`) to compute a precise per-package dependency-closure signature, so cache hashing and `--since` filtering can avoid treating a lockfile change as a repo-wide invalidation.", + "email": "email not defined" + }, + { + "packageName": "@lage-run/config", + "type": "patch", + "dependentChangeType": "patch", + "comment": "Add experimental opt-in `experimentalLockfileInvalidation` config option (`{ packageManager: \"pnpm\" }`) for smarter, per-package lockfile invalidation.", + "email": "email not defined" + }, + { + "packageName": "@lage-run/hasher", + "type": "minor", + "dependentChangeType": "patch", + "comment": "`TargetHasher` accepts an experimental `experimentalLockfileInvalidation` option. When enabled, external dependency invalidation uses a precise per-package pnpm lockfile closure signature instead of the resolved dependency list, so only packages whose closure changed get a new cache key. Unsupported lockfiles conservatively hash their complete content into every target.", + "email": "email not defined" + }, + { + "packageName": "@lage-run/cli", + "type": "patch", + "dependentChangeType": "patch", + "comment": "Support the experimental `experimentalLockfileInvalidation` config option so that `--since` filtering and cache hashing only invalidate packages actually affected by a pnpm lockfile change, instead of the whole graph. Only pnpm (`lockfileVersion 9.x`) is supported; other package managers/versions fall back to blanket invalidation.", + "email": "email not defined" + }, + { + "packageName": "workspace-tools", + "type": "patch", + "dependentChangeType": "patch", + "comment": "Add `getMergeBase` git helper to resolve the merge-base commit SHA between two refs.", + "email": "email not defined" + } + ] +} diff --git a/docs/docs/guides/cache.md b/docs/docs/guides/cache.md index e7de1d187..026121192 100644 --- a/docs/docs/guides/cache.md +++ b/docs/docs/guides/cache.md @@ -27,3 +27,44 @@ lage build --reset-cache ## Cache Options Caching capability is provided by `backfill`. All of the configuration under the `cacheOptions` key is passed to `backfill`. For the complete documentation of `cacheOptions`, see the [`backfill` configuration documentation](https://www.npmjs.com/package/backfill#configuration). + +## Experimental: smarter lockfile invalidation + +A common cause of poor cache hit rates in PR builds is the lockfile. By default, `lage` treats any +change to the lockfile as a repo-wide change, which invalidates every package's cache. Since the +pnpm lockfile changes frequently in PRs, this means a single dependency bump can cause a full +rebuild and total cache misses — even for packages that were not affected by the change. + +The experimental `experimentalLockfileInvalidation` config option makes `lage` analyze the lockfile +and only invalidate the packages whose resolved dependency closure actually changed: + +```js title="/lage.config.js" +const config = { + cacheOptions: { + environmentGlob: ["package.json", "lage.config.js", "pnpm-lock.yaml"] + }, + repoWideChanges: ["pnpm-lock.yaml"], + experimentalLockfileInvalidation: { packageManager: "pnpm" } +}; +``` + +`lage` automatically removes `pnpm-lock.yaml` from global environment and repo-wide matches while +the feature owns it, including matches from wildcard globs. You do not need to remove existing +lockfile entries from those options. + +`lage` computes a stable per-package signature that captures each workspace project's entire +resolved external dependency graph (using a memoized Merkle hash of the lockfile's shared dependency +DAG, so the added cost is roughly proportional to the size of the lockfile — computed once per run, +not per package). Only packages whose signature changed get a new cache key, so unaffected packages +keep their cache hits, including reuse from the [remote cache](./remote-cache.md) across PR branches. + +**Only pnpm is supported**, and only the latest lockfile format (`lockfileVersion 9.x`). This +feature depends on pnpm's strict, deterministic lockfile, which precisely describes every project's +resolved dependency graph (including peer-dependency resolution). Package managers with looser +lockfiles (npm, yarn) do not offer the same guarantees and are not supported. For any unsupported +package manager or lockfile version, `lage` warns and falls back to blanket invalidation. In that +fallback, every cache key includes the raw lockfile content, so builds are never under-invalidated. +Top-level pnpm settings and metadata also intentionally invalidate every package. + +See the [configuration reference](../reference/config.md#experimental-smarter-lockfile-invalidation) +for details. diff --git a/docs/docs/guides/remote-cache.md b/docs/docs/guides/remote-cache.md index 136e9c279..63f39faa3 100644 --- a/docs/docs/guides/remote-cache.md +++ b/docs/docs/guides/remote-cache.md @@ -124,3 +124,32 @@ const config = { }; module.exports = config; ``` + +## Improving remote cache hit rates on lockfile changes + +Remote cache is most valuable in PR builds, where one branch can reuse cache entries produced by +another. However, by default any lockfile change invalidates every package's cache key, so a single +dependency bump in a PR causes remote cache misses across the entire repo. + +If you use pnpm, the experimental `experimentalLockfileInvalidation` option makes cache keys change +only for the packages whose resolved dependency closure actually changed, so unaffected packages +continue to hit the remote cache: + +```js title="/lage.config.js" +const config = { + cacheOptions: { + environmentGlob: ["package.json", "lage.config.js", "pnpm-lock.yaml"] + }, + repoWideChanges: ["pnpm-lock.yaml"], + experimentalLockfileInvalidation: { packageManager: "pnpm" } +}; +``` + +`lage` automatically removes the lockfile from global cache inputs while applying its per-package +signature, so existing exact or wildcard environment globs do not defeat the optimization. + +Only pnpm (latest `lockfileVersion 9.x`) is supported; unsupported package managers or lockfile +versions fall back to raw lockfile content in every cache key. See the +[caching guide](./cache.md#experimental-smarter-lockfile-invalidation) and the +[configuration reference](../reference/config.md#experimental-smarter-lockfile-invalidation) for +details. diff --git a/docs/docs/guides/scopes.md b/docs/docs/guides/scopes.md index b51f7840c..ccc6ea6ca 100644 --- a/docs/docs/guides/scopes.md +++ b/docs/docs/guides/scopes.md @@ -53,3 +53,32 @@ In fact, this is so useful that `lage` has a special syntactic sugar for it: ## syntactic sugar for --scope build-tools --no-dependents lage build --to build-tools ``` + +## Experimental: smarter lockfile invalidation with `--since` + +When you run `lage --since `, `lage` normally treats a lockfile change as a repo-wide change +(via `repoWideChanges`) and runs **every** package. In PR builds the pnpm lockfile changes often, so +this defeats the purpose of `--since`. + +The experimental `experimentalLockfileInvalidation` config option teaches `--since` to diff the old +and new lockfile and only include the packages whose resolved dependency closure actually changed: + +```js title="/lage.config.js" +const config = { + repoWideChanges: ["pnpm-lock.yaml"], + experimentalLockfileInvalidation: { packageManager: "pnpm" } +}; +``` + +With this enabled, a lockfile change that only affects a couple of packages will only cause those +packages (and their dependents) to run under `--since`, instead of the entire graph. When the +lockfile is unchanged, this adds no lockfile parsing work. `lage` automatically ignores the lockfile +in repo-wide matches while analyzing it, including wildcard matches. Staged and unstaged lockfile +edits receive the same precise analysis as committed changes. + +**Only pnpm is supported** (latest `lockfileVersion 9.x`), because it depends on pnpm's strict, +deterministic lockfile. Unsupported package managers or lockfile versions safely fall back to the +previous blanket behavior. Missing, added, deleted, malformed, or globally significant lockfile +changes also fall back to all packages. See the +[configuration reference](../reference/config.md#experimental-smarter-lockfile-invalidation) and the +[caching guide](./cache.md#experimental-smarter-lockfile-invalidation) for details. diff --git a/docs/docs/reference/config.md b/docs/docs/reference/config.md index 84adcb6f5..282c22fc7 100644 --- a/docs/docs/reference/config.md +++ b/docs/docs/reference/config.md @@ -117,3 +117,66 @@ const config = { module.exports = config; ``` + +## Experimental: smarter lockfile invalidation + +By default, any change to your package manager's lockfile (e.g. `pnpm-lock.yaml`) is treated as a +repo-wide change: it invalidates **every** package's cache and, when using `--since`, forces +**every** package to run. This is safe but expensive — in PR builds the lockfile changes often, and +a single dependency bump ends up rebuilding the whole graph and missing the cache (including the +remote cache) for packages that were not actually affected. + +The `experimentalLockfileInvalidation` option makes `lage` analyze the lockfile to determine +**exactly which workspace packages had their resolved dependency closure changed**, and only those +packages (and their dependents) are invalidated. Everything else keeps its cache hits. + +```js title="/lage.config.js" +/** @type {import("lage").ConfigFileOptions} */ +const config = { + // ... + experimentalLockfileInvalidation: { + // Only "pnpm" is supported today. + packageManager: "pnpm" + } +}; +``` + +When enabled, `lage` takes ownership of `pnpm-lock.yaml` handling. It excludes the lockfile from +`repoWideChanges` and `cacheOptions.environmentGlob` matches, including wildcard matches, so existing +configuration does not need to change: + +```js title="/lage.config.js" +const config = { + cacheOptions: { + environmentGlob: ["package.json", "lage.config.js", "pnpm-lock.yaml"] + }, + repoWideChanges: ["pnpm-lock.yaml"], + experimentalLockfileInvalidation: { packageManager: "pnpm" } +}; +``` + +Changes to top-level pnpm settings and metadata (such as overrides, patched dependencies, and unknown +future fields) still invalidate every package because they can affect the entire install. Staged and +unstaged lockfile edits are analyzed precisely. A missing, deleted, newly added, malformed, or +unsupported lockfile safely uses the blanket fallback. + +### Supported package managers + +Only **pnpm** is supported, and only the **latest pnpm lockfile format (`lockfileVersion 9.x`)**. + +This feature relies on the lockfile being **strict and deterministic** — that is, it must fully and +unambiguously describe each workspace project's entire resolved dependency graph (including +peer-dependency resolution). pnpm's lockfile provides exactly this via its `importers` and +`snapshots` sections, which is what lets `lage` compute a precise per-package signature. Package +managers with looser or less deterministic lockfiles (npm, yarn) do not provide the same guarantees, +so they are intentionally not supported here. + +For anything unsupported — a different package manager, an older pnpm lockfile version, or a lockfile +that cannot be parsed — `lage` logs a warning and **safely falls back to blanket invalidation**. The +raw lockfile content is included in every cache key, and `--since` runs every package, so builds never +silently under-invalidate. + +:::caution Experimental +This option is experimental and may change. It is opt-in and has no effect on other `lage` commands +when disabled. See the [caching guide](../guides/cache.md) for more details. +::: diff --git a/packages/cli/package.json b/packages/cli/package.json index 95dcf5611..89379ac77 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -26,6 +26,7 @@ "@lage-run/cache": "workspace:^", "@lage-run/config": "workspace:^", "@lage-run/hasher": "workspace:^", + "@lage-run/lockfile": "workspace:^", "@lage-run/logger": "workspace:^", "@lage-run/reporters": "workspace:^", "@lage-run/rpc": "workspace:^", diff --git a/packages/cli/src/__tests__/createTargetGraph.test.ts b/packages/cli/src/__tests__/createTargetGraph.test.ts index c2b9c82b3..3945096ea 100644 --- a/packages/cli/src/__tests__/createTargetGraph.test.ts +++ b/packages/cli/src/__tests__/createTargetGraph.test.ts @@ -110,6 +110,41 @@ describe("createTargetGraph", () => { }, ]); }); + + it.each([ + { repoWideChanged: false, expectedStagedTarget: true }, + { repoWideChanged: true, expectedStagedTarget: false }, + ])("uses staged targets only when changes are not repo-wide", async ({ repoWideChanged, expectedStagedTarget }) => { + const packageInfos: PackageInfos = { + foo: stubPackage({ name: "foo", scripts: ["lint"] }), + }; + const targetGraph = await createTargetGraph({ + logger: createLogger(), + root: ROOT, + dependencies: false, + dependents: false, + enableTargetConfigMerging: true, + enablePhantomTargetOptimization: false, + ignore: [], + pipeline: { + lint: { + stagedTarget: {}, + }, + }, + repoWideChanges: ["pnpm-lock.yaml"], + scope: [], + since: "main", + outputs: [], + tasks: ["lint"], + packageInfos, + priorities: [], + changedFiles: ["pnpm-lock.yaml"], + filteredPackages: ["foo"], + repoWideChanged, + }); + + expect(targetGraph.targets.has("Δlint")).toBe(expectedStagedTarget); + }); }); const ROOT = path.resolve("/fake/root"); diff --git a/packages/cli/src/__tests__/lockfileInvalidation.test.ts b/packages/cli/src/__tests__/lockfileInvalidation.test.ts new file mode 100644 index 000000000..afa041f41 --- /dev/null +++ b/packages/cli/src/__tests__/lockfileInvalidation.test.ts @@ -0,0 +1,301 @@ +import { afterEach, describe, expect, it } from "@jest/globals"; +import createLogger from "@lage-run/logger"; +import { Monorepo } from "@lage-run/test-utilities"; +import execa from "execa"; +import fs from "fs"; +import path from "path"; +import { getPackageInfosAsync } from "workspace-tools"; +import { getFilteredPackages } from "../filter/getFilteredPackages.js"; +import { getLockfileChangedPackages } from "../filter/getLockfileChangedPackages.js"; + +const baseLockfile = `lockfileVersion: '9.0' + +importers: + + .: {} + + packages/a: + dependencies: + semver: + specifier: 7.5.4 + version: 7.5.4 + + packages/b: + dependencies: + chalk: + specifier: 4.1.2 + version: 4.1.2 + + packages/c: + dependencies: + js-yaml: + specifier: 4.1.0 + version: 4.1.0 + +snapshots: + + semver@7.5.4: {} + + chalk@4.1.2: {} + + js-yaml@4.1.0: {} +`; + +// Only packages/c's dependency (js-yaml) is bumped; a and b are untouched. +const changedLockfile = baseLockfile.replace(/4\.1\.0/g, "4.2.0"); + +const rootImporterChangedLockfile = baseLockfile + .replace( + " .: {}", + ` .: + devDependencies: + typescript: + specifier: 5.8.3 + version: 5.8.3` + ) + .replace( + "snapshots:\n", + `snapshots: + + typescript@5.8.3: {} +` + ); + +describe("experimental pnpm lockfile invalidation (--since)", () => { + let monorepo: Monorepo | undefined; + + afterEach(async () => { + await monorepo?.cleanup(); + monorepo = undefined; + }); + + async function setupWithLockfileChange(): Promise { + const repo = new Monorepo("lockfile-invalidation-since"); + await repo.init({ packages: { a: {}, b: {}, c: {} } }); + // Commit the base lockfile (this becomes HEAD~1). + await repo.commitFiles({ "pnpm-lock.yaml": baseLockfile }); + // Commit only a lockfile change that affects package c's closure (this becomes HEAD). + await repo.commitFiles({ "pnpm-lock.yaml": changedLockfile }); + return repo; + } + + async function filterSince(repo: Monorepo, since = "HEAD"): Promise { + return getFilteredPackages({ + root: repo.root, + packageInfos: await getPackageInfosAsync(repo.root), + includeDependents: false, + includeDependencies: false, + since, + sinceIgnoreGlobs: [], + scope: [], + logger: createLogger(), + repoWideChanges: ["pnpm-lock.yaml"], + experimentalLockfileInvalidation: { packageManager: "pnpm" }, + }); + } + + it("only runs packages whose closure changed when the feature is enabled", async () => { + const logger = createLogger(); + monorepo = await setupWithLockfileChange(); + + const filteredPackages = getFilteredPackages({ + root: monorepo.root, + packageInfos: await getPackageInfosAsync(monorepo.root), + includeDependents: false, + includeDependencies: false, + since: "HEAD~1", + sinceIgnoreGlobs: [], + scope: [], + logger, + repoWideChanges: ["pnpm-lock.yaml"], + experimentalLockfileInvalidation: { packageManager: "pnpm" }, + }); + + expect(filteredPackages.sort()).toEqual(["c"]); + }); + + it("owns lockfile matches from wildcard repo-wide globs", async () => { + const logger = createLogger(); + monorepo = await setupWithLockfileChange(); + + const filteredPackages = getFilteredPackages({ + root: monorepo.root, + packageInfos: await getPackageInfosAsync(monorepo.root), + includeDependents: false, + includeDependencies: false, + since: "HEAD~1", + sinceIgnoreGlobs: [], + scope: [], + logger, + repoWideChanges: ["**/*.yaml"], + experimentalLockfileInvalidation: { packageManager: "pnpm" }, + }); + + expect(filteredPackages.sort()).toEqual(["c"]); + }); + + it("falls back to blanket invalidation when the feature is disabled", async () => { + const logger = createLogger(); + monorepo = await setupWithLockfileChange(); + + const filteredPackages = getFilteredPackages({ + root: monorepo.root, + packageInfos: await getPackageInfosAsync(monorepo.root), + includeDependents: false, + includeDependencies: false, + since: "HEAD~1", + sinceIgnoreGlobs: [], + scope: [], + logger, + repoWideChanges: ["pnpm-lock.yaml"], + }); + + // Without the feature, a lockfile change is repo-wide and runs everything. + expect(filteredPackages.sort()).toEqual(["a", "b", "c"]); + }); + + it("does no lockfile analysis and behaves normally when the lockfile is unchanged", async () => { + const logger = createLogger(); + monorepo = new Monorepo("lockfile-invalidation-unchanged"); + await monorepo.init({ packages: { a: {}, b: {}, c: {} } }); + await monorepo.commitFiles({ "pnpm-lock.yaml": baseLockfile }); + // Change only package a's source; the lockfile is unchanged. + await monorepo.commitFiles({ "packages/a/src.js": "console.log('a');" }); + + const filteredPackages = getFilteredPackages({ + root: monorepo.root, + packageInfos: await getPackageInfosAsync(monorepo.root), + includeDependents: false, + includeDependencies: false, + since: "HEAD~1", + sinceIgnoreGlobs: [], + scope: [], + logger, + repoWideChanges: ["pnpm-lock.yaml"], + experimentalLockfileInvalidation: { packageManager: "pnpm" }, + }); + + expect(filteredPackages.sort()).toEqual(["a"]); + }); + + it("falls back to blanket invalidation for an unsupported lockfile version", async () => { + const logger = createLogger(); + monorepo = new Monorepo("lockfile-invalidation-unsupported"); + await monorepo.init({ packages: { a: {}, b: {}, c: {} } }); + await monorepo.commitFiles({ "pnpm-lock.yaml": "lockfileVersion: '5.4'\ndependencies: {}\n" }); + await monorepo.commitFiles({ "pnpm-lock.yaml": "lockfileVersion: '5.4'\ndependencies:\n semver: 7.5.4\n" }); + + const filteredPackages = getFilteredPackages({ + root: monorepo.root, + packageInfos: await getPackageInfosAsync(monorepo.root), + includeDependents: false, + includeDependencies: false, + since: "HEAD~1", + sinceIgnoreGlobs: [], + scope: [], + logger, + repoWideChanges: ["pnpm-lock.yaml"], + experimentalLockfileInvalidation: { packageManager: "pnpm" }, + }); + + // Unsupported version -> keep blanket behavior (all packages run). + expect(filteredPackages.sort()).toEqual(["a", "b", "c"]); + }); + + it("falls back to blanket invalidation when the root importer changes", async () => { + const logger = createLogger(); + monorepo = new Monorepo("lockfile-invalidation-root-importer"); + await monorepo.init({ packages: { a: {}, b: {}, c: {} } }); + await monorepo.commitFiles({ "pnpm-lock.yaml": baseLockfile }); + await monorepo.commitFiles({ "pnpm-lock.yaml": rootImporterChangedLockfile }); + + const filteredPackages = getFilteredPackages({ + root: monorepo.root, + packageInfos: await getPackageInfosAsync(monorepo.root), + includeDependents: false, + includeDependencies: false, + since: "HEAD~1", + sinceIgnoreGlobs: [], + scope: [], + logger, + repoWideChanges: ["pnpm-lock.yaml"], + experimentalLockfileInvalidation: { packageManager: "pnpm" }, + }); + + expect(filteredPackages.sort()).toEqual(["a", "b", "c"]); + }); + + it("precisely analyzes an unstaged lockfile change", async () => { + monorepo = new Monorepo("lockfile-invalidation-unstaged"); + await monorepo.init({ packages: { a: {}, b: {}, c: {} } }); + await monorepo.commitFiles({ "pnpm-lock.yaml": baseLockfile }); + monorepo.writeFiles({ "pnpm-lock.yaml": changedLockfile }); + + expect((await filterSince(monorepo)).sort()).toEqual(["c"]); + }); + + it("precisely analyzes a staged lockfile change", async () => { + monorepo = new Monorepo("lockfile-invalidation-staged"); + await monorepo.init({ packages: { a: {}, b: {}, c: {} } }); + await monorepo.commitFiles({ "pnpm-lock.yaml": baseLockfile }); + monorepo.writeFiles({ "pnpm-lock.yaml": changedLockfile }); + execa.sync("git", ["add", "pnpm-lock.yaml"], { cwd: monorepo.root }); + + expect((await filterSince(monorepo)).sort()).toEqual(["c"]); + }); + + it("falls back to blanket invalidation for a deleted lockfile", async () => { + monorepo = new Monorepo("lockfile-invalidation-deleted"); + await monorepo.init({ packages: { a: {}, b: {}, c: {} } }); + await monorepo.commitFiles({ "pnpm-lock.yaml": baseLockfile }); + fs.unlinkSync(path.join(monorepo.root, "pnpm-lock.yaml")); + + expect((await filterSince(monorepo)).sort()).toEqual(["a", "b", "c"]); + }); + + it("falls back to blanket invalidation for a newly added untracked lockfile", async () => { + monorepo = new Monorepo("lockfile-invalidation-untracked"); + await monorepo.init({ packages: { a: {}, b: {}, c: {} } }); + monorepo.writeFiles({ "pnpm-lock.yaml": baseLockfile }); + + expect((await filterSince(monorepo)).sort()).toEqual(["a", "b", "c"]); + }); + + it("falls back to blanket invalidation when global installation metadata changes", async () => { + monorepo = new Monorepo("lockfile-invalidation-global-metadata"); + await monorepo.init({ packages: { a: {}, b: {}, c: {} } }); + await monorepo.commitFiles({ "pnpm-lock.yaml": `settings:\n autoInstallPeers: true\n${baseLockfile}` }); + await monorepo.commitFiles({ "pnpm-lock.yaml": `settings:\n autoInstallPeers: false\n${baseLockfile}` }); + + expect((await filterSince(monorepo, "HEAD~1")).sort()).toEqual(["a", "b", "c"]); + }); + + it("falls back to blanket invalidation for malformed-but-parseable v9 lockfiles", async () => { + monorepo = new Monorepo("lockfile-invalidation-malformed"); + await monorepo.init({ packages: { a: {}, b: {}, c: {} } }); + await monorepo.commitFiles({ "pnpm-lock.yaml": "lockfileVersion: '9.0'\nimporters:\n packages/a: invalid\n" }); + await monorepo.commitFiles({ "pnpm-lock.yaml": "lockfileVersion: '9.0'\nimporters:\n packages/a: changed\n" }); + + expect((await filterSince(monorepo, "HEAD~1")).sort()).toEqual(["a", "b", "c"]); + }); + + it("falls back when a merge-base cannot be determined", async () => { + monorepo = new Monorepo("lockfile-invalidation-merge-base"); + await monorepo.init({ packages: { a: {}, b: {}, c: {} } }); + await monorepo.commitFiles({ "pnpm-lock.yaml": baseLockfile }); + + const result = getLockfileChangedPackages({ + root: monorepo.root, + since: "missing-ref", + changedFiles: ["pnpm-lock.yaml"], + packageInfos: await getPackageInfosAsync(monorepo.root), + experimentalLockfileInvalidation: { packageManager: "pnpm" }, + logger: createLogger(), + }); + + expect(result.status).toBe("fallback"); + if (result.status === "fallback") { + expect(result.reason).toContain("merge-base"); + } + }); +}); diff --git a/packages/cli/src/cache/createCacheProvider.ts b/packages/cli/src/cache/createCacheProvider.ts index 63195f5c1..3768abfc9 100644 --- a/packages/cli/src/cache/createCacheProvider.ts +++ b/packages/cli/src/cache/createCacheProvider.ts @@ -1,5 +1,6 @@ import type { CacheOptions } from "@lage-run/cache"; import { TargetHasher } from "@lage-run/hasher"; +import type { ExperimentalLockfileInvalidationOptions } from "@lage-run/lockfile"; import type { TargetLogger } from "@lage-run/reporters"; interface CreateCacheOptions { @@ -8,12 +9,13 @@ interface CreateCacheOptions { root: string; skipLocalCache: boolean; cliArgs: string[]; + experimentalLockfileInvalidation?: ExperimentalLockfileInvalidationOptions; } export async function createCache(options: CreateCacheOptions): Promise<{ hasher: TargetHasher; }> { - const { cacheOptions, root, cliArgs, logger } = options; + const { cacheOptions, root, cliArgs, logger, experimentalLockfileInvalidation } = options; const hasher = new TargetHasher({ root, @@ -21,6 +23,7 @@ export async function createCache(options: CreateCacheOptions): Promise<{ cacheKey: cacheOptions?.cacheKey, cliArgs, logger, + experimentalLockfileInvalidation, }); await hasher.initialize(); diff --git a/packages/cli/src/commands/affected/action.ts b/packages/cli/src/commands/affected/action.ts index 8c1f124ab..fba114c81 100644 --- a/packages/cli/src/commands/affected/action.ts +++ b/packages/cli/src/commands/affected/action.ts @@ -28,6 +28,7 @@ export async function affectedAction(options: AffectedOptions): Promise { scope, repoWideChanges: config.repoWideChanges, sinceIgnoreGlobs: ignore, + experimentalLockfileInvalidation: config.experimentalLockfileInvalidation, }); let output = ""; diff --git a/packages/cli/src/commands/info/action.ts b/packages/cli/src/commands/info/action.ts index 1c3994335..d50f1cf06 100644 --- a/packages/cli/src/commands/info/action.ts +++ b/packages/cli/src/commands/info/action.ts @@ -4,7 +4,7 @@ import { filterArgsForTasks } from "../run/filterArgsForTasks.js"; import type { ConfigOptions } from "@lage-run/config"; import { getConfig } from "@lage-run/config"; import { type PackageInfos, getPackageInfos, getWorkspaceManagerRoot } from "workspace-tools"; -import { getFilteredPackages } from "../../filter/getFilteredPackages.js"; +import { getChangedFilesSince, getFilteredPackages } from "../../filter/getFilteredPackages.js"; import createLogger from "@lage-run/logger"; import path from "path"; import fs from "fs"; @@ -115,6 +115,32 @@ export async function infoAction(options: InfoActionOptions, command: Command): const packageInfos = getPackageInfos(root); const { tasks, taskArgs } = filterArgsForTasks(command.args); + let changedFiles: string[] | undefined; + if (options.since) { + try { + changedFiles = getChangedFilesSince(root, options.since); + } catch (e) { + logger.warn(`An error in the git command has caused this scope run to include every package\n${e}`); + } + } + + let repoWideChanged = false; + const scope = getFilteredPackages({ + root, + packageInfos, + logger, + includeDependencies: options.dependencies, + includeDependents: options.dependents && !options.to, // --to is a short hand for --scope + --no-dependents + since: options.since, + scope: (options.scope ?? []).concat(options.to ?? []), // --to is a short hand for --scope + --no-dependents + repoWideChanges: config.repoWideChanges, + sinceIgnoreGlobs: options.ignore.concat(config.ignore), + experimentalLockfileInvalidation: config.experimentalLockfileInvalidation, + changedFiles, + onRepoWideChange: (detected) => { + repoWideChanged = detected; + }, + }); const targetGraph = await createTargetGraph({ logger, @@ -132,18 +158,10 @@ export async function infoAction(options: InfoActionOptions, command: Command): priorities: config.priorities, enableTargetConfigMerging: config.enableTargetConfigMerging, enablePhantomTargetOptimization: config.enablePhantomTargetOptimization, - }); - - const scope = getFilteredPackages({ - root, - packageInfos, - logger, - includeDependencies: options.dependencies, - includeDependents: options.dependents && !options.to, // --to is a short hand for --scope + --no-dependents - since: options.since, - scope: (options.scope ?? []).concat(options.to ?? []), // --to is a short hand for --scope + --no-dependents - repoWideChanges: config.repoWideChanges, - sinceIgnoreGlobs: options.ignore.concat(config.ignore), + experimentalLockfileInvalidation: config.experimentalLockfileInvalidation, + changedFiles, + filteredPackages: scope, + repoWideChanged, }); const pickerOptions = getBuiltInRunners({ nodeArg: options.nodeArg, npmCmd: config.npmClient, taskArgs }); diff --git a/packages/cli/src/commands/run/createTargetGraph.ts b/packages/cli/src/commands/run/createTargetGraph.ts index 79223aa69..6c7164270 100644 --- a/packages/cli/src/commands/run/createTargetGraph.ts +++ b/packages/cli/src/commands/run/createTargetGraph.ts @@ -1,10 +1,9 @@ import type { TargetLogger } from "@lage-run/reporters"; import { type Priority, type TargetGraph, WorkspaceTargetGraphBuilder } from "@lage-run/target-graph"; import type { PackageInfos } from "workspace-tools"; -import { getBranchChanges, getDefaultRemoteBranch, getStagedChanges, getUnstagedChanges, getUntrackedChanges } from "workspace-tools"; -import { getFilteredPackages } from "../../filter/getFilteredPackages.js"; +import { getChangedFilesSince, getFilteredPackages } from "../../filter/getFilteredPackages.js"; import type { PipelineDefinition } from "@lage-run/config"; -import { hasRepoChanged } from "../../filter/hasRepoChanged.js"; +import type { ExperimentalLockfileInvalidationOptions } from "@lage-run/lockfile"; interface CreateTargetGraphOptions { logger: TargetLogger; @@ -22,19 +21,10 @@ interface CreateTargetGraphOptions { priorities: Priority[]; enableTargetConfigMerging: boolean; enablePhantomTargetOptimization: boolean; -} - -function getChangedFiles(since: string, cwd: string) { - const targetBranch = since || getDefaultRemoteBranch({ cwd }); - - return [ - ...new Set([ - ...(getUntrackedChanges({ cwd }) || []), - ...(getUnstagedChanges({ cwd }) || []), - ...(getBranchChanges({ branch: targetBranch, cwd }) || []), - ...(getStagedChanges({ cwd }) || []), - ]), - ]; + experimentalLockfileInvalidation?: ExperimentalLockfileInvalidationOptions; + changedFiles?: string[]; + filteredPackages?: string[]; + repoWideChanged?: boolean; } export async function createTargetGraph(options: CreateTargetGraphOptions): Promise { @@ -54,32 +44,44 @@ export async function createTargetGraph(options: CreateTargetGraphOptions): Prom tasks, packageInfos, priorities, + experimentalLockfileInvalidation, + changedFiles: providedChangedFiles, + filteredPackages: providedFilteredPackages, + repoWideChanged: providedRepoWideChanged, } = options; const builder = new WorkspaceTargetGraphBuilder({ root, packageInfos, enableTargetConfigMerging, enablePhantomTargetOptimization }); - const packages = getFilteredPackages({ - root, - logger, - packageInfos, - includeDependencies: dependencies, - includeDependents: dependents, - since, - scope, - repoWideChanges, - sinceIgnoreGlobs: ignore, - }); - - let changedFiles: string[] = []; - - // TODO: enhancement would be for workspace-tools to implement a "getChangedPackageFromChangedFiles()" type function - // TODO: optimize this so that we don't double up the work to determine if repo has changed - if (since) { - if (!hasRepoChanged({ since, root, environmentGlob: repoWideChanges, logger })) { - changedFiles = getChangedFiles(since, root); + let changedFiles = providedChangedFiles; + if (since && changedFiles === undefined) { + try { + changedFiles = getChangedFilesSince(root, since); + } catch (e) { + logger.warn(`An error in the git command has caused this scope run to include every package\n${e}`); } } + let repoWideChanged = providedRepoWideChanged ?? false; + const packages = + providedFilteredPackages ?? + getFilteredPackages({ + root, + logger, + packageInfos, + includeDependencies: dependencies, + includeDependents: dependents, + since, + scope, + repoWideChanges, + sinceIgnoreGlobs: ignore, + experimentalLockfileInvalidation, + changedFiles, + onRepoWideChange: (detected) => { + repoWideChanged = detected; + }, + }); + const targetConfigChangedFiles = repoWideChanged ? [] : (changedFiles ?? []); + const pipelineEntries = Object.entries(pipeline); // Add lage pipeline configuration in the package.json files. @@ -110,10 +112,10 @@ export async function createTargetGraph(options: CreateTargetGraphOptions): Prom options: {}, outputs, }, - changedFiles + targetConfigChangedFiles ); } else { - builder.addTargetConfig(id, definition, changedFiles); + builder.addTargetConfig(id, definition, targetConfigChangedFiles); } } diff --git a/packages/cli/src/commands/run/runAction.ts b/packages/cli/src/commands/run/runAction.ts index 5aec69b18..63e3c8f8f 100644 --- a/packages/cli/src/commands/run/runAction.ts +++ b/packages/cli/src/commands/run/runAction.ts @@ -62,6 +62,7 @@ export async function runAction(options: RunOptions, command: Command): Promise< priorities: config.priorities, enableTargetConfigMerging: config.enableTargetConfigMerging, enablePhantomTargetOptimization: config.enablePhantomTargetOptimization, + experimentalLockfileInvalidation: config.experimentalLockfileInvalidation, }); validateTargetGraph(targetGraph, allowNoTargetRuns); @@ -78,6 +79,7 @@ export async function runAction(options: RunOptions, command: Command): Promise< cacheOptions: config.cacheOptions, cliArgs: taskArgs, skipLocalCache: options.skipLocalCache, + experimentalLockfileInvalidation: config.experimentalLockfileInvalidation, }); const scheduler = new SimpleScheduler({ diff --git a/packages/cli/src/commands/run/watchAction.ts b/packages/cli/src/commands/run/watchAction.ts index 9e936a6f4..9acb92f6d 100644 --- a/packages/cli/src/commands/run/watchAction.ts +++ b/packages/cli/src/commands/run/watchAction.ts @@ -61,6 +61,7 @@ export async function watchAction(options: RunOptions, command: Command): Promis priorities: config.priorities, enableTargetConfigMerging: config.enableTargetConfigMerging, enablePhantomTargetOptimization: config.enablePhantomTargetOptimization, + experimentalLockfileInvalidation: config.experimentalLockfileInvalidation, }); // Make sure we do not attempt writeRemoteCache in watch mode @@ -76,6 +77,7 @@ export async function watchAction(options: RunOptions, command: Command): Promis cacheOptions: config.cacheOptions, cliArgs: taskArgs, skipLocalCache: options.skipLocalCache, + experimentalLockfileInvalidation: config.experimentalLockfileInvalidation, }); const scheduler = new SimpleScheduler({ diff --git a/packages/cli/src/commands/server/lageService.ts b/packages/cli/src/commands/server/lageService.ts index f41bd24ed..f0341da85 100644 --- a/packages/cli/src/commands/server/lageService.ts +++ b/packages/cli/src/commands/server/lageService.ts @@ -79,6 +79,7 @@ async function createInitializedPromise({ cwd, logger, serverControls, nodeArg, priorities: config.priorities, enableTargetConfigMerging: config.enableTargetConfigMerging, enablePhantomTargetOptimization: config.enablePhantomTargetOptimization, + experimentalLockfileInvalidation: config.experimentalLockfileInvalidation, }); const targetHasher = new TargetHasher({ @@ -87,6 +88,7 @@ async function createInitializedPromise({ cwd, logger, serverControls, nodeArg, logger, cacheKey: config.cacheOptions?.cacheKey, cliArgs: taskArgs, + experimentalLockfileInvalidation: config.experimentalLockfileInvalidation, }); logger.info("Initializing hasher"); @@ -236,6 +238,7 @@ export function createLageService({ cwd, serverControls, logger, concurrency, ta fs.mkdirSync(path.dirname(targetHashFullPath), { recursive: true }); } + targetHasher.refreshLockfileSignatures(); fs.writeFileSync(targetHashFullPath, await targetHasher.hash(target)); } catch { throw new ConnectError(`Error writing target hash file: ${targetHashFullPath}`, Code.Internal); diff --git a/packages/cli/src/filter/getFilteredPackages.ts b/packages/cli/src/filter/getFilteredPackages.ts index 4cc86e98b..b1b616b1d 100644 --- a/packages/cli/src/filter/getFilteredPackages.ts +++ b/packages/cli/src/filter/getFilteredPackages.ts @@ -1,8 +1,20 @@ import type { PackageInfos } from "workspace-tools"; -import { getScopedPackages, getChangedPackages, getTransitiveDependents, getTransitiveDependencies } from "workspace-tools"; +import { + getScopedPackages, + getPackagesByFiles, + getTransitiveDependents, + getTransitiveDependencies, + getBranchChanges, + getStagedChanges, + getUnstagedChanges, + getUntrackedChanges, +} from "workspace-tools"; +import type { ExperimentalLockfileInvalidationOptions } from "@lage-run/lockfile"; +import { getLockfileName } from "@lage-run/lockfile"; import type { TargetLogger } from "@lage-run/reporters"; import { hasRepoChanged } from "./hasRepoChanged.js"; +import { getLockfileChangedPackages } from "./getLockfileChangedPackages.js"; export function getFilteredPackages(options: { root: string; @@ -14,8 +26,26 @@ export function getFilteredPackages(options: { repoWideChanges: string[]; includeDependents: boolean; includeDependencies: boolean; + experimentalLockfileInvalidation?: ExperimentalLockfileInvalidationOptions; + /** Complete changed-file set, if the caller already computed it. */ + changedFiles?: string[]; + /** Reports whether repo-wide changes caused every package to be selected. */ + onRepoWideChange?: (detected: boolean) => void; }): string[] { - const { scope, since, sinceIgnoreGlobs, repoWideChanges, includeDependents, includeDependencies, logger, packageInfos, root } = options; + const { + scope, + since, + sinceIgnoreGlobs, + repoWideChanges, + includeDependents, + includeDependencies, + logger, + packageInfos, + root, + experimentalLockfileInvalidation, + changedFiles: providedChangedFiles, + onRepoWideChange, + } = options; // If scoped is defined, get scoped packages const hasScopes = Array.isArray(scope) && scope.length > 0; @@ -39,16 +69,63 @@ export function getFilteredPackages(options: { } // If since is defined, get changed packages. else if (hasSince) { - try { - changedPackages = getChangedPackages({ - cwd: root, - target: since, - ignoreGlobs: sinceIgnoreGlobs, + let changedFiles: string[] | undefined = providedChangedFiles; + if (changedFiles === undefined) { + try { + changedFiles = getChangedFilesSince(root, since!); + } catch (e) { + logger.warn(`An error in the git command has caused this scope run to include every package\n${e}`); + } + } + + // When experimental lockfile invalidation is enabled, analyze the lockfile change (if any) so + // that the lockfile does not trigger a blanket invalidation. On any analysis failure, we keep + // the previous blanket behavior so builds never silently under-invalidate. + let lockfileAffectedPackages: string[] | undefined; + let effectiveIgnoreGlobs = sinceIgnoreGlobs; + let effectiveRepoWideChanges = repoWideChanges; + let effectiveChangedFiles = changedFiles; + + if (experimentalLockfileInvalidation && changedFiles !== undefined) { + const lockfileName = getLockfileName(experimentalLockfileInvalidation); + const lockfileResult = getLockfileChangedPackages({ + root, + since: since!, + changedFiles, + packageInfos, + experimentalLockfileInvalidation, + logger, + }); + + if (lockfileResult.status === "affected") { + lockfileAffectedPackages = [...lockfileResult.packages]; + // The feature now owns lockfile handling: prevent the lockfile from triggering the blanket + // "all packages" behavior in both the changed-packages and repo-wide-changes paths. + effectiveIgnoreGlobs = [...(sinceIgnoreGlobs ?? []), lockfileName]; + effectiveRepoWideChanges = repoWideChanges.filter((glob) => glob !== lockfileName); + effectiveChangedFiles = changedFiles.filter((file) => file !== lockfileName); + } else if (lockfileResult.status === "fallback") { + logger.warn( + `Experimental lockfile invalidation could not analyze the lockfile change (${lockfileResult.reason}); falling back to blanket invalidation.` + ); + } + // "unchanged": nothing to do; the lockfile is not among the changed files. + } + + if (changedFiles !== undefined) { + changedPackages = getPackagesByFiles({ + root, + files: changedFiles, + ignoreGlobs: effectiveIgnoreGlobs, + returnAllPackagesOnNoMatch: true, }); - } catch (e) { - logger.warn(`An error in the git command has caused this scope run to include every package\n${e}`); - // if getChangedPackages throws, we will assume all have changed (using changedPackage = undefined) } + + // Merge in packages whose dependency closure changed due to the lockfile. + if (lockfileAffectedPackages !== undefined && changedPackages !== undefined) { + changedPackages = [...new Set([...changedPackages, ...lockfileAffectedPackages])]; + } + filteredPackages = filterPackages({ logger, packageInfos, @@ -60,7 +137,15 @@ export function getFilteredPackages(options: { // If the defined repo-wide changes are detected the get all packages and append to the filtered packages. // This alo ensures that the modified packages are always run first. - if (hasRepoChanged({ since, root, environmentGlob: repoWideChanges, logger })) { + const repoWideChanged = hasRepoChanged({ + since, + root, + environmentGlob: effectiveRepoWideChanges, + logger, + changedFiles: effectiveChangedFiles, + }); + onRepoWideChange?.(repoWideChanged); + if (repoWideChanged) { logger.verbose( `Repo-wide changes detected, running all packages. The following changed packages and their deps (if specified) will be run first: ${filteredPackages.join( "," @@ -68,6 +153,7 @@ export function getFilteredPackages(options: { ); filteredPackages = [...new Set(filteredPackages.concat(Object.keys(packageInfos)))]; } + return filteredPackages; } else { // If neither scope or since is defined, return all packages @@ -75,6 +161,17 @@ export function getFilteredPackages(options: { } } +export function getChangedFilesSince(root: string, since: string): string[] { + return [ + ...new Set([ + ...getUntrackedChanges({ cwd: root }), + ...getUnstagedChanges({ cwd: root }), + ...getBranchChanges({ branch: since, cwd: root }), + ...getStagedChanges({ cwd: root }), + ]), + ]; +} + export function filterPackages(options: { logger: TargetLogger; packageInfos: PackageInfos; diff --git a/packages/cli/src/filter/getLockfileChangedPackages.ts b/packages/cli/src/filter/getLockfileChangedPackages.ts new file mode 100644 index 000000000..4e8556656 --- /dev/null +++ b/packages/cli/src/filter/getLockfileChangedPackages.ts @@ -0,0 +1,107 @@ +import fs from "fs"; +import path from "path"; +import { + diffPackageSignatures, + getLockfileName, + parseLockfileGraph, + splitImporterSignatures, + type ExperimentalLockfileInvalidationOptions, +} from "@lage-run/lockfile"; +import { getFileFromRef, getMergeBase, type PackageInfos } from "workspace-tools"; +import type { TargetLogger } from "@lage-run/reporters"; + +/** + * The outcome of analyzing a lockfile change for the experimental smarter invalidation feature. + * + * - `unchanged`: the lockfile did not change; no packages are affected and the caller should do no + * extra work. + * - `affected`: the lockfile changed and was analyzed successfully; only `packages` had their + * resolved dependency closure changed. + * - `fallback`: the lockfile changed but could not be analyzed (unsupported version, missing old + * version, parse error, etc.); the caller should keep the previous blanket invalidation behavior + * so that builds never silently under-invalidate. + */ +export type LockfileChangeResult = + | { status: "unchanged" } + | { status: "affected"; packages: Set } + | { status: "fallback"; reason: string }; + +/** + * Determines which workspace packages are affected by a pnpm lockfile change between the `--since` + * ref (merge-base) and the working tree. + * + * The lockfile is only analyzed when it is actually among the changed files, so this does no extra + * work when the lockfile is unchanged. + */ +export function getLockfileChangedPackages(options: { + root: string; + since: string; + changedFiles: string[]; + packageInfos: PackageInfos; + experimentalLockfileInvalidation: ExperimentalLockfileInvalidationOptions; + logger: TargetLogger; +}): LockfileChangeResult { + const { root, since, changedFiles, packageInfos, experimentalLockfileInvalidation, logger } = options; + + const lockfileName = getLockfileName(experimentalLockfileInvalidation); + + // Cheap membership check first: if the lockfile is not among the changed files, do no extra work. + if (!changedFiles.includes(lockfileName)) { + return { status: "unchanged" }; + } + + let newContent: string; + try { + newContent = fs.readFileSync(path.join(root, lockfileName), "utf8"); + } catch (e) { + return { status: "fallback", reason: `could not read current ${lockfileName}: ${e}` }; + } + + // Read the old lockfile from the merge-base so it is consistent with the changed-files set. + const mergeBase = getMergeBase({ ref: since, cwd: root, throwOnError: false }); + if (mergeBase === undefined) { + return { status: "fallback", reason: `could not determine the merge-base for ref "${since}"` }; + } + const oldRef = mergeBase; + const oldContent = getFileFromRef({ filePath: lockfileName, ref: oldRef, cwd: root, throwOnError: false }); + if (oldContent === undefined) { + return { status: "fallback", reason: `could not read ${lockfileName} at ref "${oldRef}"` }; + } + + const oldResult = parseLockfileGraph(experimentalLockfileInvalidation, oldContent); + const newResult = parseLockfileGraph(experimentalLockfileInvalidation, newContent); + + if (oldResult.status !== "success") { + const reason = oldResult.status === "unsupported" ? oldResult.reason : "no lockfile content"; + return { status: "fallback", reason: `could not analyze old ${lockfileName}: ${reason}` }; + } + if (newResult.status !== "success") { + const reason = newResult.status === "unsupported" ? newResult.reason : "no lockfile content"; + return { status: "fallback", reason: `could not analyze current ${lockfileName}: ${reason}` }; + } + + if (oldResult.graph.globalSignature !== newResult.graph.globalSignature) { + return { status: "fallback", reason: `${lockfileName} global installation metadata changed` }; + } + + const oldSignatures = splitImporterSignatures(oldResult.graph, packageInfos, root); + const newSignatures = splitImporterSignatures(newResult.graph, packageInfos, root); + const changedUnmappedImporters = diffPackageSignatures( + oldSignatures.unmappedImporterSignatures, + newSignatures.unmappedImporterSignatures + ); + if (changedUnmappedImporters.size > 0) { + return { + status: "fallback", + reason: `${lockfileName} changed unmapped importer(s): ${[...changedUnmappedImporters].join(", ")}`, + }; + } + + const changed = diffPackageSignatures(oldSignatures.packageSignatures, newSignatures.packageSignatures); + + logger.verbose( + `Experimental lockfile invalidation: ${lockfileName} changed; ${changed.size} package(s) affected: ${[...changed].join(",")}` + ); + + return { status: "affected", packages: changed }; +} diff --git a/packages/cli/src/filter/hasRepoChanged.ts b/packages/cli/src/filter/hasRepoChanged.ts index 6c175ec65..90940987c 100644 --- a/packages/cli/src/filter/hasRepoChanged.ts +++ b/packages/cli/src/filter/hasRepoChanged.ts @@ -14,8 +14,10 @@ export function hasRepoChanged(params: { /** `environmentGlob` from cache config */ environmentGlob: string[]; logger: TargetLogger; + /** Complete changed-file set, if already available. */ + changedFiles?: string[]; }): boolean { - const { since, root, environmentGlob, logger } = params; + const { since, root, environmentGlob, logger, changedFiles: providedChangedFiles } = params; if (!environmentGlob.length) { // Following old logic: if no environmentGlob, it hasn't changed @@ -23,10 +25,12 @@ export function hasRepoChanged(params: { } try { - const changedFiles = getBranchChanges({ - branch: since, - cwd: root, - }); + const changedFiles = + providedChangedFiles ?? + getBranchChanges({ + branch: since, + cwd: root, + }); if (!changedFiles.length) { return false; } diff --git a/packages/config/etc/config.api.md b/packages/config/etc/config.api.md index a869e670f..e209b028e 100644 --- a/packages/config/etc/config.api.md +++ b/packages/config/etc/config.api.md @@ -6,6 +6,7 @@ import type { Config } from 'backfill-config'; import type { CustomStorageConfig } from 'backfill-config'; +import type { ExperimentalLockfileInvalidationOptions } from '@lage-run/lockfile'; import type { LogLevel } from '@lage-run/logger'; import type { Priority } from '@lage-run/target-graph'; import type { TargetConfig } from '@lage-run/target-graph'; @@ -56,6 +57,7 @@ export interface ConfigOptions { concurrency: number; enablePhantomTargetOptimization: boolean; enableTargetConfigMerging: boolean; + experimentalLockfileInvalidation?: ExperimentalLockfileInvalidationOptions; ignore: string[]; loggerOptions: LoggerOptions; // Warning: (ae-forgotten-export) The symbol "NpmClient" needs to be exported by the entry point index.d.ts diff --git a/packages/config/package.json b/packages/config/package.json index 19ab9acfb..e23779b5f 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -19,6 +19,7 @@ "lint": "monorepo-scripts lint" }, "dependencies": { + "@lage-run/lockfile": "workspace:^", "@lage-run/logger": "workspace:^", "@lage-run/runners": "workspace:^", "@lage-run/target-graph": "workspace:^", diff --git a/packages/config/src/getConfig.ts b/packages/config/src/getConfig.ts index 8f013e423..aa534639d 100644 --- a/packages/config/src/getConfig.ts +++ b/packages/config/src/getConfig.ts @@ -34,5 +34,6 @@ export async function getConfig(cwd: string): Promise { enablePhantomTargetOptimization: config?.enablePhantomTargetOptimization ?? false, reporter: config?.reporter, reporters: config?.reporters ?? {}, + experimentalLockfileInvalidation: config?.experimentalLockfileInvalidation, }; } diff --git a/packages/config/src/types/ConfigOptions.ts b/packages/config/src/types/ConfigOptions.ts index a623515cd..4dd657d5d 100644 --- a/packages/config/src/types/ConfigOptions.ts +++ b/packages/config/src/types/ConfigOptions.ts @@ -3,6 +3,7 @@ import type { Priority } from "@lage-run/target-graph"; import type { PipelineDefinition } from "./PipelineDefinition.js"; import type { LoggerOptions } from "./LoggerOptions.js"; import type { TargetRunnerPickerOptions } from "@lage-run/runners"; +import type { ExperimentalLockfileInvalidationOptions } from "@lage-run/lockfile"; export type NpmClient = "npm" | "yarn" | "pnpm"; @@ -48,6 +49,33 @@ export interface ConfigOptions { /** Disable the `--since` flag when any of these files changed */ repoWideChanges: string[]; + /** + * **Experimental.** Opt in to smarter, per-package lockfile invalidation. + * + * By default, any change to the package manager lockfile (e.g. `pnpm-lock.yaml`) is treated as a + * repo-wide change: it invalidates every package's cache and, with `--since`, forces every package + * to run. When this option is set, Lage instead analyzes the lockfile to determine exactly which + * workspace packages had their resolved dependency closure changed, and only those packages (and + * their dependents) are invalidated — preserving cache hits (including remote cache reuse) for the + * rest. + * + * Only `pnpm` is supported today, and only the latest pnpm lockfile format (`lockfileVersion 9.x`). + * Strict, deterministic lockfiles (like pnpm's) are what make this analysis reliable. For + * unsupported package managers or lockfile versions, Lage safely falls back to the previous + * blanket invalidation behavior. + * + * When enabling this, remove the lockfile from `repoWideChanges` and from + * `cacheOptions.environmentGlob` so that the feature can own lockfile handling. + * + * Example: + * ```js + * { + * experimentalLockfileInvalidation: { packageManager: "pnpm" } + * } + * ``` + */ + experimentalLockfileInvalidation?: ExperimentalLockfileInvalidationOptions; + /** Which NPM Client to use when running npm lifecycle scripts */ npmClient: NpmClient; diff --git a/packages/hasher/etc/hasher.api.md b/packages/hasher/etc/hasher.api.md index d33862b39..049493c4e 100644 --- a/packages/hasher/etc/hasher.api.md +++ b/packages/hasher/etc/hasher.api.md @@ -5,6 +5,7 @@ ```ts import { DependencyMap } from 'workspace-tools'; +import { ExperimentalLockfileInvalidationOptions } from '@lage-run/lockfile'; import type { Logger } from '@lage-run/logger'; import { PackageInfos } from 'workspace-tools'; import { Target } from '@lage-run/target-graph'; @@ -67,6 +68,7 @@ export class TargetHasher { initialize(): Promise; // (undocumented) packageTree: PackageTree | undefined; + refreshLockfileSignatures(): void; } // @public (undocumented) @@ -77,6 +79,7 @@ interface TargetHasherOptions { cliArgs?: string[]; // (undocumented) environmentGlob: string[]; + experimentalLockfileInvalidation?: ExperimentalLockfileInvalidationOptions; // (undocumented) logger?: Logger; // (undocumented) diff --git a/packages/hasher/package.json b/packages/hasher/package.json index b8b6307fb..b0a6c4430 100644 --- a/packages/hasher/package.json +++ b/packages/hasher/package.json @@ -17,6 +17,7 @@ "lint": "monorepo-scripts lint" }, "dependencies": { + "@lage-run/lockfile": "workspace:^", "@lage-run/logger": "workspace:^", "@lage-run/target-graph": "workspace:^", "backfill-hasher": "workspace:^", diff --git a/packages/hasher/src/TargetHasher.ts b/packages/hasher/src/TargetHasher.ts index 02bdb999e..2652cc6d7 100644 --- a/packages/hasher/src/TargetHasher.ts +++ b/packages/hasher/src/TargetHasher.ts @@ -1,7 +1,16 @@ import type { Logger } from "@lage-run/logger"; import type { Target } from "@lage-run/target-graph"; +import { + type ExperimentalLockfileInvalidationOptions, + getLockfileName, + parseLockfileGraph, + splitImporterSignatures, +} from "@lage-run/lockfile"; import { resolveExternalDependencies } from "backfill-hasher"; +import crypto from "crypto"; +import fs from "fs"; import { hash } from "glob-hasher"; +import path from "path"; import { createDependencyMap, type DependencyMap, @@ -24,6 +33,13 @@ export interface TargetHasherOptions { cliArgs?: string[]; // "never" means no structured data arguments are used logger?: Logger; + /** + * **Experimental.** When set, external dependency invalidation is computed from a precise + * per-package lockfile closure signature instead of the (less reliable) resolved dependency list. + * Only pnpm (lockfileVersion 9.x) is supported; other package managers/versions fall back to the + * default behavior. + */ + experimentalLockfileInvalidation?: ExperimentalLockfileInvalidationOptions; } export interface TargetManifest { @@ -58,6 +74,32 @@ export class TargetHasher { private lockInfo: ParsedLock | undefined; private targetHashes: Record = {}; + /** + * When the experimental lockfile invalidation feature is enabled and the lockfile is supported, + * maps a workspace package name to a single stable signature capturing its entire resolved + * external dependency closure. Undefined when the feature is disabled or the lockfile is + * unsupported (in which case the default `resolveExternalDependencies` behavior is used). + */ + private lockfilePackageSignatures: Map | undefined; + + /** + * Signature for lockfile importers that do not map to a workspace package, such as the repo root + * importer. Root dev tools can affect any package script, so this signature is included in every + * target hash when available. + */ + private lockfileGlobalSignature: string | undefined; + + /** Signature used for every target when precise analysis is unavailable. */ + private lockfileFallbackSignature: string | undefined; + + /** Signature used by root targets, which can observe every workspace importer. */ + private lockfileRootSignature: string | undefined; + + /** File stat key used by long-lived server mode to detect lockfile changes cheaply. */ + private lockfileStateKey: string | undefined; + + private readonly unreadableLockfileNonce = crypto.randomUUID(); + public dependencyMap: DependencyMap = { dependencies: new Map(), dependents: new Map(), @@ -90,6 +132,7 @@ export class TargetHasher { this.fileHasher .readManifest() .then(() => globAsyncCached(environmentGlob, { cwd: root })) + .then((files) => this.excludeManagedLockfile(files)) .then((files) => this.fileHasher.hash(files)) .then((h) => (this.globalInputsHash = h)), @@ -115,17 +158,126 @@ export class TargetHasher { return this.packageTree.initialize(); }), - parseLockFile(root).then((lockInfo) => (this.lockInfo = lockInfo)), + this.options.experimentalLockfileInvalidation + ? Promise.resolve() + : parseLockFile(root).then((lockInfo) => (this.lockInfo = lockInfo)), ]); await this.initializedPromise; + this.initializeLockfileSignatures(); + if (this.logger !== undefined) { const globalInputsHash = hashStrings(Object.values(this.globalInputsHash ?? {})); this.logger.verbose(`Global inputs hash: ${globalInputsHash}`); } } + /** + * Computes the per-package lockfile closure signatures once, when the experimental lockfile + * invalidation feature is enabled. Runs after `packageInfos` is populated. On any unsupported or + * missing lockfile, hashes the raw lockfile state into every target so fallback remains + * conservative even when users remove the lockfile from `environmentGlob`. + */ + private initializeLockfileSignatures(): void { + const { experimentalLockfileInvalidation, root } = this.options; + if (!experimentalLockfileInvalidation) { + return; + } + + this.lockfilePackageSignatures = undefined; + this.lockfileGlobalSignature = undefined; + this.lockfileFallbackSignature = undefined; + this.lockfileRootSignature = undefined; + + const lockfileName = getLockfileName(experimentalLockfileInvalidation); + const lockfilePath = path.join(root, lockfileName); + let rawContent: string; + try { + const stat = fs.statSync(lockfilePath); + this.lockfileStateKey = `${stat.mtimeMs}:${stat.ctimeMs}:${stat.size}`; + rawContent = fs.readFileSync(lockfilePath, "utf8"); + } catch (e) { + const errorCode = e instanceof Error && "code" in e ? String(e.code) : undefined; + this.lockfileStateKey = errorCode === "ENOENT" ? "missing" : `unreadable:${this.unreadableLockfileNonce}`; + this.lockfileFallbackSignature = hashStrings([`lockfile:${lockfileName}:${this.lockfileStateKey}`]); + this.lockfileRootSignature = this.lockfileFallbackSignature; + this.logger?.warn( + errorCode === "ENOENT" + ? `Experimental lockfile invalidation is enabled but no ${lockfileName} was found; using conservative cache invalidation.` + : `Experimental lockfile invalidation could not read ${lockfileName} (${e}); disabling cross-run cache reuse.` + ); + return; + } + + const result = parseLockfileGraph(experimentalLockfileInvalidation, rawContent); + if (result.status === "success") { + const { packageSignatures, unmappedImporterSignatures } = splitImporterSignatures(result.graph, this.packageInfos, root); + const missingPackages = Object.keys(this.packageInfos).filter((packageName) => !packageSignatures.has(packageName)); + if (missingPackages.length > 0) { + this.setLockfileFallbackSignature( + lockfileName, + rawContent, + `no lockfile importer was found for workspace package(s): ${missingPackages.join(", ")}` + ); + return; + } + + this.lockfilePackageSignatures = new Map(packageSignatures); + const unmappedSignatures = [...unmappedImporterSignatures] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([importerId, signature]) => `${importerId}:${signature}`); + this.lockfileGlobalSignature = hashStrings([result.graph.globalSignature, ...unmappedSignatures]); + this.lockfileRootSignature = hashStrings([ + result.graph.globalSignature, + ...[...result.graph.importerSignatures] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([importerId, signature]) => `${importerId}:${signature}`), + ]); + this.logger?.verbose( + `Experimental lockfile invalidation enabled for "${experimentalLockfileInvalidation.packageManager}"; computed closure signatures for ${this.lockfilePackageSignatures.size} package(s) and ${unmappedImporterSignatures.size} unmapped importer(s).` + ); + } else { + const reason = result.status === "unsupported" ? result.reason : "lockfile content was unavailable"; + this.setLockfileFallbackSignature(lockfileName, rawContent, reason); + } + } + + private setLockfileFallbackSignature(lockfileName: string, rawContent: string, reason: string): void { + this.lockfilePackageSignatures = undefined; + this.lockfileGlobalSignature = undefined; + this.lockfileFallbackSignature = hashStrings([`lockfile:${lockfileName}`, rawContent]); + this.lockfileRootSignature = this.lockfileFallbackSignature; + this.logger?.warn( + `Experimental lockfile invalidation could not precisely analyze ${lockfileName} (${reason}); hashing the complete lockfile into every target.` + ); + } + + /** + * Refreshes experimental lockfile signatures if the configured lockfile changed. This is used by + * the long-lived worker service; normal one-shot Lage runs never pay this stat check. + */ + public refreshLockfileSignatures(): void { + const { experimentalLockfileInvalidation, root } = this.options; + if (!experimentalLockfileInvalidation) { + return; + } + + const lockfilePath = path.join(root, getLockfileName(experimentalLockfileInvalidation)); + let stateKey: string; + try { + const stat = fs.statSync(lockfilePath); + stateKey = `${stat.mtimeMs}:${stat.ctimeMs}:${stat.size}`; + } catch (e) { + const errorCode = e instanceof Error && "code" in e ? String(e.code) : undefined; + stateKey = errorCode === "ENOENT" ? "missing" : `unreadable:${this.unreadableLockfileNonce}`; + } + + if (stateKey !== this.lockfileStateKey) { + this.initializeLockfileSignatures(); + } + } + public async hash(target: Target): Promise { this.ensureInitialized(); @@ -140,6 +292,9 @@ export class TargetHasher { const fileFashes = hash(files, { cwd: root }) ?? {}; const hashes = Object.values(fileFashes) as string[]; + if (this.lockfileRootSignature !== undefined) { + hashes.push(`lockfile-root:${this.lockfileRootSignature}`); + } return hashStrings(hashes); } @@ -148,15 +303,28 @@ export class TargetHasher { // 2. add hash of target packages' internal and external deps const { dependencies, devDependencies } = this.packageInfos[target.packageName!]; - const parsedLock = this.lockInfo!; - const allDependencies: Record = { ...dependencies, ...devDependencies, }; const internalDeps = Object.keys(allDependencies).filter((dep) => this.packageInfos[dep]); - const externalDeps = resolveExternalDependencies(allDependencies, this.packageInfos, parsedLock); + + // When the experimental lockfile invalidation feature is enabled and produced a signature for + // this package, use the precise closure signature instead of the resolved dependency list. This + // captures the package's entire transitive external closure (including peer-resolved deps) in a + // single stable hash, so only packages whose closure actually changed get a new hash. + const lockfileSignature = this.lockfilePackageSignatures?.get(target.packageName!); + const lockfileGlobalDeps = this.lockfileGlobalSignature !== undefined ? [`lockfile-global:${this.lockfileGlobalSignature}`] : []; + let externalDeps: string[]; + if (this.options.experimentalLockfileInvalidation) { + externalDeps = + this.lockfileFallbackSignature !== undefined + ? [`lockfile-fallback:${this.lockfileFallbackSignature}`] + : [`lockfile:${lockfileSignature!}`, ...lockfileGlobalDeps]; + } else { + externalDeps = resolveExternalDependencies(allDependencies, this.packageInfos, this.lockInfo!); + } const resolvedDependencies = [...internalDeps, ...externalDeps].sort(); const files = getInputFiles(target, this.dependencyMap, this.packageTree!); @@ -190,12 +358,25 @@ export class TargetHasher { private async getEnvironmentGlobHashes(root: string, target: Target): Promise> { const globalFileHashes = target.environmentGlob - ? this.fileHasher.hash(await globAsyncCached(target.environmentGlob, { cwd: root })) + ? this.fileHasher.hash(this.excludeManagedLockfile(await globAsyncCached(target.environmentGlob, { cwd: root }))) : (this.globalInputsHash ?? {}); return globalFileHashes; } + private excludeManagedLockfile(files: string[]): string[] { + const { experimentalLockfileInvalidation, root } = this.options; + if (!experimentalLockfileInvalidation) { + return files; + } + + const lockfileName = getLockfileName(experimentalLockfileInvalidation); + return files.filter((file) => { + const relativePath = (path.isAbsolute(file) ? path.relative(root, file) : file).replace(/\\/g, "/").replace(/^\.\//, ""); + return relativePath !== lockfileName; + }); + } + public cleanup(): void { this.fileHasher.writeManifest(); } diff --git a/packages/hasher/src/__tests__/lockfileInvalidation.test.ts b/packages/hasher/src/__tests__/lockfileInvalidation.test.ts new file mode 100644 index 000000000..8e29a22ec --- /dev/null +++ b/packages/hasher/src/__tests__/lockfileInvalidation.test.ts @@ -0,0 +1,236 @@ +import { afterEach, describe, expect, it } from "@jest/globals"; +import { Monorepo } from "@lage-run/test-utilities"; +import type { Target } from "@lage-run/target-graph"; +import path from "path"; +import { TargetHasher } from "../TargetHasher.js"; + +const baseLockfile = `lockfileVersion: '9.0' + +importers: + + .: {} + + packages/a: + dependencies: + semver: + specifier: 7.5.4 + version: 7.5.4 + + packages/b: + dependencies: + chalk: + specifier: 4.1.2 + version: 4.1.2 + + packages/c: + dependencies: + js-yaml: + specifier: 4.1.0 + version: 4.1.0 + +snapshots: + + semver@7.5.4: {} + + chalk@4.1.2: {} + + js-yaml@4.1.0: {} +`; + +// Only packages/c's dependency (js-yaml) changes. +const changedLockfile = baseLockfile.replace(/4\.1\.0/g, "4.2.0"); + +const rootImporterChangedLockfile = baseLockfile + .replace( + " .: {}", + ` .: + devDependencies: + typescript: + specifier: 5.8.3 + version: 5.8.3` + ) + .replace( + "snapshots:\n", + `snapshots: + + typescript@5.8.3: {} +` + ); + +const unsupportedLockfile = `lockfileVersion: '8.0' +importers: + .: {} + packages/a: {} + packages/b: {} + packages/c: {} +`; + +const malformedLockfile = `lockfileVersion: '9.0' +importers: + .: {} + packages/a: + dependencies: + invalid: 1.0.0 + packages/b: {} + packages/c: {} +`; + +describe("TargetHasher with experimental pnpm lockfile invalidation", () => { + let monorepos: Monorepo[] = []; + + afterEach(async () => { + for (const monorepo of monorepos) { + await monorepo.cleanup(); + } + monorepos = []; + }); + + async function setup(lockfile: string): Promise { + const monorepo = new Monorepo("hasher-lockfile-invalidation"); + await monorepo.init({ packages: { a: {}, b: {}, c: {} } }); + monorepo.writeFiles({ "pnpm-lock.yaml": lockfile }); + monorepos.push(monorepo); + return monorepo; + } + + function createTarget(root: string, packageName: string): Target { + return { + cwd: path.join(root, "packages", packageName), + dependencies: [], + dependents: [], + depSpecs: [], + id: `${packageName}#build`, + label: `${packageName}#build`, + packageName, + task: "build", + }; + } + + function createRootTarget(root: string): Target { + return { + cache: true, + cwd: root, + dependencies: [], + dependents: [], + depSpecs: [], + id: "//#build", + inputs: ["lage.config.js"], + label: "//#build", + task: "build", + }; + } + + async function hashAll(root: string, lockfile: string, environmentGlob: string[] = []): Promise> { + const monorepo = await setup(lockfile); + const hasher = new TargetHasher({ + root: monorepo.root, + environmentGlob, + experimentalLockfileInvalidation: { packageManager: "pnpm" }, + }); + await hasher.initialize(); + const result: Record = {}; + for (const pkg of ["a", "b", "c"]) { + result[pkg] = await hasher.hash(createTarget(monorepo.root, pkg)); + } + hasher.cleanup(); + return result; + } + + async function hashRoot(lockfile: string): Promise { + const monorepo = await setup(lockfile); + const hasher = new TargetHasher({ + root: monorepo.root, + environmentGlob: [], + experimentalLockfileInvalidation: { packageManager: "pnpm" }, + }); + await hasher.initialize(); + const result = await hasher.hash(createRootTarget(monorepo.root)); + hasher.cleanup(); + return result; + } + + it("only changes the hash of packages whose lockfile closure changed", async () => { + const baseHashes = await hashAll("base", baseLockfile); + const changedHashes = await hashAll("changed", changedLockfile); + + // Only package c's closure changed in the lockfile. + expect(changedHashes.a).toEqual(baseHashes.a); + expect(changedHashes.b).toEqual(baseHashes.b); + expect(changedHashes.c).not.toEqual(baseHashes.c); + }); + + it("owns lockfile matches from wildcard environment globs", async () => { + const baseHashes = await hashAll("base", baseLockfile, ["**/*.yaml"]); + const changedHashes = await hashAll("changed", changedLockfile, ["**/*.yaml"]); + + expect(changedHashes.a).toEqual(baseHashes.a); + expect(changedHashes.b).toEqual(baseHashes.b); + expect(changedHashes.c).not.toEqual(baseHashes.c); + }); + + it("produces the same hashes for the same lockfile across hasher instances", async () => { + const first = await hashAll("first", baseLockfile); + const second = await hashAll("second", baseLockfile); + + expect(second).toEqual(first); + }); + + it("changes every package hash when the root importer closure changes", async () => { + const baseHashes = await hashAll("base", baseLockfile); + const changedHashes = await hashAll("changed", rootImporterChangedLockfile); + + expect(changedHashes.a).not.toEqual(baseHashes.a); + expect(changedHashes.b).not.toEqual(baseHashes.b); + expect(changedHashes.c).not.toEqual(baseHashes.c); + }); + + it("changes every package hash when precise lockfile analysis is unsupported", async () => { + const baseHashes = await hashAll("base", unsupportedLockfile); + const changedHashes = await hashAll("changed", `${unsupportedLockfile}\nunknownField: changed\n`); + + expect(changedHashes.a).not.toEqual(baseHashes.a); + expect(changedHashes.b).not.toEqual(baseHashes.b); + expect(changedHashes.c).not.toEqual(baseHashes.c); + }); + + it("changes every package hash when malformed v9 lockfile content changes", async () => { + const baseHashes = await hashAll("base", malformedLockfile); + const changedHashes = await hashAll("changed", `${malformedLockfile}\nunknownField: changed\n`); + + expect(changedHashes.a).not.toEqual(baseHashes.a); + expect(changedHashes.b).not.toEqual(baseHashes.b); + expect(changedHashes.c).not.toEqual(baseHashes.c); + }); + + it("changes every package hash when global installation settings change", async () => { + const baseHashes = await hashAll("base", `settings:\n autoInstallPeers: true\n${baseLockfile}`); + const changedHashes = await hashAll("changed", `settings:\n autoInstallPeers: false\n${baseLockfile}`); + + expect(changedHashes.a).not.toEqual(baseHashes.a); + expect(changedHashes.b).not.toEqual(baseHashes.b); + expect(changedHashes.c).not.toEqual(baseHashes.c); + }); + + it("includes all lockfile importers in cached root target hashes", async () => { + expect(await hashRoot(changedLockfile)).not.toEqual(await hashRoot(baseLockfile)); + expect(await hashRoot(rootImporterChangedLockfile)).not.toEqual(await hashRoot(baseLockfile)); + }); + + it("refreshes lockfile signatures for long-lived hasher instances", async () => { + const monorepo = await setup(baseLockfile); + const hasher = new TargetHasher({ + root: monorepo.root, + environmentGlob: [], + experimentalLockfileInvalidation: { packageManager: "pnpm" }, + }); + await hasher.initialize(); + const target = createTarget(monorepo.root, "c"); + const originalHash = await hasher.hash(target); + + monorepo.writeFiles({ "pnpm-lock.yaml": baseLockfile.replace(/4\.1\.0/g, "4.20.0") }); + hasher.refreshLockfileSignatures(); + + expect(await hasher.hash(target)).not.toEqual(originalHash); + hasher.cleanup(); + }); +}); diff --git a/packages/lockfile/README.md b/packages/lockfile/README.md new file mode 100644 index 000000000..b015fd4e8 --- /dev/null +++ b/packages/lockfile/README.md @@ -0,0 +1,12 @@ +# @lage-run/lockfile + +**Experimental.** Utilities for computing which workspace packages are actually affected by a +package-manager lockfile change, so that Lage's cache hashing and `--since` filtering can avoid +treating every lockfile change as a repo-wide, full-graph invalidation. + +Only **pnpm** is supported, and only the latest pnpm lockfile format (`lockfileVersion 9.0`). +Strict, deterministic lockfiles (like pnpm's) are what make this analysis reliable. For +unsupported package managers or lockfile versions, Lage falls back to its previous blanket +invalidation behavior. + +See the [caching guide](https://microsoft.github.io/lage/docs/guides/cache) for details. diff --git a/packages/lockfile/etc/lockfile.api.md b/packages/lockfile/etc/lockfile.api.md new file mode 100644 index 000000000..3fccb5afb --- /dev/null +++ b/packages/lockfile/etc/lockfile.api.md @@ -0,0 +1,77 @@ +## API Report File for "@lage-run/lockfile" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts + +import type { PackageInfos } from 'workspace-tools'; + +// @public +export function diffPackageSignatures(oldSignatures: ReadonlyMap, newSignatures: ReadonlyMap): Set; + +// @public +export interface ExperimentalLockfileInvalidationOptions { + packageManager: LockfilePackageManager; +} + +// @public +export function getLockfileName(options: ExperimentalLockfileInvalidationOptions): string; + +// @public +export function isSupportedPnpmLockfileVersion(lockfileVersion: string | number | undefined): boolean; + +// @public +export function loadLockfileGraph(options: ExperimentalLockfileInvalidationOptions, root: string): LockfileGraphResult; + +// @public +export function loadPnpmLockfileGraph(root: string): LockfileGraphResult; + +// @public +export interface LockfileGraph { + readonly globalSignature: string; + readonly importerSignatures: ReadonlyMap; +} + +// @public +export type LockfileGraphResult = { + status: "success"; + graph: LockfileGraph; +} | { + status: "no-lockfile"; +} | { + status: "unsupported"; + reason: string; +}; + +// @public +export type LockfilePackageManager = "pnpm"; + +// @public +export function mapImporterSignaturesToPackages(graph: LockfileGraph, packageInfos: PackageInfos, root: string): Map; + +// @public +export interface PackageLockfileSignatures { + // (undocumented) + readonly packageSignatures: ReadonlyMap; + // (undocumented) + readonly unmappedImporterSignatures: ReadonlyMap; +} + +// @public +export function parseLockfileGraph(options: ExperimentalLockfileInvalidationOptions, rawContent: string): LockfileGraphResult; + +// @public +export function parsePnpmLockfileGraph(rawContent: string): LockfileGraphResult; + +// @public +export const PNPM_LOCKFILE_NAME = "pnpm-lock.yaml"; + +// @public +export function splitImporterSignatures(graph: LockfileGraph, packageInfos: PackageInfos, root: string): PackageLockfileSignatures; + +// @public (undocumented) +export const supportedLockfilePackageManagers: readonly LockfilePackageManager[]; + +// (No @packageDocumentation comment for this package) + +``` diff --git a/packages/lockfile/jest.config.js b/packages/lockfile/jest.config.js new file mode 100644 index 000000000..2278faa83 --- /dev/null +++ b/packages/lockfile/jest.config.js @@ -0,0 +1 @@ +module.exports = require("@lage-run/monorepo-scripts/config/jest.config.js"); diff --git a/packages/lockfile/package.json b/packages/lockfile/package.json new file mode 100644 index 000000000..e6e3bcc6c --- /dev/null +++ b/packages/lockfile/package.json @@ -0,0 +1,31 @@ +{ + "name": "@lage-run/lockfile", + "version": "0.1.0", + "description": "Experimental pnpm lockfile analysis for smarter cache and --since invalidation in Lage", + "repository": { + "url": "https://github.com/microsoft/lage" + }, + "license": "MIT", + "main": "lib/index.js", + "types": "lib/index.d.ts", + "scripts": { + "api": "monorepo-scripts api", + "build": "yarn types && yarn transpile", + "transpile": "monorepo-scripts transpile", + "types": "yarn run -T tsc", + "test": "yarn run -T jest", + "lint": "monorepo-scripts lint" + }, + "dependencies": { + "js-yaml": "4.2.0", + "workspace-tools": "workspace:^" + }, + "devDependencies": { + "@lage-run/monorepo-scripts": "workspace:^", + "@types/js-yaml": "4.0.9" + }, + "files": [ + "lib/!(__*)", + "lib/!(__*)/**" + ] +} diff --git a/packages/lockfile/src/__fixtures__/pnpm-lock-base.yaml b/packages/lockfile/src/__fixtures__/pnpm-lock-base.yaml new file mode 100644 index 000000000..74b98b8d6 --- /dev/null +++ b/packages/lockfile/src/__fixtures__/pnpm-lock-base.yaml @@ -0,0 +1,110 @@ +lockfileVersion: "9.0" + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + .: {} + + packages/a: + dependencies: + pkg-b: + specifier: workspace:* + version: link:../b + semver: + specifier: 7.5.4 + version: 7.5.4 + + packages/b: + dependencies: + chalk: + specifier: 4.1.2 + version: 4.1.2 + + packages/c: + dependencies: + js-yaml: + specifier: 4.1.0 + version: 4.1.0 + +packages: + ansi-styles@4.3.0: + resolution: { integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg== } + engines: { node: ">=8" } + + argparse@2.0.1: + resolution: { integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== } + + chalk@4.1.2: + resolution: { integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== } + engines: { node: ">=10" } + + color-convert@2.0.1: + resolution: { integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ== } + engines: { node: ">=7.0.0" } + + color-name@1.1.4: + resolution: { integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== } + + has-flag@4.0.0: + resolution: { integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== } + engines: { node: ">=8" } + + js-yaml@4.1.0: + resolution: { integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA== } + hasBin: true + + lru-cache@6.0.0: + resolution: { integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA== } + engines: { node: ">=10" } + + semver@7.5.4: + resolution: { integrity: sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA== } + engines: { node: ">=10" } + hasBin: true + + supports-color@7.2.0: + resolution: { integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw== } + engines: { node: ">=8" } + + yallist@4.0.0: + resolution: { integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== } + +snapshots: + ansi-styles@4.3.0: + dependencies: + color-convert: 2.0.1 + + argparse@2.0.1: {} + + chalk@4.1.2: + dependencies: + ansi-styles: 4.3.0 + supports-color: 7.2.0 + + color-convert@2.0.1: + dependencies: + color-name: 1.1.4 + + color-name@1.1.4: {} + + has-flag@4.0.0: {} + + js-yaml@4.1.0: + dependencies: + argparse: 2.0.1 + + lru-cache@6.0.0: + dependencies: + yallist: 4.0.0 + + semver@7.5.4: + dependencies: + lru-cache: 6.0.0 + + supports-color@7.2.0: + dependencies: + has-flag: 4.0.0 + + yallist@4.0.0: {} diff --git a/packages/lockfile/src/__fixtures__/pnpm-lock-changed-c.yaml b/packages/lockfile/src/__fixtures__/pnpm-lock-changed-c.yaml new file mode 100644 index 000000000..108c2f27e --- /dev/null +++ b/packages/lockfile/src/__fixtures__/pnpm-lock-changed-c.yaml @@ -0,0 +1,110 @@ +lockfileVersion: "9.0" + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + .: {} + + packages/a: + dependencies: + pkg-b: + specifier: workspace:* + version: link:../b + semver: + specifier: 7.5.4 + version: 7.5.4 + + packages/b: + dependencies: + chalk: + specifier: 4.1.2 + version: 4.1.2 + + packages/c: + dependencies: + js-yaml: + specifier: 4.2.0 + version: 4.2.0 + +packages: + ansi-styles@4.3.0: + resolution: { integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg== } + engines: { node: ">=8" } + + argparse@2.0.1: + resolution: { integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== } + + chalk@4.1.2: + resolution: { integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== } + engines: { node: ">=10" } + + color-convert@2.0.1: + resolution: { integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ== } + engines: { node: ">=7.0.0" } + + color-name@1.1.4: + resolution: { integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== } + + has-flag@4.0.0: + resolution: { integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== } + engines: { node: ">=8" } + + js-yaml@4.2.0: + resolution: { integrity: sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw== } + hasBin: true + + lru-cache@6.0.0: + resolution: { integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA== } + engines: { node: ">=10" } + + semver@7.5.4: + resolution: { integrity: sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA== } + engines: { node: ">=10" } + hasBin: true + + supports-color@7.2.0: + resolution: { integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw== } + engines: { node: ">=8" } + + yallist@4.0.0: + resolution: { integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== } + +snapshots: + ansi-styles@4.3.0: + dependencies: + color-convert: 2.0.1 + + argparse@2.0.1: {} + + chalk@4.1.2: + dependencies: + ansi-styles: 4.3.0 + supports-color: 7.2.0 + + color-convert@2.0.1: + dependencies: + color-name: 1.1.4 + + color-name@1.1.4: {} + + has-flag@4.0.0: {} + + js-yaml@4.2.0: + dependencies: + argparse: 2.0.1 + + lru-cache@6.0.0: + dependencies: + yallist: 4.0.0 + + semver@7.5.4: + dependencies: + lru-cache: 6.0.0 + + supports-color@7.2.0: + dependencies: + has-flag: 4.0.0 + + yallist@4.0.0: {} diff --git a/packages/lockfile/src/__fixtures__/pnpm-lock-cycle.yaml b/packages/lockfile/src/__fixtures__/pnpm-lock-cycle.yaml new file mode 100644 index 000000000..e63563bf9 --- /dev/null +++ b/packages/lockfile/src/__fixtures__/pnpm-lock-cycle.yaml @@ -0,0 +1,24 @@ +lockfileVersion: "9.0" + +importers: + .: {} + + packages/app: + dependencies: + cyclic-a: + specifier: 1.0.0 + version: 1.0.0 + +packages: + cyclic-a@1.0.0: + resolution: { integrity: sha512-aaa } + cyclic-b@1.0.0: + resolution: { integrity: sha512-bbb } + +snapshots: + cyclic-a@1.0.0: + dependencies: + cyclic-b: 1.0.0 + cyclic-b@1.0.0: + dependencies: + cyclic-a: 1.0.0 diff --git a/packages/lockfile/src/__fixtures__/pnpm-lock-simple-base.yaml b/packages/lockfile/src/__fixtures__/pnpm-lock-simple-base.yaml new file mode 100644 index 000000000..c5e845195 --- /dev/null +++ b/packages/lockfile/src/__fixtures__/pnpm-lock-simple-base.yaml @@ -0,0 +1,62 @@ +lockfileVersion: "9.0" + +importers: + .: {} + + packages/a: + dependencies: + pkg-b: + specifier: workspace:* + version: link:../b + semver: + specifier: 7.5.4 + version: 7.5.4 + + packages/b: + dependencies: + chalk: + specifier: 4.1.2 + version: 4.1.2 + + packages/c: + dependencies: + js-yaml: + specifier: 4.1.0 + version: 4.1.0 + +packages: + ansi-styles@4.3.0: + resolution: { integrity: sha512-aaa } + chalk@4.1.2: + resolution: { integrity: sha512-bbb } + color-convert@2.0.1: + resolution: { integrity: sha512-ccc } + color-name@1.1.4: + resolution: { integrity: sha512-ddd } + has-flag@4.0.0: + resolution: { integrity: sha512-eee } + js-yaml@4.1.0: + resolution: { integrity: sha512-fff } + semver@7.5.4: + resolution: { integrity: sha512-ggg } + supports-color@7.2.0: + resolution: { integrity: sha512-hhh } + +snapshots: + ansi-styles@4.3.0: + dependencies: + color-convert: 2.0.1 + chalk@4.1.2: + dependencies: + ansi-styles: 4.3.0 + supports-color: 7.2.0 + color-convert@2.0.1: + dependencies: + color-name: 1.1.4 + color-name@1.1.4: {} + has-flag@4.0.0: {} + js-yaml@4.1.0: {} + semver@7.5.4: {} + supports-color@7.2.0: + dependencies: + has-flag: 4.0.0 diff --git a/packages/lockfile/src/__fixtures__/pnpm-lock-simple-transitive.yaml b/packages/lockfile/src/__fixtures__/pnpm-lock-simple-transitive.yaml new file mode 100644 index 000000000..d6addc2c9 --- /dev/null +++ b/packages/lockfile/src/__fixtures__/pnpm-lock-simple-transitive.yaml @@ -0,0 +1,62 @@ +lockfileVersion: "9.0" + +importers: + .: {} + + packages/a: + dependencies: + pkg-b: + specifier: workspace:* + version: link:../b + semver: + specifier: 7.5.4 + version: 7.5.4 + + packages/b: + dependencies: + chalk: + specifier: 4.1.2 + version: 4.1.2 + + packages/c: + dependencies: + js-yaml: + specifier: 4.1.0 + version: 4.1.0 + +packages: + ansi-styles@4.3.0: + resolution: { integrity: sha512-aaa } + chalk@4.1.2: + resolution: { integrity: sha512-bbb } + color-convert@2.0.1: + resolution: { integrity: sha512-ccc } + color-name@1.1.5: + resolution: { integrity: sha512-ddd2 } + has-flag@4.0.0: + resolution: { integrity: sha512-eee } + js-yaml@4.1.0: + resolution: { integrity: sha512-fff } + semver@7.5.4: + resolution: { integrity: sha512-ggg } + supports-color@7.2.0: + resolution: { integrity: sha512-hhh } + +snapshots: + ansi-styles@4.3.0: + dependencies: + color-convert: 2.0.1 + chalk@4.1.2: + dependencies: + ansi-styles: 4.3.0 + supports-color: 7.2.0 + color-convert@2.0.1: + dependencies: + color-name: 1.1.5 + color-name@1.1.5: {} + has-flag@4.0.0: {} + js-yaml@4.1.0: {} + semver@7.5.4: {} + supports-color@7.2.0: + dependencies: + has-flag: 4.0.0 diff --git a/packages/lockfile/src/__tests__/lockfileGraph.test.ts b/packages/lockfile/src/__tests__/lockfileGraph.test.ts new file mode 100644 index 000000000..997ecb0d5 --- /dev/null +++ b/packages/lockfile/src/__tests__/lockfileGraph.test.ts @@ -0,0 +1,362 @@ +import { describe, expect, it } from "@jest/globals"; +import fs from "fs"; +import path from "path"; +import { + parsePnpmLockfileGraph, + diffPackageSignatures, + mapImporterSignaturesToPackages, + splitImporterSignatures, +} from "../loadLockfileGraph.js"; +import { buildPnpmLockfileGraph, isSupportedPnpmLockfileVersion, type PnpmSnapshot } from "../pnpmLockfileGraph.js"; +import type { LockfileGraph } from "../types.js"; + +const fixturesDir = path.join(__dirname, "..", "__fixtures__"); + +function readFixture(name: string): string { + return fs.readFileSync(path.join(fixturesDir, `${name}.yaml`), "utf8"); +} + +function graphOf(name: string): LockfileGraph { + const result = parsePnpmLockfileGraph(readFixture(name)); + if (result.status !== "success") { + throw new Error(`Expected success but got: ${JSON.stringify(result)}`); + } + return result.graph; +} + +describe("isSupportedPnpmLockfileVersion", () => { + it.each([ + ["9.0", true], + ["9.1", true], + [9, true], + ["6.0", false], + ["5.4", false], + ["8.0", false], + [undefined, false], + ])("returns %s -> %s", (version, expected) => { + expect(isSupportedPnpmLockfileVersion(version as string | number | undefined)).toBe(expected); + }); +}); + +describe("parsePnpmLockfileGraph", () => { + it("parses a valid v9 lockfile and produces per-importer signatures", () => { + const graph = graphOf("pnpm-lock-base"); + expect([...graph.importerSignatures.keys()].sort()).toEqual([".", "packages/a", "packages/b", "packages/c"]); + for (const signature of graph.importerSignatures.values()) { + expect(signature).toMatch(/^[a-f0-9]{40}$/); + } + }); + + it("returns 'unsupported' for an older lockfile version", () => { + const result = parsePnpmLockfileGraph("lockfileVersion: '5.4'\ndependencies: {}\n"); + expect(result.status).toBe("unsupported"); + if (result.status === "unsupported") { + expect(result.reason).toContain("5.4"); + } + }); + + it("returns 'unsupported' for malformed content", () => { + const result = parsePnpmLockfileGraph(":\n : not yaml at all: ["); + expect(result.status).toBe("unsupported"); + }); + + it("returns 'unsupported' for empty content", () => { + const result = parsePnpmLockfileGraph(""); + expect(result.status).toBe("unsupported"); + }); + + it.each([ + ["missing importers", "lockfileVersion: '9.0'\nsnapshots: {}\n"], + [ + "invalid importer dependency", + `lockfileVersion: '9.0' +importers: + packages/a: + dependencies: + semver: 7.5.4 +snapshots: {} +`, + ], + [ + "unresolved importer dependency", + `lockfileVersion: '9.0' +importers: + packages/a: + dependencies: + semver: + specifier: 7.5.4 + version: 7.5.4 +snapshots: {} +`, + ], + [ + "unresolved snapshot dependency", + `lockfileVersion: '9.0' +importers: + packages/a: + dependencies: + semver: + specifier: 7.5.4 + version: 7.5.4 +snapshots: + semver@7.5.4: + dependencies: + missing: 1.0.0 +`, + ], + ])("returns 'unsupported' for %s", (_name, content) => { + expect(parsePnpmLockfileGraph(content).status).toBe("unsupported"); + }); + + it("produces stable signatures across repeated parses", () => { + const first = graphOf("pnpm-lock-simple-base"); + const second = graphOf("pnpm-lock-simple-base"); + expect([...second.importerSignatures]).toEqual([...first.importerSignatures]); + }); +}); + +describe("closure change detection", () => { + it("only changes the affected package's signature for a direct dependency change", () => { + const base = graphOf("pnpm-lock-base"); + const changed = graphOf("pnpm-lock-changed-c"); + + // pkg-c bumped js-yaml; a and b are untouched. + expect(changed.importerSignatures.get("packages/c")).not.toEqual(base.importerSignatures.get("packages/c")); + expect(changed.importerSignatures.get("packages/a")).toEqual(base.importerSignatures.get("packages/a")); + expect(changed.importerSignatures.get("packages/b")).toEqual(base.importerSignatures.get("packages/b")); + }); + + it("propagates a deep transitive change to the consuming package only", () => { + const base = graphOf("pnpm-lock-simple-base"); + const changed = graphOf("pnpm-lock-simple-transitive"); + + // color-name (chalk -> ansi-styles -> color-convert -> color-name) changed; only pkg-b consumes chalk. + expect(changed.importerSignatures.get("packages/b")).not.toEqual(base.importerSignatures.get("packages/b")); + expect(changed.importerSignatures.get("packages/a")).toEqual(base.importerSignatures.get("packages/a")); + expect(changed.importerSignatures.get("packages/c")).toEqual(base.importerSignatures.get("packages/c")); + }); + + it("handles cyclic dependency graphs without infinite recursion", () => { + const graph = graphOf("pnpm-lock-cycle"); + const signature = graph.importerSignatures.get("packages/app"); + expect(signature).toMatch(/^[a-f0-9]{40}$/); + }); + + it("includes package artifact metadata in signatures", () => { + const base = parsePnpmLockfileGraph(`lockfileVersion: '9.0' +importers: + packages/a: + dependencies: + semver: + specifier: 7.5.4 + version: 7.5.4 +packages: + semver@7.5.4: + resolution: + integrity: sha512-base +snapshots: + semver@7.5.4: {} +`); + const changed = parsePnpmLockfileGraph(`lockfileVersion: '9.0' +importers: + packages/a: + dependencies: + semver: + specifier: 7.5.4 + version: 7.5.4 +packages: + semver@7.5.4: + resolution: + integrity: sha512-changed +snapshots: + semver@7.5.4: {} +`); + + expect(base.status).toBe("success"); + expect(changed.status).toBe("success"); + if (base.status === "success" && changed.status === "success") { + expect(changed.graph.importerSignatures.get("packages/a")).not.toEqual(base.graph.importerSignatures.get("packages/a")); + } + }); + + it.each([ + ["peer", "react-dom@18.3.1(react@18.3.1)", "react-dom@18.3.1"], + ["patch", "react-dom@18.3.1(patch_hash=abc123)", "react-dom@18.3.1"], + ["scoped peer", "@scope/pkg@1.0.0(react@18.3.1)", "@scope/pkg@1.0.0"], + ])("associates %s snapshot keys with their package artifact metadata", (_name, snapshotKey, packageKey) => { + const createLockfile = (integrity: string) => `lockfileVersion: '9.0' +importers: + packages/a: + dependencies: + dependency: + specifier: 1.0.0 + version: ${JSON.stringify(snapshotKey)} +packages: + ${JSON.stringify(packageKey)}: + resolution: + integrity: ${integrity} +snapshots: + ${JSON.stringify(snapshotKey)}: {} +`; + const base = parsePnpmLockfileGraph(createLockfile("sha512-base")); + const changed = parsePnpmLockfileGraph(createLockfile("sha512-changed")); + + expect(base.status).toBe("success"); + expect(changed.status).toBe("success"); + if (base.status === "success" && changed.status === "success") { + expect(changed.graph.importerSignatures.get("packages/a")).not.toEqual(base.graph.importerSignatures.get("packages/a")); + } + }); + + it("resolves aliases to the actual snapshot key", () => { + const result = parsePnpmLockfileGraph(`lockfileVersion: '9.0' +importers: + packages/a: + dependencies: + alias: + specifier: npm:real-package@1.0.0 + version: real-package@1.0.0 +snapshots: + real-package@1.0.0: {} +`); + expect(result.status).toBe("success"); + }); + + it("includes top-level installation settings in the global signature", () => { + const base = parsePnpmLockfileGraph(`lockfileVersion: '9.0' +settings: + autoInstallPeers: true +importers: + packages/a: {} +`); + const changed = parsePnpmLockfileGraph(`lockfileVersion: '9.0' +settings: + autoInstallPeers: false +importers: + packages/a: {} +`); + + expect(base.status).toBe("success"); + expect(changed.status).toBe("success"); + if (base.status === "success" && changed.status === "success") { + expect(changed.graph.globalSignature).not.toEqual(base.graph.globalSignature); + } + }); + + it("propagates changes across cyclic dependency components", () => { + const base = parsePnpmLockfileGraph(`lockfileVersion: '9.0' +importers: + packages/app: + dependencies: + cyclic-b: + specifier: 1.0.0 + version: 1.0.0 +snapshots: + cyclic-a@1.0.0: + dependencies: + cyclic-b: 1.0.0 + left-pad: 1.3.0 + cyclic-b@1.0.0: + dependencies: + cyclic-a: 1.0.0 + left-pad@1.3.0: {} +`); + const changed = parsePnpmLockfileGraph(`lockfileVersion: '9.0' +importers: + packages/app: + dependencies: + cyclic-b: + specifier: 1.0.0 + version: 1.0.0 +snapshots: + cyclic-a@1.0.0: + dependencies: + cyclic-b: 1.0.0 + left-pad: 1.3.1 + cyclic-b@1.0.0: + dependencies: + cyclic-a: 1.0.0 + left-pad@1.3.1: {} +`); + + expect(base.status).toBe("success"); + expect(changed.status).toBe("success"); + if (base.status === "success" && changed.status === "success") { + expect(changed.graph.importerSignatures.get("packages/app")).not.toEqual(base.graph.importerSignatures.get("packages/app")); + } + }); + + it("handles a deeply nested dependency graph without overflowing the call stack", () => { + const nodeCount = 20_000; + const snapshots: Record = {}; + for (let index = 0; index < nodeCount; index++) { + snapshots[`dependency-${index}@1.0.0`] = index === nodeCount - 1 ? {} : { dependencies: { [`dependency-${index + 1}`]: "1.0.0" } }; + } + + const graph = buildPnpmLockfileGraph({ + lockfileVersion: "9.0", + importers: { + "packages/app": { + dependencies: { + "dependency-0": { specifier: "1.0.0", version: "1.0.0" }, + }, + }, + }, + snapshots, + }); + + expect(graph.importerSignatures.get("packages/app")).toMatch(/^[a-f0-9]{40}$/); + }); +}); + +describe("mapImporterSignaturesToPackages", () => { + const packageInfos = { + "pkg-a": { packageJsonPath: "/root/packages/a/package.json" }, + "pkg-b": { packageJsonPath: "/root/packages/b/package.json" }, + "pkg-c": { packageJsonPath: "/root/packages/c/package.json" }, + } as any; + + it("maps importer ids to workspace package names and tracks unknown importers separately", () => { + const graph = graphOf("pnpm-lock-base"); + const signatures = mapImporterSignaturesToPackages(graph, packageInfos, "/root"); + const splitSignatures = splitImporterSignatures(graph, packageInfos, "/root"); + + expect([...signatures.keys()].sort()).toEqual(["pkg-a", "pkg-b", "pkg-c"]); + expect([...splitSignatures.packageSignatures.keys()].sort()).toEqual(["pkg-a", "pkg-b", "pkg-c"]); + expect([...splitSignatures.unmappedImporterSignatures.keys()]).toEqual(["."]); + expect(signatures.has(".")).toBe(false); + }); +}); + +describe("diffPackageSignatures", () => { + it("returns packages with changed signatures", () => { + const oldSig = new Map([ + ["pkg-a", "aaa"], + ["pkg-b", "bbb"], + ["pkg-c", "ccc"], + ]); + const newSig = new Map([ + ["pkg-a", "aaa"], + ["pkg-b", "BBB"], + ["pkg-c", "ccc"], + ]); + expect([...diffPackageSignatures(oldSig, newSig)]).toEqual(["pkg-b"]); + }); + + it("treats added and removed packages as changed", () => { + const oldSig = new Map([ + ["pkg-a", "aaa"], + ["pkg-removed", "xxx"], + ]); + const newSig = new Map([ + ["pkg-a", "aaa"], + ["pkg-added", "yyy"], + ]); + expect([...diffPackageSignatures(oldSig, newSig)].sort()).toEqual(["pkg-added", "pkg-removed"]); + }); + + it("returns an empty set when nothing changed", () => { + const sig = new Map([["pkg-a", "aaa"]]); + expect(diffPackageSignatures(sig, new Map(sig)).size).toBe(0); + }); +}); diff --git a/packages/lockfile/src/index.ts b/packages/lockfile/src/index.ts new file mode 100644 index 000000000..55c53ccfd --- /dev/null +++ b/packages/lockfile/src/index.ts @@ -0,0 +1,57 @@ +import { loadPnpmLockfileGraph, parsePnpmLockfileGraph, PNPM_LOCKFILE_NAME } from "./loadLockfileGraph.js"; +import type { ExperimentalLockfileInvalidationOptions, LockfileGraphResult } from "./types.js"; + +export type { + ExperimentalLockfileInvalidationOptions, + LockfileGraph, + LockfileGraphResult, + LockfilePackageManager, + PackageLockfileSignatures, +} from "./types.js"; +export { supportedLockfilePackageManagers } from "./types.js"; +export { + diffPackageSignatures, + loadPnpmLockfileGraph, + mapImporterSignaturesToPackages, + parsePnpmLockfileGraph, + PNPM_LOCKFILE_NAME, + splitImporterSignatures, +} from "./loadLockfileGraph.js"; +export { isSupportedPnpmLockfileVersion } from "./pnpmLockfileGraph.js"; + +/** + * Returns the lockfile file name for the configured package manager. + */ +export function getLockfileName(options: ExperimentalLockfileInvalidationOptions): string { + switch (options.packageManager) { + case "pnpm": + return PNPM_LOCKFILE_NAME; + default: + return PNPM_LOCKFILE_NAME; + } +} + +/** + * Loads and analyzes the current lockfile at the repo root for the configured package manager. + */ +export function loadLockfileGraph(options: ExperimentalLockfileInvalidationOptions, root: string): LockfileGraphResult { + switch (options.packageManager) { + case "pnpm": + return loadPnpmLockfileGraph(root); + default: + return { status: "unsupported", reason: `unsupported package manager "${options.packageManager}"` }; + } +} + +/** + * Parses raw lockfile content for the configured package manager (e.g. an older lockfile obtained + * from `git show :`). + */ +export function parseLockfileGraph(options: ExperimentalLockfileInvalidationOptions, rawContent: string): LockfileGraphResult { + switch (options.packageManager) { + case "pnpm": + return parsePnpmLockfileGraph(rawContent); + default: + return { status: "unsupported", reason: `unsupported package manager "${options.packageManager}"` }; + } +} diff --git a/packages/lockfile/src/loadLockfileGraph.ts b/packages/lockfile/src/loadLockfileGraph.ts new file mode 100644 index 000000000..132c1fd13 --- /dev/null +++ b/packages/lockfile/src/loadLockfileGraph.ts @@ -0,0 +1,221 @@ +import fs from "fs"; +import { createRequire } from "module"; +import path from "path"; +import type jsYaml from "js-yaml"; +import type { PackageInfos } from "workspace-tools"; +import { buildPnpmLockfileGraph, isSupportedPnpmLockfileVersion, type PnpmLockfile } from "./pnpmLockfileGraph.js"; +import type { LockfileGraph, LockfileGraphResult, PackageLockfileSignatures } from "./types.js"; + +/** The default pnpm lockfile file name. */ +export const PNPM_LOCKFILE_NAME = "pnpm-lock.yaml"; + +const requireModule = createRequire(__filename); +let yaml: typeof jsYaml | undefined; + +function parseYaml(rawContent: string): unknown { + yaml ??= requireModule("js-yaml") as typeof jsYaml; + return yaml.load(rawContent); +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function validateStringRecord(value: unknown, context: string): void { + if (!isRecord(value)) { + throw new Error(`${context} must be an object`); + } + for (const [name, entry] of Object.entries(value)) { + if (typeof entry !== "string") { + throw new Error(`${context}.${name} must be a string`); + } + } +} + +function validateImporterDependencies(value: unknown, context: string): void { + if (!isRecord(value)) { + throw new Error(`${context} must be an object`); + } + for (const [name, entry] of Object.entries(value)) { + if (!isRecord(entry) || typeof entry.version !== "string") { + throw new Error(`${context}.${name} must contain a string version`); + } + if (entry.specifier !== undefined && typeof entry.specifier !== "string") { + throw new Error(`${context}.${name}.specifier must be a string`); + } + } +} + +function validatePnpmLockfile(doc: unknown): PnpmLockfile { + if (!isRecord(doc)) { + throw new Error("pnpm lockfile content was empty or malformed"); + } + if (!isRecord(doc.importers)) { + throw new Error("pnpm lockfile importers must be an object"); + } + if (doc.packages !== undefined && !isRecord(doc.packages)) { + throw new Error("pnpm lockfile packages must be an object"); + } + if (doc.snapshots !== undefined && !isRecord(doc.snapshots)) { + throw new Error("pnpm lockfile snapshots must be an object"); + } + + for (const [importerId, importer] of Object.entries(doc.importers)) { + if (!isRecord(importer)) { + throw new Error(`pnpm importer "${importerId}" must be an object`); + } + for (const dependencyType of ["dependencies", "devDependencies", "optionalDependencies"] as const) { + if (importer[dependencyType] !== undefined) { + validateImporterDependencies(importer[dependencyType], `pnpm importer "${importerId}".${dependencyType}`); + } + } + } + + for (const [depPath, metadata] of Object.entries((doc.packages as Record | undefined) ?? {})) { + if (!isRecord(metadata)) { + throw new Error(`pnpm package "${depPath}" must be an object`); + } + } + + for (const [depPath, snapshot] of Object.entries((doc.snapshots as Record | undefined) ?? {})) { + if (!isRecord(snapshot)) { + throw new Error(`pnpm snapshot "${depPath}" must be an object`); + } + for (const dependencyType of ["dependencies", "optionalDependencies"] as const) { + if (snapshot[dependencyType] !== undefined) { + validateStringRecord(snapshot[dependencyType], `pnpm snapshot "${depPath}".${dependencyType}`); + } + } + } + + return doc as PnpmLockfile; +} + +/** + * Parses raw pnpm lockfile content (e.g. read from disk or `git show`) into a {@link LockfileGraph}. + * + * Returns an `unsupported` result (rather than throwing) when the content cannot be parsed or the + * lockfile version is not supported, so callers can safely fall back to blanket invalidation. + */ +export function parsePnpmLockfileGraph(rawContent: string): LockfileGraphResult { + let doc: unknown; + try { + doc = parseYaml(rawContent); + } catch (e) { + return { status: "unsupported", reason: `pnpm lockfile could not be parsed as YAML: ${e}` }; + } + + if (!isRecord(doc)) { + return { status: "unsupported", reason: "pnpm lockfile content was empty or malformed" }; + } + if (!isSupportedPnpmLockfileVersion(doc.lockfileVersion as string | number | undefined)) { + return { + status: "unsupported", + reason: `unsupported pnpm lockfileVersion "${String(doc.lockfileVersion)}"; only the latest format (9.x) is supported`, + }; + } + + let lockfile: PnpmLockfile; + try { + lockfile = validatePnpmLockfile(doc); + } catch (e) { + return { status: "unsupported", reason: `pnpm lockfile schema is not supported: ${e}` }; + } + + let graph: LockfileGraph; + try { + graph = buildPnpmLockfileGraph(lockfile); + } catch (e) { + return { status: "unsupported", reason: `pnpm lockfile could not be interpreted: ${e}` }; + } + + return { status: "success", graph }; +} + +/** + * Loads and analyzes the pnpm lockfile at the repo root. + * + * Returns `no-lockfile` if no lockfile exists, `unsupported` for unsupported versions/parse errors, + * or `success` with the computed {@link LockfileGraph}. + */ +export function loadPnpmLockfileGraph(root: string): LockfileGraphResult { + const lockfilePath = path.join(root, PNPM_LOCKFILE_NAME); + let rawContent: string; + try { + rawContent = fs.readFileSync(lockfilePath, "utf8"); + } catch { + return { status: "no-lockfile" }; + } + + return parsePnpmLockfileGraph(rawContent); +} + +function toPosix(p: string): string { + return p.replace(/\\/g, "/"); +} + +/** + * Builds a map of importer id (posix-relative path from the root) to workspace package name. + */ +function createImporterIdToPackageName(packageInfos: PackageInfos, root: string): Map { + const map = new Map(); + for (const [name, info] of Object.entries(packageInfos)) { + const packageDir = path.dirname(info.packageJsonPath); + const importerId = toPosix(path.relative(root, packageDir)) || "."; + map.set(importerId, name); + } + return map; +} + +/** + * Splits a lockfile graph's per-importer signatures into workspace-package signatures and unmapped + * importer signatures. Unmapped importers (commonly the root importer `"."`) are not safe to ignore: + * root dev tools can affect every package script, so callers should treat changes to these + * signatures as global invalidation. + */ +export function splitImporterSignatures(graph: LockfileGraph, packageInfos: PackageInfos, root: string): PackageLockfileSignatures { + const importerIdToPackageName = createImporterIdToPackageName(packageInfos, root); + const packageSignatures = new Map(); + const unmappedImporterSignatures = new Map(); + + for (const [importerId, signature] of graph.importerSignatures) { + const packageName = importerIdToPackageName.get(importerId); + if (packageName !== undefined) { + packageSignatures.set(packageName, signature); + } else { + unmappedImporterSignatures.set(importerId, signature); + } + } + + return { packageSignatures, unmappedImporterSignatures }; +} + +/** + * Maps a lockfile graph's per-importer signatures to per-workspace-package signatures. + */ +export function mapImporterSignaturesToPackages(graph: LockfileGraph, packageInfos: PackageInfos, root: string): Map { + return new Map(splitImporterSignatures(graph, packageInfos, root).packageSignatures); +} + +/** + * Returns the set of workspace packages whose resolved dependency closure changed between two + * lockfile graphs. A package is considered changed if its signature differs, or if it gained or + * lost a lockfile entry. + */ +export function diffPackageSignatures(oldSignatures: ReadonlyMap, newSignatures: ReadonlyMap): Set { + const changed = new Set(); + + for (const [packageName, signature] of newSignatures) { + if (oldSignatures.get(packageName) !== signature) { + changed.add(packageName); + } + } + + for (const packageName of oldSignatures.keys()) { + if (!newSignatures.has(packageName)) { + changed.add(packageName); + } + } + + return changed; +} diff --git a/packages/lockfile/src/pnpmLockfileGraph.ts b/packages/lockfile/src/pnpmLockfileGraph.ts new file mode 100644 index 000000000..df1e1c54a --- /dev/null +++ b/packages/lockfile/src/pnpmLockfileGraph.ts @@ -0,0 +1,340 @@ +import crypto from "crypto"; +import type { LockfileGraph } from "./types.js"; + +/** Only the latest pnpm lockfile format is supported. */ +const SUPPORTED_MAJOR_LOCKFILE_VERSION = "9"; + +/** + * Resolves a dependency reference (as found in a lockfile) to its relative dep path (the key used + * in the lockfile's `snapshots` map), or `null` for references that do not map to a lockfile entry + * (e.g. workspace `link:` references). + * + * This is a native port of `@pnpm/dependency-path`'s `refToRelative`. It is reimplemented here + * (rather than depending on the package) because `@pnpm/dependency-path` is ESM-only and pulls in a + * heavy dependency tree; the logic itself is small and stable for the supported lockfile format. + */ +export function refToRelative(reference: string, pkgName: string): string | null { + if (reference.startsWith("link:")) { + return null; + } + if (reference[0] === "@") { + return reference; + } + const atIndex = reference.indexOf("@"); + if (atIndex === -1) { + return `${pkgName}@${reference}`; + } + const colonIndex = reference.indexOf(":"); + const bracketIndex = reference.indexOf("("); + if ((colonIndex === -1 || atIndex < colonIndex) && (bracketIndex === -1 || atIndex < bracketIndex)) { + return reference; + } + return `${pkgName}@${reference}`; +} + +/** A resolved snapshot in a pnpm `9.0` lockfile's `snapshots` section. */ +export interface PnpmSnapshot { + dependencies?: Record; + optionalDependencies?: Record; +} + +export type PnpmPackageMetadata = Record; + +export interface PnpmImporterDependency { + specifier?: string; + version: string; +} + +/** An importer (workspace project) entry in a pnpm `9.0` lockfile's `importers` section. */ +export interface PnpmImporter { + dependencies?: Record; + devDependencies?: Record; + optionalDependencies?: Record; + [key: string]: unknown; +} + +/** The parts of a parsed pnpm `9.0` lockfile that are relevant to closure analysis. */ +export interface PnpmLockfile { + lockfileVersion?: string | number; + importers?: Record; + packages?: Record; + snapshots?: Record; + [key: string]: unknown; +} + +/** + * Returns true if the given pnpm `lockfileVersion` is supported by the smarter invalidation logic. + * Only the latest major format (`9.x`, e.g. `"9.0"`) is supported. + */ +export function isSupportedPnpmLockfileVersion(lockfileVersion: string | number | undefined): boolean { + if (lockfileVersion === undefined) { + return false; + } + + const asString = String(lockfileVersion); + const [major] = asString.split("."); + return major === SUPPORTED_MAJOR_LOCKFILE_VERSION; +} + +function hashOrdered(parts: string[]): string { + const hasher = crypto.createHash("sha1"); + for (const part of parts) { + // Length-prefix each part so that concatenation is unambiguous. + hasher.update(String(part.length)); + hasher.update("\0"); + hasher.update(part); + } + return hasher.digest("hex"); +} + +function stableSerialize(value: unknown): string { + if (value === null || typeof value !== "object") { + return JSON.stringify(value); + } + + if (Array.isArray(value)) { + return `[${value.map((item) => stableSerialize(item)).join(",")}]`; + } + + const entries = Object.entries(value as Record) + .filter(([, entryValue]) => entryValue !== undefined) + .sort(([left], [right]) => left.localeCompare(right)); + + return `{${entries.map(([key, entryValue]) => `${JSON.stringify(key)}:${stableSerialize(entryValue)}`).join(",")}}`; +} + +function resolveChildDepPaths(dependencies: Record | undefined): string[] { + if (!dependencies) { + return []; + } + + const depPaths: string[] = []; + for (const [name, reference] of Object.entries(dependencies)) { + // `refToRelative` returns null for workspace links (e.g. `link:../b`), which are internal + // dependencies handled by Lage's package graph rather than the external lockfile closure. + const relative = refToRelative(reference, name); + if (relative) { + depPaths.push(relative); + } + } + return depPaths; +} + +function resolveImporterDepPaths(dependencies: Record | undefined): string[] { + if (!dependencies) { + return []; + } + + const depPaths: string[] = []; + for (const [name, entry] of Object.entries(dependencies)) { + const relative = refToRelative(entry.version, name); + if (relative) { + depPaths.push(relative); + } + } + return depPaths; +} + +function getChildDepPaths(snapshot: PnpmSnapshot | undefined): string[] { + return [...resolveChildDepPaths(snapshot?.dependencies), ...resolveChildDepPaths(snapshot?.optionalDependencies)].sort(); +} + +function getPackageMetadata(depPath: string, packages: Record): PnpmPackageMetadata | undefined { + const exact = packages[depPath]; + if (exact !== undefined) { + return exact; + } + + // pnpm v9 stores peer- and patch-resolved snapshots under suffixed keys such as + // `react-dom@18.3.1(react@18.3.1)` while artifact metadata remains under the base key. + const suffixIndex = depPath.indexOf("("); + return suffixIndex === -1 ? undefined : packages[depPath.slice(0, suffixIndex)]; +} + +function getNodeMetadataHash( + depPath: string, + snapshots: Record, + packages: Record +): string { + return hashOrdered([ + depPath, + stableSerialize(getPackageMetadata(depPath, packages) ?? null), + stableSerialize(snapshots[depPath] ?? null), + ]); +} + +function findStronglyConnectedComponents(depPaths: string[], getChildren: (depPath: string) => string[]): string[][] { + const sortedDepPaths = [...depPaths].sort(); + const childrenByDepPath = new Map(sortedDepPaths.map((depPath) => [depPath, getChildren(depPath)])); + const reverseChildren = new Map(sortedDepPaths.map((depPath) => [depPath, [] as string[]])); + + for (const [depPath, children] of childrenByDepPath) { + for (const child of children) { + reverseChildren.get(child)?.push(depPath); + } + } + for (const parents of reverseChildren.values()) { + parents.sort(); + } + + const visited = new Set(); + const finishOrder: string[] = []; + for (const start of sortedDepPaths) { + if (visited.has(start)) { + continue; + } + + visited.add(start); + const stack: Array<{ depPath: string; childIndex: number }> = [{ depPath: start, childIndex: 0 }]; + while (stack.length > 0) { + const frame = stack[stack.length - 1]; + const children = childrenByDepPath.get(frame.depPath) ?? []; + const child = children[frame.childIndex]; + if (child !== undefined) { + frame.childIndex++; + if (!visited.has(child)) { + visited.add(child); + stack.push({ depPath: child, childIndex: 0 }); + } + } else { + finishOrder.push(frame.depPath); + stack.pop(); + } + } + } + + const assigned = new Set(); + const components: string[][] = []; + for (let index = finishOrder.length - 1; index >= 0; index--) { + const start = finishOrder[index]; + if (assigned.has(start)) { + continue; + } + + assigned.add(start); + const component: string[] = []; + const stack = [start]; + while (stack.length > 0) { + const depPath = stack.pop()!; + component.push(depPath); + for (const parent of reverseChildren.get(depPath) ?? []) { + if (!assigned.has(parent)) { + assigned.add(parent); + stack.push(parent); + } + } + } + components.push(component.sort()); + } + + return components; +} + +/** + * Computes a stable Merkle-style hash for every resolved package (snapshot) in the lockfile. + * + * Each node's hash is derived from its own dep path, package metadata, snapshot metadata, and the + * hashes of its resolved children, so a change anywhere in a package's transitive closure changes + * its hash. Cycles are collapsed into strongly connected components so changes to any member of the + * cycle propagate to every importer that enters the component. + */ +function computeSnapshotHashes( + snapshots: Record, + packages: Record +): Map { + const allDepPaths = Object.keys(snapshots).sort(); + const knownDepPaths = new Set(allDepPaths); + const getKnownChildren = (depPath: string): string[] => { + const children = getChildDepPaths(snapshots[depPath]); + const unresolvedChild = children.find((child) => !knownDepPaths.has(child)); + if (unresolvedChild !== undefined) { + throw new Error(`snapshot "${depPath}" references missing snapshot "${unresolvedChild}"`); + } + return children; + }; + const components = findStronglyConnectedComponents(allDepPaths, getKnownChildren); + const componentByDepPath = new Map(); + + components.forEach((component, componentIndex) => { + for (const depPath of component) { + componentByDepPath.set(depPath, componentIndex); + } + }); + + const componentChildren = components.map(() => new Set()); + const componentParents = components.map(() => new Set()); + for (const [depPath, componentIndex] of componentByDepPath) { + for (const child of getKnownChildren(depPath)) { + const childComponentIndex = componentByDepPath.get(child)!; + if (childComponentIndex !== componentIndex) { + componentChildren[componentIndex].add(childComponentIndex); + componentParents[childComponentIndex].add(componentIndex); + } + } + } + + const componentHashes = new Map(); + const remainingChildren = componentChildren.map((children) => children.size); + const ready = remainingChildren.flatMap((count, componentIndex) => (count === 0 ? [componentIndex] : [])); + while (ready.length > 0) { + const componentIndex = ready.pop()!; + const memberHashes = components[componentIndex].map((depPath) => getNodeMetadataHash(depPath, snapshots, packages)); + const childHashes = [...componentChildren[componentIndex]].map((childIndex) => componentHashes.get(childIndex)!).sort(); + componentHashes.set(componentIndex, hashOrdered(["component", ...memberHashes, ...childHashes])); + + for (const parentIndex of componentParents[componentIndex]) { + remainingChildren[parentIndex]--; + if (remainingChildren[parentIndex] === 0) { + ready.push(parentIndex); + } + } + } + + if (componentHashes.size !== components.length) { + throw new Error("could not hash the pnpm snapshot component graph"); + } + + const nodeHashes = new Map(); + for (const depPath of allDepPaths) { + const componentIndex = componentByDepPath.get(depPath); + if (componentIndex !== undefined) { + nodeHashes.set(depPath, hashOrdered([depPath, componentHashes.get(componentIndex)!])); + } + } + + return nodeHashes; +} + +function computeImporterSignature(importerId: string, importer: PnpmImporter, snapshotHashes: Map): string { + const directDepPaths = [ + ...resolveImporterDepPaths(importer.dependencies), + ...resolveImporterDepPaths(importer.devDependencies), + ...resolveImporterDepPaths(importer.optionalDependencies), + ]; + + const uniqueSorted = [...new Set(directDepPaths)].sort(); + const childHashes = uniqueSorted.map((depPath) => { + const signature = snapshotHashes.get(depPath); + if (signature === undefined) { + throw new Error(`importer "${importerId}" references missing snapshot "${depPath}"`); + } + return signature; + }); + + return hashOrdered([stableSerialize(importer), ...childHashes]); +} + +/** + * Builds a {@link LockfileGraph} from a parsed pnpm lockfile. + */ +export function buildPnpmLockfileGraph(lockfile: PnpmLockfile): LockfileGraph { + const snapshotHashes = computeSnapshotHashes(lockfile.snapshots ?? {}, lockfile.packages ?? {}); + + const importerSignatures = new Map(); + for (const [importerId, importer] of Object.entries(lockfile.importers ?? {})) { + importerSignatures.set(importerId, computeImporterSignature(importerId, importer, snapshotHashes)); + } + + const { importers: _importers, packages: _packages, snapshots: _snapshots, ...globalFields } = lockfile; + return { importerSignatures, globalSignature: hashOrdered([stableSerialize(globalFields)]) }; +} diff --git a/packages/lockfile/src/types.ts b/packages/lockfile/src/types.ts new file mode 100644 index 000000000..6a3f42d2f --- /dev/null +++ b/packages/lockfile/src/types.ts @@ -0,0 +1,47 @@ +/** + * Supported package managers for experimental lockfile invalidation. + * + * Only `pnpm` is supported today. Strict, deterministic lockfiles (like pnpm's) are what make + * the per-package closure analysis reliable. Other package managers intentionally fall back to + * Lage's previous blanket invalidation behavior. + */ +export type LockfilePackageManager = "pnpm"; + +export const supportedLockfilePackageManagers: readonly LockfilePackageManager[] = ["pnpm"]; + +/** + * Experimental opt-in configuration for smarter lockfile invalidation. + */ +export interface ExperimentalLockfileInvalidationOptions { + /** The package manager whose lockfile should be analyzed. Only `"pnpm"` is supported today. */ + packageManager: LockfilePackageManager; +} + +/** + * The result of loading and analyzing a lockfile. + */ +export type LockfileGraphResult = + | { status: "success"; graph: LockfileGraph } + | { status: "no-lockfile" } + | { status: "unsupported"; reason: string }; + +/** + * A lockfile graph exposes, for each workspace project (importer), a single stable signature that + * captures that project's entire resolved external dependency closure. Two lockfiles that resolve + * a given project's dependency graph identically will produce the same signature for that project. + */ +export interface LockfileGraph { + /** Map of importer id (posix-relative path from the repo root) to a closure signature. */ + readonly importerSignatures: ReadonlyMap; + /** Signature of lockfile fields whose changes can affect every importer. */ + readonly globalSignature: string; +} + +/** + * A lockfile graph's importer signatures split into workspace package signatures and signatures for + * importers that do not correspond to a known workspace package (for example the root importer). + */ +export interface PackageLockfileSignatures { + readonly packageSignatures: ReadonlyMap; + readonly unmappedImporterSignatures: ReadonlyMap; +} diff --git a/packages/lockfile/tsconfig.json b/packages/lockfile/tsconfig.json new file mode 100644 index 000000000..b96c504ca --- /dev/null +++ b/packages/lockfile/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "@lage-run/monorepo-scripts/config/tsconfig.base.json", + "compilerOptions": { + "outDir": "./lib" + }, + "include": ["src"] +} diff --git a/packages/workspace-tools/etc/workspace-tools.api.md b/packages/workspace-tools/etc/workspace-tools.api.md index 311d73d97..e9282f73b 100644 --- a/packages/workspace-tools/etc/workspace-tools.api.md +++ b/packages/workspace-tools/etc/workspace-tools.api.md @@ -249,6 +249,12 @@ export function getFullBranchRef(branch: string, cwd: string): string | null; // @public @deprecated (undocumented) export const getInternalDeps: typeof getPackageDependencies; +// @public +export function getMergeBase(params: { + ref: string; + otherRef?: string; +} & GitCommonOptions): string | undefined; + // @public export function getPackageDependencies(info: PackageInfo, internalPackages: Set, options?: PackageDependenciesOptions): string[]; diff --git a/packages/workspace-tools/package.json b/packages/workspace-tools/package.json index e9be3d3f4..17c1e3220 100644 --- a/packages/workspace-tools/package.json +++ b/packages/workspace-tools/package.json @@ -34,7 +34,7 @@ "@lage-run/test-utilities": "workspace:^", "@types/git-url-parse": "^16.0.0", "@types/jju": "^1.4.5", - "@types/js-yaml": "^4.0.5", + "@types/js-yaml": "4.0.9", "@types/micromatch": "^4.0.0", "@types/yarnpkg__lockfile": "^1.1.5", "ts-dedent": "^2.2.0" diff --git a/packages/workspace-tools/src/git/getMergeBase.ts b/packages/workspace-tools/src/git/getMergeBase.ts new file mode 100644 index 000000000..1b8a94d03 --- /dev/null +++ b/packages/workspace-tools/src/git/getMergeBase.ts @@ -0,0 +1,26 @@ +import { git } from "./git.js"; +import type { GitCommonOptions } from "./types.js"; + +/** + * Gets the merge-base (common ancestor) commit SHA between two git refs. + * + * This corresponds to the point that `getBranchChanges` / `git diff ...` compares against, so + * it can be used to read the "old" version of a file consistently with the set of changed files. + * + * Returns undefined if no merge-base exists or the command fails. + */ +export function getMergeBase( + params: { + /** The first ref (e.g. the `--since` target branch). */ + ref: string; + /** The second ref (defaults to `HEAD`). */ + otherRef?: string; + } & GitCommonOptions +): string | undefined { + const { ref, otherRef = "HEAD", ...options } = params; + const result = git(["merge-base", ref, otherRef], { + description: `Getting merge-base of ${ref} and ${otherRef}`, + ...options, + }); + return result.success ? result.stdout.trim() : undefined; +} diff --git a/packages/workspace-tools/src/git/index.ts b/packages/workspace-tools/src/git/index.ts index 8e5f19865..90b242ae7 100644 --- a/packages/workspace-tools/src/git/index.ts +++ b/packages/workspace-tools/src/git/index.ts @@ -19,6 +19,7 @@ export { } from "./getDefaultRemoteBranch.js"; export { getFileAddedHash } from "./getFileAddedHash.js"; export { getFileFromRef } from "./getFileFromRef.js"; +export { getMergeBase } from "./getMergeBase.js"; export { getRecentCommitMessages } from "./getRecentCommitMessages.js"; export { getRemotes } from "./getRemotes.js"; export { diff --git a/scripts/worker/depcheck.js b/scripts/worker/depcheck.js index a29616f2c..63b768f2a 100644 --- a/scripts/worker/depcheck.js +++ b/scripts/worker/depcheck.js @@ -13,6 +13,7 @@ const extraIgnoreMatches = { "@lage-run/monorepo-scripts": ["@typescript-eslint/*", "eslint-plugin-file-extension-in-import-ts", "@types/*"], "@lage-run/rpc": ["@bufbuild/protoc-gen-es", "@connectrpc/protoc-gen-connect-es"], "@lage-run/e2e-tests": ["@lage-run/cli"], + "@lage-run/lockfile": ["js-yaml"], lage: ["@lage-run/cli", "@lage-run/runners"], }; diff --git a/yarn.lock b/yarn.lock index 12bbd907e..4e0a588ab 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1858,6 +1858,7 @@ __metadata: "@lage-run/cache": "workspace:^" "@lage-run/config": "workspace:^" "@lage-run/hasher": "workspace:^" + "@lage-run/lockfile": "workspace:^" "@lage-run/logger": "workspace:^" "@lage-run/monorepo-scripts": "workspace:^" "@lage-run/reporters": "workspace:^" @@ -1889,6 +1890,7 @@ __metadata: version: 0.0.0-use.local resolution: "@lage-run/config@workspace:packages/config" dependencies: + "@lage-run/lockfile": "workspace:^" "@lage-run/logger": "workspace:^" "@lage-run/monorepo-scripts": "workspace:^" "@lage-run/runners": "workspace:^" @@ -1934,6 +1936,7 @@ __metadata: version: 0.0.0-use.local resolution: "@lage-run/hasher@workspace:packages/hasher" dependencies: + "@lage-run/lockfile": "workspace:^" "@lage-run/logger": "workspace:^" "@lage-run/monorepo-scripts": "workspace:^" "@lage-run/target-graph": "workspace:^" @@ -1971,6 +1974,17 @@ __metadata: languageName: unknown linkType: soft +"@lage-run/lockfile@workspace:^, @lage-run/lockfile@workspace:packages/lockfile": + version: 0.0.0-use.local + resolution: "@lage-run/lockfile@workspace:packages/lockfile" + dependencies: + "@lage-run/monorepo-scripts": "workspace:^" + "@types/js-yaml": "npm:4.0.9" + js-yaml: "npm:4.2.0" + workspace-tools: "workspace:^" + languageName: unknown + linkType: soft + "@lage-run/logger@workspace:^, @lage-run/logger@workspace:packages/logger": version: 0.0.0-use.local resolution: "@lage-run/logger@workspace:packages/logger" @@ -2681,7 +2695,7 @@ __metadata: languageName: node linkType: hard -"@types/js-yaml@npm:^4.0.5": +"@types/js-yaml@npm:4.0.9": version: 4.0.9 resolution: "@types/js-yaml@npm:4.0.9" checksum: 10c0/24de857aa8d61526bbfbbaa383aa538283ad17363fcd5bb5148e2c7f604547db36646440e739d78241ed008702a8920665d1add5618687b6743858fae00da211 @@ -9530,7 +9544,7 @@ __metadata: "@lage-run/test-utilities": "workspace:^" "@types/git-url-parse": "npm:^16.0.0" "@types/jju": "npm:^1.4.5" - "@types/js-yaml": "npm:^4.0.5" + "@types/js-yaml": "npm:4.0.9" "@types/micromatch": "npm:^4.0.0" "@types/yarnpkg__lockfile": "npm:^1.1.5" "@yarnpkg/lockfile": "npm:1.1.0"