Move Azure cache implementation to plugin

change/change-922e3e3d-8503-4578-9876-071040126335.json

@@ -0,0 +1,53 @@
+{
+  "changes": [
+    {
+      "type": "minor",
+      "comment": "New plugin package for Azure Blob Storage cache",
+      "packageName": "@lage-run/azure-blob-cache-storage",
+      "email": "email not defined",
+      "dependentChangeType": "patch"
+    },
+    {
+      "type": "major",
+      "comment": "Move Azure Blob cache to plugin; remove built-in azure-blob provider",
+      "packageName": "backfill-cache",
+      "email": "email not defined",
+      "dependentChangeType": "patch"
+    },
+    {
+      "type": "major",
+      "comment": "Remove AzureBlobCacheStorageConfig and CustomStorageConfig from CacheStorageConfig union; add CustomCacheStoragePlugin and CustomCacheStorageConfig types",
+      "packageName": "backfill-config",
+      "email": "email not defined",
+      "dependentChangeType": "patch"
+    },
+    {
+      "type": "minor",
+      "comment": "Support custom plugin providers via isCustomPluginProvider",
+      "packageName": "backfill",
+      "email": "email not defined",
+      "dependentChangeType": "patch"
+    },
+    {
+      "type": "major",
+      "comment": "Remove Azure credential handling; credentials are now managed by the @lage-run/azure-blob-cache-storage plugin",
+      "packageName": "@lage-run/cache",
+      "email": "email not defined",
+      "dependentChangeType": "patch"
+    },
+    {
+      "type": "major",
+      "comment": "Remove AzureCredentialName export and Azure-specific type augmentation from CacheOptions",
+      "packageName": "@lage-run/config",
+      "email": "email not defined",
+      "dependentChangeType": "patch"
+    },
+    {
+      "type": "major",
+      "comment": "Migrate from CustomStorageConfig to CustomCacheStoragePlugin pattern",
+      "packageName": "@lage-run/cache-github-actions",
+      "email": "email not defined",
+      "dependentChangeType": "patch"
+    }
+  ]
+}

docs/docs/guides/remote-cache.md

@@ -12,17 +12,23 @@ The theory is that when the CI job runs, it'll produce a "last known good" cache
 
 ## Setting up remote cache - Azure Blob Storage
 
-Follow these steps to set up a remote cache.
+Azure Blob Storage cache is available as a plugin: `@lage-run/azure-blob-cache-storage`. This plugin must be installed separately.
 
-### 1. Upgrade to latest `lage`
+### 1. Install the plugin
+
+```
+yarn add @lage-run/azure-blob-cache-storage
+```
+
+### 2. Upgrade to latest `lage`
 
 See the [migration guide](../cookbook/migration.mdx) for more details.
 
 ```
 yarn upgrade lage
 ```
 
-### 2. Create `.env` and add to `.gitignore`
+### 3. Create `.env` and add to `.gitignore`
 
 Create the file:
 
@@ -39,7 +45,7 @@ lib
 dist
 ```
 
-### 3. Generate auth tokens from Azure storage account
+### 4. Generate auth tokens from Azure storage account
 
 Prerequisite is to have a working Storage Account with Blob Storage Container created. Note that container name, it'll be needed for Step 5.
 
@@ -51,7 +57,7 @@ Prerequisite is to have a working Storage Account with Blob Storage Container cr
 6. Click "show keys"
 7. Save the "connection string" - this is your **read-write** connection string (alternatively, you can create a read-write SAS connection string)
 
-### 4. Modify the `.env` file with the remote cache connection information
+### 5. Modify the `.env` file with the remote cache connection information
 
 ```txt title=".env"
 ## This is required as of right now
@@ -61,7 +67,24 @@ BACKFILL_CACHE_PROVIDER="azure-blob"
 BACKFILL_CACHE_PROVIDER_OPTIONS={"connectionString":"the **read-only** connection string","container":"CONTAINER NAME"}
 ```
 
-### 5. Create a "secret" in the CI system for a Read/Write token
+Alternatively, you can configure it directly in `lage.config.js`:
+
+```js title="lage.config.js"
+module.exports = {
+  cacheOptions: {
+    cacheStorageConfig: {
+      provider: "custom",
+      plugin: "@lage-run/azure-blob-cache-storage",
+      options: {
+        connectionString: "...",
+        container: "...",
+      },
+    },
+  },
+};
+```
+
+### 6. Create a "secret" in the CI system for a Read/Write token
 
 Here's an example snippet of Github Action with the correct environment variable set:
 
@@ -81,7 +104,7 @@ Create a secret named "BACKFILL_CACHE_PROVIDER_OPTIONS":
 
 `process.env.BACKFILL_CACHE_PROVIDER_OPTIONS`is evaluated via backfill (see [`getEnvConfig()`](https://github.com/microsoft/lage/blob/master/packages/backfill-config/src/envConfig.ts#L82) in `backfill-config`).
 
-For "azure-blob" cache provider with a non-sas/key-based `connectionString`(storage account endpoint) requiring azure identity authentication do not use `BACKFILL_CACHE_PROVIDER_OPTIONS`, instead populate the required env variables according to the desired identity/environment. (See [Azure Idenity SDK](https://learn.microsoft.com/en-us/javascript/api/overview/azure/identity-readme)) and set `credentialName` property in the `lage.config.js` under `cacheOptions.cacheStorageConfig.options.credentialName` or via env var `AZURE_IDENTITY_CREDENTIAL_NAME` Supported options are:
+For the Azure Blob cache provider with a non-sas/key-based `connectionString` (storage account endpoint) requiring [Azure Identity](https://learn.microsoft.com/en-us/javascript/api/overview/azure/identity-readme) authentication, you can pass a `credentialName` option in the plugin config or via the `AZURE_IDENTITY_CREDENTIAL_NAME` environment variable. (Do not use `BACKFILL_CACHE_PROVIDER_OPTIONS` in this case.) Supported options are:
 
 - `"azure-cli"`
 - `"managed-identity"`

docs/docs/reference/config.md

@@ -69,9 +69,10 @@ const config = {
   cacheOptions: {
     /** @see https://www.npmjs.com/package/backfill#configuration */
     cacheStorageConfig: {
-      // use this to specify a remote cache provider such as "azure-blob",
-      provider: "azure-blob",
-      // there are specific options here for each cache provider
+      // use this to specify a remote cache plugin such as "@lage-run/azure-blob-cache-storage",
+      provider: "custom",
+      plugin: "@lage-run/azure-blob-cache-storage",
+      // there are specific options here for each cache plugin
       options: {}
     },
 

packages/azure-blob-cache-storage/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "@lage-run/azure-blob-cache-storage",
+  "version": "0.1.0",
+  "description": "Azure Blob Storage cache plugin for backfill/lage",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/microsoft/lage"
+  },
+  "homepage": "https://microsoft.github.io/lage/",
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "scripts": {
+    "build": "yarn types && yarn transpile",
+    "transpile": "monorepo-scripts transpile",
+    "types": "yarn run -T tsc",
+    "lint": "monorepo-scripts lint"
+  },
+  "dependencies": {
+    "@azure/core-auth": "1.9.0",
+    "@azure/identity": "4.9.1",
+    "@azure/storage-blob": "12.27.0",
+    "backfill-cache": "workspace:^",
+    "backfill-config": "workspace:^",
+    "backfill-logger": "workspace:^",
+    "fs-extra": "8.1.0",
+    "tar-fs": "2.1.4"
+  },
+  "devDependencies": {
+    "@lage-run/monorepo-scripts": "workspace:^",
+    "@types/fs-extra": "^8.0.0",
+    "@types/tar-fs": "^2.0.1"
+  },
+  "engines": {
+    "node": ">=14"
+  },
+  "files": [
+    "lib/!(__*)",
+    "lib/!(__*)/**"
+  ]
+}

packages/backfill-cache/src/AzureBlobCacheStorage.ts → packages/azure-blob-cache-storage/src/AzureBlobCacheStorage.ts

@@ -7,7 +7,7 @@ import type { AzureBlobCacheStorageOptions } from "backfill-config";
 
 import { stat } from "fs-extra";
 import type { ContainerClient } from "@azure/storage-blob";
-import { CacheStorage } from "./CacheStorage.js";
+import { CacheStorage } from "backfill-cache";
 
 const ONE_MEGABYTE = 1024 * 1024;
 const FOUR_MEGABYTES = 4 * ONE_MEGABYTE;

packages/cache/src/CredentialCache.ts → packages/azure-blob-cache-storage/src/CredentialCache.ts

@@ -6,7 +6,13 @@ import {
   EnvironmentCredential,
   WorkloadIdentityCredential,
 } from "@azure/identity";
-import type { AzureCredentialName } from "@lage-run/config";
+
+/**
+ * Allowed credential names matching camelCase of @azure/identity credential class names
+ *  @see https://learn.microsoft.com/en-us/azure/developer/javascript/sdk/authentication/credential-chains
+ */
+export type AzureCredentialName = "environment" | "workload-identity" | "managed-identity" | "visual-studio-code" | "azure-cli";
+
 /**
  * Exhaustive credential factory map keyed by AzureCredentialName.
  * This enforces compile-time alignment with the AzureCredentialName union and provides a single source of truth.

packages/azure-blob-cache-storage/src/createPlugin.ts

@@ -0,0 +1,48 @@
+import type { Logger } from "backfill-logger";
+import type {
+  ICacheStorage,
+  CustomCacheStoragePlugin,
+  AzureBlobCacheStorageOptions,
+  AzureBlobCacheStorageConnectionStringOptions,
+} from "backfill-config";
+
+import { AzureBlobCacheStorage } from "./AzureBlobCacheStorage.js";
+import { CredentialCache, type AzureCredentialName } from "./CredentialCache.js";
+
+export type AzureBlobPluginOptions = AzureBlobCacheStorageOptions & {
+  /** Optional credential name for Azure Identity authentication. */
+  credentialName?: AzureCredentialName;
+};
+
+function isTokenConnectionString(connectionString: string) {
+  return connectionString.includes("SharedAccessSignature") || connectionString.includes("AccountKey");
+}
+
+const plugin: CustomCacheStoragePlugin<AzureBlobPluginOptions> = {
+  name: "azure-blob",
+  getProvider(logger: Logger, cwd: string, options: AzureBlobPluginOptions): ICacheStorage {
+    // Handle credential injection for connection-string-based options
+    if ("connectionString" in options && !isTokenConnectionString(options.connectionString)) {
+      const connStringOptions = options as AzureBlobCacheStorageConnectionStringOptions & { credentialName?: AzureCredentialName };
+      if (!connStringOptions.credential) {
+        const credName = connStringOptions.credentialName
+          ?? (process.env.AZURE_IDENTITY_CREDENTIAL_NAME || undefined);
+
+        if (credName != null) {
+          if (!CredentialCache.credentialNames.includes(credName as AzureCredentialName)) {
+            throw new Error(
+              `Invalid credentialName: "${credName}". Allowed values: ${CredentialCache.credentialNames.join(", ")}`
+            );
+          }
+          connStringOptions.credential = CredentialCache.getInstance(credName as AzureCredentialName);
+        } else {
+          connStringOptions.credential = CredentialCache.getInstance();
+        }
+      }
+    }
+
+    return new AzureBlobCacheStorage(options, logger, cwd);
+  },
+};
+
+export default plugin;

packages/azure-blob-cache-storage/src/index.ts

@@ -0,0 +1,4 @@
+export type { AzureCredentialName } from "./CredentialCache.js";
+export { CredentialCache } from "./CredentialCache.js";
+export type { AzureBlobPluginOptions } from "./createPlugin.js";
+export { default } from "./createPlugin.js";

packages/azure-blob-cache-storage/tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "extends": "@lage-run/monorepo-scripts/config/tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "lib"
+  },
+  "include": ["src"]
+}

packages/backfill-cache/package.json

@@ -18,7 +18,6 @@
     "test": "yarn run -T jest"
   },
   "dependencies": {
-    "@azure/storage-blob": "12.27.0",
     "@lage-run/globby": "workspace:^",
     "backfill-config": "workspace:^",
     "backfill-logger": "workspace:^",

packages/backfill-cache/src/__tests__/getCacheStorageProvider.test.ts

@@ -1,7 +1,5 @@
-import { type Logger, makeLogger } from "backfill-logger";
+import { makeLogger } from "backfill-logger";
 import { getCacheStorageProvider } from "../getCacheStorageProvider.js";
-import type { ICacheStorage } from "../CacheStorage.js";
-import { AzureBlobCacheStorage } from "../AzureBlobCacheStorage.js";
 import { LocalCacheStorage } from "../LocalCacheStorage.js";
 
 describe("getCacheStorageProvider", () => {
@@ -65,54 +63,18 @@ describe("getCacheStorageProvider", () => {
     expect(provider instanceof LocalCacheStorage).toBeTruthy();
   });
 
-  test("can get an azure-blob storage provider", () => {
-    const provider = getCacheStorageProvider(
-      {
-        provider: "azure-blob",
-        options: {
-          connectionString: "some connection string",
-          container: "some container",
+  test("throws when custom plugin cannot be loaded", () => {
+    expect(() =>
+      getCacheStorageProvider(
+        {
+          provider: "custom",
+          plugin: "nonexistent-plugin-package",
+          options: {},
         },
-      },
-      "test",
-      makeLogger("silly"),
-      "cwd"
-    );
-
-    expect(provider instanceof AzureBlobCacheStorage).toBeTruthy();
-  });
-
-  test("can get a custom storage provider as a class", () => {
-    const TestProvider = class implements ICacheStorage {
-      constructor(
-        private logger: Logger,
-        private cwd: string
-      ) {}
-
-      public fetch(hash: string) {
-        this.logger.silly(`fetching ${this.cwd} ${hash}`);
-        return Promise.resolve(true);
-      }
-
-      public put(hash: string, filesToCache: string[]) {
-        this.logger.silly(
-          `putting ${this.cwd} ${hash} ${filesToCache.length} files`
-        );
-        return Promise.resolve();
-      }
-    };
-
-    const provider = getCacheStorageProvider(
-      {
-        provider: (logger, cwd) => new TestProvider(logger, cwd),
-        name: "test-provider",
-      },
-      "test",
-      makeLogger("silly"),
-      "cwd"
-    );
-
-    expect(provider.fetch).toBeTruthy();
-    expect(provider.put).toBeTruthy();
+        "test",
+        makeLogger("silly"),
+        "cwd"
+      )
+    ).toThrow('Failed to load custom cache storage plugin "nonexistent-plugin-package"');
   });
 });