Skip to content

Commit 200ea75

Browse files
flakey5flakey5
authored
src, scripts: use kv to cache directories (#756)
* src, scripts: use kv to cache directories Signed-off-by: flakey5 <73616808+flakey5@users.noreply.github.com> * review + kv namespace id Signed-off-by: flakey5 <73616808+flakey5@users.noreply.github.com> * build directory cache workflow Signed-off-by: flakey5 <73616808+flakey5@users.noreply.github.com> * review Signed-off-by: flakey5 <73616808+flakey5@users.noreply.github.com> --------- Signed-off-by: flakey5 <73616808+flakey5@users.noreply.github.com>
1 parent df7e214 commit 200ea75

35 files changed

Lines changed: 2108 additions & 466 deletions

‎.env.example‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Your Cloudflare account tag.
2+
#
3+
# Needed for:
4+
# - Directory cache scripts
5+
CLOUDFLARE_ACCOUNT_ID=
6+
7+
# Cloudflare V4 API token.
8+
#
9+
# Needed for:
10+
# - Directory cache scripts
11+
#
12+
# Required permissions:
13+
# - `Workers KV Storage`: Edit
14+
# - `Workers R2 Storage`: Read
15+
#
16+
# See https://developers.cloudflare.com/fundamentals/api/get-started/create-token/
17+
CLOUDFLARE_API_TOKEN=
18+
19+
# S3 credentials for your R2 bucket.
20+
#
21+
# Needed for:
22+
# - Directory listings in the worker.
23+
# - Directory cache scripts
24+
#
25+
# Required permissions:
26+
# - `Object Read Only`
27+
#
28+
# See https://dash.cloudflare.com/?account=/r2/api-tokens
29+
S3_ACCESS_KEY_ID=
30+
S3_ACCESS_KEY_SECRET=

‎.github/workflows/build-directory-cache.yaml‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
name: Build Directory Cache
2+
3+
on:
4+
workflow_dispatch:
5+
6+
jobs:
7+
build-directory-cache:
8+
name: Build Directory Cache
9+
runs-on: ubuntu-latest
10+
11+
steps:
12+
- uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0
13+
with:
14+
egress-policy: audit
15+
16+
- name: Git Checkout
17+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
18+
19+
- name: Cache Dependencies
20+
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
21+
with:
22+
path: |
23+
~/.npm
24+
node_modules/.cache
25+
key: ${{ runner.os }}-npm-${{ hashFiles('**/workflows/format.yml') }}
26+
restore-keys: ${{ runner.os }}-npm-
27+
28+
- name: Setup Node
29+
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
30+
with:
31+
node-version: lts/*
32+
cache: 'npm'
33+
34+
- name: Install dependencies
35+
run: npm ci && npm update nodejs-latest-linker --save
36+
37+
- name: Build Directory Cache
38+
run: node scripts/build-directory-cache.mjs && node --run format
39+
env:
40+
CLOUDFLARE_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
41+
S3_ACCESS_KEY_ID: ${{ secrets.CF_ACCESS_KEY_ID }}
42+
S3_ACCESS_KEY_SECRET: ${{ secrets.CF_SECRET_ACCESS_KEY }}
43+
44+
- name: Commit Changes
45+
id: git_auto_commit
46+
uses: stefanzweifel/git-auto-commit-action@04702edda442b2e678b25b537cec683a1493fcb9 # v7.1.0
47+
with:
48+
commit_options: '--no-verify --no-signoff'
49+
commit_message: 'chore: update redirect links'
50+
branch: build-directory-cache
51+
create_branch: true
52+
53+
- name: Open and Merge Pull Request
54+
if: steps.git_auto_commit.outputs.changes_detected == 'true'
55+
run: |
56+
gh pr create --fill
57+
gh pr merge --squash --delete-branch --admin
58+
env:
59+
GITHUB_TOKEN: ${{ secrets.GH_BOT_TOKEN }}
60+
61+
- name: Deploy to Production
62+
if: steps.git_auto_commit.outputs.changes_detected == 'true'
63+
run: |
64+
gh workflow run deploy.yml
65+
env:
66+
GITHUB_TOKEN: ${{ secrets.GH_BOT_TOKEN }}
67+
68+
- name: Alert on Failure
69+
if: failure() && github.repository == 'nodejs/release-cloudflare-worker'
70+
uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 # 2.3.3
71+
env:
72+
SLACK_COLOR: '#DE512A'
73+
SLACK_ICON: https://github.com/nodejs.png?size=48
74+
SLACK_TITLE: Build Directory Cache failed (${{ github.ref }})
75+
SLACK_MESSAGE: The `build-directory-cache.yaml` action has failed.
76+
SLACK_USERNAME: nodejs-bot
77+
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}

‎.github/workflows/update-links.yml‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,11 @@ permissions:
66
on:
77
# Triggered by https://github.com/nodejs/node/blob/main/.github/workflows/update-release-links.yml
88
workflow_dispatch:
9+
inputs:
10+
version:
11+
description: 'Node.js version (ex/ `v20.0.0`)'
12+
required: true
13+
type: string
914

1015
concurrency:
1116
group: update-redirect-links
@@ -22,6 +27,7 @@ jobs:
2227
egress-policy: block
2328
allowed-endpoints: >
2429
api.github.com:443
30+
api.cloudflare.com:443
2531
dist-prod.07be8d2fbc940503ca1be344714cb0d1.r2.cloudflarestorage.com:443
2632
github.com:443
2733
hooks.slack.com:443
@@ -49,11 +55,13 @@ jobs:
4955
- name: Install dependencies
5056
run: npm ci && npm update nodejs-latest-linker --save
5157

52-
- name: Update Redirect Links
53-
run: node scripts/build-r2-symlinks.mjs && node --run format
58+
- name: Update Directory Cache
59+
run: node scripts/update-directory-cache.mjs "$VERSION_INPUT" && node --run format
5460
env:
55-
CF_ACCESS_KEY_ID: ${{ secrets.CF_ACCESS_KEY_ID }}
56-
CF_SECRET_ACCESS_KEY: ${{ secrets.CF_SECRET_ACCESS_KEY }}
61+
VERSION_INPUT: '${{ inputs.version }}'
62+
CLOUDFLARE_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
63+
S3_ACCESS_KEY_ID: ${{ secrets.CF_ACCESS_KEY_ID }}
64+
S3_ACCESS_KEY_SECRET: ${{ secrets.CF_SECRET_ACCESS_KEY }}
5765

5866
- name: Commit Changes
5967
id: git_auto_commit

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@ node_modules/
33
dist/
44
.dev.vars
55
.sentryclirc
6+
.env

‎e2e-tests/directory.kv.test.ts‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
import { env, createExecutionContext } from 'cloudflare:test';
2+
import { test, beforeAll, expect, vi } from 'vitest';
3+
import {
4+
populateDirectoryCacheWithDevBucket,
5+
populateR2WithDevBucket,
6+
} from './util';
7+
import worker from '../src/worker';
8+
import type { Env } from '../src/env';
9+
import { CACHE_HEADERS } from '../src/constants/cache';
10+
11+
const mockedEnv: Env = {
12+
...env,
13+
ENVIRONMENT: 'e2e-tests',
14+
CACHING: false,
15+
LOG_ERRORS: true,
16+
USE_KV: true,
17+
};
18+
19+
beforeAll(async () => {
20+
await populateR2WithDevBucket();
21+
await populateDirectoryCacheWithDevBucket();
22+
23+
vi.mock(
24+
import('../src/constants/latestVersions.json'),
25+
async importOriginal => {
26+
const original = await importOriginal();
27+
28+
// Point all `latest-` directories to one that exists in the dev bucket
29+
Object.keys(original.default).forEach(branch => {
30+
let updatedValue: string;
31+
if (branch === 'node-latest.tar.gz') {
32+
updatedValue = 'latest/node-v20.0.0.tar.gz';
33+
} else {
34+
updatedValue = 'v20.0.0';
35+
}
36+
37+
// @ts-expect-error
38+
original.default[branch] = updatedValue;
39+
});
40+
41+
return original;
42+
}
43+
);
44+
});
45+
46+
// Ensure essential endpoints are routable
47+
for (const path of ['/dist/', '/docs/', '/api/', '/download/', '/metrics/']) {
48+
test(`GET \`${path}\` returns 200`, async () => {
49+
const ctx = createExecutionContext();
50+
51+
const res = await worker.fetch(
52+
new Request(`https://localhost${path}`),
53+
mockedEnv,
54+
ctx
55+
);
56+
57+
// Consume body promise
58+
await res.text();
59+
60+
expect(res.status).toBe(200);
61+
});
62+
}
63+
64+
test('GET `/dist/unknown-directory/` returns 404', async () => {
65+
const ctx = createExecutionContext();
66+
67+
const res = await worker.fetch(
68+
new Request('https://localhost/dist/unknown-directory/'),
69+
mockedEnv,
70+
ctx
71+
);
72+
73+
expect(res.status).toBe(404);
74+
expect(res.headers.get('cache-control')).toStrictEqual(CACHE_HEADERS.failure);
75+
expect(await res.text()).toStrictEqual('Directory not found');
76+
});
77+
78+
test('GET `/dist` redirects to `/dist/`', async () => {
79+
const ctx = createExecutionContext();
80+
81+
const res = await worker.fetch(
82+
new Request('https://localhost/dist'),
83+
mockedEnv,
84+
ctx
85+
);
86+
87+
expect(res.status).toBe(301);
88+
expect(res.headers.get('location')).toStrictEqual('https://localhost/dist/');
89+
});

e2e-tests/directory.test.ts renamed to e2e-tests/directory.s3.test.ts

File renamed without changes.

‎e2e-tests/util.ts‎

Lines changed: 48 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
import { env } from 'cloudflare:test';
2+
import { join } from 'node:path';
23
import { inject } from 'vitest';
3-
import type { Env } from '../env';
4-
import type { Directory } from '../../vitest-setup';
4+
import type { Env } from '../src/env';
5+
import type { Directory } from '../vitest-setup';
6+
import type { ReadDirectoryResult, File } from '../src/providers/provider';
57

68
async function populateR2BucketDirectory(directory: Directory): Promise<void> {
79
const promises: Array<Promise<unknown>> = [];
@@ -10,7 +12,7 @@ async function populateR2BucketDirectory(directory: Directory): Promise<void> {
1012
const file = directory.files[path];
1113

1214
promises.push(
13-
env.R2_BUCKET.put(path, file.contents, {
15+
env.R2_BUCKET.put(join(directory.name, path), file.contents, {
1416
customMetadata: {
1517
// This is added by rclone when copying the release assets to the
1618
// bucket.
@@ -27,6 +29,41 @@ async function populateR2BucketDirectory(directory: Directory): Promise<void> {
2729
await Promise.all(promises);
2830
}
2931

32+
async function populateDirectoryCache(directory: Directory): Promise<void> {
33+
let hasIndexHtmlFile = false;
34+
35+
const files: File[] = Object.keys(directory.files).map(name => {
36+
const file = directory.files[name];
37+
38+
if (!hasIndexHtmlFile && name.match(/index.htm(?:l)$/)) {
39+
hasIndexHtmlFile = true;
40+
}
41+
42+
return {
43+
name,
44+
lastModified: new Date(file.lastModified),
45+
size: file.size,
46+
};
47+
});
48+
49+
const cachedDirectory: ReadDirectoryResult = {
50+
subdirectories: Object.keys(directory.subdirectories),
51+
files,
52+
hasIndexHtmlFile,
53+
lastModified: new Date(),
54+
};
55+
56+
const promises: Array<Promise<void>> = [
57+
env.DIRECTORY_CACHE.put(
58+
`${directory.name}/`,
59+
JSON.stringify(cachedDirectory)
60+
),
61+
...Object.values(directory.subdirectories).map(populateDirectoryCache),
62+
];
63+
64+
await Promise.all(promises);
65+
}
66+
3067
/**
3168
* Writes the contents of the dev bucket into the R2 bucket given in {@link env}
3269
*/
@@ -38,6 +75,14 @@ export async function populateR2WithDevBucket(): Promise<void> {
3875
await populateR2BucketDirectory(devBucket);
3976
}
4077

78+
export async function populateDirectoryCacheWithDevBucket(): Promise<void> {
79+
// Grab the contents of the dev bucket
80+
const devBucket = inject('devBucket');
81+
82+
// Write it to KV
83+
await populateDirectoryCache(devBucket);
84+
}
85+
4186
declare module 'cloudflare:test' {
4287
// eslint-disable-next-line @typescript-eslint/no-empty-object-type
4388
interface ProvidedEnv extends Env {}

‎lib/README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# `lib/`
2+
3+
Utilities used in local scripts and in the deployed worker.

‎lib/limits.mjs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
/**
2+
* Max amount of retries for requests to R2
3+
*/
4+
export const R2_RETRY_LIMIT = 5;
5+
6+
/**
7+
* Max amount of retries for requests to KV
8+
*/
9+
export const KV_RETRY_LIMIT = 5;
10+
11+
/**
12+
* Max amount of keys to be returned in a S3 request
13+
*/
14+
export const S3_MAX_KEYS = 1000;
15+
16+
/**
17+
* Max amount of keys we can have in a KV request
18+
*/
19+
export const KV_MAX_KEYS = 10_000;

0 commit comments

Comments
 (0)