Skip to content

Update Vercel queue delay cap for longer sleeps #1604

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pranaygp wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Update Vercel queue delay cap for longer sleeps #1604

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
563c84b
Update Vercel queue delay cap for longer sleeps
pranaygp Apr 3, 2026
6b2568f
Validate Vercel queue max delay override
pranaygp Apr 3, 2026
407954e
Adjust Vercel queue re-enqueue margin
pranaygp Apr 3, 2026
da34354
Merge branch 'main' into pranaygp/codex/update-vercel-queue-7d-delay
pranaygp Apr 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/strong-guests-study.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@workflow/world-vercel': patch
---

Update Vercel queue max delay for longer sleeps without having to re-enqueue as frequently
118 changes: 114 additions & 4 deletions packages/world-vercel/src/queue.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ vi.mock('./utils.js', () => ({
getHeaders: vi.fn().mockReturnValue(new Map()),
}));

import { createQueue } from './queue.js';
import { createQueue, MAX_DELAY_SECONDS } from './queue.js';

describe('createQueue', () => {
beforeEach(() => {
Expand Down Expand Up @@ -404,7 +404,7 @@ describe('createQueue', () => {
}
});

it('should clamp delaySeconds to max 23 hours for long sleeps', async () => {
it('should clamp delaySeconds to 1 hour less than 7 days for long sleeps', async () => {
mockSend.mockResolvedValue({ messageId: 'new-msg-123' });

let capturedHandler: (
Expand All @@ -422,7 +422,7 @@ describe('createQueue', () => {
try {
const queue = createQueue();
queue.createQueueHandler('__wkf_workflow_', async () => ({
timeoutSeconds: 100000,
timeoutSeconds: 700000,
}));

await capturedHandler!(
Expand All @@ -443,7 +443,7 @@ describe('createQueue', () => {
expect(mockSend).toHaveBeenCalledTimes(1);
// send(topicName, payload, options)
const sendOpts = mockSend.mock.calls[0][2];
expect(sendOpts.delaySeconds).toBe(82800); // MAX_DELAY_SECONDS
expect(sendOpts.delaySeconds).toBe(MAX_DELAY_SECONDS);
} finally {
if (originalEnv !== undefined) {
process.env.VERCEL_DEPLOYMENT_ID = originalEnv;
Expand All @@ -453,6 +453,116 @@ describe('createQueue', () => {
}
});

it('should fall back to the default max delay when the env override is invalid', async () => {
mockSend.mockResolvedValue({ messageId: 'new-msg-123' });

let capturedHandler: (
message: unknown,
metadata: unknown
) => Promise<void>;
mockHandleCallback.mockImplementation((handler) => {
capturedHandler = handler;
return async () => new Response('ok');
});

const originalDeploymentId = process.env.VERCEL_DEPLOYMENT_ID;
const originalMaxDelay = process.env.VERCEL_QUEUE_MAX_DELAY_SECONDS;
process.env.VERCEL_DEPLOYMENT_ID = 'dpl_test';
process.env.VERCEL_QUEUE_MAX_DELAY_SECONDS = 'not-a-number';

try {
const queue = createQueue();
queue.createQueueHandler('__wkf_workflow_', async () => ({
timeoutSeconds: 700000,
}));

await capturedHandler!(
{
payload: { runId: 'run-123' },
queueName: '__wkf_workflow_test',
deploymentId: 'dpl_original',
},
{
messageId: 'msg-123',
deliveryCount: 1,
createdAt: new Date(),
topicName: '__wkf_workflow_test',
consumerGroup: 'test',
}
);

const sendOpts = mockSend.mock.calls[0][2];
expect(sendOpts.delaySeconds).toBe(MAX_DELAY_SECONDS);
} finally {
if (originalDeploymentId !== undefined) {
process.env.VERCEL_DEPLOYMENT_ID = originalDeploymentId;
} else {
delete process.env.VERCEL_DEPLOYMENT_ID;
}

if (originalMaxDelay !== undefined) {
process.env.VERCEL_QUEUE_MAX_DELAY_SECONDS = originalMaxDelay;
} else {
delete process.env.VERCEL_QUEUE_MAX_DELAY_SECONDS;
}
}
});

it('should clamp oversized env overrides to the default max delay', async () => {
mockSend.mockResolvedValue({ messageId: 'new-msg-123' });

let capturedHandler: (
message: unknown,
metadata: unknown
) => Promise<void>;
mockHandleCallback.mockImplementation((handler) => {
capturedHandler = handler;
return async () => new Response('ok');
});

const originalDeploymentId = process.env.VERCEL_DEPLOYMENT_ID;
const originalMaxDelay = process.env.VERCEL_QUEUE_MAX_DELAY_SECONDS;
process.env.VERCEL_DEPLOYMENT_ID = 'dpl_test';
process.env.VERCEL_QUEUE_MAX_DELAY_SECONDS = `${MAX_DELAY_SECONDS + 1}`;

try {
const queue = createQueue();
queue.createQueueHandler('__wkf_workflow_', async () => ({
timeoutSeconds: 700000,
}));

await capturedHandler!(
{
payload: { runId: 'run-123' },
queueName: '__wkf_workflow_test',
deploymentId: 'dpl_original',
},
{
messageId: 'msg-123',
deliveryCount: 1,
createdAt: new Date(),
topicName: '__wkf_workflow_test',
consumerGroup: 'test',
}
);

const sendOpts = mockSend.mock.calls[0][2];
expect(sendOpts.delaySeconds).toBe(MAX_DELAY_SECONDS);
} finally {
if (originalDeploymentId !== undefined) {
process.env.VERCEL_DEPLOYMENT_ID = originalDeploymentId;
} else {
delete process.env.VERCEL_DEPLOYMENT_ID;
}

if (originalMaxDelay !== undefined) {
process.env.VERCEL_QUEUE_MAX_DELAY_SECONDS = originalMaxDelay;
} else {
delete process.env.VERCEL_QUEUE_MAX_DELAY_SECONDS;
}
}
});

it('should send new message without delaySeconds when handler returns timeoutSeconds: 0', async () => {
mockSend.mockResolvedValue({ messageId: 'new-msg-123' });

Expand Down
38 changes: 29 additions & 9 deletions packages/world-vercel/src/queue.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,12 @@ const MessageWrapper = z.object({
* rather than using visibility timeouts on the same message.
*
* Benefits of this approach:
* - Fresh 24-hour lifetime with each message (no message age tracking needed)
* - Fresh delay window with each message (no message age tracking needed)
* - Messages fire at the scheduled time (no short-circuit + recheck pattern)
* - Simpler conceptual model: messages are triggers with delivery schedules
*
* For sleeps > 24 hours (max delay), we use chaining:
* 1. Schedule message with max delay (~23h, leaving buffer)
* For sleeps > 7 days (max delay), we use chaining:
* 1. Schedule message with max delay (~6d, leaving a 24h re-enqueue margin)
* 2. When it fires, workflow checks if sleep is complete
* 3. If not, another delayed message is queued for remaining time
* 4. Process repeats until the full sleep duration has elapsed
Expand All @@ -48,9 +48,28 @@ const MessageWrapper = z.object({
*
* These constants can be overridden via environment variables for testing.
*/
const MAX_DELAY_SECONDS = Number(
process.env.VERCEL_QUEUE_MAX_DELAY_SECONDS || 82800 // 23 hours - leave 1h buffer before 24h retention limit
);
const SECONDS_PER_HOUR = 60 * 60;
Copy link
Copy Markdown
Member

@VaguelySerious VaguelySerious Apr 3, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should actually make this one day. Practically, there'll be no performance or cost difference, BUT if there's infra downtime, and it happens to be the "right" minute, this gives us a bigger margin for queue to re-drive before it gives up? Would have to think through this more deeply but seems like a generally safer approach

Suggested change
const SECONDS_PER_HOUR = 60 * 60;
const RE_ENQUEUE_MARGIN_SECONDS = 24 * 3600; // 24 hours

Copy link
Copy Markdown
Collaborator Author

@pranaygp pranaygp Apr 3, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Makes sense — applied in 407954ea. The re-enqueue margin is now 24 hours, which brings the effective max delayed send to 6 days and gives more room for queue redrive if there is infra turbulence near the delivery boundary.

export const RE_ENQUEUE_MARGIN_SECONDS = 24 * SECONDS_PER_HOUR; // 24 hours
export const MAX_QUEUE_DELAY_WINDOW_SECONDS = 7 * 24 * SECONDS_PER_HOUR; // 7 days
export const MAX_DELAY_SECONDS =
MAX_QUEUE_DELAY_WINDOW_SECONDS - RE_ENQUEUE_MARGIN_SECONDS;

function getMaxDelaySeconds(): number {
const rawMaxDelaySeconds = process.env.VERCEL_QUEUE_MAX_DELAY_SECONDS;
if (
rawMaxDelaySeconds === undefined ||
rawMaxDelaySeconds.trim().length === 0
) {
return MAX_DELAY_SECONDS;
}

const parsedMaxDelaySeconds = Number(rawMaxDelaySeconds);
if (!Number.isFinite(parsedMaxDelaySeconds) || parsedMaxDelaySeconds < 0) {
return MAX_DELAY_SECONDS;
}

return Math.min(Math.floor(parsedMaxDelaySeconds), MAX_DELAY_SECONDS);
}

/**
* Extract known identifiers from a queue payload and return them as VQS headers.
Expand Down Expand Up @@ -191,11 +210,12 @@ export function createQueue(config?: APIConfig): Queue {

if (typeof result?.timeoutSeconds === 'number') {
// When timeoutSeconds is 0, skip delaySeconds entirely for immediate re-enqueue.
// Otherwise, clamp to max delay (23h) - for longer sleeps, the workflow will chain
// multiple delayed messages until the full sleep duration has elapsed.
// Otherwise, clamp to the queue delay window minus a 24h buffer (6d).
// For longer sleeps, the workflow will chain multiple delayed messages until
// the full sleep duration has elapsed.
const delaySeconds =
result.timeoutSeconds > 0
? Math.min(result.timeoutSeconds, MAX_DELAY_SECONDS)
? Math.min(result.timeoutSeconds, getMaxDelaySeconds())
: undefined;

// Send new message BEFORE acknowledging current message.
Expand Down
16 changes: 8 additions & 8 deletions pnpm-lock.yaml
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion pnpm-workspace.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ catalog:
"@types/node": 22.19.0
"@vercel/functions": ^3.4.3
"@vercel/oidc": 3.2.0
"@vercel/queue": 0.1.4
"@vercel/queue": 0.1.6
"@vitest/coverage-v8": ^4.0.18
ai: 6.0.116
esbuild: ^0.27.3
Expand Down
Loading