diff --git a/packages/sdk/index.html b/packages/sdk/index.html index e780f5e..311863b 100644 --- a/packages/sdk/index.html +++ b/packages/sdk/index.html @@ -332,6 +332,7 @@

Configuration

+ @@ -474,6 +475,12 @@

Image Reference (Style Transfer)

+
+ + +
@@ -581,6 +588,39 @@

Console Logs

let decartClient = null; let decartRealtime = null; let localStream = null; + + // Captures a low-res JPEG snapshot from the local camera stream. + // Mirrors the pattern used in tryonv1 so the bouncer's prompt enhancer can see what the person is currently wearing. + async function captureReferenceFrame() { + if (!localStream) return null; + const video = document.createElement('video'); + video.srcObject = localStream; + video.muted = true; + video.playsInline = true; + await video.play(); + await new Promise((resolve) => { + if ('requestVideoFrameCallback' in video) { + video.requestVideoFrameCallback(() => resolve()); + } else { + requestAnimationFrame(() => requestAnimationFrame(() => resolve())); + } + }); + if (!video.videoWidth) { + video.pause(); + video.srcObject = null; + return null; + } + const scale = 320 / video.videoWidth; + const canvas = document.createElement('canvas'); + canvas.width = 320; + canvas.height = Math.round(video.videoHeight * scale); + canvas.getContext('2d').drawImage(video, 0, 0, canvas.width, canvas.height); + video.pause(); + video.srcObject = null; + return new Promise((resolve) => { + canvas.toBlob((blob) => resolve(blob), 'image/jpeg', 0.5); + }); + } let isConnected = false; let activeConnectionMode = null; let processedVideoUrl = null; @@ -634,6 +674,7 @@

Console Logs

connectivityReasons: document.getElementById('connectivity-reasons'), // Image reference elements referenceImage: document.getElementById('reference-image'), + includeReferenceFrame: document.getElementById('include-reference-frame'), setImage: document.getElementById('send-image'), imageStatus: document.getElementById('image-status'), imageStatusText: document.getElementById('image-status-text'), @@ -1354,7 +1395,17 @@

Console Logs

addLog('Sending reference image...', 'info'); - await decartRealtime.setImage(file); + let sampleFrameData = null; + if (elements.includeReferenceFrame.checked) { + sampleFrameData = await captureReferenceFrame(); + if (sampleFrameData) { + addLog(`Captured reference frame from camera (${(sampleFrameData.size / 1024).toFixed(1)} KB)`, 'info'); + } else { + addLog('Camera stream not available — skipping reference frame', 'warning'); + } + } + + await decartRealtime.setImage(file, sampleFrameData ? { sampleFrameData } : undefined); elements.imageStatusText.textContent = '✅ Image sent successfully!'; elements.imageStatusText.style.color = '#4CAF50'; diff --git a/packages/sdk/src/realtime/client.ts b/packages/sdk/src/realtime/client.ts index b8593c7..1cce90b 100644 --- a/packages/sdk/src/realtime/client.ts +++ b/packages/sdk/src/realtime/client.ts @@ -98,7 +98,12 @@ export type RealTimeClient = { * - `"file_..."` id (from `client.files.upload(...).id`): sent as a server-side reference. * - `null`: clear the current image. */ - setImage: (image: Blob | File | string | null, options?: ImageSetOptions) => Promise; + setImage: ( + image: Blob | File | string | null, + options?: Omit & { + sampleFrameData?: Blob | File | string | null; + }, + ) => Promise; }; export const createRealTimeClient = (opts: RealTimeClientOptions) => { @@ -246,13 +251,23 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => { return subscribeToken; }, getSubscribeToken: () => subscribeToken, - setImage: async (image: Blob | File | string | null, imgOptions?: ImageSetOptions) => { + setImage: async ( + image: Blob | File | string | null, + imgOptions?: Omit & { + sampleFrameData?: Blob | File | string | null; + }, + ) => { + const { sampleFrameData, ...rest } = imgOptions ?? {}; + const sessionOpts: ImageSetOptions = { ...rest }; + if (sampleFrameData !== undefined) { + sessionOpts.sampleFrameData = sampleFrameData === null ? null : await imageToBase64(sampleFrameData); + } if (isFileRefId(image)) { - return activeSession.setImage({ kind: "ref", ref: image }, imgOptions); + return activeSession.setImage({ kind: "ref", ref: image }, sessionOpts); } - if (image === null) return activeSession.setImage({ kind: "data", data: null }, imgOptions); + if (image === null) return activeSession.setImage({ kind: "data", data: null }, sessionOpts); const base64 = await imageToBase64(image); - return activeSession.setImage({ kind: "data", data: base64 }, imgOptions); + return activeSession.setImage({ kind: "data", data: base64 }, sessionOpts); }, }; diff --git a/packages/sdk/src/realtime/methods.ts b/packages/sdk/src/realtime/methods.ts index 56a418f..7601aea 100644 --- a/packages/sdk/src/realtime/methods.ts +++ b/packages/sdk/src/realtime/methods.ts @@ -13,6 +13,7 @@ const setInputSchema = z * - `"file_..."` id (from `client.files.upload(...).id`): sent as a server-side reference. */ image: z.union([z.instanceof(Blob), z.instanceof(File), z.string(), z.null()]).optional(), + sampleFrameData: z.union([z.instanceof(Blob), z.instanceof(File), z.string(), z.null()]).optional(), }) .refine((data) => data.prompt !== undefined || data.image !== undefined, { message: "At least one of 'prompt' or 'image' must be provided", @@ -33,8 +34,15 @@ export const realtimeMethods = ( const parsed = setInputSchema.safeParse(input); if (!parsed.success) throw parsed.error; - const { prompt, enhance, image } = parsed.data; - const options = { prompt, enhance, timeout: REALTIME_CONFIG.methods.updateTimeoutMs }; + const { prompt, enhance, image, sampleFrameData } = parsed.data; + const options: { prompt?: string; enhance: boolean; timeout: number; sampleFrameData?: string | null } = { + prompt, + enhance, + timeout: REALTIME_CONFIG.methods.updateTimeoutMs, + }; + if (sampleFrameData !== undefined) { + options.sampleFrameData = sampleFrameData === null ? null : await imageToBase64(sampleFrameData); + } if (isFileRefId(image)) { await session.setImage({ kind: "ref", ref: image }, options); diff --git a/packages/sdk/src/realtime/signaling-channel.ts b/packages/sdk/src/realtime/signaling-channel.ts index 482778e..f70f82a 100644 --- a/packages/sdk/src/realtime/signaling-channel.ts +++ b/packages/sdk/src/realtime/signaling-channel.ts @@ -173,6 +173,7 @@ export class SignalingChannel { : { type: "set_image", image_data: payload.data }; if (opts.prompt !== undefined) message.prompt = opts.prompt; if (opts.enhance !== undefined) message.enhance_prompt = opts.enhance; + if (opts.sampleFrameData !== undefined) message.sample_frame_data = opts.sampleFrameData; const ack = await this.request({ message, diff --git a/packages/sdk/src/realtime/types.ts b/packages/sdk/src/realtime/types.ts index 639ad9f..97c8a3b 100644 --- a/packages/sdk/src/realtime/types.ts +++ b/packages/sdk/src/realtime/types.ts @@ -23,6 +23,7 @@ export type SetImageMessage = { image_ref?: string; prompt?: string | null; enhance_prompt?: boolean; + sample_frame_data?: string | null; }; export type SetImagePayload = { kind: "data"; data: string | null } | { kind: "ref"; ref: string }; @@ -121,6 +122,8 @@ export type ImageSetOptions = { prompt?: string | null; enhance?: boolean; timeout?: number; + /** Optional base64-encoded sample frame (e.g. current camera frame) for prompt-enhancement context. */ + sampleFrameData?: string | null; }; // Incoming message types (from server) diff --git a/packages/sdk/src/shared/model.ts b/packages/sdk/src/shared/model.ts index 71301ca..b1dba65 100644 --- a/packages/sdk/src/shared/model.ts +++ b/packages/sdk/src/shared/model.ts @@ -3,6 +3,7 @@ import { createModelNotFoundError } from "../utils/errors"; const CANONICAL_MODEL_NAMES = [ "lucy-2.1", + "lucy-2.5", "lucy-2.1-vton", "lucy-vton-2", "lucy-vton-3", @@ -13,6 +14,7 @@ const CANONICAL_MODEL_NAMES = [ const CANONICAL_REALTIME_MODEL_NAMES = [ "lucy-2.1", + "lucy-2.5", "lucy-2.1-vton", "lucy-vton-2", "lucy-vton-3", @@ -67,6 +69,7 @@ function warnDeprecated(model: string): void { export const realtimeModels = z.union([ // Canonical names z.literal("lucy-2.1"), + z.literal("lucy-2.5"), z.literal("lucy-2.1-vton"), z.literal("lucy-vton-2"), z.literal("lucy-vton-3"), @@ -262,6 +265,7 @@ export const modelInputSchemas = { "lucy-image-2": imageEditSchema, "lucy-restyle-2": restyleSchema, "lucy-2.1": videoEdit2Schema, + "lucy-2.5": videoEdit2Schema, "lucy-2.1-vton": videoEdit2Schema, "lucy-vton-2": videoEdit2Schema, "lucy-vton-3": videoEdit2Schema, @@ -350,6 +354,14 @@ const _models = { height: 624, inputSchema: z.object({}), }, + "lucy-2.5": { + urlPath: "/v1/stream", + name: "lucy-2.5" as const, + fps: 20, + width: 1088, + height: 624, + inputSchema: z.object({}), + }, "lucy-2.1-vton": { urlPath: "/v1/stream", name: "lucy-2.1-vton" as const, @@ -634,6 +646,7 @@ export const models = { * * Available options: * - `"lucy-2.1"` - Lucy 2.1 realtime video editing + * - `"lucy-2.5"` - Lucy 2.5 realtime video editing * - `"lucy-2.1-vton"` - Lucy 2.1 virtual try-on * - `"lucy-vton-2"` - Lucy virtual try-on 2 * - `"lucy-vton-3"` - Lucy virtual try-on 3 diff --git a/packages/sdk/tests/realtime.unit.test.ts b/packages/sdk/tests/realtime.unit.test.ts index 781b69b..cff5515 100644 --- a/packages/sdk/tests/realtime.unit.test.ts +++ b/packages/sdk/tests/realtime.unit.test.ts @@ -298,6 +298,36 @@ describe("set()", () => { expect.objectContaining({ timeout: REALTIME_CONFIG.methods.updateTimeoutMs }), ); }); + + it("converts sampleFrameData Blob to base64 and forwards it on options", async () => { + const refBlob = new Blob(["ref-frame"], { type: "image/jpeg" }); + mockImageToBase64.mockImplementation(async (input) => (input === refBlob ? "refbase64" : "imgbase64")); + + await methods.set({ image: "raw", sampleFrameData: refBlob }); + + expect(mockImageToBase64).toHaveBeenCalledWith(refBlob); + expect(mockSession.setImage).toHaveBeenCalledWith( + { kind: "data", data: "imgbase64" }, + expect.objectContaining({ sampleFrameData: "refbase64" }), + ); + }); + + it("forwards null sampleFrameData without calling imageToBase64 on it", async () => { + mockImageToBase64.mockResolvedValue("imgbase64"); + await methods.set({ image: "raw", sampleFrameData: null }); + + expect(mockImageToBase64).toHaveBeenCalledTimes(1); + expect(mockSession.setImage).toHaveBeenCalledWith( + { kind: "data", data: "imgbase64" }, + expect.objectContaining({ sampleFrameData: null }), + ); + }); + + it("omits sampleFrameData from options when not provided", async () => { + await methods.set({ prompt: "a cat" }); + const call = mockSession.setImage.mock.calls[0]?.[1] as Record; + expect("sampleFrameData" in call).toBe(false); + }); }); describe("Subscribe Token", () => { diff --git a/packages/sdk/tests/unit.test.ts b/packages/sdk/tests/unit.test.ts index c1869a7..09c7750 100644 --- a/packages/sdk/tests/unit.test.ts +++ b/packages/sdk/tests/unit.test.ts @@ -2369,6 +2369,7 @@ describe("Canonical Model Names", () => { expect(canonicalRealtimeModels.options).toEqual([ "lucy-2.1", + "lucy-2.5", "lucy-2.1-vton", "lucy-vton-2", "lucy-vton-3", @@ -2440,7 +2441,7 @@ describe("Canonical Model Names", () => { it("lists all models when called without options", () => { const listedModels = listModels(); - expect(listedModels).toHaveLength(28); + expect(listedModels).toHaveLength(29); expect(listedModels.some((model) => model.kind === "realtime" && model.name === "lucy-2.1")).toBe(true); expect(listedModels.some((model) => model.kind === "video" && model.name === "lucy-clip")).toBe(true); expect(listedModels.some((model) => model.kind === "image" && model.name === "lucy-image-2")).toBe(true); @@ -2506,6 +2507,15 @@ describe("Canonical Model Names", () => { expect(model.height).toBe(624); }); + it("lucy-2.5 canonical name works", () => { + const model = models.realtime("lucy-2.5"); + expect(model.name).toBe("lucy-2.5"); + expect(model.urlPath).toBe("/v1/stream"); + expect(model.fps).toBe(20); + expect(model.width).toBe(1088); + expect(model.height).toBe(624); + }); + it("lucy-2.1-vton canonical name works", () => { const model = models.realtime("lucy-2.1-vton"); expect(model.name).toBe("lucy-2.1-vton");