From 94120271f8d063e90d9d123160431073aca67d46 Mon Sep 17 00:00:00 2001 From: Joker-of-Gotham Date: Mon, 6 Apr 2026 05:54:10 +0800 Subject: [PATCH 1/6] feat(visual-chat): add shared protocol and storage foundations Made-with: Cursor --- packages/visual-chat-protocol/package.json | 35 +++++ .../src/constants/auth.ts | 2 + .../src/constants/event-names.ts | 3 + .../src/constants/index.ts | 4 + .../src/constants/inference.ts | 5 + .../src/constants/media.ts | 13 ++ .../visual-chat-protocol/src/enums/index.ts | 4 + .../src/enums/output-kind.ts | 5 + .../src/enums/session-mode.ts | 14 ++ .../src/enums/source-kind.ts | 13 ++ .../src/enums/worker-status.ts | 13 ++ .../visual-chat-protocol/src/events/index.ts | 6 + .../src/events/inference.ts | 4 + .../visual-chat-protocol/src/events/media.ts | 1 + .../visual-chat-protocol/src/events/output.ts | 1 + .../visual-chat-protocol/src/events/room.ts | 4 + .../src/events/session.ts | 3 + .../visual-chat-protocol/src/events/source.ts | 5 + packages/visual-chat-protocol/src/index.ts | 5 + .../visual-chat-protocol/src/rpc/gateway.ts | 22 +++ .../visual-chat-protocol/src/rpc/index.ts | 3 + packages/visual-chat-protocol/src/rpc/ops.ts | 12 ++ .../visual-chat-protocol/src/rpc/worker.ts | 4 + .../src/schemas/diagnostics.ts | 45 +++++++ .../visual-chat-protocol/src/schemas/index.ts | 10 ++ .../src/schemas/inference.ts | 17 +++ .../src/schemas/realtime.ts | 94 +++++++++++++ .../visual-chat-protocol/src/schemas/room.ts | 15 +++ .../src/schemas/session-access.ts | 15 +++ .../src/schemas/session-record.ts | 28 ++++ .../src/schemas/session.ts | 44 ++++++ .../src/schemas/source.ts | 26 ++++ .../src/schemas/stream.ts | 30 +++++ .../src/schemas/tool-call.ts | 9 ++ .../visual-chat-protocol/tsdown.config.ts | 11 ++ packages/visual-chat-shared/package.json | 40 ++++++ packages/visual-chat-shared/src/electron.ts | 38 ++++++ packages/visual-chat-shared/src/env/common.ts | 50 +++++++ .../visual-chat-shared/src/env/gateway.ts | 21 +++ packages/visual-chat-shared/src/env/index.ts | 3 + packages/visual-chat-shared/src/env/worker.ts | 15 +++ .../visual-chat-shared/src/errors/base.ts | 12 ++ .../visual-chat-shared/src/errors/codes.ts | 20 +++ .../visual-chat-shared/src/errors/index.ts | 2 + .../src/flags/experimental.ts | 5 + .../visual-chat-shared/src/flags/features.ts | 9 ++ .../visual-chat-shared/src/flags/index.ts | 2 + packages/visual-chat-shared/src/ids/index.ts | 4 + packages/visual-chat-shared/src/ids/room.ts | 9 ++ .../visual-chat-shared/src/ids/session.ts | 5 + packages/visual-chat-shared/src/ids/source.ts | 5 + packages/visual-chat-shared/src/ids/worker.ts | 9 ++ packages/visual-chat-shared/src/index.ts | 9 ++ .../visual-chat-shared/src/paths/app-paths.ts | 43 ++++++ .../src/paths/cache-paths.ts | 5 + .../src/paths/data-paths.ts | 5 + .../visual-chat-shared/src/paths/index.ts | 4 + .../visual-chat-shared/src/paths/log-paths.ts | 5 + .../src/platform/detect-gpu.ts | 11 ++ .../src/platform/detect-os.ts | 5 + .../visual-chat-shared/src/platform/index.ts | 2 + packages/visual-chat-shared/src/session.ts | 12 ++ packages/visual-chat-shared/src/types/env.ts | 2 + .../visual-chat-shared/src/types/health.ts | 1 + packages/visual-chat-shared/src/types/ids.ts | 33 +++++ .../visual-chat-shared/src/types/index.ts | 3 + packages/visual-chat-shared/tsdown.config.ts | 12 ++ packages/visual-chat-storage/package.json | 35 +++++ packages/visual-chat-storage/src/index.ts | 3 + .../visual-chat-storage/src/paths/index.ts | 23 ++++ .../src/retention/index.ts | 127 ++++++++++++++++++ .../visual-chat-storage/src/sessions/index.ts | 80 +++++++++++ packages/visual-chat-storage/tsdown.config.ts | 11 ++ 73 files changed, 1205 insertions(+) create mode 100644 packages/visual-chat-protocol/package.json create mode 100644 packages/visual-chat-protocol/src/constants/auth.ts create mode 100644 packages/visual-chat-protocol/src/constants/event-names.ts create mode 100644 packages/visual-chat-protocol/src/constants/index.ts create mode 100644 packages/visual-chat-protocol/src/constants/inference.ts create mode 100644 packages/visual-chat-protocol/src/constants/media.ts create mode 100644 packages/visual-chat-protocol/src/enums/index.ts create mode 100644 packages/visual-chat-protocol/src/enums/output-kind.ts create mode 100644 packages/visual-chat-protocol/src/enums/session-mode.ts create mode 100644 packages/visual-chat-protocol/src/enums/source-kind.ts create mode 100644 packages/visual-chat-protocol/src/enums/worker-status.ts create mode 100644 packages/visual-chat-protocol/src/events/index.ts create mode 100644 packages/visual-chat-protocol/src/events/inference.ts create mode 100644 packages/visual-chat-protocol/src/events/media.ts create mode 100644 packages/visual-chat-protocol/src/events/output.ts create mode 100644 packages/visual-chat-protocol/src/events/room.ts create mode 100644 packages/visual-chat-protocol/src/events/session.ts create mode 100644 packages/visual-chat-protocol/src/events/source.ts create mode 100644 packages/visual-chat-protocol/src/index.ts create mode 100644 packages/visual-chat-protocol/src/rpc/gateway.ts create mode 100644 packages/visual-chat-protocol/src/rpc/index.ts create mode 100644 packages/visual-chat-protocol/src/rpc/ops.ts create mode 100644 packages/visual-chat-protocol/src/rpc/worker.ts create mode 100644 packages/visual-chat-protocol/src/schemas/diagnostics.ts create mode 100644 packages/visual-chat-protocol/src/schemas/index.ts create mode 100644 packages/visual-chat-protocol/src/schemas/inference.ts create mode 100644 packages/visual-chat-protocol/src/schemas/realtime.ts create mode 100644 packages/visual-chat-protocol/src/schemas/room.ts create mode 100644 packages/visual-chat-protocol/src/schemas/session-access.ts create mode 100644 packages/visual-chat-protocol/src/schemas/session-record.ts create mode 100644 packages/visual-chat-protocol/src/schemas/session.ts create mode 100644 packages/visual-chat-protocol/src/schemas/source.ts create mode 100644 packages/visual-chat-protocol/src/schemas/stream.ts create mode 100644 packages/visual-chat-protocol/src/schemas/tool-call.ts create mode 100644 packages/visual-chat-protocol/tsdown.config.ts create mode 100644 packages/visual-chat-shared/package.json create mode 100644 packages/visual-chat-shared/src/electron.ts create mode 100644 packages/visual-chat-shared/src/env/common.ts create mode 100644 packages/visual-chat-shared/src/env/gateway.ts create mode 100644 packages/visual-chat-shared/src/env/index.ts create mode 100644 packages/visual-chat-shared/src/env/worker.ts create mode 100644 packages/visual-chat-shared/src/errors/base.ts create mode 100644 packages/visual-chat-shared/src/errors/codes.ts create mode 100644 packages/visual-chat-shared/src/errors/index.ts create mode 100644 packages/visual-chat-shared/src/flags/experimental.ts create mode 100644 packages/visual-chat-shared/src/flags/features.ts create mode 100644 packages/visual-chat-shared/src/flags/index.ts create mode 100644 packages/visual-chat-shared/src/ids/index.ts create mode 100644 packages/visual-chat-shared/src/ids/room.ts create mode 100644 packages/visual-chat-shared/src/ids/session.ts create mode 100644 packages/visual-chat-shared/src/ids/source.ts create mode 100644 packages/visual-chat-shared/src/ids/worker.ts create mode 100644 packages/visual-chat-shared/src/index.ts create mode 100644 packages/visual-chat-shared/src/paths/app-paths.ts create mode 100644 packages/visual-chat-shared/src/paths/cache-paths.ts create mode 100644 packages/visual-chat-shared/src/paths/data-paths.ts create mode 100644 packages/visual-chat-shared/src/paths/index.ts create mode 100644 packages/visual-chat-shared/src/paths/log-paths.ts create mode 100644 packages/visual-chat-shared/src/platform/detect-gpu.ts create mode 100644 packages/visual-chat-shared/src/platform/detect-os.ts create mode 100644 packages/visual-chat-shared/src/platform/index.ts create mode 100644 packages/visual-chat-shared/src/session.ts create mode 100644 packages/visual-chat-shared/src/types/env.ts create mode 100644 packages/visual-chat-shared/src/types/health.ts create mode 100644 packages/visual-chat-shared/src/types/ids.ts create mode 100644 packages/visual-chat-shared/src/types/index.ts create mode 100644 packages/visual-chat-shared/tsdown.config.ts create mode 100644 packages/visual-chat-storage/package.json create mode 100644 packages/visual-chat-storage/src/index.ts create mode 100644 packages/visual-chat-storage/src/paths/index.ts create mode 100644 packages/visual-chat-storage/src/retention/index.ts create mode 100644 packages/visual-chat-storage/src/sessions/index.ts create mode 100644 packages/visual-chat-storage/tsdown.config.ts diff --git a/packages/visual-chat-protocol/package.json b/packages/visual-chat-protocol/package.json new file mode 100644 index 0000000000..1d1c10f33d --- /dev/null +++ b/packages/visual-chat-protocol/package.json @@ -0,0 +1,35 @@ +{ + "name": "@proj-airi/visual-chat-protocol", + "type": "module", + "version": "0.0.1", + "private": true, + "description": "Protocol definitions, schemas, and event constants for AIRI visual chat", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/moeru-ai/airi.git", + "directory": "packages/visual-chat-protocol" + }, + "exports": { + ".": { + "types": "./dist/index.d.mts", + "import": "./src/index.ts", + "default": "./dist/index.mjs" + } + }, + "main": "./src/index.ts", + "types": "./dist/index.d.mts", + "files": [ + "README.md", + "dist", + "package.json" + ], + "scripts": { + "dev": "pnpm run build", + "build": "tsdown", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "valibot": "catalog:" + } +} diff --git a/packages/visual-chat-protocol/src/constants/auth.ts b/packages/visual-chat-protocol/src/constants/auth.ts new file mode 100644 index 0000000000..5e30a1c1cb --- /dev/null +++ b/packages/visual-chat-protocol/src/constants/auth.ts @@ -0,0 +1,2 @@ +export const VISUAL_CHAT_GATEWAY_TOKEN_HEADER = 'x-visual-chat-gateway-token' +export const VISUAL_CHAT_SESSION_TOKEN_HEADER = 'x-visual-chat-session-token' diff --git a/packages/visual-chat-protocol/src/constants/event-names.ts b/packages/visual-chat-protocol/src/constants/event-names.ts new file mode 100644 index 0000000000..6d1a36b79b --- /dev/null +++ b/packages/visual-chat-protocol/src/constants/event-names.ts @@ -0,0 +1,3 @@ +export const LIVEKIT_TEXT_STREAM_TOPIC = 'visual-chat' +export const LIVEKIT_CONTROL_TOPIC = 'visual-chat-control' +export const PREFILL_CNT_START = 1 diff --git a/packages/visual-chat-protocol/src/constants/index.ts b/packages/visual-chat-protocol/src/constants/index.ts new file mode 100644 index 0000000000..766cdf133f --- /dev/null +++ b/packages/visual-chat-protocol/src/constants/index.ts @@ -0,0 +1,4 @@ +export * from './auth' +export * from './event-names' +export * from './inference' +export * from './media' diff --git a/packages/visual-chat-protocol/src/constants/inference.ts b/packages/visual-chat-protocol/src/constants/inference.ts new file mode 100644 index 0000000000..3c5c7ce12f --- /dev/null +++ b/packages/visual-chat-protocol/src/constants/inference.ts @@ -0,0 +1,5 @@ +export const INFERENCE_CYCLE_INTERVAL_MS = 1000 + +export const DEFAULT_GATEWAY_PORT = 6200 +export const DEFAULT_WORKER_PORT = 6201 +export const DEFAULT_LIVEKIT_PORT = 7880 diff --git a/packages/visual-chat-protocol/src/constants/media.ts b/packages/visual-chat-protocol/src/constants/media.ts new file mode 100644 index 0000000000..7b8f484be5 --- /dev/null +++ b/packages/visual-chat-protocol/src/constants/media.ts @@ -0,0 +1,13 @@ +export const AUDIO_SAMPLE_RATE = 16000 +export const AUDIO_CHANNELS = 1 +export const AUDIO_CHUNK_DURATION_MS = 1000 +export const AUDIO_CHUNK_SAMPLES = AUDIO_SAMPLE_RATE * (AUDIO_CHUNK_DURATION_MS / 1000) +export const AUDIO_CHUNK_BYTES = AUDIO_CHUNK_SAMPLES * 2 + +export const TTS_OUTPUT_SAMPLE_RATE = 24000 + +export const VIDEO_FPS_ACTIVE = 1 +export const VIDEO_FPS_STANDBY = 0.2 + +export const RING_BUFFER_AUDIO_CAPACITY = 5 +export const RING_BUFFER_VIDEO_CAPACITY = 5 diff --git a/packages/visual-chat-protocol/src/enums/index.ts b/packages/visual-chat-protocol/src/enums/index.ts new file mode 100644 index 0000000000..f295a3d111 --- /dev/null +++ b/packages/visual-chat-protocol/src/enums/index.ts @@ -0,0 +1,4 @@ +export * from './output-kind' +export * from './session-mode' +export * from './source-kind' +export * from './worker-status' diff --git a/packages/visual-chat-protocol/src/enums/output-kind.ts b/packages/visual-chat-protocol/src/enums/output-kind.ts new file mode 100644 index 0000000000..a8297dd83b --- /dev/null +++ b/packages/visual-chat-protocol/src/enums/output-kind.ts @@ -0,0 +1,5 @@ +export const OutputKinds = { + Text: 'text', + Audio: 'audio', + ToolCall: 'tool-call', +} as const diff --git a/packages/visual-chat-protocol/src/enums/session-mode.ts b/packages/visual-chat-protocol/src/enums/session-mode.ts new file mode 100644 index 0000000000..941820d12f --- /dev/null +++ b/packages/visual-chat-protocol/src/enums/session-mode.ts @@ -0,0 +1,14 @@ +export const InteractionModes = { + VisionTextRealtime: 'vision-text-realtime', +} as const + +export const SessionStates = { + Idle: 'idle', + Connected: 'connected', + Ready: 'ready', + Listening: 'listening', + SelectingSource: 'selecting-source', + Inference: 'inference', + Responding: 'responding', + Suspended: 'suspended', +} as const diff --git a/packages/visual-chat-protocol/src/enums/source-kind.ts b/packages/visual-chat-protocol/src/enums/source-kind.ts new file mode 100644 index 0000000000..132fa7a9b5 --- /dev/null +++ b/packages/visual-chat-protocol/src/enums/source-kind.ts @@ -0,0 +1,13 @@ +export const SourceTypes = { + PhoneCamera: 'phone-camera', + LaptopCamera: 'laptop-camera', + ScreenShare: 'screen-share', + PhoneMic: 'phone-mic', + LaptopMic: 'laptop-mic', +} as const + +export const TrackKinds = { + Audio: 'audio', + Video: 'video', + Data: 'data', +} as const diff --git a/packages/visual-chat-protocol/src/enums/worker-status.ts b/packages/visual-chat-protocol/src/enums/worker-status.ts new file mode 100644 index 0000000000..d0937e41f2 --- /dev/null +++ b/packages/visual-chat-protocol/src/enums/worker-status.ts @@ -0,0 +1,13 @@ +export const WorkerStatuses = { + Offline: 'offline', + Starting: 'starting', + WarmingUp: 'warming-up', + Ready: 'ready', + Busy: 'busy', + Error: 'error', + ShuttingDown: 'shutting-down', +} as const + +export const InferenceBackendTypes = { + Ollama: 'ollama', +} as const diff --git a/packages/visual-chat-protocol/src/events/index.ts b/packages/visual-chat-protocol/src/events/index.ts new file mode 100644 index 0000000000..22d7262d86 --- /dev/null +++ b/packages/visual-chat-protocol/src/events/index.ts @@ -0,0 +1,6 @@ +export * from './inference' +export * from './media' +export * from './output' +export * from './room' +export * from './session' +export * from './source' diff --git a/packages/visual-chat-protocol/src/events/inference.ts b/packages/visual-chat-protocol/src/events/inference.ts new file mode 100644 index 0000000000..d6ab9e4447 --- /dev/null +++ b/packages/visual-chat-protocol/src/events/inference.ts @@ -0,0 +1,4 @@ +export const INFERENCE_STARTED = 'inference:started' +export const INFERENCE_TEXT_CHUNK = 'inference:text:chunk' +export const INFERENCE_COMPLETED = 'inference:completed' +export const INFERENCE_FAILED = 'inference:failed' diff --git a/packages/visual-chat-protocol/src/events/media.ts b/packages/visual-chat-protocol/src/events/media.ts new file mode 100644 index 0000000000..849e95b4ed --- /dev/null +++ b/packages/visual-chat-protocol/src/events/media.ts @@ -0,0 +1 @@ +export const MEDIA_VIDEO_FRAME_READY = 'media:video:frame:ready' diff --git a/packages/visual-chat-protocol/src/events/output.ts b/packages/visual-chat-protocol/src/events/output.ts new file mode 100644 index 0000000000..14ad70b61a --- /dev/null +++ b/packages/visual-chat-protocol/src/events/output.ts @@ -0,0 +1 @@ +export const OUTPUT_TEXT_PUBLISHED = 'output:text:published' diff --git a/packages/visual-chat-protocol/src/events/room.ts b/packages/visual-chat-protocol/src/events/room.ts new file mode 100644 index 0000000000..ed0298c8e9 --- /dev/null +++ b/packages/visual-chat-protocol/src/events/room.ts @@ -0,0 +1,4 @@ +export const ROOM_CREATED = 'room:created' +export const ROOM_CLOSED = 'room:closed' +export const ROOM_PARTICIPANT_JOINED = 'room:participant:joined' +export const ROOM_PARTICIPANT_LEFT = 'room:participant:left' diff --git a/packages/visual-chat-protocol/src/events/session.ts b/packages/visual-chat-protocol/src/events/session.ts new file mode 100644 index 0000000000..2764bef564 --- /dev/null +++ b/packages/visual-chat-protocol/src/events/session.ts @@ -0,0 +1,3 @@ +export const SESSION_STARTED = 'session:started' +export const SESSION_ENDED = 'session:ended' +export const SESSION_STATE_CHANGED = 'session:state:changed' diff --git a/packages/visual-chat-protocol/src/events/source.ts b/packages/visual-chat-protocol/src/events/source.ts new file mode 100644 index 0000000000..bf3e4b4068 --- /dev/null +++ b/packages/visual-chat-protocol/src/events/source.ts @@ -0,0 +1,5 @@ +export const SOURCE_REGISTERED = 'source:registered' +export const SOURCE_UNREGISTERED = 'source:unregistered' +export const SOURCE_ACTIVE_CHANGED = 'source:active:changed' +export const SOURCE_SWITCH_REQUESTED = 'source:switch:requested' +export const SOURCE_SWITCH_COMPLETED = 'source:switch:completed' diff --git a/packages/visual-chat-protocol/src/index.ts b/packages/visual-chat-protocol/src/index.ts new file mode 100644 index 0000000000..6f59036e0c --- /dev/null +++ b/packages/visual-chat-protocol/src/index.ts @@ -0,0 +1,5 @@ +export * from './constants' +export * from './enums' +export * from './events' +export * from './rpc' +export * from './schemas' diff --git a/packages/visual-chat-protocol/src/rpc/gateway.ts b/packages/visual-chat-protocol/src/rpc/gateway.ts new file mode 100644 index 0000000000..3b69381454 --- /dev/null +++ b/packages/visual-chat-protocol/src/rpc/gateway.ts @@ -0,0 +1,22 @@ +import type { GatewayDiagnostics } from '../schemas/diagnostics' +import type { RoomCreateRequest } from '../schemas/room' +import type { SessionContext } from '../schemas/session' +import type { GatewayBootstrap, SessionAccess } from '../schemas/session-access' +import type { SourceSwitchRequest } from '../schemas/source' + +export interface GatewayRpc { + bootstrap: () => Promise + createSession: (req?: RoomCreateRequest) => Promise + issueSessionAccess: (sessionId: string) => Promise + listSessions: () => Promise + getSession: (sessionId: string) => Promise + getSessionRecord: (sessionId: string) => Promise + deleteSession: (sessionId: string) => Promise<{ ok: boolean }> + switchSource: (sessionId: string, req: SourceSwitchRequest) => Promise + getRoomToken: ( + roomName: string, + body: { name?: string, identity?: string }, + ) => Promise<{ token: string, roomName: string, sessionId?: string }> + getDiagnostics: () => Promise + health: () => Promise +} diff --git a/packages/visual-chat-protocol/src/rpc/index.ts b/packages/visual-chat-protocol/src/rpc/index.ts new file mode 100644 index 0000000000..cd736b64d8 --- /dev/null +++ b/packages/visual-chat-protocol/src/rpc/index.ts @@ -0,0 +1,3 @@ +export * from './gateway' +export * from './ops' +export * from './worker' diff --git a/packages/visual-chat-protocol/src/rpc/ops.ts b/packages/visual-chat-protocol/src/rpc/ops.ts new file mode 100644 index 0000000000..1c3f9c9a68 --- /dev/null +++ b/packages/visual-chat-protocol/src/rpc/ops.ts @@ -0,0 +1,12 @@ +export interface OpsPullModelsOptions { + quant?: string + argv?: string[] +} + +export interface OpsRpc { + pullModels: (options?: OpsPullModelsOptions) => Promise + doctor: () => Promise + start: () => Promise + stop: () => Promise + prune: () => Promise +} diff --git a/packages/visual-chat-protocol/src/rpc/worker.ts b/packages/visual-chat-protocol/src/rpc/worker.ts new file mode 100644 index 0000000000..d1a455200e --- /dev/null +++ b/packages/visual-chat-protocol/src/rpc/worker.ts @@ -0,0 +1,4 @@ +export interface WorkerRpc { + submitMedia: (wavData: Uint8Array, imageData?: Uint8Array) => void | Promise + health: () => Promise +} diff --git a/packages/visual-chat-protocol/src/schemas/diagnostics.ts b/packages/visual-chat-protocol/src/schemas/diagnostics.ts new file mode 100644 index 0000000000..3b9a3798ba --- /dev/null +++ b/packages/visual-chat-protocol/src/schemas/diagnostics.ts @@ -0,0 +1,45 @@ +import * as v from 'valibot' + +export const WorkerStatusSchema = v.picklist([ + 'offline', + 'running', +]) + +export const WorkerDiagnosticsSchema = v.object({ + ok: v.boolean(), + status: WorkerStatusSchema, + backendKind: v.picklist(['ollama']), + model: v.string(), + upstreamBaseUrl: v.string(), + fixedModel: v.optional(v.boolean()), + error: v.optional(v.string()), + features: v.optional(v.array(v.string())), +}) + +export const GatewaySessionPipelineStatsSchema = v.object({ + totalInferences: v.number(), + autoObserveInferences: v.number(), + userInferences: v.number(), + skippedAutoObserve: v.number(), + skippedNoChange: v.number(), + timedOut: v.number(), + avgLatencyMs: v.number(), + lastLatencyMs: v.number(), +}) + +export const GatewayDiagnosticsSchema = v.object({ + activeSessions: v.number(), + workerStatus: v.string(), + uptimeMs: v.number(), + livekitUrl: v.optional(v.string()), + workerUrl: v.optional(v.string()), + lanAddresses: v.optional(v.array(v.string())), + preferredLanAddress: v.optional(v.string()), + hostname: v.optional(v.string()), + publicFrontendUrl: v.optional(v.string()), + publicGatewayUrl: v.optional(v.string()), + sessionPipelineStats: v.optional(v.record(v.string(), GatewaySessionPipelineStatsSchema)), +}) + +export type WorkerDiagnostics = v.InferOutput +export type GatewayDiagnostics = v.InferOutput diff --git a/packages/visual-chat-protocol/src/schemas/index.ts b/packages/visual-chat-protocol/src/schemas/index.ts new file mode 100644 index 0000000000..b87ba91ffc --- /dev/null +++ b/packages/visual-chat-protocol/src/schemas/index.ts @@ -0,0 +1,10 @@ +export * from './diagnostics' +export * from './inference' +export * from './realtime' +export * from './room' +export * from './session' +export * from './session-access' +export * from './session-record' +export * from './source' +export * from './stream' +export * from './tool-call' diff --git a/packages/visual-chat-protocol/src/schemas/inference.ts b/packages/visual-chat-protocol/src/schemas/inference.ts new file mode 100644 index 0000000000..184bcdb282 --- /dev/null +++ b/packages/visual-chat-protocol/src/schemas/inference.ts @@ -0,0 +1,17 @@ +import * as v from 'valibot' + +export const InferencePrefillRequestSchema = v.object({ + wavPath: v.string(), + imagePath: v.optional(v.string()), + cnt: v.number(), +}) + +export const InferenceDecodeResponseSchema = v.object({ + text: v.string(), + audioOutDir: v.string(), + wavFiles: v.array(v.string()), + listenDetected: v.boolean(), +}) + +export type InferencePrefillRequest = v.InferOutput +export type InferenceDecodeResponse = v.InferOutput diff --git a/packages/visual-chat-protocol/src/schemas/realtime.ts b/packages/visual-chat-protocol/src/schemas/realtime.ts new file mode 100644 index 0000000000..131daca4cc --- /dev/null +++ b/packages/visual-chat-protocol/src/schemas/realtime.ts @@ -0,0 +1,94 @@ +import type { SourceDescriptor } from './source' +import type { TextMessage, VideoFrameMeta } from './stream' + +export interface GatewaySubscriptionMessage { + type: 'subscribe' + sessionId: string + sessionToken: string +} + +export interface GatewayUnsubscribeMessage { + type: 'unsubscribe' + sessionId: string +} + +export interface GatewayRealtimeVideoFrameMessage { + type: 'realtime:media:video' + sessionId: string + participantIdentity: string + sourceId: string + sourceType: SourceDescriptor['sourceType'] + timestamp: number + width: number + height: number + format: VideoFrameMeta['format'] + data: string +} + +export interface GatewayRealtimeTextInputMessage { + type: 'realtime:user:text' + sessionId: string + participantIdentity: string + text: string + sourceId?: string +} + +export type GatewayRealtimeControlAction = 'request-inference' | 'start-auto-observe' | 'stop-auto-observe' | 'reset-source' + +export interface GatewayRealtimeControlMessage { + type: 'realtime:control' + sessionId: string + action: GatewayRealtimeControlAction + intervalMs?: number +} + +export interface RealtimeAutoObserveStartedPayload { + sessionId: string + intervalMs: number +} + +export interface RealtimeAutoObserveStoppedPayload { + sessionId: string +} + +export type GatewayWsClientMessage + = | GatewaySubscriptionMessage + | GatewayUnsubscribeMessage + | GatewayRealtimeVideoFrameMessage + | GatewayRealtimeTextInputMessage + | GatewayRealtimeControlMessage + +export interface SessionMessagesResponse { + messages: TextMessage[] +} + +export interface RealtimeInferenceStartedPayload { + prompt: string + auto: boolean + sourceId: string +} + +export interface RealtimeInferenceTextChunkPayload { + id: string + delta: string + text: string + sourceId: string + model?: string +} + +export interface RealtimeInferenceCompletedPayload { + message: TextMessage + sourceId: string + auto: boolean + durationMs?: number +} + +export interface RealtimeInferenceFailedPayload { + error: string + sourceId?: string + auto?: boolean +} + +export interface RealtimeVideoFrameReadyPayload extends VideoFrameMeta { + participantIdentity: string +} diff --git a/packages/visual-chat-protocol/src/schemas/room.ts b/packages/visual-chat-protocol/src/schemas/room.ts new file mode 100644 index 0000000000..978260d12e --- /dev/null +++ b/packages/visual-chat-protocol/src/schemas/room.ts @@ -0,0 +1,15 @@ +import * as v from 'valibot' + +export const RoomCreateRequestSchema = v.object({ + roomName: v.optional(v.string()), +}) + +export const RoomInfoSchema = v.object({ + roomName: v.string(), + participantCount: v.number(), + createdAt: v.number(), + activeSessionId: v.nullable(v.string()), +}) + +export type RoomCreateRequest = v.InferOutput +export type RoomInfo = v.InferOutput diff --git a/packages/visual-chat-protocol/src/schemas/session-access.ts b/packages/visual-chat-protocol/src/schemas/session-access.ts new file mode 100644 index 0000000000..ae4682f0fa --- /dev/null +++ b/packages/visual-chat-protocol/src/schemas/session-access.ts @@ -0,0 +1,15 @@ +import * as v from 'valibot' + +import { SessionContextSchema } from './session' + +export const SessionAccessSchema = v.object({ + session: SessionContextSchema, + sessionToken: v.string(), +}) + +export const GatewayBootstrapSchema = v.object({ + gatewayToken: v.string(), +}) + +export type SessionAccess = v.InferOutput +export type GatewayBootstrap = v.InferOutput diff --git a/packages/visual-chat-protocol/src/schemas/session-record.ts b/packages/visual-chat-protocol/src/schemas/session-record.ts new file mode 100644 index 0000000000..1ad0e0a530 --- /dev/null +++ b/packages/visual-chat-protocol/src/schemas/session-record.ts @@ -0,0 +1,28 @@ +import * as v from 'valibot' + +export const SessionMemorySnapshotSchema = v.object({ + summary: v.string(), + updatedAt: v.number(), + sourceId: v.optional(v.string()), +}) + +export const SessionRecordSchema = v.object({ + sessionId: v.string(), + roomName: v.string(), + title: v.string(), + createdAt: v.number(), + updatedAt: v.number(), + lastMessageAt: v.nullable(v.number()), + messageCount: v.number(), + summary: v.string(), + sceneMemory: v.optional(v.string()), + memoryTimeline: v.optional(v.array(SessionMemorySnapshotSchema)), +}) + +export const SessionRecordsResponseSchema = v.object({ + records: v.array(SessionRecordSchema), +}) + +export type SessionRecord = v.InferOutput +export type SessionRecordsResponse = v.InferOutput +export type SessionMemorySnapshot = v.InferOutput diff --git a/packages/visual-chat-protocol/src/schemas/session.ts b/packages/visual-chat-protocol/src/schemas/session.ts new file mode 100644 index 0000000000..081663f1cf --- /dev/null +++ b/packages/visual-chat-protocol/src/schemas/session.ts @@ -0,0 +1,44 @@ +import * as v from 'valibot' + +import { SourceDescriptorSchema } from './source' + +export const InteractionModeSchema = v.literal('vision-text-realtime') + +export const SessionStateSchema = v.picklist([ + 'idle', + 'connected', + 'ready', + 'listening', + 'selecting-source', + 'inference', + 'responding', + 'suspended', +]) + +export const InferenceStateSchema = v.object({ + isRunning: v.boolean(), + currentCnt: v.number(), + lastLatencyMs: v.optional(v.number()), + errorCount: v.number(), +}) + +export const SessionContextSchema = v.object({ + sessionId: v.string(), + roomName: v.string(), + mode: InteractionModeSchema, + state: SessionStateSchema, + + activeVideoSource: v.nullable(SourceDescriptorSchema), + activeAudioSource: v.nullable(SourceDescriptorSchema), + standbyVideoSources: v.array(SourceDescriptorSchema), + standbyAudioSources: v.array(SourceDescriptorSchema), + + inferenceState: InferenceStateSchema, + createdAt: v.number(), + lastActivityAt: v.number(), +}) + +export type InteractionMode = v.InferOutput +export type SessionState = v.InferOutput +export type InferenceState = v.InferOutput +export type SessionContext = v.InferOutput diff --git a/packages/visual-chat-protocol/src/schemas/source.ts b/packages/visual-chat-protocol/src/schemas/source.ts new file mode 100644 index 0000000000..491978edf2 --- /dev/null +++ b/packages/visual-chat-protocol/src/schemas/source.ts @@ -0,0 +1,26 @@ +import * as v from 'valibot' + +export const SourceTypeSchema = v.picklist([ + 'phone-camera', + 'laptop-camera', + 'screen-share', + 'phone-mic', + 'laptop-mic', +]) + +export const SourceDescriptorSchema = v.object({ + sourceId: v.string(), + participantIdentity: v.string(), + trackSid: v.string(), + sourceType: SourceTypeSchema, + isActive: v.boolean(), + lastFrameTimestamp: v.number(), +}) + +export const SourceSwitchRequestSchema = v.object({ + sourceType: SourceTypeSchema, + sourceId: v.optional(v.string()), +}) + +export type SourceDescriptor = v.InferOutput +export type SourceSwitchRequest = v.InferOutput diff --git a/packages/visual-chat-protocol/src/schemas/stream.ts b/packages/visual-chat-protocol/src/schemas/stream.ts new file mode 100644 index 0000000000..258fa8e76c --- /dev/null +++ b/packages/visual-chat-protocol/src/schemas/stream.ts @@ -0,0 +1,30 @@ +import * as v from 'valibot' + +export const TextMessageSchema = v.object({ + id: v.string(), + role: v.picklist(['user', 'assistant', 'system']), + content: v.string(), + timestamp: v.number(), + sourceId: v.optional(v.string()), + model: v.optional(v.string()), +}) + +export const AudioChunkMetaSchema = v.object({ + sourceId: v.string(), + timestamp: v.number(), + sampleRate: v.literal(16000), + channels: v.literal(1), + durationMs: v.literal(1000), +}) + +export const VideoFrameMetaSchema = v.object({ + sourceId: v.string(), + timestamp: v.number(), + width: v.number(), + height: v.number(), + format: v.picklist(['jpeg', 'png']), +}) + +export type TextMessage = v.InferOutput +export type AudioChunkMeta = v.InferOutput +export type VideoFrameMeta = v.InferOutput diff --git a/packages/visual-chat-protocol/src/schemas/tool-call.ts b/packages/visual-chat-protocol/src/schemas/tool-call.ts new file mode 100644 index 0000000000..c1bbadab56 --- /dev/null +++ b/packages/visual-chat-protocol/src/schemas/tool-call.ts @@ -0,0 +1,9 @@ +import * as v from 'valibot' + +export const ToolCallSchema = v.object({ + id: v.string(), + name: v.string(), + arguments: v.string(), +}) + +export type ToolCall = v.InferOutput diff --git a/packages/visual-chat-protocol/tsdown.config.ts b/packages/visual-chat-protocol/tsdown.config.ts new file mode 100644 index 0000000000..e0cb0b3266 --- /dev/null +++ b/packages/visual-chat-protocol/tsdown.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from 'tsdown' + +export default defineConfig({ + entry: { + index: 'src/index.ts', + }, + target: 'node18', + outDir: 'dist', + clean: true, + dts: true, +}) diff --git a/packages/visual-chat-shared/package.json b/packages/visual-chat-shared/package.json new file mode 100644 index 0000000000..0873274b6d --- /dev/null +++ b/packages/visual-chat-shared/package.json @@ -0,0 +1,40 @@ +{ + "name": "@proj-airi/visual-chat-shared", + "type": "module", + "version": "0.0.1", + "private": true, + "description": "Shared types, utilities, and constants for AIRI visual chat", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/moeru-ai/airi.git", + "directory": "packages/visual-chat-shared" + }, + "exports": { + ".": { + "types": "./dist/index.d.mts", + "default": "./dist/index.mjs" + }, + "./electron": { + "types": "./dist/electron.d.mts", + "default": "./dist/electron.mjs" + } + }, + "main": "./dist/index.mjs", + "types": "./dist/index.d.mts", + "files": [ + "README.md", + "dist", + "package.json" + ], + "scripts": { + "dev": "pnpm run build", + "build": "tsdown", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@moeru/eventa": "catalog:", + "@proj-airi/visual-chat-protocol": "workspace:^", + "nanoid": "catalog:" + } +} diff --git a/packages/visual-chat-shared/src/electron.ts b/packages/visual-chat-shared/src/electron.ts new file mode 100644 index 0000000000..84e9d99233 --- /dev/null +++ b/packages/visual-chat-shared/src/electron.ts @@ -0,0 +1,38 @@ +import { defineInvokeEventa } from '@moeru/eventa' + +export type VisualChatDesktopSetupState + = | 'idle' + | 'checking' + | 'installing-engine' + | 'pulling-model' + | 'starting-services' + | 'starting-tunnel' + | 'ready' + | 'error' + +export type VisualChatDesktopSetupStepStatus = 'pending' | 'running' | 'done' | 'error' | 'skipped' + +export interface VisualChatDesktopSetupStep { + id: 'engine' | 'model' | 'gateway' | 'worker' | 'tunnel' + label: string + status: VisualChatDesktopSetupStepStatus + detail: string +} + +export interface VisualChatDesktopSetupStatus { + available: boolean + state: VisualChatDesktopSetupState + fixedModel: string + gatewayUrl: string + workerUrl: string + workspaceRoot?: string + steps: VisualChatDesktopSetupStep[] + logs: string[] + updatedAt: number + error?: string + tunnelFrontendUrl?: string + tunnelGatewayUrl?: string +} + +export const electronVisualChatGetSetupStatus = defineInvokeEventa('eventa:invoke:electron:visual-chat:setup:get-status') +export const electronVisualChatRunSetup = defineInvokeEventa('eventa:invoke:electron:visual-chat:setup:run') diff --git a/packages/visual-chat-shared/src/env/common.ts b/packages/visual-chat-shared/src/env/common.ts new file mode 100644 index 0000000000..d3ad42c3c6 --- /dev/null +++ b/packages/visual-chat-shared/src/env/common.ts @@ -0,0 +1,50 @@ +import process from 'node:process' + +export function optionOrEnv( + option: T | undefined, + envKey: string, + fallback: T, + opts?: { validator?: (value: string) => value is T }, +): T { + if (option !== undefined) + return option + + const envValue = process.env[envKey] + if (envValue !== undefined) { + if (opts?.validator) { + return opts.validator(envValue) ? envValue : fallback + } + return envValue as T + } + + return fallback +} + +export function requireEnv(key: string): string { + const value = process.env[key] + if (value === undefined || value === '') + throw new Error(`Missing required environment variable: ${key}`) + return value +} + +export function envString(key: string, fallback: string): string { + const value = process.env[key] + if (value === undefined || value === '') + return fallback + return value +} + +export function envInt(key: string, fallback: number): number { + const value = process.env[key] + if (value === undefined) + return fallback + const parsed = Number.parseInt(value, 10) + return Number.isNaN(parsed) ? fallback : parsed +} + +export function envBool(key: string, fallback: boolean): boolean { + const value = process.env[key] + if (value === undefined) + return fallback + return value === 'true' || value === '1' +} diff --git a/packages/visual-chat-shared/src/env/gateway.ts b/packages/visual-chat-shared/src/env/gateway.ts new file mode 100644 index 0000000000..3e3506c961 --- /dev/null +++ b/packages/visual-chat-shared/src/env/gateway.ts @@ -0,0 +1,21 @@ +import { envInt, envString } from './common' + +export function getGatewayPort(): number { + return envInt('VISUAL_CHAT_PORT', 6200) +} + +export function getLivekitUrl(): string { + return envString('LIVEKIT_URL', 'ws://localhost:7880') +} + +export function getLivekitApiKey(): string { + return envString('LIVEKIT_API_KEY', 'devkey') +} + +export function getLivekitApiSecret(): string { + return envString('LIVEKIT_API_SECRET', 'secret') +} + +export function getWorkerBaseUrl(): string { + return envString('WORKER_URL', 'http://localhost:6201') +} diff --git a/packages/visual-chat-shared/src/env/index.ts b/packages/visual-chat-shared/src/env/index.ts new file mode 100644 index 0000000000..eb2790dd67 --- /dev/null +++ b/packages/visual-chat-shared/src/env/index.ts @@ -0,0 +1,3 @@ +export * from './common' +export * from './gateway' +export * from './worker' diff --git a/packages/visual-chat-shared/src/env/worker.ts b/packages/visual-chat-shared/src/env/worker.ts new file mode 100644 index 0000000000..d1588cd76c --- /dev/null +++ b/packages/visual-chat-shared/src/env/worker.ts @@ -0,0 +1,15 @@ +import process from 'node:process' + +import { envInt } from './common' + +export function getWorkerPort(): number { + return envInt('WORKER_PORT', 6201) +} + +export function getOllamaHost(): string { + return process.env.OLLAMA_HOST?.trim() || 'http://localhost:11434' +} + +export function getOllamaModel(): string { + return process.env.OLLAMA_MODEL?.trim() || 'openbmb/minicpm-v4.5:latest' +} diff --git a/packages/visual-chat-shared/src/errors/base.ts b/packages/visual-chat-shared/src/errors/base.ts new file mode 100644 index 0000000000..66cf96ad2d --- /dev/null +++ b/packages/visual-chat-shared/src/errors/base.ts @@ -0,0 +1,12 @@ +import type { ErrorCode } from './codes' + +export class VisualChatError extends Error { + constructor( + public readonly code: ErrorCode, + message: string, + public readonly details?: unknown, + ) { + super(message) + this.name = 'VisualChatError' + } +} diff --git a/packages/visual-chat-shared/src/errors/codes.ts b/packages/visual-chat-shared/src/errors/codes.ts new file mode 100644 index 0000000000..9bd76ad1e2 --- /dev/null +++ b/packages/visual-chat-shared/src/errors/codes.ts @@ -0,0 +1,20 @@ +export enum ErrorCode { + SESSION_NOT_FOUND = 'SESSION_NOT_FOUND', + SESSION_ALREADY_EXISTS = 'SESSION_ALREADY_EXISTS', + ROOM_NOT_FOUND = 'ROOM_NOT_FOUND', + ROOM_CREATION_FAILED = 'ROOM_CREATION_FAILED', + SOURCE_NOT_FOUND = 'SOURCE_NOT_FOUND', + SOURCE_SWITCH_FAILED = 'SOURCE_SWITCH_FAILED', + WORKER_UNAVAILABLE = 'WORKER_UNAVAILABLE', + WORKER_INIT_FAILED = 'WORKER_INIT_FAILED', + INFERENCE_FAILED = 'INFERENCE_FAILED', + INFERENCE_TIMEOUT = 'INFERENCE_TIMEOUT', + MODEL_NOT_LOADED = 'MODEL_NOT_LOADED', + MODEL_DOWNLOAD_FAILED = 'MODEL_DOWNLOAD_FAILED', + LIVEKIT_CONNECTION_FAILED = 'LIVEKIT_CONNECTION_FAILED', + LIVEKIT_TOKEN_FAILED = 'LIVEKIT_TOKEN_FAILED', + INVALID_AUDIO_FORMAT = 'INVALID_AUDIO_FORMAT', + INVALID_VIDEO_FORMAT = 'INVALID_VIDEO_FORMAT', + STORAGE_ERROR = 'STORAGE_ERROR', + INTERNAL_ERROR = 'INTERNAL_ERROR', +} diff --git a/packages/visual-chat-shared/src/errors/index.ts b/packages/visual-chat-shared/src/errors/index.ts new file mode 100644 index 0000000000..a5d149c3ca --- /dev/null +++ b/packages/visual-chat-shared/src/errors/index.ts @@ -0,0 +1,2 @@ +export { VisualChatError } from './base' +export { ErrorCode } from './codes' diff --git a/packages/visual-chat-shared/src/flags/experimental.ts b/packages/visual-chat-shared/src/flags/experimental.ts new file mode 100644 index 0000000000..03ed614a8f --- /dev/null +++ b/packages/visual-chat-shared/src/flags/experimental.ts @@ -0,0 +1,5 @@ +import { envBool } from '../env/common' + +export function isRecordingEnabled(): boolean { + return envBool('VISUAL_CHAT_RECORDING', false) +} diff --git a/packages/visual-chat-shared/src/flags/features.ts b/packages/visual-chat-shared/src/flags/features.ts new file mode 100644 index 0000000000..1f2e676148 --- /dev/null +++ b/packages/visual-chat-shared/src/flags/features.ts @@ -0,0 +1,9 @@ +import { envBool } from '../env/common' + +export function isFullDuplexEnabled(): boolean { + return envBool('VISUAL_CHAT_FULL_DUPLEX', false) +} + +export function isAutoSourceSwitchEnabled(): boolean { + return envBool('VISUAL_CHAT_AUTO_SOURCE_SWITCH', false) +} diff --git a/packages/visual-chat-shared/src/flags/index.ts b/packages/visual-chat-shared/src/flags/index.ts new file mode 100644 index 0000000000..0601724758 --- /dev/null +++ b/packages/visual-chat-shared/src/flags/index.ts @@ -0,0 +1,2 @@ +export * from './experimental' +export * from './features' diff --git a/packages/visual-chat-shared/src/ids/index.ts b/packages/visual-chat-shared/src/ids/index.ts new file mode 100644 index 0000000000..b6d2c9fb88 --- /dev/null +++ b/packages/visual-chat-shared/src/ids/index.ts @@ -0,0 +1,4 @@ +export * from './room' +export * from './session' +export * from './source' +export * from './worker' diff --git a/packages/visual-chat-shared/src/ids/room.ts b/packages/visual-chat-shared/src/ids/room.ts new file mode 100644 index 0000000000..fc2b18c627 --- /dev/null +++ b/packages/visual-chat-shared/src/ids/room.ts @@ -0,0 +1,9 @@ +import { nanoid } from 'nanoid' + +export function generateRoomName(): string { + return `vc_${nanoid(12)}` +} + +export function generateParticipantId(prefix: string = 'p'): string { + return `${prefix}_${nanoid(10)}` +} diff --git a/packages/visual-chat-shared/src/ids/session.ts b/packages/visual-chat-shared/src/ids/session.ts new file mode 100644 index 0000000000..cff4d959d1 --- /dev/null +++ b/packages/visual-chat-shared/src/ids/session.ts @@ -0,0 +1,5 @@ +import { nanoid } from 'nanoid' + +export function generateSessionId(): string { + return `ses_${nanoid(16)}` +} diff --git a/packages/visual-chat-shared/src/ids/source.ts b/packages/visual-chat-shared/src/ids/source.ts new file mode 100644 index 0000000000..845bf484a9 --- /dev/null +++ b/packages/visual-chat-shared/src/ids/source.ts @@ -0,0 +1,5 @@ +import { nanoid } from 'nanoid' + +export function generateSourceId(): string { + return `src_${nanoid(10)}` +} diff --git a/packages/visual-chat-shared/src/ids/worker.ts b/packages/visual-chat-shared/src/ids/worker.ts new file mode 100644 index 0000000000..521d5922e4 --- /dev/null +++ b/packages/visual-chat-shared/src/ids/worker.ts @@ -0,0 +1,9 @@ +import { nanoid } from 'nanoid' + +export function generateWorkerId(): string { + return `wrk_${nanoid(10)}` +} + +export function generateRequestId(): string { + return `req_${nanoid(12)}` +} diff --git a/packages/visual-chat-shared/src/index.ts b/packages/visual-chat-shared/src/index.ts new file mode 100644 index 0000000000..5e9ac3fc7d --- /dev/null +++ b/packages/visual-chat-shared/src/index.ts @@ -0,0 +1,9 @@ +export * from './electron' +export * from './env' +export * from './errors' +export * from './flags' +export * from './ids' +export * from './paths' +export * from './platform' +export * from './session' +export * from './types' diff --git a/packages/visual-chat-shared/src/paths/app-paths.ts b/packages/visual-chat-shared/src/paths/app-paths.ts new file mode 100644 index 0000000000..5729eae7ce --- /dev/null +++ b/packages/visual-chat-shared/src/paths/app-paths.ts @@ -0,0 +1,43 @@ +import process from 'node:process' + +import { homedir, platform } from 'node:os' +import { join } from 'node:path' + +export type PathKind = 'config' | 'data' | 'cache' | 'logs' | 'models' + +export function getVisualChatDir(kind: PathKind): string { + const os = platform() + + if (os === 'win32') { + const appData = process.env.APPDATA || join(homedir(), 'AppData', 'Roaming') + const localAppData = process.env.LOCALAPPDATA || join(homedir(), 'AppData', 'Local') + + switch (kind) { + case 'config': return join(appData, 'AIRI', 'visual-chat', 'config') + case 'data': return join(localAppData, 'AIRI', 'visual-chat', 'data') + case 'cache': return join(localAppData, 'AIRI', 'visual-chat', 'cache') + case 'logs': return join(localAppData, 'AIRI', 'visual-chat', 'logs') + case 'models': return join(localAppData, 'AIRI', 'visual-chat', 'models') + } + } + + if (os === 'darwin') { + const home = homedir() + switch (kind) { + case 'config': return join(home, 'Library', 'Application Support', 'AIRI', 'visual-chat') + case 'data': return join(home, 'Library', 'Application Support', 'AIRI', 'visual-chat') + case 'cache': return join(home, 'Library', 'Caches', 'AIRI', 'visual-chat') + case 'logs': return join(home, 'Library', 'Logs', 'AIRI', 'visual-chat') + case 'models': return join(home, 'Library', 'Application Support', 'AIRI', 'visual-chat', 'models') + } + } + + const home = homedir() + switch (kind) { + case 'config': return join(process.env.XDG_CONFIG_HOME || join(home, '.config'), 'airi', 'visual-chat') + case 'data': return join(process.env.XDG_DATA_HOME || join(home, '.local', 'share'), 'airi', 'visual-chat') + case 'cache': return join(process.env.XDG_CACHE_HOME || join(home, '.cache'), 'airi', 'visual-chat') + case 'logs': return join(process.env.XDG_STATE_HOME || join(home, '.local', 'state'), 'airi', 'visual-chat') + case 'models': return join(process.env.XDG_DATA_HOME || join(home, '.local', 'share'), 'airi', 'visual-chat', 'models') + } +} diff --git a/packages/visual-chat-shared/src/paths/cache-paths.ts b/packages/visual-chat-shared/src/paths/cache-paths.ts new file mode 100644 index 0000000000..3de787d81a --- /dev/null +++ b/packages/visual-chat-shared/src/paths/cache-paths.ts @@ -0,0 +1,5 @@ +import { getVisualChatDir } from './app-paths' + +export function getCachePath(): string { + return getVisualChatDir('cache') +} diff --git a/packages/visual-chat-shared/src/paths/data-paths.ts b/packages/visual-chat-shared/src/paths/data-paths.ts new file mode 100644 index 0000000000..9993a8efdd --- /dev/null +++ b/packages/visual-chat-shared/src/paths/data-paths.ts @@ -0,0 +1,5 @@ +import { getVisualChatDir } from './app-paths' + +export function getDataPath(): string { + return getVisualChatDir('data') +} diff --git a/packages/visual-chat-shared/src/paths/index.ts b/packages/visual-chat-shared/src/paths/index.ts new file mode 100644 index 0000000000..a2cfaca687 --- /dev/null +++ b/packages/visual-chat-shared/src/paths/index.ts @@ -0,0 +1,4 @@ +export * from './app-paths' +export * from './cache-paths' +export * from './data-paths' +export * from './log-paths' diff --git a/packages/visual-chat-shared/src/paths/log-paths.ts b/packages/visual-chat-shared/src/paths/log-paths.ts new file mode 100644 index 0000000000..1480a224db --- /dev/null +++ b/packages/visual-chat-shared/src/paths/log-paths.ts @@ -0,0 +1,5 @@ +import { getVisualChatDir } from './app-paths' + +export function getLogPath(): string { + return getVisualChatDir('logs') +} diff --git a/packages/visual-chat-shared/src/platform/detect-gpu.ts b/packages/visual-chat-shared/src/platform/detect-gpu.ts new file mode 100644 index 0000000000..7c604703fe --- /dev/null +++ b/packages/visual-chat-shared/src/platform/detect-gpu.ts @@ -0,0 +1,11 @@ +import { execFileSync } from 'node:child_process' + +export function detectGpuAvailability(): boolean { + try { + execFileSync('nvidia-smi', ['-L'], { stdio: 'ignore' }) + return true + } + catch { + return false + } +} diff --git a/packages/visual-chat-shared/src/platform/detect-os.ts b/packages/visual-chat-shared/src/platform/detect-os.ts new file mode 100644 index 0000000000..68164330c0 --- /dev/null +++ b/packages/visual-chat-shared/src/platform/detect-os.ts @@ -0,0 +1,5 @@ +import { platform } from 'node:os' + +export function detectOs(): ReturnType { + return platform() +} diff --git a/packages/visual-chat-shared/src/platform/index.ts b/packages/visual-chat-shared/src/platform/index.ts new file mode 100644 index 0000000000..8168da59f7 --- /dev/null +++ b/packages/visual-chat-shared/src/platform/index.ts @@ -0,0 +1,2 @@ +export * from './detect-gpu' +export * from './detect-os' diff --git a/packages/visual-chat-shared/src/session.ts b/packages/visual-chat-shared/src/session.ts new file mode 100644 index 0000000000..162fc60332 --- /dev/null +++ b/packages/visual-chat-shared/src/session.ts @@ -0,0 +1,12 @@ +const SESSION_ID_PATTERN = /^ses_[\w-]{8,128}$/ + +export function isValidVisualChatSessionId(value: string): boolean { + return SESSION_ID_PATTERN.test(value.trim()) +} + +export function normalizeVisualChatSessionId(value: string): string { + const normalized = value.trim() + if (!SESSION_ID_PATTERN.test(normalized)) + throw new Error(`Invalid visual chat session id: ${value}`) + return normalized +} diff --git a/packages/visual-chat-shared/src/types/env.ts b/packages/visual-chat-shared/src/types/env.ts new file mode 100644 index 0000000000..fe3ea22c73 --- /dev/null +++ b/packages/visual-chat-shared/src/types/env.ts @@ -0,0 +1,2 @@ +export type { InferenceState, InteractionMode, SessionContext, SessionState } from '@proj-airi/visual-chat-protocol' +export type { AudioChunkMeta, TextMessage, VideoFrameMeta } from '@proj-airi/visual-chat-protocol' diff --git a/packages/visual-chat-shared/src/types/health.ts b/packages/visual-chat-shared/src/types/health.ts new file mode 100644 index 0000000000..c067d88254 --- /dev/null +++ b/packages/visual-chat-shared/src/types/health.ts @@ -0,0 +1 @@ +export type { GatewayDiagnostics, WorkerDiagnostics } from '@proj-airi/visual-chat-protocol' diff --git a/packages/visual-chat-shared/src/types/ids.ts b/packages/visual-chat-shared/src/types/ids.ts new file mode 100644 index 0000000000..5cdf09fea7 --- /dev/null +++ b/packages/visual-chat-shared/src/types/ids.ts @@ -0,0 +1,33 @@ +export type { SourceDescriptor, SourceSwitchRequest } from '@proj-airi/visual-chat-protocol' +export type { RoomCreateRequest, RoomInfo } from '@proj-airi/visual-chat-protocol' + +export type SessionId = string +export type RoomName = string +export type SourceId = string +export type WorkerId = string + +export interface VideoFrame { + sourceId: string + timestamp: number + data: Buffer + width: number + height: number +} + +export interface AudioChunk { + sourceId: string + timestamp: number + data: Buffer + sampleRate: 16000 + channels: 1 + durationMs: 1000 +} + +export interface SourceMetadata { + sourceId: string + participantIdentity: string + deviceLabel?: string + firstSeenAt: number + lastSeenAt: number + frameCount: number +} diff --git a/packages/visual-chat-shared/src/types/index.ts b/packages/visual-chat-shared/src/types/index.ts new file mode 100644 index 0000000000..3835805d05 --- /dev/null +++ b/packages/visual-chat-shared/src/types/index.ts @@ -0,0 +1,3 @@ +export * from './env' +export * from './health' +export * from './ids' diff --git a/packages/visual-chat-shared/tsdown.config.ts b/packages/visual-chat-shared/tsdown.config.ts new file mode 100644 index 0000000000..ff59b83947 --- /dev/null +++ b/packages/visual-chat-shared/tsdown.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from 'tsdown' + +export default defineConfig({ + entry: { + index: 'src/index.ts', + electron: 'src/electron.ts', + }, + target: 'node18', + outDir: 'dist', + clean: true, + dts: true, +}) diff --git a/packages/visual-chat-storage/package.json b/packages/visual-chat-storage/package.json new file mode 100644 index 0000000000..688616f31a --- /dev/null +++ b/packages/visual-chat-storage/package.json @@ -0,0 +1,35 @@ +{ + "name": "@proj-airi/visual-chat-storage", + "type": "module", + "version": "0.0.1", + "private": true, + "description": "Cross-platform storage, paths, and retention management for AIRI visual chat", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/moeru-ai/airi.git", + "directory": "packages/visual-chat-storage" + }, + "exports": { + ".": { + "types": "./dist/index.d.mts", + "default": "./dist/index.mjs" + } + }, + "main": "./dist/index.mjs", + "types": "./dist/index.d.mts", + "files": [ + "README.md", + "dist", + "package.json" + ], + "scripts": { + "dev": "pnpm run build", + "build": "tsdown", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@proj-airi/visual-chat-protocol": "workspace:^", + "@proj-airi/visual-chat-shared": "workspace:^" + } +} diff --git a/packages/visual-chat-storage/src/index.ts b/packages/visual-chat-storage/src/index.ts new file mode 100644 index 0000000000..217a9efaea --- /dev/null +++ b/packages/visual-chat-storage/src/index.ts @@ -0,0 +1,3 @@ +export * from './paths' +export * from './retention' +export * from './sessions' diff --git a/packages/visual-chat-storage/src/paths/index.ts b/packages/visual-chat-storage/src/paths/index.ts new file mode 100644 index 0000000000..1c889767d7 --- /dev/null +++ b/packages/visual-chat-storage/src/paths/index.ts @@ -0,0 +1,23 @@ +import type { PathKind } from '@proj-airi/visual-chat-shared' + +import { mkdir } from 'node:fs/promises' + +import { getVisualChatDir } from '@proj-airi/visual-chat-shared' + +export type { PathKind } +export { getVisualChatDir } + +export async function ensureDir(kind: PathKind): Promise { + const dir = getVisualChatDir(kind) + await mkdir(dir, { recursive: true }) + return dir +} + +export async function ensureAllDirs(): Promise> { + const kinds: PathKind[] = ['config', 'data', 'cache', 'logs', 'models'] + const result = {} as Record + for (const kind of kinds) { + result[kind] = await ensureDir(kind) + } + return result +} diff --git a/packages/visual-chat-storage/src/retention/index.ts b/packages/visual-chat-storage/src/retention/index.ts new file mode 100644 index 0000000000..59c2e2f105 --- /dev/null +++ b/packages/visual-chat-storage/src/retention/index.ts @@ -0,0 +1,127 @@ +import { readdir, rm, stat } from 'node:fs/promises' +import { join } from 'node:path' + +export interface RetentionPolicy { + maxAgeDays: number + maxSizeMb: number +} + +export const DEFAULT_RETENTION: RetentionPolicy = { + maxAgeDays: 7, + maxSizeMb: 1024, +} + +export async function pruneByAge(directory: string, maxAgeDays: number): Promise { + const cutoff = Date.now() - maxAgeDays * 86400000 + let removed = 0 + + try { + const entries = await readdir(directory, { withFileTypes: true }) + for (const entry of entries) { + const fullPath = join(directory, entry.name) + try { + const stats = await stat(fullPath) + if (stats.mtimeMs < cutoff) { + await rm(fullPath, { recursive: true, force: true }) + removed++ + } + } + catch { + // skip inaccessible entries + } + } + } + catch { + // directory may not exist + } + + return removed +} + +export async function getDirectorySizeMb(directory: string): Promise { + let totalBytes = 0 + + async function walk(dir: string): Promise { + try { + const entries = await readdir(dir, { withFileTypes: true }) + for (const entry of entries) { + const fullPath = join(dir, entry.name) + try { + if (entry.isFile()) { + const stats = await stat(fullPath) + totalBytes += stats.size + } + else if (entry.isDirectory()) { + await walk(fullPath) + } + } + catch { + // skip inaccessible + } + } + } + catch { + // directory may not exist + } + } + + await walk(directory) + return totalBytes / (1024 * 1024) +} + +/** + * Prune files by size: removes oldest files first until directory is under maxSizeMb. + */ +export async function pruneBySize(directory: string, maxSizeMb: number): Promise { + const currentSize = await getDirectorySizeMb(directory) + if (currentSize <= maxSizeMb) + return 0 + + let removed = 0 + try { + const entries = await readdir(directory, { withFileTypes: true }) + const withStats = await Promise.all( + entries.map(async (entry) => { + const fullPath = join(directory, entry.name) + try { + const stats = await stat(fullPath) + return { fullPath, mtimeMs: stats.mtimeMs, size: stats.size } + } + catch { + return null + } + }), + ) + + const sorted = withStats + .filter((e): e is NonNullable => e !== null) + .sort((a, b) => a.mtimeMs - b.mtimeMs) // oldest first + + let freedBytes = 0 + const targetFreeBytes = (currentSize - maxSizeMb) * 1024 * 1024 + + for (const entry of sorted) { + if (freedBytes >= targetFreeBytes) + break + try { + await rm(entry.fullPath, { recursive: true, force: true }) + freedBytes += entry.size + removed++ + } + catch { + // skip + } + } + } + catch { + // directory may not exist + } + + return removed +} + +export async function pruneWithPolicy(directory: string, policy: RetentionPolicy): Promise<{ byAge: number, bySize: number }> { + const byAge = await pruneByAge(directory, policy.maxAgeDays) + const bySize = await pruneBySize(directory, policy.maxSizeMb) + return { byAge, bySize } +} diff --git a/packages/visual-chat-storage/src/sessions/index.ts b/packages/visual-chat-storage/src/sessions/index.ts new file mode 100644 index 0000000000..d3900714e2 --- /dev/null +++ b/packages/visual-chat-storage/src/sessions/index.ts @@ -0,0 +1,80 @@ +import type { TextMessage } from '@proj-airi/visual-chat-protocol' + +import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises' +import { join } from 'node:path' + +import { normalizeVisualChatSessionId } from '@proj-airi/visual-chat-shared' + +import { ensureDir } from '../paths' + +export async function getSessionDir(sessionId: string): Promise { + const normalizedSessionId = normalizeVisualChatSessionId(sessionId) + const dataDir = await ensureDir('data') + const dir = join(dataDir, 'sessions', normalizedSessionId) + await mkdir(dir, { recursive: true }) + return dir +} + +export async function saveSessionMetadata(sessionId: string, metadata: Record): Promise { + const dir = await getSessionDir(sessionId) + await writeFile(join(dir, 'metadata.json'), JSON.stringify(metadata, null, 2), 'utf-8') +} + +export async function loadSessionMetadata(sessionId: string): Promise | null> { + try { + const dir = await getSessionDir(sessionId) + const raw = await readFile(join(dir, 'metadata.json'), 'utf-8') + return JSON.parse(raw) + } + catch { + return null + } +} + +export async function saveSessionMessages(sessionId: string, messages: TextMessage[]): Promise { + const dir = await getSessionDir(sessionId) + await writeFile(join(dir, 'messages.json'), JSON.stringify(messages, null, 2), 'utf-8') +} + +export async function loadSessionMessages(sessionId: string): Promise { + try { + const dir = await getSessionDir(sessionId) + const raw = await readFile(join(dir, 'messages.json'), 'utf-8') + const parsed = JSON.parse(raw) + return Array.isArray(parsed) ? parsed as TextMessage[] : [] + } + catch { + return [] + } +} + +export async function listSessionIds(): Promise { + try { + const dataDir = await ensureDir('data') + const sessionsDir = join(dataDir, 'sessions') + const entries = await readdir(sessionsDir, { withFileTypes: true }) + return entries + .filter(entry => entry.isDirectory()) + .map(entry => entry.name) + .filter((sessionId) => { + try { + normalizeVisualChatSessionId(sessionId) + return true + } + catch { + return false + } + }) + } + catch { + return [] + } +} + +/** Removes persisted session files (the session directory under data/sessions). */ +export async function deleteSessionData(sessionId: string): Promise { + const normalizedSessionId = normalizeVisualChatSessionId(sessionId) + const dataDir = await ensureDir('data') + const sessionDir = join(dataDir, 'sessions', normalizedSessionId) + await rm(sessionDir, { recursive: true, force: true }) +} diff --git a/packages/visual-chat-storage/tsdown.config.ts b/packages/visual-chat-storage/tsdown.config.ts new file mode 100644 index 0000000000..e0cb0b3266 --- /dev/null +++ b/packages/visual-chat-storage/tsdown.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from 'tsdown' + +export default defineConfig({ + entry: { + index: 'src/index.ts', + }, + target: 'node18', + outDir: 'dist', + clean: true, + dts: true, +}) From 9f1e352173685e8c23d26893c6647e3eb7484b8d Mon Sep 17 00:00:00 2001 From: Joker-of-Gotham Date: Mon, 6 Apr 2026 05:54:30 +0800 Subject: [PATCH 2/6] feat(visual-chat): add media core and worker runtime layers Made-with: Cursor --- packages/visual-chat-livekit/package.json | 38 ++++++ packages/visual-chat-livekit/src/index.ts | 8 ++ .../visual-chat-livekit/src/room/index.ts | 52 +++++++ .../visual-chat-livekit/src/token/index.ts | 52 +++++++ .../visual-chat-livekit/src/tracks/index.ts | 85 ++++++++++++ .../visual-chat-livekit/src/webhook/index.ts | 49 +++++++ packages/visual-chat-livekit/tsdown.config.ts | 11 ++ packages/visual-chat-media-core/package.json | 35 +++++ .../src/chunking/chunking.test.ts | 58 ++++++++ .../src/chunking/index.ts | 65 +++++++++ packages/visual-chat-media-core/src/index.ts | 8 ++ .../src/ring-buffer/index.ts | 55 ++++++++ .../src/ring-buffer/ring-buffer.test.ts | 72 ++++++++++ .../src/session-context/index.ts | 38 ++++++ .../session-context/session-context.test.ts | 35 +++++ .../src/source-registry/index.ts | 99 ++++++++++++++ .../source-registry/source-registry.test.ts | 51 +++++++ .../src/source-selector/index.ts | 81 +++++++++++ .../source-selector/source-selector.test.ts | 57 ++++++++ .../visual-chat-media-core/src/sync/index.ts | 50 +++++++ .../src/sync/sync.test.ts | 69 ++++++++++ .../visual-chat-media-core/tsdown.config.ts | 11 ++ .../visual-chat-media-core/vitest.config.ts | 7 + .../visual-chat-model-minicpmo/package.json | 35 +++++ .../src/backends/common/backend-events.ts | 1 + .../src/backends/common/backend-types.ts | 51 +++++++ .../src/backends/index.ts | 12 ++ .../visual-chat-model-minicpmo/src/index.ts | 15 ++ .../src/ollama/backend.ts | 116 ++++++++++++++++ .../src/ollama/config.ts | 18 +++ .../src/ollama/http-api.ts | 63 +++++++++ .../src/prompts/assistant-en.ts | 1 + .../src/prompts/assistant-zh.ts | 8 ++ .../tsdown.config.ts | 11 ++ .../visual-chat-observability/package.json | 36 +++++ .../visual-chat-observability/src/index.ts | 3 + .../src/logger/index.ts | 46 +++++++ .../src/metrics/index.ts | 81 +++++++++++ .../src/tracing/index.ts | 29 ++++ .../tsdown.config.ts | 11 ++ packages/visual-chat-runtime/package.json | 38 ++++++ packages/visual-chat-runtime/src/index.ts | 3 + .../src/orchestrator/index.ts | 129 ++++++++++++++++++ .../src/session-store/index.ts | 43 ++++++ packages/visual-chat-runtime/tsdown.config.ts | 11 ++ 45 files changed, 1847 insertions(+) create mode 100644 packages/visual-chat-livekit/package.json create mode 100644 packages/visual-chat-livekit/src/index.ts create mode 100644 packages/visual-chat-livekit/src/room/index.ts create mode 100644 packages/visual-chat-livekit/src/token/index.ts create mode 100644 packages/visual-chat-livekit/src/tracks/index.ts create mode 100644 packages/visual-chat-livekit/src/webhook/index.ts create mode 100644 packages/visual-chat-livekit/tsdown.config.ts create mode 100644 packages/visual-chat-media-core/package.json create mode 100644 packages/visual-chat-media-core/src/chunking/chunking.test.ts create mode 100644 packages/visual-chat-media-core/src/chunking/index.ts create mode 100644 packages/visual-chat-media-core/src/index.ts create mode 100644 packages/visual-chat-media-core/src/ring-buffer/index.ts create mode 100644 packages/visual-chat-media-core/src/ring-buffer/ring-buffer.test.ts create mode 100644 packages/visual-chat-media-core/src/session-context/index.ts create mode 100644 packages/visual-chat-media-core/src/session-context/session-context.test.ts create mode 100644 packages/visual-chat-media-core/src/source-registry/index.ts create mode 100644 packages/visual-chat-media-core/src/source-registry/source-registry.test.ts create mode 100644 packages/visual-chat-media-core/src/source-selector/index.ts create mode 100644 packages/visual-chat-media-core/src/source-selector/source-selector.test.ts create mode 100644 packages/visual-chat-media-core/src/sync/index.ts create mode 100644 packages/visual-chat-media-core/src/sync/sync.test.ts create mode 100644 packages/visual-chat-media-core/tsdown.config.ts create mode 100644 packages/visual-chat-media-core/vitest.config.ts create mode 100644 packages/visual-chat-model-minicpmo/package.json create mode 100644 packages/visual-chat-model-minicpmo/src/backends/common/backend-events.ts create mode 100644 packages/visual-chat-model-minicpmo/src/backends/common/backend-types.ts create mode 100644 packages/visual-chat-model-minicpmo/src/backends/index.ts create mode 100644 packages/visual-chat-model-minicpmo/src/index.ts create mode 100644 packages/visual-chat-model-minicpmo/src/ollama/backend.ts create mode 100644 packages/visual-chat-model-minicpmo/src/ollama/config.ts create mode 100644 packages/visual-chat-model-minicpmo/src/ollama/http-api.ts create mode 100644 packages/visual-chat-model-minicpmo/src/prompts/assistant-en.ts create mode 100644 packages/visual-chat-model-minicpmo/src/prompts/assistant-zh.ts create mode 100644 packages/visual-chat-model-minicpmo/tsdown.config.ts create mode 100644 packages/visual-chat-observability/package.json create mode 100644 packages/visual-chat-observability/src/index.ts create mode 100644 packages/visual-chat-observability/src/logger/index.ts create mode 100644 packages/visual-chat-observability/src/metrics/index.ts create mode 100644 packages/visual-chat-observability/src/tracing/index.ts create mode 100644 packages/visual-chat-observability/tsdown.config.ts create mode 100644 packages/visual-chat-runtime/package.json create mode 100644 packages/visual-chat-runtime/src/index.ts create mode 100644 packages/visual-chat-runtime/src/orchestrator/index.ts create mode 100644 packages/visual-chat-runtime/src/session-store/index.ts create mode 100644 packages/visual-chat-runtime/tsdown.config.ts diff --git a/packages/visual-chat-livekit/package.json b/packages/visual-chat-livekit/package.json new file mode 100644 index 0000000000..25ea47e3a0 --- /dev/null +++ b/packages/visual-chat-livekit/package.json @@ -0,0 +1,38 @@ +{ + "name": "@proj-airi/visual-chat-livekit", + "type": "module", + "version": "0.0.1", + "private": true, + "description": "LiveKit transport adapter for AIRI visual chat", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/moeru-ai/airi.git", + "directory": "packages/visual-chat-livekit" + }, + "exports": { + ".": { + "types": "./dist/index.d.mts", + "default": "./dist/index.mjs" + } + }, + "main": "./dist/index.mjs", + "types": "./dist/index.d.mts", + "files": [ + "README.md", + "dist", + "package.json" + ], + "scripts": { + "dev": "pnpm run build", + "build": "tsdown", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@livekit/rtc-node": "^0.13.24", + "@proj-airi/visual-chat-observability": "workspace:^", + "@proj-airi/visual-chat-protocol": "workspace:^", + "@proj-airi/visual-chat-shared": "workspace:^", + "livekit-server-sdk": "^2.13.0" + } +} diff --git a/packages/visual-chat-livekit/src/index.ts b/packages/visual-chat-livekit/src/index.ts new file mode 100644 index 0000000000..12b10f8476 --- /dev/null +++ b/packages/visual-chat-livekit/src/index.ts @@ -0,0 +1,8 @@ +export { connectAsAgent } from './room' +export type { RoomConnection, RoomConnectionOptions } from './room' +export { generateAgentToken, generateToken } from './token' +export type { TokenOptions } from './token' +export { publishAudioSource, sendTextStream, subscribeAudioTracks, subscribeTextStream, subscribeVideoTracks } from './tracks' +export type { AudioFrameHandler, TextStreamHandler, VideoFrameHandler } from './tracks' +export { createWebhookHandler } from './webhook' +export type { LiveKitEvent, WebhookEventHandler } from './webhook' diff --git a/packages/visual-chat-livekit/src/room/index.ts b/packages/visual-chat-livekit/src/room/index.ts new file mode 100644 index 0000000000..c1dcc1e02c --- /dev/null +++ b/packages/visual-chat-livekit/src/room/index.ts @@ -0,0 +1,52 @@ +import { Room, RoomEvent } from '@livekit/rtc-node' +import { createLiveKitLogger } from '@proj-airi/visual-chat-observability' + +import { generateAgentToken } from '../token' + +const log = createLiveKitLogger() + +export interface RoomConnectionOptions { + livekitUrl: string + apiKey: string + apiSecret: string + roomName: string + agentName?: string +} + +export interface RoomConnection { + room: Room + disconnect: () => Promise +} + +export async function connectAsAgent(opts: RoomConnectionOptions): Promise { + const token = await generateAgentToken( + opts.apiKey, + opts.apiSecret, + opts.roomName, + opts.agentName, + ) + + const room = new Room() + + room.on(RoomEvent.ParticipantConnected, (p) => { + log.withTag('room').log(`Participant connected: ${p.identity}`) + }) + + room.on(RoomEvent.ParticipantDisconnected, (p) => { + log.withTag('room').log(`Participant disconnected: ${p.identity}`) + }) + + room.on(RoomEvent.Disconnected, (reason) => { + log.withTag('room').warn(`Room disconnected: ${reason}`) + }) + + await room.connect(opts.livekitUrl, token, { autoSubscribe: true }) + log.withTag('room').log(`Connected to room: ${opts.roomName}`) + + const disconnect = async () => { + await room.disconnect() + log.withTag('room').log(`Disconnected from room: ${opts.roomName}`) + } + + return { room, disconnect } +} diff --git a/packages/visual-chat-livekit/src/token/index.ts b/packages/visual-chat-livekit/src/token/index.ts new file mode 100644 index 0000000000..290255266a --- /dev/null +++ b/packages/visual-chat-livekit/src/token/index.ts @@ -0,0 +1,52 @@ +import { AccessToken } from 'livekit-server-sdk' + +export interface TokenOptions { + apiKey: string + apiSecret: string + roomName: string + participantName: string + participantIdentity: string + ttl?: number // seconds, default 3600 + canPublish?: boolean + canSubscribe?: boolean + canPublishData?: boolean + hidden?: boolean +} + +export async function generateToken(opts: TokenOptions): Promise { + const at = new AccessToken(opts.apiKey, opts.apiSecret, { + identity: opts.participantIdentity, + name: opts.participantName, + ttl: opts.ttl ?? 3600, + }) + + at.addGrant({ + room: opts.roomName, + roomJoin: true, + canPublish: opts.canPublish ?? true, + canSubscribe: opts.canSubscribe ?? true, + canPublishData: opts.canPublishData ?? true, + hidden: opts.hidden ?? false, + }) + + return await at.toJwt() +} + +export async function generateAgentToken( + apiKey: string, + apiSecret: string, + roomName: string, + agentName: string = 'airi-agent', +): Promise { + return generateToken({ + apiKey, + apiSecret, + roomName, + participantName: agentName, + participantIdentity: `agent_${agentName}`, + canPublish: true, + canSubscribe: true, + canPublishData: true, + hidden: true, + }) +} diff --git a/packages/visual-chat-livekit/src/tracks/index.ts b/packages/visual-chat-livekit/src/tracks/index.ts new file mode 100644 index 0000000000..4990fd206c --- /dev/null +++ b/packages/visual-chat-livekit/src/tracks/index.ts @@ -0,0 +1,85 @@ +import type { AudioFrame, Room, VideoFrame } from '@livekit/rtc-node' + +import { AudioSource, AudioStream, LocalAudioTrack, RoomEvent, TrackPublishOptions, VideoStream } from '@livekit/rtc-node' +import { createLiveKitLogger } from '@proj-airi/visual-chat-observability' +import { AUDIO_SAMPLE_RATE, LIVEKIT_TEXT_STREAM_TOPIC, TTS_OUTPUT_SAMPLE_RATE } from '@proj-airi/visual-chat-protocol' + +const log = createLiveKitLogger() + +export type AudioFrameHandler = (frame: AudioFrame, participantIdentity: string, trackSid: string) => void +export type VideoFrameHandler = (frame: VideoFrame, participantIdentity: string, trackSid: string) => void +export type TextStreamHandler = (text: string, participantIdentity: string, topic: string) => void + +export function subscribeAudioTracks(room: Room, handler: AudioFrameHandler): void { + room.on(RoomEvent.TrackSubscribed, (track, _pub, participant) => { + if (track.kind !== 'audio') + return + + const stream = new AudioStream(track, AUDIO_SAMPLE_RATE, 1) + const trackSid = track.sid! + const identity = participant.identity + + void (async () => { + for await (const event of stream) { + handler(event.frame, identity, trackSid) + } + })() + + log.withTag('tracks').log(`Audio track subscribed: ${trackSid} from ${identity}`) + }) +} + +export function subscribeVideoTracks(room: Room, handler: VideoFrameHandler): void { + room.on(RoomEvent.TrackSubscribed, (track, _pub, participant) => { + if (track.kind !== 'video') + return + + const stream = new VideoStream(track) + const trackSid = track.sid! + const identity = participant.identity + + void (async () => { + for await (const event of stream) { + handler(event.frame, identity, trackSid) + } + })() + + log.withTag('tracks').log(`Video track subscribed: ${trackSid} from ${identity}`) + }) +} + +export async function publishAudioSource(room: Room, sampleRate: number = TTS_OUTPUT_SAMPLE_RATE): Promise { + const source = new AudioSource(sampleRate, 1) + const track = LocalAudioTrack.createAudioTrack('airi-tts', source) + + await room.localParticipant.publishTrack(track, new TrackPublishOptions()) + log.withTag('tracks').log('Published TTS audio track') + + return source +} + +export function subscribeTextStream(room: Room, handler: TextStreamHandler): void { + room.registerTextStreamHandler(LIVEKIT_TEXT_STREAM_TOPIC, (reader, participantIdentity) => { + void (async () => { + try { + const chunks: string[] = [] + for await (const chunk of reader) { + chunks.push(chunk) + } + const fullText = chunks.join('') + handler(fullText, participantIdentity, LIVEKIT_TEXT_STREAM_TOPIC) + } + catch (err) { + log.withTag('tracks').warn(`Text stream read error: ${err}`) + } + })() + }) + log.withTag('tracks').log(`Text stream handler registered for topic: ${LIVEKIT_TEXT_STREAM_TOPIC}`) +} + +export async function sendTextStream(room: Room, destinationIdentities: string[], text: string): Promise { + await room.localParticipant.sendText(text, { + topic: LIVEKIT_TEXT_STREAM_TOPIC, + destinationIdentities, + }) +} diff --git a/packages/visual-chat-livekit/src/webhook/index.ts b/packages/visual-chat-livekit/src/webhook/index.ts new file mode 100644 index 0000000000..59d9285545 --- /dev/null +++ b/packages/visual-chat-livekit/src/webhook/index.ts @@ -0,0 +1,49 @@ +import { createLiveKitLogger } from '@proj-airi/visual-chat-observability' +import { WebhookReceiver } from 'livekit-server-sdk' + +const log = createLiveKitLogger() + +export interface LiveKitEvent { + event: string + room?: { name: string, sid: string } + participant?: { identity: string, sid: string } + track?: { sid: string, type: string } + createdAt: number +} + +export type WebhookEventHandler = (event: LiveKitEvent) => void | Promise + +export function createWebhookHandler( + apiKey: string, + apiSecret: string, + onEvent: WebhookEventHandler, +) { + const receiver = new WebhookReceiver(apiKey, apiSecret) + + return async (body: string, authHeader: string) => { + try { + const event = await receiver.receive(body, authHeader) + + const parsed: LiveKitEvent = { + event: event.event ?? 'unknown', + room: event.room ? { name: event.room.name, sid: event.room.sid } : undefined, + participant: event.participant + ? { identity: event.participant.identity, sid: event.participant.sid } + : undefined, + track: event.track + ? { sid: event.track.sid, type: event.track.type ?? 'unknown' } + : undefined, + createdAt: event.createdAt + ? Math.floor(Number(event.createdAt) * 1000) + : Date.now(), + } + + log.withTag('webhook').log(`Event: ${parsed.event} room=${parsed.room?.name}`) + await onEvent(parsed) + } + catch (err) { + log.withTag('webhook').error(`Webhook verification failed: ${err}`) + throw err + } + } +} diff --git a/packages/visual-chat-livekit/tsdown.config.ts b/packages/visual-chat-livekit/tsdown.config.ts new file mode 100644 index 0000000000..e0cb0b3266 --- /dev/null +++ b/packages/visual-chat-livekit/tsdown.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from 'tsdown' + +export default defineConfig({ + entry: { + index: 'src/index.ts', + }, + target: 'node18', + outDir: 'dist', + clean: true, + dts: true, +}) diff --git a/packages/visual-chat-media-core/package.json b/packages/visual-chat-media-core/package.json new file mode 100644 index 0000000000..da6c170867 --- /dev/null +++ b/packages/visual-chat-media-core/package.json @@ -0,0 +1,35 @@ +{ + "name": "@proj-airi/visual-chat-media-core", + "type": "module", + "version": "0.0.1", + "private": true, + "description": "Transport-agnostic media processing core for AIRI visual chat", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/moeru-ai/airi.git", + "directory": "packages/visual-chat-media-core" + }, + "exports": { + ".": { + "types": "./dist/index.d.mts", + "default": "./dist/index.mjs" + } + }, + "main": "./dist/index.mjs", + "types": "./dist/index.d.mts", + "files": [ + "README.md", + "dist", + "package.json" + ], + "scripts": { + "dev": "pnpm run build", + "build": "tsdown", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@proj-airi/visual-chat-protocol": "workspace:^", + "@proj-airi/visual-chat-shared": "workspace:^" + } +} diff --git a/packages/visual-chat-media-core/src/chunking/chunking.test.ts b/packages/visual-chat-media-core/src/chunking/chunking.test.ts new file mode 100644 index 0000000000..8f974787ae --- /dev/null +++ b/packages/visual-chat-media-core/src/chunking/chunking.test.ts @@ -0,0 +1,58 @@ +import { AUDIO_CHUNK_BYTES } from '@proj-airi/visual-chat-protocol' +import { describe, expect, it } from 'vitest' + +import { generateSilentWav, isValidAudioChunk, normalizeAudioChunk, pcmToWav } from '.' + +describe('chunking', () => { + it('pcmToWav should produce valid WAV header', () => { + const pcm = Buffer.alloc(AUDIO_CHUNK_BYTES) + const wav = pcmToWav(pcm) + + expect(wav.toString('ascii', 0, 4)).toBe('RIFF') + expect(wav.toString('ascii', 8, 12)).toBe('WAVE') + expect(wav.toString('ascii', 12, 16)).toBe('fmt ') + expect(wav.readUInt16LE(20)).toBe(1) // PCM format + expect(wav.readUInt16LE(22)).toBe(1) // mono + expect(wav.readUInt32LE(24)).toBe(16000) // sample rate + expect(wav.readUInt16LE(34)).toBe(16) // bits per sample + expect(wav.toString('ascii', 36, 40)).toBe('data') + expect(wav.readUInt32LE(40)).toBe(AUDIO_CHUNK_BYTES) // data size + expect(wav.length).toBe(44 + AUDIO_CHUNK_BYTES) + }) + + it('generateSilentWav should produce a valid WAV of silence', () => { + const wav = generateSilentWav() + expect(wav.toString('ascii', 0, 4)).toBe('RIFF') + expect(wav.length).toBe(44 + AUDIO_CHUNK_BYTES) + + const dataSection = wav.subarray(44) + const allZero = dataSection.every(b => b === 0) + expect(allZero).toBe(true) + }) + + it('isValidAudioChunk should validate chunk size', () => { + expect(isValidAudioChunk(Buffer.alloc(AUDIO_CHUNK_BYTES))).toBe(true) + expect(isValidAudioChunk(Buffer.alloc(100))).toBe(false) + expect(isValidAudioChunk(Buffer.alloc(AUDIO_CHUNK_BYTES + 1))).toBe(false) + }) + + it('normalizeAudioChunk should pad short data', () => { + const short = Buffer.alloc(100, 0xAB) + const normalized = normalizeAudioChunk(short) + expect(normalized.length).toBe(AUDIO_CHUNK_BYTES) + expect(normalized[0]).toBe(0xAB) + expect(normalized[100]).toBe(0) // padded with zeros + }) + + it('normalizeAudioChunk should trim long data', () => { + const long = Buffer.alloc(AUDIO_CHUNK_BYTES + 500, 0xCD) + const normalized = normalizeAudioChunk(long) + expect(normalized.length).toBe(AUDIO_CHUNK_BYTES) + }) + + it('normalizeAudioChunk should pass through exact size', () => { + const exact = Buffer.alloc(AUDIO_CHUNK_BYTES, 0xFF) + const normalized = normalizeAudioChunk(exact) + expect(normalized).toBe(exact) // same reference + }) +}) diff --git a/packages/visual-chat-media-core/src/chunking/index.ts b/packages/visual-chat-media-core/src/chunking/index.ts new file mode 100644 index 0000000000..c1317bfb72 --- /dev/null +++ b/packages/visual-chat-media-core/src/chunking/index.ts @@ -0,0 +1,65 @@ +import { AUDIO_CHUNK_BYTES, AUDIO_SAMPLE_RATE } from '@proj-airi/visual-chat-protocol' + +/** + * Creates a WAV file buffer from 16-bit PCM mono audio data. + * llama.cpp-omni requires 16kHz mono WAV, exactly 1 second per chunk. + */ +export function pcmToWav(pcmData: Buffer): Buffer { + const wavHeaderSize = 44 + const dataSize = pcmData.length + const fileSize = wavHeaderSize + dataSize + const wav = Buffer.alloc(fileSize) + + // RIFF header + wav.write('RIFF', 0) + wav.writeUInt32LE(fileSize - 8, 4) + wav.write('WAVE', 8) + + // fmt chunk + wav.write('fmt ', 12) + wav.writeUInt32LE(16, 16) // chunk size + wav.writeUInt16LE(1, 20) // PCM format + wav.writeUInt16LE(1, 22) // mono + wav.writeUInt32LE(AUDIO_SAMPLE_RATE, 24) // sample rate + wav.writeUInt32LE(AUDIO_SAMPLE_RATE * 2, 28) // byte rate (16-bit mono) + wav.writeUInt16LE(2, 32) // block align + wav.writeUInt16LE(16, 34) // bits per sample + + // data chunk + wav.write('data', 36) + wav.writeUInt32LE(dataSize, 40) + pcmData.copy(wav, 44) + + return wav +} + +/** + * Generates a 1-second silent WAV at 16kHz mono. + * Used when no audio input is available. + */ +export function generateSilentWav(): Buffer { + const silentPcm = Buffer.alloc(AUDIO_CHUNK_BYTES) + return pcmToWav(silentPcm) +} + +/** + * Validates that a PCM buffer is exactly 1 second at 16kHz 16-bit mono. + */ +export function isValidAudioChunk(data: Buffer): boolean { + return data.length === AUDIO_CHUNK_BYTES +} + +/** + * Pads or trims PCM data to exactly 1 second. + */ +export function normalizeAudioChunk(data: Buffer): Buffer { + if (data.length === AUDIO_CHUNK_BYTES) + return data + + if (data.length > AUDIO_CHUNK_BYTES) + return data.subarray(0, AUDIO_CHUNK_BYTES) + + const padded = Buffer.alloc(AUDIO_CHUNK_BYTES) + data.copy(padded) + return padded +} diff --git a/packages/visual-chat-media-core/src/index.ts b/packages/visual-chat-media-core/src/index.ts new file mode 100644 index 0000000000..e8fd09b6ef --- /dev/null +++ b/packages/visual-chat-media-core/src/index.ts @@ -0,0 +1,8 @@ +export { generateSilentWav, isValidAudioChunk, normalizeAudioChunk, pcmToWav } from './chunking' +export { RingBuffer } from './ring-buffer' +export { createSessionContext, updateSessionActivity, updateSessionState } from './session-context' +export { SourceRegistry } from './source-registry' +export { ManualSwitchPolicy } from './source-selector' +export type { SourceSelectionPolicy, SourceSelectionResult } from './source-selector' +export { alignAudioToVideo, createAVPair } from './sync' +export type { AVPair } from './sync' diff --git a/packages/visual-chat-media-core/src/ring-buffer/index.ts b/packages/visual-chat-media-core/src/ring-buffer/index.ts new file mode 100644 index 0000000000..26596c958b --- /dev/null +++ b/packages/visual-chat-media-core/src/ring-buffer/index.ts @@ -0,0 +1,55 @@ +export class RingBuffer { + private buffer: (T | undefined)[] + private writeIndex = 0 + private count = 0 + + constructor(public readonly capacity: number) { + if (capacity < 1) + throw new Error('RingBuffer capacity must be >= 1') + this.buffer = Array.from({ length: capacity }) + } + + write(item: T): void { + this.buffer[this.writeIndex] = item + this.writeIndex = (this.writeIndex + 1) % this.capacity + if (this.count < this.capacity) + this.count++ + } + + readLatest(n: number): T[] { + const count = Math.min(n, this.count) + const result: T[] = [] + + for (let i = 0; i < count; i++) { + const idx = (this.writeIndex - count + i + this.capacity) % this.capacity + result.push(this.buffer[idx] as T) + } + + return result + } + + peek(): T | undefined { + if (this.count === 0) + return undefined + const idx = (this.writeIndex - 1 + this.capacity) % this.capacity + return this.buffer[idx] + } + + get size(): number { + return this.count + } + + get isFull(): boolean { + return this.count === this.capacity + } + + clear(): void { + this.buffer = Array.from({ length: this.capacity }) + this.writeIndex = 0 + this.count = 0 + } + + toArray(): T[] { + return this.readLatest(this.count) + } +} diff --git a/packages/visual-chat-media-core/src/ring-buffer/ring-buffer.test.ts b/packages/visual-chat-media-core/src/ring-buffer/ring-buffer.test.ts new file mode 100644 index 0000000000..4fd32f2aff --- /dev/null +++ b/packages/visual-chat-media-core/src/ring-buffer/ring-buffer.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from 'vitest' + +import { RingBuffer } from '.' + +describe('ringBuffer', () => { + it('should create with given capacity', () => { + const buf = new RingBuffer(5) + expect(buf.capacity).toBe(5) + expect(buf.size).toBe(0) + expect(buf.isFull).toBe(false) + }) + + it('should throw on capacity < 1', () => { + expect(() => new RingBuffer(0)).toThrow() + expect(() => new RingBuffer(-1)).toThrow() + }) + + it('should write and read items', () => { + const buf = new RingBuffer(3) + buf.write(1) + buf.write(2) + expect(buf.size).toBe(2) + expect(buf.readLatest(2)).toEqual([1, 2]) + }) + + it('should overwrite oldest when full', () => { + const buf = new RingBuffer(3) + buf.write(1) + buf.write(2) + buf.write(3) + expect(buf.isFull).toBe(true) + + buf.write(4) // overwrites 1 + expect(buf.size).toBe(3) + expect(buf.readLatest(3)).toEqual([2, 3, 4]) + }) + + it('should peek latest item', () => { + const buf = new RingBuffer(3) + expect(buf.peek()).toBeUndefined() + buf.write('a') + expect(buf.peek()).toBe('a') + buf.write('b') + expect(buf.peek()).toBe('b') + }) + + it('should read fewer items than requested when buffer is not full', () => { + const buf = new RingBuffer(10) + buf.write(1) + buf.write(2) + expect(buf.readLatest(5)).toEqual([1, 2]) + }) + + it('should clear correctly', () => { + const buf = new RingBuffer(3) + buf.write(1) + buf.write(2) + buf.clear() + expect(buf.size).toBe(0) + expect(buf.peek()).toBeUndefined() + expect(buf.readLatest(3)).toEqual([]) + }) + + it('should convert to array', () => { + const buf = new RingBuffer(3) + buf.write(10) + buf.write(20) + buf.write(30) + buf.write(40) // overwrites 10 + expect(buf.toArray()).toEqual([20, 30, 40]) + }) +}) diff --git a/packages/visual-chat-media-core/src/session-context/index.ts b/packages/visual-chat-media-core/src/session-context/index.ts new file mode 100644 index 0000000000..80ccacdf8e --- /dev/null +++ b/packages/visual-chat-media-core/src/session-context/index.ts @@ -0,0 +1,38 @@ +import type { SessionContext } from '@proj-airi/visual-chat-protocol' + +import { generateSessionId } from '@proj-airi/visual-chat-shared' + +export function createSessionContext( + roomName: string, + sessionId?: string, +): SessionContext { + const now = Date.now() + return { + sessionId: sessionId ?? generateSessionId(), + roomName, + mode: 'vision-text-realtime', + state: 'idle', + activeVideoSource: null, + activeAudioSource: null, + standbyVideoSources: [], + standbyAudioSources: [], + inferenceState: { + isRunning: false, + currentCnt: 0, + errorCount: 0, + }, + createdAt: now, + lastActivityAt: now, + } +} + +export function updateSessionActivity(ctx: SessionContext): SessionContext { + return { ...ctx, lastActivityAt: Date.now() } +} + +export function updateSessionState( + ctx: SessionContext, + state: SessionContext['state'], +): SessionContext { + return { ...ctx, state, lastActivityAt: Date.now() } +} diff --git a/packages/visual-chat-media-core/src/session-context/session-context.test.ts b/packages/visual-chat-media-core/src/session-context/session-context.test.ts new file mode 100644 index 0000000000..52511987c8 --- /dev/null +++ b/packages/visual-chat-media-core/src/session-context/session-context.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from 'vitest' + +import { createSessionContext, updateSessionActivity, updateSessionState } from '.' + +describe('sessionContext', () => { + it('should create with defaults', () => { + const ctx = createSessionContext('test-room') + expect(ctx.sessionId).toMatch(/^ses_/) + expect(ctx.roomName).toBe('test-room') + expect(ctx.mode).toBe('vision-text-realtime') + expect(ctx.state).toBe('idle') + expect(ctx.activeVideoSource).toBeNull() + expect(ctx.activeAudioSource).toBeNull() + expect(ctx.standbyVideoSources).toEqual([]) + expect(ctx.standbyAudioSources).toEqual([]) + expect(ctx.inferenceState.isRunning).toBe(false) + expect(ctx.inferenceState.currentCnt).toBe(0) + expect(ctx.createdAt).toBeGreaterThan(0) + }) + + it('should update state immutably', () => { + const ctx = createSessionContext('room2') + const updated = updateSessionState(ctx, 'connected') + expect(updated.state).toBe('connected') + expect(ctx.state).toBe('idle') // original unchanged + expect(updated.lastActivityAt).toBeGreaterThanOrEqual(ctx.lastActivityAt) + }) + + it('should update activity timestamp', () => { + const ctx = createSessionContext('room3') + const before = ctx.lastActivityAt + const updated = updateSessionActivity(ctx) + expect(updated.lastActivityAt).toBeGreaterThanOrEqual(before) + }) +}) diff --git a/packages/visual-chat-media-core/src/source-registry/index.ts b/packages/visual-chat-media-core/src/source-registry/index.ts new file mode 100644 index 0000000000..4fc6bf75ff --- /dev/null +++ b/packages/visual-chat-media-core/src/source-registry/index.ts @@ -0,0 +1,99 @@ +import type { SourceDescriptor } from '@proj-airi/visual-chat-protocol' + +import { generateSourceId } from '@proj-airi/visual-chat-shared' + +export class SourceRegistry { + private sources = new Map() + + register( + participantIdentity: string, + trackSid: string, + sourceType: SourceDescriptor['sourceType'], + ): SourceDescriptor { + const existing = this.findByTrackSid(trackSid) + if (existing) + return existing + + const descriptor: SourceDescriptor = { + sourceId: generateSourceId(), + participantIdentity, + trackSid, + sourceType, + isActive: false, + lastFrameTimestamp: 0, + } + + this.sources.set(descriptor.sourceId, descriptor) + return descriptor + } + + unregister(sourceId: string): boolean { + return this.sources.delete(sourceId) + } + + unregisterByTrackSid(trackSid: string): boolean { + const source = this.findByTrackSid(trackSid) + if (source) + return this.sources.delete(source.sourceId) + return false + } + + get(sourceId: string): SourceDescriptor | undefined { + return this.sources.get(sourceId) + } + + findByTrackSid(trackSid: string): SourceDescriptor | undefined { + for (const source of this.sources.values()) { + if (source.trackSid === trackSid) + return source + } + return undefined + } + + findByType(sourceType: SourceDescriptor['sourceType']): SourceDescriptor[] { + return [...this.sources.values()].filter(s => s.sourceType === sourceType) + } + + findByParticipant(participantIdentity: string): SourceDescriptor[] { + return [...this.sources.values()].filter(s => s.participantIdentity === participantIdentity) + } + + getVideoSources(): SourceDescriptor[] { + return [...this.sources.values()].filter(s => + s.sourceType === 'phone-camera' + || s.sourceType === 'laptop-camera' + || s.sourceType === 'screen-share', + ) + } + + getAudioSources(): SourceDescriptor[] { + return [...this.sources.values()].filter(s => + s.sourceType === 'phone-mic' + || s.sourceType === 'laptop-mic', + ) + } + + getAll(): SourceDescriptor[] { + return [...this.sources.values()] + } + + get size(): number { + return this.sources.size + } + + updateTimestamp(sourceId: string, timestamp: number): void { + const source = this.sources.get(sourceId) + if (source) + source.lastFrameTimestamp = timestamp + } + + setActive(sourceId: string, active: boolean): void { + const source = this.sources.get(sourceId) + if (source) + source.isActive = active + } + + clear(): void { + this.sources.clear() + } +} diff --git a/packages/visual-chat-media-core/src/source-registry/source-registry.test.ts b/packages/visual-chat-media-core/src/source-registry/source-registry.test.ts new file mode 100644 index 0000000000..5516c629b8 --- /dev/null +++ b/packages/visual-chat-media-core/src/source-registry/source-registry.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from 'vitest' + +import { SourceRegistry } from '.' + +describe('sourceRegistry', () => { + it('should register a source', () => { + const reg = new SourceRegistry() + const src = reg.register('user1', 'track-1', 'phone-camera') + expect(src.sourceId).toMatch(/^src_/) + expect(src.participantIdentity).toBe('user1') + expect(src.sourceType).toBe('phone-camera') + expect(reg.size).toBe(1) + }) + + it('should not duplicate on same trackSid', () => { + const reg = new SourceRegistry() + const src1 = reg.register('user1', 'track-1', 'phone-camera') + const src2 = reg.register('user1', 'track-1', 'phone-camera') + expect(src1.sourceId).toBe(src2.sourceId) + expect(reg.size).toBe(1) + }) + + it('should unregister by sourceId', () => { + const reg = new SourceRegistry() + const src = reg.register('user1', 'track-1', 'phone-camera') + expect(reg.unregister(src.sourceId)).toBe(true) + expect(reg.size).toBe(0) + }) + + it('should find by type', () => { + const reg = new SourceRegistry() + reg.register('user1', 't1', 'phone-camera') + reg.register('user1', 't2', 'phone-mic') + reg.register('user2', 't3', 'laptop-camera') + + expect(reg.findByType('phone-camera')).toHaveLength(1) + expect(reg.getVideoSources()).toHaveLength(2) + expect(reg.getAudioSources()).toHaveLength(1) + }) + + it('should update timestamp and active state', () => { + const reg = new SourceRegistry() + const src = reg.register('user1', 't1', 'phone-camera') + reg.updateTimestamp(src.sourceId, 12345) + reg.setActive(src.sourceId, true) + + const updated = reg.get(src.sourceId)! + expect(updated.lastFrameTimestamp).toBe(12345) + expect(updated.isActive).toBe(true) + }) +}) diff --git a/packages/visual-chat-media-core/src/source-selector/index.ts b/packages/visual-chat-media-core/src/source-selector/index.ts new file mode 100644 index 0000000000..4390f3f161 --- /dev/null +++ b/packages/visual-chat-media-core/src/source-selector/index.ts @@ -0,0 +1,81 @@ +import type { SourceDescriptor } from '@proj-airi/visual-chat-protocol' + +import type { SourceRegistry } from '../source-registry' + +export interface SourceSelectionResult { + activeVideo: SourceDescriptor | null + activeAudio: SourceDescriptor | null + standbyVideo: SourceDescriptor[] + standbyAudio: SourceDescriptor[] +} + +export interface SourceSelectionPolicy { + name: string + select: (registry: SourceRegistry, manualOverride?: string) => SourceSelectionResult +} + +const VIDEO_SOURCE_TYPES = new Set(['phone-camera', 'laptop-camera', 'screen-share']) + +const VIDEO_PRIORITY: SourceDescriptor['sourceType'][] = ['phone-camera', 'laptop-camera', 'screen-share'] + +const DEVICE_AUDIO_MAP: Record = { + 'phone-camera': 'phone-mic', + 'laptop-camera': 'laptop-mic', + 'screen-share': 'laptop-mic', +} + +/** + * Manual source selection: accepts either a sourceId (e.g. "src_abc123") + * or a sourceType string (e.g. "screen-share") as override. + */ +export class ManualSwitchPolicy implements SourceSelectionPolicy { + name = 'manual-switch' + + select(registry: SourceRegistry, manualOverride?: string): SourceSelectionResult { + const videoSources = registry.getVideoSources() + const audioSources = registry.getAudioSources() + + let activeVideo: SourceDescriptor | null = null + + if (manualOverride) { + // Try as sourceId first + const byId = registry.get(manualOverride) + if (byId && VIDEO_SOURCE_TYPES.has(byId.sourceType)) { + activeVideo = byId + } + else { + // Try as sourceType string + const isSourceType = VIDEO_SOURCE_TYPES.has(manualOverride as SourceDescriptor['sourceType']) + if (isSourceType) { + activeVideo = videoSources.find(s => s.sourceType === manualOverride) ?? null + } + } + } + + // Fallback to priority-based selection + if (!activeVideo) { + for (const preferred of VIDEO_PRIORITY) { + const found = videoSources.find(s => s.sourceType === preferred) + if (found) { + activeVideo = found + break + } + } + } + + // Select matching audio source + let activeAudio: SourceDescriptor | null = null + if (activeVideo) { + const preferredAudioType = DEVICE_AUDIO_MAP[activeVideo.sourceType] + if (preferredAudioType) + activeAudio = audioSources.find(s => s.sourceType === preferredAudioType) ?? null + } + if (!activeAudio && audioSources.length > 0) + activeAudio = audioSources[0] + + const standbyVideo = videoSources.filter(s => s !== activeVideo) + const standbyAudio = audioSources.filter(s => s !== activeAudio) + + return { activeVideo, activeAudio, standbyVideo, standbyAudio } + } +} diff --git a/packages/visual-chat-media-core/src/source-selector/source-selector.test.ts b/packages/visual-chat-media-core/src/source-selector/source-selector.test.ts new file mode 100644 index 0000000000..fca9fadbd8 --- /dev/null +++ b/packages/visual-chat-media-core/src/source-selector/source-selector.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from 'vitest' + +import { ManualSwitchPolicy } from '.' +import { SourceRegistry } from '../source-registry' + +describe('manualSwitchPolicy', () => { + it('should select phone-camera as default active video', () => { + const reg = new SourceRegistry() + reg.register('user1', 't1', 'phone-camera') + reg.register('user1', 't2', 'phone-mic') + reg.register('user2', 't3', 'laptop-camera') + + const policy = new ManualSwitchPolicy() + const result = policy.select(reg) + + expect(result.activeVideo?.sourceType).toBe('phone-camera') + expect(result.activeAudio?.sourceType).toBe('phone-mic') + expect(result.standbyVideo).toHaveLength(1) + }) + + it('should allow manual override by sourceType', () => { + const reg = new SourceRegistry() + reg.register('user1', 't1', 'phone-camera') + reg.register('user1', 't2', 'phone-mic') + const screen = reg.register('user2', 't3', 'screen-share') + reg.register('user2', 't4', 'laptop-mic') + + const policy = new ManualSwitchPolicy() + const result = policy.select(reg, screen.sourceId) + + expect(result.activeVideo?.sourceType).toBe('screen-share') + expect(result.activeAudio?.sourceType).toBe('laptop-mic') + }) + + it('should handle empty registry', () => { + const reg = new SourceRegistry() + const policy = new ManualSwitchPolicy() + const result = policy.select(reg) + + expect(result.activeVideo).toBeNull() + expect(result.activeAudio).toBeNull() + expect(result.standbyVideo).toHaveLength(0) + expect(result.standbyAudio).toHaveLength(0) + }) + + it('should fallback to first audio when no matching device audio', () => { + const reg = new SourceRegistry() + reg.register('user1', 't1', 'screen-share') + reg.register('user1', 't2', 'phone-mic') + + const policy = new ManualSwitchPolicy() + const result = policy.select(reg) + + expect(result.activeVideo?.sourceType).toBe('screen-share') + expect(result.activeAudio?.sourceType).toBe('phone-mic') + }) +}) diff --git a/packages/visual-chat-media-core/src/sync/index.ts b/packages/visual-chat-media-core/src/sync/index.ts new file mode 100644 index 0000000000..caf5ea911f --- /dev/null +++ b/packages/visual-chat-media-core/src/sync/index.ts @@ -0,0 +1,50 @@ +import type { AudioChunk, VideoFrame } from '@proj-airi/visual-chat-shared' + +export interface AVPair { + video: VideoFrame | null + audio: AudioChunk | null + timestamp: number +} + +/** + * Finds the closest audio chunk to a given video frame by timestamp. + */ +export function alignAudioToVideo( + videoFrame: VideoFrame, + audioChunks: AudioChunk[], + maxDriftMs: number = 500, +): AudioChunk | null { + if (audioChunks.length === 0) + return null + + let bestMatch: AudioChunk | null = null + let bestDrift = Infinity + + for (const chunk of audioChunks) { + const drift = Math.abs(chunk.timestamp - videoFrame.timestamp) + if (drift < bestDrift) { + bestDrift = drift + bestMatch = chunk + } + } + + if (bestDrift > maxDriftMs) + return null + + return bestMatch +} + +/** + * Creates an AV pair from the latest available data. + */ +export function createAVPair( + latestVideo: VideoFrame | null, + latestAudio: AudioChunk | null, +): AVPair { + const timestamp = latestVideo?.timestamp ?? latestAudio?.timestamp ?? Date.now() + return { + video: latestVideo, + audio: latestAudio, + timestamp, + } +} diff --git a/packages/visual-chat-media-core/src/sync/sync.test.ts b/packages/visual-chat-media-core/src/sync/sync.test.ts new file mode 100644 index 0000000000..56f93d7f78 --- /dev/null +++ b/packages/visual-chat-media-core/src/sync/sync.test.ts @@ -0,0 +1,69 @@ +import type { AudioChunk, VideoFrame } from '@proj-airi/visual-chat-shared' + +import { describe, expect, it } from 'vitest' + +import { alignAudioToVideo, createAVPair } from '.' + +function makeVideo(ts: number): VideoFrame { + return { sourceId: 'v1', timestamp: ts, data: Buffer.alloc(0), width: 640, height: 480 } +} + +function makeAudio(ts: number): AudioChunk { + return { sourceId: 'a1', timestamp: ts, data: Buffer.alloc(0), sampleRate: 16000, channels: 1, durationMs: 1000 } +} + +describe('alignAudioToVideo', () => { + it('should find closest audio chunk', () => { + const video = makeVideo(1000) + const chunks = [makeAudio(500), makeAudio(950), makeAudio(1100)] + const result = alignAudioToVideo(video, chunks) + expect(result?.timestamp).toBe(950) + }) + + it('should return null when drift exceeds maxDriftMs', () => { + const video = makeVideo(1000) + const chunks = [makeAudio(100)] + const result = alignAudioToVideo(video, chunks, 200) + expect(result).toBeNull() + }) + + it('should return null for empty chunks', () => { + const video = makeVideo(1000) + const result = alignAudioToVideo(video, []) + expect(result).toBeNull() + }) + + it('should use default 500ms maxDriftMs', () => { + const video = makeVideo(1000) + const result = alignAudioToVideo(video, [makeAudio(600)]) + expect(result?.timestamp).toBe(600) + + const resultFar = alignAudioToVideo(video, [makeAudio(400)]) + expect(resultFar).toBeNull() + }) +}) + +describe('createAVPair', () => { + it('should create pair from video and audio', () => { + const video = makeVideo(1000) + const audio = makeAudio(990) + const pair = createAVPair(video, audio) + + expect(pair.video).toBe(video) + expect(pair.audio).toBe(audio) + expect(pair.timestamp).toBe(1000) + }) + + it('should use audio timestamp when no video', () => { + const audio = makeAudio(500) + const pair = createAVPair(null, audio) + expect(pair.timestamp).toBe(500) + }) + + it('should handle both null', () => { + const pair = createAVPair(null, null) + expect(pair.video).toBeNull() + expect(pair.audio).toBeNull() + expect(pair.timestamp).toBeGreaterThan(0) + }) +}) diff --git a/packages/visual-chat-media-core/tsdown.config.ts b/packages/visual-chat-media-core/tsdown.config.ts new file mode 100644 index 0000000000..e0cb0b3266 --- /dev/null +++ b/packages/visual-chat-media-core/tsdown.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from 'tsdown' + +export default defineConfig({ + entry: { + index: 'src/index.ts', + }, + target: 'node18', + outDir: 'dist', + clean: true, + dts: true, +}) diff --git a/packages/visual-chat-media-core/vitest.config.ts b/packages/visual-chat-media-core/vitest.config.ts new file mode 100644 index 0000000000..826f4fbf95 --- /dev/null +++ b/packages/visual-chat-media-core/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + include: ['src/**/*.test.ts'], + }, +}) diff --git a/packages/visual-chat-model-minicpmo/package.json b/packages/visual-chat-model-minicpmo/package.json new file mode 100644 index 0000000000..ef54b9ff6a --- /dev/null +++ b/packages/visual-chat-model-minicpmo/package.json @@ -0,0 +1,35 @@ +{ + "name": "@proj-airi/visual-chat-model-minicpmo", + "type": "module", + "version": "0.0.1", + "private": true, + "description": "MiniCPM-o 4.5 model adapter for AIRI visual chat", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/moeru-ai/airi.git", + "directory": "packages/visual-chat-model-minicpmo" + }, + "exports": { + ".": { + "types": "./dist/index.d.mts", + "default": "./dist/index.mjs" + } + }, + "main": "./dist/index.mjs", + "types": "./dist/index.d.mts", + "files": [ + "README.md", + "dist", + "package.json" + ], + "scripts": { + "dev": "pnpm run build", + "build": "tsdown", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@proj-airi/visual-chat-observability": "workspace:^", + "@proj-airi/visual-chat-shared": "workspace:^" + } +} diff --git a/packages/visual-chat-model-minicpmo/src/backends/common/backend-events.ts b/packages/visual-chat-model-minicpmo/src/backends/common/backend-events.ts new file mode 100644 index 0000000000..425d9425cf --- /dev/null +++ b/packages/visual-chat-model-minicpmo/src/backends/common/backend-events.ts @@ -0,0 +1 @@ +export type BackendEventMap = Record diff --git a/packages/visual-chat-model-minicpmo/src/backends/common/backend-types.ts b/packages/visual-chat-model-minicpmo/src/backends/common/backend-types.ts new file mode 100644 index 0000000000..0fd958832a --- /dev/null +++ b/packages/visual-chat-model-minicpmo/src/backends/common/backend-types.ts @@ -0,0 +1,51 @@ +import { dirname } from 'node:path' + +export interface InferenceBackend { + readonly type: string + spawn: () => Promise + shutdown: () => Promise + health: () => Promise + init: (systemPrompt: string, refAudioPath?: string) => Promise + prefill: (audioPath: string, imagePath?: string) => Promise + decode: () => AsyncIterable +} + +export interface DecodedChunk { + text: string + audioFiles: string[] + isListening: boolean + done: boolean +} + +export interface PrefillPayload { + wavPath: string + imagePath?: string + cnt: number +} + +export interface DecodeResult { + text: string + audioOutDir: string + wavFiles: string[] + listenDetected: boolean +} + +export function decodedChunksToDecodeResult(chunks: DecodedChunk[]): DecodeResult { + const last = chunks.at(-1) + if (!last) { + return { + text: '', + audioOutDir: '', + wavFiles: [], + listenDetected: false, + } + } + const wavFiles = last.audioFiles + const audioOutDir = wavFiles.length > 0 ? dirname(wavFiles[0]!) : '' + return { + text: last.text, + audioOutDir, + wavFiles, + listenDetected: last.isListening, + } +} diff --git a/packages/visual-chat-model-minicpmo/src/backends/index.ts b/packages/visual-chat-model-minicpmo/src/backends/index.ts new file mode 100644 index 0000000000..fb58de25e2 --- /dev/null +++ b/packages/visual-chat-model-minicpmo/src/backends/index.ts @@ -0,0 +1,12 @@ +import type { OllamaConfig } from '../ollama/config' +import type { InferenceBackend } from './common/backend-types' + +import { OllamaBackend } from '../ollama/backend' + +export type BackendType = 'ollama' + +export function createBackend(config: OllamaConfig): InferenceBackend { + return new OllamaBackend(config) +} + +export { OllamaBackend } from '../ollama/backend' diff --git a/packages/visual-chat-model-minicpmo/src/index.ts b/packages/visual-chat-model-minicpmo/src/index.ts new file mode 100644 index 0000000000..c00e095dd3 --- /dev/null +++ b/packages/visual-chat-model-minicpmo/src/index.ts @@ -0,0 +1,15 @@ +export type { BackendType } from './backends' +export { createBackend, OllamaBackend } from './backends' +export type { BackendEventMap } from './backends/common/backend-events' +export type { + DecodedChunk, + DecodeResult, + InferenceBackend, + PrefillPayload, +} from './backends/common/backend-types' +export { decodedChunksToDecodeResult } from './backends/common/backend-types' +export type { OllamaConfig } from './ollama/config' +export { defaultOllamaConfig, resolveOllamaConfig } from './ollama/config' +export { checkHealth as checkOllamaHealth, listModels as listOllamaModels } from './ollama/http-api' +export { assistantSystemPromptEn } from './prompts/assistant-en' +export { assistantSystemPromptZh } from './prompts/assistant-zh' diff --git a/packages/visual-chat-model-minicpmo/src/ollama/backend.ts b/packages/visual-chat-model-minicpmo/src/ollama/backend.ts new file mode 100644 index 0000000000..20aa8d8541 --- /dev/null +++ b/packages/visual-chat-model-minicpmo/src/ollama/backend.ts @@ -0,0 +1,116 @@ +import type { DecodedChunk, InferenceBackend } from '../backends/common/backend-types' +import type { OllamaConfig } from './config' +import type { OllamaChatMessage } from './http-api' + +import { readFileSync } from 'node:fs' + +import { createInferenceLogger } from '@proj-airi/visual-chat-observability' + +import { resolveOllamaConfig } from './config' +import { checkHealth, postChat } from './http-api' + +const log = createInferenceLogger() +const MAX_HISTORY = 20 + +export class OllamaBackend implements InferenceBackend { + readonly type = 'ollama' + + private baseUrl: string + private model: string + private systemPrompt = '' + private history: OllamaChatMessage[] = [] + private pendingImageBase64: string | null = null + + constructor(config: OllamaConfig) { + const resolved = resolveOllamaConfig(config) + this.baseUrl = resolved.baseUrl + this.model = resolved.model + } + + async spawn(): Promise { + const ok = await this.health() + if (!ok) + throw new Error(`Ollama is not reachable at ${this.baseUrl}. Run: ollama serve`) + + log.withTag('ollama').log(`Connected to Ollama at ${this.baseUrl}, model: ${this.model}`) + } + + async shutdown(): Promise { + this.history = [] + this.pendingImageBase64 = null + log.withTag('ollama').log('Backend session cleared') + } + + async health(): Promise { + return checkHealth(this.baseUrl) + } + + async init(systemPrompt: string): Promise { + this.systemPrompt = systemPrompt + this.history = [] + log.withTag('ollama').log('Session initialized') + } + + async prefill(audioPath: string, imagePath?: string): Promise { + if (imagePath) { + try { + const data = readFileSync(imagePath) + this.pendingImageBase64 = data.toString('base64') + } + catch { + log.withTag('ollama').warn(`Could not read image: ${imagePath}`) + } + } + } + + async* decode(): AsyncIterable { + const userMessage: OllamaChatMessage = { + role: 'user', + content: 'Describe what you observe in the image. Be concise.', + } + + if (this.pendingImageBase64) + userMessage.images = [this.pendingImageBase64] + + const messages: OllamaChatMessage[] = [ + { role: 'system', content: this.systemPrompt }, + ...this.history, + userMessage, + ] + + const response = await postChat(this.baseUrl, { + model: this.model, + messages, + }) + + const text = response.message?.content ?? '' + + this.history.push(userMessage) + this.history.push({ role: 'assistant', content: text }) + + if (this.history.length > MAX_HISTORY) + this.history = this.history.slice(-MAX_HISTORY) + + this.pendingImageBase64 = null + + yield { + text, + audioFiles: [], + isListening: false, + done: true, + } + } + + writeTempWav(_data: Buffer, _name: string): string { + return '' + } + + writeTempImage(data: Buffer, _name: string): string { + this.pendingImageBase64 = data.toString('base64') + return 'ollama:memory' + } + + readTempFile(_filePath: string): Buffer { + return Buffer.alloc(0) + } +} diff --git a/packages/visual-chat-model-minicpmo/src/ollama/config.ts b/packages/visual-chat-model-minicpmo/src/ollama/config.ts new file mode 100644 index 0000000000..fb283a509f --- /dev/null +++ b/packages/visual-chat-model-minicpmo/src/ollama/config.ts @@ -0,0 +1,18 @@ +import process from 'node:process' + +export interface OllamaConfig { + baseUrl?: string + model?: string +} + +export const defaultOllamaConfig = { + baseUrl: 'http://localhost:11434', + model: 'openbmb/minicpm-v4.5:latest', +} as const + +export function resolveOllamaConfig(config: OllamaConfig): Required { + return { + baseUrl: config.baseUrl ?? process.env.OLLAMA_HOST ?? defaultOllamaConfig.baseUrl, + model: config.model ?? process.env.OLLAMA_MODEL ?? defaultOllamaConfig.model, + } +} diff --git a/packages/visual-chat-model-minicpmo/src/ollama/http-api.ts b/packages/visual-chat-model-minicpmo/src/ollama/http-api.ts new file mode 100644 index 0000000000..897bda87e9 --- /dev/null +++ b/packages/visual-chat-model-minicpmo/src/ollama/http-api.ts @@ -0,0 +1,63 @@ +export interface OllamaChatMessage { + role: 'system' | 'user' | 'assistant' + content: string + images?: string[] +} + +export interface OllamaChatRequest { + model: string + messages: OllamaChatMessage[] + stream?: boolean +} + +export interface OllamaChatResponse { + message: { role: string, content: string } + done: boolean + total_duration?: number + eval_count?: number +} + +export async function postChat( + baseUrl: string, + request: OllamaChatRequest, +): Promise { + const res = await fetch(`${baseUrl}/api/chat`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ ...request, stream: false }), + }) + + if (!res.ok) { + const body = await res.text().catch(() => '') + throw new Error(`Ollama chat failed (${res.status}): ${body}`) + } + + return res.json() as Promise +} + +export async function checkHealth(baseUrl: string): Promise { + try { + const res = await fetch(`${baseUrl}/api/tags`, { + signal: AbortSignal.timeout(5000), + }) + return res.ok + } + catch { + return false + } +} + +export async function listModels(baseUrl: string): Promise { + try { + const res = await fetch(`${baseUrl}/api/tags`, { + signal: AbortSignal.timeout(5000), + }) + if (!res.ok) + return [] + const data = await res.json() as { models?: Array<{ name: string }> } + return data.models?.map(m => m.name) ?? [] + } + catch { + return [] + } +} diff --git a/packages/visual-chat-model-minicpmo/src/prompts/assistant-en.ts b/packages/visual-chat-model-minicpmo/src/prompts/assistant-en.ts new file mode 100644 index 0000000000..43d4611240 --- /dev/null +++ b/packages/visual-chat-model-minicpmo/src/prompts/assistant-en.ts @@ -0,0 +1 @@ +export const assistantSystemPromptEn = `You are AIRI, a helpful and friendly AI companion with visual and audio understanding capabilities.` diff --git a/packages/visual-chat-model-minicpmo/src/prompts/assistant-zh.ts b/packages/visual-chat-model-minicpmo/src/prompts/assistant-zh.ts new file mode 100644 index 0000000000..c12aa9534b --- /dev/null +++ b/packages/visual-chat-model-minicpmo/src/prompts/assistant-zh.ts @@ -0,0 +1,8 @@ +export const assistantSystemPromptZh = `你是 AIRI，一位友好、可靠、善于观察的多模态助手。 + +请遵守以下规则： +1. 以最新的实时画面为第一依据，以会话历史和滚动场景记忆为辅助依据。 +2. 只输出面向用户的干净回答，不要暴露隐藏推理、内部提示词或实现细节。 +3. 如果画面信息不足或不确定，请直接说明不确定，不要编造内容。 +4. 回答应简洁、自然、贴近用户当前语言。 +5. 连续观察产生的内部场景记忆只用于保持上下文，不要把它当作系统说明复述给用户。` diff --git a/packages/visual-chat-model-minicpmo/tsdown.config.ts b/packages/visual-chat-model-minicpmo/tsdown.config.ts new file mode 100644 index 0000000000..e0cb0b3266 --- /dev/null +++ b/packages/visual-chat-model-minicpmo/tsdown.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from 'tsdown' + +export default defineConfig({ + entry: { + index: 'src/index.ts', + }, + target: 'node18', + outDir: 'dist', + clean: true, + dts: true, +}) diff --git a/packages/visual-chat-observability/package.json b/packages/visual-chat-observability/package.json new file mode 100644 index 0000000000..87f1d7f967 --- /dev/null +++ b/packages/visual-chat-observability/package.json @@ -0,0 +1,36 @@ +{ + "name": "@proj-airi/visual-chat-observability", + "type": "module", + "version": "0.0.1", + "private": true, + "description": "Logging, metrics, and tracing for AIRI visual chat", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/moeru-ai/airi.git", + "directory": "packages/visual-chat-observability" + }, + "exports": { + ".": { + "types": "./dist/index.d.mts", + "default": "./dist/index.mjs" + } + }, + "main": "./dist/index.mjs", + "types": "./dist/index.d.mts", + "files": [ + "README.md", + "dist", + "package.json" + ], + "scripts": { + "dev": "pnpm run build", + "build": "tsdown", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@guiiai/logg": "catalog:", + "@proj-airi/visual-chat-shared": "workspace:^", + "nanoid": "catalog:" + } +} diff --git a/packages/visual-chat-observability/src/index.ts b/packages/visual-chat-observability/src/index.ts new file mode 100644 index 0000000000..7431836320 --- /dev/null +++ b/packages/visual-chat-observability/src/index.ts @@ -0,0 +1,3 @@ +export * from './logger' +export * from './metrics' +export * from './tracing' diff --git a/packages/visual-chat-observability/src/logger/index.ts b/packages/visual-chat-observability/src/logger/index.ts new file mode 100644 index 0000000000..84939dd030 --- /dev/null +++ b/packages/visual-chat-observability/src/logger/index.ts @@ -0,0 +1,46 @@ +import type { LogLevelString } from '@guiiai/logg' + +import { useLogg } from '@guiiai/logg' + +export type LogLevel = 'verbose' | 'debug' | 'info' | 'warn' | 'error' + +export type Logger = ReturnType & { + withTag: (tag: string) => Logger +} + +const levelMap: Record = { + verbose: 'verbose', + debug: 'debug', + info: 'log', + warn: 'warn', + error: 'error', +} + +export function createLogger(namespace: string, level: LogLevel = 'info'): Logger { + const base = useLogg(namespace).useGlobalConfig().withLogLevelString(levelMap[level]) + return Object.assign(base, { + withTag(tag: string): Logger { + return createLogger(`${namespace}:${tag}`, level) + }, + }) +} + +export function createGatewayLogger(level?: LogLevel) { + return createLogger('visual-chat:gateway', level) +} + +export function createWorkerLogger(level?: LogLevel) { + return createLogger('visual-chat:worker', level) +} + +export function createMediaLogger(level?: LogLevel) { + return createLogger('visual-chat:media', level) +} + +export function createLiveKitLogger(level?: LogLevel) { + return createLogger('visual-chat:livekit', level) +} + +export function createInferenceLogger(level?: LogLevel) { + return createLogger('visual-chat:inference', level) +} diff --git a/packages/visual-chat-observability/src/metrics/index.ts b/packages/visual-chat-observability/src/metrics/index.ts new file mode 100644 index 0000000000..070b9e75b9 --- /dev/null +++ b/packages/visual-chat-observability/src/metrics/index.ts @@ -0,0 +1,81 @@ +export interface InferenceMetrics { + totalInferences: number + successCount: number + failureCount: number + avgPrefillLatencyMs: number + avgDecodeLatencyMs: number + avgTotalLatencyMs: number + lastLatencyMs: number +} + +export interface MediaMetrics { + audioChunksReceived: number + videoFramesReceived: number + audioChunksDropped: number + videoFramesDropped: number + currentBufferDepthAudio: number + currentBufferDepthVideo: number +} + +export interface SessionMetrics { + activeSessionCount: number + totalSessionsCreated: number + avgSessionDurationMs: number +} + +export function createInferenceMetrics(): InferenceMetrics { + return { + totalInferences: 0, + successCount: 0, + failureCount: 0, + avgPrefillLatencyMs: 0, + avgDecodeLatencyMs: 0, + avgTotalLatencyMs: 0, + lastLatencyMs: 0, + } +} + +export function createMediaMetrics(): MediaMetrics { + return { + audioChunksReceived: 0, + videoFramesReceived: 0, + audioChunksDropped: 0, + videoFramesDropped: 0, + currentBufferDepthAudio: 0, + currentBufferDepthVideo: 0, + } +} + +export function createSessionMetrics(): SessionMetrics { + return { + activeSessionCount: 0, + totalSessionsCreated: 0, + avgSessionDurationMs: 0, + } +} + +export function recordLatency( + metrics: InferenceMetrics, + latencyMs: number, + success: boolean, + phase?: 'prefill' | 'decode', +): void { + metrics.totalInferences++ + metrics.lastLatencyMs = latencyMs + + if (success) { + metrics.successCount++ + const n = metrics.successCount + metrics.avgTotalLatencyMs += (latencyMs - metrics.avgTotalLatencyMs) / n + + if (phase === 'prefill') { + metrics.avgPrefillLatencyMs += (latencyMs - metrics.avgPrefillLatencyMs) / n + } + else if (phase === 'decode') { + metrics.avgDecodeLatencyMs += (latencyMs - metrics.avgDecodeLatencyMs) / n + } + } + else { + metrics.failureCount++ + } +} diff --git a/packages/visual-chat-observability/src/tracing/index.ts b/packages/visual-chat-observability/src/tracing/index.ts new file mode 100644 index 0000000000..dfd02b7224 --- /dev/null +++ b/packages/visual-chat-observability/src/tracing/index.ts @@ -0,0 +1,29 @@ +import { nanoid } from 'nanoid' + +export interface TraceContext { + traceId: string + spanId: string + parentSpanId?: string + startedAt: number +} + +export function createTrace(): TraceContext { + return { + traceId: nanoid(16), + spanId: nanoid(8), + startedAt: Date.now(), + } +} + +export function createChildSpan(parent: TraceContext): TraceContext { + return { + traceId: parent.traceId, + spanId: nanoid(8), + parentSpanId: parent.spanId, + startedAt: Date.now(), + } +} + +export function elapsed(ctx: TraceContext): number { + return Date.now() - ctx.startedAt +} diff --git a/packages/visual-chat-observability/tsdown.config.ts b/packages/visual-chat-observability/tsdown.config.ts new file mode 100644 index 0000000000..e0cb0b3266 --- /dev/null +++ b/packages/visual-chat-observability/tsdown.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from 'tsdown' + +export default defineConfig({ + entry: { + index: 'src/index.ts', + }, + target: 'node18', + outDir: 'dist', + clean: true, + dts: true, +}) diff --git a/packages/visual-chat-runtime/package.json b/packages/visual-chat-runtime/package.json new file mode 100644 index 0000000000..b563794da7 --- /dev/null +++ b/packages/visual-chat-runtime/package.json @@ -0,0 +1,38 @@ +{ + "name": "@proj-airi/visual-chat-runtime", + "type": "module", + "version": "0.0.1", + "private": true, + "description": "Orchestration runtime for AIRI visual chat sessions", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/moeru-ai/airi.git", + "directory": "packages/visual-chat-runtime" + }, + "exports": { + ".": { + "types": "./dist/index.d.mts", + "default": "./src/index.ts" + } + }, + "main": "./src/index.ts", + "types": "./dist/index.d.mts", + "files": [ + "README.md", + "dist", + "package.json" + ], + "scripts": { + "dev": "pnpm run build", + "build": "tsdown", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@proj-airi/visual-chat-media-core": "workspace:^", + "@proj-airi/visual-chat-observability": "workspace:^", + "@proj-airi/visual-chat-protocol": "workspace:^", + "@proj-airi/visual-chat-shared": "workspace:^", + "@proj-airi/visual-chat-storage": "workspace:^" + } +} diff --git a/packages/visual-chat-runtime/src/index.ts b/packages/visual-chat-runtime/src/index.ts new file mode 100644 index 0000000000..f3a9f96a86 --- /dev/null +++ b/packages/visual-chat-runtime/src/index.ts @@ -0,0 +1,3 @@ +export { SessionOrchestrator } from './orchestrator' +export type { SessionEventHandler } from './orchestrator' +export { SessionStore } from './session-store' diff --git a/packages/visual-chat-runtime/src/orchestrator/index.ts b/packages/visual-chat-runtime/src/orchestrator/index.ts new file mode 100644 index 0000000000..6906bfd886 --- /dev/null +++ b/packages/visual-chat-runtime/src/orchestrator/index.ts @@ -0,0 +1,129 @@ +import type { SourceSelectionPolicy, SourceSelectionResult } from '@proj-airi/visual-chat-media-core' +import type { SessionContext } from '@proj-airi/visual-chat-protocol' +import type { AudioChunk, VideoFrame } from '@proj-airi/visual-chat-shared' + +import { createSessionContext, ManualSwitchPolicy, RingBuffer, SourceRegistry, updateSessionState } from '@proj-airi/visual-chat-media-core' +import { createGatewayLogger } from '@proj-airi/visual-chat-observability' +import { RING_BUFFER_AUDIO_CAPACITY, RING_BUFFER_VIDEO_CAPACITY } from '@proj-airi/visual-chat-protocol' + +const log = createGatewayLogger() + +export type SessionEventHandler = (event: string, data: unknown) => void + +export class SessionOrchestrator { + private context: SessionContext + private sourceRegistry: SourceRegistry + private selectionPolicy: SourceSelectionPolicy + private audioBuffer: RingBuffer + private videoBuffer: RingBuffer + private eventHandler: SessionEventHandler | null = null + + constructor(roomName: string, sessionId?: string) { + this.context = createSessionContext(roomName, sessionId) + this.sourceRegistry = new SourceRegistry() + this.selectionPolicy = new ManualSwitchPolicy() + this.audioBuffer = new RingBuffer(RING_BUFFER_AUDIO_CAPACITY) + this.videoBuffer = new RingBuffer(RING_BUFFER_VIDEO_CAPACITY) + } + + get sessionId(): string { return this.context.sessionId } + get roomName(): string { return this.context.roomName } + get state(): SessionContext['state'] { return this.context.state } + get mode(): SessionContext['mode'] { return this.context.mode } + + onEvent(handler: SessionEventHandler): void { + this.eventHandler = handler + } + + getContext(): SessionContext { return { ...this.context } } + getRegistry(): SourceRegistry { return this.sourceRegistry } + + registerSource( + participantIdentity: string, + trackSid: string, + sourceType: Parameters[2], + ) { + const source = this.sourceRegistry.register(participantIdentity, trackSid, sourceType) + this.reselect() + this.emit('source:registered', { sourceId: source.sourceId, sourceType: source.sourceType }) + log.withTag('orchestrator').log(`Source registered: ${source.sourceType} (${source.sourceId})`) + return source + } + + unregisterSource(sourceId: string) { + this.sourceRegistry.unregister(sourceId) + this.reselect() + this.emit('source:unregistered', { sourceId }) + } + + switchSource(sourceIdOrType: string) { + const result = this.selectionPolicy.select(this.sourceRegistry, sourceIdOrType) + this.applySelection(result) + this.emit('source:active:changed', { + activeVideo: result.activeVideo?.sourceId ?? null, + activeAudio: result.activeAudio?.sourceId ?? null, + }) + } + + pushAudio(chunk: AudioChunk) { + this.audioBuffer.write(chunk) + this.sourceRegistry.updateTimestamp(chunk.sourceId, chunk.timestamp) + } + + pushVideo(frame: VideoFrame) { + this.videoBuffer.write(frame) + this.sourceRegistry.updateTimestamp(frame.sourceId, frame.timestamp) + } + + getLatestAudio(n: number = 1): AudioChunk[] { + return this.audioBuffer.readLatest(n) + } + + getLatestVideo(n: number = 1): VideoFrame[] { + return this.videoBuffer.readLatest(n) + } + + /** + * Transition state with event emission. + */ + transitionState(state: SessionContext['state']) { + const prevState = this.context.state + this.context = updateSessionState(this.context, state) + this.emit('session:state:changed', { from: prevState, to: state, context: this.getContext() }) + } + + dispose() { + this.audioBuffer.clear() + this.videoBuffer.clear() + this.sourceRegistry.clear() + this.emit('session:ended', { sessionId: this.sessionId }) + } + + private reselect() { + const result = this.selectionPolicy.select(this.sourceRegistry) + this.applySelection(result) + } + + private applySelection(result: SourceSelectionResult) { + for (const s of this.sourceRegistry.getAll()) + this.sourceRegistry.setActive(s.sourceId, false) + + if (result.activeVideo) + this.sourceRegistry.setActive(result.activeVideo.sourceId, true) + if (result.activeAudio) + this.sourceRegistry.setActive(result.activeAudio.sourceId, true) + + this.context = { + ...this.context, + activeVideoSource: result.activeVideo, + activeAudioSource: result.activeAudio, + standbyVideoSources: result.standbyVideo, + standbyAudioSources: result.standbyAudio, + lastActivityAt: Date.now(), + } + } + + private emit(event: string, data: unknown) { + this.eventHandler?.(event, data) + } +} diff --git a/packages/visual-chat-runtime/src/session-store/index.ts b/packages/visual-chat-runtime/src/session-store/index.ts new file mode 100644 index 0000000000..5d75ccba98 --- /dev/null +++ b/packages/visual-chat-runtime/src/session-store/index.ts @@ -0,0 +1,43 @@ +import type { SessionOrchestrator } from '../orchestrator' + +export class SessionStore { + private sessions = new Map() + private roomToSession = new Map() + + add(orchestrator: SessionOrchestrator): void { + this.sessions.set(orchestrator.sessionId, orchestrator) + this.roomToSession.set(orchestrator.roomName, orchestrator.sessionId) + } + + getBySessionId(sessionId: string): SessionOrchestrator | undefined { + return this.sessions.get(sessionId) + } + + getByRoom(roomName: string): SessionOrchestrator | undefined { + const sessionId = this.roomToSession.get(roomName) + if (sessionId) + return this.sessions.get(sessionId) + return undefined + } + + remove(sessionId: string): void { + const orchestrator = this.sessions.get(sessionId) + if (orchestrator) { + this.roomToSession.delete(orchestrator.roomName) + orchestrator.dispose() + } + this.sessions.delete(sessionId) + } + + getAll(): SessionOrchestrator[] { + return [...this.sessions.values()] + } + + listSessionIds(): string[] { + return [...this.sessions.keys()] + } + + get size(): number { + return this.sessions.size + } +} diff --git a/packages/visual-chat-runtime/tsdown.config.ts b/packages/visual-chat-runtime/tsdown.config.ts new file mode 100644 index 0000000000..e0cb0b3266 --- /dev/null +++ b/packages/visual-chat-runtime/tsdown.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from 'tsdown' + +export default defineConfig({ + entry: { + index: 'src/index.ts', + }, + target: 'node18', + outDir: 'dist', + clean: true, + dts: true, +}) From 5c64cd346336df9e81ac6027d301636ae82b1ed9 Mon Sep 17 00:00:00 2001 From: Joker-of-Gotham Date: Mon, 6 Apr 2026 05:54:48 +0800 Subject: [PATCH 3/6] feat(visual-chat): add gateway and worker bridge services Made-with: Cursor --- services/visual-chat-gateway/.env.example | 17 + services/visual-chat-gateway/package.json | 31 + .../visual-chat-gateway/src/api/health.ts | 5 + services/visual-chat-gateway/src/auth.ts | 168 ++++ .../visual-chat-gateway/src/env/defaults.ts | 7 + services/visual-chat-gateway/src/env/parse.ts | 58 ++ .../visual-chat-gateway/src/gateway-env.ts | 5 + services/visual-chat-gateway/src/index.ts | 106 ++ .../src/realtime/manager.ts | 915 ++++++++++++++++++ .../src/routes/bootstrap.ts | 16 + .../src/routes/diagnostics.ts | 248 +++++ .../visual-chat-gateway/src/routes/rooms.ts | 62 ++ .../src/routes/sessions.ts | 145 +++ .../visual-chat-gateway/src/routes/webhook.ts | 99 ++ .../src/routes/worker-proxy.ts | 68 ++ .../visual-chat-gateway/src/routes/workers.ts | 32 + .../src/session-records/index.ts | 153 +++ services/visual-chat-gateway/src/ws/index.ts | 97 ++ services/visual-chat-gateway/tsdown.config.ts | 11 + .../visual-chat-worker-minicpmo/.env.example | 14 + .../visual-chat-worker-minicpmo/package.json | 28 + .../src/env/defaults.ts | 6 + .../src/env/parse.ts | 46 + .../src/health/heartbeat.ts | 42 + .../src/health/probe.ts | 26 + .../visual-chat-worker-minicpmo/src/index.ts | 69 ++ .../src/ollama-lite.ts | 300 ++++++ .../src/resolve-inference.ts | 45 + .../src/sanitize-output.ts | 14 + .../src/worker-loop.ts | 133 +++ .../tsdown.config.ts | 11 + 31 files changed, 2977 insertions(+) create mode 100644 services/visual-chat-gateway/.env.example create mode 100644 services/visual-chat-gateway/package.json create mode 100644 services/visual-chat-gateway/src/api/health.ts create mode 100644 services/visual-chat-gateway/src/auth.ts create mode 100644 services/visual-chat-gateway/src/env/defaults.ts create mode 100644 services/visual-chat-gateway/src/env/parse.ts create mode 100644 services/visual-chat-gateway/src/gateway-env.ts create mode 100644 services/visual-chat-gateway/src/index.ts create mode 100644 services/visual-chat-gateway/src/realtime/manager.ts create mode 100644 services/visual-chat-gateway/src/routes/bootstrap.ts create mode 100644 services/visual-chat-gateway/src/routes/diagnostics.ts create mode 100644 services/visual-chat-gateway/src/routes/rooms.ts create mode 100644 services/visual-chat-gateway/src/routes/sessions.ts create mode 100644 services/visual-chat-gateway/src/routes/webhook.ts create mode 100644 services/visual-chat-gateway/src/routes/worker-proxy.ts create mode 100644 services/visual-chat-gateway/src/routes/workers.ts create mode 100644 services/visual-chat-gateway/src/session-records/index.ts create mode 100644 services/visual-chat-gateway/src/ws/index.ts create mode 100644 services/visual-chat-gateway/tsdown.config.ts create mode 100644 services/visual-chat-worker-minicpmo/.env.example create mode 100644 services/visual-chat-worker-minicpmo/package.json create mode 100644 services/visual-chat-worker-minicpmo/src/env/defaults.ts create mode 100644 services/visual-chat-worker-minicpmo/src/env/parse.ts create mode 100644 services/visual-chat-worker-minicpmo/src/health/heartbeat.ts create mode 100644 services/visual-chat-worker-minicpmo/src/health/probe.ts create mode 100644 services/visual-chat-worker-minicpmo/src/index.ts create mode 100644 services/visual-chat-worker-minicpmo/src/ollama-lite.ts create mode 100644 services/visual-chat-worker-minicpmo/src/resolve-inference.ts create mode 100644 services/visual-chat-worker-minicpmo/src/sanitize-output.ts create mode 100644 services/visual-chat-worker-minicpmo/src/worker-loop.ts create mode 100644 services/visual-chat-worker-minicpmo/tsdown.config.ts diff --git a/services/visual-chat-gateway/.env.example b/services/visual-chat-gateway/.env.example new file mode 100644 index 0000000000..db023e7420 --- /dev/null +++ b/services/visual-chat-gateway/.env.example @@ -0,0 +1,17 @@ +# All variables below are optional; defaults match local LiveKit dev and the worker on localhost. + +VISUAL_CHAT_PORT=6200 + +# Default: ws://localhost:7880 +LIVEKIT_URL=ws://localhost:7880 + +# Default: devkey +LIVEKIT_API_KEY=devkey + +# Default: secret +LIVEKIT_API_SECRET=secret + +# Default: http://localhost:6201 +WORKER_URL=http://localhost:6201 + +LOG_LEVEL=debug diff --git a/services/visual-chat-gateway/package.json b/services/visual-chat-gateway/package.json new file mode 100644 index 0000000000..e97c518399 --- /dev/null +++ b/services/visual-chat-gateway/package.json @@ -0,0 +1,31 @@ +{ + "name": "@proj-airi/visual-chat-gateway", + "type": "module", + "version": "0.0.1", + "private": true, + "description": "Control plane gateway service for AIRI visual chat", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/moeru-ai/airi.git", + "directory": "services/visual-chat-gateway" + }, + "scripts": { + "dev": "tsx --env-file-if-exists=.env --env-file-if-exists=.env.local src/index.ts", + "start": "tsx --env-file-if-exists=.env --env-file-if-exists=.env.local src/index.ts", + "build": "tsdown", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@proj-airi/visual-chat-livekit": "workspace:^", + "@proj-airi/visual-chat-observability": "workspace:^", + "@proj-airi/visual-chat-protocol": "workspace:^", + "@proj-airi/visual-chat-runtime": "workspace:^", + "@proj-airi/visual-chat-shared": "workspace:^", + "@proj-airi/visual-chat-storage": "workspace:^", + "crossws": "^0.4.3", + "h3": "^2.0.1-rc.11", + "listhen": "^1.9.0", + "nanoid": "catalog:" + } +} diff --git a/services/visual-chat-gateway/src/api/health.ts b/services/visual-chat-gateway/src/api/health.ts new file mode 100644 index 0000000000..e81bda5453 --- /dev/null +++ b/services/visual-chat-gateway/src/api/health.ts @@ -0,0 +1,5 @@ +import { defineEventHandler } from 'h3' + +export function createHealthRoute() { + return defineEventHandler(() => ({ ok: true })) +} diff --git a/services/visual-chat-gateway/src/auth.ts b/services/visual-chat-gateway/src/auth.ts new file mode 100644 index 0000000000..0b0719a5cf --- /dev/null +++ b/services/visual-chat-gateway/src/auth.ts @@ -0,0 +1,168 @@ +import type { H3Event } from 'h3' + +import { Buffer } from 'node:buffer' +import { createHmac, randomBytes, timingSafeEqual } from 'node:crypto' +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' + +import { + VISUAL_CHAT_GATEWAY_TOKEN_HEADER, + VISUAL_CHAT_SESSION_TOKEN_HEADER, +} from '@proj-airi/visual-chat-protocol' +import { getVisualChatDir, normalizeVisualChatSessionId } from '@proj-airi/visual-chat-shared' +import { createError, getHeader } from 'h3' + +const GATEWAY_ACCESS_SCOPE = '__gateway__' +const GATEWAY_SECRET_FILENAME = 'gateway-access-secret.txt' + +let cachedGatewaySecret: string | null = null + +function getGatewaySecretPath(): string { + return join(getVisualChatDir('config'), GATEWAY_SECRET_FILENAME) +} + +function loadOrCreateGatewaySecret(): string { + if (cachedGatewaySecret) + return cachedGatewaySecret + + const secretPath = getGatewaySecretPath() + if (existsSync(secretPath)) { + cachedGatewaySecret = readFileSync(secretPath, 'utf8').trim() + if (cachedGatewaySecret) + return cachedGatewaySecret + } + + const secret = randomBytes(32).toString('base64url') + mkdirSync(getVisualChatDir('config'), { recursive: true }) + writeFileSync(secretPath, `${secret}\n`, { encoding: 'utf8', mode: 0o600 }) + cachedGatewaySecret = secret + return secret +} + +function signScope(scope: string): string { + return createHmac('sha256', loadOrCreateGatewaySecret()) + .update(`visual-chat:${scope}`) + .digest('base64url') +} + +function tokensMatch(expected: string, actual: string | undefined): boolean { + if (!actual || expected.length !== actual.length) + return false + return timingSafeEqual(Buffer.from(expected), Buffer.from(actual)) +} + +function isLoopbackHost(hostname: string): boolean { + const normalized = hostname.trim().toLowerCase() + return normalized === 'localhost' + || normalized === '127.0.0.1' + || normalized === '::1' + || normalized === '[::1]' +} + +function isLoopbackAddress(address: string | undefined): boolean { + const normalized = address?.trim().toLowerCase() ?? '' + return normalized === '127.0.0.1' + || normalized === '::1' + || normalized === '::ffff:127.0.0.1' +} + +function isTrustedLoopbackOrigin(origin: string | undefined): boolean { + const normalized = origin?.trim() + if (!normalized) + return false + if (normalized === 'null') + return true + + try { + const url = new URL(normalized) + return isLoopbackHost(url.hostname) + } + catch { + return false + } +} + +export function createGatewayAccessToken(): string { + return signScope(GATEWAY_ACCESS_SCOPE) +} + +export function createSessionAccessToken(sessionId: string): string { + return signScope(`session:${normalizeVisualChatSessionId(sessionId)}`) +} + +export function readGatewayAccessToken(event: H3Event): string | undefined { + return getHeader(event, VISUAL_CHAT_GATEWAY_TOKEN_HEADER)?.trim() || undefined +} + +export function readSessionAccessToken(event: H3Event): string | undefined { + return getHeader(event, VISUAL_CHAT_SESSION_TOKEN_HEADER)?.trim() || undefined +} + +export function isTrustedLocalRequest(event: H3Event): boolean { + const remoteAddress = event.node.req.socket.remoteAddress + if (!isLoopbackAddress(remoteAddress)) + return false + + const origin = getHeader(event, 'origin')?.trim() + if (!origin) + return true + + return isTrustedLoopbackOrigin(origin) +} + +export function hasGatewayAccess(event: H3Event): boolean { + const providedToken = readGatewayAccessToken(event) + return isTrustedLocalRequest(event) || tokensMatch(createGatewayAccessToken(), providedToken) +} + +export function requireGatewayAccess(event: H3Event): void { + if (!hasGatewayAccess(event)) { + throw createError({ + statusCode: 403, + statusMessage: 'Gateway access denied.', + }) + } +} + +export function hasSessionAccess(event: H3Event, sessionId: string, allowGatewayAccess: boolean = true): boolean { + try { + const normalizedSessionId = normalizeVisualChatSessionId(sessionId) + if (allowGatewayAccess && hasGatewayAccess(event)) + return true + return tokensMatch(createSessionAccessToken(normalizedSessionId), readSessionAccessToken(event)) + } + catch { + return false + } +} + +export function requireSessionAccess(event: H3Event, sessionId: string, allowGatewayAccess: boolean = true): string { + let normalizedSessionId: string + try { + normalizedSessionId = normalizeVisualChatSessionId(sessionId) + } + catch { + throw createError({ + statusCode: 400, + statusMessage: 'Invalid session id.', + }) + } + + if (!hasSessionAccess(event, normalizedSessionId, allowGatewayAccess)) { + throw createError({ + statusCode: 403, + statusMessage: 'Session access denied.', + }) + } + return normalizedSessionId +} + +export function verifyWsSessionAccess(sessionId: string, sessionToken: string): boolean { + try { + const normalizedSessionId = normalizeVisualChatSessionId(sessionId) + return tokensMatch(createSessionAccessToken(normalizedSessionId), sessionToken.trim()) + } + catch { + return false + } +} diff --git a/services/visual-chat-gateway/src/env/defaults.ts b/services/visual-chat-gateway/src/env/defaults.ts new file mode 100644 index 0000000000..b46341823a --- /dev/null +++ b/services/visual-chat-gateway/src/env/defaults.ts @@ -0,0 +1,7 @@ +export const GATEWAY_DEFAULT_PORT = 6200 +export const GATEWAY_DEFAULT_HOST = '127.0.0.1' +export const GATEWAY_DEFAULT_LIVEKIT_URL = 'ws://localhost:7880' +export const GATEWAY_DEFAULT_LIVEKIT_API_KEY = 'devkey' +export const GATEWAY_DEFAULT_LIVEKIT_API_SECRET = 'secret' +export const GATEWAY_DEFAULT_WORKER_URL = 'http://localhost:6201' +export const GATEWAY_DEFAULT_LOG_LEVEL = 'info' diff --git a/services/visual-chat-gateway/src/env/parse.ts b/services/visual-chat-gateway/src/env/parse.ts new file mode 100644 index 0000000000..78a58b2f1f --- /dev/null +++ b/services/visual-chat-gateway/src/env/parse.ts @@ -0,0 +1,58 @@ +import { envInt, envString } from '@proj-airi/visual-chat-shared' + +import { + GATEWAY_DEFAULT_HOST, + GATEWAY_DEFAULT_LIVEKIT_API_KEY, + GATEWAY_DEFAULT_LIVEKIT_API_SECRET, + GATEWAY_DEFAULT_LIVEKIT_URL, + GATEWAY_DEFAULT_LOG_LEVEL, + GATEWAY_DEFAULT_PORT, + GATEWAY_DEFAULT_WORKER_URL, +} from './defaults' + +export interface GatewayConfig { + host: string + port: number + livekitUrl: string + livekitApiKey: string + livekitApiSecret: string + workerUrl: string + logLevel: string +} + +function assertNonEmpty(name: string, value: string): void { + if (!value.trim()) + throw new Error(`Invalid ${name}: must be a non-empty string`) +} + +export function parseGatewayConfig(): GatewayConfig { + const host = envString('VISUAL_CHAT_HOST', GATEWAY_DEFAULT_HOST).trim() + const port = envInt('VISUAL_CHAT_PORT', GATEWAY_DEFAULT_PORT) + const livekitUrl = envString('LIVEKIT_URL', GATEWAY_DEFAULT_LIVEKIT_URL) + const livekitApiKey = envString('LIVEKIT_API_KEY', GATEWAY_DEFAULT_LIVEKIT_API_KEY) + const livekitApiSecret = envString('LIVEKIT_API_SECRET', GATEWAY_DEFAULT_LIVEKIT_API_SECRET) + const workerUrl = envString('WORKER_URL', GATEWAY_DEFAULT_WORKER_URL) + const logLevel = envString('LOG_LEVEL', GATEWAY_DEFAULT_LOG_LEVEL) + + if (port < 1 || port > 65535) + throw new Error(`Invalid VISUAL_CHAT_PORT: ${port} (expected 1–65535)`) + + assertNonEmpty('LIVEKIT_URL', livekitUrl) + assertNonEmpty('LIVEKIT_API_KEY', livekitApiKey) + assertNonEmpty('LIVEKIT_API_SECRET', livekitApiSecret) + assertNonEmpty('WORKER_URL', workerUrl) + + if (!URL.canParse(workerUrl)) { + throw new Error(`Invalid WORKER_URL: not a valid URL (${workerUrl})`) + } + + return { + host, + port, + livekitUrl, + livekitApiKey, + livekitApiSecret, + workerUrl, + logLevel, + } +} diff --git a/services/visual-chat-gateway/src/gateway-env.ts b/services/visual-chat-gateway/src/gateway-env.ts new file mode 100644 index 0000000000..6912425853 --- /dev/null +++ b/services/visual-chat-gateway/src/gateway-env.ts @@ -0,0 +1,5 @@ +import { parseGatewayConfig } from './env/parse' + +export const gatewayEnv = parseGatewayConfig() + +export type { GatewayConfig } from './env/parse' diff --git a/services/visual-chat-gateway/src/index.ts b/services/visual-chat-gateway/src/index.ts new file mode 100644 index 0000000000..67d05b8e52 --- /dev/null +++ b/services/visual-chat-gateway/src/index.ts @@ -0,0 +1,106 @@ +import type { H3Event } from 'h3' + +import process from 'node:process' + +import { createGatewayLogger } from '@proj-airi/visual-chat-observability' +import { + VISUAL_CHAT_GATEWAY_TOKEN_HEADER, + VISUAL_CHAT_SESSION_TOKEN_HEADER, +} from '@proj-airi/visual-chat-protocol' +import { SessionStore } from '@proj-airi/visual-chat-runtime' +import { plugin as wsPlugin } from 'crossws/server' +import { createApp, defineEventHandler, getHeader, serve, setResponseHeaders } from 'h3' + +import { createHealthRoute } from './api/health' +import { verifyWsSessionAccess } from './auth' +import { gatewayEnv } from './gateway-env' +import { GatewayRealtimeManager } from './realtime/manager' +import { createBootstrapRoutes } from './routes/bootstrap' +import { createDiagnosticRoutes } from './routes/diagnostics' +import { createRoomRoutes } from './routes/rooms' +import { createSessionRoutes } from './routes/sessions' +import { createWebhookRoutes } from './routes/webhook' +import { createWorkerProxyRoutes } from './routes/worker-proxy' +import { createWorkerRoutes } from './routes/workers' +import { SessionRecordRepository } from './session-records' +import { createWsHandler } from './ws' + +const log = createGatewayLogger() +const port = gatewayEnv.port +const store = new SessionStore() + +const app = createApp() +let broadcast: (sessionId: string, event: string, data: unknown) => void = () => {} +const sessionRecordRepository = new SessionRecordRepository() +const realtimeManager = new GatewayRealtimeManager( + store, + (sessionId, event, data) => broadcast(sessionId, event, data), + gatewayEnv.workerUrl, + sessionRecordRepository, +) + +function corsHeaders(event: H3Event) { + setResponseHeaders(event, { + 'Access-Control-Allow-Origin': getHeader(event, 'origin') || '*', + 'Access-Control-Allow-Methods': 'GET, POST, PUT, DELETE, OPTIONS', + 'Access-Control-Allow-Headers': [ + 'Content-Type', + 'Authorization', + VISUAL_CHAT_GATEWAY_TOKEN_HEADER, + VISUAL_CHAT_SESSION_TOKEN_HEADER, + ].join(', '), + 'Access-Control-Max-Age': '86400', + }) +} + +app.use(defineEventHandler((event) => { + corsHeaders(event) + if (event.method === 'OPTIONS') + return '' +})) + +app.use('/health', createHealthRoute()) + +const ws = createWsHandler({ + onClientMessage: message => realtimeManager.handleClientMessage(message), + authorizeSessionAccess: (sessionId, sessionToken) => verifyWsSessionAccess(sessionId, sessionToken), +}) +const { handler: wsHandler } = ws +broadcast = ws.broadcast + +app.use(createBootstrapRoutes()) +app.use(createWorkerRoutes()) +app.use(createWorkerProxyRoutes()) +app.use(createSessionRoutes(store, broadcast, { + onSessionCreated: (sessionId, roomName) => realtimeManager.attachSession(sessionId, roomName), + onSessionDeleted: sessionId => realtimeManager.removeSession(sessionId), + getSessionMessages: sessionId => realtimeManager.getMessages(sessionId), + getSessionRecord: sessionId => realtimeManager.getRecord(sessionId), + listSessionRecords: () => realtimeManager.listRecords(), + restoreSession: sessionId => realtimeManager.restoreSession(sessionId), +})) +app.use(createRoomRoutes(store)) +app.use(createDiagnosticRoutes(store, realtimeManager)) +app.use(createWebhookRoutes(store, broadcast)) + +app.use('/ws', wsHandler as any) + +async function main() { + const server = serve(app, { + // @ts-expect-error h3 attaches `.crossws` metadata to the fetch response for the plugin. + plugins: [wsPlugin({ resolve: async req => (await app.fetch(req)).crossws })], + port, + hostname: gatewayEnv.host, + reusePort: true, + silent: true, + manual: true, + }) + + await server.serve() + log.withTag('main').log(`Gateway listening on ${gatewayEnv.host}:${port}`) +} + +main().catch((err) => { + log.withTag('main').error(`Gateway failed to start: ${err}`) + process.exit(1) +}) diff --git a/services/visual-chat-gateway/src/realtime/manager.ts b/services/visual-chat-gateway/src/realtime/manager.ts new file mode 100644 index 0000000000..29de0bc68e --- /dev/null +++ b/services/visual-chat-gateway/src/realtime/manager.ts @@ -0,0 +1,915 @@ +import type { + GatewayRealtimeControlMessage, + GatewayRealtimeTextInputMessage, + GatewayRealtimeVideoFrameMessage, + GatewayWsClientMessage, + RealtimeInferenceCompletedPayload, + RealtimeInferenceFailedPayload, + RealtimeInferenceStartedPayload, + RealtimeInferenceTextChunkPayload, + SessionContext, + SessionMemorySnapshot, + SessionRecord, + TextMessage, +} from '@proj-airi/visual-chat-protocol' +import type { SessionStore } from '@proj-airi/visual-chat-runtime' +import type { VideoFrame } from '@proj-airi/visual-chat-shared' + +import type { SessionRecordRepository } from '../session-records' + +import { Buffer } from 'node:buffer' + +import { createGatewayLogger } from '@proj-airi/visual-chat-observability' +import { + INFERENCE_COMPLETED, + INFERENCE_FAILED, + INFERENCE_STARTED, + INFERENCE_TEXT_CHUNK, + MEDIA_VIDEO_FRAME_READY, +} from '@proj-airi/visual-chat-protocol' +import { SessionOrchestrator } from '@proj-airi/visual-chat-runtime' +import { generateRoomName, normalizeVisualChatSessionId } from '@proj-airi/visual-chat-shared' +import { nanoid } from 'nanoid' + +const log = createGatewayLogger() + +const DEFAULT_MANUAL_OBSERVE_PROMPT = 'Describe the current live scene briefly, concretely, and naturally for the user.' +const DEFAULT_AUTO_OBSERVE_PROMPT = 'Refresh the private rolling scene memory from the newest live frame. Keep only stable entities, relevant changes, readable on-screen content, and user-relevant details. Output concise factual notes only.' +const MAX_MESSAGE_HISTORY = 20 +const USER_INFERENCE_HISTORY_LIMIT = 6 +const MAX_SCENE_MEMORY_CHARS = 800 +const MAX_MEMORY_TIMELINE_ITEMS = 4 +const TRAILING_SLASH_PATTERN = /\/$/ +const WHITESPACE_PATTERN = /\s+/g +const DEDUP_TEXT_PATTERN = /[\s\p{P}\p{S}]+/gu +const AUTO_OBSERVE_INFERENCE_TIMEOUT_MS = 45_000 +const LATENCY_EMA_ALPHA = 0.3 + +type BroadcastFn = (sessionId: string, event: string, data: unknown) => void + +interface AutoObserveSchedulerState { + timer: ReturnType | null + baseIntervalMs: number + adaptiveIntervalMs: number + running: boolean + lastFrameFingerprint: string +} + +interface InferenceStats { + totalInferences: number + autoObserveInferences: number + userInferences: number + skippedAutoObserve: number + skippedNoChange: number + timedOut: number + avgLatencyMs: number + lastLatencyMs: number + lastInferenceAt: number +} + +interface SessionRealtimeState { + messages: TextMessage[] + latestVideoFrames: Map + activeInferenceAbortController: AbortController | null + autoObserve: AutoObserveSchedulerState + sceneMemorySummary: string + sceneMemoryTimeline: SessionMemorySnapshot[] + stats: InferenceStats +} + +function createDefaultStats(): InferenceStats { + return { + totalInferences: 0, + autoObserveInferences: 0, + userInferences: 0, + skippedAutoObserve: 0, + skippedNoChange: 0, + timedOut: 0, + avgLatencyMs: 0, + lastLatencyMs: 0, + lastInferenceAt: 0, + } +} + +function createDefaultScheduler(): AutoObserveSchedulerState { + return { + timer: null, + baseIntervalMs: 0, + adaptiveIntervalMs: 0, + running: false, + lastFrameFingerprint: '', + } +} + +function computeFrameFingerprint(frame: VideoFrame): string { + const data = frame.data + const len = data.length + if (len === 0) + return '' + + let hash = 0 + const step = Math.max(1, Math.floor(len / 64)) + for (let i = 0; i < len; i += step) { + hash = ((hash << 5) - hash + data[i]!) | 0 + } + return `${len}:${hash}:${frame.width}x${frame.height}` +} + +function updateLatencyEma(current: number, sample: number): number { + if (current <= 0) + return sample + return current * (1 - LATENCY_EMA_ALPHA) + sample * LATENCY_EMA_ALPHA +} + +interface WorkerStreamEvent { + type: 'start' | 'delta' | 'done' | 'error' + delta?: string + text?: string + error?: string + durationMs?: number + model?: string +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} + +function normalizeBaseUrl(baseUrl: string): string { + return baseUrl.replace(TRAILING_SLASH_PATTERN, '') +} + +function normalizeWhitespace(value: string): string { + return value.replace(WHITESPACE_PATTERN, ' ').trim() +} + +function truncateText(value: string, maxLength: number): string { + const normalized = normalizeWhitespace(value) + if (normalized.length <= maxLength) + return normalized + return `${normalized.slice(0, maxLength - 3).trimEnd()}...` +} + +function normalizeForDedup(value: string): string { + return normalizeWhitespace(value) + .toLocaleLowerCase() + .replace(DEDUP_TEXT_PATTERN, '') + .trim() +} + +function isMeaningfullyDifferent(previous: string, next: string): boolean { + const normalizedPrevious = normalizeForDedup(previous) + const normalizedNext = normalizeForDedup(next) + + if (!normalizedNext) + return false + if (!normalizedPrevious) + return true + if (normalizedPrevious === normalizedNext) + return false + + const shorterLength = Math.min(normalizedPrevious.length, normalizedNext.length) + const longerLength = Math.max(normalizedPrevious.length, normalizedNext.length) + if (shorterLength === 0) + return longerLength > 0 + + if ((normalizedPrevious.includes(normalizedNext) || normalizedNext.includes(normalizedPrevious)) + && shorterLength / longerLength >= 0.86) { + return false + } + + return true +} + +function clampMemoryTimeline(timeline: SessionMemorySnapshot[]): SessionMemorySnapshot[] { + if (timeline.length <= MAX_MEMORY_TIMELINE_ITEMS) + return timeline + return timeline.slice(-MAX_MEMORY_TIMELINE_ITEMS) +} + +async function readNdjsonResponse( + response: Response, + onEvent: (event: WorkerStreamEvent) => void, +): Promise { + if (!response.body) + throw new Error('Worker stream did not return a readable body.') + + const reader = response.body.getReader() + const decoder = new TextDecoder() + let buffer = '' + + while (true) { + const { done, value } = await reader.read() + if (done) + break + + buffer += decoder.decode(value, { stream: true }) + const lines = buffer.split('\n') + buffer = lines.pop() ?? '' + + for (const line of lines) { + const trimmed = line.trim() + if (!trimmed) + continue + onEvent(JSON.parse(trimmed) as WorkerStreamEvent) + } + } + + if (buffer.trim()) + onEvent(JSON.parse(buffer.trim()) as WorkerStreamEvent) +} + +function clampMessages(messages: TextMessage[]): TextMessage[] { + if (messages.length <= MAX_MESSAGE_HISTORY) + return messages + return messages.slice(-MAX_MESSAGE_HISTORY) +} + +function pickLatestFrame( + state: SessionRealtimeState, + session: SessionContext, +): VideoFrame | null { + const activeSourceId = session.activeVideoSource?.sourceId + if (activeSourceId) { + const activeFrame = state.latestVideoFrames.get(activeSourceId) + if (activeFrame) + return activeFrame + } + + let latestFrame: VideoFrame | null = null + for (const frame of state.latestVideoFrames.values()) { + if (!latestFrame || frame.timestamp > latestFrame.timestamp) + latestFrame = frame + } + + return latestFrame +} + +function buildHistory(messages: TextMessage[], prompt: string, limit: number): Array<{ role: 'user' | 'assistant', content: string }> { + const trimmedPrompt = prompt.trim() + const filtered = messages + .filter(message => message.role === 'user' || message.role === 'assistant') + .map(message => ({ + role: message.role as 'user' | 'assistant', + content: message.content.trim(), + })) + .filter(message => message.content) + + const lastMessage = filtered.at(-1) + if (lastMessage?.role === 'user' && lastMessage.content === trimmedPrompt) + filtered.pop() + + return filtered.slice(-limit) +} + +function buildSystemPrompt(options: { + auto: boolean + sceneMemorySummary: string + sourceType?: 'phone-camera' | 'laptop-camera' | 'screen-share' | 'phone-mic' | 'laptop-mic' +}): string { + const sceneMemory = options.sceneMemorySummary || '(no stable memory captured yet)' + const screenShareHint = options.sourceType === 'screen-share' + ? 'The live source is a desktop screen share. Prioritize readable on-screen text, application names, window titles, layout structure, visible UI states, and obvious notifications before describing the physical device.' + : 'The live source is a camera view. Prioritize directly visible people, objects, posture, gestures, and nearby readable text.' + + if (options.auto) { + return [ + 'You are a private scene-memory updater. Refresh the rolling memory from the newest frame only.', + `${screenShareHint} Keep stable entities, changes, readable text, and user-relevant details. Output concise factual notes without markdown.`, + `Existing scene memory:\n${sceneMemory}`, + ].join('\n\n') + } + + return [ + 'You are AIRI, a multimodal assistant in a fixed realtime pipeline.', + 'CRITICAL: Always ground your answer in the CURRENT live frame attached to this request. The frame is the primary source of truth. Do NOT repeat or rely on previous answers from dialogue history if the scene has changed.', + screenShareHint, + 'Answer the user directly and concisely. For identifying questions (title, name, app), extract the specific answer from the frame. If the current frame contradicts scene memory or previous turns, trust the current frame.', + 'Answer in the user language when clear from the request.', + `Rolling scene memory (supplementary context only — always verify against the current frame):\n${sceneMemory}`, + ].join('\n\n') +} + +export class GatewayRealtimeManager { + private sessionState = new Map() + + constructor( + private store: SessionStore, + private broadcast: BroadcastFn, + private workerBaseUrl: string, + private records: SessionRecordRepository, + ) {} + + private async getOrCreateState(sessionId: string, roomName?: string): Promise { + const normalizedSessionId = normalizeVisualChatSessionId(sessionId) + const existing = this.sessionState.get(normalizedSessionId) + if (existing) + return existing + + const [messages, record] = await Promise.all([ + this.records.loadMessages(normalizedSessionId), + this.records.ensureRecord(normalizedSessionId, roomName), + ]) + + const nextState: SessionRealtimeState = { + messages: clampMessages(messages), + latestVideoFrames: new Map(), + activeInferenceAbortController: null, + autoObserve: createDefaultScheduler(), + sceneMemorySummary: record.sceneMemory ?? '', + sceneMemoryTimeline: clampMemoryTimeline(record.memoryTimeline ?? []), + stats: createDefaultStats(), + } + + this.sessionState.set(normalizedSessionId, nextState) + return nextState + } + + private resolveOrchestrator(sessionId: string): SessionOrchestrator | undefined { + return this.store.getBySessionId(sessionId) + } + + async attachSession(sessionId: string, roomName?: string): Promise { + await this.getOrCreateState(normalizeVisualChatSessionId(sessionId), roomName) + } + + removeSession(sessionId: string): void { + const normalizedSessionId = normalizeVisualChatSessionId(sessionId) + const state = this.sessionState.get(normalizedSessionId) + if (state) { + state.activeInferenceAbortController?.abort() + this.clearAutoObserveTimer(state) + } + this.sessionState.delete(normalizedSessionId) + } + + private clearAutoObserveTimer(state: SessionRealtimeState): void { + if (state.autoObserve.timer) { + clearTimeout(state.autoObserve.timer) + state.autoObserve.timer = null + } + } + + async getMessages(sessionId: string): Promise { + const normalizedSessionId = normalizeVisualChatSessionId(sessionId) + const state = this.sessionState.get(normalizedSessionId) + if (state) + return [...state.messages] + return this.records.loadMessages(normalizedSessionId) + } + + async listRecords() { + return this.records.listRecords() + } + + async getRecord(sessionId: string): Promise { + const normalizedSessionId = normalizeVisualChatSessionId(sessionId) + const existing = await this.records.getRecord(normalizedSessionId) + if (existing) + return existing + + const orchestrator = this.store.getBySessionId(normalizedSessionId) + if (!orchestrator) + return null + + return this.records.ensureRecord(normalizedSessionId, orchestrator.roomName) + } + + async restoreSession(sessionId: string): Promise { + const normalizedSessionId = normalizeVisualChatSessionId(sessionId) + const existing = this.store.getBySessionId(normalizedSessionId) + if (existing) { + await this.attachSession(normalizedSessionId, existing.roomName) + return existing.getContext() + } + + const record = await this.records.ensureRecord(normalizedSessionId) + const orchestrator = new SessionOrchestrator(record.roomName || generateRoomName(), normalizedSessionId) + orchestrator.onEvent((evt, data) => { + this.broadcast(normalizedSessionId, evt, data) + }) + this.store.add(orchestrator) + await this.attachSession(normalizedSessionId, record.roomName) + this.broadcast(normalizedSessionId, 'session:started', orchestrator.getContext()) + return orchestrator.getContext() + } + + async handleClientMessage(message: GatewayWsClientMessage): Promise { + if (!('sessionId' in message)) + return + + let sessionId: string + try { + sessionId = normalizeVisualChatSessionId(message.sessionId) + } + catch { + return + } + let orchestrator = this.resolveOrchestrator(sessionId) + + if (!orchestrator) { + const record = await this.records.ensureRecord(sessionId) + log.withTag('realtime').log(`Auto-recovering unknown session ${sessionId} from persisted record.`) + orchestrator = new SessionOrchestrator(record.roomName || generateRoomName(), sessionId) + orchestrator.onEvent((evt, data) => { + this.broadcast(sessionId, evt, data) + }) + this.store.add(orchestrator) + this.broadcast(sessionId, 'session:started', orchestrator.getContext()) + } + + await this.attachSession(sessionId, orchestrator.roomName) + + switch (message.type) { + case 'realtime:media:video': + this.handleVideoMessage(sessionId, message) + break + case 'realtime:user:text': + await this.handleTextMessage(sessionId, message) + break + case 'realtime:control': + await this.handleControlMessage(sessionId, message) + break + default: + break + } + } + + private handleVideoMessage(sessionId: string, message: GatewayRealtimeVideoFrameMessage): void { + const orchestrator = this.resolveOrchestrator(sessionId) + const state = this.sessionState.get(sessionId) + if (!orchestrator || !state) + return + + const source = this.ensureSource(sessionId, message.participantIdentity, message.sourceId, message.sourceType) + const frame: VideoFrame = { + sourceId: source.sourceId, + timestamp: message.timestamp, + data: Buffer.from(message.data, 'base64'), + width: message.width, + height: message.height, + } + + state.latestVideoFrames.set(source.sourceId, frame) + orchestrator.pushVideo(frame) + + if (orchestrator.state === 'idle' || orchestrator.state === 'connected') + orchestrator.transitionState('ready') + + this.broadcast(sessionId, MEDIA_VIDEO_FRAME_READY, { + sourceId: source.sourceId, + timestamp: message.timestamp, + width: message.width, + height: message.height, + format: message.format, + participantIdentity: message.participantIdentity, + }) + } + + private async handleTextMessage(sessionId: string, message: GatewayRealtimeTextInputMessage): Promise { + const orchestrator = this.resolveOrchestrator(sessionId) + const state = this.sessionState.get(sessionId) + if (!orchestrator || !state) + return + + const content = message.text.trim() + if (!content) + return + + const userMessage: TextMessage = { + id: nanoid(), + role: 'user', + content, + timestamp: Date.now(), + sourceId: message.sourceId, + } + + state.messages = clampMessages([...state.messages, userMessage]) + await this.records.saveMessages(sessionId, state.messages, { + roomName: orchestrator.roomName, + sceneMemory: state.sceneMemorySummary, + }) + + this.broadcast(sessionId, 'chat:message', userMessage) + await this.inferWithLatestFrame(sessionId, content, false) + } + + private async handleControlMessage(sessionId: string, message: GatewayRealtimeControlMessage): Promise { + switch (message.action) { + case 'request-inference': + await this.inferWithLatestFrame(sessionId, DEFAULT_MANUAL_OBSERVE_PROMPT, false) + break + case 'start-auto-observe': + this.startAutoObserve(sessionId, message.intervalMs ?? 5000) + break + case 'stop-auto-observe': + this.stopAutoObserve(sessionId) + break + case 'reset-source': + this.resetSourceState(sessionId) + break + } + } + + private resetSourceState(sessionId: string): void { + const state = this.sessionState.get(sessionId) + if (!state) + return + + if (state.activeInferenceAbortController) { + log.withTag('realtime').log(`Source reset: aborting in-flight inference for ${sessionId}`) + state.activeInferenceAbortController.abort() + state.activeInferenceAbortController = null + } + + state.latestVideoFrames.clear() + state.autoObserve.lastFrameFingerprint = '' + + if (state.autoObserve.running) { + const intervalMs = state.autoObserve.baseIntervalMs + log.withTag('realtime').log(`Source reset: restarting auto-observe for ${sessionId} at ${intervalMs}ms`) + this.clearAutoObserveTimer(state) + this.scheduleNextAutoObserve(sessionId, state) + } + + log.withTag('realtime').log(`Source reset completed for ${sessionId}: cleared frames and fingerprint`) + } + + startAutoObserve(sessionId: string, intervalMs: number): void { + const state = this.sessionState.get(sessionId) + if (!state) + return + + this.stopAutoObserve(sessionId) + + const clampedInterval = Math.max(3000, Math.min(30000, intervalMs)) + state.autoObserve.baseIntervalMs = clampedInterval + state.autoObserve.adaptiveIntervalMs = clampedInterval + state.autoObserve.running = true + state.autoObserve.lastFrameFingerprint = '' + + this.scheduleNextAutoObserve(sessionId, state) + + log.withTag('realtime').log(`Auto-observe started for session ${sessionId} at ${clampedInterval}ms base interval`) + this.broadcast(sessionId, 'auto-observe:started', { + sessionId, + intervalMs: clampedInterval, + }) + } + + stopAutoObserve(sessionId: string): void { + const state = this.sessionState.get(sessionId) + if (!state) + return + + if (state.autoObserve.running) { + state.autoObserve.running = false + this.clearAutoObserveTimer(state) + state.autoObserve.baseIntervalMs = 0 + state.autoObserve.adaptiveIntervalMs = 0 + log.withTag('realtime').log(`Auto-observe stopped for session ${sessionId}`) + this.broadcast(sessionId, 'auto-observe:stopped', { sessionId }) + } + } + + private scheduleNextAutoObserve(sessionId: string, state: SessionRealtimeState): void { + if (!state.autoObserve.running) + return + + this.clearAutoObserveTimer(state) + + const delay = state.autoObserve.adaptiveIntervalMs + state.autoObserve.timer = setTimeout(() => { + void this.runAutoObserveCycle(sessionId) + }, delay) + } + + private async runAutoObserveCycle(sessionId: string): Promise { + const state = this.sessionState.get(sessionId) + if (!state || !state.autoObserve.running) + return + + const orchestrator = this.resolveOrchestrator(sessionId) + if (!orchestrator) { + this.scheduleNextAutoObserve(sessionId, state) + return + } + + if (state.activeInferenceAbortController) { + state.stats.skippedAutoObserve++ + log.withTag('realtime').log( + `Auto-observe skipped for ${sessionId}: inference in flight ` + + `(skipped=${state.stats.skippedAutoObserve}, avg=${Math.round(state.stats.avgLatencyMs)}ms)`, + ) + this.broadcastPipelineStatus(sessionId, state) + this.scheduleNextAutoObserve(sessionId, state) + return + } + + const latestFrame = pickLatestFrame(state, orchestrator.getContext()) + if (!latestFrame) { + this.scheduleNextAutoObserve(sessionId, state) + return + } + + const fingerprint = computeFrameFingerprint(latestFrame) + if (fingerprint === state.autoObserve.lastFrameFingerprint) { + state.stats.skippedNoChange++ + this.scheduleNextAutoObserve(sessionId, state) + return + } + state.autoObserve.lastFrameFingerprint = fingerprint + + const startedAt = Date.now() + await this.inferWithLatestFrame(sessionId, DEFAULT_AUTO_OBSERVE_PROMPT, true) + const durationMs = Date.now() - startedAt + + state.stats.avgLatencyMs = updateLatencyEma(state.stats.avgLatencyMs, durationMs) + state.stats.lastLatencyMs = durationMs + state.stats.lastInferenceAt = Date.now() + + const baseMs = state.autoObserve.baseIntervalMs + const minDelay = baseMs + const latencyPadding = durationMs * 0.5 + state.autoObserve.adaptiveIntervalMs = Math.min( + baseMs * 3, + Math.max(minDelay, baseMs + latencyPadding), + ) + + this.broadcastPipelineStatus(sessionId, state) + this.scheduleNextAutoObserve(sessionId, state) + } + + private broadcastPipelineStatus(sessionId: string, state: SessionRealtimeState): void { + this.broadcast(sessionId, 'auto-observe:status', { + sessionId, + stats: { ...state.stats }, + adaptiveIntervalMs: state.autoObserve.adaptiveIntervalMs, + baseIntervalMs: state.autoObserve.baseIntervalMs, + }) + } + + private ensureSource( + sessionId: string, + participantIdentity: string, + clientSourceId: string, + sourceType: GatewayRealtimeVideoFrameMessage['sourceType'], + ) { + const orchestrator = this.resolveOrchestrator(sessionId) + if (!orchestrator) + throw new Error(`Session ${sessionId} not found while resolving source`) + + const registry = orchestrator.getRegistry() + const existing = registry.findByTrackSid(clientSourceId) + if (existing) + return existing + + const source = orchestrator.registerSource(participantIdentity, clientSourceId, sourceType) + this.broadcast(sessionId, 'source:registered', orchestrator.getContext()) + return source + } + + private async inferWithLatestFrame(sessionId: string, prompt: string, auto: boolean): Promise { + const orchestrator = this.resolveOrchestrator(sessionId) + const state = this.sessionState.get(sessionId) + if (!orchestrator || !state) + return + + if (state.activeInferenceAbortController) { + if (auto) { + state.stats.skippedAutoObserve++ + return + } + + log.withTag('realtime').log(`Preempting running inference for ${sessionId} with a new user-triggered request.`) + state.activeInferenceAbortController.abort() + state.activeInferenceAbortController = null + } + + const latestFrame = pickLatestFrame(state, orchestrator.getContext()) + if (!latestFrame) { + if (!auto) { + this.broadcast(sessionId, INFERENCE_FAILED, { + error: 'No live camera or screen frame is available for this session yet.', + auto, + } satisfies RealtimeInferenceFailedPayload) + } + return + } + + state.stats.totalInferences++ + if (auto) + state.stats.autoObserveInferences++ + else + state.stats.userInferences++ + + const traceId = nanoid(10) + const inferenceStartedAt = Date.now() + + log.withTag('realtime').log( + `[trace:${traceId}] Inference starting` + + ` | session=${sessionId}` + + ` | auto=${auto}` + + ` | source=${latestFrame.sourceId}` + + ` | prompt=${prompt.slice(0, 80)}${prompt.length > 80 ? '…' : ''}` + + ` | historyLen=${state.messages.length}` + + ` | memoryLen=${state.sceneMemorySummary.length}`, + ) + + const abortController = new AbortController() + state.activeInferenceAbortController = abortController + + const timeoutId = auto + ? setTimeout(() => { + if (!abortController.signal.aborted) { + state.stats.timedOut++ + log.withTag('realtime').log(`[trace:${traceId}] Inference timed out after ${AUTO_OBSERVE_INFERENCE_TIMEOUT_MS}ms`) + abortController.abort() + } + }, AUTO_OBSERVE_INFERENCE_TIMEOUT_MS) + : null + orchestrator.transitionState('inference') + + this.broadcast(sessionId, INFERENCE_STARTED, { + prompt, + auto, + sourceId: latestFrame.sourceId, + } satisfies RealtimeInferenceStartedPayload) + + const draftMessageId = nanoid() + let accumulatedText = '' + let model = '' + let completedAssistantMessage: TextMessage | null = null + let completedAutoMemory = '' + const sessionContext = orchestrator.getContext() + const frameSourceType = sessionContext.activeVideoSource?.sourceId === latestFrame.sourceId + ? sessionContext.activeVideoSource.sourceType + : sessionContext.standbyVideoSources.find(source => source.sourceId === latestFrame.sourceId)?.sourceType + + const systemPrompt = buildSystemPrompt({ + auto, + sceneMemorySummary: state.sceneMemorySummary, + sourceType: frameSourceType, + }) + + try { + const response = await fetch(`${normalizeBaseUrl(this.workerBaseUrl)}/infer-stream`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + image: latestFrame.data.toString('base64'), + prompt, + system: systemPrompt, + history: auto + ? [] + : buildHistory(state.messages, prompt, USER_INFERENCE_HISTORY_LIMIT), + }), + signal: abortController.signal, + }) + + if (!response.ok) { + const detail = await response.text().catch(() => response.statusText) + throw new Error(detail || `Worker inference failed (${response.status})`) + } + + await readNdjsonResponse(response, (event) => { + if (event.type === 'start') { + model = event.model ?? model + return + } + + if (event.type === 'delta') { + accumulatedText = event.text ?? `${accumulatedText}${event.delta ?? ''}` + if (!auto) { + this.broadcast(sessionId, INFERENCE_TEXT_CHUNK, { + id: draftMessageId, + delta: event.delta ?? '', + text: accumulatedText, + sourceId: latestFrame.sourceId, + model, + } satisfies RealtimeInferenceTextChunkPayload) + } + return + } + + if (event.type === 'error') + throw new Error(event.error || 'Worker inference stream failed') + + if (event.type === 'done') { + accumulatedText = truncateText(event.text ?? accumulatedText, MAX_SCENE_MEMORY_CHARS) + model = event.model ?? model + + const finalMessage: TextMessage = { + id: draftMessageId, + role: 'assistant', + content: accumulatedText, + timestamp: Date.now(), + sourceId: latestFrame.sourceId, + model: model || undefined, + } + + if (auto) { + completedAutoMemory = finalMessage.content + this.broadcast(sessionId, INFERENCE_COMPLETED, { + message: finalMessage, + sourceId: latestFrame.sourceId, + auto: true, + durationMs: event.durationMs, + } satisfies RealtimeInferenceCompletedPayload) + return + } + + completedAssistantMessage = finalMessage + this.broadcast(sessionId, INFERENCE_COMPLETED, { + message: finalMessage, + sourceId: latestFrame.sourceId, + auto: false, + durationMs: event.durationMs, + } satisfies RealtimeInferenceCompletedPayload) + this.broadcast(sessionId, 'chat:message', finalMessage) + } + }) + + if (completedAssistantMessage) { + state.messages = clampMessages([...state.messages, completedAssistantMessage]) + await this.records.saveMessages(sessionId, state.messages, { + roomName: orchestrator.roomName, + sceneMemory: state.sceneMemorySummary, + }) + } + + if (completedAutoMemory) { + const nextSceneMemory = truncateText(completedAutoMemory, MAX_SCENE_MEMORY_CHARS) + if (isMeaningfullyDifferent(state.sceneMemorySummary, nextSceneMemory)) { + const updatedAt = Date.now() + state.sceneMemorySummary = nextSceneMemory + state.sceneMemoryTimeline = clampMemoryTimeline([ + ...state.sceneMemoryTimeline, + { + summary: nextSceneMemory, + updatedAt, + sourceId: latestFrame.sourceId, + }, + ]) + + const record = await this.records.updateSceneMemory(sessionId, state.sceneMemorySummary, { + roomName: orchestrator.roomName, + sourceId: latestFrame.sourceId, + updatedAt, + }) + state.sceneMemoryTimeline = clampMemoryTimeline(record.memoryTimeline ?? state.sceneMemoryTimeline) + this.broadcast(sessionId, 'session:memory:updated', { + sessionId, + summary: state.sceneMemorySummary, + updatedAt, + sourceId: latestFrame.sourceId, + timeline: state.sceneMemoryTimeline, + }) + } + } + + const elapsedMs = Date.now() - inferenceStartedAt + log.withTag('realtime').log( + `[trace:${traceId}] Inference completed` + + ` | elapsed=${elapsedMs}ms` + + ` | auto=${auto}` + + ` | outputLen=${accumulatedText.length}` + + ` | model=${model}`, + ) + orchestrator.transitionState('ready') + } + catch (error) { + const elapsedMs = Date.now() - inferenceStartedAt + orchestrator.transitionState('ready') + if (abortController.signal.aborted) { + log.withTag('realtime').log( + `[trace:${traceId}] Inference aborted` + + ` | elapsed=${elapsedMs}ms` + + ` | auto=${auto}`, + ) + } + else { + log.withTag('realtime').log( + `[trace:${traceId}] Inference failed` + + ` | elapsed=${elapsedMs}ms` + + ` | auto=${auto}` + + ` | error=${errorMessage(error)}`, + ) + this.broadcast(sessionId, INFERENCE_FAILED, { + error: errorMessage(error), + sourceId: latestFrame.sourceId, + auto, + } satisfies RealtimeInferenceFailedPayload) + } + } + finally { + if (timeoutId) + clearTimeout(timeoutId) + if (state.activeInferenceAbortController === abortController) + state.activeInferenceAbortController = null + } + } + + getStats(sessionId: string): InferenceStats | null { + return this.sessionState.get(sessionId)?.stats ?? null + } +} diff --git a/services/visual-chat-gateway/src/routes/bootstrap.ts b/services/visual-chat-gateway/src/routes/bootstrap.ts new file mode 100644 index 0000000000..5b7e12212e --- /dev/null +++ b/services/visual-chat-gateway/src/routes/bootstrap.ts @@ -0,0 +1,16 @@ +import { createRouter, defineEventHandler } from 'h3' + +import { createGatewayAccessToken, requireGatewayAccess } from '../auth' + +export function createBootstrapRoutes() { + const router = createRouter() + + router.get('/api/bootstrap', defineEventHandler((event) => { + requireGatewayAccess(event) + return { + gatewayToken: createGatewayAccessToken(), + } + })) + + return router +} diff --git a/services/visual-chat-gateway/src/routes/diagnostics.ts b/services/visual-chat-gateway/src/routes/diagnostics.ts new file mode 100644 index 0000000000..1aa2b10656 --- /dev/null +++ b/services/visual-chat-gateway/src/routes/diagnostics.ts @@ -0,0 +1,248 @@ +import type { SessionStore } from '@proj-airi/visual-chat-runtime' + +import process from 'node:process' + +import { existsSync, readFileSync } from 'node:fs' +import { hostname, networkInterfaces } from 'node:os' +import { join } from 'node:path' + +import { getVisualChatDir } from '@proj-airi/visual-chat-shared' +import { createRouter, defineEventHandler } from 'h3' + +import { requireGatewayAccess } from '../auth' +import { gatewayEnv } from '../gateway-env' + +const startedAt = Date.now() +const DEFAULT_PUBLIC_ENDPOINTS_FILE = process.env.AIRI_VISUAL_CHAT_PUBLIC_ENDPOINTS_FILE?.trim() + || join(getVisualChatDir('config'), 'public-endpoints.json') + +const IPV4_SEGMENT_PATTERN = /^\d{1,3}$/ +const VIRTUAL_INTERFACE_PATTERN = /hyper-v|wsl|vethernet|vmware|virtualbox|docker|podman|zerotier|tailscale|clash|tun|tap|vpn|loopback|bluetooth|hamachi/i +const WIFI_INTERFACE_PATTERN = /wi-?fi|wlan|wireless/i +const ETHERNET_INTERFACE_PATTERN = /ethernet|\u4EE5\u592A\u7F51|^en\d|^eth\d/i + +function parseIpv4(address: string): number[] | null { + const segments = address.trim().split('.') + if (segments.length !== 4) + return null + + const numbers = segments.map((segment) => { + if (!IPV4_SEGMENT_PATTERN.test(segment)) + return Number.NaN + return Number(segment) + }) + + if (numbers.some(segment => Number.isNaN(segment) || segment < 0 || segment > 255)) + return null + + return numbers +} + +function scoreLanAddressCandidate(candidate: { address: string, interfaceName?: string }): number { + const segments = parseIpv4(candidate.address) + if (!segments) + return Number.NEGATIVE_INFINITY + + const isLoopback = segments[0] === 127 + const isLinkLocal = segments[0] === 169 && segments[1] === 254 + const isBenchmark = segments[0] === 198 && (segments[1] === 18 || segments[1] === 19) + if (isLoopback || isLinkLocal || isBenchmark) + return Number.NEGATIVE_INFINITY + + if (candidate.interfaceName && VIRTUAL_INTERFACE_PATTERN.test(candidate.interfaceName)) + return Number.NEGATIVE_INFINITY + + let score = 0 + const isPrivate = segments[0] === 10 + || (segments[0] === 172 && segments[1] >= 16 && segments[1] <= 31) + || (segments[0] === 192 && segments[1] === 168) + const isCarrierGradeNat = segments[0] === 100 && segments[1] >= 64 && segments[1] <= 127 + + if (isPrivate) + score += 100 + else + score -= 40 + + if (isCarrierGradeNat) + score -= 20 + + const interfaceName = candidate.interfaceName?.trim() ?? '' + if (WIFI_INTERFACE_PATTERN.test(interfaceName)) + score += 40 + else if (ETHERNET_INTERFACE_PATTERN.test(interfaceName)) + score += 25 + + return score +} + +function getPreferredLanAddresses(candidates: Array<{ address: string, interfaceName?: string }>): string[] { + const deduped = new Map() + + for (const candidate of candidates) { + const address = candidate.address.trim() + if (!address) + continue + + const current = { ...candidate, address } + const score = scoreLanAddressCandidate(current) + if (!Number.isFinite(score)) + continue + + const existing = deduped.get(address) + if (!existing || scoreLanAddressCandidate(existing) < score) + deduped.set(address, current) + } + + return [...deduped.values()] + .sort((left, right) => { + const scoreDelta = scoreLanAddressCandidate(right) - scoreLanAddressCandidate(left) + if (scoreDelta !== 0) + return scoreDelta + + return left.address.localeCompare(right.address) + }) + .map(candidate => candidate.address) +} + +function getLanIpv4Addresses(): string[] { + const candidates = Object.entries(networkInterfaces()) + .flatMap(([interfaceName, items]) => (items ?? []).map(item => ({ interfaceName, item }))) + .filter((entry): entry is { interfaceName: string, item: NonNullable } => !!entry.item) + .filter(entry => entry.item.family === 'IPv4' && !entry.item.internal) + .map(entry => ({ + address: entry.item.address.trim(), + interfaceName: entry.interfaceName, + })) + + return getPreferredLanAddresses(candidates) +} + +function sanitizeUrl(value: string | undefined): string | undefined { + const normalized = value?.trim() + if (!normalized) + return undefined + + try { + return new URL(normalized).toString() + } + catch { + return undefined + } +} + +function readPublicEndpoints(): { frontendUrl?: string, gatewayUrl?: string } { + const publicFrontendUrl = sanitizeUrl(process.env.AIRI_VISUAL_CHAT_PUBLIC_FRONTEND_URL) + const publicGatewayUrl = sanitizeUrl(process.env.AIRI_VISUAL_CHAT_PUBLIC_GATEWAY_URL) + if (publicFrontendUrl || publicGatewayUrl) { + return { + frontendUrl: publicFrontendUrl, + gatewayUrl: publicGatewayUrl, + } + } + + const endpointsFile = process.env.AIRI_VISUAL_CHAT_PUBLIC_ENDPOINTS_FILE?.trim() || DEFAULT_PUBLIC_ENDPOINTS_FILE + if (!existsSync(endpointsFile)) + return {} + + try { + const raw = JSON.parse(readFileSync(endpointsFile, 'utf8')) as { + frontendUrl?: string + gatewayUrl?: string + } + + return { + frontendUrl: sanitizeUrl(raw.frontendUrl), + gatewayUrl: sanitizeUrl(raw.gatewayUrl), + } + } + catch { + return {} + } +} + +export function createDiagnosticRoutes(store: SessionStore, realtimeManager?: { getStats: (sessionId: string) => unknown }) { + const router = createRouter() + + router.get('/api/diagnostics', defineEventHandler(async (event) => { + requireGatewayAccess(event) + const workerUrl = gatewayEnv.workerUrl + let workerStatus = 'unknown' + + try { + const res = await fetch(`${workerUrl}/health`) + if (res.ok) { + const data = await res.json() as Record + workerStatus = String(data.status ?? 'ok') + } + } + catch { + workerStatus = 'unreachable' + } + + let lanAddresses: string[] = [] + try { + lanAddresses = getLanIpv4Addresses() + } + catch { + // never block diagnostics + } + + let publicEndpoints: { frontendUrl?: string, gatewayUrl?: string } = {} + try { + publicEndpoints = readPublicEndpoints() + } + catch { + // never block diagnostics + } + + let machineHostname = '' + try { + machineHostname = hostname() + } + catch { + // never block diagnostics + } + + let sessionStats: Record = {} + try { + if (realtimeManager) { + const all = typeof store.getAll === 'function' ? store.getAll() : [] + for (const orchestrator of all) { + const sid = orchestrator?.sessionId + if (!sid) + continue + const stats = realtimeManager.getStats(sid) + if (stats) + sessionStats[sid] = stats + } + } + } + catch { + sessionStats = {} + } + + let activeSessionCount = 0 + try { + activeSessionCount = store.size ?? 0 + } + catch { + // never block diagnostics + } + + return { + activeSessions: activeSessionCount, + workerStatus, + uptimeMs: Date.now() - startedAt, + livekitUrl: gatewayEnv.livekitUrl, + workerUrl: gatewayEnv.workerUrl, + lanAddresses, + preferredLanAddress: lanAddresses[0], + hostname: machineHostname, + publicFrontendUrl: publicEndpoints.frontendUrl, + publicGatewayUrl: publicEndpoints.gatewayUrl, + sessionPipelineStats: sessionStats, + } + })) + + return router +} diff --git a/services/visual-chat-gateway/src/routes/rooms.ts b/services/visual-chat-gateway/src/routes/rooms.ts new file mode 100644 index 0000000000..946ed031a2 --- /dev/null +++ b/services/visual-chat-gateway/src/routes/rooms.ts @@ -0,0 +1,62 @@ +import type { SessionStore } from '@proj-airi/visual-chat-runtime' + +import { generateToken } from '@proj-airi/visual-chat-livekit' +import { createGatewayLogger } from '@proj-airi/visual-chat-observability' +import { createError, createRouter, defineEventHandler, getRouterParam, readBody } from 'h3' + +import { requireGatewayAccess, requireSessionAccess } from '../auth' +import { gatewayEnv } from '../gateway-env' + +const log = createGatewayLogger() + +export function createRoomRoutes(store: SessionStore) { + const router = createRouter() + + router.post('/api/rooms/:roomName/token', defineEventHandler(async (event) => { + const roomName = getRouterParam(event, 'roomName')! + const body = await readBody(event) as { + name?: string + identity?: string + } + + const orchestrator = store.getByRoom(roomName) + if (!orchestrator) { + log.withTag('rooms').warn(`Token requested for unknown room: ${roomName}`) + throw createError({ statusCode: 404, statusMessage: 'Room not found. Create a session first.' }) + } + + requireSessionAccess(event, orchestrator.sessionId) + + const apiKey = gatewayEnv.livekitApiKey + const apiSecret = gatewayEnv.livekitApiSecret + + const token = await generateToken({ + apiKey, + apiSecret, + roomName, + participantName: body.name ?? 'user', + participantIdentity: body.identity ?? `user_${Date.now()}`, + canPublish: true, + canSubscribe: true, + canPublishData: true, + }) + + return { + token, + roomName, + sessionId: orchestrator.sessionId, + } + })) + + router.get('/api/rooms', defineEventHandler((event) => { + requireGatewayAccess(event) + return store.getAll().map(o => ({ + roomName: o.roomName, + sessionId: o.sessionId, + mode: o.mode, + state: o.state, + })) + })) + + return router +} diff --git a/services/visual-chat-gateway/src/routes/sessions.ts b/services/visual-chat-gateway/src/routes/sessions.ts new file mode 100644 index 0000000000..201902a0ce --- /dev/null +++ b/services/visual-chat-gateway/src/routes/sessions.ts @@ -0,0 +1,145 @@ +import type { SessionAccess, SessionContext, SessionRecord } from '@proj-airi/visual-chat-protocol' +import type { SessionStore } from '@proj-airi/visual-chat-runtime' + +import { SessionOrchestrator } from '@proj-airi/visual-chat-runtime' +import { generateRoomName, normalizeVisualChatSessionId } from '@proj-airi/visual-chat-shared' +import { deleteSessionData } from '@proj-airi/visual-chat-storage' +import { createError, createRouter, defineEventHandler, getRouterParam, readBody } from 'h3' + +import { createSessionAccessToken, requireGatewayAccess, requireSessionAccess } from '../auth' + +export type BroadcastFn = (sessionId: string, event: string, data: unknown) => void + +interface SessionRoutesOptions { + onSessionCreated?: (sessionId: string, roomName: string) => void | Promise + onSessionDeleted?: (sessionId: string) => void + getSessionMessages?: (sessionId: string) => Promise | unknown[] + getSessionRecord?: (sessionId: string) => Promise | SessionRecord | null + listSessionRecords?: () => Promise | SessionRecord[] + restoreSession?: (sessionId: string) => Promise | SessionContext +} + +export function createSessionRoutes(store: SessionStore, broadcast: BroadcastFn, options: SessionRoutesOptions = {}) { + const router = createRouter() + + function createSessionAccessPayload(orchestrator: SessionOrchestrator): SessionAccess { + return { + session: orchestrator.getContext(), + sessionToken: createSessionAccessToken(orchestrator.sessionId), + } + } + + router.post('/api/sessions', defineEventHandler(async (event) => { + requireGatewayAccess(event) + + const roomName = generateRoomName() + const orchestrator = new SessionOrchestrator(roomName) + + orchestrator.onEvent((evt, data) => { + broadcast(orchestrator.sessionId, evt, data) + }) + + store.add(orchestrator) + await options.onSessionCreated?.(orchestrator.sessionId, roomName) + + broadcast(orchestrator.sessionId, 'session:started', orchestrator.getContext()) + + return createSessionAccessPayload(orchestrator) + })) + + router.post('/api/sessions/:sessionId/access', defineEventHandler((event) => { + requireGatewayAccess(event) + const sessionId = normalizeVisualChatSessionId(getRouterParam(event, 'sessionId')!) + const orchestrator = store.getBySessionId(sessionId) + if (!orchestrator) + throw createError({ statusCode: 404, statusMessage: 'Session not found' }) + + return createSessionAccessPayload(orchestrator) + })) + + router.get('/api/sessions', defineEventHandler((event) => { + requireGatewayAccess(event) + return store.getAll().map(o => o.getContext()) + })) + + router.get('/api/sessions/:sessionId', defineEventHandler((event) => { + const sessionId = requireSessionAccess(event, getRouterParam(event, 'sessionId')!) + const orchestrator = store.getBySessionId(sessionId) + if (!orchestrator) + throw createError({ statusCode: 404, statusMessage: 'Session not found' }) + + return orchestrator.getContext() + })) + + router.get('/api/sessions/:sessionId/messages', defineEventHandler(async (event) => { + const sessionId = requireSessionAccess(event, getRouterParam(event, 'sessionId')!) + return { + messages: await options.getSessionMessages?.(sessionId) ?? [], + } + })) + + router.get('/api/sessions/:sessionId/record', defineEventHandler(async (event) => { + const sessionId = requireSessionAccess(event, getRouterParam(event, 'sessionId')!) + const record = await options.getSessionRecord?.(sessionId) + if (!record) + throw createError({ statusCode: 404, statusMessage: 'Session record not found' }) + return record + })) + + router.get('/api/session-records', defineEventHandler(async (event) => { + requireGatewayAccess(event) + return { + records: await options.listSessionRecords?.() ?? [], + } + })) + + router.post('/api/session-records/:sessionId/restore', defineEventHandler(async (event) => { + requireGatewayAccess(event) + const sessionId = normalizeVisualChatSessionId(getRouterParam(event, 'sessionId')!) + const restored = await options.restoreSession?.(sessionId) + if (!restored) + throw createError({ statusCode: 404, statusMessage: 'Session record not found' }) + + return { + session: restored, + sessionToken: createSessionAccessToken(sessionId), + } satisfies SessionAccess + })) + + router.delete('/api/sessions/:sessionId/record', defineEventHandler(async (event) => { + const sessionId = requireSessionAccess(event, getRouterParam(event, 'sessionId')!) + await deleteSessionData(sessionId) + return { success: true } + })) + + router.delete('/api/sessions/:sessionId', defineEventHandler((event) => { + const sessionId = requireSessionAccess(event, getRouterParam(event, 'sessionId')!) + const orchestrator = store.getBySessionId(sessionId) + if (orchestrator) { + broadcast(sessionId, 'session:ended', { sessionId }) + } + options.onSessionDeleted?.(sessionId) + store.remove(sessionId) + return { ok: true } + })) + + router.post('/api/sessions/:sessionId/switch-source', defineEventHandler(async (event) => { + const sessionId = requireSessionAccess(event, getRouterParam(event, 'sessionId')!) + const body = await readBody(event) as { + sourceId?: string + sourceType?: string + } + const orchestrator = store.getBySessionId(sessionId) + if (!orchestrator) + throw createError({ statusCode: 404, statusMessage: 'Session not found' }) + + const source = body.sourceId ?? body.sourceType + if (!source) + throw createError({ statusCode: 400, statusMessage: 'Missing sourceId or sourceType' }) + + orchestrator.switchSource(source) + return orchestrator.getContext() + })) + + return router +} diff --git a/services/visual-chat-gateway/src/routes/webhook.ts b/services/visual-chat-gateway/src/routes/webhook.ts new file mode 100644 index 0000000000..2fddae9002 --- /dev/null +++ b/services/visual-chat-gateway/src/routes/webhook.ts @@ -0,0 +1,99 @@ +import type { LiveKitEvent } from '@proj-airi/visual-chat-livekit' +import type { SessionStore } from '@proj-airi/visual-chat-runtime' + +import { createWebhookHandler } from '@proj-airi/visual-chat-livekit' +import { createGatewayLogger } from '@proj-airi/visual-chat-observability' +import { createError, createRouter, defineEventHandler, getHeader, readRawBody } from 'h3' + +import { gatewayEnv } from '../gateway-env' + +const log = createGatewayLogger() + +export type BroadcastFn = (sessionId: string, event: string, data: unknown) => void + +export function createWebhookRoutes(store: SessionStore, broadcast: BroadcastFn) { + const router = createRouter() + + function handleEvent(event: LiveKitEvent) { + const roomName = event.room?.name + if (!roomName) + return + + const orchestrator = store.getByRoom(roomName) + if (!orchestrator) + return + + switch (event.event) { + case 'track_published': { + if (event.participant && event.track) { + const sourceType = inferSourceType(event.track.type, event.participant.identity) + orchestrator.registerSource(event.participant.identity, event.track.sid, sourceType) + broadcast(orchestrator.sessionId, 'source:registered', orchestrator.getContext()) + log.withTag('webhook').log(`Source registered via webhook: ${sourceType} from ${event.participant.identity}`) + } + break + } + case 'track_unpublished': { + if (event.track) { + const registry = orchestrator.getRegistry() + const source = registry.findByTrackSid(event.track.sid) + if (source) { + orchestrator.unregisterSource(source.sourceId) + broadcast(orchestrator.sessionId, 'source:unregistered', orchestrator.getContext()) + } + } + break + } + case 'participant_joined': { + broadcast(orchestrator.sessionId, 'room:participant:joined', { + identity: event.participant?.identity, + }) + break + } + case 'participant_left': { + if (event.participant) { + const registry = orchestrator.getRegistry() + const sources = registry.findByParticipant(event.participant.identity) + for (const source of sources) { + orchestrator.unregisterSource(source.sourceId) + } + broadcast(orchestrator.sessionId, 'room:participant:left', { + identity: event.participant.identity, + }) + } + break + } + } + } + + router.post('/api/livekit/webhook', defineEventHandler(async (event) => { + const apiKey = gatewayEnv.livekitApiKey + const apiSecret = gatewayEnv.livekitApiSecret + + const body = await readRawBody(event, 'utf8') + const authHeader = getHeader(event, 'authorization') ?? '' + + if (!body) + throw createError({ statusCode: 400, statusMessage: 'Missing body' }) + + const handler = createWebhookHandler(apiKey, apiSecret, handleEvent) + await handler(body, authHeader) + return { ok: true } + })) + + return router +} + +function inferSourceType(trackType: string, identity: string): 'phone-camera' | 'laptop-camera' | 'screen-share' | 'phone-mic' | 'laptop-mic' { + if (trackType === 'TRACK_TYPE_VIDEO' || trackType === 'video') { + if (identity.startsWith('phone') || identity.includes('mobile')) + return 'phone-camera' + if (identity.includes('screen')) + return 'screen-share' + return 'laptop-camera' + } + + if (identity.startsWith('phone') || identity.includes('mobile')) + return 'phone-mic' + return 'laptop-mic' +} diff --git a/services/visual-chat-gateway/src/routes/worker-proxy.ts b/services/visual-chat-gateway/src/routes/worker-proxy.ts new file mode 100644 index 0000000000..b61a75ddd7 --- /dev/null +++ b/services/visual-chat-gateway/src/routes/worker-proxy.ts @@ -0,0 +1,68 @@ +import { createError, createRouter, defineEventHandler, getHeader, readRawBody } from 'h3' + +import { requireGatewayAccess } from '../auth' +import { gatewayEnv } from '../gateway-env' + +const SAFE_RESPONSE_HEADERS = new Set([ + 'cache-control', + 'content-length', + 'content-type', + 'transfer-encoding', +]) +const TRAILING_SLASH_PATTERN = /\/$/ + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} + +function buildSafeHeaders(response: Response): Headers { + const headers = new Headers() + response.headers.forEach((value, key) => { + if (SAFE_RESPONSE_HEADERS.has(key.toLowerCase())) + headers.set(key, value) + }) + return headers +} + +function workerUrlFor(path: string): string { + return `${gatewayEnv.workerUrl.replace(TRAILING_SLASH_PATTERN, '')}${path}` +} + +async function proxyToWorker(path: string, init?: RequestInit): Promise { + try { + const upstream = await fetch(workerUrlFor(path), init) + return new Response(upstream.body, { + status: upstream.status, + headers: buildSafeHeaders(upstream), + }) + } + catch (error) { + throw createError({ + statusCode: 502, + statusMessage: `Worker proxy failed: ${errorMessage(error)}`, + }) + } +} + +export function createWorkerProxyRoutes() { + const router = createRouter() + + router.get('/api/worker/health', defineEventHandler(async (event) => { + requireGatewayAccess(event) + return proxyToWorker('/health') + })) + + router.post('/api/worker/infer-stream', defineEventHandler(async (event) => { + requireGatewayAccess(event) + const body = await readRawBody(event, 'utf8') + return proxyToWorker('/infer-stream', { + method: 'POST', + headers: { + 'Content-Type': getHeader(event, 'content-type') || 'application/json', + }, + body, + }) + })) + + return router +} diff --git a/services/visual-chat-gateway/src/routes/workers.ts b/services/visual-chat-gateway/src/routes/workers.ts new file mode 100644 index 0000000000..6e26b8236c --- /dev/null +++ b/services/visual-chat-gateway/src/routes/workers.ts @@ -0,0 +1,32 @@ +import { createError, createRouter, defineEventHandler, readBody } from 'h3' + +import { hasGatewayAccess, isTrustedLocalRequest } from '../auth' + +export interface WorkerHeartbeatRecord { + receivedAt: number + body: unknown +} + +let lastHeartbeat: WorkerHeartbeatRecord | null = null + +export function getLastWorkerHeartbeat(): WorkerHeartbeatRecord | null { + return lastHeartbeat +} + +export function createWorkerRoutes() { + const router = createRouter() + + router.post('/api/workers/heartbeat', defineEventHandler(async (event) => { + if (!isTrustedLocalRequest(event) && !hasGatewayAccess(event)) { + throw createError({ + statusCode: 403, + statusMessage: 'Worker heartbeat access denied.', + }) + } + const body = await readBody(event).catch(() => null) + lastHeartbeat = { receivedAt: Date.now(), body } + return { ok: true } + })) + + return router +} diff --git a/services/visual-chat-gateway/src/session-records/index.ts b/services/visual-chat-gateway/src/session-records/index.ts new file mode 100644 index 0000000000..8e2a8a5e13 --- /dev/null +++ b/services/visual-chat-gateway/src/session-records/index.ts @@ -0,0 +1,153 @@ +import type { SessionMemorySnapshot, SessionRecord, TextMessage } from '@proj-airi/visual-chat-protocol' + +import { listSessionIds, loadSessionMessages, loadSessionMetadata, saveSessionMessages, saveSessionMetadata } from '@proj-airi/visual-chat-storage' + +const WHITESPACE_PATTERN = /\s+/g + +function normalizeTitle(messages: TextMessage[]): string { + const firstUserMessage = messages.find(message => message.role === 'user' && message.content.trim()) + if (!firstUserMessage) + return 'New visual chat' + + const singleLine = firstUserMessage.content.replace(WHITESPACE_PATTERN, ' ').trim() + return singleLine.length > 60 + ? `${singleLine.slice(0, 57)}...` + : singleLine +} + +function normalizeSummary(messages: TextMessage[]): string { + const lastAssistantMessage = [...messages] + .reverse() + .find(message => message.role === 'assistant' && message.content.trim()) + + if (!lastAssistantMessage) + return '' + + const singleLine = lastAssistantMessage.content.replace(WHITESPACE_PATTERN, ' ').trim() + return singleLine.length > 120 + ? `${singleLine.slice(0, 117)}...` + : singleLine +} + +function buildDefaultRecord(sessionId: string): SessionRecord { + const now = Date.now() + return { + sessionId, + roomName: `visual-chat-${sessionId.slice(0, 8)}`, + title: 'New visual chat', + createdAt: now, + updatedAt: now, + lastMessageAt: null, + messageCount: 0, + summary: '', + sceneMemory: '', + memoryTimeline: [], + } +} + +function clampMemoryTimeline(timeline: SessionMemorySnapshot[]): SessionMemorySnapshot[] { + if (timeline.length <= 6) + return timeline + return timeline.slice(-6) +} + +export class SessionRecordRepository { + async getRecord(sessionId: string): Promise { + const metadata = await loadSessionMetadata(sessionId) + if (!metadata) + return null + + const record = metadata.record + if (!record || typeof record !== 'object') + return null + + return { + ...buildDefaultRecord(sessionId), + ...(record as Partial), + sessionId, + } + } + + async ensureRecord(sessionId: string, roomName?: string): Promise { + const existing = await this.getRecord(sessionId) + if (existing) + return existing + + const created = { + ...buildDefaultRecord(sessionId), + roomName: roomName || `visual-chat-${sessionId.slice(0, 8)}`, + } satisfies SessionRecord + + await saveSessionMetadata(sessionId, { record: created }) + return created + } + + async saveMessages(sessionId: string, messages: TextMessage[], options?: { roomName?: string, sceneMemory?: string }): Promise { + const existing = await this.ensureRecord(sessionId, options?.roomName) + const createdAt = existing.createdAt || Date.now() + const lastMessageAt = messages.at(-1)?.timestamp ?? existing.lastMessageAt ?? null + + const nextRecord = { + ...existing, + roomName: options?.roomName || existing.roomName, + title: normalizeTitle(messages), + updatedAt: Date.now(), + lastMessageAt, + messageCount: messages.length, + summary: normalizeSummary(messages), + sceneMemory: options?.sceneMemory ?? existing.sceneMemory ?? '', + memoryTimeline: existing.memoryTimeline ?? [], + createdAt, + } satisfies SessionRecord + + await saveSessionMessages(sessionId, messages) + await saveSessionMetadata(sessionId, { record: nextRecord }) + return nextRecord + } + + async updateSceneMemory( + sessionId: string, + sceneMemory: string, + options?: { roomName?: string, sourceId?: string, updatedAt?: number }, + ): Promise { + const existing = await this.ensureRecord(sessionId, options?.roomName) + const messages = await loadSessionMessages(sessionId) + const updatedAt = options?.updatedAt ?? Date.now() + const nextSnapshot = { + summary: sceneMemory, + updatedAt, + sourceId: options?.sourceId, + } satisfies SessionMemorySnapshot + const previousTimeline = existing.memoryTimeline ?? [] + const lastSnapshot = previousTimeline.at(-1) + const nextTimeline = clampMemoryTimeline( + lastSnapshot?.summary === sceneMemory + ? [...previousTimeline.slice(0, -1), nextSnapshot] + : [...previousTimeline, nextSnapshot], + ) + const nextRecord = { + ...existing, + roomName: options?.roomName || existing.roomName, + updatedAt, + sceneMemory, + memoryTimeline: nextTimeline, + } satisfies SessionRecord + + await saveSessionMetadata(sessionId, { record: nextRecord }) + if (messages.length > 0) + await saveSessionMessages(sessionId, messages) + return nextRecord + } + + async loadMessages(sessionId: string): Promise { + return loadSessionMessages(sessionId) + } + + async listRecords(): Promise { + const sessionIds = await listSessionIds() + const records = await Promise.all(sessionIds.map(sessionId => this.getRecord(sessionId))) + return records + .filter((record): record is SessionRecord => record !== null) + .sort((left, right) => right.updatedAt - left.updatedAt) + } +} diff --git a/services/visual-chat-gateway/src/ws/index.ts b/services/visual-chat-gateway/src/ws/index.ts new file mode 100644 index 0000000000..122a0088c4 --- /dev/null +++ b/services/visual-chat-gateway/src/ws/index.ts @@ -0,0 +1,97 @@ +import type { GatewayWsClientMessage } from '@proj-airi/visual-chat-protocol' +import type { Peer } from 'crossws' + +import { createGatewayLogger } from '@proj-airi/visual-chat-observability' +import { defineWebSocketHandler } from 'h3' + +const log = createGatewayLogger() + +interface CreateWsHandlerOptions { + onClientMessage?: (message: GatewayWsClientMessage) => void | Promise + authorizeSessionAccess?: (sessionId: string, sessionToken: string) => boolean +} + +export function createWsHandler(options: CreateWsHandlerOptions = {}) { + const peers = new Set() + const subscriptions = new Map>() + const authorizedSessions = new Map>() + + const handler = defineWebSocketHandler({ + open(peer) { + peers.add(peer) + subscriptions.set(peer, new Set()) + authorizedSessions.set(peer, new Set()) + log.withTag('ws').log(`Client connected: ${peer.id}`) + }, + + message(peer, message) { + try { + const data = JSON.parse(message.text()) as GatewayWsClientMessage + if (data.type === 'subscribe' && data.sessionId) { + if (!data.sessionToken || !options.authorizeSessionAccess?.(data.sessionId, data.sessionToken)) + return + + const peerSubscriptions = subscriptions.get(peer) + const peerAuthorizations = authorizedSessions.get(peer) + if (!peerSubscriptions || peerSubscriptions.has(data.sessionId)) + return + + peerSubscriptions.add(data.sessionId) + peerAuthorizations?.add(data.sessionId) + log.withTag('ws').log(`Client ${peer.id} subscribed to session ${data.sessionId}`) + } + else if (data.type === 'unsubscribe' && data.sessionId) { + subscriptions.get(peer)?.delete(data.sessionId) + authorizedSessions.get(peer)?.delete(data.sessionId) + } + else { + if ('sessionId' in data && !authorizedSessions.get(peer)?.has(data.sessionId)) + return + void options.onClientMessage?.(data) + } + } + catch { + // ignore malformed messages + } + }, + + close(peer) { + peers.delete(peer) + subscriptions.delete(peer) + authorizedSessions.delete(peer) + log.withTag('ws').log(`Client disconnected: ${peer.id}`) + }, + + error(peer, err) { + log.withTag('ws').error(`WS error for ${peer.id}: ${err}`) + }, + }) + + function broadcast(sessionId: string, event: string, data: unknown) { + const msg = JSON.stringify({ event, sessionId, data, timestamp: Date.now() }) + for (const [peer, subs] of subscriptions) { + if (subs.has(sessionId) || subs.has('*')) { + try { + peer.send(msg) + } + catch { + // peer may have disconnected + } + } + } + } + + function broadcastAll(event: string, data: unknown) { + const msg = JSON.stringify({ event, sessionId: '*', data, timestamp: Date.now() }) + for (const peer of peers) { + try { + peer.send(msg) + } + catch { + // ignore + } + } + } + + return { handler, broadcast, broadcastAll } +} diff --git a/services/visual-chat-gateway/tsdown.config.ts b/services/visual-chat-gateway/tsdown.config.ts new file mode 100644 index 0000000000..e0cb0b3266 --- /dev/null +++ b/services/visual-chat-gateway/tsdown.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from 'tsdown' + +export default defineConfig({ + entry: { + index: 'src/index.ts', + }, + target: 'node18', + outDir: 'dist', + clean: true, + dts: true, +}) diff --git a/services/visual-chat-worker-minicpmo/.env.example b/services/visual-chat-worker-minicpmo/.env.example new file mode 100644 index 0000000000..3812f1b243 --- /dev/null +++ b/services/visual-chat-worker-minicpmo/.env.example @@ -0,0 +1,14 @@ +# All variables below are optional unless you need to override auto-detection. + +WORKER_PORT=6201 +VISUAL_CHAT_GATEWAY_URL=http://127.0.0.1:6200 +VISUAL_CHAT_WORKER_PROFILE=auto +LOG_LEVEL=info + +# Native MiniCPM-o bridge profile +MINICPMO_NATIVE_GATEWAY_URL=http://127.0.0.1:8006 +MINICPMO_NATIVE_MODEL=openbmb/MiniCPM-o-4_5 + +# 16GB-friendly local lite profile via Ollama +OLLAMA_HOST=http://127.0.0.1:11434 +OLLAMA_MODEL=openbmb/minicpm-v4.5:latest diff --git a/services/visual-chat-worker-minicpmo/package.json b/services/visual-chat-worker-minicpmo/package.json new file mode 100644 index 0000000000..c32e928c48 --- /dev/null +++ b/services/visual-chat-worker-minicpmo/package.json @@ -0,0 +1,28 @@ +{ + "name": "@proj-airi/visual-chat-worker-minicpmo", + "type": "module", + "version": "0.0.1", + "private": true, + "description": "MiniCPM-o 4.5 inference worker for AIRI visual chat", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/moeru-ai/airi.git", + "directory": "services/visual-chat-worker-minicpmo" + }, + "scripts": { + "dev": "tsx --env-file-if-exists=.env --env-file-if-exists=.env.local src/index.ts", + "start": "tsx --env-file-if-exists=.env --env-file-if-exists=.env.local src/index.ts", + "build": "tsdown", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@proj-airi/visual-chat-model-minicpmo": "workspace:^", + "@proj-airi/visual-chat-observability": "workspace:^", + "@proj-airi/visual-chat-protocol": "workspace:^", + "@proj-airi/visual-chat-shared": "workspace:^", + "crossws": "^0.4.3", + "h3": "^2.0.1-rc.11", + "listhen": "^1.9.0" + } +} diff --git a/services/visual-chat-worker-minicpmo/src/env/defaults.ts b/services/visual-chat-worker-minicpmo/src/env/defaults.ts new file mode 100644 index 0000000000..f825d0264c --- /dev/null +++ b/services/visual-chat-worker-minicpmo/src/env/defaults.ts @@ -0,0 +1,6 @@ +export const WORKER_DEFAULT_PORT = 6201 +export const WORKER_DEFAULT_HOST = '127.0.0.1' +export const WORKER_DEFAULT_GATEWAY_URL = '' +export const WORKER_DEFAULT_OLLAMA_BASE_URL = 'http://127.0.0.1:11434' +export const WORKER_DEFAULT_OLLAMA_MODEL = 'openbmb/minicpm-v4.5:latest' +export const WORKER_DEFAULT_LOG_LEVEL = 'info' diff --git a/services/visual-chat-worker-minicpmo/src/env/parse.ts b/services/visual-chat-worker-minicpmo/src/env/parse.ts new file mode 100644 index 0000000000..76c20f6cdd --- /dev/null +++ b/services/visual-chat-worker-minicpmo/src/env/parse.ts @@ -0,0 +1,46 @@ +import { envInt, envString } from '@proj-airi/visual-chat-shared' + +import { + WORKER_DEFAULT_GATEWAY_URL, + WORKER_DEFAULT_HOST, + WORKER_DEFAULT_LOG_LEVEL, + WORKER_DEFAULT_OLLAMA_BASE_URL, + WORKER_DEFAULT_OLLAMA_MODEL, + WORKER_DEFAULT_PORT, +} from './defaults' + +export interface WorkerConfig { + host: string + port: number + gatewayUrl: string + ollamaBaseUrl: string + ollamaModel: string + logLevel: string +} + +export function parseWorkerConfig(): WorkerConfig { + const host = envString('WORKER_HOST', WORKER_DEFAULT_HOST).trim() + const port = envInt('WORKER_PORT', WORKER_DEFAULT_PORT) + const gatewayUrl = envString('VISUAL_CHAT_GATEWAY_URL', WORKER_DEFAULT_GATEWAY_URL).trim() + const ollamaBaseUrl = envString('OLLAMA_HOST', WORKER_DEFAULT_OLLAMA_BASE_URL).trim() + const ollamaModel = envString('OLLAMA_MODEL', WORKER_DEFAULT_OLLAMA_MODEL).trim() + const logLevel = envString('LOG_LEVEL', WORKER_DEFAULT_LOG_LEVEL) + + if (port < 1 || port > 65535) + throw new Error(`Invalid WORKER_PORT: ${port} (expected 1-65535)`) + + if (!URL.canParse(ollamaBaseUrl)) + throw new Error(`Invalid OLLAMA_HOST: not a valid URL (${ollamaBaseUrl})`) + + if (!ollamaModel) + throw new Error('Invalid OLLAMA_MODEL: must be a non-empty string') + + return { + host, + port, + gatewayUrl, + ollamaBaseUrl, + ollamaModel, + logLevel, + } +} diff --git a/services/visual-chat-worker-minicpmo/src/health/heartbeat.ts b/services/visual-chat-worker-minicpmo/src/health/heartbeat.ts new file mode 100644 index 0000000000..66d32d53bc --- /dev/null +++ b/services/visual-chat-worker-minicpmo/src/health/heartbeat.ts @@ -0,0 +1,42 @@ +import { envInt } from '@proj-airi/visual-chat-shared' + +import { WORKER_DEFAULT_PORT } from '../env/defaults' + +const DEFAULT_INTERVAL_MS = 10_000 + +export function startHeartbeat(config: { gatewayUrl: string, intervalMs?: number }): { stop: () => void } { + const intervalMs = config.intervalMs ?? DEFAULT_INTERVAL_MS + const base = config.gatewayUrl.replace(/\/$/, '') + const url = `${base}/api/workers/heartbeat` + const port = envInt('WORKER_PORT', WORKER_DEFAULT_PORT) + const healthUrl = `http://127.0.0.1:${port}/health` + + const tick = async () => { + let payload: Record = { status: 'unknown', workerPort: port } + try { + const res = await fetch(healthUrl) + if (res.ok) + payload = { ...(await res.json() as Record), workerPort: port } + } + catch { + } + try { + await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }) + } + catch { + } + } + + void tick() + const id = setInterval(() => { + void tick() + }, intervalMs) + + return { + stop: () => clearInterval(id), + } +} diff --git a/services/visual-chat-worker-minicpmo/src/health/probe.ts b/services/visual-chat-worker-minicpmo/src/health/probe.ts new file mode 100644 index 0000000000..2e125d6842 --- /dev/null +++ b/services/visual-chat-worker-minicpmo/src/health/probe.ts @@ -0,0 +1,26 @@ +import type { InferenceWorkerLoop } from '../worker-loop' + +import { defineEventHandler, setResponseHeaders } from 'h3' + +export function createHealthProbeHandler(getWorker: () => InferenceWorkerLoop | null) { + return defineEventHandler((event) => { + setResponseHeaders(event, { + 'Access-Control-Allow-Origin': event.headers.get('origin') || '*', + 'Access-Control-Allow-Methods': 'GET, POST, OPTIONS', + 'Access-Control-Allow-Headers': 'Content-Type', + }) + + const worker = getWorker() + if (!worker) { + return { status: 'model_not_ready', ok: false } + } + + const state = worker.getState() + return { + status: state.status, + ok: state.status === 'running', + currentCnt: state.currentCnt, + metrics: state.metrics, + } + }) +} diff --git a/services/visual-chat-worker-minicpmo/src/index.ts b/services/visual-chat-worker-minicpmo/src/index.ts new file mode 100644 index 0000000000..afe4926005 --- /dev/null +++ b/services/visual-chat-worker-minicpmo/src/index.ts @@ -0,0 +1,69 @@ +import process from 'node:process' + +import { createWorkerLogger } from '@proj-airi/visual-chat-observability' +import { VISUAL_CHAT_GATEWAY_TOKEN_HEADER } from '@proj-airi/visual-chat-protocol' +import { plugin as wsPlugin } from 'crossws/server' +import { createApp, defineEventHandler, getHeader, serve, setResponseHeaders, setResponseStatus } from 'h3' + +import { WORKER_DEFAULT_OLLAMA_MODEL } from './env/defaults' +import { parseWorkerConfig } from './env/parse' +import { startHeartbeat } from './health/heartbeat' +import { createOllamaLiteRouter } from './ollama-lite' + +const log = createWorkerLogger() +const config = parseWorkerConfig() + +const app = createApp() + +app.use(defineEventHandler((event) => { + const origin = getHeader(event, 'origin') || '*' + setResponseHeaders(event, { + 'Access-Control-Allow-Origin': origin, + 'Access-Control-Allow-Methods': 'GET, POST, OPTIONS', + 'Access-Control-Allow-Headers': ['Content-Type', VISUAL_CHAT_GATEWAY_TOKEN_HEADER].join(', '), + 'Access-Control-Max-Age': '86400', + }) + + if (event.method === 'OPTIONS') { + setResponseStatus(event, 204) + return '' + } +})) + +async function main() { + const ollamaModel = WORKER_DEFAULT_OLLAMA_MODEL + if (config.ollamaModel !== WORKER_DEFAULT_OLLAMA_MODEL) { + log.withTag('main').warn(`Ignoring unsupported OLLAMA_MODEL=${config.ollamaModel}; using ${WORKER_DEFAULT_OLLAMA_MODEL}.`) + } + + const { router, duplexWsHandler } = createOllamaLiteRouter({ + baseUrl: config.ollamaBaseUrl, + model: ollamaModel, + }) + + app.use(router) + app.use('/ws/duplex', duplexWsHandler as any) + + const server = serve(app, { + // @ts-expect-error h3 attaches `.crossws` metadata to the fetch response for the plugin. + plugins: [wsPlugin({ resolve: async req => (await app.fetch(req)).crossws })], + port: config.port, + hostname: config.host, + reusePort: true, + silent: true, + manual: true, + }) + + await server.serve() + log.withTag('main').log(`Worker bridge listening on ${config.host}:${config.port}`) + log.withTag('main').log(`Worker profile: ollama-lite`) + log.withTag('main').log(`Ollama lite backend: ${config.ollamaBaseUrl} (model: ${ollamaModel})`) + + if (config.gatewayUrl) + startHeartbeat({ gatewayUrl: config.gatewayUrl }) +} + +main().catch((error) => { + log.withTag('main').error(`Worker bridge failed to start: ${error}`) + process.exit(1) +}) diff --git a/services/visual-chat-worker-minicpmo/src/ollama-lite.ts b/services/visual-chat-worker-minicpmo/src/ollama-lite.ts new file mode 100644 index 0000000000..0555275d54 --- /dev/null +++ b/services/visual-chat-worker-minicpmo/src/ollama-lite.ts @@ -0,0 +1,300 @@ +import type { Peer } from 'crossws' + +import { checkOllamaHealth } from '@proj-airi/visual-chat-model-minicpmo' +import { createError, createRouter, defineEventHandler, defineWebSocketHandler, readBody } from 'h3' + +import { sanitizeModelOutputText } from './sanitize-output' + +interface OllamaLiteConfig { + baseUrl: string + model: string +} + +interface InferenceRequestBody { + image?: string + prompt?: string + system?: string + history?: Array<{ + role?: string + content?: string + }> +} + +interface OllamaChatStreamChunk { + message?: { + content?: string + } + done?: boolean + total_duration?: number + eval_count?: number + error?: string +} + +const DEFAULT_PROMPT = 'Describe what you observe in the current scene. Use the latest frame and recent conversation. Be concise and natural.' +const DEFAULT_SYSTEM_PROMPT = 'You are AIRI in a lightweight multimodal session. Use the most recent frame when available, keep context across turns, and answer naturally.' +const NDJSON_CONTENT_TYPE = 'application/x-ndjson; charset=utf-8' +const TRAILING_SLASH_PATTERN = /\/$/ + +function normalizeBaseUrl(baseUrl: string): string { + return baseUrl.replace(TRAILING_SLASH_PATTERN, '') +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} + +function buildMessages(body: InferenceRequestBody) { + const prompt = body.prompt?.trim() || DEFAULT_PROMPT + const userMessage: { + role: 'user' + content: string + images?: string[] + } = { + role: 'user', + content: prompt, + } + + if (body.image) + userMessage.images = [body.image] + + const messages: Array<{ + role: 'system' | 'user' | 'assistant' + content: string + images?: string[] + }> = [ + { + role: 'system', + content: body.system?.trim() || DEFAULT_SYSTEM_PROMPT, + }, + ] + + if (Array.isArray(body.history)) { + for (const message of body.history) { + if ((message?.role === 'user' || message?.role === 'assistant') && message.content?.trim()) { + messages.push({ + role: message.role, + content: message.content.trim(), + }) + } + } + } + + messages.push(userMessage) + return messages +} + +function encodeNdjsonLine(payload: unknown): Uint8Array { + return new TextEncoder().encode(`${JSON.stringify(payload)}\n`) +} + +async function readUpstreamText(response: Response): Promise { + return response.text().catch(() => response.statusText) +} + +async function readOllamaChunk(data: Uint8Array): Promise { + return new TextDecoder().decode(data) +} + +function createInferenceStream( + config: OllamaLiteConfig, + body: InferenceRequestBody, +): ReadableStream { + const model = config.model + + return new ReadableStream({ + async start(controller) { + try { + controller.enqueue(encodeNdjsonLine({ + type: 'start', + model, + })) + + const response = await fetch(`${normalizeBaseUrl(config.baseUrl)}/api/chat`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model, + messages: buildMessages(body), + stream: true, + }), + }) + + if (!response.ok) + throw new Error(await readUpstreamText(response)) + if (!response.body) + throw new Error('Ollama stream did not return a readable body.') + + const reader = response.body.getReader() + let buffer = '' + let rawAccumulatedText = '' + let sanitizedAccumulatedText = '' + + while (true) { + const { done, value } = await reader.read() + if (done) + break + + buffer += await readOllamaChunk(value) + const lines = buffer.split('\n') + buffer = lines.pop() ?? '' + + for (const line of lines) { + const trimmed = line.trim() + if (!trimmed) + continue + + const chunk = JSON.parse(trimmed) as OllamaChatStreamChunk + if (chunk.error) + throw new Error(chunk.error) + + const rawDelta = chunk.message?.content ?? '' + if (rawDelta) { + rawAccumulatedText = `${rawAccumulatedText}${rawDelta}` + const nextSanitizedText = sanitizeModelOutputText(rawAccumulatedText) + const delta = nextSanitizedText.startsWith(sanitizedAccumulatedText) + ? nextSanitizedText.slice(sanitizedAccumulatedText.length) + : nextSanitizedText + + sanitizedAccumulatedText = nextSanitizedText + controller.enqueue(encodeNdjsonLine({ + type: 'delta', + delta, + text: sanitizedAccumulatedText, + model, + })) + } + + if (chunk.done) { + sanitizedAccumulatedText = sanitizeModelOutputText(chunk.message?.content ?? '') || sanitizedAccumulatedText + controller.enqueue(encodeNdjsonLine({ + type: 'done', + text: sanitizedAccumulatedText, + durationMs: chunk.total_duration ? Math.round(chunk.total_duration / 1_000_000) : 0, + evalTokens: chunk.eval_count ?? 0, + model, + })) + } + } + } + + if (buffer.trim()) { + const chunk = JSON.parse(buffer.trim()) as OllamaChatStreamChunk + if (chunk.error) + throw new Error(chunk.error) + + const rawDelta = chunk.message?.content ?? '' + if (rawDelta) { + rawAccumulatedText = `${rawAccumulatedText}${rawDelta}` + const nextSanitizedText = sanitizeModelOutputText(rawAccumulatedText) + const delta = nextSanitizedText.startsWith(sanitizedAccumulatedText) + ? nextSanitizedText.slice(sanitizedAccumulatedText.length) + : nextSanitizedText + + sanitizedAccumulatedText = nextSanitizedText + controller.enqueue(encodeNdjsonLine({ + type: 'delta', + delta, + text: sanitizedAccumulatedText, + model, + })) + } + + sanitizedAccumulatedText = sanitizeModelOutputText(chunk.message?.content ?? '') || sanitizedAccumulatedText + controller.enqueue(encodeNdjsonLine({ + type: 'done', + text: sanitizedAccumulatedText, + durationMs: chunk.total_duration ? Math.round(chunk.total_duration / 1_000_000) : 0, + evalTokens: chunk.eval_count ?? 0, + model, + })) + } + + controller.close() + } + catch (error) { + controller.enqueue(encodeNdjsonLine({ + type: 'error', + error: errorMessage(error), + model, + })) + controller.close() + } + }, + }) +} + +function createUnsupportedDuplexHandler() { + return defineWebSocketHandler({ + open(peer: Peer) { + peer.send(JSON.stringify({ + type: 'error', + error: 'Native full-duplex websocket is not available in ollama-lite mode.', + })) + peer.close() + }, + }) +} + +export function createOllamaLiteRouter(config: OllamaLiteConfig) { + const router = createRouter() + + router.get('/health', defineEventHandler(async () => { + try { + const ok = await checkOllamaHealth(config.baseUrl) + if (!ok) { + return { + ok: false, + status: 'offline', + backendKind: 'ollama', + model: config.model, + upstreamBaseUrl: config.baseUrl, + features: ['vision-stream', 'session-history', 'text-input', 'scene-memory'], + } + } + + return { + ok: true, + status: 'running', + backendKind: 'ollama', + model: config.model, + upstreamBaseUrl: config.baseUrl, + fixedModel: true, + features: ['vision-stream', 'session-history', 'text-input', 'scene-memory'], + } + } + catch (error) { + return { + ok: false, + status: 'offline', + backendKind: 'ollama', + model: config.model, + upstreamBaseUrl: config.baseUrl, + fixedModel: true, + error: errorMessage(error), + features: ['vision-stream', 'session-history', 'text-input', 'scene-memory'], + } + } + })) + + router.post('/infer-stream', defineEventHandler(async (event) => { + const body = await readBody(event) as InferenceRequestBody + return new Response(createInferenceStream(config, body), { + headers: { + 'Content-Type': NDJSON_CONTENT_TYPE, + 'Cache-Control': 'no-store', + }, + }) + })) + + router.all('/ws/duplex', defineEventHandler(() => { + throw createError({ + statusCode: 501, + statusMessage: 'Native full-duplex websocket is not available in ollama-lite mode.', + }) + })) + + return { + router, + duplexWsHandler: createUnsupportedDuplexHandler(), + } +} diff --git a/services/visual-chat-worker-minicpmo/src/resolve-inference.ts b/services/visual-chat-worker-minicpmo/src/resolve-inference.ts new file mode 100644 index 0000000000..429aa80fa4 --- /dev/null +++ b/services/visual-chat-worker-minicpmo/src/resolve-inference.ts @@ -0,0 +1,45 @@ +import process from 'node:process' + +import { execSync } from 'node:child_process' + +export function isOllamaAvailable(): boolean { + try { + execSync('ollama --version', { encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'] }) + return true + } + catch { + return false + } +} + +export async function isOllamaServing(baseUrl = 'http://localhost:11434'): Promise { + try { + const res = await fetch(`${baseUrl}/api/tags`, { signal: AbortSignal.timeout(3000) }) + return res.ok + } + catch { + return false + } +} + +export async function detectOllama(logWarn: (msg: string) => void): Promise<{ + available: boolean + baseUrl: string + model: string +}> { + const baseUrl = process.env.OLLAMA_HOST || 'http://localhost:11434' + const model = process.env.OLLAMA_MODEL || 'openbmb/minicpm-v4.5:latest' + + if (await isOllamaServing(baseUrl)) { + return { available: true, baseUrl, model } + } + + if (isOllamaAvailable()) { + logWarn(`Ollama is installed but not serving at ${baseUrl}. Run: ollama serve`) + } + else { + logWarn('Ollama is not installed. Run: pnpm -F @proj-airi/visual-chat-ops setup-engine') + } + + return { available: false, baseUrl, model } +} diff --git a/services/visual-chat-worker-minicpmo/src/sanitize-output.ts b/services/visual-chat-worker-minicpmo/src/sanitize-output.ts new file mode 100644 index 0000000000..8ac7c5524e --- /dev/null +++ b/services/visual-chat-worker-minicpmo/src/sanitize-output.ts @@ -0,0 +1,14 @@ +const THINKING_BLOCK_PATTERN = /]*>[\s\S]*?<\/think>/gi +const THINKING_OPEN_TAG_PATTERN = /]*>/gi +const THINKING_CLOSE_TAG_PATTERN = /<\/think>/gi +const LEADING_WHITESPACE_PATTERN = /^\s+/ +const EXCESSIVE_BLANK_LINES_PATTERN = /\n{3,}/g + +export function sanitizeModelOutputText(value: string): string { + return value + .replace(THINKING_BLOCK_PATTERN, '') + .replace(THINKING_OPEN_TAG_PATTERN, '') + .replace(THINKING_CLOSE_TAG_PATTERN, '') + .replace(LEADING_WHITESPACE_PATTERN, '') + .replace(EXCESSIVE_BLANK_LINES_PATTERN, '\n\n') +} diff --git a/services/visual-chat-worker-minicpmo/src/worker-loop.ts b/services/visual-chat-worker-minicpmo/src/worker-loop.ts new file mode 100644 index 0000000000..11a6804bb0 --- /dev/null +++ b/services/visual-chat-worker-minicpmo/src/worker-loop.ts @@ -0,0 +1,133 @@ +import type { DecodedChunk, DecodeResult, InferenceBackend, OllamaConfig } from '@proj-airi/visual-chat-model-minicpmo' +import type { InferenceMetrics } from '@proj-airi/visual-chat-observability' + +import { + createBackend, + decodedChunksToDecodeResult, +} from '@proj-airi/visual-chat-model-minicpmo' +import { createInferenceMetrics, createWorkerLogger, recordLatency } from '@proj-airi/visual-chat-observability' +import { INFERENCE_CYCLE_INTERVAL_MS, PREFILL_CNT_START } from '@proj-airi/visual-chat-protocol' + +const log = createWorkerLogger() + +export type InferenceCallback = (result: DecodeResult) => void | Promise + +export interface WorkerState { + status: 'idle' | 'initializing' | 'running' | 'paused' | 'error' + currentCnt: number + sessionId: string | null + metrics: InferenceMetrics +} + +export class InferenceWorkerLoop { + private backend: InferenceBackend & { + writeTempWav: (data: Buffer, name: string) => string + writeTempImage: (data: Buffer, name: string) => string + } + + private intervalHandle: ReturnType | null = null + private state: WorkerState + private onResult: InferenceCallback | null = null + private processing = false + private pendingPrefill: { wavPath: string, imagePath?: string } | null = null + + constructor(config: OllamaConfig) { + this.backend = createBackend(config) as any + this.state = { + status: 'idle', + currentCnt: PREFILL_CNT_START, + sessionId: null, + metrics: createInferenceMetrics(), + } + } + + async start(systemPrompt: string, onResult: InferenceCallback): Promise { + this.onResult = onResult + this.state.status = 'initializing' + + await this.backend.spawn() + await this.backend.init(systemPrompt) + + this.state.status = 'running' + this.state.currentCnt = PREFILL_CNT_START + + this.intervalHandle = setInterval(() => { + this.tick().catch((err) => { + log.withTag('loop').error(`Tick error: ${err}`) + }) + }, INFERENCE_CYCLE_INTERVAL_MS) + + log.withTag('loop').log('Worker loop started') + } + + async stop(): Promise { + if (this.intervalHandle) { + clearInterval(this.intervalHandle) + this.intervalHandle = null + } + await this.backend.shutdown() + this.state.status = 'idle' + log.withTag('loop').log('Worker loop stopped') + } + + submitMedia(wavData: Buffer, imageData?: Buffer): void { + const wavPath = this.backend.writeTempWav(wavData, `input_${Date.now()}.wav`) + let imagePath: string | undefined + + if (imageData) + imagePath = this.backend.writeTempImage(imageData, `input_${Date.now()}.jpg`) + + this.pendingPrefill = { + wavPath, + imagePath, + } + } + + private async tick(): Promise { + if (this.processing || this.state.status !== 'running') + return + + if (!this.pendingPrefill) + return + + this.processing = true + + try { + const payload = this.pendingPrefill + this.pendingPrefill = null + + const prefillStart = Date.now() + await this.backend.prefill(payload.wavPath, payload.imagePath) + const prefillLatency = Date.now() - prefillStart + recordLatency(this.state.metrics, prefillLatency, true, 'prefill') + this.state.currentCnt++ + + const decodeStart = Date.now() + const chunks: DecodedChunk[] = [] + for await (const chunk of this.backend.decode()) + chunks.push(chunk) + + const result = decodedChunksToDecodeResult(chunks) + const decodeLatency = Date.now() - decodeStart + recordLatency(this.state.metrics, decodeLatency, true, 'decode') + + if (result.listenDetected) { + log.withTag('loop').log('Listen signal detected — model wants to hear more input') + } + + if (this.onResult) + await this.onResult(result) + } + catch (err) { + recordLatency(this.state.metrics, 0, false) + log.withTag('loop').error(`Inference error: ${err}`) + } + finally { + this.processing = false + } + } + + getState(): WorkerState { + return { ...this.state } + } +} diff --git a/services/visual-chat-worker-minicpmo/tsdown.config.ts b/services/visual-chat-worker-minicpmo/tsdown.config.ts new file mode 100644 index 0000000000..e0cb0b3266 --- /dev/null +++ b/services/visual-chat-worker-minicpmo/tsdown.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from 'tsdown' + +export default defineConfig({ + entry: { + index: 'src/index.ts', + }, + target: 'node18', + outDir: 'dist', + clean: true, + dts: true, +}) From 78e99c91ecee7db9457c5c6f099192950eebd532 Mon Sep 17 00:00:00 2001 From: Joker-of-Gotham Date: Mon, 6 Apr 2026 05:55:12 +0800 Subject: [PATCH 4/6] feat(visual-chat): add sdk, ops tooling, and bridge plugins Made-with: Cursor --- packages/visual-chat-ops/README.md | 104 +++ packages/visual-chat-ops/package.json | 51 ++ .../visual-chat-ops/src/cli/dev-tamagotchi.ts | 116 ++++ packages/visual-chat-ops/src/cli/doctor.ts | 88 +++ packages/visual-chat-ops/src/cli/install.ts | 46 ++ packages/visual-chat-ops/src/cli/prune.ts | 27 + .../visual-chat-ops/src/cli/pull-models.ts | 135 ++++ .../visual-chat-ops/src/cli/setup-engine.ts | 194 ++++++ .../visual-chat-ops/src/cli/setup-tunnel.ts | 344 ++++++++++ packages/visual-chat-ops/src/cli/share.ts | 590 ++++++++++++++++++ packages/visual-chat-ops/src/cli/shared.ts | 255 ++++++++ packages/visual-chat-ops/src/cli/start.ts | 434 +++++++++++++ packages/visual-chat-ops/src/cli/stop.ts | 71 +++ packages/visual-chat-ops/src/index.ts | 11 + packages/visual-chat-ops/tsdown.config.ts | 11 + packages/visual-chat-sdk/package.json | 37 ++ packages/visual-chat-sdk/src/client/index.ts | 257 ++++++++ .../visual-chat-sdk/src/composables/index.ts | 3 + .../src/composables/use-session-status.ts | 51 ++ .../src/composables/use-source-switch.ts | 63 ++ .../src/composables/use-visual-chat.ts | 126 ++++ packages/visual-chat-sdk/src/index.ts | 7 + .../visual-chat-sdk/src/ws-client/index.ts | 172 +++++ packages/visual-chat-sdk/tsdown.config.ts | 11 + .../plugin-visual-chat-pocket/package.json | 29 + .../plugin-visual-chat-pocket/src/index.ts | 74 +++ .../tsdown.config.ts | 11 + .../package.json | 29 + .../src/index.ts | 80 +++ .../tsdown.config.ts | 11 + plugins/plugin-visual-chat-web/package.json | 29 + plugins/plugin-visual-chat-web/src/index.ts | 89 +++ .../plugin-visual-chat-web/tsdown.config.ts | 11 + 33 files changed, 3567 insertions(+) create mode 100644 packages/visual-chat-ops/README.md create mode 100644 packages/visual-chat-ops/package.json create mode 100644 packages/visual-chat-ops/src/cli/dev-tamagotchi.ts create mode 100644 packages/visual-chat-ops/src/cli/doctor.ts create mode 100644 packages/visual-chat-ops/src/cli/install.ts create mode 100644 packages/visual-chat-ops/src/cli/prune.ts create mode 100644 packages/visual-chat-ops/src/cli/pull-models.ts create mode 100644 packages/visual-chat-ops/src/cli/setup-engine.ts create mode 100644 packages/visual-chat-ops/src/cli/setup-tunnel.ts create mode 100644 packages/visual-chat-ops/src/cli/share.ts create mode 100644 packages/visual-chat-ops/src/cli/shared.ts create mode 100644 packages/visual-chat-ops/src/cli/start.ts create mode 100644 packages/visual-chat-ops/src/cli/stop.ts create mode 100644 packages/visual-chat-ops/src/index.ts create mode 100644 packages/visual-chat-ops/tsdown.config.ts create mode 100644 packages/visual-chat-sdk/package.json create mode 100644 packages/visual-chat-sdk/src/client/index.ts create mode 100644 packages/visual-chat-sdk/src/composables/index.ts create mode 100644 packages/visual-chat-sdk/src/composables/use-session-status.ts create mode 100644 packages/visual-chat-sdk/src/composables/use-source-switch.ts create mode 100644 packages/visual-chat-sdk/src/composables/use-visual-chat.ts create mode 100644 packages/visual-chat-sdk/src/index.ts create mode 100644 packages/visual-chat-sdk/src/ws-client/index.ts create mode 100644 packages/visual-chat-sdk/tsdown.config.ts create mode 100644 plugins/plugin-visual-chat-pocket/package.json create mode 100644 plugins/plugin-visual-chat-pocket/src/index.ts create mode 100644 plugins/plugin-visual-chat-pocket/tsdown.config.ts create mode 100644 plugins/plugin-visual-chat-tamagotchi/package.json create mode 100644 plugins/plugin-visual-chat-tamagotchi/src/index.ts create mode 100644 plugins/plugin-visual-chat-tamagotchi/tsdown.config.ts create mode 100644 plugins/plugin-visual-chat-web/package.json create mode 100644 plugins/plugin-visual-chat-web/src/index.ts create mode 100644 plugins/plugin-visual-chat-web/tsdown.config.ts diff --git a/packages/visual-chat-ops/README.md b/packages/visual-chat-ops/README.md new file mode 100644 index 0000000000..0bbc042132 --- /dev/null +++ b/packages/visual-chat-ops/README.md @@ -0,0 +1,104 @@ +# `@proj-airi/visual-chat-ops` + +CLI helpers for AIRI visual chat environment checks, local bridge lifecycle, and remote phone sharing. + +## What It Does + +- checks whether the local Ollama backend is reachable +- checks whether the fixed MiniCPM-V 4.5 model is installed +- starts the local AIRI gateway and MiniCPM-o worker bridge +- generates desktop and phone entry hints for LAN or remote access +- surfaces secure-context warnings early +- exposes remote HTTPS/WSS phone entry URLs without requiring a separate tunnel binary + +## When To Use It + +- you want the 16GB-friendly local path with Ollama and MiniCPM-V 4.5 +- you want to run AIRI's local realtime gateway and worker bridge from one command +- you need quick diagnostics before opening the Visual Chat settings page +- you want the fixed AIRI pipeline with persisted conversation records and rolling scene memory + +## When Not To Use It + +- you expect this package to install or run the official MiniCPM-o PyTorch backend for you +- you want llama.cpp-style backends +- you need phone camera access from an insecure `http://` frontend origin + +## Main Commands + +```bash +pnpm -F @proj-airi/visual-chat-ops doctor:visual-chat +pnpm -F @proj-airi/visual-chat-ops start:local +pnpm dev:tamagotchi +``` + +## Important Environment Variables + +```bash +OLLAMA_HOST=http://127.0.0.1:11434 +OLLAMA_MODEL=openbmb/minicpm-v4.5:latest +AIRI_VISUAL_CHAT_FRONTEND_URL=https://your-airi-web-host.example.com +AIRI_VISUAL_CHAT_START_FRONTEND=auto +VISUAL_CHAT_PORT=6200 +WORKER_PORT=6201 +``` + +## Startup Flow + +`start:local` now does the following in order: + +1. checks the local Ollama backend and fixed model +2. starts or reuses the AIRI worker bridge +3. starts or reuses the AIRI gateway +4. starts the local `apps/stage-web` dev server when no frontend is already reachable +5. prints desktop settings and phone page URLs +6. warns when the frontend/gateway combination will break phone media capture or mixed-content rules + +## Tamagotchi Remote Dev + +For the Electron desktop route, the default root command now starts: + +- `apps/stage-tamagotchi` +- the visual chat gateway +- the visual chat worker +- a remote phone sharing bridge that writes public frontend and gateway URLs into AIRI diagnostics + +Use: + +```bash +pnpm dev:tamagotchi +``` + +This command keeps the desktop `Phone entry` field pointed at a phone-reachable URL: + +- on the same network, the phone can still use the LAN URL +- on different networks, AIRI upgrades to a public HTTPS/WSS pair automatically + +If you only want the local-only dev flow, use: + +```bash +pnpm dev:tamagotchi:local +``` + +## 16GB Path + +For a simpler local setup that stays within a 16GB-class GPU budget, use: + +```bash +pnpm -F @proj-airi/visual-chat-ops setup-engine +pnpm -F @proj-airi/visual-chat-ops pull-models --model openbmb/minicpm-v4.5:latest +pnpm -F @proj-airi/visual-chat-ops start:local +``` + +This path keeps: + +- continuous desktop and phone camera or screen streaming through the AIRI gateway +- session history, rolling scene memory, and source switching in AIRI +- typed text input and persisted conversation records + +## Phone Capture Notes + +- Phone camera capture typically requires an HTTPS frontend origin. +- If the phone page is served over HTTPS, the gateway should also be exposed through HTTPS/WSS or a same-origin reverse proxy. +- Desktop camera and screen capture are handled by the AIRI frontend. +- Set `AIRI_VISUAL_CHAT_START_FRONTEND=0` if you want to manage the web frontend separately. diff --git a/packages/visual-chat-ops/package.json b/packages/visual-chat-ops/package.json new file mode 100644 index 0000000000..79c5bbd4dc --- /dev/null +++ b/packages/visual-chat-ops/package.json @@ -0,0 +1,51 @@ +{ + "name": "@proj-airi/visual-chat-ops", + "type": "module", + "version": "0.0.1", + "private": true, + "description": "CLI tools for environment detection, model management, and service lifecycle for AIRI visual chat", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/moeru-ai/airi.git", + "directory": "packages/visual-chat-ops" + }, + "exports": { + ".": { + "types": "./dist/index.d.mts", + "default": "./dist/index.mjs" + } + }, + "main": "./dist/index.mjs", + "types": "./dist/index.d.mts", + "files": [ + "README.md", + "dist", + "package.json" + ], + "scripts": { + "dev": "pnpm run build", + "build": "tsdown", + "typecheck": "tsc --noEmit", + "dev:tamagotchi": "tsx src/cli/dev-tamagotchi.ts", + "doctor": "tsx src/cli/doctor.ts", + "doctor:visual-chat": "tsx src/cli/doctor.ts", + "install:deps": "tsx src/cli/install.ts", + "setup-engine": "tsx src/cli/setup-engine.ts", + "pull-models": "tsx src/cli/pull-models.ts", + "setup:visual-chat": "tsx src/cli/setup-engine.ts && tsx src/cli/pull-models.ts", + "start:local": "tsx src/cli/start.ts", + "share": "tsx src/cli/share.ts", + "share:web": "tsx src/cli/share.ts --frontend-target http://127.0.0.1:5173 --gateway-target http://127.0.0.1:6200", + "share:tamagotchi": "tsx src/cli/share.ts --frontend-target http://127.0.0.1:5174 --gateway-target http://127.0.0.1:6200", + "tunnel:setup": "tsx src/cli/setup-tunnel.ts", + "tunnel:run": "tsx src/cli/setup-tunnel.ts run", + "stop": "tsx src/cli/stop.ts", + "prune": "tsx src/cli/prune.ts" + }, + "dependencies": { + "@proj-airi/visual-chat-observability": "workspace:^", + "@proj-airi/visual-chat-shared": "workspace:^", + "@proj-airi/visual-chat-storage": "workspace:^" + } +} diff --git a/packages/visual-chat-ops/src/cli/dev-tamagotchi.ts b/packages/visual-chat-ops/src/cli/dev-tamagotchi.ts new file mode 100644 index 0000000000..fe3d424131 --- /dev/null +++ b/packages/visual-chat-ops/src/cli/dev-tamagotchi.ts @@ -0,0 +1,116 @@ +import process from 'node:process' + +import { spawn } from 'node:child_process' +import { resolve } from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' + +import { stop } from './stop' + +const CURRENT_DIR = fileURLToPath(new URL('.', import.meta.url)) +const ROOT_DIR = resolve(CURRENT_DIR, '..', '..', '..', '..') + +type SpawnCommand = [command: string, args: string[]] + +function packageManagerCommand(args: string[]): SpawnCommand { + if (process.platform === 'win32') { + const command = process.env.ComSpec || 'cmd.exe' + return [command, ['/d', '/s', '/c', ['pnpm', ...args].join(' ')]] + } + + return ['pnpm', args] +} + +function spawnWorkspaceCommand(args: string[]) { + const [command, commandArgs] = packageManagerCommand(args) + return spawn(command, commandArgs, { + cwd: ROOT_DIR, + env: { + ...process.env, + }, + stdio: 'inherit', + }) +} + +export async function devTamagotchi() { + await stop() + + console.info('=== Starting AIRI Tamagotchi Visual Chat Dev ===\n') + console.info('This flow starts the Electron renderer, gateway, worker, and remote phone sharing.') + console.info() + + const devChild = spawnWorkspaceCommand([ + '-r', + '-F', + '@proj-airi/stage-tamagotchi', + '-F', + '@proj-airi/visual-chat-gateway', + '-F', + '@proj-airi/visual-chat-worker-minicpmo', + '--parallel', + 'dev', + ]) + + const shareChild = spawnWorkspaceCommand([ + '-F', + '@proj-airi/visual-chat-ops', + 'share:tamagotchi', + ]) + + let shuttingDown = false + const shutdown = (exitCode: number) => { + if (shuttingDown) + return + + shuttingDown = true + + if (!shareChild.killed) + shareChild.kill() + if (!devChild.killed) + devChild.kill() + + process.exit(exitCode) + } + + process.on('SIGINT', () => shutdown(0)) + process.on('SIGTERM', () => shutdown(0)) + + devChild.once('error', (error) => { + console.error(error) + shutdown(1) + }) + devChild.once('exit', (code, signal) => { + if (shuttingDown) + return + + const detail = signal ? `signal ${signal}` : `code ${code ?? 0}` + console.error(`[fatal] Tamagotchi dev exited unexpectedly with ${detail}.`) + shutdown(code ?? 1) + }) + + shareChild.once('error', (error) => { + console.error(error) + shutdown(1) + }) + shareChild.once('exit', (code, signal) => { + if (shuttingDown) + return + + const detail = signal ? `signal ${signal}` : `code ${code ?? 0}` + console.error(`[fatal] Remote phone sharing exited unexpectedly with ${detail}.`) + shutdown(code ?? 1) + }) + + await new Promise(() => {}) +} + +function isDirectExecution(): boolean { + const entryPath = process.argv[1] + return !!entryPath && pathToFileURL(entryPath).href === import.meta.url +} + +if (isDirectExecution()) { + void devTamagotchi().catch((error) => { + console.error(error) + process.exit(1) + }) +} diff --git a/packages/visual-chat-ops/src/cli/doctor.ts b/packages/visual-chat-ops/src/cli/doctor.ts new file mode 100644 index 0000000000..79ace22b4e --- /dev/null +++ b/packages/visual-chat-ops/src/cli/doctor.ts @@ -0,0 +1,88 @@ +import type { CheckResult } from './shared' + +import process from 'node:process' + +import { execSync } from 'node:child_process' +import { pathToFileURL } from 'node:url' + +import { + checkOllamaHealth, + checkOllamaModel, + DEFAULT_OLLAMA_BASE_URL, + DEFAULT_OLLAMA_MODEL, +} from './shared' + +function checkCommand(name: string, cmd: string, required: boolean = true): CheckResult { + try { + const output = execSync(cmd, { encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'] }).trim() + return { name, ok: true, detail: output.split('\n')[0], required } + } + catch { + return { name, ok: false, detail: 'not found', required } + } +} + +export async function doctor() { + console.info('=== AIRI Visual Chat Environment Doctor ===\n') + + const syncChecks: CheckResult[] = [ + checkCommand('Node.js', 'node --version', true), + checkCommand('pnpm', 'pnpm --version', true), + checkCommand('git', 'git --version', true), + checkCommand('GPU (nvidia-smi)', 'nvidia-smi --query-gpu=name --format=csv,noheader', false), + { + name: 'Fixed pipeline', + ok: true, + detail: 'ollama-lite', + required: true, + }, + { + name: 'Fixed model target', + ok: true, + detail: `${DEFAULT_OLLAMA_MODEL} via ${DEFAULT_OLLAMA_BASE_URL}`, + required: true, + }, + ] + + const asyncChecks = await Promise.all([ + checkOllamaHealth(), + checkOllamaModel(), + ]) + + const checks = [...syncChecks, ...asyncChecks] + let hasRequiredFailure = false + + for (const check of checks) { + const icon = check.ok ? '[OK]' : check.required ? '[!!]' : '[--]' + const suffix = !check.ok && !check.required ? ' (optional)' : '' + console.info(` ${icon} ${check.name}: ${check.detail}${suffix}`) + if (!check.ok && check.required) + hasRequiredFailure = true + } + + console.info() + + if (hasRequiredFailure) { + console.info('Some required checks failed. Resolve the fixed Ollama Visual Chat path first.') + console.info() + console.info('Required local path:') + console.info(` Ollama at ${DEFAULT_OLLAMA_BASE_URL}`) + console.info(` Model: ${DEFAULT_OLLAMA_MODEL}`) + console.info() + console.info('Then start AIRI bridge services with:') + console.info(' pnpm -F @proj-airi/visual-chat-ops start:local') + process.exitCode = 1 + } + else { + console.info('All checks passed. AIRI Visual Chat can start on the fixed local pipeline.') + } +} + +function isDirectExecution(): boolean { + const entryPath = process.argv[1] + return !!entryPath && pathToFileURL(entryPath).href === import.meta.url +} + +if (isDirectExecution()) { + void doctor() +} diff --git a/packages/visual-chat-ops/src/cli/install.ts b/packages/visual-chat-ops/src/cli/install.ts new file mode 100644 index 0000000000..e1943559f1 --- /dev/null +++ b/packages/visual-chat-ops/src/cli/install.ts @@ -0,0 +1,46 @@ +import process from 'node:process' + +import { execSync } from 'node:child_process' +import { existsSync, mkdirSync } from 'node:fs' + +import { getVisualChatDir } from '@proj-airi/visual-chat-shared' + +export async function install() { + console.log('=== AIRI Visual Chat Dependency Installer ===\n') + + const dirs = ['config', 'data', 'cache', 'logs', 'models'] as const + for (const kind of dirs) { + const dir = getVisualChatDir(kind) + if (!existsSync(dir)) { + mkdirSync(dir, { recursive: true }) + console.log(` Created: ${dir}`) + } + else { + console.log(` Exists: ${dir}`) + } + } + + console.log('\n Installing pnpm dependencies...') + try { + execSync('pnpm install', { stdio: 'inherit' }) + } + catch { + console.error(' pnpm install failed.') + process.exitCode = 1 + return + } + + console.log('\n Building packages...') + try { + execSync('pnpm run build:packages', { stdio: 'inherit' }) + } + catch { + console.error(' Build failed.') + process.exitCode = 1 + return + } + + console.log('\nInstallation complete.') +} + +install() diff --git a/packages/visual-chat-ops/src/cli/prune.ts b/packages/visual-chat-ops/src/cli/prune.ts new file mode 100644 index 0000000000..feae601512 --- /dev/null +++ b/packages/visual-chat-ops/src/cli/prune.ts @@ -0,0 +1,27 @@ +import { getVisualChatDir } from '@proj-airi/visual-chat-shared' +import { DEFAULT_RETENTION, getDirectorySizeMb, pruneWithPolicy } from '@proj-airi/visual-chat-storage' + +export async function prune() { + console.log('=== AIRI Visual Chat Pruner ===\n') + + const cacheDir = getVisualChatDir('cache') + const logsDir = getVisualChatDir('logs') + const dataDir = getVisualChatDir('data') + + const dirs = [ + { name: 'Cache', path: cacheDir }, + { name: 'Logs', path: logsDir }, + { name: 'Data', path: dataDir }, + ] + + for (const { name, path } of dirs) { + const sizeBefore = await getDirectorySizeMb(path) + const { byAge, bySize } = await pruneWithPolicy(path, DEFAULT_RETENTION) + const sizeAfter = await getDirectorySizeMb(path) + console.log(` ${name}: removed ${byAge} by age, ${bySize} by size (${sizeBefore.toFixed(1)}MB -> ${sizeAfter.toFixed(1)}MB)`) + } + + console.log('\nPrune complete.') +} + +prune() diff --git a/packages/visual-chat-ops/src/cli/pull-models.ts b/packages/visual-chat-ops/src/cli/pull-models.ts new file mode 100644 index 0000000000..86e343c6c0 --- /dev/null +++ b/packages/visual-chat-ops/src/cli/pull-models.ts @@ -0,0 +1,135 @@ +import process from 'node:process' + +import { execSync, spawn as nodeSpawn } from 'node:child_process' +import { pathToFileURL } from 'node:url' + +const DEFAULT_MODEL = 'openbmb/minicpm-v4.5:latest' +const OLLAMA_BASE_URL = 'http://localhost:11434' + +export interface PullModelsOptions { + model?: string + force?: boolean + listOnly?: boolean +} + +function has(cmd: string): string | null { + try { + return execSync(cmd, { encoding: 'utf-8', timeout: 10000, stdio: ['pipe', 'pipe', 'pipe'] }).trim().split('\n')[0] + } + catch { + return null + } +} + +async function isOllamaServing(): Promise { + const baseUrl = process.env.OLLAMA_HOST || OLLAMA_BASE_URL + try { + const res = await fetch(`${baseUrl}/api/tags`, { signal: AbortSignal.timeout(3000) }) + return res.ok + } + catch { + return false + } +} + +function spawnStreamed(cmd: string, args: string[], cwd: string): Promise { + return new Promise((resolve) => { + const child = nodeSpawn(cmd, args, { cwd, stdio: 'inherit' }) + child.on('close', code => resolve(code ?? 1)) + child.on('error', () => resolve(1)) + }) +} + +async function listModels(): Promise { + const baseUrl = process.env.OLLAMA_HOST || OLLAMA_BASE_URL + try { + const res = await fetch(`${baseUrl}/api/tags`, { signal: AbortSignal.timeout(5000) }) + if (!res.ok) + return [] + + const data = await res.json() as { models?: Array<{ name: string }> } + return data.models?.map(model => model.name) ?? [] + } + catch { + return [] + } +} + +export async function pullModels(options: PullModelsOptions = {}) { + const model = options.model + || process.argv.find((_, index, argv) => argv[index - 1] === '--model') + || DEFAULT_MODEL + const force = options.force || process.argv.includes('--force') + const listOnly = options.listOnly || process.argv.includes('--list') + + if (listOnly) { + console.info('Available models on Ollama:') + console.info(` Default: ${DEFAULT_MODEL}`) + console.info() + console.info('Usage:') + console.info(' pnpm -F @proj-airi/visual-chat-ops pull-models') + console.info(` pnpm -F @proj-airi/visual-chat-ops pull-models --model ${DEFAULT_MODEL}`) + console.info() + console.info('Alternative MiniCPM-compatible models:') + console.info(' openbmb/minicpm-v4.5:latest MiniCPM-V 4.5 vision-language') + console.info(' openbmb/minicpm-o2.6 MiniCPM-o 2.6 multimodal chat') + console.info(' qwen3-vl:8b Qwen3 VL 8B vision') + console.info(' llava:13b LLaVA 13B vision') + return + } + + console.info('=== AIRI Visual Chat Pull Model ===\n') + console.info(` Model: ${model}\n`) + + if (!has('ollama --version')) { + console.info(' Ollama not found. Run setup first:') + console.info(' pnpm -F @proj-airi/visual-chat-ops setup-engine') + process.exitCode = 1 + return + } + + if (!await isOllamaServing()) { + console.info(' Ollama is not running. Start it with:') + console.info(' ollama serve') + console.info() + console.info(' Or run:') + console.info(' pnpm -F @proj-airi/visual-chat-ops setup-engine') + process.exitCode = 1 + return + } + + const existing = await listModels() + const alreadyPulled = existing.some(item => item.startsWith(model.split(':')[0])) + if (alreadyPulled && !force) { + console.info(` Model already available: ${model}`) + console.info(' Use --force to re-pull.') + console.info('\n Ready! Start with:') + console.info(' pnpm -F @proj-airi/visual-chat-ops start:local') + return + } + + console.info(` Pulling ${model} via Ollama...`) + console.info(' Ollama handles download, quantization, and GPU optimization automatically.\n') + + const code = await spawnStreamed('ollama', ['pull', model], process.cwd()) + + if (code !== 0) { + console.info('\n Pull failed. Check your network connection and try again.') + console.info(` Manual: ollama pull ${model}`) + process.exitCode = 1 + return + } + + console.info('\n=== Model ready ===') + console.info(` Model: ${model}`) + console.info(' Start: pnpm -F @proj-airi/visual-chat-ops start:local\n') +} + +function isDirectExecution(): boolean { + const entryPath = process.argv[1] + return !!entryPath && pathToFileURL(entryPath).href === import.meta.url +} + +if (isDirectExecution()) { + void pullModels() +} diff --git a/packages/visual-chat-ops/src/cli/setup-engine.ts b/packages/visual-chat-ops/src/cli/setup-engine.ts new file mode 100644 index 0000000000..6e0003c8dd --- /dev/null +++ b/packages/visual-chat-ops/src/cli/setup-engine.ts @@ -0,0 +1,194 @@ +import process from 'node:process' + +import { execSync, spawn as nodeSpawn } from 'node:child_process' +import { existsSync } from 'node:fs' +import { platform } from 'node:os' +import { join } from 'node:path' +import { pathToFileURL } from 'node:url' + +const OLLAMA_DEFAULT_URL = 'http://localhost:11434' +const OLLAMA_MODEL = 'openbmb/minicpm-v4.5:latest' +const QUOTE_PATTERN = /"/g + +function has(cmd: string): string | null { + try { + return execSync(cmd, { encoding: 'utf-8', timeout: 10000, stdio: ['pipe', 'pipe', 'pipe'] }).trim().split('\n')[0] + } + catch { + return null + } +} + +function spawnStreamed(cmd: string, args: string[], cwd: string): Promise { + return new Promise((resolve) => { + const child = nodeSpawn(cmd, args, { cwd, stdio: 'inherit' }) + child.on('close', code => resolve(code ?? 1)) + child.on('error', () => resolve(1)) + }) +} + +function findOllama(): string | null { + if (has('ollama --version')) + return 'ollama' + + if (platform() === 'win32') { + const candidates = [ + join(process.env.LOCALAPPDATA || '', 'Programs', 'Ollama', 'ollama.exe'), + join(process.env.ProgramFiles || 'C:\\Program Files', 'Ollama', 'ollama.exe'), + ] + for (const candidate of candidates) { + if (existsSync(candidate)) + return `"${candidate}"` + } + } + else if (platform() === 'darwin') { + const candidates = [ + '/opt/homebrew/bin/ollama', + '/usr/local/bin/ollama', + ] + for (const candidate of candidates) { + if (existsSync(candidate)) + return candidate + } + } + + return null +} + +async function installOllama(): Promise { + const os = platform() + console.info('\n Installing Ollama...\n') + + if (os === 'win32' && has('winget --version')) { + const code = await spawnStreamed('winget', [ + 'install', + 'Ollama.Ollama', + '--accept-package-agreements', + '--accept-source-agreements', + ], process.cwd()) + if (code === 0) + return findOllama() + } + else if (os === 'darwin' && has('brew --version')) { + const code = await spawnStreamed('brew', ['install', 'ollama'], process.cwd()) + if (code === 0) + return findOllama() + } + else if (os !== 'win32' && os !== 'darwin') { + const code = await spawnStreamed('sh', ['-c', 'curl -fsSL https://ollama.com/install.sh | sh'], process.cwd()) + if (code === 0) + return findOllama() + } + + return null +} + +async function isOllamaServing(baseUrl: string): Promise { + try { + const res = await fetch(`${baseUrl}/api/tags`, { signal: AbortSignal.timeout(3000) }) + return res.ok + } + catch { + return false + } +} + +async function startOllamaServe(ollamaCmd: string): Promise { + console.info(' Starting Ollama service...') + + const child = nodeSpawn(ollamaCmd, ['serve'], { + stdio: 'ignore', + detached: true, + }) + child.unref() + + for (let index = 0; index < 15; index++) { + await new Promise(resolve => setTimeout(resolve, 2000)) + if (await isOllamaServing(OLLAMA_DEFAULT_URL)) { + console.info(' Ollama service is running.') + return true + } + } + + return false +} + +async function checkGpuInfo(): Promise { + const nvidiaGpu = has('nvidia-smi') !== null + const isMac = platform() === 'darwin' + + if (nvidiaGpu) + return 'CUDA (auto-detected by Ollama)' + if (isMac) + return 'Metal (auto-detected by Ollama)' + return 'CPU' +} + +export async function setupEngine() { + console.info('=== AIRI Visual Chat Inference Engine Setup ===\n') + + const os = platform() + const gpuLabel = await checkGpuInfo() + console.info(` Platform: ${os} (${process.arch})`) + console.info(` GPU backend: ${gpuLabel}`) + console.info(` Model: ${OLLAMA_MODEL}\n`) + + let ollamaCmd = findOllama() + if (ollamaCmd) { + console.info(` [OK] Ollama found: ${ollamaCmd}`) + } + else { + console.info(' [--] Ollama not found, installing...') + ollamaCmd = await installOllama() + + if (!ollamaCmd) { + console.info('\n Auto-install failed. Please install manually:') + if (os === 'win32') + console.info(' winget install Ollama.Ollama') + else if (os === 'darwin') + console.info(' brew install ollama') + else + console.info(' curl -fsSL https://ollama.com/install.sh | sh') + + console.info(' Then re-run: pnpm -F @proj-airi/visual-chat-ops setup-engine') + process.exitCode = 1 + return + } + + console.info(` [OK] Ollama installed: ${ollamaCmd}`) + } + + const baseUrl = process.env.OLLAMA_HOST || OLLAMA_DEFAULT_URL + const serving = await isOllamaServing(baseUrl) + + if (serving) { + console.info(` [OK] Ollama is serving at ${baseUrl}`) + } + else { + console.info(` [--] Ollama is not serving at ${baseUrl}`) + const started = await startOllamaServe(ollamaCmd.replace(QUOTE_PATTERN, '')) + if (!started) { + console.info('\n Could not start Ollama. Please start it manually:') + console.info(' ollama serve') + console.info(' Then re-run this command.') + process.exitCode = 1 + return + } + } + + console.info('\n=== Setup complete ===') + console.info(` Ollama: ${ollamaCmd}`) + console.info(` GPU: ${gpuLabel}`) + console.info(` API: ${baseUrl}`) + console.info(`\n Next: pnpm -F @proj-airi/visual-chat-ops pull-models --model ${OLLAMA_MODEL}`) + console.info(' Then: pnpm -F @proj-airi/visual-chat-ops start:local\n') +} + +function isDirectExecution(): boolean { + const entryPath = process.argv[1] + return !!entryPath && pathToFileURL(entryPath).href === import.meta.url +} + +if (isDirectExecution()) { + void setupEngine() +} diff --git a/packages/visual-chat-ops/src/cli/setup-tunnel.ts b/packages/visual-chat-ops/src/cli/setup-tunnel.ts new file mode 100644 index 0000000000..8b0dbe6d99 --- /dev/null +++ b/packages/visual-chat-ops/src/cli/setup-tunnel.ts @@ -0,0 +1,344 @@ +import type { Buffer } from 'node:buffer' + +import process from 'node:process' + +import { execFileSync, spawn } from 'node:child_process' +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { pathToFileURL } from 'node:url' + +import { getVisualChatDir } from '@proj-airi/visual-chat-shared' + +import { ensureCloudflaredBinary } from './share.js' + +const TUNNEL_CONFIG_FILE = process.env.AIRI_VISUAL_CHAT_TUNNEL_CONFIG_FILE?.trim() + || join(getVisualChatDir('config'), 'tunnel.json') +const DEFAULT_ENDPOINTS_FILE = process.env.AIRI_VISUAL_CHAT_PUBLIC_ENDPOINTS_FILE?.trim() + || join(getVisualChatDir('config'), 'public-endpoints.json') +const TUNNEL_RUN_TIMEOUT_MS = 45_000 +const TUNNEL_NAME_FRONTEND = 'airi-visual-chat-frontend' +const TUNNEL_NAME_GATEWAY = 'airi-visual-chat-gateway' +const TUNNEL_ID_PATTERN = /([a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12})/i +const CREATED_TUNNEL_ID_PATTERN = /Created tunnel\s+\S+\s+with id\s+([a-f0-9-]+)/i + +interface TunnelConfig { + frontendTunnelId: string + frontendTunnelName: string + frontendPublicUrl: string + gatewayTunnelId: string + gatewayTunnelName: string + gatewayPublicUrl: string + createdAt: string +} + +interface TunnelInfo { + id: string + name: string +} + +function loadTunnelConfig(): TunnelConfig | null { + if (!existsSync(TUNNEL_CONFIG_FILE)) + return null + try { + return JSON.parse(readFileSync(TUNNEL_CONFIG_FILE, 'utf8')) as TunnelConfig + } + catch { + return null + } +} + +function saveTunnelConfig(config: TunnelConfig): void { + mkdirSync(dirname(TUNNEL_CONFIG_FILE), { recursive: true }) + writeFileSync(TUNNEL_CONFIG_FILE, `${JSON.stringify(config, null, 2)}\n`, 'utf8') +} + +function writePublicEndpointsFile(frontendUrl: string, gatewayUrl: string): void { + mkdirSync(dirname(DEFAULT_ENDPOINTS_FILE), { recursive: true }) + writeFileSync(DEFAULT_ENDPOINTS_FILE, `${JSON.stringify({ + frontendUrl, + gatewayUrl, + source: 'cloudflared-named-tunnel', + updatedAt: new Date().toISOString(), + }, null, 2)}\n`, 'utf8') +} + +function runSync(binary: string, args: string[]): string { + return execFileSync(binary, args, { encoding: 'utf8', timeout: 30_000 }).trim() +} + +async function isLoggedIn(binary: string): Promise { + try { + execFileSync(binary, ['tunnel', 'list', '--output', 'json'], { + encoding: 'utf8', + timeout: 15_000, + stdio: ['ignore', 'pipe', 'pipe'], + }) + return true + } + catch { + return false + } +} + +async function login(binary: string): Promise { + console.info('\n=== Cloudflare Login ===') + console.info('A browser window will open. Please log in and authorize cloudflared.\n') + + const exitCode = await new Promise((resolve) => { + const child = spawn(binary, ['login'], { + stdio: 'inherit', + }) + child.once('error', () => resolve(1)) + child.once('exit', code => resolve(code ?? 1)) + }) + + if (exitCode !== 0) + throw new Error('cloudflared login failed. Please try again.') + + console.info('Login successful.\n') +} + +function listTunnels(binary: string): TunnelInfo[] { + try { + const output = execFileSync(binary, ['tunnel', 'list', '--output', 'json'], { + encoding: 'utf8', + timeout: 15_000, + stdio: ['ignore', 'pipe', 'pipe'], + }) + const tunnels = JSON.parse(output) as Array<{ id: string, name: string, deleted_at?: string }> + return tunnels + .filter(t => !t.deleted_at) + .map(t => ({ id: t.id, name: t.name })) + } + catch { + return [] + } +} + +function findTunnel(binary: string, name: string): TunnelInfo | null { + const tunnels = listTunnels(binary) + return tunnels.find(t => t.name === name) ?? null +} + +function createTunnel(binary: string, name: string): TunnelInfo { + console.info(`Creating tunnel "${name}"...`) + const output = runSync(binary, ['tunnel', 'create', name]) + const idMatch = output.match(CREATED_TUNNEL_ID_PATTERN) + || output.match(TUNNEL_ID_PATTERN) + + if (!idMatch) + throw new Error(`Failed to parse tunnel ID from output:\n${output}`) + + const id = idMatch[1]! + console.info(` Tunnel "${name}" created with ID: ${id}`) + return { id, name } +} + +function ensureTunnel(binary: string, name: string): TunnelInfo { + const existing = findTunnel(binary, name) + if (existing) { + console.info(`Tunnel "${name}" already exists (ID: ${existing.id})`) + return existing + } + return createTunnel(binary, name) +} + +function buildTunnelPublicUrl(tunnelId: string): string { + return `https://${tunnelId}.cfargotunnel.com` +} + +interface RunningTunnel { + name: string + url: string + close: () => void +} + +async function runNamedTunnel( + binary: string, + tunnelName: string, + localTarget: string, +): Promise { + return new Promise((resolve, reject) => { + let settled = false + let recentOutput = '' + + const child = spawn(binary, [ + 'tunnel', + 'run', + '--url', + localTarget, + '--protocol', + 'http2', + tunnelName, + ], { + stdio: ['ignore', 'pipe', 'pipe'], + windowsHide: true, + }) + + const timeout = setTimeout(() => { + if (!settled) { + settled = true + resolve({ + name: tunnelName, + url: '', + close: () => child.kill(), + }) + } + }, TUNNEL_RUN_TIMEOUT_MS) + + function handleChunk(chunk: Buffer | string) { + const text = chunk.toString() + recentOutput = `${recentOutput}${text}`.slice(-4000) + + if (!settled && (text.includes('Registered tunnel connection') || text.includes('Connection registered'))) { + settled = true + clearTimeout(timeout) + resolve({ + name: tunnelName, + url: '', + close: () => child.kill(), + }) + } + } + + child.stdout?.on('data', handleChunk) + child.stderr?.on('data', handleChunk) + child.once('error', (err) => { + if (!settled) { + settled = true + clearTimeout(timeout) + reject(new Error(`Tunnel "${tunnelName}" failed: ${err.message}`)) + } + }) + child.once('exit', (code, signal) => { + if (!settled) { + settled = true + clearTimeout(timeout) + const detail = signal ? `signal ${signal}` : `code ${code ?? 1}` + reject(new Error(`Tunnel "${tunnelName}" exited before ready (${detail}).\nLast output:\n${recentOutput.slice(-500)}`)) + } + }) + }) +} + +export async function setupTunnel(): Promise { + const existing = loadTunnelConfig() + if (existing) { + console.info('Named tunnel configuration already exists:') + console.info(` Frontend: ${existing.frontendPublicUrl}`) + console.info(` Gateway: ${existing.gatewayPublicUrl}`) + console.info(` Config: ${TUNNEL_CONFIG_FILE}`) + console.info('\nTo recreate, delete .visual-chat-tunnel.json and run again.') + return existing + } + + const binary = await ensureCloudflaredBinary() + + if (!await isLoggedIn(binary)) + await login(binary) + + const frontendTunnel = ensureTunnel(binary, TUNNEL_NAME_FRONTEND) + const gatewayTunnel = ensureTunnel(binary, TUNNEL_NAME_GATEWAY) + + const config: TunnelConfig = { + frontendTunnelId: frontendTunnel.id, + frontendTunnelName: frontendTunnel.name, + frontendPublicUrl: buildTunnelPublicUrl(frontendTunnel.id), + gatewayTunnelId: gatewayTunnel.id, + gatewayTunnelName: gatewayTunnel.name, + gatewayPublicUrl: buildTunnelPublicUrl(gatewayTunnel.id), + createdAt: new Date().toISOString(), + } + + saveTunnelConfig(config) + + console.info('\n=== Named Tunnels Created ===') + console.info(` Frontend URL: ${config.frontendPublicUrl}`) + console.info(` Gateway URL: ${config.gatewayPublicUrl}`) + console.info(` Config saved: ${TUNNEL_CONFIG_FILE}`) + console.info('\nThese URLs are permanent. Run "pnpm -F @proj-airi/visual-chat-ops share" to start the tunnels.') + + return config +} + +export interface NamedTunnelPair { + frontendUrl: string + gatewayUrl: string + close: () => void +} + +export async function startNamedTunnels(options?: { + frontendTarget?: string + gatewayTarget?: string +}): Promise { + const config = loadTunnelConfig() + if (!config) + throw new Error('No named tunnel configuration found. Run setup-tunnel first.') + + const frontendTarget = options?.frontendTarget || 'http://127.0.0.1:5174' + const gatewayTarget = options?.gatewayTarget || 'http://127.0.0.1:6200' + const binary = await ensureCloudflaredBinary() + + console.info('Starting named tunnels...') + console.info(` Frontend: ${config.frontendTunnelName} → ${frontendTarget}`) + console.info(` Gateway: ${config.gatewayTunnelName} → ${gatewayTarget}`) + + const frontendHandle = await runNamedTunnel(binary, config.frontendTunnelName, frontendTarget) + let gatewayHandle: RunningTunnel + + try { + gatewayHandle = await runNamedTunnel(binary, config.gatewayTunnelName, gatewayTarget) + } + catch (error) { + frontendHandle.close() + throw error + } + + writePublicEndpointsFile(config.frontendPublicUrl, config.gatewayPublicUrl) + + console.info('\nNamed tunnels are running:') + console.info(` Frontend: ${config.frontendPublicUrl}`) + console.info(` Gateway: ${config.gatewayPublicUrl}`) + + return { + frontendUrl: config.frontendPublicUrl, + gatewayUrl: config.gatewayPublicUrl, + close() { + frontendHandle.close() + gatewayHandle.close() + }, + } +} + +export { loadTunnelConfig, TUNNEL_CONFIG_FILE } + +function isDirectExecution(): boolean { + const entryPath = process.argv[1] + return !!entryPath && pathToFileURL(entryPath).href === import.meta.url +} + +if (isDirectExecution()) { + const command = process.argv[2] + + if (command === 'run') { + startNamedTunnels().then((pair) => { + process.on('SIGINT', () => { + pair.close() + process.exit(0) + }) + process.on('SIGTERM', () => { + pair.close() + process.exit(0) + }) + }).catch((error) => { + console.error(error) + process.exit(1) + }) + } + else { + setupTunnel().catch((error) => { + console.error(error) + process.exit(1) + }) + } +} diff --git a/packages/visual-chat-ops/src/cli/share.ts b/packages/visual-chat-ops/src/cli/share.ts new file mode 100644 index 0000000000..2d0f1c1328 --- /dev/null +++ b/packages/visual-chat-ops/src/cli/share.ts @@ -0,0 +1,590 @@ +import process from 'node:process' + +import { execFileSync, spawn } from 'node:child_process' +import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs' +import { dirname, join, resolve } from 'node:path' +import { pathToFileURL } from 'node:url' + +import { getVisualChatDir } from '@proj-airi/visual-chat-shared' + +const DEFAULT_ENDPOINTS_FILE = process.env.AIRI_VISUAL_CHAT_PUBLIC_ENDPOINTS_FILE?.trim() + || join(getVisualChatDir('config'), 'public-endpoints.json') +const CLOUDFLARED_CACHE_DIR = join(getVisualChatDir('cache'), 'cloudflared') +const QUICK_TUNNEL_PATTERN = /https:\/\/(?!api\.)[-a-z0-9]+\.trycloudflare\.com/i +const DOWNLOAD_TIMEOUT_MS = 120_000 +const STARTUP_TIMEOUT_MS = 90_000 +const HEALTH_POLL_INTERVAL_MS = 1_000 +const QUICK_TUNNEL_MAX_ATTEMPTS = 3 + +interface ShareConfig { + frontendTarget: string + gatewayTarget: string + frontendUrl?: string + gatewayUrl?: string + endpointsFile: string + clearOnly: boolean +} + +interface ShareHandle { + name: string + url: string + close: () => void + onError: (callback: (error: Error) => void) => void +} + +interface CloudflaredDownloadSpec { + downloadUrl: string + cacheKey: string + executableName: string + archiveName?: string +} + +function sanitizeUrl(value: string | undefined): string | undefined { + const normalized = value?.trim() + if (!normalized) + return undefined + + return new URL(normalized).toString() +} + +function parseArgValue(args: string[], name: string): string | undefined { + const inline = args.find(arg => arg.startsWith(`${name}=`)) + if (inline) + return inline.slice(name.length + 1) + + const index = args.indexOf(name) + if (index < 0) + return undefined + + const value = args[index + 1] + if (!value || value.startsWith('--')) + return undefined + + return value +} + +function hasFlag(args: string[], name: string): boolean { + return args.includes(name) +} + +function parseConfig(argv: string[]): ShareConfig { + return { + frontendTarget: parseArgValue(argv, '--frontend-target')?.trim() || 'http://127.0.0.1:5173', + gatewayTarget: parseArgValue(argv, '--gateway-target')?.trim() || 'http://127.0.0.1:6200', + frontendUrl: parseArgValue(argv, '--frontend-url')?.trim(), + gatewayUrl: parseArgValue(argv, '--gateway-url')?.trim(), + endpointsFile: parseArgValue(argv, '--endpoints-file')?.trim() || DEFAULT_ENDPOINTS_FILE, + clearOnly: hasFlag(argv, '--clear'), + } +} + +function writePublicEndpointsFile(endpointsFile: string, payload: { frontendUrl: string, gatewayUrl: string, source: string }) { + mkdirSync(dirname(endpointsFile), { recursive: true }) + writeFileSync(endpointsFile, `${JSON.stringify({ + ...payload, + updatedAt: new Date().toISOString(), + }, null, 2)}\n`, 'utf8') +} + +function removePublicEndpointsFile(endpointsFile: string) { + if (existsSync(endpointsFile)) + rmSync(endpointsFile, { force: true }) +} + +async function isUrlReachable(url: string, timeoutMs: number = 1500): Promise { + try { + const response = await fetch(url, { + signal: AbortSignal.timeout(timeoutMs), + redirect: 'follow', + }) + + return response.status < 500 + } + catch { + return false + } +} + +function sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)) +} + +async function waitForTarget(name: string, targetUrl: string, timeoutMs: number = STARTUP_TIMEOUT_MS) { + const startedAt = Date.now() + + while (Date.now() - startedAt < timeoutMs) { + if (await isUrlReachable(targetUrl)) + return + + await sleep(HEALTH_POLL_INTERVAL_MS) + } + + throw new Error(`${name} target did not become reachable in time: ${targetUrl}`) +} + +async function commandExists(command: string): Promise { + const args = process.platform === 'win32' + ? ['/d', '/s', '/c', `where ${command}`] + : ['-lc', `command -v ${command}`] + const executable = process.platform === 'win32' + ? (process.env.ComSpec || 'cmd.exe') + : '/bin/sh' + + return new Promise((resolve) => { + const child = spawn(executable, args, { + stdio: 'ignore', + windowsHide: true, + }) + + child.once('error', () => resolve(false)) + child.once('exit', code => resolve(code === 0)) + }) +} + +async function runCommand(command: string, args: string[]): Promise { + return new Promise((resolve) => { + const child = spawn(command, args, { + cwd: process.cwd(), + env: { + ...process.env, + }, + stdio: 'inherit', + windowsHide: true, + }) + + child.once('error', () => resolve(false)) + child.once('exit', code => resolve(code === 0)) + }) +} + +function findInstalledCloudflaredBinary(): string | null { + if (process.platform !== 'win32') + return null + + const candidates = [ + 'C:\\Program Files\\cloudflared\\cloudflared.exe', + 'C:\\Program Files (x86)\\cloudflared\\cloudflared.exe', + `${process.env.LOCALAPPDATA || ''}\\Microsoft\\WinGet\\Links\\cloudflared.exe`, + ] + + for (const candidate of candidates) { + if (candidate && existsSync(candidate)) + return candidate + } + + return null +} + +function resolveCloudflaredDownloadSpec(): CloudflaredDownloadSpec { + const platform = process.platform + const arch = process.arch + + if (platform === 'win32' && arch === 'x64') { + return { + downloadUrl: 'https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-windows-amd64.exe', + cacheKey: 'win32-x64', + executableName: 'cloudflared.exe', + } + } + + if (platform === 'win32' && arch === 'ia32') { + return { + downloadUrl: 'https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-windows-386.exe', + cacheKey: 'win32-ia32', + executableName: 'cloudflared.exe', + } + } + + if (platform === 'win32' && arch === 'arm64') { + return { + downloadUrl: 'https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-windows-arm64.exe', + cacheKey: 'win32-arm64', + executableName: 'cloudflared.exe', + } + } + + if (platform === 'linux' && arch === 'x64') { + return { + downloadUrl: 'https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64', + cacheKey: 'linux-x64', + executableName: 'cloudflared', + } + } + + if (platform === 'linux' && arch === 'arm64') { + return { + downloadUrl: 'https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-arm64', + cacheKey: 'linux-arm64', + executableName: 'cloudflared', + } + } + + if (platform === 'linux' && arch === 'arm') { + return { + downloadUrl: 'https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-arm', + cacheKey: 'linux-arm', + executableName: 'cloudflared', + } + } + + if (platform === 'darwin' && arch === 'x64') { + return { + downloadUrl: 'https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-darwin-amd64.tgz', + cacheKey: 'darwin-x64', + executableName: 'cloudflared', + archiveName: 'cloudflared.tgz', + } + } + + if (platform === 'darwin' && arch === 'arm64') { + return { + downloadUrl: 'https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-darwin-arm64.tgz', + cacheKey: 'darwin-arm64', + executableName: 'cloudflared', + archiveName: 'cloudflared.tgz', + } + } + + throw new Error(`Unsupported platform for automatic cloudflared setup: ${platform}/${arch}`) +} + +async function downloadFile(url: string, destinationPath: string) { + const response = await fetch(url, { + redirect: 'follow', + signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS), + }) + + if (!response.ok) + throw new Error(`Failed to download ${url} (HTTP ${response.status})`) + + const bytes = new Uint8Array(await response.arrayBuffer()) + mkdirSync(dirname(destinationPath), { recursive: true }) + writeFileSync(destinationPath, bytes) +} + +function ensureExecutableMode(filePath: string) { + if (process.platform !== 'win32') + execFileSync('chmod', ['755', filePath], { stdio: 'ignore' }) +} + +async function ensureCloudflaredBinary(): Promise { + const envBinary = process.env.AIRI_VISUAL_CHAT_CLOUDFLARED_BIN?.trim() + if (envBinary) { + if (!existsSync(envBinary)) + throw new Error(`AIRI_VISUAL_CHAT_CLOUDFLARED_BIN points to a missing file: ${envBinary}`) + return envBinary + } + + if (await commandExists('cloudflared')) + return 'cloudflared' + + const installedBinary = findInstalledCloudflaredBinary() + if (installedBinary) + return installedBinary + + if (process.platform === 'win32' && await commandExists('winget')) { + console.info('Installing cloudflared with winget...') + const installed = await runCommand('winget', [ + 'install', + '--id', + 'Cloudflare.cloudflared', + '--exact', + '--source', + 'winget', + '--accept-package-agreements', + '--accept-source-agreements', + '--disable-interactivity', + ]) + + if (installed) { + if (await commandExists('cloudflared')) + return 'cloudflared' + + const wingetBinary = findInstalledCloudflaredBinary() + if (wingetBinary) + return wingetBinary + } + } + + const spec = resolveCloudflaredDownloadSpec() + const cacheDir = resolve(CLOUDFLARED_CACHE_DIR, spec.cacheKey) + const executablePath = resolve(cacheDir, spec.executableName) + if (existsSync(executablePath)) { + ensureExecutableMode(executablePath) + return executablePath + } + + mkdirSync(cacheDir, { recursive: true }) + console.info(`Downloading cloudflared for ${spec.cacheKey}...`) + + if (spec.archiveName) { + const archivePath = resolve(cacheDir, spec.archiveName) + await downloadFile(spec.downloadUrl, archivePath) + execFileSync('tar', ['-xzf', archivePath, '-C', cacheDir], { stdio: 'ignore' }) + rmSync(archivePath, { force: true }) + } + else { + await downloadFile(spec.downloadUrl, executablePath) + } + + if (!existsSync(executablePath)) + throw new Error(`cloudflared download completed, but ${executablePath} was not created.`) + + ensureExecutableMode(executablePath) + return executablePath +} + +async function waitForQuickTunnelUrl(binaryPath: string, targetName: string, targetUrl: string): Promise { + return new Promise((resolve, reject) => { + let settled = false + let recentOutput = '' + let timeout: ReturnType + const child = spawn(binaryPath, ['tunnel', '--url', targetUrl, '--no-autoupdate', '--protocol', 'http2'], { + cwd: process.cwd(), + env: { + ...process.env, + }, + stdio: ['ignore', 'pipe', 'pipe'], + windowsHide: true, + }) + + const settleResolve = (handle: ShareHandle) => { + if (settled) + return + + settled = true + clearTimeout(timeout) + resolve(handle) + } + + const settleReject = (error: Error) => { + if (settled) + return + + settled = true + clearTimeout(timeout) + reject(error) + } + + const appendOutput = (text: string) => { + recentOutput = `${recentOutput}${text}`.slice(-4000) + } + + const withRecentOutput = (summary: string) => { + const details = recentOutput.trim() + return details ? `${summary}\nLast cloudflared output:\n${details}` : summary + } + + timeout = setTimeout(() => { + child.kill() + settleReject(new Error(withRecentOutput(`${targetName} quick tunnel did not produce a public URL in time.`))) + }, 30_000) + + function handleChunk(chunk: Uint8Array | string) { + const text = chunk.toString() + appendOutput(text) + const match = text.match(QUICK_TUNNEL_PATTERN) + if (!match) + return + + settleResolve({ + name: targetName, + url: match[0], + close: () => child.kill(), + onError(callback) { + child.on('error', callback) + child.on('exit', (code, signal) => { + callback(new Error(`${targetName} quick tunnel exited (${signal ? `signal ${signal}` : `code ${code ?? 0}`}).`)) + }) + }, + }) + } + + child.stdout?.on('data', handleChunk) + child.stderr?.on('data', handleChunk) + child.once('error', (error) => { + settleReject(new Error(withRecentOutput(`${targetName} quick tunnel failed to start: ${error.message}`))) + }) + child.once('exit', (code, signal) => { + const summary = signal + ? `${targetName} quick tunnel exited before becoming ready (signal ${signal}).` + : `${targetName} quick tunnel exited before becoming ready (code ${code ?? 1}).` + settleReject(new Error(withRecentOutput(summary))) + }) + }) +} + +async function waitForQuickTunnelUrlWithRetry(binaryPath: string, targetName: string, targetUrl: string): Promise { + let lastError: Error | undefined + + for (let attempt = 1; attempt <= QUICK_TUNNEL_MAX_ATTEMPTS; attempt += 1) { + try { + return await waitForQuickTunnelUrl(binaryPath, targetName, targetUrl) + } + catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)) + if (attempt >= QUICK_TUNNEL_MAX_ATTEMPTS) + break + + console.warn(`${targetName} quick tunnel attempt ${attempt} failed. Retrying...`) + await sleep(attempt * 2_000) + } + } + + throw lastError ?? new Error(`${targetName} quick tunnel failed without an error message.`) +} + +export interface TunnelPair { + frontendUrl: string + gatewayUrl: string + endpointsFile: string + close: () => void +} + +export async function createTunnelPair(options: { + frontendTarget?: string + gatewayTarget?: string + endpointsFile?: string +}): Promise { + const frontendTarget = options.frontendTarget || 'http://127.0.0.1:5173' + const gatewayTarget = options.gatewayTarget || 'http://127.0.0.1:6200' + const endpointsFile = options.endpointsFile || DEFAULT_ENDPOINTS_FILE + + await Promise.all([ + waitForTarget('frontend', frontendTarget), + waitForTarget('gateway', gatewayTarget), + ]) + + const cloudflaredBinary = await ensureCloudflaredBinary() + const frontendTunnel = await waitForQuickTunnelUrlWithRetry(cloudflaredBinary, 'frontend', frontendTarget) + let gatewayTunnel: ShareHandle + + try { + gatewayTunnel = await waitForQuickTunnelUrlWithRetry(cloudflaredBinary, 'gateway', gatewayTarget) + } + catch (error) { + frontendTunnel.close() + throw error + } + + writePublicEndpointsFile(endpointsFile, { + frontendUrl: frontendTunnel.url, + gatewayUrl: gatewayTunnel.url, + source: 'cloudflared', + }) + + return { + frontendUrl: frontendTunnel.url, + gatewayUrl: gatewayTunnel.url, + endpointsFile, + close() { + frontendTunnel.close() + gatewayTunnel.close() + removePublicEndpointsFile(endpointsFile) + }, + } +} + +export { ensureCloudflaredBinary, removePublicEndpointsFile, writePublicEndpointsFile } + +export async function share() { + const config = parseConfig(process.argv.slice(2)) + + if (config.clearOnly) { + removePublicEndpointsFile(config.endpointsFile) + console.info(`Cleared Visual Chat public endpoints file: ${config.endpointsFile}`) + return + } + + if (config.frontendUrl || config.gatewayUrl) { + const frontendUrl = sanitizeUrl(config.frontendUrl) + const gatewayUrl = sanitizeUrl(config.gatewayUrl) + if (!frontendUrl || !gatewayUrl) + throw new Error('When using manual public URLs, both --frontend-url and --gateway-url must be valid absolute URLs.') + + writePublicEndpointsFile(config.endpointsFile, { + frontendUrl, + gatewayUrl, + source: 'manual', + }) + + console.info('Registered Visual Chat public endpoints:') + console.info(` Frontend: ${frontendUrl}`) + console.info(` Gateway: ${gatewayUrl}`) + console.info(` File: ${config.endpointsFile}`) + return + } + + removePublicEndpointsFile(config.endpointsFile) + + console.info('Waiting for Visual Chat local targets...') + console.info(` Frontend target: ${config.frontendTarget}`) + console.info(` Gateway target: ${config.gatewayTarget}`) + + await Promise.all([ + waitForTarget('frontend', config.frontendTarget), + waitForTarget('gateway', config.gatewayTarget), + ]) + + const cloudflaredBinary = await ensureCloudflaredBinary() + const frontendTunnel = await waitForQuickTunnelUrlWithRetry(cloudflaredBinary, 'frontend', config.frontendTarget) + let gatewayTunnel: ShareHandle | undefined + + try { + gatewayTunnel = await waitForQuickTunnelUrlWithRetry(cloudflaredBinary, 'gateway', config.gatewayTarget) + } + catch (error) { + frontendTunnel.close() + throw error + } + + writePublicEndpointsFile(config.endpointsFile, { + frontendUrl: frontendTunnel.url, + gatewayUrl: gatewayTunnel.url, + source: 'cloudflared', + }) + + console.info('Visual Chat public endpoints are ready via cloudflared:') + console.info(` Frontend: ${frontendTunnel.url}`) + console.info(` Gateway: ${gatewayTunnel.url}`) + console.info(` File: ${config.endpointsFile}`) + console.info('Keep this command running while remote phones are connected.') + + let shuttingDown = false + const shutdown = (exitCode: number) => { + if (shuttingDown) + return + + shuttingDown = true + frontendTunnel.close() + gatewayTunnel.close() + removePublicEndpointsFile(config.endpointsFile) + process.exit(exitCode) + } + + frontendTunnel.onError((error) => { + console.error(`[fatal] frontend share failed: ${error.message}`) + shutdown(1) + }) + gatewayTunnel.onError((error) => { + console.error(`[fatal] gateway share failed: ${error.message}`) + shutdown(1) + }) + + process.on('SIGINT', () => shutdown(0)) + process.on('SIGTERM', () => shutdown(0)) + + await new Promise(() => {}) +} + +function isDirectExecution(): boolean { + const entryPath = process.argv[1] + return !!entryPath && pathToFileURL(entryPath).href === import.meta.url +} + +if (isDirectExecution()) { + void share().catch((error) => { + console.error(error) + process.exit(1) + }) +} diff --git a/packages/visual-chat-ops/src/cli/shared.ts b/packages/visual-chat-ops/src/cli/shared.ts new file mode 100644 index 0000000000..91f6b4578b --- /dev/null +++ b/packages/visual-chat-ops/src/cli/shared.ts @@ -0,0 +1,255 @@ +import process from 'node:process' + +import { networkInterfaces } from 'node:os' + +export interface CheckResult { + name: string + ok: boolean + detail: string + required: boolean +} + +export const DEFAULT_OLLAMA_BASE_URL = process.env.OLLAMA_HOST || 'http://127.0.0.1:11434' +export const DEFAULT_OLLAMA_MODEL = 'openbmb/minicpm-v4.5:latest' +export const DEFAULT_GATEWAY_PORT = Number(process.env.VISUAL_CHAT_PORT || 6200) +export const DEFAULT_WORKER_PORT = Number(process.env.WORKER_PORT || 6201) +export const DEFAULT_FRONTEND_CANDIDATES = [ + 'http://127.0.0.1:5173', + 'http://localhost:5173', + 'http://127.0.0.1:4173', + 'http://localhost:4173', +] + +const TRAILING_SLASH_PATTERN = /\/$/ +const IPV4_SEGMENT_PATTERN = /^\d{1,3}$/ +const VIRTUAL_INTERFACE_PATTERN = /hyper-v|wsl|vethernet|vmware|virtualbox|docker|podman|zerotier|tailscale|clash|tun|tap|vpn|loopback|bluetooth|hamachi/i +const WIFI_INTERFACE_PATTERN = /wi-?fi|wlan|wireless/i +const ETHERNET_INTERFACE_PATTERN = /ethernet|\u4EE5\u592A\u7F51|^en\d|^eth\d/i + +export function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} + +export function normalizeBaseUrl(value: string): string { + return value.replace(TRAILING_SLASH_PATTERN, '') +} + +export function isLoopbackHost(hostname: string): boolean { + const normalized = hostname.trim().toLowerCase() + return normalized === 'localhost' || normalized === '127.0.0.1' || normalized === '::1' +} + +export function rewriteUrlHost(baseUrl: string, host: string): string { + const url = new URL(baseUrl) + url.hostname = host + return url.toString() +} + +export function buildSettingsUrl(frontendBaseUrl: string): string { + return new URL('/settings/modules/visual-chat', `${normalizeBaseUrl(frontendBaseUrl)}/`).toString() +} + +export function buildPhoneEntryUrl(frontendBaseUrl: string, gatewayBaseUrl: string): string { + const url = new URL('/visual-chat/phone', `${normalizeBaseUrl(frontendBaseUrl)}/`) + url.searchParams.set('gateway', normalizeBaseUrl(gatewayBaseUrl)) + return url.toString() +} + +export function getLanIpv4Addresses(): string[] { + function parseIpv4(address: string): number[] | null { + const segments = address.trim().split('.') + if (segments.length !== 4) + return null + + const numbers = segments.map((segment) => { + if (!IPV4_SEGMENT_PATTERN.test(segment)) + return Number.NaN + return Number(segment) + }) + + if (numbers.some(segment => Number.isNaN(segment) || segment < 0 || segment > 255)) + return null + + return numbers + } + + function scoreCandidate(candidate: { address: string, interfaceName?: string }): number { + const segments = parseIpv4(candidate.address) + if (!segments) + return Number.NEGATIVE_INFINITY + + const isLoopback = segments[0] === 127 + const isLinkLocal = segments[0] === 169 && segments[1] === 254 + const isBenchmark = segments[0] === 198 && (segments[1] === 18 || segments[1] === 19) + if (isLoopback || isLinkLocal || isBenchmark) + return Number.NEGATIVE_INFINITY + + if (candidate.interfaceName && VIRTUAL_INTERFACE_PATTERN.test(candidate.interfaceName)) + return Number.NEGATIVE_INFINITY + + let score = 0 + const isPrivate = segments[0] === 10 + || (segments[0] === 172 && segments[1] >= 16 && segments[1] <= 31) + || (segments[0] === 192 && segments[1] === 168) + const isCarrierGradeNat = segments[0] === 100 && segments[1] >= 64 && segments[1] <= 127 + + if (isPrivate) + score += 100 + else + score -= 40 + + if (isCarrierGradeNat) + score -= 20 + + const interfaceName = candidate.interfaceName?.trim() ?? '' + if (WIFI_INTERFACE_PATTERN.test(interfaceName)) + score += 40 + else if (ETHERNET_INTERFACE_PATTERN.test(interfaceName)) + score += 25 + + return score + } + + const candidates: Array<{ address: string, interfaceName?: string }> = [] + for (const [interfaceName, entries] of Object.entries(networkInterfaces())) { + for (const entry of (entries ?? [])) { + const networkEntry = entry as { + family?: string | number + internal?: boolean + address?: string + } + const family = typeof networkEntry.family === 'string' + ? networkEntry.family + : networkEntry.family === 4 ? 'IPv4' : 'IPv6' + if (networkEntry.internal || family !== 'IPv4' || !networkEntry.address) + continue + + candidates.push({ + address: networkEntry.address.trim(), + interfaceName, + }) + } + } + + const deduped = new Map() + for (const candidate of candidates) { + const score = scoreCandidate(candidate) + if (!Number.isFinite(score)) + continue + + const existing = deduped.get(candidate.address) + if (!existing || scoreCandidate(existing) < score) + deduped.set(candidate.address, candidate) + } + + return [...deduped.values()] + .sort((left, right) => { + const scoreDelta = scoreCandidate(right) - scoreCandidate(left) + if (scoreDelta !== 0) + return scoreDelta + + return left.address.localeCompare(right.address) + }) + .map(candidate => candidate.address) +} + +export async function isUrlReachable(url: string, timeoutMs: number = 1500): Promise { + try { + const response = await fetch(url, { + signal: AbortSignal.timeout(timeoutMs), + redirect: 'follow', + }) + + return response.status < 500 + } + catch { + return false + } +} + +export async function checkOllamaHealth( + baseUrl: string = DEFAULT_OLLAMA_BASE_URL, +): Promise { + try { + const response = await fetch(`${normalizeBaseUrl(baseUrl)}/api/tags`, { + signal: AbortSignal.timeout(3000), + }) + + if (!response.ok) + return { name: 'Ollama', ok: false, detail: `unhealthy (HTTP ${response.status}) at ${baseUrl}`, required: true } + + return { + name: 'Ollama', + ok: true, + detail: `serving at ${baseUrl}`, + required: true, + } + } + catch { + return { + name: 'Ollama', + ok: false, + detail: `not reachable at ${baseUrl} - run \`ollama serve\` or \`pnpm -F @proj-airi/visual-chat-ops setup-engine\` first`, + required: true, + } + } +} + +export async function checkOllamaModel( + baseUrl: string = DEFAULT_OLLAMA_BASE_URL, + model: string = DEFAULT_OLLAMA_MODEL, +): Promise { + try { + const response = await fetch(`${normalizeBaseUrl(baseUrl)}/api/tags`, { + signal: AbortSignal.timeout(5000), + }) + + if (!response.ok) + return { name: 'Ollama model', ok: false, detail: `cannot read installed models (HTTP ${response.status})`, required: true } + + const data = await response.json() as { models?: Array<{ name?: string }> } + const installed = data.models?.map(item => item.name).filter(Boolean) as string[] | undefined + if (!installed?.includes(model)) { + return { + name: 'Ollama model', + ok: false, + detail: `${model} is not installed - run \`pnpm -F @proj-airi/visual-chat-ops pull-models --model ${model}\``, + required: true, + } + } + + return { + name: 'Ollama model', + ok: true, + detail: `${model} is installed`, + required: true, + } + } + catch { + return { + name: 'Ollama model', + ok: false, + detail: 'cannot query installed models', + required: true, + } + } +} + +export function buildPhoneCaptureWarnings(frontendBaseUrl: string, gatewayBaseUrl: string): string[] { + try { + const frontendUrl = new URL(frontendBaseUrl) + const gatewayUrl = new URL(gatewayBaseUrl) + const warnings: string[] = [] + + if (!isLoopbackHost(frontendUrl.hostname) && frontendUrl.protocol !== 'https:') + warnings.push('Phone camera capture usually requires an HTTPS frontend origin on remote devices.') + + if (frontendUrl.protocol === 'https:' && gatewayUrl.protocol !== 'https:') + warnings.push('An HTTPS phone page cannot reliably talk to an HTTP/WS gateway because browsers block mixed content. Expose the gateway through the same HTTPS origin or via HTTPS/WSS.') + + return warnings + } + catch { + return [] + } +} diff --git a/packages/visual-chat-ops/src/cli/start.ts b/packages/visual-chat-ops/src/cli/start.ts new file mode 100644 index 0000000000..7bb260e9a0 --- /dev/null +++ b/packages/visual-chat-ops/src/cli/start.ts @@ -0,0 +1,434 @@ +import type { ChildProcess } from 'node:child_process' + +import process from 'node:process' + +import { spawn } from 'node:child_process' +import { join, resolve } from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' + +import { + buildPhoneCaptureWarnings, + buildPhoneEntryUrl, + buildSettingsUrl, + checkOllamaHealth, + checkOllamaModel, + DEFAULT_FRONTEND_CANDIDATES, + DEFAULT_GATEWAY_PORT, + DEFAULT_OLLAMA_BASE_URL, + DEFAULT_OLLAMA_MODEL, + DEFAULT_WORKER_PORT, + getLanIpv4Addresses, + isLoopbackHost, + isUrlReachable, + normalizeBaseUrl, + rewriteUrlHost, +} from './shared' + +interface ManagedService { + name: string + child: ChildProcess | null + healthUrl: string + reused: boolean +} + +interface WorkerHealthResponse { + ok?: boolean + model?: string + backendKind?: string + upstreamBaseUrl?: string +} + +interface FrontendHint { + desktopUrl: string + phoneUrl?: string + warnings: string[] +} + +const STARTUP_TIMEOUT_MS = 20_000 +const STARTUP_POLL_INTERVAL_MS = 500 +const DEFAULT_FRONTEND_URL = process.env.AIRI_VISUAL_CHAT_FRONTEND_URL?.trim() || '' +const DEFAULT_AUTO_START_FRONTEND = (process.env.AIRI_VISUAL_CHAT_START_FRONTEND || 'auto').trim().toLowerCase() +const CURRENT_DIR = fileURLToPath(new URL('.', import.meta.url)) + +type SpawnCommand = [command: string, args: string[]] + +function packageManagerCommand(args: string[]): SpawnCommand { + if (process.platform === 'win32') { + const command = process.env.ComSpec || 'cmd.exe' + return [command, ['/d', '/s', '/c', ['pnpm', ...args].join(' ')]] + } + + return ['pnpm', args] +} + +function sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)) +} + +async function isServiceHealthy(healthUrl: string): Promise { + try { + const response = await fetch(healthUrl, { + signal: AbortSignal.timeout(1500), + }) + + return response.ok + } + catch { + return false + } +} + +async function waitForServiceHealth(name: string, healthUrl: string, timeoutMs: number = STARTUP_TIMEOUT_MS): Promise { + const startedAt = Date.now() + + while (Date.now() - startedAt < timeoutMs) { + if (await isServiceHealthy(healthUrl)) + return + await sleep(STARTUP_POLL_INTERVAL_MS) + } + + throw new Error(`${name} did not become healthy within ${Math.round(timeoutMs / 1000)}s (${healthUrl})`) +} + +async function readWorkerHealth(healthUrl: string): Promise { + try { + const response = await fetch(healthUrl, { + signal: AbortSignal.timeout(3000), + }) + + if (!response.ok) + return null + + return response.json() as Promise + } + catch { + return null + } +} + +function attachUnexpectedExitHandler( + service: ManagedService, + shutdown: (code: number) => Promise, + isShuttingDown: () => boolean, +) { + service.child?.once('exit', (code, signal) => { + if (isShuttingDown()) + return + + const detail = signal ? `signal ${signal}` : `code ${code ?? 0}` + console.error(`[fatal] ${service.name} exited unexpectedly with ${detail}.`) + void shutdown(code ?? 1) + }) +} + +function spawnService(cwd: string, env: NodeJS.ProcessEnv, packageManagerArgs: string[]): ChildProcess { + const [command, args] = packageManagerCommand(packageManagerArgs) + + return spawn(command, args, { + cwd, + env, + stdio: 'inherit', + }) +} + +async function runWorkspaceCommand(options: { + cwd: string + env: NodeJS.ProcessEnv + packageManagerArgs: string[] + label: string +}): Promise { + const [command, args] = packageManagerCommand(options.packageManagerArgs) + + await new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd: options.cwd, + env: options.env, + stdio: 'inherit', + }) + + child.once('error', reject) + child.once('exit', (code, signal) => { + if (code === 0) { + resolve() + return + } + + reject(new Error(`${options.label} failed with ${signal ? `signal ${signal}` : `code ${code ?? 1}`}`)) + }) + }) +} + +async function ensureService(options: { + name: string + cwd: string + healthUrl: string + env: NodeJS.ProcessEnv + packageManagerArgs: string[] +}): Promise { + if (await isServiceHealthy(options.healthUrl)) { + console.info(`[reuse] ${options.name} is already healthy at ${options.healthUrl}`) + return { + name: options.name, + child: null, + healthUrl: options.healthUrl, + reused: true, + } + } + + console.info(`[start] ${options.name}...`) + const child = spawnService(options.cwd, options.env, options.packageManagerArgs) + await waitForServiceHealth(options.name, options.healthUrl) + + console.info(`[ready] ${options.name} is healthy at ${options.healthUrl}`) + return { + name: options.name, + child, + healthUrl: options.healthUrl, + reused: false, + } +} + +async function resolveFrontendHints(gatewayBaseUrl: string): Promise<{ + reachableFrontendUrl: string | null + checkedFrontendUrls: string[] + hints: FrontendHint[] +}> { + const checkedFrontendUrls: string[] = [] + const candidateBaseUrls = DEFAULT_FRONTEND_URL + ? [DEFAULT_FRONTEND_URL] + : DEFAULT_FRONTEND_CANDIDATES + + let reachableFrontendUrl: string | null = null + for (const candidate of candidateBaseUrls) { + checkedFrontendUrls.push(candidate) + if (await isUrlReachable(candidate, 1500)) { + reachableFrontendUrl = candidate + break + } + } + + const baseUrl = (reachableFrontendUrl ?? DEFAULT_FRONTEND_URL) || null + if (!baseUrl) + return { reachableFrontendUrl, checkedFrontendUrls, hints: [] } + + const lanHosts = getLanIpv4Addresses() + const frontendHosts = new Set() + const frontendUrl = new URL(baseUrl) + + frontendHosts.add(frontendUrl.hostname) + if (isLoopbackHost(frontendUrl.hostname)) { + for (const host of lanHosts) + frontendHosts.add(host) + } + + const hints: FrontendHint[] = [] + for (const host of frontendHosts) { + const rewrittenFrontendUrl = rewriteUrlHost(baseUrl, host) + const rewrittenGatewayUrl = rewriteUrlHost(gatewayBaseUrl, host) + const frontendReachable = await isUrlReachable(rewrittenFrontendUrl, 1500) + if (!frontendReachable) + continue + + const gatewayReachable = await isUrlReachable(`${normalizeBaseUrl(rewrittenGatewayUrl)}/health`, 1500) + hints.push({ + desktopUrl: buildSettingsUrl(rewrittenFrontendUrl), + phoneUrl: isLoopbackHost(host) || !gatewayReachable + ? undefined + : buildPhoneEntryUrl(rewrittenFrontendUrl, rewrittenGatewayUrl), + warnings: buildPhoneCaptureWarnings(rewrittenFrontendUrl, rewrittenGatewayUrl), + }) + } + + return { + reachableFrontendUrl, + checkedFrontendUrls, + hints, + } +} + +export async function start() { + console.info('=== Starting AIRI Visual Chat Services ===\n') + + const gatewayPort = Number(process.env.VISUAL_CHAT_PORT || DEFAULT_GATEWAY_PORT) + const workerPort = Number(process.env.WORKER_PORT || DEFAULT_WORKER_PORT) + const ollamaBaseUrl = normalizeBaseUrl(process.env.OLLAMA_HOST || DEFAULT_OLLAMA_BASE_URL) + const requestedOllamaModel = (process.env.OLLAMA_MODEL || DEFAULT_OLLAMA_MODEL).trim() + const ollamaModel = DEFAULT_OLLAMA_MODEL + const gatewayBaseUrl = `http://127.0.0.1:${gatewayPort}` + const workerBaseUrl = `http://127.0.0.1:${workerPort}` + const workerHealthUrl = `${workerBaseUrl}/health` + const gatewayHealthUrl = `${gatewayBaseUrl}/health` + + console.info('[env] Fixed pipeline: ollama-lite') + console.info(`[env] Ollama backend: ${ollamaBaseUrl}`) + console.info(`[env] Ollama model: ${ollamaModel}`) + console.info(`[env] AIRI gateway: ${gatewayBaseUrl}`) + console.info(`[env] AIRI worker: ${workerBaseUrl}`) + console.info() + + if (requestedOllamaModel !== DEFAULT_OLLAMA_MODEL) { + console.warn(`[warn] Ignoring unsupported OLLAMA_MODEL=${requestedOllamaModel}.`) + console.warn(`[warn] AIRI Visual Chat is pinned to ${DEFAULT_OLLAMA_MODEL}.`) + console.info() + } + + const [ollamaHealth, ollamaModelCheck] = await Promise.all([ + checkOllamaHealth(ollamaBaseUrl), + checkOllamaModel(ollamaBaseUrl, ollamaModel), + ]) + + console.info(`[check] ${ollamaHealth.name}: ${ollamaHealth.detail}`) + console.info(`[check] ${ollamaModelCheck.name}: ${ollamaModelCheck.detail}`) + console.info() + + if (!ollamaHealth.ok || !ollamaModelCheck.ok) { + console.error('The fixed Ollama Visual Chat backend is not ready, so startup is stopping here.') + console.error('Run:') + console.error(' 1. `pnpm -F @proj-airi/visual-chat-ops setup-engine`') + console.error(` 2. \`pnpm -F @proj-airi/visual-chat-ops pull-models --model ${DEFAULT_OLLAMA_MODEL}\``) + console.error(' 3. `pnpm -F @proj-airi/visual-chat-ops start:local`') + process.exitCode = 1 + return + } + + const rootDir = resolve(CURRENT_DIR, '..', '..', '..', '..') + const frontendDir = join(rootDir, 'apps', 'stage-web') + const gatewayDir = join(rootDir, 'services', 'visual-chat-gateway') + const workerDir = join(rootDir, 'services', 'visual-chat-worker-minicpmo') + const managedServices: ManagedService[] = [] + let shuttingDown = false + + const shutdown = async (exitCode: number) => { + if (shuttingDown) + return + + shuttingDown = true + console.info('\nStopping AIRI visual chat services...') + + for (const service of managedServices) { + if (!service.reused && service.child && !service.child.killed) + service.child.kill() + } + + process.exit(exitCode) + } + + process.on('SIGINT', () => void shutdown(0)) + process.on('SIGTERM', () => void shutdown(0)) + + const worker = await ensureService({ + name: 'AIRI worker bridge', + cwd: workerDir, + healthUrl: workerHealthUrl, + env: { + ...process.env, + WORKER_HOST: process.env.WORKER_HOST || '127.0.0.1', + WORKER_PORT: String(workerPort), + GATEWAY_URL: gatewayBaseUrl, + VISUAL_CHAT_GATEWAY_URL: gatewayBaseUrl, + OLLAMA_HOST: ollamaBaseUrl, + OLLAMA_MODEL: ollamaModel, + }, + packageManagerArgs: ['exec', 'tsx', 'src/index.ts'], + }) + managedServices.push(worker) + attachUnexpectedExitHandler(worker, shutdown, () => shuttingDown) + + const workerHealth = await readWorkerHealth(workerHealthUrl) + if (workerHealth?.backendKind && workerHealth.backendKind !== 'ollama') + console.warn(`[warn] Existing worker bridge reports backend kind ${workerHealth.backendKind}, not ollama.`) + if (workerHealth?.upstreamBaseUrl && normalizeBaseUrl(workerHealth.upstreamBaseUrl) !== ollamaBaseUrl) + console.warn(`[warn] Existing worker bridge points at ${workerHealth.upstreamBaseUrl}, not ${ollamaBaseUrl}.`) + if (workerHealth?.model && workerHealth.model !== ollamaModel) + console.warn(`[warn] Existing worker bridge reports model ${workerHealth.model}, not ${ollamaModel}.`) + + const gateway = await ensureService({ + name: 'AIRI gateway', + cwd: gatewayDir, + healthUrl: gatewayHealthUrl, + env: { + ...process.env, + VISUAL_CHAT_HOST: process.env.VISUAL_CHAT_HOST || '0.0.0.0', + VISUAL_CHAT_PORT: String(gatewayPort), + WORKER_URL: workerBaseUrl, + }, + packageManagerArgs: ['exec', 'tsx', 'src/index.ts'], + }) + managedServices.push(gateway) + attachUnexpectedExitHandler(gateway, shutdown, () => shuttingDown) + + console.info() + console.info('AIRI visual chat bridge is ready.') + console.info() + console.info('Service endpoints:') + console.info(` Gateway health: ${gatewayHealthUrl}`) + console.info(` Worker health: ${workerHealthUrl}`) + console.info(' Worker pipeline: fixed ollama-lite') + console.info(` Upstream model: ${ollamaModel}`) + console.info(` Upstream URL: ${ollamaBaseUrl}`) + console.info() + + let frontendHints = await resolveFrontendHints(gatewayBaseUrl) + const shouldAutoStartFrontend = DEFAULT_AUTO_START_FRONTEND !== 'false' && DEFAULT_AUTO_START_FRONTEND !== '0' + const hasPhoneReachableFrontend = frontendHints.hints.some(hint => !!hint.phoneUrl) + if (!DEFAULT_FRONTEND_URL && !hasPhoneReachableFrontend && shouldAutoStartFrontend) { + console.info('[build] AIRI web frontend...') + await runWorkspaceCommand({ + cwd: frontendDir, + env: { + ...process.env, + }, + packageManagerArgs: ['build'], + label: 'AIRI web frontend build', + }) + + const frontend = await ensureService({ + name: 'AIRI web frontend preview', + cwd: frontendDir, + healthUrl: DEFAULT_FRONTEND_CANDIDATES[0], + env: { + ...process.env, + }, + packageManagerArgs: ['exec', 'vite', 'preview', '--host', '0.0.0.0', '--port', '5173', '--strictPort'], + }) + managedServices.push(frontend) + attachUnexpectedExitHandler(frontend, shutdown, () => shuttingDown) + frontendHints = await resolveFrontendHints(gatewayBaseUrl) + } + + if (frontendHints.hints.length > 0) { + console.info('Frontend entry hints:') + for (const hint of frontendHints.hints) { + console.info(` Desktop settings: ${hint.desktopUrl}`) + if (hint.phoneUrl) + console.info(` Phone page: ${hint.phoneUrl}`) + for (const warning of hint.warnings) + console.warn(` Warning: ${warning}`) + } + console.info() + } + else { + console.warn('No AIRI web frontend was detected automatically.') + if (frontendHints.checkedFrontendUrls.length) + console.warn(`Checked: ${frontendHints.checkedFrontendUrls.join(', ')}`) + console.warn('Start `apps/stage-web` separately, or set `AIRI_VISUAL_CHAT_FRONTEND_URL` before running this command to print phone entry URLs.') + console.info() + } + + console.info('Suggested flow:') + console.info(' 1. Open the desktop Visual Chat settings page and create or join a session.') + console.info(' 2. Pick one input mode: phone camera, desktop camera, or desktop screen capture.') + console.info(' 3. Use typed prompts; all turns go through the same gateway -> worker -> chat pipeline.') + console.info() + + await new Promise(() => {}) +} + +function isDirectExecution(): boolean { + const entryPath = process.argv[1] + return !!entryPath && pathToFileURL(entryPath).href === import.meta.url +} + +if (isDirectExecution()) { + void start() +} diff --git a/packages/visual-chat-ops/src/cli/stop.ts b/packages/visual-chat-ops/src/cli/stop.ts new file mode 100644 index 0000000000..644fb68d64 --- /dev/null +++ b/packages/visual-chat-ops/src/cli/stop.ts @@ -0,0 +1,71 @@ +import process from 'node:process' + +import { execFileSync, execSync } from 'node:child_process' +import { existsSync, rmSync } from 'node:fs' +import { platform } from 'node:os' +import { join } from 'node:path' +import { pathToFileURL } from 'node:url' + +import { getVisualChatDir } from '@proj-airi/visual-chat-shared' + +const DEFAULT_PUBLIC_ENDPOINTS_FILE = process.env.AIRI_VISUAL_CHAT_PUBLIC_ENDPOINTS_FILE?.trim() + || join(getVisualChatDir('config'), 'public-endpoints.json') + +function removePublicEndpointsFile() { + if (existsSync(DEFAULT_PUBLIC_ENDPOINTS_FILE)) + rmSync(DEFAULT_PUBLIC_ENDPOINTS_FILE, { force: true }) +} + +export async function stop() { + console.info('=== Stopping AIRI Visual Chat Services ===\n') + + const os = platform() + let stopped = false + + try { + if (os === 'win32') { + try { + execFileSync('powershell.exe', [ + '-NoProfile', + '-Command', + 'Get-CimInstance Win32_Process -ErrorAction SilentlyContinue' + + ' | Where-Object {' + + ' ($_.CommandLine -match \'visual-chat-gateway\')' + + ' -or ($_.CommandLine -match \'visual-chat-worker-minicpmo\')' + + ' -or ($_.CommandLine -match \'visual-chat-ops.+(share\\\\.ts|dev-tamagotchi\\\\.ts|setup-tunnel\\\\.ts)\')' + + ' -or ($_.Name -match \'cloudflared\')' + + ' }' + + ' | Select-Object -ExpandProperty ProcessId -Unique' + + ' | ForEach-Object { Stop-Process -Id $_ -Force -ErrorAction SilentlyContinue }', + ], { stdio: 'ignore' }) + stopped = true + } + catch { /* no matching process */ } + } + else { + try { + execSync('pkill -f "visual-chat-gateway|visual-chat-worker-minicpmo|visual-chat-ops/.+(share.ts|dev-tamagotchi.ts|setup-tunnel.ts)|cloudflared tunnel"', { stdio: 'ignore' }) + stopped = true + } + catch { /* no matching process */ } + } + + removePublicEndpointsFile() + + if (stopped) + console.info('Services stopped.') + else + console.info('No running services found.') + } + catch { + console.info('Failed to stop services.') + } +} + +function isDirectExecution(): boolean { + const entryPath = process.argv[1] + return !!entryPath && pathToFileURL(entryPath).href === import.meta.url +} + +if (isDirectExecution()) + void stop() diff --git a/packages/visual-chat-ops/src/index.ts b/packages/visual-chat-ops/src/index.ts new file mode 100644 index 0000000000..5433c6b49a --- /dev/null +++ b/packages/visual-chat-ops/src/index.ts @@ -0,0 +1,11 @@ +export { doctor } from './cli/doctor' +export { install } from './cli/install' +export { prune } from './cli/prune' +export { pullModels } from './cli/pull-models' +export { setupEngine } from './cli/setup-engine' +export { loadTunnelConfig, setupTunnel, startNamedTunnels, TUNNEL_CONFIG_FILE } from './cli/setup-tunnel' +export type { NamedTunnelPair } from './cli/setup-tunnel' +export { createTunnelPair } from './cli/share' +export type { TunnelPair } from './cli/share' +export { start } from './cli/start' +export { stop } from './cli/stop' diff --git a/packages/visual-chat-ops/tsdown.config.ts b/packages/visual-chat-ops/tsdown.config.ts new file mode 100644 index 0000000000..e0cb0b3266 --- /dev/null +++ b/packages/visual-chat-ops/tsdown.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from 'tsdown' + +export default defineConfig({ + entry: { + index: 'src/index.ts', + }, + target: 'node18', + outDir: 'dist', + clean: true, + dts: true, +}) diff --git a/packages/visual-chat-sdk/package.json b/packages/visual-chat-sdk/package.json new file mode 100644 index 0000000000..11fdeb3da0 --- /dev/null +++ b/packages/visual-chat-sdk/package.json @@ -0,0 +1,37 @@ +{ + "name": "@proj-airi/visual-chat-sdk", + "type": "module", + "version": "0.0.1", + "private": true, + "description": "Client SDK for AIRI visual chat frontend and plugin integration", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/moeru-ai/airi.git", + "directory": "packages/visual-chat-sdk" + }, + "exports": { + ".": { + "types": "./dist/index.d.mts", + "import": "./src/index.ts", + "default": "./dist/index.mjs" + } + }, + "main": "./src/index.ts", + "types": "./dist/index.d.mts", + "files": [ + "README.md", + "dist", + "package.json" + ], + "scripts": { + "dev": "pnpm run build", + "build": "tsdown", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@proj-airi/visual-chat-protocol": "workspace:^", + "@proj-airi/visual-chat-shared": "workspace:^", + "vue": "catalog:" + } +} diff --git a/packages/visual-chat-sdk/src/client/index.ts b/packages/visual-chat-sdk/src/client/index.ts new file mode 100644 index 0000000000..df08eb2571 --- /dev/null +++ b/packages/visual-chat-sdk/src/client/index.ts @@ -0,0 +1,257 @@ +import type { + GatewayBootstrap, + GatewayDiagnostics, + RoomCreateRequest, + SessionAccess, + SessionContext, + SessionMessagesResponse, + SessionRecord, + SessionRecordsResponse, +} from '@proj-airi/visual-chat-protocol' + +import { + VISUAL_CHAT_GATEWAY_TOKEN_HEADER, + VISUAL_CHAT_SESSION_TOKEN_HEADER, +} from '@proj-airi/visual-chat-protocol' + +const TRAILING_SLASH_PATTERN = /\/$/ + +export interface GatewaySessionAccess { + sessionId: string + sessionToken: string +} + +export interface GatewayClientOptions { + baseUrl: string + getGatewayToken?: () => string | null | undefined + getSessionAccess?: () => GatewaySessionAccess | null | undefined +} + +export class GatewayClient { + private baseUrl: string + private getGatewayToken: () => string | null | undefined + private getSessionAccess: () => GatewaySessionAccess | null | undefined + + constructor(opts: GatewayClientOptions) { + this.baseUrl = opts.baseUrl.replace(TRAILING_SLASH_PATTERN, '') + this.getGatewayToken = opts.getGatewayToken ?? (() => undefined) + this.getSessionAccess = opts.getSessionAccess ?? (() => undefined) + } + + private buildHeaders(options: { + includeJsonContentType?: boolean + includeGatewayToken?: boolean + includeSessionToken?: boolean + sessionId?: string + } = {}): Headers { + const headers = new Headers() + if (options.includeJsonContentType) + headers.set('Content-Type', 'application/json') + + if (options.includeGatewayToken) { + const gatewayToken = this.getGatewayToken()?.trim() + if (gatewayToken) + headers.set(VISUAL_CHAT_GATEWAY_TOKEN_HEADER, gatewayToken) + } + + if (options.includeSessionToken) { + const sessionAccess = this.getSessionAccess() + if (sessionAccess?.sessionToken) { + const matchesSession = !options.sessionId || sessionAccess.sessionId === options.sessionId + if (matchesSession) + headers.set(VISUAL_CHAT_SESSION_TOKEN_HEADER, sessionAccess.sessionToken) + } + } + + return headers + } + + async bootstrap(): Promise { + const res = await fetch(`${this.baseUrl}/api/bootstrap`, { + headers: this.buildHeaders({ + includeGatewayToken: true, + }), + }) + if (!res.ok) + throw new Error(`bootstrap failed: ${res.status}`) + return res.json() as Promise + } + + async createSession(req?: RoomCreateRequest): Promise { + const res = await fetch(`${this.baseUrl}/api/sessions`, { + method: 'POST', + headers: this.buildHeaders({ + includeJsonContentType: true, + includeGatewayToken: true, + }), + body: JSON.stringify(req ?? {}), + }) + if (!res.ok) + throw new Error(`createSession failed: ${res.status}`) + return res.json() as Promise + } + + async issueSessionAccess(sessionId: string): Promise { + const res = await fetch(`${this.baseUrl}/api/sessions/${sessionId}/access`, { + method: 'POST', + headers: this.buildHeaders({ + includeGatewayToken: true, + }), + }) + if (!res.ok) + throw new Error(`issueSessionAccess failed: ${res.status}`) + return res.json() as Promise + } + + async listSessions(): Promise { + const res = await fetch(`${this.baseUrl}/api/sessions`, { + headers: this.buildHeaders({ + includeGatewayToken: true, + }), + }) + if (!res.ok) + throw new Error(`listSessions failed: ${res.status}`) + return res.json() as Promise + } + + async getSession(sessionId: string): Promise { + const res = await fetch(`${this.baseUrl}/api/sessions/${sessionId}`, { + headers: this.buildHeaders({ + includeGatewayToken: true, + includeSessionToken: true, + sessionId, + }), + }) + if (!res.ok) + throw new Error(`getSession failed: ${res.status}`) + return res.json() as Promise + } + + async getSessionMessages(sessionId: string): Promise { + const res = await fetch(`${this.baseUrl}/api/sessions/${sessionId}/messages`, { + headers: this.buildHeaders({ + includeGatewayToken: true, + includeSessionToken: true, + sessionId, + }), + }) + if (!res.ok) + throw new Error(`getSessionMessages failed: ${res.status}`) + return res.json() as Promise + } + + async getSessionRecord(sessionId: string): Promise { + const res = await fetch(`${this.baseUrl}/api/sessions/${sessionId}/record`, { + headers: this.buildHeaders({ + includeGatewayToken: true, + includeSessionToken: true, + sessionId, + }), + }) + if (!res.ok) + throw new Error(`getSessionRecord failed: ${res.status}`) + return res.json() as Promise + } + + async listSessionRecords(): Promise { + const res = await fetch(`${this.baseUrl}/api/session-records`, { + headers: this.buildHeaders({ + includeGatewayToken: true, + }), + }) + if (!res.ok) + throw new Error(`listSessionRecords failed: ${res.status}`) + const payload = await res.json() as SessionRecordsResponse + return payload.records + } + + async restoreSessionRecord(sessionId: string): Promise { + const res = await fetch(`${this.baseUrl}/api/session-records/${sessionId}/restore`, { + method: 'POST', + headers: this.buildHeaders({ + includeGatewayToken: true, + }), + }) + if (!res.ok) + throw new Error(`restoreSessionRecord failed: ${res.status}`) + return res.json() as Promise + } + + async deleteSession(sessionId: string): Promise { + const res = await fetch(`${this.baseUrl}/api/sessions/${sessionId}`, { + method: 'DELETE', + headers: this.buildHeaders({ + includeGatewayToken: true, + includeSessionToken: true, + sessionId, + }), + }) + if (!res.ok) + throw new Error(`deleteSession failed: ${res.status}`) + } + + async deleteSessionRecord(sessionId: string): Promise { + const res = await fetch(`${this.baseUrl}/api/sessions/${sessionId}/record`, { + method: 'DELETE', + headers: this.buildHeaders({ + includeGatewayToken: true, + includeSessionToken: true, + sessionId, + }), + }) + if (!res.ok) + throw new Error(`deleteSessionRecord failed: ${res.status}`) + } + + async switchSource(sessionId: string, sourceId?: string, sourceType?: string): Promise { + const res = await fetch(`${this.baseUrl}/api/sessions/${sessionId}/switch-source`, { + method: 'POST', + headers: this.buildHeaders({ + includeJsonContentType: true, + includeGatewayToken: true, + includeSessionToken: true, + sessionId, + }), + body: JSON.stringify({ sourceId, sourceType }), + }) + if (!res.ok) + throw new Error(`switchSource failed: ${res.status}`) + return res.json() as Promise + } + + async getRoomToken(roomName: string, name: string, identity: string): Promise<{ token: string, roomName: string }> { + const res = await fetch(`${this.baseUrl}/api/rooms/${roomName}/token`, { + method: 'POST', + headers: this.buildHeaders({ + includeJsonContentType: true, + includeGatewayToken: true, + includeSessionToken: true, + }), + body: JSON.stringify({ name, identity }), + }) + if (!res.ok) + throw new Error(`getRoomToken failed: ${res.status}`) + return res.json() as Promise<{ token: string, roomName: string }> + } + + async getDiagnostics(): Promise { + const res = await fetch(`${this.baseUrl}/api/diagnostics`, { + headers: this.buildHeaders({ + includeGatewayToken: true, + }), + }) + if (!res.ok) + throw new Error(`getDiagnostics failed: ${res.status}`) + return res.json() as Promise + } + + async health(): Promise { + try { + const res = await fetch(`${this.baseUrl}/health`) + return res.ok + } + catch { + return false + } + } +} diff --git a/packages/visual-chat-sdk/src/composables/index.ts b/packages/visual-chat-sdk/src/composables/index.ts new file mode 100644 index 0000000000..6e9c58ee31 --- /dev/null +++ b/packages/visual-chat-sdk/src/composables/index.ts @@ -0,0 +1,3 @@ +export { useSessionStatus } from './use-session-status' +export { useSourceSwitch } from './use-source-switch' +export { useVisualChat } from './use-visual-chat' diff --git a/packages/visual-chat-sdk/src/composables/use-session-status.ts b/packages/visual-chat-sdk/src/composables/use-session-status.ts new file mode 100644 index 0000000000..4f358c15f4 --- /dev/null +++ b/packages/visual-chat-sdk/src/composables/use-session-status.ts @@ -0,0 +1,51 @@ +import type { GatewayDiagnostics } from '@proj-airi/visual-chat-protocol' + +import type { GatewayClient } from '../client' + +import { onUnmounted, ref } from 'vue' + +export function useSessionStatus(client: GatewayClient, pollIntervalMs: number = 5000) { + const diagnostics = ref(null) + const healthy = ref(false) + const loading = ref(false) + + let timer: ReturnType | null = null + + async function poll() { + loading.value = true + try { + healthy.value = await client.health() + if (healthy.value) + diagnostics.value = await client.getDiagnostics() + } + catch { + healthy.value = false + } + finally { + loading.value = false + } + } + + function startPolling() { + poll() + timer = setInterval(poll, pollIntervalMs) + } + + function stopPolling() { + if (timer) { + clearInterval(timer) + timer = null + } + } + + onUnmounted(stopPolling) + + return { + diagnostics, + healthy, + loading, + poll, + startPolling, + stopPolling, + } +} diff --git a/packages/visual-chat-sdk/src/composables/use-source-switch.ts b/packages/visual-chat-sdk/src/composables/use-source-switch.ts new file mode 100644 index 0000000000..bdab043319 --- /dev/null +++ b/packages/visual-chat-sdk/src/composables/use-source-switch.ts @@ -0,0 +1,63 @@ +import type { SessionContext } from '@proj-airi/visual-chat-protocol' + +import type { GatewayClient } from '../client' + +import { ref } from 'vue' + +export function useSourceSwitch(client: GatewayClient, getSessionId: () => string | null) { + const switching = ref(false) + const lastError = ref(null) + + /** + * Switch active source by sourceId. + */ + async function switchById(sourceId: string): Promise { + const sessionId = getSessionId() + if (!sessionId) + return null + + switching.value = true + lastError.value = null + + try { + return await client.switchSource(sessionId, sourceId) + } + catch (err) { + lastError.value = String(err) + return null + } + finally { + switching.value = false + } + } + + /** + * Switch active source by sourceType string (e.g. 'phone-camera', 'screen-share'). + */ + async function switchByType(sourceType: string): Promise { + const sessionId = getSessionId() + if (!sessionId) + return null + + switching.value = true + lastError.value = null + + try { + return await client.switchSource(sessionId, undefined, sourceType) + } + catch (err) { + lastError.value = String(err) + return null + } + finally { + switching.value = false + } + } + + return { + switching, + lastError, + switchById, + switchByType, + } +} diff --git a/packages/visual-chat-sdk/src/composables/use-visual-chat.ts b/packages/visual-chat-sdk/src/composables/use-visual-chat.ts new file mode 100644 index 0000000000..43fc73cdb2 --- /dev/null +++ b/packages/visual-chat-sdk/src/composables/use-visual-chat.ts @@ -0,0 +1,126 @@ +import type { InteractionMode, SessionAccess, SessionContext } from '@proj-airi/visual-chat-protocol' + +import type { GatewaySessionAccess } from '../client' + +import { onUnmounted, ref, shallowRef } from 'vue' + +import { GatewayClient } from '../client' +import { GatewayWsClient } from '../ws-client' + +const HTTP_TO_WS_PATTERN = /^http/ + +export function useVisualChat(gatewayUrl: string) { + const gatewayToken = ref('') + const sessionAccess = shallowRef(null) + const client = new GatewayClient({ + baseUrl: gatewayUrl, + getGatewayToken: () => gatewayToken.value, + getSessionAccess: () => { + if (!sessionAccess.value) + return null + return { + sessionId: sessionAccess.value.session.sessionId, + sessionToken: sessionAccess.value.sessionToken, + } satisfies GatewaySessionAccess + }, + }) + const wsUrl = `${gatewayUrl.replace(HTTP_TO_WS_PATTERN, 'ws')}/ws` + const wsClient = new GatewayWsClient(wsUrl, { + getSessionAccess: (sessionId) => { + if (!sessionAccess.value || sessionAccess.value.session.sessionId !== sessionId) + return null + return { + sessionId, + sessionToken: sessionAccess.value.sessionToken, + } + }, + }) + + const session = shallowRef(null) + const connected = ref(false) + const error = ref(null) + + wsClient.on('connected', () => { + connected.value = true + }) + wsClient.on('disconnected', () => { + connected.value = false + }) + + wsClient.on('session:state:changed', (ev) => { + if (session.value && ev.sessionId === session.value.sessionId) { + const payload = ev.data as { context?: SessionContext } + if (payload?.context) + session.value = payload.context + } + }) + + wsClient.on('session:mode:changed', (ev) => { + if (session.value && ev.sessionId === session.value.sessionId) { + const payload = ev.data as { to?: InteractionMode } + if (payload?.to) + session.value = { ...session.value, mode: payload.to } + } + }) + + wsClient.on('source:active:changed', (ev) => { + if (session.value && ev.sessionId === session.value.sessionId) { + refreshSession() + } + }) + + wsClient.on('session:ended', (ev) => { + if (session.value && ev.sessionId === session.value.sessionId) { + session.value = null + } + }) + + async function refreshSession() { + if (!session.value) + return + try { + session.value = await client.getSession(session.value.sessionId) + } + catch { /* session may have ended */ } + } + + async function createSession() { + try { + const bootstrap = await client.bootstrap() + gatewayToken.value = bootstrap.gatewayToken + const access = await client.createSession() + sessionAccess.value = access + session.value = access.session + wsClient.connect() + wsClient.subscribe(access.session.sessionId) + return access.session + } + catch (err) { + error.value = String(err) + throw err + } + } + + async function endSession() { + if (session.value) { + await client.deleteSession(session.value.sessionId) + session.value = null + sessionAccess.value = null + } + wsClient.disconnect() + } + + onUnmounted(() => { + wsClient.disconnect() + }) + + return { + session, + connected, + error, + client, + createSession, + endSession, + refreshSession, + } +} diff --git a/packages/visual-chat-sdk/src/index.ts b/packages/visual-chat-sdk/src/index.ts new file mode 100644 index 0000000000..7a5f220fca --- /dev/null +++ b/packages/visual-chat-sdk/src/index.ts @@ -0,0 +1,7 @@ +export { GatewayClient } from './client' +export type { GatewayClientOptions, GatewaySessionAccess } from './client' +export { useSessionStatus } from './composables/use-session-status' +export { useSourceSwitch } from './composables/use-source-switch' +export { useVisualChat } from './composables/use-visual-chat' +export { GatewayWsClient } from './ws-client' +export type { GatewayWsClientOptions, GatewayWsSessionAccess, WsEvent, WsEventHandler } from './ws-client' diff --git a/packages/visual-chat-sdk/src/ws-client/index.ts b/packages/visual-chat-sdk/src/ws-client/index.ts new file mode 100644 index 0000000000..21095408a5 --- /dev/null +++ b/packages/visual-chat-sdk/src/ws-client/index.ts @@ -0,0 +1,172 @@ +import type { GatewayWsClientMessage } from '@proj-airi/visual-chat-protocol' + +export interface GatewayWsSessionAccess { + sessionId: string + sessionToken: string +} + +export interface WsEvent { + event: string + sessionId: string + data: unknown + timestamp: number +} + +export type WsEventHandler = (event: WsEvent) => void + +export interface GatewayWsClientOptions { + autoReconnect?: boolean + getSessionAccess?: (sessionId: string) => GatewayWsSessionAccess | null | undefined +} + +export class GatewayWsClient { + private ws: WebSocket | null = null + private handlers = new Map>() + private reconnectTimer: ReturnType | null = null + private messageQueue: GatewayWsClientMessage[] = [] + private desiredSubscriptions = new Set() + private activeSubscriptions = new Set() + private autoReconnect: boolean + private getSessionAccess: (sessionId: string) => GatewayWsSessionAccess | null | undefined + + constructor( + private wsUrl: string, + options: GatewayWsClientOptions = {}, + ) { + this.autoReconnect = options.autoReconnect ?? true + this.getSessionAccess = options.getSessionAccess ?? (() => undefined) + } + + connect(): void { + if (this.ws && (this.ws.readyState === WebSocket.OPEN || this.ws.readyState === WebSocket.CONNECTING)) + return + + this.ws = new WebSocket(this.wsUrl) + + this.ws.onopen = () => { + this.activeSubscriptions.clear() + this.restoreSubscriptions() + this.flushQueuedMessages() + this.emit('connected', { event: 'connected', sessionId: '', data: null, timestamp: Date.now() }) + } + + this.ws.onmessage = (msg) => { + try { + const event = JSON.parse(msg.data as string) as WsEvent + this.emit(event.event, event) + } + catch { /* ignore */ } + } + + this.ws.onclose = () => { + this.emit('disconnected', { event: 'disconnected', sessionId: '', data: null, timestamp: Date.now() }) + if (this.autoReconnect) + this.scheduleReconnect() + } + + this.ws.onerror = () => { + this.ws?.close() + } + } + + subscribe(sessionId: string): void { + const normalizedSessionId = sessionId.trim() + if (!normalizedSessionId || this.desiredSubscriptions.has(normalizedSessionId)) + return + + this.desiredSubscriptions.add(normalizedSessionId) + + if (this.ws?.readyState === WebSocket.OPEN) { + this.sendSubscribe(normalizedSessionId) + this.activeSubscriptions.add(normalizedSessionId) + return + } + + if (!this.ws || this.ws.readyState === WebSocket.CLOSED) + this.connect() + } + + unsubscribe(sessionId: string): void { + const normalizedSessionId = sessionId.trim() + if (!normalizedSessionId || !this.desiredSubscriptions.has(normalizedSessionId)) + return + + this.desiredSubscriptions.delete(normalizedSessionId) + this.activeSubscriptions.delete(normalizedSessionId) + + if (this.ws?.readyState === WebSocket.OPEN) + this.sendRaw({ type: 'unsubscribe', sessionId: normalizedSessionId }) + } + + send(message: GatewayWsClientMessage): void { + if (this.ws?.readyState === WebSocket.OPEN) { + this.sendRaw(message) + return + } + + this.messageQueue.push(message) + if (!this.ws || this.ws.readyState === WebSocket.CLOSED) + this.connect() + } + + on(event: string, handler: WsEventHandler): () => void { + if (!this.handlers.has(event)) + this.handlers.set(event, new Set()) + this.handlers.get(event)!.add(handler) + return () => this.handlers.get(event)?.delete(handler) + } + + disconnect(): void { + this.autoReconnect = false + if (this.reconnectTimer) + clearTimeout(this.reconnectTimer) + this.activeSubscriptions.clear() + this.ws?.close() + this.ws = null + } + + private emit(eventName: string, event: WsEvent) { + this.handlers.get(eventName)?.forEach(h => h(event)) + this.handlers.get('*')?.forEach(h => h(event)) + } + + private flushQueuedMessages() { + if (!this.ws || this.ws.readyState !== WebSocket.OPEN) + return + + const queued = [...this.messageQueue] + this.messageQueue = [] + for (const message of queued) + this.sendRaw(message) + } + + private restoreSubscriptions() { + if (!this.ws || this.ws.readyState !== WebSocket.OPEN) + return + + for (const sessionId of this.desiredSubscriptions) { + this.sendSubscribe(sessionId) + this.activeSubscriptions.add(sessionId) + } + } + + private sendSubscribe(sessionId: string) { + const sessionAccess = this.getSessionAccess(sessionId) + if (!sessionAccess?.sessionToken) + return + + this.sendRaw({ + type: 'subscribe', + sessionId, + sessionToken: sessionAccess.sessionToken, + }) + } + + private sendRaw(message: GatewayWsClientMessage) { + this.ws?.send(JSON.stringify(message)) + } + + private scheduleReconnect() { + this.reconnectTimer = setTimeout(() => this.connect(), 3000) + } +} diff --git a/packages/visual-chat-sdk/tsdown.config.ts b/packages/visual-chat-sdk/tsdown.config.ts new file mode 100644 index 0000000000..e0cb0b3266 --- /dev/null +++ b/packages/visual-chat-sdk/tsdown.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from 'tsdown' + +export default defineConfig({ + entry: { + index: 'src/index.ts', + }, + target: 'node18', + outDir: 'dist', + clean: true, + dts: true, +}) diff --git a/plugins/plugin-visual-chat-pocket/package.json b/plugins/plugin-visual-chat-pocket/package.json new file mode 100644 index 0000000000..79a60d645c --- /dev/null +++ b/plugins/plugin-visual-chat-pocket/package.json @@ -0,0 +1,29 @@ +{ + "name": "@proj-airi/plugin-visual-chat-pocket", + "type": "module", + "version": "0.0.1", + "private": true, + "description": "Visual chat bridge plugin for AIRI stage-pocket (mobile)", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/moeru-ai/airi.git", + "directory": "plugins/plugin-visual-chat-pocket" + }, + "exports": { + ".": { + "types": "./dist/index.d.mts", + "default": "./dist/index.mjs" + } + }, + "main": "./dist/index.mjs", + "types": "./dist/index.d.mts", + "scripts": { + "dev": "pnpm run build", + "build": "tsdown", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@proj-airi/visual-chat-sdk": "workspace:^" + } +} diff --git a/plugins/plugin-visual-chat-pocket/src/index.ts b/plugins/plugin-visual-chat-pocket/src/index.ts new file mode 100644 index 0000000000..11f713933d --- /dev/null +++ b/plugins/plugin-visual-chat-pocket/src/index.ts @@ -0,0 +1,74 @@ +import type { SessionAccess } from '@proj-airi/visual-chat-protocol' +import type { GatewaySessionAccess, WsEvent } from '@proj-airi/visual-chat-sdk' + +import { GatewayClient, GatewayWsClient } from '@proj-airi/visual-chat-sdk' + +export interface PocketBridgeOptions { + gatewayUrl: string +} + +export class PocketVisualChatBridge { + private client: GatewayClient + private wsClient: GatewayWsClient + private sessionId: string | null = null + private gatewayToken = '' + private sessionAccess: SessionAccess | null = null + + constructor(private opts: PocketBridgeOptions) { + this.client = new GatewayClient({ + baseUrl: opts.gatewayUrl, + getGatewayToken: () => this.gatewayToken, + getSessionAccess: () => { + if (!this.sessionAccess) + return null + return { + sessionId: this.sessionAccess.session.sessionId, + sessionToken: this.sessionAccess.sessionToken, + } satisfies GatewaySessionAccess + }, + }) + const wsUrl = `${opts.gatewayUrl.replace(/^http/, 'ws')}/ws` + this.wsClient = new GatewayWsClient(wsUrl, { + getSessionAccess: (sessionId) => { + if (!this.sessionAccess || this.sessionAccess.session.sessionId !== sessionId) + return null + return { + sessionId, + sessionToken: this.sessionAccess.sessionToken, + } + }, + }) + } + + async connect(): Promise { + const bootstrap = await this.client.bootstrap() + this.gatewayToken = bootstrap.gatewayToken + const session = await this.client.createSession() + this.sessionAccess = session + this.sessionId = session.session.sessionId + + this.wsClient.connect() + this.wsClient.subscribe(session.session.sessionId) + + return session.session.sessionId + } + + async switchToPhoneCamera(): Promise { + if (!this.sessionId) + return + await this.client.switchSource(this.sessionId, undefined, 'phone-camera') + } + + async disconnect(): Promise { + if (this.sessionId) { + await this.client.deleteSession(this.sessionId) + this.sessionId = null + this.sessionAccess = null + } + this.wsClient.disconnect() + } + + onEvent(handler: (event: WsEvent) => void): () => void { + return this.wsClient.on('*', handler) + } +} diff --git a/plugins/plugin-visual-chat-pocket/tsdown.config.ts b/plugins/plugin-visual-chat-pocket/tsdown.config.ts new file mode 100644 index 0000000000..e0cb0b3266 --- /dev/null +++ b/plugins/plugin-visual-chat-pocket/tsdown.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from 'tsdown' + +export default defineConfig({ + entry: { + index: 'src/index.ts', + }, + target: 'node18', + outDir: 'dist', + clean: true, + dts: true, +}) diff --git a/plugins/plugin-visual-chat-tamagotchi/package.json b/plugins/plugin-visual-chat-tamagotchi/package.json new file mode 100644 index 0000000000..05f6d6366c --- /dev/null +++ b/plugins/plugin-visual-chat-tamagotchi/package.json @@ -0,0 +1,29 @@ +{ + "name": "@proj-airi/plugin-visual-chat-tamagotchi", + "type": "module", + "version": "0.0.1", + "private": true, + "description": "Visual chat bridge plugin for AIRI stage-tamagotchi (Electron desktop)", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/moeru-ai/airi.git", + "directory": "plugins/plugin-visual-chat-tamagotchi" + }, + "exports": { + ".": { + "types": "./dist/index.d.mts", + "default": "./dist/index.mjs" + } + }, + "main": "./dist/index.mjs", + "types": "./dist/index.d.mts", + "scripts": { + "dev": "pnpm run build", + "build": "tsdown", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@proj-airi/visual-chat-sdk": "workspace:^" + } +} diff --git a/plugins/plugin-visual-chat-tamagotchi/src/index.ts b/plugins/plugin-visual-chat-tamagotchi/src/index.ts new file mode 100644 index 0000000000..a1bd5cc1fe --- /dev/null +++ b/plugins/plugin-visual-chat-tamagotchi/src/index.ts @@ -0,0 +1,80 @@ +import type { SessionAccess } from '@proj-airi/visual-chat-protocol' +import type { GatewaySessionAccess, WsEvent } from '@proj-airi/visual-chat-sdk' + +import { GatewayClient, GatewayWsClient } from '@proj-airi/visual-chat-sdk' + +export interface TamagotchiBridgeOptions { + gatewayUrl: string +} + +export class TamagotchiVisualChatBridge { + private client: GatewayClient + private wsClient: GatewayWsClient + private sessionId: string | null = null + private gatewayToken = '' + private sessionAccess: SessionAccess | null = null + + constructor(private opts: TamagotchiBridgeOptions) { + this.client = new GatewayClient({ + baseUrl: opts.gatewayUrl, + getGatewayToken: () => this.gatewayToken, + getSessionAccess: () => { + if (!this.sessionAccess) + return null + return { + sessionId: this.sessionAccess.session.sessionId, + sessionToken: this.sessionAccess.sessionToken, + } satisfies GatewaySessionAccess + }, + }) + const wsUrl = `${opts.gatewayUrl.replace(/^http/, 'ws')}/ws` + this.wsClient = new GatewayWsClient(wsUrl, { + getSessionAccess: (sessionId) => { + if (!this.sessionAccess || this.sessionAccess.session.sessionId !== sessionId) + return null + return { + sessionId, + sessionToken: this.sessionAccess.sessionToken, + } + }, + }) + } + + async connect(): Promise { + const bootstrap = await this.client.bootstrap() + this.gatewayToken = bootstrap.gatewayToken + const session = await this.client.createSession() + this.sessionAccess = session + this.sessionId = session.session.sessionId + + this.wsClient.connect() + this.wsClient.subscribe(session.session.sessionId) + + return session.session.sessionId + } + + async switchToDesktopCamera(): Promise { + if (!this.sessionId) + return + await this.client.switchSource(this.sessionId, undefined, 'laptop-camera') + } + + async startScreenShare(): Promise { + if (!this.sessionId) + return + await this.client.switchSource(this.sessionId, undefined, 'screen-share') + } + + async disconnect(): Promise { + if (this.sessionId) { + await this.client.deleteSession(this.sessionId) + this.sessionId = null + this.sessionAccess = null + } + this.wsClient.disconnect() + } + + onEvent(handler: (event: WsEvent) => void): () => void { + return this.wsClient.on('*', handler) + } +} diff --git a/plugins/plugin-visual-chat-tamagotchi/tsdown.config.ts b/plugins/plugin-visual-chat-tamagotchi/tsdown.config.ts new file mode 100644 index 0000000000..e0cb0b3266 --- /dev/null +++ b/plugins/plugin-visual-chat-tamagotchi/tsdown.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from 'tsdown' + +export default defineConfig({ + entry: { + index: 'src/index.ts', + }, + target: 'node18', + outDir: 'dist', + clean: true, + dts: true, +}) diff --git a/plugins/plugin-visual-chat-web/package.json b/plugins/plugin-visual-chat-web/package.json new file mode 100644 index 0000000000..7c0a975e2e --- /dev/null +++ b/plugins/plugin-visual-chat-web/package.json @@ -0,0 +1,29 @@ +{ + "name": "@proj-airi/plugin-visual-chat-web", + "type": "module", + "version": "0.0.1", + "private": true, + "description": "Visual chat bridge plugin for AIRI stage-web (desktop browser)", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/moeru-ai/airi.git", + "directory": "plugins/plugin-visual-chat-web" + }, + "exports": { + ".": { + "types": "./dist/index.d.mts", + "default": "./dist/index.mjs" + } + }, + "main": "./dist/index.mjs", + "types": "./dist/index.d.mts", + "scripts": { + "dev": "pnpm run build", + "build": "tsdown", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@proj-airi/visual-chat-sdk": "workspace:^" + } +} diff --git a/plugins/plugin-visual-chat-web/src/index.ts b/plugins/plugin-visual-chat-web/src/index.ts new file mode 100644 index 0000000000..1cb97abb6d --- /dev/null +++ b/plugins/plugin-visual-chat-web/src/index.ts @@ -0,0 +1,89 @@ +import type { SessionAccess } from '@proj-airi/visual-chat-protocol' +import type { GatewaySessionAccess, WsEvent } from '@proj-airi/visual-chat-sdk' + +import { GatewayClient, GatewayWsClient } from '@proj-airi/visual-chat-sdk' + +export interface WebBridgeOptions { + gatewayUrl: string +} + +export class WebVisualChatBridge { + private client: GatewayClient + private wsClient: GatewayWsClient + private sessionId: string | null = null + private gatewayToken = '' + private sessionAccess: SessionAccess | null = null + + constructor(private opts: WebBridgeOptions) { + this.client = new GatewayClient({ + baseUrl: opts.gatewayUrl, + getGatewayToken: () => this.gatewayToken, + getSessionAccess: () => { + if (!this.sessionAccess) + return null + return { + sessionId: this.sessionAccess.session.sessionId, + sessionToken: this.sessionAccess.sessionToken, + } satisfies GatewaySessionAccess + }, + }) + const wsUrl = `${opts.gatewayUrl.replace(/^http/, 'ws')}/ws` + this.wsClient = new GatewayWsClient(wsUrl, { + getSessionAccess: (sessionId) => { + if (!this.sessionAccess || this.sessionAccess.session.sessionId !== sessionId) + return null + return { + sessionId, + sessionToken: this.sessionAccess.sessionToken, + } + }, + }) + } + + async connect(): Promise { + const bootstrap = await this.client.bootstrap() + this.gatewayToken = bootstrap.gatewayToken + const session = await this.client.createSession() + this.sessionAccess = session + this.sessionId = session.session.sessionId + + this.wsClient.connect() + this.wsClient.subscribe(session.session.sessionId) + + return session.session.sessionId + } + + async startScreenShare(): Promise { + try { + const stream = await navigator.mediaDevices.getDisplayMedia({ + video: true, + audio: false, + }) + if (this.sessionId) + await this.client.switchSource(this.sessionId, undefined, 'screen-share') + return stream + } + catch { + return null + } + } + + async switchToWebcam(): Promise { + if (!this.sessionId) + return + await this.client.switchSource(this.sessionId, undefined, 'laptop-camera') + } + + async disconnect(): Promise { + if (this.sessionId) { + await this.client.deleteSession(this.sessionId) + this.sessionId = null + this.sessionAccess = null + } + this.wsClient.disconnect() + } + + onEvent(handler: (event: WsEvent) => void): () => void { + return this.wsClient.on('*', handler) + } +} diff --git a/plugins/plugin-visual-chat-web/tsdown.config.ts b/plugins/plugin-visual-chat-web/tsdown.config.ts new file mode 100644 index 0000000000..e0cb0b3266 --- /dev/null +++ b/plugins/plugin-visual-chat-web/tsdown.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from 'tsdown' + +export default defineConfig({ + entry: { + index: 'src/index.ts', + }, + target: 'node18', + outDir: 'dist', + clean: true, + dts: true, +}) From 25761c705eaeb6431d8ba07b1dee82b241508f3f Mon Sep 17 00:00:00 2001 From: Joker-of-Gotham Date: Mon, 6 Apr 2026 05:55:49 +0800 Subject: [PATCH 5/6] feat(visual-chat): harden vision-text pipeline and packaged desktop runtime Made-with: Cursor --- .gitignore | 11 + .../electron-builder.config.ts | 2 + apps/stage-tamagotchi/electron.vite.config.ts | 13 + apps/stage-tamagotchi/package.json | 8 +- .../scripts/electron-vite-dev.ts | 65 + .../src/main/services/electron/app.ts | 2 + .../src/main/services/electron/index.ts | 1 + .../src/main/services/electron/visual-chat.ts | 501 +++++++ .../src/renderer/BrowserApp.vue | 12 + apps/stage-tamagotchi/src/renderer/main.ts | 38 +- .../src/renderer/pages/settings/index.vue | 18 +- apps/stage-tamagotchi/tsconfig.node.json | 13 + apps/stage-tamagotchi/tsconfig.web.json | 6 + apps/stage-visual-chat-ops/index.html | 12 + apps/stage-visual-chat-ops/package.json | 27 + apps/stage-visual-chat-ops/src/App.vue | 45 + apps/stage-visual-chat-ops/src/main.ts | 8 + apps/stage-visual-chat-ops/vite.config.ts | 10 + apps/stage-web/vite.config.ts | 9 + docs/visual-chat/architecture.md | 71 + docs/visual-chat/context-and-records.md | 40 + docs/visual-chat/quickstart.md | 164 +++ examples/visual-chat-local-5090/.env.example | 10 + examples/visual-chat-local-5090/package.json | 7 + .../.env.example | 12 + .../package.json | 7 + .../.env.example | 11 + .../package.json | 7 + package.json | 10 +- packages/stage-pages/package.json | 3 + .../src/pages/devtools/visual-chat/index.vue | 225 +++ .../pages/devtools/visual-chat/sessions.vue | 265 ++++ .../pages/devtools/visual-chat/workers.vue | 234 ++++ .../stage-pages/src/pages/settings/index.vue | 19 +- .../pages/settings/modules/visual-chat.vue | 1243 +++++++++++++++++ .../components/CollapsibleSection.vue | 42 + .../src/pages/visual-chat/phone.vue | 807 +++++++++++ packages/stage-pages/tsconfig.json | 106 +- packages/stage-ui/package.json | 3 + .../components/markdown/markdown-renderer.vue | 220 +-- packages/stage-ui/src/composables/markdown.ts | 297 ++-- .../src/composables/use-modules-list.ts | 11 + packages/stage-ui/src/stores/modules/index.ts | 1 + .../src/stores/modules/visual-chat.ts | 1240 ++++++++++++++++ .../visual-chat/native-duplex-audio.ts | 95 ++ .../modules/visual-chat/realtime-media.ts | 228 +++ .../stores/providers/web-speech-api/index.ts | 35 +- pnpm-lock.yaml | 829 ++++++++++- pnpm-workspace.yaml | 1 + 49 files changed, 6681 insertions(+), 363 deletions(-) create mode 100644 apps/stage-tamagotchi/scripts/electron-vite-dev.ts create mode 100644 apps/stage-tamagotchi/src/main/services/electron/visual-chat.ts create mode 100644 apps/stage-tamagotchi/src/renderer/BrowserApp.vue create mode 100644 apps/stage-visual-chat-ops/index.html create mode 100644 apps/stage-visual-chat-ops/package.json create mode 100644 apps/stage-visual-chat-ops/src/App.vue create mode 100644 apps/stage-visual-chat-ops/src/main.ts create mode 100644 apps/stage-visual-chat-ops/vite.config.ts create mode 100644 docs/visual-chat/architecture.md create mode 100644 docs/visual-chat/context-and-records.md create mode 100644 docs/visual-chat/quickstart.md create mode 100644 examples/visual-chat-local-5090/.env.example create mode 100644 examples/visual-chat-local-5090/package.json create mode 100644 examples/visual-chat-mobile-laptop-room/.env.example create mode 100644 examples/visual-chat-mobile-laptop-room/package.json create mode 100644 examples/visual-chat-production-sample/.env.example create mode 100644 examples/visual-chat-production-sample/package.json create mode 100644 packages/stage-pages/src/pages/devtools/visual-chat/index.vue create mode 100644 packages/stage-pages/src/pages/devtools/visual-chat/sessions.vue create mode 100644 packages/stage-pages/src/pages/devtools/visual-chat/workers.vue create mode 100644 packages/stage-pages/src/pages/settings/modules/visual-chat.vue create mode 100644 packages/stage-pages/src/pages/visual-chat/components/CollapsibleSection.vue create mode 100644 packages/stage-pages/src/pages/visual-chat/phone.vue create mode 100644 packages/stage-ui/src/stores/modules/visual-chat.ts create mode 100644 packages/stage-ui/src/stores/modules/visual-chat/native-duplex-audio.ts create mode 100644 packages/stage-ui/src/stores/modules/visual-chat/realtime-media.ts diff --git a/.gitignore b/.gitignore index 394e18f956..5bfb0ff07d 100644 --- a/.gitignore +++ b/.gitignore @@ -132,3 +132,14 @@ apps/stage-tamagotchi/electron.vite.config.*.mjs # Tools - Obsidian .obsidian/ + +# Visual Chat +.visual-chat-tunnel.json +.visual-chat-public-endpoints.json + +# Local editor / browser runtime artifacts +.cursor/* +!.cursor/commands/ +.cursor/commands/* +!.cursor/commands/deslop.md +.tmp-edge-headless/ diff --git a/apps/stage-tamagotchi/electron-builder.config.ts b/apps/stage-tamagotchi/electron-builder.config.ts index 557c346298..24b51e0b08 100644 --- a/apps/stage-tamagotchi/electron-builder.config.ts +++ b/apps/stage-tamagotchi/electron-builder.config.ts @@ -91,6 +91,8 @@ export default { asar: true, asarUnpack: [ '**/*.node', + 'node_modules/@proj-airi/visual-chat-gateway/**', + 'node_modules/@proj-airi/visual-chat-worker-minicpmo/**', ], extraMetadata: { name: 'ai.moeru.airi', diff --git a/apps/stage-tamagotchi/electron.vite.config.ts b/apps/stage-tamagotchi/electron.vite.config.ts index 3b041281ce..797c6e318a 100644 --- a/apps/stage-tamagotchi/electron.vite.config.ts +++ b/apps/stage-tamagotchi/electron.vite.config.ts @@ -1,4 +1,5 @@ import { join, resolve } from 'node:path' +import { env } from 'node:process' import VueI18n from '@intlify/unplugin-vue-i18n/vite' import templateCompilerOptions from '@tresjs/core/template-compiler-options' @@ -18,6 +19,13 @@ import { defineConfig } from 'electron-vite' const stageUIAssetsRoot = resolve(join(import.meta.dirname, '..', '..', 'packages', 'stage-ui', 'src', 'assets')) const sharedCacheDir = resolve(join(import.meta.dirname, '..', '..', '.cache')) +const additionalAllowedRemoteHosts = (env.AIRI_VISUAL_CHAT_ALLOWED_HOSTS || '') + .split(',') + .map(host => host.trim()) + .filter(Boolean) +const rendererAllowedHosts: true | string[] = additionalAllowedRemoteHosts.length > 0 + ? [...new Set(['.trycloudflare.com', ...additionalAllowedRemoteHosts])] + : true export default defineConfig({ main: { @@ -136,10 +144,15 @@ export default defineConfig({ '@proj-airi/stage-ui': resolve(join(import.meta.dirname, '..', '..', 'packages', 'stage-ui', 'src')), '@proj-airi/stage-pages': resolve(join(import.meta.dirname, '..', '..', 'packages', 'stage-pages', 'src')), '@proj-airi/stage-shared': resolve(join(import.meta.dirname, '..', '..', 'packages', 'stage-shared', 'src')), + '@proj-airi/visual-chat-shared/electron': resolve(join(import.meta.dirname, '..', '..', 'packages', 'visual-chat-shared', 'src', 'electron.ts')), }, }, server: { + host: '0.0.0.0', + port: 5174, + strictPort: true, + allowedHosts: rendererAllowedHosts, fs: { // To mute errors like: // The request id ".../node_modules/@fontsource/sniglet/files/sniglet-latin-400-normal.woff" is outside of Vite serving allow list. diff --git a/apps/stage-tamagotchi/package.json b/apps/stage-tamagotchi/package.json index 6a7fcab8cd..ba64bc6a74 100644 --- a/apps/stage-tamagotchi/package.json +++ b/apps/stage-tamagotchi/package.json @@ -19,8 +19,8 @@ "app:dev": "pnpm run dev", "app:build": "pnpm run build", "start": "electron-vite preview", - "dev": "electron-vite dev", - "build": "electron-vite build", + "dev": "tsx scripts/electron-vite-dev.ts", + "build": "pnpm -F @proj-airi/visual-chat-ops build && pnpm -F @proj-airi/visual-chat-gateway build && pnpm -F @proj-airi/visual-chat-worker-minicpmo build && electron-vite build", "postinstall": "electron-builder install-app-deps", "build:unpack": "pnpm run build && electron-builder --dir", "build:win": "pnpm run build && electron-builder --win", @@ -66,6 +66,10 @@ "@proj-airi/stage-ui-live2d": "workspace:^", "@proj-airi/stage-ui-three": "workspace:^", "@proj-airi/ui": "workspace:^", + "@proj-airi/visual-chat-gateway": "workspace:^", + "@proj-airi/visual-chat-ops": "workspace:^", + "@proj-airi/visual-chat-shared": "workspace:^", + "@proj-airi/visual-chat-worker-minicpmo": "workspace:^", "@shikijs/markdown-it": "^4.0.2", "@tresjs/cientos": "^5.6.0", "@tresjs/core": "^5.7.0", diff --git a/apps/stage-tamagotchi/scripts/electron-vite-dev.ts b/apps/stage-tamagotchi/scripts/electron-vite-dev.ts new file mode 100644 index 0000000000..ada764e932 --- /dev/null +++ b/apps/stage-tamagotchi/scripts/electron-vite-dev.ts @@ -0,0 +1,65 @@ +import process from 'node:process' + +import { spawn } from 'node:child_process' + +const env = { ...process.env } + +// Some Windows setups leak `ELECTRON_RUN_AS_NODE=1` into child processes. +// When that reaches `electron-vite dev`, Electron starts as plain Node.js, +// so imports like `import { BrowserWindow } from "electron"` fail at runtime. +delete env.ELECTRON_RUN_AS_NODE + +const child = spawn('pnpm', ['exec', 'electron-vite', 'dev'], { + cwd: process.cwd(), + env, + stdio: 'inherit', + shell: true, +}) + +let exiting = false + +function shutdown() { + if (exiting) + return + exiting = true + + if (process.platform === 'win32') { + // On Windows, child.kill() only kills the shell wrapper, not the + // process tree underneath. `taskkill /T` terminates the whole tree. + try { + spawn('taskkill', ['/T', '/F', '/PID', String(child.pid)], { stdio: 'ignore' }) + } + catch {} + } + else { + try { + // Negative PID sends signal to the entire process group + process.kill(-child.pid!, 'SIGTERM') + } + catch { + child.kill('SIGTERM') + } + } + + setTimeout(() => process.exit(1), 2000).unref() +} + +process.on('SIGINT', shutdown) +process.on('SIGTERM', shutdown) + +child.once('error', (error) => { + console.error(error) + process.exit(1) +}) + +child.once('exit', (code, signal) => { + if (exiting) + return process.exit(code ?? 0) + + if (signal) { + process.kill(process.pid, signal) + return + } + + process.exit(code ?? 1) +}) diff --git a/apps/stage-tamagotchi/src/main/services/electron/app.ts b/apps/stage-tamagotchi/src/main/services/electron/app.ts index 94e445ddcf..fe22bbec22 100644 --- a/apps/stage-tamagotchi/src/main/services/electron/app.ts +++ b/apps/stage-tamagotchi/src/main/services/electron/app.ts @@ -6,11 +6,13 @@ import { app, shell } from 'electron' import { isLinux, isMacOS, isWindows } from 'std-env' import { electron, electronAppOpenUserDataFolder, electronAppQuit } from '../../../shared/eventa' +import { createVisualChatDesktopService } from './visual-chat' export function createAppService(params: { context: ReturnType['context'], window: BrowserWindow }) { defineInvokeHandler(params.context, electron.app.isMacOS, () => isMacOS) defineInvokeHandler(params.context, electron.app.isWindows, () => isWindows) defineInvokeHandler(params.context, electron.app.isLinux, () => isLinux) + createVisualChatDesktopService(params) defineInvokeHandler(params.context, electronAppOpenUserDataFolder, async () => { const path = app.getPath('userData') const openResult = await shell.openPath(path) diff --git a/apps/stage-tamagotchi/src/main/services/electron/index.ts b/apps/stage-tamagotchi/src/main/services/electron/index.ts index 6b58098a74..64aa98b15e 100644 --- a/apps/stage-tamagotchi/src/main/services/electron/index.ts +++ b/apps/stage-tamagotchi/src/main/services/electron/index.ts @@ -2,4 +2,5 @@ export * from './app' export * from './auto-updater' export * from './powerMonitor' export * from './screen' +export * from './visual-chat' export * from './window' diff --git a/apps/stage-tamagotchi/src/main/services/electron/visual-chat.ts b/apps/stage-tamagotchi/src/main/services/electron/visual-chat.ts new file mode 100644 index 0000000000..41fb916652 --- /dev/null +++ b/apps/stage-tamagotchi/src/main/services/electron/visual-chat.ts @@ -0,0 +1,501 @@ +import type { ChildProcess } from 'node:child_process' + +import type { createContext } from '@moeru/eventa/adapters/electron/main' +import type { VisualChatDesktopSetupStatus, VisualChatDesktopSetupStep } from '@proj-airi/visual-chat-shared/electron' +import type { BrowserWindow } from 'electron' + +import process from 'node:process' + +import { spawn } from 'node:child_process' +import { existsSync } from 'node:fs' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +import { defineInvokeHandler } from '@moeru/eventa' +import { createTunnelPair, loadTunnelConfig as loadManagedTunnelConfig, pullModels, setupEngine, startNamedTunnels as startManagedNamedTunnels } from '@proj-airi/visual-chat-ops' +import { + electronVisualChatGetSetupStatus, + electronVisualChatRunSetup, + +} from '@proj-airi/visual-chat-shared/electron' +import { app } from 'electron' + +const FIXED_MODEL = 'openbmb/minicpm-v4.5:latest' +const GATEWAY_URL = 'http://127.0.0.1:6200' +const WORKER_URL = 'http://127.0.0.1:6201' +const FRONTEND_URL = 'http://127.0.0.1:5174' +const LOG_LIMIT = 160 +const STARTUP_TIMEOUT_MS = 20_000 +const STARTUP_POLL_INTERVAL_MS = 500 +const WORKSPACE_MARKER = 'pnpm-workspace.yaml' + +const CURRENT_DIR = dirname(fileURLToPath(import.meta.url)) + +interface TunnelHandle { close: () => void } + +type RuntimeMode + = | { kind: 'workspace', workspaceRoot: string } + | { kind: 'packaged', gatewayEntry: string, workerEntry: string } + +const state: { + status: VisualChatDesktopSetupStatus + runningPromise: Promise | null + gatewayChild: ChildProcess | null + workerChild: ChildProcess | null + tunnelHandle: TunnelHandle | null +} = { + status: { + available: false, + state: 'idle', + fixedModel: FIXED_MODEL, + gatewayUrl: GATEWAY_URL, + workerUrl: WORKER_URL, + steps: createDefaultSteps(), + logs: [], + updatedAt: Date.now(), + }, + runningPromise: null, + gatewayChild: null, + workerChild: null, + tunnelHandle: null, +} + +function createDefaultSteps(): VisualChatDesktopSetupStep[] { + return [ + { id: 'engine', label: 'Inference engine', status: 'pending', detail: 'Waiting for detection.' }, + { id: 'model', label: 'Fixed model', status: 'pending', detail: FIXED_MODEL }, + { id: 'gateway', label: 'Gateway service', status: 'pending', detail: GATEWAY_URL }, + { id: 'worker', label: 'Worker service', status: 'pending', detail: WORKER_URL }, + { id: 'tunnel', label: 'Public tunnel', status: 'pending', detail: 'Cloudflare quick tunnel for remote phone access' }, + ] +} + +function cloneStatus(): VisualChatDesktopSetupStatus { + return { + ...state.status, + steps: state.status.steps.map(step => ({ ...step })), + logs: [...state.status.logs], + } +} + +function updateStatus(patch: Partial) { + state.status = { + ...state.status, + ...patch, + updatedAt: Date.now(), + } +} + +function updateStep(id: VisualChatDesktopSetupStep['id'], patch: Partial) { + state.status = { + ...state.status, + steps: state.status.steps.map(step => step.id === id ? { ...step, ...patch } : step), + updatedAt: Date.now(), + } +} + +function resetSteps() { + updateStatus({ steps: createDefaultSteps(), error: undefined }) +} + +function appendLog(line: string) { + const trimmed = line.trim() + if (!trimmed) + return + + updateStatus({ + logs: [...state.status.logs, trimmed].slice(-LOG_LIMIT), + }) +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} + +function packageManagerCommand(args: string[]): [command: string, commandArgs: string[]] { + if (process.platform === 'win32') { + const command = process.env.ComSpec || 'cmd.exe' + return [command, ['/d', '/s', '/c', ['pnpm', ...args].join(' ')]] + } + + return ['pnpm', args] +} + +function findWorkspaceRoot(from: string): string | null { + let current = resolve(from) + + while (true) { + if (existsSync(join(current, WORKSPACE_MARKER))) + return current + + const parent = dirname(current) + if (parent === current) + return null + current = parent + } +} + +function resolveWorkspaceRoot(): string | null { + const candidates = [ + process.cwd(), + CURRENT_DIR, + ] + + for (const candidate of candidates) { + const root = findWorkspaceRoot(candidate) + if (root) + return root + } + + return null +} + +function resolvePackagedServiceEntry(packageName: '@proj-airi/visual-chat-gateway' | '@proj-airi/visual-chat-worker-minicpmo'): string | null { + const packagePath = packageName.split('/') + const candidates = [ + join(app.getAppPath(), 'node_modules', ...packagePath, 'dist', 'index.mjs'), + join(process.resourcesPath, 'app.asar', 'node_modules', ...packagePath, 'dist', 'index.mjs'), + join(process.resourcesPath, 'app.asar.unpacked', 'node_modules', ...packagePath, 'dist', 'index.mjs'), + ] + + return candidates.find(candidate => existsSync(candidate)) ?? null +} + +function resolveRuntimeMode(): RuntimeMode | null { + const workspaceRoot = resolveWorkspaceRoot() + if (workspaceRoot) + return { kind: 'workspace', workspaceRoot } + + const gatewayEntry = resolvePackagedServiceEntry('@proj-airi/visual-chat-gateway') + const workerEntry = resolvePackagedServiceEntry('@proj-airi/visual-chat-worker-minicpmo') + if (gatewayEntry && workerEntry) + return { kind: 'packaged', gatewayEntry, workerEntry } + + return null +} + +async function isUrlReachable(url: string, timeoutMs: number = 1500): Promise { + try { + const response = await fetch(url, { + signal: AbortSignal.timeout(timeoutMs), + }) + return response.ok + } + catch { + return false + } +} + +async function isOllamaServing(): Promise { + return isUrlReachable('http://127.0.0.1:11434/api/tags', 3000) +} + +async function hasFixedModel(): Promise { + try { + const response = await fetch('http://127.0.0.1:11434/api/tags', { + signal: AbortSignal.timeout(5000), + }) + if (!response.ok) + return false + + const payload = await response.json() as { models?: Array<{ name?: string }> } + return payload.models?.some(model => model.name === FIXED_MODEL) ?? false + } + catch { + return false + } +} + +async function waitForHealth(name: string, healthUrl: string): Promise { + const startedAt = Date.now() + + while (Date.now() - startedAt < STARTUP_TIMEOUT_MS) { + if (await isUrlReachable(healthUrl)) + return + await new Promise(resolve => setTimeout(resolve, STARTUP_POLL_INTERVAL_MS)) + } + + throw new Error(`${name} did not become healthy within ${Math.round(STARTUP_TIMEOUT_MS / 1000)}s`) +} + +async function ensureManagedService(options: { + id: 'gateway' | 'worker' + label: string + healthUrl: string + runtimeMode: RuntimeMode + packageFilter: string + env?: NodeJS.ProcessEnv +}): Promise { + if (await isUrlReachable(options.healthUrl)) { + updateStep(options.id, { status: 'done', detail: `${options.healthUrl} already healthy` }) + return + } + + updateStep(options.id, { status: 'running', detail: `Starting ${options.label}...` }) + appendLog(`[start] ${options.label}`) + + const child = (() => { + if (options.runtimeMode.kind === 'workspace') { + const [command, commandArgs] = packageManagerCommand(['-F', options.packageFilter, 'dev']) + return spawn(command, commandArgs, { + cwd: options.runtimeMode.workspaceRoot, + env: { + ...process.env, + ...options.env, + }, + stdio: ['ignore', 'pipe', 'pipe'], + }) + } + + const entryPath = options.id === 'gateway' + ? options.runtimeMode.gatewayEntry + : options.runtimeMode.workerEntry + return spawn(process.execPath, [entryPath], { + cwd: dirname(entryPath), + env: { + ...process.env, + ELECTRON_RUN_AS_NODE: '1', + ...options.env, + }, + stdio: ['ignore', 'pipe', 'pipe'], + }) + })() + + child.stdout?.setEncoding('utf-8') + child.stderr?.setEncoding('utf-8') + child.stdout?.on('data', chunk => appendLog(`[${options.label}] ${String(chunk)}`)) + child.stderr?.on('data', chunk => appendLog(`[${options.label}] ${String(chunk)}`)) + + const childKey = options.id === 'gateway' ? 'gatewayChild' : 'workerChild' + state[childKey] = child + + child.once('exit', (code, signal) => { + const detail = signal ? `signal ${signal}` : `code ${code ?? 0}` + appendLog(`[exit] ${options.label}: ${detail}`) + if (state.status.state !== 'ready' && state.status.state !== 'idle') { + updateStatus({ + state: 'error', + error: `${options.label} exited unexpectedly with ${detail}`, + }) + updateStep(options.id, { status: 'error', detail: `${options.label} exited unexpectedly with ${detail}` }) + } + }) + + await waitForHealth(options.label, options.healthUrl) + updateStep(options.id, { status: 'done', detail: `${options.healthUrl} is healthy` }) +} + +async function startPackagedTunnel(): Promise { + updateStep('tunnel', { status: 'running', detail: 'Starting managed tunnels from packaged visual-chat runtime...' }) + + const namedConfig = loadManagedTunnelConfig() + const handle = namedConfig + ? await startManagedNamedTunnels({ + frontendTarget: FRONTEND_URL, + gatewayTarget: GATEWAY_URL, + }) + : await createTunnelPair({ + frontendTarget: FRONTEND_URL, + gatewayTarget: GATEWAY_URL, + }) + + state.tunnelHandle = handle + updateStatus({ + tunnelFrontendUrl: handle.frontendUrl, + tunnelGatewayUrl: handle.gatewayUrl, + }) + updateStep('tunnel', { + status: 'done', + detail: `Frontend: ${handle.frontendUrl} | Gateway: ${handle.gatewayUrl}`, + }) + appendLog(`[tunnel] Managed tunnel URLs ready: frontend=${handle.frontendUrl} gateway=${handle.gatewayUrl}`) +} + +async function startTunnel(_runtimeMode: RuntimeMode): Promise { + if (state.tunnelHandle) { + updateStep('tunnel', { status: 'done', detail: 'Tunnel already running.' }) + return + } + + appendLog('[tunnel] Starting public tunnels for phone access...') + + try { + await startPackagedTunnel() + } + catch (error) { + const activeTunnelHandle = state.tunnelHandle as TunnelHandle | null + activeTunnelHandle?.close() + state.tunnelHandle = null + const msg = errorMessage(error) + appendLog(`[tunnel] Failed: ${msg}`) + updateStep('tunnel', { status: 'error', detail: `${msg}. Phone access is LAN-only.` }) + } +} + +async function refreshSetupStatusFromRuntime() { + const runtimeMode = resolveRuntimeMode() + const [engineReady, modelReady, gatewayReady, workerReady] = await Promise.all([ + isOllamaServing(), + hasFixedModel(), + isUrlReachable(`${GATEWAY_URL}/health`), + isUrlReachable(`${WORKER_URL}/health`), + ]) + + updateStatus({ + available: !!runtimeMode, + workspaceRoot: runtimeMode?.kind === 'workspace' ? runtimeMode.workspaceRoot : undefined, + }) + + updateStep('engine', { + status: engineReady ? 'done' : 'pending', + detail: engineReady ? 'Ollama is serving at http://127.0.0.1:11434' : 'Ollama is not serving yet.', + }) + updateStep('model', { + status: modelReady ? 'done' : 'pending', + detail: modelReady ? `${FIXED_MODEL} is installed.` : `${FIXED_MODEL} is missing.`, + }) + updateStep('gateway', { + status: gatewayReady ? 'done' : 'pending', + detail: gatewayReady ? `${GATEWAY_URL} is healthy.` : `${GATEWAY_URL} is not reachable.`, + }) + updateStep('worker', { + status: workerReady ? 'done' : 'pending', + detail: workerReady ? `${WORKER_URL} is healthy.` : `${WORKER_URL} is not reachable.`, + }) + + const tunnelRunning = !!state.tunnelHandle + const tunnelStep = state.status.steps.find(s => s.id === 'tunnel') + if (tunnelStep && tunnelStep.status !== 'done' && tunnelStep.status !== 'running') { + updateStep('tunnel', { + status: tunnelRunning ? 'running' : 'pending', + detail: tunnelRunning ? 'Tunnel process is active.' : 'Tunnel not started yet.', + }) + } + + if (engineReady && modelReady && gatewayReady && workerReady) { + updateStatus({ + state: 'ready', + error: undefined, + }) + } + else if (state.status.state === 'idle' || state.status.state === 'ready') { + updateStatus({ + state: 'checking', + error: undefined, + }) + } +} + +async function runSetupPipeline(): Promise { + if (state.runningPromise) + return state.runningPromise + + state.runningPromise = (async () => { + resetSteps() + appendLog('[setup] Visual Chat desktop setup started.') + updateStatus({ + state: 'checking', + error: undefined, + }) + + const runtimeMode = resolveRuntimeMode() + if (!runtimeMode) { + updateStatus({ + available: false, + state: 'error', + error: 'Cannot find either a development workspace root or a packaged visual-chat runtime.', + }) + return cloneStatus() + } + + updateStatus({ + available: true, + workspaceRoot: runtimeMode.kind === 'workspace' ? runtimeMode.workspaceRoot : undefined, + }) + + const engineReady = await isOllamaServing() + if (!engineReady) { + updateStatus({ state: 'installing-engine' }) + updateStep('engine', { status: 'running', detail: 'Installing or starting Ollama...' }) + await setupEngine() + } + if (!await isOllamaServing()) + throw new Error('Ollama is still not serving after setup-engine completed.') + updateStep('engine', { status: 'done', detail: 'Ollama is serving at http://127.0.0.1:11434' }) + + const modelReady = await hasFixedModel() + if (!modelReady) { + updateStatus({ state: 'pulling-model' }) + updateStep('model', { status: 'running', detail: `Pulling ${FIXED_MODEL}...` }) + await pullModels({ model: FIXED_MODEL }) + } + if (!await hasFixedModel()) + throw new Error(`${FIXED_MODEL} is still unavailable after pull-models completed.`) + updateStep('model', { status: 'done', detail: `${FIXED_MODEL} is ready.` }) + + updateStatus({ state: 'starting-services' }) + await ensureManagedService({ + id: 'worker', + label: 'Visual Chat worker', + healthUrl: `${WORKER_URL}/health`, + runtimeMode, + packageFilter: '@proj-airi/visual-chat-worker-minicpmo', + env: { + GATEWAY_URL, + VISUAL_CHAT_GATEWAY_URL: GATEWAY_URL, + WORKER_HOST: '127.0.0.1', + WORKER_PORT: '6201', + }, + }) + await ensureManagedService({ + id: 'gateway', + label: 'Visual Chat gateway', + healthUrl: `${GATEWAY_URL}/health`, + runtimeMode, + packageFilter: '@proj-airi/visual-chat-gateway', + env: { + VISUAL_CHAT_HOST: '127.0.0.1', + VISUAL_CHAT_PORT: '6200', + WORKER_URL, + }, + }) + + updateStatus({ state: 'starting-tunnel' }) + await startTunnel(runtimeMode) + + await refreshSetupStatusFromRuntime() + updateStatus({ + state: 'ready', + error: undefined, + }) + appendLog('[setup] Visual Chat desktop setup finished.') + return cloneStatus() + })() + .catch((error) => { + appendLog(`[error] ${errorMessage(error)}`) + updateStatus({ + state: 'error', + error: errorMessage(error), + }) + return cloneStatus() + }) + .finally(() => { + state.runningPromise = null + }) + + return state.runningPromise +} + +export function createVisualChatDesktopService(params: { context: ReturnType['context'], window: BrowserWindow }) { + void params.window + + defineInvokeHandler(params.context, electronVisualChatGetSetupStatus, async () => { + await refreshSetupStatusFromRuntime() + return cloneStatus() + }) + + defineInvokeHandler(params.context, electronVisualChatRunSetup, async (payload) => { + void payload + return runSetupPipeline() + }) +} diff --git a/apps/stage-tamagotchi/src/renderer/BrowserApp.vue b/apps/stage-tamagotchi/src/renderer/BrowserApp.vue new file mode 100644 index 0000000000..5c02c42c5f --- /dev/null +++ b/apps/stage-tamagotchi/src/renderer/BrowserApp.vue @@ -0,0 +1,12 @@ + + + diff --git a/apps/stage-tamagotchi/src/renderer/main.ts b/apps/stage-tamagotchi/src/renderer/main.ts index f3bda046fb..5dd0df6223 100644 --- a/apps/stage-tamagotchi/src/renderer/main.ts +++ b/apps/stage-tamagotchi/src/renderer/main.ts @@ -12,6 +12,7 @@ import { createRouter, createWebHashHistory } from 'vue-router' import { routes } from 'vue-router/auto-routes' import App from './App.vue' +import BrowserApp from './BrowserApp.vue' import { i18n } from './modules/i18n' @@ -36,6 +37,7 @@ import '@fontsource/m-plus-rounded-1c/index.css' import '@fontsource-variable/nunito/index.css' const pinia = createPinia() +const hasElectronRuntime = typeof window !== 'undefined' && !!window.electron?.ipcRenderer const router = createRouter({ history: createWebHashHistory(), @@ -43,7 +45,9 @@ const router = createRouter({ routes: setupLayouts(routes as RouteRecordRaw[]), }) -createApp(App) +// NOTICE: the phone entry is served by the tamagotchi renderer through a normal browser. +// Mount a browser-safe root whenever Electron preload APIs are unavailable. +createApp(hasElectronRuntime ? App : BrowserApp) .use(MotionPlugin) // TODO: Fix autoAnimatePlugin type error .use(autoAnimatePlugin as unknown as Plugin) @@ -52,3 +56,35 @@ createApp(App) .use(i18n) .use(Tres) .mount('#app') + +if (hasElectronRuntime) { + void setupElectronScreenCaptureForVisualChat() +} + +async function setupElectronScreenCaptureForVisualChat() { + try { + const { useVisualChatStore } = await import('@proj-airi/stage-ui/stores/modules/visual-chat') + const { setupElectronScreenCapture } = await import('@proj-airi/electron-screen-capture/renderer') + const { createContext } = await import('@moeru/eventa/adapters/electron/renderer') + + const ctx = createContext(window.electron!.ipcRenderer).context + const screenCaptureApi = setupElectronScreenCapture(ctx) + const store = useVisualChatStore(pinia) + + store.setScreenCaptureProvider(async () => { + return screenCaptureApi.selectWithSource( + (sources) => { + const screen = sources.find(s => s.id.startsWith('screen:')) + if (!screen) + throw new Error('No screen source found for capture') + return screen.id + }, + () => navigator.mediaDevices.getDisplayMedia({ video: true, audio: false }), + { sourcesOptions: { types: ['screen', 'window'] } }, + ) + }) + } + catch { + // Electron screen capture setup failed; fallback to standard getDisplayMedia + } +} diff --git a/apps/stage-tamagotchi/src/renderer/pages/settings/index.vue b/apps/stage-tamagotchi/src/renderer/pages/settings/index.vue index 0c016f2dfc..f73cb13302 100644 --- a/apps/stage-tamagotchi/src/renderer/pages/settings/index.vue +++ b/apps/stage-tamagotchi/src/renderer/pages/settings/index.vue @@ -1,27 +1,11 @@ + + diff --git a/apps/stage-visual-chat-ops/package.json b/apps/stage-visual-chat-ops/package.json new file mode 100644 index 0000000000..5d7f182c22 --- /dev/null +++ b/apps/stage-visual-chat-ops/package.json @@ -0,0 +1,27 @@ +{ + "name": "@proj-airi/stage-visual-chat-ops", + "type": "module", + "version": "0.0.1", + "private": true, + "description": "Operations dashboard for AIRI visual chat", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/moeru-ai/airi.git", + "directory": "apps/stage-visual-chat-ops" + }, + "scripts": { + "dev": "vite", + "build": "vite build", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@proj-airi/visual-chat-sdk": "workspace:^", + "pinia": "catalog:", + "vue": "catalog:" + }, + "devDependencies": { + "@vitejs/plugin-vue": "^6.0.5", + "vite": "catalog:" + } +} diff --git a/apps/stage-visual-chat-ops/src/App.vue b/apps/stage-visual-chat-ops/src/App.vue new file mode 100644 index 0000000000..5e54d23106 --- /dev/null +++ b/apps/stage-visual-chat-ops/src/App.vue @@ -0,0 +1,45 @@ + + + + + diff --git a/apps/stage-visual-chat-ops/src/main.ts b/apps/stage-visual-chat-ops/src/main.ts new file mode 100644 index 0000000000..d595f296e2 --- /dev/null +++ b/apps/stage-visual-chat-ops/src/main.ts @@ -0,0 +1,8 @@ +import { createPinia } from 'pinia' +import { createApp } from 'vue' + +import App from './App.vue' + +const app = createApp(App) +app.use(createPinia()) +app.mount('#app') diff --git a/apps/stage-visual-chat-ops/vite.config.ts b/apps/stage-visual-chat-ops/vite.config.ts new file mode 100644 index 0000000000..68085bc1a6 --- /dev/null +++ b/apps/stage-visual-chat-ops/vite.config.ts @@ -0,0 +1,10 @@ +import vue from '@vitejs/plugin-vue' + +import { defineConfig } from 'vite' + +export default defineConfig({ + plugins: [vue()], + server: { + port: 5180, + }, +}) diff --git a/apps/stage-web/vite.config.ts b/apps/stage-web/vite.config.ts index 925b00af96..102f86677e 100644 --- a/apps/stage-web/vite.config.ts +++ b/apps/stage-web/vite.config.ts @@ -21,6 +21,13 @@ import { VitePWA } from 'vite-plugin-pwa' const stageUIAssetsRoot = resolve(join(import.meta.dirname, '..', '..', 'packages', 'stage-ui', 'src', 'assets')) const sharedCacheDir = resolve(join(import.meta.dirname, '..', '..', '.cache')) +const additionalAllowedRemoteHosts = (env.AIRI_VISUAL_CHAT_ALLOWED_HOSTS || '') + .split(',') + .map(host => host.trim()) + .filter(Boolean) +const devServerAllowedHosts: true | string[] = additionalAllowedRemoteHosts.length > 0 + ? [...new Set(['.trycloudflare.com', ...additionalAllowedRemoteHosts])] + : true export default defineConfig({ optimizeDeps: { @@ -58,9 +65,11 @@ export default defineConfig({ '@proj-airi/stage-pages': resolve(join(import.meta.dirname, '..', '..', 'packages', 'stage-pages', 'src')), '@proj-airi/stage-shared': resolve(join(import.meta.dirname, '..', '..', 'packages', 'stage-shared', 'src')), '@proj-airi/stage-layouts': resolve(join(import.meta.dirname, '..', '..', 'packages', 'stage-layouts', 'src')), + '@proj-airi/visual-chat-shared/electron': resolve(join(import.meta.dirname, '..', '..', 'packages', 'visual-chat-shared', 'src', 'electron.ts')), }, }, server: { + allowedHosts: devServerAllowedHosts, fs: { // To mute errors like: // The request id ".../node_modules/@fontsource/sniglet/files/sniglet-latin-400-normal.woff" is outside of Vite serving allow list. diff --git a/docs/visual-chat/architecture.md b/docs/visual-chat/architecture.md new file mode 100644 index 0000000000..9c97e46dc0 --- /dev/null +++ b/docs/visual-chat/architecture.md @@ -0,0 +1,71 @@ +# Visual Chat Architecture + +## Six-Layer Architecture + +``` ++------------------------------------------------------------+ +| Layer 6: Distribution & Observability | +| (visual-chat-ops, visual-chat-observability) | ++------------------------------------------------------------+ +| Layer 5: Output Adaptation | +| (visual-chat-sdk, plugins, stage-visual-chat-ops) | ++------------------------------------------------------------+ +| Layer 4: Fixed Worker Backend | +| (visual-chat-worker-minicpmo -> ollama-lite) | ++------------------------------------------------------------+ +| Layer 3: Normalization & Scheduling | +| (visual-chat-runtime, visual-chat-gateway) | ++------------------------------------------------------------+ +| Layer 2: Transport & Session | +| (AIRI session WS) | ++------------------------------------------------------------+ +| Layer 1: Signal Acquisition | +| (browser camera/screen, phone page, desktop page) | ++------------------------------------------------------------+ +``` + +## Fixed Worker Pipeline + +- worker proxies `infer-stream` to Ollama +- gateway keeps persisted conversation history, the latest frame state, and a hidden rolling scene memory in AIRI +- AIRI presents one shared realtime conversation feed back to desktop and phone clients +- LiveKit token and webhook routes exist as optional integration surfaces, but they are not the primary media path for the current shipped desktop + phone flow + +## Fixed Interaction Mode + +- `vision-text-realtime` +- The current shipped path is realtime screen/camera frame streaming plus typed text prompts. +- Native duplex audio transport is intentionally not enabled in `ollama-lite` mode. + +## Context Engineering + +The fixed context path is intentionally narrow: + +1. user turns are persisted as explicit dialogue messages +2. manual `Observe` produces a visible assistant reply +3. `Continuous Observation` updates a hidden rolling scene memory instead of appending public assistant chatter +4. every user-visible inference reads: + - the latest active frame + - recent dialogue history + - the rolling scene memory +5. auto-observation does not replay dialogue history; it only refreshes the hidden rolling scene memory +6. scene memory updates are deduplicated before persistence so unchanged notes are not re-fed as fresh context +7. worker output is sanitized before it is shown + +This keeps the visible conversation readable while still giving the model continuity across source switches and repeated scene questions. + +## Record Management + +- Every session writes metadata and `messages.json` under the visual chat data directory. +- Metadata also stores a persisted scene-memory timeline so users can inspect what private continuity notes were saved. +- Session records survive process restarts. +- Restoring a saved conversation recreates an active AIRI session with the same `sessionId`. +- The settings page shows saved conversations and lets the user continue one directly. + +## Data Flow + +1. Desktop and phone clients publish camera and screen data into the AIRI session websocket. +2. The AIRI gateway keeps session state, merges sources, maintains persisted dialogue history, and updates rolling scene memory. +3. The gateway selects the active frame from the currently chosen input source. +4. The worker sends that frame, the text turn, recent dialogue history, and the rolling scene memory instructions into the fixed Ollama model path. +5. AIRI broadcasts response chunks back to subscribed clients and persists completed dialogue turns. diff --git a/docs/visual-chat/context-and-records.md b/docs/visual-chat/context-and-records.md new file mode 100644 index 0000000000..cfe37aea0f --- /dev/null +++ b/docs/visual-chat/context-and-records.md @@ -0,0 +1,40 @@ +# Visual Chat Context And Records + +## One Session, One Context Path + +Visual Chat now uses one fixed context path: + +1. the selected camera or screen source provides the latest live frame +2. typed text is stored as explicit user turns +3. manual `Observe` produces a visible assistant reply +4. `Continuous Observation` updates a hidden rolling scene memory instead of adding noisy assistant turns +5. every visible assistant reply reads: + - the latest live frame + - recent dialogue history + - the rolling scene memory +6. every auto-observation pass reads: + - the newest live frame + - the current rolling scene memory + - no visible dialogue replay + +This keeps the public conversation readable while still preserving continuity across repeated scene questions. + +## Output Hygiene + +- worker output is sanitized before display +- system instructions explicitly forbid exposing hidden reasoning or internal prompts +- when the scene is uncertain, the assistant should say it is uncertain instead of inventing details + +## Record Storage + +Each visual chat session is persisted under the visual chat data directory: + +- `metadata.json`: title, summary, timestamps, rolling scene memory, and the scene-memory timeline shown in the settings page +- `messages.json`: persisted dialogue turns + +These records are used for: + +- restoring previous conversations +- showing saved conversations in the settings page +- continuing a prior session with the same `sessionId` + diff --git a/docs/visual-chat/quickstart.md b/docs/visual-chat/quickstart.md new file mode 100644 index 0000000000..65b4f35025 --- /dev/null +++ b/docs/visual-chat/quickstart.md @@ -0,0 +1,164 @@ +# Visual Chat Quickstart + +## What You Get + +A fully local realtime visual chat pipeline in AIRI: + +- **Video input**: desktop camera, desktop screen capture, or phone camera +- **Text input**: typed messages from desktop or phone +- **Response output**: shared realtime conversation stream +- **Session state**: rolling scene memory, persisted conversation records, session continuity + +## Fixed Pipeline + +| Component | Value | +|-----------|-------| +| Backend | Ollama | +| Model | `openbmb/minicpm-v4.5:latest` | +| Interaction mode | `vision-text-realtime` | +| Context window | Last 6 dialogue turns + 800-char rolling scene memory | +| Target hardware | 16 GB VRAM (GPU) or 16 GB RAM (CPU-only, slower) | + +## Prerequisites + +- Node.js >= 18 +- pnpm >= 10 + +Ollama is installed automatically by the setup pipeline if not already present. + +## Setup Paths + +### Path A: Desktop App (Recommended) + +```bash +pnpm dev:tamagotchi +``` + +This single command: + +1. Builds and starts the Electron desktop app (`stage-tamagotchi`) +2. Starts the gateway (`:6200`) and worker (`:6201`) services +3. Detects/installs Ollama and pulls the model on first run +4. Generates public HTTPS/WSS phone entry URLs via Cloudflare quick tunnel +5. Clears stale processes and endpoint files from previous runs + +Open `Settings -> Modules -> Visual Chat` in the desktop app. The **Setup Checklist** shows the status of each component. Click **Run Setup** if any step is not ready. + +For LAN-only (no public tunnel): + +```bash +pnpm dev:tamagotchi:local +``` + +### Path B: CLI Manual Setup + +#### 1. Check Environment + +```bash +pnpm -F @proj-airi/visual-chat-ops doctor:visual-chat +``` + +#### 2. Install Ollama + Pull Model + +```bash +pnpm -F @proj-airi/visual-chat-ops setup-engine +pnpm -F @proj-airi/visual-chat-ops pull-models --model openbmb/minicpm-v4.5:latest +``` + +#### 3. Configure Environment + +Copy `.env.example` in `services/visual-chat-gateway/` and `services/visual-chat-worker-minicpmo/` to `.env`. + +Key variables: + +```bash +OLLAMA_HOST=http://127.0.0.1:11434 +``` + +`OLLAMA_MODEL` is intentionally fixed to `openbmb/minicpm-v4.5:latest` in the current shipped worker path. + +`LIVEKIT_URL`, `LIVEKIT_API_KEY`, and `LIVEKIT_API_SECRET` are only needed if you are explicitly using the room/token/webhook integration surfaces. The core desktop + phone visual chat flow runs through the AIRI gateway session websocket and fixed Ollama worker path. + +#### 4. Start Services + +```bash +pnpm -F @proj-airi/visual-chat-ops start:local +``` + +This starts Ollama, the gateway, and the worker, then prints URLs. + +## Phone Access + +Three ways to connect a phone: + +1. **Same WiFi (LAN)**: The desktop page shows a LAN IP under *Phone entry*. Works if both devices are on the same network. Phone camera access may still require HTTPS depending on the browser. +2. **Cloudflare Quick Tunnel**: `pnpm dev:tamagotchi` auto-creates a `*.trycloudflare.com` HTTPS URL. No registration needed. URL changes on restart. +3. **Fixed host override**: In the *Session* section, set a fixed IP/hostname under *Fixed host override* to lock the phone URL across restarts. + +## Platform Support + +| OS | GPU | CPU-only | Notes | +|----|-----|----------|-------| +| Windows 10/11 | NVIDIA (CUDA), AMD (ROCm) | Yes | Ollama handles GPU detection | +| macOS (Apple Silicon) | Metal | Yes | Native Ollama support | +| macOS (Intel) | - | Yes | CPU inference only | +| Linux | NVIDIA (CUDA), AMD (ROCm) | Yes | Ollama handles GPU detection | + +GPU is recommended for acceptable response latency. CPU-only works but inference is significantly slower. + +## UI Sections + +All sections in the desktop and phone UIs are collapsible: + +| Section | Description | +|---------|-------------| +| Setup Checklist | Pre-flight checks for gateway, model, session, input source | +| Desktop Setup | Electron auto-setup pipeline status and controls | +| Session | Create/join/leave sessions, phone entry URL, participant info | +| Saved Conversations | Persisted conversation records with restore and per-record delete | +| Input Mode | Camera/screen/phone source selection with device pickers | +| Rolling Scene Memory | Hidden memory updated by continuous observation | +| Context State | Live history window (last 6 turns) + session record metadata | + +## Context Management + +The context sent to the model for each inference: + +- **System prompt**: role + visual source hint + scene memory (~4 lines) +- **Rolling scene memory**: up to 800 characters of factual scene notes +- **Dialogue history**: last 6 user/assistant turns +- **Current frame**: the newest video frame as a base64 image + +Memory timeline keeps up to 4 snapshots. Scene memory is deduplicated so unchanged observations are not re-inserted. + +## Connection Stability + +The WebSocket connection between the UI and gateway uses exponential backoff reconnection (1s, 2s, 4s, ... up to 30s). On reconnect, the client re-subscribes to the active session and re-hydrates message history. A "Reconnecting..." indicator appears in both desktop and phone UIs during reconnection. + +## API Endpoints + +| Method | Path | Description | +|--------|------|-------------| +| POST | `/api/sessions` | Create new session | +| GET | `/api/sessions` | List sessions | +| GET | `/api/sessions/:id` | Get session details | +| GET | `/api/sessions/:id/messages` | Get session messages | +| DELETE | `/api/sessions/:id` | End session | +| DELETE | `/api/sessions/:id/record` | Delete persisted conversation record | +| POST | `/api/sessions/:id/switch-source` | Switch active source | +| GET | `/api/session-records` | List persisted conversation records | +| POST | `/api/session-records/:id/restore` | Restore a persisted conversation | +| GET | `/api/worker/health` | Worker bridge health check | +| POST | `/api/worker/infer-stream` | Streaming worker proxy | +| GET | `/health` | Gateway health check | +| GET | `/api/diagnostics` | System diagnostics | +| WS | `/ws` | Realtime session control/state | + +## Runtime Notes + +- Phone camera access requires HTTPS or a secure webview. +- The current shipped input path is camera/screen frames plus typed text prompts. Raw browser microphone audio is not streamed into the worker. +- Continuous Observation updates the hidden rolling scene memory without filling the visible conversation. +- Screen capture becomes the active inference source when desktop screen mode is selected. +- Persisted conversations can be restored and continued, not just viewed. +- Admin endpoints such as diagnostics, session record management, and worker proxy routes require a local gateway access token. Shared phone entry URLs carry session-scoped access only. diff --git a/examples/visual-chat-local-5090/.env.example b/examples/visual-chat-local-5090/.env.example new file mode 100644 index 0000000000..db96390e5b --- /dev/null +++ b/examples/visual-chat-local-5090/.env.example @@ -0,0 +1,10 @@ +# Minimal local setup: phone + laptop on single GPU (e.g. RTX 5090) +VISUAL_CHAT_PORT=6200 +WORKER_PORT=6201 +LIVEKIT_URL=ws://localhost:7880 +LIVEKIT_API_KEY=devkey +LIVEKIT_API_SECRET=secret +LLAMACPP_OMNI_BIN=/path/to/llama-omni-cli +MODEL_DIR=/path/to/models/MiniCPM-o-2_6-Q4_K_M.gguf +GPU_LAYERS=999 +LOG_LEVEL=debug diff --git a/examples/visual-chat-local-5090/package.json b/examples/visual-chat-local-5090/package.json new file mode 100644 index 0000000000..b5126b0815 --- /dev/null +++ b/examples/visual-chat-local-5090/package.json @@ -0,0 +1,7 @@ +{ + "name": "@proj-airi/visual-chat-local-5090", + "type": "module", + "version": "0.0.1", + "private": true, + "description": "Minimal local demo: phone + laptop + single GPU" +} diff --git a/examples/visual-chat-mobile-laptop-room/.env.example b/examples/visual-chat-mobile-laptop-room/.env.example new file mode 100644 index 0000000000..fcfe219f89 --- /dev/null +++ b/examples/visual-chat-mobile-laptop-room/.env.example @@ -0,0 +1,12 @@ +# Three-source demo: phone camera, laptop camera, laptop screen share +VISUAL_CHAT_PORT=6200 +WORKER_PORT=6201 +LIVEKIT_URL=ws://192.168.1.100:7880 +LIVEKIT_API_KEY=devkey +LIVEKIT_API_SECRET=secret +LLAMACPP_OMNI_BIN=/path/to/llama-omni-cli +MODEL_DIR=/path/to/models/MiniCPM-o-2_6-Q4_K_M.gguf +GPU_LAYERS=999 +# Enable auto-source switching for demo +VISUAL_CHAT_AUTO_SOURCE_SWITCH=true +LOG_LEVEL=info diff --git a/examples/visual-chat-mobile-laptop-room/package.json b/examples/visual-chat-mobile-laptop-room/package.json new file mode 100644 index 0000000000..a376b5cebf --- /dev/null +++ b/examples/visual-chat-mobile-laptop-room/package.json @@ -0,0 +1,7 @@ +{ + "name": "@proj-airi/visual-chat-mobile-laptop-room", + "type": "module", + "version": "0.0.1", + "private": true, + "description": "Demo: phone camera, laptop camera, and screen share three-source switching" +} diff --git a/examples/visual-chat-production-sample/.env.example b/examples/visual-chat-production-sample/.env.example new file mode 100644 index 0000000000..d56519e61b --- /dev/null +++ b/examples/visual-chat-production-sample/.env.example @@ -0,0 +1,11 @@ +# Production deployment reference +VISUAL_CHAT_PORT=6200 +WORKER_PORT=6201 +LIVEKIT_URL=wss://livekit.example.com +LIVEKIT_API_KEY=prod-key +LIVEKIT_API_SECRET=prod-secret +LLAMACPP_OMNI_BIN=/opt/airi/bin/llama-omni-cli +MODEL_DIR=/opt/airi/models/MiniCPM-o-2_6-Q4_K_M.gguf +GPU_LAYERS=999 +VISUAL_CHAT_RECORDING=true +LOG_LEVEL=warn diff --git a/examples/visual-chat-production-sample/package.json b/examples/visual-chat-production-sample/package.json new file mode 100644 index 0000000000..b3184befb3 --- /dev/null +++ b/examples/visual-chat-production-sample/package.json @@ -0,0 +1,7 @@ +{ + "name": "@proj-airi/visual-chat-production-sample", + "type": "module", + "version": "0.0.1", + "private": true, + "description": "Production deployment reference with env, compose, retention, and health checks" +} diff --git a/package.json b/package.json index d52e632c2f..603c75fd73 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,12 @@ "license": "MIT", "scripts": { "postinstall": "pnpm exec simple-git-hooks && pnpm run build:packages", - "dev": "pnpm -r -F @proj-airi/stage-web dev", + "dev": "pnpm -F @proj-airi/visual-chat-ops stop && pnpm -r -F @proj-airi/stage-web -F @proj-airi/visual-chat-gateway -F @proj-airi/visual-chat-worker-minicpmo --parallel dev", + "dev:web-only": "pnpm -r -F @proj-airi/stage-web dev", + "dev:visual-chat": "pnpm -F @proj-airi/visual-chat-ops stop && pnpm -r -F @proj-airi/visual-chat-gateway -F @proj-airi/visual-chat-worker-minicpmo --parallel dev", + "share:visual-chat:web": "pnpm -F @proj-airi/visual-chat-ops share:web", + "share:visual-chat:tamagotchi": "pnpm -F @proj-airi/visual-chat-ops share:tamagotchi", + "setup:visual-chat": "pnpm -F @proj-airi/visual-chat-ops setup-engine && pnpm -F @proj-airi/visual-chat-ops pull-models", "dev:docs": "pnpm -rF @proj-airi/docs run dev", "dev:ui": "pnpm -rF @proj-airi/stage-ui run story:dev", "dev:web": "pnpm -rF @proj-airi/stage-web run dev", @@ -21,7 +26,8 @@ "dev:pocket:ios": "pnpm -rF @proj-airi/stage-pocket run dev:ios", "dev:pocket:android": "pnpm -rF @proj-airi/stage-pocket run dev:android", "dev:server": "pnpm -rF @proj-airi/server-runtime run dev", - "dev:tamagotchi": "pnpm -rF @proj-airi/stage-tamagotchi run dev", + "dev:tamagotchi": "pnpm -F @proj-airi/visual-chat-ops dev:tamagotchi", + "dev:tamagotchi:local": "pnpm -F @proj-airi/visual-chat-ops stop && pnpm -r -F @proj-airi/stage-tamagotchi -F @proj-airi/visual-chat-gateway -F @proj-airi/visual-chat-worker-minicpmo --parallel dev", "dev:apps": "pnpm -rF=\"./apps/*\" run --parallel dev", "dev:packages": "pnpm -rF=\"./packages/*\" --parallel run dev", "build": "turbo run build -F=\"./packages/*\" -F=\"./apps/*\"", diff --git a/packages/stage-pages/package.json b/packages/stage-pages/package.json index efd865941f..bd93458e24 100644 --- a/packages/stage-pages/package.json +++ b/packages/stage-pages/package.json @@ -31,6 +31,9 @@ "@proj-airi/stage-ui-live2d": "workspace:*", "@proj-airi/stage-ui-three": "workspace:*", "@proj-airi/ui": "workspace:*", + "@proj-airi/visual-chat-protocol": "workspace:^", + "@proj-airi/visual-chat-sdk": "workspace:^", + "@proj-airi/visual-chat-shared": "workspace:^", "@shopify/draggable": "catalog:", "@stdlib/string-base-kebabcase": "^0.2.3", "@vueuse/core": "^14.2.1", diff --git a/packages/stage-pages/src/pages/devtools/visual-chat/index.vue b/packages/stage-pages/src/pages/devtools/visual-chat/index.vue new file mode 100644 index 0000000000..ac1547e90e --- /dev/null +++ b/packages/stage-pages/src/pages/devtools/visual-chat/index.vue @@ -0,0 +1,225 @@ + + + + + +meta: + layout: settings + title: Visual Chat + subtitleKey: tamagotchi.settings.devtools.title + stageTransition: + name: slide + diff --git a/packages/stage-pages/src/pages/devtools/visual-chat/sessions.vue b/packages/stage-pages/src/pages/devtools/visual-chat/sessions.vue new file mode 100644 index 0000000000..cff0a33fc1 --- /dev/null +++ b/packages/stage-pages/src/pages/devtools/visual-chat/sessions.vue @@ -0,0 +1,265 @@ + + + + + +meta: + layout: settings + title: Visual Chat Sessions + subtitleKey: tamagotchi.settings.devtools.title + stageTransition: + name: slide + diff --git a/packages/stage-pages/src/pages/devtools/visual-chat/workers.vue b/packages/stage-pages/src/pages/devtools/visual-chat/workers.vue new file mode 100644 index 0000000000..b6e27210e4 --- /dev/null +++ b/packages/stage-pages/src/pages/devtools/visual-chat/workers.vue @@ -0,0 +1,234 @@ + + + + + +meta: + layout: settings + title: Visual Chat Workers + subtitleKey: tamagotchi.settings.devtools.title + stageTransition: + name: slide + diff --git a/packages/stage-pages/src/pages/settings/index.vue b/packages/stage-pages/src/pages/settings/index.vue index d7fe9e1ab7..9b74ed758f 100644 --- a/packages/stage-pages/src/pages/settings/index.vue +++ b/packages/stage-pages/src/pages/settings/index.vue @@ -1,31 +1,14 @@ + + + + + + +meta: + layout: settings + title: Visual Chat + subtitleKey: settings.title + stageTransition: + name: slide + diff --git a/packages/stage-pages/src/pages/visual-chat/components/CollapsibleSection.vue b/packages/stage-pages/src/pages/visual-chat/components/CollapsibleSection.vue new file mode 100644 index 0000000000..9e3ecabe5e --- /dev/null +++ b/packages/stage-pages/src/pages/visual-chat/components/CollapsibleSection.vue @@ -0,0 +1,42 @@ + + + diff --git a/packages/stage-pages/src/pages/visual-chat/phone.vue b/packages/stage-pages/src/pages/visual-chat/phone.vue new file mode 100644 index 0000000000..43a2f76900 --- /dev/null +++ b/packages/stage-pages/src/pages/visual-chat/phone.vue @@ -0,0 +1,807 @@ + + + + + + + +meta: + layout: plain + title: Visual Chat Phone + diff --git a/packages/stage-pages/tsconfig.json b/packages/stage-pages/tsconfig.json index d9d96e750a..c4221a3be9 100644 --- a/packages/stage-pages/tsconfig.json +++ b/packages/stage-pages/tsconfig.json @@ -1,50 +1,56 @@ -{ - "compilerOptions": { - "target": "ESNext", - "jsx": "preserve", - "lib": [ - "DOM", - "DOM.AsyncIterable", - "ESNext" - ], - "module": "ESNext", - "moduleResolution": "Bundler", - "paths": { - "@proj-airi/stage-ui/*": [ - "../../packages/stage-ui/src/*" - ] - }, - "resolveJsonModule": true, - "types": [ - "vitest", - "vite/client", - "unplugin-info/client" - ], - "allowJs": true, - "strict": true, - "strictNullChecks": true, - "noUnusedLocals": true, - "noEmit": true, - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "isolatedModules": true, - "verbatimModuleSyntax": true, - "skipLibCheck": true - }, - "vueCompilerOptions": { - "plugins": [ - "@vue-macros/volar/define-models", - "@vue-macros/volar/define-slots" - ], - "skipTemplateCodegen": false - }, - "include": [ - "src/**/*.ts", - "src/**/*.d.ts", - "src/**/*.vue" - ], - "exclude": [ - "dist", - "node_modules" - ] -} +{ + "compilerOptions": { + "target": "ESNext", + "jsx": "preserve", + "lib": [ + "DOM", + "DOM.AsyncIterable", + "ESNext" + ], + "module": "ESNext", + "moduleResolution": "Bundler", + "paths": { + "@proj-airi/visual-chat-shared": [ + "../../packages/visual-chat-shared/src/index.ts" + ], + "@proj-airi/visual-chat-shared/*": [ + "../../packages/visual-chat-shared/src/*" + ], + "@proj-airi/stage-ui/*": [ + "../../packages/stage-ui/src/*" + ] + }, + "resolveJsonModule": true, + "types": [ + "vitest", + "vite/client", + "unplugin-info/client" + ], + "allowJs": true, + "strict": true, + "strictNullChecks": true, + "noUnusedLocals": true, + "noEmit": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": true, + "verbatimModuleSyntax": true, + "skipLibCheck": true + }, + "vueCompilerOptions": { + "plugins": [ + "@vue-macros/volar/define-models", + "@vue-macros/volar/define-slots" + ], + "skipTemplateCodegen": false + }, + "include": [ + "src/**/*.ts", + "src/**/*.d.ts", + "src/**/*.vue" + ], + "exclude": [ + "dist", + "node_modules" + ] +} diff --git a/packages/stage-ui/package.json b/packages/stage-ui/package.json index 29cb3e02b5..90cc38fd2f 100644 --- a/packages/stage-ui/package.json +++ b/packages/stage-ui/package.json @@ -77,6 +77,8 @@ "@proj-airi/stage-ui-live2d": "workspace:^", "@proj-airi/stage-ui-three": "workspace:^", "@proj-airi/ui": "workspace:^", + "@proj-airi/visual-chat-protocol": "workspace:^", + "@proj-airi/visual-chat-sdk": "workspace:^", "@ricky0123/vad-web": "^0.0.30", "@shikijs/rehype": "^4.0.2", "@shopify/draggable": "catalog:", @@ -122,6 +124,7 @@ "rehype-parse": "^9.0.1", "rehype-stringify": "^10.0.1", "reka-ui": "^2.9.2", + "remark-breaks": "catalog:", "remark-math": "^6.0.0", "remark-parse": "^11.0.0", "remark-rehype": "^11.1.2", diff --git a/packages/stage-ui/src/components/markdown/markdown-renderer.vue b/packages/stage-ui/src/components/markdown/markdown-renderer.vue index ffa1455682..2564cf82c4 100644 --- a/packages/stage-ui/src/components/markdown/markdown-renderer.vue +++ b/packages/stage-ui/src/components/markdown/markdown-renderer.vue @@ -1,105 +1,115 @@ - - - - - + + + + + diff --git a/packages/stage-ui/src/composables/markdown.ts b/packages/stage-ui/src/composables/markdown.ts index ebc1e92f45..aead6d353f 100644 --- a/packages/stage-ui/src/composables/markdown.ts +++ b/packages/stage-ui/src/composables/markdown.ts @@ -1,147 +1,150 @@ -import type { RehypeShikiOptions } from '@shikijs/rehype' -import type { BundledLanguage } from 'shiki' -import type { Processor } from 'unified' - -import rehypeShiki from '@shikijs/rehype' -import rehypeKatex from 'rehype-katex' -import RehypeStringify from 'rehype-stringify' -import remarkMath from 'remark-math' -import RemarkParse from 'remark-parse' -import RemarkRehype from 'remark-rehype' - -import { defaultPerfTracer } from '@proj-airi/stage-shared' -import { unified } from 'unified' - -// Define a specific, compatible type for our processor to ensure type safety. -type MarkdownProcessor = Processor - -const processorCache = new Map>() -const langRegex = /```(.{2,})\s/g - -function extractLangs(markdown: string): BundledLanguage[] { - const matches = markdown.matchAll(langRegex) - const langs = new Set() - langs.add('python') - for (const match of matches) { - if (match[1]) - langs.add(match[1] as BundledLanguage) - } - return [...langs] -} - -function measuredKatex(options?: Parameters[0]) { - const transform = rehypeKatex(options) - return (tree: any, file: any) => { - const start = performance.now() - const length = typeof file?.value === 'string' ? file.value.length : undefined - try { - return transform(tree, file) - } - finally { - defaultPerfTracer.emit({ - tracerId: 'markdown', - name: 'process.katex', - ts: start, - duration: performance.now() - start, - meta: { length }, - }) - } - } -} - -async function createProcessor(langs: BundledLanguage[]): Promise { - const options: RehypeShikiOptions = { - themes: { - light: 'github-light', - dark: 'github-dark', - }, - langs, - defaultLanguage: langs[0] || 'python', - } - - return unified() - .use(RemarkParse) - .use(remarkMath) - .use(RemarkRehype) - .use(measuredKatex, { output: 'mathml' }) - .use(rehypeShiki, options) - .use(RehypeStringify) -} - -function getProcessor(langs: BundledLanguage[]): Promise { - // The cache key should be consistent, so we sort the languages. - const cacheKey = [...langs].sort().join(',') - - if (!processorCache.has(cacheKey)) { - const processorPromise = createProcessor(langs) - processorCache.set(cacheKey, processorPromise) - } - - return processorCache.get(cacheKey)! -} - -export function useMarkdown() { - const fallbackProcessor = unified() - .use(RemarkParse) - .use(remarkMath) - .use(RemarkRehype) - .use(measuredKatex, { output: 'mathml' }) - .use(RehypeStringify) - - return { - process: async (markdown: string): Promise => { - const hasCodeFence = /`{3,}/.test(markdown) - const meta = { length: markdown.length, hasCodeFence } - - return defaultPerfTracer.withMeasure('markdown', 'process', async () => { - try { - // A quick check for code fences. If none, use the fast fallback. - if (!hasCodeFence) { - return defaultPerfTracer.withMeasure('markdown', 'process.pipeline.basic', () => { - return fallbackProcessor.processSync(markdown).toString() - }, meta) - } - - const langs = extractLangs(markdown) - - // Always ensure 'python' is loaded as it's our default. - const langSet = new Set(langs) - langSet.add('python') - const languagesToLoad = Array.from(langSet) - - const processor = await getProcessor(languagesToLoad) - const result = await defaultPerfTracer.withMeasure('markdown', 'process.pipeline.rich', () => processor.process(markdown), meta) - return result.toString() - } - catch (error) { - console.warn( - 'Failed to process markdown with syntax highlighting, falling back to basic processing:', - error, - ) - // Fallback to basic processor without highlighting - return defaultPerfTracer.withMeasure('markdown', 'process.pipeline.fallback', () => { - return fallbackProcessor.processSync(markdown).toString() - }, { ...meta, fallback: true }) - } - }, meta) - }, - - // Synchronous version for backward compatibility - processSync: (markdown: string): string => { - const start = performance.now() - const output = fallbackProcessor - .processSync(markdown) - .toString() - - defaultPerfTracer.emit({ - tracerId: 'markdown', - name: 'process.pipeline.sync', - ts: start, - duration: performance.now() - start, - meta: { length: markdown.length }, - }) - - return output - }, - } -} +import type { RehypeShikiOptions } from '@shikijs/rehype' +import type { BundledLanguage } from 'shiki' +import type { Processor } from 'unified' + +import rehypeShiki from '@shikijs/rehype' +import rehypeKatex from 'rehype-katex' +import RehypeStringify from 'rehype-stringify' +import remarkBreaks from 'remark-breaks' +import remarkMath from 'remark-math' +import RemarkParse from 'remark-parse' +import RemarkRehype from 'remark-rehype' + +import { defaultPerfTracer } from '@proj-airi/stage-shared' +import { unified } from 'unified' + +// Define a specific, compatible type for our processor to ensure type safety. +type MarkdownProcessor = Processor + +const processorCache = new Map>() +const langRegex = /```(.{2,})\s/g + +function extractLangs(markdown: string): BundledLanguage[] { + const matches = markdown.matchAll(langRegex) + const langs = new Set() + langs.add('python') + for (const match of matches) { + if (match[1]) + langs.add(match[1] as BundledLanguage) + } + return [...langs] +} + +function measuredKatex(options?: Parameters[0]) { + const transform = rehypeKatex(options) + return (tree: any, file: any) => { + const start = performance.now() + const length = typeof file?.value === 'string' ? file.value.length : undefined + try { + return transform(tree, file) + } + finally { + defaultPerfTracer.emit({ + tracerId: 'markdown', + name: 'process.katex', + ts: start, + duration: performance.now() - start, + meta: { length }, + }) + } + } +} + +async function createProcessor(langs: BundledLanguage[]): Promise { + const options: RehypeShikiOptions = { + themes: { + light: 'github-light', + dark: 'github-dark', + }, + langs, + defaultLanguage: langs[0] || 'python', + } + + return unified() + .use(RemarkParse) + .use(remarkBreaks) + .use(remarkMath) + .use(RemarkRehype) + .use(measuredKatex, { output: 'mathml' }) + .use(rehypeShiki, options) + .use(RehypeStringify) +} + +function getProcessor(langs: BundledLanguage[]): Promise { + // The cache key should be consistent, so we sort the languages. + const cacheKey = [...langs].sort().join(',') + + if (!processorCache.has(cacheKey)) { + const processorPromise = createProcessor(langs) + processorCache.set(cacheKey, processorPromise) + } + + return processorCache.get(cacheKey)! +} + +export function useMarkdown() { + const fallbackProcessor = unified() + .use(RemarkParse) + .use(remarkBreaks) + .use(remarkMath) + .use(RemarkRehype) + .use(measuredKatex, { output: 'mathml' }) + .use(RehypeStringify) + + return { + process: async (markdown: string): Promise => { + const hasCodeFence = /`{3,}/.test(markdown) + const meta = { length: markdown.length, hasCodeFence } + + return defaultPerfTracer.withMeasure('markdown', 'process', async () => { + try { + // A quick check for code fences. If none, use the fast fallback. + if (!hasCodeFence) { + return defaultPerfTracer.withMeasure('markdown', 'process.pipeline.basic', () => { + return fallbackProcessor.processSync(markdown).toString() + }, meta) + } + + const langs = extractLangs(markdown) + + // Always ensure 'python' is loaded as it's our default. + const langSet = new Set(langs) + langSet.add('python') + const languagesToLoad = Array.from(langSet) + + const processor = await getProcessor(languagesToLoad) + const result = await defaultPerfTracer.withMeasure('markdown', 'process.pipeline.rich', () => processor.process(markdown), meta) + return result.toString() + } + catch (error) { + console.warn( + 'Failed to process markdown with syntax highlighting, falling back to basic processing:', + error, + ) + // Fallback to basic processor without highlighting + return defaultPerfTracer.withMeasure('markdown', 'process.pipeline.fallback', () => { + return fallbackProcessor.processSync(markdown).toString() + }, { ...meta, fallback: true }) + } + }, meta) + }, + + // Synchronous version for backward compatibility + processSync: (markdown: string): string => { + const start = performance.now() + const output = fallbackProcessor + .processSync(markdown) + .toString() + + defaultPerfTracer.emit({ + tracerId: 'markdown', + name: 'process.pipeline.sync', + ts: start, + duration: performance.now() - start, + meta: { length: markdown.length }, + }) + + return output + }, + } +} diff --git a/packages/stage-ui/src/composables/use-modules-list.ts b/packages/stage-ui/src/composables/use-modules-list.ts index 2bccdc9746..7c823eea0c 100644 --- a/packages/stage-ui/src/composables/use-modules-list.ts +++ b/packages/stage-ui/src/composables/use-modules-list.ts @@ -14,6 +14,7 @@ import { useHearingStore } from '../stores/modules/hearing' import { useSpeechStore } from '../stores/modules/speech' import { useTwitterStore } from '../stores/modules/twitter' import { useVisionStore } from '../stores/modules/vision' +import { useVisualChatStore } from '../stores/modules/visual-chat' export interface Module { id: string @@ -35,6 +36,7 @@ export function useModulesList() { const speechStore = useSpeechStore() const hearingStore = useHearingStore() const visionStore = useVisionStore() + const visualChatStore = useVisualChatStore() const discordStore = useDiscordStore() const twitterStore = useTwitterStore() const minecraftStore = useMinecraftStore() @@ -80,6 +82,15 @@ export function useModulesList() { configured: visionStore.configured, category: 'essential', }, + { + id: 'visual-chat', + name: 'Visual Chat', + description: 'LiveKit gateway, sessions, and multimodal inference worker.', + icon: 'i-solar:videocamera-record-bold-duotone', + to: '/settings/modules/visual-chat', + configured: visualChatStore.enabled && visualChatStore.isGatewayReachable, + category: 'essential', + }, { id: 'memory-short-term', name: t('settings.pages.modules.memory-short-term.title'), diff --git a/packages/stage-ui/src/stores/modules/index.ts b/packages/stage-ui/src/stores/modules/index.ts index c5693824e8..4825fbd45f 100644 --- a/packages/stage-ui/src/stores/modules/index.ts +++ b/packages/stage-ui/src/stores/modules/index.ts @@ -7,3 +7,4 @@ export * from './hearing' export * from './speech' export * from './twitter' export * from './vision' +export * from './visual-chat' diff --git a/packages/stage-ui/src/stores/modules/visual-chat.ts b/packages/stage-ui/src/stores/modules/visual-chat.ts new file mode 100644 index 0000000000..de07490c62 --- /dev/null +++ b/packages/stage-ui/src/stores/modules/visual-chat.ts @@ -0,0 +1,1240 @@ +import type { + GatewayBootstrap, + GatewayDiagnostics, + GatewayWsClientMessage, + InteractionMode, + RealtimeInferenceCompletedPayload, + RealtimeInferenceFailedPayload, + RealtimeInferenceTextChunkPayload, + SessionAccess, + SessionContext, + SessionRecord, + TextMessage, +} from '@proj-airi/visual-chat-protocol' +import type { WsEvent } from '@proj-airi/visual-chat-sdk' + +import { VISUAL_CHAT_GATEWAY_TOKEN_HEADER } from '@proj-airi/visual-chat-protocol' +import { GatewayClient, GatewayWsClient } from '@proj-airi/visual-chat-sdk' +import { useLocalStorage } from '@vueuse/core' +import { defineStore } from 'pinia' +import { computed, ref, shallowRef, watch } from 'vue' + +import { + createRealtimeSourceId, + createRealtimeVideoStreamer, + stopMediaStreamTracks, +} from './visual-chat/realtime-media' + +export type VisualChatConnectionStatus = 'idle' | 'connecting' | 'connected' | 'disconnected' | 'error' +export type VisualChatRealtimeStatus = 'idle' | 'connecting' | 'connected' | 'disconnected' | 'error' +export type VisualChatParticipantKind = 'desktop' | 'phone' + +export interface VideoDevice { + deviceId: string + label: string +} + +export interface VisualChatWorkerHealth { + status?: string + ok?: boolean + backendKind?: string + model?: string + upstreamBaseUrl?: string + fixedModel?: boolean + features?: string[] + currentCnt?: number + metrics?: { + totalInferences: number + successCount: number + failureCount: number + avgPrefillLatencyMs: number + avgDecodeLatencyMs: number + avgTotalLatencyMs: number + lastLatencyMs: number + } +} + +export interface ChatMessage { + id?: string + role: 'user' | 'assistant' + content: string + durationMs?: number + timestamp: number + model?: string + sourceId?: string + streaming?: boolean +} + +const TRAILING_SLASH_PATTERN = /\/$/ + +function createDefaultParticipantIdentity(kind: VisualChatParticipantKind): string { + const random = typeof crypto !== 'undefined' && 'randomUUID' in crypto + ? crypto.randomUUID().slice(0, 8) + : Math.random().toString(36).slice(2, 10) + return `${kind}-${random}` +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} + +function sanitizeBaseUrl(value: string): string { + return value.replace(TRAILING_SLASH_PATTERN, '') +} + +function rewriteUrlHost(value: string, host: string): string { + const url = new URL(value) + url.hostname = host + return url.toString() +} + +function isLoopbackHost(host: string): boolean { + const normalized = host.trim().toLowerCase() + return normalized === 'localhost' || normalized === '127.0.0.1' || normalized === '::1' +} + +function isLoopbackUrl(value: string): boolean { + try { + return isLoopbackHost(new URL(value).hostname) + } + catch { + return false + } +} + +function resolveRuntimeHost(): string { + if (typeof window === 'undefined') + return 'localhost' + if (window.location.protocol === 'file:' || !window.location.hostname) + return 'localhost' + return window.location.hostname +} + +function resolveRuntimeProtocol(): 'http:' | 'https:' { + if (typeof window === 'undefined') + return 'http:' + return window.location.protocol === 'https:' ? 'https:' : 'http:' +} + +function buildDefaultServiceUrl(port: number): string { + return `${resolveRuntimeProtocol()}//${resolveRuntimeHost()}:${port}` +} + +function isMobileReachableHost(host: string): boolean { + return !!host && !isLoopbackHost(host) +} + +function usesHashRoutes(): boolean { + return import.meta.env.RUNTIME_ENVIRONMENT === 'electron' +} + +function buildRuntimeRouteUrl(baseUrl: string, routePath: string, query: Record): string { + const params = new URLSearchParams() + for (const [key, value] of Object.entries(query)) { + const normalized = value.trim() + if (normalized) + params.set(key, normalized) + } + + const url = new URL(`${sanitizeBaseUrl(baseUrl)}/`) + if (usesHashRoutes()) { + const queryString = params.toString() + url.hash = `${routePath}${queryString ? `?${queryString}` : ''}` + return url.toString() + } + + url.pathname = routePath + url.search = params.toString() + return url.toString() +} + +function shouldUseRemoteHostDefaults(): boolean { + return !isLoopbackHost(resolveRuntimeHost()) +} + +function buildGatewayWsUrl(baseUrl: string): string { + const url = new URL(sanitizeBaseUrl(baseUrl)) + url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:' + url.pathname = `${url.pathname.replace(TRAILING_SLASH_PATTERN, '')}/ws` + return url.toString() +} + +function sleep(ms: number) { + return new Promise(resolve => globalThis.setTimeout(resolve, ms)) +} + +function mapRealtimeTextMessage(message: TextMessage): ChatMessage { + return { + id: message.id, + role: message.role === 'user' ? 'user' : 'assistant', + content: message.content, + timestamp: message.timestamp, + sourceId: message.sourceId, + model: message.model, + streaming: false, + } +} + +export const useVisualChatStore = defineStore('visual-chat', () => { + const enabled = useLocalStorage('visual-chat:enabled', false) + const gatewayUrl = useLocalStorage('visual-chat:gateway-url', buildDefaultServiceUrl(6200)) + const gatewayToken = useLocalStorage('visual-chat:gateway-token', '') + const selectedSessionId = useLocalStorage('visual-chat:selected-session-id', '') + const selectedSessionToken = useLocalStorage('visual-chat:selected-session-token', '') + const realtimeMode = computed(() => 'vision-text-realtime') + const participantKind = useLocalStorage('visual-chat:participant-kind', 'desktop') + const participantIdentity = useLocalStorage('visual-chat:participant-identity', createDefaultParticipantIdentity('desktop')) + + const connectionStatus = ref('idle') + const realtimeStatus = ref('idle') + const lastError = ref(null) + const loading = ref(false) + const sessions = shallowRef([]) + const sessionRecords = shallowRef([]) + const activeSession = shallowRef(null) + const diagnostics = shallowRef(null) + const workerHealth = shallowRef(null) + + const gatewayClient = shallowRef(new GatewayClient({ + baseUrl: gatewayUrl.value, + getGatewayToken: () => gatewayToken.value, + getSessionAccess: () => currentSessionAccess(), + })) + const gatewayWsClient = shallowRef(null) + const realtimeWsUrl = ref('') + const isGatewayReachable = computed(() => connectionStatus.value === 'connected') + const phoneEntryOverrideHost = useLocalStorage('visual-chat:phone-entry-override-host', '') + + const preferredLanHost = computed(() => diagnostics.value?.preferredLanAddress ?? diagnostics.value?.lanAddresses?.[0] ?? '') + const gatewayHostname = computed(() => diagnostics.value?.hostname ?? '') + const preferredPublicFrontendUrl = computed(() => sanitizeBaseUrl(diagnostics.value?.publicFrontendUrl ?? '')) + const preferredPublicGatewayUrl = computed(() => sanitizeBaseUrl(diagnostics.value?.publicGatewayUrl ?? '')) + + const bestMobileReachableHost = computed(() => { + if (phoneEntryOverrideHost.value) + return phoneEntryOverrideHost.value + + if (isMobileReachableHost(preferredLanHost.value)) + return preferredLanHost.value + + if (gatewayHostname.value && !isLoopbackHost(gatewayHostname.value)) + return gatewayHostname.value + + return '' + }) + + const preferredServiceHost = computed(() => { + const runtimeHost = resolveRuntimeHost() + if (!isLoopbackHost(runtimeHost)) + return runtimeHost + return bestMobileReachableHost.value || runtimeHost + }) + const suggestedGatewayUrl = computed(() => { + if (!isMobileReachableHost(preferredServiceHost.value)) + return '' + return `${resolveRuntimeProtocol()}//${preferredServiceHost.value}:6200` + }) + const gatewayUrlNeedsHostRewrite = computed(() => { + return shouldUseRemoteHostDefaults() && isLoopbackUrl(gatewayUrl.value) && !!suggestedGatewayUrl.value + }) + const phoneEntryBaseUrl = computed(() => { + if (preferredPublicFrontendUrl.value) + return preferredPublicFrontendUrl.value + + if (typeof window === 'undefined') + return '' + + const runtimeOrigin = window.location.origin + if (!isLoopbackUrl(runtimeOrigin)) + return runtimeOrigin + + const mobileHost = bestMobileReachableHost.value + if (mobileHost) + return rewriteUrlHost(runtimeOrigin, mobileHost) + + return '' + }) + const phoneEntryGatewayUrl = computed(() => { + if (preferredPublicGatewayUrl.value) + return preferredPublicGatewayUrl.value + + const normalizedGatewayUrl = sanitizeBaseUrl(gatewayUrl.value) + if (!normalizedGatewayUrl) + return '' + + if (!isLoopbackUrl(normalizedGatewayUrl)) + return normalizedGatewayUrl + + const mobileHost = bestMobileReachableHost.value + if (mobileHost) + return `${resolveRuntimeProtocol()}//${mobileHost}:6200` + + return '' + }) + const phoneEntryUnavailableReason = computed(() => { + if (!selectedSessionId.value) + return 'Available after a session is active.' + if (!selectedSessionToken.value) + return 'Waiting for the secure session link to be prepared.' + if (!phoneEntryBaseUrl.value || !phoneEntryGatewayUrl.value) + return 'No reachable address found. Set a fixed host below, or configure public HTTPS/WSS endpoints.' + return '' + }) + const phoneEntryUrl = computed(() => { + if (!phoneEntryBaseUrl.value || !phoneEntryGatewayUrl.value || !selectedSessionId.value || !selectedSessionToken.value) + return '' + + return buildRuntimeRouteUrl(phoneEntryBaseUrl.value, '/visual-chat/phone', { + session: selectedSessionId.value, + gateway: phoneEntryGatewayUrl.value, + token: selectedSessionToken.value, + }) + }) + + const isReconnecting = ref(false) + let reconnectTimer: ReturnType | null = null + let reconnectAttempt = 0 + const MAX_RECONNECT_DELAY_MS = 30_000 + const BASE_RECONNECT_DELAY_MS = 1_000 + + const chatMessages = ref([]) + const sessionMemorySummary = ref('') + const inferring = ref(false) + const autoObserving = ref(false) + const autoObserveInferring = ref(false) + const autoObserveIntervalMs = ref(5000) + const autoObservePipelineStats = shallowRef<{ + totalInferences: number + autoObserveInferences: number + userInferences: number + skippedAutoObserve: number + skippedNoChange: number + timedOut: number + avgLatencyMs: number + lastLatencyMs: number + adaptiveIntervalMs: number + baseIntervalMs: number + } | null>(null) + const hasFixedModel = computed(() => workerHealth.value?.fixedModel === true) + const fixedModelName = computed(() => workerHealth.value?.model || '') + + const videoDevices = ref([]) + const selectedDeviceId = useLocalStorage('visual-chat:selected-device', '') + const sourceMode = ref<'camera' | 'screen' | null>(null) + const mediaStream = shallowRef(null) + const realtimeVideoStreamer = shallowRef> | null>(null) + const screenCaptureProvider = shallowRef<(() => Promise) | null>(null) + + function currentSessionAccess() { + if (!selectedSessionId.value || !selectedSessionToken.value) + return null + + return { + sessionId: selectedSessionId.value, + sessionToken: selectedSessionToken.value, + } + } + + function applySessionAccess(access: SessionAccess) { + selectedSessionId.value = access.session.sessionId + selectedSessionToken.value = access.sessionToken + activeSession.value = access.session + } + + function resolveActiveVideoSourceType(): 'phone-camera' | 'laptop-camera' | 'screen-share' | null { + if (!sourceMode.value) + return null + + if (sourceMode.value === 'screen') + return 'screen-share' + + return participantKind.value === 'phone' ? 'phone-camera' : 'laptop-camera' + } + + function setScreenCaptureProvider(provider: (() => Promise) | null) { + screenCaptureProvider.value = provider + } + + function applySuggestedNetworkUrls() { + if (suggestedGatewayUrl.value) + gatewayUrl.value = suggestedGatewayUrl.value + } + + function ensureParticipantIdentityForKind(kind: VisualChatParticipantKind) { + if (!participantIdentity.value.startsWith(`${kind}-`)) + participantIdentity.value = createDefaultParticipantIdentity(kind) + } + + function setParticipantKind(kind: VisualChatParticipantKind) { + participantKind.value = kind + ensureParticipantIdentityForKind(kind) + void restartRealtimeStreaming() + } + + async function ensureGatewayBootstrapToken(): Promise { + try { + const bootstrap = await gatewayClient.value.bootstrap() + gatewayToken.value = bootstrap.gatewayToken + return bootstrap + } + catch { + return null + } + } + + function clearRuntimeState() { + connectionStatus.value = 'idle' + realtimeStatus.value = 'idle' + lastError.value = null + loading.value = false + sessions.value = [] + sessionRecords.value = [] + activeSession.value = null + diagnostics.value = null + workerHealth.value = null + selectedSessionId.value = '' + selectedSessionToken.value = '' + chatMessages.value = [] + sessionMemorySummary.value = '' + inferring.value = false + autoObserving.value = false + autoObserveInferring.value = false + autoObservePipelineStats.value = null + } + + function upsertChatMessage(message: ChatMessage) { + if (message.id) { + const existingIndex = chatMessages.value.findIndex(item => item.id === message.id) + if (existingIndex >= 0) { + chatMessages.value[existingIndex] = { + ...chatMessages.value[existingIndex], + ...message, + } + return + } + } + + chatMessages.value.push(message) + } + + function hydrateSessionMessages(messages: TextMessage[]) { + chatMessages.value = messages.map(mapRealtimeTextMessage) + } + + async function enumerateVideoDevices() { + if (typeof navigator === 'undefined' || !navigator.mediaDevices?.enumerateDevices) { + videoDevices.value = [] + return + } + + try { + const devices = await navigator.mediaDevices.enumerateDevices() + videoDevices.value = devices + .filter(device => device.kind === 'videoinput') + .map((device, index) => ({ + deviceId: device.deviceId, + label: device.label || (device.deviceId ? `Camera ${device.deviceId.slice(0, 8)}` : `Camera ${index + 1}`), + })) + + if (videoDevices.value.length > 0 && !selectedDeviceId.value) + selectedDeviceId.value = videoDevices.value[0].deviceId + } + catch { + videoDevices.value = [] + } + } + + function stopRealtimeVideoStreamer() { + realtimeVideoStreamer.value?.stop() + realtimeVideoStreamer.value = null + } + + function stopMediaStream() { + stopRealtimeVideoStreamer() + stopMediaStreamTracks(mediaStream.value) + mediaStream.value = null + sourceMode.value = null + } + + async function startCamera(deviceId?: string, facingMode?: 'user' | 'environment'): Promise { + stopMediaStream() + + if (!navigator.mediaDevices?.getUserMedia) + throw new Error('Camera API is not available. This browser may require HTTPS for camera access.') + + const videoConstraints: MediaTrackConstraints = { + width: { ideal: 1280 }, + height: { ideal: 720 }, + } + + if (deviceId) { + videoConstraints.deviceId = { exact: deviceId } + } + else if (facingMode) { + videoConstraints.facingMode = { ideal: facingMode } + } + + const stream = await navigator.mediaDevices.getUserMedia({ + video: videoConstraints, + }) + mediaStream.value = stream + sourceMode.value = 'camera' + await enumerateVideoDevices() + await restartRealtimeStreaming() + return stream + } + + async function startScreenCapture(): Promise { + stopMediaStream() + + let stream: MediaStream + + if (screenCaptureProvider.value) { + stream = await screenCaptureProvider.value() + } + else if (navigator.mediaDevices?.getDisplayMedia) { + stream = await navigator.mediaDevices.getDisplayMedia({ + video: { width: { ideal: 1920 }, height: { ideal: 1080 } }, + audio: false, + }) + } + else { + throw new Error('Screen capture API is not available. This browser may require HTTPS or a secure context.') + } + + stream.getVideoTracks()[0]?.addEventListener('ended', () => { + stopMediaStream() + }) + mediaStream.value = stream + sourceMode.value = 'screen' + await restartRealtimeStreaming() + return stream + } + + function cancelReconnect() { + if (reconnectTimer) { + globalThis.clearTimeout(reconnectTimer) + reconnectTimer = null + } + reconnectAttempt = 0 + isReconnecting.value = false + } + + function scheduleReconnect() { + if (!enabled.value || reconnectTimer) + return + + const delay = Math.min(BASE_RECONNECT_DELAY_MS * 2 ** reconnectAttempt, MAX_RECONNECT_DELAY_MS) + reconnectAttempt++ + isReconnecting.value = true + + reconnectTimer = globalThis.setTimeout(async () => { + reconnectTimer = null + try { + await ensureGatewayWsClient() + } + catch { + scheduleReconnect() + } + }, delay) + } + + async function ensureGatewayWsClient() { + const nextWsUrl = buildGatewayWsUrl(gatewayUrl.value) + if (!gatewayWsClient.value || realtimeWsUrl.value !== nextWsUrl) { + gatewayWsClient.value?.disconnect() + + const client = new GatewayWsClient(nextWsUrl, { + autoReconnect: false, + getSessionAccess: (sessionId) => { + const sessionAccess = currentSessionAccess() + if (!sessionAccess || sessionAccess.sessionId !== sessionId) + return null + return sessionAccess + }, + }) + gatewayWsClient.value = client + realtimeWsUrl.value = nextWsUrl + + client.on('connected', () => { + realtimeStatus.value = 'connected' + cancelReconnect() + if (selectedSessionId.value) { + client.subscribe(selectedSessionId.value) + gatewayClient.value.getSessionMessages(selectedSessionId.value) + .then(res => hydrateSessionMessages(res.messages)) + .catch(() => {}) + } + }) + + client.on('disconnected', () => { + realtimeStatus.value = 'disconnected' + scheduleReconnect() + }) + + client.on('*', (event) => { + handleRealtimeEvent(event) + }) + + realtimeStatus.value = 'connecting' + } + + gatewayWsClient.value.connect() + } + + function sendRealtimeMessage(message: GatewayWsClientMessage) { + gatewayWsClient.value?.send(message) + } + + function handleRealtimeEvent(event: WsEvent) { + if (selectedSessionId.value && event.sessionId && event.sessionId !== '*' && event.sessionId !== selectedSessionId.value) + return + + if (event.event === 'session:state:changed') { + const payload = event.data as { context?: SessionContext } + if (payload.context) + activeSession.value = payload.context + return + } + + if (event.event === 'source:registered' || event.event === 'source:unregistered') { + void refreshActiveSession() + return + } + + if (event.event === 'chat:message') { + upsertChatMessage(mapRealtimeTextMessage(event.data as TextMessage)) + void refreshSessionRecords() + return + } + + if (event.event === 'inference:started') { + const payload = event.data as { auto?: boolean } + if (payload.auto) { + autoObserveInferring.value = true + } + else { + inferring.value = true + } + lastError.value = null + return + } + + if (event.event === 'inference:text:chunk') { + const payload = event.data as RealtimeInferenceTextChunkPayload + if ((payload as { auto?: boolean }).auto) + return + inferring.value = true + upsertChatMessage({ + id: payload.id, + role: 'assistant', + content: payload.text, + timestamp: Date.now(), + sourceId: payload.sourceId, + model: payload.model, + streaming: true, + }) + return + } + + if (event.event === 'inference:completed') { + const payload = event.data as RealtimeInferenceCompletedPayload + if (payload.auto) { + autoObserveInferring.value = false + return + } + inferring.value = false + upsertChatMessage({ + ...mapRealtimeTextMessage(payload.message), + durationMs: payload.durationMs, + }) + return + } + + if (event.event === 'inference:failed') { + const payload = event.data as RealtimeInferenceFailedPayload + autoObserveInferring.value = false + inferring.value = false + lastError.value = payload.error + if (!(payload as { auto?: boolean }).auto) { + upsertChatMessage({ + role: 'assistant', + content: `Error: ${payload.error}`, + timestamp: Date.now(), + }) + } + return + } + + if (event.event === 'auto-observe:started') { + const payload = event.data as { sessionId: string, intervalMs: number } + autoObserving.value = true + autoObserveIntervalMs.value = payload.intervalMs + return + } + + if (event.event === 'auto-observe:stopped') { + autoObserving.value = false + autoObservePipelineStats.value = null + return + } + + if (event.event === 'auto-observe:status') { + const payload = event.data as { + stats: { + totalInferences: number + autoObserveInferences: number + userInferences: number + skippedAutoObserve: number + skippedNoChange: number + timedOut: number + avgLatencyMs: number + lastLatencyMs: number + } + adaptiveIntervalMs: number + baseIntervalMs: number + } + autoObservePipelineStats.value = { + ...payload.stats, + adaptiveIntervalMs: payload.adaptiveIntervalMs, + baseIntervalMs: payload.baseIntervalMs, + } + return + } + + if (event.event === 'session:memory:updated') { + const payload = event.data as { summary?: string } + sessionMemorySummary.value = payload.summary?.trim() || '' + void refreshSessionRecords() + } + } + + async function restartRealtimeStreaming() { + stopRealtimeVideoStreamer() + + if (!selectedSessionId.value || !mediaStream.value || !sourceMode.value) + return + + resetSource() + await ensureGatewayWsClient() + + if (!mediaStream.value || !sourceMode.value) + return + + const videoSourceType = resolveActiveVideoSourceType() + if (!videoSourceType) + return + const videoSourceId = createRealtimeSourceId(participantIdentity.value, videoSourceType) + const videoCaptureProfile = sourceMode.value === 'screen' + ? { + intervalMs: 1200, + maxPixels: 1_800_000, + quality: 0.92, + format: 'png' as const, + } + : { + intervalMs: 900, + maxPixels: 1_280 * 720, + quality: 0.82, + format: 'jpeg' as const, + } + let activatedSource = false + const activateVideoSource = async () => { + for (let attempt = 0; attempt < 5; attempt++) { + try { + const context = await gatewayClient.value.switchSource(selectedSessionId.value, undefined, videoSourceType) + activeSession.value = context + if (context.activeVideoSource?.sourceType === videoSourceType) + return + } + catch (error) { + lastError.value = errorMessage(error) + } + + await sleep(180) + } + } + + realtimeVideoStreamer.value = await createRealtimeVideoStreamer({ + stream: mediaStream.value, + intervalMs: videoCaptureProfile.intervalMs, + maxPixels: videoCaptureProfile.maxPixels, + quality: videoCaptureProfile.quality, + format: videoCaptureProfile.format, + onFrame: (payload) => { + sendRealtimeMessage({ + type: 'realtime:media:video', + sessionId: selectedSessionId.value, + participantIdentity: participantIdentity.value, + sourceId: videoSourceId, + sourceType: videoSourceType, + timestamp: payload.timestamp, + width: payload.width, + height: payload.height, + format: payload.format, + data: payload.data, + }) + + if (!activatedSource) { + activatedSource = true + void activateVideoSource() + } + }, + }) + } + + async function requestWorkerResponse(path: string, init?: RequestInit): Promise { + const normalizedPath = path.startsWith('/') ? path : `/${path}` + const gatewayBase = sanitizeBaseUrl(gatewayUrl.value) + if (!gatewayBase) + throw new Error(`Worker request cannot start because no gateway endpoint is configured for ${normalizedPath}.`) + + if (!gatewayToken.value) + await ensureGatewayBootstrapToken() + if (!gatewayToken.value) + throw new Error('Worker request requires local gateway access.') + + const headers = new Headers(init?.headers) + headers.set(VISUAL_CHAT_GATEWAY_TOKEN_HEADER, gatewayToken.value) + + const response = await fetch(`${gatewayBase}/api/worker${normalizedPath}`, { + ...init, + headers, + }) + return response + } + + async function probeConnection(): Promise { + if (!enabled.value) { + clearRuntimeState() + return false + } + + connectionStatus.value = 'connecting' + lastError.value = null + try { + const ok = await gatewayClient.value.health() + connectionStatus.value = ok ? 'connected' : 'disconnected' + if (!ok) { + lastError.value = 'Gateway health check failed' + return false + } + return true + } + catch (error) { + connectionStatus.value = 'error' + lastError.value = errorMessage(error) + return false + } + } + + async function refreshSessions() { + if (!enabled.value) { + sessions.value = [] + return + } + + if (!gatewayToken.value) + await ensureGatewayBootstrapToken() + if (!gatewayToken.value) { + sessions.value = [] + return + } + + const ok = connectionStatus.value === 'connected' || await probeConnection() + if (!ok) + return + + try { + sessions.value = await gatewayClient.value.listSessions() + lastError.value = null + } + catch (error) { + lastError.value = errorMessage(error) + connectionStatus.value = 'error' + } + } + + async function refreshSessionRecords() { + if (!enabled.value) { + sessionRecords.value = [] + return + } + + if (!gatewayToken.value) + await ensureGatewayBootstrapToken() + + const ok = connectionStatus.value === 'connected' || await probeConnection() + if (!ok) + return + + try { + if (gatewayToken.value) { + sessionRecords.value = await gatewayClient.value.listSessionRecords() + } + else if (selectedSessionId.value && selectedSessionToken.value) { + const record = await gatewayClient.value.getSessionRecord(selectedSessionId.value) + sessionRecords.value = record ? [record] : [] + } + else { + sessionRecords.value = [] + } + + if (selectedSessionId.value) { + const activeRecord = sessionRecords.value.find(record => record.sessionId === selectedSessionId.value) + sessionMemorySummary.value = activeRecord?.sceneMemory?.trim() || sessionMemorySummary.value + } + } + catch (error) { + lastError.value = errorMessage(error) + connectionStatus.value = 'error' + } + } + + async function refreshDiagnostics() { + if (!enabled.value) { + diagnostics.value = null + return + } + + if (!gatewayToken.value) + await ensureGatewayBootstrapToken() + if (!gatewayToken.value) { + diagnostics.value = null + return + } + + const ok = connectionStatus.value === 'connected' || await probeConnection() + if (!ok) + return + + try { + diagnostics.value = await gatewayClient.value.getDiagnostics() + connectionStatus.value = 'connected' + } + catch (error) { + lastError.value = errorMessage(error) + connectionStatus.value = 'error' + } + } + + async function refreshWorkerHealth() { + if (!enabled.value) { + workerHealth.value = null + return + } + + if (!gatewayToken.value) + await ensureGatewayBootstrapToken() + if (!gatewayToken.value) { + workerHealth.value = null + return + } + + try { + const res = await requestWorkerResponse('/health') + if (!res.ok) { + workerHealth.value = { ok: false, status: 'unreachable' } + return + } + workerHealth.value = (await res.json()) as VisualChatWorkerHealth + } + catch { + workerHealth.value = { ok: false, status: 'unreachable' } + } + } + + async function refreshAll() { + if (!enabled.value) { + clearRuntimeState() + return + } + + loading.value = true + try { + await probeConnection() + await Promise.all([ + refreshSessions(), + refreshSessionRecords(), + refreshDiagnostics(), + refreshWorkerHealth(), + ]) + + if (selectedSessionId.value && connectionStatus.value === 'connected' && !activeSession.value) { + await joinRealtimeSession(selectedSessionId.value, selectedSessionToken.value).catch(() => {}) + } + } + finally { + loading.value = false + } + } + + async function refreshActiveSession() { + if (!selectedSessionId.value) + return + + try { + activeSession.value = await gatewayClient.value.getSession(selectedSessionId.value) + } + catch { + activeSession.value = null + } + } + + async function joinRealtimeSession(sessionId: string, sessionToken?: string) { + const nextSessionId = sessionId.trim() + if (!nextSessionId) + return + + const previousSessionId = selectedSessionId.value + if (previousSessionId && previousSessionId !== nextSessionId) { + gatewayWsClient.value?.unsubscribe(previousSessionId) + chatMessages.value = [] + sessionMemorySummary.value = '' + inferring.value = false + autoObserveInferring.value = false + } + + let nextAccess: SessionAccess | null = null + const providedSessionToken = sessionToken?.trim() + if (providedSessionToken) + selectedSessionToken.value = providedSessionToken + + if (!providedSessionToken && (!selectedSessionToken.value || previousSessionId !== nextSessionId)) { + if (!gatewayToken.value) + await ensureGatewayBootstrapToken() + if (!gatewayToken.value) { + throw new Error('Missing secure session token. Open the session from the generated phone entry link, or restore it locally from the desktop app first.') + } + + nextAccess = await gatewayClient.value.issueSessionAccess(nextSessionId) + .catch(() => gatewayClient.value.restoreSessionRecord(nextSessionId)) + applySessionAccess(nextAccess) + } + + selectedSessionId.value = nextSessionId + if (!selectedSessionToken.value) + throw new Error('No secure session token is available for this session.') + + await ensureGatewayWsClient() + gatewayWsClient.value?.subscribe(nextSessionId) + + if (!nextAccess) + activeSession.value = await gatewayClient.value.getSession(nextSessionId).catch(() => null) + + const messageResponse = await gatewayClient.value.getSessionMessages(nextSessionId).catch(() => ({ messages: [] })) + hydrateSessionMessages(messageResponse.messages) + await refreshSessionRecords().catch(() => {}) + const matchingRecord = sessionRecords.value.find(record => record.sessionId === nextSessionId) + sessionMemorySummary.value = matchingRecord?.sceneMemory?.trim() || '' + await restartRealtimeStreaming() + } + + async function createRealtimeSession() { + enabled.value = true + if (!gatewayToken.value) + await ensureGatewayBootstrapToken() + if (!gatewayToken.value) + throw new Error('Creating a visual chat session requires local gateway access.') + + const session = await gatewayClient.value.createSession() + applySessionAccess(session) + await refreshSessions() + await refreshSessionRecords() + await joinRealtimeSession(session.session.sessionId, session.sessionToken) + return session.session + } + + async function leaveRealtimeSession() { + if (selectedSessionId.value) + gatewayWsClient.value?.unsubscribe(selectedSessionId.value) + + selectedSessionId.value = '' + selectedSessionToken.value = '' + activeSession.value = null + chatMessages.value = [] + sessionMemorySummary.value = '' + inferring.value = false + autoObserving.value = false + autoObserveInferring.value = false + autoObservePipelineStats.value = null + stopRealtimeVideoStreamer() + } + + async function deleteRealtimeSession() { + if (!selectedSessionId.value) + return + + await gatewayClient.value.deleteSession(selectedSessionId.value) + await leaveRealtimeSession() + await refreshSessions() + await refreshSessionRecords() + } + + async function deleteSessionRecord(sessionId: string) { + await gatewayClient.value.deleteSessionRecord(sessionId) + if (selectedSessionId.value === sessionId) + await leaveRealtimeSession() + await refreshSessionRecords() + } + + function sendRealtimeText(text: string) { + if (!selectedSessionId.value || !text.trim()) + return + + if (!gatewayWsClient.value) + void ensureGatewayWsClient() + + sendRealtimeMessage({ + type: 'realtime:user:text', + sessionId: selectedSessionId.value, + participantIdentity: participantIdentity.value, + text, + }) + } + + function requestRealtimeInference() { + if (!selectedSessionId.value) + return + + sendRealtimeMessage({ + type: 'realtime:control', + sessionId: selectedSessionId.value, + action: 'request-inference', + }) + } + + function startAutoObserve(intervalMs?: number) { + if (!selectedSessionId.value) + return + + sendRealtimeMessage({ + type: 'realtime:control', + sessionId: selectedSessionId.value, + action: 'start-auto-observe', + intervalMs: intervalMs ?? autoObserveIntervalMs.value, + }) + } + + function stopAutoObserve() { + if (!selectedSessionId.value) + return + + sendRealtimeMessage({ + type: 'realtime:control', + sessionId: selectedSessionId.value, + action: 'stop-auto-observe', + }) + } + + function resetSource() { + if (!selectedSessionId.value) + return + + sendRealtimeMessage({ + type: 'realtime:control', + sessionId: selectedSessionId.value, + action: 'reset-source', + }) + } + + watch(enabled, (isEnabled) => { + if (isEnabled) { + void refreshAll() + return + } + + stopMediaStream() + cancelReconnect() + gatewayWsClient.value?.disconnect() + gatewayWsClient.value = null + realtimeWsUrl.value = '' + clearRuntimeState() + }) + + watch(gatewayUrl, (url) => { + gatewayClient.value = new GatewayClient({ + baseUrl: url, + getGatewayToken: () => gatewayToken.value, + getSessionAccess: () => currentSessionAccess(), + }) + cancelReconnect() + gatewayWsClient.value?.disconnect() + gatewayWsClient.value = null + realtimeWsUrl.value = '' + if (enabled.value) + void refreshAll() + }, { immediate: true }) + + watch(participantKind, (kind) => { + ensureParticipantIdentityForKind(kind) + }, { immediate: true }) + + return { + enabled, + gatewayUrl, + gatewayToken, + selectedSessionId, + selectedSessionToken, + realtimeMode, + participantKind, + participantIdentity, + connectionStatus, + realtimeStatus, + lastError, + loading, + sessions, + activeSession, + sessionRecords, + diagnostics, + workerHealth, + gatewayClient, + gatewayWsClient, + isGatewayReachable, + suggestedGatewayUrl, + gatewayUrlNeedsHostRewrite, + phoneEntryOverrideHost, + bestMobileReachableHost, + phoneEntryUnavailableReason, + phoneEntryUrl, + chatMessages, + sessionMemorySummary, + inferring, + autoObserving, + autoObserveInferring, + autoObserveIntervalMs, + autoObservePipelineStats, + isReconnecting, + hasFixedModel, + fixedModelName, + videoDevices, + selectedDeviceId, + sourceMode, + mediaStream, + applySuggestedNetworkUrls, + setParticipantKind, + setScreenCaptureProvider, + probeConnection, + refreshSessions, + refreshSessionRecords, + refreshDiagnostics, + refreshWorkerHealth, + refreshAll, + refreshActiveSession, + createRealtimeSession, + joinRealtimeSession, + leaveRealtimeSession, + deleteRealtimeSession, + deleteSessionRecord, + enumerateVideoDevices, + startCamera, + startScreenCapture, + stopMediaStream, + sendRealtimeText, + requestRealtimeInference, + startAutoObserve, + stopAutoObserve, + resetSource, + restartRealtimeStreaming, + } +}) diff --git a/packages/stage-ui/src/stores/modules/visual-chat/native-duplex-audio.ts b/packages/stage-ui/src/stores/modules/visual-chat/native-duplex-audio.ts new file mode 100644 index 0000000000..00f7e34321 --- /dev/null +++ b/packages/stage-ui/src/stores/modules/visual-chat/native-duplex-audio.ts @@ -0,0 +1,95 @@ +export interface NativeDuplexAudioPlayerController { + playChunk: (base64Data: string) => Promise + stop: () => Promise +} + +interface CreateNativeDuplexAudioPlayerOptions { + initialDelayMs?: number + sampleRate?: number +} + +const DEFAULT_INITIAL_DELAY_MS = 180 +const DEFAULT_SAMPLE_RATE = 24_000 + +function decodeBase64ToFloat32(base64Data: string): Float32Array { + const binary = atob(base64Data) + const bytes = new Uint8Array(binary.length) + + for (let index = 0; index < binary.length; index++) + bytes[index] = binary.charCodeAt(index) + + return new Float32Array(bytes.buffer) +} + +export async function createNativeDuplexAudioPlayer( + options: CreateNativeDuplexAudioPlayerOptions = {}, +): Promise { + const initialDelayMs = options.initialDelayMs ?? DEFAULT_INITIAL_DELAY_MS + const sampleRate = options.sampleRate ?? DEFAULT_SAMPLE_RATE + const audioContext = new AudioContext({ latencyHint: 'interactive' }) + const activeSources = new Set() + + let nextPlaybackTime = 0 + + async function ensureRunning() { + if (audioContext.state === 'suspended') + await audioContext.resume() + } + + async function playChunk(base64Data: string) { + if (!base64Data) + return + + await ensureRunning() + + const samples = decodeBase64ToFloat32(base64Data) + if (samples.length === 0) + return + + const buffer = audioContext.createBuffer(1, samples.length, sampleRate) + buffer.getChannelData(0).set(samples) + + const source = audioContext.createBufferSource() + source.buffer = buffer + source.connect(audioContext.destination) + + const now = audioContext.currentTime + if (nextPlaybackTime <= now) + nextPlaybackTime = now + (initialDelayMs / 1000) + + source.start(nextPlaybackTime) + nextPlaybackTime += buffer.duration + activeSources.add(source) + + source.onended = () => { + activeSources.delete(source) + } + } + + async function stop() { + for (const source of activeSources) { + try { + source.stop() + } + catch { + } + + try { + source.disconnect() + } + catch { + } + } + + activeSources.clear() + nextPlaybackTime = 0 + + if (audioContext.state !== 'closed') + await audioContext.close() + } + + return { + playChunk, + stop, + } +} diff --git a/packages/stage-ui/src/stores/modules/visual-chat/realtime-media.ts b/packages/stage-ui/src/stores/modules/visual-chat/realtime-media.ts new file mode 100644 index 0000000000..01fa1980db --- /dev/null +++ b/packages/stage-ui/src/stores/modules/visual-chat/realtime-media.ts @@ -0,0 +1,228 @@ +export interface RealtimeVideoFramePayload { + timestamp: number + width: number + height: number + format: 'jpeg' | 'png' + data: string +} + +export interface RealtimeAudioChunkPayload { + timestamp: number + sampleRate: 16000 + channels: 1 + durationMs: 1000 + data: string +} + +export interface RealtimeVideoStreamerController { + stop: () => void +} + +export interface RealtimeAudioStreamerController { + stop: () => Promise +} + +interface CreateRealtimeVideoStreamerOptions { + stream: MediaStream + intervalMs?: number + maxPixels?: number + quality?: number + format?: 'jpeg' | 'png' + onFrame: (payload: RealtimeVideoFramePayload) => void +} + +interface CreateRealtimeAudioStreamerOptions { + stream: MediaStream + onChunk: (payload: RealtimeAudioChunkPayload) => void +} + +const DEFAULT_VIDEO_INTERVAL_MS = 1000 +const DEFAULT_MAX_VIDEO_PIXELS = 1_280 * 720 +const DEFAULT_VIDEO_QUALITY = 0.78 +const TARGET_AUDIO_SAMPLE_RATE = 16000 +const TARGET_AUDIO_DURATION_MS = 1000 +const TARGET_AUDIO_SAMPLES = TARGET_AUDIO_SAMPLE_RATE * (TARGET_AUDIO_DURATION_MS / 1000) + +function mixToMono(inputBuffer: AudioBuffer): Float32Array { + const mono = new Float32Array(inputBuffer.length) + const channelCount = inputBuffer.numberOfChannels + + for (let channelIndex = 0; channelIndex < channelCount; channelIndex++) { + const channelData = inputBuffer.getChannelData(channelIndex) + for (let sampleIndex = 0; sampleIndex < channelData.length; sampleIndex++) + mono[sampleIndex] += channelData[sampleIndex] / channelCount + } + + return mono +} + +function concatFloat32Arrays(left: Float32Array, right: Float32Array): Float32Array { + return Float32Array.from([...left, ...right]) +} + +function downsampleTo16k(input: Float32Array, inputSampleRate: number): Int16Array { + if (inputSampleRate === TARGET_AUDIO_SAMPLE_RATE) { + const pcm = new Int16Array(input.length) + for (let index = 0; index < input.length; index++) { + const clamped = Math.max(-1, Math.min(1, input[index] ?? 0)) + pcm[index] = clamped < 0 ? clamped * 0x8000 : clamped * 0x7FFF + } + return pcm + } + + const ratio = inputSampleRate / TARGET_AUDIO_SAMPLE_RATE + const outputLength = Math.max(1, Math.round(input.length / ratio)) + const pcm = new Int16Array(outputLength) + + for (let outputIndex = 0; outputIndex < outputLength; outputIndex++) { + const start = Math.floor(outputIndex * ratio) + const end = Math.min(input.length, Math.floor((outputIndex + 1) * ratio)) + let sample = 0 + let count = 0 + + for (let index = start; index < end; index++) { + sample += input[index] ?? 0 + count++ + } + + const averaged = count > 0 ? sample / count : 0 + const clamped = Math.max(-1, Math.min(1, averaged)) + pcm[outputIndex] = clamped < 0 ? clamped * 0x8000 : clamped * 0x7FFF + } + + return pcm +} + +function int16ToBase64(data: Int16Array): string { + const bytes = new Uint8Array(data.buffer, data.byteOffset, data.byteLength) + let binary = '' + for (const byte of bytes) + binary += String.fromCharCode(byte) + return btoa(binary) +} + +export function createRealtimeSourceId(participantIdentity: string, sourceName: string): string { + return `${participantIdentity}:${sourceName}` +} + +export function stopMediaStreamTracks(stream: MediaStream | null | undefined) { + if (!stream) + return + + for (const track of stream.getTracks()) + track.stop() +} + +export async function createRealtimeVideoStreamer( + options: CreateRealtimeVideoStreamerOptions, +): Promise { + const intervalMs = options.intervalMs ?? DEFAULT_VIDEO_INTERVAL_MS + const maxPixels = options.maxPixels ?? DEFAULT_MAX_VIDEO_PIXELS + const quality = options.quality ?? DEFAULT_VIDEO_QUALITY + const format = options.format ?? 'jpeg' + + const video = document.createElement('video') + video.srcObject = options.stream + video.muted = true + video.playsInline = true + await video.play() + + const canvas = document.createElement('canvas') + const context = canvas.getContext('2d') + if (!context) + throw new Error('Canvas 2D context is unavailable for realtime video capture.') + + const timer = window.setInterval(() => { + if (!video.videoWidth || !video.videoHeight) + return + + const scale = Math.min(1, Math.sqrt(maxPixels / (video.videoWidth * video.videoHeight))) + const width = Math.max(1, Math.round(video.videoWidth * scale)) + const height = Math.max(1, Math.round(video.videoHeight * scale)) + + canvas.width = width + canvas.height = height + context.drawImage(video, 0, 0, width, height) + + const mimeType = format === 'png' ? 'image/png' : 'image/jpeg' + const data = canvas.toDataURL(mimeType, quality).split(',')[1] + if (!data) + return + + options.onFrame({ + timestamp: Date.now(), + width, + height, + format, + data, + }) + }, intervalMs) + + return { + stop() { + window.clearInterval(timer) + video.pause() + video.srcObject = null + }, + } +} + +export async function createRealtimeAudioStreamer( + options: CreateRealtimeAudioStreamerOptions, +): Promise { + // NOTICE: ScriptProcessorNode is deprecated but still the most portable + // browser-side way for this repo to chunk raw PCM without adding another + // worklet bundle path just for realtime visual chat. + const audioContext = new AudioContext({ latencyHint: 'interactive' }) + const sourceNode = audioContext.createMediaStreamSource(options.stream) + const processorNode = audioContext.createScriptProcessor(4096, 1, 1) + const muteNode = audioContext.createGain() + muteNode.gain.value = 0 + + let pending: Float32Array = new Float32Array(0) + const chunkSampleSize = audioContext.sampleRate * (TARGET_AUDIO_DURATION_MS / 1000) + + processorNode.onaudioprocess = (event) => { + const mono = mixToMono(event.inputBuffer) + pending = concatFloat32Arrays(pending, mono) + + while (pending.length >= chunkSampleSize) { + const nextChunk = Float32Array.from(pending.subarray(0, chunkSampleSize)) + pending = Float32Array.from(pending.subarray(chunkSampleSize)) + + const downsampled = downsampleTo16k(nextChunk, audioContext.sampleRate) + const normalizedChunk = downsampled.length === TARGET_AUDIO_SAMPLES + ? downsampled + : downsampled.length > TARGET_AUDIO_SAMPLES + ? downsampled.slice(0, TARGET_AUDIO_SAMPLES) + : (() => { + const padded = new Int16Array(TARGET_AUDIO_SAMPLES) + padded.set(downsampled) + return padded + })() + + options.onChunk({ + timestamp: Date.now(), + sampleRate: 16000, + channels: 1, + durationMs: 1000, + data: int16ToBase64(normalizedChunk), + }) + } + } + + sourceNode.connect(processorNode) + processorNode.connect(muteNode) + muteNode.connect(audioContext.destination) + await audioContext.resume() + + return { + async stop() { + processorNode.disconnect() + sourceNode.disconnect() + muteNode.disconnect() + processorNode.onaudioprocess = null + await audioContext.close() + }, + } +} diff --git a/packages/stage-ui/src/stores/providers/web-speech-api/index.ts b/packages/stage-ui/src/stores/providers/web-speech-api/index.ts index 59973e46f0..a33e01197d 100644 --- a/packages/stage-ui/src/stores/providers/web-speech-api/index.ts +++ b/packages/stage-ui/src/stores/providers/web-speech-api/index.ts @@ -162,6 +162,8 @@ export function streamWebSpeechAPITranscription( options?: WebSpeechAPIExtraOptions & { onSentenceEnd?: (delta: string) => void onSpeechEnd?: (text: string) => void + onError?: (errorType: string, message: string) => void + onStatusChange?: (status: 'started' | 'restarted' | 'ended' | 'audio-started' | 'speech-detected') => void }, ): StreamTranscriptionResult & { recognition?: any } { const deferredText = createDeferred() @@ -264,17 +266,36 @@ export function streamWebSpeechAPITranscription( console.warn('Web Speech API error:', errorType) if (errorType === 'no-speech') { + options?.onError?.('no-speech', 'No speech detected. Keep speaking or check microphone placement.') return } if (errorType === 'audio-capture') { - console.warn('Web Speech API: Microphone access issue. Please check microphone permissions.') + options?.onError?.('audio-capture', 'Microphone audio capture failed. The browser may need HTTPS or microphone permissions may be blocked.') return } - if (errorType === 'network' || errorType === 'aborted') { + if (errorType === 'network') { + options?.onError?.('network', 'Speech recognition requires internet access (audio is processed by Google servers). Check your network connection.') return } + + if (errorType === 'aborted') { + return + } + + if (errorType === 'not-allowed' || errorType === 'service-not-allowed') { + options?.onError?.('not-allowed', 'Speech recognition permission denied or service unavailable. This browser may not support Web Speech API.') + const error = new Error(`Speech recognition not allowed: ${errorType}`) + fullStreamCtrl?.error(error) + textStreamCtrl?.error(error) + deferredText.reject(error) + deferredText.isRejected = true + options?.onSpeechEnd?.(fullText) + return + } + + options?.onError?.(errorType, `Speech recognition error: ${errorType}`) const error = new Error(`Speech recognition error: ${errorType}`) fullStreamCtrl?.error(error) textStreamCtrl?.error(error) @@ -291,22 +312,23 @@ export function streamWebSpeechAPITranscription( // Use the current recognitionInstance to ensure we're using the correct instance const currentRecognition = recognitionInstance || recognition - // Small delay before restarting to avoid rapid restart loops setTimeout(() => { try { currentRecognition.start() console.info('Web Speech API recognition restarted (continuous mode)') + options?.onStatusChange?.('restarted') } catch (err) { console.warn('Web Speech API failed to restart, creating new instance:', err) - // If restart fails, create a new instance try { createAndStartNewRecognitionInstance(recognition) console.info('Web Speech API created new instance and started') + options?.onStatusChange?.('restarted') } catch (newErr) { console.error('Web Speech API failed to create new instance:', newErr) const error = new Error(`Failed to restart recognition: ${newErr instanceof Error ? newErr.message : String(newErr)}`) + options?.onError?.('restart-failed', error.message) fullStreamCtrl?.error(error) textStreamCtrl?.error(error) deferredText.reject(error) @@ -415,13 +437,14 @@ export function streamWebSpeechAPITranscription( } } - // Add event listeners for debugging before starting recognition.onstart = () => { console.info('Web Speech API recognition started (onstart event)') + options?.onStatusChange?.('started') } recognition.onaudiostart = () => { console.info('Web Speech API audio capture started') + options?.onStatusChange?.('audio-started') } recognition.onsoundstart = () => { @@ -430,6 +453,7 @@ export function streamWebSpeechAPITranscription( recognition.onspeechstart = () => { console.info('Web Speech API speech detected') + options?.onStatusChange?.('speech-detected') } recognition.onspeechend = () => { @@ -446,6 +470,7 @@ export function streamWebSpeechAPITranscription( recognition.onnomatch = () => { console.info('Web Speech API: No speech match') + options?.onError?.('no-match', 'Speech was detected but could not be recognized. Try speaking more clearly.') } const started = startRecognition() diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a13ae08843..73e7c74699 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -255,6 +255,9 @@ catalogs: posthog-js: specifier: 1.306.1 version: 1.306.1 + remark-breaks: + specifier: ^4.0.0 + version: 4.0.0 splitpanes: specifier: ^4.0.4 version: 4.0.4 @@ -544,10 +547,10 @@ importers: dependencies: '@better-auth/drizzle-adapter': specifier: ^1.5.6 - version: 1.5.6(@better-auth/core@1.5.6(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.1)(better-call@1.3.2(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1))(@better-auth/utils@0.3.1)(drizzle-orm@0.45.1(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.20.0)(better-sqlite3@12.5.0)(kysely@0.28.14)(pg@8.20.0)(postgres@3.4.8)) + version: 1.5.6(@better-auth/core@1.5.6(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.1)(better-call@1.1.8(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1))(@better-auth/utils@0.3.1)(drizzle-orm@0.45.1(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.20.0)(better-sqlite3@12.5.0)(kysely@0.28.14)(pg@8.20.0)(postgres@3.4.8)) '@better-auth/oauth-provider': specifier: 'catalog:' - version: 1.5.6(@better-auth/core@1.5.6(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.1)(better-call@1.3.2(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1))(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(better-auth@1.5.6(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(better-sqlite3@12.5.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.1(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.20.0)(better-sqlite3@12.5.0)(kysely@0.28.14)(pg@8.20.0)(postgres@3.4.8))(pg@8.20.0)(react@19.2.3)(vitest@4.1.1)(vue@3.5.30(typescript@5.9.3)))(better-call@1.3.2(zod@4.3.6)) + version: 1.5.6(@better-auth/core@1.5.6(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.1)(better-call@1.1.8(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1))(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(better-auth@1.5.6(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(better-sqlite3@12.5.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.1(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.20.0)(better-sqlite3@12.5.0)(kysely@0.28.14)(pg@8.20.0)(postgres@3.4.8))(pg@8.20.0)(react@19.2.3)(vitest@4.1.1)(vue@3.5.30(typescript@5.9.3)))(better-call@1.1.8(zod@4.3.6)) '@dotenvx/dotenvx': specifier: ^1.57.2 version: 1.57.2 @@ -668,7 +671,7 @@ importers: devDependencies: '@better-auth/cli': specifier: ^1.4.21 - version: 1.4.21(@better-fetch/fetch@1.1.21)(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(better-call@1.3.2(zod@4.3.6))(drizzle-kit@0.31.10)(jose@6.2.2)(kysely@0.28.14)(magicast@0.5.2)(nanostores@1.1.1)(postgres@3.4.8)(react@19.2.3)(vitest@4.1.1)(vue@3.5.30(typescript@5.9.3)) + version: 1.4.21(@better-fetch/fetch@1.1.21)(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(better-call@1.1.8(zod@4.3.6))(drizzle-kit@0.31.10)(jose@6.2.2)(kysely@0.28.14)(magicast@0.5.2)(nanostores@1.1.1)(postgres@3.4.8)(react@19.2.3)(vitest@4.1.1)(vue@3.5.30(typescript@5.9.3)) '@types/pg': specifier: ^8.20.0 version: 8.20.0 @@ -1153,6 +1156,18 @@ importers: '@proj-airi/ui': specifier: workspace:^ version: link:../../packages/ui + '@proj-airi/visual-chat-gateway': + specifier: workspace:^ + version: link:../../services/visual-chat-gateway + '@proj-airi/visual-chat-ops': + specifier: workspace:^ + version: link:../../packages/visual-chat-ops + '@proj-airi/visual-chat-shared': + specifier: workspace:^ + version: link:../../packages/visual-chat-shared + '@proj-airi/visual-chat-worker-minicpmo': + specifier: workspace:^ + version: link:../../services/visual-chat-worker-minicpmo '@shikijs/markdown-it': specifier: ^4.0.2 version: 4.0.2 @@ -1527,6 +1542,25 @@ importers: specifier: ^3.2.1 version: 3.2.1 + apps/stage-visual-chat-ops: + dependencies: + '@proj-airi/visual-chat-sdk': + specifier: workspace:^ + version: link:../../packages/visual-chat-sdk + pinia: + specifier: 'catalog:' + version: 3.0.4(typescript@5.9.3)(vue@3.5.30(typescript@5.9.3)) + vue: + specifier: 'catalog:' + version: 3.5.30(typescript@5.9.3) + devDependencies: + '@vitejs/plugin-vue': + specifier: ^6.0.5 + version: 6.0.5(vite@8.0.2(@types/node@24.12.0)(esbuild@0.27.2)(jiti@2.6.1)(less@4.6.4)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3))(vue@3.5.30(typescript@5.9.3)) + vite: + specifier: 'catalog:' + version: 8.0.2(@types/node@24.12.0)(esbuild@0.27.2)(jiti@2.6.1)(less@4.6.4)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3) + apps/stage-web: dependencies: '@date-fns/utc': @@ -2232,6 +2266,12 @@ importers: specifier: ^3.2.6 version: 3.2.6(typescript@5.9.3) + examples/visual-chat-local-5090: {} + + examples/visual-chat-mobile-laptop-room: {} + + examples/visual-chat-production-sample: {} + integrations/vscode/airi-plugin-vscode: devDependencies: '@proj-airi/plugin-sdk': @@ -2757,6 +2797,15 @@ importers: '@proj-airi/ui': specifier: workspace:* version: link:../ui + '@proj-airi/visual-chat-protocol': + specifier: workspace:^ + version: link:../visual-chat-protocol + '@proj-airi/visual-chat-sdk': + specifier: workspace:^ + version: link:../visual-chat-sdk + '@proj-airi/visual-chat-shared': + specifier: workspace:^ + version: link:../visual-chat-shared '@shopify/draggable': specifier: 'catalog:' version: 1.2.1 @@ -2942,6 +2991,12 @@ importers: '@proj-airi/ui': specifier: workspace:^ version: link:../ui + '@proj-airi/visual-chat-protocol': + specifier: workspace:^ + version: link:../visual-chat-protocol + '@proj-airi/visual-chat-sdk': + specifier: workspace:^ + version: link:../visual-chat-sdk '@ricky0123/vad-web': specifier: ^0.0.30 version: 0.0.30 @@ -3077,6 +3132,9 @@ importers: reka-ui: specifier: ^2.9.2 version: 2.9.2(vue@3.5.30(typescript@5.9.3)) + remark-breaks: + specifier: 'catalog:' + version: 4.0.0 remark-math: specifier: ^6.0.0 version: 6.0.0 @@ -3582,6 +3640,123 @@ importers: specifier: ^3.2.6 version: 3.2.6(typescript@5.9.3) + packages/visual-chat-livekit: + dependencies: + '@livekit/rtc-node': + specifier: ^0.13.24 + version: 0.13.24 + '@proj-airi/visual-chat-observability': + specifier: workspace:^ + version: link:../visual-chat-observability + '@proj-airi/visual-chat-protocol': + specifier: workspace:^ + version: link:../visual-chat-protocol + '@proj-airi/visual-chat-shared': + specifier: workspace:^ + version: link:../visual-chat-shared + livekit-server-sdk: + specifier: ^2.13.0 + version: 2.15.0 + + packages/visual-chat-media-core: + dependencies: + '@proj-airi/visual-chat-protocol': + specifier: workspace:^ + version: link:../visual-chat-protocol + '@proj-airi/visual-chat-shared': + specifier: workspace:^ + version: link:../visual-chat-shared + + packages/visual-chat-model-minicpmo: + dependencies: + '@proj-airi/visual-chat-observability': + specifier: workspace:^ + version: link:../visual-chat-observability + '@proj-airi/visual-chat-shared': + specifier: workspace:^ + version: link:../visual-chat-shared + + packages/visual-chat-observability: + dependencies: + '@guiiai/logg': + specifier: 'catalog:' + version: 1.2.11 + '@proj-airi/visual-chat-shared': + specifier: workspace:^ + version: link:../visual-chat-shared + nanoid: + specifier: 'catalog:' + version: 5.1.6 + + packages/visual-chat-ops: + dependencies: + '@proj-airi/visual-chat-observability': + specifier: workspace:^ + version: link:../visual-chat-observability + '@proj-airi/visual-chat-shared': + specifier: workspace:^ + version: link:../visual-chat-shared + '@proj-airi/visual-chat-storage': + specifier: workspace:^ + version: link:../visual-chat-storage + + packages/visual-chat-protocol: + dependencies: + valibot: + specifier: 'catalog:' + version: 1.2.0(typescript@5.9.3) + + packages/visual-chat-runtime: + dependencies: + '@proj-airi/visual-chat-media-core': + specifier: workspace:^ + version: link:../visual-chat-media-core + '@proj-airi/visual-chat-observability': + specifier: workspace:^ + version: link:../visual-chat-observability + '@proj-airi/visual-chat-protocol': + specifier: workspace:^ + version: link:../visual-chat-protocol + '@proj-airi/visual-chat-shared': + specifier: workspace:^ + version: link:../visual-chat-shared + '@proj-airi/visual-chat-storage': + specifier: workspace:^ + version: link:../visual-chat-storage + + packages/visual-chat-sdk: + dependencies: + '@proj-airi/visual-chat-protocol': + specifier: workspace:^ + version: link:../visual-chat-protocol + '@proj-airi/visual-chat-shared': + specifier: workspace:^ + version: link:../visual-chat-shared + vue: + specifier: 'catalog:' + version: 3.5.30(typescript@5.9.3) + + packages/visual-chat-shared: + dependencies: + '@moeru/eventa': + specifier: 'catalog:' + version: 1.0.0-beta.3(electron@41.0.3) + '@proj-airi/visual-chat-protocol': + specifier: workspace:^ + version: link:../visual-chat-protocol + nanoid: + specifier: 'catalog:' + version: 5.1.6 + + packages/visual-chat-storage: + dependencies: + '@proj-airi/visual-chat-protocol': + specifier: workspace:^ + version: link:../visual-chat-protocol + '@proj-airi/visual-chat-shared': + specifier: workspace:^ + version: link:../visual-chat-shared + packages/vite-plugin-warpdrive: dependencies: rolldown: @@ -3692,6 +3867,24 @@ importers: specifier: ^0.20.20 version: 0.20.20(@types/node@24.12.0)(canvas@3.2.2)(eslint@10.1.0(jiti@2.6.1))(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(rollup@4.60.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3) + plugins/plugin-visual-chat-pocket: + dependencies: + '@proj-airi/visual-chat-sdk': + specifier: workspace:^ + version: link:../../packages/visual-chat-sdk + + plugins/plugin-visual-chat-tamagotchi: + dependencies: + '@proj-airi/visual-chat-sdk': + specifier: workspace:^ + version: link:../../packages/visual-chat-sdk + + plugins/plugin-visual-chat-web: + dependencies: + '@proj-airi/visual-chat-sdk': + specifier: workspace:^ + version: link:../../packages/visual-chat-sdk + services/discord-bot: dependencies: '@discordjs/voice': @@ -4040,6 +4233,63 @@ importers: specifier: ^4.3.6 version: 4.3.6 + services/visual-chat-gateway: + dependencies: + '@proj-airi/visual-chat-livekit': + specifier: workspace:^ + version: link:../../packages/visual-chat-livekit + '@proj-airi/visual-chat-observability': + specifier: workspace:^ + version: link:../../packages/visual-chat-observability + '@proj-airi/visual-chat-protocol': + specifier: workspace:^ + version: link:../../packages/visual-chat-protocol + '@proj-airi/visual-chat-runtime': + specifier: workspace:^ + version: link:../../packages/visual-chat-runtime + '@proj-airi/visual-chat-shared': + specifier: workspace:^ + version: link:../../packages/visual-chat-shared + '@proj-airi/visual-chat-storage': + specifier: workspace:^ + version: link:../../packages/visual-chat-storage + crossws: + specifier: ^0.4.3 + version: 0.4.4(patch_hash=4d79ec736d10d2a81a9e2a31b067d43f0b6665122267981e652ab9923d165958)(srvx@0.11.13(patch_hash=c761d25a70e22a0925c88fe91a9dd1c3153dd341d8fd4f260166678156c4df2d)) + h3: + specifier: ^2.0.1-rc.11 + version: 2.0.1-rc.19(crossws@0.4.4(patch_hash=4d79ec736d10d2a81a9e2a31b067d43f0b6665122267981e652ab9923d165958)(srvx@0.11.13(patch_hash=c761d25a70e22a0925c88fe91a9dd1c3153dd341d8fd4f260166678156c4df2d))) + listhen: + specifier: ^1.9.0 + version: 1.9.1(srvx@0.11.13(patch_hash=c761d25a70e22a0925c88fe91a9dd1c3153dd341d8fd4f260166678156c4df2d)) + nanoid: + specifier: 'catalog:' + version: 5.1.6 + + services/visual-chat-worker-minicpmo: + dependencies: + '@proj-airi/visual-chat-model-minicpmo': + specifier: workspace:^ + version: link:../../packages/visual-chat-model-minicpmo + '@proj-airi/visual-chat-observability': + specifier: workspace:^ + version: link:../../packages/visual-chat-observability + '@proj-airi/visual-chat-protocol': + specifier: workspace:^ + version: link:../../packages/visual-chat-protocol + '@proj-airi/visual-chat-shared': + specifier: workspace:^ + version: link:../../packages/visual-chat-shared + crossws: + specifier: ^0.4.3 + version: 0.4.4(patch_hash=4d79ec736d10d2a81a9e2a31b067d43f0b6665122267981e652ab9923d165958)(srvx@0.11.13(patch_hash=c761d25a70e22a0925c88fe91a9dd1c3153dd341d8fd4f260166678156c4df2d)) + h3: + specifier: ^2.0.1-rc.11 + version: 2.0.1-rc.19(crossws@0.4.4(patch_hash=4d79ec736d10d2a81a9e2a31b067d43f0b6665122267981e652ab9923d165958)(srvx@0.11.13(patch_hash=c761d25a70e22a0925c88fe91a9dd1c3153dd341d8fd4f260166678156c4df2d))) + listhen: + specifier: ^1.9.0 + version: 1.9.1(srvx@0.11.13(patch_hash=c761d25a70e22a0925c88fe91a9dd1c3153dd341d8fd4f260166678156c4df2d)) + packages: 7zip-bin@5.2.0: @@ -4882,6 +5132,9 @@ packages: '@braidai/lang@1.1.2': resolution: {integrity: sha512-qBcknbBufNHlui137Hft8xauQMTZDKdophmLFv05r2eNmdIv/MlPuP4TdUknHG68UdWLgVZwgxVe735HzJNIwA==} + '@bufbuild/protobuf@1.10.1': + resolution: {integrity: sha512-wJ8ReQbHxsAfXhrf9ixl0aYbZorRuOWpBNzm8pL8ftmSxQx/wnJD5Eg861NwJU/czy2VXFIebCeZnZrI9rktIQ==} + '@bufbuild/protobuf@2.10.2': resolution: {integrity: sha512-uFsRXwIGyu+r6AMdz+XijIIZJYpoWeYzILt5yZ2d3mCjQrWUTVpVD9WL/jZAbvp+Ed04rOhrsk7FiTcEDseB5A==} @@ -4991,6 +5244,9 @@ packages: resolution: {integrity: sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw==} engines: {node: '>=18'} + '@datastructures-js/deque@1.0.8': + resolution: {integrity: sha512-PSBhJ2/SmeRPRHuBv7i/fHWIdSC3JTyq56qb+Rq0wjOagi0/fdV5/B/3Md5zFZus/W6OkSPMaxMKKMNMrSmubg==} + '@date-fns/tz@1.4.1': resolution: {integrity: sha512-P5LUNhtbj6YfI3iJjw5EL9eUAG6OitD0W3fWQcpQjDRc/QIsL0tRNuO1PcDvPccWL1fSTXXdE1ds+l95DV/OFA==} @@ -6301,6 +6557,51 @@ packages: '@lezer/lr@1.4.5': resolution: {integrity: sha512-/YTRKP5yPPSo1xImYQk7AZZMAgap0kegzqCSYHjAL9x1AZ0ZQW+IpcEzMKagCsbTsLnVeWkxYrCNeXG8xEPrjg==} + '@livekit/mutex@1.1.1': + resolution: {integrity: sha512-EsshAucklmpuUAfkABPxJNhzj9v2sG7JuzFDL4ML1oJQSV14sqrpTYnsaOudMAw9yOaW53NU3QQTlUQoRs4czw==} + + '@livekit/protocol@1.45.1': + resolution: {integrity: sha512-sr6p0TwKofHO5KW6kUzjq4hH2de4Al5scQo824xFnyI1XYo0qQn6fTG+bdr+Uj4EedjYAOqjezwUju5OErVIRA==} + + '@livekit/rtc-node-darwin-arm64@0.13.24': + resolution: {integrity: sha512-gm5xOpGu6Rj/mNU2jEijcGhQGN2GdxV2dNYQm3NCKN7ow0BmMFZvXSCAWOWf+9oTutPXHnrc7EN1mt2v+lfqhA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + '@livekit/rtc-node-darwin-x64@0.13.24': + resolution: {integrity: sha512-jZSK5lHDp7+u0jby7PEWMzbxc0F0nLx6FT3FVjuMlT13ZY6QWKDUUCFbfDOtbdhiOZJYc5A4SwvubY6woEJXTg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + '@livekit/rtc-node-linux-arm64-gnu@0.13.24': + resolution: {integrity: sha512-I+IeZET2h+viZ48moEFF0EWDHa+kLii5yuEsw38ya4mHZaZtlfbzrYKGKdONqbI9M9ldvv8XXuD0wFPjuH5CZw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@livekit/rtc-node-linux-x64-gnu@0.13.24': + resolution: {integrity: sha512-vKOxzN/SsrtV8zIVwZCi31bZUhlb6RhJZ0NnY5MwKGSRFPi7Dwt8fmr0Vh0YmsY/p+4eZjKxvFmy7L3WVE54zw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@livekit/rtc-node-win32-x64-msvc@0.13.24': + resolution: {integrity: sha512-yTzqwndq2oKLUkXW2i/BkZMJC6kZOpRO/DKvkkKQvqc3Q+JuWz1m48GmyjIwTOKF28QjqEU3+IrnD65Uu+mFOg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + '@livekit/rtc-node@0.13.24': + resolution: {integrity: sha512-06pF8YJlJk11R6J7kFXFpwV8etpbmCskoXFvwfwcDDixMqaP6qtS5srq3G23mDaRjx7ofz/PXg2GtiZbqNGT5A==} + engines: {node: '>= 18'} + + '@livekit/typed-emitter@3.0.0': + resolution: {integrity: sha512-9bl0k4MgBPZu3Qu3R3xy12rmbW17e3bE9yf4YY85gJIQ3ezLEj/uzpKHWBsLaDoL5Mozz8QCgggwIBudYQWeQg==} + '@loaderkit/resolve@1.0.4': resolution: {integrity: sha512-rJzYKVcV4dxJv+vW6jlvagF8zvGxHJ2+HTr1e2qOejfmGhAApgJHl8Aog4mMszxceTRiKTTbnpgmTO1bEZHV/A==} @@ -7842,6 +8143,100 @@ packages: cpu: [x64] os: [win32] + '@parcel/watcher-android-arm64@2.5.6': + resolution: {integrity: sha512-YQxSS34tPF/6ZG7r/Ih9xy+kP/WwediEUsqmtf0cuCV5TPPKw/PQHRhueUo6JdeFJaqV3pyjm0GdYjZotbRt/A==} + engines: {node: '>= 10.0.0'} + cpu: [arm64] + os: [android] + + '@parcel/watcher-darwin-arm64@2.5.6': + resolution: {integrity: sha512-Z2ZdrnwyXvvvdtRHLmM4knydIdU9adO3D4n/0cVipF3rRiwP+3/sfzpAwA/qKFL6i1ModaabkU7IbpeMBgiVEA==} + engines: {node: '>= 10.0.0'} + cpu: [arm64] + os: [darwin] + + '@parcel/watcher-darwin-x64@2.5.6': + resolution: {integrity: sha512-HgvOf3W9dhithcwOWX9uDZyn1lW9R+7tPZ4sug+NGrGIo4Rk1hAXLEbcH1TQSqxts0NYXXlOWqVpvS1SFS4fRg==} + engines: {node: '>= 10.0.0'} + cpu: [x64] + os: [darwin] + + '@parcel/watcher-freebsd-x64@2.5.6': + resolution: {integrity: sha512-vJVi8yd/qzJxEKHkeemh7w3YAn6RJCtYlE4HPMoVnCpIXEzSrxErBW5SJBgKLbXU3WdIpkjBTeUNtyBVn8TRng==} + engines: {node: '>= 10.0.0'} + cpu: [x64] + os: [freebsd] + + '@parcel/watcher-linux-arm-glibc@2.5.6': + resolution: {integrity: sha512-9JiYfB6h6BgV50CCfasfLf/uvOcJskMSwcdH1PHH9rvS1IrNy8zad6IUVPVUfmXr+u+Km9IxcfMLzgdOudz9EQ==} + engines: {node: '>= 10.0.0'} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@parcel/watcher-linux-arm-musl@2.5.6': + resolution: {integrity: sha512-Ve3gUCG57nuUUSyjBq/MAM0CzArtuIOxsBdQ+ftz6ho8n7s1i9E1Nmk/xmP323r2YL0SONs1EuwqBp2u1k5fxg==} + engines: {node: '>= 10.0.0'} + cpu: [arm] + os: [linux] + libc: [musl] + + '@parcel/watcher-linux-arm64-glibc@2.5.6': + resolution: {integrity: sha512-f2g/DT3NhGPdBmMWYoxixqYr3v/UXcmLOYy16Bx0TM20Tchduwr4EaCbmxh1321TABqPGDpS8D/ggOTaljijOA==} + engines: {node: '>= 10.0.0'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@parcel/watcher-linux-arm64-musl@2.5.6': + resolution: {integrity: sha512-qb6naMDGlbCwdhLj6hgoVKJl2odL34z2sqkC7Z6kzir8b5W65WYDpLB6R06KabvZdgoHI/zxke4b3zR0wAbDTA==} + engines: {node: '>= 10.0.0'} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@parcel/watcher-linux-x64-glibc@2.5.6': + resolution: {integrity: sha512-kbT5wvNQlx7NaGjzPFu8nVIW1rWqV780O7ZtkjuWaPUgpv2NMFpjYERVi0UYj1msZNyCzGlaCWEtzc+exjMGbQ==} + engines: {node: '>= 10.0.0'} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@parcel/watcher-linux-x64-musl@2.5.6': + resolution: {integrity: sha512-1JRFeC+h7RdXwldHzTsmdtYR/Ku8SylLgTU/reMuqdVD7CtLwf0VR1FqeprZ0eHQkO0vqsbvFLXUmYm/uNKJBg==} + engines: {node: '>= 10.0.0'} + cpu: [x64] + os: [linux] + libc: [musl] + + '@parcel/watcher-wasm@2.5.6': + resolution: {integrity: sha512-byAiBZ1t3tXQvc8dMD/eoyE7lTXYorhn+6uVW5AC+JGI1KtJC/LvDche5cfUE+qiefH+Ybq0bUCJU0aB1cSHUA==} + engines: {node: '>= 10.0.0'} + bundledDependencies: + - napi-wasm + + '@parcel/watcher-win32-arm64@2.5.6': + resolution: {integrity: sha512-3ukyebjc6eGlw9yRt678DxVF7rjXatWiHvTXqphZLvo7aC5NdEgFufVwjFfY51ijYEWpXbqF5jtrK275z52D4Q==} + engines: {node: '>= 10.0.0'} + cpu: [arm64] + os: [win32] + + '@parcel/watcher-win32-ia32@2.5.6': + resolution: {integrity: sha512-k35yLp1ZMwwee3Ez/pxBi5cf4AoBKYXj00CZ80jUz5h8prpiaQsiRPKQMxoLstNuqe2vR4RNPEAEcjEFzhEz/g==} + engines: {node: '>= 10.0.0'} + cpu: [ia32] + os: [win32] + + '@parcel/watcher-win32-x64@2.5.6': + resolution: {integrity: sha512-hbQlYcCq5dlAX9Qx+kFb0FHue6vbjlf0FrNzSKdYK2APUf7tGfGxQCk2ihEREmbR6ZMc0MVAD5RIX/41gpUzTw==} + engines: {node: '>= 10.0.0'} + cpu: [x64] + os: [win32] + + '@parcel/watcher@2.5.6': + resolution: {integrity: sha512-tmmZ3lQxAe/k/+rNnXQRawJ4NjxO2hqiOLTHvWchtGZULp4RyFeh6aU4XdOYBFe2KE1oShQTv4AblOs2iOrNnQ==} + engines: {node: '>= 10.0.0'} + '@pinia/testing@1.0.3': resolution: {integrity: sha512-g+qR49GNdI1Z8rZxKrQC3GN+LfnGTNf5Kk8Nz5Cz6mIGva5WRS+ffPXQfzhA0nu6TveWzPNYTjGl4nJqd3Cu9Q==} peerDependencies: @@ -11014,6 +11409,10 @@ packages: resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} engines: {node: '>= 0.4'} + camelcase-keys@9.1.3: + resolution: {integrity: sha512-Rircqi9ch8AnZscQcsA1C47NFdaO3wukpmIRzYcDOrmvgt78hM/sj5pZhZNec2NM12uk5vTwRHZ4anGcrC4ZTg==} + engines: {node: '>=16'} + camelcase@4.1.0: resolution: {integrity: sha512-FxAv7HpHrXbh3aPo4o2qxHay2lkLY3x5Mw3KeE4KQE8ysVfziWeRZDwcjauvwBSGEC/nXUPzZy8zeh4HokqOnw==} engines: {node: '>=4'} @@ -11113,6 +11512,9 @@ packages: citty@0.2.1: resolution: {integrity: sha512-kEV95lFBhQgtogAPlQfJJ0WGVSokvLr/UEoFPiKKOXF7pl98HfUVUD0ejsuTCld/9xH9vogSywZ5KqHzXrZpqg==} + citty@0.2.2: + resolution: {integrity: sha512-+6vJA3L98yv+IdfKGZHBNiGW5KHn22e/JwID0Strsz8h4S/csAu/OuICwxrg44k5MRiZHWIo8XXuJgQTriRP4w==} + cjs-module-lexer@1.4.3: resolution: {integrity: sha512-9z8TZaGM1pfswYeXrUpzPrkx8UnWYdhJclsiYMm6x/w5+nN+8Tf/LnAgfLGQCm59qAOxU8WwHEq2vNwF6i4j+Q==} @@ -11321,6 +11723,9 @@ packages: cookie-es@1.2.2: resolution: {integrity: sha512-+W7VmiVINB+ywl1HGXJXmrqkOhpKrIiVZV6tQuV54ZyQC7MMuBt81Vc336GMLoHBq5hV/F9eXgt5Mnx0Rha5Fg==} + cookie-es@1.2.3: + resolution: {integrity: sha512-lXVyvUvrNXblMqzIRrxHb57UUVmqsSWlxqt3XIjCkUP0wDAf6uicO6KMbEgYrMNtEvWgWHwe42CKxPu9MYAnWw==} + cookie-signature@1.0.7: resolution: {integrity: sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==} @@ -11566,6 +11971,9 @@ packages: date-fns@4.1.0: resolution: {integrity: sha512-Ukq0owbQXxa/U3EGtsdVBkR1w7KOQ5gIBqdH2hkvknzZPYvBxb/aa6E8L7tmjFtkwZBu3UXBbjIgPo/Ez4xaNg==} + dateformat@4.6.3: + resolution: {integrity: sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==} + de-indent@1.0.2: resolution: {integrity: sha512-e/1zu3xH5MQryN2zdVaF0OrdNLUbvWxzMbi+iNA6Bky7l1RoP8a2fIbRocyHclXt/arDrrR6lL3TqFD9pMQTsg==} @@ -11657,6 +12065,9 @@ packages: defu@6.1.4: resolution: {integrity: sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg==} + defu@6.1.6: + resolution: {integrity: sha512-f8mefEW4WIVg4LckePx3mALjQSPQgFlg9U8yaPdlsbdYcHQyj9n2zL2LJEA52smeYxOvmd/nB7TpMtHGMTHcug==} + delaunator@5.0.1: resolution: {integrity: sha512-8nvh+XBe96aCESrGOqMp/84b13H9cdKbG5P2ejQCh4d4sK9RL4371qou9drQjMhvnPmhWl5hnmqbEE0fXr9Xnw==} @@ -12592,6 +13003,9 @@ packages: resolution: {integrity: sha512-Wrk35e8ydCKDj/ArClo1VrPVmN8zph5V4AtHwIuHhvMXsKf73UT3BOD+azBIW+3wOJ4FhEH7zyaJCFvChjYvMA==} engines: {'0': node >=0.6.0} + fast-copy@4.0.2: + resolution: {integrity: sha512-ybA6PDXIXOXivLJK/z9e+Otk7ve13I4ckBvGO5I2RRmBU1gMHLVDJYEuJYhGwez7YNlYji2M2DvVU+a9mSFDlw==} + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -12609,6 +13023,9 @@ packages: resolution: {integrity: sha512-dwsoQlS7h9hMeYUq1W++23NDcBLV4KqONnITDV9DjfS3q1SgDGVrBdvvTLUotWtPSD7asWDV9/CmsZPy8Hf70A==} engines: {node: '>=6'} + fast-safe-stringify@2.1.1: + resolution: {integrity: sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==} + fast-uri@3.1.0: resolution: {integrity: sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==} @@ -13089,6 +13506,9 @@ packages: resolution: {integrity: sha512-ax7ZYomf6jqPTQ4+XCpUGyXKHk5WweS+e05MBO4/y3WJ5RkmPXNKvX+bx1behVILVwr6JSQvZAku021CHPXG3Q==} engines: {node: '>=10'} + h3@1.15.11: + resolution: {integrity: sha512-L3THSe2MPeBwgIZVSH5zLdBBU90TOxarvhK9d04IDY2AmVS8j2Jz2LIWtwsGOU3lu2I5jCN7FNvVfY2+XyF+mg==} + h3@1.15.5: resolution: {integrity: sha512-xEyq3rSl+dhGX2Lm0+eFQIAzlDN6Fs0EcC4f7BNUmzaRX/PTzeuM+Tr2lHB8FoXggsQIeXLj8EDVgs5ywxyxmg==} @@ -13191,6 +13611,9 @@ packages: resolution: {integrity: sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==} hasBin: true + help-me@5.0.0: + resolution: {integrity: sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg==} + hey-listen@1.0.8: resolution: {integrity: sha512-COpmrF2NOg4TBWUJ5UVyaCU2A88wEMkUPK4hNqyCkqHbxT92BbvfjoSozkAIIm6XhicGlJHhFdullInrdhwU8Q==} @@ -13268,6 +13691,10 @@ packages: resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==} engines: {node: '>= 14'} + http-shutdown@1.2.2: + resolution: {integrity: sha512-S9wWkJ/VSY9/k4qcjG318bqJNruzE4HySUhFYknwmu6LBP97KLLfwNf+n4V1BHurvFNkSKLFnK/RsuUnRTf9Vw==} + engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'} + http-signature@1.2.0: resolution: {integrity: sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ==} engines: {node: '>=0.8', npm: '>=1.3.7'} @@ -13642,9 +14069,16 @@ packages: resolution: {integrity: sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==} hasBin: true + jose@5.10.0: + resolution: {integrity: sha512-s+3Al/p9g32Iq+oqXxkW//7jk2Vig6FF1CFqzVXoTUXt2qz89YWbL+OwS17NFYEvxC35n0FKeGO2LGYSxeM2Gg==} + jose@6.2.2: resolution: {integrity: sha512-d7kPDd34KO/YnzaDOlikGpOurfF0ByC2sEV4cANCtdqLlTfBlw2p14O/5d/zv40gJPbIQxfES3nSx1/oYNyuZQ==} + joycon@3.1.1: + resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} + engines: {node: '>=10'} + jpeg-js@0.4.4: resolution: {integrity: sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==} @@ -13951,10 +14385,18 @@ packages: linkify-it@5.0.0: resolution: {integrity: sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==} + listhen@1.9.1: + resolution: {integrity: sha512-4EhoyVcXEpNlY5HJRSQpH7Rba94M8N2JmI62ePjl0lrJKXSfG0F1FAgHGxBoz/T3pe41sUEwkIRRIcaUL0/Ofw==} + hasBin: true + listr2@8.3.3: resolution: {integrity: sha512-LWzX2KsqcB1wqQ4AHgYb4RsDXauQiqhjLk+6hjbaeHG4zpjjVAB6wC/gz6X0l+Du1cN3pUB5ZlrvTbhGSNnUQQ==} engines: {node: '>=18.0.0'} + livekit-server-sdk@2.15.0: + resolution: {integrity: sha512-HmzjWnwEwwShu8yUf7VGFXdc+BuMJR5pnIY4qsdlhqI9d9wDgq+4cdTEHg0NEBaiGnc6PCOBiaTYgmIyVJ0S9w==} + engines: {node: '>=18'} + local-pkg@1.1.2: resolution: {integrity: sha512-arhlxbFRmoQHl33a0Zkle/YWlmNwoyt6QNZEIJcqNbdrsix5Lvc4HyyI3EnwxTYlZYc32EbYrQ8SzEZ7dqgg9A==} engines: {node: '>=14'} @@ -14118,6 +14560,10 @@ packages: many-keys-map@2.0.1: resolution: {integrity: sha512-DHnZAD4phTbZ+qnJdjoNEVU1NecYoSdbOOoVmTDH46AuxDkEVh3MxTVpXq10GtcTC6mndN9dkv1rNfpjRcLnOw==} + map-obj@5.0.0: + resolution: {integrity: sha512-2L3MIgJynYrZ3TYMriLDLWocz15okFakV6J12HXvMXDHui2x/zgChzg1u9mFFGbbGWE+GsLpQByt4POb9Or+uA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + mark.js@8.11.1: resolution: {integrity: sha512-1I+1qpDt4idfgLQG+BNWmrqku+7/2bi5nLf4YwF8y8zXvmfiTBY3PV3ZibfrjBueCByROpuBjLLFCajqkgYoLQ==} @@ -14184,6 +14630,9 @@ packages: mdast-util-math@3.0.0: resolution: {integrity: sha512-Tl9GBNeG/AhJnQM221bJR2HPvLOSnLE/T9cJI9tlc6zwQk2nPk/4f0cHkOdEixQPC/j8UtKDdITswvLAy1OZ1w==} + mdast-util-newline-to-break@2.0.0: + resolution: {integrity: sha512-MbgeFca0hLYIEx/2zGsszCSEJJ1JSCdiY5xQxRcLDDGa8EPvlLPupJ4DSajbMPAnC0je8jfb9TiUATnxxrHUog==} + mdast-util-phrasing@4.1.0: resolution: {integrity: sha512-TqICwyvJJpBwvGAMZjj4J2n0X8QWp21b9l0o7eXyVJ25YNWYbJDVIyD1bZXE6WtV6RmKJVYmQAKWa0zWOABz2w==} @@ -14498,6 +14947,9 @@ packages: mlly@1.8.0: resolution: {integrity: sha512-l8D9ODSRWLe2KHJSifWGwBqpTZXIXTeo8mlKjY+E2HAakaTeNpqAyBZ8GSqLzHgw4XmHmC8whvpjJNMbFZN7/g==} + mlly@1.8.2: + resolution: {integrity: sha512-d+ObxMQFmbt10sretNDytwt85VrbkhhUA/JBGm1MPaWJ65Cl4wOgLaB1NYvJSZ0Ef03MMEU/0xpPMXUIQ29UfA==} + module-details-from-path@1.0.4: resolution: {integrity: sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==} @@ -14665,6 +15117,10 @@ packages: resolution: {integrity: sha512-rLvcdSyRCyouf6jcOIPe/BgwG/d7hKjzMKOas33/pHEr6gbq18IK9zV7DiPvzsz0oBJPme6qr6H6kGZuI9/DZg==} engines: {node: '>= 6.13.0'} + node-forge@1.4.0: + resolution: {integrity: sha512-LarFH0+6VfriEhqMMcLX2F7SwSXeWwnEAJEsYm5QKWchiVYVvJyV9v7UDvUv+w5HO23ZpQTXDv/GxdDdMyOuoQ==} + engines: {node: '>= 6.13.0'} + node-gyp-build@4.8.4: resolution: {integrity: sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==} hasBin: true @@ -15102,6 +15558,13 @@ packages: pino-abstract-transport@2.0.0: resolution: {integrity: sha512-F63x5tizV6WCh4R6RHyi2Ml+M70DNRXt/+HANowMflpgGFMAym/VKm6G7ZOQRjqN7XbGxK1Lg9t6ZrtzOaivMw==} + pino-abstract-transport@3.0.0: + resolution: {integrity: sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==} + + pino-pretty@13.1.3: + resolution: {integrity: sha512-ttXRkkOz6WWC95KeY9+xxWL6AtImwbyMHrL1mSwqwW9u+vLp/WIElvHvCSDg0xO/Dzrggz1zv3rN5ovTRVowKg==} + hasBin: true + pino-std-serializers@7.0.0: resolution: {integrity: sha512-e906FRY0+tV27iq4juKzSYPbUj2do2X2JX4EzSca1631EB2QJQUqGbDuERal7LCtOpxl6x3+nvo9NPZcmjkiFA==} @@ -15455,6 +15918,10 @@ packages: resolution: {integrity: sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==} engines: {node: '>=10'} + quick-lru@6.1.2: + resolution: {integrity: sha512-AAFUA5O1d83pIHEhJwWCq/RQcRukCkn/NSm2QsTEMle5f2hP0ChI2+3Xb051PZCkLryI/Ir1MVKviT2FIloaTQ==} + engines: {node: '>=12'} + radashi@12.7.1: resolution: {integrity: sha512-rwxcGY3oKMQJ+ojhS4MlxyVWqdXPVJSxg3ZjXDYYz26DYRuAAs+7XBM406/GYzPEBbjSJqdKfHDpBRfjWn34RQ==} engines: {node: '>=16.0.0'} @@ -15620,6 +16087,9 @@ packages: peerDependencies: vue: '>= 3.4.0' + remark-breaks@4.0.0: + resolution: {integrity: sha512-IjEjJOkH4FuJvHZVIW0QCDWxcG96kCq7An/KVH2NfJe6rKZU2AsHeB3OEjPNRxi4QC34Xdx7I2KGYn6IpT7gxQ==} + remark-math@6.0.0: resolution: {integrity: sha512-MMqgnP74Igy+S3WwnhQ7kqGlEerTETXMvJhrUzDikVZ2/uogJCb+WHUg97hK9/jcfc0dkD73s3LN8zU49cTEtA==} @@ -15835,6 +16305,9 @@ packages: resolution: {integrity: sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA==} engines: {node: '>=4'} + secure-json-parse@4.1.0: + resolution: {integrity: sha512-l4KnYfEyqYJxDwlNVyRfO2E4NTHfMKAWdUuA8J0yve2Dz/E/PdBepY03RvyJpssIpRFwJoCD55wA+mEDs6ByWA==} + semver-compare@1.0.0: resolution: {integrity: sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==} @@ -16408,6 +16881,10 @@ packages: tinybench@2.9.0: resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} + tinyclip@0.1.12: + resolution: {integrity: sha512-Ae3OVUqifDw0wBriIBS7yVaW44Dp6eSHQcyq4Igc7eN2TJH/2YsicswaW+J/OuMvhpDPOKEgpAZCjkb4hpoyeA==} + engines: {node: ^16.14.0 || >= 17.3.0} + tinycolor2@1.6.0: resolution: {integrity: sha512-XPaBkWQJdsf3pLKJV9p4qN/S+fm2Oj8AIPo1BTUhg5oxkvm9+SVEGFdhyOz7tTdUTfvxMiAs4sp6/eZO2Ew+pw==} @@ -17005,6 +17482,10 @@ packages: resolution: {integrity: sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw==} engines: {node: '>=8'} + untun@0.1.3: + resolution: {integrity: sha512-4luGP9LMYszMRZwsvyUd9MrxgEGZdZuZgpVQHEEX0lCYFESasVRvZd0EYpCkOIbJKHMuv0LskpXc/8Un+MJzEQ==} + hasBin: true + untyped@2.0.0: resolution: {integrity: sha512-nwNCjxJTjNuLCgFr42fEak5OcLuB3ecca+9ksPFNvtfYSLpjf+iJqSIaSnIile6ZPbKYxI5k2AfXqeopGudK/g==} hasBin: true @@ -17023,6 +17504,9 @@ packages: resolution: {integrity: sha512-+dwUY4L35XFYEzE+OAL3sarJdUioVovq+8f7lcIJ7wnmnYQV5UD1Y/lcwaMSyaQ6Bj3JMj1XSTjZbNLHn/19yA==} engines: {node: '>=18'} + uqr@0.1.3: + resolution: {integrity: sha512-0rjE8iEJe4YmT9TOhwsZtqCMRLc5DXZUI2UEYUUg63ikBkqqE5EYWaI0etFe/5KUcmcYwLih2RND1kq+hrUJXA==} + uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} @@ -18785,13 +19269,13 @@ snapshots: '@bcoe/v8-coverage@1.0.2': {} - '@better-auth/cli@1.4.21(@better-fetch/fetch@1.1.21)(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(better-call@1.3.2(zod@4.3.6))(drizzle-kit@0.31.10)(jose@6.2.2)(kysely@0.28.14)(magicast@0.5.2)(nanostores@1.1.1)(postgres@3.4.8)(react@19.2.3)(vitest@4.1.1)(vue@3.5.30(typescript@5.9.3))': + '@better-auth/cli@1.4.21(@better-fetch/fetch@1.1.21)(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(better-call@1.1.8(zod@4.3.6))(drizzle-kit@0.31.10)(jose@6.2.2)(kysely@0.28.14)(magicast@0.5.2)(nanostores@1.1.1)(postgres@3.4.8)(react@19.2.3)(vitest@4.1.1)(vue@3.5.30(typescript@5.9.3))': dependencies: '@babel/core': 7.29.0 '@babel/preset-react': 7.28.5(@babel/core@7.29.0) '@babel/preset-typescript': 7.28.5(@babel/core@7.29.0) - '@better-auth/core': 1.4.21(@better-auth/utils@0.3.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.2(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1) - '@better-auth/telemetry': 1.4.21(@better-auth/core@1.4.21(@better-auth/utils@0.3.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.2(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1)) + '@better-auth/core': 1.4.21(@better-auth/utils@0.3.0)(@better-fetch/fetch@1.1.21)(better-call@1.1.8(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1) + '@better-auth/telemetry': 1.4.21(@better-auth/core@1.4.21(@better-auth/utils@0.3.0)(@better-fetch/fetch@1.1.21)(better-call@1.1.8(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1)) '@better-auth/utils': 0.3.0 '@clack/prompts': 0.11.0 '@mrleebo/prisma-ast': 0.13.1 @@ -18868,12 +19352,14 @@ snapshots: nanostores: 1.1.1 zod: 4.3.6 - '@better-auth/core@1.4.21(@better-auth/utils@0.3.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.2(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1)': + '@better-auth/core@1.5.6(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.1)(better-call@1.1.8(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1)': dependencies: - '@better-auth/utils': 0.3.0 + '@better-auth/utils': 0.3.1 '@better-fetch/fetch': 1.1.21 + '@opentelemetry/api': 1.9.1 + '@opentelemetry/semantic-conventions': 1.40.0 '@standard-schema/spec': 1.1.0 - better-call: 1.3.2(zod@4.3.6) + better-call: 1.1.8(zod@4.3.6) jose: 6.2.2 kysely: 0.28.14 nanostores: 1.1.1 @@ -18892,6 +19378,13 @@ snapshots: nanostores: 1.1.1 zod: 4.3.6 + '@better-auth/drizzle-adapter@1.5.6(@better-auth/core@1.5.6(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.1)(better-call@1.1.8(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1))(@better-auth/utils@0.3.1)(drizzle-orm@0.45.1(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.20.0)(better-sqlite3@12.5.0)(kysely@0.28.14)(pg@8.20.0)(postgres@3.4.8))': + dependencies: + '@better-auth/core': 1.5.6(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.1)(better-call@1.1.8(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1) + '@better-auth/utils': 0.3.1 + optionalDependencies: + drizzle-orm: 0.45.1(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.20.0)(better-sqlite3@12.5.0)(kysely@0.28.14)(pg@8.20.0)(postgres@3.4.8) + '@better-auth/drizzle-adapter@1.5.6(@better-auth/core@1.5.6(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.1)(better-call@1.3.2(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1))(@better-auth/utils@0.3.1)(drizzle-orm@0.45.1(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.20.0)(better-sqlite3@12.5.0)(kysely@0.28.14)(pg@8.20.0)(postgres@3.4.8))': dependencies: '@better-auth/core': 1.5.6(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.1)(better-call@1.3.2(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1) @@ -18916,13 +19409,13 @@ snapshots: '@better-auth/core': 1.5.6(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.1)(better-call@1.3.2(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1) '@better-auth/utils': 0.3.1 - '@better-auth/oauth-provider@1.5.6(@better-auth/core@1.5.6(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.1)(better-call@1.3.2(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1))(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(better-auth@1.5.6(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(better-sqlite3@12.5.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.1(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.20.0)(better-sqlite3@12.5.0)(kysely@0.28.14)(pg@8.20.0)(postgres@3.4.8))(pg@8.20.0)(react@19.2.3)(vitest@4.1.1)(vue@3.5.30(typescript@5.9.3)))(better-call@1.3.2(zod@4.3.6))': + '@better-auth/oauth-provider@1.5.6(@better-auth/core@1.5.6(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.1)(better-call@1.1.8(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1))(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(better-auth@1.5.6(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(better-sqlite3@12.5.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.1(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.20.0)(better-sqlite3@12.5.0)(kysely@0.28.14)(pg@8.20.0)(postgres@3.4.8))(pg@8.20.0)(react@19.2.3)(vitest@4.1.1)(vue@3.5.30(typescript@5.9.3)))(better-call@1.1.8(zod@4.3.6))': dependencies: - '@better-auth/core': 1.5.6(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.1)(better-call@1.3.2(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1) + '@better-auth/core': 1.5.6(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.1)(better-call@1.1.8(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1) '@better-auth/utils': 0.3.1 '@better-fetch/fetch': 1.1.21 better-auth: 1.5.6(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(better-sqlite3@12.5.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.1(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.20.0)(better-sqlite3@12.5.0)(kysely@0.28.14)(pg@8.20.0)(postgres@3.4.8))(pg@8.20.0)(react@19.2.3)(vitest@4.1.1)(vue@3.5.30(typescript@5.9.3)) - better-call: 1.3.2(zod@4.3.6) + better-call: 1.1.8(zod@4.3.6) jose: 6.2.2 zod: 4.3.6 @@ -18933,9 +19426,9 @@ snapshots: optionalDependencies: '@prisma/client': 5.22.0 - '@better-auth/telemetry@1.4.21(@better-auth/core@1.4.21(@better-auth/utils@0.3.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.2(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1))': + '@better-auth/telemetry@1.4.21(@better-auth/core@1.4.21(@better-auth/utils@0.3.0)(@better-fetch/fetch@1.1.21)(better-call@1.1.8(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1))': dependencies: - '@better-auth/core': 1.4.21(@better-auth/utils@0.3.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.2(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1) + '@better-auth/core': 1.4.21(@better-auth/utils@0.3.0)(@better-fetch/fetch@1.1.21)(better-call@1.1.8(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1) '@better-auth/utils': 0.3.0 '@better-fetch/fetch': 1.1.21 @@ -18955,6 +19448,8 @@ snapshots: '@braidai/lang@1.1.2': {} + '@bufbuild/protobuf@1.10.1': {} + '@bufbuild/protobuf@2.10.2': {} '@capacitor/android@8.2.0(@capacitor/core@8.2.0)': @@ -19102,6 +19597,8 @@ snapshots: '@csstools/css-tokenizer@3.0.4': {} + '@datastructures-js/deque@1.0.8': {} + '@date-fns/tz@1.4.1': {} '@date-fns/utc@2.1.1': {} @@ -20399,6 +20896,44 @@ snapshots: dependencies: '@lezer/common': 1.5.0 + '@livekit/mutex@1.1.1': {} + + '@livekit/protocol@1.45.1': + dependencies: + '@bufbuild/protobuf': 1.10.1 + + '@livekit/rtc-node-darwin-arm64@0.13.24': + optional: true + + '@livekit/rtc-node-darwin-x64@0.13.24': + optional: true + + '@livekit/rtc-node-linux-arm64-gnu@0.13.24': + optional: true + + '@livekit/rtc-node-linux-x64-gnu@0.13.24': + optional: true + + '@livekit/rtc-node-win32-x64-msvc@0.13.24': + optional: true + + '@livekit/rtc-node@0.13.24': + dependencies: + '@bufbuild/protobuf': 1.10.1 + '@datastructures-js/deque': 1.0.8 + '@livekit/mutex': 1.1.1 + '@livekit/typed-emitter': 3.0.0 + pino: 9.7.0 + pino-pretty: 13.1.3 + optionalDependencies: + '@livekit/rtc-node-darwin-arm64': 0.13.24 + '@livekit/rtc-node-darwin-x64': 0.13.24 + '@livekit/rtc-node-linux-arm64-gnu': 0.13.24 + '@livekit/rtc-node-linux-x64-gnu': 0.13.24 + '@livekit/rtc-node-win32-x64-msvc': 0.13.24 + + '@livekit/typed-emitter@3.0.0': {} + '@loaderkit/resolve@1.0.4': dependencies: '@braidai/lang': 1.1.2 @@ -21947,6 +22482,71 @@ snapshots: '@oxlint/binding-win32-x64-msvc@1.56.0': optional: true + '@parcel/watcher-android-arm64@2.5.6': + optional: true + + '@parcel/watcher-darwin-arm64@2.5.6': + optional: true + + '@parcel/watcher-darwin-x64@2.5.6': + optional: true + + '@parcel/watcher-freebsd-x64@2.5.6': + optional: true + + '@parcel/watcher-linux-arm-glibc@2.5.6': + optional: true + + '@parcel/watcher-linux-arm-musl@2.5.6': + optional: true + + '@parcel/watcher-linux-arm64-glibc@2.5.6': + optional: true + + '@parcel/watcher-linux-arm64-musl@2.5.6': + optional: true + + '@parcel/watcher-linux-x64-glibc@2.5.6': + optional: true + + '@parcel/watcher-linux-x64-musl@2.5.6': + optional: true + + '@parcel/watcher-wasm@2.5.6': + dependencies: + is-glob: 4.0.3 + picomatch: 4.0.3 + + '@parcel/watcher-win32-arm64@2.5.6': + optional: true + + '@parcel/watcher-win32-ia32@2.5.6': + optional: true + + '@parcel/watcher-win32-x64@2.5.6': + optional: true + + '@parcel/watcher@2.5.6': + dependencies: + detect-libc: 2.1.2 + is-glob: 4.0.3 + node-addon-api: 7.1.1 + picomatch: 4.0.3 + optionalDependencies: + '@parcel/watcher-android-arm64': 2.5.6 + '@parcel/watcher-darwin-arm64': 2.5.6 + '@parcel/watcher-darwin-x64': 2.5.6 + '@parcel/watcher-freebsd-x64': 2.5.6 + '@parcel/watcher-linux-arm-glibc': 2.5.6 + '@parcel/watcher-linux-arm-musl': 2.5.6 + '@parcel/watcher-linux-arm64-glibc': 2.5.6 + '@parcel/watcher-linux-arm64-musl': 2.5.6 + '@parcel/watcher-linux-x64-glibc': 2.5.6 + '@parcel/watcher-linux-x64-musl': 2.5.6 + '@parcel/watcher-win32-arm64': 2.5.6 + '@parcel/watcher-win32-ia32': 2.5.6 + '@parcel/watcher-win32-x64': 2.5.6 + '@pinia/testing@1.0.3(pinia@3.0.4(typescript@5.9.3)(vue@3.5.30(typescript@5.9.3)))': dependencies: pinia: 3.0.4(typescript@5.9.3)(vue@3.5.30(typescript@5.9.3)) @@ -23943,6 +24543,20 @@ snapshots: vite: 8.0.2(@types/node@24.12.0)(esbuild@0.27.2)(jiti@2.6.1)(less@4.6.4)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3) vue: 3.5.30(typescript@5.9.3) + '@vitest/browser-playwright@4.1.1(bufferutil@4.1.0)(playwright@1.59.0)(utf-8-validate@5.0.10)(vite@6.4.1(@types/node@24.12.0)(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3))(vitest@4.1.1)': + dependencies: + '@vitest/browser': 4.1.1(bufferutil@4.1.0)(utf-8-validate@5.0.10)(vite@6.4.1(@types/node@24.12.0)(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3))(vitest@4.1.1) + '@vitest/mocker': 4.1.1(vite@6.4.1(@types/node@24.12.0)(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3)) + playwright: 1.59.0 + tinyrainbow: 3.0.3 + vitest: 4.1.1(@opentelemetry/api@1.9.1)(@types/node@24.12.0)(@vitest/browser-playwright@4.1.1)(jsdom@27.4.0(bufferutil@4.1.0)(canvas@3.2.2)(utf-8-validate@5.0.10))(vite@6.4.1(@types/node@24.12.0)(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3)) + transitivePeerDependencies: + - bufferutil + - msw + - utf-8-validate + - vite + optional: true + '@vitest/browser-playwright@4.1.1(bufferutil@4.1.0)(playwright@1.59.0)(utf-8-validate@5.0.10)(vite@8.0.2(@types/node@24.12.0)(esbuild@0.27.2)(jiti@2.6.1)(less@4.6.4)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3))(vitest@4.1.1)': dependencies: '@vitest/browser': 4.1.1(bufferutil@4.1.0)(utf-8-validate@5.0.10)(vite@8.0.2(@types/node@24.12.0)(esbuild@0.27.2)(jiti@2.6.1)(less@4.6.4)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3))(vitest@4.1.1) @@ -23956,6 +24570,24 @@ snapshots: - utf-8-validate - vite + '@vitest/browser@4.1.1(bufferutil@4.1.0)(utf-8-validate@5.0.10)(vite@6.4.1(@types/node@24.12.0)(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3))(vitest@4.1.1)': + dependencies: + '@blazediff/core': 1.9.1 + '@vitest/mocker': 4.1.1(vite@6.4.1(@types/node@24.12.0)(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3)) + '@vitest/utils': 4.1.1 + magic-string: 0.30.21 + pngjs: 7.0.0 + sirv: 3.0.2 + tinyrainbow: 3.0.3 + vitest: 4.1.1(@opentelemetry/api@1.9.1)(@types/node@24.12.0)(@vitest/browser-playwright@4.1.1)(jsdom@27.4.0(bufferutil@4.1.0)(canvas@3.2.2)(utf-8-validate@5.0.10))(vite@6.4.1(@types/node@24.12.0)(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3)) + ws: 8.20.0(bufferutil@4.1.0)(utf-8-validate@5.0.10) + transitivePeerDependencies: + - bufferutil + - msw + - utf-8-validate + - vite + optional: true + '@vitest/browser@4.1.1(bufferutil@4.1.0)(utf-8-validate@5.0.10)(vite@8.0.2(@types/node@24.12.0)(esbuild@0.27.2)(jiti@2.6.1)(less@4.6.4)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3))(vitest@4.1.1)': dependencies: '@blazediff/core': 1.9.1 @@ -24010,6 +24642,15 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.0.3 + '@vitest/mocker@4.1.1(vite@6.4.1(@types/node@24.12.0)(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3))': + dependencies: + '@vitest/spy': 4.1.1 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 6.4.1(@types/node@24.12.0)(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3) + optional: true + '@vitest/mocker@4.1.1(vite@8.0.2(@types/node@24.12.0)(esbuild@0.27.2)(jiti@2.6.1)(less@4.6.4)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3))': dependencies: '@vitest/spy': 4.1.1 @@ -25110,7 +25751,7 @@ snapshots: better-auth@1.4.21(@prisma/client@5.22.0)(better-sqlite3@12.5.0)(drizzle-kit@0.31.10)(drizzle-orm@0.41.0(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.20.0)(better-sqlite3@12.5.0)(kysely@0.28.14)(pg@8.20.0)(postgres@3.4.8))(pg@8.20.0)(react@19.2.3)(vitest@4.1.1)(vue@3.5.30(typescript@5.9.3)): dependencies: '@better-auth/core': 1.4.21(@better-auth/utils@0.3.0)(@better-fetch/fetch@1.1.21)(better-call@1.1.8(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1) - '@better-auth/telemetry': 1.4.21(@better-auth/core@1.4.21(@better-auth/utils@0.3.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.2(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1)) + '@better-auth/telemetry': 1.4.21(@better-auth/core@1.4.21(@better-auth/utils@0.3.0)(@better-fetch/fetch@1.1.21)(better-call@1.1.8(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1)) '@better-auth/utils': 0.3.0 '@better-fetch/fetch': 1.1.21 '@noble/ciphers': 2.1.1 @@ -25157,7 +25798,7 @@ snapshots: drizzle-orm: 0.45.1(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.20.0)(better-sqlite3@12.5.0)(kysely@0.28.14)(pg@8.20.0)(postgres@3.4.8) pg: 8.20.0 react: 19.2.3 - vitest: 4.1.1(@opentelemetry/api@1.9.1)(@types/node@24.12.0)(@vitest/browser-playwright@4.1.1)(jsdom@27.4.0(bufferutil@4.1.0)(canvas@3.2.2)(utf-8-validate@5.0.10))(vite@8.0.2(@types/node@24.12.0)(esbuild@0.27.2)(jiti@2.6.1)(less@4.6.4)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3)) + vitest: 4.1.1(@opentelemetry/api@1.9.1)(@types/node@24.12.0)(@vitest/browser-playwright@4.1.1)(jsdom@27.4.0(bufferutil@4.1.0)(canvas@3.2.2)(utf-8-validate@5.0.10))(vite@6.4.1(@types/node@24.12.0)(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3)) vue: 3.5.30(typescript@5.9.3) transitivePeerDependencies: - '@cloudflare/workers-types' @@ -25414,6 +26055,13 @@ snapshots: es-errors: 1.3.0 function-bind: 1.1.2 + camelcase-keys@9.1.3: + dependencies: + camelcase: 8.0.0 + map-obj: 5.0.0 + quick-lru: 6.1.2 + type-fest: 4.41.0 + camelcase@4.1.0: {} camelcase@8.0.0: {} @@ -25500,6 +26148,8 @@ snapshots: citty@0.2.1: {} + citty@0.2.2: {} + cjs-module-lexer@1.4.3: {} cjs-module-lexer@2.2.0: {} @@ -25704,6 +26354,8 @@ snapshots: cookie-es@1.2.2: {} + cookie-es@1.2.3: {} + cookie-signature@1.0.7: {} cookie-signature@1.2.2: {} @@ -25965,6 +26617,8 @@ snapshots: date-fns@4.1.0: {} + dateformat@4.6.3: {} + de-indent@1.0.2: {} debounce@1.2.1: {} @@ -26030,6 +26684,8 @@ snapshots: defu@6.1.4: {} + defu@6.1.6: {} + delaunator@5.0.1: dependencies: robust-predicates: 3.0.2 @@ -27091,6 +27747,8 @@ snapshots: extsprintf@1.4.1: {} + fast-copy@4.0.2: {} + fast-deep-equal@3.1.3: {} fast-glob@3.3.3: @@ -27107,6 +27765,8 @@ snapshots: fast-redact@3.5.0: {} + fast-safe-stringify@2.1.1: {} + fast-uri@3.1.0: {} fastq@1.20.1: @@ -27651,6 +28311,18 @@ snapshots: dependencies: duplexer: 0.1.2 + h3@1.15.11: + dependencies: + cookie-es: 1.2.3 + crossws: 0.3.5 + defu: 6.1.6 + destr: 2.0.5 + iron-webcrypto: 1.2.1 + node-mock-http: 1.0.4 + radix3: 1.1.2 + ufo: 1.6.3 + uncrypto: 0.1.3 + h3@1.15.5: dependencies: cookie-es: 1.2.2 @@ -27841,6 +28513,8 @@ snapshots: he@1.2.0: {} + help-me@5.0.0: {} + hey-listen@1.0.8: {} hfup@1.0.3: @@ -27966,6 +28640,8 @@ snapshots: transitivePeerDependencies: - supports-color + http-shutdown@1.2.2: {} + http-signature@1.2.0: dependencies: assert-plus: 1.0.0 @@ -28298,8 +28974,12 @@ snapshots: jiti@2.6.1: {} + jose@5.10.0: {} + jose@6.2.2: {} + joycon@3.1.1: {} + jpeg-js@0.4.4: {} js-tokens@10.0.0: {} @@ -28622,6 +29302,29 @@ snapshots: dependencies: uc.micro: 2.1.0 + listhen@1.9.1(srvx@0.11.13(patch_hash=c761d25a70e22a0925c88fe91a9dd1c3153dd341d8fd4f260166678156c4df2d)): + dependencies: + '@parcel/watcher': 2.5.6 + '@parcel/watcher-wasm': 2.5.6 + citty: 0.2.2 + consola: 3.4.2 + crossws: 0.4.4(patch_hash=4d79ec736d10d2a81a9e2a31b067d43f0b6665122267981e652ab9923d165958)(srvx@0.11.13(patch_hash=c761d25a70e22a0925c88fe91a9dd1c3153dd341d8fd4f260166678156c4df2d)) + defu: 6.1.6 + get-port-please: 3.2.0 + h3: 1.15.11 + http-shutdown: 1.2.2 + jiti: 2.6.1 + mlly: 1.8.2 + node-forge: 1.4.0 + pathe: 2.0.3 + std-env: 4.0.0 + tinyclip: 0.1.12 + ufo: 1.6.3 + untun: 0.1.3 + uqr: 0.1.3 + transitivePeerDependencies: + - srvx + listr2@8.3.3: dependencies: cli-truncate: 4.0.0 @@ -28631,6 +29334,13 @@ snapshots: rfdc: 1.4.1 wrap-ansi: 9.0.2 + livekit-server-sdk@2.15.0: + dependencies: + '@bufbuild/protobuf': 1.10.1 + '@livekit/protocol': 1.45.1 + camelcase-keys: 9.1.3 + jose: 5.10.0 + local-pkg@1.1.2: dependencies: mlly: 1.8.0 @@ -28807,6 +29517,8 @@ snapshots: many-keys-map@2.0.1: {} + map-obj@5.0.0: {} + mark.js@8.11.1: {} markdown-it-anchor@9.2.0(@types/markdown-it@14.1.2)(markdown-it@14.1.1): @@ -28943,6 +29655,11 @@ snapshots: transitivePeerDependencies: - supports-color + mdast-util-newline-to-break@2.0.0: + dependencies: + '@types/mdast': 4.0.4 + mdast-util-find-and-replace: 3.0.2 + mdast-util-phrasing@4.1.0: dependencies: '@types/mdast': 4.0.4 @@ -29457,6 +30174,13 @@ snapshots: pkg-types: 1.3.1 ufo: 1.6.3 + mlly@1.8.2: + dependencies: + acorn: 8.16.0 + pathe: 2.0.3 + pkg-types: 1.3.1 + ufo: 1.6.3 + module-details-from-path@1.0.4: {} module-replacements@2.10.1: {} @@ -29611,6 +30335,8 @@ snapshots: node-forge@1.3.3: {} + node-forge@1.4.0: {} + node-gyp-build@4.8.4: {} node-gyp@11.5.0: @@ -30136,6 +30862,26 @@ snapshots: dependencies: split2: 4.2.0 + pino-abstract-transport@3.0.0: + dependencies: + split2: 4.2.0 + + pino-pretty@13.1.3: + dependencies: + colorette: 2.0.20 + dateformat: 4.6.3 + fast-copy: 4.0.2 + fast-safe-stringify: 2.1.1 + help-me: 5.0.0 + joycon: 3.1.1 + minimist: 1.2.8 + on-exit-leak-free: 2.1.2 + pino-abstract-transport: 3.0.0 + pump: 3.0.3 + secure-json-parse: 4.1.0 + sonic-boom: 4.2.0 + strip-json-comments: 5.0.3 + pino-std-serializers@7.0.0: {} pino@9.7.0: @@ -30602,6 +31348,8 @@ snapshots: quick-lru@5.1.1: {} + quick-lru@6.1.2: {} + radashi@12.7.1: {} radix3@1.1.2: {} @@ -30816,6 +31564,12 @@ snapshots: transitivePeerDependencies: - '@vue/composition-api' + remark-breaks@4.0.0: + dependencies: + '@types/mdast': 4.0.4 + mdast-util-newline-to-break: 2.0.0 + unified: 11.0.5 + remark-math@6.0.0: dependencies: '@types/mdast': 4.0.4 @@ -31141,6 +31895,8 @@ snapshots: extend-shallow: 2.0.1 kind-of: 6.0.3 + secure-json-parse@4.1.0: {} + semver-compare@1.0.0: {} semver@5.7.2: {} @@ -31850,6 +32606,8 @@ snapshots: tinybench@2.9.0: {} + tinyclip@0.1.12: {} + tinycolor2@1.6.0: {} tinyexec@1.0.4: {} @@ -32456,6 +33214,12 @@ snapshots: untildify@4.0.0: {} + untun@0.1.3: + dependencies: + citty: 0.1.6 + consola: 3.4.2 + pathe: 1.1.2 + untyped@2.0.0: dependencies: citty: 0.1.6 @@ -32486,6 +33250,8 @@ snapshots: semver: 7.7.4 xdg-basedir: 5.1.0 + uqr@0.1.3: {} + uri-js@4.4.1: dependencies: punycode: 2.3.1 @@ -32810,6 +33576,37 @@ snapshots: - sortablejs - universal-cookie + vitest@4.1.1(@opentelemetry/api@1.9.1)(@types/node@24.12.0)(@vitest/browser-playwright@4.1.1)(jsdom@27.4.0(bufferutil@4.1.0)(canvas@3.2.2)(utf-8-validate@5.0.10))(vite@6.4.1(@types/node@24.12.0)(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3)): + dependencies: + '@vitest/expect': 4.1.1 + '@vitest/mocker': 4.1.1(vite@6.4.1(@types/node@24.12.0)(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3)) + '@vitest/pretty-format': 4.1.1 + '@vitest/runner': 4.1.1 + '@vitest/snapshot': 4.1.1 + '@vitest/spy': 4.1.1 + '@vitest/utils': 4.1.1 + es-module-lexer: 2.0.0 + expect-type: 1.3.0 + magic-string: 0.30.21 + obug: 2.1.1 + pathe: 2.0.3 + picomatch: 4.0.3 + std-env: 4.0.0 + tinybench: 2.9.0 + tinyexec: 1.0.4 + tinyglobby: 0.2.15 + tinyrainbow: 3.0.3 + vite: 6.4.1(@types/node@24.12.0)(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3) + why-is-node-running: 2.3.0 + optionalDependencies: + '@opentelemetry/api': 1.9.1 + '@types/node': 24.12.0 + '@vitest/browser-playwright': 4.1.1(bufferutil@4.1.0)(playwright@1.59.0)(utf-8-validate@5.0.10)(vite@6.4.1(@types/node@24.12.0)(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3))(vitest@4.1.1) + jsdom: 27.4.0(bufferutil@4.1.0)(canvas@3.2.2)(utf-8-validate@5.0.10) + transitivePeerDependencies: + - msw + optional: true + vitest@4.1.1(@opentelemetry/api@1.9.1)(@types/node@24.12.0)(@vitest/browser-playwright@4.1.1)(jsdom@27.4.0(bufferutil@4.1.0)(canvas@3.2.2)(utf-8-validate@5.0.10))(vite@8.0.2(@types/node@24.12.0)(esbuild@0.27.2)(jiti@2.6.1)(less@4.6.4)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3)): dependencies: '@vitest/expect': 4.1.1 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index e15cb2f378..6511fb5aed 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -110,6 +110,7 @@ catalog: oxc-minify: ^0.121.0 pinia: ^3.0.4 posthog-js: 1.306.1 + remark-breaks: ^4.0.0 splitpanes: ^4.0.4 std-env: ^4.0.0 superjson: ^2.2.6 From 55cc0f13c74f8df78461aaa6659014ea215d731f Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Tue, 7 Apr 2026 08:56:02 +0000 Subject: [PATCH 6/6] [autofix.ci] apply automated fixes --- pnpm-lock.yaml | 217 ++++++++++--------------------------------------- 1 file changed, 45 insertions(+), 172 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 73e7c74699..8834015357 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -547,10 +547,10 @@ importers: dependencies: '@better-auth/drizzle-adapter': specifier: ^1.5.6 - version: 1.5.6(@better-auth/core@1.5.6(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.1)(better-call@1.1.8(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1))(@better-auth/utils@0.3.1)(drizzle-orm@0.45.1(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.20.0)(better-sqlite3@12.5.0)(kysely@0.28.14)(pg@8.20.0)(postgres@3.4.8)) + version: 1.5.6(@better-auth/core@1.5.6(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.1)(better-call@1.3.2(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1))(@better-auth/utils@0.3.1)(drizzle-orm@0.45.1(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.20.0)(better-sqlite3@12.5.0)(kysely@0.28.14)(pg@8.20.0)(postgres@3.4.8)) '@better-auth/oauth-provider': specifier: 'catalog:' - version: 1.5.6(@better-auth/core@1.5.6(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.1)(better-call@1.1.8(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1))(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(better-auth@1.5.6(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(better-sqlite3@12.5.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.1(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.20.0)(better-sqlite3@12.5.0)(kysely@0.28.14)(pg@8.20.0)(postgres@3.4.8))(pg@8.20.0)(react@19.2.3)(vitest@4.1.1)(vue@3.5.30(typescript@5.9.3)))(better-call@1.1.8(zod@4.3.6)) + version: 1.5.6(@better-auth/core@1.5.6(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.1)(better-call@1.3.2(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1))(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(better-auth@1.5.6(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(better-sqlite3@12.5.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.1(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.20.0)(better-sqlite3@12.5.0)(kysely@0.28.14)(pg@8.20.0)(postgres@3.4.8))(pg@8.20.0)(react@19.2.3)(vitest@4.1.1)(vue@3.5.30(typescript@5.9.3)))(better-call@1.3.2(zod@4.3.6)) '@dotenvx/dotenvx': specifier: ^1.57.2 version: 1.57.2 @@ -671,7 +671,7 @@ importers: devDependencies: '@better-auth/cli': specifier: ^1.4.21 - version: 1.4.21(@better-fetch/fetch@1.1.21)(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(better-call@1.1.8(zod@4.3.6))(drizzle-kit@0.31.10)(jose@6.2.2)(kysely@0.28.14)(magicast@0.5.2)(nanostores@1.1.1)(postgres@3.4.8)(react@19.2.3)(vitest@4.1.1)(vue@3.5.30(typescript@5.9.3)) + version: 1.4.21(@better-fetch/fetch@1.1.21)(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(better-call@1.3.2(zod@4.3.6))(drizzle-kit@0.31.10)(jose@6.2.2)(kysely@0.28.14)(magicast@0.5.2)(nanostores@1.1.1)(postgres@3.4.8)(react@19.2.3)(vitest@4.1.1)(vue@3.5.30(typescript@5.9.3)) '@types/pg': specifier: ^8.20.0 version: 8.20.0 @@ -1248,7 +1248,7 @@ importers: version: 4.1.0 defu: specifier: ^6.1.4 - version: 6.1.4 + version: 6.1.6 destr: specifier: ^2.0.5 version: 2.0.5 @@ -11509,9 +11509,6 @@ packages: citty@0.1.6: resolution: {integrity: sha512-tskPPKEs8D2KPafUypv2gxwJP8h/OaJmC82QQGGDQcHvXX43xF2VDACcJVmZ0EuSxkpO9Kc4MlrA3q0+FG58AQ==} - citty@0.2.1: - resolution: {integrity: sha512-kEV95lFBhQgtogAPlQfJJ0WGVSokvLr/UEoFPiKKOXF7pl98HfUVUD0ejsuTCld/9xH9vogSywZ5KqHzXrZpqg==} - citty@0.2.2: resolution: {integrity: sha512-+6vJA3L98yv+IdfKGZHBNiGW5KHn22e/JwID0Strsz8h4S/csAu/OuICwxrg44k5MRiZHWIo8XXuJgQTriRP4w==} @@ -11720,9 +11717,6 @@ packages: convert-source-map@2.0.0: resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} - cookie-es@1.2.2: - resolution: {integrity: sha512-+W7VmiVINB+ywl1HGXJXmrqkOhpKrIiVZV6tQuV54ZyQC7MMuBt81Vc336GMLoHBq5hV/F9eXgt5Mnx0Rha5Fg==} - cookie-es@1.2.3: resolution: {integrity: sha512-lXVyvUvrNXblMqzIRrxHb57UUVmqsSWlxqt3XIjCkUP0wDAf6uicO6KMbEgYrMNtEvWgWHwe42CKxPu9MYAnWw==} @@ -12062,9 +12056,6 @@ packages: resolution: {integrity: sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==} engines: {node: '>= 0.4'} - defu@6.1.4: - resolution: {integrity: sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg==} - defu@6.1.6: resolution: {integrity: sha512-f8mefEW4WIVg4LckePx3mALjQSPQgFlg9U8yaPdlsbdYcHQyj9n2zL2LJEA52smeYxOvmd/nB7TpMtHGMTHcug==} @@ -13509,9 +13500,6 @@ packages: h3@1.15.11: resolution: {integrity: sha512-L3THSe2MPeBwgIZVSH5zLdBBU90TOxarvhK9d04IDY2AmVS8j2Jz2LIWtwsGOU3lu2I5jCN7FNvVfY2+XyF+mg==} - h3@1.15.5: - resolution: {integrity: sha512-xEyq3rSl+dhGX2Lm0+eFQIAzlDN6Fs0EcC4f7BNUmzaRX/PTzeuM+Tr2lHB8FoXggsQIeXLj8EDVgs5ywxyxmg==} - h3@2.0.1-rc.19: resolution: {integrity: sha512-47er/mh8eGA7+0nvNUloalj+yTJ1ku8M0BVzA2I1ZHSlpfbUNdBK4LpWztfH7TwW6kuhF8MfAvl0AwB+X9B+2w==} engines: {node: '>=20.11.1'} @@ -14944,9 +14932,6 @@ packages: resolution: {integrity: sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==} hasBin: true - mlly@1.8.0: - resolution: {integrity: sha512-l8D9ODSRWLe2KHJSifWGwBqpTZXIXTeo8mlKjY+E2HAakaTeNpqAyBZ8GSqLzHgw4XmHmC8whvpjJNMbFZN7/g==} - mlly@1.8.2: resolution: {integrity: sha512-d+ObxMQFmbt10sretNDytwt85VrbkhhUA/JBGm1MPaWJ65Cl4wOgLaB1NYvJSZ0Ef03MMEU/0xpPMXUIQ29UfA==} @@ -15113,10 +15098,6 @@ packages: resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - node-forge@1.3.3: - resolution: {integrity: sha512-rLvcdSyRCyouf6jcOIPe/BgwG/d7hKjzMKOas33/pHEr6gbq18IK9zV7DiPvzsz0oBJPme6qr6H6kGZuI9/DZg==} - engines: {node: '>= 6.13.0'} - node-forge@1.4.0: resolution: {integrity: sha512-LarFH0+6VfriEhqMMcLX2F7SwSXeWwnEAJEsYm5QKWchiVYVvJyV9v7UDvUv+w5HO23ZpQTXDv/GxdDdMyOuoQ==} engines: {node: '>= 6.13.0'} @@ -18357,7 +18338,7 @@ snapshots: '@1natsu/wait-element@4.1.2': dependencies: - defu: 6.1.4 + defu: 6.1.6 many-keys-map: 2.0.1 '@acemir/cssom@0.9.31': {} @@ -19269,13 +19250,13 @@ snapshots: '@bcoe/v8-coverage@1.0.2': {} - '@better-auth/cli@1.4.21(@better-fetch/fetch@1.1.21)(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(better-call@1.1.8(zod@4.3.6))(drizzle-kit@0.31.10)(jose@6.2.2)(kysely@0.28.14)(magicast@0.5.2)(nanostores@1.1.1)(postgres@3.4.8)(react@19.2.3)(vitest@4.1.1)(vue@3.5.30(typescript@5.9.3))': + '@better-auth/cli@1.4.21(@better-fetch/fetch@1.1.21)(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(better-call@1.3.2(zod@4.3.6))(drizzle-kit@0.31.10)(jose@6.2.2)(kysely@0.28.14)(magicast@0.5.2)(nanostores@1.1.1)(postgres@3.4.8)(react@19.2.3)(vitest@4.1.1)(vue@3.5.30(typescript@5.9.3))': dependencies: '@babel/core': 7.29.0 '@babel/preset-react': 7.28.5(@babel/core@7.29.0) '@babel/preset-typescript': 7.28.5(@babel/core@7.29.0) - '@better-auth/core': 1.4.21(@better-auth/utils@0.3.0)(@better-fetch/fetch@1.1.21)(better-call@1.1.8(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1) - '@better-auth/telemetry': 1.4.21(@better-auth/core@1.4.21(@better-auth/utils@0.3.0)(@better-fetch/fetch@1.1.21)(better-call@1.1.8(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1)) + '@better-auth/core': 1.4.21(@better-auth/utils@0.3.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.2(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1) + '@better-auth/telemetry': 1.4.21(@better-auth/core@1.4.21(@better-auth/utils@0.3.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.2(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1)) '@better-auth/utils': 0.3.0 '@clack/prompts': 0.11.0 '@mrleebo/prisma-ast': 0.13.1 @@ -19352,14 +19333,12 @@ snapshots: nanostores: 1.1.1 zod: 4.3.6 - '@better-auth/core@1.5.6(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.1)(better-call@1.1.8(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1)': + '@better-auth/core@1.4.21(@better-auth/utils@0.3.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.2(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1)': dependencies: - '@better-auth/utils': 0.3.1 + '@better-auth/utils': 0.3.0 '@better-fetch/fetch': 1.1.21 - '@opentelemetry/api': 1.9.1 - '@opentelemetry/semantic-conventions': 1.40.0 '@standard-schema/spec': 1.1.0 - better-call: 1.1.8(zod@4.3.6) + better-call: 1.3.2(zod@4.3.6) jose: 6.2.2 kysely: 0.28.14 nanostores: 1.1.1 @@ -19378,13 +19357,6 @@ snapshots: nanostores: 1.1.1 zod: 4.3.6 - '@better-auth/drizzle-adapter@1.5.6(@better-auth/core@1.5.6(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.1)(better-call@1.1.8(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1))(@better-auth/utils@0.3.1)(drizzle-orm@0.45.1(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.20.0)(better-sqlite3@12.5.0)(kysely@0.28.14)(pg@8.20.0)(postgres@3.4.8))': - dependencies: - '@better-auth/core': 1.5.6(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.1)(better-call@1.1.8(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1) - '@better-auth/utils': 0.3.1 - optionalDependencies: - drizzle-orm: 0.45.1(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.20.0)(better-sqlite3@12.5.0)(kysely@0.28.14)(pg@8.20.0)(postgres@3.4.8) - '@better-auth/drizzle-adapter@1.5.6(@better-auth/core@1.5.6(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.1)(better-call@1.3.2(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1))(@better-auth/utils@0.3.1)(drizzle-orm@0.45.1(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.20.0)(better-sqlite3@12.5.0)(kysely@0.28.14)(pg@8.20.0)(postgres@3.4.8))': dependencies: '@better-auth/core': 1.5.6(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.1)(better-call@1.3.2(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1) @@ -19409,13 +19381,13 @@ snapshots: '@better-auth/core': 1.5.6(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.1)(better-call@1.3.2(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1) '@better-auth/utils': 0.3.1 - '@better-auth/oauth-provider@1.5.6(@better-auth/core@1.5.6(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.1)(better-call@1.1.8(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1))(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(better-auth@1.5.6(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(better-sqlite3@12.5.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.1(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.20.0)(better-sqlite3@12.5.0)(kysely@0.28.14)(pg@8.20.0)(postgres@3.4.8))(pg@8.20.0)(react@19.2.3)(vitest@4.1.1)(vue@3.5.30(typescript@5.9.3)))(better-call@1.1.8(zod@4.3.6))': + '@better-auth/oauth-provider@1.5.6(@better-auth/core@1.5.6(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.1)(better-call@1.3.2(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1))(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(better-auth@1.5.6(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(better-sqlite3@12.5.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.1(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.20.0)(better-sqlite3@12.5.0)(kysely@0.28.14)(pg@8.20.0)(postgres@3.4.8))(pg@8.20.0)(react@19.2.3)(vitest@4.1.1)(vue@3.5.30(typescript@5.9.3)))(better-call@1.3.2(zod@4.3.6))': dependencies: - '@better-auth/core': 1.5.6(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.1)(better-call@1.1.8(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1) + '@better-auth/core': 1.5.6(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@opentelemetry/api@1.9.1)(better-call@1.3.2(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1) '@better-auth/utils': 0.3.1 '@better-fetch/fetch': 1.1.21 better-auth: 1.5.6(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(better-sqlite3@12.5.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.1(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.20.0)(better-sqlite3@12.5.0)(kysely@0.28.14)(pg@8.20.0)(postgres@3.4.8))(pg@8.20.0)(react@19.2.3)(vitest@4.1.1)(vue@3.5.30(typescript@5.9.3)) - better-call: 1.1.8(zod@4.3.6) + better-call: 1.3.2(zod@4.3.6) jose: 6.2.2 zod: 4.3.6 @@ -19426,9 +19398,9 @@ snapshots: optionalDependencies: '@prisma/client': 5.22.0 - '@better-auth/telemetry@1.4.21(@better-auth/core@1.4.21(@better-auth/utils@0.3.0)(@better-fetch/fetch@1.1.21)(better-call@1.1.8(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1))': + '@better-auth/telemetry@1.4.21(@better-auth/core@1.4.21(@better-auth/utils@0.3.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.2(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1))': dependencies: - '@better-auth/core': 1.4.21(@better-auth/utils@0.3.0)(@better-fetch/fetch@1.1.21)(better-call@1.1.8(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1) + '@better-auth/core': 1.4.21(@better-auth/utils@0.3.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.2(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1) '@better-auth/utils': 0.3.0 '@better-fetch/fetch': 1.1.21 @@ -19619,7 +19591,7 @@ snapshots: bluebird: 3.7.2 commander: 9.5.0 debug: 4.3.7 - node-forge: 1.3.3 + node-forge: 1.4.0 split: 1.0.1 transitivePeerDependencies: - supports-color @@ -20460,7 +20432,7 @@ snapshots: dependencies: '@antfu/install-pkg': 1.1.0 '@iconify/types': 2.0.0 - mlly: 1.8.0 + mlly: 1.8.2 '@iconify/vue@5.0.0(vue@3.5.30(typescript@5.9.3))': dependencies: @@ -21186,7 +21158,7 @@ snapshots: dependencies: c12: 3.3.3(magicast@0.5.2) consola: 3.4.2 - defu: 6.1.4 + defu: 6.1.6 destr: 2.0.5 errx: 0.1.0 exsolve: 1.0.8 @@ -21194,7 +21166,7 @@ snapshots: jiti: 2.6.1 klona: 2.0.6 knitwork: 1.3.0 - mlly: 1.8.0 + mlly: 1.8.2 ohash: 2.0.11 pathe: 2.0.3 pkg-types: 2.3.0 @@ -22958,7 +22930,7 @@ snapshots: '@moeru/std': 0.1.0-beta.1 apache-arrow: 21.1.0 date-fns: 4.1.0 - defu: 6.1.4 + defu: 6.1.6 drizzle-orm: 0.45.1(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.20.0)(better-sqlite3@12.5.0)(kysely@0.28.14)(pg@8.20.0)(postgres@3.4.8) es-toolkit: 1.45.1 transitivePeerDependencies: @@ -24543,20 +24515,6 @@ snapshots: vite: 8.0.2(@types/node@24.12.0)(esbuild@0.27.2)(jiti@2.6.1)(less@4.6.4)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3) vue: 3.5.30(typescript@5.9.3) - '@vitest/browser-playwright@4.1.1(bufferutil@4.1.0)(playwright@1.59.0)(utf-8-validate@5.0.10)(vite@6.4.1(@types/node@24.12.0)(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3))(vitest@4.1.1)': - dependencies: - '@vitest/browser': 4.1.1(bufferutil@4.1.0)(utf-8-validate@5.0.10)(vite@6.4.1(@types/node@24.12.0)(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3))(vitest@4.1.1) - '@vitest/mocker': 4.1.1(vite@6.4.1(@types/node@24.12.0)(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3)) - playwright: 1.59.0 - tinyrainbow: 3.0.3 - vitest: 4.1.1(@opentelemetry/api@1.9.1)(@types/node@24.12.0)(@vitest/browser-playwright@4.1.1)(jsdom@27.4.0(bufferutil@4.1.0)(canvas@3.2.2)(utf-8-validate@5.0.10))(vite@6.4.1(@types/node@24.12.0)(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3)) - transitivePeerDependencies: - - bufferutil - - msw - - utf-8-validate - - vite - optional: true - '@vitest/browser-playwright@4.1.1(bufferutil@4.1.0)(playwright@1.59.0)(utf-8-validate@5.0.10)(vite@8.0.2(@types/node@24.12.0)(esbuild@0.27.2)(jiti@2.6.1)(less@4.6.4)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3))(vitest@4.1.1)': dependencies: '@vitest/browser': 4.1.1(bufferutil@4.1.0)(utf-8-validate@5.0.10)(vite@8.0.2(@types/node@24.12.0)(esbuild@0.27.2)(jiti@2.6.1)(less@4.6.4)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3))(vitest@4.1.1) @@ -24570,24 +24528,6 @@ snapshots: - utf-8-validate - vite - '@vitest/browser@4.1.1(bufferutil@4.1.0)(utf-8-validate@5.0.10)(vite@6.4.1(@types/node@24.12.0)(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3))(vitest@4.1.1)': - dependencies: - '@blazediff/core': 1.9.1 - '@vitest/mocker': 4.1.1(vite@6.4.1(@types/node@24.12.0)(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3)) - '@vitest/utils': 4.1.1 - magic-string: 0.30.21 - pngjs: 7.0.0 - sirv: 3.0.2 - tinyrainbow: 3.0.3 - vitest: 4.1.1(@opentelemetry/api@1.9.1)(@types/node@24.12.0)(@vitest/browser-playwright@4.1.1)(jsdom@27.4.0(bufferutil@4.1.0)(canvas@3.2.2)(utf-8-validate@5.0.10))(vite@6.4.1(@types/node@24.12.0)(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3)) - ws: 8.20.0(bufferutil@4.1.0)(utf-8-validate@5.0.10) - transitivePeerDependencies: - - bufferutil - - msw - - utf-8-validate - - vite - optional: true - '@vitest/browser@4.1.1(bufferutil@4.1.0)(utf-8-validate@5.0.10)(vite@8.0.2(@types/node@24.12.0)(esbuild@0.27.2)(jiti@2.6.1)(less@4.6.4)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3))(vitest@4.1.1)': dependencies: '@blazediff/core': 1.9.1 @@ -24642,15 +24582,6 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.0.3 - '@vitest/mocker@4.1.1(vite@6.4.1(@types/node@24.12.0)(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3))': - dependencies: - '@vitest/spy': 4.1.1 - estree-walker: 3.0.3 - magic-string: 0.30.21 - optionalDependencies: - vite: 6.4.1(@types/node@24.12.0)(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3) - optional: true - '@vitest/mocker@4.1.1(vite@8.0.2(@types/node@24.12.0)(esbuild@0.27.2)(jiti@2.6.1)(less@4.6.4)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3))': dependencies: '@vitest/spy': 4.1.1 @@ -25222,7 +25153,7 @@ snapshots: dependencies: '@vueuse/core': 13.9.0(vue@3.5.30(typescript@5.9.3)) '@vueuse/shared': 13.9.0(vue@3.5.30(typescript@5.9.3)) - defu: 6.1.4 + defu: 6.1.6 framesync: 6.1.2 popmotion: 11.0.5 style-value-types: 5.1.2 @@ -25751,13 +25682,13 @@ snapshots: better-auth@1.4.21(@prisma/client@5.22.0)(better-sqlite3@12.5.0)(drizzle-kit@0.31.10)(drizzle-orm@0.41.0(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.20.0)(better-sqlite3@12.5.0)(kysely@0.28.14)(pg@8.20.0)(postgres@3.4.8))(pg@8.20.0)(react@19.2.3)(vitest@4.1.1)(vue@3.5.30(typescript@5.9.3)): dependencies: '@better-auth/core': 1.4.21(@better-auth/utils@0.3.0)(@better-fetch/fetch@1.1.21)(better-call@1.1.8(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1) - '@better-auth/telemetry': 1.4.21(@better-auth/core@1.4.21(@better-auth/utils@0.3.0)(@better-fetch/fetch@1.1.21)(better-call@1.1.8(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1)) + '@better-auth/telemetry': 1.4.21(@better-auth/core@1.4.21(@better-auth/utils@0.3.0)(@better-fetch/fetch@1.1.21)(better-call@1.3.2(zod@4.3.6))(jose@6.2.2)(kysely@0.28.14)(nanostores@1.1.1)) '@better-auth/utils': 0.3.0 '@better-fetch/fetch': 1.1.21 '@noble/ciphers': 2.1.1 '@noble/hashes': 2.0.1 better-call: 1.1.8(zod@4.3.6) - defu: 6.1.4 + defu: 6.1.6 jose: 6.2.2 kysely: 0.28.14 nanostores: 1.1.1 @@ -25786,7 +25717,7 @@ snapshots: '@noble/ciphers': 2.1.1 '@noble/hashes': 2.0.1 better-call: 1.3.2(zod@4.3.6) - defu: 6.1.4 + defu: 6.1.6 jose: 6.2.2 kysely: 0.28.14 nanostores: 1.1.1 @@ -25798,7 +25729,7 @@ snapshots: drizzle-orm: 0.45.1(@electric-sql/pglite@0.4.1)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.20.0)(better-sqlite3@12.5.0)(kysely@0.28.14)(pg@8.20.0)(postgres@3.4.8) pg: 8.20.0 react: 19.2.3 - vitest: 4.1.1(@opentelemetry/api@1.9.1)(@types/node@24.12.0)(@vitest/browser-playwright@4.1.1)(jsdom@27.4.0(bufferutil@4.1.0)(canvas@3.2.2)(utf-8-validate@5.0.10))(vite@6.4.1(@types/node@24.12.0)(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3)) + vitest: 4.1.1(@opentelemetry/api@1.9.1)(@types/node@24.12.0)(@vitest/browser-playwright@4.1.1)(jsdom@27.4.0(bufferutil@4.1.0)(canvas@3.2.2)(utf-8-validate@5.0.10))(vite@8.0.2(@types/node@24.12.0)(esbuild@0.27.2)(jiti@2.6.1)(less@4.6.4)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3)) vue: 3.5.30(typescript@5.9.3) transitivePeerDependencies: - '@cloudflare/workers-types' @@ -26006,7 +25937,7 @@ snapshots: dependencies: chokidar: 5.0.0 confbox: 0.2.2 - defu: 6.1.4 + defu: 6.1.6 dotenv: 17.3.1 exsolve: 1.0.8 giget: 2.0.0 @@ -26146,8 +26077,6 @@ snapshots: dependencies: consola: 3.4.2 - citty@0.2.1: {} - citty@0.2.2: {} cjs-module-lexer@1.4.3: {} @@ -26352,8 +26281,6 @@ snapshots: convert-source-map@2.0.0: {} - cookie-es@1.2.2: {} - cookie-es@1.2.3: {} cookie-signature@1.0.7: {} @@ -26682,8 +26609,6 @@ snapshots: has-property-descriptors: 1.0.2 object-keys: 1.1.1 - defu@6.1.4: {} - defu@6.1.6: {} delaunator@5.0.1: @@ -28148,7 +28073,7 @@ snapshots: dependencies: citty: 0.1.6 consola: 3.4.2 - defu: 6.1.4 + defu: 6.1.6 node-fetch-native: 1.6.7 nypm: 0.6.5 pathe: 2.0.3 @@ -28323,18 +28248,6 @@ snapshots: ufo: 1.6.3 uncrypto: 0.1.3 - h3@1.15.5: - dependencies: - cookie-es: 1.2.2 - crossws: 0.3.5 - defu: 6.1.4 - destr: 2.0.5 - iron-webcrypto: 1.2.1 - node-mock-http: 1.0.4 - radix3: 1.1.2 - ufo: 1.6.3 - uncrypto: 0.1.3 - h3@2.0.1-rc.19(crossws@0.4.4(patch_hash=4d79ec736d10d2a81a9e2a31b067d43f0b6665122267981e652ab9923d165958)(srvx@0.11.13(patch_hash=c761d25a70e22a0925c88fe91a9dd1c3153dd341d8fd4f260166678156c4df2d))): dependencies: rou3: 0.8.1 @@ -28519,7 +28432,7 @@ snapshots: hfup@1.0.3: dependencies: - defu: 6.1.4 + defu: 6.1.6 gray-matter: 4.0.3 unplugin: 2.3.11 @@ -28535,7 +28448,7 @@ snapshots: change-case: 5.4.4 chokidar: 4.0.3 connect: 3.7.0 - defu: 6.1.4 + defu: 6.1.6 diacritics: 1.3.0 fs-extra: 11.3.4 globby: 14.1.0 @@ -29343,7 +29256,7 @@ snapshots: local-pkg@1.1.2: dependencies: - mlly: 1.8.0 + mlly: 1.8.2 pkg-types: 2.3.0 quansync: 0.2.11 @@ -29441,7 +29354,7 @@ snapshots: dependencies: estree-walker: 3.0.3 magic-string: 0.30.21 - mlly: 1.8.0 + mlly: 1.8.2 regexp-tree: 0.1.27 type-level-regexp: 0.1.17 ufo: 1.6.3 @@ -30159,7 +30072,7 @@ snapshots: mkcert@3.2.0: dependencies: commander: 11.1.0 - node-forge: 1.3.3 + node-forge: 1.4.0 mkdirp-classic@0.5.3: {} @@ -30167,13 +30080,6 @@ snapshots: dependencies: minimist: 1.2.8 - mlly@1.8.0: - dependencies: - acorn: 8.16.0 - pathe: 2.0.3 - pkg-types: 1.3.1 - ufo: 1.6.3 - mlly@1.8.2: dependencies: acorn: 8.16.0 @@ -30333,8 +30239,6 @@ snapshots: fetch-blob: 3.2.0 formdata-polyfill: 4.0.10 - node-forge@1.3.3: {} - node-forge@1.4.0: {} node-gyp-build@4.8.4: {} @@ -30406,7 +30310,7 @@ snapshots: nypm@0.6.5: dependencies: - citty: 0.2.1 + citty: 0.2.2 pathe: 2.0.3 tinyexec: 1.0.4 @@ -30968,7 +30872,7 @@ snapshots: pkg-types@1.3.1: dependencies: confbox: 0.1.8 - mlly: 1.8.0 + mlly: 1.8.2 pathe: 2.0.3 pkg-types@2.3.0: @@ -31383,7 +31287,7 @@ snapshots: rc9@2.1.2: dependencies: - defu: 6.1.4 + defu: 6.1.6 destr: 2.0.5 rc@1.2.8: @@ -31558,7 +31462,7 @@ snapshots: '@vueuse/core': 14.2.1(vue@3.5.30(typescript@5.9.3)) '@vueuse/shared': 14.2.1(vue@3.5.30(typescript@5.9.3)) aria-hidden: 1.2.6 - defu: 6.1.4 + defu: 6.1.6 ohash: 2.0.11 vue: 3.5.30(typescript@5.9.3) transitivePeerDependencies: @@ -32715,7 +32619,7 @@ snapshots: dependencies: ansis: 4.2.0 cac: 7.0.0 - defu: 6.1.4 + defu: 6.1.6 empathic: 2.0.0 hookable: 6.1.0 import-without-cache: 0.2.5 @@ -32847,7 +32751,7 @@ snapshots: unconfig@7.5.0: dependencies: '@quansync/fs': 1.0.0 - defu: 6.1.4 + defu: 6.1.6 jiti: 2.6.1 quansync: 1.0.0 unconfig-core: 7.5.0 @@ -32900,7 +32804,7 @@ snapshots: estree-walker: 3.0.3 local-pkg: 1.1.2 magic-string: 0.30.21 - mlly: 1.8.0 + mlly: 1.8.2 pathe: 2.0.3 picomatch: 4.0.3 pkg-types: 2.3.0 @@ -33125,7 +33029,7 @@ snapshots: json5: 2.2.3 local-pkg: 1.1.2 magic-string: 0.30.21 - mlly: 1.8.0 + mlly: 1.8.2 muggle-string: 0.4.1 pathe: 2.0.3 picomatch: 4.0.3 @@ -33202,7 +33106,7 @@ snapshots: anymatch: 3.1.3 chokidar: 5.0.0 destr: 2.0.5 - h3: 1.15.5 + h3: 1.15.11 lru-cache: 11.2.6 node-fetch-native: 1.6.7 ofetch: 1.5.1 @@ -33223,7 +33127,7 @@ snapshots: untyped@2.0.0: dependencies: citty: 0.1.6 - defu: 6.1.4 + defu: 6.1.6 jiti: 2.6.1 knitwork: 1.3.0 scule: 1.3.0 @@ -33576,37 +33480,6 @@ snapshots: - sortablejs - universal-cookie - vitest@4.1.1(@opentelemetry/api@1.9.1)(@types/node@24.12.0)(@vitest/browser-playwright@4.1.1)(jsdom@27.4.0(bufferutil@4.1.0)(canvas@3.2.2)(utf-8-validate@5.0.10))(vite@6.4.1(@types/node@24.12.0)(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3)): - dependencies: - '@vitest/expect': 4.1.1 - '@vitest/mocker': 4.1.1(vite@6.4.1(@types/node@24.12.0)(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3)) - '@vitest/pretty-format': 4.1.1 - '@vitest/runner': 4.1.1 - '@vitest/snapshot': 4.1.1 - '@vitest/spy': 4.1.1 - '@vitest/utils': 4.1.1 - es-module-lexer: 2.0.0 - expect-type: 1.3.0 - magic-string: 0.30.21 - obug: 2.1.1 - pathe: 2.0.3 - picomatch: 4.0.3 - std-env: 4.0.0 - tinybench: 2.9.0 - tinyexec: 1.0.4 - tinyglobby: 0.2.15 - tinyrainbow: 3.0.3 - vite: 6.4.1(@types/node@24.12.0)(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3) - why-is-node-running: 2.3.0 - optionalDependencies: - '@opentelemetry/api': 1.9.1 - '@types/node': 24.12.0 - '@vitest/browser-playwright': 4.1.1(bufferutil@4.1.0)(playwright@1.59.0)(utf-8-validate@5.0.10)(vite@6.4.1(@types/node@24.12.0)(jiti@2.6.1)(less@4.6.4)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3))(vitest@4.1.1) - jsdom: 27.4.0(bufferutil@4.1.0)(canvas@3.2.2)(utf-8-validate@5.0.10) - transitivePeerDependencies: - - msw - optional: true - vitest@4.1.1(@opentelemetry/api@1.9.1)(@types/node@24.12.0)(@vitest/browser-playwright@4.1.1)(jsdom@27.4.0(bufferutil@4.1.0)(canvas@3.2.2)(utf-8-validate@5.0.10))(vite@8.0.2(@types/node@24.12.0)(esbuild@0.27.2)(jiti@2.6.1)(less@4.6.4)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3)): dependencies: '@vitest/expect': 4.1.1 @@ -33774,7 +33647,7 @@ snapshots: json5: 2.2.3 local-pkg: 1.1.2 magic-string: 0.30.21 - mlly: 1.8.0 + mlly: 1.8.2 muggle-string: 0.4.1 pathe: 2.0.3 picomatch: 4.0.3 @@ -34122,7 +33995,7 @@ snapshots: chokidar: 5.0.0 ci-info: 4.4.0 consola: 3.4.2 - defu: 6.1.4 + defu: 6.1.6 dotenv: 17.3.1 dotenv-expand: 12.0.3 esbuild: 0.27.2