diff --git a/Cargo.toml b/Cargo.toml index 69b6f01ae..2ecee6a20 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,11 +24,11 @@ async-trait = "0.1" serenity = { version = "0.12", default-features = false, features = ["client", "gateway", "model", "rustls_backend", "cache"], optional = true } [features] -# Default: core only (Discord + Slack). Gateway ships as separate binary. +# Default: core only (Discord + Slack). Platform adapters are opt-in. default = ["discord", "slack", "secrets-aws", "agentcore", "config-s3", "pre-seed"] -# Opt-in: compile all gateway adapters into a single unified binary -unified = ["telegram", "line", "feishu", "googlechat", "wecom", "teams"] +# Opt-in: compile all platform adapters into a single unified binary +unified = ["telegram", "line", "feishu", "googlechat", "wecom", "teams", "vtuber"] # Core adapters discord = ["dep:serenity", "openab-core/discord"] @@ -40,13 +40,14 @@ agentcore = ["openab-core/agentcore"] config-s3 = ["openab-core/config-s3"] pre-seed = ["openab-core/pre-seed"] -# Gateway adapters (each pulls in the gateway crate + axum for embedded server) +# Platform adapters (each pulls in the gateway crate + axum for embedded server) telegram = ["dep:openab-gateway", "dep:axum", "openab-gateway/telegram"] line = ["dep:openab-gateway", "dep:axum", "openab-gateway/line"] feishu = ["dep:openab-gateway", "dep:axum", "openab-gateway/feishu"] googlechat = ["dep:openab-gateway", "dep:axum", "openab-gateway/googlechat"] wecom = ["dep:openab-gateway", "dep:axum", "openab-gateway/wecom"] teams = ["dep:openab-gateway", "dep:axum", "openab-gateway/teams"] +vtuber = ["dep:openab-gateway", "dep:axum", "openab-gateway/vtuber"] [dev-dependencies] tempfile = "3.27.0" diff --git a/crates/openab-gateway/Cargo.toml b/crates/openab-gateway/Cargo.toml index 26ee00db9..c9f499f9c 100644 --- a/crates/openab-gateway/Cargo.toml +++ b/crates/openab-gateway/Cargo.toml @@ -35,10 +35,11 @@ urlencoding = "2" wiremock = "0.6" [features] -default = ["telegram", "line", "feishu", "googlechat", "wecom", "teams"] +default = ["telegram", "line", "feishu", "googlechat", "wecom", "teams", "vtuber"] telegram = [] line = [] feishu = [] googlechat = [] wecom = [] teams = [] +vtuber = [] diff --git a/crates/openab-gateway/src/adapters/mod.rs b/crates/openab-gateway/src/adapters/mod.rs index f58f870a2..7c58c2135 100644 --- a/crates/openab-gateway/src/adapters/mod.rs +++ b/crates/openab-gateway/src/adapters/mod.rs @@ -12,3 +12,5 @@ pub mod googlechat; pub mod wecom; #[cfg(feature = "teams")] pub mod teams; +#[cfg(feature = "vtuber")] +pub mod vtuber; diff --git a/crates/openab-gateway/src/adapters/teams.rs b/crates/openab-gateway/src/adapters/teams.rs index c8dec8509..3eb9fd71b 100644 --- a/crates/openab-gateway/src/adapters/teams.rs +++ b/crates/openab-gateway/src/adapters/teams.rs @@ -275,7 +275,9 @@ impl TeamsAdapter { } // B2: Validate channel endorsements — key must endorse the activity's channelId - let channel_id = activity.channel_id.as_deref() + let channel_id = activity + .channel_id + .as_deref() .ok_or_else(|| anyhow::anyhow!("activity missing channelId"))?; if key.endorsements.is_empty() { anyhow::bail!("JWK has no endorsements — cannot verify channelId={channel_id}"); @@ -301,9 +303,13 @@ impl TeamsAdapter { let token_data = decode::(token, &decoding_key, &validation)?; // B1: Validate serviceUrl claim matches activity's serviceUrl - let activity_service_url = activity.service_url.as_deref() + let activity_service_url = activity + .service_url + .as_deref() .ok_or_else(|| anyhow::anyhow!("activity missing serviceUrl"))?; - let token_service_url = token_data.claims.get("serviceurl") + let token_service_url = token_data + .claims + .get("serviceurl") .and_then(|v| v.as_str()) .ok_or_else(|| anyhow::anyhow!("JWT missing serviceurl claim"))?; if token_service_url != activity_service_url { @@ -799,7 +805,9 @@ mod tests { async fn jwt_rejects_garbage_token() { let adapter = TeamsAdapter::new(make_config(vec![])); let activity = make_activity_with_tenant(Some("t1")); - let result = adapter.validate_jwt("Bearer not.a.valid.jwt", &activity).await; + let result = adapter + .validate_jwt("Bearer not.a.valid.jwt", &activity) + .await; assert!(result.is_err()); } diff --git a/crates/openab-gateway/src/adapters/vtuber.rs b/crates/openab-gateway/src/adapters/vtuber.rs new file mode 100644 index 000000000..05fe0b8f4 --- /dev/null +++ b/crates/openab-gateway/src/adapters/vtuber.rs @@ -0,0 +1,739 @@ +//! VTuber platform adapter. +//! +//! OpenAI-compatible POST /v1/chat/completions (SSE) streams agent replies as +//! `chat.completion.chunk` deltas. + +use crate::schema::*; +use axum::extract::State; +use axum::http::{header::AUTHORIZATION, HeaderMap, StatusCode}; +use axum::response::sse::{Event as SseEvent, KeepAlive, Sse}; +use axum::response::IntoResponse; +use axum::Json; +use futures_util::stream::Stream; +use serde::Deserialize; +use serde_json::json; +use std::collections::HashMap; +use std::convert::Infallible; +use std::sync::Arc; +use std::time::Duration; +use subtle::ConstantTimeEq; +use tokio::sync::{mpsc, Mutex, OwnedMutexGuard}; +use tracing::{info, warn}; +use uuid::Uuid; + +/// Fixed channel ID for VTuber session reuse. +/// All /v1/chat/completions requests share this channel so the OAB +/// session pool (`[pool]`) reuses the same warm agent process. +const VTUBER_PERSISTENT_CHANNEL: &str = "vtb_persistent"; + +// --------------------------------------------------------------------------- +// Tool-status suppression +// --------------------------------------------------------------------------- + +fn parse_pure_tool_status(text: &str) -> Option<(String, &'static str)> { + let normalized = text + .trim() + .trim_matches('`') + .trim() + .trim_start_matches('✅') + .trim_start_matches('✔') + .trim_start_matches('✓') + .trim() + .trim_matches('`') + .trim(); + + if normalized.is_empty() || normalized.len() > 64 { + return None; + } + + let mut chars = normalized.chars(); + let first = chars.next()?; + if !first.is_ascii_alphabetic() { + return None; + } + if !chars.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-' || c == ' ') { + return None; + } + + let lower = normalized.to_ascii_lowercase(); + if lower == "toolsearch" + || lower == "tool search" + || lower.ends_with("search") + || lower.ends_with("tool") + { + return Some((normalized.to_string(), "done")); + } + + None +} + +// --------------------------------------------------------------------------- +// OpenAI-compatible /v1/chat/completions (SSE) +// --------------------------------------------------------------------------- + +pub enum ReplyChunk { + Snapshot(String), + Done, +} + +pub type ReplyRegistry = Arc>>>; + +const REPLY_FIRST_TIMEOUT: Duration = Duration::from_secs(180); +const DEFAULT_REPLY_TAIL_IDLE: Duration = Duration::from_millis(1500); +const MAX_IN_FLIGHT_REQUESTS: usize = 32; + +fn reply_tail_idle_timeout() -> Duration { + std::env::var("VTUBER_REPLY_TAIL_IDLE_MS") + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|v| *v > 0) + .map(Duration::from_millis) + .unwrap_or(DEFAULT_REPLY_TAIL_IDLE) +} + +pub struct VtuberConfig { + pub auth_key: Option, + pub default_model: String, +} + +impl VtuberConfig { + pub fn from_env() -> Option { + let enabled = std::env::var("VTUBER_ENABLED") + .map(|v| v == "true" || v == "1") + .unwrap_or(false); + if !enabled { + return None; + } + let auth_key = std::env::var("VTUBER_AUTH_KEY").ok(); + if auth_key.is_none() { + warn!("VTUBER_AUTH_KEY not set — /v1/chat/completions is UNAUTHENTICATED"); + } + let default_model = + std::env::var("VTUBER_DEFAULT_MODEL").unwrap_or_else(|_| "openab".into()); + Some(Self { + auth_key, + default_model, + }) + } +} + +#[derive(Deserialize)] +pub struct ChatMessage { + #[serde(default)] + pub role: String, + #[serde(default)] + pub content: String, +} + +#[derive(Deserialize)] +pub struct ChatRequest { + #[serde(default)] + pub model: Option, + #[serde(default)] + pub messages: Vec, + #[serde(default)] + pub stream: Option, +} + +#[cfg(test)] +fn flatten_messages(messages: &[ChatMessage]) -> String { + let mut out = String::new(); + for m in messages { + if m.content.trim().is_empty() { + continue; + } + let label = match m.role.as_str() { + "system" => "System", + "assistant" => "Assistant", + "user" | "" => "User", + other => other, + }; + if !out.is_empty() { + out.push_str("\n\n"); + } + out.push_str(label); + out.push_str(": "); + out.push_str(&m.content); + } + out +} + +fn persistent_session_prompt(messages: &[ChatMessage]) -> String { + let mut out = String::new(); + for m in messages { + if !matches!(m.role.as_str(), "system" | "developer") || m.content.trim().is_empty() { + continue; + } + append_labeled_message(&mut out, m); + } + + if let Some(m) = messages.iter().rev().find(|m| { + !matches!(m.role.as_str(), "assistant" | "system" | "developer") + && !m.content.trim().is_empty() + }) { + append_labeled_message(&mut out, m); + } + + out +} + +fn append_labeled_message(out: &mut String, message: &ChatMessage) { + let label = match message.role.as_str() { + "system" => "System", + "developer" => "Developer", + "assistant" => "Assistant", + "user" | "" => "User", + other => other, + }; + if !out.is_empty() { + out.push_str("\n\n"); + } + out.push_str(label); + out.push_str(": "); + out.push_str(&message.content); +} + +fn delta_suffix(full: &str, sent_len: usize) -> (String, usize) { + match full.get(sent_len..) { + Some(suffix) => (suffix.to_string(), full.len()), + None => (full.to_string(), full.len()), + } +} + +fn constant_time_eq(a: &str, b: &str) -> bool { + a.as_bytes().ct_eq(b.as_bytes()).into() +} + +pub async fn chat_completions( + State(state): State>, + headers: HeaderMap, + Json(req): Json, +) -> axum::response::Response { + let Some(ref cfg) = state.vtuber else { + return ( + StatusCode::SERVICE_UNAVAILABLE, + "vtuber adapter not configured", + ) + .into_response(); + }; + + if let Some(expected) = cfg.auth_key.as_ref() { + let provided = headers + .get(AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + .and_then(|s| s.strip_prefix("Bearer ")); + let ok = provided + .map(|provided| constant_time_eq(provided, expected)) + .unwrap_or(false); + if !ok { + return (StatusCode::UNAUTHORIZED, "invalid api key").into_response(); + } + } + + if req.stream != Some(true) { + return ( + StatusCode::BAD_REQUEST, + "only streaming mode is supported; set stream: true", + ) + .into_response(); + } + + // Serialise requests — one agent turn at a time on the shared VTuber session. + let Ok(request_guard) = state.vtuber_request_lock.clone().try_lock_owned() else { + let mut resp = axum::response::Response::new("agent is busy, retry in a moment".into()); + *resp.status_mut() = StatusCode::TOO_MANY_REQUESTS; + resp.headers_mut() + .insert(axum::http::header::RETRY_AFTER, "3".parse().unwrap()); + return resp; + }; + + let prompt = persistent_session_prompt(&req.messages); + if prompt.trim().is_empty() { + return ( + StatusCode::BAD_REQUEST, + "messages must contain non-empty content", + ) + .into_response(); + } + let model = req + .model + .clone() + .unwrap_or_else(|| cfg.default_model.clone()); + + // Persistent channel: reuse the same ACP session across VTuber turns. + let channel_id = VTUBER_PERSISTENT_CHANNEL.to_string(); + let (tx, rx) = mpsc::unbounded_channel::(); + { + let mut pending = state.vtuber_pending.lock().await; + if pending.len() >= MAX_IN_FLIGHT_REQUESTS { + let mut resp = axum::response::Response::new("too many in-flight requests".into()); + *resp.status_mut() = StatusCode::TOO_MANY_REQUESTS; + resp.headers_mut() + .insert(axum::http::header::RETRY_AFTER, "3".parse().unwrap()); + return resp; + } + pending.insert(channel_id.clone(), tx); + } + + let event = GatewayEvent::new( + "vtuber", + ChannelInfo { + id: channel_id.clone(), + channel_type: "dm".into(), + thread_id: None, + }, + SenderInfo { + id: "vtuber".into(), + name: "vtuber".into(), + display_name: "VTuber".into(), + is_bot: false, + }, + &prompt, + &format!("vtbmsg_{}", Uuid::new_v4()), + Vec::new(), + ); + match serde_json::to_string(&event) { + Ok(json) => { + let _ = state.event_tx.send(json); + } + Err(e) => { + state.vtuber_pending.lock().await.remove(&channel_id); + warn!("vtuber: failed to serialize event: {e}"); + return (StatusCode::INTERNAL_SERVER_ERROR, "internal error").into_response(); + } + } + info!(channel = %channel_id, "vtuber: chat request dispatched"); + + let stream = reply_stream( + rx, + model, + channel_id, + state.vtuber_pending.clone(), + reply_tail_idle_timeout(), + Some(request_guard), + ); + Sse::new(stream) + .keep_alive(KeepAlive::default()) + .into_response() +} + +fn chunk_event( + id: &str, + created: i64, + model: &str, + delta: serde_json::Value, + finish: Option<&str>, +) -> SseEvent { + let payload = json!({ + "id": id, "object": "chat.completion.chunk", "created": created, "model": model, + "choices": [{ "index": 0, "delta": delta, "finish_reason": finish }], + }); + SseEvent::default().data(payload.to_string()) +} + +struct StreamState { + rx: mpsc::UnboundedReceiver, + sent_len: usize, + phase: u8, + id: String, + created: i64, + model: String, + channel_id: String, + registry: ReplyRegistry, + seen_snapshot: bool, + tail_idle: Duration, + _request_guard: Option>, +} + +fn reply_stream( + rx: mpsc::UnboundedReceiver, + model: String, + channel_id: String, + registry: ReplyRegistry, + tail_idle: Duration, + request_guard: Option>, +) -> impl Stream> { + let init = StreamState { + rx, + sent_len: 0, + phase: 0, + id: format!("chatcmpl-{}", Uuid::new_v4()), + created: chrono::Utc::now().timestamp(), + model, + channel_id, + registry, + seen_snapshot: false, + tail_idle, + _request_guard: request_guard, + }; + futures_util::stream::unfold(init, |mut s| async move { + loop { + match s.phase { + 0 => { + s.phase = 1; + let ev = chunk_event( + &s.id, + s.created, + &s.model, + json!({"role":"assistant"}), + None, + ); + return Some((Ok(ev), s)); + } + 1 => match tokio::time::timeout( + if s.seen_snapshot { + s.tail_idle + } else { + REPLY_FIRST_TIMEOUT + }, + s.rx.recv(), + ) + .await + { + Ok(Some(ReplyChunk::Snapshot(full))) => { + let (delta, new_len) = delta_suffix(&full, s.sent_len); + if delta.is_empty() { + continue; + } + s.sent_len = new_len; + s.seen_snapshot = true; + let ev = chunk_event( + &s.id, + s.created, + &s.model, + json!({"content": delta}), + None, + ); + return Some((Ok(ev), s)); + } + Ok(Some(ReplyChunk::Done)) | Ok(None) => { + s.phase = 2; + continue; + } + Err(_) => { + if s.seen_snapshot { + info!(channel = %s.channel_id, "vtuber: reply stream idle, closing"); + } else { + warn!(channel = %s.channel_id, "vtuber: no reply — session may be dead; next request triggers respawn"); + } + s.phase = 2; + continue; + } + }, + 2 => { + s.phase = 3; + let ev = chunk_event(&s.id, s.created, &s.model, json!({}), Some("stop")); + return Some((Ok(ev), s)); + } + 3 => { + s.phase = 4; + s.registry.lock().await.remove(&s.channel_id); + return Some((Ok(SseEvent::default().data("[DONE]")), s)); + } + _ => return None, + } + } + }) +} + +pub async fn handle_reply(reply: &GatewayReply, registry: &ReplyRegistry) { + let key = reply.channel.id.as_str(); + let full = reply.content.text.clone(); + if full == "…" || full == "draft" { + return; + } + let is_pure_tool_status = parse_pure_tool_status(&full).is_some(); + + let mut map = registry.lock().await; + let Some(tx) = map.get(key) else { + return; + }; + + match reply.command.as_deref() { + Some("edit_message") => { + if is_pure_tool_status { + return; + } + if tx.send(ReplyChunk::Snapshot(full)).is_err() { + map.remove(key); + } + } + None => { + if !is_pure_tool_status { + let _ = tx.send(ReplyChunk::Snapshot(full)); + } + let _ = tx.send(ReplyChunk::Done); + map.remove(key); + } + _ => {} + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use futures_util::StreamExt; + + // ----------------------------------------------------------------------- + // Helper: spin up a minimal gateway with the VTuber chat route. + // ----------------------------------------------------------------------- + async fn start_gateway_with_state() -> (String, Arc) { + let (event_tx, _) = tokio::sync::broadcast::channel::(256); + let state = Arc::new(crate::AppState { + telegram_bot_token: None, + telegram_secret_token: None, + telegram_rich_messages: true, + telegram_streaming: None, + line_channel_secret: None, + line_access_token: None, + teams: None, + teams_service_urls: Mutex::new(HashMap::new()), + feishu: None, + google_chat: None, + wecom: None, + telegram_trusted_source_only: false, + vtuber: Some(VtuberConfig { + auth_key: Some("test-key".into()), + default_model: "openab".into(), + }), + vtuber_pending: Arc::new(Mutex::new(HashMap::new())), + vtuber_request_lock: Arc::new(tokio::sync::Mutex::new(())), + ws_token: Some("oab-token".into()), + event_tx, + reply_token_cache: Arc::new(std::sync::Mutex::new(HashMap::new())), + line_webhook_semaphore: Arc::new(tokio::sync::Semaphore::new(8)), + client: reqwest::Client::new(), + }); + + let app = axum::Router::new() + .route( + "/v1/chat/completions", + axum::routing::post(chat_completions), + ) + .with_state(state.clone()); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap().to_string(); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + + (addr, state) + } + + async fn start_gateway() -> String { + let (addr, _) = start_gateway_with_state().await; + addr + } + + // ----------------------------------------------------------------------- + // Integration: Tier-1 explicitly requires streaming mode. + // ----------------------------------------------------------------------- + #[tokio::test] + async fn chat_completions_rejects_non_streaming_requests() { + let addr = start_gateway().await; + let url = format!("http://{}/v1/chat/completions", addr); + let client = reqwest::Client::new(); + let bodies = [ + serde_json::json!({ + "model": "openab", + "messages": [{"role": "user", "content": "hello"}] + }), + serde_json::json!({ + "model": "openab", + "stream": false, + "messages": [{"role": "user", "content": "hello"}] + }), + ]; + + for body in bodies { + let resp = client + .post(&url) + .bearer_auth("test-key") + .json(&body) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + let text = resp.text().await.unwrap(); + assert!(text.contains("only streaming mode is supported")); + } + } + + #[tokio::test] + async fn chat_completions_rejects_when_in_flight_cap_is_reached() { + let (addr, state) = start_gateway_with_state().await; + for i in 0..MAX_IN_FLIGHT_REQUESTS { + let (tx, _rx) = mpsc::unbounded_channel::(); + state + .vtuber_pending + .lock() + .await + .insert(format!("existing_{i}"), tx); + } + + let resp = reqwest::Client::new() + .post(format!("http://{addr}/v1/chat/completions")) + .bearer_auth("test-key") + .json(&serde_json::json!({ + "model": "openab", + "stream": true, + "messages": [{"role": "user", "content": "hello"}] + })) + .send() + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS); + assert_eq!( + resp.headers() + .get(axum::http::header::RETRY_AFTER) + .and_then(|v| v.to_str().ok()), + Some("3") + ); + } + + #[tokio::test] + async fn chat_completions_serializes_for_stream_lifetime() { + let addr = start_gateway().await; + let url = format!("http://{}/v1/chat/completions", addr); + let client = reqwest::Client::new(); + let body = serde_json::json!({ + "model": "openab", + "stream": true, + "messages": [{"role": "user", "content": "hello"}] + }); + + let first = client + .post(&url) + .bearer_auth("test-key") + .json(&body) + .send() + .await + .unwrap(); + assert_eq!(first.status(), StatusCode::OK); + + let second = client + .post(&url) + .bearer_auth("test-key") + .json(&body) + .send() + .await + .unwrap(); + assert_eq!(second.status(), StatusCode::TOO_MANY_REQUESTS); + assert_eq!( + second + .headers() + .get(axum::http::header::RETRY_AFTER) + .and_then(|v| v.to_str().ok()), + Some("3") + ); + + drop(first); + } + + // ----------------------------------------------------------------------- + // Unit tests + // ----------------------------------------------------------------------- + + #[test] + fn auth_key_comparison_matches_only_exact_value() { + assert!(constant_time_eq("test-key", "test-key")); + assert!(!constant_time_eq("test-key", "test-kez")); + assert!(!constant_time_eq("test", "test-key")); + } + + #[test] + fn flatten_labels_roles_and_skips_empty() { + let msgs = vec![ + ChatMessage { + role: "system".into(), + content: "be concise".into(), + }, + ChatMessage { + role: "user".into(), + content: " ".into(), + }, + ChatMessage { + role: "assistant".into(), + content: "hello".into(), + }, + ChatMessage { + role: "user".into(), + content: "hi".into(), + }, + ]; + assert_eq!( + flatten_messages(&msgs), + "System: be concise\n\nAssistant: hello\n\nUser: hi" + ); + } + + #[test] + fn persistent_session_prompt_uses_system_and_latest_user_only() { + let msgs = vec![ + ChatMessage { + role: "system".into(), + content: "be 小光".into(), + }, + ChatMessage { + role: "user".into(), + content: "hi".into(), + }, + ChatMessage { + role: "assistant".into(), + content: "hello".into(), + }, + ChatMessage { + role: "user".into(), + content: "what now?".into(), + }, + ]; + + assert_eq!( + persistent_session_prompt(&msgs), + "System: be 小光\n\nUser: what now?" + ); + } + + #[tokio::test] + async fn reply_stream_finishes_after_snapshot_idle() { + let (tx, rx) = mpsc::unbounded_channel::(); + let registry: ReplyRegistry = Arc::new(Mutex::new(HashMap::new())); + registry.lock().await.insert("ch_idle".into(), tx.clone()); + + let mut stream = Box::pin(reply_stream( + rx, + "openab".into(), + "ch_idle".into(), + registry.clone(), + Duration::from_millis(10), + None, + )); + + assert!( + stream.next().await.is_some(), + "role chunk should be emitted first" + ); + tx.send(ReplyChunk::Snapshot("hello".into())).unwrap(); + assert!( + stream.next().await.is_some(), + "content chunk should be emitted" + ); + + let finish = tokio::time::timeout(Duration::from_secs(1), stream.next()).await; + assert!(finish.is_ok(), "finish chunk should arrive after tail idle"); + assert!(finish.unwrap().is_some()); + + let done = tokio::time::timeout(Duration::from_secs(1), stream.next()).await; + assert!(done.is_ok(), "[DONE] should arrive after finish chunk"); + assert!(done.unwrap().is_some()); + + assert!( + !registry.lock().await.contains_key("ch_idle"), + "stream completion should remove pending registry entry" + ); + } +} diff --git a/crates/openab-gateway/src/lib.rs b/crates/openab-gateway/src/lib.rs index 50750e744..3b082786a 100644 --- a/crates/openab-gateway/src/lib.rs +++ b/crates/openab-gateway/src/lib.rs @@ -42,6 +42,15 @@ pub struct AppState { pub google_chat: Option, #[cfg(feature = "wecom")] pub wecom: Option, + #[cfg(feature = "vtuber")] + pub vtuber: Option, + /// In-flight OpenAI-compatible requests awaiting their streamed reply, + /// keyed by the per-request `channel.id`. See `adapters::vtuber`. + #[cfg(feature = "vtuber")] + pub vtuber_pending: adapters::vtuber::ReplyRegistry, + /// Serialises /v1/chat/completions requests on the shared VTuber session. + #[cfg(feature = "vtuber")] + pub vtuber_request_lock: Arc>, pub ws_token: Option, pub event_tx: broadcast::Sender, pub reply_token_cache: ReplyTokenCache, @@ -78,6 +87,12 @@ impl AppState { google_chat: None, #[cfg(feature = "wecom")] wecom: None, + #[cfg(feature = "vtuber")] + vtuber: None, + #[cfg(feature = "vtuber")] + vtuber_pending: Arc::new(Mutex::new(HashMap::new())), + #[cfg(feature = "vtuber")] + vtuber_request_lock: Arc::new(tokio::sync::Mutex::new(())), ws_token: None, event_tx, reply_token_cache: Arc::new(std::sync::Mutex::new(HashMap::new())), @@ -162,6 +177,10 @@ impl AppState { let wecom = adapters::wecom::WecomConfig::from_env() .map(adapters::wecom::WecomAdapter::new); + // VTuber (OpenAI-compatible) + #[cfg(feature = "vtuber")] + let vtuber = adapters::vtuber::VtuberConfig::from_env(); + let client = reqwest::Client::builder() .timeout(std::time::Duration::from_secs(30)) .build() @@ -184,6 +203,12 @@ impl AppState { google_chat, #[cfg(feature = "wecom")] wecom, + #[cfg(feature = "vtuber")] + vtuber, + #[cfg(feature = "vtuber")] + vtuber_pending: Arc::new(Mutex::new(HashMap::new())), + #[cfg(feature = "vtuber")] + vtuber_request_lock: Arc::new(tokio::sync::Mutex::new(())), ws_token, event_tx, reply_token_cache: Arc::new(std::sync::Mutex::new(HashMap::new())), @@ -419,6 +444,16 @@ pub async fn serve(config: ServeConfig) -> anyhow::Result<()> { #[cfg(not(feature = "wecom"))] let wecom: Option<()> = None; + // VTuber (OpenAI-compatible) adapter + #[cfg(feature = "vtuber")] + let vtuber = adapters::vtuber::VtuberConfig::from_env(); + #[cfg(feature = "vtuber")] + if vtuber.is_some() { + let path = std::env::var("VTUBER_PATH").unwrap_or_else(|_| "/v1/chat/completions".into()); + info!("vtuber adapter enabled at {path}"); + app = app.route(&path, post(adapters::vtuber::chat_completions)); + } + let client = reqwest::Client::builder() .timeout(std::time::Duration::from_secs(30)) .build() @@ -445,6 +480,12 @@ pub async fn serve(config: ServeConfig) -> anyhow::Result<()> { google_chat, #[cfg(feature = "wecom")] wecom, + #[cfg(feature = "vtuber")] + vtuber, + #[cfg(feature = "vtuber")] + vtuber_pending: Arc::new(Mutex::new(HashMap::new())), + #[cfg(feature = "vtuber")] + vtuber_request_lock: Arc::new(tokio::sync::Mutex::new(())), ws_token, event_tx, reply_token_cache, @@ -652,6 +693,14 @@ async fn handle_oab_connection(state: Arc, socket: axum::extract::ws:: warn!("reply for wecom but adapter not configured"); } } + #[cfg(feature = "vtuber")] + "vtuber" => { + adapters::vtuber::handle_reply( + &reply, + &state_for_recv.vtuber_pending, + ) + .await; + } other => warn!(platform = other, "unknown reply platform"), } } diff --git a/docs/config-reference.md b/docs/config-reference.md index 2f5509a14..def59540f 100644 --- a/docs/config-reference.md +++ b/docs/config-reference.md @@ -692,6 +692,7 @@ Each platform is auto-enabled when its env vars are present: | Google Chat | `GOOGLE_CHAT_ENABLED=true` | `GOOGLE_CHAT_SA_KEY_JSON`, `GOOGLE_CHAT_SA_KEY_FILE`, `GOOGLE_CHAT_ACCESS_TOKEN`, `GOOGLE_CHAT_AUDIENCE`, `GOOGLE_CHAT_WEBHOOK_PATH` | | WeCom | `WECOM_CORP_ID` | _(see wecom config)_ | | Teams | `TEAMS_APP_ID` | `TEAMS_WEBHOOK_PATH` | +| VTuber | `VTUBER_ENABLED=true` | `VTUBER_AUTH_KEY`, `VTUBER_DEFAULT_MODEL`, `VTUBER_PATH`, `VTUBER_REPLY_TAIL_IDLE_MS` | > ⚠️ **Production checklist**: Set `GATEWAY_ALLOW_ALL_CHANNELS=false` and `GATEWAY_ALLOW_ALL_USERS=false` with explicit allowlists. The defaults are permissive for development convenience. > diff --git a/docs/vtuber.md b/docs/vtuber.md new file mode 100644 index 000000000..9485ebe8d --- /dev/null +++ b/docs/vtuber.md @@ -0,0 +1,142 @@ +# VTuber (OpenAI-compatible) Setup + +Expose an **OpenAI-compatible `/v1/chat/completions` (SSE)** endpoint from the +unified OpenAB binary, so any character "skin" that already speaks OpenAI chat +completions (AniCompanion, Open-LLM-VTuber, ChatVRM, ...) gets a real agent: +tool use, code, MCP, memory, and the same configured ACP backend. + +The skin connects directly to the unified OpenAB HTTP listener in the same +OpenAB process: + +- Chat: `POST /v1/chat/completions` streams SSE responses. +- Agent work: the route dispatches to the configured ACP agent in OpenAB. + +Unlike chat-platform adapters (LINE, Telegram, ...), this is **not a webhook**: +the skin opens an HTTP request and the reply streams back on that same +connection. + +## Prerequisites + +- An OpenAB image/binary built with the `unified` feature. +- An ACP agent configured in the same OpenAB process. +- A public URL, tunnel, or localhost endpoint reachable by the skin. +- A skin that supports an OpenAI-compatible backend, for example + AniCompanion -> Settings -> Agent backend -> OpenAI-compatible. + +## 1. Enable the VTuber Adapter + +Set these environment variables on the OpenAB process: + +```bash +VTUBER_ENABLED=true +VTUBER_AUTH_KEY="$(openssl rand -hex 32)" +VTUBER_DEFAULT_MODEL=openab +GATEWAY_LISTEN=0.0.0.0:8080 +``` + +Example Docker run with a Kiro-backed OpenAB image: + +```bash +docker run -d --name openab-vtuber \ + -e VTUBER_ENABLED=true \ + -e VTUBER_AUTH_KEY="$VTUBER_AUTH_KEY" \ + -e VTUBER_DEFAULT_MODEL=openab \ + -e KIRO_API_KEY="$KIRO_API_KEY" \ + -p 8080:8080 \ + ghcr.io/openabdev/openab:beta-kiro +``` + +For Kubernetes or Zeabur, put the same environment variables on the OpenAB +service. No companion container or adapter config block is needed in unified +mode. + +## 2. Configure the Agent + +Use the normal OpenAB agent configuration. The VTuber adapter submits incoming +skin messages directly to OpenAB's in-process dispatcher. + +```toml +[agent] +command = "kiro-cli" +args = ["acp", "--trust-all-tools"] +``` + +Streaming is handled by the VTuber adapter's SSE endpoint; it does not require +separate streaming settings. + +The adapter reuses one warm ACP session for VTuber traffic. Requests are +serialized; if a turn is already streaming, another request receives `429` with +`Retry-After: 3`. + +## 3. Point the Skin at OpenAB + +In the skin's OpenAI-compatible backend settings: + +- **Endpoint / Base URL**: `https://your-openab-host` (the adapter serves + `/v1/chat/completions`) +- **API Key**: the `VTUBER_AUTH_KEY` value (sent as `Authorization: Bearer `) +- **Model**: anything; OpenAB routes to the configured ACP agent and echoes the + model name back in OpenAI-compatible chunks + +## 4. Test + +```bash +curl -N https://your-openab-host/v1/chat/completions \ + -H "Authorization: Bearer $VTUBER_AUTH_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model":"openab","stream":true, + "messages":[{"role":"user","content":"hi 小光"}]}' +``` + +You should see `data: {...chat.completion.chunk...}` lines ending with +`data: [DONE]`. + +## Emotion Tags + +Inline `[emotion]` tags (for example AniCompanion's `[happy]`, `[sad]`, +`[curious]`, ...) are the skin's own convention. The skin's persona/system +prompt instructs the agent to emit them, and the skin parses and strips them +before TTS. The adapter passes them through verbatim. + +## Environment Variables + +| Variable | Required | Description | +|---|---|---| +| `VTUBER_ENABLED` | Yes | `true`/`1` to enable the adapter in the unified binary | +| `VTUBER_AUTH_KEY` | Recommended | Bearer key required on requests. If unset, the endpoint is unauthenticated and logs a warning | +| `VTUBER_DEFAULT_MODEL` | No | Model name echoed back when the request omits one (default `openab`) | +| `VTUBER_PATH` | No | Route path (default `/v1/chat/completions`) | +| `VTUBER_REPLY_TAIL_IDLE_MS` | No | Tail-idle close delay after the first content snapshot (default `1500`) | +| `GATEWAY_LISTEN` | No | Bind address for the unified HTTP listener (default `0.0.0.0:8080`) | + +## Notes & Limitations + +- **One shared warm session.** The adapter reuses a stable VTuber session to + avoid per-turn agent cold starts. OpenAI-compatible skins may still send full + `messages[]`; the adapter forwards system/developer instructions plus the + latest user turn so assistant history is not duplicated inside the persistent + ACP session. +- **One turn at a time.** Concurrent requests return `429` with `Retry-After: 3` + instead of queueing behind the active VTuber turn. +- **No agent output => no chat output.** If the configured ACP agent cannot + answer, the SSE request eventually closes after the reply timeout. +- **Tags are not motion.** Mapping `[emotion]` tags to VRM expressions, Live2D + parameters, or VTube Studio actions is the skin's job. + +## Troubleshooting + +**No response / stream hangs then closes:** +- Confirm `VTUBER_ENABLED=true` is set on the OpenAB process. +- Confirm the ACP agent is configured and authenticated. +- Check OpenAB logs for `unified: vtuber adapter enabled`. + +**`401 invalid api key`:** +- The `Authorization: Bearer ` value must match `VTUBER_AUTH_KEY`. + +**Reply arrives all at once instead of streaming:** +- Confirm the skin is calling `/v1/chat/completions` with `stream: true`. +- Confirm no proxy in front of OpenAB buffers SSE responses. + +## References + +- [RFC: VTuber adapter](https://github.com/openabdev/openab/issues/1233) diff --git a/src/main.rs b/src/main.rs index b7daf5656..ed2088340 100644 --- a/src/main.rs +++ b/src/main.rs @@ -123,7 +123,14 @@ fn has_unified_platform_env() -> bool { || (cfg!(feature = "feishu") && std::env::var("FEISHU_APP_ID").is_ok()) || (cfg!(feature = "wecom") && std::env::var("WECOM_CORP_ID").is_ok()) || (cfg!(feature = "teams") && std::env::var("TEAMS_APP_ID").is_ok()) - || (cfg!(feature = "googlechat") && std::env::var("GOOGLE_CHAT_ENABLED").map(|v| v == "true" || v == "1").unwrap_or(false)) + || (cfg!(feature = "googlechat") + && std::env::var("GOOGLE_CHAT_ENABLED") + .map(|v| v == "true" || v == "1") + .unwrap_or(false)) + || (cfg!(feature = "vtuber") + && std::env::var("VTUBER_ENABLED") + .map(|v| v == "true" || v == "1") + .unwrap_or(false)) } #[tokio::main] @@ -145,7 +152,11 @@ async fn main() -> anyhow::Result<()> { return Ok(()); } #[cfg(feature = "agentcore")] - Commands::AgentcoreBridge { runtime_arn, region, command } => { + Commands::AgentcoreBridge { + runtime_arn, + region, + command, + } => { return acp::agentcore::run_bridge(&runtime_arn, ®ion, &command).await; } Commands::Set { key, value, thread } => { @@ -562,7 +573,7 @@ async fn main() -> anyhow::Result<()> { feature = "teams", ))] let _unified_handle = { - use openab_core::gateway::{GatewayEventContext, process_gateway_event}; + use openab_core::gateway::{process_gateway_event, GatewayEventContext}; if has_unified_platform_env() || cfg.telegram.is_some() { let listen_addr = std::env::var("GATEWAY_LISTEN") @@ -606,8 +617,8 @@ async fn main() -> anyhow::Result<()> { let gw_state = Arc::new(gw_state_inner); // Build axum router with platform webhook routes - let mut app = axum::Router::new() - .route("/health", axum::routing::get(|| async { "ok" })); + let mut app = + axum::Router::new().route("/health", axum::routing::get(|| async { "ok" })); #[cfg(feature = "telegram")] if gw_state.telegram_bot_token.is_some() { @@ -616,13 +627,19 @@ async fn main() -> anyhow::Result<()> { .unwrap_or_else(|_| "/webhook/telegram".into()) }); info!(path = %path, "unified: telegram adapter enabled"); - app = app.route(&path, axum::routing::post(openab_gateway::adapters::telegram::webhook)); + app = app.route( + &path, + axum::routing::post(openab_gateway::adapters::telegram::webhook), + ); } #[cfg(feature = "line")] { info!("unified: line adapter enabled"); - app = app.route("/webhook/line", axum::routing::post(openab_gateway::adapters::line::webhook)); + app = app.route( + "/webhook/line", + axum::routing::post(openab_gateway::adapters::line::webhook), + ); } #[cfg(feature = "feishu")] @@ -630,23 +647,35 @@ async fn main() -> anyhow::Result<()> { let path = std::env::var("FEISHU_WEBHOOK_PATH") .unwrap_or_else(|_| "/webhook/feishu".into()); info!(path = %path, "unified: feishu adapter enabled"); - app = app.route(&path, axum::routing::post(openab_gateway::adapters::feishu::webhook)); + app = app.route( + &path, + axum::routing::post(openab_gateway::adapters::feishu::webhook), + ); } #[cfg(feature = "wecom")] if let Some(ref w) = gw_state.wecom { info!(path = %w.config.webhook_path, "unified: wecom adapter enabled"); app = app - .route(&w.config.webhook_path, axum::routing::get(openab_gateway::adapters::wecom::verify)) - .route(&w.config.webhook_path, axum::routing::post(openab_gateway::adapters::wecom::webhook)); + .route( + &w.config.webhook_path, + axum::routing::get(openab_gateway::adapters::wecom::verify), + ) + .route( + &w.config.webhook_path, + axum::routing::post(openab_gateway::adapters::wecom::webhook), + ); } #[cfg(feature = "teams")] if gw_state.teams.is_some() { - let path = std::env::var("TEAMS_WEBHOOK_PATH") - .unwrap_or_else(|_| "/webhook/teams".into()); + let path = + std::env::var("TEAMS_WEBHOOK_PATH").unwrap_or_else(|_| "/webhook/teams".into()); info!(path = %path, "unified: teams adapter enabled"); - app = app.route(&path, axum::routing::post(openab_gateway::adapters::teams::webhook)); + app = app.route( + &path, + axum::routing::post(openab_gateway::adapters::teams::webhook), + ); } #[cfg(feature = "googlechat")] @@ -654,17 +683,31 @@ async fn main() -> anyhow::Result<()> { let path = std::env::var("GOOGLE_CHAT_WEBHOOK_PATH") .unwrap_or_else(|_| "/webhook/googlechat".into()); info!(path = %path, "unified: googlechat adapter enabled"); - app = app.route(&path, axum::routing::post(openab_gateway::adapters::googlechat::webhook)); + app = app.route( + &path, + axum::routing::post(openab_gateway::adapters::googlechat::webhook), + ); + } + + #[cfg(feature = "vtuber")] + if gw_state.vtuber.is_some() { + let path = + std::env::var("VTUBER_PATH").unwrap_or_else(|_| "/v1/chat/completions".into()); + info!(path = %path, "unified: vtuber adapter enabled"); + app = app.route( + &path, + axum::routing::post(openab_gateway::adapters::vtuber::chat_completions), + ); } let app = app.with_state(gw_state.clone()); // Bridge task: receive events from adapters via event_tx, dispatch to core let unified_adapter: Arc = Arc::new( - unified_adapter::UnifiedGatewayAdapter::new(gw_state.clone()) + unified_adapter::UnifiedGatewayAdapter::new(gw_state.clone()), ); - // Read security gating from env (mirrors [gateway] config section) + // Read security gating from env for unified platform adapters. let gw_allow_all_channels = std::env::var("GATEWAY_ALLOW_ALL_CHANNELS") .map(|v| v != "0" && !v.eq_ignore_ascii_case("false")) .unwrap_or(true); @@ -862,14 +905,17 @@ async fn main() -> anyhow::Result<()> { let reminder_store = remind::ReminderStore::load(reminder_path); // Construct ambient dispatcher if enabled and channels configured. - let ambient_dispatcher = if cfg.ambient.enabled && !cfg.ambient.discord.channels.is_empty() { + let ambient_dispatcher = if cfg.ambient.enabled && !cfg.ambient.discord.channels.is_empty() + { info!( channels = ?cfg.ambient.discord.channels, flush_interval = cfg.ambient.flush_interval_seconds, flush_max_messages = cfg.ambient.flush_max_messages, "ambient mode enabled" ); - Some(Arc::new(openab_core::ambient::AmbientDispatcher::new(cfg.ambient.clone()))) + Some(Arc::new(openab_core::ambient::AmbientDispatcher::new( + cfg.ambient.clone(), + ))) } else { None }; @@ -1069,6 +1115,7 @@ mod tests { std::env::remove_var("WECOM_CORP_ID"); std::env::remove_var("TEAMS_APP_ID"); std::env::remove_var("GOOGLE_CHAT_ENABLED"); + std::env::remove_var("VTUBER_ENABLED"); } // Case 1: no env vars → false @@ -1090,6 +1137,16 @@ mod tests { std::env::set_var("TELEGRAM_BOT_TOKEN", "test-token"); assert_eq!(has_unified_platform_env(), cfg!(feature = "telegram")); + // Case 5: VTUBER_ENABLED=true → true only if feature compiled + clear_all(); + std::env::set_var("VTUBER_ENABLED", "true"); + assert_eq!(has_unified_platform_env(), cfg!(feature = "vtuber")); + + // Case 6: VTUBER_ENABLED=yes (invalid) → false + clear_all(); + std::env::set_var("VTUBER_ENABLED", "yes"); + assert!(!has_unified_platform_env()); + // Cleanup clear_all(); } diff --git a/src/unified_adapter.rs b/src/unified_adapter.rs index 0d6043f54..12b812e5a 100644 --- a/src/unified_adapter.rs +++ b/src/unified_adapter.rs @@ -89,8 +89,19 @@ impl UnifiedGatewayAdapter { .await; } } + #[cfg(feature = "vtuber")] + "vtuber" => { + openab_gateway::adapters::vtuber::handle_reply( + reply, + &self.gw_state.vtuber_pending, + ) + .await; + } other => { - tracing::warn!(platform = other, "unified adapter: unknown platform, cannot route reply"); + tracing::warn!( + platform = other, + "unified adapter: unknown platform, cannot route reply" + ); } } } @@ -138,8 +149,13 @@ impl ChatAdapter for UnifiedGatewayAdapter { self.dispatch_reply(&reply).await; Ok(MessageRef { channel: channel.clone(), - message_id: format!("unified_{:x}", std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH).unwrap_or_default().as_nanos()), + message_id: format!( + "unified_{:x}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + ), }) } @@ -195,8 +211,13 @@ impl ChatAdapter for UnifiedGatewayAdapter { self.dispatch_reply(&reply).await; Ok(MessageRef { channel: channel.clone(), - message_id: format!("unified_{:x}", std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH).unwrap_or_default().as_nanos()), + message_id: format!( + "unified_{:x}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + ), }) }