diff --git a/agent/codex/session.go b/agent/codex/session.go index c620306b9b..9f508b79df 100644 --- a/agent/codex/session.go +++ b/agent/codex/session.go @@ -408,14 +408,17 @@ func (cs *codexSession) handleEvent(raw map[string]any) { case "turn.failed": errMsg := "" + errInfo := "" if errObj, ok := raw["error"].(map[string]any); ok { errMsg, _ = errObj["message"].(string) + errInfo, _ = errObj["codex_error_info"].(string) } if errMsg == "" { errMsg = "turn failed (no details)" } - slog.Warn("codexSession: turn failed", "error", errMsg) - evt := core.Event{Type: core.EventError, Error: fmt.Errorf("%s", errMsg)} + errKind := codexErrorKind(errInfo, errMsg) + slog.Warn("codexSession: turn failed", "error", errMsg, "codex_error_info", errInfo, "error_kind", errKind) + evt := core.Event{Type: core.EventError, Error: fmt.Errorf("%s", errMsg), ErrorKind: errKind} select { case cs.events <- evt: case <-cs.ctx.Done(): @@ -435,6 +438,36 @@ func (cs *codexSession) handleEvent(raw map[string]any) { } } +func codexErrorKind(errorInfo string, message string) core.ErrorKind { + switch strings.TrimSpace(errorInfo) { + case "server_overloaded": + return core.ErrorKindOverloaded + case "rate_limit_exceeded", "rate_limit": + return core.ErrorKindRateLimit + } + + msg := strings.ToLower(message) + if strings.Contains(msg, "at capacity") || strings.Contains(msg, "overloaded") { + return core.ErrorKindOverloaded + } + if strings.Contains(msg, "rate limit") || strings.Contains(msg, "rate_limit") { + return core.ErrorKindRateLimit + } + if strings.Contains(msg, "stream disconnected before completion") || + strings.Contains(msg, "stream closed before response.completed") || + strings.Contains(msg, "you can retry your request") || + strings.Contains(msg, "processing your request") || + strings.Contains(msg, "unexpected status 502") || + strings.Contains(msg, "unexpected status 503") || + strings.Contains(msg, "unexpected status 504") || + strings.Contains(msg, "bad gateway") || + strings.Contains(msg, "service unavailable") || + strings.Contains(msg, "gateway timeout") { + return core.ErrorKindOverloaded + } + return core.ErrorKindUnknown +} + // flushPendingAsThinking emits all buffered agent_messages as EventThinking. func (cs *codexSession) flushPendingAsThinking() { if cs.ctx.Err() != nil { diff --git a/agent/codex/session_test.go b/agent/codex/session_test.go index c789557e26..a5e74415ad 100644 --- a/agent/codex/session_test.go +++ b/agent/codex/session_test.go @@ -609,7 +609,11 @@ func TestSend_HandlesLargeJSONLines(t *testing.T) { if err != nil { t.Fatalf("newCodexSession: %v", err) } - defer cs.Close() + defer func() { + if err := cs.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + }() if err := cs.Send("hello", "", nil, nil); err != nil { t.Fatalf("Send: %v", err) @@ -644,6 +648,93 @@ func TestSend_HandlesLargeJSONLines(t *testing.T) { } } +func TestSend_TurnFailedServerOverloadedIsRetriable(t *testing.T) { + workDir := t.TempDir() + binDir := filepath.Join(workDir, "bin") + if err := os.MkdirAll(binDir, 0o755); err != nil { + t.Fatalf("mkdir bin: %v", err) + } + + shellScript := "#!/bin/sh\n" + + "printf '%s\\n' '{\"type\":\"thread.started\",\"thread_id\":\"thread-overloaded\"}'\n" + + "printf '%s\\n' '{\"type\":\"turn.failed\",\"error\":{\"message\":\"Selected model is at capacity. Please try a different model.\",\"codex_error_info\":\"server_overloaded\"}}'\n" + powershellScript := "[Console]::Out.WriteLine('{\"type\":\"thread.started\",\"thread_id\":\"thread-overloaded\"}')\n" + + "[Console]::Out.WriteLine('{\"type\":\"turn.failed\",\"error\":{\"message\":\"Selected model is at capacity. Please try a different model.\",\"codex_error_info\":\"server_overloaded\"}}')\n" + writeFakeCodexScript(t, binDir, shellScript, powershellScript) + + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + + cs, err := newCodexSession(context.Background(), "codex", nil, workDir, "", "", "", "", "", nil, "", "", "") + if err != nil { + t.Fatalf("newCodexSession: %v", err) + } + defer func() { + if err := cs.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + }() + + if err := cs.Send("hello", "", nil, nil); err != nil { + t.Fatalf("Send: %v", err) + } + + timeout := time.After(5 * time.Second) + for { + select { + case evt := <-cs.Events(): + if evt.Type != core.EventError { + continue + } + if evt.ErrorKind != core.ErrorKindOverloaded { + t.Fatalf("ErrorKind = %q, want %q", evt.ErrorKind, core.ErrorKindOverloaded) + } + if evt.Error == nil || !strings.Contains(evt.Error.Error(), "Selected model is at capacity") { + t.Fatalf("Error = %v, want capacity message", evt.Error) + } + return + case <-timeout: + t.Fatal("timed out waiting for retriable error event") + } + } +} + +func TestCodexErrorKind_RetriableStreamFailures(t *testing.T) { + tests := []struct { + name string + message string + want core.ErrorKind + }{ + { + name: "generic processing error with retry hint", + message: "stream disconnected before completion: An error occurred while processing your request. You can retry your request, or contact us through our help center at help.openai.com if the error persists. Please include the request ID 0a043d4a-c04a-4ace-becd-deb4cc1f5dd2 in your message.", + want: core.ErrorKindOverloaded, + }, + { + name: "stream closed before response completed", + message: "stream disconnected before completion: stream closed before response.completed", + want: core.ErrorKindOverloaded, + }, + { + name: "gateway error", + message: "unexpected status 502 Bad Gateway, url: https://aiapi.uu.cc/v1/responses", + want: core.ErrorKindOverloaded, + }, + { + name: "non transient error remains unknown", + message: "authentication failed: invalid api key", + want: core.ErrorKindUnknown, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := codexErrorKind("", tc.message); got != tc.want { + t.Fatalf("codexErrorKind() = %q, want %q", got, tc.want) + } + }) + } +} + func TestWaitForArgsFile_WaitsForNonEmptyContent(t *testing.T) { workDir := t.TempDir() argsFile := filepath.Join(workDir, "args.txt") diff --git a/core/engine.go b/core/engine.go index 5abc4f5971..232efa35e2 100644 --- a/core/engine.go +++ b/core/engine.go @@ -3720,59 +3720,20 @@ func (e *Engine) processInteractiveMessageWith(p Platform, msg *Message, session } } - // Start typing indicator if platform supports it. - // Ownership is transferred to processInteractiveEvents which manages - // stopping/restarting it across queued message turns. - var stopTyping func() - if ti, ok := p.(TypingIndicator); ok { - stopTyping = ti.StartTyping(e.ctx, msg.ReplyCtx) - } - defer func() { - // Stop typing if ownership was NOT transferred to processInteractiveEvents - // (i.e. an early return before that call). - if stopTyping != nil { - stopTyping() - } - }() - // Stop the unsolicited reader (if running) and hand off event channel // ownership to this foreground turn. Only drain events when the previous // turn ended abnormally (eventsNeedResync=true, the default). e.stopUnsolicitedReader(state) - state.mu.Lock() - needResync := state.eventsNeedResync - state.mu.Unlock() - if needResync { - drainEvents(state.agentSession.Events()) - } promptContent := e.buildSenderPrompt(msg.Content, msg.UserID, msg.UserName, msg.Platform, msg.SessionKey, msg.ChannelKey) - sendStart := time.Now() state.mu.Lock() state.currentMessageID = msg.MessageID state.fromVoice = msg.FromVoice state.sideText = "" - as := state.agentSession // capture under lock to avoid race with cleanup state.mu.Unlock() - // Run Send concurrently with processInteractiveEvents. Some agents block inside - // Send until the prompt turn finishes (e.g. ACP session/prompt); they may emit - // EventPermissionRequest while blocked — the event loop must run in parallel. - sendDone := make(chan error, 1) - go func() { - if as == nil { - sendDone <- fmt.Errorf("agent session became nil") - return - } - sendDone <- as.Send(promptContent, msg.MessageID, msg.Images, msg.Files) - }() - - e.processInteractiveEvents(state, session, sessions, interactiveKey, msg.MessageID, turnStart, stopTyping, sendDone, msg.ReplyCtx) - if elapsed := time.Since(sendStart); elapsed >= slowAgentSend { - slog.Warn("slow agent send", "elapsed", elapsed, "session", msg.SessionKey, "content_len", len(msg.Content)) - } - stopTyping = nil // ownership transferred; prevent defer from double-stopping + e.processInteractiveTurnWithRetry(state, session, sessions, interactiveKey, promptContent, msg.MessageID, msg.Images, msg.Files, msg.ReplyCtx, turnStart, msg.SessionKey, len(msg.Content)) // Guard against a narrow race: a message may have been queued between // processInteractiveEvents observing an empty queue and returning here @@ -3794,6 +3755,160 @@ func (e *Engine) processInteractiveMessageWith(p Platform, msg *Message, session } } +type interactiveRetryTurn struct { + kind ErrorKind + err error + promptContent string + msgID string + images []ImageAttachment + files []FileAttachment + replyCtx any + logSessionKey string + contentLen int + notify func(string) bool + finalizeNotice func(ProgressCardState, CardStatus) +} + +func (e *Engine) processInteractiveTurnWithRetry(state *interactiveState, session *Session, sessions *SessionManager, sessionKey string, promptContent string, msgID string, images []ImageAttachment, files []FileAttachment, replyCtx any, turnStart time.Time, logSessionKey string, contentLen int) { + maxAttempts := RetriableErrorMaxAttempts + if maxAttempts < 1 { + maxAttempts = 1 + } + + var stopTyping func() + defer func() { + if stopTyping != nil { + stopTyping() + } + }() + + for attempt := 1; ; attempt++ { + if state.isStopped() { + return + } + + state.mu.Lock() + p := state.platform + as := state.agentSession // capture under lock to avoid race with cleanup + needResync := state.eventsNeedResync + state.mu.Unlock() + + if as == nil || !as.Alive() { + if stopTyping != nil { + stopTyping() + stopTyping = nil + } + e.send(p, replyCtx, fmt.Sprintf(e.i18n.T(MsgError), "agent session ended")) + return + } + + if needResync { + drainEvents(as.Events()) + } + + if stopTyping == nil { + if ti, ok := p.(TypingIndicator); ok { + stopTyping = ti.StartTyping(e.ctx, replyCtx) + } + } + + sendStart := time.Now() + // Run Send concurrently with processInteractiveEvents. Some agents block inside + // Send until the prompt turn finishes (e.g. ACP session/prompt); they may emit + // EventPermissionRequest while blocked — the event loop must run in parallel. + sendDone := make(chan error, 1) + go func(agentSession AgentSession) { + if agentSession == nil { + sendDone <- fmt.Errorf("agent session became nil") + return + } + sendDone <- agentSession.Send(promptContent, msgID, images, files) + }(as) + + retryTurn := e.processInteractiveEvents(state, session, sessions, sessionKey, msgID, turnStart, stopTyping, sendDone, replyCtx) + stopTyping = nil // ownership transferred; prevent defer from double-stopping + + if elapsed := time.Since(sendStart); elapsed >= slowAgentSend { + slog.Warn("slow agent send", "elapsed", elapsed, "session", logSessionKey, "content_len", contentLen, "attempt", attempt) + } + + if retryTurn == nil || !retryTurn.kind.IsRetriable() { + return + } + retryKind := retryTurn.kind + retryErr := retryTurn.err + if retryTurn.promptContent != "" { + promptContent = retryTurn.promptContent + msgID = retryTurn.msgID + images = retryTurn.images + files = retryTurn.files + replyCtx = retryTurn.replyCtx + logSessionKey = retryTurn.logSessionKey + contentLen = retryTurn.contentLen + } + + if attempt >= maxAttempts { + slog.Error("retriable agent error exhausted", "error", retryErr, "kind", retryKind, "session", logSessionKey, "attempts", attempt) + if retryTurn.finalizeNotice != nil { + retryTurn.finalizeNotice(ProgressCardStateFailed, CardStatusError) + } + state.mu.Lock() + p := state.platform + state.mu.Unlock() + if retryErr == nil { + retryErr = fmt.Errorf("agent returned retriable error: %s", retryKind) + } + if retryErr != nil { + e.send(p, replyCtx, fmt.Sprintf(e.i18n.T(MsgError), retryErr)) + } + return + } + + delay := RetriableErrorDelay(attempt) + slog.Warn("retrying agent turn after retriable error", "error", retryErr, "kind", retryKind, "session", logSessionKey, "attempt", attempt, "max_attempts", maxAttempts, "delay", delay) + if attempt == 1 || attempt%5 == 0 { + state.mu.Lock() + p := state.platform + state.mu.Unlock() + notice := fmt.Sprintf(e.i18n.T(MsgRetriableAgentError), delay.Round(time.Second), attempt+1, maxAttempts) + if retryTurn.notify == nil || !retryTurn.notify(notice) { + e.send(p, replyCtx, notice) + } + } + + timer := time.NewTimer(delay) + stopCh := state.stopSignal() + select { + case <-timer.C: + case <-e.ctx.Done(): + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + if retryTurn.finalizeNotice != nil { + retryTurn.finalizeNotice(ProgressCardStateCompleted, CardStatusDone) + } + return + case <-stopCh: + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + if retryTurn.finalizeNotice != nil { + retryTurn.finalizeNotice(ProgressCardStateCompleted, CardStatusDone) + } + return + } + if retryTurn.finalizeNotice != nil { + retryTurn.finalizeNotice(ProgressCardStateCompleted, CardStatusDone) + } + } +} + // getOrCreateWorkspaceAgent returns (or creates) a per-workspace agent and session manager. // workspace must be a normalized path (from resolveWorkspace or normalizeWorkspacePath). func (e *Engine) getOrCreateWorkspaceAgent(workspace string) (Agent, *SessionManager, error) { @@ -4680,7 +4795,7 @@ var agentErrorHandlers = []agentErrorHandler{ {"Session not found", MsgSessionNotFound}, } -func (e *Engine) processInteractiveEvents(state *interactiveState, session *Session, sessions *SessionManager, sessionKey string, msgID string, turnStart time.Time, stopTypingFn func(), sendDone <-chan error, replyCtx any) { +func (e *Engine) processInteractiveEvents(state *interactiveState, session *Session, sessions *SessionManager, sessionKey string, msgID string, turnStart time.Time, stopTypingFn func(), sendDone <-chan error, replyCtx any) (retryTurn *interactiveRetryTurn) { if msgID != "" { state.mu.Lock() state.currentMessageID = msgID @@ -4700,6 +4815,7 @@ func (e *Engine) processInteractiveEvents(state *interactiveState, session *Sess var partialText string triggerAutoCompress := false pendingSend := sendDone + var currentRetryTurn *interactiveRetryTurn // stopTyping tracks the current turn's typing indicator so it can be // stopped when a queued message starts a new turn. @@ -5847,6 +5963,15 @@ func (e *Engine) processInteractiveEvents(state *interactiveState, session *Sess } queuedPrompt := e.buildSenderPrompt(queued.content, queued.userID, queued.userName, queued.msgPlatform, queued.msgSessionKey, queued.channelKey) + currentRetryTurn = &interactiveRetryTurn{ + promptContent: queuedPrompt, + msgID: queued.messageID, + images: queued.images, + files: queued.files, + replyCtx: queued.replyCtx, + logSessionKey: sessionKey, + contentLen: len(queued.content), + } state.mu.Lock() as := state.agentSession // capture under lock to avoid race with cleanup @@ -5960,11 +6085,81 @@ func (e *Engine) processInteractiveEvents(state *interactiveState, session *Sess return case EventError: - cp.Finalize(ProgressCardStateFailed) - sp.discard() state.mu.Lock() state.eventsNeedResync = true state.mu.Unlock() + if event.ErrorKind.IsRetriable() { + if pendingSend != nil { + if err := <-pendingSend; err != nil { + slog.Debug("async send error after retriable EventError", "error", err) + } + } + slog.Warn("agent retriable error", "error", event.Error, "kind", event.ErrorKind, "session_key", sessionKey) + var lastRetryNotice string + notifyRetry := func(notice string) bool { + notice = strings.TrimSpace(notice) + if notice == "" { + return false + } + lastRetryNotice = notice + if hasRichCard && cardMessageID != nil { + if updater, ok := p.(MessageUpdater); ok { + statusFooter := joinStatusFooterLines( + notice, + e.composeRichStatusFooter(true, turnStart, e.agent, state.agentSession, state.workspaceDir), + ) + card := buildResolvedRichCard(CardStatusWorking, "", toolSteps, partialText, true, statusFooter) + if err := updater.UpdateMessage(e.ctx, cardMessageID, card); err == nil { + return true + } else { + slog.Debug("rich card: failed to update retriable notice", "platform", p.Name(), "error", err) + } + } + } + if cp.AppendStructuredImmediate(ProgressCardEntry{Kind: ProgressEntryInfo, Text: notice}, notice) { + return true + } + return sp.updateStatusFooter(CardStatusWorking, notice) + } + finalizeRetryNotice := func(progressState ProgressCardState, cardStatus CardStatus) { + if hasRichCard && cardMessageID != nil { + if updater, ok := p.(MessageUpdater); ok { + if cardStatus == "" { + cardStatus = CardStatusDone + } + statusFooter := e.composeRichStatusFooter(cardStatus != CardStatusDone && cardStatus != CardStatusError, turnStart, e.agent, state.agentSession, state.workspaceDir) + if lastRetryNotice != "" { + statusFooter = joinStatusFooterLines(lastRetryNotice, statusFooter) + } + card := buildResolvedRichCard(cardStatus, "", toolSteps, partialText, false, statusFooter) + if err := updater.UpdateMessage(e.ctx, cardMessageID, card); err != nil { + slog.Debug("rich card: failed to finalize retriable notice", "platform", p.Name(), "error", err) + } + } + } + if progressState == "" { + progressState = ProgressCardStateCompleted + } + cp.Finalize(progressState) + if lastRetryNotice != "" { + if cardStatus == "" { + cardStatus = CardStatusDone + } + sp.updateStatusFooter(cardStatus, lastRetryNotice) + } + } + retryTurn := &interactiveRetryTurn{kind: event.ErrorKind, err: event.Error} + if currentRetryTurn != nil { + *retryTurn = *currentRetryTurn + retryTurn.kind = event.ErrorKind + retryTurn.err = event.Error + } + retryTurn.notify = notifyRetry + retryTurn.finalizeNotice = finalizeRetryNotice + return retryTurn + } + cp.Finalize(ProgressCardStateFailed) + sp.discard() if hasRichCard && cardMessageID != nil { errCard := buildResolvedRichCard(CardStatusError, "", toolSteps, partialText, false, e.composeRichStatusFooter(false, turnStart, e.agent, state.agentSession, state.workspaceDir)) if updater, ok := p.(MessageUpdater); ok { @@ -6065,6 +6260,7 @@ channelClosed: } } } + return } func mergeRichToolResult(steps []ToolStep, event Event, result string, maxLen int) []ToolStep { @@ -6180,22 +6376,8 @@ func (e *Engine) drainPendingMessages(state *interactiveState, session *Session, session.AddHistory("user", queued.content) - sendDone := make(chan error, 1) - go func() { - if as == nil { - sendDone <- fmt.Errorf("agent session became nil") - return - } - sendDone <- as.Send(prompt, queued.messageID, queued.images, queued.files) - }() - - var stopTyping func() - if ti, ok := queued.platform.(TypingIndicator); ok { - stopTyping = ti.StartTyping(e.ctx, queued.replyCtx) - } - slog.Info("processing queued message", "session", sessionKey) - e.processInteractiveEvents(state, session, sessions, sessionKey, queued.messageID, time.Now(), stopTyping, sendDone, queued.replyCtx) + e.processInteractiveTurnWithRetry(state, session, sessions, sessionKey, prompt, queued.messageID, queued.images, queued.files, queued.replyCtx, time.Now(), sessionKey, len(queued.content)) } } @@ -7613,6 +7795,17 @@ func appendReplyFooter(content, footer string) string { return content + "\n\n*" + footer + "*" } +func joinStatusFooterLines(lines ...string) string { + parts := make([]string, 0, len(lines)) + for _, line := range lines { + line = strings.TrimSpace(line) + if line != "" { + parts = append(parts, line) + } + } + return strings.Join(parts, "\n") +} + func appendFinalMetadataToSegment(segment, fullResponse string) string { segment = strings.TrimRight(segment, "\n ") if segment == "" { diff --git a/core/engine_test.go b/core/engine_test.go index ce806d11d4..2623d3bfae 100644 --- a/core/engine_test.go +++ b/core/engine_test.go @@ -649,6 +649,22 @@ func waitDeleteModePhase(t *testing.T, e *Engine, sessionKey, targetPhase string t.Fatalf("timed out waiting for delete mode phase %q", targetPhase) } +// waitForRefreshedCards waits until the stub platform has observed at least +// minCount refreshes or the timeout expires. +func waitForRefreshedCards(t *testing.T, p *stubCardPlatform, minCount int) []*Card { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + refreshed := p.getRefreshedCards() + if len(refreshed) >= minCount { + return refreshed + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("timed out waiting for %d refreshed cards", minCount) + return nil +} + type stubProviderAgent struct { stubAgent providers []ProviderConfig @@ -1089,6 +1105,410 @@ func TestProcessInteractiveEvents_DoesNotSuppressDifferentFinalText(t *testing.T } } +func TestProcessInteractiveEvents_ReturnsRetriableErrorWithoutSendingRawError(t *testing.T) { + p := &stubPlatformEngine{n: "test"} + e := NewEngine("test", &stubAgent{}, []Platform{p}, "", LangEnglish) + sessionKey := "test:user1" + session := e.sessions.GetOrCreateActive(sessionKey) + agentSession := newControllableSession("s1") + state := &interactiveState{ + agentSession: agentSession, + platform: p, + replyCtx: "ctx-1", + } + e.interactiveStates[sessionKey] = state + + agentSession.events <- Event{ + Type: EventError, + Error: errors.New("Selected model is at capacity. Please try a different model."), + ErrorKind: ErrorKindOverloaded, + } + retryTurn := e.processInteractiveEvents(state, session, e.sessions, sessionKey, "m1", time.Now(), nil, nil, nil) + + if retryTurn == nil { + t.Fatal("retryTurn = nil, want retriable turn") + } + if retryTurn.kind != ErrorKindOverloaded { + t.Fatalf("retryKind = %q, want %q", retryTurn.kind, ErrorKindOverloaded) + } + if retryTurn.err == nil || !strings.Contains(retryTurn.err.Error(), "capacity") { + t.Fatalf("retryErr = %v, want capacity error", retryTurn.err) + } + if sent := p.getSent(); len(sent) != 0 { + t.Fatalf("sent = %#v, want no raw error sent before retry", sent) + } + state.mu.Lock() + needResync := state.eventsNeedResync + state.mu.Unlock() + if !needResync { + t.Fatal("eventsNeedResync = false, want true after retriable error") + } +} + +func TestProcessInteractiveTurnWithRetry_ReplaysQueuedPromptAfterOverloaded(t *testing.T) { + oldInitialDelay := RetriableErrorInitialDelay + oldRetryDelay := RetriableErrorRetryDelay + oldMaxAttempts := RetriableErrorMaxAttempts + RetriableErrorInitialDelay = time.Millisecond + RetriableErrorRetryDelay = time.Millisecond + RetriableErrorMaxAttempts = 3 + t.Cleanup(func() { + RetriableErrorInitialDelay = oldInitialDelay + RetriableErrorRetryDelay = oldRetryDelay + RetriableErrorMaxAttempts = oldMaxAttempts + }) + + p := &stubPlatformEngine{n: "test"} + e := NewEngine("test", &stubAgent{}, []Platform{p}, "", LangEnglish) + sessionKey := "test:user1" + session := e.sessions.GetOrCreateActive(sessionKey) + agentSession := newQueuedRetryAgentSession("s1") + state := &interactiveState{ + agentSession: agentSession, + platform: p, + replyCtx: "ctx-1", + pendingMessages: []queuedMessage{ + {platform: p, replyCtx: "ctx-queued", content: "queued-msg", messageID: "queued-1"}, + }, + } + e.interactiveStates[sessionKey] = state + + e.processInteractiveTurnWithRetry(state, session, e.sessions, sessionKey, "initial-msg", "m1", nil, nil, "ctx-1", time.Now(), sessionKey, len("initial-msg")) + + calls := agentSession.prompts() + if len(calls) != 3 { + t.Fatalf("prompts = %#v, want initial + queued + queued retry", calls) + } + if calls[0] != "initial-msg" { + t.Fatalf("first prompt = %q, want initial-msg", calls[0]) + } + if !strings.Contains(calls[1], "queued-msg") || !strings.Contains(calls[2], "queued-msg") { + t.Fatalf("queued prompts = %#v, want both retries to target queued-msg", calls[1:]) + } + if strings.Contains(calls[2], "initial-msg") { + t.Fatalf("retry prompt = %q, should not replay initial prompt", calls[2]) + } +} + +func TestProcessInteractiveTurnWithRetry_StopCancelsRetryDelay(t *testing.T) { + oldInitialDelay := RetriableErrorInitialDelay + oldRetryDelay := RetriableErrorRetryDelay + oldMaxAttempts := RetriableErrorMaxAttempts + RetriableErrorInitialDelay = time.Hour + RetriableErrorRetryDelay = time.Hour + RetriableErrorMaxAttempts = 3 + t.Cleanup(func() { + RetriableErrorInitialDelay = oldInitialDelay + RetriableErrorRetryDelay = oldRetryDelay + RetriableErrorMaxAttempts = oldMaxAttempts + }) + + p := &stubPlatformEngine{n: "test"} + e := NewEngine("test", &stubAgent{}, []Platform{p}, "", LangEnglish) + sessionKey := "test:user1" + session := e.sessions.GetOrCreateActive(sessionKey) + agentSession := newAlwaysRetryAgentSession("s1") + state := &interactiveState{ + agentSession: agentSession, + platform: p, + replyCtx: "ctx-1", + } + e.interactiveStates[sessionKey] = state + + done := make(chan struct{}) + go func() { + e.processInteractiveTurnWithRetry(state, session, e.sessions, sessionKey, "hello", "m1", nil, nil, "ctx-1", time.Now(), sessionKey, len("hello")) + close(done) + }() + + deadline := time.Now().Add(2 * time.Second) + for agentSession.sendCount() == 0 && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + state.markStopped() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("retry delay did not stop after state.markStopped") + } + if got := agentSession.sendCount(); got != 1 { + t.Fatalf("sendCount = %d, want no retry after stop", got) + } +} + +func TestProcessInteractiveTurnWithRetry_ReplaysPromptAfterOverloaded(t *testing.T) { + oldInitialDelay := RetriableErrorInitialDelay + oldRetryDelay := RetriableErrorRetryDelay + oldMaxAttempts := RetriableErrorMaxAttempts + RetriableErrorInitialDelay = time.Millisecond + RetriableErrorRetryDelay = time.Millisecond + RetriableErrorMaxAttempts = 2 + t.Cleanup(func() { + RetriableErrorInitialDelay = oldInitialDelay + RetriableErrorRetryDelay = oldRetryDelay + RetriableErrorMaxAttempts = oldMaxAttempts + }) + + p := &stubPlatformEngine{n: "test"} + e := NewEngine("test", &stubAgent{}, []Platform{p}, "", LangEnglish) + sessionKey := "test:user1" + session := e.sessions.GetOrCreateActive(sessionKey) + agentSession := newRetryOnceAgentSession("s1") + state := &interactiveState{ + agentSession: agentSession, + platform: p, + replyCtx: "ctx-1", + eventsNeedResync: true, + } + e.interactiveStates[sessionKey] = state + + e.processInteractiveTurnWithRetry(state, session, e.sessions, sessionKey, "hello", "m1", nil, nil, "ctx-1", time.Now(), sessionKey, len("hello")) + + if got := agentSession.sendCount(); got != 2 { + t.Fatalf("sendCount = %d, want 2", got) + } + sent := p.getSent() + if len(sent) != 2 { + t.Fatalf("sent = %#v, want retry notice and final response", sent) + } + if !strings.Contains(sent[0], "Retrying") { + t.Fatalf("retry notice = %q, want English retry notice", sent[0]) + } + if sent[1] != "ok after retry" { + t.Fatalf("final response = %q, want ok after retry", sent[1]) + } +} + +func TestProcessInteractiveTurnWithRetry_RichCardNoticeUpdatesCard(t *testing.T) { + oldInitialDelay := RetriableErrorInitialDelay + oldRetryDelay := RetriableErrorRetryDelay + oldMaxAttempts := RetriableErrorMaxAttempts + RetriableErrorInitialDelay = time.Millisecond + RetriableErrorRetryDelay = time.Millisecond + RetriableErrorMaxAttempts = 2 + t.Cleanup(func() { + RetriableErrorInitialDelay = oldInitialDelay + RetriableErrorRetryDelay = oldRetryDelay + RetriableErrorMaxAttempts = oldMaxAttempts + }) + + p := &stubCompactProgressPlatform{ + stubPlatformEngine: stubPlatformEngine{n: "feishu"}, + style: "card", + supportPayload: true, + } + e := NewEngine("test", &stubAgent{}, []Platform{p}, "", LangEnglish) + e.SetDisplayConfig(DisplayCfg{ + ThinkingMessages: true, + ThinkingMaxLen: 300, + ToolMaxLen: 500, + ToolMessages: true, + Mode: "full", + CardMode: "rich", + }) + sessionKey := "feishu:user-rich-retry" + session := e.sessions.GetOrCreateActive(sessionKey) + agentSession := newRichCardRetryOnceAgentSession("s-rich-retry") + state := &interactiveState{ + agentSession: agentSession, + platform: p, + replyCtx: "ctx-rich-retry", + } + e.interactiveStates[sessionKey] = state + + e.processInteractiveTurnWithRetry(state, session, e.sessions, sessionKey, "hello", "m-rich-retry", nil, nil, "ctx-rich-retry", time.Now(), sessionKey, len("hello")) + + if got := agentSession.sendCount(); got != 2 { + t.Fatalf("sendCount = %d, want 2", got) + } + for _, sent := range p.getSent() { + if strings.Contains(sent, "Retrying") || strings.Contains(sent, "rate-limited") { + t.Fatalf("retry notice was sent as standalone message: %#v", p.getSent()) + } + } + rendered := strings.Join(append(p.getPreviewStarts(), p.getPreviewEdits()...), "\n") + if !strings.Contains(rendered, "Retrying in") || !strings.Contains(rendered, "attempt 2/2") { + t.Fatalf("rich card updates should contain retry notice, got %q", rendered) + } + if !strings.Contains(rendered, "rich status=done") { + t.Fatalf("rich card retry notice should be finalized before replay, got %q", rendered) + } + if !strings.Contains(rendered, "ok after retry") { + t.Fatalf("rich card updates should contain final response, got %q", rendered) + } +} + +func TestProcessInteractiveTurnWithRetry_ProgressCardNoticeUpdatesCard(t *testing.T) { + oldInitialDelay := RetriableErrorInitialDelay + oldRetryDelay := RetriableErrorRetryDelay + oldMaxAttempts := RetriableErrorMaxAttempts + RetriableErrorInitialDelay = time.Millisecond + RetriableErrorRetryDelay = time.Millisecond + RetriableErrorMaxAttempts = 2 + t.Cleanup(func() { + RetriableErrorInitialDelay = oldInitialDelay + RetriableErrorRetryDelay = oldRetryDelay + RetriableErrorMaxAttempts = oldMaxAttempts + }) + + p := &stubCompactProgressPlatform{ + stubPlatformEngine: stubPlatformEngine{n: "feishu"}, + style: "card", + supportPayload: true, + } + e := NewEngine("test", &stubAgent{}, []Platform{p}, "", LangEnglish) + sessionKey := "feishu:user-progress-card-retry" + session := e.sessions.GetOrCreateActive(sessionKey) + agentSession := newRichCardRetryOnceAgentSession("s-progress-card-retry") + state := &interactiveState{ + agentSession: agentSession, + platform: p, + replyCtx: "ctx-progress-card-retry", + } + e.interactiveStates[sessionKey] = state + + e.processInteractiveTurnWithRetry(state, session, e.sessions, sessionKey, "hello", "m-progress-card-retry", nil, nil, "ctx-progress-card-retry", time.Now(), sessionKey, len("hello")) + + for _, sent := range p.getSent() { + if strings.Contains(sent, "Retrying") || strings.Contains(sent, "rate-limited") { + t.Fatalf("retry notice was sent as standalone message: %#v", p.getSent()) + } + } + edits := p.getPreviewEdits() + if len(edits) == 0 { + t.Fatal("preview edits = 0, want retry notice to update progress card") + } + var sawRetry bool + var sawCompleted bool + for _, edit := range edits { + payload, ok := ParseProgressCardPayload(edit) + if !ok { + continue + } + if payload.State == ProgressCardStateCompleted { + sawCompleted = true + } + for _, item := range payload.Items { + if item.Kind == ProgressEntryInfo && strings.Contains(item.Text, "Retrying in") && strings.Contains(item.Text, "attempt 2/2") { + sawRetry = true + } + } + } + if !sawRetry { + t.Fatalf("progress card edits should contain retry info item, got %#v", edits) + } + if !sawCompleted { + t.Fatalf("progress card retry notice should be finalized before replay, got %#v", edits) + } +} + +func TestProcessInteractiveTurnWithRetry_ProgressCardRetryExhaustionFinalizesFailed(t *testing.T) { + oldInitialDelay := RetriableErrorInitialDelay + oldRetryDelay := RetriableErrorRetryDelay + oldMaxAttempts := RetriableErrorMaxAttempts + RetriableErrorInitialDelay = time.Millisecond + RetriableErrorRetryDelay = time.Millisecond + RetriableErrorMaxAttempts = 1 + t.Cleanup(func() { + RetriableErrorInitialDelay = oldInitialDelay + RetriableErrorRetryDelay = oldRetryDelay + RetriableErrorMaxAttempts = oldMaxAttempts + }) + + p := &stubCompactProgressPlatform{ + stubPlatformEngine: stubPlatformEngine{n: "feishu"}, + style: "card", + supportPayload: true, + } + e := NewEngine("test", &stubAgent{}, []Platform{p}, "", LangEnglish) + sessionKey := "feishu:user-progress-card-retry-exhausted" + session := e.sessions.GetOrCreateActive(sessionKey) + agentSession := newRichCardAlwaysRetryAgentSession("s-progress-card-retry-exhausted") + state := &interactiveState{ + agentSession: agentSession, + platform: p, + replyCtx: "ctx-progress-card-retry-exhausted", + } + e.interactiveStates[sessionKey] = state + + e.processInteractiveTurnWithRetry(state, session, e.sessions, sessionKey, "hello", "m-progress-card-retry-exhausted", nil, nil, "ctx-progress-card-retry-exhausted", time.Now(), sessionKey, len("hello")) + + edits := p.getPreviewEdits() + var sawFailed bool + for _, edit := range edits { + payload, ok := ParseProgressCardPayload(edit) + if !ok { + continue + } + if payload.State == ProgressCardStateFailed { + sawFailed = true + } + } + if !sawFailed { + t.Fatalf("exhausted retry progress card should be finalized failed, edits=%#v", edits) + } +} + +func TestProcessInteractiveTurnWithRetry_ProgressCardRetryNoticeBypassesThrottle(t *testing.T) { + oldInitialDelay := RetriableErrorInitialDelay + oldRetryDelay := RetriableErrorRetryDelay + oldMaxAttempts := RetriableErrorMaxAttempts + RetriableErrorInitialDelay = time.Millisecond + RetriableErrorRetryDelay = time.Millisecond + RetriableErrorMaxAttempts = 2 + t.Cleanup(func() { + RetriableErrorInitialDelay = oldInitialDelay + RetriableErrorRetryDelay = oldRetryDelay + RetriableErrorMaxAttempts = oldMaxAttempts + }) + + p := &stubThrottledProgressPlatform{ + stubCompactProgressPlatform: stubCompactProgressPlatform{ + stubPlatformEngine: stubPlatformEngine{n: "discord"}, + style: "card", + supportPayload: true, + }, + throttle: time.Hour, + } + e := NewEngine("test", &stubAgent{}, []Platform{p}, "", LangEnglish) + sessionKey := "discord:user-progress-card-retry" + session := e.sessions.GetOrCreateActive(sessionKey) + agentSession := newRichCardRetryOnceAgentSession("s-progress-card-retry-throttle") + state := &interactiveState{ + agentSession: agentSession, + platform: p, + replyCtx: "ctx-progress-card-retry-throttle", + } + e.interactiveStates[sessionKey] = state + + e.processInteractiveTurnWithRetry(state, session, e.sessions, sessionKey, "hello", "m-progress-card-retry-throttle", nil, nil, "ctx-progress-card-retry-throttle", time.Now(), sessionKey, len("hello")) + + edits := p.getPreviewEdits() + var sawRetry bool + var sawCompleted bool + for _, edit := range edits { + payload, ok := ParseProgressCardPayload(edit) + if !ok { + continue + } + if payload.State == ProgressCardStateCompleted { + sawCompleted = true + } + for _, item := range payload.Items { + if item.Kind == ProgressEntryInfo && strings.Contains(item.Text, "Retrying in") { + sawRetry = true + } + } + } + if !sawRetry { + t.Fatalf("retry notice should bypass progress edit throttle, edits=%#v", edits) + } + if !sawCompleted { + t.Fatalf("retry notice should finalize progress card before replay, edits=%#v", edits) + } +} + func TestProcessInteractiveEvents_StripsAgentFooterWhenEnabled(t *testing.T) { p := &stubPlatformEngine{n: "telegram"} e := NewEngine("test", &stubAgent{}, []Platform{p}, "", LangEnglish) @@ -4141,10 +4561,7 @@ func TestDeleteMode_ConfirmAndSubmitDeletesSelectedSessions(t *testing.T) { if got, want := strings.Join(agent.deleted, ","), "session-1,session-3"; got != want { t.Fatalf("deleted = %q, want %q", got, want) } - refreshed := p.getRefreshedCards() - if len(refreshed) == 0 { - t.Fatal("expected refreshed result card via RefreshCard") - } + refreshed := waitForRefreshedCards(t, p, 1) pushedCard := refreshed[len(refreshed)-1] if !strings.Contains(pushedCard.RenderText(), "Session deleted: One") { t.Fatalf("result text = %q, want delete result", pushedCard.RenderText()) @@ -4176,10 +4593,7 @@ func TestDeleteMode_SubmitReportsMissingSelectedSessions(t *testing.T) { } // Wait for async deletion to complete. waitDeleteModePhase(t, e, msg.SessionKey, "result") - refreshed := p.getRefreshedCards() - if len(refreshed) == 0 { - t.Fatal("expected refreshed result card via RefreshCard") - } + refreshed := waitForRefreshedCards(t, p, 1) pushedCard := refreshed[len(refreshed)-1] resultText := pushedCard.RenderText() if !strings.Contains(resultText, "Session deleted: One") { @@ -4281,10 +4695,8 @@ func TestDeleteMode_SubmitBlocksActiveSession(t *testing.T) { if len(agent.deleted) != 0 { t.Fatalf("deleted = %v, want none", agent.deleted) } - if len(p.getRefreshedCards()) == 0 { - t.Fatal("expected refreshed result card via RefreshCard") - } - pushedCard := p.getRefreshedCards()[len(p.getRefreshedCards())-1] + refreshed := waitForRefreshedCards(t, p, 1) + pushedCard := refreshed[len(refreshed)-1] if !strings.Contains(pushedCard.RenderText(), "Cannot delete the currently active session") { t.Fatalf("result text = %q, want active-session warning", pushedCard.RenderText()) } @@ -4357,10 +4769,7 @@ func TestDeleteMode_FormSubmitShowsConfirmThenDeletes(t *testing.T) { if got, want := strings.Join(agent.deleted, ","), "session-1,session-3"; got != want { t.Fatalf("deleted = %q, want %q", got, want) } - refreshed := p.getRefreshedCards() - if len(refreshed) == 0 { - t.Fatal("expected pushed result card via RefreshCard") - } + refreshed := waitForRefreshedCards(t, p, 1) pushedCard := refreshed[len(refreshed)-1] if !strings.Contains(pushedCard.RenderText(), "Session deleted: One") { t.Fatalf("result text = %q, want delete result", pushedCard.RenderText()) @@ -7166,6 +7575,236 @@ type controllableAgentSession struct { usageErr error } +type retryOnceAgentSession struct { + sessionID string + events chan Event + sends int + mu sync.Mutex +} + +type queuedRetryAgentSession struct { + sessionID string + events chan Event + promptList []string + queuedSeen int + mu sync.Mutex +} + +type alwaysRetryAgentSession struct { + sessionID string + events chan Event + sends int + mu sync.Mutex +} + +type richCardRetryOnceAgentSession struct { + sessionID string + events chan Event + sends int + mu sync.Mutex +} + +type richCardAlwaysRetryAgentSession struct { + sessionID string + events chan Event + sends int + mu sync.Mutex +} + +func newRetryOnceAgentSession(id string) *retryOnceAgentSession { + return &retryOnceAgentSession{ + sessionID: id, + events: make(chan Event, 8), + } +} + +func newQueuedRetryAgentSession(id string) *queuedRetryAgentSession { + return &queuedRetryAgentSession{ + sessionID: id, + events: make(chan Event, 8), + } +} + +func newAlwaysRetryAgentSession(id string) *alwaysRetryAgentSession { + return &alwaysRetryAgentSession{ + sessionID: id, + events: make(chan Event, 8), + } +} + +func newRichCardRetryOnceAgentSession(id string) *richCardRetryOnceAgentSession { + return &richCardRetryOnceAgentSession{ + sessionID: id, + events: make(chan Event, 8), + } +} + +func newRichCardAlwaysRetryAgentSession(id string) *richCardAlwaysRetryAgentSession { + return &richCardAlwaysRetryAgentSession{ + sessionID: id, + events: make(chan Event, 8), + } +} + +func (s *retryOnceAgentSession) Send(_ string, _ string, _ []ImageAttachment, _ []FileAttachment) error { + s.mu.Lock() + s.sends++ + sendNo := s.sends + s.mu.Unlock() + if sendNo == 1 { + s.events <- Event{ + Type: EventError, + Error: errors.New("Selected model is at capacity. Please try a different model."), + ErrorKind: ErrorKindOverloaded, + } + return nil + } + s.events <- Event{Type: EventResult, Content: "ok after retry", Done: true} + return nil +} + +func (s *retryOnceAgentSession) RespondPermission(_ string, _ PermissionResult) error { return nil } +func (s *retryOnceAgentSession) Events() <-chan Event { return s.events } +func (s *retryOnceAgentSession) CurrentSessionID() string { return s.sessionID } +func (s *retryOnceAgentSession) Alive() bool { return true } +func (s *retryOnceAgentSession) Close() error { + close(s.events) + return nil +} + +func (s *retryOnceAgentSession) sendCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.sends +} + +func (s *queuedRetryAgentSession) Send(prompt string, _ string, _ []ImageAttachment, _ []FileAttachment) error { + s.mu.Lock() + s.promptList = append(s.promptList, prompt) + isQueued := strings.Contains(prompt, "queued-msg") + if isQueued { + s.queuedSeen++ + } + queuedSeen := s.queuedSeen + s.mu.Unlock() + + if !isQueued { + s.events <- Event{Type: EventResult, Content: "initial ok", Done: true} + return nil + } + if queuedSeen == 1 { + s.events <- Event{ + Type: EventError, + Error: errors.New("Selected model is at capacity. Please try a different model."), + ErrorKind: ErrorKindOverloaded, + } + return nil + } + s.events <- Event{Type: EventResult, Content: "queued ok", Done: true} + return nil +} + +func (s *queuedRetryAgentSession) RespondPermission(_ string, _ PermissionResult) error { return nil } +func (s *queuedRetryAgentSession) Events() <-chan Event { return s.events } +func (s *queuedRetryAgentSession) CurrentSessionID() string { return s.sessionID } +func (s *queuedRetryAgentSession) Alive() bool { return true } +func (s *queuedRetryAgentSession) Close() error { + close(s.events) + return nil +} + +func (s *queuedRetryAgentSession) prompts() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.promptList...) +} + +func (s *alwaysRetryAgentSession) Send(_ string, _ string, _ []ImageAttachment, _ []FileAttachment) error { + s.mu.Lock() + s.sends++ + s.mu.Unlock() + s.events <- Event{ + Type: EventError, + Error: errors.New("Selected model is at capacity. Please try a different model."), + ErrorKind: ErrorKindOverloaded, + } + return nil +} + +func (s *alwaysRetryAgentSession) RespondPermission(_ string, _ PermissionResult) error { return nil } +func (s *alwaysRetryAgentSession) Events() <-chan Event { return s.events } +func (s *alwaysRetryAgentSession) CurrentSessionID() string { return s.sessionID } +func (s *alwaysRetryAgentSession) Alive() bool { return true } +func (s *alwaysRetryAgentSession) Close() error { + close(s.events) + return nil +} + +func (s *alwaysRetryAgentSession) sendCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.sends +} + +func (s *richCardRetryOnceAgentSession) Send(_ string, _ string, _ []ImageAttachment, _ []FileAttachment) error { + s.mu.Lock() + s.sends++ + sendNo := s.sends + s.mu.Unlock() + if sendNo == 1 { + s.events <- Event{Type: EventThinking, Content: "Inspecting retry path"} + s.events <- Event{ + Type: EventError, + Error: errors.New("Selected model is at capacity. Please try a different model."), + ErrorKind: ErrorKindOverloaded, + } + return nil + } + s.events <- Event{Type: EventText, Content: "ok after retry"} + s.events <- Event{Type: EventResult, Content: "ok after retry", Done: true} + return nil +} + +func (s *richCardRetryOnceAgentSession) RespondPermission(_ string, _ PermissionResult) error { + return nil +} +func (s *richCardRetryOnceAgentSession) Events() <-chan Event { return s.events } +func (s *richCardRetryOnceAgentSession) CurrentSessionID() string { return s.sessionID } +func (s *richCardRetryOnceAgentSession) Alive() bool { return true } +func (s *richCardRetryOnceAgentSession) Close() error { + close(s.events) + return nil +} +func (s *richCardRetryOnceAgentSession) sendCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.sends +} + +func (s *richCardAlwaysRetryAgentSession) Send(_ string, _ string, _ []ImageAttachment, _ []FileAttachment) error { + s.mu.Lock() + s.sends++ + s.mu.Unlock() + s.events <- Event{Type: EventThinking, Content: "Inspecting retry path"} + s.events <- Event{ + Type: EventError, + Error: errors.New("Selected model is at capacity. Please try a different model."), + ErrorKind: ErrorKindOverloaded, + } + return nil +} + +func (s *richCardAlwaysRetryAgentSession) RespondPermission(_ string, _ PermissionResult) error { + return nil +} +func (s *richCardAlwaysRetryAgentSession) Events() <-chan Event { return s.events } +func (s *richCardAlwaysRetryAgentSession) CurrentSessionID() string { return s.sessionID } +func (s *richCardAlwaysRetryAgentSession) Alive() bool { return true } +func (s *richCardAlwaysRetryAgentSession) Close() error { + close(s.events) + return nil +} + func newControllableSession(id string) *controllableAgentSession { return &controllableAgentSession{ sessionID: id, diff --git a/core/i18n.go b/core/i18n.go index 410588a041..982b88b6bf 100644 --- a/core/i18n.go +++ b/core/i18n.go @@ -181,6 +181,7 @@ const ( MsgToolAllowFailed MsgKey = "tool_allow_failed" MsgToolAllowedNew MsgKey = "tool_allowed_new" MsgError MsgKey = "error" + MsgRetriableAgentError MsgKey = "retriable_agent_error" MsgSessionNotFound MsgKey = "session_not_found" MsgFailedToStartAgentSession MsgKey = "failed_to_start_agent_session" MsgFailedToDeleteSession MsgKey = "failed_to_delete_session" @@ -378,31 +379,31 @@ const ( MsgCronIDLabel MsgKey = "cron_id_label" MsgCronFailedSuffix MsgKey = "cron_failed_suffix" - MsgTimerNotAvailable MsgKey = "timer_not_available" - MsgTimerUsage MsgKey = "timer_usage" - MsgTimerAddUsage MsgKey = "timer_add_usage" - MsgTimerAdded MsgKey = "timer_added" - MsgTimerAddedExec MsgKey = "timer_added_exec" - MsgTimerAddExecUsage MsgKey = "timer_addexec_usage" - MsgTimerEmpty MsgKey = "timer_empty" - MsgTimerListTitle MsgKey = "timer_list_title" - MsgTimerListFooter MsgKey = "timer_list_footer" - MsgTimerDelUsage MsgKey = "timer_del_usage" - MsgTimerMuteUsage MsgKey = "timer_mute_usage" - MsgTimerDeleted MsgKey = "timer_deleted" - MsgTimerNotFound MsgKey = "timer_not_found" - MsgTimerMuted MsgKey = "timer_muted" - MsgTimerUnmuted MsgKey = "timer_unmuted" - MsgTimerCardHint MsgKey = "timer_card_hint" - MsgTimerBtnMute MsgKey = "timer_btn_mute" - MsgTimerBtnUnmute MsgKey = "timer_btn_unmute" - MsgTimerBtnDelete MsgKey = "timer_btn_delete" - MsgTimerIDLabel MsgKey = "timer_id_label" - MsgTimerScheduledLabel MsgKey = "timer_scheduled_label" - MsgTimerFailedSuffix MsgKey = "timer_failed_suffix" - MsgCommandsTagAgent MsgKey = "commands_tag_agent" - MsgCommandsTagShell MsgKey = "commands_tag_shell" - MsgUpgradeTimeoutSuffix MsgKey = "upgrade_timeout_suffix" + MsgTimerNotAvailable MsgKey = "timer_not_available" + MsgTimerUsage MsgKey = "timer_usage" + MsgTimerAddUsage MsgKey = "timer_add_usage" + MsgTimerAdded MsgKey = "timer_added" + MsgTimerAddedExec MsgKey = "timer_added_exec" + MsgTimerAddExecUsage MsgKey = "timer_addexec_usage" + MsgTimerEmpty MsgKey = "timer_empty" + MsgTimerListTitle MsgKey = "timer_list_title" + MsgTimerListFooter MsgKey = "timer_list_footer" + MsgTimerDelUsage MsgKey = "timer_del_usage" + MsgTimerMuteUsage MsgKey = "timer_mute_usage" + MsgTimerDeleted MsgKey = "timer_deleted" + MsgTimerNotFound MsgKey = "timer_not_found" + MsgTimerMuted MsgKey = "timer_muted" + MsgTimerUnmuted MsgKey = "timer_unmuted" + MsgTimerCardHint MsgKey = "timer_card_hint" + MsgTimerBtnMute MsgKey = "timer_btn_mute" + MsgTimerBtnUnmute MsgKey = "timer_btn_unmute" + MsgTimerBtnDelete MsgKey = "timer_btn_delete" + MsgTimerIDLabel MsgKey = "timer_id_label" + MsgTimerScheduledLabel MsgKey = "timer_scheduled_label" + MsgTimerFailedSuffix MsgKey = "timer_failed_suffix" + MsgCommandsTagAgent MsgKey = "commands_tag_agent" + MsgCommandsTagShell MsgKey = "commands_tag_shell" + MsgUpgradeTimeoutSuffix MsgKey = "upgrade_timeout_suffix" MsgCronScheduleLabel MsgKey = "cron_schedule_label" MsgCronNextRunLabel MsgKey = "cron_next_run_label" @@ -800,6 +801,13 @@ var messages = map[MsgKey]map[Language]string{ LangJapanese: "❌ エラー: %v", LangSpanish: "❌ Error: %v", }, + MsgRetriableAgentError: { + LangEnglish: "⚠️ Upstream model is busy or rate-limited. Retrying in %s (attempt %d/%d).", + LangChinese: "⚠️ 上游模型繁忙或限流,将在 %s 后自动重试(第 %d/%d 次)。", + LangTraditionalChinese: "⚠️ 上游模型繁忙或限流,將在 %s 後自動重試(第 %d/%d 次)。", + LangJapanese: "⚠️ 上流モデルが混雑またはレート制限中です。%s 後に自動再試行します(%d/%d 回目)。", + LangSpanish: "⚠️ El modelo upstream está ocupado o limitado. Reintentando en %s (intento %d/%d).", + }, MsgBackgroundAutoDenied: { LangEnglish: "⚠️ Background task requested permission for `%s` but was auto-denied (no active user turn). Send a message or use `/yolo` to approve future requests.", LangChinese: "⚠️ 后台任务请求使用工具 `%s` 的权限,但已自动拒绝(当前无活跃会话)。请发送消息或使用 `/yolo` 授权后续请求。", diff --git a/core/message.go b/core/message.go index e9614723d0..b3341fdbd7 100644 --- a/core/message.go +++ b/core/message.go @@ -391,6 +391,24 @@ const ( EventThinking EventType = "thinking" // thinking/processing status ) +// ErrorKind classifies agent errors that need special handling by the engine. +type ErrorKind string + +const ( + ErrorKindUnknown ErrorKind = "" + ErrorKindRateLimit ErrorKind = "rate_limit" + ErrorKindOverloaded ErrorKind = "overloaded" +) + +func (k ErrorKind) IsRetriable() bool { + switch k { + case ErrorKindRateLimit, ErrorKindOverloaded: + return true + default: + return false + } +} + // UserQuestion represents a structured question from AskUserQuestion. type UserQuestion struct { Question string `json:"question"` @@ -421,6 +439,7 @@ type Event struct { Questions []UserQuestion // populated when ToolName == "AskUserQuestion" Done bool Error error + ErrorKind ErrorKind InputTokens int // token usage from agent result events OutputTokens int CacheCreationInputTokens int // cache-write tokens (new content written to cache) diff --git a/core/progress_compact.go b/core/progress_compact.go index d2d7a821cd..12bcbe1b74 100644 --- a/core/progress_compact.go +++ b/core/progress_compact.go @@ -378,6 +378,15 @@ func (w *compactProgressWriter) AppendEvent(kind ProgressCardEntryKind, text str // AppendStructured appends one structured progress event and updates the in-place message. func (w *compactProgressWriter) AppendStructured(item ProgressCardEntry, fallback string) bool { + return w.appendStructured(item, fallback, false) +} + +// AppendStructuredImmediate appends an event and forces an immediate edit. +func (w *compactProgressWriter) AppendStructuredImmediate(item ProgressCardEntry, fallback string) bool { + return w.appendStructured(item, fallback, true) +} + +func (w *compactProgressWriter) appendStructured(item ProgressCardEntry, fallback string, forceUpdate bool) bool { if !w.enabled || w.failed { return false } @@ -477,7 +486,7 @@ func (w *compactProgressWriter) AppendStructured(item ProgressCardEntry, fallbac return true } - if w.minUpdateInterval > 0 && time.Since(w.lastUpdateAt) < w.minUpdateInterval { + if !forceUpdate && w.minUpdateInterval > 0 && time.Since(w.lastUpdateAt) < w.minUpdateInterval { return true } diff --git a/core/retriable_error.go b/core/retriable_error.go new file mode 100644 index 0000000000..96b84724f0 --- /dev/null +++ b/core/retriable_error.go @@ -0,0 +1,16 @@ +package core + +import "time" + +var ( + RetriableErrorInitialDelay = 30 * time.Second + RetriableErrorRetryDelay = 60 * time.Second + RetriableErrorMaxAttempts = 30 +) + +func RetriableErrorDelay(attempt int) time.Duration { + if attempt <= 1 { + return RetriableErrorInitialDelay + } + return RetriableErrorRetryDelay +} diff --git a/core/streaming.go b/core/streaming.go index 8c4aa1adc7..850dd15f9d 100644 --- a/core/streaming.go +++ b/core/streaming.go @@ -536,6 +536,58 @@ func (sp *streamPreview) setStatus(status CardStatus) { } } +func (sp *streamPreview) updateStatusFooter(status CardStatus, statusFooter string) bool { + statusFooter = strings.TrimSpace(statusFooter) + if statusFooter == "" { + return false + } + + sp.mu.Lock() + defer sp.mu.Unlock() + + sp.pendingStatus = status + if sp.previewMsgID == nil || sp.degraded { + return false + } + + body := sp.fullText + maxChars := sp.cfg.MaxChars + if maxChars > 0 && len([]rune(body)) > maxChars { + body = string([]rune(body)[:maxChars]) + "…" + } + if sp.transform != nil { + body = sp.transform(body) + } + + if sfu, ok := sp.platform.(StatusFooterUpdater); ok { + if err := sfu.UpdateMessageWithStatusFooter(sp.ctx, sp.previewMsgID, body, statusFooter); err == nil { + if statusUpdater, ok := sp.platform.(PreviewStatusUpdater); ok { + statusUpdater.SetPreviewStatus(sp.previewMsgID, status) + } + return true + } else { + slog.Debug("stream preview status footer update failed, falling back", "error", err) + } + } + + updater, ok := sp.platform.(MessageUpdater) + if !ok { + return false + } + content := appendReplyFooter(body, statusFooter) + if err := updater.UpdateMessage(sp.ctx, sp.previewMsgID, content); err != nil { + slog.Debug("stream preview inline status footer update failed", "error", err) + return false + } + sp.lastSentText = content + sp.lastSentViaUpdate = true + sp.lastSentAt = time.Now() + if statusUpdater, ok := sp.platform.(PreviewStatusUpdater); ok { + statusUpdater.SetPreviewStatus(sp.previewMsgID, status) + } + return true +} + // detachPreview clears the preview message handle so that finish() won't // delete it. Call this after freeze() when the frozen preview should remain // visible as a permanent message (e.g. text before the first tool call).