fix(sdks): windowed degraded detection for pool backoff

docs/guides/client-pool.md

@@ -49,10 +49,30 @@ because it is the only part of the pool that is gated by a distributed lock:
 ### Lifecycle model
 
 Each pool instance moves through `NOT_STARTED → STARTING → RUNNING → DRAINING → STOPPED`.
-Health is tracked separately as `HEALTHY | DEGRADED | DRAINING | STOPPED`; after
-`degraded_threshold` consecutive create failures the pool enters `DEGRADED` and applies
-exponential backoff before retrying warmup. Callers do not need to observe these states
-directly — `snapshot()` exposes them for diagnostics.
+Health is tracked separately as `HEALTHY | DEGRADED | DRAINING | STOPPED`. Create failures
+are counted inside a sliding time window (`failure_window`): once `degraded_threshold`
+failures fall inside the window, the pool enters `DEGRADED` and applies exponential backoff
+before retrying warmup. Detection is rate-based — a successful create never resets the
+window, so a pool with a sustained high failure rate stays paused even when successes are
+interleaved. The pool returns to `HEALTHY` only once the window drains and no backoff is
+active. Callers do not need to observe these states directly — `snapshot()` exposes them
+for diagnostics.
+
+::: warning Behavior change from the next release
+Starting with the next release after Kotlin `java/sandbox` v1.0.18, Go `sdks/sandbox/go`
+v1.0.5, and Python `opensandbox` v0.1.15, degraded detection changes from **consecutive**
+failures (any success reset the count) to **rate-based** detection over the sliding
+`failure_window` (default `60 s`). Concretely:
+
+- A successful warmup no longer resets the failure count or cancels an active backoff.
+- While `DEGRADED`, an expired backoff window is renewed automatically as long as the
+  failure window is still hot — the pool stays paused until the failure rate actually
+  drops below the threshold, instead of retrying every backoff step.
+- Recovery is time-based: the pool returns to `HEALTHY` when the failure window drains.
+- The new `failure_window` knob (default `60 s`) is added to all three SDKs; the existing
+  `degraded_threshold` semantics change from "consecutive failures" to "failures inside
+  the window".
+:::
 
 ![Client pool lifecycle state machine](/images/client-pool-lifecycle.svg)
 
@@ -97,7 +117,8 @@ canonical reference; refer to the per-language builder or constructor for exact
 | `warmup_concurrency`                  | `max(1, ceil(max_idle * 0.2))`     | Warmup worker pool size                                                          |
 | `primary_lock_ttl`                    | `60 s`                             | Leader-lock TTL; must exceed `warmup_ready_timeout` + preparer time              |
 | `reconcile_interval`                  | `30 s`                             | Interval between reconcile ticks                                                 |
-| `degraded_threshold`                  | `3`                                | Consecutive create failures before entering `DEGRADED` with backoff              |
+| `degraded_threshold`                  | `3`                                | Create failures inside `failure_window` before entering `DEGRADED` with backoff |
+| `failure_window`                      | `60 s`                             | Sliding time window over which create failures are counted for degraded detection |
 | `acquire_ready_timeout`               | `30 s`                             | Max wait for the returned sandbox to become ready                                |
 | `acquire_health_check_polling_interval` | `200 ms`                         | Ready-poll interval during acquire                                               |
 | `acquire_health_check`                | `null`                             | Custom readiness predicate for acquire                                           |
@@ -277,7 +298,7 @@ _ = result
 Every SDK exposes read-only accessors:
 
 - `snapshot()` — pool phase, health, counters (idle size, in-flight warmups,
-  consecutive failures, last error).
+  failures inside the sliding window, last error).
 - `snapshot_idle_entries()` — the current idle sandbox IDs with expiry timestamps.
 - `resize(max_idle)` — change the target buffer size at runtime.
 - `release_all_idle()` — drain the currently visible idle buffer and best-effort kill

sdks/sandbox/go/pool.go

@@ -127,6 +127,9 @@ func (p *DefaultSandboxPool) Start(ctx context.Context) error {
 			"warmup_ready_timeout", p.config.WarmupReadyTimeout)
 	}
 	p.reconciler = newReconcileState(p.config.DegradedThreshold)
+	if p.config.FailureWindow > 0 {
+		p.reconciler.failureWindow = p.config.FailureWindow
+	}
 	p.ticker = time.NewTicker(p.config.ReconcileInterval)
 	p.done = make(chan struct{})
 	p.doneClosed = false

sdks/sandbox/go/pool_builder.go

@@ -37,6 +37,7 @@ func NewSandboxPoolBuilder() *SandboxPoolBuilder {
 			PrimaryLockTTL:                    60 * time.Second,
 			ReconcileInterval:                 30 * time.Second,
 			DegradedThreshold:                 3,
+			FailureWindow:                     60 * time.Second,
 			AcquireReadyTimeout:               30 * time.Second,
 			WarmupReadyTimeout:                30 * time.Second,
 			AcquireHealthCheckPollingInterval: 200 * time.Millisecond,
@@ -104,13 +105,20 @@ func (b *SandboxPoolBuilder) PrimaryLockTTL(d time.Duration) *SandboxPoolBuilder
 	return b
 }
 
-// DegradedThreshold sets the number of consecutive failures before the pool
-// is considered degraded.
+// DegradedThreshold sets the number of failures inside the failure window
+// before the pool is considered degraded.
 func (b *SandboxPoolBuilder) DegradedThreshold(n int) *SandboxPoolBuilder {
 	b.config.DegradedThreshold = n
 	return b
 }
 
+// FailureWindow sets the sliding time window over which create failures are
+// counted for degraded detection. Must be positive. Default: 60s.
+func (b *SandboxPoolBuilder) FailureWindow(d time.Duration) *SandboxPoolBuilder {
+	b.config.FailureWindow = d
+	return b
+}
+
 // AcquireReadyTimeout sets the timeout for health checks during Acquire.
 func (b *SandboxPoolBuilder) AcquireReadyTimeout(d time.Duration) *SandboxPoolBuilder {
 	b.config.AcquireReadyTimeout = d
@@ -228,6 +236,9 @@ func (b *SandboxPoolBuilder) Build() (*DefaultSandboxPool, error) {
 	if b.config.DegradedThreshold <= 0 {
 		return nil, fmt.Errorf("opensandbox: pool builder: DegradedThreshold must be positive")
 	}
+	if b.config.FailureWindow <= 0 {
+		return nil, fmt.Errorf("opensandbox: pool builder: FailureWindow must be positive")
+	}
 	if b.config.SandboxCreator == nil && b.config.CreationSpec.Image == "" && b.config.CreationSpec.SnapshotID == "" {
 		return nil, fmt.Errorf("opensandbox: pool builder: CreationSpec (with Image or SnapshotID) is required when no SandboxCreator is set")
 	}

sdks/sandbox/go/pool_reconciler.go

@@ -24,88 +24,161 @@ import (
 const (
 	reconcileBackoffBase = 30 * time.Second
 	reconcileMaxBackoff  = 24 * time.Hour
+	// defaultFailureWindow is the default sliding time window over which create
+	// failures are counted for degraded detection.
+	defaultFailureWindow = 60 * time.Second
 )
 
 // reconcileState tracks the health and backoff state of the reconcile loop.
+//
+// Degradation is detected by failure *rate* rather than consecutive failures: every failure is
+// recorded with a timestamp and ages out of failureWindow. The pool enters PoolDegraded and opens
+// an exponential backoff window once degradedThreshold failures are present inside the window.
+// Successful creates never reset the window (recovery is time-based, not success-based), and an
+// active backoff window is never cancelled by a success.
+//
+// While PoolDegraded, an expired backoff window is renewed automatically whenever the failure
+// window is still hot, so the pool stays paused until the failure rate actually drops below the
+// threshold. The pool returns to PoolHealthy only once the failure window drains and no backoff
+// is active.
 type reconcileState struct {
 	mu                sync.Mutex
 	degradedThreshold int
+	failureWindow     time.Duration
 	failureCount      int
+	failureTimes      []time.Time
 	backoffAttempts   int
 	backoffUntil      time.Time
 	lastError         string
 	healthState       PoolHealthState
+	now               func() time.Time
 }
 
-// newReconcileState creates a new reconcileState with the given degraded threshold.
+// newReconcileState creates a new reconcileState with the given degraded threshold and the
+// default failure window.
 func newReconcileState(degradedThreshold int) *reconcileState {
 	return &reconcileState{
 		degradedThreshold: degradedThreshold,
+		failureWindow:     defaultFailureWindow,
 		healthState:       PoolHealthy,
+		now:               time.Now,
 	}
 }
 
-// recordSuccess resets the failure state and marks the pool as healthy.
-func (s *reconcileState) recordSuccess() {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	s.failureCount = 0
-	s.backoffAttempts = 0
-	s.backoffUntil = time.Time{}
-	s.lastError = ""
-	s.healthState = PoolHealthy
-}
-
 // recordFailure records a single failure. Delegates to recordFailures.
 func (s *reconcileState) recordFailure(err error) {
 	s.recordFailures(1, err)
 }
 
-// recordFailures records count failures in one call. If the cumulative count
-// reaches or exceeds the degraded threshold, the pool transitions to degraded
-// state and backoffAttempts is incremented (escalating the backoff duration).
-// In production, this is called once per reconcile tick, so backoff escalates
-// per failing tick, not per individual failed sandbox creation.
+// recordFailures records count failures in one call. Failures are recorded with the current
+// timestamp and age out of the sliding failureWindow; the pool transitions to degraded and opens
+// an exponential backoff window when the windowed count reaches or exceeds the degraded
+// threshold. Failures recorded while a backoff window is already active do not advance the
+// exponential delay (only the counter and last error are updated).
 func (s *reconcileState) recordFailures(count int, err error) {
 	if count <= 0 {
 		return
 	}
 	s.mu.Lock()
 	defer s.mu.Unlock()
-	s.failureCount += count
+	now := s.now()
+	for i := 0; i < count; i++ {
+		s.failureTimes = append(s.failureTimes, now)
+	}
+	s.pruneExpired(now)
 	if err != nil {
 		s.lastError = err.Error()
 	}
-	if s.failureCount >= s.degradedThreshold {
-		s.healthState = PoolDegraded
-		s.backoffAttempts++
-		shift := s.backoffAttempts - 1
-		if shift > 15 {
-			shift = 15
-		}
-		backoff := reconcileBackoffBase * (1 << shift)
-		if backoff > reconcileMaxBackoff {
-			backoff = reconcileMaxBackoff
-		}
-		s.backoffUntil = time.Now().Add(backoff)
+	if s.failureCount < s.degradedThreshold {
+		return
 	}
+	if s.healthState == PoolDegraded && !s.backoffUntil.IsZero() && now.Before(s.backoffUntil) {
+		return
+	}
+	s.activateNextBackoff(now)
 }
 
 // shouldBackoff returns true if the reconciler is in a backoff period.
+//
+// Advances the state machine on the read path: while PoolDegraded, an expired backoff window is
+// renewed when the failure window is still hot, so the pool stays paused until the failure rate
+// falls below the threshold; once the failure window drains and no backoff is active, the pool
+// recovers to PoolHealthy.
 func (s *reconcileState) shouldBackoff() bool {
 	s.mu.Lock()
 	defer s.mu.Unlock()
-	return time.Now().Before(s.backoffUntil)
+	return s.refreshLocked(s.now())
 }
 
-// snapshot returns a point-in-time view of the reconcile health state.
+// refreshLocked advances the state machine at the current time and reports whether create
+// attempts should be suppressed: prunes the sliding failure window, renews an expired backoff
+// window while the window is still hot, and recovers to healthy once the window drains.
+// Callers must hold s.mu.
+func (s *reconcileState) refreshLocked(now time.Time) bool {
+	s.pruneExpired(now)
+	if s.healthState != PoolDegraded || s.backoffUntil.IsZero() {
+		return false
+	}
+	if now.Before(s.backoffUntil) {
+		return true
+	}
+	if s.failureCount >= s.degradedThreshold {
+		s.activateNextBackoff(now)
+		return true
+	}
+	s.recover()
+	return false
+}
+
+// snapshot returns a point-in-time view of the reconcile health state. It advances the state
+// machine first (prune/renew/recover) so readers never observe a stale DEGRADED state or stale
+// failure count after the window has drained.
 func (s *reconcileState) snapshot() (PoolHealthState, int, bool, string) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
-	backoffActive := time.Now().Before(s.backoffUntil)
+	backoffActive := s.refreshLocked(s.now())
 	return s.healthState, s.failureCount, backoffActive, s.lastError
 }
 
+func (s *reconcileState) activateNextBackoff(now time.Time) {
+	s.healthState = PoolDegraded
+	s.backoffAttempts++
+	shift := s.backoffAttempts - 1
+	if shift > 30 {
+		shift = 30
+	}
+	// Cap the delay in seconds before constructing the duration: with sustained
+	// renewals backoffAttempts grows without bound, and 30s << 29 already overflows
+	// the int64 nanosecond range of time.Duration (wrapping negative and defeating
+	// the max check below).
+	maxSeconds := int64(reconcileMaxBackoff / time.Second)
+	delaySeconds := int64(reconcileBackoffBase/time.Second) << shift
+	if delaySeconds > maxSeconds {
+		delaySeconds = maxSeconds
+	}
+	s.backoffUntil = now.Add(time.Duration(delaySeconds) * time.Second)
+}
+
+func (s *reconcileState) recover() {
+	s.healthState = PoolHealthy
+	s.backoffUntil = time.Time{}
+	s.backoffAttempts = 0
+	s.lastError = ""
+}
+
+func (s *reconcileState) pruneExpired(now time.Time) {
+	cutoff := now.Add(-s.failureWindow)
+	kept := 0
+	for _, ts := range s.failureTimes {
+		if !ts.Before(cutoff) {
+			s.failureTimes[kept] = ts
+			kept++
+		}
+	}
+	s.failureTimes = s.failureTimes[:kept]
+	s.failureCount = kept
+}
+
 // reconcileTick performs a single reconciliation pass. It is designed to be
 // called periodically by the pool's background loop.
 //
@@ -207,7 +280,9 @@ func reconcileTick(
 			removedCount++
 		}
 		if !shrinkErr && removedCount > 0 {
-			state.recordSuccess()
+			logger.Debug("reconcile: shrunk excess idle",
+				"pool_name", poolName,
+				"removed", removedCount)
 		}
 		return
 	}
@@ -282,7 +357,8 @@ func reconcileTick(
 		state.recordFailures(failCount, lastCreateErr)
 	}
 
-	// Place created sandboxes into idle pool; record success per-putIdle.
+	// Place created sandboxes into idle pool. Successful puts do not reset the failure
+	// window; recovery is time-based (see reconcileState).
 	for i, id := range createdIDs {
 		renewed, renewErr := store.RenewPrimaryLock(ctx, poolName, ownerID, lockTTL)
 		if renewErr != nil || !renewed {
@@ -309,7 +385,6 @@ func reconcileTick(
 				"orphan_count", len(createdIDs)-i)
 			return
 		}
-		state.recordSuccess()
 	}
 
 	if len(createdIDs) > 0 {