feat: retain working Lantern servers across config updates

backend/radiance.go

@@ -262,7 +262,7 @@ func (r *LocalBackend) Start() {
 				span.End() // point-in-time marker — config was received at this timestamp
 			}
 		}
-		if err := r.setServers(list, true); err != nil {
+		if err := r.updateServers(list); err != nil {
 			slog.Error("setting servers in manager", "error", err)
 		}
 		if err := r.RunOfflineURLTests(); err != nil && !errors.Is(err, vpn.ErrTunnelAlreadyConnected) {
@@ -613,9 +613,40 @@ func (r *LocalBackend) RevokePrivateServerInvite(ip string, port int, accessToke
 	return r.srvManager.RevokePrivateServerInvite(ip, port, accessToken, inviteName)
 }
 
-func (r *LocalBackend) setServers(list servers.ServerList, isLantern bool) error {
-	if err := r.srvManager.SetServers(list, isLantern); err != nil {
-		return fmt.Errorf("failed to set servers in ServerManager: %w", err)
+// maxRetainedLanternServers caps the number of working Lantern servers retained
+// across config updates.
+const maxRetainedLanternServers = 60
+
+func (r *LocalBackend) updateServers(list servers.ServerList) error {
+	existing := r.srvManager.AllServers()
+	// prune any servers from the incoming list that already exists to avoid deleting
+	// selection history results and closing existing connections
+	existingTags := serverTagSet(existing)
+	list.Servers = slices.DeleteFunc(list.Servers, func(srv *servers.Server) bool {
+		_, exists := existingTags[srv.Tag]
+		return exists
+	})
+
+	tagsToEvict := lanternServersToEvict(existing, len(list.Servers), maxRetainedLanternServers)
+
+	if len(tagsToEvict) > 0 {
+		slog.Debug(
+			"Evicting retained Lantern servers to make room for new config batch",
+			"count", len(tagsToEvict),
+			"tags", tagsToEvict,
+		)
+		if _, err := r.srvManager.RemoveServers(tagsToEvict); err != nil {
+			return fmt.Errorf("remove retained Lantern servers: %w", err)
+		}
+	}
+
+	slog.Debug(
+		"Adding new Lantern servers from config update",
+		"count", len(list.Servers),
+		"tags", slices.Collect(maps.Keys(serverTagSet(list.Servers))),
+	)
+	if err := r.srvManager.AddServers(list, false); err != nil {
+		return fmt.Errorf("add Lantern servers: %w", err)
 	}
 	// updateOutbounds evicts any outbound absent from the list; include all
 	// servers so user-added outbounds aren't removed on a Lantern config update.
@@ -629,6 +660,66 @@ func (r *LocalBackend) setServers(list servers.ServerList, isLantern bool) error
 	return nil
 }
 
+func serverTagSet(list []*servers.Server) map[string]struct{} {
+	tags := make(map[string]struct{}, len(list))
+	for _, srv := range list {
+		tags[srv.Tag] = struct{}{}
+	}
+	return tags
+}
+
+// lanternServersToEvict returns the Lantern server tags to remove before the
+// next config batch is added. Hard-demoted servers are removed first. Remaining
+// candidates are evicted oldest-first by SelectionHistory.UpdatedAt; missing
+// history sorts oldest.
+func lanternServersToEvict(
+	existing []*servers.Server,
+	incomingCount, limit int,
+) []string {
+	tagsToEvict := make([]string, 0)
+	retentionCandidates := make([]*servers.Server, 0, len(existing))
+
+	for _, srv := range existing {
+		if !srv.IsLantern {
+			continue
+		}
+		if isHardDemoted(srv) {
+			tagsToEvict = append(tagsToEvict, srv.Tag)
+			continue
+		}
+		retentionCandidates = append(retentionCandidates, srv)
+	}
+
+	retentionBudget := max(limit-incomingCount, 0)
+	if len(retentionCandidates) <= retentionBudget {
+		return tagsToEvict
+	}
+
+	slices.SortFunc(retentionCandidates, compareSelectionAge)
+
+	overflow := len(retentionCandidates) - retentionBudget
+	for _, srv := range retentionCandidates[:overflow] {
+		tagsToEvict = append(tagsToEvict, srv.Tag)
+	}
+
+	return tagsToEvict
+}
+
+func isHardDemoted(srv *servers.Server) bool {
+	return srv.SelectionHistory != nil && srv.SelectionHistory.HardDemoted
+}
+
+func compareSelectionAge(a, b *servers.Server) int {
+	return selectionUpdatedAt(a).Compare(selectionUpdatedAt(b))
+}
+
+func selectionUpdatedAt(srv *servers.Server) time.Time {
+	if srv.SelectionHistory == nil {
+		return time.Time{}
+	}
+	return srv.SelectionHistory.UpdatedAt
+}
+
 // clearSelectedIfMissing reverts the persisted selection to auto-select when
 // the selected server is no longer present in the manager.
 func (r *LocalBackend) clearSelectedIfMissing() {

backend/radiance_test.go

@@ -0,0 +1,120 @@
+package backend
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/getlantern/radiance/servers"
+)
+
+func TestExhaustionGate_AllowRateLimitsBelowGap(t *testing.T) {
+	prev := defaultExhaustionRefetchGap
+	defaultExhaustionRefetchGap = 50 * time.Millisecond
+	t.Cleanup(func() { defaultExhaustionRefetchGap = prev })
+
+	var g exhaustionGate
+	require.True(t, g.allow(), "first allow must pass on a zero gate")
+	assert.False(t, g.allow(), "second allow inside the gap must be rate-limited")
+	assert.False(t, g.allow(), "third allow inside the gap must still be rate-limited")
+
+	time.Sleep(defaultExhaustionRefetchGap + 10*time.Millisecond)
+	assert.True(t, g.allow(), "allow after the gap elapses must pass again")
+	assert.False(t, g.allow(), "post-recovery allow must re-arm the gate")
+}
+
+func newTestServer(tag string, isLantern, hardDemoted bool, updatedAt time.Time) *servers.Server {
+	srv := &servers.Server{
+		Tag:       tag,
+		IsLantern: isLantern,
+	}
+	if hardDemoted || !updatedAt.IsZero() {
+		srv.SelectionHistory = &servers.SelectionHistory{
+			HardDemoted: hardDemoted,
+			UpdatedAt:   updatedAt,
+		}
+	}
+	return srv
+}
+
+func TestLanternServersToEvict(t *testing.T) {
+	baseTime := time.Unix(0, 0).UTC()
+
+	tests := []struct {
+		name     string
+		existing []*servers.Server
+		incoming int
+		limit    int
+		want     []string
+	}{
+		{
+			name: "evicts only hard-demoted lantern servers",
+			existing: []*servers.Server{
+				newTestServer("demoted", true, true, baseTime),
+				newTestServer("working", true, false, baseTime),
+				newTestServer("users-demoted", false, true, baseTime),
+			},
+			limit: 60,
+			want:  []string{"demoted"},
+		},
+		{
+			name: "hard-demoted lantern server is evicted regardless of incoming config",
+			existing: []*servers.Server{
+				newTestServer("demoted", true, true, baseTime),
+			},
+			incoming: 1,
+			limit:    60,
+			want:     []string{"demoted"},
+		},
+		{
+			name: "under the limit nothing is evicted",
+			existing: []*servers.Server{
+				newTestServer("a", true, false, baseTime),
+				newTestServer("b", true, false, baseTime),
+			},
+			incoming: 2,
+			limit:    60,
+		},
+		{
+			name: "over the limit evicts oldest working servers and keeps the newest",
+			existing: []*servers.Server{
+				newTestServer("old", true, false, baseTime.Add(1*time.Hour)),
+				newTestServer("mid", true, false, baseTime.Add(2*time.Hour)),
+				newTestServer("new", true, false, baseTime.Add(3*time.Hour)),
+			},
+			incoming: 1,
+			limit:    3,
+			want:     []string{"old"},
+		},
+		{
+			name: "incoming at the limit evicts all existing working servers",
+			existing: []*servers.Server{
+				newTestServer("a", true, false, baseTime.Add(1*time.Hour)),
+				newTestServer("b", true, false, baseTime.Add(2*time.Hour)),
+			},
+			incoming: 3,
+			limit:    3,
+			want:     []string{"a", "b"},
+		},
+		{
+			name: "server with no selection history sorts oldest",
+			existing: []*servers.Server{
+				newTestServer("no-history", true, false, time.Time{}),
+				newTestServer("probed", true, false, baseTime.Add(5*time.Hour)),
+			},
+			incoming: 1,
+			limit:    2,
+			want:     []string{"no-history"},
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			got := lanternServersToEvict(tt.existing, tt.incoming, tt.limit)
+			assert.ElementsMatch(t, tt.want, got)
+		})
+	}
+}

cmd/lantern/servers.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"slices"
 	"strings"
 
 	C "github.com/getlantern/common"
@@ -24,6 +25,7 @@ type ServersCmd struct {
 type ServersListCmd struct {
 	Latency bool `arg:"--latency" help:"include latest latency from selection history"`
 	JSON    bool `arg:"--json" help:"output JSON"`
+	Limit   int  `arg:"--limit" help:"limit number of servers to list"`
 }
 
 type ServersShowCmd struct {
@@ -92,9 +94,9 @@ func runServers(ctx context.Context, c *ipc.Client, cmd *ServersCmd) error {
 	case cmd.PrivateServer != nil:
 		return runPrivateServer(ctx, c, cmd.PrivateServer)
 	case cmd.List != nil:
-		return serversList(ctx, c, cmd.List.Latency, cmd.List.JSON)
+		return serversList(ctx, c, cmd.List.Latency, cmd.List.JSON, cmd.List.Limit)
 	default:
-		return serversList(ctx, c, false, false)
+		return serversList(ctx, c, false, false, 0)
 	}
 }
 
@@ -116,7 +118,7 @@ func runPrivateServer(ctx context.Context, c *ipc.Client, cmd *PrivateServerCmd)
 	}
 }
 
-func serversList(ctx context.Context, c *ipc.Client, showLatency, asJSON bool) error {
+func serversList(ctx context.Context, c *ipc.Client, showLatency, asJSON bool, limit int) error {
 	srvs, err := c.Servers(ctx)
 	if err != nil {
 		return err
@@ -137,9 +139,38 @@ func serversList(ctx context.Context, c *ipc.Client, showLatency, asJSON bool) e
 		fmt.Println("No servers available")
 		return nil
 	}
-	for _, s := range srvs {
+	latestDelay := func(s *servers.Server) uint32 {
+		if s.SelectionHistory != nil {
+			return s.SelectionHistory.LatestSuccessDelay()
+		}
+		return 0
+	}
+	slices.SortFunc(srvs, func(a, b *servers.Server) int {
+		ad := latestDelay(a)
+		bd := latestDelay(b)
+
+		switch {
+		case ad == 0 && bd == 0:
+			return 0
+		case ad == 0:
+			return 1
+		case bd == 0:
+			return -1
+		case ad < bd:
+			return -1
+		case ad > bd:
+			return 1
+		default:
+			return 0
+		}
+	})
+	for i, s := range srvs {
+		if limit > 0 && i >= limit {
+			break
+		}
 		printServerEntry(s, showLatency)
 	}
+	fmt.Printf("%d total server(s)\n", len(srvs))
 	return nil
 }
 

cmd/lanternd/lanternd.go

@@ -199,9 +199,12 @@ func spawnChild(args []string, dataPath, logPath, logLevel string) (*childProces
 	go func() {
 		defer stdoutPipe.Close()
 		var w io.Writer = os.Stdout
-		if h, ok := logger.Handler().(*rlog.Handler); ok {
-			w = h.Writer()
-		}
+		// TODO: the child process outputs to both stdout and the file logger, so we end up with
+		// 	duplicate log lines. we'll come back to this later and fix it, but for now just
+		// 	write to stdout to avoid the duplication.
+		// if h, ok := logger.Handler().(*rlog.Handler); ok {
+		// 	w = h.Writer()
+		// }
 		scanner := bufio.NewScanner(stdoutPipe)
 		for scanner.Scan() {
 			if s := scanner.Text(); s != "" {

events/events.go

@@ -121,12 +121,13 @@ func (e *Subscription[T]) Unsubscribe() {
 func Emit[T Event](evt T) {
 	subscriptionsMu.RLock()
 	defer subscriptionsMu.RUnlock()
-	if subs, ok := subscriptions[reflect.TypeFor[T]()]; ok {
+	evtType := reflect.TypeFor[T]()
+	if subs, ok := subscriptions[evtType]; ok {
 		for _, cb := range subs {
 			go func() {
 				defer func() {
 					if r := recover(); r != nil {
-						slog.Error("Panic in event callback", "error", r)
+						slog.Error("Panic in event callback", "error", r, "event", evtType.String())
 					}
 				}()
 				cb(evt)

go.mod

@@ -36,7 +36,7 @@ require (
 	github.com/getlantern/domainfront v0.0.0-20260419161617-0bff0b2169f4
 	github.com/getlantern/keepcurrent v0.0.0-20260616120552-f204338b01a3
 	github.com/getlantern/kindling v0.0.0-20260611181428-9a360f63ad5a
-	github.com/getlantern/lantern-box v0.0.93
+	github.com/getlantern/lantern-box v0.0.95
 	github.com/getlantern/pluriconfig v0.0.0-20251126214241-8cc8bc561535
 	github.com/getlantern/publicip v0.0.0-20260328175246-2c460fe80c6b
 	github.com/getlantern/semconv v0.0.0-20260327040646-21845dda05cb