Deterministic state syncs

consensus/bor/bor.go

@@ -1760,12 +1760,32 @@ func (c *Bor) CommitStates(
 	// Wait for heimdall to be synced before fetching state sync events
 	c.spanStore.waitUntilHeimdallIsSynced(c.ctx)
 
-	eventRecords, err = c.HeimdallClient.StateSyncEvents(c.ctx, from, to.Unix())
-	if err != nil {
-		log.Error("Error occurred when fetching state sync events", "fromID", from, "to", to.Unix(), "err", err)
+	if c.config.IsDeterministicStateSync(header.Number) {
+		log.Info("Using deterministic state sync", "cutoff", to.Unix())
+
+		eventRecords, err = c.HeimdallClient.StateSyncEventsByTime(c.ctx, from, to.Unix())
+		if err != nil {
+			// Liveness-over-safety tradeoff (matches pre-fork behavior):
+			// FetchWithRetry already retries aggressively and MultiHeimdallClient
+			// provides failover, so errors reaching here are persistent. Returning
+			// empty lets the proposer build a block with 0 state syncs. If other
+			// validators succeed, they will derive a different state root and reject
+			// this block — the proposer misses a slot but no silent divergence
+			// occurs. Skipped events are retried at the next sprint since `from`
+			// is derived from the on-chain LastStateId.
+			log.Error("Error fetching deterministic state sync events", "fromID", from, "to", to.Unix(), "err", err)
 
-		stateSyncs := make([]*types.StateSyncData, 0)
-		return stateSyncs, nil
+			return make([]*types.StateSyncData, 0), nil
+		}
+	} else {
+		eventRecords, err = c.HeimdallClient.StateSyncEvents(c.ctx, from, to.Unix())
+		if err != nil {
+			// Pre-fork: preserve existing behavior (returning empty, no error)
+			log.Error("Error occurred when fetching state sync events", "fromID", from, "to", to.Unix(), "err", err)
+
+			stateSyncs := make([]*types.StateSyncData, 0)
+			return stateSyncs, nil
+		}
 	}
 
 	// This if statement checks if there are any state sync record overrides configured for the current block number.

consensus/bor/bor_test.go

@@ -119,6 +119,15 @@
 func (f *failingHeimdallClient) FetchStatus(ctx context.Context) (*ctypes.SyncInfo, error) {
 	return nil, errors.New("fetch status failed")
 }
+func (f *failingHeimdallClient) GetBlockHeightByTime(_ context.Context, _ int64) (int64, error) {
+	return 0, errors.New("get block height by time failed")
+}
+func (f *failingHeimdallClient) StateSyncEventsAtHeight(_ context.Context, _ uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) {
+	return nil, errors.New("state sync events at height failed")
+}
+func (f *failingHeimdallClient) StateSyncEventsByTime(_ context.Context, _ uint64, _ int64) ([]*clerk.EventRecordWithTime, error) {
+	return nil, errors.New("state sync events by time failed")
+}
 
 // newStateDBForTest creates a fresh state database for testing.
 func newStateDBForTest(t *testing.T, root common.Hash) *state.StateDB {
@@ -2974,6 +2983,15 @@
 func (m *mockHeimdallClient) FetchStatus(ctx context.Context) (*ctypes.SyncInfo, error) {
 	return &ctypes.SyncInfo{CatchingUp: false}, nil
 }
+func (m *mockHeimdallClient) GetBlockHeightByTime(_ context.Context, _ int64) (int64, error) {
+	return 0, nil
+}
+func (m *mockHeimdallClient) StateSyncEventsAtHeight(_ context.Context, _ uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) {
+	return nil, nil
+}
+func (m *mockHeimdallClient) StateSyncEventsByTime(_ context.Context, _ uint64, _ int64) ([]*clerk.EventRecordWithTime, error) {
+	return m.events, nil
+}
 func TestEncodeSigHeader_WithBaseFee(t *testing.T) {
 	t.Parallel()
 	h := &types.Header{
@@ -5259,3 +5277,182 @@
 		require.NotErrorIs(t, err, errMissingGiuglianoFields)
 	}
 }
+
+// trackingHeimdallClient records which IHeimdallClient methods were called.
+// It returns configurable results and tracks call counts for assertions.
+type trackingHeimdallClient struct {
+	// Call counters
+	stateSyncEventsCalled       int
+	stateSyncEventsByTimeCalled int
+
+	// Configurable return values
+	events          []*clerk.EventRecordWithTime
+	eventsErr       error
+	eventsByTime    []*clerk.EventRecordWithTime
+	eventsByTimeErr error
+}
+
+func (t *trackingHeimdallClient) Close() {}
+func (t *trackingHeimdallClient) StateSyncEvents(context.Context, uint64, int64) ([]*clerk.EventRecordWithTime, error) {
+	t.stateSyncEventsCalled++
+	return t.events, t.eventsErr
+}
+func (t *trackingHeimdallClient) GetSpan(context.Context, uint64) (*borTypes.Span, error) {
+	return nil, nil
+}
+func (t *trackingHeimdallClient) GetLatestSpan(context.Context) (*borTypes.Span, error) {
+	return nil, nil
+}
+func (t *trackingHeimdallClient) FetchCheckpoint(context.Context, int64) (*checkpoint.Checkpoint, error) {
+	return nil, nil
+}
+func (t *trackingHeimdallClient) FetchCheckpointCount(context.Context) (int64, error) {
+	return 0, nil
+}
+func (t *trackingHeimdallClient) FetchMilestone(context.Context) (*milestone.Milestone, error) {
+	return nil, nil
+}
+func (t *trackingHeimdallClient) FetchMilestoneCount(context.Context) (int64, error) {
+	return 0, nil
+}
+func (t *trackingHeimdallClient) FetchStatus(context.Context) (*ctypes.SyncInfo, error) {
+	return &ctypes.SyncInfo{CatchingUp: false}, nil
+}
+func (t *trackingHeimdallClient) StateSyncEventsByTime(context.Context, uint64, int64) ([]*clerk.EventRecordWithTime, error) {
+	t.stateSyncEventsByTimeCalled++
+	return t.eventsByTime, t.eventsByTimeErr
+}
+
+// deterministicBorConfig returns a BorConfig with DeterministicStateSyncBlock set.
+func deterministicBorConfig(forkBlock int64) *params.BorConfig {
+	return &params.BorConfig{
+		Sprint:                      map[string]uint64{"0": 16},
+		Period:                      map[string]uint64{"0": 2},
+		IndoreBlock:                 big.NewInt(0),
+		StateSyncConfirmationDelay:  map[string]uint64{"0": 0},
+		RioBlock:                    big.NewInt(1000000),
+		DeterministicStateSyncBlock: big.NewInt(forkBlock),
+	}
+}
+
+func TestCommitStates_DeterministicForkSwitch(t *testing.T) {
+	t.Parallel()
+
+	addr1 := common.HexToAddress("0x1")
+	sp := &fakeSpanner{vals: []*valset.Validator{{Address: addr1, VotingPower: 1}}}
+	mockGC := &mockGenesisContractForCommitStatesIndore{lastStateID: 0}
+
+	// Fork activates at block 100
+	borCfg := deterministicBorConfig(100)
+	genesisTime := uint64(time.Now().Unix()) - 200
+	chain, b := newChainAndBorForTest(t, sp, borCfg, true, addr1, genesisTime)
+	b.GenesisContractsClient = mockGC
+
+	genesis := chain.HeaderChain().GetHeaderByNumber(0)
+	now := time.Now()
+
+	// Pre-fork: block 16 should use StateSyncEvents (old legacy path)
+	tracker := &trackingHeimdallClient{
+		events: []*clerk.EventRecordWithTime{},
+	}
+	b.SetHeimdallClient(tracker)
+
+	stateDb := newStateDBForTest(t, genesis.Root)
+	h := &types.Header{Number: big.NewInt(16), ParentHash: genesis.Hash(), Time: uint64(now.Unix())}
+
+	_, err := b.CommitStates(stateDb, h, statefull.ChainContext{Chain: chain.HeaderChain(), Bor: b})
+	require.NoError(t, err)
+	require.Equal(t, 1, tracker.stateSyncEventsCalled, "pre-fork should call StateSyncEvents")
+	require.Equal(t, 0, tracker.stateSyncEventsByTimeCalled, "pre-fork should not call StateSyncEventsByTime")
+
+	// Post-fork: block 112 should use StateSyncEventsByTime (deterministic state sync)
+	tracker2 := &trackingHeimdallClient{
+		eventsByTime: []*clerk.EventRecordWithTime{},
+	}
+	b.SetHeimdallClient(tracker2)
+
+	stateDb2 := newStateDBForTest(t, genesis.Root)
+	h2 := &types.Header{Number: big.NewInt(112), ParentHash: genesis.Hash(), Time: uint64(now.Unix())}
+
+	_, err = b.CommitStates(stateDb2, h2, statefull.ChainContext{Chain: chain.HeaderChain(), Bor: b})
+	require.NoError(t, err)
+	require.Equal(t, 0, tracker2.stateSyncEventsCalled, "post-fork should not call StateSyncEvents")
+	require.Equal(t, 1, tracker2.stateSyncEventsByTimeCalled, "post-fork should call StateSyncEventsByTime")
+}
+
+func TestCommitStates_ResilientPostFork(t *testing.T) {
+	t.Parallel()
+
+	addr1 := common.HexToAddress("0x1")
+	sp := &fakeSpanner{vals: []*valset.Validator{{Address: addr1, VotingPower: 1}}}
+	mockGC := &mockGenesisContractForCommitStatesIndore{lastStateID: 0}
+
+	// Fork activates at block 0 so all blocks are post-fork
+	borCfg := deterministicBorConfig(0)
+	genesisTime := uint64(time.Now().Unix()) - 200
+	chain, b := newChainAndBorForTest(t, sp, borCfg, true, addr1, genesisTime)
+	b.GenesisContractsClient = mockGC
+
+	genesis := chain.HeaderChain().GetHeaderByNumber(0)
+	now := time.Now()
+
+	// StateSyncEventsByTime returns an error
+	tracker := &trackingHeimdallClient{
+		eventsByTimeErr: errors.New("heimdall state sync by time failed"),
+	}
+	b.SetHeimdallClient(tracker)
+
+	stateDb := newStateDBForTest(t, genesis.Root)
+	h := &types.Header{Number: big.NewInt(16), ParentHash: genesis.Hash(), Time: uint64(now.Unix())}
+
+	result, err := b.CommitStates(stateDb, h, statefull.ChainContext{Chain: chain.HeaderChain(), Bor: b})
+
+	// Post-fork errors are resilient: log + return empty, no error
+	require.NoError(t, err, "post-fork should not return error on StateSyncEventsByTime failure")
+	require.Empty(t, result, "post-fork should return empty on StateSyncEventsByTime failure")
+
+	// StateSyncEventsByTime should have been called
+	require.Equal(t, 1, tracker.stateSyncEventsByTimeCalled,
+		"StateSyncEventsByTime should have been called once")
+	// Must not fallback to StateSyncEvents
+	require.Equal(t, 0, tracker.stateSyncEventsCalled,
+		"post-fork should NOT fall back to StateSyncEvents on error")
+}
+
+func TestCommitStates_ResilientPostFork_ReturnsEmptyOnError(t *testing.T) {
+	t.Parallel()
+
+	addr1 := common.HexToAddress("0x1")
+	sp := &fakeSpanner{vals: []*valset.Validator{{Address: addr1, VotingPower: 1}}}
+	mockGC := &mockGenesisContractForCommitStatesIndore{lastStateID: 0}
+
+	borCfg := deterministicBorConfig(0)
+	genesisTime := uint64(time.Now().Unix()) - 200
+	chain, b := newChainAndBorForTest(t, sp, borCfg, true, addr1, genesisTime)
+	b.GenesisContractsClient = mockGC
+
+	genesis := chain.HeaderChain().GetHeaderByNumber(0)
+	now := time.Now()
+
+	// StateSyncEventsByTime fails with an HTTP error
+	tracker := &trackingHeimdallClient{
+		eventsByTimeErr: errors.New("HTTP 503: service unavailable"),
+	}
+	b.SetHeimdallClient(tracker)
+
+	stateDb := newStateDBForTest(t, genesis.Root)
+	h := &types.Header{Number: big.NewInt(16), ParentHash: genesis.Hash(), Time: uint64(now.Unix())}
+
+	result, err := b.CommitStates(stateDb, h, statefull.ChainContext{Chain: chain.HeaderChain(), Bor: b})
+
+	// Post-fork is resilient: returns empty on error, does not propagate
+	require.NoError(t, err, "post-fork should not return error on StateSyncEventsByTime failure")
+	require.Empty(t, result, "post-fork should return empty on StateSyncEventsByTime failure")
+
+	// StateSyncEventsByTime should have been called
+	require.Equal(t, 1, tracker.stateSyncEventsByTimeCalled,
+		"StateSyncEventsByTime should have been called")
+	// Old path should not have been called as fallback
+	require.Equal(t, 0, tracker.stateSyncEventsCalled,
+		"post-fork should not fall back to StateSyncEvents")
+}

consensus/bor/heimdall.go

@@ -14,6 +14,7 @@ import (
 //go:generate mockgen -source=heimdall.go -destination=../../tests/bor/mocks/IHeimdallClient.go -package=mocks
 type IHeimdallClient interface {
 	StateSyncEvents(ctx context.Context, fromID uint64, to int64) ([]*clerk.EventRecordWithTime, error)
+	StateSyncEventsByTime(ctx context.Context, fromID uint64, toTime int64) ([]*clerk.EventRecordWithTime, error)
 	GetSpan(ctx context.Context, spanID uint64) (*types.Span, error)
 	GetLatestSpan(ctx context.Context) (*types.Span, error)
 	FetchCheckpoint(ctx context.Context, number int64) (*checkpoint.Checkpoint, error)

consensus/bor/heimdall/client.go

@@ -95,6 +95,8 @@ const (
 	fetchLatestSpan = "bor/spans/latest"
 
 	fetchStatus = "/status"
+
+	fetchStateSyncsByTimePath = "clerk/state-syncs-by-time"
 )
 
 // StateSyncEvents fetches the state sync events from heimdall
@@ -265,6 +267,63 @@ func (h *HeimdallClient) FetchStatus(ctx context.Context) (*ctypes.SyncInfo, err
 	return response, nil
 }
 
+// StateSyncsByTimeResponse uses the proto-generated response type from heimdall-v2.
+type StateSyncsByTimeResponse = clerkTypes.StateSyncsByTimeResponse
+
+// StateSyncEventsByTime fetches state sync events using the combined endpoint that
+// resolves the Heimdall height from the cutoff time internally.
+func (h *HeimdallClient) StateSyncEventsByTime(ctx context.Context, fromID uint64, toTime int64) ([]*clerk.EventRecordWithTime, error) {
+	// Global timeout bounding the entire paginated fetch, matching the gRPC
+	// implementation's stateSyncTotalTimeout (1 minute).
+	ctx, cancel := context.WithTimeout(ctx, 1*time.Minute)
+	defer cancel()
+
+	ctx = WithRequestType(ctx, StateSyncByTimeRequest)
+
+	eventRecords := make([]*clerk.EventRecordWithTime, 0)
+
+	for {
+		u, err := stateSyncsByTimeURL(h.urlString, fromID, toTime)
+		if err != nil {
+			return nil, err
+		}
+
+		log.Debug("Fetching state sync events by time", "queryParams", u.RawQuery)
+
+		response, err := FetchWithRetry[StateSyncsByTimeResponse](ctx, h.client, u, h.closeCh)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, e := range response.EventRecords {
+			record := &clerk.EventRecordWithTime{
+				EventRecord: clerk.EventRecord{
+					ID:       e.Id,
+					ChainID:  e.BorChainId,
+					Contract: common.HexToAddress(e.Contract),
+					Data:     e.Data,
+					LogIndex: e.LogIndex,
+					TxHash:   common.HexToHash(e.TxHash),
+				},
+				Time: e.RecordTime,
+			}
+			eventRecords = append(eventRecords, record)
+		}
+
+		if len(response.EventRecords) < stateFetchLimit {
+			break
+		}
+
+		fromID += uint64(stateFetchLimit)
+	}
+
+	sort.SliceStable(eventRecords, func(i, j int) bool {
+		return eventRecords[i].ID < eventRecords[j].ID
+	})
+
+	return eventRecords, nil
+}
+
 func FetchOnce[T any](ctx context.Context, client http.Client, url *url.URL, closeCh chan struct{}) (*T, error) {
 	request := &Request{client: client, url: url, start: time.Now()}
 	return Fetch[T](ctx, request)
@@ -437,6 +496,15 @@ func statusURL(urlString string) (*url.URL, error) {
 	return makeURL(urlString, fetchStatus, "")
 }
 
+func stateSyncsByTimeURL(urlString string, fromID uint64, toTime int64) (*url.URL, error) {
+	t := time.Unix(toTime, 0).UTC()
+	params := url.Values{}
+	params.Set("from_id", fmt.Sprintf("%d", fromID))
+	params.Set("to_time", t.Format(time.RFC3339Nano))
+	params.Set("pagination.limit", fmt.Sprintf("%d", stateFetchLimit))
+	return makeURL(urlString, fetchStateSyncsByTimePath, params.Encode())
+}
+
 func makeURL(urlString, rawPath, rawQuery string) (*url.URL, error) {
 	u, err := url.Parse(urlString)
 	if err != nil {

consensus/bor/heimdall/failover_client.go

@@ -29,6 +29,7 @@ const (
 // running into Go's covariant-slice restriction.
 type Endpoint interface {
 	StateSyncEvents(ctx context.Context, fromID uint64, to int64) ([]*clerk.EventRecordWithTime, error)
+	StateSyncEventsByTime(ctx context.Context, fromID uint64, toTime int64) ([]*clerk.EventRecordWithTime, error)
 	GetSpan(ctx context.Context, spanID uint64) (*types.Span, error)
 	GetLatestSpan(ctx context.Context) (*types.Span, error)
 	FetchCheckpoint(ctx context.Context, number int64) (*checkpoint.Checkpoint, error)
@@ -109,6 +110,17 @@ func (f *MultiHeimdallClient) StateSyncEvents(ctx context.Context, fromID uint64
 	})
 }
 
+func (f *MultiHeimdallClient) StateSyncEventsByTime(ctx context.Context, fromID uint64, toTime int64) ([]*clerk.EventRecordWithTime, error) {
+	// Set a 1-minute global timeout for the paginated fetch BEFORE callWithFailover
+	// applies its per-attempt attemptTimeout cap.
+	ctx, cancel := context.WithTimeout(ctx, 1*time.Minute)
+	defer cancel()
+
+	return callWithFailover(f, ctx, func(ctx context.Context, c Endpoint) ([]*clerk.EventRecordWithTime, error) {
+		return c.StateSyncEventsByTime(ctx, fromID, toTime)
+	})
+}
+
 func (f *MultiHeimdallClient) GetSpan(ctx context.Context, spanID uint64) (*types.Span, error) {
 	return callWithFailover(f, ctx, func(ctx context.Context, c Endpoint) (*types.Span, error) {
 		return c.GetSpan(ctx, spanID)