Skip to content

fix: backfill historical keyper sets on startup #712

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jannikluhn wants to merge 2 commits into
base: main
Choose a base branch
from
Open

fix: backfill historical keyper sets on startup #712

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bfd715d
fix: backfill historical keyper sets on startup
jannikluhn Jun 1, 2026
574b7b4
fixup! fix: backfill historical keyper sets on startup
jannikluhn Jun 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions rolling-shutter/chainobserver/db/keyper/keyper.sqlc.gen.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 5 additions & 1 deletion rolling-shutter/chainobserver/db/keyper/sql/queries/keyper.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,8 @@ ORDER BY activation_block_number DESC LIMIT 1;

-- name: GetKeyperSets :many
SELECT * FROM keyper_set
ORDER BY activation_block_number ASC;
ORDER BY activation_block_number ASC;

-- name: GetKeyperSetIndices :many
SELECT keyper_config_index FROM keyper_set
ORDER BY keyper_config_index ASC;
7 changes: 7 additions & 0 deletions rolling-shutter/keyperimpl/gnosis/keyper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
sequencerBindings "github.com/shutter-network/gnosh-contracts/gnoshcontracts/sequencer"
validatorRegistryBindings "github.com/shutter-network/gnosh-contracts/gnoshcontracts/validatorregistry"

obskeyper "github.com/shutter-network/rolling-shutter/rolling-shutter/chainobserver/db/keyper"
"github.com/shutter-network/rolling-shutter/rolling-shutter/eonkeypublisher"
"github.com/shutter-network/rolling-shutter/rolling-shutter/keyper"
"github.com/shutter-network/rolling-shutter/rolling-shutter/keyper/epochkghandler"
Expand All @@ -22,6 +23,7 @@ import (
"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/broker"
"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync"
syncevent "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync/event"
chainsyncer "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync/syncer"
"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/db"
"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/service"
"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/slotticker"
Expand Down Expand Up @@ -109,6 +111,10 @@ func (kpr *Keyper) Start(ctx context.Context, runner service.Runner) error {
return errors.Wrap(err, "can't instantiate keyper core")
}

ksIndices, err := obskeyper.New(kpr.dbpool).GetKeyperSetIndices(ctx)
if err != nil {
return errors.Wrap(err, "failed to load keyper set indices")
}
kpr.chainSyncClient, err = chainsync.NewClient(
ctx,
chainsync.WithClientURL(kpr.config.Gnosis.Node.EthereumURL),
Expand All @@ -118,6 +124,7 @@ func (kpr *Keyper) Start(ctx context.Context, runner service.Runner) error {
chainsync.WithSyncNewKeyperSet(kpr.channelNewKeyperSet),
chainsync.WithPrivateKey(kpr.config.Gnosis.Node.PrivateKey.Key),
chainsync.WithLogger(gethLog.NewLogger(slog.Default().Handler())),
chainsync.WithKeyperSetKnownRanges(chainsyncer.IndicesToRanges(ksIndices)),
)
if err != nil {
return err
Expand Down
6 changes: 6 additions & 0 deletions rolling-shutter/keyperimpl/optimism/keyper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/broker"
"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync"
syncevent "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync/event"
chainsyncer "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync/syncer"
"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/configuration"
"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/db"
"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/identitypreimage"
Expand Down Expand Up @@ -78,13 +79,18 @@ func (kpr *Keyper) Start(ctx context.Context, runner service.Runner) error {
if err != nil {
return errors.Wrap(err, "can't instantiate keyper core")
}
ksIndices, err := obskeyper.New(kpr.dbpool).GetKeyperSetIndices(ctx)
if err != nil {
return errors.Wrap(err, "failed to load keyper set indices")
}
// TODO: wrap the logger and pass in
kpr.l2Client, err = chainsync.NewClient(
ctx,
chainsync.WithClientURL(kpr.config.Optimism.JSONRPCURL),
chainsync.WithSyncNewBlock(kpr.newBlock),
chainsync.WithSyncNewKeyperSet(kpr.newKeyperSet),
chainsync.WithPrivateKey(kpr.config.Optimism.PrivateKey.Key),
chainsync.WithKeyperSetKnownRanges(chainsyncer.IndicesToRanges(ksIndices)),
)
if err != nil {
return err
Expand Down
7 changes: 7 additions & 0 deletions rolling-shutter/keyperimpl/primev/keyper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
providerregistry "github.com/primev/mev-commit/contracts-abi/clients/ProviderRegistry"
"github.com/rs/zerolog/log"

obskeyper "github.com/shutter-network/rolling-shutter/rolling-shutter/chainobserver/db/keyper"
"github.com/shutter-network/rolling-shutter/rolling-shutter/eonkeypublisher"
"github.com/shutter-network/rolling-shutter/rolling-shutter/keyper"
"github.com/shutter-network/rolling-shutter/rolling-shutter/keyper/epochkghandler"
Expand All @@ -19,6 +20,7 @@ import (
"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/broker"
"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync"
syncevent "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync/event"
chainsyncer "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync/syncer"
"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/db"
"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/service"
"github.com/shutter-network/rolling-shutter/rolling-shutter/p2p"
Expand Down Expand Up @@ -77,6 +79,10 @@ func (k *Keyper) Start(ctx context.Context, runner service.Runner) error {
return errors.Wrap(err, "can't instantiate keyper core")
}

ksIndices, err := obskeyper.New(k.dbpool).GetKeyperSetIndices(ctx)
if err != nil {
return errors.Wrap(err, "failed to load keyper set indices")
}
k.chainSyncClient, err = chainsync.NewClient(
ctx,
chainsync.WithClientURL(k.config.Chain.Node.EthereumURL),
Expand All @@ -86,6 +92,7 @@ func (k *Keyper) Start(ctx context.Context, runner service.Runner) error {
chainsync.WithSyncNewBlock(k.channelNewBlock),
chainsync.WithPrivateKey(k.config.Chain.Node.PrivateKey.Key),
chainsync.WithLogger(gethLog.NewLogger(slog.Default().Handler())),
chainsync.WithKeyperSetKnownRanges(chainsyncer.IndicesToRanges(ksIndices)),
)
if err != nil {
return err
Expand Down
7 changes: 7 additions & 0 deletions rolling-shutter/keyperimpl/shutterservice/keyper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
triggerRegistryV1Bindings "github.com/shutter-network/contracts/v2/bindings/shuttereventtriggerregistryv1"
registryBindings "github.com/shutter-network/contracts/v2/bindings/shutterregistry"

obskeyper "github.com/shutter-network/rolling-shutter/rolling-shutter/chainobserver/db/keyper"
"github.com/shutter-network/rolling-shutter/rolling-shutter/eonkeypublisher"
"github.com/shutter-network/rolling-shutter/rolling-shutter/keyper"
"github.com/shutter-network/rolling-shutter/rolling-shutter/keyper/epochkghandler"
Expand All @@ -22,6 +23,7 @@ import (
"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/broker"
"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync"
syncevent "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync/event"
chainsyncer "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync/syncer"
"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/db"
"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/service"
"github.com/shutter-network/rolling-shutter/rolling-shutter/p2p"
Expand Down Expand Up @@ -83,6 +85,10 @@ func (kpr *Keyper) Start(ctx context.Context, runner service.Runner) error {
if err != nil {
return errors.Wrap(err, "can't instantiate keyper core")
}
ksIndices, err := obskeyper.New(kpr.dbpool).GetKeyperSetIndices(ctx)
if err != nil {
return errors.Wrap(err, "failed to load keyper set indices")
}
kpr.chainSyncClient, err = chainsync.NewClient(
ctx,
chainsync.WithClientURL(kpr.config.Chain.Node.EthereumURL),
Expand All @@ -92,6 +98,7 @@ func (kpr *Keyper) Start(ctx context.Context, runner service.Runner) error {
chainsync.WithSyncNewKeyperSet(kpr.channelNewKeyperSet),
chainsync.WithPrivateKey(kpr.config.Chain.Node.PrivateKey.Key),
chainsync.WithLogger(gethLog.NewLogger(slog.Default().Handler())),
chainsync.WithKeyperSetKnownRanges(chainsyncer.IndicesToRanges(ksIndices)),
)
if err != nil {
return err
Expand Down
27 changes: 18 additions & 9 deletions rolling-shutter/medley/chainsync/options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,11 @@ type options struct {
syncStart *number.BlockNumber
privKey *ecdsa.PrivateKey

handlerShutterState event.ShutterStateHandler
handlerKeyperSet event.KeyperSetHandler
handlerEonPublicKey event.EonPublicKeyHandler
handlerBlock event.BlockHandler
handlerShutterState event.ShutterStateHandler
handlerKeyperSet event.KeyperSetHandler
handlerEonPublicKey event.EonPublicKeyHandler
handlerBlock event.BlockHandler
knownKeyperSetRanges []syncer.IndexRange
}

func (o *options) verify() error {
Expand Down Expand Up @@ -85,11 +86,12 @@ func (o *options) apply(ctx context.Context, c *Client) error {
return err
}
c.kssync = &syncer.KeyperSetSyncer{
Client: client,
Contract: c.KeyperSetManager,
Log: c.log,
StartBlock: o.syncStart,
Handler: o.handlerKeyperSet,
Client: client,
Contract: c.KeyperSetManager,
Log: c.log,
StartBlock: o.syncStart,
Handler: o.handlerKeyperSet,
KnownRanges: o.knownKeyperSetRanges,
}
if o.handlerKeyperSet != nil {
c.services = append(c.services, c.kssync)
Expand Down Expand Up @@ -214,6 +216,13 @@ func WithSyncNewKeyperSet(handler event.KeyperSetHandler) Option {
}
}

func WithKeyperSetKnownRanges(ranges []syncer.IndexRange) Option {
return func(o *options) error {
o.knownKeyperSetRanges = ranges
return nil
}
}

func WithSyncNewBlock(handler event.BlockHandler) Option {
return func(o *options) error {
o.handlerBlock = handler
Expand Down
39 changes: 14 additions & 25 deletions rolling-shutter/medley/chainsync/syncer/keyperset.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,12 @@ func makeCallError(attrName string, err error) error {
const channelSize = 10

type KeyperSetSyncer struct {
Client client.Client
Contract *bindings.KeyperSetManager
Log log.Logger
StartBlock *number.BlockNumber
Handler event.KeyperSetHandler
Client client.Client
Contract *bindings.KeyperSetManager
Log log.Logger
StartBlock *number.BlockNumber
Handler event.KeyperSetHandler
KnownRanges []IndexRange

keyperAddedCh chan *bindings.KeyperSetManagerKeyperSetAdded
}
Expand Down Expand Up @@ -89,34 +90,22 @@ func (s *KeyperSetSyncer) getInitialKeyperSets(ctx context.Context) ([]*event.Ke
if err := guardCallOpts(opts, false); err != nil {
return nil, err
}
bn := s.StartBlock.ToUInt64Ptr()
if bn == nil {
// this should not be the case
return nil, errors.New("start block is 'latest'")
}

initialKeyperSets := []*event.KeyperSet{}
// this blocknumber specifies the argument to the contract
// getter
ks, err := s.GetKeyperSetForBlock(ctx, opts, s.StartBlock)
if err != nil {
return nil, err
}
initialKeyperSets = append(initialKeyperSets, ks)

numKS, err := s.Contract.GetNumKeyperSets(opts)
if err != nil {
return nil, err
}

for i := ks.Eon + 1; i < numKS; i++ {
ks, err = s.GetKeyperSetByIndex(ctx, opts, i)
if err != nil {
return nil, err
var initialKeyperSets []*event.KeyperSet
for _, r := range complementRanges(s.KnownRanges, numKS) {

@ylembachar ylembachar Jun 2, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we perhaps avoid complementRanges and just loop over the contract indices? It would be simpler to read and understand, and would reduce the amount of logic and test coverage needed here.

Since numKS is the contract count, we can do for i := uint64(0); i < numKS; i++, skip indices already present in the DB, and fetch the rest. That also guarantees we never fetch outside [0, numKS), which can happen with the current logic if the known DB ranges are ahead of the contract snapshot.

for i := r.Start; i <= r.End; i++ {
ks, err := s.GetKeyperSetByIndex(ctx, opts, i)
if err != nil {
return nil, err
}
initialKeyperSets = append(initialKeyperSets, ks)
}
initialKeyperSets = append(initialKeyperSets, ks)
}

return initialKeyperSets, nil
}

Expand Down
48 changes: 48 additions & 0 deletions rolling-shutter/medley/chainsync/syncer/range.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package syncer

// IndexRange is a closed interval [Start, End] of keyper set indices.
type IndexRange struct {
Start uint64
End uint64
}

// IndicesToRanges converts a sorted slice of int64 indices into contiguous ranges.
// Indices must be non-negative; this matches the int64 type returned by sqlc for bigint columns.
func IndicesToRanges(indices []int64) []IndexRange {
if len(indices) == 0 {
return nil
}
var ranges []IndexRange
start := uint64(indices[0]) //nolint:gosec
end := start
for _, idx := range indices[1:] {
v := uint64(idx) //nolint:gosec
if v == end+1 {
end = v
} else {
ranges = append(ranges, IndexRange{Start: start, End: end})
start = v
end = v
}
}
return append(ranges, IndexRange{Start: start, End: end})
}

// complementRanges returns the ranges in [0, total) not covered by known.
// known must be sorted and non-overlapping.
func complementRanges(known []IndexRange, total uint64) []IndexRange {
var result []IndexRange
cursor := uint64(0)
for _, r := range known {
if r.Start > cursor {
result = append(result, IndexRange{Start: cursor, End: r.Start - 1})
}
if r.End+1 > cursor {
cursor = r.End + 1
}
}
if cursor < total {
result = append(result, IndexRange{Start: cursor, End: total - 1})
}
return result
}
Loading