Skip to content
Open
Show file tree
Hide file tree
Changes from 9 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
File renamed without changes.
File renamed without changes.
File renamed without changes.
49 changes: 34 additions & 15 deletions pkg/clientinfo/performance.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,15 +37,12 @@ type PerformanceMetrics struct {
registry *Registry
cancel context.CancelFunc

// Counters track cumulative counts of events
countersMutex sync.RWMutex
counters map[string]*counter

// Histograms track distributions of values (like durations)
histogramsMutex sync.RWMutex
histograms map[string]*histogram

// Gauges track current values (like queue sizes)
gaugesMutex sync.RWMutex
gauges map[string]*gauge
}
Expand Down Expand Up @@ -102,7 +99,16 @@ func (pm *PerformanceMetrics) Stop() {
// registerAllMetrics registers all performance metrics with 0 values
// so they appear in the /metrics endpoint even before operations occur.
func (pm *PerformanceMetrics) registerAllMetrics() {
// Register all counter metrics with 0 initial value
pm.registerCounterMetrics()
pm.registerWalletActionMetrics()
pm.registerHistogramMetrics()
pm.registerGaugeMetrics()
}

// registerCounterMetrics registers all counter metrics with 0 initial values.
// Map entries are populated before observers are registered so that observer
// callbacks never read the map while it is being written concurrently.
func (pm *PerformanceMetrics) registerCounterMetrics() {
counters := []string{
MetricDKGJoinedTotal,
MetricDKGFailedTotal,
Expand All @@ -119,6 +125,9 @@ func (pm *PerformanceMetrics) registerAllMetrics() {
MetricRedemptionProofSubmissionsTotal,
MetricRedemptionProofSubmissionsSuccessTotal,
MetricRedemptionProofSubmissionsFailedTotal,
MetricDepositSweepProofSubmissionsTotal,
MetricDepositSweepProofSubmissionsSuccessTotal,
MetricDepositSweepProofSubmissionsFailedTotal,
MetricWalletActionsTotal,
MetricWalletActionSuccessTotal,
MetricWalletActionFailedTotal,
Expand Down Expand Up @@ -147,14 +156,12 @@ func (pm *PerformanceMetrics) registerAllMetrics() {
counters = append(counters, NetworkJoinFailureMetricName(reason))
}

// First, initialize all counters in the map
pm.countersMutex.Lock()
for _, name := range counters {
pm.counters[name] = &counter{value: 0}
}
pm.countersMutex.Unlock()

// Then, register observers (this prevents concurrent map read/write)
for _, name := range counters {
metricName := name // Capture for closure
pm.registry.ObserveApplicationSource(
Expand All @@ -175,7 +182,11 @@ func (pm *PerformanceMetrics) registerAllMetrics() {
)
}

// Register per-action type wallet metrics
}

// registerWalletActionMetrics registers per-action-type wallet counters and
// duration histograms with 0 initial values.
func (pm *PerformanceMetrics) registerWalletActionMetrics() {
// For each action type, register: total, success_total, failed_total, duration_seconds
for _, actionType := range GetAllWalletActionTypes() {
actionCounters := []string{
Expand Down Expand Up @@ -236,8 +247,12 @@ func (pm *PerformanceMetrics) registerAllMetrics() {
)
}

// Register all duration/histogram metrics with 0 initial values
// Note: These use the actual metric names as used in the codebase
}

// registerHistogramMetrics registers standalone duration/histogram metrics with
// 0 initial values.
func (pm *PerformanceMetrics) registerHistogramMetrics() {
// These use the actual metric names as used in the codebase.
durationMetrics := []string{
MetricDKGDurationSeconds,
MetricSigningDurationSeconds,
Expand All @@ -249,7 +264,6 @@ func (pm *PerformanceMetrics) registerAllMetrics() {
MetricNetworkHandshakeDurationSeconds,
}

// First, initialize all histograms in the map
pm.histogramsMutex.Lock()
for _, name := range durationMetrics {
pm.histograms[name] = &histogram{
Expand All @@ -258,7 +272,6 @@ func (pm *PerformanceMetrics) registerAllMetrics() {
}
pm.histogramsMutex.Unlock()

// Then, register observers (this prevents concurrent map read/write)
for _, name := range durationMetrics {
metricName := name
sources := map[string]Source{
Expand Down Expand Up @@ -295,7 +308,10 @@ func (pm *PerformanceMetrics) registerAllMetrics() {
pm.registry.ObserveApplicationSource("performance", sources)
}

// Register all gauge metrics with 0 initial value
}

// registerGaugeMetrics registers all gauge metrics with 0 initial values.
func (pm *PerformanceMetrics) registerGaugeMetrics() {
gauges := []string{
MetricWalletDispatcherActiveActions,
MetricIncomingMessageQueueSize,
Expand All @@ -309,14 +325,12 @@ func (pm *PerformanceMetrics) registerAllMetrics() {
MetricSwapUtilizationPercent,
}

// First, initialize all gauges in the map
pm.gaugesMutex.Lock()
for _, name := range gauges {
pm.gauges[name] = &gauge{value: 0}
}
pm.gaugesMutex.Unlock()

// Then, register observers (this prevents concurrent map read/write)
for _, name := range gauges {
metricName := name // Capture for closure
pm.registry.ObserveApplicationSource(
Expand Down Expand Up @@ -433,7 +447,7 @@ func (pm *PerformanceMetrics) SetGauge(name string, value float64) {
// observeSystemMetrics periodically collects and updates system metrics
// including CPU utilization, memory usage, and goroutine count.
func (pm *PerformanceMetrics) observeSystemMetrics(ctx context.Context) {
ticker := time.NewTicker(60 * time.Second) // Update every 10 seconds
ticker := time.NewTicker(60 * time.Second) // Update every 60 seconds
defer ticker.Stop()

var lastMemStats runtime.MemStats
Expand Down Expand Up @@ -634,6 +648,11 @@ const (
MetricRedemptionProofSubmissionsSuccessTotal = "redemption_proof_submissions_success_total"
MetricRedemptionProofSubmissionsFailedTotal = "redemption_proof_submissions_failed_total"

// Deposit Sweep Proof Submission Metrics (SPV maintainer)
MetricDepositSweepProofSubmissionsTotal = "deposit_sweep_proof_submissions_total"
MetricDepositSweepProofSubmissionsSuccessTotal = "deposit_sweep_proof_submissions_success_total"
MetricDepositSweepProofSubmissionsFailedTotal = "deposit_sweep_proof_submissions_failed_total"

// Wallet Action Metrics (aggregate)
MetricWalletActionsTotal = "wallet_actions_total"
MetricWalletActionSuccessTotal = "wallet_action_success_total"
Expand Down
24 changes: 10 additions & 14 deletions pkg/maintainer/spv/deposit_sweep.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/keep-network/keep-core/pkg/bitcoin"
"github.com/keep-network/keep-core/pkg/chain"
"github.com/keep-network/keep-core/pkg/clientinfo"
)

// SubmitDepositSweepProof prepares deposit sweep proof for the given
Expand All @@ -26,7 +27,7 @@ func SubmitDepositSweepProof(
btcChain,
spvChain,
bitcoin.AssembleSpvProof,
getGlobalMetricsRecorder(),
getMetricsRecorder(),
)
}

Expand All @@ -42,12 +43,12 @@ func submitDepositSweepProof(
) error {
// Record proof submission attempt
if metricsRecorder != nil {
metricsRecorder.IncrementCounter("deposit_sweep_proof_submissions_total", 1)
metricsRecorder.IncrementCounter(clientinfo.MetricDepositSweepProofSubmissionsTotal, 1)
}

if requiredConfirmations == 0 {
if metricsRecorder != nil {
metricsRecorder.IncrementCounter("deposit_sweep_proof_submissions_failed_total", 1)
metricsRecorder.IncrementCounter(clientinfo.MetricDepositSweepProofSubmissionsFailedTotal, 1)
}
return fmt.Errorf(
"provided required confirmations count must be greater than 0",
Expand All @@ -61,7 +62,7 @@ func submitDepositSweepProof(
)
if err != nil {
if metricsRecorder != nil {
metricsRecorder.IncrementCounter("deposit_sweep_proof_submissions_failed_total", 1)
metricsRecorder.IncrementCounter(clientinfo.MetricDepositSweepProofSubmissionsFailedTotal, 1)
}
return fmt.Errorf(
"failed to assemble transaction spv proof: [%v]",
Expand All @@ -76,7 +77,7 @@ func submitDepositSweepProof(
)
if err != nil {
if metricsRecorder != nil {
metricsRecorder.IncrementCounter("deposit_sweep_proof_submissions_failed_total", 1)
metricsRecorder.IncrementCounter(clientinfo.MetricDepositSweepProofSubmissionsFailedTotal, 1)
}
return fmt.Errorf(
"error while parsing transaction inputs: [%v]",
Expand All @@ -91,7 +92,7 @@ func submitDepositSweepProof(
vault,
); err != nil {
if metricsRecorder != nil {
metricsRecorder.IncrementCounter("deposit_sweep_proof_submissions_failed_total", 1)
metricsRecorder.IncrementCounter(clientinfo.MetricDepositSweepProofSubmissionsFailedTotal, 1)
}
return fmt.Errorf(
"failed to submit deposit sweep proof with reimbursement: [%v]",
Expand All @@ -101,7 +102,7 @@ func submitDepositSweepProof(

// Record successful proof submission
if metricsRecorder != nil {
metricsRecorder.IncrementCounter("deposit_sweep_proof_submissions_success_total", 1)
metricsRecorder.IncrementCounter(clientinfo.MetricDepositSweepProofSubmissionsSuccessTotal, 1)
}

return nil
Expand All @@ -118,17 +119,12 @@ func parseDepositSweepTransactionInputs(
common.Address,
error,
) {
// Represents the main UTXO of the deposit sweep transaction. Nil if there
// was no main UTXO.
var mainUTXO *bitcoin.UnspentTransactionOutput = nil

// Stores the vault address of the deposits. Each deposit should have the
// same value of vault. The zero-filled value indicates there was no vault
// value set for the deposits.
// Each deposit must have the same vault value. The zero-filled value
// indicates there was no vault set for the deposits.
var vault = common.Address{}

// This flag checks if at least one deposit input has been found during
// deposit processing.
var depositAlreadyProcessed = false

// Perform a sanity check: a deposit sweep transaction must have exactly one
Expand Down
2 changes: 1 addition & 1 deletion pkg/maintainer/spv/deposit_sweep_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ func TestSubmitDepositSweepProof(t *testing.T) {
btcChain,
spvChain,
mockSpvProofAssembler,
getGlobalMetricsRecorder(),
getMetricsRecorder(),
)
if err != nil {
t.Fatal(err)
Expand Down
9 changes: 1 addition & 8 deletions pkg/maintainer/spv/redemptions.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,6 @@ import (
"github.com/keep-network/keep-core/pkg/tbtc"
)

// getGlobalMetricsRecorder returns the global metrics recorder if set.
func getGlobalMetricsRecorder() interface {
IncrementCounter(name string, value float64)
} {
return getMetricsRecorder()
}

// SubmitRedemptionProof prepares redemption proof for the given transaction
// and submits it to the on-chain contract. If the number of required
// confirmations is `0`, an error is returned.
Expand All @@ -31,7 +24,7 @@ func SubmitRedemptionProof(
btcChain,
spvChain,
bitcoin.AssembleSpvProof,
getGlobalMetricsRecorder(),
getMetricsRecorder(),
)
}

Expand Down
2 changes: 1 addition & 1 deletion pkg/maintainer/spv/redemptions_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ func TestSubmitRedemptionProof(t *testing.T) {
btcChain,
spvChain,
mockSpvProofAssembler,
getGlobalMetricsRecorder(),
getMetricsRecorder(),
)
if err != nil {
t.Fatal(err)
Expand Down
2 changes: 1 addition & 1 deletion pkg/protocol/state/sync_machine.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ func (sm *SyncMachine) Execute(startBlockHeight uint64) (SyncState, uint64, erro
err := sm.blockCounter.WaitForBlockHeight(startBlockHeight)
if err != nil {
cancelCtx()
return nil, 0, fmt.Errorf("failed to wait for the execution start block")
return nil, 0, fmt.Errorf("failed to wait for the execution start block: [%w]", err)
}

lastStateEndBlockHeight := startBlockHeight
Expand Down
22 changes: 7 additions & 15 deletions pkg/tbtc/coordination.go
Original file line number Diff line number Diff line change
Expand Up @@ -380,9 +380,6 @@ func (ce *coordinationExecutor) coordinate(

startTime := time.Now()

// Record duration metric once at the end using defer
var coordinationFailed bool

seed, err := ce.getSeed(window.coordinationBlock)
if err != nil {
return nil, fmt.Errorf("failed to compute coordination seed: [%v]", err)
Expand Down Expand Up @@ -431,7 +428,6 @@ func (ce *coordinationExecutor) coordinate(
// no point to keep the context active as retransmissions do not
// occur anyway.
cancelCtx()
coordinationFailed = true
if ce.metricsRecorder != nil {
ce.metricsRecorder.IncrementCounter(clientinfo.MetricCoordinationFailedTotal, 1)
}
Expand All @@ -455,7 +451,6 @@ func (ce *coordinationExecutor) coordinate(
append(actionsChecklist, ActionNoop),
)
if err != nil {
coordinationFailed = true
// Record as leader timeout observation, not as a failure of this node.
// The actual failure is on the leader's side.
if ce.metricsRecorder != nil {
Expand Down Expand Up @@ -498,7 +493,7 @@ func (ce *coordinationExecutor) coordinate(
execLogger.Infof("coordination completed with result: [%s]", result)

// Record successful coordination counter
if ce.metricsRecorder != nil && !coordinationFailed {
if ce.metricsRecorder != nil {
ce.metricsRecorder.IncrementCounter(clientinfo.MetricCoordinationProceduresExecutedTotal, 1)
ce.metricsRecorder.RecordDuration(clientinfo.MetricCoordinationDurationSeconds, time.Since(startTime))
}
Expand Down Expand Up @@ -608,15 +603,12 @@ func (ce *coordinationExecutor) getActionsChecklist(
// proposal generator performs a full-history chain scan.
if coordinationBlock < DepositSweepEveryWindowActivationBlock {
if windowIndex%frequencyWindows == 0 {
actions = append(actions, ActionDepositSweep)
}

if windowIndex%frequencyWindows == 0 {
actions = append(actions, ActionMovedFundsSweep)
}

if windowIndex%frequencyWindows == 0 {
actions = append(actions, ActionMovingFunds)
actions = append(
actions,
ActionDepositSweep,
ActionMovedFundsSweep,
ActionMovingFunds,
)
}
} else {
actions = append(actions, ActionDepositSweep)
Expand Down
3 changes: 0 additions & 3 deletions pkg/tbtc/coordination_window_metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -218,16 +218,13 @@ func (cwm *coordinationWindowMetrics) recordWalletCoordination(
wm.WalletsFailed++
}

// Track leader
leaderStr := leader.String()
wm.Leaders[leaderStr]++

// Track action type
if actionType != "" {
wm.ActionTypes[actionType]++
}

// Track faults
faultDetails := make([]faultDetail, 0, len(faults))
for _, fault := range faults {
faultTypeStr := fault.faultType.String()
Expand Down
11 changes: 7 additions & 4 deletions pkg/tbtc/deposit_sweep.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,13 +51,16 @@ const (
depositSweepBroadcastCheckDelay = 1 * time.Minute
)

// DepositKey identifies a deposit by the outpoint of its funding transaction.
type DepositKey struct {
FundingTxHash bitcoin.Hash
FundingOutputIndex uint32
}

// DepositSweepProposal represents a deposit sweep proposal issued by a
// wallet's coordination leader.
type DepositSweepProposal struct {
DepositsKeys []struct {
FundingTxHash bitcoin.Hash
FundingOutputIndex uint32
}
DepositsKeys []DepositKey
SweepTxFee *big.Int
DepositsRevealBlocks []*big.Int
}
Expand Down
Loading
Loading