Skip to content

replay: adding better metrics #7583

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ibhatt-jumptrading merged 3 commits into from
Dec 16, 2025
+108 −10
Merged

replay: adding better metrics #7583

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
da1aa8c
replay: adding counters for replay-blocking conditions
ibhatt-jumptrading Dec 15, 2025
2250022
replay: adding reasm tracking metrics
ibhatt-jumptrading Dec 15, 2025
5590120
wip
ibhatt-jumptrading Dec 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions book/api/metrics-generated.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -526,8 +526,15 @@
| <span class="metrics-name">replay_&#8203;reset_&#8203;slot</span> | gauge | The slot at which we last reset the replay stage, or 0 if unknown |
| <span class="metrics-name">replay_&#8203;max_&#8203;live_&#8203;banks</span> | gauge | The maximum number of banks we can have alive |
| <span class="metrics-name">replay_&#8203;live_&#8203;banks</span> | gauge | The number of banks we currently have alive |
| <span class="metrics-name">replay_&#8203;reasm_&#8203;free</span> | gauge | The number of free FEC sets in the reassembly queue |
| <span class="metrics-name">replay_&#8203;reasm_&#8203;latest_&#8203;slot</span> | gauge | Slot of the latest FEC set in the reassembly queue that can be replayed |
| <span class="metrics-name">replay_&#8203;reasm_&#8203;latest_&#8203;fec_&#8203;idx</span> | gauge | FEC set index of the latest FEC set in the reassembly queue that can be replayed |
| <span class="metrics-name">replay_&#8203;slots_&#8203;total</span> | counter | Count of slots replayed successfully |
| <span class="metrics-name">replay_&#8203;transactions_&#8203;total</span> | counter | Count of transactions processed overall on the current fork |
| <span class="metrics-name">replay_&#8203;sched_&#8203;full</span> | counter | Times where sched is full and a FEC set can't be processed |
| <span class="metrics-name">replay_&#8203;reasm_&#8203;empty</span> | counter | Times where reasm is empty and a FEC set can't be processed |
| <span class="metrics-name">replay_&#8203;leader_&#8203;bid_&#8203;wait</span> | counter | Times where replay is blocked by the the PoH tile not sending an end of leader message |
| <span class="metrics-name">replay_&#8203;banks_&#8203;full</span> | counter | Times where banks are full and a FEC set can't be processed |
| <span class="metrics-name">replay_&#8203;progcache_&#8203;rooted</span> | counter | Number of program cache entries rooted |
| <span class="metrics-name">replay_&#8203;progcache_&#8203;gc_&#8203;root</span> | counter | Number of program cache entries garbage collected while rooting |
| <span class="metrics-name">replay_&#8203;accdb_&#8203;rooted</span> | counter | Number of account database entries rooted |
Expand Down
7 changes: 7 additions & 0 deletions src/disco/metrics/generated/fd_metrics_replay.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,15 @@ const fd_metrics_meta_t FD_METRICS_REPLAY[FD_METRICS_REPLAY_TOTAL] = {
DECLARE_METRIC( REPLAY_RESET_SLOT, GAUGE ),
DECLARE_METRIC( REPLAY_MAX_LIVE_BANKS, GAUGE ),
DECLARE_METRIC( REPLAY_LIVE_BANKS, GAUGE ),
DECLARE_METRIC( REPLAY_REASM_FREE, GAUGE ),
DECLARE_METRIC( REPLAY_REASM_LATEST_SLOT, GAUGE ),
DECLARE_METRIC( REPLAY_REASM_LATEST_FEC_IDX, GAUGE ),
DECLARE_METRIC( REPLAY_SLOTS_TOTAL, COUNTER ),
DECLARE_METRIC( REPLAY_TRANSACTIONS_TOTAL, COUNTER ),
DECLARE_METRIC( REPLAY_SCHED_FULL, COUNTER ),
DECLARE_METRIC( REPLAY_REASM_EMPTY, COUNTER ),
DECLARE_METRIC( REPLAY_LEADER_BID_WAIT, COUNTER ),
DECLARE_METRIC( REPLAY_BANKS_FULL, COUNTER ),
DECLARE_METRIC( REPLAY_PROGCACHE_ROOTED, COUNTER ),
DECLARE_METRIC( REPLAY_PROGCACHE_GC_ROOT, COUNTER ),
DECLARE_METRIC( REPLAY_ACCDB_ROOTED, COUNTER ),
Expand Down
56 changes: 49 additions & 7 deletions src/disco/metrics/generated/fd_metrics_replay.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,43 +96,85 @@
#define FD_METRICS_GAUGE_REPLAY_LIVE_BANKS_DESC "The number of banks we currently have alive"
#define FD_METRICS_GAUGE_REPLAY_LIVE_BANKS_CVT (FD_METRICS_CONVERTER_NONE)

#define FD_METRICS_COUNTER_REPLAY_SLOTS_TOTAL_OFF (125UL)
#define FD_METRICS_GAUGE_REPLAY_REASM_FREE_OFF (125UL)
#define FD_METRICS_GAUGE_REPLAY_REASM_FREE_NAME "replay_reasm_free"
#define FD_METRICS_GAUGE_REPLAY_REASM_FREE_TYPE (FD_METRICS_TYPE_GAUGE)
#define FD_METRICS_GAUGE_REPLAY_REASM_FREE_DESC "The number of free FEC sets in the reassembly queue"
#define FD_METRICS_GAUGE_REPLAY_REASM_FREE_CVT (FD_METRICS_CONVERTER_NONE)

#define FD_METRICS_GAUGE_REPLAY_REASM_LATEST_SLOT_OFF (126UL)
#define FD_METRICS_GAUGE_REPLAY_REASM_LATEST_SLOT_NAME "replay_reasm_latest_slot"
#define FD_METRICS_GAUGE_REPLAY_REASM_LATEST_SLOT_TYPE (FD_METRICS_TYPE_GAUGE)
#define FD_METRICS_GAUGE_REPLAY_REASM_LATEST_SLOT_DESC "Slot of the latest FEC set in the reassembly queue that can be replayed"
#define FD_METRICS_GAUGE_REPLAY_REASM_LATEST_SLOT_CVT (FD_METRICS_CONVERTER_NONE)

#define FD_METRICS_GAUGE_REPLAY_REASM_LATEST_FEC_IDX_OFF (127UL)
#define FD_METRICS_GAUGE_REPLAY_REASM_LATEST_FEC_IDX_NAME "replay_reasm_latest_fec_idx"
#define FD_METRICS_GAUGE_REPLAY_REASM_LATEST_FEC_IDX_TYPE (FD_METRICS_TYPE_GAUGE)
#define FD_METRICS_GAUGE_REPLAY_REASM_LATEST_FEC_IDX_DESC "FEC set index of the latest FEC set in the reassembly queue that can be replayed"
#define FD_METRICS_GAUGE_REPLAY_REASM_LATEST_FEC_IDX_CVT (FD_METRICS_CONVERTER_NONE)

#define FD_METRICS_COUNTER_REPLAY_SLOTS_TOTAL_OFF (128UL)
#define FD_METRICS_COUNTER_REPLAY_SLOTS_TOTAL_NAME "replay_slots_total"
#define FD_METRICS_COUNTER_REPLAY_SLOTS_TOTAL_TYPE (FD_METRICS_TYPE_COUNTER)
#define FD_METRICS_COUNTER_REPLAY_SLOTS_TOTAL_DESC "Count of slots replayed successfully"
#define FD_METRICS_COUNTER_REPLAY_SLOTS_TOTAL_CVT (FD_METRICS_CONVERTER_NONE)

#define FD_METRICS_COUNTER_REPLAY_TRANSACTIONS_TOTAL_OFF (126UL)
#define FD_METRICS_COUNTER_REPLAY_TRANSACTIONS_TOTAL_OFF (129UL)
#define FD_METRICS_COUNTER_REPLAY_TRANSACTIONS_TOTAL_NAME "replay_transactions_total"
#define FD_METRICS_COUNTER_REPLAY_TRANSACTIONS_TOTAL_TYPE (FD_METRICS_TYPE_COUNTER)
#define FD_METRICS_COUNTER_REPLAY_TRANSACTIONS_TOTAL_DESC "Count of transactions processed overall on the current fork"
#define FD_METRICS_COUNTER_REPLAY_TRANSACTIONS_TOTAL_CVT (FD_METRICS_CONVERTER_NONE)

#define FD_METRICS_COUNTER_REPLAY_PROGCACHE_ROOTED_OFF (127UL)
#define FD_METRICS_COUNTER_REPLAY_SCHED_FULL_OFF (130UL)
#define FD_METRICS_COUNTER_REPLAY_SCHED_FULL_NAME "replay_sched_full"
#define FD_METRICS_COUNTER_REPLAY_SCHED_FULL_TYPE (FD_METRICS_TYPE_COUNTER)
#define FD_METRICS_COUNTER_REPLAY_SCHED_FULL_DESC "Times where sched is full and a FEC set can't be processed"
#define FD_METRICS_COUNTER_REPLAY_SCHED_FULL_CVT (FD_METRICS_CONVERTER_NONE)

#define FD_METRICS_COUNTER_REPLAY_REASM_EMPTY_OFF (131UL)
#define FD_METRICS_COUNTER_REPLAY_REASM_EMPTY_NAME "replay_reasm_empty"
#define FD_METRICS_COUNTER_REPLAY_REASM_EMPTY_TYPE (FD_METRICS_TYPE_COUNTER)
#define FD_METRICS_COUNTER_REPLAY_REASM_EMPTY_DESC "Times where reasm is empty and a FEC set can't be processed"
#define FD_METRICS_COUNTER_REPLAY_REASM_EMPTY_CVT (FD_METRICS_CONVERTER_NONE)

#define FD_METRICS_COUNTER_REPLAY_LEADER_BID_WAIT_OFF (132UL)
#define FD_METRICS_COUNTER_REPLAY_LEADER_BID_WAIT_NAME "replay_leader_bid_wait"
#define FD_METRICS_COUNTER_REPLAY_LEADER_BID_WAIT_TYPE (FD_METRICS_TYPE_COUNTER)
#define FD_METRICS_COUNTER_REPLAY_LEADER_BID_WAIT_DESC "Times where replay is blocked by the the PoH tile not sending an end of leader message"
#define FD_METRICS_COUNTER_REPLAY_LEADER_BID_WAIT_CVT (FD_METRICS_CONVERTER_NONE)

#define FD_METRICS_COUNTER_REPLAY_BANKS_FULL_OFF (133UL)
#define FD_METRICS_COUNTER_REPLAY_BANKS_FULL_NAME "replay_banks_full"
#define FD_METRICS_COUNTER_REPLAY_BANKS_FULL_TYPE (FD_METRICS_TYPE_COUNTER)
#define FD_METRICS_COUNTER_REPLAY_BANKS_FULL_DESC "Times where banks are full and a FEC set can't be processed"
#define FD_METRICS_COUNTER_REPLAY_BANKS_FULL_CVT (FD_METRICS_CONVERTER_NONE)

#define FD_METRICS_COUNTER_REPLAY_PROGCACHE_ROOTED_OFF (134UL)
#define FD_METRICS_COUNTER_REPLAY_PROGCACHE_ROOTED_NAME "replay_progcache_rooted"
#define FD_METRICS_COUNTER_REPLAY_PROGCACHE_ROOTED_TYPE (FD_METRICS_TYPE_COUNTER)
#define FD_METRICS_COUNTER_REPLAY_PROGCACHE_ROOTED_DESC "Number of program cache entries rooted"
#define FD_METRICS_COUNTER_REPLAY_PROGCACHE_ROOTED_CVT (FD_METRICS_CONVERTER_NONE)

#define FD_METRICS_COUNTER_REPLAY_PROGCACHE_GC_ROOT_OFF (128UL)
#define FD_METRICS_COUNTER_REPLAY_PROGCACHE_GC_ROOT_OFF (135UL)
#define FD_METRICS_COUNTER_REPLAY_PROGCACHE_GC_ROOT_NAME "replay_progcache_gc_root"
#define FD_METRICS_COUNTER_REPLAY_PROGCACHE_GC_ROOT_TYPE (FD_METRICS_TYPE_COUNTER)
#define FD_METRICS_COUNTER_REPLAY_PROGCACHE_GC_ROOT_DESC "Number of program cache entries garbage collected while rooting"
#define FD_METRICS_COUNTER_REPLAY_PROGCACHE_GC_ROOT_CVT (FD_METRICS_CONVERTER_NONE)

#define FD_METRICS_COUNTER_REPLAY_ACCDB_ROOTED_OFF (129UL)
#define FD_METRICS_COUNTER_REPLAY_ACCDB_ROOTED_OFF (136UL)
#define FD_METRICS_COUNTER_REPLAY_ACCDB_ROOTED_NAME "replay_accdb_rooted"
#define FD_METRICS_COUNTER_REPLAY_ACCDB_ROOTED_TYPE (FD_METRICS_TYPE_COUNTER)
#define FD_METRICS_COUNTER_REPLAY_ACCDB_ROOTED_DESC "Number of account database entries rooted"
#define FD_METRICS_COUNTER_REPLAY_ACCDB_ROOTED_CVT (FD_METRICS_CONVERTER_NONE)

#define FD_METRICS_COUNTER_REPLAY_ACCDB_GC_ROOT_OFF (130UL)
#define FD_METRICS_COUNTER_REPLAY_ACCDB_GC_ROOT_OFF (137UL)
#define FD_METRICS_COUNTER_REPLAY_ACCDB_GC_ROOT_NAME "replay_accdb_gc_root"
#define FD_METRICS_COUNTER_REPLAY_ACCDB_GC_ROOT_TYPE (FD_METRICS_TYPE_COUNTER)
#define FD_METRICS_COUNTER_REPLAY_ACCDB_GC_ROOT_DESC "Number of account database entries garbage collected"
#define FD_METRICS_COUNTER_REPLAY_ACCDB_GC_ROOT_CVT (FD_METRICS_CONVERTER_NONE)

#define FD_METRICS_REPLAY_TOTAL (19UL)
#define FD_METRICS_REPLAY_TOTAL (26UL)
extern const fd_metrics_meta_t FD_METRICS_REPLAY[FD_METRICS_REPLAY_TOTAL];

#endif /* HEADER_fd_src_disco_metrics_generated_fd_metrics_replay_h */
9 changes: 9 additions & 0 deletions src/disco/metrics/metrics.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -794,9 +794,18 @@ metric introduced.
<gauge name="MaxLiveBanks" summary="The maximum number of banks we can have alive" />
<gauge name="LiveBanks" summary="The number of banks we currently have alive" />

<gauge name="ReasmFree" summary="The number of free FEC sets in the reassembly queue" />
<gauge name="ReasmLatestSlot" summary="Slot of the latest FEC set in the reassembly queue that can be replayed" />
<gauge name="ReasmLatestFecIdx" summary="FEC set index of the latest FEC set in the reassembly queue that can be replayed" />

<counter name="SlotsTotal" summary="Count of slots replayed successfully" />
<counter name="TransactionsTotal" summary="Count of transactions processed overall on the current fork" />

<counter name="SchedFull" summary="Times where sched is full and a FEC set can't be processed" />
<counter name="ReasmEmpty" summary="Times where reasm is empty and a FEC set can't be processed" />
<counter name="LeaderBidWait" summary="Times where replay is blocked by the PoH tile not sending an end of leader message" />
<counter name="BanksFull" summary="Times where banks are full and a FEC set can't be processed" />

<counter name="ProgcacheRooted" summary="Number of program cache entries rooted" />
<counter name="ProgcacheGcRoot" summary="Number of program cache entries garbage collected while rooting" />

Expand Down
39 changes: 36 additions & 3 deletions src/discof/replay/fd_replay_tile.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -406,6 +406,14 @@ struct fd_replay_tile {

ulong slots_total;
ulong transactions_total;

ulong reasm_latest_slot;
ulong reasm_latest_fec_idx;

ulong sched_full;
ulong reasm_empty;
ulong leader_bid_wait;
ulong banks_full;
} metrics;

uchar __attribute__((aligned(FD_MULTI_EPOCH_LEADERS_ALIGN))) mleaders_mem[ FD_MULTI_EPOCH_LEADERS_FOOTPRINT ];
Expand Down Expand Up @@ -470,9 +478,20 @@ metrics_write( fd_replay_tile_t * ctx ) {
ulong live_banks = fd_banks_pool_max( bank_pool ) - fd_banks_pool_free( bank_pool );
FD_MGAUGE_SET( REPLAY, LIVE_BANKS, live_banks );

ulong reasm_free = fd_reasm_free( ctx->reasm );
FD_MGAUGE_SET( REPLAY, REASM_FREE, reasm_free );

FD_MCNT_SET( REPLAY, SLOTS_TOTAL, ctx->metrics.slots_total );
FD_MCNT_SET( REPLAY, TRANSACTIONS_TOTAL, ctx->metrics.transactions_total );

FD_MGAUGE_SET( REPLAY, REASM_LATEST_SLOT, ctx->metrics.reasm_latest_slot );
FD_MGAUGE_SET( REPLAY, REASM_LATEST_FEC_IDX, ctx->metrics.reasm_latest_fec_idx );

FD_MCNT_SET( REPLAY, SCHED_FULL, ctx->metrics.sched_full );
FD_MCNT_SET( REPLAY, REASM_EMPTY, ctx->metrics.reasm_empty );
FD_MCNT_SET( REPLAY, LEADER_BID_WAIT, ctx->metrics.leader_bid_wait );
FD_MCNT_SET( REPLAY, BANKS_FULL, ctx->metrics.banks_full );

FD_MCNT_SET( REPLAY, PROGCACHE_ROOTED, ctx->progcache_admin->metrics.root_cnt );
FD_MCNT_SET( REPLAY, PROGCACHE_GC_ROOT, ctx->progcache_admin->metrics.gc_root_cnt );

Expand Down Expand Up @@ -1600,8 +1619,18 @@ replay( fd_replay_tile_t * ctx,
static int
can_process_fec( fd_replay_tile_t * ctx ) {
fd_reasm_fec_t * fec;
if( FD_UNLIKELY( !fd_sched_can_ingest( ctx->sched, 1UL ) ) ) return 0;
if( FD_UNLIKELY( (fec = fd_reasm_peek( ctx->reasm ))==NULL ) ) return 0;
if( FD_UNLIKELY( !fd_sched_can_ingest( ctx->sched, 1UL ) ) ) {
ctx->metrics.sched_full++;
return 0;
}

if( FD_UNLIKELY( (fec = fd_reasm_peek( ctx->reasm ))==NULL ) ) {
ctx->metrics.reasm_empty++;
return 0;
}

ctx->metrics.reasm_latest_slot = fec->slot;
ctx->metrics.reasm_latest_fec_idx = fec->fec_set_idx;

if( FD_UNLIKELY( ctx->is_leader && fec->fec_set_idx==0U && fd_reasm_parent( ctx->reasm, fec )->bank_idx==ctx->leader_bank->idx ) ) {
/* There's a race that's exceedingly rare, where we receive the
Expand All @@ -1618,12 +1647,16 @@ can_process_fec( fd_replay_tile_t * ctx ) {
ordering invariants in banks and sched. */
FD_TEST( ctx->recv_block_id );
FD_TEST( !ctx->recv_poh );
ctx->metrics.leader_bid_wait++;
return 0;
}

/* If fec_set_idx is 0, we need a new bank for a new slot. Banks must
not be full in this case. */
if( FD_UNLIKELY( fd_banks_is_full( ctx->banks ) && fec->fec_set_idx==0 ) ) return 0;
if( FD_UNLIKELY( fd_banks_is_full( ctx->banks ) && fec->fec_set_idx==0 ) ) {
ctx->metrics.banks_full++;
return 0;
}

/* Otherwise, banks may not be full, so we can always create a new
bank if needed. Or, if banks are full, the current fec set's
Expand Down
Loading