replay: adding counters for replay-blocking conditions

Author: ibhatt-jumptrading

book/api/metrics-generated.md

@@ -528,6 +528,10 @@
 | <span class="metrics-name">replay_&#8203;live_&#8203;banks</span> | gauge | The number of banks we currently have alive |
 | <span class="metrics-name">replay_&#8203;slots_&#8203;total</span> | counter | Count of slots replayed successfully |
 | <span class="metrics-name">replay_&#8203;transactions_&#8203;total</span> | counter | Count of transactions processed overall on the current fork |
+| <span class="metrics-name">replay_&#8203;sched_&#8203;full</span> | counter | Times where sched is full and a FEC set can't be processed |
+| <span class="metrics-name">replay_&#8203;reasm_&#8203;empty</span> | counter | Times where reasm is empty and a FEC set can't be processed |
+| <span class="metrics-name">replay_&#8203;leader_&#8203;bid_&#8203;wait</span> | counter | Times where replay is blocked by the the PoH tile not sending an end of leader message |
+| <span class="metrics-name">replay_&#8203;banks_&#8203;full</span> | counter | Times where banks are full and a FEC set can't be processed |
 | <span class="metrics-name">replay_&#8203;progcache_&#8203;rooted</span> | counter | Number of program cache entries rooted |
 | <span class="metrics-name">replay_&#8203;progcache_&#8203;gc_&#8203;root</span> | counter | Number of program cache entries garbage collected while rooting |
 | <span class="metrics-name">replay_&#8203;accdb_&#8203;rooted</span> | counter | Number of account database entries rooted |

src/disco/metrics/generated/fd_metrics_replay.c

@@ -17,6 +17,10 @@ const fd_metrics_meta_t FD_METRICS_REPLAY[FD_METRICS_REPLAY_TOTAL] = {
     DECLARE_METRIC( REPLAY_LIVE_BANKS, GAUGE ),
     DECLARE_METRIC( REPLAY_SLOTS_TOTAL, COUNTER ),
     DECLARE_METRIC( REPLAY_TRANSACTIONS_TOTAL, COUNTER ),
+    DECLARE_METRIC( REPLAY_SCHED_FULL, COUNTER ),
+    DECLARE_METRIC( REPLAY_REASM_EMPTY, COUNTER ),
+    DECLARE_METRIC( REPLAY_LEADER_BID_WAIT, COUNTER ),
+    DECLARE_METRIC( REPLAY_BANKS_FULL, COUNTER ),
     DECLARE_METRIC( REPLAY_PROGCACHE_ROOTED, COUNTER ),
     DECLARE_METRIC( REPLAY_PROGCACHE_GC_ROOT, COUNTER ),
     DECLARE_METRIC( REPLAY_ACCDB_ROOTED, COUNTER ),

src/disco/metrics/generated/fd_metrics_replay.h

@@ -108,31 +108,55 @@
 #define FD_METRICS_COUNTER_REPLAY_TRANSACTIONS_TOTAL_DESC "Count of transactions processed overall on the current fork"
 #define FD_METRICS_COUNTER_REPLAY_TRANSACTIONS_TOTAL_CVT  (FD_METRICS_CONVERTER_NONE)
 
-#define FD_METRICS_COUNTER_REPLAY_PROGCACHE_ROOTED_OFF  (127UL)
+#define FD_METRICS_COUNTER_REPLAY_SCHED_FULL_OFF  (127UL)
+#define FD_METRICS_COUNTER_REPLAY_SCHED_FULL_NAME "replay_sched_full"
+#define FD_METRICS_COUNTER_REPLAY_SCHED_FULL_TYPE (FD_METRICS_TYPE_COUNTER)
+#define FD_METRICS_COUNTER_REPLAY_SCHED_FULL_DESC "Times where sched is full and a FEC set can't be processed"
+#define FD_METRICS_COUNTER_REPLAY_SCHED_FULL_CVT  (FD_METRICS_CONVERTER_NONE)
+
+#define FD_METRICS_COUNTER_REPLAY_REASM_EMPTY_OFF  (128UL)
+#define FD_METRICS_COUNTER_REPLAY_REASM_EMPTY_NAME "replay_reasm_empty"
+#define FD_METRICS_COUNTER_REPLAY_REASM_EMPTY_TYPE (FD_METRICS_TYPE_COUNTER)
+#define FD_METRICS_COUNTER_REPLAY_REASM_EMPTY_DESC "Times where reasm is empty and a FEC set can't be processed"
+#define FD_METRICS_COUNTER_REPLAY_REASM_EMPTY_CVT  (FD_METRICS_CONVERTER_NONE)
+
+#define FD_METRICS_COUNTER_REPLAY_LEADER_BID_WAIT_OFF  (129UL)
+#define FD_METRICS_COUNTER_REPLAY_LEADER_BID_WAIT_NAME "replay_leader_bid_wait"
+#define FD_METRICS_COUNTER_REPLAY_LEADER_BID_WAIT_TYPE (FD_METRICS_TYPE_COUNTER)
+#define FD_METRICS_COUNTER_REPLAY_LEADER_BID_WAIT_DESC "Times where replay is blocked by the the PoH tile not sending an end of leader message"
+#define FD_METRICS_COUNTER_REPLAY_LEADER_BID_WAIT_CVT  (FD_METRICS_CONVERTER_NONE)
+
+#define FD_METRICS_COUNTER_REPLAY_BANKS_FULL_OFF  (130UL)
+#define FD_METRICS_COUNTER_REPLAY_BANKS_FULL_NAME "replay_banks_full"
+#define FD_METRICS_COUNTER_REPLAY_BANKS_FULL_TYPE (FD_METRICS_TYPE_COUNTER)
+#define FD_METRICS_COUNTER_REPLAY_BANKS_FULL_DESC "Times where banks are full and a FEC set can't be processed"
+#define FD_METRICS_COUNTER_REPLAY_BANKS_FULL_CVT  (FD_METRICS_CONVERTER_NONE)
+
+#define FD_METRICS_COUNTER_REPLAY_PROGCACHE_ROOTED_OFF  (131UL)
 #define FD_METRICS_COUNTER_REPLAY_PROGCACHE_ROOTED_NAME "replay_progcache_rooted"
 #define FD_METRICS_COUNTER_REPLAY_PROGCACHE_ROOTED_TYPE (FD_METRICS_TYPE_COUNTER)
 #define FD_METRICS_COUNTER_REPLAY_PROGCACHE_ROOTED_DESC "Number of program cache entries rooted"
 #define FD_METRICS_COUNTER_REPLAY_PROGCACHE_ROOTED_CVT  (FD_METRICS_CONVERTER_NONE)
 
-#define FD_METRICS_COUNTER_REPLAY_PROGCACHE_GC_ROOT_OFF  (128UL)
+#define FD_METRICS_COUNTER_REPLAY_PROGCACHE_GC_ROOT_OFF  (132UL)
 #define FD_METRICS_COUNTER_REPLAY_PROGCACHE_GC_ROOT_NAME "replay_progcache_gc_root"
 #define FD_METRICS_COUNTER_REPLAY_PROGCACHE_GC_ROOT_TYPE (FD_METRICS_TYPE_COUNTER)
 #define FD_METRICS_COUNTER_REPLAY_PROGCACHE_GC_ROOT_DESC "Number of program cache entries garbage collected while rooting"
 #define FD_METRICS_COUNTER_REPLAY_PROGCACHE_GC_ROOT_CVT  (FD_METRICS_CONVERTER_NONE)
 
-#define FD_METRICS_COUNTER_REPLAY_ACCDB_ROOTED_OFF  (129UL)
+#define FD_METRICS_COUNTER_REPLAY_ACCDB_ROOTED_OFF  (133UL)
 #define FD_METRICS_COUNTER_REPLAY_ACCDB_ROOTED_NAME "replay_accdb_rooted"
 #define FD_METRICS_COUNTER_REPLAY_ACCDB_ROOTED_TYPE (FD_METRICS_TYPE_COUNTER)
 #define FD_METRICS_COUNTER_REPLAY_ACCDB_ROOTED_DESC "Number of account database entries rooted"
 #define FD_METRICS_COUNTER_REPLAY_ACCDB_ROOTED_CVT  (FD_METRICS_CONVERTER_NONE)
 
-#define FD_METRICS_COUNTER_REPLAY_ACCDB_GC_ROOT_OFF  (130UL)
+#define FD_METRICS_COUNTER_REPLAY_ACCDB_GC_ROOT_OFF  (134UL)
 #define FD_METRICS_COUNTER_REPLAY_ACCDB_GC_ROOT_NAME "replay_accdb_gc_root"
 #define FD_METRICS_COUNTER_REPLAY_ACCDB_GC_ROOT_TYPE (FD_METRICS_TYPE_COUNTER)
 #define FD_METRICS_COUNTER_REPLAY_ACCDB_GC_ROOT_DESC "Number of account database entries garbage collected"
 #define FD_METRICS_COUNTER_REPLAY_ACCDB_GC_ROOT_CVT  (FD_METRICS_CONVERTER_NONE)
 
-#define FD_METRICS_REPLAY_TOTAL (19UL)
+#define FD_METRICS_REPLAY_TOTAL (23UL)
 extern const fd_metrics_meta_t FD_METRICS_REPLAY[FD_METRICS_REPLAY_TOTAL];
 
 #endif /* HEADER_fd_src_disco_metrics_generated_fd_metrics_replay_h */

src/disco/metrics/metrics.xml

@@ -797,6 +797,11 @@ metric introduced.
   <counter name="SlotsTotal" summary="Count of slots replayed successfully" />
   <counter name="TransactionsTotal" summary="Count of transactions processed overall on the current fork" />
 
+  <counter name="SchedFull" summary="Times where sched is full and a FEC set can't be processed" />
+  <counter name="ReasmEmpty" summary="Times where reasm is empty and a FEC set can't be processed" />
+  <counter name="LeaderBidWait" summary="Times where replay is blocked by the the PoH tile not sending an end of leader message" />
+  <counter name="BanksFull" summary="Times where banks are full and a FEC set can't be processed" />
+
   <counter name="ProgcacheRooted" summary="Number of program cache entries rooted" />
   <counter name="ProgcacheGcRoot" summary="Number of program cache entries garbage collected while rooting" />
 

src/discof/replay/fd_replay_tile.c

@@ -406,6 +406,11 @@ struct fd_replay_tile {
 
     ulong slots_total;
     ulong transactions_total;
+
+    ulong sched_full;
+    ulong reasm_empty;
+    ulong leader_bid_wait;
+    ulong banks_full;
   } metrics;
 
   uchar __attribute__((aligned(FD_MULTI_EPOCH_LEADERS_ALIGN))) mleaders_mem[ FD_MULTI_EPOCH_LEADERS_FOOTPRINT ];
@@ -473,6 +478,11 @@ metrics_write( fd_replay_tile_t * ctx ) {
   FD_MCNT_SET( REPLAY, SLOTS_TOTAL, ctx->metrics.slots_total );
   FD_MCNT_SET( REPLAY, TRANSACTIONS_TOTAL, ctx->metrics.transactions_total );
 
+  FD_MCNT_SET( REPLAY, SCHED_FULL, ctx->metrics.sched_full );
+  FD_MCNT_SET( REPLAY, REASM_EMPTY, ctx->metrics.reasm_empty );
+  FD_MCNT_SET( REPLAY, LEADER_BID_WAIT, ctx->metrics.leader_bid_wait );
+  FD_MCNT_SET( REPLAY, BANKS_FULL, ctx->metrics.banks_full );
+
   FD_MCNT_SET( REPLAY, PROGCACHE_ROOTED,  ctx->progcache_admin->metrics.root_cnt );
   FD_MCNT_SET( REPLAY, PROGCACHE_GC_ROOT, ctx->progcache_admin->metrics.gc_root_cnt );
 
@@ -1600,8 +1610,15 @@ replay( fd_replay_tile_t *  ctx,
 static int
 can_process_fec( fd_replay_tile_t * ctx ) {
   fd_reasm_fec_t * fec;
-  if( FD_UNLIKELY( !fd_sched_can_ingest( ctx->sched, 1UL ) ) ) return 0;
-  if( FD_UNLIKELY( (fec = fd_reasm_peek( ctx->reasm ))==NULL ) ) return 0;
+  if( FD_UNLIKELY( !fd_sched_can_ingest( ctx->sched, 1UL ) ) ) {
+    ctx->metrics.sched_full++;
+    return 0;
+  }
+
+  if( FD_UNLIKELY( (fec = fd_reasm_peek( ctx->reasm ))==NULL ) ) {
+    ctx->metrics.reasm_empty++;
+    return 0;
+  }
 
   if( FD_UNLIKELY( ctx->is_leader && fec->fec_set_idx==0U && fd_reasm_parent( ctx->reasm, fec )->bank_idx==ctx->leader_bank->idx ) ) {
     /* There's a race that's exceedingly rare, where we receive the
@@ -1618,12 +1635,16 @@ can_process_fec( fd_replay_tile_t * ctx ) {
        ordering invariants in banks and sched. */
     FD_TEST( ctx->recv_block_id );
     FD_TEST( !ctx->recv_poh );
+    ctx->metrics.leader_bid_wait++;
     return 0;
   }
 
   /* If fec_set_idx is 0, we need a new bank for a new slot.  Banks must
      not be full in this case. */
-  if( FD_UNLIKELY( fd_banks_is_full( ctx->banks ) && fec->fec_set_idx==0 ) ) return 0;
+  if( FD_UNLIKELY( fd_banks_is_full( ctx->banks ) && fec->fec_set_idx==0 ) ) {
+    ctx->metrics.banks_full++;
+    return 0;
+  }
 
   /* Otherwise, banks may not be full, so we can always create a new
      bank if needed.  Or, if banks are full, the current fec set's
@@ -1897,6 +1918,8 @@ after_credit( fd_replay_tile_t *  ctx,
     *charge_busy = 1;
     *opt_poll_in = 0;
     return;
+  } else {
+
   }
 
   *charge_busy = replay( ctx, stem );