runtime: removing txn accounts from executor (#7388)

Author: ibhatt-jumptrading

src/discof/bank/fd_bank_tile.c

@@ -274,7 +274,7 @@ handle_microblock( fd_bank_ctx_t *     ctx,
        that first the non-alt accounts are laid out, then the writable
        alt accounts, and finally the read-only alt accounts. */
     fd_txn_t * txn_descriptor = TXN( txn_in->txn );
-    fd_acct_addr_t const * writable_alt = fd_type_pun_const( txn_out->accounts.account_keys+txn_descriptor->acct_addr_cnt );
+    fd_acct_addr_t const * writable_alt = fd_type_pun_const( txn_out->accounts.keys+txn_descriptor->acct_addr_cnt );
     if( FD_LIKELY( ctx->enable_rebates ) ) fd_pack_rebate_sum_add_txn( ctx->rebater, txn, &writable_alt, 1UL );
 
     /* The VM will stop executing and fail an instruction immediately if
@@ -390,7 +390,7 @@ handle_bundle( fd_bank_ctx_t *     ctx,
       continue;
     }
 
-    writable_alt[i] = fd_type_pun_const( txn_out->accounts.account_keys+TXN( txn_in->txn )->acct_addr_cnt );
+    writable_alt[i] = fd_type_pun_const( txn_out->accounts.keys+TXN( txn_in->txn )->acct_addr_cnt );
   }
 
   /* If all of the transactions in the bundle executed successfully, we
@@ -597,15 +597,15 @@ unprivileged_init( fd_topo_t *      topo,
     }
   }
 
-  ctx->runtime->accdb = accdb;
-  ctx->runtime->funk = fd_accdb_user_v1_funk( accdb );
-  ctx->runtime->progcache = progcache;
-  ctx->runtime->status_cache = txncache;
-  ctx->runtime->log.log_collector = ctx->log_collector;
+  ctx->runtime->accdb                    = accdb;
+  ctx->runtime->funk                     = fd_accdb_user_v1_funk( accdb );
+  ctx->runtime->progcache                = progcache;
+  ctx->runtime->status_cache             = txncache;
+  ctx->runtime->log.log_collector        = ctx->log_collector;
   ctx->runtime->log.enable_log_collector = 0;
-  ctx->runtime->log.capture_ctx = NULL;
-  ctx->runtime->log.dumping_mem = NULL;
-  ctx->runtime->log.tracing_mem = NULL;
+  ctx->runtime->log.capture_ctx          = NULL;
+  ctx->runtime->log.dumping_mem          = NULL;
+  ctx->runtime->log.tracing_mem          = NULL;
 
   ulong banks_obj_id = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "banks" );
   FD_TEST( banks_obj_id!=ULONG_MAX );

src/flamenco/fd_flamenco_base.h

@@ -116,6 +116,11 @@ struct fd_account_meta {
 };
 typedef struct fd_account_meta fd_account_meta_t;
 
+FD_FN_PURE static inline uchar *
+fd_account_data( fd_account_meta_t const * acc ) {
+  return (uchar *)( acc+1 );
+}
+
 FD_PROTOTYPES_BEGIN
 
 /* fd_acct_addr_cstr converts the given Solana address into a base58-

src/flamenco/log_collector/fd_log_collector.h

@@ -387,7 +387,7 @@ fd_log_collector_program_invoke( fd_exec_instr_ctx_t * ctx ) {
     return;
   }
 
-  fd_pubkey_t const * program_id_pubkey = &ctx->txn_out->accounts.account_keys[ ctx->instr->program_id ];
+  fd_pubkey_t const * program_id_pubkey = &ctx->txn_out->accounts.keys[ ctx->instr->program_id ];
   /* Cache ctx->program_id_base58 */
   fd_base58_encode_32( program_id_pubkey->uc, NULL, ctx->program_id_base58 );
   /* Max msg_sz: 22 - 4 + 44 + 10 = 72 < 127 => we can use printf */

src/flamenco/progcache/fd_prog_load.c

@@ -18,7 +18,7 @@ fd_get_executable_program_content_for_v4_loader( fd_txn_account_t const * progra
   int err;
 
   /* Get the current loader v4 state. This implicitly also checks the dlen. */
-  fd_loader_v4_state_t const * state = fd_loader_v4_get_state( program_acc, &err );
+  fd_loader_v4_state_t const * state = fd_loader_v4_get_state( program_acc->meta, &err );
   if( FD_UNLIKELY( err ) ) {
     return NULL;
   }

src/flamenco/progcache/test_progcache_common.c

@@ -140,8 +140,6 @@ create_test_account( test_env_t * env,
     fd_txn_account_set_data( acc, data, data_len );
   }
 
-  acc->starting_lamports = 1UL;
-  acc->starting_dlen     = data_len;
   fd_txn_account_set_lamports( acc, 1UL );
   fd_txn_account_set_executable( acc, executable );
   fd_txn_account_set_owner( acc, &owner );

src/flamenco/rewards/fd_rewards.c

@@ -822,7 +822,7 @@ calculate_rewards_and_distribute_vote_rewards( fd_bank_t *                    ba
       FD_LOG_ERR(( "Adding lamports to vote account would cause overflow" ));
     }
 
-    fd_hashes_update_lthash( vote_rec, prev_hash,bank, capture_ctx );
+    fd_hashes_update_lthash( vote_rec->pubkey, vote_rec->meta, prev_hash,bank, capture_ctx );
     fd_txn_account_mutable_fini( vote_rec, accdb, &prepare );
 
     distributed_rewards = fd_ulong_sat_add( distributed_rewards, rewards );
@@ -877,7 +877,7 @@ distribute_epoch_reward_to_stake_acc( fd_bank_t *               bank,
   fd_txn_account_set_slot( stake_acc_rec, fd_bank_slot_get( bank ) );
 
   fd_stake_state_v2_t stake_state[1] = {0};
-  if( fd_stake_get_state( stake_acc_rec, stake_state ) != 0 ) {
+  if( fd_stake_get_state( stake_acc_rec->meta, stake_state ) != 0 ) {
     FD_LOG_DEBUG(( "failed to read stake state for %s", FD_BASE58_ENC_32_ALLOCA( stake_pubkey ) ));
     return 1;
   }
@@ -927,7 +927,7 @@ distribute_epoch_reward_to_stake_acc( fd_bank_t *               bank,
     FD_LOG_ERR(( "write_stake_state failed" ));
   }
 
-  fd_hashes_update_lthash( stake_acc_rec, prev_hash, bank, capture_ctx );
+  fd_hashes_update_lthash( stake_acc_rec->pubkey, stake_acc_rec->meta, prev_hash, bank, capture_ctx );
   fd_txn_account_mutable_fini( stake_acc_rec, accdb, &prepare );
 
   return 0;

src/flamenco/runtime/context/fd_exec_instr_ctx.c

@@ -7,7 +7,7 @@ fd_exec_instr_ctx_find_idx_of_instr_account( fd_exec_instr_ctx_t const * ctx,
                                              fd_pubkey_t const *         pubkey ) {
   for( int i=0; i<ctx->instr->acct_cnt; i++ ) {
     ushort idx_in_txn = ctx->instr->accounts[ i ].index_in_transaction;
-    if( memcmp( pubkey->uc, ctx->txn_out->accounts.account_keys[ idx_in_txn ].uc, sizeof(fd_pubkey_t) )==0 ) {
+    if( memcmp( pubkey->uc, ctx->txn_out->accounts.keys[ idx_in_txn ].uc, sizeof(fd_pubkey_t) )==0 ) {
       return i;
     }
   }
@@ -46,11 +46,9 @@ fd_exec_instr_ctx_try_borrow_account( fd_exec_instr_ctx_t const * ctx,
                                       fd_borrowed_account_t *     account ) {
   /* Get the account from the transaction context using idx_in_txn.
      https://github.com/anza-xyz/agave/blob/v2.1.14/sdk/src/transaction_context.rs#L600-L602 */
-  fd_txn_account_t * txn_account = NULL;
   int err = fd_runtime_get_account_at_index( ctx->txn_in,
                                              ctx->txn_out,
                                              idx_in_txn,
-                                             &txn_account,
                                              NULL );
   if( FD_UNLIKELY( err ) ) {
     /* Return a MissingAccount error if the account is not found.
@@ -59,19 +57,23 @@ fd_exec_instr_ctx_try_borrow_account( fd_exec_instr_ctx_t const * ctx,
     return FD_EXECUTOR_INSTR_ERR_MISSING_ACC;
   }
 
+  fd_account_meta_t * meta = ctx->txn_out->accounts.metas[idx_in_txn];
+
   /* Return an AccountBorrowFailed error if the write is not acquirable.
      https://github.com/anza-xyz/agave/blob/v2.1.14/sdk/src/transaction_context.rs#L605 */
-  int borrow_res = fd_txn_account_try_borrow_mut( txn_account );
-  if( FD_UNLIKELY( !borrow_res ) ) {
+  if( FD_UNLIKELY( ctx->runtime->accounts.refcnt[idx_in_txn]!=0UL ) ) {
     return FD_EXECUTOR_INSTR_ERR_ACC_BORROW_FAILED;
   }
+  ctx->runtime->accounts.refcnt[idx_in_txn]++;
 
   /* Create a BorrowedAccount upon success.
      https://github.com/anza-xyz/agave/blob/v2.1.14/sdk/src/transaction_context.rs#L606 */
   fd_borrowed_account_init( account,
-                            txn_account,
+                            &ctx->txn_out->accounts.keys[idx_in_txn],
+                            meta,
                             ctx,
-                            idx_in_instr );
+                            idx_in_instr,
+                            &ctx->runtime->accounts.refcnt[idx_in_txn] );
   return FD_EXECUTOR_INSTR_SUCCESS;
 }
 
@@ -101,7 +103,7 @@ fd_exec_instr_ctx_try_borrow_instr_account_with_key( fd_exec_instr_ctx_t const *
                                                      fd_borrowed_account_t *     account ) {
   for( ushort i=0; i<ctx->instr->acct_cnt; i++ ) {
     ushort idx_in_txn = ctx->instr->accounts[ i ].index_in_transaction;
-    if( memcmp( pubkey->uc, ctx->txn_out->accounts.account_keys[ idx_in_txn ].uc, sizeof(fd_pubkey_t) )==0 ) {
+    if( memcmp( pubkey->uc, ctx->txn_out->accounts.keys[ idx_in_txn ].uc, sizeof(fd_pubkey_t) )==0 ) {
       return fd_exec_instr_ctx_try_borrow_instr_account( ctx, i, account );
     }
   }
@@ -148,7 +150,7 @@ fd_exec_instr_ctx_any_signed( fd_exec_instr_ctx_t const * ctx,
     ushort idx_in_txn = ctx->instr->accounts[ j ].index_in_transaction;
     is_signer |=
         ( ( !!fd_instr_acc_is_signer_idx( ctx->instr, j, NULL ) ) &
-        ( 0==memcmp( pubkey->key, ctx->txn_out->accounts.account_keys[ idx_in_txn ].key, sizeof(fd_pubkey_t) ) ) );
+        ( 0==memcmp( pubkey->key, ctx->txn_out->accounts.keys[ idx_in_txn ].key, sizeof(fd_pubkey_t) ) ) );
   }
   return is_signer;
 }

src/flamenco/runtime/fd_borrowed_account.c

@@ -4,7 +4,6 @@ int
 fd_borrowed_account_get_data_mut( fd_borrowed_account_t * borrowed_acct,
                                   uchar * *               data_out,
                                   ulong *                 dlen_out ) {
-  fd_txn_account_t * acct = borrowed_acct->acct;
 
   /* https://github.com/anza-xyz/agave/blob/v2.1.14/sdk/src/transaction_context.rs#L824 */
   int err;
@@ -14,17 +13,16 @@ fd_borrowed_account_get_data_mut( fd_borrowed_account_t * borrowed_acct,
   }
 
   if ( data_out != NULL )
-    *data_out = fd_txn_account_get_data_mut( acct );
+    *data_out = fd_account_data( borrowed_acct->meta );
   if ( dlen_out != NULL )
-    *dlen_out = fd_txn_account_get_data_len( acct );
+    *dlen_out = borrowed_acct->meta->dlen;
 
   return FD_EXECUTOR_INSTR_SUCCESS;
 }
 
 int
 fd_borrowed_account_set_owner( fd_borrowed_account_t * borrowed_acct,
                                fd_pubkey_t const *     owner ) {
-  fd_txn_account_t * acct = borrowed_acct->acct;
 
   /* Only the owner can assign a new owner
      https://github.com/anza-xyz/agave/blob/v2.1.14/sdk/src/transaction_context.rs#L741 */
@@ -46,15 +44,15 @@ fd_borrowed_account_set_owner( fd_borrowed_account_t * borrowed_acct,
 
   /* Don't copy the account if the owner does not change
      https://github.com/anza-xyz/agave/blob/v2.1.14/sdk/src/transaction_context.rs#L757 */
-  if( !memcmp( fd_txn_account_get_owner( acct ), owner, sizeof( fd_pubkey_t ) ) ) {
+  if( !memcmp( borrowed_acct->meta->owner, owner, sizeof(fd_pubkey_t) ) ) {
     return FD_EXECUTOR_INSTR_SUCCESS;
   }
 
   /* Agave self.touch() is a no-op */
 
   /* Copy into owner
      https://github.com/anza-xyz/agave/blob/v2.1.14/sdk/src/transaction_context.rs#L761 */
-  fd_txn_account_set_owner( acct, owner );
+  fd_memcpy( borrowed_acct->meta->owner, owner, sizeof(fd_pubkey_t) );
   return FD_EXECUTOR_INSTR_SUCCESS;
 }
 
@@ -63,12 +61,11 @@ fd_borrowed_account_set_owner( fd_borrowed_account_t * borrowed_acct,
 int
 fd_borrowed_account_set_lamports( fd_borrowed_account_t * borrowed_acct,
                                   ulong                   lamports ) {
-  fd_txn_account_t * acct = borrowed_acct->acct;
 
   /* An account not owned by the program cannot have its blanace decrease
      https://github.com/anza-xyz/agave/blob/v2.1.14/sdk/src/transaction_context.rs#L775 */
   if( FD_UNLIKELY( (!fd_borrowed_account_is_owned_by_current_program( borrowed_acct )) &&
-                   (lamports<fd_txn_account_get_lamports( acct )) ) ) {
+                   (lamports<borrowed_acct->meta->lamports) ) ) {
     return FD_EXECUTOR_INSTR_ERR_EXTERNAL_ACCOUNT_LAMPORT_SPEND;
   }
 
@@ -80,21 +77,20 @@ fd_borrowed_account_set_lamports( fd_borrowed_account_t * borrowed_acct,
 
   /* Don't copy the account if the lamports do not change
      https://github.com/anza-xyz/agave/blob/v2.1.14/sdk/src/transaction_context.rs#L787 */
-  if( fd_txn_account_get_lamports( acct )==lamports ) {
+  if( borrowed_acct->meta->lamports==lamports ) {
     return FD_EXECUTOR_INSTR_SUCCESS;
   }
 
   /* Agave self.touch() is a no-op */
 
-  fd_txn_account_set_lamports( acct, lamports );
+  borrowed_acct->meta->lamports = lamports;
   return FD_EXECUTOR_INSTR_SUCCESS;
 }
 
 int
 fd_borrowed_account_set_data_from_slice( fd_borrowed_account_t * borrowed_acct,
                                          uchar const *           data,
                                          ulong                   data_sz ) {
-  fd_txn_account_t * acct = borrowed_acct->acct;
 
   /* https://github.com/anza-xyz/agave/blob/v2.1.14/sdk/src/transaction_context.rs#L865 */
   int err;
@@ -115,16 +111,16 @@ fd_borrowed_account_set_data_from_slice( fd_borrowed_account_t * borrowed_acct,
   }
 
   /* AccountSharedData::set_data_from_slice() */
-  fd_txn_account_set_data( acct, data, data_sz );
+  borrowed_acct->meta->dlen = (uint)data_sz;
+  fd_memcpy( fd_account_data( borrowed_acct->meta ), data, data_sz );
 
   return FD_EXECUTOR_INSTR_SUCCESS;
 }
 
 int
 fd_borrowed_account_set_data_length( fd_borrowed_account_t * borrowed_acct,
                                      ulong                   new_len ) {
-  fd_txn_account_t * acct = borrowed_acct->acct;
-  int                err  = FD_EXECUTOR_INSTR_SUCCESS;
+  int err = FD_EXECUTOR_INSTR_SUCCESS;
 
   /* https://github.com/anza-xyz/agave/blob/v2.1.14/sdk/src/transaction_context.rs#L883 */
   if( FD_UNLIKELY( !fd_borrowed_account_can_data_be_resized( borrowed_acct, new_len, &err ) ) ) {
@@ -136,7 +132,7 @@ fd_borrowed_account_set_data_length( fd_borrowed_account_t * borrowed_acct,
     return err;
   }
 
-  ulong old_len = fd_txn_account_get_data_len( acct );
+  ulong old_len = borrowed_acct->meta->dlen;
 
   /* Don't copy the account if the length does not change
      https://github.com/anza-xyz/agave/blob/v2.1.14/sdk/src/transaction_context.rs#L886 */
@@ -153,19 +149,18 @@ fd_borrowed_account_set_data_length( fd_borrowed_account_t * borrowed_acct,
 
   /* Resize the account
      https://github.com/anza-xyz/agave/blob/v2.1.14/sdk/src/transaction_context.rs#L891 */
-  fd_txn_account_resize( acct, new_len );
+  fd_account_meta_resize( borrowed_acct->meta, new_len );
+
   return FD_EXECUTOR_INSTR_SUCCESS;
 }
 
 int
 fd_borrowed_account_set_executable( fd_borrowed_account_t * borrowed_acct,
                                     int                     is_executable ) {
-  fd_txn_account_t * acct = borrowed_acct->acct;
-
   /* To become executable an account must be rent exempt
      https://github.com/anza-xyz/agave/blob/v2.1.14/sdk/src/transaction_context.rs#L1003-L1006 */
   fd_rent_t const * rent = fd_bank_rent_query( borrowed_acct->instr_ctx->bank );
-  if( FD_UNLIKELY( fd_txn_account_get_lamports( acct )<fd_rent_exempt_minimum_balance( rent, fd_txn_account_get_data_len( acct ) ) ) ) {
+  if( FD_UNLIKELY( borrowed_acct->meta->lamports<fd_rent_exempt_minimum_balance( rent, borrowed_acct->meta->dlen ) ) ) {
     return FD_EXECUTOR_INSTR_ERR_EXECUTABLE_ACCOUNT_NOT_RENT_EXEMPT;
   }
 
@@ -190,7 +185,7 @@ fd_borrowed_account_set_executable( fd_borrowed_account_t * borrowed_acct,
   /* Agave self.touch() is a no-op */
 
   /* https://github.com/anza-xyz/agave/blob/v2.1.14/sdk/src/transaction_context.rs#L1027 */
-  fd_txn_account_set_executable( acct, is_executable );
+  borrowed_acct->meta->executable = !!is_executable;
 
   return FD_EXECUTOR_INSTR_SUCCESS;
 }
@@ -200,8 +195,7 @@ fd_borrowed_account_update_accounts_resize_delta( fd_borrowed_account_t * borrow
                                                   ulong                   new_len,
                                                   int *                   err ) {
   fd_exec_instr_ctx_t const * instr_ctx  = borrowed_acct->instr_ctx;
-  fd_txn_account_t *          acct       = borrowed_acct->acct;
-  ulong                       size_delta = fd_ulong_sat_sub( new_len, fd_txn_account_get_data_len( acct ) );
+  ulong                       size_delta = fd_ulong_sat_sub( new_len, borrowed_acct->meta->dlen );
 
   /* TODO: The size delta should never exceed the value of ULONG_MAX so this
      could be replaced with a normal addition. However to match execution with
@@ -215,11 +209,10 @@ int
 fd_borrowed_account_can_data_be_resized( fd_borrowed_account_t const * borrowed_acct,
                                          ulong                         new_length,
                                          int *                         err ) {
-  fd_txn_account_t * acct = borrowed_acct->acct;
 
   /* Only the owner can change the length of the data
      https://github.com/anza-xyz/agave/blob/v2.1.14/sdk/src/transaction_context.rs#L1095 */
-  if( FD_UNLIKELY( (fd_txn_account_get_data_len( acct )!=new_length) &
+  if( FD_UNLIKELY( (borrowed_acct->meta->dlen!=new_length) &
                    (!fd_borrowed_account_is_owned_by_current_program( borrowed_acct )) ) ) {
     *err = FD_EXECUTOR_INSTR_ERR_ACC_DATA_SIZE_CHANGED;
     return 0;
@@ -234,7 +227,7 @@ fd_borrowed_account_can_data_be_resized( fd_borrowed_account_t const * borrowed_
 
   /* The resize can not exceed the per-transaction maximum
      https://github.com/anza-xyz/agave/blob/v2.1.14/sdk/src/transaction_context.rs#L1104-L1108 */
-  ulong length_delta              = fd_ulong_sat_sub( new_length, fd_txn_account_get_data_len( acct ) );
+  ulong length_delta              = fd_ulong_sat_sub( new_length, borrowed_acct->meta->dlen );
   ulong new_accounts_resize_delta = fd_ulong_sat_add( borrowed_acct->instr_ctx->txn_out->details.accounts_resize_delta, length_delta );
   if( FD_UNLIKELY( new_accounts_resize_delta > MAX_PERMITTED_ACCOUNT_DATA_ALLOCS_PER_TXN ) ) {
     *err = FD_EXECUTOR_INSTR_ERR_MAX_ACCS_DATA_ALLOCS_EXCEEDED;