txn: change parser to parse txns with >64 instrs

src/ballet/txn/fd_txn.h

@@ -85,18 +85,19 @@
    to 33. FIXME: We should petition to limit this to approx 8. */
 #define FD_TXN_ADDR_TABLE_LOOKUP_MAX (127UL)
 
-/* FD_TXN_INSTR_MAX: The (inclusive) maximum number of instructions a transaction
-   can have.  As of Solana 1.15.0, this is limited to 64. */
+/* FD_TXN_INSTR_MAX: The (inclusive) maximum number of instructions a
+   transaction can have.  As of Solana 1.15.0, this is limited to 64.
+   A transaction can have more than 64 instructions, but if more than
+   that many instructions are executed, a transaction will fail. */
 #define FD_TXN_INSTR_MAX             (64UL)
 
-
 /* FD_TXN_MAX_SZ: The maximum amount of memory (in bytes) that a fd_txn can
    take up, including the instruction array and any address tables.  The
    worst-case transaction is a V0 transaction with only two account
    addresses (a program and a fee payer), and tons of empty instructions (no
    accounts, no data) and as many address table lookups loading a single
    account as possible. */
-#define FD_TXN_MAX_SZ                (852UL)
+#define FD_TXN_MAX_SZ                (854UL)
 
 
 /* FD_TXN_MTU: The maximum size (in bytes, inclusive) of a serialized
@@ -285,13 +286,24 @@ struct fd_txn {
       */
 
   /* instr_cnt: The number of instructions in this transaction.
-     instr_cnt in [0, FD_TXN_INSTR_MAX]. */
+     instr_cnt in [0, FD_TXN_INSTR_MAX].  There can be more than
+     FD_TXN_INSTR_MAX instructions in a transaction, but they will
+     not be parsed or counted here. */
   ushort      instr_cnt;
 
+  /* has_unparsed_instrs: 1 if the transaction has more than
+     FD_TXN_INSTR_MAX instructions, 0 otherwise.  */
+  uchar       has_unparsed_instrs;
+  uchar       _padding_reserved_2; /* explicitly declare what the compiler would
+                                     insert anyways */
+
   /* instr: The array of instructions in this transaction. It's a "flexible array
      member" since C does not allow the pretty typical 0-len array at the end
      of the struct trick.
-     Indexed [0, instr_cnt). */
+     Indexed [0, min(instr_cnt, FD_TXN_INSTR_MAX)).  All transactions
+     past the 64th instruction are not parsed because they will
+     automatically fail to execute in the runtime due to protocol
+     limits. */
   fd_txn_instr_t instr[ ];
 
   /* Logically, there's another field here:
@@ -705,14 +717,13 @@ fd_txn_parse_core( uchar const             * payload,
                    ulong                     payload_sz,
                    void                    * out_buf,
                    fd_txn_parse_counters_t * counters_opt,
-                   ulong *                   payload_sz_opt,
-                   ulong                     instr_max );
+                   ulong *                   payload_sz_opt );
 
 
 /* fd_txn_parse: Convenient wrapper around fd_txn_parse_core that eliminates some optional arguments */
 static inline ulong
 fd_txn_parse( uchar const * payload, ulong payload_sz, void * out_buf, fd_txn_parse_counters_t * counters_opt ) {
-  return fd_txn_parse_core( payload, payload_sz, out_buf, counters_opt, NULL, FD_TXN_INSTR_MAX );
+  return fd_txn_parse_core( payload, payload_sz, out_buf, counters_opt, NULL );
 }
 
 /* fd_txn_is_writable: Is the account at the supplied index writable

src/ballet/txn/fd_txn_parse.c

@@ -8,8 +8,7 @@ fd_txn_parse_core( uchar const             * payload,
                    ulong                     payload_sz,
                    void                    * out_buf,
                    fd_txn_parse_counters_t * counters_opt,
-                   ulong *                   payload_sz_opt,
-                   ulong                     instr_max ) {
+                   ulong *                   payload_sz_opt ) {
   ulong i = 0UL;
   /* This code does non-trivial parsing of untrusted user input, which
      is a potentially dangerous thing.  The main invariants we need to
@@ -122,10 +121,7 @@ fd_txn_parse_core( uchar const             * payload,
   ushort instr_cnt = (ushort)0;
   READ_CHECKED_COMPACT_U16( bytes_consumed,                instr_cnt,                i );     i+=bytes_consumed;
 
-  /* FIXME: compile-time max after static_instruction_limit. */
-  CHECK( (ulong)instr_cnt<=instr_max            );
-
-  CHECK_LEFT( MIN_INSTR_SZ*instr_cnt            );
+  CHECK_LEFT( MIN_INSTR_SZ*instr_cnt );
   /* If it has >0 instructions, it must have at least one other account
      address (the program id) that can't be the fee payer */
   CHECK( (ulong)acct_addr_cnt>(!!instr_cnt) );
@@ -145,7 +141,8 @@ fd_txn_parse_core( uchar const             * payload,
     /* Need to assign addr_table_lookup_cnt,
        addr_table_adtl_writable_cnt, addr_table_adtl_cnt,
        _padding_reserved_1 later */
-    parsed->instr_cnt                     = instr_cnt;
+    parsed->instr_cnt                     = fd_ushort_min( instr_cnt, FD_TXN_INSTR_MAX );
+    parsed->has_unparsed_instrs           = (instr_cnt>FD_TXN_INSTR_MAX) ? 1 : 0;
   }
 
   uchar max_acct = 0UL;
@@ -169,7 +166,11 @@ fd_txn_parse_core( uchar const             * payload,
        program ID can't come from a table. */
     CHECK( (0UL < (ulong)program_id) & ((ulong)program_id < (ulong)acct_addr_cnt) );
 
-    if( parsed ){
+    /* We only care to parse the first FD_TXN_INSTR_MAX (64)
+       instructions because will never execute more than that many
+       instructions in a single transaction since it will lead to an
+       execution failure.  We ignore the rest of the instructions. */
+    if( parsed && j<FD_TXN_INSTR_MAX ) {
       parsed->instr[ j ].program_id          = program_id;
       parsed->instr[ j ]._padding_reserved_1 = (uchar)0;
       parsed->instr[ j ].acct_cnt            = acct_cnt;
@@ -234,18 +235,13 @@ fd_txn_parse_core( uchar const             * payload,
     parsed->addr_table_adtl_writable_cnt  = (uchar)addr_table_adtl_writable_cnt;
     parsed->addr_table_adtl_cnt           = (uchar)addr_table_adtl_cnt;
     parsed->_padding_reserved_1           = (uchar)0;
+    parsed->_padding_reserved_2           = (uchar)0;
   }
 
   if( FD_LIKELY( counters_opt   ) ) counters_opt->success_cnt++;
   if( FD_LIKELY( payload_sz_opt ) ) *payload_sz_opt = i;
 
   ulong footprint = fd_txn_footprint( instr_cnt, addr_table_cnt );
-  /* FIXME remove this check when static_instruction_limit is activated on all networks. */
-  if( FD_UNLIKELY( instr_cnt>FD_TXN_INSTR_MAX && footprint>FD_TXN_MAX_SZ ) ) {
-    uchar const * sig = payload+parsed->signature_off;
-    FD_LOG_HEXDUMP_WARNING(( "txnsig", sig, FD_TXN_SIGNATURE_SZ ));
-    FD_LOG_CRIT(( "instr_cnt %u footprint %lu", instr_cnt, footprint ));
-  }
   return footprint;
 
   #undef CHECK

src/discof/replay/fd_replay_tile.c

@@ -1710,7 +1710,7 @@ process_fec_set( fd_replay_tile_t *  ctx,
        safe because we know that the old entry for this bank index has
        already been pruned away. */
     fd_block_id_ele_t * block_id_ele = &ctx->block_id_arr[ reasm_fec->bank_idx ];
-    if( FD_LIKELY( fd_block_id_map_ele_query( ctx->block_id_map, &block_id_ele->block_id, NULL, ctx->block_id_arr ) ) ) {
+    if( FD_LIKELY( fd_block_id_map_ele_query( ctx->block_id_map, &block_id_ele->block_id, NULL, ctx->block_id_arr )==block_id_ele ) ) {
       FD_TEST( fd_block_id_map_ele_remove( ctx->block_id_map, &block_id_ele->block_id, NULL, ctx->block_id_arr ) );
     }
     block_id_ele->block_id_seen = 0;

src/discof/replay/fd_sched.c

@@ -1463,8 +1463,7 @@ fd_sched_parse_txn( fd_sched_t * sched, fd_sched_block_t * block, fd_sched_alut_
                                     fd_ulong_min( FD_TXN_MTU, block->fec_buf_sz-block->fec_buf_soff ),
                                     txn,
                                     NULL,
-                                    &pay_sz,
-                                    FD_TXN_INSTR_MAX*2UL );
+                                    &pay_sz );
 
   if( FD_UNLIKELY( !pay_sz || !txn_sz ) ) {
     /* Can't parse out a full transaction. */

src/flamenco/gossip/fd_gossip_msg_parse.c

@@ -178,7 +178,7 @@ fd_gossip_msg_crds_vote_parse( fd_gossip_view_crds_value_t * crds_val,
   CHECK( crds_val->vote->index<FD_GOSSIP_VOTE_IDX_MAX );
   CHECK_LEFT( 32U ); crds_val->pubkey_off  = CUR_OFFSET                                         ; INC( 32U );
   ulong transaction_sz;
-  CHECK( fd_txn_parse_core( CURSOR, BYTES_REMAINING, NULL, NULL, &transaction_sz, FD_TXN_INSTR_MAX )!=0UL );
+  CHECK( fd_txn_parse_core( CURSOR, BYTES_REMAINING, NULL, NULL, &transaction_sz )!=0UL );
   crds_val->vote->txn_off = CUR_OFFSET;
   crds_val->vote->txn_sz  = transaction_sz;
   INC( transaction_sz );

src/flamenco/runtime/fd_executor.c

@@ -1714,6 +1714,17 @@ fd_execute_txn( fd_runtime_t *      runtime,
     }
   }
 
+  /* If the transaction has more than FD_TXN_INSTR_MAX instructions,
+     we know that the transaction has to fail.  Firedancer's txn parsing
+     logic ignores any instructions past the 64th one.  If the txn has
+     yet to fail at this point and we know that there were unparsed
+     transactions, we want to fail the transaction with an instruction
+     error. */
+  if( FD_UNLIKELY( txn->has_unparsed_instrs ) ) {
+    txn_out->err.exec_err = FD_EXECUTOR_INSTR_ERR_MAX_INSN_TRACE_LENS_EXCEEDED;
+    return FD_RUNTIME_TXN_ERR_INSTRUCTION_ERROR;
+  }
+
   /* TODO: This function needs to be split out of fd_execute_txn and be placed
       into the replay tile once it is implemented. */
   return fd_executor_txn_check( runtime, bank, txn_out );

src/flamenco/runtime/tests/run_backtest_all.sh

@@ -96,3 +96,4 @@ src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-376969880-simd-339
 src/flamenco/runtime/tests/run_ledger_backtest.sh -l breakpoint-385786458 -y 1 -m 2000000 -e 385786458
 src/flamenco/runtime/tests/run_ledger_backtest.sh -l localnet-deprecate-rent-exemption-threshold -y 1 -m 1000 -e 260
 src/flamenco/runtime/tests/run_ledger_backtest.sh -l localnet-static-instruction-limit -y 1 -m 1000 -e 191
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l devnet-427963044 -y 1 -m 2000000 -e 427963085

src/flamenco/runtime/tests/run_backtest_ci.sh

@@ -29,3 +29,4 @@ src/flamenco/runtime/tests/run_ledger_backtest.sh -l localnet-stake-v3.0.0 -y 1
 src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-378539412 -y 5 -m 2000000 -e 378539445
 src/flamenco/runtime/tests/run_ledger_backtest.sh -l devnet-422969842 -y 1 -m 2000000 -e 422969848
 src/flamenco/runtime/tests/run_ledger_backtest.sh -l breakpoint-385786458 -y 1 -m 2000000 -e 385786458
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l devnet-427963044 -y 1 -m 2000000 -e 427963085