TXMv2 changes for Hedera

pkg/config/toml/defaults/Hedera_Testnet.toml

@@ -13,22 +13,29 @@ FinalizedBlockOffset = 2
 Enabled = true
 
 [GasEstimator]
-Mode = 'SuggestedPrice'
-# Since Hedera dont have mempool and there's no way for a node to front run or a user to bribe a node to submit the transaction earlier than it's consensus timestamp,
-# But they have automated congesting pricing throttling which would mean at high sustained level the gasPrice itself could be increased to prevent malicious behaviour.
-# Disabling the Bumpthreshold as TXM now implicity handles the bumping after checking on-chain nonce & re-broadcast for Hedera chain type
-BumpThreshold = 0
-BumpMin = '10 gwei'
-BumpPercent = 20
-# Dynamic gas estimation is a must Hedera, since Hedera consumes 80% of gaslimit by default, we will end up overpaying for gas
+EIP1559DynamicFees = true
+Mode = 'FeeHistory'
+BumpThreshold = 1 # Force retry after each block.
 EstimateLimit = true
+LimitTransfer = 25_000
+LimitDefault = 200_000
+LimitMax = 200_000
 
-[Transactions]
-# To hit throttling you'd need to maintain 15 m gas /sec over a prolonged period of time.
-# Because Hedera's block times are every 2 secs it's less less likely to happen as compared to other chains
-# Setting this to little higher even though Hedera has High TPS, We have seen 10-12s to get the trasaction mined & 20-25s incase of failures
-# Accounting for Node syncs & avoid re-sending txns before fetching the receipt, setting to 2m
-ResendAfterThreshold = '2m'
+[GasEstimator.FeeHistory]
+CacheTimeout = '4s'
+
+[GasEstimator.BlockHistory]
+# This is used by the FeeHistory estimator in chains that don't have a mempool.
+BlockHistorySize = 0
+
+[Transactions.TransactionManagerV2]
+Enabled = true
+BlockTime = '3s'
+
+[Transactions.AutoPurge]
+Enabled = true
+Threshold = 5
+MinAttempts = 10000
 
 
 [NodePool]

pkg/txm/attempt_builder.go

@@ -10,29 +10,34 @@ import (
 
 	"github.com/smartcontractkit/chainlink-common/pkg/logger"
 	"github.com/smartcontractkit/chainlink-evm/pkg/assets"
+	"github.com/smartcontractkit/chainlink-evm/pkg/config/chaintype"
 	"github.com/smartcontractkit/chainlink-evm/pkg/gas"
 	"github.com/smartcontractkit/chainlink-evm/pkg/keys"
 	"github.com/smartcontractkit/chainlink-evm/pkg/txm/types"
 )
 
-// maxBumpThreshold controls the maximum number of bumps for an attempt.
-const maxBumpThreshold = 5
+const (
+	maxBumpThreshold   = 5              // maxBumpThreshold controls the maximum number of bumps for an attempt.
+	hederaWeiToTinybar = 10_000_000_001 // hederaWeiToTinybar is the minimum allowed value for a transfer in Hedera plus 1. Hedera uses HBAR instead of ETH
+)
 
 type attemptBuilder struct {
 	gas.EvmFeeEstimator
 	priceMaxKey         func(common.Address) *assets.Wei
 	keystore            keys.TxSigner
 	emptyTxLimitDefault uint64
 	feeBoost            bool
+	chainType           chaintype.ChainType
 }
 
-func NewAttemptBuilder(priceMaxKey func(common.Address) *assets.Wei, estimator gas.EvmFeeEstimator, keystore keys.TxSigner, emptyTxLimitDefault uint64, feeBoost bool) *attemptBuilder {
+func NewAttemptBuilder(priceMaxKey func(common.Address) *assets.Wei, estimator gas.EvmFeeEstimator, keystore keys.TxSigner, emptyTxLimitDefault uint64, feeBoost bool, chainType chaintype.ChainType) *attemptBuilder {
 	return &attemptBuilder{
 		priceMaxKey:         priceMaxKey,
 		EvmFeeEstimator:     estimator,
 		keystore:            keystore,
 		emptyTxLimitDefault: emptyTxLimitDefault,
 		feeBoost:            feeBoost,
+		chainType:           chainType,
 	}
 }
 
@@ -75,9 +80,11 @@ func (a *attemptBuilder) NewBumpAttempt(ctx context.Context, lggr logger.Logger,
 }
 
 func (a *attemptBuilder) NewAgnosticBumpAttempt(ctx context.Context, lggr logger.Logger, tx *types.Transaction, dynamic bool) (attempt *types.Attempt, err error) {
+	if a.chainType == chaintype.ChainHedera {
+		return a.newHederaAttempt(ctx, lggr, tx, a.priceMaxKey(tx.FromAddress), dynamic)
+	}
 	// if the transaction is purgeable or feeBoost is enabled, NewAttempt will return the max fee instantly, so there is no need to bump
 	attempt, err = a.NewAttempt(ctx, lggr, tx, dynamic)
-
 	if err != nil {
 		return
 	}
@@ -99,6 +106,45 @@ func (a *attemptBuilder) NewAgnosticBumpAttempt(ctx context.Context, lggr logger
 	return attempt, nil
 }
 
+// newHederaAttempt is used to build a new attempt for Hedera.
+// Hedera is a special case. It doesn't have a mempool but can reject an attempt for unknown reasons, even though the RPC returns success.
+// The network binds transactions with unique IDs and a timestamp. If the timestamp exceeds a threshold it will auto-reject the
+// transaction no matter how many times we retry. To bypass this case, we fetch a new market price and bump the fee by 1 per attempt
+// to forcefully generate a new hash. We avoid max pricing purgeable transactions for the same reason.
+func (a *attemptBuilder) newHederaAttempt(ctx context.Context, lggr logger.Logger, tx *types.Transaction, maxPrice *assets.Wei, dynamic bool) (*types.Attempt, error) {
+	gasLimit := tx.SpecifiedGasLimit
+	if tx.IsPurgeable {
+		gasLimit = a.emptyTxLimitDefault
+	}
+	fee, estimatedGasLimit, err := a.GetFee(ctx, tx.Data, gasLimit, maxPrice, &tx.FromAddress, &tx.ToAddress)
+	if err != nil {
+		return nil, err
+	}
+	txType := evmtypes.LegacyTxType
+	if dynamic {
+		txType = evmtypes.DynamicFeeTxType
+	}
+
+	attempt, err := a.newCustomAttempt(ctx, tx, fee, estimatedGasLimit, byte(txType), lggr)
+	if err != nil {
+		return nil, err
+	}
+	for range tx.AttemptCount {
+		if attempt.Fee.ValidDynamic() && maxPrice.Cmp(attempt.Fee.GasFeeCap) > 0 {
+			fee.GasFeeCap = attempt.Fee.GasFeeCap.Add(assets.NewWeiI(1)) // Hedera doesn't have a mempool so maxPriorityFeePerGas is always 0.
+		} else if attempt.Fee.GasPrice != nil && maxPrice.Cmp(attempt.Fee.GasPrice) > 0 {
+			fee.GasPrice = attempt.Fee.GasPrice.Add(assets.NewWeiI(1))
+		} else {
+			break
+		}
+		attempt, err = a.newCustomAttempt(ctx, tx, fee, estimatedGasLimit, byte(txType), lggr)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return attempt, nil
+}
+
 func (a *attemptBuilder) newCustomAttempt(
 	ctx context.Context,
 	tx *types.Transaction,
@@ -136,6 +182,10 @@ func (a *attemptBuilder) newLegacyAttempt(ctx context.Context, tx *types.Transac
 		toAddress = tx.ToAddress
 		value = tx.Value
 	}
+	if a.chainType == chaintype.ChainHedera && tx.IsPurgeable {
+		value = big.NewInt(hederaWeiToTinybar)
+		toAddress = tx.FromAddress
+	}
 	if tx.Nonce == nil {
 		return nil, fmt.Errorf("failed to create attempt for txID: %v: nonce empty", tx.ID)
 	}
@@ -174,6 +224,10 @@ func (a *attemptBuilder) newDynamicFeeAttempt(ctx context.Context, tx *types.Tra
 		toAddress = tx.ToAddress
 		value = tx.Value
 	}
+	if a.chainType == chaintype.ChainHedera && tx.IsPurgeable {
+		value = big.NewInt(10_000_000_001)
+		toAddress = tx.FromAddress
+	}
 	if tx.Nonce == nil {
 		return nil, fmt.Errorf("failed to create attempt for txID: %v: nonce empty", tx.ID)
 	}

pkg/txm/attempt_builder_test.go

@@ -22,7 +22,7 @@ import (
 )
 
 func TestAttemptBuilder_newLegacyAttempt(t *testing.T) {
-	ab := NewAttemptBuilder(nil, nil, keystest.TxSigner(nil), 100, false)
+	ab := NewAttemptBuilder(nil, nil, keystest.TxSigner(nil), 100, false, "")
 	address := testutils.NewAddress()
 	lggr := logger.Test(t)
 	var gasLimit uint64 = 100
@@ -57,7 +57,7 @@ func TestAttemptBuilder_newLegacyAttempt(t *testing.T) {
 }
 
 func TestAttemptBuilder_newDynamicFeeAttempt(t *testing.T) {
-	ab := NewAttemptBuilder(nil, nil, keystest.TxSigner(nil), 100, false)
+	ab := NewAttemptBuilder(nil, nil, keystest.TxSigner(nil), 100, false, "")
 	address := testutils.NewAddress()
 
 	lggr := logger.Test(t)
@@ -100,7 +100,7 @@ func TestAttemptBuilder_NewAttempt(t *testing.T) {
 	var nonce uint64 = 1
 	var specifiedGasLimit uint64 = 200
 	var emptyGasLimit uint64 = 100
-	ab := NewAttemptBuilder(priceMaxKey, mockEstimator, keystest.TxSigner(nil), emptyGasLimit, false)
+	ab := NewAttemptBuilder(priceMaxKey, mockEstimator, keystest.TxSigner(nil), emptyGasLimit, false, "")
 	address := testutils.NewAddress()
 	lggr := logger.Test(t)
 
@@ -163,7 +163,7 @@ func TestAttemptBuilder_NewAttempt(t *testing.T) {
 
 	t.Run("uses SpecifiedGasLimit when feeBoost is enabled and tx is not purgeable", func(t *testing.T) {
 		boostEstimator := mocks.NewEvmFeeEstimator(t)
-		boostAb := NewAttemptBuilder(priceMaxKey, boostEstimator, keystest.TxSigner(nil), emptyGasLimit, true)
+		boostAb := NewAttemptBuilder(priceMaxKey, boostEstimator, keystest.TxSigner(nil), emptyGasLimit, true, "")
 		tx := &types.Transaction{ID: 10, FromAddress: address, Nonce: &nonce, SpecifiedGasLimit: specifiedGasLimit}
 		boostEstimator.On("GetMaxFee", mock.Anything, mock.Anything, specifiedGasLimit, mock.Anything, mock.Anything, mock.Anything).
 			Return(gas.EvmFee{GasPrice: assets.NewWeiI(100)}, specifiedGasLimit, nil).Once()
@@ -175,7 +175,7 @@ func TestAttemptBuilder_NewAttempt(t *testing.T) {
 
 	t.Run("uses emptyTxLimitDefault when feeBoost is enabled and tx is purgeable", func(t *testing.T) {
 		boostEstimator := mocks.NewEvmFeeEstimator(t)
-		boostAb := NewAttemptBuilder(priceMaxKey, boostEstimator, keystest.TxSigner(nil), emptyGasLimit, true)
+		boostAb := NewAttemptBuilder(priceMaxKey, boostEstimator, keystest.TxSigner(nil), emptyGasLimit, true, "")
 		tx := &types.Transaction{ID: 10, FromAddress: address, IsPurgeable: true, Nonce: &nonce, SpecifiedGasLimit: specifiedGasLimit}
 		boostEstimator.On("GetMaxFee", mock.Anything, mock.Anything, emptyGasLimit, mock.Anything, mock.Anything, mock.Anything).
 			Return(gas.EvmFee{GasPrice: assets.NewWeiI(100)}, emptyGasLimit, nil).Once()
@@ -196,7 +196,7 @@ func TestAttemptBuilder_NewAgnosticBumpAttempt(t *testing.T) {
 
 	t.Run("returns original attempt when AttemptCount is 0", func(t *testing.T) {
 		mockEstimator := mocks.NewEvmFeeEstimator(t)
-		ab := NewAttemptBuilder(priceMaxKey, mockEstimator, keystest.TxSigner(nil), 100, false)
+		ab := NewAttemptBuilder(priceMaxKey, mockEstimator, keystest.TxSigner(nil), 100, false, "")
 
 		tx := &types.Transaction{
 			ID:           10,
@@ -220,7 +220,7 @@ func TestAttemptBuilder_NewAgnosticBumpAttempt(t *testing.T) {
 
 	t.Run("bumps once when AttemptCount is 1", func(t *testing.T) {
 		mockEstimator := mocks.NewEvmFeeEstimator(t)
-		ab := NewAttemptBuilder(priceMaxKey, mockEstimator, keystest.TxSigner(nil), 100, false)
+		ab := NewAttemptBuilder(priceMaxKey, mockEstimator, keystest.TxSigner(nil), 100, false, "")
 
 		tx := &types.Transaction{
 			ID:           10,
@@ -246,7 +246,7 @@ func TestAttemptBuilder_NewAgnosticBumpAttempt(t *testing.T) {
 
 	t.Run("bumps N times when AttemptCount is N", func(t *testing.T) {
 		mockEstimator := mocks.NewEvmFeeEstimator(t)
-		ab := NewAttemptBuilder(priceMaxKey, mockEstimator, keystest.TxSigner(nil), 100, false)
+		ab := NewAttemptBuilder(priceMaxKey, mockEstimator, keystest.TxSigner(nil), 100, false, "")
 
 		tx := &types.Transaction{
 			ID:           10,
@@ -277,7 +277,7 @@ func TestAttemptBuilder_NewAgnosticBumpAttempt(t *testing.T) {
 
 	t.Run("returns last valid attempt when BumpFee fails", func(t *testing.T) {
 		mockEstimator := mocks.NewEvmFeeEstimator(t)
-		ab := NewAttemptBuilder(priceMaxKey, mockEstimator, keystest.TxSigner(nil), 100, false)
+		ab := NewAttemptBuilder(priceMaxKey, mockEstimator, keystest.TxSigner(nil), 100, false, "")
 
 		tx := &types.Transaction{
 			ID:           10,
@@ -306,7 +306,7 @@ func TestAttemptBuilder_NewAgnosticBumpAttempt(t *testing.T) {
 
 	t.Run("caps bumps at maxBumpThreshold", func(t *testing.T) {
 		mockEstimator := mocks.NewEvmFeeEstimator(t)
-		ab := NewAttemptBuilder(priceMaxKey, mockEstimator, keystest.TxSigner(nil), 100, false)
+		ab := NewAttemptBuilder(priceMaxKey, mockEstimator, keystest.TxSigner(nil), 100, false, "")
 
 		tx := &types.Transaction{
 			ID:           10,
@@ -331,7 +331,7 @@ func TestAttemptBuilder_NewAgnosticBumpAttempt(t *testing.T) {
 
 	t.Run("returns max percentile attempt when transaction is purgeable", func(t *testing.T) {
 		mockEstimator := mocks.NewEvmFeeEstimator(t)
-		ab := NewAttemptBuilder(priceMaxKey, mockEstimator, keystest.TxSigner(nil), 100, false)
+		ab := NewAttemptBuilder(priceMaxKey, mockEstimator, keystest.TxSigner(nil), 100, false, "")
 
 		tx := &types.Transaction{
 			ID:           10,

pkg/txm/integration-tests/integration_test.go

@@ -99,7 +99,7 @@ func setupTestnetTXM(
 	require.NoError(t, err, "failed to add private key to keystore")
 
 	// AttemptBuilder
-	ab := txm.NewAttemptBuilder(configs.PriceMaxKey, estimator, keystore, configs.LimitTransfer(), false)
+	ab := txm.NewAttemptBuilder(configs.PriceMaxKey, estimator, keystore, configs.LimitTransfer(), false, "")
 
 	// InMemory storage
 	store := storage.NewInMemoryStoreManager(lggr, chainID)
@@ -158,7 +158,7 @@ func setupDevnetTXM(
 	require.NoError(t, keystore.Add(privateKeyHex), "failed to add private key to keystore")
 
 	// AttemptBuilder
-	ab := txm.NewAttemptBuilder(configs.PriceMaxKey, estimator, keystore, configs.LimitDefault(), false)
+	ab := txm.NewAttemptBuilder(configs.PriceMaxKey, estimator, keystore, configs.LimitDefault(), false, "")
 
 	// InMemory storage
 	store := storage.NewInMemoryStoreManager(lggr, chainID)

pkg/txm/stuck_tx_detector.go

@@ -16,6 +16,8 @@ import (
 	"github.com/smartcontractkit/chainlink-evm/pkg/txm/types"
 )
 
+const maxHederaAttemptsThreshold = 4
+
 type StuckTxDetectorConfig struct {
 	BlockTime             time.Duration
 	StuckTxBlockThreshold uint32
@@ -42,12 +44,14 @@ func NewStuckTxDetector(lggr logger.Logger, chaintype chaintype.ChainType, confi
 func (s *stuckTxDetector) DetectStuckTransaction(ctx context.Context, tx *types.Transaction) (bool, error) {
 	//nolint:gocritic //placeholder for upcoming chaintypes
 	switch s.chainType {
+	case chaintype.ChainHedera:
+		return s.hederaDetection(tx), nil
 	default:
 		return s.timeBasedDetection(tx), nil
 	}
 }
 
-// timeBasedDetection marks a transaction if:
+// timeBasedDetection marks a transaction purgeable if:
 //   - LastBroadcastAt is nil
 //   - Total attempt count is equal or greater than the maxAttemptsThreshold
 //
@@ -80,6 +84,14 @@ func (s *stuckTxDetector) timeBasedDetection(tx *types.Transaction) bool {
 	return false
 }
 
+// hederaDetection mars a transaction as purgeable starting from maxHederaAttemptsThreshold.
+// Hedera is a unique chain that is not EVM based, but it provides an RPC endpoint that mimics EVM RPC calls.
+// This means that the RPC will respond unreliably and can drop your requests without an error.
+// To bypass that we want to optimistically broadcast transactions and then fallback to purgeable transactions to clear the nonce.
+func (s *stuckTxDetector) hederaDetection(tx *types.Transaction) bool {
+	return tx.AttemptCount >= maxHederaAttemptsThreshold
+}
+
 type APIResponse struct {
 	Status string      `json:"status,omitempty"`
 	Hash   common.Hash `json:"hash,omitempty"`

pkg/txm/txm_test.go

@@ -427,7 +427,7 @@ func TestFlow_ResendTransaction(t *testing.T) {
 	mockEstimator := mocks.NewEvmFeeEstimator(t)
 	defaultGasLimit := uint64(100000)
 	keystore := &keystest.FakeChainStore{}
-	attemptBuilder := txm.NewAttemptBuilder(func(address common.Address) *assets.Wei { return assets.NewWeiI(1) }, mockEstimator, keystore, 22000, false)
+	attemptBuilder := txm.NewAttemptBuilder(func(address common.Address) *assets.Wei { return assets.NewWeiI(1) }, mockEstimator, keystore, 22000, false, "")
 	stuckTxDetector := txm.NewStuckTxDetector(logger.Test(t), "", txm.StuckTxDetectorConfig{BlockTime: config.BlockTime, StuckTxBlockThreshold: uint32(config.RetryBlockThreshold + 1)})
 	tm := txm.NewTxm(logger.Test(t), testutils.FixtureChainID, client, attemptBuilder, txStoreManager, stuckTxDetector, config, keystore, nil, txm.NewNoopTxmMetrics())
 	initialNonce := uint64(0)
@@ -501,7 +501,7 @@ func TestFlow_ErrorHandler(t *testing.T) {
 	config := txm.Config{EIP1559: true, EmptyTxLimitDefault: 22000, RetryBlockThreshold: 0, BlockTime: 2 * time.Second}
 	mockEstimator := mocks.NewEvmFeeEstimator(t)
 	keystore := &keystest.FakeChainStore{}
-	attemptBuilder := txm.NewAttemptBuilder(func(address common.Address) *assets.Wei { return assets.NewWeiI(1) }, mockEstimator, keystore, 22000, false)
+	attemptBuilder := txm.NewAttemptBuilder(func(address common.Address) *assets.Wei { return assets.NewWeiI(1) }, mockEstimator, keystore, 22000, false, "")
 	stuckTxDetector := txm.NewStuckTxDetector(lggr, "", txm.StuckTxDetectorConfig{BlockTime: config.BlockTime, StuckTxBlockThreshold: uint32(config.RetryBlockThreshold + 1)})
 	errorHandler := dualbroadcast.NewErrorHandler()
 	tm := txm.NewTxm(lggr, testutils.FixtureChainID, client, attemptBuilder, txStoreManager, stuckTxDetector, config, keystore, errorHandler, txm.NewNoopTxmMetrics())

pkg/txmgr/builder.go

@@ -142,7 +142,7 @@ func NewTxmV2(
 	}
 
 	feeBoost := txmV2Config.DualBroadcast() != nil && *txmV2Config.DualBroadcast() && txmV2Config.FeeBoost()
-	attemptBuilder := txm.NewAttemptBuilder(fCfg.PriceMaxKey, estimator, keyStore, gasEstimatorConfig.LimitTransfer(), feeBoost)
+	attemptBuilder := txm.NewAttemptBuilder(fCfg.PriceMaxKey, estimator, keyStore, gasEstimatorConfig.LimitTransfer(), feeBoost, chainConfig.ChainType())
 	inMemoryStoreManager := storage.NewInMemoryStoreManager(lggr, chainID)
 	readRequestsToMultipleNodes := false
 	if txmV2Config.ReadRequestsToMultipleNodes() != nil && *txmV2Config.ReadRequestsToMultipleNodes() {