Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .github/workflows/integration_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,14 @@ jobs:
# replace the github.com/evstack/ev-abci module with the local version
go mod edit -replace github.com/evstack/ev-abci=$GO_EXECUTION_ABCI_DIR

# pin dependency versions to avoid compatibility issues
go mod edit -replace github.com/libp2p/go-libp2p-quic-transport=github.com/libp2p/go-libp2p-quic-transport@v0.33.1
go mod edit -replace github.com/libp2p/go-libp2p=github.com/libp2p/go-libp2p@v0.43.0
go mod edit -replace github.com/quic-go/quic-go=github.com/quic-go/quic-go@v0.54.1
go mod edit -replace github.com/quic-go/webtransport-go=github.com/quic-go/webtransport-go@v0.9.0
go mod edit -replace github.com/multiformats/go-multiaddr=github.com/multiformats/go-multiaddr@v0.16.1
go mod edit -replace buf.build/go/protovalidate=buf.build/go/protovalidate@v0.12.0

# download dependencies and update go.mod/go.sum
go mod tidy

Expand Down
12 changes: 11 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,17 @@ RUN ignite app install github.com/ignite/apps/evolve@${IGNITE_EVOLVE_APP_VERSION

RUN go mod edit -replace github.com/evstack/ev-node=github.com/evstack/ev-node@${EVNODE_VERSION} && \
go mod edit -replace github.com/evstack/ev-abci=/workspace/ev-abci && \
go mod tidy
go mod edit -replace github.com/libp2p/go-libp2p-quic-transport=github.com/libp2p/go-libp2p-quic-transport@v0.33.1 && \
go mod edit -replace github.com/libp2p/go-libp2p=github.com/libp2p/go-libp2p@v0.43.0 && \
go mod edit -replace github.com/quic-go/quic-go=github.com/quic-go/quic-go@v0.54.1 && \
go mod edit -replace github.com/quic-go/webtransport-go=github.com/quic-go/webtransport-go@v0.9.0 && \
go mod edit -replace github.com/multiformats/go-multiaddr=github.com/multiformats/go-multiaddr@v0.16.1 && \
go mod edit -replace buf.build/go/protovalidate=buf.build/go/protovalidate@v0.12.0 && \
go mod tidy && \
go mod download

# Verify pinned module versions are effective in build context
RUN go list -m all | grep -E 'github.com/libp2p/go-libp2p|github.com/multiformats/go-multiaddr'

# TODO: replace this with proper ignite flag to skip IBC registration when available
# Patch out IBC registration (comment out the call and its error handling)
Expand Down
13 changes: 13 additions & 0 deletions Dockerfile.cosmos-sdk
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,19 @@ RUN if [ "$ENABLE_IBC" = "false" ]; then \
echo "IBC enabled, leaving registration intact."; \
fi

# Pin dependency versions to avoid compatibility issues
RUN go mod edit -replace github.com/libp2p/go-libp2p-quic-transport=github.com/libp2p/go-libp2p-quic-transport@v0.33.1 && \
go mod edit -replace github.com/libp2p/go-libp2p=github.com/libp2p/go-libp2p@v0.43.0 && \
go mod edit -replace github.com/quic-go/quic-go=github.com/quic-go/quic-go@v0.54.1 && \
go mod edit -replace github.com/quic-go/webtransport-go=github.com/quic-go/webtransport-go@v0.9.0 && \
go mod edit -replace github.com/multiformats/go-multiaddr=github.com/multiformats/go-multiaddr@v0.16.1 && \
go mod edit -replace buf.build/go/protovalidate=buf.build/go/protovalidate@v0.12.0 && \
go mod tidy && \
go mod download

# Verify pinned module versions are effective in build context
RUN go list -m all | grep -E 'github.com/libp2p/go-libp2p|github.com/multiformats/go-multiaddr'

# Build the Cosmos SDK binary
RUN ignite chain build --skip-proto

Expand Down
4 changes: 4 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ replace github.com/celestiaorg/go-header => github.com/julienrbrt/go-header v0.0
replace (
github.com/btcsuite/btcd => github.com/btcsuite/btcd v0.22.0-beta
github.com/cosmos/cosmos-sdk => github.com/cosmos/cosmos-sdk v0.50.14
github.com/libp2p/go-libp2p => github.com/libp2p/go-libp2p v0.43.0
github.com/libp2p/go-libp2p-quic-transport => github.com/libp2p/go-libp2p-quic-transport v0.33.1
github.com/quic-go/quic-go => github.com/quic-go/quic-go v0.54.1
github.com/quic-go/webtransport-go => github.com/quic-go/webtransport-go v0.9.0
)

exclude github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2
Expand Down
11 changes: 4 additions & 7 deletions modules/migrationmngr/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -116,14 +116,11 @@ The core of the migration is handled by returning `abci.ValidatorUpdate` message
- **Removing Old Validators**: The existing validators that are not part of the new sequencer/attester set have their power reduced to `0`.
- **Adding New Participants**: The new sequencer and/or attesters are added to the consensus set with a power of `1`.

### IBC-Aware Migrations
### Migration Execution

A critical feature of the migration manager is its awareness of the Inter-Blockchain Communication (IBC) protocol. A sudden, drastic change in the validator set can cause IBC light clients on other chains to fail their verification checks, leading to a broken connection.
The migration is performed in a single, atomic step. At the migration start height, all old validators are removed, and the new sequencer/attesters are added in one atomic update.

To prevent this, the module first checks if IBC is enabled by verifying the existence of the IBC module's store key.

- **If IBC is Enabled**: The migration is "smoothed" over a period of blocks (currently `30` blocks, defined by `IBCSmoothingFactor`). In each block during this period, a fraction of the old validators are removed, and a fraction of the new attesters are added. This gradual change ensures that IBC light clients can safely update their trusted validator sets without interruption. On the first block of the migration, all validators are set to have an equal power of `1` to prevent any single validator from having a disproportionate amount of power during the transition.
- **If IBC is Not Enabled**: The migration can be performed "immediately" in a single block. All old validators are removed, and the new sequencer/attesters are added in one atomic update at the migration start height.
**Note:** IBC light client updates must be performed at height H+1 (one block after the migration) to ensure proper verification of the validator set changes.

### The Chain Halt: A Coordinated Upgrade

Expand All @@ -143,7 +140,7 @@ If you are a validator or node operator on a chain using this module, you must b

1. **Monitor Governance**: The migration will be initiated by a governance proposal. Stay informed about upcoming proposals. The proposal will define the target block height for the migration.

2. **Prepare for the Chain Halt**: Your node **will stop** at a predictable block height, calculated from the migration's start height (`block_height` in the proposal). With IBC enabled, the halt is at `block_height + 31`; without IBC, it is `block_height + 2`. This is expected. Check your node's logs for the specific "MIGRATE" error message.
2. **Prepare for the Chain Halt**: Your node **will stop** at a predictable block height, calculated from the migration's start height (`block_height` in the proposal). The halt occurs at `block_height + 2`. This is expected. Check your node's logs for the specific "MIGRATE" error message.

3. **Perform the Upgrade**: Once the chain has halted, you must perform the following steps:
a. **Install the New Binary**: You will need to replace your current node software (e.g., `gmd`) with the new version required for the rollup.
Expand Down
13 changes: 4 additions & 9 deletions modules/migrationmngr/depinject.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,10 @@ func init() {
type ModuleInputs struct {
depinject.In

Config *modulev1.Module
Cdc codec.Codec
StoreService store.KVStoreService
AddressCodec address.Codec
// optional, used to detect if IBC module is enabled.
// When IBC module is present, use `depinject.Provide(IBCStoreKey(ibcStoreKey))`
IBCStoreKey keeper.IbcKVStoreKeyAlias `optional:"true"`

Config *modulev1.Module
Cdc codec.Codec
StoreService store.KVStoreService
AddressCodec address.Codec
StakingKeeper types.StakingKeeper
}

Expand All @@ -59,7 +55,6 @@ func ProvideModule(in ModuleInputs) ModuleOutputs {
in.StoreService,
in.AddressCodec,
in.StakingKeeper,
in.IBCStoreKey,
authority.String(),
)
m := NewAppModule(in.Cdc, k)
Expand Down
14 changes: 3 additions & 11 deletions modules/migrationmngr/keeper/abci.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,17 +77,9 @@ func (k Keeper) EndBlock(ctx context.Context) ([]abci.ValidatorUpdate, error) {
}

var updates []abci.ValidatorUpdate
if !k.isIBCEnabled(ctx) {
// if IBC is not enabled, we can migrate immediately
// but only return updates on the first block of migration (start height)
if uint64(sdkCtx.BlockHeight()) == start {
updates, err = k.migrateNow(ctx, migration, validatorSet)
if err != nil {
return nil, err
}
}
} else {
updates, err = k.migrateOver(sdkCtx, migration, validatorSet)
// Always perform immediate migration updates at the start height.
if uint64(sdkCtx.BlockHeight()) == start {
updates, err = k.migrateNow(ctx, migration, validatorSet)
if err != nil {
return nil, err
}
Expand Down
4 changes: 1 addition & 3 deletions modules/migrationmngr/keeper/grpc_query_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,6 @@ func initFixture(tb testing.TB) *fixture {
storeService,
addressCodec,
stakingKeeper,
nil,
sdk.AccAddress(address.Module(types.ModuleName)).String(),
)

Expand Down Expand Up @@ -144,7 +143,6 @@ func TestIsMigrating_IBCEnabled(t *testing.T) {
storeService,
addressCodec,
stakingKeeper,
func() *storetypes.KVStoreKey { return key },
sdk.AccAddress(address.Module(types.ModuleName)).String(),
)

Expand All @@ -159,7 +157,7 @@ func TestIsMigrating_IBCEnabled(t *testing.T) {
require.NoError(t, err)
require.True(t, resp.IsMigrating)
require.Equal(t, uint64(1), resp.StartBlockHeight)
require.Equal(t, 1+keeper.IBCSmoothingFactor, resp.EndBlockHeight)
require.Equal(t, uint64(2), resp.EndBlockHeight)
}

func TestSequencer_Migrating(t *testing.T) {
Expand Down
42 changes: 2 additions & 40 deletions modules/migrationmngr/keeper/keeper.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,24 +8,18 @@ import (
"cosmossdk.io/core/address"
corestore "cosmossdk.io/core/store"
"cosmossdk.io/log"
storetypes "cosmossdk.io/store/types"
"github.com/cosmos/cosmos-sdk/codec"
sdk "github.com/cosmos/cosmos-sdk/types"

"github.com/evstack/ev-abci/modules/migrationmngr/types"
)

// IbcStoreKey is the store key used for IBC-related data.
// It is an alias for storetypes.StoreKey to allow depinject to resolve it as a dependency (as runtime assumes 1 module = 1 store key maximum).
type IbcKVStoreKeyAlias = func() *storetypes.KVStoreKey

type Keeper struct {
storeService corestore.KVStoreService
cdc codec.BinaryCodec
addressCodec address.Codec
authority string

ibcStoreKey IbcKVStoreKeyAlias
stakingKeeper types.StakingKeeper

Schema collections.Schema
Expand All @@ -40,7 +34,6 @@ func NewKeeper(
storeService corestore.KVStoreService,
addressCodec address.Codec,
stakingKeeper types.StakingKeeper,
ibcStoreKey IbcKVStoreKeyAlias,
authority string,
) Keeper {
// ensure that authority is a valid account address
Expand All @@ -55,7 +48,6 @@ func NewKeeper(
authority: authority,
addressCodec: addressCodec,
stakingKeeper: stakingKeeper,
ibcStoreKey: ibcStoreKey,
Sequencer: collections.NewItem(
sb,
types.SequencerKey,
Expand Down Expand Up @@ -103,42 +95,12 @@ func (k Keeper) IsMigrating(ctx context.Context) (start, end uint64, ok bool) {
return 0, 0, false
}

// smoothen the migration over IBCSmoothingFactor blocks, in order to migrate the validator set to the sequencer or attesters network when IBC is enabled.
migrationEndHeight := migration.BlockHeight + IBCSmoothingFactor

// If IBC is not enabled, the migration can be done in one block.
if !k.isIBCEnabled(ctx) {
migrationEndHeight = migration.BlockHeight + 1
}
// Migration is performed in a single step.
migrationEndHeight := migration.BlockHeight + 1

sdkCtx := sdk.UnwrapSDKContext(ctx)
currentHeight := uint64(sdkCtx.BlockHeight())
migrationInProgress := currentHeight >= migration.BlockHeight && currentHeight <= migrationEndHeight

return migration.BlockHeight, migrationEndHeight, migrationInProgress
}

// isIBCEnabled checks if IBC is enabled on the chain.
// In order to not import the IBC module, we only check if the IBC store exists,
// but not the ibc params. This should be sufficient for our use case.
func (k Keeper) isIBCEnabled(ctx context.Context) bool {
enabled := true

if k.ibcStoreKey == nil {
return false
}

sdkCtx := sdk.UnwrapSDKContext(ctx)

ms := sdkCtx.MultiStore().CacheMultiStore()
defer func() {
if r := recover(); r != nil {
// If we panic, it means the store does not exist, so IBC is not enabled.
enabled = false
}
}()
ms.GetKVStore(k.ibcStoreKey())

// has not panicked, so store exists
return enabled
}
125 changes: 0 additions & 125 deletions modules/migrationmngr/keeper/migration.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,15 @@ package keeper

import (
"context"
"errors"

"cosmossdk.io/collections"
abci "github.com/cometbft/cometbft/abci/types"
sdkerrors "github.com/cosmos/cosmos-sdk/types/errors"
stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types"

"github.com/evstack/ev-abci/modules/migrationmngr/types"
)

// IBCSmoothingFactor is the factor used to smooth the migration process when IBC is enabled. It determines how many blocks the migration will take.
var IBCSmoothingFactor uint64 = 30

// migrateNow migrates the chain to evolve immediately.
// this method is used when ibc is not enabled, so no migration smoothing is needed.
func (k Keeper) migrateNow(
ctx context.Context,
migrationData types.EvolveMigration,
Expand Down Expand Up @@ -116,122 +110,3 @@ func migrateToAttesters(

return initialValUpdates, nil
}

// migrateOver migrates the chain to evolve over a period of blocks.
// this is to ensure ibc light client verification keep working while changing the whole validator set.
// the migration step is tracked in store.
func (k Keeper) migrateOver(
ctx context.Context,
migrationData types.EvolveMigration,
lastValidatorSet []stakingtypes.Validator,
) (initialValUpdates []abci.ValidatorUpdate, err error) {
step, err := k.MigrationStep.Get(ctx)
if err != nil && !errors.Is(err, collections.ErrNotFound) {
return nil, sdkerrors.ErrInvalidRequest.Wrapf("failed to get migration step: %v", err)
}

if step >= IBCSmoothingFactor {
// migration complete, just return the final set (same as migrateNow)
if err := k.MigrationStep.Remove(ctx); err != nil {
return nil, sdkerrors.ErrInvalidRequest.Wrapf("failed to remove migration step: %v", err)
}
return k.migrateNow(ctx, migrationData, lastValidatorSet)
}

switch len(migrationData.Attesters) {
case 0:
// no attesters, migrate to a single sequencer over smoothing period
// remove all validators except the sequencer, add sequencer at the end
seq := migrationData.Sequencer
var oldValsToRemove []stakingtypes.Validator
for _, val := range lastValidatorSet {
if !val.ConsensusPubkey.Equal(seq.ConsensusPubkey) {
oldValsToRemove = append(oldValsToRemove, val)
}
}
removePerStep := (len(oldValsToRemove) + int(IBCSmoothingFactor) - 1) / int(IBCSmoothingFactor)
startRemove := int(step) * removePerStep
endRemove := min(startRemove+removePerStep, len(oldValsToRemove))
for _, val := range oldValsToRemove[startRemove:endRemove] {
powerUpdate := val.ABCIValidatorUpdateZero()
initialValUpdates = append(initialValUpdates, powerUpdate)
}
default:
// attesters present, migrate as before
attesterPubKeys := make(map[string]struct{})
for _, attester := range migrationData.Attesters {
attesterPubKeys[attester.ConsensusPubkey.String()] = struct{}{}
}
var oldValsToRemove []stakingtypes.Validator
for _, val := range lastValidatorSet {
if _, ok := attesterPubKeys[val.ConsensusPubkey.String()]; !ok {
oldValsToRemove = append(oldValsToRemove, val)
}
}
lastValPubKeys := make(map[string]struct{})
for _, val := range lastValidatorSet {
lastValPubKeys[val.ConsensusPubkey.String()] = struct{}{}
}
var newAttestersToAdd []types.Attester
for _, attester := range migrationData.Attesters {
if _, ok := lastValPubKeys[attester.ConsensusPubkey.String()]; !ok {
newAttestersToAdd = append(newAttestersToAdd, attester)
}
}
removePerStep := (len(oldValsToRemove) + int(IBCSmoothingFactor) - 1) / int(IBCSmoothingFactor)
addPerStep := (len(newAttestersToAdd) + int(IBCSmoothingFactor) - 1) / int(IBCSmoothingFactor)
startRemove := int(step) * removePerStep
endRemove := min(startRemove+removePerStep, len(oldValsToRemove))
for _, val := range oldValsToRemove[startRemove:endRemove] {
powerUpdate := val.ABCIValidatorUpdateZero()
initialValUpdates = append(initialValUpdates, powerUpdate)
}
startAdd := int(step) * addPerStep
endAdd := min(startAdd+addPerStep, len(newAttestersToAdd))
for _, attester := range newAttestersToAdd[startAdd:endAdd] {
pk, err := attester.TmConsPublicKey()
if err != nil {
return nil, sdkerrors.ErrInvalidRequest.Wrapf("failed to get attester pubkey: %v", err)
}
attesterUpdate := abci.ValidatorUpdate{
PubKey: pk,
Power: 1,
}
initialValUpdates = append(initialValUpdates, attesterUpdate)
}
}

// increment and persist the step
if err := k.MigrationStep.Set(ctx, step+1); err != nil {
return nil, sdkerrors.ErrInvalidRequest.Wrapf("failed to set migration step: %v", err)
}

// the first time, we set the whole validator set to the same validator power. This is to avoid a validator ends up with >= 33% or worse >= 66%
// vp during the migration.
// TODO: add a test
if step == 0 {
// Create a map of existing updates for O(1) lookup
existingUpdates := make(map[string]bool)
for _, powerUpdate := range initialValUpdates {
existingUpdates[powerUpdate.PubKey.String()] = true
}

// set the whole validator set to the same power
for _, val := range lastValidatorSet {
valPubKey, err := val.CmtConsPublicKey()
if err != nil {
return nil, sdkerrors.ErrInvalidRequest.Wrapf("failed to get validator pubkey: %v", err)
}

if !existingUpdates[valPubKey.String()] {
powerUpdate := abci.ValidatorUpdate{
PubKey: valPubKey,
Power: 1,
}
initialValUpdates = append(initialValUpdates, powerUpdate)
}
}
}

return initialValUpdates, nil
}
Loading
Loading