Extend nitro-testnode to run consensus and execution separately

README.md

@@ -100,6 +100,11 @@ To fund the address 0x1111222233334444555566667777888899990000 on l2, use:
 ./test-node.bash script send-l2 --to address_0x1111222233334444555566667777888899990000
 ```
 
+To get private key of an account you can:
+```bash
+./test-node.bash script print-private-key --account funnel
+```
+
 For help and further scripts, see:
 
 ```bash

docker-compose.yaml

@@ -154,6 +154,99 @@ services:
       - "consensus:/consensus"
       - "config:/config"
 
+  execution-follower-node:
+    pid: host # allow debugging
+    image: nitro-node-dev-testnode
+    entrypoint: /usr/local/bin/nitro
+    ports:
+      - "127.0.0.1:7247:8547"
+      - "127.0.0.1:7248:8548"
+      - "127.0.0.1:9682:9682"
+    volumes:
+      - "seqdata:/home/user/.arbitrum/local/nitro"
+      - "l1keystore:/home/user/l1keystore"
+      - "config:/config"
+      - "tokenbridge-data:/tokenbridge-data"
+    command:
+      - --conf.file=/config/consensus_config.json
+      - --node.feed.output.enable
+      - --http.api=net,web3,eth,txpool,debug,timeboost,auctioneer
+      - --node.seq-coordinator.my-url=http://sequencer:8547
+      - --graphql.enable
+      - --graphql.vhosts=*
+      - --graphql.corsdomain=*
+      - --ws.addr=0.0.0.0
+      - --ws.port=9682
+      - --ws.api=net,web3,eth,txpool,debug,nitroexecution
+      - --execution.rpc-server.enable
+      - --execution.rpc-server.public
+      - --execution.rpc-server.authenticated=false
+      - --execution.consensus-rpc-client.url=ws://consensus-follower-node:8552
+      - --execution.consensus-rpc-client.jwtsecret=/config/jwt.hex
+      - --execution.consensus-rpc-client.retries=3
+      - --execution.consensus-rpc-client.connection-wait=15s
+      - --execution.consensus-rpc-client.retry-delay=1s
+    depends_on:
+      - geth
+
+  consensus-follower-node:
+    pid: host # allow debugging
+    image: nitro-node-dev-testnode
+    entrypoint: /usr/local/bin/nitro
+    ports:
+      - "127.0.0.1:7147:8547"
+      - "127.0.0.1:7148:8548"
+      - "127.0.0.1:8552:8552"
+    volumes:
+      - "seqdata:/home/user/.arbitrum/local/nitro"
+      - "l1keystore:/home/user/l1keystore"
+      - "config:/config"
+      - "tokenbridge-data:/tokenbridge-data"
+    command:
+      - --conf.file=/config/consensus_config.json
+      - --http.api=net,web3,eth,txpool,debug,timeboost,auctioneer
+      - --node.seq-coordinator.my-url=http://sequencer:8547
+      - --http.api=net,web3,eth,txpool,debug,timeboost,auctioneer
+      - --graphql.enable
+      - --graphql.vhosts=*
+      - --graphql.corsdomain=*
+      - --ws.addr=0.0.0.0
+      - --ws.port=8552
+      - --ws.api=net,web3,eth,txpool,debug,nitroconsensus
+      - --node.rpc-server.enable
+      - --node.rpc-server.public
+      - --node.rpc-server.authenticated=false
+      - --node.execution-rpc-client.url=ws://execution-follower-node:9682
+      - --node.execution-rpc-client.jwtsecret=/config/jwt.hex
+      - --node.execution-rpc-client.retries=3
+      - --node.execution-rpc-client.connection-wait=15s
+      - --node.execution-rpc-client.retry-delay=1s
+    depends_on:
+      - geth
+
+  regular-follower-node:
+    pid: host # allow debugging
+    image: nitro-node-dev-testnode
+    entrypoint: /usr/local/bin/nitro
+    ports:
+      - "127.0.0.1:7447:8547"
+      - "127.0.0.1:7548:8548"
+    volumes:
+      - "seqdata:/home/user/.arbitrum/local/nitro"
+      - "l1keystore:/home/user/l1keystore"
+      - "config:/config"
+      - "tokenbridge-data:/tokenbridge-data"
+    command:
+      - --conf.file=/config/consensus_config.json
+      - --http.api=net,web3,eth,txpool,debug,timeboost,auctioneer
+      - --node.seq-coordinator.my-url=http://sequencer:8547
+      - --http.api=net,web3,eth,txpool,debug,timeboost,auctioneer
+      - --graphql.enable
+      - --graphql.vhosts=*
+      - --graphql.corsdomain=*
+    depends_on:
+      - geth
+
   sequencer:
     pid: host # allow debugging
     image: nitro-node-dev-testnode

scripts/config.ts

@@ -268,9 +268,6 @@ function writeConfigs(argv: any) {
                     "enable": true,
                     "urls": ["http://das-mirror:9877"],
                 },
-                // TODO Fix das config to not need this redundant config
-                "parent-chain-node-url": argv.l1url,
-                "sequencer-inbox-address": "not_set"
             }
         },
         "execution": {
@@ -292,11 +289,8 @@ function writeConfigs(argv: any) {
         },
     }
 
-    baseConfig.node["data-availability"]["sequencer-inbox-address"] = ethers.utils.hexlify(getChainInfo()[0]["rollup"]["sequencer-inbox"]);
-
     if (argv.referenceDA) {
         (baseConfig as any).node["da"] = {
-            "mode": "external",
             "external-provider": {
                 "enable": true,
                 "with-writer": false,
@@ -309,6 +303,11 @@ function writeConfigs(argv: any) {
 
     const baseConfJSON = JSON.stringify(baseConfig)
 
+    // setup consensus config
+    let consensusConfig = JSON.parse(baseConfJSON)
+    consensusConfig.persistent.chain = "consensus-local"
+    fs.writeFileSync(path.join(consts.configpath, "consensus_config.json"), JSON.stringify(consensusConfig))
+
     if (argv.simple) {
         let simpleConfig = JSON.parse(baseConfJSON)
         simpleConfig.node.staker.enable = true
@@ -480,8 +479,10 @@ function writeL2DASCommitteeConfig(argv: any) {
                 "enable": true,
                 "enable-expiry": true
             },
-            "sequencer-inbox-address": sequencerInboxAddr,
-            "parent-chain-node-url": argv.l1url
+        },
+        "parent-chain": {
+            "node-url": argv.l1url,
+            "sequencer-inbox-address": sequencerInboxAddr
         },
         "enable-rest": true,
         "enable-rpc": true,
@@ -504,8 +505,6 @@ function writeL2DASMirrorConfig(argv: any, sequencerInboxAddr: string) {
                 "enable": true,
                 "enable-expiry": false
             },
-            "sequencer-inbox-address": sequencerInboxAddr,
-            "parent-chain-node-url": argv.l1url,
             "rest-aggregator": {
                 "enable": true,
                 "sync-to-storage": {
@@ -517,6 +516,10 @@ function writeL2DASMirrorConfig(argv: any, sequencerInboxAddr: string) {
                 "urls": ["http://das-committee-a:9877", "http://das-committee-b:9877"],
             }
         },
+        "parent-chain": {
+            "node-url": argv.l1url,
+            "sequencer-inbox-address": sequencerInboxAddr
+        },
         "enable-rest": true,
         "enable-rpc": false,
         "log-level": "INFO",

test-node.bash

@@ -50,6 +50,8 @@ blockscout=false
 tokenbridge=false
 l3node=false
 consensusclient=false
+run_consensus_and_execution_in_different_processes=false
+follower_node=false
 redundantsequencers=0
 l3_custom_fee_token=false
 l3_custom_fee_token_pricer=false
@@ -174,6 +176,14 @@ while [[ $# -gt 0 ]]; do
             build_utils=true
             shift
             ;;
+        --run-consensus-and-execution-in-different-processes)
+            run_consensus_and_execution_in_different_processes=true
+            shift
+            ;;
+        --follower-node)
+            follower_node=true
+            shift
+            ;;
         --validate)
             simple=false
             validate=true
@@ -321,6 +331,7 @@ while [[ $# -gt 0 ]]; do
             echo --l2-referenceda  run the L2 with reference external data availability provider
             echo --l2-timeboost    run the L2 with Timeboost enabled, including auctioneer and bid validator
             echo --batchposters    batch posters [0-3]
+            echo --follower-node   run a follower node 
             echo --redundantsequencers redundant sequencers [0-3]
             echo --detach          detach from nodes after running them
             echo --blockscout      build or launch blockscout
@@ -337,16 +348,52 @@ while [[ $# -gt 0 ]]; do
             echo --no-build-dev-blockscout  don\'t rebuild dev blockscout docker image
             echo --build-utils         rebuild scripts, rollupcreator, token bridge docker images
             echo --no-build-utils      don\'t rebuild scripts, rollupcreator, token bridge docker images
+            echo --run-consensus-and-execution-in-different-processes  run consensus and execution node in different processed communicating over rpc
             echo --force-build-utils   force rebuilding utils, useful if NITRO_CONTRACTS_BRANCH or TOKEN_BRIDGE_BRANCH changes
             echo
             echo script runs inside a separate docker. For SCRIPT-ARGS, run $0 script --help
             exit 0
     esac
 done
 
+# TODO: We can't run a full execution client and a consensus in different processes. Needs https://github.com/OffchainLabs/nitro/pull/4264
+# NODES=""
+# INITIAL_SEQ_NODES=""
+
+# # TODO: what to do in case when l3node is enabled or l2timeboost is enabled? sequencer seems to be hardcoded there.
+# if $run_consensus_and_execution_in_different_processes && $simple; then
+#     echo == Setting up node to run in split mode using RPC
+#     NODES="$NODES sequencer consensus-follower-node execution-follower-node"
+#     INITIAL_SEQ_NODES="$INITIAL_SEQ_NODES sequencer consensus-follower-node execution-follower-node"
+#     if [ $redundantsequencers -gt 0 ]; then
+#         echo !!! Redundant sequencers not allowed when running in split mode !!!
+#     fi
+# else
+#     NODES="$NODES sequencer"
+#     INITIAL_SEQ_NODES="$INITIAL_SEQ_NODES sequencer"
+#     if [ $redundantsequencers -gt 0 ]; then
+#         NODES="$NODES sequencer_b"
+#         INITIAL_SEQ_NODES="$INITIAL_SEQ_NODES sequencer_b"
+#     fi
+#     if [ $redundantsequencers -gt 1 ]; then
+#         NODES="$NODES sequencer_c"
+#     fi
+#     if [ $redundantsequencers -gt 2 ]; then
+#         NODES="$NODES sequencer_d"
+#     fi
+# fi
+
 NODES="sequencer"
 INITIAL_SEQ_NODES="sequencer"
 
+if $run_consensus_and_execution_in_different_processes && $simple; then
+    NODES="$NODES consensus-follower-node execution-follower-node"
+fi
+
+if $follower_node && $simple; then
+    NODES="$NODES regular-follower-node"
+fi
+
 if ! $simple; then
     NODES="$NODES redis"
 fi
@@ -598,10 +645,13 @@ if $force_init; then
         run_script redis-init --redundancy $redundantsequencers
     fi
 
-    echo == Funding l2 funnel and dev key
+    echo == Spinning up sequencer nodes
     docker compose up --wait $INITIAL_SEQ_NODES
+    echo == Sleeping for 45s allow for parent chain to recieve the contract creation tx and process it
     sleep 45 # in case we need to create a smart contract wallet, allow for parent chain to recieve the contract creation tx and process it
+    echo == Funding l2 funnel and dev key
     run_script bridge-funds --ethamount 100000 --wait
+    echo == Funding l2owner
     run_script send-l2 --ethamount 100 --to l2owner --wait
     rollupAddress=`docker compose run --rm --entrypoint sh poster -c "jq -r '.[0].rollup.rollup' /config/deployed_chain_info.json | tail -n 1 | tr -d '\r\n'"`