Develop to main

.github/CODEOWNERS

@@ -0,0 +1,2 @@
+* @lidofinance/lido-valset-oracles
+.github @lidofinance/review-gh-workflows

.github/workflows/test.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Set up node
         uses: actions/setup-node@v3
         with:
-          node-version: 'lts/*'
+          node-version: '16'
       - name: Get yarn cache directory path
         id: yarn-cache-dir-path
         run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT

.github/workflows/tests_and_checks.yml

@@ -10,7 +10,7 @@ jobs:
       - name: Set up node
         uses: actions/setup-node@v3.5.0
         with:
-          node-version: 'lts/*'
+          node-version: '16'
           cache: 'yarn'
       - name: Install dependencies
         run: yarn install --immutable

README.md

@@ -51,13 +51,14 @@ Options are configured via environment variables.
 
 | Variable                                      | Required | Default/Example       | Description                                                                                                                                                                                                                                                                     |
 |-----------------------------------------------|----------|-----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| EXECUTION_NODE                                | Yes      | http://1.2.3.4:8545   | Ethereum Execution Node endpoint                                                                                                                                                                                                                                                |
-| CONSENSUS_NODE                                | Yes      | http://1.2.3.4:5051   | Ethereum Consensus Node endpoint                                                                                                                                                                                                                                                |
-| JWT_SECRET_PATH                               | No       | /data/neth/jwt.hex    | Path to JWT secret hex file for Nethermind RPC authentication. If provided, will generate a new JWT token for each execution node request                                                                                                                                       |
+| EXECUTION_NODE                                | Yes      | http://1.2.3.4:8545   | Ethereum Execution Node endpoint. Accepts a single URL or a comma-separated list (e.g. `http://primary:8545,http://backup:8545`). Each request tries the endpoints in order; on a 5xx, 408, 429, network or timeout error the daemon moves to the next URL. Other 4xx and validation errors are terminal (not retried across endpoints). Voluntary-exit submissions are an exception: they broadcast to every CL URL in parallel. **Cross-endpoint consistency.** At startup the daemon issues `eth_chainId` to every EL URL and `/eth/v1/config/deposit_contract` to every CL URL, and refuses to boot if the chain ids disagree. Misconfigured mixed-network URLs (e.g. mainnet + testnet pasted together) are caught at boot, not in production.        |
+| CONSENSUS_NODE                                | Yes      | http://1.2.3.4:5051   | Ethereum Consensus Node endpoint. Same multi-URL semantics as `EXECUTION_NODE`.                                                                                                                                                                                                                                                          |
+| JWT_SECRET_PATH                               | No       | /data/neth/jwt.hex    | Path to JWT secret hex file for Nethermind RPC authentication. If provided, will generate a new JWT token for each execution node request. The same secret is sent to every URL configured in `EXECUTION_NODE`.                                                                                                                          |
 | LOCATOR_ADDRESS                               | Yes      | 0x123                 | Address of the Locator contract [Hoodi](https://docs.lido.fi/deployed-contracts/hoodi/) / [Mainnet](https://docs.lido.fi/deployed-contracts/)                                                                                                                                   |
-| STAKING_MODULE_ID                             | Yes      | 123                   | Staking Module ID for which operator ID is set, currently only one exists - ([NodeOperatorsRegistry](https://github.com/lidofinance/lido-dao#contracts)) with id `1`                                                                                                            |
-| OPERATOR_ID                                   | Yes      | 123                   | Operator ID in the Node Operators registry, easiest to get from Operators UI: [Hoodi](https://operators-hoodi.testnet.fi/)/[Mainnet](https://operators.lido.fi)                                                                                                                 |
-| OPERATOR_IDENTIFIERS                          | No       | [0,1,2]               | Alternative to OPERATOR_ID. Array of Operator IDs to handle exits for multiple operators simultaneously                                                                                                                                                                         |
+| EJECTOR_SCOPE                                 | Yes*     | {"1":[123]}           | Preferred module-to-operators mapping: object keys are `stakingModuleId`, array values are `operatorIds`. Example: `{"1":[123]}` means staking module `1`, operator `123`; `{"1":[123],"4":[0,1]}` adds staking module `4` operators `0` and `1`. Replaces deprecated `STAKING_MODULE_ID`, `OPERATOR_ID`, and `OPERATOR_IDENTIFIERS`. |
+| STAKING_MODULE_ID                             | No       | 123                   | Deprecated legacy single-module config. Use `EJECTOR_SCOPE` instead.                                                                                                                                                                                                            |
+| OPERATOR_ID                                   | No       | 123                   | Deprecated legacy single-operator config. Use `EJECTOR_SCOPE` instead.                                                                                                                                                                                                          |
+| OPERATOR_IDENTIFIERS                          | No       | [0,1,2]               | Deprecated legacy multi-operator config for one `STAKING_MODULE_ID`. Use `EJECTOR_SCOPE` instead.                                                                                                                                                                               |
 | MESSAGES_LOCATION                             | No       | messages              | Local folder or external storage bucket url to load json exit message files from. Required if you are using exit messages mode                                                                                                                                                  |
 | VALIDATOR_EXIT_WEBHOOK                        | No       | http://webhook        | POST validator info to an endpoint instead of sending out an exit message in order to initiate an exit. Required if you are using webhook mode                                                                                                                                  |
 | ORACLE_ADDRESSES_ALLOWLIST                    | Yes      | ["0x123"]             | Allowed Oracle addresses to accept transactions. List can be obtained from HashConsensus contract on [Hoodi](https://hoodi.etherscan.io/address/0x32EC59a78abaca3f91527aeB2008925D5AaC1eFC)/[Mainnet](https://etherscan.io/address/0xD624B08C83bAECF0807Dd2c6880C3154a5F0B288)  |
@@ -67,13 +68,15 @@ Options are configured via environment variables.
 | MESSAGES_PASSWORD                             | No       | password              | Password to decrypt encrypted exit messages with. Needed only if you encrypt your exit messages                                                                                                                                                                                 |
 | MESSAGES_PASSWORD_FILE                        | No       | password_inside.txt   | Path to a file with password inside to decrypt exit messages with. Needed only if you have encrypted exit messages. If used, MESSAGES_PASSWORD (not MESSAGES_PASSWORD_FILE) needs to be added to LOGGER_SECRETS in order to be sanitized                                        |
 | BLOCKS_PRELOAD                                | No       | 50000                 | Amount of blocks to load events from on start. Increase if daemon was not running for some time. Defaults to a week of blocks                                                                                                                                                   |
+| VALIDATORS_BATCH_SIZE                         | No       | 1000                  | Validator IDs per Consensus API validators request batch.                                                                                                                                                                                                                       |
+| VOTING_EVENTS_FRAME_BLOCKS                    | No       | 216000                | How many blocks of Easy-Track motion events to look back through when security-verifying voluntary-exit requests. Default ~30 days. Lower values reduce RPC load but may skip older matching motion or voting events. |
 | JOB_INTERVAL                                  | No       | 384000                | Time interval in milliseconds to run checks. Defaults to time of 1 epoch                                                                                                                                                                                                        |
 | HTTP_PORT                                     | No       | 8989                  | Port to serve metrics and health check on                                                                                                                                                                                                                                       |
 | RUN_METRICS                                   | No       | false                 | Enable metrics endpoint                                                                                                                                                                                                                                                         |
 | RUN_HEALTH_CHECK                              | No       | true                  | Enable health check endpoint                                                                                                                                                                                                                                                    |
 | LOGGER_LEVEL                                  | No       | info                  | Severity level from which to start showing errors eg info will hide debug messages                                                                                                                                                                                              |
 | LOGGER_FORMAT                                 | No       | simple                | Simple or JSON log output: simple/json                                                                                                                                                                                                                                          |
-| LOGGER_SECRETS                                | No       | ["MESSAGES_PASSWORD"] | JSON string array of either env var keys to sanitize in logs or exact values                                                                                                                                                                                                    |
+| LOGGER_SECRETS                                | No       | ["MESSAGES_PASSWORD"] | JSON array of env var names or literal values to redact from logs. `EXECUTION_NODE` and `CONSENSUS_NODE` also redact each URL from a comma-separated RPC list. |
 | DRY_RUN                                       | No       | false                 | Run the service without actually sending out exit messages                                                                                                                                                                                                                      |
 | TRUST_MODE                                    | No       | false                 | Skip security checks for exit requests. Please, use it only if you running your own RPC nodes.                                                                                                                                                                                  |
 | DISABLE_SECURITY_DONT_USE_IN_PRODUCTION       | No       | false                 | Deprecated: Use TRUST_MODE instead. Skip security checks for exit requests                                                                                                                                                                                                      |

docker-compose.test.yml

@@ -8,6 +8,7 @@ services:
       - EXECUTION_NODE=${EXECUTION_NODE}
       - CONSENSUS_NODE=${CONSENSUS_NODE}
       - LOCATOR_ADDRESS=${LOCATOR_ADDRESS}
+      - EJECTOR_SCOPE
       - STAKING_MODULE_ID=${STAKING_MODULE_ID}
       - OPERATOR_ID=${OPERATOR_ID}
       - MESSAGES_LOCATION=${MESSAGES_LOCATION}

docker-compose.yml

@@ -9,6 +9,7 @@ services:
       - EXECUTION_NODE=${EXECUTION_NODE}
       - CONSENSUS_NODE=${CONSENSUS_NODE}
       - LOCATOR_ADDRESS=${LOCATOR_ADDRESS}
+      - EJECTOR_SCOPE
       - STAKING_MODULE_ID=${STAKING_MODULE_ID}
       - OPERATOR_ID=${OPERATOR_ID}
       - MESSAGES_LOCATION=${MESSAGES_LOCATION}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "validator-ejector",
-  "version": "1.9.0",
+  "version": "2.0.0",
   "private": true,
   "main": "index.js",
   "license": "MIT",
@@ -26,8 +26,10 @@
   "type": "module",
   "scripts": {
     "dev": "node --loader ts-node/esm --disable-warning=ExperimentalWarning src/index.ts",
-    "test": "vitest",
-    "coverage": "vitest run --coverage",
+    "test": "vitest run --config vite.config.ts",
+    "test:watch": "vitest --config vite.config.ts",
+    "test:e2e": "vitest run --config vite.e2e.config.ts",
+    "coverage": "vitest run --coverage --config vite.config.ts",
     "lint": "eslint --ext ts .",
     "start": "node dist/src/index.js",
     "build": "tsc --build",

sample.env

@@ -1,9 +1,26 @@
+# Single URL or comma-separated list. Each request tries the endpoints in
+# order; on a 5xx, 408, 429, network or timeout error the daemon moves to the
+# next URL. Other 4xx and validation errors are terminal.
 EXECUTION_NODE=http://1.2.3.4:8545
+# Optional: when set, includes a Bearer token on every EL request.
+# IMPORTANT: the same JWT secret is sent to *every* URL in EXECUTION_NODE.
+# If your EL nodes use different jwt.hex files (e.g. Nethermind + Geth),
+# you must keep them synced manually, or skip JWT (use unauthenticated RPC).
 JWT_SECRET_PATH=/data/neth/jwt.hex
+# Single URL or comma-separated list (same fallback semantics as EXECUTION_NODE).
+# Voluntary-exit submissions broadcast to every configured CL URL in parallel.
 CONSENSUS_NODE=http://1.2.3.4:5051
 LOCATOR_ADDRESS=0x1eDf09b5023DC86737b59dE68a8130De878984f5
-STAKING_MODULE_ID=1
-OPERATOR_ID=123
+
+# Preferred scope config: {"stakingModuleId":[operatorIds]}.
+# For multi-module setups:
+# EJECTOR_SCOPE={"1":[123],"4":[0,1]}
+EJECTOR_SCOPE={"1":[123]}
+
+# Deprecated legacy alternative:
+# STAKING_MODULE_ID=1
+# OPERATOR_ID=123
+# OPERATOR_IDENTIFIERS=[0,1,2]
 
 MESSAGES_LOCATION=messages
 MESSAGES_PASSWORD=pass
@@ -16,6 +33,7 @@ SUBMIT_TX_HASH_ALLOWLIST=[]
 EASY_TRACK_ADDRESS=0xF0211b7660680B49De1A7E9f25C65660F0a13Fea
 
 BLOCKS_PRELOAD=50000 # 7 days of blocks
+VALIDATORS_BATCH_SIZE=1000
 
 HTTP_PORT=8989
 RUN_METRICS=true
@@ -26,3 +44,4 @@ LOGGER_FORMAT=simple
 LOGGER_SECRETS=["MESSAGES_PASSWORD","EXECUTION_NODE", "CONSENSUS_NODE"]
 
 DRY_RUN=false
+TRUST_MODE=false

sample.infra.env

@@ -1,8 +1,21 @@
+# EXECUTION_NODE and CONSENSUS_NODE accept a single URL or a comma-separated list.
+# Each request tries endpoints in order; on a 5xx, 408, 429, network or timeout
+# error the daemon moves to the next URL. Other 4xx and validation errors are
+# terminal.
+# Voluntary-exit submissions broadcast to every configured CL URL in parallel.
 EXECUTION_NODE=http://1.2.3.4:8545
 CONSENSUS_NODE=http://1.2.3.4:5051
 LOCATOR_ADDRESS=0x1eDf09b5023DC86737b59dE68a8130De878984f5
-STAKING_MODULE_ID=123
-OPERATOR_ID=123
+
+# Preferred scope config: {"stakingModuleId":[operatorIds]}.
+# For multi-module setups:
+# EJECTOR_SCOPE={"1":[123],"4":[0,1]}
+EJECTOR_SCOPE={"123":[123]}
+
+# Deprecated legacy alternative:
+# STAKING_MODULE_ID=123
+# OPERATOR_ID=123
+# OPERATOR_IDENTIFIERS=[0,1,2]
 
 MESSAGES_LOCATION=messages
 MESSAGES_PASSWORD=pass
@@ -15,6 +28,7 @@ SUBMIT_TX_HASH_ALLOWLIST=[]
 EASY_TRACK_ADDRESS=0xF0211b7660680B49De1A7E9f25C65660F0a13Fea
 
 BLOCKS_PRELOAD=50000 # 7 days of blocks
+VALIDATORS_BATCH_SIZE=1000
 
 HTTP_PORT=8989
 RUN_METRICS=true
@@ -25,3 +39,4 @@ LOGGER_FORMAT=json
 LOGGER_SECRETS=["MESSAGES_PASSWORD","EXECUTION_NODE", "CONSENSUS_NODE"]
 
 DRY_RUN=true
+TRUST_MODE=false

src/app/app.spec.ts

@@ -1,6 +1,5 @@
 import { makeLogger } from '../lib/index.js'
 import nock from 'nock'
-import { makeConfig as mC } from '../services/config/service.js'
 import { mockEthServer } from '../test/mock-eth-server.js'
 import * as ELMocks from '../services/execution-api/fixtures.js'
 import * as CLMocks from '../services/consensus-api/fixtures.js'
@@ -12,7 +11,7 @@ dotenv.config()
 const mockConfig = async (config) => {
   const { makeConfig } = (await vi.importActual(
     '../services/config/service.js'
-  )) as { makeConfig: typeof mC }
+  )) as typeof import('../services/config/service.js')
 
   vi.doMock('../services/config/service.js', () => {
     return {

src/app/interface.ts

@@ -6,6 +6,7 @@ import type { ExecutionApiService } from '../services/execution-api/service.js'
 import type { ConsensusApiService } from '../services/consensus-api/service.js'
 import type { AppInfoReaderService } from '../services/app-info-reader/service.js'
 import type { MessageReloader } from '../services/message-reloader/message-reloader.js'
+import type { ConsistencyChecker } from '../services/consistency/service.js'
 
 export interface Dependencies {
   config: ConfigService
@@ -17,4 +18,5 @@ export interface Dependencies {
   executionApi: ExecutionApiService
   consensusApi: ConsensusApiService
   appInfoReader: AppInfoReaderService
+  consistencyChecker: ConsistencyChecker
 }

src/app/module.ts

@@ -32,6 +32,7 @@ import { makeApp } from './service.js'
 import { makeMessageReloader } from '../services/message-reloader/message-reloader.js'
 import { makeForkVersionResolver } from '../services/fork-version-resolver/service.js'
 import { makeExitLogsService } from '../services/exit-logs/service.js'
+import { makeConsistencyChecker } from '../services/consistency/service.js'
 
 dotenv.config()
 
@@ -74,6 +75,12 @@ export const makeAppModule = async () => {
     jwtService
   )
 
+  const consistencyChecker = makeConsistencyChecker(
+    executionHttp,
+    logger,
+    config
+  )
+
   const consensusApi = makeConsensusApi(
     makeRequest([
       retry(3),
@@ -158,6 +165,7 @@ export const makeAppModule = async () => {
     executionApi,
     consensusApi,
     appInfoReader,
+    consistencyChecker,
   })
 
   return {