Develop to Main

.gitignore

@@ -4,5 +4,6 @@ node_modules
 /messages
 dist
 .jest
-
+jwt.hex
 /.idea
+artifacts

README.md

@@ -53,16 +53,17 @@ Options are configured via environment variables.
 |--------------------------------| -------- | --------------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | EXECUTION_NODE                 | Yes      | http://1.2.3.4:8545   | Ethereum Execution Node endpoint                                                                                                                                                                                                                        |
 | CONSENSUS_NODE                 | Yes      | http://1.2.3.4:5051   | Ethereum Consensus Node endpoint                                                                                                                                                                                                                        |
+| JWT_SECRET_PATH                | No       | /data/neth/jwt.hex    | Path to JWT secret hex file for Nethermind RPC authentication. If provided, will generate a new JWT token for each execution node request                                                                                                                |
 | LOCATOR_ADDRESS                | Yes      | 0x123                 | Address of the Locator contract [Goerli](https://docs.lido.fi/deployed-contracts/goerli/) / [Mainnet](https://docs.lido.fi/deployed-contracts/)                                                                                                         |
 | STAKING_MODULE_ID              | Yes      | 123                   | Staking Module ID for which operator ID is set, currently only one exists - ([NodeOperatorsRegistry](https://github.com/lidofinance/lido-dao#contracts)) with id `1`                                                                                    |
 | OPERATOR_ID                    | Yes      | 123                   | Operator ID in the Node Operators registry, easiest to get from Operators UI: [Goerli](https://operators.testnet.fi)/[Mainnet](https://operators.lido.fi)                                                                                               |
+| OPERATOR_IDENTIFIERS           | No       | [0,1,2]               | Alternative to OPERATOR_ID. Array of Operator IDs to handle exits for multiple operators simultaneously                                                                                                                                                 |
 | MESSAGES_LOCATION              | No       | messages              | Local folder or external storage bucket url to load json exit message files from. Required if you are using exit messages mode                                                                                                                          |
 | VALIDATOR_EXIT_WEBHOOK         | No       | http://webhook        | POST validator info to an endpoint instead of sending out an exit message in order to initiate an exit. Required if you are using webhook mode                                                                                                          |
 | ORACLE_ADDRESSES_ALLOWLIST     | Yes      | ["0x123"]             | Allowed Oracle addresses to accept transactions from [Goerli](https://testnet.testnet.fi/#/lido-testnet-prater/0x24d8451bc07e7af4ba94f69acdd9ad3c6579d9fb/) / [Mainnet](https://mainnet.lido.fi/#/lido-dao/0x442af784a788a5bd6f42a01ebe9f287a871243fb/) |
 | MESSAGES_PASSWORD              | No       | password              | Password to decrypt encrypted exit messages with. Needed only if you encrypt your exit messages                                                                                                                                                         |
 | MESSAGES_PASSWORD_FILE         | No       | password_inside.txt   | Path to a file with password inside to decrypt exit messages with. Needed only if you have encrypted exit messages. If used, MESSAGES_PASSWORD (not MESSAGES_PASSWORD_FILE) needs to be added to LOGGER_SECRETS in order to be sanitized                |
 | BLOCKS_PRELOAD                 | No       | 50000                 | Amount of blocks to load events from on start. Increase if daemon was not running for some time. Defaults to a week of blocks                                                                                                                           |
-| BLOCKS_LOOP                    | No       | 900                   | Amount of blocks to load events from on every poll. Defaults to 3 hours of blocks                                                                                                                                                                       |
 | JOB_INTERVAL                   | No       | 384000                | Time interval in milliseconds to run checks. Defaults to time of 1 epoch                                                                                                                                                                                |
 | HTTP_PORT                      | No       | 8989                  | Port to serve metrics and health check on                                                                                                                                                                                                               |
 | RUN_METRICS                    | No       | false                 | Enable metrics endpoint                                                                                                                                                                                                                                 |
@@ -129,7 +130,7 @@ Available metrics:
 - exit_messages: ['valid'] - Exit messages and their validity: JSON parseability, structure and signature
 - exit_actions: ['result'] - Statuses of initiated validator exits
 - event_security_verification: ['result'] - Statuses of exit event security verifications
-- polling_last_blocks_duration_seconds: ['eventsNumber'] - Duration of pooling last blocks in microseconds
+- polling_last_blocks_duration_seconds: ['eventsNumber'] - Duration of polling last blocks in microseconds
 - execution_request_duration_seconds: ['result', 'status', 'domain'] - Execution node request duration in microseconds
 - consensus_request_duration_seconds: ['result', 'status', 'domain'] - Consensus node request duration in microseconds
 - job_duration_seconds: ['name', 'interval', 'result'] - Duration of Ejector cycle cron job
@@ -168,3 +169,4 @@ ln -s /opt/homebrew/bin/python3 /usr/local/bin/python
 ```
 
 More info here - https://github.com/ChainSafe/lodestar/issues/4767#issuecomment-1640631566
+

package.json

@@ -1,6 +1,7 @@
 {
   "name": "validator-ejector",
-  "version": "1.7.0",
+  "version": "1.7.1",
+  "private": true,
   "main": "index.js",
   "license": "MIT",
   "dependencies": {
@@ -16,14 +17,15 @@
     "@lodestar/utils": "1.2.2",
     "dotenv": "16.0.3",
     "ethers": "5.7.2",
-    "lido-nanolib": "1.4.0",
+    "jsonwebtoken": "9.0.0",
+    "lru-cache": "10.4.3",
     "node-fetch": "3.3.0",
     "prom-client": "14.1.0",
     "typescript": "4.9.3"
   },
   "type": "module",
   "scripts": {
-    "dev": "ts-node src/index.ts",
+    "dev": "node --loader ts-node/esm --disable-warning=ExperimentalWarning src/index.ts",
     "test": "vitest",
     "coverage": "vitest run --coverage",
     "lint": "eslint --ext ts .",
@@ -32,17 +34,18 @@
     "encrypt": "ts-node encryptor/encrypt.ts"
   },
   "devDependencies": {
+    "@types/jsonwebtoken": "9.0.1",
     "@types/node": "18.11.11",
     "@typescript-eslint/eslint-plugin": "5.45.1",
     "@typescript-eslint/parser": "5.45.1",
     "eslint": "8.29.0",
     "eslint-config-prettier": "8.5.0",
     "eslint-plugin-prettier": "4.2.1",
-    "nock": "^13.3.1",
+    "nock": "13.3.1",
     "prettier": "2.8.0",
     "ts-node": "10.9.1",
-    "vite": "^4.3.3",
-    "vitest": "^0.30.1"
+    "vite": "4.5.13",
+    "vitest": "0.30.1"
   },
   "jest": {
     "extensionsToTreatAsEsm": [
@@ -63,5 +66,6 @@
       "dotenv/config"
     ],
     "testEnvironment": "node"
-  }
+  },
+  "packageManager": "yarn@1.22.22+sha512.a6b2f7906b721bba3d67d4aff083df04dad64c399707841b7acf00f6b133b7ac24255f2652fa22ae3534329dc6180534e98d17432037ff6fd140556e2bb3137e"
 }

sample.env

@@ -1,4 +1,5 @@
 EXECUTION_NODE=http://1.2.3.4:8545
+JWT_SECRET_PATH=/data/neth/jwt.hex
 CONSENSUS_NODE=http://1.2.3.4:5051
 LOCATOR_ADDRESS=0x1eDf09b5023DC86737b59dE68a8130De878984f5
 STAKING_MODULE_ID=1

src/app/app.spec.ts

@@ -1,4 +1,4 @@
-import { makeLogger } from 'lido-nanolib'
+import { makeLogger } from '../lib/index.js'
 import nock from 'nock'
 import { makeConfig as mC } from '../services/config/service.js'
 import { mockEthServer } from '../test/mock-eth-server.js'

src/app/interface.ts

@@ -1,4 +1,4 @@
-import type { JobRunnerService, LoggerService } from 'lido-nanolib'
+import type { JobRunnerService, LoggerService } from '../lib/index.js'
 import type { ConfigService } from '../services/config/service.js'
 import type { MetricsService } from '../services/prom/service.js'
 import type { HttpHandlerService } from '../services/http-handler/service.js'

src/app/module.ts

@@ -1,13 +1,13 @@
-import { makeLogger } from 'lido-nanolib'
-import { makeRequest } from 'lido-nanolib'
+import { makeLogger } from '../lib/index.js'
+import { makeRequest } from '../lib/index.js'
 import {
   logger as loggerMiddleware,
   notOkError,
   retry,
   abort,
   prom,
-} from 'lido-nanolib'
-import { makeJobRunner } from 'lido-nanolib'
+} from '../lib/index.js'
+import { makeJobRunner } from '../lib/index.js'
 
 import dotenv from 'dotenv'
 
@@ -18,6 +18,7 @@ import {
 } from '../services/config/service.js'
 import { makeConsensusApi } from '../services/consensus-api/service.js'
 import { makeExecutionApi } from '../services/execution-api/service.js'
+import { makeJwtService } from '../services/jwt/service.js'
 import { makeMetrics, register } from '../services/prom/service.js'
 import { makeLocalFileReader } from '../services/local-file-reader/service.js'
 import { makeMessagesProcessor } from '../services/messages-processor/service.js'
@@ -30,6 +31,7 @@ import { makeGsStore } from '../services/gs-store/service.js'
 import { makeApp } from './service.js'
 import { makeMessageReloader } from '../services/message-reloader/message-reloader.js'
 import { makeForkVersionResolver } from '../services/fork-version-resolver/service.js'
+import { makeExitLogsService } from '../services/exit-logs/service.js'
 
 dotenv.config()
 
@@ -49,19 +51,31 @@ export const makeAppModule = async () => {
 
   const metrics = makeMetrics({ PREFIX: config.PROM_PREFIX })
 
+  const jwtService = makeJwtService(logger, {
+    JWT_SECRET_PATH: config.JWT_SECRET_PATH || '',
+  })
+
+  if (config.JWT_SECRET_PATH) {
+    await jwtService.initialize()
+  }
+
+  const executionHttp = makeRequest([
+    retry(3),
+    loggerMiddleware(logger),
+    prom(metrics.executionRequestDurationSeconds),
+    notOkError(),
+    abort(30_000),
+  ])
+
   const executionApi = makeExecutionApi(
-    makeRequest([
-      retry(3),
-      loggerMiddleware(logger),
-      prom(metrics.executionRequestDurationSeconds),
-      notOkError(),
-      abort(30_000),
-    ]),
+    executionHttp,
     logger,
     config,
-    metrics
+    jwtService
   )
 
+  const exitLogs = makeExitLogsService(logger, executionApi, config, metrics)
+
   const consensusApi = makeConsensusApi(
     makeRequest([
       retry(3),
@@ -110,6 +124,7 @@ export const makeAppModule = async () => {
     config,
     messageReloader,
     executionApi,
+    exitLogs,
     consensusApi,
     messagesProcessor,
     webhookProcessor,

src/app/service.ts

@@ -1,5 +1,6 @@
 import type { Dependencies } from './interface.js'
 import { MessageStorage } from '../services/job-processor/message-storage.js'
+import { getHeapStatistics } from 'v8'
 
 export const makeApp = ({
   config,
@@ -11,14 +12,22 @@ export const makeApp = ({
   consensusApi,
   appInfoReader,
 }: Dependencies) => {
-  const { OPERATOR_ID, BLOCKS_PRELOAD, BLOCKS_LOOP, JOB_INTERVAL } = config
+  const { OPERATOR_ID, BLOCKS_PRELOAD, JOB_INTERVAL, OPERATOR_IDENTIFIERS } =
+    config
 
   let ejectorCycleTimer: NodeJS.Timer | null = null
 
   const run = async () => {
     const version = await appInfoReader.getVersion()
     const mode = config.MESSAGES_LOCATION ? 'message' : 'webhook'
-    logger.info(`Validator Ejector v${version} started in ${mode} mode`, config)
+
+    const { heap_size_limit } = getHeapStatistics()
+    const heapLimit = Math.round(heap_size_limit / 1024 / 1024).toString()
+
+    logger.info(`Validator Ejector v${version} started in ${mode} mode`, {
+      ...config,
+      heapLimit,
+    })
 
     metrics.buildInfo
       .labels({
@@ -35,23 +44,25 @@ export const makeApp = ({
     const messageStorage = new MessageStorage()
 
     logger.info(
-      `Starting, searching only for requests for operator ${OPERATOR_ID}`
+      `Starting, searching only for requests for operators ${
+        OPERATOR_ID ?? OPERATOR_IDENTIFIERS
+      }`
     )
 
     logger.info(`Loading initial events for ${BLOCKS_PRELOAD} last blocks`)
+    const fetchTimeStart = performance.now()
+
     await job.once({
-      eventsNumber: BLOCKS_PRELOAD,
       messageStorage: messageStorage,
     })
 
-    logger.info(
-      `Starting ${
-        JOB_INTERVAL / 1000
-      } seconds polling for ${BLOCKS_LOOP} last blocks`
-    )
+    const fetchTimeEnd = performance.now()
+    const fetchTime = Math.ceil(fetchTimeEnd - fetchTimeStart) / 1000
+    logger.info(`Initial events loaded in ${fetchTime} seconds`)
+
+    logger.info(`Starting ${JOB_INTERVAL / 1000} seconds polling`)
 
-    ejectorCycleTimer = job.pooling({
-      eventsNumber: BLOCKS_LOOP,
+    ejectorCycleTimer = job.loop({
       messageStorage: messageStorage,
     })
   }

src/index.ts

@@ -1,4 +1,4 @@
-import { makeLogger } from 'lido-nanolib'
+import { makeLogger } from './lib/index.js'
 import { makeAppModule } from './app/module.js'
 
 const bootstrap = async () => {

src/lib/index.ts

@@ -0,0 +1,5 @@
+export * from './job/index.js'
+export * from './logger/index.js'
+export * from './request/index.js'
+export * from './request/middlewares.js'
+export * from './validator/index.js'

src/lib/job/index.ts

@@ -0,0 +1,53 @@
+import type { Histogram } from 'prom-client'
+
+import { makeLogger } from '../logger/index.js'
+
+const jobLockMap: Record<string, boolean> = {}
+
+export type JobRunnerService = ReturnType<typeof makeJobRunner>
+
+export const makeJobRunner = <Initial>(
+  name: string,
+  {
+    config,
+    logger,
+    metric,
+    handler,
+  }: {
+    config: { JOB_INTERVAL: number }
+    logger: ReturnType<typeof makeLogger>
+    metric: Histogram
+    handler: (handlerValue: Initial) => Promise<void>
+  }
+) => {
+  const once = async (handlerValue: Initial) => {
+    if (jobLockMap[name]) return
+    jobLockMap[name] = true
+
+    const end = metric.startTimer({
+      name,
+      interval: config.JOB_INTERVAL,
+    })
+
+    try {
+      logger.debug('Job started', { name })
+      await handler(handlerValue)
+      end({ result: 'success' })
+    } catch (error) {
+      logger.warn('Job ended with error', error)
+      end({ result: 'error' })
+    } finally {
+      jobLockMap[name] = false
+      logger.debug('Job ended', { name })
+    }
+  }
+
+  const loop = (handlerValue: Initial) => {
+    return setInterval(() => once(handlerValue), config.JOB_INTERVAL)
+  }
+
+  return {
+    once,
+    loop,
+  }
+}

src/lib/logger/index.ts

@@ -0,0 +1,47 @@
+import { printer } from './printer.js'
+import { serializeErrorWithCause } from './serialize-error.js'
+import type { LoggerOptions, Logger } from './types'
+
+export const LOG_LEVELS = ['error', 'warn', 'log', 'info', 'debug']
+
+export type LoggerService = ReturnType<typeof makeLogger>
+
+export const makeLogger = (options: LoggerOptions) => {
+  const {
+    level,
+    format,
+    silent,
+    causeDepth = 3,
+    sanitizer = { secrets: [], replacer: '*' },
+  }: LoggerOptions = options
+  return LOG_LEVELS.reduce((logger, logLevel) => {
+    logger[logLevel] = (message: string, details?: any) => {
+      const logLevelOrder = LOG_LEVELS.indexOf(logLevel)
+      if (LOG_LEVELS.indexOf(level) < logLevelOrder) return
+      const output: {
+        message: string
+        level: string
+        timestamp: number
+
+        details?: unknown
+      } = {
+        message,
+        details,
+        level: logLevel,
+        timestamp: Math.floor(Date.now() / 1000),
+      }
+
+      if (details instanceof Error) {
+        output.details = {
+          ...details,
+          ...serializeErrorWithCause(details, causeDepth),
+        }
+      }
+
+      const print = format === 'simple' ? printer.simple : printer.json
+
+      if (!silent) print(output, logLevel, sanitizer)
+    }
+    return logger
+  }, {}) as Logger
+}