added rpc manager for rpc failover and fallbacks

bun.lock

@@ -14,6 +14,7 @@
         "@opentelemetry/semantic-conventions": "^1.37.0",
         "@openzeppelin/merkle-tree": "1.0.8",
         "@rhinestone/module-sdk": "0.2.7",
+        "async-mutex": "^0.5.0",
         "axios": "^1.9.0",
         "bullmq": "^5.8.2",
         "cors": "^2.8.5",
@@ -578,6 +579,8 @@
 
     "assertion-error": ["assertion-error@2.0.1", "", {}, "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA=="],
 
+    "async-mutex": ["async-mutex@0.5.0", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-1A94B18jkJ3DYq284ohPxoXbfTA5HsQ7/Mf4DEhcyLx3Bz27Rh59iScbB6EPiP+B+joue6YCxcMXSbFC1tZKwA=="],
+
     "asynckit": ["asynckit@0.4.0", "", {}, "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="],
 
     "atomic-sleep": ["atomic-sleep@1.0.0", "", {}, "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ=="],

package.json

@@ -22,14 +22,15 @@
   "dependencies": {
     "@chainlink/env-enc": "^1.0.5",
     "@lifi/sdk": "^3.7.9",
-    "@openzeppelin/merkle-tree": "1.0.8",
-    "@rhinestone/module-sdk": "0.2.7",
     "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/auto-instrumentations-node": "^0.67.0",
     "@opentelemetry/exporter-trace-otlp-http": "^0.205.0",
     "@opentelemetry/resources": "^2.1.0",
     "@opentelemetry/sdk-node": "^0.205.0",
     "@opentelemetry/semantic-conventions": "^1.37.0",
-    "@opentelemetry/auto-instrumentations-node": "^0.67.0",
+    "@openzeppelin/merkle-tree": "1.0.8",
+    "@rhinestone/module-sdk": "0.2.7",
+    "async-mutex": "^0.5.0",
     "axios": "^1.9.0",
     "bullmq": "^5.8.2",
     "cors": "^2.8.5",

src/api/bootstrap.ts

@@ -4,6 +4,7 @@ import { parsePort } from "@/common";
 import { Logger } from "@/core/logger";
 import { HealthCheckService } from "@/health-check";
 import { NodeService } from "@/node";
+import { RpcManagerService } from "@/rpc-manager";
 import { type ClusterWorkerMessage, decodeMessage } from "@/workers";
 import cors from "cors";
 import express from "express";
@@ -23,6 +24,7 @@ export async function bootstrap() {
   const chainsService = Container.get(ChainsService);
   const healthCheckService = Container.get(HealthCheckService);
   const nodeService = Container.get(NodeService);
+  const rpcManagerService = Container.get(RpcManagerService);
 
   await nodeService.readVersion();
 
@@ -61,6 +63,19 @@ export async function bootstrap() {
         healthCheckService.setResult(data);
         break;
       }
+
+      case "rpcManagerConfig": {
+        rpcManagerService.setup(data, false);
+        break;
+      }
+
+      case "rpcProviderSyncConfig": {
+        rpcManagerService.syncProviders(
+          data.chainId,
+          data.rpcProviderSyncConfigs,
+        );
+        break;
+      }
     }
   });
 }

src/master/bootstrap.ts

@@ -4,6 +4,7 @@ import { Logger } from "@/core/logger";
 import { ExecutorService } from "@/executor";
 import { HealthCheckService } from "@/health-check";
 import { NodeService } from "@/node";
+import { RpcChainConfig, RpcManagerService } from "@/rpc-manager";
 import { SimulatorService } from "@/simulator";
 import { WorkersService } from "@/workers";
 import { entries, map } from "remeda";
@@ -18,10 +19,88 @@ export async function bootstrap() {
   const healthCheckService = Container.get(HealthCheckService);
   const nodeService = Container.get(NodeService);
   const workersService = Container.get(WorkersService);
+  const rpcManagerService = Container.get(RpcManagerService);
 
   await chainsService.initialize();
-  await batcherService.initialize();
   await workersService.initialize();
+
+  const chainsSettings = chainsService.getChainsSettings();
+
+  const rpcChainConfigs: RpcChainConfig[] = [];
+
+  for (const { chainId, rpcs } of chainsSettings) {
+    const rpcChainConfig: RpcChainConfig =
+      rpcManagerService.prepareRpcChainConfig(chainId, rpcs);
+
+    rpcChainConfigs.push(rpcChainConfig);
+  }
+
+  const rpcConfigs = {
+    chains: rpcChainConfigs,
+  };
+
+  rpcManagerService
+    .on("setup", (rpcConfigs) => {
+      workersService.notifyClusterWorkers({
+        type: "rpcManagerConfig",
+        data: rpcConfigs,
+      });
+
+      for (const rpcConfig of rpcConfigs.chains) {
+        workersService.notifyThreadWorkers(
+          {
+            type: "rpcManagerConfig",
+            data: rpcConfig,
+          },
+          {
+            chainId: rpcConfig.chainId,
+            workerType: "simulator",
+          },
+        );
+
+        workersService.notifyThreadWorkers(
+          {
+            type: "rpcManagerConfig",
+            data: rpcConfig,
+          },
+          {
+            chainId: rpcConfig.chainId,
+            workerType: "executor",
+          },
+        );
+      }
+    })
+    .on("sync", (rpcProviderSyncConfigs) => {
+      workersService.notifyClusterWorkers({
+        type: "rpcProviderSyncConfig",
+        data: rpcProviderSyncConfigs,
+      });
+
+      workersService.notifyThreadWorkers(
+        {
+          type: "rpcProviderSyncConfig",
+          data: rpcProviderSyncConfigs,
+        },
+        {
+          chainId: rpcProviderSyncConfigs.chainId,
+          workerType: "simulator",
+        },
+      );
+
+      workersService.notifyThreadWorkers(
+        {
+          type: "rpcProviderSyncConfig",
+          data: rpcProviderSyncConfigs,
+        },
+        {
+          chainId: rpcProviderSyncConfigs.chainId,
+          workerType: "executor",
+        },
+      );
+    })
+    .setup(rpcConfigs, true);
+
+  await batcherService.initialize();
   await nodeService.initialize();
 
   await healthCheckService