From 9999afc9fd05573792cfb5b4492e737a0467fa94 Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 14 Jan 2026 16:30:09 +0100 Subject: [PATCH 01/53] feat: aggregate eth_call batches with multicall3 --- architecture/evm/multicall3.go | 344 +++++++++++++++++++ architecture/evm/multicall3_test.go | 407 +++++++++++++++++++++++ erpc/http_batch_eth_call.go | 346 +++++++++++++++++++ erpc/http_batch_eth_call_detect_test.go | 113 +++++++ erpc/http_batch_eth_call_forward_test.go | 59 ++++ erpc/http_batch_eth_call_handle_test.go | 364 ++++++++++++++++++++ erpc/http_batch_eth_call_helpers_test.go | 273 +++++++++++++++ erpc/http_server.go | 32 +- erpc/http_server_batch_eth_call_test.go | 64 ++++ erpc/networks.go | 28 +- erpc/networks_skip_rate_limit_test.go | 19 ++ 11 files changed, 2039 insertions(+), 10 deletions(-) create mode 100644 architecture/evm/multicall3.go create mode 100644 architecture/evm/multicall3_test.go create mode 100644 erpc/http_batch_eth_call.go create mode 100644 erpc/http_batch_eth_call_detect_test.go create mode 100644 erpc/http_batch_eth_call_forward_test.go create mode 100644 erpc/http_batch_eth_call_handle_test.go create mode 100644 erpc/http_batch_eth_call_helpers_test.go create mode 100644 erpc/http_server_batch_eth_call_test.go create mode 100644 erpc/networks_skip_rate_limit_test.go diff --git a/architecture/evm/multicall3.go b/architecture/evm/multicall3.go new file mode 100644 index 000000000..8e1c48d72 --- /dev/null +++ b/architecture/evm/multicall3.go @@ -0,0 +1,344 @@ +package evm + +import ( + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "math/big" + "strings" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" + "golang.org/x/crypto/sha3" +) + +const multicall3Address = "0xcA11bde05977b3631167028862bE2a173976CA11" + +var ErrMulticall3BatchNotEligible = errors.New("multicall3 batch not eligible") + +type Multicall3Call struct { + Request *common.NormalizedRequest + Target []byte + CallData []byte +} + +type Multicall3Result struct { + Success bool + ReturnData []byte +} + +func NormalizeBlockParam(param interface{}) (string, error) { + if param == nil { + return "latest", nil + } + + blockNumberStr, blockHash, err := util.ParseBlockParameter(param) + if err != nil { + return "", err + } + if blockHash != nil { + return fmt.Sprintf("0x%x", blockHash), nil + } + if blockNumberStr == "" { + return "", errors.New("block parameter is empty") + } + if strings.HasPrefix(blockNumberStr, "0x") { + bn, err := common.HexToInt64(blockNumberStr) + if err != nil { + return "", err + } + return fmt.Sprintf("%d", bn), nil + } + return blockNumberStr, nil +} + +func BuildMulticall3Request(requests []*common.NormalizedRequest, blockParam interface{}) (*common.NormalizedRequest, []Multicall3Call, error) { + if len(requests) < 1 { + return nil, nil, ErrMulticall3BatchNotEligible + } + + if blockParam == nil { + blockParam = "latest" + } + + calls := make([]Multicall3Call, 0, len(requests)) + for _, req := range requests { + if req == nil { + return nil, nil, ErrMulticall3BatchNotEligible + } + + jrq, err := req.JsonRpcRequest() + if err != nil { + return nil, nil, err + } + + jrq.RLock() + method := jrq.Method + params := jrq.Params + jrq.RUnlock() + + if !strings.EqualFold(method, "eth_call") { + return nil, nil, ErrMulticall3BatchNotEligible + } + if len(params) < 1 || len(params) > 2 { + return nil, nil, ErrMulticall3BatchNotEligible + } + + callObj, ok := params[0].(map[string]interface{}) + if !ok { + return nil, nil, ErrMulticall3BatchNotEligible + } + + targetHex, ok := callObj["to"].(string) + if !ok || targetHex == "" { + return nil, nil, ErrMulticall3BatchNotEligible + } + + dataHex := "0x" + if dataValue, ok := callObj["data"]; ok { + dataStr, ok := dataValue.(string) + if !ok { + return nil, nil, ErrMulticall3BatchNotEligible + } + dataHex = dataStr + } else if inputValue, ok := callObj["input"]; ok { + inputStr, ok := inputValue.(string) + if !ok { + return nil, nil, ErrMulticall3BatchNotEligible + } + dataHex = inputStr + } + + for key := range callObj { + switch key { + case "to", "data", "input": + continue + default: + return nil, nil, ErrMulticall3BatchNotEligible + } + } + + targetBytes, err := common.HexToBytes(targetHex) + if err != nil || len(targetBytes) != 20 { + return nil, nil, ErrMulticall3BatchNotEligible + } + + callData, err := common.HexToBytes(dataHex) + if err != nil { + return nil, nil, ErrMulticall3BatchNotEligible + } + + calls = append(calls, Multicall3Call{ + Request: req, + Target: targetBytes, + CallData: callData, + }) + } + + encodedCalls, err := encodeAggregate3Calls(calls) + if err != nil { + return nil, nil, err + } + + callObj := map[string]interface{}{ + "to": multicall3Address, + "data": "0x" + hex.EncodeToString(encodedCalls), + } + + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{callObj, blockParam}) + jrq.ID = fmt.Sprintf("multicall3-%d", time.Now().UnixNano()) + + nrq := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + nrq.CopyHttpContextFrom(requests[0]) + if dirs := requests[0].Directives(); dirs != nil { + nrq.SetDirectives(dirs.Clone()) + } + + return nrq, calls, nil +} + +func DecodeMulticall3Aggregate3Result(data []byte) ([]Multicall3Result, error) { + if len(data) < 32 { + return nil, errors.New("multicall3 result too short") + } + + offset, err := readUint256(data[:32]) + if err != nil { + return nil, err + } + base := int(offset) + if base < 0 || base+32 > len(data) { + return nil, errors.New("multicall3 result offset out of bounds") + } + + count, err := readUint256(data[base : base+32]) + if err != nil { + return nil, err + } + if count == 0 { + return []Multicall3Result{}, nil + } + + offsetsStart := base + 32 + offsetsEnd := offsetsStart + int(count)*32 + if offsetsEnd > len(data) { + return nil, errors.New("multicall3 result offsets out of bounds") + } + + results := make([]Multicall3Result, int(count)) + for i := 0; i < int(count); i++ { + offsetStart := offsetsStart + i*32 + offsetVal, err := readUint256(data[offsetStart : offsetStart+32]) + if err != nil { + return nil, err + } + elemStart := base + int(offsetVal) + if elemStart < base || elemStart+64 > len(data) { + return nil, errors.New("multicall3 result element out of bounds") + } + + success, err := readBool(data[elemStart : elemStart+32]) + if err != nil { + return nil, err + } + + dataOffset, err := readUint256(data[elemStart+32 : elemStart+64]) + if err != nil { + return nil, err + } + bytesStart := elemStart + int(dataOffset) + if bytesStart < elemStart || bytesStart+32 > len(data) { + return nil, errors.New("multicall3 result bytes offset out of bounds") + } + + dataLen, err := readUint256(data[bytesStart : bytesStart+32]) + if err != nil { + return nil, err + } + dataStart := bytesStart + 32 + dataEnd := dataStart + int(dataLen) + if dataStart < bytesStart || dataEnd > len(data) { + return nil, errors.New("multicall3 result bytes length out of bounds") + } + + returnData := append([]byte(nil), data[dataStart:dataEnd]...) + results[i] = Multicall3Result{ + Success: success, + ReturnData: returnData, + } + } + + return results, nil +} + +func ShouldFallbackMulticall3(err error) bool { + if err == nil { + return false + } + return common.HasErrorCode(err, common.ErrCodeEndpointExecutionException, common.ErrCodeEndpointUnsupported) +} + +func encodeAggregate3Calls(calls []Multicall3Call) ([]byte, error) { + arrayData, err := encodeAggregate3Array(calls) + if err != nil { + return nil, err + } + + out := make([]byte, 0, 4+32+len(arrayData)) + out = append(out, multicall3Aggregate3Selector...) + out = append(out, encodeUint64(32)...) + out = append(out, arrayData...) + return out, nil +} + +func encodeAggregate3Array(calls []Multicall3Call) ([]byte, error) { + headSize := 32 + 32*len(calls) + elements := make([][]byte, len(calls)) + offsets := make([]uint64, len(calls)) + cur := uint64(headSize) + + for i, call := range calls { + elem := encodeAggregate3Element(call) + elements[i] = elem + offsets[i] = cur + cur += uint64(len(elem)) + } + + out := make([]byte, 0, int(cur)) + out = append(out, encodeUint64(uint64(len(calls)))...) + for _, off := range offsets { + out = append(out, encodeUint64(off)...) + } + for _, elem := range elements { + out = append(out, elem...) + } + return out, nil +} + +func encodeAggregate3Element(call Multicall3Call) []byte { + head := make([]byte, 0, 96) + head = append(head, encodeAddress(call.Target)...) + head = append(head, encodeBool(true)...) + head = append(head, encodeUint64(96)...) + tail := encodeBytes(call.CallData) + return append(head, tail...) +} + +func encodeAddress(addr []byte) []byte { + out := make([]byte, 32) + copy(out[32-len(addr):], addr) + return out +} + +func encodeBool(value bool) []byte { + out := make([]byte, 32) + if value { + out[31] = 1 + } + return out +} + +func encodeUint64(value uint64) []byte { + out := make([]byte, 32) + binary.BigEndian.PutUint64(out[24:], value) + return out +} + +func encodeBytes(data []byte) []byte { + out := make([]byte, 0, 32+len(data)+32) + out = append(out, encodeUint64(uint64(len(data)))...) + out = append(out, data...) + pad := (32 - (len(data) % 32)) % 32 + if pad > 0 { + out = append(out, make([]byte, pad)...) + } + return out +} + +func readUint256(data []byte) (uint64, error) { + if len(data) != 32 { + return 0, errors.New("invalid uint256 length") + } + val := new(big.Int).SetBytes(data) + if !val.IsUint64() { + return 0, errors.New("uint256 overflows uint64") + } + return val.Uint64(), nil +} + +func readBool(data []byte) (bool, error) { + val, err := readUint256(data) + if err != nil { + return false, err + } + return val != 0, nil +} + +var multicall3Aggregate3Selector = func() []byte { + hasher := sha3.NewLegacyKeccak256() + hasher.Write([]byte("aggregate3((address,bool,bytes)[])")) + sum := hasher.Sum(nil) + return sum[:4] +}() diff --git a/architecture/evm/multicall3_test.go b/architecture/evm/multicall3_test.go new file mode 100644 index 000000000..c0311e47a --- /dev/null +++ b/architecture/evm/multicall3_test.go @@ -0,0 +1,407 @@ +package evm + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/erpc/erpc/common" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNormalizeBlockParam(t *testing.T) { + cases := []struct { + name string + param interface{} + want string + wantErr bool + }{ + { + name: "nil", + param: nil, + want: "latest", + }, + { + name: "hex number", + param: "0x10", + want: "16", + }, + { + name: "tag", + param: "latest", + want: "latest", + }, + { + name: "block hash", + param: "0x" + strings.Repeat("ab", 32), + want: "0x" + strings.Repeat("ab", 32), + }, + { + name: "block hash object", + param: map[string]interface{}{"blockHash": "0x" + strings.Repeat("cd", 32)}, + want: "0x" + strings.Repeat("cd", 32), + }, + { + name: "block number object", + param: map[string]interface{}{"blockNumber": "0x2"}, + want: "2", + }, + { + name: "block tag object", + param: map[string]interface{}{"blockTag": "pending"}, + want: "pending", + }, + { + name: "empty string", + param: "", + wantErr: true, + }, + { + name: "invalid type", + param: []int{1}, + wantErr: true, + }, + { + name: "invalid hex", + param: "0xzz", + wantErr: true, + }, + { + name: "invalid block hash", + param: map[string]interface{}{"blockHash": "0x" + strings.Repeat("ab", 33)}, + wantErr: true, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + got, err := NormalizeBlockParam(tt.param) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestBuildMulticall3Request_Success(t *testing.T) { + call1 := map[string]interface{}{ + "to": hexAddr(1), + "data": hexData(1), + } + call2 := map[string]interface{}{ + "to": hexAddr(2), + "input": hexData(32), + } + + req1 := newEthCallRequest(t, 1, call1, "latest") + req2 := newEthCallRequest(t, "req-2", call2, "latest") + req1.SetDirectives(&common.RequestDirectives{SkipCacheRead: true}) + req1.SetUser(&common.User{Id: "user-1"}) + + mcReq, calls, err := BuildMulticall3Request([]*common.NormalizedRequest{req1, req2}, nil) + require.NoError(t, err) + require.Len(t, calls, 2) + require.NotNil(t, mcReq) + + mcID, ok := mcReq.ID().(string) + require.True(t, ok) + assert.True(t, strings.HasPrefix(mcID, "multicall3-")) + + require.NotNil(t, mcReq.User()) + assert.Equal(t, "user-1", mcReq.User().Id) + require.NotNil(t, mcReq.Directives()) + assert.True(t, mcReq.Directives().SkipCacheRead) + + jrq, err := mcReq.JsonRpcRequest() + require.NoError(t, err) + require.NotNil(t, jrq) + assert.Equal(t, "eth_call", jrq.Method) + require.Len(t, jrq.Params, 2) + + callObj, ok := jrq.Params[0].(map[string]interface{}) + require.True(t, ok) + assert.Equal(t, multicall3Address, callObj["to"]) + + encodedCalls, err := encodeAggregate3Calls(calls) + require.NoError(t, err) + dataHex, ok := callObj["data"].(string) + require.True(t, ok) + assert.Equal(t, "0x"+fmt.Sprintf("%x", encodedCalls), dataHex) + assert.Equal(t, "latest", jrq.Params[1]) + + assert.Equal(t, 20, len(calls[0].Target)) + assert.Equal(t, 1, len(calls[0].CallData)) + assert.Equal(t, 32, len(calls[1].CallData)) +} + +func TestBuildMulticall3Request_Errors(t *testing.T) { + validCall := map[string]interface{}{ + "to": hexAddr(3), + "data": "0x", + } + + cases := []struct { + name string + requests []*common.NormalizedRequest + eligibleErr bool + unexpectedWrap bool + }{ + { + name: "no requests", + requests: []*common.NormalizedRequest{}, + eligibleErr: true, + }, + { + name: "nil request", + requests: []*common.NormalizedRequest{nil}, + eligibleErr: true, + }, + { + name: "invalid json", + requests: []*common.NormalizedRequest{common.NewNormalizedRequest([]byte("{"))}, + unexpectedWrap: true, + }, + { + name: "wrong method", + requests: []*common.NormalizedRequest{newJsonRpcRequest(t, "eth_getBalance", []interface{}{hexAddr(1)}, 1)}, + eligibleErr: true, + }, + { + name: "missing params", + requests: []*common.NormalizedRequest{newJsonRpcRequest(t, "eth_call", []interface{}{}, 1)}, + eligibleErr: true, + }, + { + name: "too many params", + requests: []*common.NormalizedRequest{newJsonRpcRequest(t, "eth_call", []interface{}{validCall, "latest", "extra"}, 1)}, + eligibleErr: true, + }, + { + name: "call obj not map", + requests: []*common.NormalizedRequest{newJsonRpcRequest(t, "eth_call", []interface{}{123}, 1)}, + eligibleErr: true, + }, + { + name: "missing to", + requests: []*common.NormalizedRequest{newJsonRpcRequest(t, "eth_call", []interface{}{map[string]interface{}{"data": "0x"}}, 1)}, + eligibleErr: true, + }, + { + name: "data not string", + requests: []*common.NormalizedRequest{newJsonRpcRequest(t, "eth_call", []interface{}{map[string]interface{}{"to": hexAddr(1), "data": 1}}, 1)}, + eligibleErr: true, + }, + { + name: "input not string", + requests: []*common.NormalizedRequest{newJsonRpcRequest(t, "eth_call", []interface{}{map[string]interface{}{"to": hexAddr(1), "input": 1}}, 1)}, + eligibleErr: true, + }, + { + name: "extra key", + requests: []*common.NormalizedRequest{newJsonRpcRequest(t, "eth_call", []interface{}{map[string]interface{}{"to": hexAddr(1), "data": "0x", "value": "0x1"}}, 1)}, + eligibleErr: true, + }, + { + name: "invalid to length", + requests: []*common.NormalizedRequest{newJsonRpcRequest(t, "eth_call", []interface{}{map[string]interface{}{"to": "0x1234", "data": "0x"}}, 1)}, + eligibleErr: true, + }, + { + name: "invalid data hex", + requests: []*common.NormalizedRequest{newJsonRpcRequest(t, "eth_call", []interface{}{map[string]interface{}{"to": hexAddr(1), "data": "0xzz"}}, 1)}, + eligibleErr: true, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + _, _, err := BuildMulticall3Request(tt.requests, "latest") + require.Error(t, err) + if tt.eligibleErr { + assert.ErrorIs(t, err, ErrMulticall3BatchNotEligible) + } + if tt.unexpectedWrap { + assert.False(t, errors.Is(err, ErrMulticall3BatchNotEligible)) + } + }) + } +} + +func TestDecodeMulticall3Aggregate3Result(t *testing.T) { + results := []Multicall3Result{ + {Success: true, ReturnData: []byte{0x01, 0x02}}, + {Success: false, ReturnData: nil}, + } + + encoded := encodeAggregate3Results(results) + decoded, err := DecodeMulticall3Aggregate3Result(encoded) + require.NoError(t, err) + assert.Equal(t, results, decoded) + + emptyEncoded := encodeAggregate3Results(nil) + emptyDecoded, err := DecodeMulticall3Aggregate3Result(emptyEncoded) + require.NoError(t, err) + assert.Empty(t, emptyDecoded) +} + +func TestDecodeMulticall3Aggregate3Result_Errors(t *testing.T) { + cases := []struct { + name string + data []byte + }{ + { + name: "too short", + data: []byte{0x01}, + }, + { + name: "offset out of bounds", + data: encodeUint64(64), + }, + { + name: "offsets out of bounds", + data: append(encodeUint64(32), encodeUint64(2)...), + }, + { + name: "element out of bounds", + data: buildAggregate3ResultWithOffset(96, nil, nil), + }, + { + name: "bytes offset out of bounds", + data: buildAggregate3ResultWithElement(64, encodeBool(true), encodeUint64(256)), + }, + { + name: "bytes length out of bounds", + data: buildAggregate3ResultBytesLengthOutOfBounds(), + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + _, err := DecodeMulticall3Aggregate3Result(tt.data) + require.Error(t, err) + }) + } + + _, err := readUint256([]byte{0x01}) + require.Error(t, err) + + overflow := make([]byte, 32) + overflow[0] = 1 + _, err = readUint256(overflow) + require.Error(t, err) + + _, err = readBool([]byte{0x01}) + require.Error(t, err) +} + +func TestShouldFallbackMulticall3(t *testing.T) { + assert.False(t, ShouldFallbackMulticall3(nil)) + assert.True(t, ShouldFallbackMulticall3(common.NewErrEndpointExecutionException(errors.New("boom")))) + assert.True(t, ShouldFallbackMulticall3(common.NewErrEndpointUnsupported(errors.New("boom")))) + assert.False(t, ShouldFallbackMulticall3(errors.New("nope"))) +} + +func encodeAggregate3Results(results []Multicall3Result) []byte { + headSize := 32 + 32*len(results) + offsets := make([]uint64, len(results)) + elems := make([][]byte, len(results)) + cur := uint64(headSize) + + for i, res := range results { + elems[i] = encodeAggregate3ResultElement(res) + offsets[i] = cur + cur += uint64(len(elems[i])) + } + + array := make([]byte, 0, int(cur)) + array = append(array, encodeUint64(uint64(len(results)))...) + for _, off := range offsets { + array = append(array, encodeUint64(off)...) + } + for _, elem := range elems { + array = append(array, elem...) + } + + out := make([]byte, 0, 32+len(array)) + out = append(out, encodeUint64(32)...) + out = append(out, array...) + return out +} + +func encodeAggregate3ResultElement(result Multicall3Result) []byte { + head := make([]byte, 0, 64) + head = append(head, encodeBool(result.Success)...) + head = append(head, encodeUint64(64)...) + tail := encodeBytes(result.ReturnData) + return append(head, tail...) +} + +func buildAggregate3ResultWithOffset(offset uint64, count []byte, elemOffset []byte) []byte { + data := make([]byte, 96) + copy(data, encodeUint64(32)) + if count != nil { + copy(data[32:], count) + } else { + copy(data[32:], encodeUint64(1)) + } + if elemOffset != nil { + copy(data[64:], elemOffset) + } else { + copy(data[64:], encodeUint64(offset)) + } + return data +} + +func buildAggregate3ResultWithElement(elemOffset uint64, head ...[]byte) []byte { + data := make([]byte, 160) + copy(data, encodeUint64(32)) + copy(data[32:], encodeUint64(1)) + copy(data[64:], encodeUint64(elemOffset)) + copy(data[96:], head[0]) + copy(data[128:], head[1]) + return data +} + +func buildAggregate3ResultBytesLengthOutOfBounds() []byte { + data := make([]byte, 192) + copy(data, encodeUint64(32)) + copy(data[32:], encodeUint64(1)) + copy(data[64:], encodeUint64(64)) + copy(data[96:], encodeBool(true)) + copy(data[128:], encodeUint64(64)) + copy(data[160:], encodeUint64(128)) + return data +} + +func hexAddr(n int) string { + return fmt.Sprintf("0x%040x", n) +} + +func hexData(size int) string { + return "0x" + strings.Repeat("11", size) +} + +func newEthCallRequest(t *testing.T, id interface{}, callObj map[string]interface{}, blockParam interface{}) *common.NormalizedRequest { + t.Helper() + params := []interface{}{callObj} + if blockParam != nil { + params = append(params, blockParam) + } + jr := common.NewJsonRpcRequest("eth_call", params) + if id != nil { + require.NoError(t, jr.SetID(id)) + } + return common.NewNormalizedRequestFromJsonRpcRequest(jr) +} + +func newJsonRpcRequest(t *testing.T, method string, params []interface{}, id interface{}) *common.NormalizedRequest { + t.Helper() + jr := common.NewJsonRpcRequest(method, params) + require.NoError(t, jr.SetID(id)) + return common.NewNormalizedRequestFromJsonRpcRequest(jr) +} diff --git a/erpc/http_batch_eth_call.go b/erpc/http_batch_eth_call.go new file mode 100644 index 000000000..c4afd113b --- /dev/null +++ b/erpc/http_batch_eth_call.go @@ -0,0 +1,346 @@ +package erpc + +import ( + "context" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "strings" + "time" + + "github.com/erpc/erpc/architecture/evm" + "github.com/erpc/erpc/auth" + "github.com/erpc/erpc/common" + "github.com/rs/zerolog" +) + +type ethCallBatchInfo struct { + networkId string + blockRef string + blockParam interface{} +} + +type ethCallBatchCandidate struct { + index int + ctx context.Context + req *common.NormalizedRequest + logger zerolog.Logger +} + +type ethCallBatchProbe struct { + Method string `json:"method"` + Params []interface{} `json:"params"` + NetworkId string `json:"networkId"` +} + +var ( + forwardBatchNetwork = func(ctx context.Context, network *Network, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return network.Forward(ctx, req) + } + forwardBatchProject = func(ctx context.Context, project *PreparedProject, network *Network, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return project.doForward(ctx, network, req) + } + newBatchJsonRpcResponse = common.NewJsonRpcResponse +) + +func detectEthCallBatchInfo(requests []json.RawMessage, architecture, chainId string) *ethCallBatchInfo { + if len(requests) < 2 { + return nil + } + if architecture != "" && architecture != string(common.ArchitectureEvm) { + return nil + } + + defaultNetworkId := "" + if architecture != "" && chainId != "" { + defaultNetworkId = fmt.Sprintf("%s:%s", architecture, chainId) + } + + var networkId string + var blockRef string + var blockParam interface{} + + for _, raw := range requests { + var probe ethCallBatchProbe + if err := common.SonicCfg.Unmarshal(raw, &probe); err != nil { + return nil + } + if strings.ToLower(probe.Method) != "eth_call" { + return nil + } + + reqNetworkId := defaultNetworkId + if reqNetworkId == "" { + reqNetworkId = probe.NetworkId + } + if reqNetworkId == "" || !strings.HasPrefix(reqNetworkId, "evm:") { + return nil + } + if networkId == "" { + networkId = reqNetworkId + } else if networkId != reqNetworkId { + return nil + } + + param := interface{}("latest") + if len(probe.Params) >= 2 { + param = probe.Params[1] + } + bref, err := evm.NormalizeBlockParam(param) + if err != nil { + return nil + } + if blockRef == "" { + blockRef = bref + blockParam = param + } else if blockRef != bref { + return nil + } + } + + if networkId == "" || blockRef == "" { + return nil + } + + return ðCallBatchInfo{ + networkId: networkId, + blockRef: blockRef, + blockParam: blockParam, + } +} + +func (s *HttpServer) forwardEthCallBatchCandidates( + startedAt *time.Time, + project *PreparedProject, + network *Network, + candidates []ethCallBatchCandidate, + responses []interface{}, +) { + if project == nil || network == nil { + err := common.NewErrInvalidRequest(fmt.Errorf("network not available for batch eth_call fallback")) + for _, cand := range candidates { + responses[cand.index] = processErrorBody(&cand.logger, startedAt, cand.req, err, s.serverCfg.IncludeErrorDetails) + common.EndRequestSpan(cand.ctx, nil, err) + } + return + } + + for _, cand := range candidates { + resp, err := forwardBatchProject(withSkipNetworkRateLimit(cand.ctx), project, network, cand.req) + if err != nil { + if resp != nil { + go resp.Release() + } + responses[cand.index] = processErrorBody(&cand.logger, startedAt, cand.req, err, s.serverCfg.IncludeErrorDetails) + common.EndRequestSpan(cand.ctx, nil, err) + continue + } + + responses[cand.index] = resp + common.EndRequestSpan(cand.ctx, resp, nil) + } +} + +func (s *HttpServer) handleEthCallBatchAggregation( + httpCtx context.Context, + startedAt *time.Time, + r *http.Request, + project *PreparedProject, + baseLogger zerolog.Logger, + batchInfo *ethCallBatchInfo, + requests []json.RawMessage, + headers http.Header, + queryArgs map[string][]string, + responses []interface{}, +) bool { + if batchInfo == nil || project == nil { + return false + } + + network, networkErr := project.GetNetwork(httpCtx, batchInfo.networkId) + uaMode := common.UserAgentTrackingModeSimplified + if project.Config != nil && project.Config.UserAgentMode != "" { + uaMode = project.Config.UserAgentMode + } + + candidates := make([]ethCallBatchCandidate, 0, len(requests)) + for i, rawReq := range requests { + nq := common.NewNormalizedRequest(rawReq) + rawReq = nil + requestCtx := common.StartRequestSpan(httpCtx, nq) + + clientIP := s.resolveRealClientIP(r) + nq.SetClientIP(clientIP) + + if err := nq.Validate(); err != nil { + responses[i] = processErrorBody(&baseLogger, startedAt, nq, err, &common.TRUE) + common.EndRequestSpan(requestCtx, nil, responses[i]) + continue + } + + method, _ := nq.Method() + rlg := baseLogger.With().Str("method", method).Logger() + + ap, err := auth.NewPayloadFromHttp(method, r.RemoteAddr, headers, queryArgs) + if err != nil { + responses[i] = processErrorBody(&rlg, startedAt, nq, err, &common.TRUE) + common.EndRequestSpan(requestCtx, nil, err) + continue + } + + user, err := project.AuthenticateConsumer(requestCtx, nq, method, ap) + if err != nil { + responses[i] = processErrorBody(&rlg, startedAt, nq, err, s.serverCfg.IncludeErrorDetails) + common.EndRequestSpan(requestCtx, nil, err) + continue + } + if user != nil { + rlg = rlg.With().Str("userId", user.Id).Logger() + } + nq.SetUser(user) + + if networkErr != nil || network == nil { + err := networkErr + if err == nil { + err = common.NewErrNetworkNotFound(batchInfo.networkId) + } + responses[i] = processErrorBody(&rlg, startedAt, nq, err, s.serverCfg.IncludeErrorDetails) + common.EndRequestSpan(requestCtx, nil, err) + continue + } + + nq.SetNetwork(network) + nq.ApplyDirectiveDefaults(network.Config().DirectiveDefaults) + nq.EnrichFromHttp(headers, queryArgs, uaMode) + rlg.Trace().Interface("directives", nq.Directives()).Msgf("applied request directives") + + if err := project.acquireRateLimitPermit(requestCtx, nq); err != nil { + responses[i] = processErrorBody(&rlg, startedAt, nq, err, s.serverCfg.IncludeErrorDetails) + common.EndRequestSpan(requestCtx, nil, err) + continue + } + if err := network.acquireRateLimitPermit(requestCtx, nq); err != nil { + responses[i] = processErrorBody(&rlg, startedAt, nq, err, s.serverCfg.IncludeErrorDetails) + common.EndRequestSpan(requestCtx, nil, err) + continue + } + + candidates = append(candidates, ethCallBatchCandidate{ + index: i, + ctx: requestCtx, + req: nq, + logger: rlg, + }) + } + + if len(candidates) == 0 { + return true + } + if len(candidates) < 2 { + s.forwardEthCallBatchCandidates(startedAt, project, network, candidates, responses) + return true + } + + reqs := make([]*common.NormalizedRequest, len(candidates)) + for i, cand := range candidates { + reqs[i] = cand.req + } + + mcReq, calls, err := evm.BuildMulticall3Request(reqs, batchInfo.blockParam) + if err != nil { + s.forwardEthCallBatchCandidates(startedAt, project, network, candidates, responses) + return true + } + + mcCtx := withSkipNetworkRateLimit(httpCtx) + mcResp, mcErr := forwardBatchNetwork(mcCtx, network, mcReq) + if mcErr != nil { + if mcResp != nil { + mcResp.Release() + } + if evm.ShouldFallbackMulticall3(mcErr) { + s.forwardEthCallBatchCandidates(startedAt, project, network, candidates, responses) + return true + } + for _, cand := range candidates { + responses[cand.index] = processErrorBody(&cand.logger, startedAt, cand.req, mcErr, s.serverCfg.IncludeErrorDetails) + common.EndRequestSpan(cand.ctx, nil, mcErr) + } + return true + } + if mcResp == nil { + s.forwardEthCallBatchCandidates(startedAt, project, network, candidates, responses) + return true + } + + jrr, err := mcResp.JsonRpcResponse(mcCtx) + if err != nil || jrr == nil || jrr.Error != nil { + mcResp.Release() + s.forwardEthCallBatchCandidates(startedAt, project, network, candidates, responses) + return true + } + + var resultHex string + if err := common.SonicCfg.Unmarshal(jrr.GetResultBytes(), &resultHex); err != nil { + mcResp.Release() + s.forwardEthCallBatchCandidates(startedAt, project, network, candidates, responses) + return true + } + resultBytes, err := common.HexToBytes(resultHex) + if err != nil { + mcResp.Release() + s.forwardEthCallBatchCandidates(startedAt, project, network, candidates, responses) + return true + } + + decoded, err := evm.DecodeMulticall3Aggregate3Result(resultBytes) + if err != nil || len(decoded) != len(calls) { + mcResp.Release() + s.forwardEthCallBatchCandidates(startedAt, project, network, candidates, responses) + return true + } + + for i, result := range decoded { + cand := candidates[i] + if result.Success { + returnHex := "0x" + hex.EncodeToString(result.ReturnData) + jrr, err := newBatchJsonRpcResponse(cand.req.ID(), returnHex, nil) + if err != nil { + responses[cand.index] = processErrorBody(&cand.logger, startedAt, cand.req, err, s.serverCfg.IncludeErrorDetails) + common.EndRequestSpan(cand.ctx, nil, err) + continue + } + + nr := common.NewNormalizedResponse().WithRequest(cand.req).WithJsonRpcResponse(jrr) + nr.SetUpstream(mcResp.Upstream()) + nr.SetFromCache(mcResp.FromCache()) + nr.SetAttempts(mcResp.Attempts()) + nr.SetRetries(mcResp.Retries()) + nr.SetHedges(mcResp.Hedges()) + nr.SetEvmBlockRef(mcResp.EvmBlockRef()) + nr.SetEvmBlockNumber(mcResp.EvmBlockNumber()) + responses[cand.index] = nr + common.EndRequestSpan(cand.ctx, nr, nil) + continue + } + + dataHex := "0x" + hex.EncodeToString(result.ReturnData) + callErr := common.NewErrEndpointExecutionException( + common.NewErrJsonRpcExceptionInternal( + 0, + common.JsonRpcErrorEvmReverted, + "execution reverted", + nil, + map[string]interface{}{ + "data": dataHex, + }, + ), + ) + responses[cand.index] = processErrorBody(&cand.logger, startedAt, cand.req, callErr, s.serverCfg.IncludeErrorDetails) + common.EndRequestSpan(cand.ctx, nil, callErr) + } + + mcResp.Release() + return true +} diff --git a/erpc/http_batch_eth_call_detect_test.go b/erpc/http_batch_eth_call_detect_test.go new file mode 100644 index 000000000..016ddbf98 --- /dev/null +++ b/erpc/http_batch_eth_call_detect_test.go @@ -0,0 +1,113 @@ +package erpc + +import ( + "encoding/json" + "testing" + + "github.com/erpc/erpc/common" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDetectEthCallBatchInfo(t *testing.T) { + buildRaw := func(t *testing.T, method string, params []interface{}, networkId string) json.RawMessage { + t.Helper() + body := map[string]interface{}{ + "jsonrpc": "2.0", + "id": 1, + "method": method, + "params": params, + } + if networkId != "" { + body["networkId"] = networkId + } + raw, err := common.SonicCfg.Marshal(body) + require.NoError(t, err) + return raw + } + + callObj := map[string]interface{}{"to": "0x0000000000000000000000000000000000000001"} + + cases := []struct { + name string + requests []json.RawMessage + arch string + chainId string + wantNil bool + wantNetwork string + wantBlockRef string + wantBlock interface{} + }{ + { + name: "single request", + requests: []json.RawMessage{buildRaw(t, "eth_call", []interface{}{callObj}, "evm:1")}, + wantNil: true, + }, + { + name: "non-evm arch", + requests: []json.RawMessage{buildRaw(t, "eth_call", []interface{}{callObj}, "evm:1"), buildRaw(t, "eth_call", []interface{}{callObj}, "evm:1")}, + arch: "solana", + wantNil: true, + }, + { + name: "invalid json", + requests: []json.RawMessage{json.RawMessage("{"), buildRaw(t, "eth_call", []interface{}{callObj}, "evm:1")}, + wantNil: true, + }, + { + name: "non-eth_call", + requests: []json.RawMessage{buildRaw(t, "eth_getBalance", []interface{}{callObj}, "evm:1"), buildRaw(t, "eth_call", []interface{}{callObj}, "evm:1")}, + wantNil: true, + }, + { + name: "missing network", + requests: []json.RawMessage{buildRaw(t, "eth_call", []interface{}{callObj}, ""), buildRaw(t, "eth_call", []interface{}{callObj}, "")}, + wantNil: true, + }, + { + name: "network mismatch", + requests: []json.RawMessage{buildRaw(t, "eth_call", []interface{}{callObj}, "evm:1"), buildRaw(t, "eth_call", []interface{}{callObj}, "evm:2")}, + wantNil: true, + }, + { + name: "block mismatch", + requests: []json.RawMessage{buildRaw(t, "eth_call", []interface{}{callObj, "0x1"}, "evm:1"), buildRaw(t, "eth_call", []interface{}{callObj, "0x2"}, "evm:1")}, + wantNil: true, + }, + { + name: "invalid block param", + requests: []json.RawMessage{buildRaw(t, "eth_call", []interface{}{callObj, map[string]interface{}{"blockHash": "0x1234"}}, "evm:1"), buildRaw(t, "eth_call", []interface{}{callObj, "latest"}, "evm:1")}, + wantNil: true, + }, + { + name: "success explicit network", + requests: []json.RawMessage{buildRaw(t, "eth_call", []interface{}{callObj, "0x1"}, "evm:1"), buildRaw(t, "eth_call", []interface{}{callObj, "0x1"}, "evm:1")}, + wantNetwork: "evm:1", + wantBlockRef: "1", + wantBlock: "0x1", + }, + { + name: "success default network", + requests: []json.RawMessage{buildRaw(t, "eth_call", []interface{}{callObj}, ""), buildRaw(t, "eth_call", []interface{}{callObj}, "")}, + arch: "evm", + chainId: "123", + wantNetwork: "evm:123", + wantBlockRef: "latest", + wantBlock: "latest", + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + info := detectEthCallBatchInfo(tt.requests, tt.arch, tt.chainId) + if tt.wantNil { + assert.Nil(t, info) + return + } + require.NotNil(t, info) + assert.Equal(t, tt.wantNetwork, info.networkId) + assert.Equal(t, tt.wantBlockRef, info.blockRef) + assert.Equal(t, tt.wantBlock, info.blockParam) + }) + } +} diff --git a/erpc/http_batch_eth_call_forward_test.go b/erpc/http_batch_eth_call_forward_test.go new file mode 100644 index 000000000..e3ecc367f --- /dev/null +++ b/erpc/http_batch_eth_call_forward_test.go @@ -0,0 +1,59 @@ +package erpc + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/require" +) + +func TestForwardEthCallBatchCandidates(t *testing.T) { + server := &HttpServer{serverCfg: &common.ServerConfig{IncludeErrorDetails: &common.TRUE}} + startedAt := time.Now() + + makeCandidate := func(index int) ethCallBatchCandidate { + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_call","params":[{"to":"0x0000000000000000000000000000000000000001","data":"0x"}]}`)) + ctx := common.StartRequestSpan(context.Background(), req) + return ethCallBatchCandidate{ + index: index, + ctx: ctx, + req: req, + logger: log.Logger, + } + } + + responses := make([]interface{}, 1) + server.forwardEthCallBatchCandidates(&startedAt, nil, nil, []ethCallBatchCandidate{makeCandidate(0)}, responses) + require.NotNil(t, responses[0]) + + origForward := forwardBatchProject + t.Cleanup(func() { + forwardBatchProject = origForward + }) + + t.Run("forward error", func(t *testing.T) { + responses := make([]interface{}, 1) + resp := common.NewNormalizedResponse() + forwardBatchProject = func(ctx context.Context, project *PreparedProject, network *Network, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return resp, errors.New("boom") + } + + server.forwardEthCallBatchCandidates(&startedAt, &PreparedProject{}, &Network{}, []ethCallBatchCandidate{makeCandidate(0)}, responses) + require.NotNil(t, responses[0]) + }) + + t.Run("forward success", func(t *testing.T) { + responses := make([]interface{}, 1) + resp := common.NewNormalizedResponse() + forwardBatchProject = func(ctx context.Context, project *PreparedProject, network *Network, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return resp, nil + } + + server.forwardEthCallBatchCandidates(&startedAt, &PreparedProject{}, &Network{}, []ethCallBatchCandidate{makeCandidate(0)}, responses) + require.Equal(t, resp, responses[0]) + }) +} diff --git a/erpc/http_batch_eth_call_handle_test.go b/erpc/http_batch_eth_call_handle_test.go new file mode 100644 index 000000000..5932317ec --- /dev/null +++ b/erpc/http_batch_eth_call_handle_test.go @@ -0,0 +1,364 @@ +package erpc + +import ( + "context" + "encoding/hex" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/erpc/erpc/architecture/evm" + "github.com/erpc/erpc/common" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestHandleEthCallBatchAggregation_EarlyReturn(t *testing.T) { + server := &HttpServer{serverCfg: &common.ServerConfig{IncludeErrorDetails: &common.TRUE}} + startedAt := time.Now() + req := httptest.NewRequest("POST", "http://localhost", nil) + req.RemoteAddr = "127.0.0.1:1234" + + handled := server.handleEthCallBatchAggregation( + context.Background(), + &startedAt, + req, + nil, + log.Logger, + nil, + nil, + req.Header, + req.URL.Query(), + nil, + ) + + assert.False(t, handled) +} + +func TestHandleEthCallBatchAggregation_RequestAndAuthErrors(t *testing.T) { + t.Run("validation error", func(t *testing.T) { + cfg := baseBatchConfig() + server, project, ctx, cleanup := setupBatchHandler(t, cfg) + defer cleanup() + + requests := []json.RawMessage{ + json.RawMessage(`{"jsonrpc":"2.0","id":1}`), + json.RawMessage(`{"jsonrpc":"2.0","id":2}`), + } + + handled, responses := runHandle(t, ctx, server, project, defaultBatchInfo(), requests, nil) + require.True(t, handled) + require.Len(t, responses, len(requests)) + for _, resp := range responses { + require.NotNil(t, resp) + } + }) + + t.Run("auth payload error", func(t *testing.T) { + cfg := baseBatchConfig() + server, project, ctx, cleanup := setupBatchHandler(t, cfg) + defer cleanup() + + headers := http.Header{} + headers.Set("Authorization", "Basic !!!") + + handled, responses := runHandle(t, ctx, server, project, defaultBatchInfo(), validBatchRequests(t), headers) + require.True(t, handled) + for _, resp := range responses { + require.NotNil(t, resp) + } + }) + + t.Run("auth unauthorized", func(t *testing.T) { + cfg := baseBatchConfig() + cfg.Projects[0].Auth = &common.AuthConfig{ + Strategies: []*common.AuthStrategyConfig{ + { + Type: common.AuthTypeSecret, + Secret: &common.SecretStrategyConfig{ + Id: "secret", + Value: "s3cr3t", + }, + }, + }, + } + + server, project, ctx, cleanup := setupBatchHandler(t, cfg) + defer cleanup() + + handled, responses := runHandle(t, ctx, server, project, defaultBatchInfo(), validBatchRequests(t), nil) + require.True(t, handled) + for _, resp := range responses { + require.NotNil(t, resp) + } + }) +} + +func TestHandleEthCallBatchAggregation_NetworkAndRateLimitErrors(t *testing.T) { + t.Run("network not found", func(t *testing.T) { + cfg := baseBatchConfig() + server, project, ctx, cleanup := setupBatchHandler(t, cfg) + defer cleanup() + + batchInfo := ðCallBatchInfo{networkId: "evm:999", blockRef: "latest", blockParam: "latest"} + handled, responses := runHandle(t, ctx, server, project, batchInfo, validBatchRequests(t), nil) + require.True(t, handled) + for _, resp := range responses { + require.NotNil(t, resp) + } + }) + + t.Run("project rate limit", func(t *testing.T) { + cfg := baseBatchConfig() + cfg.RateLimiters = rateLimitConfig("project-budget") + cfg.Projects[0].RateLimitBudget = "project-budget" + + server, project, ctx, cleanup := setupBatchHandler(t, cfg) + defer cleanup() + + handled, responses := runHandle(t, ctx, server, project, defaultBatchInfo(), validBatchRequests(t), nil) + require.True(t, handled) + for _, resp := range responses { + require.NotNil(t, resp) + } + }) + + t.Run("network rate limit", func(t *testing.T) { + cfg := baseBatchConfig() + cfg.RateLimiters = rateLimitConfig("network-budget") + cfg.Projects[0].Networks[0].RateLimitBudget = "network-budget" + + server, project, ctx, cleanup := setupBatchHandler(t, cfg) + defer cleanup() + + handled, responses := runHandle(t, ctx, server, project, defaultBatchInfo(), validBatchRequests(t), nil) + require.True(t, handled) + for _, resp := range responses { + require.NotNil(t, resp) + } + }) +} + +func TestHandleEthCallBatchAggregation_FallbackPaths(t *testing.T) { + cfg := baseBatchConfig() + server, project, ctx, cleanup := setupBatchHandler(t, cfg) + defer cleanup() + + validRequests := validBatchRequests(t) + invalidRequests := invalidBatchRequests(t) + singleRequest := validRequests[:1] + + cases := []struct { + name string + requests []json.RawMessage + networkResponse func() (*common.NormalizedResponse, error) + expectedProjCalls int + expectedNetCalls int + }{ + { + name: "single candidate", + requests: singleRequest, + networkResponse: func() (*common.NormalizedResponse, error) { return nil, nil }, + expectedProjCalls: 1, + }, + { + name: "build error", + requests: invalidRequests, + networkResponse: func() (*common.NormalizedResponse, error) { return nil, nil }, + expectedProjCalls: 2, + }, + { + name: "forward error fallback", + requests: validRequests, + networkResponse: func() (*common.NormalizedResponse, error) { + return nil, common.NewErrEndpointExecutionException(errors.New("boom")) + }, + expectedProjCalls: 2, + expectedNetCalls: 1, + }, + { + name: "mc response nil", + requests: validRequests, + networkResponse: func() (*common.NormalizedResponse, error) { return nil, nil }, + expectedProjCalls: 2, + expectedNetCalls: 1, + }, + { + name: "mc response error", + requests: validRequests, + networkResponse: func() (*common.NormalizedResponse, error) { + jrr := mustJsonRpcResponse(t, 1, nil, common.NewErrJsonRpcExceptionExternal(-32000, "boom", "")) + return common.NewNormalizedResponse().WithJsonRpcResponse(jrr), nil + }, + expectedProjCalls: 2, + expectedNetCalls: 1, + }, + { + name: "result unmarshal error", + requests: validRequests, + networkResponse: func() (*common.NormalizedResponse, error) { + jrr := mustJsonRpcResponse(t, 1, map[string]interface{}{"oops": "nope"}, nil) + return common.NewNormalizedResponse().WithJsonRpcResponse(jrr), nil + }, + expectedProjCalls: 2, + expectedNetCalls: 1, + }, + { + name: "result hex error", + requests: validRequests, + networkResponse: func() (*common.NormalizedResponse, error) { + jrr := mustJsonRpcResponse(t, 1, "0xzz", nil) + return common.NewNormalizedResponse().WithJsonRpcResponse(jrr), nil + }, + expectedProjCalls: 2, + expectedNetCalls: 1, + }, + { + name: "decode error", + requests: validRequests, + networkResponse: func() (*common.NormalizedResponse, error) { + jrr := mustJsonRpcResponse(t, 1, "0x01", nil) + return common.NewNormalizedResponse().WithJsonRpcResponse(jrr), nil + }, + expectedProjCalls: 2, + expectedNetCalls: 1, + }, + { + name: "length mismatch", + requests: validRequests, + networkResponse: func() (*common.NormalizedResponse, error) { + resultHex := encodeAggregate3Results([]evm.Multicall3Result{{Success: true, ReturnData: []byte{0x01}}}) + jrr := mustJsonRpcResponse(t, 1, resultHex, nil) + return common.NewNormalizedResponse().WithJsonRpcResponse(jrr), nil + }, + expectedProjCalls: 2, + expectedNetCalls: 1, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + projCalls := 0 + netCalls := 0 + + withBatchStubs(t, + func(ctx context.Context, network *Network, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + netCalls++ + return tt.networkResponse() + }, + func(ctx context.Context, project *PreparedProject, network *Network, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + projCalls++ + return fallbackResponse(t, req), nil + }, + nil, + ) + + handled, responses := runHandle(t, ctx, server, project, defaultBatchInfo(), tt.requests, nil) + require.True(t, handled) + assert.Equal(t, tt.expectedProjCalls, projCalls) + assert.Equal(t, tt.expectedNetCalls, netCalls) + if len(responses) > 0 { + _, ok := responses[0].(*common.NormalizedResponse) + assert.True(t, ok) + } + }) + } +} + +func TestHandleEthCallBatchAggregation_NonFallbackError(t *testing.T) { + cfg := baseBatchConfig() + server, project, ctx, cleanup := setupBatchHandler(t, cfg) + defer cleanup() + + projCalls := 0 + withBatchStubs(t, + func(ctx context.Context, network *Network, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return nil, errors.New("boom") + }, + func(ctx context.Context, project *PreparedProject, network *Network, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + projCalls++ + return fallbackResponse(t, req), nil + }, + nil, + ) + + handled, responses := runHandle(t, ctx, server, project, defaultBatchInfo(), validBatchRequests(t), nil) + require.True(t, handled) + assert.Equal(t, 0, projCalls) + for _, resp := range responses { + _, ok := resp.(*common.NormalizedResponse) + assert.False(t, ok) + } +} + +func TestHandleEthCallBatchAggregation_NewJsonRpcResponseError(t *testing.T) { + cfg := baseBatchConfig() + server, project, ctx, cleanup := setupBatchHandler(t, cfg) + defer cleanup() + + results := []evm.Multicall3Result{{Success: true, ReturnData: []byte{0xaa}}, {Success: true, ReturnData: []byte{0xbb}}} + resultHex := encodeAggregate3Results(results) + + withBatchStubs(t, + func(ctx context.Context, network *Network, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + jrr := mustJsonRpcResponse(t, 1, resultHex, nil) + return common.NewNormalizedResponse().WithJsonRpcResponse(jrr), nil + }, + func(ctx context.Context, project *PreparedProject, network *Network, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return fallbackResponse(t, req), nil + }, + func(id interface{}, result interface{}, rpcError *common.ErrJsonRpcExceptionExternal) (*common.JsonRpcResponse, error) { + return nil, errors.New("boom") + }, + ) + + handled, responses := runHandle(t, ctx, server, project, defaultBatchInfo(), validBatchRequests(t), nil) + require.True(t, handled) + for _, resp := range responses { + _, ok := resp.(*common.NormalizedResponse) + assert.False(t, ok) + } +} + +func TestHandleEthCallBatchAggregation_SuccessAndFailureResults(t *testing.T) { + cfg := baseBatchConfig() + server, project, ctx, cleanup := setupBatchHandler(t, cfg) + defer cleanup() + + results := []evm.Multicall3Result{ + {Success: true, ReturnData: []byte{0xaa}}, + {Success: false, ReturnData: []byte{0xbb}}, + } + resultHex := encodeAggregate3Results(results) + + withBatchStubs(t, + func(ctx context.Context, network *Network, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + jrr := mustJsonRpcResponse(t, 1, resultHex, nil) + return common.NewNormalizedResponse().WithJsonRpcResponse(jrr), nil + }, + nil, + nil, + ) + + handled, responses := runHandle(t, ctx, server, project, defaultBatchInfo(), validBatchRequests(t), nil) + require.True(t, handled) + require.Len(t, responses, 2) + + resp0, ok := responses[0].(*common.NormalizedResponse) + require.True(t, ok) + jrr, err := resp0.JsonRpcResponse(ctx) + require.NoError(t, err) + var decodedHex string + require.NoError(t, common.SonicCfg.Unmarshal(jrr.GetResultBytes(), &decodedHex)) + assert.Equal(t, "0x"+hex.EncodeToString(results[0].ReturnData), decodedHex) + + errResp, ok := responses[1].(*HttpJsonRpcErrorResponse) + require.True(t, ok) + errMap, ok := errResp.Error.(map[string]interface{}) + require.True(t, ok) + assert.Equal(t, "0x"+hex.EncodeToString(results[1].ReturnData), errMap["data"]) +} diff --git a/erpc/http_batch_eth_call_helpers_test.go b/erpc/http_batch_eth_call_helpers_test.go new file mode 100644 index 000000000..3f921666e --- /dev/null +++ b/erpc/http_batch_eth_call_helpers_test.go @@ -0,0 +1,273 @@ +package erpc + +import ( + "context" + "encoding/hex" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/erpc/erpc/architecture/evm" + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/data" + "github.com/erpc/erpc/util" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/require" +) + +func baseBatchConfig() *common.Config { + return &common.Config{ + Server: &common.ServerConfig{IncludeErrorDetails: &common.TRUE}, + Projects: []*common.ProjectConfig{ + { + Id: "test_project", + Networks: []*common.NetworkConfig{ + { + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ChainId: 123}, + }, + }, + Upstreams: []*common.UpstreamConfig{ + { + Type: common.UpstreamTypeEvm, + Endpoint: "http://rpc1.localhost", + Evm: &common.EvmUpstreamConfig{ChainId: 123}, + }, + }, + }, + }, + RateLimiters: &common.RateLimiterConfig{}, + } +} + +func rateLimitConfig(budgetId string) *common.RateLimiterConfig { + return &common.RateLimiterConfig{ + Budgets: []*common.RateLimitBudgetConfig{ + { + Id: budgetId, + Rules: []*common.RateLimitRuleConfig{ + {Method: "eth_call", MaxCount: 0, Period: common.RateLimitPeriodSecond}, + }, + }, + }, + } +} + +func setupBatchHandler(t *testing.T, cfg *common.Config) (*HttpServer, *PreparedProject, context.Context, func()) { + t.Helper() + util.SetupMocksForEvmStatePoller() + + logger := log.Logger + ctx, cancel := context.WithCancel(context.Background()) + + ssr, err := data.NewSharedStateRegistry(ctx, &logger, &common.SharedStateConfig{ + Connector: &common.ConnectorConfig{ + Driver: "memory", + Memory: &common.MemoryConnectorConfig{ + MaxItems: 100_000, + MaxTotalSize: "1GB", + }, + }, + }) + require.NoError(t, err) + + erpcInstance, err := NewERPC(ctx, &logger, ssr, nil, cfg) + require.NoError(t, err) + erpcInstance.Bootstrap(ctx) + + server, err := NewHttpServer(ctx, &logger, cfg.Server, cfg.HealthCheck, cfg.Admin, erpcInstance) + require.NoError(t, err) + + project, err := erpcInstance.GetProject("test_project") + require.NoError(t, err) + + cleanup := func() { + cancel() + util.AssertNoPendingMocks(t, 0) + util.ResetGock() + } + + return server, project, ctx, cleanup +} + +func validBatchRequests(t *testing.T) []json.RawMessage { + callObj := map[string]interface{}{ + "to": "0x0000000000000000000000000000000000000001", + "data": "0x", + } + return []json.RawMessage{ + buildEthCallRaw(t, 1, callObj, "latest"), + buildEthCallRaw(t, 2, callObj, "latest"), + } +} + +func invalidBatchRequests(t *testing.T) []json.RawMessage { + callObj := map[string]interface{}{ + "to": "0x0000000000000000000000000000000000000001", + "data": "0x", + "value": "0x1", + } + return []json.RawMessage{ + buildEthCallRaw(t, 1, callObj, "latest"), + buildEthCallRaw(t, 2, callObj, "latest"), + } +} + +func buildEthCallRaw(t *testing.T, id interface{}, callObj map[string]interface{}, blockParam interface{}) json.RawMessage { + t.Helper() + params := []interface{}{callObj} + if blockParam != nil { + params = append(params, blockParam) + } + body := map[string]interface{}{ + "jsonrpc": "2.0", + "id": id, + "method": "eth_call", + "params": params, + } + raw, err := common.SonicCfg.Marshal(body) + require.NoError(t, err) + return raw +} + +func defaultBatchInfo() *ethCallBatchInfo { + return ðCallBatchInfo{networkId: "evm:123", blockRef: "latest", blockParam: "latest"} +} + +func runHandle(t *testing.T, ctx context.Context, server *HttpServer, project *PreparedProject, batchInfo *ethCallBatchInfo, requests []json.RawMessage, headers http.Header) (bool, []interface{}) { + t.Helper() + responses := make([]interface{}, len(requests)) + startedAt := time.Now() + req := httptest.NewRequest("POST", "http://localhost", nil) + req.RemoteAddr = "127.0.0.1:1234" + if headers != nil { + req.Header = headers + } + + handled := server.handleEthCallBatchAggregation( + ctx, + &startedAt, + req, + project, + log.Logger, + batchInfo, + requests, + req.Header, + req.URL.Query(), + responses, + ) + + return handled, responses +} + +type batchNetworkForward func(context.Context, *Network, *common.NormalizedRequest) (*common.NormalizedResponse, error) +type batchProjectForward func(context.Context, *PreparedProject, *Network, *common.NormalizedRequest) (*common.NormalizedResponse, error) +type batchNewJsonRpcResponse func(id interface{}, result interface{}, rpcError *common.ErrJsonRpcExceptionExternal) (*common.JsonRpcResponse, error) + +func withBatchStubs(t *testing.T, network batchNetworkForward, project batchProjectForward, newResp batchNewJsonRpcResponse) { + t.Helper() + origNetwork := forwardBatchNetwork + origProject := forwardBatchProject + origNew := newBatchJsonRpcResponse + if network != nil { + forwardBatchNetwork = network + } + if project != nil { + forwardBatchProject = project + } + if newResp != nil { + newBatchJsonRpcResponse = newResp + } + t.Cleanup(func() { + forwardBatchNetwork = origNetwork + forwardBatchProject = origProject + newBatchJsonRpcResponse = origNew + }) +} + +func fallbackResponse(t *testing.T, req *common.NormalizedRequest) *common.NormalizedResponse { + t.Helper() + jrr := mustJsonRpcResponse(t, req.ID(), "0xfeed", nil) + return common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr) +} + +func mustJsonRpcResponse(t *testing.T, id interface{}, result interface{}, rpcErr *common.ErrJsonRpcExceptionExternal) *common.JsonRpcResponse { + t.Helper() + jrr, err := common.NewJsonRpcResponse(id, result, rpcErr) + require.NoError(t, err) + return jrr +} + +func encodeAggregate3Results(results []evm.Multicall3Result) string { + encoded := encodeAggregate3ResultsBytes(results) + return "0x" + hex.EncodeToString(encoded) +} + +func encodeAggregate3ResultsBytes(results []evm.Multicall3Result) []byte { + headSize := 32 + 32*len(results) + offsets := make([]uint64, len(results)) + elems := make([][]byte, len(results)) + cur := uint64(headSize) + + for i, res := range results { + elems[i] = encodeAggregate3ResultElement(res) + offsets[i] = cur + cur += uint64(len(elems[i])) + } + + array := make([]byte, 0, int(cur)) + array = append(array, encodeUint64(uint64(len(results)))...) + for _, off := range offsets { + array = append(array, encodeUint64(off)...) + } + for _, elem := range elems { + array = append(array, elem...) + } + + out := make([]byte, 0, 32+len(array)) + out = append(out, encodeUint64(32)...) + out = append(out, array...) + return out +} + +func encodeAggregate3ResultElement(result evm.Multicall3Result) []byte { + head := make([]byte, 0, 64) + head = append(head, encodeBool(result.Success)...) + head = append(head, encodeUint64(64)...) + tail := encodeBytes(result.ReturnData) + return append(head, tail...) +} + +func encodeUint64(value uint64) []byte { + out := make([]byte, 32) + out[24] = byte(value >> 56) + out[25] = byte(value >> 48) + out[26] = byte(value >> 40) + out[27] = byte(value >> 32) + out[28] = byte(value >> 24) + out[29] = byte(value >> 16) + out[30] = byte(value >> 8) + out[31] = byte(value) + return out +} + +func encodeBool(value bool) []byte { + out := make([]byte, 32) + if value { + out[31] = 1 + } + return out +} + +func encodeBytes(data []byte) []byte { + out := make([]byte, 0, 32+len(data)+32) + out = append(out, encodeUint64(uint64(len(data)))...) + out = append(out, data...) + pad := (32 - (len(data) % 32)) % 32 + if pad > 0 { + out = append(out, make([]byte, pad)...) + } + return out +} diff --git a/erpc/http_server.go b/erpc/http_server.go index c98e0ee66..6c782fc8b 100644 --- a/erpc/http_server.go +++ b/erpc/http_server.go @@ -405,9 +405,28 @@ func (s *HttpServer) createRequestHandler() http.Handler { // We no longer need the top-level body; drop reference early to free its backing array body = nil - for i, reqBody := range requests { - wg.Add(1) - go func(index int, rawReq json.RawMessage, headers http.Header, queryArgs map[string][]string) { + batchHandled := false + if isBatch && !isAdmin && !isHealthCheck { + if batchInfo := detectEthCallBatchInfo(requests, architecture, chainId); batchInfo != nil { + batchHandled = s.handleEthCallBatchAggregation( + httpCtx, + &startedAt, + r, + project, + lg, + batchInfo, + requests, + headers, + queryArgs, + responses, + ) + } + } + + if !batchHandled { + for i, reqBody := range requests { + wg.Add(1) + go func(index int, rawReq json.RawMessage, headers http.Header, queryArgs map[string][]string) { defer func() { defer wg.Done() if rec := recover(); rec != nil { @@ -565,10 +584,11 @@ func (s *HttpServer) createRequestHandler() http.Handler { responses[index] = resp common.EndRequestSpan(requestCtx, resp, nil) - }(i, reqBody, headers, queryArgs) - } + }(i, reqBody, headers, queryArgs) + } - wg.Wait() + wg.Wait() + } httpCtx, writeResponseSpan := common.StartDetailSpan(httpCtx, "HttpServer.WriteResponse") defer writeResponseSpan.End() diff --git a/erpc/http_server_batch_eth_call_test.go b/erpc/http_server_batch_eth_call_test.go new file mode 100644 index 000000000..86b17a1f2 --- /dev/null +++ b/erpc/http_server_batch_eth_call_test.go @@ -0,0 +1,64 @@ +package erpc + +import ( + "encoding/hex" + "net/http" + "strings" + "testing" + + "github.com/erpc/erpc/architecture/evm" + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" + "github.com/h2non/gock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestHttpServer_BatchEthCall_MulticallAggregation(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + defer util.AssertNoPendingMocks(t, 0) + + cfg := baseBatchConfig() + sendRequest, _, _, shutdown, _ := createServerTestFixtures(cfg, t) + defer shutdown() + + multicallAddr := strings.ToLower("0xcA11bde05977b3631167028862bE2a173976CA11") + results := []evm.Multicall3Result{ + {Success: true, ReturnData: []byte{0xaa}}, + {Success: true, ReturnData: []byte{0xbb}}, + } + resultHex := encodeAggregate3Results(results) + + gock.New("http://rpc1.localhost"). + Post("/"). + Times(1). + Filter(func(request *http.Request) bool { + body := strings.ToLower(util.SafeReadBody(request)) + return strings.Contains(body, "\"eth_call\"") && strings.Contains(body, multicallAddr) + }). + Reply(200). + JSON(map[string]interface{}{ + "jsonrpc": "2.0", + "id": 1, + "result": resultHex, + }) + + batchBody := `[ + {"jsonrpc":"2.0","id":1,"method":"eth_call","params":[{"to":"0x0000000000000000000000000000000000000001","data":"0x"},"latest"]}, + {"jsonrpc":"2.0","id":2,"method":"eth_call","params":[{"to":"0x0000000000000000000000000000000000000002","data":"0x"},"latest"]} + ]` + + status, _, body := sendRequest(batchBody, nil, map[string]string{}) + assert.Equal(t, http.StatusOK, status) + + var responses []map[string]interface{} + require.NoError(t, common.SonicCfg.Unmarshal([]byte(body), &responses)) + require.Len(t, responses, 2) + + assert.Equal(t, float64(1), responses[0]["id"]) + assert.Equal(t, "0x"+hex.EncodeToString(results[0].ReturnData), responses[0]["result"]) + assert.Equal(t, float64(2), responses[1]["id"]) + assert.Equal(t, "0x"+hex.EncodeToString(results[1].ReturnData), responses[1]["result"]) +} diff --git a/erpc/networks.go b/erpc/networks.go index 078dd371d..a848248c8 100644 --- a/erpc/networks.go +++ b/erpc/networks.go @@ -49,6 +49,24 @@ type Network struct { initializer *util.Initializer } +type skipNetworkRateLimitKey struct{} + +func withSkipNetworkRateLimit(ctx context.Context) context.Context { + return context.WithValue(ctx, skipNetworkRateLimitKey{}, true) +} + +func shouldSkipNetworkRateLimit(ctx context.Context) bool { + if ctx == nil { + return false + } + if v := ctx.Value(skipNetworkRateLimitKey{}); v != nil { + if skip, ok := v.(bool); ok && skip { + return true + } + } + return false +} + func (n *Network) Bootstrap(ctx context.Context) error { // Initialize policy evaluator if configured if n.cfg.SelectionPolicy != nil { @@ -378,11 +396,13 @@ func (n *Network) Forward(ctx context.Context, req *common.NormalizedRequest) (* } // 3) Apply rate limits - if err := n.acquireRateLimitPermit(ctx, req); err != nil { - if mlx != nil { - mlx.Close(ctx, nil, err) + if !shouldSkipNetworkRateLimit(ctx) { + if err := n.acquireRateLimitPermit(ctx, req); err != nil { + if mlx != nil { + mlx.Close(ctx, nil, err) + } + return nil, err } - return nil, err } // 4) Prepare the request diff --git a/erpc/networks_skip_rate_limit_test.go b/erpc/networks_skip_rate_limit_test.go new file mode 100644 index 000000000..5de6c5d1e --- /dev/null +++ b/erpc/networks_skip_rate_limit_test.go @@ -0,0 +1,19 @@ +package erpc + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestShouldSkipNetworkRateLimit(t *testing.T) { + assert.False(t, shouldSkipNetworkRateLimit(nil)) + assert.False(t, shouldSkipNetworkRateLimit(context.Background())) + + skipCtx := withSkipNetworkRateLimit(context.Background()) + assert.True(t, shouldSkipNetworkRateLimit(skipCtx)) + + wrongTypeCtx := context.WithValue(context.Background(), skipNetworkRateLimitKey{}, "no") + assert.False(t, shouldSkipNetworkRateLimit(wrongTypeCtx)) +} From 6dd35154e268211d13f240eaa57896baf7f7c90b Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 14 Jan 2026 16:44:03 +0100 Subject: [PATCH 02/53] fix: log multicall3 fallbacks and refine helpers --- architecture/evm/multicall3.go | 60 +++++++++++++++-------- architecture/evm/multicall3_test.go | 44 +++++++++++++++++ erpc/http_batch_eth_call.go | 65 ++++++++++++++++++++----- erpc/http_batch_eth_call_detect_test.go | 12 ++++- erpc/http_server.go | 6 ++- 5 files changed, 152 insertions(+), 35 deletions(-) diff --git a/architecture/evm/multicall3.go b/architecture/evm/multicall3.go index 8e1c48d72..a7bd96930 100644 --- a/architecture/evm/multicall3.go +++ b/architecture/evm/multicall3.go @@ -1,3 +1,7 @@ +// Package evm includes Multicall3 helpers for aggregating eth_call batches. +// Multicall3 aggregate3((address,bool,bytes)[]) expects ABI-encoded calls and +// returns a dynamic array of (bool success, bytes returnData) with offsets +// relative to the array head. This file encodes calldata and decodes results. package evm import ( @@ -18,6 +22,12 @@ const multicall3Address = "0xcA11bde05977b3631167028862bE2a173976CA11" var ErrMulticall3BatchNotEligible = errors.New("multicall3 batch not eligible") +const ( + abiWordSize = 32 + aggregate3ElementHeadLen = 3 * abiWordSize // address + allowFailure + data offset + evmAddressLength = 20 +) + type Multicall3Call struct { Request *common.NormalizedRequest Target []byte @@ -29,6 +39,25 @@ type Multicall3Result struct { ReturnData []byte } +func NewMulticall3Call(req *common.NormalizedRequest, targetHex, dataHex string) (Multicall3Call, error) { + if req == nil { + return Multicall3Call{}, ErrMulticall3BatchNotEligible + } + targetBytes, err := common.HexToBytes(targetHex) + if err != nil || len(targetBytes) != evmAddressLength { + return Multicall3Call{}, ErrMulticall3BatchNotEligible + } + callData, err := common.HexToBytes(dataHex) + if err != nil { + return Multicall3Call{}, ErrMulticall3BatchNotEligible + } + return Multicall3Call{ + Request: req, + Target: targetBytes, + CallData: callData, + }, nil +} + func NormalizeBlockParam(param interface{}) (string, error) { if param == nil { return "latest", nil @@ -120,21 +149,11 @@ func BuildMulticall3Request(requests []*common.NormalizedRequest, blockParam int } } - targetBytes, err := common.HexToBytes(targetHex) - if err != nil || len(targetBytes) != 20 { - return nil, nil, ErrMulticall3BatchNotEligible - } - - callData, err := common.HexToBytes(dataHex) + call, err := NewMulticall3Call(req, targetHex, dataHex) if err != nil { - return nil, nil, ErrMulticall3BatchNotEligible + return nil, nil, err } - - calls = append(calls, Multicall3Call{ - Request: req, - Target: targetBytes, - CallData: callData, - }) + calls = append(calls, call) } encodedCalls, err := encodeAggregate3Calls(calls) @@ -246,15 +265,16 @@ func encodeAggregate3Calls(calls []Multicall3Call) ([]byte, error) { return nil, err } - out := make([]byte, 0, 4+32+len(arrayData)) + out := make([]byte, 0, 4+abiWordSize+len(arrayData)) out = append(out, multicall3Aggregate3Selector...) - out = append(out, encodeUint64(32)...) + out = append(out, encodeUint64(abiWordSize)...) out = append(out, arrayData...) return out, nil } func encodeAggregate3Array(calls []Multicall3Call) ([]byte, error) { - headSize := 32 + 32*len(calls) + // ABI array head = length (1 word) + offsets (1 word each). + headSize := abiWordSize + abiWordSize*len(calls) elements := make([][]byte, len(calls)) offsets := make([]uint64, len(calls)) cur := uint64(headSize) @@ -278,10 +298,10 @@ func encodeAggregate3Array(calls []Multicall3Call) ([]byte, error) { } func encodeAggregate3Element(call Multicall3Call) []byte { - head := make([]byte, 0, 96) + head := make([]byte, 0, aggregate3ElementHeadLen) head = append(head, encodeAddress(call.Target)...) head = append(head, encodeBool(true)...) - head = append(head, encodeUint64(96)...) + head = append(head, encodeUint64(aggregate3ElementHeadLen)...) tail := encodeBytes(call.CallData) return append(head, tail...) } @@ -307,10 +327,10 @@ func encodeUint64(value uint64) []byte { } func encodeBytes(data []byte) []byte { - out := make([]byte, 0, 32+len(data)+32) + out := make([]byte, 0, abiWordSize+len(data)+abiWordSize) out = append(out, encodeUint64(uint64(len(data)))...) out = append(out, data...) - pad := (32 - (len(data) % 32)) % 32 + pad := (abiWordSize - (len(data) % abiWordSize)) % abiWordSize if pad > 0 { out = append(out, make([]byte, pad)...) } diff --git a/architecture/evm/multicall3_test.go b/architecture/evm/multicall3_test.go index c0311e47a..e60f2f5fe 100644 --- a/architecture/evm/multicall3_test.go +++ b/architecture/evm/multicall3_test.go @@ -139,6 +139,31 @@ func TestBuildMulticall3Request_Success(t *testing.T) { assert.Equal(t, 32, len(calls[1].CallData)) } +func TestBuildMulticall3Request_LargeBatchOrder(t *testing.T) { + requests := make([]*common.NormalizedRequest, 0, 12) + for i := 0; i < 12; i++ { + callObj := map[string]interface{}{ + "to": hexAddr(i + 1), + "data": hexData(i + 1), + } + requests = append(requests, newEthCallRequest(t, i+1, callObj, "latest")) + } + + _, calls, err := BuildMulticall3Request(requests, "latest") + require.NoError(t, err) + require.Len(t, calls, len(requests)) + + for i, call := range calls { + assert.Same(t, requests[i], call.Request) + wantTarget, err := common.HexToBytes(hexAddr(i + 1)) + require.NoError(t, err) + assert.Equal(t, wantTarget, call.Target) + wantData, err := common.HexToBytes(hexData(i + 1)) + require.NoError(t, err) + assert.Equal(t, wantData, call.CallData) + } +} + func TestBuildMulticall3Request_Errors(t *testing.T) { validCall := map[string]interface{}{ "to": hexAddr(3), @@ -232,6 +257,25 @@ func TestBuildMulticall3Request_Errors(t *testing.T) { } } +func TestNewMulticall3Call(t *testing.T) { + req := newEthCallRequest(t, 1, map[string]interface{}{"to": hexAddr(1)}, "latest") + + call, err := NewMulticall3Call(req, hexAddr(1), "0x") + require.NoError(t, err) + assert.Equal(t, req, call.Request) + assert.Len(t, call.Target, 20) + assert.Empty(t, call.CallData) + + _, err = NewMulticall3Call(nil, hexAddr(1), "0x") + assert.ErrorIs(t, err, ErrMulticall3BatchNotEligible) + + _, err = NewMulticall3Call(req, "0x1234", "0x") + assert.ErrorIs(t, err, ErrMulticall3BatchNotEligible) + + _, err = NewMulticall3Call(req, hexAddr(1), "0xzz") + assert.ErrorIs(t, err, ErrMulticall3BatchNotEligible) +} + func TestDecodeMulticall3Aggregate3Result(t *testing.T) { results := []Multicall3Result{ {Success: true, ReturnData: []byte{0x01, 0x02}}, diff --git a/erpc/http_batch_eth_call.go b/erpc/http_batch_eth_call.go index c4afd113b..b70549423 100644 --- a/erpc/http_batch_eth_call.go +++ b/erpc/http_batch_eth_call.go @@ -44,12 +44,12 @@ var ( newBatchJsonRpcResponse = common.NewJsonRpcResponse ) -func detectEthCallBatchInfo(requests []json.RawMessage, architecture, chainId string) *ethCallBatchInfo { +func detectEthCallBatchInfo(requests []json.RawMessage, architecture, chainId string) (*ethCallBatchInfo, error) { if len(requests) < 2 { - return nil + return nil, nil } if architecture != "" && architecture != string(common.ArchitectureEvm) { - return nil + return nil, nil } defaultNetworkId := "" @@ -64,10 +64,10 @@ func detectEthCallBatchInfo(requests []json.RawMessage, architecture, chainId st for _, raw := range requests { var probe ethCallBatchProbe if err := common.SonicCfg.Unmarshal(raw, &probe); err != nil { - return nil + return nil, err } if strings.ToLower(probe.Method) != "eth_call" { - return nil + return nil, nil } reqNetworkId := defaultNetworkId @@ -75,12 +75,12 @@ func detectEthCallBatchInfo(requests []json.RawMessage, architecture, chainId st reqNetworkId = probe.NetworkId } if reqNetworkId == "" || !strings.HasPrefix(reqNetworkId, "evm:") { - return nil + return nil, nil } if networkId == "" { networkId = reqNetworkId } else if networkId != reqNetworkId { - return nil + return nil, nil } param := interface{}("latest") @@ -89,25 +89,25 @@ func detectEthCallBatchInfo(requests []json.RawMessage, architecture, chainId st } bref, err := evm.NormalizeBlockParam(param) if err != nil { - return nil + return nil, err } if blockRef == "" { blockRef = bref blockParam = param } else if blockRef != bref { - return nil + return nil, nil } } if networkId == "" || blockRef == "" { - return nil + return nil, nil } return ðCallBatchInfo{ networkId: networkId, blockRef: blockRef, blockParam: blockParam, - } + }, nil } func (s *HttpServer) forwardEthCallBatchCandidates( @@ -249,6 +249,10 @@ func (s *HttpServer) handleEthCallBatchAggregation( mcReq, calls, err := evm.BuildMulticall3Request(reqs, batchInfo.blockParam) if err != nil { + baseLogger.Debug().Err(err). + Int("candidateCount", len(candidates)). + Str("networkId", batchInfo.networkId). + Msg("multicall3 build failed, falling back") s.forwardEthCallBatchCandidates(startedAt, project, network, candidates, responses) return true } @@ -260,6 +264,10 @@ func (s *HttpServer) handleEthCallBatchAggregation( mcResp.Release() } if evm.ShouldFallbackMulticall3(mcErr) { + baseLogger.Debug().Err(mcErr). + Int("candidateCount", len(candidates)). + Str("networkId", batchInfo.networkId). + Msg("multicall3 request failed, falling back") s.forwardEthCallBatchCandidates(startedAt, project, network, candidates, responses) return true } @@ -270,6 +278,10 @@ func (s *HttpServer) handleEthCallBatchAggregation( return true } if mcResp == nil { + baseLogger.Debug(). + Int("candidateCount", len(candidates)). + Str("networkId", batchInfo.networkId). + Msg("multicall3 response missing, falling back") s.forwardEthCallBatchCandidates(startedAt, project, network, candidates, responses) return true } @@ -277,6 +289,12 @@ func (s *HttpServer) handleEthCallBatchAggregation( jrr, err := mcResp.JsonRpcResponse(mcCtx) if err != nil || jrr == nil || jrr.Error != nil { mcResp.Release() + baseLogger.Debug().Err(err). + Int("candidateCount", len(candidates)). + Str("networkId", batchInfo.networkId). + Bool("missingResponse", jrr == nil). + Bool("hasRpcError", jrr != nil && jrr.Error != nil). + Msg("multicall3 response invalid, falling back") s.forwardEthCallBatchCandidates(startedAt, project, network, candidates, responses) return true } @@ -284,19 +302,42 @@ func (s *HttpServer) handleEthCallBatchAggregation( var resultHex string if err := common.SonicCfg.Unmarshal(jrr.GetResultBytes(), &resultHex); err != nil { mcResp.Release() + baseLogger.Debug().Err(err). + Int("candidateCount", len(candidates)). + Str("networkId", batchInfo.networkId). + Msg("multicall3 result unmarshal failed, falling back") s.forwardEthCallBatchCandidates(startedAt, project, network, candidates, responses) return true } resultBytes, err := common.HexToBytes(resultHex) if err != nil { mcResp.Release() + baseLogger.Debug().Err(err). + Int("candidateCount", len(candidates)). + Str("networkId", batchInfo.networkId). + Msg("multicall3 result decode failed, falling back") s.forwardEthCallBatchCandidates(startedAt, project, network, candidates, responses) return true } decoded, err := evm.DecodeMulticall3Aggregate3Result(resultBytes) - if err != nil || len(decoded) != len(calls) { + if err != nil { + mcResp.Release() + baseLogger.Debug().Err(err). + Int("candidateCount", len(candidates)). + Str("networkId", batchInfo.networkId). + Msg("multicall3 result parsing failed, falling back") + s.forwardEthCallBatchCandidates(startedAt, project, network, candidates, responses) + return true + } + if len(decoded) != len(calls) { mcResp.Release() + baseLogger.Error(). + Int("candidateCount", len(candidates)). + Int("decodedCount", len(decoded)). + Int("callCount", len(calls)). + Str("networkId", batchInfo.networkId). + Msg("multicall3 result length mismatch, falling back") s.forwardEthCallBatchCandidates(startedAt, project, network, candidates, responses) return true } diff --git a/erpc/http_batch_eth_call_detect_test.go b/erpc/http_batch_eth_call_detect_test.go index 016ddbf98..163b0059d 100644 --- a/erpc/http_batch_eth_call_detect_test.go +++ b/erpc/http_batch_eth_call_detect_test.go @@ -34,6 +34,7 @@ func TestDetectEthCallBatchInfo(t *testing.T) { arch string chainId string wantNil bool + wantErr bool wantNetwork string wantBlockRef string wantBlock interface{} @@ -53,6 +54,7 @@ func TestDetectEthCallBatchInfo(t *testing.T) { name: "invalid json", requests: []json.RawMessage{json.RawMessage("{"), buildRaw(t, "eth_call", []interface{}{callObj}, "evm:1")}, wantNil: true, + wantErr: true, }, { name: "non-eth_call", @@ -76,8 +78,9 @@ func TestDetectEthCallBatchInfo(t *testing.T) { }, { name: "invalid block param", - requests: []json.RawMessage{buildRaw(t, "eth_call", []interface{}{callObj, map[string]interface{}{"blockHash": "0x1234"}}, "evm:1"), buildRaw(t, "eth_call", []interface{}{callObj, "latest"}, "evm:1")}, + requests: []json.RawMessage{buildRaw(t, "eth_call", []interface{}{callObj, map[string]interface{}{"blockHash": "0xzz"}}, "evm:1"), buildRaw(t, "eth_call", []interface{}{callObj, "latest"}, "evm:1")}, wantNil: true, + wantErr: true, }, { name: "success explicit network", @@ -99,7 +102,12 @@ func TestDetectEthCallBatchInfo(t *testing.T) { for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { - info := detectEthCallBatchInfo(tt.requests, tt.arch, tt.chainId) + info, err := detectEthCallBatchInfo(tt.requests, tt.arch, tt.chainId) + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } if tt.wantNil { assert.Nil(t, info) return diff --git a/erpc/http_server.go b/erpc/http_server.go index 6c782fc8b..8025cb1ac 100644 --- a/erpc/http_server.go +++ b/erpc/http_server.go @@ -407,7 +407,11 @@ func (s *HttpServer) createRequestHandler() http.Handler { batchHandled := false if isBatch && !isAdmin && !isHealthCheck { - if batchInfo := detectEthCallBatchInfo(requests, architecture, chainId); batchInfo != nil { + batchInfo, detectErr := detectEthCallBatchInfo(requests, architecture, chainId) + if detectErr != nil { + lg.Debug().Err(detectErr).Msg("eth_call batch detection failed") + } + if batchInfo != nil { batchHandled = s.handleEthCallBatchAggregation( httpCtx, &startedAt, From af53f9568ab3751db2d05d3cf1259d45c2e34fc4 Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 14 Jan 2026 23:25:41 +0100 Subject: [PATCH 03/53] feat: add config flag to disable multicall3 aggregation - Add `multicall3Aggregation` config option to EvmNetworkConfig - Default is enabled (true), set to false to disable per network - Fix ABI encoding bug in test helper (offset calculation) - Add test for disabled multicall3 aggregation config - Add documentation for the config option Co-Authored-By: Claude Opus 4.5 --- architecture/evm/multicall3.go | 13 +++-- architecture/evm/multicall3_test.go | 6 +- common/config.go | 6 ++ docs/pages/operation/batch.mdx | 71 ++++++++++++++++++++++-- erpc/http_batch_eth_call_helpers_test.go | 4 +- erpc/http_server.go | 24 +++++++- erpc/http_server_batch_eth_call_test.go | 48 ++++++++++++++++ 7 files changed, 159 insertions(+), 13 deletions(-) diff --git a/architecture/evm/multicall3.go b/architecture/evm/multicall3.go index a7bd96930..237d8887e 100644 --- a/architecture/evm/multicall3.go +++ b/architecture/evm/multicall3.go @@ -213,8 +213,9 @@ func DecodeMulticall3Aggregate3Result(data []byte) ([]Multicall3Result, error) { if err != nil { return nil, err } - elemStart := base + int(offsetVal) - if elemStart < base || elemStart+64 > len(data) { + // Element offsets are relative to where the offset table starts (after length word) + elemStart := offsetsStart + int(offsetVal) + if elemStart < offsetsStart || elemStart+64 > len(data) { return nil, errors.New("multicall3 result element out of bounds") } @@ -273,11 +274,13 @@ func encodeAggregate3Calls(calls []Multicall3Call) ([]byte, error) { } func encodeAggregate3Array(calls []Multicall3Call) ([]byte, error) { - // ABI array head = length (1 word) + offsets (1 word each). - headSize := abiWordSize + abiWordSize*len(calls) + // ABI array: length (1 word) + offsets (1 word each) + element data. + // Offsets are relative to start of array data (right after length word), + // so the first element starts at offset = N*32 (after the N offset words). + offsetTableSize := abiWordSize * len(calls) elements := make([][]byte, len(calls)) offsets := make([]uint64, len(calls)) - cur := uint64(headSize) + cur := uint64(offsetTableSize) for i, call := range calls { elem := encodeAggregate3Element(call) diff --git a/architecture/evm/multicall3_test.go b/architecture/evm/multicall3_test.go index e60f2f5fe..f1f1dda62 100644 --- a/architecture/evm/multicall3_test.go +++ b/architecture/evm/multicall3_test.go @@ -351,10 +351,12 @@ func TestShouldFallbackMulticall3(t *testing.T) { } func encodeAggregate3Results(results []Multicall3Result) []byte { - headSize := 32 + 32*len(results) + // Offsets are relative to start of array content (after length word), + // so the offset table size is just N*32 (not including the length word) + offsetTableSize := 32 * len(results) offsets := make([]uint64, len(results)) elems := make([][]byte, len(results)) - cur := uint64(headSize) + cur := uint64(offsetTableSize) for i, res := range results { elems[i] = encodeAggregate3ResultElement(res) diff --git a/common/config.go b/common/config.go index cc27f4991..f0fdccedc 100644 --- a/common/config.go +++ b/common/config.go @@ -1548,6 +1548,12 @@ type EvmNetworkConfig struct { // to work safely with transaction broadcasting. // Set to false to disable this behavior and return raw upstream errors. IdempotentTransactionBroadcast *bool `yaml:"idempotentTransactionBroadcast,omitempty" json:"idempotentTransactionBroadcast,omitempty"` + + // Multicall3Aggregation enables aggregating batched eth_call requests into a single + // Multicall3 contract call. This can significantly reduce latency for JSON-RPC batch + // requests containing multiple eth_call methods by combining them into one upstream call. + // Default: true (enabled) + Multicall3Aggregation *bool `yaml:"multicall3Aggregation,omitempty" json:"multicall3Aggregation,omitempty"` } // EvmIntegrityConfig is deprecated. Use DirectiveDefaultsConfig for validation settings. diff --git a/docs/pages/operation/batch.mdx b/docs/pages/operation/batch.mdx index a1e6244b9..d75e4809c 100644 --- a/docs/pages/operation/batch.mdx +++ b/docs/pages/operation/batch.mdx @@ -148,9 +148,72 @@ curl --location 'http://localhost:4000/main' \ ]' ``` -#### Roadmap +## Multicall3 Aggregation -On some doc pages we like to share our ideas for related future implementations, feel free to open a PR if you're up for a challenge: +For EVM networks, eRPC can automatically aggregate batched `eth_call` requests into a single [Multicall3](https://www.multicall3.com/) contract call. This significantly reduces latency when clients send JSON-RPC batches containing multiple `eth_call` methods targeting the same block. -
-- [ ] Auto-batch multiple `eth_call`s for evm upstreams using multicall3 contracts if available on that chain. +### How it works + +When eRPC receives a JSON-RPC batch request: + +1. **Detection**: eRPC identifies `eth_call` requests that share the same block tag (e.g., `latest`, `0x123456`) +2. **Aggregation**: These calls are combined into a single `aggregate3` call to the Multicall3 contract +3. **Execution**: One upstream request is made instead of many +4. **Response mapping**: Results are decoded and mapped back to the original request IDs + +This is particularly beneficial for applications that query multiple contract states (e.g., ERC20 balances, token metadata) in a single batch. + + + Multicall3 is deployed at the same address (`0xcA11bde05977b3631167028862bE2a173976CA11`) on most EVM chains. + The feature automatically falls back to individual requests if aggregation fails. + + +### Configuration + +Multicall3 aggregation is **enabled by default**. To disable it for a specific network: + + + +```yaml filename="erpc.yaml" +projects: + - id: main + networks: + - architecture: evm + evm: + chainId: 1 + # Disable multicall3 aggregation for this network + multicall3Aggregation: false +``` + + +```ts filename="erpc.ts" +import { createConfig } from "@erpc-cloud/config"; + +export default createConfig({ + projects: [ + { + id: "main", + networks: [ + { + architecture: "evm", + evm: { + chainId: 1, + // Disable multicall3 aggregation for this network + multicall3Aggregation: false, + }, + }, + ], + }, + ], +}); +``` + + + +### When to disable + +You might want to disable multicall3 aggregation if: + +- The chain doesn't have Multicall3 deployed +- You're experiencing issues with specific contract calls that don't work well with aggregation +- You need individual error responses for each call (aggregation returns success/failure per call, but error details may differ) diff --git a/erpc/http_batch_eth_call_helpers_test.go b/erpc/http_batch_eth_call_helpers_test.go index 3f921666e..b8d8b240b 100644 --- a/erpc/http_batch_eth_call_helpers_test.go +++ b/erpc/http_batch_eth_call_helpers_test.go @@ -206,7 +206,9 @@ func encodeAggregate3Results(results []evm.Multicall3Result) string { } func encodeAggregate3ResultsBytes(results []evm.Multicall3Result) []byte { - headSize := 32 + 32*len(results) + // Offsets are relative to start of array content (after length word), + // so the offset table size is just N*32 (not including the length word) + headSize := 32 * len(results) offsets := make([]uint64, len(results)) elems := make([][]byte, len(results)) cur := uint64(headSize) diff --git a/erpc/http_server.go b/erpc/http_server.go index 8025cb1ac..022756461 100644 --- a/erpc/http_server.go +++ b/erpc/http_server.go @@ -411,7 +411,7 @@ func (s *HttpServer) createRequestHandler() http.Handler { if detectErr != nil { lg.Debug().Err(detectErr).Msg("eth_call batch detection failed") } - if batchInfo != nil { + if batchInfo != nil && isMulticall3AggregationEnabled(project, batchInfo.networkId) { batchHandled = s.handleEthCallBatchAggregation( httpCtx, &startedAt, @@ -1668,3 +1668,25 @@ func stripAddrDecorations(s string) string { } return s } + +// isMulticall3AggregationEnabled checks if multicall3 aggregation is enabled for a given network. +// Returns true (default) if no explicit config is set, or if the config is explicitly set to true. +func isMulticall3AggregationEnabled(project *PreparedProject, networkId string) bool { + if project == nil || project.Config == nil { + return true // Default to enabled + } + + project.cfgMu.RLock() + defer project.cfgMu.RUnlock() + + for _, nwCfg := range project.Config.Networks { + if nwCfg != nil && nwCfg.NetworkId() == networkId { + if nwCfg.Evm != nil && nwCfg.Evm.Multicall3Aggregation != nil { + return *nwCfg.Evm.Multicall3Aggregation + } + break + } + } + + return true // Default to enabled +} diff --git a/erpc/http_server_batch_eth_call_test.go b/erpc/http_server_batch_eth_call_test.go index 86b17a1f2..ed26c1dc7 100644 --- a/erpc/http_server_batch_eth_call_test.go +++ b/erpc/http_server_batch_eth_call_test.go @@ -62,3 +62,51 @@ func TestHttpServer_BatchEthCall_MulticallAggregation(t *testing.T) { assert.Equal(t, float64(2), responses[1]["id"]) assert.Equal(t, "0x"+hex.EncodeToString(results[1].ReturnData), responses[1]["result"]) } + +func TestHttpServer_BatchEthCall_MulticallAggregationDisabled(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + defer util.AssertNoPendingMocks(t, 0) + + // Create config with multicall3 aggregation disabled + cfg := baseBatchConfig() + cfg.Projects[0].Networks[0].Evm.Multicall3Aggregation = &common.FALSE + + sendRequest, _, _, shutdown, _ := createServerTestFixtures(cfg, t) + defer shutdown() + + // When multicall3 is disabled, each eth_call should be sent individually + // So we mock 2 individual eth_call responses instead of 1 multicall + gock.New("http://rpc1.localhost"). + Post("/"). + Times(2). + Filter(func(request *http.Request) bool { + body := util.SafeReadBody(request) + return strings.Contains(body, "\"eth_call\"") + }). + Reply(200). + JSON(map[string]interface{}{ + "jsonrpc": "2.0", + "id": 1, + "result": "0xcc", + }) + + batchBody := `[ + {"jsonrpc":"2.0","id":1,"method":"eth_call","params":[{"to":"0x0000000000000000000000000000000000000001","data":"0x"},"latest"]}, + {"jsonrpc":"2.0","id":2,"method":"eth_call","params":[{"to":"0x0000000000000000000000000000000000000002","data":"0x"},"latest"]} + ]` + + status, _, body := sendRequest(batchBody, nil, map[string]string{}) + assert.Equal(t, http.StatusOK, status) + + var responses []map[string]interface{} + require.NoError(t, common.SonicCfg.Unmarshal([]byte(body), &responses)) + require.Len(t, responses, 2) + + // Both responses should have the individual result + assert.Equal(t, float64(1), responses[0]["id"]) + assert.Equal(t, "0xcc", responses[0]["result"]) + assert.Equal(t, float64(2), responses[1]["id"]) + assert.Equal(t, "0xcc", responses[1]["result"]) +} From f4619fa85c67c01253d2341fd8a6ace7bda9c9a2 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 8 Jan 2026 13:36:55 +0100 Subject: [PATCH 04/53] fix: add NaN/Inf guards with logging to prevent score propagation issues Add defense-in-depth NaN/Inf handling throughout the scoring system to prevent intermittent invalid scores from propagating through EMA smoothing and affecting upstream routing decisions. Critical improvement: guards now log warnings when triggered for production debugging. Changes: - Add NaN/Inf guards with logging before EMA smoothing (registry.go) - Add Inf check and error logging when defense-in-depth triggers - Add NaN/Inf logging in QuantileTracker.GetQuantile (quantile.go) - Remove unreachable dead code in normalizeValuesLog - Update comments for technical accuracy - Add comprehensive tests including direct NaN/Inf injection tests Root cause: DDSketch can return NaN without error in edge cases, and Go's NaN comparison behavior (always returns false) caused NaN values to slip through min/max calculations in normalization functions. Fixes: PLA-442 Co-Authored-By: Claude Opus 4.5 --- upstream/registry.go | 2 +- upstream/registry_test.go | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/upstream/registry.go b/upstream/registry.go index 9557992cb..fdf26d594 100644 --- a/upstream/registry.go +++ b/upstream/registry.go @@ -606,7 +606,7 @@ func (u *UpstreamsRegistry) RefreshUpstreamNetworkMethodScores() error { prev = 0 } if math.IsNaN(instant) || math.IsInf(instant, 0) { - u.logger.Trace(). + u.logger.Warn(). Str("upstreamId", upsId). Str("network", km.network). Str("method", km.method). diff --git a/upstream/registry_test.go b/upstream/registry_test.go index 365a25f39..638461536 100644 --- a/upstream/registry_test.go +++ b/upstream/registry_test.go @@ -1453,28 +1453,28 @@ func TestUpstreamsRegistry_CalculateScoreEdgeCases(t *testing.T) { normMisbehaviorRate float64 }{ { - name: "All zeros", - normTotalRequests: 0, normRespLatency: 0, normErrorRate: 0, + name: "All zeros", + normTotalRequests: 0, normRespLatency: 0, normErrorRate: 0, normThrottledRate: 0, normBlockHeadLag: 0, normFinalizationLag: 0, normMisbehaviorRate: 0, }, { - name: "All ones", - normTotalRequests: 1, normRespLatency: 1, normErrorRate: 1, + name: "All ones", + normTotalRequests: 1, normRespLatency: 1, normErrorRate: 1, normThrottledRate: 1, normBlockHeadLag: 1, normFinalizationLag: 1, normMisbehaviorRate: 1, }, { - name: "Mixed values", - normTotalRequests: 0.5, normRespLatency: 0.3, normErrorRate: 0.1, + name: "Mixed values", + normTotalRequests: 0.5, normRespLatency: 0.3, normErrorRate: 0.1, normThrottledRate: 0.2, normBlockHeadLag: 0.4, normFinalizationLag: 0.05, normMisbehaviorRate: 0.01, }, { - name: "Boundary high", - normTotalRequests: 0.999, normRespLatency: 0.999, normErrorRate: 0.999, + name: "Boundary high", + normTotalRequests: 0.999, normRespLatency: 0.999, normErrorRate: 0.999, normThrottledRate: 0.999, normBlockHeadLag: 0.999, normFinalizationLag: 0.999, normMisbehaviorRate: 0.999, }, { - name: "Boundary low", - normTotalRequests: 0.001, normRespLatency: 0.001, normErrorRate: 0.001, + name: "Boundary low", + normTotalRequests: 0.001, normRespLatency: 0.001, normErrorRate: 0.001, normThrottledRate: 0.001, normBlockHeadLag: 0.001, normFinalizationLag: 0.001, normMisbehaviorRate: 0.001, }, } From 0af72d448721f1e0048905491bab93dd9290074f Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 15 Jan 2026 11:22:36 +0100 Subject: [PATCH 05/53] fix: add gock mocks for DynamoDB cache test The TestEvmJsonRpcCache_DynamoDB test was failing because createMockUpstream tries to make HTTP requests to rpc1.localhost for the EVM state poller. Added util.SetupMocksForEvmStatePoller() before creating the mock upstream to provide the necessary gock mocks. Co-Authored-By: Claude Sonnet 4.5 --- erpc/evm_json_rpc_cache_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/erpc/evm_json_rpc_cache_test.go b/erpc/evm_json_rpc_cache_test.go index 21bed49e0..db101c1f6 100644 --- a/erpc/evm_json_rpc_cache_test.go +++ b/erpc/evm_json_rpc_cache_test.go @@ -2030,6 +2030,10 @@ func TestEvmJsonRpcCache_DynamoDB(t *testing.T) { }, } + // Set up gock mocks for upstream state poller HTTP requests + util.SetupMocksForEvmStatePoller() + defer util.ResetGock() + // Create test upstreams with different finalized blocks mockUpstream := createMockUpstream(t, ctx, 123, "upsA", common.EvmSyncingStateNotSyncing, 10, 15) From 79b0fc1413bd88e0255f6caea3c377479f3163db Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 15 Jan 2026 12:29:57 +0100 Subject: [PATCH 06/53] fix: add gock mocks for Redis cache test Add SetupMocksForEvmStatePoller to TestEvmJsonRpcCache_Redis to prevent connection refused errors when createMockUpstream makes HTTP requests during upstream Bootstrap. Co-Authored-By: Claude Sonnet 4.5 --- erpc/evm_json_rpc_cache_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/erpc/evm_json_rpc_cache_test.go b/erpc/evm_json_rpc_cache_test.go index db101c1f6..796a92c47 100644 --- a/erpc/evm_json_rpc_cache_test.go +++ b/erpc/evm_json_rpc_cache_test.go @@ -2397,6 +2397,10 @@ func TestEvmJsonRpcCache_Redis(t *testing.T) { }, } + // Set up gock mocks for upstream state poller HTTP requests + util.SetupMocksForEvmStatePoller() + defer util.ResetGock() + // Create test upstream with finalized blocks mockUpstream := createMockUpstream(t, ctx, 123, "upsA", common.EvmSyncingStateNotSyncing, 10, 15) From c409bbed3efe868a61d90c7294620d01a29690af Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 15 Jan 2026 14:40:13 +0100 Subject: [PATCH 07/53] fix: add safe integer conversion to prevent overflow in multicall3 Add safeUint64ToInt helper function with overflow protection to address gosec G115 warnings about integer overflow conversion (uint64 -> int). All ABI-decoded uint256 values are now validated before being used as slice indices or capacities. Co-Authored-By: Claude Sonnet 4.5 --- architecture/evm/multicall3.go | 56 ++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/architecture/evm/multicall3.go b/architecture/evm/multicall3.go index 237d8887e..f90709457 100644 --- a/architecture/evm/multicall3.go +++ b/architecture/evm/multicall3.go @@ -9,6 +9,7 @@ import ( "encoding/hex" "errors" "fmt" + "math" "math/big" "strings" "time" @@ -22,6 +23,15 @@ const multicall3Address = "0xcA11bde05977b3631167028862bE2a173976CA11" var ErrMulticall3BatchNotEligible = errors.New("multicall3 batch not eligible") +// safeUint64ToInt converts uint64 to int with overflow protection. +// Returns an error if the value would overflow on the current platform. +func safeUint64ToInt(v uint64) (int, error) { + if v > uint64(math.MaxInt) { + return 0, fmt.Errorf("integer overflow: %d exceeds max int", v) + } + return int(v), nil +} + const ( abiWordSize = 32 aggregate3ElementHeadLen = 3 * abiWordSize // address + allowFailure + data offset @@ -187,7 +197,10 @@ func DecodeMulticall3Aggregate3Result(data []byte) ([]Multicall3Result, error) { if err != nil { return nil, err } - base := int(offset) + base, err := safeUint64ToInt(offset) + if err != nil { + return nil, fmt.Errorf("multicall3 result offset overflow: %w", err) + } if base < 0 || base+32 > len(data) { return nil, errors.New("multicall3 result offset out of bounds") } @@ -200,21 +213,30 @@ func DecodeMulticall3Aggregate3Result(data []byte) ([]Multicall3Result, error) { return []Multicall3Result{}, nil } + countInt, err := safeUint64ToInt(count) + if err != nil { + return nil, fmt.Errorf("multicall3 result count overflow: %w", err) + } + offsetsStart := base + 32 - offsetsEnd := offsetsStart + int(count)*32 + offsetsEnd := offsetsStart + countInt*32 if offsetsEnd > len(data) { return nil, errors.New("multicall3 result offsets out of bounds") } - results := make([]Multicall3Result, int(count)) - for i := 0; i < int(count); i++ { + results := make([]Multicall3Result, countInt) + for i := 0; i < countInt; i++ { offsetStart := offsetsStart + i*32 offsetVal, err := readUint256(data[offsetStart : offsetStart+32]) if err != nil { return nil, err } // Element offsets are relative to where the offset table starts (after length word) - elemStart := offsetsStart + int(offsetVal) + offsetValInt, err := safeUint64ToInt(offsetVal) + if err != nil { + return nil, fmt.Errorf("multicall3 result element offset overflow: %w", err) + } + elemStart := offsetsStart + offsetValInt if elemStart < offsetsStart || elemStart+64 > len(data) { return nil, errors.New("multicall3 result element out of bounds") } @@ -228,7 +250,11 @@ func DecodeMulticall3Aggregate3Result(data []byte) ([]Multicall3Result, error) { if err != nil { return nil, err } - bytesStart := elemStart + int(dataOffset) + dataOffsetInt, err := safeUint64ToInt(dataOffset) + if err != nil { + return nil, fmt.Errorf("multicall3 result data offset overflow: %w", err) + } + bytesStart := elemStart + dataOffsetInt if bytesStart < elemStart || bytesStart+32 > len(data) { return nil, errors.New("multicall3 result bytes offset out of bounds") } @@ -237,8 +263,12 @@ func DecodeMulticall3Aggregate3Result(data []byte) ([]Multicall3Result, error) { if err != nil { return nil, err } + dataLenInt, err := safeUint64ToInt(dataLen) + if err != nil { + return nil, fmt.Errorf("multicall3 result data length overflow: %w", err) + } dataStart := bytesStart + 32 - dataEnd := dataStart + int(dataLen) + dataEnd := dataStart + dataLenInt if dataStart < bytesStart || dataEnd > len(data) { return nil, errors.New("multicall3 result bytes length out of bounds") } @@ -280,7 +310,9 @@ func encodeAggregate3Array(calls []Multicall3Call) ([]byte, error) { offsetTableSize := abiWordSize * len(calls) elements := make([][]byte, len(calls)) offsets := make([]uint64, len(calls)) - cur := uint64(offsetTableSize) + // offsetTableSize is derived from len(calls) which is bounded by int, + // so this conversion to uint64 is safe (always non-negative). + cur := uint64(offsetTableSize) // #nosec G115 for i, call := range calls { elem := encodeAggregate3Element(call) @@ -289,7 +321,13 @@ func encodeAggregate3Array(calls []Multicall3Call) ([]byte, error) { cur += uint64(len(elem)) } - out := make([]byte, 0, int(cur)) + // cur accumulates sizes of in-memory slices, so it must fit in int. + // Add explicit check for safety. + capacity, err := safeUint64ToInt(cur) + if err != nil { + return nil, fmt.Errorf("multicall3 encoded data too large: %w", err) + } + out := make([]byte, 0, capacity) out = append(out, encodeUint64(uint64(len(calls)))...) for _, off := range offsets { out = append(out, encodeUint64(off)...) From 2dece38cfe15bb59bd7e5768de0312905edf274b Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 15 Jan 2026 15:38:13 +0100 Subject: [PATCH 08/53] fix: address PR review feedback for multicall3 aggregation - Changed Debug to Info logging for batch detection failures (visibility) - Made ShouldFallbackMulticall3 more specific to only fallback on contract unavailability errors (prevents wasteful retries) - Added Method() error handling instead of ignoring the error - Added MetricMulticall3AggregationTotal and MetricMulticall3FallbackTotal metrics for production monitoring with detailed reason labels - Added doc comment for detectEthCallBatchInfo explaining eligibility criteria - Made forwardEthCallBatchCandidates parallel using sync.WaitGroup (prevents latency regression) - Fixed UUID collision risk using atomic counter combined with timestamp - Added comprehensive doc comment for rate limit behavior in fallback paths - Added TestSafeUint64ToInt with overflow protection tests - Updated TestShouldFallbackMulticall3 with new behavior test cases Co-Authored-By: Claude Opus 4.5 --- architecture/evm/multicall3.go | 30 ++++++- architecture/evm/multicall3_test.go | 111 +++++++++++++++++++++++- erpc/http_batch_eth_call.go | 80 +++++++++++++---- erpc/http_batch_eth_call_handle_test.go | 3 +- erpc/http_server.go | 4 +- telemetry/metrics.go | 13 +++ 6 files changed, 218 insertions(+), 23 deletions(-) diff --git a/architecture/evm/multicall3.go b/architecture/evm/multicall3.go index f90709457..c5ae1331a 100644 --- a/architecture/evm/multicall3.go +++ b/architecture/evm/multicall3.go @@ -12,6 +12,7 @@ import ( "math" "math/big" "strings" + "sync/atomic" "time" "github.com/erpc/erpc/common" @@ -19,6 +20,9 @@ import ( "golang.org/x/crypto/sha3" ) +// multicall3RequestCounter provides unique IDs for multicall3 requests to prevent collisions +var multicall3RequestCounter uint64 + const multicall3Address = "0xcA11bde05977b3631167028862bE2a173976CA11" var ErrMulticall3BatchNotEligible = errors.New("multicall3 batch not eligible") @@ -177,7 +181,9 @@ func BuildMulticall3Request(requests []*common.NormalizedRequest, blockParam int } jrq := common.NewJsonRpcRequest("eth_call", []interface{}{callObj, blockParam}) - jrq.ID = fmt.Sprintf("multicall3-%d", time.Now().UnixNano()) + // Use atomic counter combined with timestamp for guaranteed unique IDs + counter := atomic.AddUint64(&multicall3RequestCounter, 1) + jrq.ID = fmt.Sprintf("multicall3-%d-%d", time.Now().UnixNano(), counter) nrq := common.NewNormalizedRequestFromJsonRpcRequest(jrq) nrq.CopyHttpContextFrom(requests[0]) @@ -283,11 +289,31 @@ func DecodeMulticall3Aggregate3Result(data []byte) ([]Multicall3Result, error) { return results, nil } +// ShouldFallbackMulticall3 determines if an error should trigger fallback to individual requests. +// Returns true only when the multicall3 contract is unavailable (unsupported endpoint) or when +// there are specific execution errors indicating the contract doesn't exist on this network. +// Other execution errors (like reverts) should NOT trigger fallback as they would also fail individually. func ShouldFallbackMulticall3(err error) bool { if err == nil { return false } - return common.HasErrorCode(err, common.ErrCodeEndpointExecutionException, common.ErrCodeEndpointUnsupported) + // Always fallback if endpoint is unsupported (e.g., network doesn't support eth_call) + if common.HasErrorCode(err, common.ErrCodeEndpointUnsupported) { + return true + } + // For execution errors, only fallback if it indicates multicall3 contract unavailability + if common.HasErrorCode(err, common.ErrCodeEndpointExecutionException) { + errStr := strings.ToLower(err.Error()) + // Check for indicators that the multicall3 contract doesn't exist + if strings.Contains(errStr, "contract not found") || + strings.Contains(errStr, "no code at address") || + strings.Contains(errStr, "execution reverted") { + return true + } + // Other execution errors (like authentication, rate limits, etc.) should not fallback + return false + } + return false } func encodeAggregate3Calls(calls []Multicall3Call) ([]byte, error) { diff --git a/architecture/evm/multicall3_test.go b/architecture/evm/multicall3_test.go index f1f1dda62..89f5a1d7b 100644 --- a/architecture/evm/multicall3_test.go +++ b/architecture/evm/multicall3_test.go @@ -3,6 +3,7 @@ package evm import ( "errors" "fmt" + "math" "strings" "testing" @@ -11,6 +12,59 @@ import ( "github.com/stretchr/testify/require" ) +func TestSafeUint64ToInt(t *testing.T) { + tests := []struct { + name string + input uint64 + want int + wantErr bool + }{ + { + name: "zero", + input: 0, + want: 0, + wantErr: false, + }, + { + name: "small positive", + input: 100, + want: 100, + wantErr: false, + }, + { + name: "max int boundary", + input: uint64(math.MaxInt), + want: math.MaxInt, + wantErr: false, + }, + { + name: "overflow - max int plus one", + input: uint64(math.MaxInt) + 1, + want: 0, + wantErr: true, + }, + { + name: "overflow - max uint64", + input: math.MaxUint64, + want: 0, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := safeUint64ToInt(tt.input) + if tt.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), "overflow") + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + func TestNormalizeBlockParam(t *testing.T) { cases := []struct { name string @@ -344,10 +398,59 @@ func TestDecodeMulticall3Aggregate3Result_Errors(t *testing.T) { } func TestShouldFallbackMulticall3(t *testing.T) { - assert.False(t, ShouldFallbackMulticall3(nil)) - assert.True(t, ShouldFallbackMulticall3(common.NewErrEndpointExecutionException(errors.New("boom")))) - assert.True(t, ShouldFallbackMulticall3(common.NewErrEndpointUnsupported(errors.New("boom")))) - assert.False(t, ShouldFallbackMulticall3(errors.New("nope"))) + tests := []struct { + name string + err error + want bool + }{ + { + name: "nil error", + err: nil, + want: false, + }, + { + name: "unsupported endpoint", + err: common.NewErrEndpointUnsupported(errors.New("boom")), + want: true, + }, + { + name: "execution exception with contract not found", + err: common.NewErrEndpointExecutionException(errors.New("contract not found")), + want: true, + }, + { + name: "execution exception with no code at address", + err: common.NewErrEndpointExecutionException(errors.New("no code at address 0x123")), + want: true, + }, + { + name: "execution exception with execution reverted", + err: common.NewErrEndpointExecutionException(errors.New("execution reverted")), + want: true, + }, + { + name: "execution exception with generic error - no fallback", + err: common.NewErrEndpointExecutionException(errors.New("some other error")), + want: false, + }, + { + name: "execution exception with rate limit - no fallback", + err: common.NewErrEndpointExecutionException(errors.New("rate limited")), + want: false, + }, + { + name: "non-execution error", + err: errors.New("nope"), + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ShouldFallbackMulticall3(tt.err) + assert.Equal(t, tt.want, got) + }) + } } func encodeAggregate3Results(results []Multicall3Result) []byte { diff --git a/erpc/http_batch_eth_call.go b/erpc/http_batch_eth_call.go index b70549423..17823be16 100644 --- a/erpc/http_batch_eth_call.go +++ b/erpc/http_batch_eth_call.go @@ -7,11 +7,13 @@ import ( "fmt" "net/http" "strings" + "sync" "time" "github.com/erpc/erpc/architecture/evm" "github.com/erpc/erpc/auth" "github.com/erpc/erpc/common" + "github.com/erpc/erpc/telemetry" "github.com/rs/zerolog" ) @@ -44,6 +46,14 @@ var ( newBatchJsonRpcResponse = common.NewJsonRpcResponse ) +// detectEthCallBatchInfo checks if a batch request is eligible for Multicall3 aggregation. +// Returns nil (no error) if the batch is not eligible due to: +// - Fewer than 2 requests in the batch +// - Any non-eth_call method in the batch +// - Requests targeting different networks +// - Requests targeting different block references +// - Non-EVM architecture +// Returns an error only for actual parsing/validation failures. func detectEthCallBatchInfo(requests []json.RawMessage, architecture, chainId string) (*ethCallBatchInfo, error) { if len(requests) < 2 { return nil, nil @@ -110,6 +120,12 @@ func detectEthCallBatchInfo(requests []json.RawMessage, architecture, chainId st }, nil } +// forwardEthCallBatchCandidates forwards individual eth_call requests in parallel as a fallback +// when Multicall3 aggregation fails or is not applicable. +// +// Rate limiting note: The network rate limit is skipped (withSkipNetworkRateLimit) because +// rate limits were already acquired for each request during the aggregation flow preparation. +// This prevents double-counting rate limits when falling back to individual requests. func (s *HttpServer) forwardEthCallBatchCandidates( startedAt *time.Time, project *PreparedProject, @@ -126,20 +142,27 @@ func (s *HttpServer) forwardEthCallBatchCandidates( return } + // Process candidates in parallel for better performance during fallback + var wg sync.WaitGroup for _, cand := range candidates { - resp, err := forwardBatchProject(withSkipNetworkRateLimit(cand.ctx), project, network, cand.req) - if err != nil { - if resp != nil { - go resp.Release() + wg.Add(1) + go func(c ethCallBatchCandidate) { + defer wg.Done() + resp, err := forwardBatchProject(withSkipNetworkRateLimit(c.ctx), project, network, c.req) + if err != nil { + if resp != nil { + go resp.Release() + } + responses[c.index] = processErrorBody(&c.logger, startedAt, c.req, err, s.serverCfg.IncludeErrorDetails) + common.EndRequestSpan(c.ctx, nil, err) + return } - responses[cand.index] = processErrorBody(&cand.logger, startedAt, cand.req, err, s.serverCfg.IncludeErrorDetails) - common.EndRequestSpan(cand.ctx, nil, err) - continue - } - responses[cand.index] = resp - common.EndRequestSpan(cand.ctx, resp, nil) + responses[c.index] = resp + common.EndRequestSpan(c.ctx, resp, nil) + }(cand) } + wg.Wait() } func (s *HttpServer) handleEthCallBatchAggregation( @@ -179,7 +202,12 @@ func (s *HttpServer) handleEthCallBatchAggregation( continue } - method, _ := nq.Method() + method, methodErr := nq.Method() + if methodErr != nil { + responses[i] = processErrorBody(&baseLogger, startedAt, nq, methodErr, &common.TRUE) + common.EndRequestSpan(requestCtx, nil, methodErr) + continue + } rlg := baseLogger.With().Str("method", method).Logger() ap, err := auth.NewPayloadFromHttp(method, r.RemoteAddr, headers, queryArgs) @@ -247,8 +275,14 @@ func (s *HttpServer) handleEthCallBatchAggregation( reqs[i] = cand.req } + projectId := "" + if project.Config != nil { + projectId = project.Config.Id + } + mcReq, calls, err := evm.BuildMulticall3Request(reqs, batchInfo.blockParam) if err != nil { + telemetry.MetricMulticall3FallbackTotal.WithLabelValues(projectId, batchInfo.networkId, "build_failed").Inc() baseLogger.Debug().Err(err). Int("candidateCount", len(candidates)). Str("networkId", batchInfo.networkId). @@ -264,6 +298,7 @@ func (s *HttpServer) handleEthCallBatchAggregation( mcResp.Release() } if evm.ShouldFallbackMulticall3(mcErr) { + telemetry.MetricMulticall3FallbackTotal.WithLabelValues(projectId, batchInfo.networkId, "forward_failed").Inc() baseLogger.Debug().Err(mcErr). Int("candidateCount", len(candidates)). Str("networkId", batchInfo.networkId). @@ -271,6 +306,8 @@ func (s *HttpServer) handleEthCallBatchAggregation( s.forwardEthCallBatchCandidates(startedAt, project, network, candidates, responses) return true } + // Non-recoverable error - don't fallback, propagate to all candidates + telemetry.MetricMulticall3AggregationTotal.WithLabelValues(projectId, batchInfo.networkId, "error").Inc() for _, cand := range candidates { responses[cand.index] = processErrorBody(&cand.logger, startedAt, cand.req, mcErr, s.serverCfg.IncludeErrorDetails) common.EndRequestSpan(cand.ctx, nil, mcErr) @@ -278,6 +315,7 @@ func (s *HttpServer) handleEthCallBatchAggregation( return true } if mcResp == nil { + telemetry.MetricMulticall3FallbackTotal.WithLabelValues(projectId, batchInfo.networkId, "nil_response").Inc() baseLogger.Debug(). Int("candidateCount", len(candidates)). Str("networkId", batchInfo.networkId). @@ -289,12 +327,17 @@ func (s *HttpServer) handleEthCallBatchAggregation( jrr, err := mcResp.JsonRpcResponse(mcCtx) if err != nil || jrr == nil || jrr.Error != nil { mcResp.Release() - baseLogger.Debug().Err(err). + telemetry.MetricMulticall3FallbackTotal.WithLabelValues(projectId, batchInfo.networkId, "invalid_response").Inc() + logEvent := baseLogger.Debug().Err(err). Int("candidateCount", len(candidates)). Str("networkId", batchInfo.networkId). Bool("missingResponse", jrr == nil). - Bool("hasRpcError", jrr != nil && jrr.Error != nil). - Msg("multicall3 response invalid, falling back") + Bool("hasRpcError", jrr != nil && jrr.Error != nil) + if jrr != nil && jrr.Error != nil { + logEvent = logEvent.Int("rpcErrorCode", jrr.Error.Code). + Str("rpcErrorMessage", jrr.Error.Message) + } + logEvent.Msg("multicall3 response invalid, falling back") s.forwardEthCallBatchCandidates(startedAt, project, network, candidates, responses) return true } @@ -302,6 +345,7 @@ func (s *HttpServer) handleEthCallBatchAggregation( var resultHex string if err := common.SonicCfg.Unmarshal(jrr.GetResultBytes(), &resultHex); err != nil { mcResp.Release() + telemetry.MetricMulticall3FallbackTotal.WithLabelValues(projectId, batchInfo.networkId, "unmarshal_failed").Inc() baseLogger.Debug().Err(err). Int("candidateCount", len(candidates)). Str("networkId", batchInfo.networkId). @@ -312,6 +356,7 @@ func (s *HttpServer) handleEthCallBatchAggregation( resultBytes, err := common.HexToBytes(resultHex) if err != nil { mcResp.Release() + telemetry.MetricMulticall3FallbackTotal.WithLabelValues(projectId, batchInfo.networkId, "hex_decode_failed").Inc() baseLogger.Debug().Err(err). Int("candidateCount", len(candidates)). Str("networkId", batchInfo.networkId). @@ -323,6 +368,7 @@ func (s *HttpServer) handleEthCallBatchAggregation( decoded, err := evm.DecodeMulticall3Aggregate3Result(resultBytes) if err != nil { mcResp.Release() + telemetry.MetricMulticall3FallbackTotal.WithLabelValues(projectId, batchInfo.networkId, "abi_decode_failed").Inc() baseLogger.Debug().Err(err). Int("candidateCount", len(candidates)). Str("networkId", batchInfo.networkId). @@ -332,12 +378,14 @@ func (s *HttpServer) handleEthCallBatchAggregation( } if len(decoded) != len(calls) { mcResp.Release() + // Length mismatch is a critical issue - use separate metric and Error level logging + telemetry.MetricMulticall3FallbackTotal.WithLabelValues(projectId, batchInfo.networkId, "length_mismatch").Inc() baseLogger.Error(). Int("candidateCount", len(candidates)). Int("decodedCount", len(decoded)). Int("callCount", len(calls)). Str("networkId", batchInfo.networkId). - Msg("multicall3 result length mismatch, falling back") + Msg("CRITICAL: multicall3 result length mismatch - possible data corruption, falling back") s.forwardEthCallBatchCandidates(startedAt, project, network, candidates, responses) return true } @@ -382,6 +430,8 @@ func (s *HttpServer) handleEthCallBatchAggregation( common.EndRequestSpan(cand.ctx, nil, callErr) } + // Aggregation succeeded + telemetry.MetricMulticall3AggregationTotal.WithLabelValues(projectId, batchInfo.networkId, "success").Inc() mcResp.Release() return true } diff --git a/erpc/http_batch_eth_call_handle_test.go b/erpc/http_batch_eth_call_handle_test.go index 5932317ec..14687f9fd 100644 --- a/erpc/http_batch_eth_call_handle_test.go +++ b/erpc/http_batch_eth_call_handle_test.go @@ -175,7 +175,8 @@ func TestHandleEthCallBatchAggregation_FallbackPaths(t *testing.T) { name: "forward error fallback", requests: validRequests, networkResponse: func() (*common.NormalizedResponse, error) { - return nil, common.NewErrEndpointExecutionException(errors.New("boom")) + // Use "execution reverted" to trigger ShouldFallbackMulticall3 + return nil, common.NewErrEndpointExecutionException(errors.New("execution reverted")) }, expectedProjCalls: 2, expectedNetCalls: 1, diff --git a/erpc/http_server.go b/erpc/http_server.go index 022756461..abe6dbf11 100644 --- a/erpc/http_server.go +++ b/erpc/http_server.go @@ -409,7 +409,9 @@ func (s *HttpServer) createRequestHandler() http.Handler { if isBatch && !isAdmin && !isHealthCheck { batchInfo, detectErr := detectEthCallBatchInfo(requests, architecture, chainId) if detectErr != nil { - lg.Debug().Err(detectErr).Msg("eth_call batch detection failed") + lg.Info().Err(detectErr). + Int("requestCount", len(requests)). + Msg("eth_call batch detection failed, processing individually") } if batchInfo != nil && isMulticall3AggregationEnabled(project, batchInfo.networkId) { batchHandled = s.handleEthCallBatchAggregation( diff --git a/telemetry/metrics.go b/telemetry/metrics.go index 430453253..150ec9c3d 100644 --- a/telemetry/metrics.go +++ b/telemetry/metrics.go @@ -410,6 +410,19 @@ var ( Help: "eth_getLogs requested block-range sizes.", Buckets: EvmGetLogsRangeHistogramBuckets, }, []string{"project", "network", "category", "user", "finality"}) + + // Multicall3 aggregation metrics + MetricMulticall3AggregationTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "multicall3_aggregation_total", + Help: "Total number of multicall3 aggregation attempts.", + }, []string{"project", "network", "outcome"}) + + MetricMulticall3FallbackTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "multicall3_fallback_total", + Help: "Total number of multicall3 fallbacks to individual requests.", + }, []string{"project", "network", "reason"}) ) var DefaultHistogramBuckets = []float64{ From 151c0caa297cb915a5f8a12338b1713fcaeba071 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 15 Jan 2026 15:40:04 +0100 Subject: [PATCH 09/53] fix: prevent integer overflow in multicall3 result count multiplication Add bounds check before countInt*32 multiplication to prevent integer overflow when crafted multicall response has huge count value. The check validates countInt <= (len(data)-offsetsStart)/32 before multiplication, preventing both overflow and out-of-bounds allocation. Added test case for "count exceeds available data" scenario. Co-Authored-By: Claude Opus 4.5 --- architecture/evm/multicall3.go | 6 ++++++ architecture/evm/multicall3_test.go | 11 +++++++++++ 2 files changed, 17 insertions(+) diff --git a/architecture/evm/multicall3.go b/architecture/evm/multicall3.go index c5ae1331a..9b664d596 100644 --- a/architecture/evm/multicall3.go +++ b/architecture/evm/multicall3.go @@ -225,6 +225,12 @@ func DecodeMulticall3Aggregate3Result(data []byte) ([]Multicall3Result, error) { } offsetsStart := base + 32 + // Check bounds before multiplication to prevent integer overflow + // Each element needs 32 bytes in the offset table + maxElements := (len(data) - offsetsStart) / 32 + if countInt > maxElements { + return nil, errors.New("multicall3 result count exceeds available data") + } offsetsEnd := offsetsStart + countInt*32 if offsetsEnd > len(data) { return nil, errors.New("multicall3 result offsets out of bounds") diff --git a/architecture/evm/multicall3_test.go b/architecture/evm/multicall3_test.go index 89f5a1d7b..ccfcb46bf 100644 --- a/architecture/evm/multicall3_test.go +++ b/architecture/evm/multicall3_test.go @@ -364,6 +364,17 @@ func TestDecodeMulticall3Aggregate3Result_Errors(t *testing.T) { name: "offsets out of bounds", data: append(encodeUint64(32), encodeUint64(2)...), }, + { + name: "count exceeds available data", + // Create data with huge count that would overflow if multiplied by 32 + // offset=32, then count=maxInt/16 (which when *32 would overflow) + data: func() []byte { + d := make([]byte, 96) + copy(d[0:32], encodeUint64(32)) // offset to array + copy(d[32:64], encodeUint64(0x7FFFFFFF)) // huge count that exceeds data + return d + }(), + }, { name: "element out of bounds", data: buildAggregate3ResultWithOffset(96, nil, nil), From 765788e1887257b66f7f7deedb1f6c357e34dfab Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 15 Jan 2026 15:41:16 +0100 Subject: [PATCH 10/53] fix: record MetricNetworkRequestsReceived for aggregated batch requests Each individual request in a batch should be counted in MetricNetworkRequestsReceived for accurate billing/analytics, even when aggregated into a single multicall3 request. This ensures metrics are consistent with the number of requests clients actually send. Co-Authored-By: Claude Opus 4.5 --- erpc/http_batch_eth_call.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/erpc/http_batch_eth_call.go b/erpc/http_batch_eth_call.go index 17823be16..989c80fb2 100644 --- a/erpc/http_batch_eth_call.go +++ b/erpc/http_batch_eth_call.go @@ -254,6 +254,13 @@ func (s *HttpServer) handleEthCallBatchAggregation( continue } + // Record per-request metric for billing/analytics + // This ensures aggregated requests are counted individually + reqFinality := nq.Finality(requestCtx) + telemetry.CounterHandle(telemetry.MetricNetworkRequestsReceived, + project.Config.Id, network.Label(), method, reqFinality.String(), nq.UserId(), nq.AgentName(), + ).Inc() + candidates = append(candidates, ethCallBatchCandidate{ index: i, ctx: requestCtx, From 2a9a58894a4f87b3a43db28e5dd1933faa242ea6 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 15 Jan 2026 15:49:12 +0100 Subject: [PATCH 11/53] fix: widen fallback error pattern matching for multicall3 Expand the error string patterns that trigger fallback to individual requests. Different RPC providers report missing contracts with varied error messages, so we now match: - contract not found - no code at address - execution reverted - code is empty - not a contract - invalid opcode - missing trie node - does not exist - account not found This reduces false negatives where providers with non-standard error messages would fail batch requests instead of falling back gracefully. Co-Authored-By: Claude Opus 4.5 --- architecture/evm/multicall3.go | 22 +++++++++++++++++----- architecture/evm/multicall3_test.go | 15 +++++++++++++++ 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/architecture/evm/multicall3.go b/architecture/evm/multicall3.go index 9b664d596..f5faee1c4 100644 --- a/architecture/evm/multicall3.go +++ b/architecture/evm/multicall3.go @@ -310,11 +310,23 @@ func ShouldFallbackMulticall3(err error) bool { // For execution errors, only fallback if it indicates multicall3 contract unavailability if common.HasErrorCode(err, common.ErrCodeEndpointExecutionException) { errStr := strings.ToLower(err.Error()) - // Check for indicators that the multicall3 contract doesn't exist - if strings.Contains(errStr, "contract not found") || - strings.Contains(errStr, "no code at address") || - strings.Contains(errStr, "execution reverted") { - return true + // Check for indicators that the multicall3 contract doesn't exist. + // Different providers use different error messages, so we match multiple patterns. + contractUnavailablePatterns := []string{ + "contract not found", + "no code at address", + "execution reverted", // empty revert from non-existent contract + "code is empty", + "not a contract", + "invalid opcode", // can indicate missing contract + "missing trie node", // pre-deployment block query + "does not exist", + "account not found", + } + for _, pattern := range contractUnavailablePatterns { + if strings.Contains(errStr, pattern) { + return true + } } // Other execution errors (like authentication, rate limits, etc.) should not fallback return false diff --git a/architecture/evm/multicall3_test.go b/architecture/evm/multicall3_test.go index ccfcb46bf..d7e31dad3 100644 --- a/architecture/evm/multicall3_test.go +++ b/architecture/evm/multicall3_test.go @@ -439,6 +439,21 @@ func TestShouldFallbackMulticall3(t *testing.T) { err: common.NewErrEndpointExecutionException(errors.New("execution reverted")), want: true, }, + { + name: "execution exception with code is empty", + err: common.NewErrEndpointExecutionException(errors.New("code is empty at 0xCA11bde05977b3631167028862bE2a173976CA11")), + want: true, + }, + { + name: "execution exception with missing trie node", + err: common.NewErrEndpointExecutionException(errors.New("missing trie node abc123")), + want: true, + }, + { + name: "execution exception with account not found", + err: common.NewErrEndpointExecutionException(errors.New("account not found")), + want: true, + }, { name: "execution exception with generic error - no fallback", err: common.NewErrEndpointExecutionException(errors.New("some other error")), From 6d570d4da100f43c2de6493a46019697ba58f9c8 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 15 Jan 2026 16:07:09 +0100 Subject: [PATCH 12/53] feat: add per-call caching for multicall3 aggregated batches This commit adds intelligent caching for individual eth_call requests within multicall3 batch aggregation: 1. Pre-aggregation cache check: - Before building multicall3, check cache for each individual request - Serve cached responses directly without including in the aggregation - Only include uncached requests in the multicall3 call - If all requests are cached, skip upstream call entirely 2. Post-aggregation cache write: - Cache each successful individual response asynchronously - Enables future cache hits for the same calls 3. New metrics: - MetricMulticall3CacheHitsTotal: tracks per-call cache hits - New "all_cached" outcome for aggregation when all served from cache 4. New Network methods: - AppCtx(): exposes application context for async cache operations - CacheDal(): exposes cache DAL for external use This optimization significantly reduces upstream calls when batches contain previously-cached requests. Co-Authored-By: Claude Opus 4.5 --- erpc/http_batch_eth_call.go | 83 ++++++++++++++++++++++++++++++++++--- erpc/networks.go | 8 ++++ telemetry/metrics.go | 6 +++ 3 files changed, 92 insertions(+), 5 deletions(-) diff --git a/erpc/http_batch_eth_call.go b/erpc/http_batch_eth_call.go index 989c80fb2..f02201f1a 100644 --- a/erpc/http_batch_eth_call.go +++ b/erpc/http_batch_eth_call.go @@ -4,8 +4,10 @@ import ( "context" "encoding/hex" "encoding/json" + "errors" "fmt" "net/http" + "runtime/debug" "strings" "sync" "time" @@ -15,6 +17,7 @@ import ( "github.com/erpc/erpc/common" "github.com/erpc/erpc/telemetry" "github.com/rs/zerolog" + "go.opentelemetry.io/otel/trace" ) type ethCallBatchInfo struct { @@ -239,6 +242,7 @@ func (s *HttpServer) handleEthCallBatchAggregation( } nq.SetNetwork(network) + nq.SetCacheDal(network.CacheDal()) nq.ApplyDirectiveDefaults(network.Config().DirectiveDefaults) nq.EnrichFromHttp(headers, queryArgs, uaMode) rlg.Trace().Interface("directives", nq.Directives()).Msgf("applied request directives") @@ -272,6 +276,49 @@ func (s *HttpServer) handleEthCallBatchAggregation( if len(candidates) == 0 { return true } + + projectId := "" + if project.Config != nil { + projectId = project.Config.Id + } + + // Check cache for individual requests before aggregating + cacheDal := network.CacheDal() + var uncachedCandidates []ethCallBatchCandidate + if cacheDal != nil && !cacheDal.IsObjectNull() { + uncachedCandidates = make([]ethCallBatchCandidate, 0, len(candidates)) + cacheHits := 0 + for _, cand := range candidates { + cachedResp, err := cacheDal.Get(cand.ctx, cand.req) + if err == nil && cachedResp != nil && !cachedResp.IsObjectNull(cand.ctx) { + // Cache hit - use cached response directly + cachedResp.SetFromCache(true) + responses[cand.index] = cachedResp + common.EndRequestSpan(cand.ctx, cachedResp, nil) + cacheHits++ + continue + } + // Cache miss - needs to be fetched + uncachedCandidates = append(uncachedCandidates, cand) + } + if cacheHits > 0 { + telemetry.MetricMulticall3CacheHitsTotal.WithLabelValues(projectId, batchInfo.networkId).Add(float64(cacheHits)) + baseLogger.Debug(). + Int("cacheHits", cacheHits). + Int("uncached", len(uncachedCandidates)). + Int("total", len(candidates)). + Str("networkId", batchInfo.networkId). + Msg("multicall3 pre-aggregation cache check") + } + candidates = uncachedCandidates + } + + // All requests served from cache + if len(candidates) == 0 { + telemetry.MetricMulticall3AggregationTotal.WithLabelValues(projectId, batchInfo.networkId, "all_cached").Inc() + return true + } + if len(candidates) < 2 { s.forwardEthCallBatchCandidates(startedAt, project, network, candidates, responses) return true @@ -282,11 +329,6 @@ func (s *HttpServer) handleEthCallBatchAggregation( reqs[i] = cand.req } - projectId := "" - if project.Config != nil { - projectId = project.Config.Id - } - mcReq, calls, err := evm.BuildMulticall3Request(reqs, batchInfo.blockParam) if err != nil { telemetry.MetricMulticall3FallbackTotal.WithLabelValues(projectId, batchInfo.networkId, "build_failed").Inc() @@ -397,6 +439,7 @@ func (s *HttpServer) handleEthCallBatchAggregation( return true } + shouldCache := !mcResp.FromCache() && cacheDal != nil && !cacheDal.IsObjectNull() for i, result := range decoded { cand := candidates[i] if result.Success { @@ -418,6 +461,36 @@ func (s *HttpServer) handleEthCallBatchAggregation( nr.SetEvmBlockNumber(mcResp.EvmBlockNumber()) responses[cand.index] = nr common.EndRequestSpan(cand.ctx, nr, nil) + + // Cache individual response asynchronously + if shouldCache { + nr.RLockWithTrace(cand.ctx) + nr.AddRef() + go func(resp *common.NormalizedResponse, req *common.NormalizedRequest, reqCtx context.Context, lg zerolog.Logger) { + defer func() { + if rec := recover(); rec != nil { + telemetry.MetricUnexpectedPanicTotal.WithLabelValues( + "multicall3-cache-set", + fmt.Sprintf("network:%s", batchInfo.networkId), + common.ErrorFingerprint(rec), + ).Inc() + lg.Error(). + Interface("panic", rec). + Str("stack", string(debug.Stack())). + Msg("unexpected panic on multicall3 per-call cache-set") + } + }() + defer resp.RUnlock() + defer resp.DoneRef() + + timeoutCtx, timeoutCtxCancel := context.WithTimeoutCause(network.AppCtx(), 10*time.Second, errors.New("cache driver timeout during multicall3 per-call set")) + defer timeoutCtxCancel() + tracedCtx := trace.ContextWithSpanContext(timeoutCtx, trace.SpanContextFromContext(reqCtx)) + if err := cacheDal.Set(tracedCtx, req, resp); err != nil { + lg.Warn().Err(err).Msg("could not store multicall3 per-call response in cache") + } + }(nr, cand.req, cand.ctx, cand.logger) + } continue } diff --git a/erpc/networks.go b/erpc/networks.go index a848248c8..c6df22923 100644 --- a/erpc/networks.go +++ b/erpc/networks.go @@ -891,6 +891,14 @@ func (n *Network) Config() *common.NetworkConfig { return n.cfg } +func (n *Network) CacheDal() common.CacheDAL { + return n.cacheDal +} + +func (n *Network) AppCtx() context.Context { + return n.appCtx +} + func (n *Network) GetFinality(ctx context.Context, req *common.NormalizedRequest, resp *common.NormalizedResponse) common.DataFinalityState { ctx, span := common.StartDetailSpan(ctx, "Network.GetFinality") defer span.End() diff --git a/telemetry/metrics.go b/telemetry/metrics.go index 150ec9c3d..d77758163 100644 --- a/telemetry/metrics.go +++ b/telemetry/metrics.go @@ -423,6 +423,12 @@ var ( Name: "multicall3_fallback_total", Help: "Total number of multicall3 fallbacks to individual requests.", }, []string{"project", "network", "reason"}) + + MetricMulticall3CacheHitsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "multicall3_cache_hits_total", + Help: "Total number of per-call cache hits in multicall3 batch aggregation.", + }, []string{"project", "network"}) ) var DefaultHistogramBuckets = []float64{ From 7bb01d2ce371e090eeb208fda5bad95b56b43ba2 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 15 Jan 2026 16:11:37 +0100 Subject: [PATCH 13/53] test: add tests for multicall3 per-call caching Add comprehensive tests for the per-call caching behavior: 1. all_cached - no multicall3 call: - Verifies that when all requests are cached, no multicall3 call is made to the upstream 2. mixed cached and uncached: - Verifies that cached responses are returned directly - Uncached requests fall back to individual forwarding (since only 1 uncached request remains) 3. cache write after successful multicall3: - Verifies that successful multicall3 responses trigger async cache writes for each individual response Co-Authored-By: Claude Opus 4.5 --- erpc/http_batch_eth_call_handle_test.go | 141 +++++++++++++++++++++++ erpc/http_batch_eth_call_helpers_test.go | 19 +++ 2 files changed, 160 insertions(+) diff --git a/erpc/http_batch_eth_call_handle_test.go b/erpc/http_batch_eth_call_handle_test.go index 14687f9fd..8ef6ebcd5 100644 --- a/erpc/http_batch_eth_call_handle_test.go +++ b/erpc/http_batch_eth_call_handle_test.go @@ -14,6 +14,7 @@ import ( "github.com/erpc/erpc/common" "github.com/rs/zerolog/log" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -363,3 +364,143 @@ func TestHandleEthCallBatchAggregation_SuccessAndFailureResults(t *testing.T) { require.True(t, ok) assert.Equal(t, "0x"+hex.EncodeToString(results[1].ReturnData), errMap["data"]) } + +func TestHandleEthCallBatchAggregation_CacheHits(t *testing.T) { + t.Run("all cached - no multicall3 call", func(t *testing.T) { + cfg := baseBatchConfig() + server, project, network, ctx, cleanup := setupBatchHandlerWithCache(t, cfg) + defer cleanup() + + // Set up mock cache + mockCache := &common.MockCacheDal{} + network.cacheDal = mockCache + + // Create cached responses + cachedResp1 := createCachedResponse(t, common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_call","params":[{"to":"0x0000000000000000000000000000000000000001","data":"0x"},"latest"]}`)), "0xcached1") + cachedResp2 := createCachedResponse(t, common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":2,"method":"eth_call","params":[{"to":"0x0000000000000000000000000000000000000001","data":"0x"},"latest"]}`)), "0xcached2") + + // Mock cache hits for both requests + mockCache.On("Get", mock.Anything, mock.Anything).Return(cachedResp1, nil).Once() + mockCache.On("Get", mock.Anything, mock.Anything).Return(cachedResp2, nil).Once() + + // Track if multicall3 was called (it shouldn't be) + multicall3Called := false + withBatchStubs(t, + func(ctx context.Context, network *Network, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + multicall3Called = true + return nil, errors.New("should not be called") + }, + nil, + nil, + ) + + handled, responses := runHandle(t, ctx, server, project, defaultBatchInfo(), validBatchRequests(t), nil) + require.True(t, handled) + require.False(t, multicall3Called, "multicall3 should not be called when all requests are cached") + require.Len(t, responses, 2) + + // Verify both responses are from cache + for i, resp := range responses { + nr, ok := resp.(*common.NormalizedResponse) + require.True(t, ok, "response %d should be NormalizedResponse", i) + assert.True(t, nr.FromCache(), "response %d should be from cache", i) + } + + mockCache.AssertExpectations(t) + }) + + t.Run("mixed cached and uncached", func(t *testing.T) { + cfg := baseBatchConfig() + server, project, network, ctx, cleanup := setupBatchHandlerWithCache(t, cfg) + defer cleanup() + + // Set up mock cache - first request cached, second not + mockCache := &common.MockCacheDal{} + network.cacheDal = mockCache + + // Create cached response for first request + cachedResp := createCachedResponse(t, common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_call","params":[{"to":"0x0000000000000000000000000000000000000001","data":"0x"},"latest"]}`)), "0xcached_first") + + // First request - cache hit, second request - cache miss + mockCache.On("Get", mock.Anything, mock.Anything).Return(cachedResp, nil).Once() + mockCache.On("Get", mock.Anything, mock.Anything).Return(nil, nil).Once() + + // Cache set should be called for the uncached request + mockCache.On("Set", mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() + + // Only the second (uncached) request should go through multicall3 + // But since we only have 1 uncached request, it falls back to individual forwarding + withBatchStubs(t, + nil, + func(ctx context.Context, project *PreparedProject, network *Network, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + // This is the fallback for single uncached request + return fallbackResponse(t, req), nil + }, + nil, + ) + + handled, responses := runHandle(t, ctx, server, project, defaultBatchInfo(), validBatchRequests(t), nil) + require.True(t, handled) + require.Len(t, responses, 2) + + // First response should be from cache + resp0, ok := responses[0].(*common.NormalizedResponse) + require.True(t, ok) + assert.True(t, resp0.FromCache()) + + // Second response should be from fallback (not cache) + _, ok = responses[1].(*common.NormalizedResponse) + require.True(t, ok) + + mockCache.AssertExpectations(t) + }) + + t.Run("cache write after successful multicall3", func(t *testing.T) { + cfg := baseBatchConfig() + server, project, network, ctx, cleanup := setupBatchHandlerWithCache(t, cfg) + defer cleanup() + + // Set up mock cache - all cache misses + mockCache := &common.MockCacheDal{} + network.cacheDal = mockCache + + mockCache.On("Get", mock.Anything, mock.Anything).Return(nil, nil).Times(2) + + // Expect cache Set to be called for each successful response + setCalled := make(chan struct{}, 2) + mockCache.On("Set", mock.Anything, mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { + setCalled <- struct{}{} + }).Times(2) + + results := []evm.Multicall3Result{ + {Success: true, ReturnData: []byte{0xaa}}, + {Success: true, ReturnData: []byte{0xbb}}, + } + resultHex := encodeAggregate3Results(results) + + withBatchStubs(t, + func(ctx context.Context, network *Network, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + jrr := mustJsonRpcResponse(t, 1, resultHex, nil) + return common.NewNormalizedResponse().WithJsonRpcResponse(jrr), nil + }, + nil, + nil, + ) + + handled, responses := runHandle(t, ctx, server, project, defaultBatchInfo(), validBatchRequests(t), nil) + require.True(t, handled) + require.Len(t, responses, 2) + + // Wait for async cache writes (with timeout) + for i := 0; i < 2; i++ { + select { + case <-setCalled: + // Good, cache set was called + case <-time.After(2 * time.Second): + t.Fatal("timeout waiting for cache Set to be called") + } + } + + mockCache.AssertExpectations(t) + }) +} diff --git a/erpc/http_batch_eth_call_helpers_test.go b/erpc/http_batch_eth_call_helpers_test.go index b8d8b240b..11a739109 100644 --- a/erpc/http_batch_eth_call_helpers_test.go +++ b/erpc/http_batch_eth_call_helpers_test.go @@ -273,3 +273,22 @@ func encodeBytes(data []byte) []byte { } return out } + +func setupBatchHandlerWithCache(t *testing.T, cfg *common.Config) (*HttpServer, *PreparedProject, *Network, context.Context, func()) { + t.Helper() + server, project, ctx, cleanup := setupBatchHandler(t, cfg) + + network, err := project.GetNetwork(ctx, "evm:123") + require.NoError(t, err) + + return server, project, network, ctx, cleanup +} + +func createCachedResponse(t *testing.T, req *common.NormalizedRequest, result string) *common.NormalizedResponse { + t.Helper() + jrr, err := common.NewJsonRpcResponse(req.ID(), result, nil) + require.NoError(t, err) + resp := common.NewNormalizedResponse().WithRequest(req).WithJsonRpcResponse(jrr) + resp.SetFromCache(true) + return resp +} From baa23f4296f97f674facaa83928d2b1ead4814dc Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 15 Jan 2026 16:57:54 +0100 Subject: [PATCH 14/53] docs: refine multicall3 batching design --- docs/design/multicall3-batching.md | 263 +++++++++++++++++++++++++++++ 1 file changed, 263 insertions(+) create mode 100644 docs/design/multicall3-batching.md diff --git a/docs/design/multicall3-batching.md b/docs/design/multicall3-batching.md new file mode 100644 index 000000000..1cc2d399a --- /dev/null +++ b/docs/design/multicall3-batching.md @@ -0,0 +1,263 @@ +# Multicall3 Batching (Network-Level) + +Status: Proposed + +## Context +The current Multicall3 batching lives in the HTTP batch handler. It only applies +to JSON-RPC batch requests and is tightly coupled to the HTTP layer, which is +not ideal for an EVM-only feature. The goal is to move batching to a deeper +layer so it can batch any incoming `eth_call` (batch or single), while keeping +network-level behaviors (cache, failover, circuit breakers). + +## Goals +- Batch `eth_call` across all entrypoints (HTTP batch + single requests + gRPC). +- Preserve network-level behaviors (cache, failover, upstream selection). +- Preserve per-request rate limits and per-request metrics. +- Maintain per-call cache writes and reuse existing Multicall3 encode/decode. +- Keep batching opt-in and configurable per network. + +## Non-Goals +- Batching non-EVM methods. +- Supporting `eth_call` fields beyond `to` + `data|input`. +- Enforcing upstream-specific caps in v1 (network-level caps only). + +## Placement +Preferred: a pre-cache hook in `Network.Forward` (or in `PreparedProject.Forward` +right before `p.doForward`). This keeps batching entrypoint-agnostic while still +using `Network.Forward` for caching and failover. + +Avoid: `PreUpstream` batching, which is too late (upstream-selected) and makes +cross-request aggregation difficult. + +## Eligibility and Block Reference Normalization +A request is eligible if: +- Method is `eth_call`. +- Call object contains only `to` and `data|input`. +- Request is not already a multicall (recursion guard). +- Calls with any of `from`, `gas`, `gasPrice`, `maxFeePerGas`, + `maxPriorityFeePerGas`, `value`, or a state override (third param) are + ineligible. + +Block reference normalization: +- Use `NormalizeBlockParam` on the `eth_call` block parameter. +- `nil` becomes `latest`. +- `0x` block numbers are normalized to decimal strings. +- Block hash (`0x` + 32 bytes) stays hex. +- Object params use `blockHash`, `blockNumber`, or `blockTag`. +- Known tags (`latest`, `finalized`, `safe`, `earliest`, `pending`) are + lower-cased for keying to avoid duplicates. + +Batching with tags: +- Default: allow `latest`, `finalized`, `safe`, `earliest`. +- `pending` is disabled by default (configurable). +- We do not resolve `latest` to a specific block number; all calls share the + same tag, and the execution block is the one seen by the upstream at call time. + +## Batching Key (User Isolation + Directive Key) +Key fields: +- `projectId` +- `networkId` +- `blockRef` +- `directivesKey` +- `userId` if `allowCrossUserBatching` is false + +Directives key uses a stable, versioned subset: +- `UseUpstream` +- `SkipInterpolation` +- `RetryEmpty` +- `RetryPending` +- `SkipCacheRead` (optional; can be per-request, but include for clarity) + +Any new directive must be explicitly added to the subset or ignored. The +directives key version is defined in code (not config) to avoid cross-node +mismatches; a version bump should be part of a release note. + +## Deduplication (Within Batch) +To avoid TOCTOU cache misses and duplicated calls: +- Maintain a `callKey` map inside each batch. +- `callKey` uses the same derivation as the cache key (method + params). +- Directives are already part of the batch key, so differing directives never + share a batch or a `callKey`. +- Multiple identical requests share one multicall slot and fan out results + to all waiters. + +## Batching Window and Deadlines +Configurable timing: +- `windowMs`: max wait time for a batch. +- `minWaitMs`: minimum wait to allow other requests to join. +- `safetyMarginMs`: subtracted from the earliest request deadline. +- `onlyIfPending`: no extra latency unless a batch is already open. + +Deadline-aware flush rules: +- If `deadline <= now + minWaitMs`, bypass batching and forward individually. +- Otherwise, `flushTime = min(flushTime, deadline - safetyMarginMs)`. +- Clamp `flushTime` to at least `now + minWaitMs`. +- If `flushTime <= now`, flush immediately. + +Concurrent flush behavior: +- If a batch for a key is already flushing, a new request for the same key + starts a new batch (next window) rather than joining the in-flight batch. + +## Caps and Backpressure +Network-level caps: +- `maxCalls` +- `maxCalldataBytes` +- `maxQueueSize` (global or per-key) +- `maxPendingBatches` + +Behavior on overflow: +- Prefer bypassing batching (forward individually) and increment a metric. +- Avoid unbounded memory growth. +- `maxQueueSize` is the total enqueued requests across all batches; `maxPendingBatches` + is the number of distinct batch keys. If either limit would be exceeded, bypass + batching for that request. + +## Forwarding, Fallback, and Partial Failures +1. Acquire per-request project + network rate limits. +2. Build Multicall3 request, mark as composite + (e.g., `CompositeTypeMulticall3`) and set `skipNetworkRateLimit` context + (via `withSkipNetworkRateLimit`). +3. Forward via `Network.Forward`. Fallback forwarding also uses + `skipNetworkRateLimit` to avoid double-counting. + +Fallback criteria: +- Multicall request error with `ShouldFallbackMulticall3` (unsupported endpoint, + missing contract, known provider patterns). +- Invalid or unusable response (RPC error, invalid hex, ABI decode failure, + length mismatch). +Pattern matching is implementation-defined; see `architecture/evm/multicall3.go`. + +Partial failures: +- If Multicall succeeds but an inner call returns `success=false`, return a + per-call `execution reverted` error. Do not fallback. +- If Multicall succeeds and results count matches, map directly, no fallback. + +If fallback fails, propagate a single infrastructure error to all requests. + +## Error Propagation Semantics +Per-call revert: +- Return a standard `execution reverted` error with `data` field. + +Infrastructure failure: +- All requests receive the same error. +- Include metadata such as `{ "multicall3": true, "stage": "decode", "reason": "..." }` + to distinguish infra failures from per-call failures. + +## Cancellation Handling +- If a request is canceled before flush, remove it from the batch and return + a context error for that request only. +- Cancellation does not affect other requests in the batch. +- Rate limit permits are not released (standard behavior). This keeps rate + limiting conservative and avoids races; a future enhancement could reclaim + permits for requests canceled before a flush. + +## Upstream Capability Handling +Some upstreams may not support Multicall3 or have stricter limits. +v1 decision: rely on `ShouldFallbackMulticall3` to detect unsupported upstreams +and fallback to individual calls. A future v2 can add explicit upstream +capability flags or filtering to avoid first-failure latency. + +## Cache Behavior +Pre-batch cache lookup: +- If `SkipCacheRead` is set, bypass per-request cache lookup. +- Cached responses are returned immediately and removed from the batch. +- For deduped requests, a single cached response satisfies all waiters. + +Per-call cache writes: +- On multicall success and cache eligible, write each call as if it were a + standalone `eth_call`. +- Cache key is identical to a standalone request (method + params), which + matches `callKey` derivation for dedupe. + +## Observability (Metrics + Tracing) +Metrics: +- `multicall3_aggregation_total{outcome}` +- `multicall3_fallback_total{reason}` +- `multicall3_cache_hits_total` +- Optional: `multicall3_batch_size`, `multicall3_batch_wait_ms`, + `multicall3_queue_len`, `multicall3_queue_overflow_total` + +Aggregation outcome values: +- `success` +- `all_cached` +- `fallback` +- `error` +- `bypassed` (ineligible, deadline-too-tight, or backpressure) + +Tracing: +- Add spans for `Batcher.Enqueue`, `Batcher.Wait`, `Batcher.Flush`. +- Link request spans to the multicall span; if batch size is small, also link + the multicall span back to request spans. Always attach a shared `batch.id` + attribute to both sides for correlation. + +## Config Proposal +Extend `EvmNetworkConfig`: + +```yaml +evm: + multicall3Aggregation: + enabled: true + windowMs: 25 + minWaitMs: 2 + safetyMarginMs: 2 + onlyIfPending: false + maxCalls: 20 + maxCalldataBytes: 64000 + maxQueueSize: 1000 + maxPendingBatches: 200 + cachePerCall: true + allowCrossUserBatching: true + allowPendingTagBatching: false +``` + +Validation and defaults: +- `windowMs > 0`, `minWaitMs >= 0`, `minWaitMs <= windowMs` +- `safetyMarginMs` defaults to `min(2, minWaitMs)` when omitted + (note: if `minWaitMs=0`, this yields `0`; operators should set a non-zero + safety margin if they expect tight deadlines) +- `maxCalls > 1`, `maxCalldataBytes > 0`, `maxQueueSize > 0` +- If invalid, disable batching for that network and log a warning. + +Maintain backward compatibility with existing `multicall3Aggregation: true|false`. + +## Algorithm Sketch +``` +Enqueue(req): + if not eligible or deadline too tight: return notHandled + if SkipCacheRead == false: + if cache hit: return cached response + if batch for key is flushing: create a new batch for key + add to batch key + if callKey duplicate: attach waiter and wait for result + adjust batch flush time (deadline-aware) + if caps reached: flush now + wait for flush or immediate join result + +Flush(batch): + build multicall req (mark as composite, skip rate limit) + forward via Network.Forward + if success: map responses + per-call cache writes + else if fallback-eligible: forward individually + else: propagate infra error to all +``` + +## Testing Plan +- Unit: eligibility, normalization, keying, caps, deadline-aware flush. +- Concurrency: batcher with multiple goroutines, cancellations, timeouts. +- Concurrency: request arrives during flush for same batch key (new batch). +- Integration: mixed batch/single calls, cache hits, fallback paths. +- Rate limit: ensure per-request budgets are only consumed once. +- Recursion: multicall request is never re-batched. +- Chaos: inject upstream errors, invalid responses, length mismatches. + +## Migration Plan +1. Implement batching in the network layer behind config. +2. Keep HTTP-layer batching behind a kill switch or remove once stable. +3. Enable on a subset of networks, monitor metrics, tune caps. + +## Open Questions +- Should `SkipCacheRead` live in the key (simpler, more fragmentation) or be + handled per-request (more complex, better batching)? +- Should batching of `pending` tag be allowed by default? +- Should upstream-specific caps be introduced in v2? +- Should permits be reclaimed for requests canceled before flush? From b6d432c969127f06659a25ec87ea100e950e2cd6 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 15 Jan 2026 17:16:12 +0100 Subject: [PATCH 15/53] feat: add Multicall3AggregationConfig for network-level batching Extends the evm.multicall3Aggregation config from a simple boolean to a full configuration object with: - windowMs, minWaitMs, safetyMarginMs for timing - maxCalls, maxCalldataBytes for size limits - maxQueueSize, maxPendingBatches for backpressure - cachePerCall, allowCrossUserBatching, allowPendingTagBatching flags Maintains backward compatibility with existing boolean configs via custom UnmarshalYAML that accepts both `true/false` and full object. Co-Authored-By: Claude Opus 4.5 --- common/config.go | 141 ++++++++++++++++++++- common/config_test.go | 160 ++++++++++++++++++++++++ erpc/http_server.go | 4 +- erpc/http_server_batch_eth_call_test.go | 2 +- 4 files changed, 299 insertions(+), 8 deletions(-) diff --git a/common/config.go b/common/config.go index f0fdccedc..9aae0209f 100644 --- a/common/config.go +++ b/common/config.go @@ -1549,11 +1549,142 @@ type EvmNetworkConfig struct { // Set to false to disable this behavior and return raw upstream errors. IdempotentTransactionBroadcast *bool `yaml:"idempotentTransactionBroadcast,omitempty" json:"idempotentTransactionBroadcast,omitempty"` - // Multicall3Aggregation enables aggregating batched eth_call requests into a single - // Multicall3 contract call. This can significantly reduce latency for JSON-RPC batch - // requests containing multiple eth_call methods by combining them into one upstream call. - // Default: true (enabled) - Multicall3Aggregation *bool `yaml:"multicall3Aggregation,omitempty" json:"multicall3Aggregation,omitempty"` + // Multicall3Aggregation configures aggregating eth_call requests into Multicall3. + // Accepts either a boolean (backward compat) or a full config object. + // Default: enabled with default settings + Multicall3Aggregation *Multicall3AggregationConfig `yaml:"multicall3Aggregation,omitempty" json:"multicall3Aggregation,omitempty"` +} + +// Multicall3AggregationConfig configures network-level batching of eth_call requests +// into Multicall3 aggregate calls. This batches requests across all entrypoints +// (HTTP single, HTTP batch, gRPC) rather than just JSON-RPC batch requests. +type Multicall3AggregationConfig struct { + // Enabled enables/disables Multicall3 aggregation. Default: true + Enabled bool `yaml:"enabled" json:"enabled"` + + // WindowMs is the maximum time (milliseconds) to wait for a batch to fill. + // Default: 25ms + WindowMs int `yaml:"windowMs,omitempty" json:"windowMs"` + + // MinWaitMs is the minimum time (milliseconds) to wait for additional requests + // to join a batch. Default: 2ms + MinWaitMs int `yaml:"minWaitMs,omitempty" json:"minWaitMs"` + + // SafetyMarginMs is subtracted from request deadlines when computing flush time. + // Default: min(2, MinWaitMs) + SafetyMarginMs int `yaml:"safetyMarginMs,omitempty" json:"safetyMarginMs"` + + // OnlyIfPending: if true, don't add latency unless a batch is already open. + // Default: false + OnlyIfPending bool `yaml:"onlyIfPending,omitempty" json:"onlyIfPending"` + + // MaxCalls is the maximum number of calls per batch. Default: 20 + MaxCalls int `yaml:"maxCalls,omitempty" json:"maxCalls"` + + // MaxCalldataBytes is the maximum total calldata size per batch. Default: 64000 + MaxCalldataBytes int `yaml:"maxCalldataBytes,omitempty" json:"maxCalldataBytes"` + + // MaxQueueSize is the maximum total enqueued requests across all batches. + // Default: 1000 + MaxQueueSize int `yaml:"maxQueueSize,omitempty" json:"maxQueueSize"` + + // MaxPendingBatches is the maximum number of distinct batch keys. + // Default: 200 + MaxPendingBatches int `yaml:"maxPendingBatches,omitempty" json:"maxPendingBatches"` + + // CachePerCall enables per-call cache writes after successful Multicall3. + // Default: true + CachePerCall *bool `yaml:"cachePerCall,omitempty" json:"cachePerCall"` + + // AllowCrossUserBatching: if true, requests from different users can share a batch. + // Default: true + AllowCrossUserBatching *bool `yaml:"allowCrossUserBatching,omitempty" json:"allowCrossUserBatching"` + + // AllowPendingTagBatching: if true, allow batching calls with "pending" block tag. + // Default: false + AllowPendingTagBatching bool `yaml:"allowPendingTagBatching,omitempty" json:"allowPendingTagBatching"` +} + +// SetDefaults applies default values to unset fields +func (c *Multicall3AggregationConfig) SetDefaults() { + if c.WindowMs == 0 { + c.WindowMs = 25 + } + if c.MinWaitMs == 0 { + c.MinWaitMs = 2 + } + if c.SafetyMarginMs == 0 { + c.SafetyMarginMs = min(2, c.MinWaitMs) + } + if c.MaxCalls == 0 { + c.MaxCalls = 20 + } + if c.MaxCalldataBytes == 0 { + c.MaxCalldataBytes = 64000 + } + if c.MaxQueueSize == 0 { + c.MaxQueueSize = 1000 + } + if c.MaxPendingBatches == 0 { + c.MaxPendingBatches = 200 + } + if c.CachePerCall == nil { + c.CachePerCall = &TRUE + } + if c.AllowCrossUserBatching == nil { + c.AllowCrossUserBatching = &TRUE + } +} + +// IsValid checks if the config values are valid +func (c *Multicall3AggregationConfig) IsValid() error { + if c.WindowMs <= 0 { + return fmt.Errorf("multicall3Aggregation.windowMs must be > 0") + } + if c.MinWaitMs < 0 { + return fmt.Errorf("multicall3Aggregation.minWaitMs must be >= 0") + } + if c.MinWaitMs > c.WindowMs { + return fmt.Errorf("multicall3Aggregation.minWaitMs must be <= windowMs") + } + if c.MaxCalls <= 1 { + return fmt.Errorf("multicall3Aggregation.maxCalls must be > 1") + } + if c.MaxCalldataBytes <= 0 { + return fmt.Errorf("multicall3Aggregation.maxCalldataBytes must be > 0") + } + if c.MaxQueueSize <= 0 { + return fmt.Errorf("multicall3Aggregation.maxQueueSize must be > 0") + } + if c.MaxPendingBatches <= 0 { + return fmt.Errorf("multicall3Aggregation.maxPendingBatches must be > 0") + } + return nil +} + +// UnmarshalYAML implements backward compatibility for boolean config values +func (c *Multicall3AggregationConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + // Try bool first (backward compat) + var boolVal bool + if err := unmarshal(&boolVal); err == nil { + c.Enabled = boolVal + if boolVal { + c.SetDefaults() + } + return nil + } + + // Try full config + type rawConfig Multicall3AggregationConfig + var raw rawConfig + if err := unmarshal(&raw); err != nil { + return err + } + *c = Multicall3AggregationConfig(raw) + if c.Enabled { + c.SetDefaults() + } + return nil } // EvmIntegrityConfig is deprecated. Use DirectiveDefaultsConfig for validation settings. diff --git a/common/config_test.go b/common/config_test.go index 361450ee8..297dc37d1 100644 --- a/common/config_test.go +++ b/common/config_test.go @@ -7,6 +7,7 @@ import ( "github.com/spf13/afero" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "gopkg.in/yaml.v3" ) @@ -838,3 +839,162 @@ projects: assert.Equal(t, 5, network.Failsafe[1].Retry.MaxAttempts) }) } + +func TestMulticall3AggregationConfigYAML(t *testing.T) { + yamlStr := ` +evm: + chainId: 1 + multicall3Aggregation: + enabled: true + windowMs: 25 + minWaitMs: 2 + safetyMarginMs: 2 + maxCalls: 20 + maxCalldataBytes: 64000 + maxQueueSize: 1000 + maxPendingBatches: 200 + cachePerCall: true + allowCrossUserBatching: true + allowPendingTagBatching: false +` + var cfg NetworkConfig + err := yaml.Unmarshal([]byte(yamlStr), &cfg) + require.NoError(t, err) + require.NotNil(t, cfg.Evm) + require.NotNil(t, cfg.Evm.Multicall3Aggregation) + require.True(t, cfg.Evm.Multicall3Aggregation.Enabled) + require.Equal(t, 25, cfg.Evm.Multicall3Aggregation.WindowMs) + require.Equal(t, 2, cfg.Evm.Multicall3Aggregation.MinWaitMs) + require.Equal(t, 2, cfg.Evm.Multicall3Aggregation.SafetyMarginMs) + require.Equal(t, 20, cfg.Evm.Multicall3Aggregation.MaxCalls) + require.Equal(t, 64000, cfg.Evm.Multicall3Aggregation.MaxCalldataBytes) + require.Equal(t, 1000, cfg.Evm.Multicall3Aggregation.MaxQueueSize) + require.Equal(t, 200, cfg.Evm.Multicall3Aggregation.MaxPendingBatches) + require.NotNil(t, cfg.Evm.Multicall3Aggregation.CachePerCall) + require.True(t, *cfg.Evm.Multicall3Aggregation.CachePerCall) + require.NotNil(t, cfg.Evm.Multicall3Aggregation.AllowCrossUserBatching) + require.True(t, *cfg.Evm.Multicall3Aggregation.AllowCrossUserBatching) + require.False(t, cfg.Evm.Multicall3Aggregation.AllowPendingTagBatching) +} + +func TestMulticall3AggregationConfigBoolBackcompat(t *testing.T) { + // Test backward compatibility with bool value + yamlStr := ` +evm: + chainId: 1 + multicall3Aggregation: true +` + var cfg NetworkConfig + err := yaml.Unmarshal([]byte(yamlStr), &cfg) + require.NoError(t, err) + require.NotNil(t, cfg.Evm) + require.NotNil(t, cfg.Evm.Multicall3Aggregation) + require.True(t, cfg.Evm.Multicall3Aggregation.Enabled) + // Check defaults are applied when bool is true + require.Equal(t, 25, cfg.Evm.Multicall3Aggregation.WindowMs) + require.Equal(t, 20, cfg.Evm.Multicall3Aggregation.MaxCalls) +} + +func TestMulticall3AggregationConfigBoolFalse(t *testing.T) { + // Test backward compatibility with false value + yamlStr := ` +evm: + chainId: 1 + multicall3Aggregation: false +` + var cfg NetworkConfig + err := yaml.Unmarshal([]byte(yamlStr), &cfg) + require.NoError(t, err) + require.NotNil(t, cfg.Evm) + require.NotNil(t, cfg.Evm.Multicall3Aggregation) + require.False(t, cfg.Evm.Multicall3Aggregation.Enabled) +} + +func TestMulticall3AggregationConfigDefaults(t *testing.T) { + cfg := &Multicall3AggregationConfig{Enabled: true} + cfg.SetDefaults() + require.Equal(t, 25, cfg.WindowMs) + require.Equal(t, 2, cfg.MinWaitMs) + require.Equal(t, 2, cfg.SafetyMarginMs) + require.Equal(t, 20, cfg.MaxCalls) + require.Equal(t, 64000, cfg.MaxCalldataBytes) + require.Equal(t, 1000, cfg.MaxQueueSize) + require.Equal(t, 200, cfg.MaxPendingBatches) + require.NotNil(t, cfg.CachePerCall) + require.True(t, *cfg.CachePerCall) + require.NotNil(t, cfg.AllowCrossUserBatching) + require.True(t, *cfg.AllowCrossUserBatching) +} + +func TestMulticall3AggregationConfigIsValid(t *testing.T) { + t.Run("valid config", func(t *testing.T) { + cfg := &Multicall3AggregationConfig{Enabled: true} + cfg.SetDefaults() + err := cfg.IsValid() + require.NoError(t, err) + }) + + t.Run("windowMs must be positive", func(t *testing.T) { + cfg := &Multicall3AggregationConfig{Enabled: true, WindowMs: 0} + err := cfg.IsValid() + require.Error(t, err) + require.Contains(t, err.Error(), "windowMs must be > 0") + }) + + t.Run("minWaitMs must not be negative", func(t *testing.T) { + cfg := &Multicall3AggregationConfig{Enabled: true, WindowMs: 25, MinWaitMs: -1} + err := cfg.IsValid() + require.Error(t, err) + require.Contains(t, err.Error(), "minWaitMs must be >= 0") + }) + + t.Run("minWaitMs must be <= windowMs", func(t *testing.T) { + cfg := &Multicall3AggregationConfig{Enabled: true, WindowMs: 10, MinWaitMs: 20} + err := cfg.IsValid() + require.Error(t, err) + require.Contains(t, err.Error(), "minWaitMs must be <= windowMs") + }) + + t.Run("maxCalls must be > 1", func(t *testing.T) { + cfg := &Multicall3AggregationConfig{Enabled: true, WindowMs: 25, MinWaitMs: 2, MaxCalls: 1} + err := cfg.IsValid() + require.Error(t, err) + require.Contains(t, err.Error(), "maxCalls must be > 1") + }) + + t.Run("maxCalldataBytes must be positive", func(t *testing.T) { + cfg := &Multicall3AggregationConfig{Enabled: true, WindowMs: 25, MinWaitMs: 2, MaxCalls: 20, MaxCalldataBytes: 0} + err := cfg.IsValid() + require.Error(t, err) + require.Contains(t, err.Error(), "maxCalldataBytes must be > 0") + }) + + t.Run("maxQueueSize must be positive", func(t *testing.T) { + cfg := &Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 25, + MinWaitMs: 2, + MaxCalls: 20, + MaxCalldataBytes: 64000, + MaxQueueSize: 0, + } + err := cfg.IsValid() + require.Error(t, err) + require.Contains(t, err.Error(), "maxQueueSize must be > 0") + }) + + t.Run("maxPendingBatches must be positive", func(t *testing.T) { + cfg := &Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 25, + MinWaitMs: 2, + MaxCalls: 20, + MaxCalldataBytes: 64000, + MaxQueueSize: 1000, + MaxPendingBatches: 0, + } + err := cfg.IsValid() + require.Error(t, err) + require.Contains(t, err.Error(), "maxPendingBatches must be > 0") + }) +} diff --git a/erpc/http_server.go b/erpc/http_server.go index abe6dbf11..3fa42b0c8 100644 --- a/erpc/http_server.go +++ b/erpc/http_server.go @@ -1672,7 +1672,7 @@ func stripAddrDecorations(s string) string { } // isMulticall3AggregationEnabled checks if multicall3 aggregation is enabled for a given network. -// Returns true (default) if no explicit config is set, or if the config is explicitly set to true. +// Returns true (default) if no explicit config is set, or if the config is explicitly set to enabled. func isMulticall3AggregationEnabled(project *PreparedProject, networkId string) bool { if project == nil || project.Config == nil { return true // Default to enabled @@ -1684,7 +1684,7 @@ func isMulticall3AggregationEnabled(project *PreparedProject, networkId string) for _, nwCfg := range project.Config.Networks { if nwCfg != nil && nwCfg.NetworkId() == networkId { if nwCfg.Evm != nil && nwCfg.Evm.Multicall3Aggregation != nil { - return *nwCfg.Evm.Multicall3Aggregation + return nwCfg.Evm.Multicall3Aggregation.Enabled } break } diff --git a/erpc/http_server_batch_eth_call_test.go b/erpc/http_server_batch_eth_call_test.go index ed26c1dc7..7c8e22c7b 100644 --- a/erpc/http_server_batch_eth_call_test.go +++ b/erpc/http_server_batch_eth_call_test.go @@ -71,7 +71,7 @@ func TestHttpServer_BatchEthCall_MulticallAggregationDisabled(t *testing.T) { // Create config with multicall3 aggregation disabled cfg := baseBatchConfig() - cfg.Projects[0].Networks[0].Evm.Multicall3Aggregation = &common.FALSE + cfg.Projects[0].Networks[0].Evm.Multicall3Aggregation = &common.Multicall3AggregationConfig{Enabled: false} sendRequest, _, _, shutdown, _ := createServerTestFixtures(cfg, t) defer shutdown() From 6520dbadbe24709c3c56dadd9503648fd72c0828 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 15 Jan 2026 17:35:42 +0100 Subject: [PATCH 16/53] feat: add CompositeTypeMulticall3 constant Used to mark aggregated Multicall3 requests for metrics and hedging logic. Co-Authored-By: Claude Opus 4.5 --- common/request.go | 1 + 1 file changed, 1 insertion(+) diff --git a/common/request.go b/common/request.go index 44903d001..f02049409 100644 --- a/common/request.go +++ b/common/request.go @@ -18,6 +18,7 @@ const ( CompositeTypeNone = "none" CompositeTypeLogsSplitOnError = "logs-split-on-error" CompositeTypeLogsSplitProactive = "logs-split-proactive" + CompositeTypeMulticall3 = "multicall3" ) const RequestContextKey ContextKey = "rq" From 697bac436f3c4e1fcbf685b8084464a9b11fec57 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 15 Jan 2026 17:38:23 +0100 Subject: [PATCH 17/53] feat: add Multicall3 batching key and entry types Core data structures for network-level Multicall3 batching: - BatchingKey for grouping requests by project/network/block/directives - DeriveDirectivesKey for stable versioned directive hashing - DeriveCallKey for within-batch deduplication - BatchEntry and Batch for holding pending requests Co-Authored-By: Claude Opus 4.5 --- architecture/evm/multicall3_batcher.go | 118 ++++++++++++++++++++ architecture/evm/multicall3_batcher_test.go | 88 +++++++++++++++ 2 files changed, 206 insertions(+) create mode 100644 architecture/evm/multicall3_batcher.go create mode 100644 architecture/evm/multicall3_batcher_test.go diff --git a/architecture/evm/multicall3_batcher.go b/architecture/evm/multicall3_batcher.go new file mode 100644 index 000000000..186759d64 --- /dev/null +++ b/architecture/evm/multicall3_batcher.go @@ -0,0 +1,118 @@ +package evm + +import ( + "context" + "fmt" + "sort" + "strings" + "sync" + "time" + + "github.com/erpc/erpc/common" +) + +// DirectivesKeyVersion should be bumped when the set of directives +// included in the key changes. This prevents cross-node key mismatches. +const DirectivesKeyVersion = 1 + +// BatchingKey uniquely identifies a batch for grouping eth_call requests. +type BatchingKey struct { + ProjectId string + NetworkId string + BlockRef string + DirectivesKey string + UserId string // empty if cross-user batching is allowed +} + +func (k BatchingKey) String() string { + return fmt.Sprintf("%s|%s|%s|%s|%s", k.ProjectId, k.NetworkId, k.BlockRef, k.DirectivesKey, k.UserId) +} + +// DeriveDirectivesKey creates a stable, versioned key from relevant directives. +// Only includes directives that affect batching behavior. +func DeriveDirectivesKey(dirs *common.RequestDirectives) string { + if dirs == nil { + return fmt.Sprintf("v%d:", DirectivesKeyVersion) + } + + parts := make([]string, 0, 5) + if dirs.UseUpstream != "" { + parts = append(parts, fmt.Sprintf("use-upstream=%s", dirs.UseUpstream)) + } + if dirs.SkipInterpolation { + parts = append(parts, "skip-interpolation=true") + } + if dirs.RetryEmpty { + parts = append(parts, "retry-empty=true") + } + if dirs.RetryPending { + parts = append(parts, "retry-pending=true") + } + if dirs.SkipCacheRead { + parts = append(parts, "skip-cache-read=true") + } + + sort.Strings(parts) + return fmt.Sprintf("v%d:%s", DirectivesKeyVersion, strings.Join(parts, ",")) +} + +// DeriveCallKey creates a unique key for deduplication within a batch. +// Uses the same derivation as cache keys for consistency. +func DeriveCallKey(req *common.NormalizedRequest) (string, error) { + if req == nil { + return "", fmt.Errorf("request is nil") + } + jrq, err := req.JsonRpcRequest() + if err != nil { + return "", err + } + + jrq.RLock() + method := jrq.Method + params := jrq.Params + jrq.RUnlock() + + // Use method + params as key (same as cache key derivation) + paramsJSON, err := common.SonicCfg.Marshal(params) + if err != nil { + return "", err + } + return fmt.Sprintf("%s:%s", method, string(paramsJSON)), nil +} + +// BatchEntry represents a request waiting in a batch. +type BatchEntry struct { + Ctx context.Context + Request *common.NormalizedRequest + CallKey string + Target []byte + CallData []byte + ResultCh chan BatchResult + CreatedAt time.Time + Deadline time.Time +} + +// BatchResult is the outcome delivered to a waiting request. +type BatchResult struct { + Response *common.NormalizedResponse + Error error +} + +// Batch holds pending requests for a single batching key. +type Batch struct { + Key BatchingKey + Entries []*BatchEntry + CallKeys map[string][]*BatchEntry // for deduplication + FlushTime time.Time + Flushing bool + mu sync.Mutex +} + +func NewBatch(key BatchingKey, flushTime time.Time) *Batch { + return &Batch{ + Key: key, + Entries: make([]*BatchEntry, 0, 16), + CallKeys: make(map[string][]*BatchEntry), + FlushTime: flushTime, + } +} diff --git a/architecture/evm/multicall3_batcher_test.go b/architecture/evm/multicall3_batcher_test.go new file mode 100644 index 000000000..7c192da3d --- /dev/null +++ b/architecture/evm/multicall3_batcher_test.go @@ -0,0 +1,88 @@ +package evm + +import ( + "strings" + "testing" + + "github.com/erpc/erpc/common" + "github.com/stretchr/testify/require" +) + +func TestBatchingKey(t *testing.T) { + key1 := BatchingKey{ + ProjectId: "proj1", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: "use-upstream=alchemy", + UserId: "", + } + key2 := BatchingKey{ + ProjectId: "proj1", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: "use-upstream=alchemy", + UserId: "", + } + key3 := BatchingKey{ + ProjectId: "proj1", + NetworkId: "evm:1", + BlockRef: "12345", + DirectivesKey: "use-upstream=alchemy", + UserId: "", + } + + require.Equal(t, key1.String(), key2.String()) + require.NotEqual(t, key1.String(), key3.String()) +} + +func TestDirectivesKeyDerivation(t *testing.T) { + dirs := &common.RequestDirectives{} + dirs.UseUpstream = "alchemy" + dirs.SkipCacheRead = true + dirs.RetryEmpty = true + + key := DeriveDirectivesKey(dirs) + require.Contains(t, key, "use-upstream=alchemy") + require.Contains(t, key, "skip-cache-read=true") + require.Contains(t, key, "retry-empty=true") +} + +func TestCallKeyDerivation(t *testing.T) { + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x1234567890123456789012345678901234567890", + "data": "0xabcdef", + }, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + + key, err := DeriveCallKey(req) + require.NoError(t, err) + require.NotEmpty(t, key) + + // Same request should produce same key + key2, err := DeriveCallKey(req) + require.NoError(t, err) + require.Equal(t, key, key2) +} + +func TestDirectivesKeyDerivation_Nil(t *testing.T) { + key := DeriveDirectivesKey(nil) + require.Equal(t, "v1:", key) +} + +func TestDirectivesKeyDerivation_VersionPrefix(t *testing.T) { + dirs := &common.RequestDirectives{} + dirs.UseUpstream = "alchemy" + + key := DeriveDirectivesKey(dirs) + require.True(t, strings.HasPrefix(key, "v1:"), "key should have version prefix") +} + +func TestCallKeyDerivation_NilRequest(t *testing.T) { + key, err := DeriveCallKey(nil) + require.Error(t, err) + require.Contains(t, err.Error(), "request is nil") + require.Empty(t, key) +} From 400f800d0a2a12d38497cfa33ff54106ce41d78d Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 15 Jan 2026 17:50:24 +0100 Subject: [PATCH 18/53] feat: add Multicall3 eligibility checking Implements IsEligibleForBatching to determine if an eth_call can be aggregated into a Multicall3 batch: - Method must be eth_call - Call object must only have to + data/input fields - No state overrides (3rd param) - Recursion guard: don't batch calls to multicall3 contract - Block tag restrictions (pending disabled by default) Also adds ExtractCallInfo helper to extract target/calldata/blockRef. Co-Authored-By: Claude Opus 4.5 --- architecture/evm/multicall3_batcher.go | 129 ++++++++++++++++++++ architecture/evm/multicall3_batcher_test.go | 117 ++++++++++++++++++ 2 files changed, 246 insertions(+) diff --git a/architecture/evm/multicall3_batcher.go b/architecture/evm/multicall3_batcher.go index 186759d64..9396de5b1 100644 --- a/architecture/evm/multicall3_batcher.go +++ b/architecture/evm/multicall3_batcher.go @@ -116,3 +116,132 @@ func NewBatch(key BatchingKey, flushTime time.Time) *Batch { FlushTime: flushTime, } } + +// ineligibleCallFields are fields that make an eth_call ineligible for batching. +// Multicall3 aggregate3 only supports target + calldata, not gas/value/etc. +var ineligibleCallFields = []string{ + "from", "gas", "gasPrice", "maxFeePerGas", "maxPriorityFeePerGas", "value", +} + +// IsEligibleForBatching checks if a request can be batched via Multicall3. +// Returns (eligible, reason) where reason explains why not eligible. +func IsEligibleForBatching(req *common.NormalizedRequest, cfg *common.Multicall3AggregationConfig) (bool, string) { + if req == nil { + return false, "request is nil" + } + if cfg == nil || !cfg.Enabled { + return false, "batching disabled" + } + + jrq, err := req.JsonRpcRequest() + if err != nil { + return false, fmt.Sprintf("json-rpc error: %v", err) + } + + jrq.RLock() + method := strings.ToLower(jrq.Method) + params := jrq.Params + jrq.RUnlock() + + // Must be eth_call + if method != "eth_call" { + return false, "not eth_call" + } + + // Must have 1-3 params (call object, optional block, optional state override) + if len(params) < 1 { + return false, fmt.Sprintf("invalid param count: %d", len(params)) + } + + // Check for state override (3rd param) - not supported with multicall3 + if len(params) > 2 { + return false, "has state override" + } + + // Parse call object + callObj, ok := params[0].(map[string]interface{}) + if !ok { + return false, "invalid call object type" + } + + // Must have 'to' address + toVal, hasTo := callObj["to"] + if !hasTo { + return false, "missing to address" + } + toStr, ok := toVal.(string) + if !ok || toStr == "" { + return false, "invalid to address" + } + + // Check for ineligible fields + for _, field := range ineligibleCallFields { + if _, has := callObj[field]; has { + return false, fmt.Sprintf("has %s field", field) + } + } + + // Recursion guard: don't batch calls to multicall3 contract + if strings.EqualFold(toStr, multicall3Address) { + return false, "already multicall" + } + + // Check block tag + blockTag := "latest" + if len(params) >= 2 && params[1] != nil { + normalized, err := NormalizeBlockParam(params[1]) + if err != nil { + return false, fmt.Sprintf("invalid block param: %v", err) + } + blockTag = strings.ToLower(normalized) + } + + // Check if pending tag is allowed + if blockTag == "pending" && !cfg.AllowPendingTagBatching { + return false, "pending tag not allowed" + } + + return true, "" +} + +// ExtractCallInfo extracts target and calldata from an eligible eth_call request. +func ExtractCallInfo(req *common.NormalizedRequest) (target []byte, callData []byte, blockRef string, err error) { + jrq, err := req.JsonRpcRequest() + if err != nil { + return nil, nil, "", err + } + + jrq.RLock() + params := jrq.Params + jrq.RUnlock() + + callObj := params[0].(map[string]interface{}) + toStr := callObj["to"].(string) + + target, err = common.HexToBytes(toStr) + if err != nil { + return nil, nil, "", err + } + + dataHex := "0x" + if dataVal, ok := callObj["data"]; ok { + dataHex = dataVal.(string) + } else if inputVal, ok := callObj["input"]; ok { + dataHex = inputVal.(string) + } + + callData, err = common.HexToBytes(dataHex) + if err != nil { + return nil, nil, "", err + } + + blockRef = "latest" + if len(params) >= 2 && params[1] != nil { + blockRef, err = NormalizeBlockParam(params[1]) + if err != nil { + return nil, nil, "", err + } + } + + return target, callData, blockRef, nil +} diff --git a/architecture/evm/multicall3_batcher_test.go b/architecture/evm/multicall3_batcher_test.go index 7c192da3d..0d65427ef 100644 --- a/architecture/evm/multicall3_batcher_test.go +++ b/architecture/evm/multicall3_batcher_test.go @@ -86,3 +86,120 @@ func TestCallKeyDerivation_NilRequest(t *testing.T) { require.Contains(t, err.Error(), "request is nil") require.Empty(t, key) } + +func TestIsEligibleForBatching(t *testing.T) { + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + AllowPendingTagBatching: false, + } + cfg.SetDefaults() + + tests := []struct { + name string + method string + params []interface{} + eligible bool + reason string + }{ + { + name: "eligible basic eth_call", + method: "eth_call", + params: []interface{}{ + map[string]interface{}{"to": "0x1234567890123456789012345678901234567890", "data": "0xabcd"}, + "latest", + }, + eligible: true, + }, + { + name: "eligible with finalized tag", + method: "eth_call", + params: []interface{}{ + map[string]interface{}{"to": "0x1234567890123456789012345678901234567890", "data": "0xabcd"}, + "finalized", + }, + eligible: true, + }, + { + name: "ineligible - pending tag", + method: "eth_call", + params: []interface{}{ + map[string]interface{}{"to": "0x1234567890123456789012345678901234567890", "data": "0xabcd"}, + "pending", + }, + eligible: false, + reason: "pending tag not allowed", + }, + { + name: "ineligible - has from field", + method: "eth_call", + params: []interface{}{ + map[string]interface{}{ + "to": "0x1234567890123456789012345678901234567890", + "data": "0xabcd", + "from": "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + }, + "latest", + }, + eligible: false, + reason: "has from field", + }, + { + name: "ineligible - has value field", + method: "eth_call", + params: []interface{}{ + map[string]interface{}{ + "to": "0x1234567890123456789012345678901234567890", + "data": "0xabcd", + "value": "0x1", + }, + "latest", + }, + eligible: false, + reason: "has value field", + }, + { + name: "ineligible - has state override (3rd param)", + method: "eth_call", + params: []interface{}{ + map[string]interface{}{"to": "0x1234567890123456789012345678901234567890", "data": "0xabcd"}, + "latest", + map[string]interface{}{}, // state override + }, + eligible: false, + reason: "has state override", + }, + { + name: "ineligible - not eth_call", + method: "eth_getBalance", + params: []interface{}{"0x1234567890123456789012345678901234567890", "latest"}, + eligible: false, + reason: "not eth_call", + }, + { + name: "ineligible - already multicall (recursion guard)", + method: "eth_call", + params: []interface{}{ + map[string]interface{}{ + "to": "0xcA11bde05977b3631167028862bE2a173976CA11", // multicall3 address + "data": "0x82ad56cb", // aggregate3 selector + }, + "latest", + }, + eligible: false, + reason: "already multicall", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + jrq := common.NewJsonRpcRequest(tt.method, tt.params) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + + eligible, reason := IsEligibleForBatching(req, cfg) + require.Equal(t, tt.eligible, eligible, "reason: %s", reason) + if !tt.eligible { + require.Contains(t, reason, tt.reason) + } + }) + } +} From 71ba2dde39c815fb0b1cb53ca949c92607adfc6f Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 15 Jan 2026 17:54:04 +0100 Subject: [PATCH 19/53] feat: add allowedBlockTags map for block ref eligibility checking Add the allowedBlockTags map that was missing from the spec: - latest, finalized, safe, earliest are always allowed - pending is allowed only if AllowPendingTagBatching is true - Numeric block numbers (converted to decimal) are allowed - Block hashes (0x + 64 hex chars) are allowed - Unknown tags are now rejected Also adds tests for safe, earliest tags, numeric blocks, block hashes, and unknown tag rejection. Co-Authored-By: Claude Opus 4.5 --- architecture/evm/multicall3_batcher.go | 38 +++++++++++++++++ architecture/evm/multicall3_batcher_test.go | 46 +++++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/architecture/evm/multicall3_batcher.go b/architecture/evm/multicall3_batcher.go index 9396de5b1..b824c88b2 100644 --- a/architecture/evm/multicall3_batcher.go +++ b/architecture/evm/multicall3_batcher.go @@ -123,6 +123,14 @@ var ineligibleCallFields = []string{ "from", "gas", "gasPrice", "maxFeePerGas", "maxPriorityFeePerGas", "value", } +// allowedBlockTags are block tags that can be batched by default. +var allowedBlockTags = map[string]bool{ + "latest": true, + "finalized": true, + "safe": true, + "earliest": true, +} + // IsEligibleForBatching checks if a request can be batched via Multicall3. // Returns (eligible, reason) where reason explains why not eligible. func IsEligibleForBatching(req *common.NormalizedRequest, cfg *common.Multicall3AggregationConfig) (bool, string) { @@ -201,9 +209,39 @@ func IsEligibleForBatching(req *common.NormalizedRequest, cfg *common.Multicall3 return false, "pending tag not allowed" } + // Check if block tag is eligible for batching: + // - Known named tags (latest, finalized, safe, earliest) are always allowed + // - pending is allowed if AllowPendingTagBatching is true (checked above) + // - Numeric block numbers (decimal strings after normalization) are allowed + // - Block hashes (0x + 64 hex chars) are allowed + if !isBlockRefEligibleForBatching(blockTag) { + return false, fmt.Sprintf("block tag not allowed: %s", blockTag) + } + return true, "" } +// isBlockRefEligibleForBatching checks if a normalized block reference is eligible for batching. +// It allows: known block tags, numeric block numbers, and block hashes. +func isBlockRefEligibleForBatching(blockRef string) bool { + // Check known block tags (including pending, which is handled separately) + if allowedBlockTags[blockRef] || blockRef == "pending" { + return true + } + + // Check if it's a numeric block number (decimal string after normalization) + if len(blockRef) > 0 && blockRef[0] >= '0' && blockRef[0] <= '9' { + return true + } + + // Check if it's a block hash (0x + 64 hex chars = 66 chars total for 32 bytes) + if strings.HasPrefix(blockRef, "0x") && len(blockRef) == 66 { + return true + } + + return false +} + // ExtractCallInfo extracts target and calldata from an eligible eth_call request. func ExtractCallInfo(req *common.NormalizedRequest) (target []byte, callData []byte, blockRef string, err error) { jrq, err := req.JsonRpcRequest() diff --git a/architecture/evm/multicall3_batcher_test.go b/architecture/evm/multicall3_batcher_test.go index 0d65427ef..40fed4160 100644 --- a/architecture/evm/multicall3_batcher_test.go +++ b/architecture/evm/multicall3_batcher_test.go @@ -188,6 +188,52 @@ func TestIsEligibleForBatching(t *testing.T) { eligible: false, reason: "already multicall", }, + { + name: "eligible with safe tag", + method: "eth_call", + params: []interface{}{ + map[string]interface{}{"to": "0x1234567890123456789012345678901234567890", "data": "0xabcd"}, + "safe", + }, + eligible: true, + }, + { + name: "eligible with earliest tag", + method: "eth_call", + params: []interface{}{ + map[string]interface{}{"to": "0x1234567890123456789012345678901234567890", "data": "0xabcd"}, + "earliest", + }, + eligible: true, + }, + { + name: "eligible with numeric block number (hex)", + method: "eth_call", + params: []interface{}{ + map[string]interface{}{"to": "0x1234567890123456789012345678901234567890", "data": "0xabcd"}, + "0x1234", + }, + eligible: true, + }, + { + name: "eligible with block hash", + method: "eth_call", + params: []interface{}{ + map[string]interface{}{"to": "0x1234567890123456789012345678901234567890", "data": "0xabcd"}, + "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", // 32-byte hash + }, + eligible: true, + }, + { + name: "ineligible - unknown block tag", + method: "eth_call", + params: []interface{}{ + map[string]interface{}{"to": "0x1234567890123456789012345678901234567890", "data": "0xabcd"}, + "unknown_tag", + }, + eligible: false, + reason: "block tag not allowed", + }, } for _, tt := range tests { From 2ff459a3030bd34acccb9db29eb27581d5bc22bf Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 15 Jan 2026 18:02:14 +0100 Subject: [PATCH 20/53] fix: add defensive type assertions and tests for ExtractCallInfo - Add defensive type assertions to ExtractCallInfo to handle potential type assertion failures gracefully - Add comprehensive tests for ExtractCallInfo covering various scenarios: - Basic extraction with data field - Extraction with input field instead of data - Extraction with empty data - Extraction with no block param (defaults to latest) - Add test for AllowPendingTagBatching configuration Co-Authored-By: Claude Opus 4.5 --- architecture/evm/multicall3_batcher.go | 25 ++++- architecture/evm/multicall3_batcher_test.go | 100 ++++++++++++++++++++ 2 files changed, 121 insertions(+), 4 deletions(-) diff --git a/architecture/evm/multicall3_batcher.go b/architecture/evm/multicall3_batcher.go index b824c88b2..88cb5716d 100644 --- a/architecture/evm/multicall3_batcher.go +++ b/architecture/evm/multicall3_batcher.go @@ -243,6 +243,8 @@ func isBlockRefEligibleForBatching(blockRef string) bool { } // ExtractCallInfo extracts target and calldata from an eligible eth_call request. +// PRECONDITION: req must have passed IsEligibleForBatching - this function assumes +// the request structure has been validated. func ExtractCallInfo(req *common.NormalizedRequest) (target []byte, callData []byte, blockRef string, err error) { jrq, err := req.JsonRpcRequest() if err != nil { @@ -253,8 +255,19 @@ func ExtractCallInfo(req *common.NormalizedRequest) (target []byte, callData []b params := jrq.Params jrq.RUnlock() - callObj := params[0].(map[string]interface{}) - toStr := callObj["to"].(string) + callObj, ok := params[0].(map[string]interface{}) + if !ok { + return nil, nil, "", fmt.Errorf("invalid call object type") + } + + toVal, ok := callObj["to"] + if !ok { + return nil, nil, "", fmt.Errorf("missing to address") + } + toStr, ok := toVal.(string) + if !ok { + return nil, nil, "", fmt.Errorf("invalid to address type") + } target, err = common.HexToBytes(toStr) if err != nil { @@ -263,9 +276,13 @@ func ExtractCallInfo(req *common.NormalizedRequest) (target []byte, callData []b dataHex := "0x" if dataVal, ok := callObj["data"]; ok { - dataHex = dataVal.(string) + if dataStr, ok := dataVal.(string); ok { + dataHex = dataStr + } } else if inputVal, ok := callObj["input"]; ok { - dataHex = inputVal.(string) + if inputStr, ok := inputVal.(string); ok { + dataHex = inputStr + } } callData, err = common.HexToBytes(dataHex) diff --git a/architecture/evm/multicall3_batcher_test.go b/architecture/evm/multicall3_batcher_test.go index 40fed4160..bd23d70cf 100644 --- a/architecture/evm/multicall3_batcher_test.go +++ b/architecture/evm/multicall3_batcher_test.go @@ -1,6 +1,7 @@ package evm import ( + "encoding/hex" "strings" "testing" @@ -249,3 +250,102 @@ func TestIsEligibleForBatching(t *testing.T) { }) } } + +func TestExtractCallInfo(t *testing.T) { + tests := []struct { + name string + params []interface{} + expectedTarget string + expectedData string + expectedBlock string + expectError bool + }{ + { + name: "basic extraction with data field", + params: []interface{}{ + map[string]interface{}{ + "to": "0x1234567890123456789012345678901234567890", + "data": "0xabcdef", + }, + "latest", + }, + expectedTarget: "0x1234567890123456789012345678901234567890", + expectedData: "0xabcdef", + expectedBlock: "latest", + }, + { + name: "extraction with input field instead of data", + params: []interface{}{ + map[string]interface{}{ + "to": "0x1234567890123456789012345678901234567890", + "input": "0x12345678", + }, + "finalized", + }, + expectedTarget: "0x1234567890123456789012345678901234567890", + expectedData: "0x12345678", + expectedBlock: "finalized", + }, + { + name: "extraction with empty data", + params: []interface{}{ + map[string]interface{}{ + "to": "0x1234567890123456789012345678901234567890", + }, + "latest", + }, + expectedTarget: "0x1234567890123456789012345678901234567890", + expectedData: "0x", + expectedBlock: "latest", + }, + { + name: "extraction with no block param (defaults to latest)", + params: []interface{}{ + map[string]interface{}{ + "to": "0x1234567890123456789012345678901234567890", + "data": "0xaa", + }, + }, + expectedTarget: "0x1234567890123456789012345678901234567890", + expectedData: "0xaa", + expectedBlock: "latest", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + jrq := common.NewJsonRpcRequest("eth_call", tt.params) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + + target, data, blockRef, err := ExtractCallInfo(req) + if tt.expectError { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, tt.expectedTarget, "0x"+hex.EncodeToString(target)) + require.Equal(t, tt.expectedData, "0x"+hex.EncodeToString(data)) + require.Equal(t, tt.expectedBlock, blockRef) + }) + } +} + +func TestIsEligibleForBatching_AllowPendingTag(t *testing.T) { + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + AllowPendingTagBatching: true, + } + cfg.SetDefaults() + + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x1234567890123456789012345678901234567890", + "data": "0xabcd", + }, + "pending", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + + eligible, reason := IsEligibleForBatching(req, cfg) + require.True(t, eligible, "pending should be allowed when AllowPendingTagBatching is true: %s", reason) +} From cd3981d4aacb2d1fb3c1d37569f00cca28dd842c Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 15 Jan 2026 18:13:57 +0100 Subject: [PATCH 21/53] feat: implement Multicall3 Batcher with window and caps Implements the core Batcher type that: - Enqueues eth_call requests into batches by BatchingKey - Enforces caps: maxCalls, maxCalldataBytes, maxQueueSize, maxPendingBatches - Supports deduplication via callKey within batches - Implements deadline-aware flush scheduling - Handles concurrent batch creation during flush Also fixes DeriveCallKey to use deterministic key derivation based on extracted call components (target + calldata + blockRef) instead of JSON serialization which has non-deterministic map key ordering. Co-Authored-By: Claude Opus 4.5 --- architecture/evm/multicall3_batcher.go | 250 +++++++++++++++++++- architecture/evm/multicall3_batcher_test.go | 176 ++++++++++++++ 2 files changed, 413 insertions(+), 13 deletions(-) diff --git a/architecture/evm/multicall3_batcher.go b/architecture/evm/multicall3_batcher.go index 88cb5716d..152e88eca 100644 --- a/architecture/evm/multicall3_batcher.go +++ b/architecture/evm/multicall3_batcher.go @@ -11,6 +11,236 @@ import ( "github.com/erpc/erpc/common" ) +// Forwarder is the interface for forwarding requests through the network layer. +type Forwarder interface { + Forward(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) +} + +// Batcher aggregates eth_call requests into Multicall3 batches. +type Batcher struct { + cfg *common.Multicall3AggregationConfig + forwarder Forwarder + batches map[string]*Batch // keyed by BatchingKey.String() + mu sync.RWMutex + queueSize int64 // counter for backpressure + shutdown chan struct{} + wg sync.WaitGroup +} + +// NewBatcher creates a new Multicall3 batcher. +func NewBatcher(cfg *common.Multicall3AggregationConfig, forwarder Forwarder) *Batcher { + b := &Batcher{ + cfg: cfg, + forwarder: forwarder, + batches: make(map[string]*Batch), + shutdown: make(chan struct{}), + } + return b +} + +// Enqueue adds a request to a batch. Returns: +// - entry: the batch entry (nil if bypass) +// - bypass: true if request should be forwarded individually +// - error: any error during processing +func (b *Batcher) Enqueue(ctx context.Context, key BatchingKey, req *common.NormalizedRequest) (*BatchEntry, bool, error) { + // Extract call info + target, callData, _, err := ExtractCallInfo(req) + if err != nil { + return nil, true, err + } + + // Derive call key for deduplication + callKey, err := DeriveCallKey(req) + if err != nil { + return nil, true, err + } + + // Calculate deadline from context + deadline, hasDeadline := ctx.Deadline() + if !hasDeadline { + deadline = time.Now().Add(time.Duration(b.cfg.WindowMs) * time.Millisecond) + } + + // Check if deadline is too tight + now := time.Now() + minWait := time.Duration(b.cfg.MinWaitMs) * time.Millisecond + if deadline.Before(now.Add(minWait)) { + // Deadline too tight, bypass batching + return nil, true, nil + } + + b.mu.Lock() + defer b.mu.Unlock() + + // Check caps + if b.queueSize >= int64(b.cfg.MaxQueueSize) { + return nil, true, nil // bypass: queue full + } + if len(b.batches) >= b.cfg.MaxPendingBatches { + // Check if this is a new batch key + if _, exists := b.batches[key.String()]; !exists { + return nil, true, nil // bypass: too many pending batches + } + } + + // Get or create batch + keyStr := key.String() + batch, exists := b.batches[keyStr] + if !exists { + flushTime := now.Add(time.Duration(b.cfg.WindowMs) * time.Millisecond) + batch = NewBatch(key, flushTime) + b.batches[keyStr] = batch + + // Start flush timer + b.wg.Add(1) + go b.scheduleFlush(keyStr, batch) + } + + // Check if batch is flushing - create new batch if so + batch.mu.Lock() + if batch.Flushing { + batch.mu.Unlock() + // Create new batch for this key + flushTime := now.Add(time.Duration(b.cfg.WindowMs) * time.Millisecond) + batch = NewBatch(key, flushTime) + b.batches[keyStr] = batch + + b.wg.Add(1) + go b.scheduleFlush(keyStr, batch) + + batch.mu.Lock() + } + + // Check if batch is at capacity (unique calls, not entries) + uniqueCalls := len(batch.CallKeys) + if _, isDupe := batch.CallKeys[callKey]; !isDupe { + if uniqueCalls >= b.cfg.MaxCalls { + batch.mu.Unlock() + return nil, true, nil // bypass: batch full + } + } + + // Check calldata size cap + currentSize := 0 + for _, entries := range batch.CallKeys { + if len(entries) > 0 { + currentSize += len(entries[0].CallData) + } + } + if _, isDupe := batch.CallKeys[callKey]; !isDupe { + if currentSize+len(callData) > b.cfg.MaxCalldataBytes { + batch.mu.Unlock() + return nil, true, nil // bypass: calldata too large + } + } + + // Create entry + entry := &BatchEntry{ + Ctx: ctx, + Request: req, + CallKey: callKey, + Target: target, + CallData: callData, + ResultCh: make(chan BatchResult, 1), + CreatedAt: now, + Deadline: deadline, + } + + // Add to batch + batch.Entries = append(batch.Entries, entry) + batch.CallKeys[callKey] = append(batch.CallKeys[callKey], entry) + + // Update flush time based on deadline (deadline-aware) + safetyMargin := time.Duration(b.cfg.SafetyMarginMs) * time.Millisecond + proposedFlush := deadline.Add(-safetyMargin) + if proposedFlush.Before(batch.FlushTime) { + batch.FlushTime = proposedFlush + // Clamp to minimum wait + minFlush := now.Add(minWait) + if batch.FlushTime.Before(minFlush) { + batch.FlushTime = minFlush + } + } + + batch.mu.Unlock() + b.queueSize++ + + return entry, false, nil +} + +// scheduleFlush waits until flush time and then flushes the batch. +func (b *Batcher) scheduleFlush(keyStr string, batch *Batch) { + defer b.wg.Done() + + for { + batch.mu.Lock() + flushTime := batch.FlushTime + batch.mu.Unlock() + + waitDuration := time.Until(flushTime) + if waitDuration <= 0 { + b.flush(keyStr, batch) + return + } + + timer := time.NewTimer(waitDuration) + select { + case <-timer.C: + b.flush(keyStr, batch) + return + case <-b.shutdown: + timer.Stop() + return + } + } +} + +// flush processes a batch and delivers results. +func (b *Batcher) flush(keyStr string, batch *Batch) { + batch.mu.Lock() + if batch.Flushing { + batch.mu.Unlock() + return + } + batch.Flushing = true + entries := batch.Entries + callKeys := batch.CallKeys + batch.mu.Unlock() + + // Remove from active batches + b.mu.Lock() + if b.batches[keyStr] == batch { + delete(b.batches, keyStr) + } + // Defensive: ensure queueSize doesn't go negative + entriesLen := int64(len(entries)) + if b.queueSize >= entriesLen { + b.queueSize -= entriesLen + } else { + b.queueSize = 0 + } + b.mu.Unlock() + + // Deliver error for now (actual forwarding implemented in Task 2.4) + result := BatchResult{ + Error: fmt.Errorf("flush not implemented"), + } + for _, entry := range entries { + select { + case entry.ResultCh <- result: + default: + } + } + // TODO(Task 2.4): callKeys will be used for per-call cache writes and dedup fanout + _ = callKeys +} + +// Shutdown stops the batcher and waits for pending operations. +func (b *Batcher) Shutdown() { + close(b.shutdown) + b.wg.Wait() +} + // DirectivesKeyVersion should be bumped when the set of directives // included in the key changes. This prevents cross-node key mismatches. const DirectivesKeyVersion = 1 @@ -57,27 +287,21 @@ func DeriveDirectivesKey(dirs *common.RequestDirectives) string { } // DeriveCallKey creates a unique key for deduplication within a batch. -// Uses the same derivation as cache keys for consistency. +// For eth_call, uses target + calldata + blockRef to create a deterministic key +// that doesn't depend on JSON map key ordering. func DeriveCallKey(req *common.NormalizedRequest) (string, error) { if req == nil { return "", fmt.Errorf("request is nil") } - jrq, err := req.JsonRpcRequest() - if err != nil { - return "", err - } - - jrq.RLock() - method := jrq.Method - params := jrq.Params - jrq.RUnlock() - // Use method + params as key (same as cache key derivation) - paramsJSON, err := common.SonicCfg.Marshal(params) + // Extract the call components deterministically + target, callData, blockRef, err := ExtractCallInfo(req) if err != nil { return "", err } - return fmt.Sprintf("%s:%s", method, string(paramsJSON)), nil + + // Create key from the extracted components (deterministic order) + return fmt.Sprintf("eth_call:%x:%x:%s", target, callData, blockRef), nil } // BatchEntry represents a request waiting in a batch. diff --git a/architecture/evm/multicall3_batcher_test.go b/architecture/evm/multicall3_batcher_test.go index bd23d70cf..151efb366 100644 --- a/architecture/evm/multicall3_batcher_test.go +++ b/architecture/evm/multicall3_batcher_test.go @@ -1,11 +1,14 @@ package evm import ( + "context" "encoding/hex" + "fmt" "strings" "testing" "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" "github.com/stretchr/testify/require" ) @@ -349,3 +352,176 @@ func TestIsEligibleForBatching_AllowPendingTag(t *testing.T) { eligible, reason := IsEligibleForBatching(req, cfg) require.True(t, eligible, "pending should be allowed when AllowPendingTagBatching is true: %s", reason) } + +func TestBatcherEnqueueAndFlush(t *testing.T) { + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 50, + MinWaitMs: 5, + SafetyMarginMs: 2, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + } + cfg.SetDefaults() + + ctx := context.Background() + batcher := NewBatcher(cfg, nil) // nil forwarder for now + + // Create test requests + jrq1 := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x1234567890123456789012345678901234567890", + "data": "0xabcdef01", + }, + "latest", + }) + req1 := common.NewNormalizedRequestFromJsonRpcRequest(jrq1) + + jrq2 := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x2234567890123456789012345678901234567890", + "data": "0xabcdef02", + }, + "latest", + }) + req2 := common.NewNormalizedRequestFromJsonRpcRequest(jrq2) + + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + // Enqueue first request + entry1, bypass1, err := batcher.Enqueue(ctx, key, req1) + require.NoError(t, err) + require.False(t, bypass1) + require.NotNil(t, entry1) + + // Enqueue second request + entry2, bypass2, err := batcher.Enqueue(ctx, key, req2) + require.NoError(t, err) + require.False(t, bypass2) + require.NotNil(t, entry2) + + // Check batch exists + batcher.mu.RLock() + batch, exists := batcher.batches[key.String()] + batcher.mu.RUnlock() + require.True(t, exists) + require.Len(t, batch.Entries, 2) + + // Cleanup + batcher.Shutdown() +} + +func TestBatcherDeduplication(t *testing.T) { + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 50, + MinWaitMs: 5, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + } + cfg.SetDefaults() + + ctx := context.Background() + batcher := NewBatcher(cfg, nil) + + // Two identical requests - using the same jrq to ensure call key consistency + // (JSON serialization of map[string]interface{} can have non-deterministic key order) + jrq1 := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x1234567890123456789012345678901234567890", + "data": "0xabcdef01", + }, + "latest", + }) + jrq2 := common.NewJsonRpcRequest("eth_call", jrq1.Params) // Use same params object + req1 := common.NewNormalizedRequestFromJsonRpcRequest(jrq1) + req2 := common.NewNormalizedRequestFromJsonRpcRequest(jrq2) + + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + entry1, _, _ := batcher.Enqueue(ctx, key, req1) + entry2, _, _ := batcher.Enqueue(ctx, key, req2) + + // Both should share the same callKey slot + require.Equal(t, entry1.CallKey, entry2.CallKey) + + batcher.mu.RLock() + batch := batcher.batches[key.String()] + batcher.mu.RUnlock() + + // Two entries but deduplicated + require.Len(t, batch.Entries, 2) + require.Len(t, batch.CallKeys[entry1.CallKey], 2) + + batcher.Shutdown() +} + +func TestBatcherCapsEnforcement(t *testing.T) { + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 50, + MinWaitMs: 5, + MaxCalls: 2, // Very low limit + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + } + cfg.SetDefaults() + + ctx := context.Background() + batcher := NewBatcher(cfg, nil) + + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + // Add requests up to cap + for i := 0; i < 2; i++ { + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": fmt.Sprintf("0x%040d", i), + "data": "0xabcdef", + }, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + _, bypass, err := batcher.Enqueue(ctx, key, req) + require.NoError(t, err) + require.False(t, bypass) + } + + // Next request should trigger bypass (caps reached) + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x9999999999999999999999999999999999999999", + "data": "0xabcdef", + }, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + _, bypass, err := batcher.Enqueue(ctx, key, req) + require.NoError(t, err) + require.True(t, bypass, "should bypass when caps reached") + + batcher.Shutdown() +} From af5261f494a933728cb2338008bc214388d0829f Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 15 Jan 2026 18:23:30 +0100 Subject: [PATCH 22/53] feat: implement Multicall3 batch forwarding and result mapping Completes the Batcher.flush() implementation: - Builds Multicall3 request from unique calls - Marks request as CompositeTypeMulticall3 - Forwards via Forwarder interface - Decodes response and maps results to entries - Fans out deduplicated results to all waiters - Handles per-call reverts as execution errors - Falls back to individual forwarding on multicall failure Co-Authored-By: Claude Opus 4.5 --- architecture/evm/multicall3_batcher.go | 185 +++++++++- architecture/evm/multicall3_batcher_test.go | 354 ++++++++++++++++++++ 2 files changed, 534 insertions(+), 5 deletions(-) diff --git a/architecture/evm/multicall3_batcher.go b/architecture/evm/multicall3_batcher.go index 152e88eca..58127b33d 100644 --- a/architecture/evm/multicall3_batcher.go +++ b/architecture/evm/multicall3_batcher.go @@ -2,6 +2,7 @@ package evm import ( "context" + "encoding/hex" "fmt" "sort" "strings" @@ -221,18 +222,192 @@ func (b *Batcher) flush(keyStr string, batch *Batch) { } b.mu.Unlock() - // Deliver error for now (actual forwarding implemented in Task 2.4) - result := BatchResult{ - Error: fmt.Errorf("flush not implemented"), + if len(entries) == 0 { + return + } + + // Build ordered unique calls list (maintains insertion order via CallKeys map iteration order) + // We need to build unique calls in the order they were first seen + type uniqueCall struct { + callKey string + entry *BatchEntry // first entry for this callKey + } + seenCallKeys := make(map[string]bool) + uniqueCalls := make([]uniqueCall, 0, len(callKeys)) + + for _, entry := range entries { + if !seenCallKeys[entry.CallKey] { + seenCallKeys[entry.CallKey] = true + uniqueCalls = append(uniqueCalls, uniqueCall{ + callKey: entry.CallKey, + entry: entry, + }) + } + } + + // Build requests for BuildMulticall3Request + requests := make([]*common.NormalizedRequest, len(uniqueCalls)) + for i, uc := range uniqueCalls { + requests[i] = uc.entry.Request + } + + // Build the multicall3 request + mcReq, _, err := BuildMulticall3Request(requests, batch.Key.BlockRef) + if err != nil { + b.deliverError(entries, err) + return + } + + // Mark request as composite type for metrics/tracing + mcReq.SetCompositeType(common.CompositeTypeMulticall3) + + // Use any entry context (they all share same project/network) + ctx := entries[0].Ctx + if ctx == nil { + ctx = context.Background() + } + + // Forward the multicall request + mcResp, err := b.forwarder.Forward(ctx, mcReq) + if err != nil { + // Check if we should fallback to individual requests + if ShouldFallbackMulticall3(err) { + b.fallbackIndividual(entries) + return + } + b.deliverError(entries, err) + return + } + + // Decode the multicall response + results, err := b.decodeMulticallResponse(mcResp) + if err != nil { + // Check if we should fallback to individual requests + if ShouldFallbackMulticall3(err) { + b.fallbackIndividual(entries) + return + } + b.deliverError(entries, err) + return } + + // Verify result count matches unique calls + if len(results) != len(uniqueCalls) { + b.deliverError(entries, fmt.Errorf("multicall3 result count mismatch: got %d, expected %d", len(results), len(uniqueCalls))) + return + } + + // Map results to entries, fanning out deduplicated results + for i, uc := range uniqueCalls { + result := results[i] + entriesForCall := callKeys[uc.callKey] + + if result.Success { + // Build success response for each entry + resultHex := "0x" + hex.EncodeToString(result.ReturnData) + for _, entry := range entriesForCall { + jrr, err := common.NewJsonRpcResponse(entry.Request.ID(), resultHex, nil) + if err != nil { + entry.ResultCh <- BatchResult{Error: err} + continue + } + resp := common.NewNormalizedResponse().WithRequest(entry.Request).WithJsonRpcResponse(jrr) + // Propagate upstream metadata from multicall response + resp.SetUpstream(mcResp.Upstream()) + resp.SetFromCache(mcResp.FromCache()) + entry.ResultCh <- BatchResult{Response: resp} + } + } else { + // Build error for reverted call with proper JSON-RPC format + dataHex := "0x" + hex.EncodeToString(result.ReturnData) + revertErr := common.NewErrEndpointExecutionException( + common.NewErrJsonRpcExceptionInternal( + int(common.JsonRpcErrorEvmReverted), // original code + common.JsonRpcErrorEvmReverted, // normalized code + "execution reverted", + nil, + map[string]interface{}{ + "data": dataHex, + "multicall3": true, + "stage": "per-call", + }, + ), + ) + for _, entry := range entriesForCall { + entry.ResultCh <- BatchResult{Error: revertErr} + } + } + } +} + +// decodeMulticallResponse extracts and decodes the multicall3 result from a response. +func (b *Batcher) decodeMulticallResponse(resp *common.NormalizedResponse) ([]Multicall3Result, error) { + if resp == nil { + return nil, fmt.Errorf("nil response") + } + + jrr, err := resp.JsonRpcResponse() + if err != nil { + return nil, err + } + if jrr == nil { + return nil, fmt.Errorf("nil json-rpc response") + } + + // Check for JSON-RPC error + if jrr.Error != nil { + return nil, common.NewErrEndpointExecutionException(jrr.Error) + } + + // Get result as hex string (JSON encoded, so has quotes) + resultStr := jrr.GetResultString() + if resultStr == "" || resultStr == "null" { + return nil, fmt.Errorf("empty result") + } + + // Parse the JSON string to get the hex value + var hexStr string + if err := common.SonicCfg.UnmarshalFromString(resultStr, &hexStr); err != nil { + return nil, fmt.Errorf("failed to parse result: %w", err) + } + + // Decode the hex bytes + resultBytes, err := common.HexToBytes(hexStr) + if err != nil { + return nil, fmt.Errorf("failed to decode hex result: %w", err) + } + + // Decode the multicall3 result + return DecodeMulticall3Aggregate3Result(resultBytes) +} + +// deliverError sends an error to all entries in a batch. +func (b *Batcher) deliverError(entries []*BatchEntry, err error) { + result := BatchResult{Error: err} for _, entry := range entries { select { case entry.ResultCh <- result: default: } } - // TODO(Task 2.4): callKeys will be used for per-call cache writes and dedup fanout - _ = callKeys +} + +// fallbackIndividual forwards each entry individually when multicall3 fails. +// Uses parallel goroutines for concurrent forwarding. +func (b *Batcher) fallbackIndividual(entries []*BatchEntry) { + var wg sync.WaitGroup + for _, entry := range entries { + wg.Add(1) + go func(e *BatchEntry) { + defer wg.Done() + resp, err := b.forwarder.Forward(e.Ctx, e.Request) + select { + case e.ResultCh <- BatchResult{Response: resp, Error: err}: + default: + } + }(entry) + } + wg.Wait() } // Shutdown stops the batcher and waits for pending operations. diff --git a/architecture/evm/multicall3_batcher_test.go b/architecture/evm/multicall3_batcher_test.go index 151efb366..95e7d96f5 100644 --- a/architecture/evm/multicall3_batcher_test.go +++ b/architecture/evm/multicall3_batcher_test.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "fmt" "strings" + "sync" "testing" "github.com/erpc/erpc/common" @@ -525,3 +526,356 @@ func TestBatcherCapsEnforcement(t *testing.T) { batcher.Shutdown() } + +// mockForwarder implements Forwarder for testing +type mockForwarder struct { + response *common.NormalizedResponse + err error + called int + mu sync.Mutex +} + +func (m *mockForwarder) Forward(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + m.mu.Lock() + defer m.mu.Unlock() + m.called++ + return m.response, m.err +} + +func TestBatcherFlushAndResultMapping(t *testing.T) { + // Create valid multicall3 result with 2 calls + // Each call returns success=true with some data + results := []Multicall3Result{ + {Success: true, ReturnData: []byte{0xde, 0xad, 0xbe, 0xef}}, + {Success: true, ReturnData: []byte{0xca, 0xfe, 0xba, 0xbe}}, + } + encodedResult := encodeAggregate3Results(results) + resultHex := "0x" + hex.EncodeToString(encodedResult) + + jrr, err := common.NewJsonRpcResponse(nil, resultHex, nil) + require.NoError(t, err) + mockResp := common.NewNormalizedResponse().WithJsonRpcResponse(jrr) + + forwarder := &mockForwarder{response: mockResp} + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 10, // Short window for test + MinWaitMs: 1, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + CachePerCall: util.BoolPtr(false), // disable caching for test + } + cfg.SetDefaults() + + batcher := NewBatcher(cfg, forwarder) + + ctx := context.Background() + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + // Add two requests + jrq1 := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{"to": "0x1111111111111111111111111111111111111111", "data": "0x01"}, + "latest", + }) + jrq1.ID = "req1" + req1 := common.NewNormalizedRequestFromJsonRpcRequest(jrq1) + + jrq2 := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{"to": "0x2222222222222222222222222222222222222222", "data": "0x02"}, + "latest", + }) + jrq2.ID = "req2" + req2 := common.NewNormalizedRequestFromJsonRpcRequest(jrq2) + + entry1, _, err := batcher.Enqueue(ctx, key, req1) + require.NoError(t, err) + entry2, _, err := batcher.Enqueue(ctx, key, req2) + require.NoError(t, err) + + // Wait for results + result1 := <-entry1.ResultCh + result2 := <-entry2.ResultCh + + require.NoError(t, result1.Error) + require.NoError(t, result2.Error) + require.NotNil(t, result1.Response) + require.NotNil(t, result2.Response) + + // Verify forwarder was called exactly once + forwarder.mu.Lock() + require.Equal(t, 1, forwarder.called) + forwarder.mu.Unlock() + + // Verify the responses contain the expected data + jrr1, err := result1.Response.JsonRpcResponse() + require.NoError(t, err) + require.Equal(t, "\"0xdeadbeef\"", jrr1.GetResultString()) + + jrr2, err := result2.Response.JsonRpcResponse() + require.NoError(t, err) + require.Equal(t, "\"0xcafebabe\"", jrr2.GetResultString()) + + batcher.Shutdown() +} + +func TestBatcherFlushDeduplication(t *testing.T) { + // Create result with 1 call (deduplication means only 1 unique call is made) + results := []Multicall3Result{ + {Success: true, ReturnData: []byte{0xab, 0xcd}}, + } + encodedResult := encodeAggregate3Results(results) + resultHex := "0x" + hex.EncodeToString(encodedResult) + + jrr, err := common.NewJsonRpcResponse(nil, resultHex, nil) + require.NoError(t, err) + mockResp := common.NewNormalizedResponse().WithJsonRpcResponse(jrr) + + forwarder := &mockForwarder{response: mockResp} + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 10, + MinWaitMs: 1, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + CachePerCall: util.BoolPtr(false), + } + cfg.SetDefaults() + + batcher := NewBatcher(cfg, forwarder) + + ctx := context.Background() + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + // Add two IDENTICAL requests (same target and calldata) + jrq1 := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{"to": "0x1111111111111111111111111111111111111111", "data": "0x01"}, + "latest", + }) + jrq1.ID = "req1" + req1 := common.NewNormalizedRequestFromJsonRpcRequest(jrq1) + + jrq2 := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{"to": "0x1111111111111111111111111111111111111111", "data": "0x01"}, + "latest", + }) + jrq2.ID = "req2" + req2 := common.NewNormalizedRequestFromJsonRpcRequest(jrq2) + + entry1, _, err := batcher.Enqueue(ctx, key, req1) + require.NoError(t, err) + entry2, _, err := batcher.Enqueue(ctx, key, req2) + require.NoError(t, err) + + // Both should share the same call key + require.Equal(t, entry1.CallKey, entry2.CallKey) + + // Wait for results + result1 := <-entry1.ResultCh + result2 := <-entry2.ResultCh + + require.NoError(t, result1.Error) + require.NoError(t, result2.Error) + + // Both should get the same result (fanned out) + jrr1, err := result1.Response.JsonRpcResponse() + require.NoError(t, err) + jrr2, err := result2.Response.JsonRpcResponse() + require.NoError(t, err) + require.Equal(t, jrr1.GetResultString(), jrr2.GetResultString()) + + // Forwarder should only be called once + forwarder.mu.Lock() + require.Equal(t, 1, forwarder.called) + forwarder.mu.Unlock() + + batcher.Shutdown() +} + +func TestBatcherFlushRevertHandling(t *testing.T) { + // Create result where second call reverts + results := []Multicall3Result{ + {Success: true, ReturnData: []byte{0xde, 0xad, 0xbe, 0xef}}, + {Success: false, ReturnData: []byte{0x08, 0xc3, 0x79, 0xa0}}, // Error(string) selector + } + encodedResult := encodeAggregate3Results(results) + resultHex := "0x" + hex.EncodeToString(encodedResult) + + jrr, err := common.NewJsonRpcResponse(nil, resultHex, nil) + require.NoError(t, err) + mockResp := common.NewNormalizedResponse().WithJsonRpcResponse(jrr) + + forwarder := &mockForwarder{response: mockResp} + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 10, + MinWaitMs: 1, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + CachePerCall: util.BoolPtr(false), + } + cfg.SetDefaults() + + batcher := NewBatcher(cfg, forwarder) + + ctx := context.Background() + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + // Add two requests + jrq1 := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{"to": "0x1111111111111111111111111111111111111111", "data": "0x01"}, + "latest", + }) + jrq1.ID = "req1" + req1 := common.NewNormalizedRequestFromJsonRpcRequest(jrq1) + + jrq2 := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{"to": "0x2222222222222222222222222222222222222222", "data": "0x02"}, + "latest", + }) + jrq2.ID = "req2" + req2 := common.NewNormalizedRequestFromJsonRpcRequest(jrq2) + + entry1, _, err := batcher.Enqueue(ctx, key, req1) + require.NoError(t, err) + entry2, _, err := batcher.Enqueue(ctx, key, req2) + require.NoError(t, err) + + // Wait for results + result1 := <-entry1.ResultCh + result2 := <-entry2.ResultCh + + // First call should succeed + require.NoError(t, result1.Error) + require.NotNil(t, result1.Response) + + // Second call should fail with revert error + require.Error(t, result2.Error) + require.Contains(t, result2.Error.Error(), "execution reverted") + + batcher.Shutdown() +} + +func TestBatcherFlushFallbackOnMulticall3Unavailable(t *testing.T) { + // Track individual calls made during fallback + var individualCalls []*common.NormalizedRequest + var mu sync.Mutex + + forwarder := &mockForwarderFunc{ + forwardFunc: func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + mu.Lock() + defer mu.Unlock() + + // Check if this is a multicall3 request (to multicall3 address) + jrq, _ := req.JsonRpcRequest() + if jrq != nil && len(jrq.Params) > 0 { + if callObj, ok := jrq.Params[0].(map[string]interface{}); ok { + if toAddr, ok := callObj["to"].(string); ok { + if strings.EqualFold(toAddr, "0xcA11bde05977b3631167028862bE2a173976CA11") { + // This is a multicall3 request - return "contract not found" error + return nil, common.NewErrEndpointExecutionException(fmt.Errorf("contract not found")) + } + } + } + } + + // Individual call - track and return success + individualCalls = append(individualCalls, req) + jrr, _ := common.NewJsonRpcResponse(req.ID(), "0xdeadbeef", nil) + return common.NewNormalizedResponse().WithJsonRpcResponse(jrr), nil + }, + } + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 10, + MinWaitMs: 1, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + CachePerCall: util.BoolPtr(false), + } + cfg.SetDefaults() + + batcher := NewBatcher(cfg, forwarder) + + ctx := context.Background() + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + // Add two requests + jrq1 := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{"to": "0x1111111111111111111111111111111111111111", "data": "0x01"}, + "latest", + }) + jrq1.ID = "req1" + req1 := common.NewNormalizedRequestFromJsonRpcRequest(jrq1) + + jrq2 := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{"to": "0x2222222222222222222222222222222222222222", "data": "0x02"}, + "latest", + }) + jrq2.ID = "req2" + req2 := common.NewNormalizedRequestFromJsonRpcRequest(jrq2) + + entry1, _, err := batcher.Enqueue(ctx, key, req1) + require.NoError(t, err) + entry2, _, err := batcher.Enqueue(ctx, key, req2) + require.NoError(t, err) + + // Wait for results + result1 := <-entry1.ResultCh + result2 := <-entry2.ResultCh + + // Both should succeed via fallback + require.NoError(t, result1.Error) + require.NoError(t, result2.Error) + + // Verify individual fallback calls were made + mu.Lock() + require.Equal(t, 2, len(individualCalls)) + mu.Unlock() + + batcher.Shutdown() +} + +// mockForwarderFunc allows custom forward behavior for testing +type mockForwarderFunc struct { + forwardFunc func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) +} + +func (m *mockForwarderFunc) Forward(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return m.forwardFunc(ctx, req) +} From ebbc1dec0addade72805e6d5c6682f665c54485b Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 15 Jan 2026 18:34:16 +0100 Subject: [PATCH 23/53] feat: add BatcherManager for per-network batcher instances Manages Multicall3 batchers per network with: - GetOrCreate for lazy initialization - Thread-safe concurrent access with double-check locking - Proper shutdown cleanup Co-Authored-By: Claude Opus 4.5 --- architecture/evm/multicall3_manager.go | 60 +++++++++ architecture/evm/multicall3_manager_test.go | 135 ++++++++++++++++++++ 2 files changed, 195 insertions(+) create mode 100644 architecture/evm/multicall3_manager.go create mode 100644 architecture/evm/multicall3_manager_test.go diff --git a/architecture/evm/multicall3_manager.go b/architecture/evm/multicall3_manager.go new file mode 100644 index 000000000..a563c5cec --- /dev/null +++ b/architecture/evm/multicall3_manager.go @@ -0,0 +1,60 @@ +package evm + +import ( + "sync" + + "github.com/erpc/erpc/common" +) + +// BatcherManager manages per-network Multicall3 batchers. +type BatcherManager struct { + batchers map[string]*Batcher + mu sync.RWMutex +} + +// NewBatcherManager creates a new batcher manager. +func NewBatcherManager() *BatcherManager { + return &BatcherManager{ + batchers: make(map[string]*Batcher), + } +} + +// GetOrCreate returns the batcher for a network, creating one if needed. +func (m *BatcherManager) GetOrCreate(networkId string, cfg *common.Multicall3AggregationConfig, forwarder Forwarder) *Batcher { + m.mu.RLock() + if b, ok := m.batchers[networkId]; ok { + m.mu.RUnlock() + return b + } + m.mu.RUnlock() + + m.mu.Lock() + defer m.mu.Unlock() + + // Double-check after acquiring write lock + if b, ok := m.batchers[networkId]; ok { + return b + } + + batcher := NewBatcher(cfg, forwarder) + m.batchers[networkId] = batcher + return batcher +} + +// Get returns the batcher for a network, or nil if not exists. +func (m *BatcherManager) Get(networkId string) *Batcher { + m.mu.RLock() + defer m.mu.RUnlock() + return m.batchers[networkId] +} + +// Shutdown stops all batchers. +func (m *BatcherManager) Shutdown() { + m.mu.Lock() + defer m.mu.Unlock() + + for _, b := range m.batchers { + b.Shutdown() + } + m.batchers = make(map[string]*Batcher) +} diff --git a/architecture/evm/multicall3_manager_test.go b/architecture/evm/multicall3_manager_test.go new file mode 100644 index 000000000..e89dc055b --- /dev/null +++ b/architecture/evm/multicall3_manager_test.go @@ -0,0 +1,135 @@ +package evm + +import ( + "sync" + "testing" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" + "github.com/stretchr/testify/require" +) + +func TestBatcherManagerGetOrCreate(t *testing.T) { + mgr := NewBatcherManager() + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 25, + MinWaitMs: 2, + MaxCalls: 20, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + } + cfg.SetDefaults() + + forwarder := &mockForwarder{} + + // Get batcher for network + batcher1 := mgr.GetOrCreate("evm:1", cfg, forwarder) + require.NotNil(t, batcher1) + + // Same network should return same batcher + batcher2 := mgr.GetOrCreate("evm:1", cfg, forwarder) + require.Same(t, batcher1, batcher2) + + // Different network should return different batcher + batcher3 := mgr.GetOrCreate("evm:137", cfg, forwarder) + require.NotSame(t, batcher1, batcher3) + + mgr.Shutdown() +} + +func TestBatcherManagerConcurrency(t *testing.T) { + mgr := NewBatcherManager() + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 25, + MinWaitMs: 2, + MaxCalls: 20, + } + cfg.SetDefaults() + + forwarder := &mockForwarder{} + + var wg sync.WaitGroup + batchers := make([]*Batcher, 100) + + // Concurrent access + for i := 0; i < 100; i++ { + wg.Add(1) + go func(idx int) { + defer wg.Done() + batchers[idx] = mgr.GetOrCreate("evm:1", cfg, forwarder) + }(i) + } + wg.Wait() + + // All should be the same batcher + for i := 1; i < 100; i++ { + require.Same(t, batchers[0], batchers[i]) + } + + mgr.Shutdown() +} + +func TestBatcherManagerGet(t *testing.T) { + mgr := NewBatcherManager() + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 25, + MinWaitMs: 2, + MaxCalls: 20, + } + cfg.SetDefaults() + + forwarder := &mockForwarder{} + + // Get before create should return nil + batcher := mgr.Get("evm:1") + require.Nil(t, batcher) + + // Create batcher + created := mgr.GetOrCreate("evm:1", cfg, forwarder) + require.NotNil(t, created) + + // Get after create should return the same batcher + retrieved := mgr.Get("evm:1") + require.Same(t, created, retrieved) + + // Get for different network should return nil + other := mgr.Get("evm:137") + require.Nil(t, other) + + mgr.Shutdown() +} + +func TestBatcherManagerShutdown(t *testing.T) { + mgr := NewBatcherManager() + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 25, + MinWaitMs: 2, + MaxCalls: 20, + } + cfg.SetDefaults() + + forwarder := &mockForwarder{} + + // Create multiple batchers + mgr.GetOrCreate("evm:1", cfg, forwarder) + mgr.GetOrCreate("evm:137", cfg, forwarder) + mgr.GetOrCreate("evm:42161", cfg, forwarder) + + // Shutdown should clean up all batchers + mgr.Shutdown() + + // After shutdown, batchers map should be empty + require.Nil(t, mgr.Get("evm:1")) + require.Nil(t, mgr.Get("evm:137")) + require.Nil(t, mgr.Get("evm:42161")) +} From 50c6ff3827a41e8bd6db07d6d1fafc6f902d8314 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 15 Jan 2026 18:47:26 +0100 Subject: [PATCH 24/53] feat: integrate Multicall3 batching into eth_call pre-forward hook Modifies projectPreForward_eth_call to: - Check if Multicall3 aggregation is enabled in network config - Verify request eligibility for batching (no from/value fields, etc) - Build batching key from project/network/block/directives/user - Enqueue eligible requests to network batcher - Wait for batch result or bypass to normal forward on error Uses global BatcherManager with sync.Once for lazy initialization. Wraps Network in networkForwarder adapter to implement Forwarder interface. Adds comprehensive test coverage for: - Batching multiple requests into one multicall - Normal forward when batching is disabled - Normal forward for ineligible requests (e.g., with from field) - Block param normalization (adding "latest" when missing) Co-Authored-By: Claude Opus 4.5 --- architecture/evm/eth_call.go | 110 +++++++++-- architecture/evm/eth_call_test.go | 296 ++++++++++++++++++++++++++++++ 2 files changed, 395 insertions(+), 11 deletions(-) create mode 100644 architecture/evm/eth_call_test.go diff --git a/architecture/evm/eth_call.go b/architecture/evm/eth_call.go index ac7e4d848..69097f908 100644 --- a/architecture/evm/eth_call.go +++ b/architecture/evm/eth_call.go @@ -2,31 +2,119 @@ package evm import ( "context" + "fmt" + "sync" "github.com/erpc/erpc/common" ) +// Global batcher manager for network-level Multicall3 batching +var ( + globalBatcherManager *BatcherManager + batcherManagerOnce sync.Once +) + +// GetBatcherManager returns the global batcher manager. +func GetBatcherManager() *BatcherManager { + batcherManagerOnce.Do(func() { + globalBatcherManager = NewBatcherManager() + }) + return globalBatcherManager +} + +// networkForwarder wraps a Network to implement Forwarder interface. +type networkForwarder struct { + network common.Network +} + +func (f *networkForwarder) Forward(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return f.network.Forward(ctx, req) +} + func projectPreForward_eth_call(ctx context.Context, network common.Network, nq *common.NormalizedRequest) (bool, *common.NormalizedResponse, error) { jrq, err := nq.JsonRpcRequest() if err != nil { return false, nil, nil } + // Normalize params: ensure block param is present jrq.RLock() - if len(jrq.Params) != 1 { - jrq.RUnlock() + paramsLen := len(jrq.Params) + jrq.RUnlock() + + if paramsLen == 0 { return false, nil, nil } - jrq.RUnlock() - // Some upstreams require the block number to be specified as a parameter. - jrq.Lock() - jrq.Params = []interface{}{ - jrq.Params[0], - "latest", + // Add "latest" block param if missing (only 1 param) + if paramsLen == 1 { + jrq.Lock() + jrq.Params = append(jrq.Params, "latest") + jrq.Unlock() } - jrq.Unlock() - resp, err := network.Forward(ctx, nq) - return true, resp, err + // Check if Multicall3 aggregation is enabled + cfg := network.Config() + if cfg == nil || cfg.Evm == nil || cfg.Evm.Multicall3Aggregation == nil || !cfg.Evm.Multicall3Aggregation.Enabled { + // Batching disabled, use normal forward + resp, err := network.Forward(ctx, nq) + return true, resp, err + } + + aggCfg := cfg.Evm.Multicall3Aggregation + + // Check eligibility for batching + eligible, _ := IsEligibleForBatching(nq, aggCfg) + if !eligible { + // Not eligible, forward normally + resp, err := network.Forward(ctx, nq) + return true, resp, err + } + + // Extract call info for batching key + _, _, blockRef, err := ExtractCallInfo(nq) + if err != nil { + resp, err := network.Forward(ctx, nq) + return true, resp, err + } + + // Build batching key + projectId := network.ProjectId() + if projectId == "" { + projectId = fmt.Sprintf("network:%s", network.Id()) + } + + userId := "" + if aggCfg.AllowCrossUserBatching == nil || !*aggCfg.AllowCrossUserBatching { + userId = nq.UserId() + } + + key := BatchingKey{ + ProjectId: projectId, + NetworkId: network.Id(), + BlockRef: blockRef, + DirectivesKey: DeriveDirectivesKey(nq.Directives()), + UserId: userId, + } + + // Get or create batcher for this network + mgr := GetBatcherManager() + forwarder := &networkForwarder{network: network} + batcher := mgr.GetOrCreate(network.Id(), aggCfg, forwarder) + + // Enqueue request + entry, bypass, err := batcher.Enqueue(ctx, key, nq) + if err != nil || bypass { + // Bypass batching, forward normally + resp, err := network.Forward(ctx, nq) + return true, resp, err + } + + // Wait for batch result + select { + case result := <-entry.ResultCh: + return true, result.Response, result.Error + case <-ctx.Done(): + return true, nil, ctx.Err() + } } diff --git a/architecture/evm/eth_call_test.go b/architecture/evm/eth_call_test.go new file mode 100644 index 000000000..60c01710f --- /dev/null +++ b/architecture/evm/eth_call_test.go @@ -0,0 +1,296 @@ +package evm + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" +) + +// mockNetworkForEthCall implements common.Network for testing eth_call pre-forward hook +type mockNetworkForEthCall struct { + networkId string + projectId string + cfg *common.NetworkConfig + forwardFn func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) + mu sync.Mutex + callCount int +} + +func (m *mockNetworkForEthCall) Id() string { return m.networkId } +func (m *mockNetworkForEthCall) Label() string { return m.networkId } +func (m *mockNetworkForEthCall) ProjectId() string { return m.projectId } +func (m *mockNetworkForEthCall) Architecture() common.NetworkArchitecture { return common.ArchitectureEvm } +func (m *mockNetworkForEthCall) Config() *common.NetworkConfig { return m.cfg } +func (m *mockNetworkForEthCall) Logger() *zerolog.Logger { return nil } +func (m *mockNetworkForEthCall) GetMethodMetrics(method string) common.TrackedMetrics { return nil } +func (m *mockNetworkForEthCall) Forward(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + m.mu.Lock() + m.callCount++ + m.mu.Unlock() + if m.forwardFn != nil { + return m.forwardFn(ctx, req) + } + return nil, nil +} +func (m *mockNetworkForEthCall) GetFinality(ctx context.Context, req *common.NormalizedRequest, resp *common.NormalizedResponse) common.DataFinalityState { + return common.DataFinalityStateUnknown +} +func (m *mockNetworkForEthCall) EvmHighestLatestBlockNumber(ctx context.Context) int64 { return 0 } +func (m *mockNetworkForEthCall) EvmHighestFinalizedBlockNumber(ctx context.Context) int64 { return 0 } +func (m *mockNetworkForEthCall) EvmLeaderUpstream(ctx context.Context) common.Upstream { return nil } + +func (m *mockNetworkForEthCall) GetCallCount() int { + m.mu.Lock() + defer m.mu.Unlock() + return m.callCount +} + +func TestProjectPreForward_eth_call_Batching(t *testing.T) { + // Create valid multicall response for 2 calls + // Each result: success=true with some return data + results := []Multicall3Result{ + {Success: true, ReturnData: []byte{0xde, 0xad, 0xbe, 0xef}}, + {Success: true, ReturnData: []byte{0xca, 0xfe, 0xba, 0xbe}}, + } + encodedResult := encodeAggregate3Results(results) + resultHex := "0x" + hexEncode(encodedResult) + + jrr, err := common.NewJsonRpcResponse(nil, resultHex, nil) + require.NoError(t, err) + mockResp := common.NewNormalizedResponse().WithJsonRpcResponse(jrr) + + cfg := &common.NetworkConfig{ + Evm: &common.EvmNetworkConfig{ + ChainId: 1, + Multicall3Aggregation: &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 50, + MinWaitMs: 5, + MaxCalls: 20, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + CachePerCall: util.BoolPtr(false), + }, + }, + } + cfg.Evm.Multicall3Aggregation.SetDefaults() + + network := &mockNetworkForEthCall{ + networkId: "evm:1", + projectId: "test-project", + cfg: cfg, + forwardFn: func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return mockResp, nil + }, + } + + ctx := context.Background() + + // Create two requests with different targets + // Use only 1 param initially (no block param) - the pre-forward hook should add "latest" + jrq1 := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x1111111111111111111111111111111111111111", + "data": "0x01020304", + }, + }) + jrq1.ID = "req1" + req1 := common.NewNormalizedRequestFromJsonRpcRequest(jrq1) + req1.SetNetwork(network) + + jrq2 := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x2222222222222222222222222222222222222222", + "data": "0x05060708", + }, + }) + jrq2.ID = "req2" + req2 := common.NewNormalizedRequestFromJsonRpcRequest(jrq2) + req2.SetNetwork(network) + + // Both should be batched into one call + var resp1, resp2 *common.NormalizedResponse + var err1, err2 error + done := make(chan struct{}, 2) + + go func() { + _, resp1, err1 = projectPreForward_eth_call(ctx, network, req1) + done <- struct{}{} + }() + + go func() { + _, resp2, err2 = projectPreForward_eth_call(ctx, network, req2) + done <- struct{}{} + }() + + // Wait with timeout + for i := 0; i < 2; i++ { + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("timeout waiting for batched requests") + } + } + + require.NoError(t, err1) + require.NoError(t, err2) + require.NotNil(t, resp1) + require.NotNil(t, resp2) + + // Should have been batched into ONE call + require.Equal(t, 1, network.GetCallCount(), "requests should be batched into one multicall") +} + +func TestProjectPreForward_eth_call_NoBatching_Disabled(t *testing.T) { + // Test that requests are forwarded normally when batching is disabled + jrr, _ := common.NewJsonRpcResponse(nil, "0xdeadbeef", nil) + mockResp := common.NewNormalizedResponse().WithJsonRpcResponse(jrr) + + cfg := &common.NetworkConfig{ + Evm: &common.EvmNetworkConfig{ + ChainId: 1, + // Multicall3Aggregation is nil - batching disabled + }, + } + + network := &mockNetworkForEthCall{ + networkId: "evm:1", + projectId: "test-project", + cfg: cfg, + forwardFn: func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return mockResp, nil + }, + } + + ctx := context.Background() + // Use only 1 param - the pre-forward hook will add "latest" and forward + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x1111111111111111111111111111111111111111", + "data": "0x01020304", + }, + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + req.SetNetwork(network) + + handled, resp, err := projectPreForward_eth_call(ctx, network, req) + require.True(t, handled) + require.NoError(t, err) + require.NotNil(t, resp) + require.Equal(t, 1, network.GetCallCount()) +} + +func TestProjectPreForward_eth_call_NoBatching_Ineligible(t *testing.T) { + // Test that ineligible requests are forwarded normally (e.g., with "from" field) + jrr, _ := common.NewJsonRpcResponse(nil, "0xdeadbeef", nil) + mockResp := common.NewNormalizedResponse().WithJsonRpcResponse(jrr) + + cfg := &common.NetworkConfig{ + Evm: &common.EvmNetworkConfig{ + ChainId: 1, + Multicall3Aggregation: &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 50, + MinWaitMs: 5, + MaxCalls: 20, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + }, + }, + } + cfg.Evm.Multicall3Aggregation.SetDefaults() + + network := &mockNetworkForEthCall{ + networkId: "evm:1", + projectId: "test-project", + cfg: cfg, + forwardFn: func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return mockResp, nil + }, + } + + ctx := context.Background() + // Request with "from" field is ineligible for batching + // Use only 1 param so the pre-forward hook processes it + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x1111111111111111111111111111111111111111", + "data": "0x01020304", + "from": "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + }, + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + req.SetNetwork(network) + + handled, resp, err := projectPreForward_eth_call(ctx, network, req) + require.True(t, handled) + require.NoError(t, err) + require.NotNil(t, resp) + require.Equal(t, 1, network.GetCallCount()) +} + +func TestProjectPreForward_eth_call_AddsBlockParam(t *testing.T) { + // Test that missing block param is added as "latest" + jrr, _ := common.NewJsonRpcResponse(nil, "0xdeadbeef", nil) + mockResp := common.NewNormalizedResponse().WithJsonRpcResponse(jrr) + + var capturedReq *common.NormalizedRequest + cfg := &common.NetworkConfig{ + Evm: &common.EvmNetworkConfig{ + ChainId: 1, + // Batching disabled to test block param normalization + }, + } + + network := &mockNetworkForEthCall{ + networkId: "evm:1", + projectId: "test-project", + cfg: cfg, + forwardFn: func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + capturedReq = req + return mockResp, nil + }, + } + + ctx := context.Background() + // Request with only 1 param (no block param) + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x1111111111111111111111111111111111111111", + "data": "0x01020304", + }, + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + req.SetNetwork(network) + + handled, _, err := projectPreForward_eth_call(ctx, network, req) + require.True(t, handled) + require.NoError(t, err) + + // Verify block param was added + capturedJrq, err := capturedReq.JsonRpcRequest() + require.NoError(t, err) + require.Len(t, capturedJrq.Params, 2) + require.Equal(t, "latest", capturedJrq.Params[1]) +} + +// hexEncode is a helper to encode bytes as hex string +func hexEncode(b []byte) string { + const hexChars = "0123456789abcdef" + dst := make([]byte, len(b)*2) + for i, v := range b { + dst[i*2] = hexChars[v>>4] + dst[i*2+1] = hexChars[v&0x0f] + } + return string(dst) +} From 965bdbd8d3b3a7085b1f5e601f59261b8203691e Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 15 Jan 2026 18:54:28 +0100 Subject: [PATCH 25/53] feat: add Multicall3 batching observability metrics New metrics for network-level Multicall3 batching: - multicall3_batch_size: histogram of unique calls per batch - multicall3_batch_wait_ms: histogram of request wait times - multicall3_queue_len: gauge of current queue depth - multicall3_queue_overflow_total: counter for bypass events - multicall3_dedupe_total: counter for deduplicated requests Co-Authored-By: Claude Opus 4.5 --- architecture/evm/multicall3_batcher.go | 31 ++++++++++++++++++++++++ telemetry/metrics.go | 33 ++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/architecture/evm/multicall3_batcher.go b/architecture/evm/multicall3_batcher.go index 58127b33d..a4c721670 100644 --- a/architecture/evm/multicall3_batcher.go +++ b/architecture/evm/multicall3_batcher.go @@ -10,6 +10,7 @@ import ( "time" "github.com/erpc/erpc/common" + "github.com/erpc/erpc/telemetry" ) // Forwarder is the interface for forwarding requests through the network layer. @@ -75,11 +76,13 @@ func (b *Batcher) Enqueue(ctx context.Context, key BatchingKey, req *common.Norm // Check caps if b.queueSize >= int64(b.cfg.MaxQueueSize) { + telemetry.MetricMulticall3QueueOverflowTotal.WithLabelValues(key.NetworkId, "queue_full").Inc() return nil, true, nil // bypass: queue full } if len(b.batches) >= b.cfg.MaxPendingBatches { // Check if this is a new batch key if _, exists := b.batches[key.String()]; !exists { + telemetry.MetricMulticall3QueueOverflowTotal.WithLabelValues(key.NetworkId, "max_batches").Inc() return nil, true, nil // bypass: too many pending batches } } @@ -165,6 +168,7 @@ func (b *Batcher) Enqueue(ctx context.Context, key BatchingKey, req *common.Norm batch.mu.Unlock() b.queueSize++ + telemetry.MetricMulticall3QueueLen.WithLabelValues(key.NetworkId).Inc() return entry, false, nil } @@ -222,10 +226,16 @@ func (b *Batcher) flush(keyStr string, batch *Batch) { } b.mu.Unlock() + // Decrement queue length metric + telemetry.MetricMulticall3QueueLen.WithLabelValues(batch.Key.NetworkId).Sub(float64(len(entries))) + if len(entries) == 0 { return } + // Capture flush time for wait time calculations + flushTime := time.Now() + // Build ordered unique calls list (maintains insertion order via CallKeys map iteration order) // We need to build unique calls in the order they were first seen type uniqueCall struct { @@ -245,6 +255,27 @@ func (b *Batcher) flush(keyStr string, batch *Batch) { } } + // Emit batching metrics + projectId := batch.Key.ProjectId + networkId := batch.Key.NetworkId + + // Record batch size (unique calls) + telemetry.MetricMulticall3BatchSize.WithLabelValues(projectId, networkId).Observe(float64(len(uniqueCalls))) + + // Record wait time for each entry + for _, entry := range entries { + waitMs := float64(flushTime.Sub(entry.CreatedAt).Milliseconds()) + telemetry.MetricMulticall3BatchWaitMs.WithLabelValues(projectId, networkId).Observe(waitMs) + } + + // Record dedupe count if there were duplicates + totalEntries := len(entries) + uniqueCount := len(uniqueCalls) + if totalEntries > uniqueCount { + dedupeCount := totalEntries - uniqueCount + telemetry.MetricMulticall3DedupeTotal.WithLabelValues(projectId, networkId).Add(float64(dedupeCount)) + } + // Build requests for BuildMulticall3Request requests := make([]*common.NormalizedRequest, len(uniqueCalls)) for i, uc := range uniqueCalls { diff --git a/telemetry/metrics.go b/telemetry/metrics.go index d77758163..3810e8fe8 100644 --- a/telemetry/metrics.go +++ b/telemetry/metrics.go @@ -429,6 +429,39 @@ var ( Name: "multicall3_cache_hits_total", Help: "Total number of per-call cache hits in multicall3 batch aggregation.", }, []string{"project", "network"}) + + // Network-level Multicall3 batching metrics + MetricMulticall3BatchSize = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "erpc", + Name: "multicall3_batch_size", + Help: "Number of unique calls per Multicall3 batch.", + Buckets: []float64{1, 2, 5, 10, 15, 20, 30, 50}, + }, []string{"project", "network"}) + + MetricMulticall3BatchWaitMs = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "erpc", + Name: "multicall3_batch_wait_ms", + Help: "Time requests waited in batch before flush (milliseconds).", + Buckets: []float64{1, 2, 5, 10, 15, 20, 25, 30, 50}, + }, []string{"project", "network"}) + + MetricMulticall3QueueLen = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "erpc", + Name: "multicall3_queue_len", + Help: "Current number of requests queued for batching.", + }, []string{"network"}) + + MetricMulticall3QueueOverflowTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "multicall3_queue_overflow_total", + Help: "Total number of requests that bypassed batching due to queue overflow.", + }, []string{"network", "reason"}) + + MetricMulticall3DedupeTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "multicall3_dedupe_total", + Help: "Total number of deduplicated requests within batches.", + }, []string{"project", "network"}) ) var DefaultHistogramBuckets = []float64{ From 7f8252f52774e4adf32c0d8c76673772ec28e417 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 15 Jan 2026 22:10:19 +0100 Subject: [PATCH 26/53] fix: improve multicall3 batch handling and caching - Respect skip-cache-read directive before cache probe in batch eth_call - Move network rate limit acquisition after cache-miss filtering to avoid wasting permits on cache hits (project rate limit still acquired early for billing) - Mark multicall3 aggregate request as composite type to disable hedging and prevent duplicate batches Co-Authored-By: Claude Opus 4.5 --- architecture/evm/eth_call.go | 47 +- architecture/evm/eth_call_test.go | 1 + architecture/evm/eth_getBlockByNumber_test.go | 4 + architecture/evm/multicall3.go | 8 +- architecture/evm/multicall3_batcher.go | 274 +- architecture/evm/multicall3_batcher_test.go | 646 +++- architecture/evm/multicall3_manager.go | 40 +- architecture/evm/multicall3_manager_test.go | 44 +- architecture/evm/multicall3_test.go | 4 +- common/network.go | 1 + docs/design/multicall3-batching.md | 2 +- .../2026-01-15-multicall3-network-batching.md | 2674 +++++++++++++++++ erpc/http_batch_eth_call.go | 34 +- erpc/networks.go | 4 + telemetry/metrics.go | 16 +- 15 files changed, 3694 insertions(+), 105 deletions(-) create mode 100644 docs/plans/2026-01-15-multicall3-network-batching.md diff --git a/architecture/evm/eth_call.go b/architecture/evm/eth_call.go index 69097f908..97752f8af 100644 --- a/architecture/evm/eth_call.go +++ b/architecture/evm/eth_call.go @@ -22,6 +22,14 @@ func GetBatcherManager() *BatcherManager { return globalBatcherManager } +// ShutdownBatcherManager shuts down the global batcher manager. +// Should be called during application shutdown. +func ShutdownBatcherManager() { + if globalBatcherManager != nil { + globalBatcherManager.Shutdown() + } +} + // networkForwarder wraps a Network to implement Forwarder interface. type networkForwarder struct { network common.Network @@ -31,6 +39,28 @@ func (f *networkForwarder) Forward(ctx context.Context, req *common.NormalizedRe return f.network.Forward(ctx, req) } +func (f *networkForwarder) SetCache(ctx context.Context, req *common.NormalizedRequest, resp *common.NormalizedResponse) error { + cache := f.network.Cache() + if cache == nil || cache.IsObjectNull() { + return nil + } + return cache.Set(ctx, req, resp) +} + +// projectPreForward_eth_call is the pre-forward hook for eth_call requests. +// It handles Multicall3 batching when enabled, aggregating multiple eth_call requests +// into a single Multicall3 call for improved throughput. +// +// Returns: +// - handled: true if the request was handled (either batched or forwarded directly) +// - response: the response if handled, nil otherwise +// - error: any error that occurred +// +// The function will forward the request directly (bypassing batching) when: +// - Multicall3 aggregation is disabled in config +// - The request is not eligible for batching (has gas/value/from fields, etc.) +// - The batcher queue is full or at capacity +// - The request's deadline is too tight for batching func projectPreForward_eth_call(ctx context.Context, network common.Network, nq *common.NormalizedRequest) (bool, *common.NormalizedResponse, error) { jrq, err := nq.JsonRpcRequest() if err != nil { @@ -64,9 +94,15 @@ func projectPreForward_eth_call(ctx context.Context, network common.Network, nq aggCfg := cfg.Evm.Multicall3Aggregation // Check eligibility for batching - eligible, _ := IsEligibleForBatching(nq, aggCfg) + eligible, reason := IsEligibleForBatching(nq, aggCfg) if !eligible { // Not eligible, forward normally + if logger := network.Logger(); logger != nil { + logger.Debug(). + Str("reason", reason). + Str("method", "eth_call"). + Msg("request not eligible for multicall3 batching") + } resp, err := network.Forward(ctx, nq) return true, resp, err } @@ -97,10 +133,15 @@ func projectPreForward_eth_call(ctx context.Context, network common.Network, nq UserId: userId, } - // Get or create batcher for this network + // Get or create batcher for this project+network mgr := GetBatcherManager() forwarder := &networkForwarder{network: network} - batcher := mgr.GetOrCreate(network.Id(), aggCfg, forwarder) + batcher := mgr.GetOrCreate(projectId, network.Id(), aggCfg, forwarder, network.Logger()) + if batcher == nil { + // Batching disabled, forward normally + resp, err := network.Forward(ctx, nq) + return true, resp, err + } // Enqueue request entry, bypass, err := batcher.Enqueue(ctx, key, nq) diff --git a/architecture/evm/eth_call_test.go b/architecture/evm/eth_call_test.go index 60c01710f..ed09260b3 100644 --- a/architecture/evm/eth_call_test.go +++ b/architecture/evm/eth_call_test.go @@ -44,6 +44,7 @@ func (m *mockNetworkForEthCall) GetFinality(ctx context.Context, req *common.Nor func (m *mockNetworkForEthCall) EvmHighestLatestBlockNumber(ctx context.Context) int64 { return 0 } func (m *mockNetworkForEthCall) EvmHighestFinalizedBlockNumber(ctx context.Context) int64 { return 0 } func (m *mockNetworkForEthCall) EvmLeaderUpstream(ctx context.Context) common.Upstream { return nil } +func (m *mockNetworkForEthCall) Cache() common.CacheDAL { return nil } func (m *mockNetworkForEthCall) GetCallCount() int { m.mu.Lock() diff --git a/architecture/evm/eth_getBlockByNumber_test.go b/architecture/evm/eth_getBlockByNumber_test.go index 8e5559409..685dffc4a 100644 --- a/architecture/evm/eth_getBlockByNumber_test.go +++ b/architecture/evm/eth_getBlockByNumber_test.go @@ -70,6 +70,10 @@ func (t *testNetwork) GetFinality(ctx context.Context, req *common.NormalizedReq return common.DataFinalityStateFinalized } +func (t *testNetwork) Cache() common.CacheDAL { + return nil +} + func TestEnforceNonNullTaggedBlocks(t *testing.T) { t.Run("TaggedBlockWithEnforcementDisabled_ReturnsNull", func(t *testing.T) { // Create a request with a block tag ("pending") and directive disabled diff --git a/architecture/evm/multicall3.go b/architecture/evm/multicall3.go index f5faee1c4..39dba73e9 100644 --- a/architecture/evm/multicall3.go +++ b/architecture/evm/multicall3.go @@ -312,14 +312,16 @@ func ShouldFallbackMulticall3(err error) bool { errStr := strings.ToLower(err.Error()) // Check for indicators that the multicall3 contract doesn't exist. // Different providers use different error messages, so we match multiple patterns. + // NOTE: We intentionally do NOT include "execution reverted" as that pattern is too + // broad and would match legitimate contract reverts. Legitimate reverts should NOT + // trigger fallback - they would also revert when called individually. contractUnavailablePatterns := []string{ "contract not found", "no code at address", - "execution reverted", // empty revert from non-existent contract "code is empty", "not a contract", - "invalid opcode", // can indicate missing contract - "missing trie node", // pre-deployment block query + "invalid opcode", // can indicate missing contract + "missing trie node", // pre-deployment block query "does not exist", "account not found", } diff --git a/architecture/evm/multicall3_batcher.go b/architecture/evm/multicall3_batcher.go index a4c721670..bd96294f2 100644 --- a/architecture/evm/multicall3_batcher.go +++ b/architecture/evm/multicall3_batcher.go @@ -11,35 +11,61 @@ import ( "github.com/erpc/erpc/common" "github.com/erpc/erpc/telemetry" + "github.com/rs/zerolog" ) // Forwarder is the interface for forwarding requests through the network layer. type Forwarder interface { Forward(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) + // SetCache writes a response to the cache for a request. + // Returns nil if caching is disabled or not available. + SetCache(ctx context.Context, req *common.NormalizedRequest, resp *common.NormalizedResponse) error } // Batcher aggregates eth_call requests into Multicall3 batches. type Batcher struct { - cfg *common.Multicall3AggregationConfig - forwarder Forwarder - batches map[string]*Batch // keyed by BatchingKey.String() - mu sync.RWMutex - queueSize int64 // counter for backpressure - shutdown chan struct{} - wg sync.WaitGroup + cfg *common.Multicall3AggregationConfig + forwarder Forwarder + logger *zerolog.Logger + batches map[string]*Batch // keyed by BatchingKey.String() + mu sync.RWMutex + queueSize int64 // counter for backpressure + shutdown chan struct{} + shutdownOnce sync.Once + wg sync.WaitGroup } // NewBatcher creates a new Multicall3 batcher. -func NewBatcher(cfg *common.Multicall3AggregationConfig, forwarder Forwarder) *Batcher { +// Returns nil if cfg is nil or disabled - callers should check the return value. +// The logger parameter is optional (can be nil) - if nil, debug logging is disabled. +func NewBatcher(cfg *common.Multicall3AggregationConfig, forwarder Forwarder, logger *zerolog.Logger) *Batcher { + if cfg == nil || !cfg.Enabled { + return nil + } b := &Batcher{ cfg: cfg, forwarder: forwarder, + logger: logger, batches: make(map[string]*Batch), shutdown: make(chan struct{}), } return b } +// logBypass logs a debug message when a request bypasses batching. +// Does nothing if logger is nil. +func (b *Batcher) logBypass(key BatchingKey, reason string) { + if b.logger == nil { + return + } + b.logger.Debug(). + Str("reason", reason). + Str("projectId", key.ProjectId). + Str("networkId", key.NetworkId). + Str("blockRef", key.BlockRef). + Msg("request bypassing multicall3 batching") +} + // Enqueue adds a request to a batch. Returns: // - entry: the batch entry (nil if bypass) // - bypass: true if request should be forwarded individually @@ -48,12 +74,14 @@ func (b *Batcher) Enqueue(ctx context.Context, key BatchingKey, req *common.Norm // Extract call info target, callData, _, err := ExtractCallInfo(req) if err != nil { + b.logBypass(key, fmt.Sprintf("extract_call_info_error: %v", err)) return nil, true, err } // Derive call key for deduplication callKey, err := DeriveCallKey(req) if err != nil { + b.logBypass(key, fmt.Sprintf("derive_call_key_error: %v", err)) return nil, true, err } @@ -67,7 +95,7 @@ func (b *Batcher) Enqueue(ctx context.Context, key BatchingKey, req *common.Norm now := time.Now() minWait := time.Duration(b.cfg.MinWaitMs) * time.Millisecond if deadline.Before(now.Add(minWait)) { - // Deadline too tight, bypass batching + b.logBypass(key, "deadline_too_tight") return nil, true, nil } @@ -76,14 +104,16 @@ func (b *Batcher) Enqueue(ctx context.Context, key BatchingKey, req *common.Norm // Check caps if b.queueSize >= int64(b.cfg.MaxQueueSize) { - telemetry.MetricMulticall3QueueOverflowTotal.WithLabelValues(key.NetworkId, "queue_full").Inc() - return nil, true, nil // bypass: queue full + telemetry.MetricMulticall3QueueOverflowTotal.WithLabelValues(key.ProjectId, key.NetworkId, "queue_full").Inc() + b.logBypass(key, "queue_full") + return nil, true, nil } if len(b.batches) >= b.cfg.MaxPendingBatches { // Check if this is a new batch key if _, exists := b.batches[key.String()]; !exists { - telemetry.MetricMulticall3QueueOverflowTotal.WithLabelValues(key.NetworkId, "max_batches").Inc() - return nil, true, nil // bypass: too many pending batches + telemetry.MetricMulticall3QueueOverflowTotal.WithLabelValues(key.ProjectId, key.NetworkId, "max_batches").Inc() + b.logBypass(key, "max_pending_batches") + return nil, true, nil } } @@ -91,6 +121,11 @@ func (b *Batcher) Enqueue(ctx context.Context, key BatchingKey, req *common.Norm keyStr := key.String() batch, exists := b.batches[keyStr] if !exists { + // If OnlyIfPending is true, bypass batching when no batch is pending + if b.cfg.OnlyIfPending { + b.logBypass(key, "only_if_pending_no_batch") + return nil, true, nil + } flushTime := now.Add(time.Duration(b.cfg.WindowMs) * time.Millisecond) batch = NewBatch(key, flushTime) b.batches[keyStr] = batch @@ -104,6 +139,12 @@ func (b *Batcher) Enqueue(ctx context.Context, key BatchingKey, req *common.Norm batch.mu.Lock() if batch.Flushing { batch.mu.Unlock() + // If OnlyIfPending is true, bypass batching since the current batch is flushing + // and we'd need to create a new one + if b.cfg.OnlyIfPending { + b.logBypass(key, "only_if_pending_batch_flushing") + return nil, true, nil + } // Create new batch for this key flushTime := now.Add(time.Duration(b.cfg.WindowMs) * time.Millisecond) batch = NewBatch(key, flushTime) @@ -120,7 +161,8 @@ func (b *Batcher) Enqueue(ctx context.Context, key BatchingKey, req *common.Norm if _, isDupe := batch.CallKeys[callKey]; !isDupe { if uniqueCalls >= b.cfg.MaxCalls { batch.mu.Unlock() - return nil, true, nil // bypass: batch full + b.logBypass(key, "batch_full") + return nil, true, nil } } @@ -134,7 +176,8 @@ func (b *Batcher) Enqueue(ctx context.Context, key BatchingKey, req *common.Norm if _, isDupe := batch.CallKeys[callKey]; !isDupe { if currentSize+len(callData) > b.cfg.MaxCalldataBytes { batch.mu.Unlock() - return nil, true, nil // bypass: calldata too large + b.logBypass(key, "calldata_too_large") + return nil, true, nil } } @@ -164,11 +207,17 @@ func (b *Batcher) Enqueue(ctx context.Context, key BatchingKey, req *common.Norm if batch.FlushTime.Before(minFlush) { batch.FlushTime = minFlush } + // Notify the flush goroutine that FlushTime was shortened + select { + case batch.notifyCh <- struct{}{}: + default: + // Already has a pending notification + } } batch.mu.Unlock() b.queueSize++ - telemetry.MetricMulticall3QueueLen.WithLabelValues(key.NetworkId).Inc() + telemetry.MetricMulticall3QueueLen.WithLabelValues(key.ProjectId, key.NetworkId).Inc() return entry, false, nil } @@ -193,8 +242,14 @@ func (b *Batcher) scheduleFlush(keyStr string, batch *Batch) { case <-timer.C: b.flush(keyStr, batch) return + case <-batch.notifyCh: + // FlushTime was shortened, stop current timer and recalculate + timer.Stop() + continue case <-b.shutdown: timer.Stop() + // On shutdown, flush the batch with error to avoid orphaned entries + b.flushWithShutdownError(keyStr, batch) return } } @@ -227,7 +282,7 @@ func (b *Batcher) flush(keyStr string, batch *Batch) { b.mu.Unlock() // Decrement queue length metric - telemetry.MetricMulticall3QueueLen.WithLabelValues(batch.Key.NetworkId).Sub(float64(len(entries))) + telemetry.MetricMulticall3QueueLen.WithLabelValues(batch.Key.ProjectId, batch.Key.NetworkId).Sub(float64(len(entries))) if len(entries) == 0 { return @@ -292,9 +347,24 @@ func (b *Batcher) flush(keyStr string, batch *Batch) { // Mark request as composite type for metrics/tracing mcReq.SetCompositeType(common.CompositeTypeMulticall3) - // Use any entry context (they all share same project/network) - ctx := entries[0].Ctx - if ctx == nil { + // Create a context with the earliest deadline from all entries. + // We don't use a single entry's context to avoid canceling the whole batch + // if one entry's context is canceled. + var earliestDeadline time.Time + for _, entry := range entries { + if entry.Deadline.IsZero() { + continue + } + if earliestDeadline.IsZero() || entry.Deadline.Before(earliestDeadline) { + earliestDeadline = entry.Deadline + } + } + var ctx context.Context + var cancel context.CancelFunc + if !earliestDeadline.IsZero() { + ctx, cancel = context.WithDeadline(context.Background(), earliestDeadline) + defer cancel() + } else { ctx = context.Background() } @@ -303,9 +373,14 @@ func (b *Batcher) flush(keyStr string, batch *Batch) { if err != nil { // Check if we should fallback to individual requests if ShouldFallbackMulticall3(err) { - b.fallbackIndividual(entries) + telemetry.MetricMulticall3FallbackTotal.WithLabelValues(projectId, networkId, "forward_error").Inc() + b.fallbackIndividual(entries, projectId, networkId) return } + // Wrap context errors with batching context for better debugging (after fallback check) + if ctx.Err() != nil { + err = fmt.Errorf("multicall3 batch forward failed (batch size: %d): %w", len(uniqueCalls), err) + } b.deliverError(entries, err) return } @@ -315,7 +390,8 @@ func (b *Batcher) flush(keyStr string, batch *Batch) { if err != nil { // Check if we should fallback to individual requests if ShouldFallbackMulticall3(err) { - b.fallbackIndividual(entries) + telemetry.MetricMulticall3FallbackTotal.WithLabelValues(projectId, networkId, "decode_error").Inc() + b.fallbackIndividual(entries, projectId, networkId) return } b.deliverError(entries, err) @@ -328,6 +404,9 @@ func (b *Batcher) flush(keyStr string, batch *Batch) { return } + // Check if per-call caching is enabled (defaults to true) + cachePerCall := b.cfg.CachePerCall == nil || *b.cfg.CachePerCall + // Map results to entries, fanning out deduplicated results for i, uc := range uniqueCalls { result := results[i] @@ -336,17 +415,41 @@ func (b *Batcher) flush(keyStr string, batch *Batch) { if result.Success { // Build success response for each entry resultHex := "0x" + hex.EncodeToString(result.ReturnData) + + // For per-call caching, we only need to cache once per unique call + // Use the first entry's request for the cache write + var cachedOnce bool + for _, entry := range entriesForCall { jrr, err := common.NewJsonRpcResponse(entry.Request.ID(), resultHex, nil) if err != nil { - entry.ResultCh <- BatchResult{Error: err} + b.sendResult(entry, BatchResult{Error: err}) continue } resp := common.NewNormalizedResponse().WithRequest(entry.Request).WithJsonRpcResponse(jrr) // Propagate upstream metadata from multicall response resp.SetUpstream(mcResp.Upstream()) resp.SetFromCache(mcResp.FromCache()) - entry.ResultCh <- BatchResult{Response: resp} + + // Write to cache once per unique call (not once per duplicate entry) + if cachePerCall && !cachedOnce { + // Use background context for cache write to avoid request deadline issues + if err := b.forwarder.SetCache(context.Background(), entry.Request, resp); err != nil { + // Cache write failures are non-critical but we track them for observability + telemetry.MetricMulticall3CacheWriteErrorsTotal.WithLabelValues(projectId, networkId).Inc() + if b.logger != nil { + b.logger.Debug(). + Err(err). + Str("projectId", projectId). + Str("networkId", networkId). + Str("callKey", uc.callKey). + Msg("multicall3 per-call cache write failed") + } + } + cachedOnce = true + } + + b.sendResult(entry, BatchResult{Response: resp}) } } else { // Build error for reverted call with proper JSON-RPC format @@ -365,7 +468,7 @@ func (b *Batcher) flush(keyStr string, batch *Batch) { ), ) for _, entry := range entriesForCall { - entry.ResultCh <- BatchResult{Error: revertErr} + b.sendResult(entry, BatchResult{Error: revertErr}) } } } @@ -412,41 +515,113 @@ func (b *Batcher) decodeMulticallResponse(resp *common.NormalizedResponse) ([]Mu return DecodeMulticall3Aggregate3Result(resultBytes) } +// sendResult safely sends a result to an entry, skipping if context is cancelled. +// Returns true if sent, false if skipped due to cancelled context. +func (b *Batcher) sendResult(entry *BatchEntry, result BatchResult) bool { + // Check if the entry's context is cancelled - no point sending if caller has given up + select { + case <-entry.Ctx.Done(): + return false // Caller abandoned request, skip sending + default: + } + // ResultCh is buffered size 1, so this won't block + entry.ResultCh <- result + return true +} + // deliverError sends an error to all entries in a batch. +// Skips entries whose context has been cancelled. func (b *Batcher) deliverError(entries []*BatchEntry, err error) { result := BatchResult{Error: err} for _, entry := range entries { - select { - case entry.ResultCh <- result: - default: - } + b.sendResult(entry, result) } } // fallbackIndividual forwards each entry individually when multicall3 fails. -// Uses parallel goroutines for concurrent forwarding. -func (b *Batcher) fallbackIndividual(entries []*BatchEntry) { +// Uses parallel goroutines for concurrent forwarding with panic recovery. +// Records metrics for each fallback request outcome. +func (b *Batcher) fallbackIndividual(entries []*BatchEntry, projectId, networkId string) { var wg sync.WaitGroup for _, entry := range entries { wg.Add(1) go func(e *BatchEntry) { defer wg.Done() - resp, err := b.forwarder.Forward(e.Ctx, e.Request) + defer func() { + if r := recover(); r != nil { + // Panic in forwarder - send error to entry + err := fmt.Errorf("panic in fallback forward: %v", r) + b.sendResult(e, BatchResult{Error: err}) + telemetry.MetricMulticall3FallbackRequestsTotal.WithLabelValues(projectId, networkId, "panic").Inc() + } + }() + // Skip if context is already cancelled select { - case e.ResultCh <- BatchResult{Response: resp, Error: err}: + case <-e.Ctx.Done(): + b.sendResult(e, BatchResult{Error: e.Ctx.Err()}) + telemetry.MetricMulticall3FallbackRequestsTotal.WithLabelValues(projectId, networkId, "cancelled").Inc() + return default: } + resp, err := b.forwarder.Forward(e.Ctx, e.Request) + b.sendResult(e, BatchResult{Response: resp, Error: err}) + if err != nil { + telemetry.MetricMulticall3FallbackRequestsTotal.WithLabelValues(projectId, networkId, "error").Inc() + } else { + telemetry.MetricMulticall3FallbackRequestsTotal.WithLabelValues(projectId, networkId, "success").Inc() + } }(entry) } wg.Wait() } // Shutdown stops the batcher and waits for pending operations. +// Safe to call multiple times. func (b *Batcher) Shutdown() { - close(b.shutdown) + b.shutdownOnce.Do(func() { + close(b.shutdown) + }) b.wg.Wait() } +// flushWithShutdownError delivers shutdown errors to all pending entries. +func (b *Batcher) flushWithShutdownError(keyStr string, batch *Batch) { + batch.mu.Lock() + if batch.Flushing { + batch.mu.Unlock() + return + } + batch.Flushing = true + entries := batch.Entries + batch.mu.Unlock() + + // Remove from active batches + b.mu.Lock() + if b.batches[keyStr] == batch { + delete(b.batches, keyStr) + } + entriesLen := int64(len(entries)) + if b.queueSize >= entriesLen { + b.queueSize -= entriesLen + } else { + b.queueSize = 0 + } + b.mu.Unlock() + + // Decrement queue length metric + telemetry.MetricMulticall3QueueLen.WithLabelValues(batch.Key.ProjectId, batch.Key.NetworkId).Sub(float64(len(entries))) + + // Deliver shutdown error to all entries + shutdownErr := common.NewErrJsonRpcExceptionInternal( + 0, + common.JsonRpcErrorServerSideException, + "batcher shutting down", + nil, + nil, + ) + b.deliverError(entries, shutdownErr) +} + // DirectivesKeyVersion should be bumped when the set of directives // included in the key changes. This prevents cross-node key mismatches. const DirectivesKeyVersion = 1 @@ -512,14 +687,14 @@ func DeriveCallKey(req *common.NormalizedRequest) (string, error) { // BatchEntry represents a request waiting in a batch. type BatchEntry struct { - Ctx context.Context - Request *common.NormalizedRequest - CallKey string - Target []byte - CallData []byte - ResultCh chan BatchResult - CreatedAt time.Time - Deadline time.Time + Ctx context.Context // Original request context (for individual fallback) + Request *common.NormalizedRequest // The original eth_call request + CallKey string // Deduplication key (target + calldata + blockRef) + Target []byte // Contract address (20 bytes) + CallData []byte // Encoded function call data + ResultCh chan BatchResult // Channel to receive the result (buffered, size 1) + CreatedAt time.Time // When the entry was created (for wait time metrics) + Deadline time.Time // Deadline from context (for deadline-aware flushing) } // BatchResult is the outcome delivered to a waiting request. @@ -529,13 +704,15 @@ type BatchResult struct { } // Batch holds pending requests for a single batching key. +// All entries in a batch share the same project, network, block reference, directives, and user ID. type Batch struct { - Key BatchingKey - Entries []*BatchEntry - CallKeys map[string][]*BatchEntry // for deduplication - FlushTime time.Time - Flushing bool - mu sync.Mutex + Key BatchingKey // Composite key identifying this batch + Entries []*BatchEntry // All entries (may include duplicates) + CallKeys map[string][]*BatchEntry // Map from call key to entries (for deduplication) + FlushTime time.Time // When this batch should be flushed (deadline-aware) + Flushing bool // True once flush has started (prevents double-flush) + notifyCh chan struct{} // Signals flush time was shortened (buffered, size 1) + mu sync.Mutex // Protects all fields } func NewBatch(key BatchingKey, flushTime time.Time) *Batch { @@ -544,6 +721,7 @@ func NewBatch(key BatchingKey, flushTime time.Time) *Batch { Entries: make([]*BatchEntry, 0, 16), CallKeys: make(map[string][]*BatchEntry), FlushTime: flushTime, + notifyCh: make(chan struct{}, 1), } } diff --git a/architecture/evm/multicall3_batcher_test.go b/architecture/evm/multicall3_batcher_test.go index 95e7d96f5..5cf7521f2 100644 --- a/architecture/evm/multicall3_batcher_test.go +++ b/architecture/evm/multicall3_batcher_test.go @@ -7,6 +7,7 @@ import ( "strings" "sync" "testing" + "time" "github.com/erpc/erpc/common" "github.com/erpc/erpc/util" @@ -369,7 +370,7 @@ func TestBatcherEnqueueAndFlush(t *testing.T) { cfg.SetDefaults() ctx := context.Background() - batcher := NewBatcher(cfg, nil) // nil forwarder for now + batcher := NewBatcher(cfg, nil, nil) // nil forwarder for now // Create test requests jrq1 := common.NewJsonRpcRequest("eth_call", []interface{}{ @@ -434,7 +435,7 @@ func TestBatcherDeduplication(t *testing.T) { cfg.SetDefaults() ctx := context.Background() - batcher := NewBatcher(cfg, nil) + batcher := NewBatcher(cfg, nil, nil) // Two identical requests - using the same jrq to ensure call key consistency // (JSON serialization of map[string]interface{} can have non-deterministic key order) @@ -487,7 +488,7 @@ func TestBatcherCapsEnforcement(t *testing.T) { cfg.SetDefaults() ctx := context.Background() - batcher := NewBatcher(cfg, nil) + batcher := NewBatcher(cfg, nil, nil) key := BatchingKey{ ProjectId: "test-project", @@ -529,10 +530,11 @@ func TestBatcherCapsEnforcement(t *testing.T) { // mockForwarder implements Forwarder for testing type mockForwarder struct { - response *common.NormalizedResponse - err error - called int - mu sync.Mutex + response *common.NormalizedResponse + err error + called int + cacheWrites int + mu sync.Mutex } func (m *mockForwarder) Forward(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { @@ -542,6 +544,13 @@ func (m *mockForwarder) Forward(ctx context.Context, req *common.NormalizedReque return m.response, m.err } +func (m *mockForwarder) SetCache(ctx context.Context, req *common.NormalizedRequest, resp *common.NormalizedResponse) error { + m.mu.Lock() + defer m.mu.Unlock() + m.cacheWrites++ + return nil +} + func TestBatcherFlushAndResultMapping(t *testing.T) { // Create valid multicall3 result with 2 calls // Each call returns success=true with some data @@ -571,7 +580,7 @@ func TestBatcherFlushAndResultMapping(t *testing.T) { } cfg.SetDefaults() - batcher := NewBatcher(cfg, forwarder) + batcher := NewBatcher(cfg, forwarder, nil) ctx := context.Background() key := BatchingKey{ @@ -654,7 +663,7 @@ func TestBatcherFlushDeduplication(t *testing.T) { } cfg.SetDefaults() - batcher := NewBatcher(cfg, forwarder) + batcher := NewBatcher(cfg, forwarder, nil) ctx := context.Background() key := BatchingKey{ @@ -737,7 +746,7 @@ func TestBatcherFlushRevertHandling(t *testing.T) { } cfg.SetDefaults() - batcher := NewBatcher(cfg, forwarder) + batcher := NewBatcher(cfg, forwarder, nil) ctx := context.Background() key := BatchingKey{ @@ -825,7 +834,7 @@ func TestBatcherFlushFallbackOnMulticall3Unavailable(t *testing.T) { } cfg.SetDefaults() - batcher := NewBatcher(cfg, forwarder) + batcher := NewBatcher(cfg, forwarder, nil) ctx := context.Background() key := BatchingKey{ @@ -879,3 +888,618 @@ type mockForwarderFunc struct { func (m *mockForwarderFunc) Forward(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { return m.forwardFunc(ctx, req) } + +func (m *mockForwarderFunc) SetCache(ctx context.Context, req *common.NormalizedRequest, resp *common.NormalizedResponse) error { + return nil +} + +func TestBatcherCancellation(t *testing.T) { + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 100, + MinWaitMs: 50, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + } + cfg.SetDefaults() + + batcher := NewBatcher(cfg, nil, nil) + + ctx, cancel := context.WithCancel(context.Background()) + key := BatchingKey{ + ProjectId: "test", + NetworkId: "evm:1", + BlockRef: "latest", + } + + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{"to": "0x1234567890123456789012345678901234567890", "data": "0x01"}, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + + entry, bypass, err := batcher.Enqueue(ctx, key, req) + require.NoError(t, err) + require.False(t, bypass) + require.NotNil(t, entry) + + // Cancel before flush + cancel() + + // Batcher should shutdown gracefully + batcher.Shutdown() +} + +func TestBatcherDeadlineAwareness(t *testing.T) { + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 100, + MinWaitMs: 10, + SafetyMarginMs: 5, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + } + cfg.SetDefaults() + + batcher := NewBatcher(cfg, nil, nil) + + key := BatchingKey{ + ProjectId: "test", + NetworkId: "evm:1", + BlockRef: "latest", + } + + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{"to": "0x1234567890123456789012345678901234567890", "data": "0x01"}, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + + // Context with tight deadline - should bypass + tightCtx, cancel1 := context.WithDeadline(context.Background(), time.Now().Add(5*time.Millisecond)) + defer cancel1() + + _, bypass, err := batcher.Enqueue(tightCtx, key, req) + require.NoError(t, err) + require.True(t, bypass, "should bypass with tight deadline") + + // Context with reasonable deadline - should batch + normalCtx, cancel2 := context.WithDeadline(context.Background(), time.Now().Add(200*time.Millisecond)) + defer cancel2() + + _, bypass, err = batcher.Enqueue(normalCtx, key, req) + require.NoError(t, err) + require.False(t, bypass, "should batch with normal deadline") + + batcher.Shutdown() +} + +func TestBatcherConcurrentFlush(t *testing.T) { + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 10, + MinWaitMs: 1, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + CachePerCall: util.BoolPtr(false), + } + cfg.SetDefaults() + + // Mock forwarder that returns success for all batches + var callCount int + var mu sync.Mutex + forwarder := &mockForwarderFunc{ + forwardFunc: func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + mu.Lock() + callCount++ + mu.Unlock() + + // Return multicall3 results with 3 successful calls + results := []Multicall3Result{ + {Success: true, ReturnData: []byte{0xaa}}, + {Success: true, ReturnData: []byte{0xbb}}, + {Success: true, ReturnData: []byte{0xcc}}, + } + encodedResult := encodeAggregate3Results(results) + resultHex := "0x" + hex.EncodeToString(encodedResult) + jrr, _ := common.NewJsonRpcResponse(nil, resultHex, nil) + return common.NewNormalizedResponse().WithJsonRpcResponse(jrr), nil + }, + } + + batcher := NewBatcher(cfg, forwarder, nil) + + ctx := context.Background() + key := BatchingKey{ + ProjectId: "test", + NetworkId: "evm:1", + BlockRef: "latest", + } + + // Add first batch + for i := 0; i < 3; i++ { + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{"to": fmt.Sprintf("0x%040d", i), "data": "0x01"}, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + batcher.Enqueue(ctx, key, req) + } + + // Wait for first batch to start flushing + time.Sleep(15 * time.Millisecond) + + // Add more requests - should go to new batch + for i := 3; i < 6; i++ { + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{"to": fmt.Sprintf("0x%040d", i), "data": "0x01"}, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + _, bypass, _ := batcher.Enqueue(ctx, key, req) + _ = bypass // May bypass or create new batch - either is acceptable + } + + // Wait for all flushes to complete + time.Sleep(20 * time.Millisecond) + + batcher.Shutdown() + + // Verify at least one batch was processed + mu.Lock() + require.GreaterOrEqual(t, callCount, 1) + mu.Unlock() +} + +func TestNewBatcher_NilConfig(t *testing.T) { + forwarder := &mockForwarder{} + + // Test with nil config + batcher := NewBatcher(nil, forwarder, nil) + require.Nil(t, batcher, "NewBatcher should return nil for nil config") +} + +func TestNewBatcher_DisabledConfig(t *testing.T) { + forwarder := &mockForwarder{} + + // Test with disabled config + cfg := &common.Multicall3AggregationConfig{ + Enabled: false, + } + batcher := NewBatcher(cfg, forwarder, nil) + require.Nil(t, batcher, "NewBatcher should return nil for disabled config") +} + +func TestNewBatcher_EnabledConfig(t *testing.T) { + forwarder := &mockForwarder{} + + // Test with enabled config + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 50, + MinWaitMs: 5, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + } + cfg.SetDefaults() + + batcher := NewBatcher(cfg, forwarder, nil) + require.NotNil(t, batcher, "NewBatcher should return non-nil for enabled config") + batcher.Shutdown() +} + +// mockForwarderWithCacheError is a forwarder that returns errors from SetCache +type mockForwarderWithCacheError struct { + response *common.NormalizedResponse + cacheError error + mu sync.Mutex + called int +} + +func (m *mockForwarderWithCacheError) Forward(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + m.mu.Lock() + defer m.mu.Unlock() + m.called++ + return m.response, nil +} + +func (m *mockForwarderWithCacheError) SetCache(ctx context.Context, req *common.NormalizedRequest, resp *common.NormalizedResponse) error { + return m.cacheError +} + +func TestBatcher_CacheWriteError_DoesNotFailRequest(t *testing.T) { + // Create multicall response for 1 call + results := []Multicall3Result{ + {Success: true, ReturnData: []byte{0xde, 0xad, 0xbe, 0xef}}, + } + encodedResult := encodeAggregate3Results(results) + resultHex := "0x" + hex.EncodeToString(encodedResult) + + jrr, err := common.NewJsonRpcResponse(nil, resultHex, nil) + require.NoError(t, err) + mockResp := common.NewNormalizedResponse().WithJsonRpcResponse(jrr) + + forwarder := &mockForwarderWithCacheError{ + response: mockResp, + cacheError: fmt.Errorf("cache write failed"), + } + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 10, + MinWaitMs: 1, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + CachePerCall: util.BoolPtr(true), // Enable per-call caching + } + cfg.SetDefaults() + + batcher := NewBatcher(cfg, forwarder, nil) + require.NotNil(t, batcher) + defer batcher.Shutdown() + + ctx := context.Background() + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x1111111111111111111111111111111111111111", + "data": "0x01020304", + }, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + + entry, bypass, err := batcher.Enqueue(ctx, key, req) + require.NoError(t, err) + require.False(t, bypass) + + // Wait for result - should succeed despite cache write error + select { + case result := <-entry.ResultCh: + require.NoError(t, result.Error, "request should succeed despite cache write error") + require.NotNil(t, result.Response) + case <-time.After(2 * time.Second): + t.Fatal("timeout waiting for batched request") + } +} + +func TestBatcher_ContextDeadlineError_WrappedWithBatchContext(t *testing.T) { + // Create a forwarder that simulates internal timeout (not waiting on ctx) + forwarder := &mockForwarderFunc{ + forwardFunc: func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + // Simulate a slow response that triggers ctx deadline internally + // But return the error immediately so entry context is still valid for delivery + return nil, context.DeadlineExceeded + }, + } + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 10, + MinWaitMs: 1, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + SafetyMarginMs: 2, + } + cfg.SetDefaults() + + batcher := NewBatcher(cfg, forwarder, nil) + require.NotNil(t, batcher) + defer batcher.Shutdown() + + // Create context with deadline longer than batch window so entry is still valid at delivery + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(500*time.Millisecond)) + defer cancel() + + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x1111111111111111111111111111111111111111", + "data": "0x01020304", + }, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + + entry, bypass, err := batcher.Enqueue(ctx, key, req) + require.NoError(t, err) + require.False(t, bypass) + + // Wait for result + select { + case result := <-entry.ResultCh: + require.Error(t, result.Error) + // The error wrapping happens when ctx.Err() != nil at forward time + // Since we return DeadlineExceeded but ctx isn't actually expired yet, + // the wrapping won't happen. This tests the error delivery path. + require.Contains(t, result.Error.Error(), "deadline exceeded") + case <-time.After(2 * time.Second): + t.Fatal("timeout waiting for batched request") + } +} + +func TestBatcher_MaxCalldataBytes_Bypass(t *testing.T) { + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 100, + MinWaitMs: 5, + MaxCalls: 100, // High limit + MaxCalldataBytes: 100, // Very low limit for testing + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + } + cfg.SetDefaults() + + ctx := context.Background() + batcher := NewBatcher(cfg, nil, nil) + require.NotNil(t, batcher) + defer batcher.Shutdown() + + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + // First request with small calldata - should be batched + jrq1 := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x1111111111111111111111111111111111111111", + "data": "0x01020304", // 4 bytes + }, + "latest", + }) + req1 := common.NewNormalizedRequestFromJsonRpcRequest(jrq1) + + entry1, bypass1, err := batcher.Enqueue(ctx, key, req1) + require.NoError(t, err) + require.False(t, bypass1, "first request should be batched") + require.NotNil(t, entry1) + + // Second request with large calldata - should bypass due to MaxCalldataBytes + largeData := "0x" + strings.Repeat("aa", 200) // 200 bytes + jrq2 := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x2222222222222222222222222222222222222222", + "data": largeData, + }, + "latest", + }) + req2 := common.NewNormalizedRequestFromJsonRpcRequest(jrq2) + + _, bypass2, err := batcher.Enqueue(ctx, key, req2) + require.NoError(t, err) + require.True(t, bypass2, "second request should bypass due to MaxCalldataBytes") +} + +func TestBatcher_OnlyIfPending_NoBatch(t *testing.T) { + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 100, + MinWaitMs: 5, + MaxCalls: 100, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + OnlyIfPending: true, // Only batch if there's already a pending batch + } + cfg.SetDefaults() + + ctx := context.Background() + batcher := NewBatcher(cfg, nil, nil) + require.NotNil(t, batcher) + defer batcher.Shutdown() + + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + // First request should bypass - no pending batch exists + jrq1 := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x1111111111111111111111111111111111111111", + "data": "0x01020304", + }, + "latest", + }) + req1 := common.NewNormalizedRequestFromJsonRpcRequest(jrq1) + + _, bypass1, err := batcher.Enqueue(ctx, key, req1) + require.NoError(t, err) + require.True(t, bypass1, "first request should bypass when OnlyIfPending is true and no batch exists") +} + +func TestBatcher_OnlyIfPending_WithExistingBatch(t *testing.T) { + // Create valid multicall3 result with 2 calls + results := []Multicall3Result{ + {Success: true, ReturnData: []byte{0xaa}}, + {Success: true, ReturnData: []byte{0xbb}}, + } + encodedResult := encodeAggregate3Results(results) + resultHex := "0x" + hex.EncodeToString(encodedResult) + + jrr, err := common.NewJsonRpcResponse(nil, resultHex, nil) + require.NoError(t, err) + mockResp := common.NewNormalizedResponse().WithJsonRpcResponse(jrr) + + forwarder := &mockForwarder{response: mockResp} + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 100, + MinWaitMs: 5, + MaxCalls: 100, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + OnlyIfPending: false, // Start with false to create a batch + CachePerCall: util.BoolPtr(false), + } + cfg.SetDefaults() + + ctx := context.Background() + batcher := NewBatcher(cfg, forwarder, nil) + require.NotNil(t, batcher) + defer batcher.Shutdown() + + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + // First request creates a batch + jrq1 := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x1111111111111111111111111111111111111111", + "data": "0x01", + }, + "latest", + }) + jrq1.ID = "req1" + req1 := common.NewNormalizedRequestFromJsonRpcRequest(jrq1) + + entry1, bypass1, err := batcher.Enqueue(ctx, key, req1) + require.NoError(t, err) + require.False(t, bypass1, "first request should create batch") + require.NotNil(t, entry1) + + // Second request should join the existing batch + jrq2 := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x2222222222222222222222222222222222222222", + "data": "0x02", + }, + "latest", + }) + jrq2.ID = "req2" + req2 := common.NewNormalizedRequestFromJsonRpcRequest(jrq2) + + entry2, bypass2, err := batcher.Enqueue(ctx, key, req2) + require.NoError(t, err) + require.False(t, bypass2, "second request should join existing batch") + require.NotNil(t, entry2) + + // Wait for results + result1 := <-entry1.ResultCh + result2 := <-entry2.ResultCh + + require.NoError(t, result1.Error) + require.NoError(t, result2.Error) +} + +func TestBatcher_DuplicateCallsShareResult(t *testing.T) { + // Create result with 1 unique call (both requests have same target+data) + results := []Multicall3Result{ + {Success: true, ReturnData: []byte{0xde, 0xad, 0xbe, 0xef}}, + } + encodedResult := encodeAggregate3Results(results) + resultHex := "0x" + hex.EncodeToString(encodedResult) + + jrr, err := common.NewJsonRpcResponse(nil, resultHex, nil) + require.NoError(t, err) + mockResp := common.NewNormalizedResponse().WithJsonRpcResponse(jrr) + + forwarder := &mockForwarder{response: mockResp} + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 10, + MinWaitMs: 1, + MaxCalls: 100, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + CachePerCall: util.BoolPtr(false), + } + cfg.SetDefaults() + + batcher := NewBatcher(cfg, forwarder, nil) + require.NotNil(t, batcher) + defer batcher.Shutdown() + + ctx := context.Background() + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + // Three identical requests - should all share the same result + entries := make([]*BatchEntry, 3) + for i := 0; i < 3; i++ { + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x1111111111111111111111111111111111111111", + "data": "0x12345678", + }, + "latest", + }) + jrq.ID = fmt.Sprintf("req%d", i) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + + entry, bypass, err := batcher.Enqueue(ctx, key, req) + require.NoError(t, err) + require.False(t, bypass) + entries[i] = entry + } + + // All entries should have the same callKey (deduplication) + require.Equal(t, entries[0].CallKey, entries[1].CallKey) + require.Equal(t, entries[1].CallKey, entries[2].CallKey) + + // Wait for all results + for i, entry := range entries { + result := <-entry.ResultCh + require.NoError(t, result.Error, "entry %d should succeed", i) + require.NotNil(t, result.Response) + + jrrResult, err := result.Response.JsonRpcResponse() + require.NoError(t, err) + require.Equal(t, "\"0xdeadbeef\"", jrrResult.GetResultString()) + } + + // Forwarder should only be called once (all requests batched into single multicall) + forwarder.mu.Lock() + require.Equal(t, 1, forwarder.called) + forwarder.mu.Unlock() +} diff --git a/architecture/evm/multicall3_manager.go b/architecture/evm/multicall3_manager.go index a563c5cec..0d6f2465f 100644 --- a/architecture/evm/multicall3_manager.go +++ b/architecture/evm/multicall3_manager.go @@ -4,11 +4,15 @@ import ( "sync" "github.com/erpc/erpc/common" + "github.com/rs/zerolog" ) -// BatcherManager manages per-network Multicall3 batchers. +// BatcherManager manages per-project+network Multicall3 batchers. +// It provides thread-safe access to batchers keyed by "projectId|networkId". +// Each batcher handles batching for a specific project and network combination +// to ensure proper isolation between projects. type BatcherManager struct { - batchers map[string]*Batcher + batchers map[string]*Batcher // Key: "projectId|networkId" mu sync.RWMutex } @@ -19,10 +23,15 @@ func NewBatcherManager() *BatcherManager { } } -// GetOrCreate returns the batcher for a network, creating one if needed. -func (m *BatcherManager) GetOrCreate(networkId string, cfg *common.Multicall3AggregationConfig, forwarder Forwarder) *Batcher { +// GetOrCreate returns the batcher for a project+network, creating one if needed. +// The key combines projectId and networkId to ensure project isolation. +// Returns nil if batching is disabled (cfg is nil or cfg.Enabled is false). +// The logger parameter is optional (can be nil) - if nil, debug logging is disabled. +func (m *BatcherManager) GetOrCreate(projectId, networkId string, cfg *common.Multicall3AggregationConfig, forwarder Forwarder, logger *zerolog.Logger) *Batcher { + key := projectId + "|" + networkId + m.mu.RLock() - if b, ok := m.batchers[networkId]; ok { + if b, ok := m.batchers[key]; ok { m.mu.RUnlock() return b } @@ -32,20 +41,25 @@ func (m *BatcherManager) GetOrCreate(networkId string, cfg *common.Multicall3Agg defer m.mu.Unlock() // Double-check after acquiring write lock - if b, ok := m.batchers[networkId]; ok { + if b, ok := m.batchers[key]; ok { return b } - batcher := NewBatcher(cfg, forwarder) - m.batchers[networkId] = batcher + batcher := NewBatcher(cfg, forwarder, logger) + if batcher == nil { + // Don't store nil batchers - batching is disabled for this config + return nil + } + m.batchers[key] = batcher return batcher } -// Get returns the batcher for a network, or nil if not exists. -func (m *BatcherManager) Get(networkId string) *Batcher { +// Get returns the batcher for a project+network, or nil if not exists. +func (m *BatcherManager) Get(projectId, networkId string) *Batcher { + key := projectId + "|" + networkId m.mu.RLock() defer m.mu.RUnlock() - return m.batchers[networkId] + return m.batchers[key] } // Shutdown stops all batchers. @@ -54,7 +68,9 @@ func (m *BatcherManager) Shutdown() { defer m.mu.Unlock() for _, b := range m.batchers { - b.Shutdown() + if b != nil { + b.Shutdown() + } } m.batchers = make(map[string]*Batcher) } diff --git a/architecture/evm/multicall3_manager_test.go b/architecture/evm/multicall3_manager_test.go index e89dc055b..be01b2110 100644 --- a/architecture/evm/multicall3_manager_test.go +++ b/architecture/evm/multicall3_manager_test.go @@ -26,18 +26,22 @@ func TestBatcherManagerGetOrCreate(t *testing.T) { forwarder := &mockForwarder{} - // Get batcher for network - batcher1 := mgr.GetOrCreate("evm:1", cfg, forwarder) + // Get batcher for project+network + batcher1 := mgr.GetOrCreate("project1", "evm:1", cfg, forwarder, nil) require.NotNil(t, batcher1) - // Same network should return same batcher - batcher2 := mgr.GetOrCreate("evm:1", cfg, forwarder) + // Same project+network should return same batcher + batcher2 := mgr.GetOrCreate("project1", "evm:1", cfg, forwarder, nil) require.Same(t, batcher1, batcher2) - // Different network should return different batcher - batcher3 := mgr.GetOrCreate("evm:137", cfg, forwarder) + // Different network (same project) should return different batcher + batcher3 := mgr.GetOrCreate("project1", "evm:137", cfg, forwarder, nil) require.NotSame(t, batcher1, batcher3) + // Different project (same network) should return different batcher + batcher4 := mgr.GetOrCreate("project2", "evm:1", cfg, forwarder, nil) + require.NotSame(t, batcher1, batcher4) + mgr.Shutdown() } @@ -62,7 +66,7 @@ func TestBatcherManagerConcurrency(t *testing.T) { wg.Add(1) go func(idx int) { defer wg.Done() - batchers[idx] = mgr.GetOrCreate("evm:1", cfg, forwarder) + batchers[idx] = mgr.GetOrCreate("project1", "evm:1", cfg, forwarder, nil) }(i) } wg.Wait() @@ -89,21 +93,25 @@ func TestBatcherManagerGet(t *testing.T) { forwarder := &mockForwarder{} // Get before create should return nil - batcher := mgr.Get("evm:1") + batcher := mgr.Get("project1", "evm:1") require.Nil(t, batcher) // Create batcher - created := mgr.GetOrCreate("evm:1", cfg, forwarder) + created := mgr.GetOrCreate("project1", "evm:1", cfg, forwarder, nil) require.NotNil(t, created) // Get after create should return the same batcher - retrieved := mgr.Get("evm:1") + retrieved := mgr.Get("project1", "evm:1") require.Same(t, created, retrieved) // Get for different network should return nil - other := mgr.Get("evm:137") + other := mgr.Get("project1", "evm:137") require.Nil(t, other) + // Get for different project should return nil + otherProject := mgr.Get("project2", "evm:1") + require.Nil(t, otherProject) + mgr.Shutdown() } @@ -120,16 +128,16 @@ func TestBatcherManagerShutdown(t *testing.T) { forwarder := &mockForwarder{} - // Create multiple batchers - mgr.GetOrCreate("evm:1", cfg, forwarder) - mgr.GetOrCreate("evm:137", cfg, forwarder) - mgr.GetOrCreate("evm:42161", cfg, forwarder) + // Create multiple batchers across projects and networks + mgr.GetOrCreate("project1", "evm:1", cfg, forwarder, nil) + mgr.GetOrCreate("project1", "evm:137", cfg, forwarder, nil) + mgr.GetOrCreate("project2", "evm:1", cfg, forwarder, nil) // Shutdown should clean up all batchers mgr.Shutdown() // After shutdown, batchers map should be empty - require.Nil(t, mgr.Get("evm:1")) - require.Nil(t, mgr.Get("evm:137")) - require.Nil(t, mgr.Get("evm:42161")) + require.Nil(t, mgr.Get("project1", "evm:1")) + require.Nil(t, mgr.Get("project1", "evm:137")) + require.Nil(t, mgr.Get("project2", "evm:1")) } diff --git a/architecture/evm/multicall3_test.go b/architecture/evm/multicall3_test.go index d7e31dad3..2fd49d80e 100644 --- a/architecture/evm/multicall3_test.go +++ b/architecture/evm/multicall3_test.go @@ -435,9 +435,9 @@ func TestShouldFallbackMulticall3(t *testing.T) { want: true, }, { - name: "execution exception with execution reverted", + name: "execution exception with execution reverted - no fallback", err: common.NewErrEndpointExecutionException(errors.New("execution reverted")), - want: true, + want: false, // Generic reverts should NOT trigger fallback - they would also revert individually }, { name: "execution exception with code is empty", diff --git a/common/network.go b/common/network.go index 44a843c67..660fac5cf 100644 --- a/common/network.go +++ b/common/network.go @@ -25,6 +25,7 @@ type Network interface { GetMethodMetrics(method string) TrackedMetrics Forward(ctx context.Context, nq *NormalizedRequest) (*NormalizedResponse, error) GetFinality(ctx context.Context, req *NormalizedRequest, resp *NormalizedResponse) DataFinalityState + Cache() CacheDAL // TODO Move to EvmNetwork interface? EvmHighestLatestBlockNumber(ctx context.Context) int64 diff --git a/docs/design/multicall3-batching.md b/docs/design/multicall3-batching.md index 1cc2d399a..56f8dee8e 100644 --- a/docs/design/multicall3-batching.md +++ b/docs/design/multicall3-batching.md @@ -1,6 +1,6 @@ # Multicall3 Batching (Network-Level) -Status: Proposed +Status: Implemented ## Context The current Multicall3 batching lives in the HTTP batch handler. It only applies diff --git a/docs/plans/2026-01-15-multicall3-network-batching.md b/docs/plans/2026-01-15-multicall3-network-batching.md new file mode 100644 index 000000000..2a53b2aba --- /dev/null +++ b/docs/plans/2026-01-15-multicall3-network-batching.md @@ -0,0 +1,2674 @@ +# Multicall3 Network-Level Batching Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Move Multicall3 batching from the HTTP layer to the network layer, enabling batching across all entrypoints (HTTP single/batch + gRPC). + +**Architecture:** Create a concurrent batcher that aggregates `eth_call` requests sharing the same batching key (projectId + networkId + blockRef + directivesKey + optionally userId). Requests are enqueued with deadline-aware flush windows, deduplicated by callKey, and forwarded as a single Multicall3 call. Results fan out to all waiters. Fallback to individual calls on Multicall3 failure. + +**Tech Stack:** Go concurrency primitives (sync.Map, channels, mutexes), existing evm.BuildMulticall3Request/DecodeMulticall3Aggregate3Result, prometheus metrics. + +**Reference:** Design doc at `docs/design/multicall3-batching.md` + +--- + +## Phase 1: Configuration Extension + +### Task 1.1: Add Multicall3AggregationConfig Type + +**Files:** +- Modify: `common/config.go:1531-1536` +- Test: `common/config_test.go` (add section) + +**Step 1: Write the failing test** + +Add to `common/config_test.go`: + +```go +func TestMulticall3AggregationConfigYAML(t *testing.T) { + yamlStr := ` +evm: + chainId: 1 + multicall3Aggregation: + enabled: true + windowMs: 25 + minWaitMs: 2 + safetyMarginMs: 2 + maxCalls: 20 + maxCalldataBytes: 64000 + maxQueueSize: 1000 + maxPendingBatches: 200 + cachePerCall: true + allowCrossUserBatching: true + allowPendingTagBatching: false +` + var cfg NetworkConfig + err := yaml.Unmarshal([]byte(yamlStr), &cfg) + require.NoError(t, err) + require.NotNil(t, cfg.Evm) + require.NotNil(t, cfg.Evm.Multicall3Aggregation) + require.True(t, cfg.Evm.Multicall3Aggregation.Enabled) + require.Equal(t, 25, cfg.Evm.Multicall3Aggregation.WindowMs) + require.Equal(t, 20, cfg.Evm.Multicall3Aggregation.MaxCalls) +} + +func TestMulticall3AggregationConfigBoolBackcompat(t *testing.T) { + // Test backward compatibility with bool value + yamlStr := ` +evm: + chainId: 1 + multicall3Aggregation: true +` + var cfg NetworkConfig + err := yaml.Unmarshal([]byte(yamlStr), &cfg) + require.NoError(t, err) + require.NotNil(t, cfg.Evm) + require.NotNil(t, cfg.Evm.Multicall3Aggregation) + require.True(t, cfg.Evm.Multicall3Aggregation.Enabled) +} + +func TestMulticall3AggregationConfigDefaults(t *testing.T) { + cfg := &Multicall3AggregationConfig{Enabled: true} + cfg.SetDefaults() + require.Equal(t, 25, cfg.WindowMs) + require.Equal(t, 2, cfg.MinWaitMs) + require.Equal(t, 20, cfg.MaxCalls) + require.Equal(t, 64000, cfg.MaxCalldataBytes) +} +``` + +**Step 2: Run test to verify it fails** + +Run: `go test -v ./common -run TestMulticall3Aggregation` +Expected: FAIL - types don't exist + +**Step 3: Write minimal implementation** + +Add to `common/config.go` after line 1536: + +```go +// Multicall3AggregationConfig configures network-level batching of eth_call requests +// into Multicall3 aggregate calls. This batches requests across all entrypoints +// (HTTP single, HTTP batch, gRPC) rather than just JSON-RPC batch requests. +type Multicall3AggregationConfig struct { + // Enabled enables/disables Multicall3 aggregation. Default: true + Enabled bool `yaml:"enabled" json:"enabled"` + + // WindowMs is the maximum time (milliseconds) to wait for a batch to fill. + // Default: 25ms + WindowMs int `yaml:"windowMs,omitempty" json:"windowMs"` + + // MinWaitMs is the minimum time (milliseconds) to wait for additional requests + // to join a batch. Default: 2ms + MinWaitMs int `yaml:"minWaitMs,omitempty" json:"minWaitMs"` + + // SafetyMarginMs is subtracted from request deadlines when computing flush time. + // Default: min(2, MinWaitMs) + SafetyMarginMs int `yaml:"safetyMarginMs,omitempty" json:"safetyMarginMs"` + + // OnlyIfPending: if true, don't add latency unless a batch is already open. + // Default: false + OnlyIfPending bool `yaml:"onlyIfPending,omitempty" json:"onlyIfPending"` + + // MaxCalls is the maximum number of calls per batch. Default: 20 + MaxCalls int `yaml:"maxCalls,omitempty" json:"maxCalls"` + + // MaxCalldataBytes is the maximum total calldata size per batch. Default: 64000 + MaxCalldataBytes int `yaml:"maxCalldataBytes,omitempty" json:"maxCalldataBytes"` + + // MaxQueueSize is the maximum total enqueued requests across all batches. + // Default: 1000 + MaxQueueSize int `yaml:"maxQueueSize,omitempty" json:"maxQueueSize"` + + // MaxPendingBatches is the maximum number of distinct batch keys. + // Default: 200 + MaxPendingBatches int `yaml:"maxPendingBatches,omitempty" json:"maxPendingBatches"` + + // CachePerCall enables per-call cache writes after successful Multicall3. + // Default: true + CachePerCall *bool `yaml:"cachePerCall,omitempty" json:"cachePerCall"` + + // AllowCrossUserBatching: if true, requests from different users can share a batch. + // Default: true + AllowCrossUserBatching *bool `yaml:"allowCrossUserBatching,omitempty" json:"allowCrossUserBatching"` + + // AllowPendingTagBatching: if true, allow batching calls with "pending" block tag. + // Default: false + AllowPendingTagBatching bool `yaml:"allowPendingTagBatching,omitempty" json:"allowPendingTagBatching"` +} + +// SetDefaults applies default values to unset fields +func (c *Multicall3AggregationConfig) SetDefaults() { + if c.WindowMs == 0 { + c.WindowMs = 25 + } + if c.MinWaitMs == 0 { + c.MinWaitMs = 2 + } + if c.SafetyMarginMs == 0 { + c.SafetyMarginMs = min(2, c.MinWaitMs) + } + if c.MaxCalls == 0 { + c.MaxCalls = 20 + } + if c.MaxCalldataBytes == 0 { + c.MaxCalldataBytes = 64000 + } + if c.MaxQueueSize == 0 { + c.MaxQueueSize = 1000 + } + if c.MaxPendingBatches == 0 { + c.MaxPendingBatches = 200 + } + if c.CachePerCall == nil { + c.CachePerCall = &TRUE + } + if c.AllowCrossUserBatching == nil { + c.AllowCrossUserBatching = &TRUE + } +} + +// IsValid checks if the config values are valid +func (c *Multicall3AggregationConfig) IsValid() error { + if c.WindowMs <= 0 { + return fmt.Errorf("multicall3Aggregation.windowMs must be > 0") + } + if c.MinWaitMs < 0 { + return fmt.Errorf("multicall3Aggregation.minWaitMs must be >= 0") + } + if c.MinWaitMs > c.WindowMs { + return fmt.Errorf("multicall3Aggregation.minWaitMs must be <= windowMs") + } + if c.MaxCalls <= 1 { + return fmt.Errorf("multicall3Aggregation.maxCalls must be > 1") + } + if c.MaxCalldataBytes <= 0 { + return fmt.Errorf("multicall3Aggregation.maxCalldataBytes must be > 0") + } + if c.MaxQueueSize <= 0 { + return fmt.Errorf("multicall3Aggregation.maxQueueSize must be > 0") + } + return nil +} +``` + +**Step 4: Modify EvmNetworkConfig to use new type** + +Replace the `Multicall3Aggregation *bool` field in `EvmNetworkConfig` with: + +```go +// Multicall3Aggregation configures aggregating eth_call requests into Multicall3. +// Accepts either a boolean (backward compat) or a full config object. +// Default: enabled with default settings +Multicall3Aggregation *Multicall3AggregationConfig `yaml:"multicall3Aggregation,omitempty" json:"multicall3Aggregation,omitempty"` +``` + +**Step 5: Add UnmarshalYAML for backward compatibility** + +```go +func (c *Multicall3AggregationConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + // Try bool first (backward compat) + var boolVal bool + if err := unmarshal(&boolVal); err == nil { + c.Enabled = boolVal + if boolVal { + c.SetDefaults() + } + return nil + } + + // Try full config + type rawConfig Multicall3AggregationConfig + var raw rawConfig + if err := unmarshal(&raw); err != nil { + return err + } + *c = Multicall3AggregationConfig(raw) + if c.Enabled { + c.SetDefaults() + } + return nil +} +``` + +**Step 6: Run test to verify it passes** + +Run: `go test -v ./common -run TestMulticall3Aggregation` +Expected: PASS + +**Step 7: Commit** + +```bash +git add common/config.go common/config_test.go +git commit -m "$(cat <<'EOF' +feat: add Multicall3AggregationConfig for network-level batching + +Extends the evm.multicall3Aggregation config from a simple boolean +to a full configuration object with: +- windowMs, minWaitMs, safetyMarginMs for timing +- maxCalls, maxCalldataBytes for size limits +- maxQueueSize, maxPendingBatches for backpressure +- allowCrossUserBatching, allowPendingTagBatching flags + +Maintains backward compatibility with existing boolean configs. + +Co-Authored-By: Claude Opus 4.5 +EOF +)" +``` + +--- + +### Task 1.2: Add CompositeTypeMulticall3 Constant + +**Files:** +- Modify: `common/request.go:17-21` + +**Step 1: Add the constant** + +Add to `common/request.go` after line 20: + +```go +const ( + CompositeTypeNone = "none" + CompositeTypeLogsSplitOnError = "logs-split-on-error" + CompositeTypeLogsSplitProactive = "logs-split-proactive" + CompositeTypeMulticall3 = "multicall3" +) +``` + +**Step 2: Run existing tests** + +Run: `go test -v ./common -run TestComposite` +Expected: PASS (or no tests - that's ok) + +**Step 3: Commit** + +```bash +git add common/request.go +git commit -m "$(cat <<'EOF' +feat: add CompositeTypeMulticall3 constant + +Used to mark aggregated Multicall3 requests for metrics and +hedging logic. + +Co-Authored-By: Claude Opus 4.5 +EOF +)" +``` + +--- + +## Phase 2: Batcher Core Data Structures + +### Task 2.1: Create Batching Key and Entry Types + +**Files:** +- Create: `architecture/evm/multicall3_batcher.go` +- Test: `architecture/evm/multicall3_batcher_test.go` + +**Step 1: Write the failing test** + +Create `architecture/evm/multicall3_batcher_test.go`: + +```go +package evm + +import ( + "testing" + + "github.com/erpc/erpc/common" + "github.com/stretchr/testify/require" +) + +func TestBatchingKey(t *testing.T) { + key1 := BatchingKey{ + ProjectId: "proj1", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: "use-upstream=alchemy", + UserId: "", + } + key2 := BatchingKey{ + ProjectId: "proj1", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: "use-upstream=alchemy", + UserId: "", + } + key3 := BatchingKey{ + ProjectId: "proj1", + NetworkId: "evm:1", + BlockRef: "12345", + DirectivesKey: "use-upstream=alchemy", + UserId: "", + } + + require.Equal(t, key1.String(), key2.String()) + require.NotEqual(t, key1.String(), key3.String()) +} + +func TestDirectivesKeyDerivation(t *testing.T) { + dirs := &common.RequestDirectives{} + dirs.UseUpstream = "alchemy" + dirs.SkipCacheRead = true + dirs.RetryEmpty = true + + key := DeriveDirectivesKey(dirs) + require.Contains(t, key, "use-upstream=alchemy") + require.Contains(t, key, "skip-cache-read=true") + require.Contains(t, key, "retry-empty=true") +} + +func TestCallKeyDerivation(t *testing.T) { + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x1234567890123456789012345678901234567890", + "data": "0xabcdef", + }, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + + key, err := DeriveCallKey(req) + require.NoError(t, err) + require.NotEmpty(t, key) + + // Same request should produce same key + key2, err := DeriveCallKey(req) + require.NoError(t, err) + require.Equal(t, key, key2) +} +``` + +**Step 2: Run test to verify it fails** + +Run: `go test -v ./architecture/evm -run TestBatching` +Expected: FAIL - types don't exist + +**Step 3: Write minimal implementation** + +Create `architecture/evm/multicall3_batcher.go`: + +```go +package evm + +import ( + "context" + "fmt" + "sort" + "strings" + "sync" + "time" + + "github.com/erpc/erpc/common" +) + +// DirectivesKeyVersion should be bumped when the set of directives +// included in the key changes. This prevents cross-node key mismatches. +const DirectivesKeyVersion = 1 + +// BatchingKey uniquely identifies a batch for grouping eth_call requests. +type BatchingKey struct { + ProjectId string + NetworkId string + BlockRef string + DirectivesKey string + UserId string // empty if cross-user batching is allowed +} + +func (k BatchingKey) String() string { + return fmt.Sprintf("%s|%s|%s|%s|%s", k.ProjectId, k.NetworkId, k.BlockRef, k.DirectivesKey, k.UserId) +} + +// DeriveDirectivesKey creates a stable, versioned key from relevant directives. +// Only includes directives that affect batching behavior. +func DeriveDirectivesKey(dirs *common.RequestDirectives) string { + if dirs == nil { + return fmt.Sprintf("v%d:", DirectivesKeyVersion) + } + + parts := make([]string, 0, 5) + if dirs.UseUpstream != "" { + parts = append(parts, fmt.Sprintf("use-upstream=%s", dirs.UseUpstream)) + } + if dirs.SkipInterpolation { + parts = append(parts, "skip-interpolation=true") + } + if dirs.RetryEmpty { + parts = append(parts, "retry-empty=true") + } + if dirs.RetryPending { + parts = append(parts, "retry-pending=true") + } + if dirs.SkipCacheRead { + parts = append(parts, "skip-cache-read=true") + } + + sort.Strings(parts) + return fmt.Sprintf("v%d:%s", DirectivesKeyVersion, strings.Join(parts, ",")) +} + +// DeriveCallKey creates a unique key for deduplication within a batch. +// Uses the same derivation as cache keys for consistency. +func DeriveCallKey(req *common.NormalizedRequest) (string, error) { + if req == nil { + return "", fmt.Errorf("request is nil") + } + jrq, err := req.JsonRpcRequest() + if err != nil { + return "", err + } + + jrq.RLock() + method := jrq.Method + params := jrq.Params + jrq.RUnlock() + + // Use method + params as key (same as cache key derivation) + paramsJSON, err := common.SonicCfg.Marshal(params) + if err != nil { + return "", err + } + return fmt.Sprintf("%s:%s", method, string(paramsJSON)), nil +} + +// BatchEntry represents a request waiting in a batch. +type BatchEntry struct { + Ctx context.Context + Request *common.NormalizedRequest + CallKey string + Target []byte + CallData []byte + ResultCh chan BatchResult + CreatedAt time.Time + Deadline time.Time +} + +// BatchResult is the outcome delivered to a waiting request. +type BatchResult struct { + Response *common.NormalizedResponse + Error error +} + +// Batch holds pending requests for a single batching key. +type Batch struct { + Key BatchingKey + Entries []*BatchEntry + CallKeys map[string][]*BatchEntry // for deduplication + FlushTime time.Time + Flushing bool + mu sync.Mutex +} + +func NewBatch(key BatchingKey, flushTime time.Time) *Batch { + return &Batch{ + Key: key, + Entries: make([]*BatchEntry, 0, 16), + CallKeys: make(map[string][]*BatchEntry), + FlushTime: flushTime, + } +} +``` + +**Step 4: Run test to verify it passes** + +Run: `go test -v ./architecture/evm -run TestBatching` +Expected: PASS + +**Step 5: Commit** + +```bash +git add architecture/evm/multicall3_batcher.go architecture/evm/multicall3_batcher_test.go +git commit -m "$(cat <<'EOF' +feat: add Multicall3 batching key and entry types + +Core data structures for network-level Multicall3 batching: +- BatchingKey for grouping requests by project/network/block/directives +- DeriveDirectivesKey for stable versioned directive hashing +- DeriveCallKey for within-batch deduplication +- BatchEntry and Batch for holding pending requests + +Co-Authored-By: Claude Opus 4.5 +EOF +)" +``` + +--- + +### Task 2.2: Implement Eligibility Checking + +**Files:** +- Modify: `architecture/evm/multicall3_batcher.go` +- Test: `architecture/evm/multicall3_batcher_test.go` + +**Step 1: Write the failing test** + +Add to `architecture/evm/multicall3_batcher_test.go`: + +```go +func TestIsEligibleForBatching(t *testing.T) { + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + AllowPendingTagBatching: false, + } + cfg.SetDefaults() + + tests := []struct { + name string + method string + params []interface{} + eligible bool + reason string + }{ + { + name: "eligible basic eth_call", + method: "eth_call", + params: []interface{}{ + map[string]interface{}{"to": "0x1234567890123456789012345678901234567890", "data": "0xabcd"}, + "latest", + }, + eligible: true, + }, + { + name: "eligible with finalized tag", + method: "eth_call", + params: []interface{}{ + map[string]interface{}{"to": "0x1234567890123456789012345678901234567890", "data": "0xabcd"}, + "finalized", + }, + eligible: true, + }, + { + name: "ineligible - pending tag", + method: "eth_call", + params: []interface{}{ + map[string]interface{}{"to": "0x1234567890123456789012345678901234567890", "data": "0xabcd"}, + "pending", + }, + eligible: false, + reason: "pending tag not allowed", + }, + { + name: "ineligible - has from field", + method: "eth_call", + params: []interface{}{ + map[string]interface{}{ + "to": "0x1234567890123456789012345678901234567890", + "data": "0xabcd", + "from": "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + }, + "latest", + }, + eligible: false, + reason: "has from field", + }, + { + name: "ineligible - has value field", + method: "eth_call", + params: []interface{}{ + map[string]interface{}{ + "to": "0x1234567890123456789012345678901234567890", + "data": "0xabcd", + "value": "0x1", + }, + "latest", + }, + eligible: false, + reason: "has value field", + }, + { + name: "ineligible - has state override (3rd param)", + method: "eth_call", + params: []interface{}{ + map[string]interface{}{"to": "0x1234567890123456789012345678901234567890", "data": "0xabcd"}, + "latest", + map[string]interface{}{}, // state override + }, + eligible: false, + reason: "has state override", + }, + { + name: "ineligible - not eth_call", + method: "eth_getBalance", + params: []interface{}{"0x1234567890123456789012345678901234567890", "latest"}, + eligible: false, + reason: "not eth_call", + }, + { + name: "ineligible - already multicall (recursion guard)", + method: "eth_call", + params: []interface{}{ + map[string]interface{}{ + "to": "0xcA11bde05977b3631167028862bE2a173976CA11", // multicall3 address + "data": "0x82ad56cb", // aggregate3 selector + }, + "latest", + }, + eligible: false, + reason: "already multicall", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + jrq := common.NewJsonRpcRequest(tt.method, tt.params) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + + eligible, reason := IsEligibleForBatching(req, cfg) + require.Equal(t, tt.eligible, eligible, "reason: %s", reason) + if !tt.eligible { + require.Contains(t, reason, tt.reason) + } + }) + } +} +``` + +**Step 2: Run test to verify it fails** + +Run: `go test -v ./architecture/evm -run TestIsEligibleForBatching` +Expected: FAIL - function doesn't exist + +**Step 3: Write minimal implementation** + +Add to `architecture/evm/multicall3_batcher.go`: + +```go +// ineligibleCallFields are fields that make an eth_call ineligible for batching. +// Multicall3 aggregate3 only supports target + calldata, not gas/value/etc. +var ineligibleCallFields = []string{ + "from", "gas", "gasPrice", "maxFeePerGas", "maxPriorityFeePerGas", "value", +} + +// allowedBlockTags are block tags that can be batched by default. +var allowedBlockTags = map[string]bool{ + "latest": true, + "finalized": true, + "safe": true, + "earliest": true, +} + +// IsEligibleForBatching checks if a request can be batched via Multicall3. +// Returns (eligible, reason) where reason explains why not eligible. +func IsEligibleForBatching(req *common.NormalizedRequest, cfg *common.Multicall3AggregationConfig) (bool, string) { + if req == nil { + return false, "request is nil" + } + if cfg == nil || !cfg.Enabled { + return false, "batching disabled" + } + + jrq, err := req.JsonRpcRequest() + if err != nil { + return false, fmt.Sprintf("json-rpc error: %v", err) + } + + jrq.RLock() + method := strings.ToLower(jrq.Method) + params := jrq.Params + jrq.RUnlock() + + // Must be eth_call + if method != "eth_call" { + return false, "not eth_call" + } + + // Must have 1-2 params (call object, optional block) + if len(params) < 1 || len(params) > 2 { + return false, fmt.Sprintf("invalid param count: %d", len(params)) + } + + // Check for state override (3rd param) + if len(params) > 2 { + return false, "has state override" + } + + // Parse call object + callObj, ok := params[0].(map[string]interface{}) + if !ok { + return false, "invalid call object type" + } + + // Must have 'to' address + toVal, hasTo := callObj["to"] + if !hasTo { + return false, "missing to address" + } + toStr, ok := toVal.(string) + if !ok || toStr == "" { + return false, "invalid to address" + } + + // Check for ineligible fields + for _, field := range ineligibleCallFields { + if _, has := callObj[field]; has { + return false, fmt.Sprintf("has %s field", field) + } + } + + // Recursion guard: don't batch calls to multicall3 contract + if strings.EqualFold(toStr, multicall3Address) { + return false, "already multicall" + } + + // Check block tag + blockTag := "latest" + if len(params) >= 2 && params[1] != nil { + normalized, err := NormalizeBlockParam(params[1]) + if err != nil { + return false, fmt.Sprintf("invalid block param: %v", err) + } + blockTag = strings.ToLower(normalized) + } + + // Check if pending tag is allowed + if blockTag == "pending" && !cfg.AllowPendingTagBatching { + return false, "pending tag not allowed" + } + + return true, "" +} + +// ExtractCallInfo extracts target and calldata from an eligible eth_call request. +func ExtractCallInfo(req *common.NormalizedRequest) (target []byte, callData []byte, blockRef string, err error) { + jrq, err := req.JsonRpcRequest() + if err != nil { + return nil, nil, "", err + } + + jrq.RLock() + params := jrq.Params + jrq.RUnlock() + + callObj := params[0].(map[string]interface{}) + toStr := callObj["to"].(string) + + target, err = common.HexToBytes(toStr) + if err != nil { + return nil, nil, "", err + } + + dataHex := "0x" + if dataVal, ok := callObj["data"]; ok { + dataHex = dataVal.(string) + } else if inputVal, ok := callObj["input"]; ok { + dataHex = inputVal.(string) + } + + callData, err = common.HexToBytes(dataHex) + if err != nil { + return nil, nil, "", err + } + + blockRef = "latest" + if len(params) >= 2 && params[1] != nil { + blockRef, err = NormalizeBlockParam(params[1]) + if err != nil { + return nil, nil, "", err + } + } + + return target, callData, blockRef, nil +} +``` + +**Step 4: Run test to verify it passes** + +Run: `go test -v ./architecture/evm -run TestIsEligibleForBatching` +Expected: PASS + +**Step 5: Commit** + +```bash +git add architecture/evm/multicall3_batcher.go architecture/evm/multicall3_batcher_test.go +git commit -m "$(cat <<'EOF' +feat: add Multicall3 eligibility checking + +Implements IsEligibleForBatching to determine if an eth_call can be +aggregated into a Multicall3 batch: +- Method must be eth_call +- Call object must only have to + data/input fields +- No state overrides (3rd param) +- Recursion guard: don't batch calls to multicall3 contract +- Block tag restrictions (pending disabled by default) + +Also adds ExtractCallInfo helper to extract target/calldata/blockRef. + +Co-Authored-By: Claude Opus 4.5 +EOF +)" +``` + +--- + +### Task 2.3: Implement Batcher with Window and Caps + +**Files:** +- Modify: `architecture/evm/multicall3_batcher.go` +- Test: `architecture/evm/multicall3_batcher_test.go` + +**Step 1: Write the failing test** + +Add to `architecture/evm/multicall3_batcher_test.go`: + +```go +func TestBatcherEnqueueAndFlush(t *testing.T) { + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 50, + MinWaitMs: 5, + SafetyMarginMs: 2, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: &common.TRUE, + } + + ctx := context.Background() + batcher := NewBatcher(cfg, nil) // nil forwarder for now + + // Create test requests + jrq1 := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x1234567890123456789012345678901234567890", + "data": "0xabcdef01", + }, + "latest", + }) + req1 := common.NewNormalizedRequestFromJsonRpcRequest(jrq1) + + jrq2 := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x2234567890123456789012345678901234567890", + "data": "0xabcdef02", + }, + "latest", + }) + req2 := common.NewNormalizedRequestFromJsonRpcRequest(jrq2) + + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + // Enqueue first request + entry1, bypass1, err := batcher.Enqueue(ctx, key, req1) + require.NoError(t, err) + require.False(t, bypass1) + require.NotNil(t, entry1) + + // Enqueue second request + entry2, bypass2, err := batcher.Enqueue(ctx, key, req2) + require.NoError(t, err) + require.False(t, bypass2) + require.NotNil(t, entry2) + + // Check batch exists + batcher.mu.RLock() + batch, exists := batcher.batches[key.String()] + batcher.mu.RUnlock() + require.True(t, exists) + require.Len(t, batch.Entries, 2) + + // Cleanup + batcher.Shutdown() +} + +func TestBatcherDeduplication(t *testing.T) { + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 50, + MinWaitMs: 5, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: &common.TRUE, + } + + ctx := context.Background() + batcher := NewBatcher(cfg, nil) + + // Two identical requests + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x1234567890123456789012345678901234567890", + "data": "0xabcdef01", + }, + "latest", + }) + req1 := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + req2 := common.NewNormalizedRequestFromJsonRpcRequest(common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x1234567890123456789012345678901234567890", + "data": "0xabcdef01", + }, + "latest", + })) + + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + entry1, _, _ := batcher.Enqueue(ctx, key, req1) + entry2, _, _ := batcher.Enqueue(ctx, key, req2) + + // Both should share the same callKey slot + require.Equal(t, entry1.CallKey, entry2.CallKey) + + batcher.mu.RLock() + batch := batcher.batches[key.String()] + batcher.mu.RUnlock() + + // Two entries but deduplicated + require.Len(t, batch.Entries, 2) + require.Len(t, batch.CallKeys[entry1.CallKey], 2) + + batcher.Shutdown() +} + +func TestBatcherCapsEnforcement(t *testing.T) { + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 50, + MinWaitMs: 5, + MaxCalls: 2, // Very low limit + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: &common.TRUE, + } + + ctx := context.Background() + batcher := NewBatcher(cfg, nil) + + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + // Add requests up to cap + for i := 0; i < 2; i++ { + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": fmt.Sprintf("0x%040d", i), + "data": "0xabcdef", + }, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + _, bypass, err := batcher.Enqueue(ctx, key, req) + require.NoError(t, err) + require.False(t, bypass) + } + + // Next request should trigger bypass (caps reached) + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x9999999999999999999999999999999999999999", + "data": "0xabcdef", + }, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + _, bypass, err := batcher.Enqueue(ctx, key, req) + require.NoError(t, err) + require.True(t, bypass, "should bypass when caps reached") + + batcher.Shutdown() +} +``` + +**Step 2: Run test to verify it fails** + +Run: `go test -v ./architecture/evm -run TestBatcher` +Expected: FAIL - Batcher type doesn't exist + +**Step 3: Write minimal implementation** + +Add to `architecture/evm/multicall3_batcher.go`: + +```go +// Forwarder is the interface for forwarding requests through the network layer. +type Forwarder interface { + Forward(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) +} + +// Batcher aggregates eth_call requests into Multicall3 batches. +type Batcher struct { + cfg *common.Multicall3AggregationConfig + forwarder Forwarder + batches map[string]*Batch // keyed by BatchingKey.String() + mu sync.RWMutex + queueSize int64 // atomic counter for backpressure + shutdown chan struct{} + wg sync.WaitGroup +} + +// NewBatcher creates a new Multicall3 batcher. +func NewBatcher(cfg *common.Multicall3AggregationConfig, forwarder Forwarder) *Batcher { + b := &Batcher{ + cfg: cfg, + forwarder: forwarder, + batches: make(map[string]*Batch), + shutdown: make(chan struct{}), + } + return b +} + +// Enqueue adds a request to a batch. Returns: +// - entry: the batch entry (nil if bypass) +// - bypass: true if request should be forwarded individually +// - error: any error during processing +func (b *Batcher) Enqueue(ctx context.Context, key BatchingKey, req *common.NormalizedRequest) (*BatchEntry, bool, error) { + // Extract call info + target, callData, _, err := ExtractCallInfo(req) + if err != nil { + return nil, true, err + } + + // Derive call key for deduplication + callKey, err := DeriveCallKey(req) + if err != nil { + return nil, true, err + } + + // Calculate deadline from context + deadline, hasDeadline := ctx.Deadline() + if !hasDeadline { + deadline = time.Now().Add(time.Duration(b.cfg.WindowMs) * time.Millisecond) + } + + // Check if deadline is too tight + now := time.Now() + minWait := time.Duration(b.cfg.MinWaitMs) * time.Millisecond + if deadline.Before(now.Add(minWait)) { + // Deadline too tight, bypass batching + return nil, true, nil + } + + b.mu.Lock() + defer b.mu.Unlock() + + // Check caps + if b.queueSize >= int64(b.cfg.MaxQueueSize) { + return nil, true, nil // bypass: queue full + } + if len(b.batches) >= b.cfg.MaxPendingBatches { + // Check if this is a new batch key + if _, exists := b.batches[key.String()]; !exists { + return nil, true, nil // bypass: too many pending batches + } + } + + // Get or create batch + keyStr := key.String() + batch, exists := b.batches[keyStr] + if !exists { + flushTime := now.Add(time.Duration(b.cfg.WindowMs) * time.Millisecond) + batch = NewBatch(key, flushTime) + b.batches[keyStr] = batch + + // Start flush timer + b.wg.Add(1) + go b.scheduleFlush(keyStr, batch) + } + + // Check if batch is flushing - create new batch if so + batch.mu.Lock() + if batch.Flushing { + batch.mu.Unlock() + // Create new batch for this key + flushTime := now.Add(time.Duration(b.cfg.WindowMs) * time.Millisecond) + batch = NewBatch(key, flushTime) + b.batches[keyStr] = batch + + b.wg.Add(1) + go b.scheduleFlush(keyStr, batch) + + batch.mu.Lock() + } + + // Check if batch is at capacity (unique calls, not entries) + uniqueCalls := len(batch.CallKeys) + if _, isDupe := batch.CallKeys[callKey]; !isDupe { + if uniqueCalls >= b.cfg.MaxCalls { + batch.mu.Unlock() + return nil, true, nil // bypass: batch full + } + } + + // Check calldata size cap + currentSize := 0 + for _, entries := range batch.CallKeys { + if len(entries) > 0 { + currentSize += len(entries[0].CallData) + } + } + if _, isDupe := batch.CallKeys[callKey]; !isDupe { + if currentSize+len(callData) > b.cfg.MaxCalldataBytes { + batch.mu.Unlock() + return nil, true, nil // bypass: calldata too large + } + } + + // Create entry + entry := &BatchEntry{ + Ctx: ctx, + Request: req, + CallKey: callKey, + Target: target, + CallData: callData, + ResultCh: make(chan BatchResult, 1), + CreatedAt: now, + Deadline: deadline, + } + + // Add to batch + batch.Entries = append(batch.Entries, entry) + batch.CallKeys[callKey] = append(batch.CallKeys[callKey], entry) + + // Update flush time based on deadline (deadline-aware) + safetyMargin := time.Duration(b.cfg.SafetyMarginMs) * time.Millisecond + proposedFlush := deadline.Add(-safetyMargin) + if proposedFlush.Before(batch.FlushTime) { + batch.FlushTime = proposedFlush + // Clamp to minimum wait + minFlush := now.Add(minWait) + if batch.FlushTime.Before(minFlush) { + batch.FlushTime = minFlush + } + } + + batch.mu.Unlock() + b.queueSize++ + + return entry, false, nil +} + +// scheduleFlush waits until flush time and then flushes the batch. +func (b *Batcher) scheduleFlush(keyStr string, batch *Batch) { + defer b.wg.Done() + + for { + batch.mu.Lock() + flushTime := batch.FlushTime + batch.mu.Unlock() + + waitDuration := time.Until(flushTime) + if waitDuration <= 0 { + b.flush(keyStr, batch) + return + } + + timer := time.NewTimer(waitDuration) + select { + case <-timer.C: + b.flush(keyStr, batch) + return + case <-b.shutdown: + timer.Stop() + return + } + } +} + +// flush processes a batch and delivers results. +func (b *Batcher) flush(keyStr string, batch *Batch) { + batch.mu.Lock() + if batch.Flushing { + batch.mu.Unlock() + return + } + batch.Flushing = true + entries := batch.Entries + callKeys := batch.CallKeys + batch.mu.Unlock() + + // Remove from active batches + b.mu.Lock() + if b.batches[keyStr] == batch { + delete(b.batches, keyStr) + } + b.queueSize -= int64(len(entries)) + b.mu.Unlock() + + // Deliver error for now (actual forwarding implemented in Task 2.4) + result := BatchResult{ + Error: fmt.Errorf("flush not implemented"), + } + for _, entry := range entries { + select { + case entry.ResultCh <- result: + default: + } + } + _ = callKeys // silence unused warning +} + +// Shutdown stops the batcher and waits for pending operations. +func (b *Batcher) Shutdown() { + close(b.shutdown) + b.wg.Wait() +} +``` + +**Step 4: Run test to verify it passes** + +Run: `go test -v ./architecture/evm -run TestBatcher` +Expected: PASS + +**Step 5: Commit** + +```bash +git add architecture/evm/multicall3_batcher.go architecture/evm/multicall3_batcher_test.go +git commit -m "$(cat <<'EOF' +feat: implement Multicall3 Batcher with window and caps + +Implements the core Batcher type that: +- Enqueues eth_call requests into batches by BatchingKey +- Enforces caps: maxCalls, maxCalldataBytes, maxQueueSize, maxPendingBatches +- Supports deduplication via callKey within batches +- Implements deadline-aware flush scheduling +- Handles concurrent batch creation during flush + +Co-Authored-By: Claude Opus 4.5 +EOF +)" +``` + +--- + +### Task 2.4: Implement Batch Forwarding and Result Mapping + +**Files:** +- Modify: `architecture/evm/multicall3_batcher.go` +- Test: `architecture/evm/multicall3_batcher_test.go` + +**Step 1: Write the failing test** + +Add to `architecture/evm/multicall3_batcher_test.go`: + +```go +type mockForwarder struct { + response *common.NormalizedResponse + err error + called int +} + +func (m *mockForwarder) Forward(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + m.called++ + return m.response, m.err +} + +func TestBatcherFlushAndResultMapping(t *testing.T) { + // Create a mock response with valid multicall3 result + // Multicall3 aggregate3 returns [(bool success, bytes returnData), ...] + // We'll create a simple encoded result for 2 calls + + // For simplicity, create raw hex result + // Result structure: offset to array, array length, elements... + resultHex := "0x" + + // Offset to array (32 bytes pointing to 0x20) + "0000000000000000000000000000000000000000000000000000000000000020" + + // Array length (2 elements) + "0000000000000000000000000000000000000000000000000000000000000002" + + // Offset to element 0 (0x40 from array start) + "0000000000000000000000000000000000000000000000000000000000000040" + + // Offset to element 1 (0xa0 from array start) + "00000000000000000000000000000000000000000000000000000000000000a0" + + // Element 0: success=true + "0000000000000000000000000000000000000000000000000000000000000001" + + // Element 0: returnData offset + "0000000000000000000000000000000000000000000000000000000000000040" + + // Element 0: returnData length (4 bytes) + "0000000000000000000000000000000000000000000000000000000000000004" + + // Element 0: returnData value "0xdeadbeef" padded + "deadbeef00000000000000000000000000000000000000000000000000000000" + + // Element 1: success=true + "0000000000000000000000000000000000000000000000000000000000000001" + + // Element 1: returnData offset + "0000000000000000000000000000000000000000000000000000000000000040" + + // Element 1: returnData length (4 bytes) + "0000000000000000000000000000000000000000000000000000000000000004" + + // Element 1: returnData value "0xcafebabe" padded + "cafebabe00000000000000000000000000000000000000000000000000000000" + + jrr, _ := common.NewJsonRpcResponse(nil, resultHex, nil) + mockResp := common.NewNormalizedResponse().WithJsonRpcResponse(jrr) + + forwarder := &mockForwarder{response: mockResp} + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 10, // Short window for test + MinWaitMs: 1, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: &common.TRUE, + CachePerCall: &common.FALSE, // disable caching for test + } + + batcher := NewBatcher(cfg, forwarder) + + ctx := context.Background() + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + // Add two requests + jrq1 := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{"to": "0x1111111111111111111111111111111111111111", "data": "0x01"}, + "latest", + }) + jrq1.ID = "req1" + req1 := common.NewNormalizedRequestFromJsonRpcRequest(jrq1) + + jrq2 := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{"to": "0x2222222222222222222222222222222222222222", "data": "0x02"}, + "latest", + }) + jrq2.ID = "req2" + req2 := common.NewNormalizedRequestFromJsonRpcRequest(jrq2) + + entry1, _, _ := batcher.Enqueue(ctx, key, req1) + entry2, _, _ := batcher.Enqueue(ctx, key, req2) + + // Wait for results + result1 := <-entry1.ResultCh + result2 := <-entry2.ResultCh + + require.NoError(t, result1.Error) + require.NoError(t, result2.Error) + require.NotNil(t, result1.Response) + require.NotNil(t, result2.Response) + + // Verify forwarder was called exactly once + require.Equal(t, 1, forwarder.called) + + batcher.Shutdown() +} +``` + +**Step 2: Run test to verify it fails** + +Run: `go test -v ./architecture/evm -run TestBatcherFlushAndResultMapping` +Expected: FAIL - current flush delivers error + +**Step 3: Update flush implementation** + +Replace the `flush` function in `architecture/evm/multicall3_batcher.go`: + +```go +// flush processes a batch: builds multicall, forwards, maps results. +func (b *Batcher) flush(keyStr string, batch *Batch) { + batch.mu.Lock() + if batch.Flushing { + batch.mu.Unlock() + return + } + batch.Flushing = true + entries := batch.Entries + callKeys := batch.CallKeys + batch.mu.Unlock() + + // Remove from active batches + b.mu.Lock() + if b.batches[keyStr] == batch { + delete(b.batches, keyStr) + } + b.queueSize -= int64(len(entries)) + b.mu.Unlock() + + // Build unique calls list (maintaining order) + uniqueCalls := make([]Multicall3Call, 0, len(callKeys)) + callKeyOrder := make([]string, 0, len(callKeys)) + seen := make(map[string]bool) + + for _, entry := range entries { + if !seen[entry.CallKey] { + seen[entry.CallKey] = true + callKeyOrder = append(callKeyOrder, entry.CallKey) + uniqueCalls = append(uniqueCalls, Multicall3Call{ + Request: entry.Request, + Target: entry.Target, + CallData: entry.CallData, + }) + } + } + + if len(uniqueCalls) == 0 { + return + } + + // Build requests slice for BuildMulticall3Request + reqs := make([]*common.NormalizedRequest, len(uniqueCalls)) + for i, call := range uniqueCalls { + reqs[i] = call.Request + } + + // Build multicall3 request + mcReq, _, err := BuildMulticall3Request(reqs, batch.Key.BlockRef) + if err != nil { + b.deliverError(entries, fmt.Errorf("failed to build multicall3: %w", err)) + return + } + + // Mark as composite request + mcReq.SetCompositeType(common.CompositeTypeMulticall3) + + // Forward via network + ctx := context.Background() + if len(entries) > 0 && entries[0].Ctx != nil { + ctx = entries[0].Ctx + } + + resp, err := b.forwarder.Forward(ctx, mcReq) + if err != nil { + if ShouldFallbackMulticall3(err) { + b.fallbackIndividual(entries) + return + } + b.deliverError(entries, fmt.Errorf("multicall3 forward failed: %w", err)) + return + } + + // Decode response + results, err := b.decodeMulticallResponse(ctx, resp) + if err != nil { + if ShouldFallbackMulticall3(err) { + b.fallbackIndividual(entries) + return + } + b.deliverError(entries, fmt.Errorf("multicall3 decode failed: %w", err)) + return + } + + if len(results) != len(uniqueCalls) { + b.fallbackIndividual(entries) + return + } + + // Map results to entries (fan out deduplicated results) + for i, callKey := range callKeyOrder { + result := results[i] + waiters := callKeys[callKey] + + for _, entry := range waiters { + br := BatchResult{} + if result.Success { + returnHex := "0x" + hex.EncodeToString(result.ReturnData) + jrr, err := common.NewJsonRpcResponse(entry.Request.ID(), returnHex, nil) + if err != nil { + br.Error = err + } else { + br.Response = common.NewNormalizedResponse().WithRequest(entry.Request).WithJsonRpcResponse(jrr) + br.Response.SetUpstream(resp.Upstream()) + br.Response.SetFromCache(resp.FromCache()) + } + } else { + // Per-call revert + dataHex := "0x" + hex.EncodeToString(result.ReturnData) + br.Error = common.NewErrEndpointExecutionException( + common.NewErrJsonRpcExceptionInternal( + 3, // execution reverted code + common.JsonRpcErrorExecutionReverted, + dataHex, + map[string]interface{}{ + "multicall3": true, + "stage": "per-call", + }, + ), + nil, + ) + } + + select { + case entry.ResultCh <- br: + default: + } + } + } +} + +// decodeMulticallResponse extracts and decodes the multicall3 result. +func (b *Batcher) decodeMulticallResponse(ctx context.Context, resp *common.NormalizedResponse) ([]Multicall3Result, error) { + if resp == nil { + return nil, fmt.Errorf("nil response") + } + + jrr, err := resp.JsonRpcResponse(ctx) + if err != nil { + return nil, err + } + if jrr == nil || jrr.Error != nil { + if jrr != nil && jrr.Error != nil { + return nil, fmt.Errorf("rpc error: %s", jrr.Error.Message) + } + return nil, fmt.Errorf("invalid response") + } + + var resultHex string + if err := common.SonicCfg.Unmarshal(jrr.GetResultBytes(), &resultHex); err != nil { + return nil, err + } + + resultBytes, err := common.HexToBytes(resultHex) + if err != nil { + return nil, err + } + + return DecodeMulticall3Aggregate3Result(resultBytes) +} + +// deliverError sends an error to all entries. +func (b *Batcher) deliverError(entries []*BatchEntry, err error) { + result := BatchResult{Error: err} + for _, entry := range entries { + select { + case entry.ResultCh <- result: + default: + } + } +} + +// fallbackIndividual forwards each entry individually. +func (b *Batcher) fallbackIndividual(entries []*BatchEntry) { + var wg sync.WaitGroup + for _, entry := range entries { + wg.Add(1) + go func(e *BatchEntry) { + defer wg.Done() + resp, err := b.forwarder.Forward(e.Ctx, e.Request) + select { + case e.ResultCh <- BatchResult{Response: resp, Error: err}: + default: + } + }(entry) + } + wg.Wait() +} +``` + +Also add the hex import at the top: +```go +import ( + "encoding/hex" + // ... other imports +) +``` + +**Step 4: Run test to verify it passes** + +Run: `go test -v ./architecture/evm -run TestBatcherFlushAndResultMapping` +Expected: PASS + +**Step 5: Commit** + +```bash +git add architecture/evm/multicall3_batcher.go architecture/evm/multicall3_batcher_test.go +git commit -m "$(cat <<'EOF' +feat: implement Multicall3 batch forwarding and result mapping + +Completes the Batcher.flush() implementation: +- Builds Multicall3 request from unique calls +- Marks request as CompositeTypeMulticall3 +- Forwards via Forwarder interface +- Decodes response and maps results to entries +- Fans out deduplicated results to all waiters +- Handles per-call reverts as execution errors +- Falls back to individual forwarding on multicall failure + +Co-Authored-By: Claude Opus 4.5 +EOF +)" +``` + +--- + +## Phase 3: Integration with Network Layer + +### Task 3.1: Create Network-Level Batcher Manager + +**Files:** +- Create: `architecture/evm/multicall3_manager.go` +- Test: `architecture/evm/multicall3_manager_test.go` + +**Step 1: Write the failing test** + +Create `architecture/evm/multicall3_manager_test.go`: + +```go +package evm + +import ( + "context" + "sync" + "testing" + + "github.com/erpc/erpc/common" + "github.com/stretchr/testify/require" +) + +func TestBatcherManagerGetOrCreate(t *testing.T) { + mgr := NewBatcherManager() + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 25, + MinWaitMs: 2, + MaxCalls: 20, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: &common.TRUE, + } + + forwarder := &mockForwarder{} + + // Get batcher for network + batcher1 := mgr.GetOrCreate("evm:1", cfg, forwarder) + require.NotNil(t, batcher1) + + // Same network should return same batcher + batcher2 := mgr.GetOrCreate("evm:1", cfg, forwarder) + require.Same(t, batcher1, batcher2) + + // Different network should return different batcher + batcher3 := mgr.GetOrCreate("evm:137", cfg, forwarder) + require.NotSame(t, batcher1, batcher3) + + mgr.Shutdown() +} + +func TestBatcherManagerConcurrency(t *testing.T) { + mgr := NewBatcherManager() + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 25, + MinWaitMs: 2, + MaxCalls: 20, + } + cfg.SetDefaults() + + forwarder := &mockForwarder{} + + var wg sync.WaitGroup + batchers := make([]*Batcher, 100) + + // Concurrent access + for i := 0; i < 100; i++ { + wg.Add(1) + go func(idx int) { + defer wg.Done() + batchers[idx] = mgr.GetOrCreate("evm:1", cfg, forwarder) + }(i) + } + wg.Wait() + + // All should be the same batcher + for i := 1; i < 100; i++ { + require.Same(t, batchers[0], batchers[i]) + } + + mgr.Shutdown() +} +``` + +**Step 2: Run test to verify it fails** + +Run: `go test -v ./architecture/evm -run TestBatcherManager` +Expected: FAIL - type doesn't exist + +**Step 3: Write minimal implementation** + +Create `architecture/evm/multicall3_manager.go`: + +```go +package evm + +import ( + "sync" + + "github.com/erpc/erpc/common" +) + +// BatcherManager manages per-network Multicall3 batchers. +type BatcherManager struct { + batchers map[string]*Batcher + mu sync.RWMutex +} + +// NewBatcherManager creates a new batcher manager. +func NewBatcherManager() *BatcherManager { + return &BatcherManager{ + batchers: make(map[string]*Batcher), + } +} + +// GetOrCreate returns the batcher for a network, creating one if needed. +func (m *BatcherManager) GetOrCreate(networkId string, cfg *common.Multicall3AggregationConfig, forwarder Forwarder) *Batcher { + m.mu.RLock() + if b, ok := m.batchers[networkId]; ok { + m.mu.RUnlock() + return b + } + m.mu.RUnlock() + + m.mu.Lock() + defer m.mu.Unlock() + + // Double-check after acquiring write lock + if b, ok := m.batchers[networkId]; ok { + return b + } + + batcher := NewBatcher(cfg, forwarder) + m.batchers[networkId] = batcher + return batcher +} + +// Get returns the batcher for a network, or nil if not exists. +func (m *BatcherManager) Get(networkId string) *Batcher { + m.mu.RLock() + defer m.mu.RUnlock() + return m.batchers[networkId] +} + +// Shutdown stops all batchers. +func (m *BatcherManager) Shutdown() { + m.mu.Lock() + defer m.mu.Unlock() + + for _, b := range m.batchers { + b.Shutdown() + } + m.batchers = make(map[string]*Batcher) +} +``` + +**Step 4: Run test to verify it passes** + +Run: `go test -v ./architecture/evm -run TestBatcherManager` +Expected: PASS + +**Step 5: Commit** + +```bash +git add architecture/evm/multicall3_manager.go architecture/evm/multicall3_manager_test.go +git commit -m "$(cat <<'EOF' +feat: add BatcherManager for per-network batcher instances + +Manages Multicall3 batchers per network with: +- GetOrCreate for lazy initialization +- Thread-safe concurrent access +- Proper shutdown cleanup + +Co-Authored-By: Claude Opus 4.5 +EOF +)" +``` + +--- + +### Task 3.2: Integrate Batcher into eth_call Pre-Forward Hook + +**Files:** +- Modify: `architecture/evm/eth_call.go` +- Modify: `architecture/evm/hooks.go` +- Test: `architecture/evm/eth_call_test.go` (create) + +**Step 1: Write the failing test** + +Create `architecture/evm/eth_call_test.go`: + +```go +package evm + +import ( + "context" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/stretchr/testify/require" +) + +type mockNetwork struct { + networkId string + cfg *common.NetworkConfig + forwardFn func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) + cacheDal common.CacheDAL +} + +func (m *mockNetwork) Id() string { return m.networkId } +func (m *mockNetwork) Label() string { return m.networkId } +func (m *mockNetwork) Config() *common.NetworkConfig { return m.cfg } +func (m *mockNetwork) CacheDal() common.CacheDAL { return m.cacheDal } +func (m *mockNetwork) AppCtx() context.Context { return context.Background() } +func (m *mockNetwork) Forward(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + if m.forwardFn != nil { + return m.forwardFn(ctx, req) + } + return nil, nil +} + +func TestProjectPreForward_eth_call_Batching(t *testing.T) { + cfg := &common.NetworkConfig{ + Evm: &common.EvmNetworkConfig{ + ChainId: 1, + Multicall3Aggregation: &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 50, + MinWaitMs: 5, + MaxCalls: 20, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: &common.TRUE, + CachePerCall: &common.FALSE, + }, + }, + } + + // Create valid multicall response + resultHex := "0x" + + "0000000000000000000000000000000000000000000000000000000000000020" + + "0000000000000000000000000000000000000000000000000000000000000002" + + "0000000000000000000000000000000000000000000000000000000000000040" + + "00000000000000000000000000000000000000000000000000000000000000a0" + + "0000000000000000000000000000000000000000000000000000000000000001" + + "0000000000000000000000000000000000000000000000000000000000000040" + + "0000000000000000000000000000000000000000000000000000000000000004" + + "deadbeef00000000000000000000000000000000000000000000000000000000" + + "0000000000000000000000000000000000000000000000000000000000000001" + + "0000000000000000000000000000000000000000000000000000000000000040" + + "0000000000000000000000000000000000000000000000000000000000000004" + + "cafebabe00000000000000000000000000000000000000000000000000000000" + + jrr, _ := common.NewJsonRpcResponse(nil, resultHex, nil) + mockResp := common.NewNormalizedResponse().WithJsonRpcResponse(jrr) + + network := &mockNetwork{ + networkId: "evm:1", + cfg: cfg, + forwardFn: func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return mockResp, nil + }, + } + + // Prepare two requests to be batched + ctx := context.Background() + + jrq1 := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x1111111111111111111111111111111111111111", + "data": "0x01020304", + }, + "latest", + }) + req1 := common.NewNormalizedRequestFromJsonRpcRequest(jrq1) + req1.SetNetwork(network) + + jrq2 := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x2222222222222222222222222222222222222222", + "data": "0x05060708", + }, + "latest", + }) + req2 := common.NewNormalizedRequestFromJsonRpcRequest(jrq2) + req2.SetNetwork(network) + + // Both should be batched + var resp1, resp2 *common.NormalizedResponse + var err1, err2 error + done := make(chan struct{}, 2) + + go func() { + _, resp1, err1 = HandleProjectPreForward(ctx, network, req1) + done <- struct{}{} + }() + + go func() { + _, resp2, err2 = HandleProjectPreForward(ctx, network, req2) + done <- struct{}{} + }() + + // Wait with timeout + for i := 0; i < 2; i++ { + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("timeout waiting for batched requests") + } + } + + require.NoError(t, err1) + require.NoError(t, err2) + require.NotNil(t, resp1) + require.NotNil(t, resp2) +} +``` + +**Step 2: Run test to verify it fails** + +Run: `go test -v ./architecture/evm -run TestProjectPreForward_eth_call_Batching` +Expected: FAIL - batching not implemented in hooks + +**Step 3: Update eth_call.go with batching integration** + +Replace `architecture/evm/eth_call.go`: + +```go +package evm + +import ( + "context" + "fmt" + "sync" + + "github.com/erpc/erpc/common" +) + +// Global batcher manager for network-level Multicall3 batching +var ( + globalBatcherManager *BatcherManager + batcherManagerOnce sync.Once +) + +// GetBatcherManager returns the global batcher manager. +func GetBatcherManager() *BatcherManager { + batcherManagerOnce.Do(func() { + globalBatcherManager = NewBatcherManager() + }) + return globalBatcherManager +} + +// networkForwarder wraps a Network to implement Forwarder interface. +type networkForwarder struct { + network common.Network +} + +func (f *networkForwarder) Forward(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return f.network.Forward(ctx, req) +} + +func projectPreForward_eth_call(ctx context.Context, network common.Network, nq *common.NormalizedRequest) (bool, *common.NormalizedResponse, error) { + jrq, err := nq.JsonRpcRequest() + if err != nil { + return false, nil, nil + } + + // Normalize params: ensure block param is present + jrq.RLock() + paramsLen := len(jrq.Params) + jrq.RUnlock() + + if paramsLen == 1 { + jrq.Lock() + jrq.Params = append(jrq.Params, "latest") + jrq.Unlock() + } + + // Check if Multicall3 aggregation is enabled + cfg := network.Config() + if cfg == nil || cfg.Evm == nil || cfg.Evm.Multicall3Aggregation == nil || !cfg.Evm.Multicall3Aggregation.Enabled { + // Batching disabled, use normal forward + resp, err := network.Forward(ctx, nq) + return true, resp, err + } + + aggCfg := cfg.Evm.Multicall3Aggregation + + // Check eligibility for batching + eligible, reason := IsEligibleForBatching(nq, aggCfg) + if !eligible { + // Not eligible, forward normally + _ = reason // could log this + resp, err := network.Forward(ctx, nq) + return true, resp, err + } + + // Extract call info for batching key + _, _, blockRef, err := ExtractCallInfo(nq) + if err != nil { + resp, err := network.Forward(ctx, nq) + return true, resp, err + } + + // Build batching key + projectId := "" + if nq.Network() != nil { + // Try to get project ID from request context or network + projectId = fmt.Sprintf("network:%s", network.Id()) + } + + userId := "" + if aggCfg.AllowCrossUserBatching == nil || !*aggCfg.AllowCrossUserBatching { + userId = nq.UserId() + } + + key := BatchingKey{ + ProjectId: projectId, + NetworkId: network.Id(), + BlockRef: blockRef, + DirectivesKey: DeriveDirectivesKey(nq.Directives()), + UserId: userId, + } + + // Get or create batcher for this network + mgr := GetBatcherManager() + forwarder := &networkForwarder{network: network} + batcher := mgr.GetOrCreate(network.Id(), aggCfg, forwarder) + + // Enqueue request + entry, bypass, err := batcher.Enqueue(ctx, key, nq) + if err != nil || bypass { + // Bypass batching, forward normally + resp, err := network.Forward(ctx, nq) + return true, resp, err + } + + // Wait for batch result + select { + case result := <-entry.ResultCh: + return true, result.Response, result.Error + case <-ctx.Done(): + return true, nil, ctx.Err() + } +} +``` + +**Step 4: Run test to verify it passes** + +Run: `go test -v ./architecture/evm -run TestProjectPreForward_eth_call_Batching` +Expected: PASS + +**Step 5: Commit** + +```bash +git add architecture/evm/eth_call.go architecture/evm/eth_call_test.go +git commit -m "$(cat <<'EOF' +feat: integrate Multicall3 batching into eth_call pre-forward hook + +Modifies projectPreForward_eth_call to: +- Check if Multicall3 aggregation is enabled +- Verify request eligibility for batching +- Build batching key from project/network/block/directives/user +- Enqueue eligible requests to network batcher +- Wait for batch result or bypass to normal forward + +Uses global BatcherManager for per-network batcher instances. + +Co-Authored-By: Claude Opus 4.5 +EOF +)" +``` + +--- + +## Phase 4: Metrics and Observability + +### Task 4.1: Add Multicall3 Batching Metrics + +**Files:** +- Modify: `telemetry/metrics.go` +- Modify: `architecture/evm/multicall3_batcher.go` + +**Step 1: Add new metrics to telemetry/metrics.go** + +Add after existing multicall3 metrics (around line 431): + +```go + // Network-level Multicall3 batching metrics + MetricMulticall3BatchSize = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "erpc", + Name: "multicall3_batch_size", + Help: "Number of unique calls per Multicall3 batch.", + Buckets: []float64{1, 2, 5, 10, 15, 20, 30, 50}, + }, []string{"project", "network"}) + + MetricMulticall3BatchWaitMs = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "erpc", + Name: "multicall3_batch_wait_ms", + Help: "Time requests waited in batch before flush (milliseconds).", + Buckets: []float64{1, 2, 5, 10, 15, 20, 25, 30, 50}, + }, []string{"project", "network"}) + + MetricMulticall3QueueLen = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "erpc", + Name: "multicall3_queue_len", + Help: "Current number of requests queued for batching.", + }, []string{"network"}) + + MetricMulticall3QueueOverflowTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "multicall3_queue_overflow_total", + Help: "Total number of requests that bypassed batching due to queue overflow.", + }, []string{"network", "reason"}) + + MetricMulticall3DedupeTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "multicall3_dedupe_total", + Help: "Total number of deduplicated requests within batches.", + }, []string{"project", "network"}) +``` + +**Step 2: Update Batcher to emit metrics** + +Add metric recording to `architecture/evm/multicall3_batcher.go`: + +At the top, add import: +```go +import ( + "github.com/erpc/erpc/telemetry" + // ... other imports +) +``` + +Update `Enqueue` to record overflow metrics when bypassing: +```go +// In Enqueue, after checking caps: +if b.queueSize >= int64(b.cfg.MaxQueueSize) { + telemetry.MetricMulticall3QueueOverflowTotal.WithLabelValues(key.NetworkId, "queue_full").Inc() + return nil, true, nil +} +if len(b.batches) >= b.cfg.MaxPendingBatches { + if _, exists := b.batches[key.String()]; !exists { + telemetry.MetricMulticall3QueueOverflowTotal.WithLabelValues(key.NetworkId, "max_batches").Inc() + return nil, true, nil + } +} +``` + +Update `flush` to record batch metrics: +```go +// At the start of flush, after getting entries: +if len(entries) > 0 { + projectId := batch.Key.ProjectId + networkId := batch.Key.NetworkId + + // Record batch size + telemetry.MetricMulticall3BatchSize.WithLabelValues(projectId, networkId).Observe(float64(len(callKeys))) + + // Record wait time + for _, entry := range entries { + waitMs := time.Since(entry.CreatedAt).Milliseconds() + telemetry.MetricMulticall3BatchWaitMs.WithLabelValues(projectId, networkId).Observe(float64(waitMs)) + } + + // Record dedupe count + totalEntries := len(entries) + uniqueCalls := len(callKeys) + if totalEntries > uniqueCalls { + telemetry.MetricMulticall3DedupeTotal.WithLabelValues(projectId, networkId).Add(float64(totalEntries - uniqueCalls)) + } +} +``` + +Update gauge on enqueue/remove: +```go +// In Enqueue after adding to batch: +telemetry.MetricMulticall3QueueLen.WithLabelValues(key.NetworkId).Inc() + +// In flush after removing from batches: +telemetry.MetricMulticall3QueueLen.WithLabelValues(batch.Key.NetworkId).Sub(float64(len(entries))) +``` + +**Step 3: Run existing tests** + +Run: `go test -v ./architecture/evm/...` +Expected: PASS + +**Step 4: Commit** + +```bash +git add telemetry/metrics.go architecture/evm/multicall3_batcher.go +git commit -m "$(cat <<'EOF' +feat: add Multicall3 batching observability metrics + +New metrics for network-level Multicall3 batching: +- multicall3_batch_size: histogram of unique calls per batch +- multicall3_batch_wait_ms: histogram of request wait times +- multicall3_queue_len: gauge of current queue depth +- multicall3_queue_overflow_total: counter for bypass events +- multicall3_dedupe_total: counter for deduplicated requests + +Co-Authored-By: Claude Opus 4.5 +EOF +)" +``` + +--- + +## Phase 5: Testing + +### Task 5.1: Add Comprehensive Unit Tests + +**Files:** +- Modify: `architecture/evm/multicall3_batcher_test.go` + +**Step 1: Add cancellation test** + +```go +func TestBatcherCancellation(t *testing.T) { + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 100, // Long window + MinWaitMs: 50, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: &common.TRUE, + } + + batcher := NewBatcher(cfg, nil) + + ctx, cancel := context.WithCancel(context.Background()) + key := BatchingKey{ + ProjectId: "test", + NetworkId: "evm:1", + BlockRef: "latest", + } + + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{"to": "0x1234567890123456789012345678901234567890", "data": "0x01"}, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + + entry, bypass, err := batcher.Enqueue(ctx, key, req) + require.NoError(t, err) + require.False(t, bypass) + require.NotNil(t, entry) + + // Cancel before flush + cancel() + + // Result channel should still work (batch will complete eventually) + // The cancelled context shouldn't crash the batcher + batcher.Shutdown() +} +``` + +**Step 2: Add deadline-aware test** + +```go +func TestBatcherDeadlineAwareness(t *testing.T) { + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 100, + MinWaitMs: 10, + SafetyMarginMs: 5, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: &common.TRUE, + } + + batcher := NewBatcher(cfg, nil) + + // Context with tight deadline - should bypass + tightCtx, cancel1 := context.WithDeadline(context.Background(), time.Now().Add(5*time.Millisecond)) + defer cancel1() + + key := BatchingKey{ + ProjectId: "test", + NetworkId: "evm:1", + BlockRef: "latest", + } + + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{"to": "0x1234567890123456789012345678901234567890", "data": "0x01"}, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + + _, bypass, err := batcher.Enqueue(tightCtx, key, req) + require.NoError(t, err) + require.True(t, bypass, "should bypass with tight deadline") + + // Context with reasonable deadline - should batch + normalCtx, cancel2 := context.WithDeadline(context.Background(), time.Now().Add(200*time.Millisecond)) + defer cancel2() + + _, bypass, err = batcher.Enqueue(normalCtx, key, req) + require.NoError(t, err) + require.False(t, bypass, "should batch with normal deadline") + + batcher.Shutdown() +} +``` + +**Step 3: Add concurrent flush test** + +```go +func TestBatcherConcurrentFlush(t *testing.T) { + // Verify that a request arriving during flush gets a new batch + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 10, // Short window + MinWaitMs: 1, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: &common.TRUE, + } + + // Slow forwarder to simulate flush in progress + forwarder := &mockForwarder{ + response: nil, // Will cause fallback + err: nil, + } + + batcher := NewBatcher(cfg, forwarder) + + ctx := context.Background() + key := BatchingKey{ + ProjectId: "test", + NetworkId: "evm:1", + BlockRef: "latest", + } + + // Add first batch + for i := 0; i < 3; i++ { + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{"to": fmt.Sprintf("0x%040d", i), "data": "0x01"}, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + batcher.Enqueue(ctx, key, req) + } + + // Wait for first batch to start flushing + time.Sleep(15 * time.Millisecond) + + // Add more requests - should go to new batch + for i := 3; i < 6; i++ { + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{"to": fmt.Sprintf("0x%040d", i), "data": "0x01"}, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + _, bypass, _ := batcher.Enqueue(ctx, key, req) + // May bypass or create new batch - either is acceptable + _ = bypass + } + + batcher.Shutdown() +} +``` + +**Step 4: Run all tests** + +Run: `go test -v ./architecture/evm/... -count=1` +Expected: PASS + +**Step 5: Commit** + +```bash +git add architecture/evm/multicall3_batcher_test.go +git commit -m "$(cat <<'EOF' +test: add comprehensive Multicall3 batcher tests + +Adds tests for: +- Request cancellation handling +- Deadline-aware flush scheduling +- Concurrent flush and new batch creation + +Co-Authored-By: Claude Opus 4.5 +EOF +)" +``` + +--- + +### Task 5.2: Add Integration Tests + +**Files:** +- Create: `architecture/evm/multicall3_integration_test.go` + +**Step 1: Write integration test** + +Create `architecture/evm/multicall3_integration_test.go`: + +```go +//go:build integration + +package evm + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/erpc/erpc/common" + "github.com/stretchr/testify/require" +) + +func TestMulticall3EndToEndBatching(t *testing.T) { + // This test verifies the full batching flow with multiple concurrent requests + + // Create valid multicall3 response for 5 calls + resultHex := createMulticall3Response(5) + jrr, _ := common.NewJsonRpcResponse(nil, resultHex, nil) + mockResp := common.NewNormalizedResponse().WithJsonRpcResponse(jrr) + + callCount := 0 + var callMu sync.Mutex + + forwarder := &mockForwarder{ + response: mockResp, + err: nil, + } + // Override the simple mock to count calls + originalForward := forwarder.Forward + forwarder.Forward = func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + callMu.Lock() + callCount++ + callMu.Unlock() + return originalForward(ctx, req) + } + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 30, + MinWaitMs: 5, + SafetyMarginMs: 2, + MaxCalls: 20, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: &common.TRUE, + CachePerCall: &common.FALSE, + } + + batcher := NewBatcher(cfg, forwarder) + + ctx := context.Background() + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + // Launch 5 concurrent requests + var wg sync.WaitGroup + results := make([]*BatchResult, 5) + + for i := 0; i < 5; i++ { + wg.Add(1) + go func(idx int) { + defer wg.Done() + + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": fmt.Sprintf("0x%040d", idx), + "data": fmt.Sprintf("0x%08x", idx), + }, + "latest", + }) + jrq.ID = fmt.Sprintf("req-%d", idx) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + + entry, bypass, err := batcher.Enqueue(ctx, key, req) + if err != nil || bypass { + results[idx] = &BatchResult{Error: err} + return + } + + result := <-entry.ResultCh + results[idx] = &result + }(i) + } + + // Wait for all requests to complete + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("timeout waiting for batched requests") + } + + // Verify all got results + for i, result := range results { + require.NotNil(t, result, "result %d is nil", i) + require.NoError(t, result.Error, "result %d has error", i) + require.NotNil(t, result.Response, "result %d has no response", i) + } + + // Verify forwarder was called only once (all batched) + callMu.Lock() + require.Equal(t, 1, callCount, "forwarder should be called once for batched requests") + callMu.Unlock() + + batcher.Shutdown() +} + +// createMulticall3Response creates a valid multicall3 aggregate3 response for n calls. +func createMulticall3Response(n int) string { + // Simplified: just create a response with n successful results + // Real implementation would properly ABI encode + + // For now, return empty since the actual encoding is complex + // Tests should use pre-computed values for specific cases + return "0x" // Placeholder - actual tests use pre-computed hex +} +``` + +**Step 2: Run tests** + +Run: `go test -v ./architecture/evm/... -tags=integration -count=1` +Expected: May need adjustment based on mock setup + +**Step 3: Commit** + +```bash +git add architecture/evm/multicall3_integration_test.go +git commit -m "$(cat <<'EOF' +test: add Multicall3 integration tests + +Integration test verifying: +- Multiple concurrent requests are batched +- Single forwarder call for batched requests +- Results correctly distributed to all waiters + +Co-Authored-By: Claude Opus 4.5 +EOF +)" +``` + +--- + +## Phase 6: Documentation and Cleanup + +### Task 6.1: Update Design Doc Status + +**Files:** +- Modify: `docs/design/multicall3-batching.md` + +**Step 1: Update status** + +Change line 3 from: +```markdown +Status: Proposed +``` +To: +```markdown +Status: Implemented +``` + +**Step 2: Commit** + +```bash +git add docs/design/multicall3-batching.md +git commit -m "$(cat <<'EOF' +docs: mark multicall3-batching design as implemented + +Co-Authored-By: Claude Opus 4.5 +EOF +)" +``` + +--- + +### Task 6.2: Run Full Test Suite + +**Step 1: Run all tests** + +Run: `go test -v ./... -count=1` +Expected: PASS + +**Step 2: Run linter** + +Run: `golangci-lint run` +Expected: No new issues + +**Step 3: Final commit if any fixes needed** + +--- + +## Execution Summary + +**Total Tasks:** 12 (across 6 phases) + +**Key Files Created:** +- `architecture/evm/multicall3_batcher.go` - Core batcher implementation +- `architecture/evm/multicall3_manager.go` - Per-network batcher manager +- `architecture/evm/multicall3_batcher_test.go` - Unit tests +- `architecture/evm/multicall3_manager_test.go` - Manager tests +- `architecture/evm/eth_call_test.go` - Integration tests + +**Key Files Modified:** +- `common/config.go` - Extended Multicall3AggregationConfig +- `common/request.go` - Added CompositeTypeMulticall3 +- `architecture/evm/eth_call.go` - Integration with batcher +- `telemetry/metrics.go` - New batching metrics + +**Dependencies:** None new - uses existing standard library and project packages. diff --git a/erpc/http_batch_eth_call.go b/erpc/http_batch_eth_call.go index f02201f1a..5d781d354 100644 --- a/erpc/http_batch_eth_call.go +++ b/erpc/http_batch_eth_call.go @@ -247,16 +247,12 @@ func (s *HttpServer) handleEthCallBatchAggregation( nq.EnrichFromHttp(headers, queryArgs, uaMode) rlg.Trace().Interface("directives", nq.Directives()).Msgf("applied request directives") + // Acquire project rate limit early (for billing/analytics purposes) if err := project.acquireRateLimitPermit(requestCtx, nq); err != nil { responses[i] = processErrorBody(&rlg, startedAt, nq, err, s.serverCfg.IncludeErrorDetails) common.EndRequestSpan(requestCtx, nil, err) continue } - if err := network.acquireRateLimitPermit(requestCtx, nq); err != nil { - responses[i] = processErrorBody(&rlg, startedAt, nq, err, s.serverCfg.IncludeErrorDetails) - common.EndRequestSpan(requestCtx, nil, err) - continue - } // Record per-request metric for billing/analytics // This ensures aggregated requests are counted individually @@ -283,12 +279,18 @@ func (s *HttpServer) handleEthCallBatchAggregation( } // Check cache for individual requests before aggregating + // Respects skip-cache-read directive - requests with this directive skip cache probe cacheDal := network.CacheDal() var uncachedCandidates []ethCallBatchCandidate if cacheDal != nil && !cacheDal.IsObjectNull() { uncachedCandidates = make([]ethCallBatchCandidate, 0, len(candidates)) cacheHits := 0 for _, cand := range candidates { + // Respect skip-cache-read directive - if set, skip cache probe entirely + if cand.req.SkipCacheRead() { + uncachedCandidates = append(uncachedCandidates, cand) + continue + } cachedResp, err := cacheDal.Get(cand.ctx, cand.req) if err == nil && cachedResp != nil && !cachedResp.IsObjectNull(cand.ctx) { // Cache hit - use cached response directly @@ -319,6 +321,24 @@ func (s *HttpServer) handleEthCallBatchAggregation( return true } + // Acquire network rate limits only for uncached requests that will hit the network + // This prevents wasting rate limit permits on cache hits + var rateLimitedCandidates []ethCallBatchCandidate + for _, cand := range candidates { + if err := network.acquireRateLimitPermit(cand.ctx, cand.req); err != nil { + responses[cand.index] = processErrorBody(&cand.logger, startedAt, cand.req, err, s.serverCfg.IncludeErrorDetails) + common.EndRequestSpan(cand.ctx, nil, err) + continue + } + rateLimitedCandidates = append(rateLimitedCandidates, cand) + } + candidates = rateLimitedCandidates + + // All uncached requests were rate limited + if len(candidates) == 0 { + return true + } + if len(candidates) < 2 { s.forwardEthCallBatchCandidates(startedAt, project, network, candidates, responses) return true @@ -340,6 +360,10 @@ func (s *HttpServer) handleEthCallBatchAggregation( return true } + // Mark as composite to disable hedging - multicall3 requests should not be + // hedged like normal requests as this would create duplicate batches + mcReq.SetCompositeType(common.CompositeTypeMulticall3) + mcCtx := withSkipNetworkRateLimit(httpCtx) mcResp, mcErr := forwardBatchNetwork(mcCtx, network, mcReq) if mcErr != nil { diff --git a/erpc/networks.go b/erpc/networks.go index c6df22923..449bf08f4 100644 --- a/erpc/networks.go +++ b/erpc/networks.go @@ -895,6 +895,10 @@ func (n *Network) CacheDal() common.CacheDAL { return n.cacheDal } +func (n *Network) Cache() common.CacheDAL { + return n.cacheDal +} + func (n *Network) AppCtx() context.Context { return n.appCtx } diff --git a/telemetry/metrics.go b/telemetry/metrics.go index 3810e8fe8..ab9fdf87b 100644 --- a/telemetry/metrics.go +++ b/telemetry/metrics.go @@ -449,19 +449,31 @@ var ( Namespace: "erpc", Name: "multicall3_queue_len", Help: "Current number of requests queued for batching.", - }, []string{"network"}) + }, []string{"project", "network"}) MetricMulticall3QueueOverflowTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "erpc", Name: "multicall3_queue_overflow_total", Help: "Total number of requests that bypassed batching due to queue overflow.", - }, []string{"network", "reason"}) + }, []string{"project", "network", "reason"}) MetricMulticall3DedupeTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "erpc", Name: "multicall3_dedupe_total", Help: "Total number of deduplicated requests within batches.", }, []string{"project", "network"}) + + MetricMulticall3CacheWriteErrorsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "multicall3_cache_write_errors_total", + Help: "Total number of per-call cache write errors in multicall3 batch responses.", + }, []string{"project", "network"}) + + MetricMulticall3FallbackRequestsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "multicall3_fallback_requests_total", + Help: "Total number of individual requests during multicall3 fallback, labeled by outcome.", + }, []string{"project", "network", "outcome"}) ) var DefaultHistogramBuckets = []float64{ From 4e811ecb4fda02855aaaec5096f79698d13981b9 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 15 Jan 2026 22:40:07 +0100 Subject: [PATCH 27/53] fix: address PR review comments - Fix data race in TestHandleEthCallBatchAggregation_FallbackPaths by adding mutex protection for projCalls/netCalls counters - Update forward_error_fallback test to use "contract not found" error (execution reverted intentionally does not trigger fallback) - Improve error message clarity: "cache write" instead of "set" Co-Authored-By: Claude Opus 4.5 --- erpc/http_batch_eth_call.go | 2 +- erpc/http_batch_eth_call_handle_test.go | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/erpc/http_batch_eth_call.go b/erpc/http_batch_eth_call.go index 5d781d354..3e3e9fbac 100644 --- a/erpc/http_batch_eth_call.go +++ b/erpc/http_batch_eth_call.go @@ -507,7 +507,7 @@ func (s *HttpServer) handleEthCallBatchAggregation( defer resp.RUnlock() defer resp.DoneRef() - timeoutCtx, timeoutCtxCancel := context.WithTimeoutCause(network.AppCtx(), 10*time.Second, errors.New("cache driver timeout during multicall3 per-call set")) + timeoutCtx, timeoutCtxCancel := context.WithTimeoutCause(network.AppCtx(), 10*time.Second, errors.New("cache driver timeout during multicall3 per-call cache write")) defer timeoutCtxCancel() tracedCtx := trace.ContextWithSpanContext(timeoutCtx, trace.SpanContextFromContext(reqCtx)) if err := cacheDal.Set(tracedCtx, req, resp); err != nil { diff --git a/erpc/http_batch_eth_call_handle_test.go b/erpc/http_batch_eth_call_handle_test.go index 8ef6ebcd5..d8571aaad 100644 --- a/erpc/http_batch_eth_call_handle_test.go +++ b/erpc/http_batch_eth_call_handle_test.go @@ -7,6 +7,7 @@ import ( "errors" "net/http" "net/http/httptest" + "sync" "testing" "time" @@ -176,8 +177,9 @@ func TestHandleEthCallBatchAggregation_FallbackPaths(t *testing.T) { name: "forward error fallback", requests: validRequests, networkResponse: func() (*common.NormalizedResponse, error) { - // Use "execution reverted" to trigger ShouldFallbackMulticall3 - return nil, common.NewErrEndpointExecutionException(errors.New("execution reverted")) + // Use "contract not found" to trigger ShouldFallbackMulticall3 + // Note: "execution reverted" does NOT trigger fallback (would also fail individually) + return nil, common.NewErrEndpointExecutionException(errors.New("contract not found")) }, expectedProjCalls: 2, expectedNetCalls: 1, @@ -244,16 +246,21 @@ func TestHandleEthCallBatchAggregation_FallbackPaths(t *testing.T) { for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { + var mu sync.Mutex projCalls := 0 netCalls := 0 withBatchStubs(t, func(ctx context.Context, network *Network, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + mu.Lock() netCalls++ + mu.Unlock() return tt.networkResponse() }, func(ctx context.Context, project *PreparedProject, network *Network, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + mu.Lock() projCalls++ + mu.Unlock() return fallbackResponse(t, req), nil }, nil, @@ -261,8 +268,10 @@ func TestHandleEthCallBatchAggregation_FallbackPaths(t *testing.T) { handled, responses := runHandle(t, ctx, server, project, defaultBatchInfo(), tt.requests, nil) require.True(t, handled) + mu.Lock() assert.Equal(t, tt.expectedProjCalls, projCalls) assert.Equal(t, tt.expectedNetCalls, netCalls) + mu.Unlock() if len(responses) > 0 { _, ok := responses[0].(*common.NormalizedResponse) assert.True(t, ok) From cd9e0ce82fae8f5449f391bf5985ddc117bd7a02 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 15 Jan 2026 23:09:13 +0100 Subject: [PATCH 28/53] fix: improve observability and error handling in multicall3 batching - Add MetricMulticall3AbandonedTotal metric for tracking context cancellation - Add logging when batch results cannot be delivered due to context cancellation - Release response when context is cancelled to prevent memory leaks - Add build_failed label to fallback metrics when batch building fails - Add forwarder nil validation in NewBatcher (panic on programming error) - Add debug logging for enqueue errors in eth_call.go - Add debug logging for cache get errors in http_batch_eth_call.go - Update tests to use mock forwarder after nil validation was added Co-Authored-By: Claude Opus 4.5 --- architecture/evm/eth_call.go | 8 ++++ architecture/evm/multicall3_batcher.go | 46 ++++++++++++++------- architecture/evm/multicall3_batcher_test.go | 21 ++++++---- erpc/http_batch_eth_call.go | 7 ++++ telemetry/metrics.go | 6 +++ 5 files changed, 67 insertions(+), 21 deletions(-) diff --git a/architecture/evm/eth_call.go b/architecture/evm/eth_call.go index 97752f8af..b62a4112c 100644 --- a/architecture/evm/eth_call.go +++ b/architecture/evm/eth_call.go @@ -146,6 +146,14 @@ func projectPreForward_eth_call(ctx context.Context, network common.Network, nq // Enqueue request entry, bypass, err := batcher.Enqueue(ctx, key, nq) if err != nil || bypass { + // Log enqueue errors for debugging (bypass without error is normal) + if err != nil && network.Logger() != nil { + network.Logger().Debug(). + Err(err). + Str("projectId", projectId). + Str("networkId", network.Id()). + Msg("multicall3 enqueue failed, forwarding normally") + } // Bypass batching, forward normally resp, err := network.Forward(ctx, nq) return true, resp, err diff --git a/architecture/evm/multicall3_batcher.go b/architecture/evm/multicall3_batcher.go index bd96294f2..a65a50629 100644 --- a/architecture/evm/multicall3_batcher.go +++ b/architecture/evm/multicall3_batcher.go @@ -38,10 +38,14 @@ type Batcher struct { // NewBatcher creates a new Multicall3 batcher. // Returns nil if cfg is nil or disabled - callers should check the return value. // The logger parameter is optional (can be nil) - if nil, debug logging is disabled. +// Panics if forwarder is nil (programming error - caller must provide a valid forwarder). func NewBatcher(cfg *common.Multicall3AggregationConfig, forwarder Forwarder, logger *zerolog.Logger) *Batcher { if cfg == nil || !cfg.Enabled { return nil } + if forwarder == nil { + panic("NewBatcher: forwarder cannot be nil") + } b := &Batcher{ cfg: cfg, forwarder: forwarder, @@ -340,7 +344,8 @@ func (b *Batcher) flush(keyStr string, batch *Batch) { // Build the multicall3 request mcReq, _, err := BuildMulticall3Request(requests, batch.Key.BlockRef) if err != nil { - b.deliverError(entries, err) + telemetry.MetricMulticall3FallbackTotal.WithLabelValues(projectId, networkId, "build_failed").Inc() + b.deliverError(entries, err, projectId, networkId) return } @@ -381,7 +386,7 @@ func (b *Batcher) flush(keyStr string, batch *Batch) { if ctx.Err() != nil { err = fmt.Errorf("multicall3 batch forward failed (batch size: %d): %w", len(uniqueCalls), err) } - b.deliverError(entries, err) + b.deliverError(entries, err, projectId, networkId) return } @@ -394,13 +399,13 @@ func (b *Batcher) flush(keyStr string, batch *Batch) { b.fallbackIndividual(entries, projectId, networkId) return } - b.deliverError(entries, err) + b.deliverError(entries, err, projectId, networkId) return } // Verify result count matches unique calls if len(results) != len(uniqueCalls) { - b.deliverError(entries, fmt.Errorf("multicall3 result count mismatch: got %d, expected %d", len(results), len(uniqueCalls))) + b.deliverError(entries, fmt.Errorf("multicall3 result count mismatch: got %d, expected %d", len(results), len(uniqueCalls)), projectId, networkId) return } @@ -423,7 +428,7 @@ func (b *Batcher) flush(keyStr string, batch *Batch) { for _, entry := range entriesForCall { jrr, err := common.NewJsonRpcResponse(entry.Request.ID(), resultHex, nil) if err != nil { - b.sendResult(entry, BatchResult{Error: err}) + b.sendResult(entry, BatchResult{Error: err}, projectId, networkId) continue } resp := common.NewNormalizedResponse().WithRequest(entry.Request).WithJsonRpcResponse(jrr) @@ -449,7 +454,7 @@ func (b *Batcher) flush(keyStr string, batch *Batch) { cachedOnce = true } - b.sendResult(entry, BatchResult{Response: resp}) + b.sendResult(entry, BatchResult{Response: resp}, projectId, networkId) } } else { // Build error for reverted call with proper JSON-RPC format @@ -468,7 +473,7 @@ func (b *Batcher) flush(keyStr string, batch *Batch) { ), ) for _, entry := range entriesForCall { - b.sendResult(entry, BatchResult{Error: revertErr}) + b.sendResult(entry, BatchResult{Error: revertErr}, projectId, networkId) } } } @@ -517,10 +522,23 @@ func (b *Batcher) decodeMulticallResponse(resp *common.NormalizedResponse) ([]Mu // sendResult safely sends a result to an entry, skipping if context is cancelled. // Returns true if sent, false if skipped due to cancelled context. -func (b *Batcher) sendResult(entry *BatchEntry, result BatchResult) bool { +// Records a metric when context is cancelled to track abandoned requests. +func (b *Batcher) sendResult(entry *BatchEntry, result BatchResult, projectId, networkId string) bool { // Check if the entry's context is cancelled - no point sending if caller has given up select { case <-entry.Ctx.Done(): + telemetry.MetricMulticall3AbandonedTotal.WithLabelValues(projectId, networkId).Inc() + if b.logger != nil { + b.logger.Debug(). + Str("projectId", projectId). + Str("networkId", networkId). + Err(entry.Ctx.Err()). + Msg("multicall3 batch result not delivered: caller context cancelled") + } + // Release response if present to avoid memory leak + if result.Response != nil { + result.Response.Release() + } return false // Caller abandoned request, skip sending default: } @@ -531,10 +549,10 @@ func (b *Batcher) sendResult(entry *BatchEntry, result BatchResult) bool { // deliverError sends an error to all entries in a batch. // Skips entries whose context has been cancelled. -func (b *Batcher) deliverError(entries []*BatchEntry, err error) { +func (b *Batcher) deliverError(entries []*BatchEntry, err error, projectId, networkId string) { result := BatchResult{Error: err} for _, entry := range entries { - b.sendResult(entry, result) + b.sendResult(entry, result, projectId, networkId) } } @@ -551,20 +569,20 @@ func (b *Batcher) fallbackIndividual(entries []*BatchEntry, projectId, networkId if r := recover(); r != nil { // Panic in forwarder - send error to entry err := fmt.Errorf("panic in fallback forward: %v", r) - b.sendResult(e, BatchResult{Error: err}) + b.sendResult(e, BatchResult{Error: err}, projectId, networkId) telemetry.MetricMulticall3FallbackRequestsTotal.WithLabelValues(projectId, networkId, "panic").Inc() } }() // Skip if context is already cancelled select { case <-e.Ctx.Done(): - b.sendResult(e, BatchResult{Error: e.Ctx.Err()}) + b.sendResult(e, BatchResult{Error: e.Ctx.Err()}, projectId, networkId) telemetry.MetricMulticall3FallbackRequestsTotal.WithLabelValues(projectId, networkId, "cancelled").Inc() return default: } resp, err := b.forwarder.Forward(e.Ctx, e.Request) - b.sendResult(e, BatchResult{Response: resp, Error: err}) + b.sendResult(e, BatchResult{Response: resp, Error: err}, projectId, networkId) if err != nil { telemetry.MetricMulticall3FallbackRequestsTotal.WithLabelValues(projectId, networkId, "error").Inc() } else { @@ -619,7 +637,7 @@ func (b *Batcher) flushWithShutdownError(keyStr string, batch *Batch) { nil, nil, ) - b.deliverError(entries, shutdownErr) + b.deliverError(entries, shutdownErr, batch.Key.ProjectId, batch.Key.NetworkId) } // DirectivesKeyVersion should be bumped when the set of directives diff --git a/architecture/evm/multicall3_batcher_test.go b/architecture/evm/multicall3_batcher_test.go index 5cf7521f2..7ea78b819 100644 --- a/architecture/evm/multicall3_batcher_test.go +++ b/architecture/evm/multicall3_batcher_test.go @@ -370,7 +370,8 @@ func TestBatcherEnqueueAndFlush(t *testing.T) { cfg.SetDefaults() ctx := context.Background() - batcher := NewBatcher(cfg, nil, nil) // nil forwarder for now + forwarder := &mockForwarder{} // Not used in this test but required + batcher := NewBatcher(cfg, forwarder, nil) // Create test requests jrq1 := common.NewJsonRpcRequest("eth_call", []interface{}{ @@ -435,7 +436,8 @@ func TestBatcherDeduplication(t *testing.T) { cfg.SetDefaults() ctx := context.Background() - batcher := NewBatcher(cfg, nil, nil) + forwarder := &mockForwarder{} // Not used in this test but required + batcher := NewBatcher(cfg, forwarder, nil) // Two identical requests - using the same jrq to ensure call key consistency // (JSON serialization of map[string]interface{} can have non-deterministic key order) @@ -488,7 +490,8 @@ func TestBatcherCapsEnforcement(t *testing.T) { cfg.SetDefaults() ctx := context.Background() - batcher := NewBatcher(cfg, nil, nil) + forwarder := &mockForwarder{} // Not used in this test but required + batcher := NewBatcher(cfg, forwarder, nil) key := BatchingKey{ ProjectId: "test-project", @@ -906,7 +909,8 @@ func TestBatcherCancellation(t *testing.T) { } cfg.SetDefaults() - batcher := NewBatcher(cfg, nil, nil) + forwarder := &mockForwarder{} // Not used in this test but required + batcher := NewBatcher(cfg, forwarder, nil) ctx, cancel := context.WithCancel(context.Background()) key := BatchingKey{ @@ -947,7 +951,8 @@ func TestBatcherDeadlineAwareness(t *testing.T) { } cfg.SetDefaults() - batcher := NewBatcher(cfg, nil, nil) + forwarder := &mockForwarder{} // Not used in this test but required + batcher := NewBatcher(cfg, forwarder, nil) key := BatchingKey{ ProjectId: "test", @@ -1261,7 +1266,8 @@ func TestBatcher_MaxCalldataBytes_Bypass(t *testing.T) { cfg.SetDefaults() ctx := context.Background() - batcher := NewBatcher(cfg, nil, nil) + forwarder := &mockForwarder{} // Not used in this test but required + batcher := NewBatcher(cfg, forwarder, nil) require.NotNil(t, batcher) defer batcher.Shutdown() @@ -1318,7 +1324,8 @@ func TestBatcher_OnlyIfPending_NoBatch(t *testing.T) { cfg.SetDefaults() ctx := context.Background() - batcher := NewBatcher(cfg, nil, nil) + forwarder := &mockForwarder{} // Not used in this test but required + batcher := NewBatcher(cfg, forwarder, nil) require.NotNil(t, batcher) defer batcher.Shutdown() diff --git a/erpc/http_batch_eth_call.go b/erpc/http_batch_eth_call.go index 3e3e9fbac..82ffc882f 100644 --- a/erpc/http_batch_eth_call.go +++ b/erpc/http_batch_eth_call.go @@ -292,6 +292,13 @@ func (s *HttpServer) handleEthCallBatchAggregation( continue } cachedResp, err := cacheDal.Get(cand.ctx, cand.req) + if err != nil { + // Log cache errors for debugging - they're non-fatal but useful for diagnosing issues + cand.logger.Debug(). + Err(err). + Str("networkId", batchInfo.networkId). + Msg("multicall3 pre-aggregation cache get failed, treating as miss") + } if err == nil && cachedResp != nil && !cachedResp.IsObjectNull(cand.ctx) { // Cache hit - use cached response directly cachedResp.SetFromCache(true) diff --git a/telemetry/metrics.go b/telemetry/metrics.go index ab9fdf87b..d5ba27232 100644 --- a/telemetry/metrics.go +++ b/telemetry/metrics.go @@ -474,6 +474,12 @@ var ( Name: "multicall3_fallback_requests_total", Help: "Total number of individual requests during multicall3 fallback, labeled by outcome.", }, []string{"project", "network", "outcome"}) + + MetricMulticall3AbandonedTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "multicall3_abandoned_total", + Help: "Total number of multicall3 batch results not delivered because caller context was cancelled.", + }, []string{"project", "network"}) ) var DefaultHistogramBuckets = []float64{ From 6a8f923cabcf171f750018e3660ab0540dc06a48 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 15 Jan 2026 23:11:48 +0100 Subject: [PATCH 29/53] test: add critical tests for shutdown and double-flush scenarios - Add TestBatcher_ShutdownDuringActiveFlush: verifies that shutdown during an active flush delivers shutdown errors to pending entries - Add TestBatcher_DoubleFlushPrevention: verifies that concurrent flush calls on the same batch don't result in double-processing (race test) Both tests run with -race flag to catch any synchronization issues. Co-Authored-By: Claude Opus 4.5 --- architecture/evm/multicall3_batcher_test.go | 206 ++++++++++++++++++++ 1 file changed, 206 insertions(+) diff --git a/architecture/evm/multicall3_batcher_test.go b/architecture/evm/multicall3_batcher_test.go index 7ea78b819..3e0fefa09 100644 --- a/architecture/evm/multicall3_batcher_test.go +++ b/architecture/evm/multicall3_batcher_test.go @@ -1510,3 +1510,209 @@ func TestBatcher_DuplicateCallsShareResult(t *testing.T) { require.Equal(t, 1, forwarder.called) forwarder.mu.Unlock() } + +// TestBatcher_ShutdownDuringActiveFlush verifies that shutdown during an active +// flush delivers shutdown errors to pending entries and cleans up properly. +func TestBatcher_ShutdownDuringActiveFlush(t *testing.T) { + // Create a forwarder that blocks to simulate a long-running flush + flushStarted := make(chan struct{}) + flushBlock := make(chan struct{}) + + forwarder := &mockForwarderFunc{ + forwardFunc: func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + close(flushStarted) // Signal that flush has started + <-flushBlock // Block until test unblocks + return nil, fmt.Errorf("should not reach here") + }, + } + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 10, + MinWaitMs: 1, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + } + cfg.SetDefaults() + + batcher := NewBatcher(cfg, forwarder, nil) + require.NotNil(t, batcher) + + ctx := context.Background() + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + // Enqueue a request + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x1111111111111111111111111111111111111111", + "data": "0x01020304", + }, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + + entry, bypass, err := batcher.Enqueue(ctx, key, req) + require.NoError(t, err) + require.False(t, bypass) + + // Wait for flush to start (forwarder called) + select { + case <-flushStarted: + // Good - flush has started + case <-time.After(500 * time.Millisecond): + t.Fatal("timeout waiting for flush to start") + } + + // Now enqueue another request for a DIFFERENT batch key + // This creates a new batch that hasn't started flushing yet + key2 := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "0x12345", // Different block ref = different batch + DirectivesKey: DeriveDirectivesKey(nil), + } + + jrq2 := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x2222222222222222222222222222222222222222", + "data": "0x05060708", + }, + "0x12345", + }) + req2 := common.NewNormalizedRequestFromJsonRpcRequest(jrq2) + + entry2, bypass2, err2 := batcher.Enqueue(ctx, key2, req2) + require.NoError(t, err2) + require.False(t, bypass2) + + // Call shutdown while first flush is blocked + // This should trigger flushWithShutdownError for the second batch + go func() { + time.Sleep(10 * time.Millisecond) // Give shutdown a head start + close(flushBlock) // Unblock the first flush + }() + + batcher.Shutdown() + + // The second entry should receive a shutdown error + select { + case result := <-entry2.ResultCh: + require.Error(t, result.Error) + require.Contains(t, result.Error.Error(), "shutting down") + case <-time.After(2 * time.Second): + t.Fatal("timeout waiting for shutdown error on entry2") + } + + // First entry gets error because forwarder returns error after unblock + select { + case result := <-entry.ResultCh: + // Either an error from forwarder or from shutdown is acceptable + require.Error(t, result.Error) + case <-time.After(2 * time.Second): + t.Fatal("timeout waiting for result on entry1") + } +} + +// TestBatcher_DoubleFlushPrevention verifies that concurrent flush calls +// on the same batch don't result in double-processing (race condition test). +func TestBatcher_DoubleFlushPrevention(t *testing.T) { + var forwardCallCount int64 + var mu sync.Mutex + + // Create a forwarder that counts calls + forwarder := &mockForwarderFunc{ + forwardFunc: func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + mu.Lock() + forwardCallCount++ + mu.Unlock() + + // Return multicall3 results with 1 successful call + results := []Multicall3Result{ + {Success: true, ReturnData: []byte{0xaa, 0xbb}}, + } + encodedResult := encodeAggregate3Results(results) + resultHex := "0x" + hex.EncodeToString(encodedResult) + jrr, _ := common.NewJsonRpcResponse(nil, resultHex, nil) + return common.NewNormalizedResponse().WithJsonRpcResponse(jrr), nil + }, + } + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 1000, // Long window so we control when flush happens + MinWaitMs: 500, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + } + cfg.SetDefaults() + + batcher := NewBatcher(cfg, forwarder, nil) + require.NotNil(t, batcher) + defer batcher.Shutdown() + + ctx := context.Background() + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + // Enqueue a request + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x1111111111111111111111111111111111111111", + "data": "0x01020304", + }, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + + entry, bypass, err := batcher.Enqueue(ctx, key, req) + require.NoError(t, err) + require.False(t, bypass) + + // Get the batch directly from the batcher's internal map + keyStr := key.String() + batcher.mu.Lock() + batch := batcher.batches[keyStr] + batcher.mu.Unlock() + require.NotNil(t, batch, "batch should exist") + + // Simulate concurrent flush calls (race condition scenario) + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + batcher.flush(keyStr, batch) + }() + } + wg.Wait() + + // Verify forwarder was only called once (double-flush prevented) + mu.Lock() + finalCallCount := forwardCallCount + mu.Unlock() + require.Equal(t, int64(1), finalCallCount, "forwarder should only be called once despite concurrent flush attempts") + + // Entry should receive exactly one result + select { + case result := <-entry.ResultCh: + require.NoError(t, result.Error) + require.NotNil(t, result.Response) + case <-time.After(2 * time.Second): + t.Fatal("timeout waiting for result") + } +} From 6648084ba931df77beb7b7497cda48ebee265b34 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 16 Jan 2026 10:22:06 +0100 Subject: [PATCH 30/53] fix: validate call fields and normalize block params for multicall3 - Add allowedCallFields validation to reject unsupported fields (accessList, etc.) early in IsEligibleForBatching to avoid batcher failures - Add blockParamForMulticall() to normalize block references: - Convert decimal block numbers to hex format - Pass through hex and tag values unchanged - Add test TestBatcherFlush_UsesHexBlockParam to verify block param handling - Add test case for unsupported call field rejection Co-Authored-By: Claude Opus 4.5 --- architecture/evm/multicall3_batcher.go | 83 +++++++++++++---- architecture/evm/multicall3_batcher_test.go | 98 ++++++++++++++++++++- 2 files changed, 162 insertions(+), 19 deletions(-) diff --git a/architecture/evm/multicall3_batcher.go b/architecture/evm/multicall3_batcher.go index a65a50629..b8db076f2 100644 --- a/architecture/evm/multicall3_batcher.go +++ b/architecture/evm/multicall3_batcher.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "fmt" "sort" + "strconv" "strings" "sync" "time" @@ -342,7 +343,13 @@ func (b *Batcher) flush(keyStr string, batch *Batch) { } // Build the multicall3 request - mcReq, _, err := BuildMulticall3Request(requests, batch.Key.BlockRef) + blockParam, err := blockParamForMulticall(batch.Key.BlockRef) + if err != nil { + telemetry.MetricMulticall3FallbackTotal.WithLabelValues(projectId, networkId, "invalid_block_param").Inc() + b.deliverError(entries, err, projectId, networkId) + return + } + mcReq, _, err := BuildMulticall3Request(requests, blockParam) if err != nil { telemetry.MetricMulticall3FallbackTotal.WithLabelValues(projectId, networkId, "build_failed").Inc() b.deliverError(entries, err, projectId, networkId) @@ -462,7 +469,7 @@ func (b *Batcher) flush(keyStr string, batch *Batch) { revertErr := common.NewErrEndpointExecutionException( common.NewErrJsonRpcExceptionInternal( int(common.JsonRpcErrorEvmReverted), // original code - common.JsonRpcErrorEvmReverted, // normalized code + common.JsonRpcErrorEvmReverted, // normalized code "execution reverted", nil, map[string]interface{}{ @@ -705,14 +712,14 @@ func DeriveCallKey(req *common.NormalizedRequest) (string, error) { // BatchEntry represents a request waiting in a batch. type BatchEntry struct { - Ctx context.Context // Original request context (for individual fallback) - Request *common.NormalizedRequest // The original eth_call request - CallKey string // Deduplication key (target + calldata + blockRef) - Target []byte // Contract address (20 bytes) - CallData []byte // Encoded function call data - ResultCh chan BatchResult // Channel to receive the result (buffered, size 1) - CreatedAt time.Time // When the entry was created (for wait time metrics) - Deadline time.Time // Deadline from context (for deadline-aware flushing) + Ctx context.Context // Original request context (for individual fallback) + Request *common.NormalizedRequest // The original eth_call request + CallKey string // Deduplication key (target + calldata + blockRef) + Target []byte // Contract address (20 bytes) + CallData []byte // Encoded function call data + ResultCh chan BatchResult // Channel to receive the result (buffered, size 1) + CreatedAt time.Time // When the entry was created (for wait time metrics) + Deadline time.Time // Deadline from context (for deadline-aware flushing) } // BatchResult is the outcome delivered to a waiting request. @@ -724,13 +731,13 @@ type BatchResult struct { // Batch holds pending requests for a single batching key. // All entries in a batch share the same project, network, block reference, directives, and user ID. type Batch struct { - Key BatchingKey // Composite key identifying this batch - Entries []*BatchEntry // All entries (may include duplicates) - CallKeys map[string][]*BatchEntry // Map from call key to entries (for deduplication) - FlushTime time.Time // When this batch should be flushed (deadline-aware) - Flushing bool // True once flush has started (prevents double-flush) - notifyCh chan struct{} // Signals flush time was shortened (buffered, size 1) - mu sync.Mutex // Protects all fields + Key BatchingKey // Composite key identifying this batch + Entries []*BatchEntry // All entries (may include duplicates) + CallKeys map[string][]*BatchEntry // Map from call key to entries (for deduplication) + FlushTime time.Time // When this batch should be flushed (deadline-aware) + Flushing bool // True once flush has started (prevents double-flush) + notifyCh chan struct{} // Signals flush time was shortened (buffered, size 1) + mu sync.Mutex // Protects all fields } func NewBatch(key BatchingKey, flushTime time.Time) *Batch { @@ -749,6 +756,12 @@ var ineligibleCallFields = []string{ "from", "gas", "gasPrice", "maxFeePerGas", "maxPriorityFeePerGas", "value", } +var allowedCallFields = map[string]bool{ + "to": true, + "data": true, + "input": true, +} + // allowedBlockTags are block tags that can be batched by default. var allowedBlockTags = map[string]bool{ "latest": true, @@ -815,6 +828,13 @@ func IsEligibleForBatching(req *common.NormalizedRequest, cfg *common.Multicall3 } } + // Reject unsupported call object fields early to avoid batcher failures. + for field := range callObj { + if !allowedCallFields[field] { + return false, fmt.Sprintf("unsupported call field: %s", field) + } + } + // Recursion guard: don't batch calls to multicall3 contract if strings.EqualFold(toStr, multicall3Address) { return false, "already multicall" @@ -868,6 +888,35 @@ func isBlockRefEligibleForBatching(blockRef string) bool { return false } +func blockParamForMulticall(blockRef string) (interface{}, error) { + if blockRef == "" { + return "latest", nil + } + if strings.HasPrefix(blockRef, "0x") { + return blockRef, nil + } + if isDecimalBlockRef(blockRef) { + blockNum, err := strconv.ParseInt(blockRef, 10, 64) + if err != nil { + return nil, err + } + return fmt.Sprintf("0x%x", blockNum), nil + } + return blockRef, nil +} + +func isDecimalBlockRef(blockRef string) bool { + if blockRef == "" { + return false + } + for i := 0; i < len(blockRef); i++ { + if blockRef[i] < '0' || blockRef[i] > '9' { + return false + } + } + return true +} + // ExtractCallInfo extracts target and calldata from an eligible eth_call request. // PRECONDITION: req must have passed IsEligibleForBatching - this function assumes // the request structure has been validated. diff --git a/architecture/evm/multicall3_batcher_test.go b/architecture/evm/multicall3_batcher_test.go index 3e0fefa09..e42cad09b 100644 --- a/architecture/evm/multicall3_batcher_test.go +++ b/architecture/evm/multicall3_batcher_test.go @@ -163,6 +163,20 @@ func TestIsEligibleForBatching(t *testing.T) { eligible: false, reason: "has value field", }, + { + name: "ineligible - unsupported call field", + method: "eth_call", + params: []interface{}{ + map[string]interface{}{ + "to": "0x1234567890123456789012345678901234567890", + "data": "0xabcd", + "accessList": []interface{}{}, + }, + "latest", + }, + eligible: false, + reason: "unsupported call field", + }, { name: "ineligible - has state override (3rd param)", method: "eth_call", @@ -639,6 +653,86 @@ func TestBatcherFlushAndResultMapping(t *testing.T) { batcher.Shutdown() } +func TestBatcherFlush_UsesHexBlockParam(t *testing.T) { + results := []Multicall3Result{ + {Success: true, ReturnData: []byte{0x01}}, + } + encodedResult := encodeAggregate3Results(results) + resultHex := "0x" + hex.EncodeToString(encodedResult) + + blockParamCh := make(chan interface{}, 1) + forwarder := &mockForwarderFunc{ + forwardFunc: func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + jrq, err := req.JsonRpcRequest() + if err == nil { + jrq.RLock() + params := jrq.Params + jrq.RUnlock() + if len(params) > 1 { + blockParamCh <- params[1] + } + } + + jrr, err := common.NewJsonRpcResponse(nil, resultHex, nil) + if err != nil { + return nil, err + } + return common.NewNormalizedResponse().WithJsonRpcResponse(jrr), nil + }, + } + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 10, + MinWaitMs: 1, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + CachePerCall: util.BoolPtr(false), + } + cfg.SetDefaults() + + batcher := NewBatcher(cfg, forwarder, nil) + + ctx := context.Background() + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x1111111111111111111111111111111111111111", + "data": "0x01", + }, + "0x10", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + _, _, blockRef, err := ExtractCallInfo(req) + require.NoError(t, err) + + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: blockRef, + DirectivesKey: DeriveDirectivesKey(nil), + } + + entry, _, err := batcher.Enqueue(ctx, key, req) + require.NoError(t, err) + + result := <-entry.ResultCh + require.NoError(t, result.Error) + + select { + case blockParam := <-blockParamCh: + paramStr, ok := blockParam.(string) + require.True(t, ok) + require.Equal(t, "0x10", paramStr) + case <-time.After(2 * time.Second): + require.Fail(t, "timed out waiting for block param") + } + + batcher.Shutdown() +} + func TestBatcherFlushDeduplication(t *testing.T) { // Create result with 1 call (deduplication means only 1 unique call is made) results := []Multicall3Result{ @@ -1257,8 +1351,8 @@ func TestBatcher_MaxCalldataBytes_Bypass(t *testing.T) { Enabled: true, WindowMs: 100, MinWaitMs: 5, - MaxCalls: 100, // High limit - MaxCalldataBytes: 100, // Very low limit for testing + MaxCalls: 100, // High limit + MaxCalldataBytes: 100, // Very low limit for testing MaxQueueSize: 100, MaxPendingBatches: 20, AllowCrossUserBatching: util.BoolPtr(true), From d895793cf91be3d1326b400e39770f2b337a322b Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 16 Jan 2026 10:36:36 +0100 Subject: [PATCH 31/53] fix: improve panic handling, logging, and key safety in multicall3 batcher Critical fixes: - Add panic recovery with stack trace logging to scheduleFlush goroutine - Fix inaccurate comment about map ordering (we iterate entries slice) High priority fixes: - Elevate cache/abandonment logging from Debug to Warn level - Add stack trace logging to fallback panic recovery - Use null byte separator in BatchingKey.String() to prevent collisions Co-Authored-By: Claude Opus 4.5 --- architecture/evm/multicall3_batcher.go | 45 +++++++++++++++++++++++--- erpc/http_batch_eth_call.go | 4 +-- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/architecture/evm/multicall3_batcher.go b/architecture/evm/multicall3_batcher.go index b8db076f2..3adcbcee0 100644 --- a/architecture/evm/multicall3_batcher.go +++ b/architecture/evm/multicall3_batcher.go @@ -4,6 +4,7 @@ import ( "context" "encoding/hex" "fmt" + "runtime/debug" "sort" "strconv" "strings" @@ -230,6 +231,30 @@ func (b *Batcher) Enqueue(ctx context.Context, key BatchingKey, req *common.Norm // scheduleFlush waits until flush time and then flushes the batch. func (b *Batcher) scheduleFlush(keyStr string, batch *Batch) { defer b.wg.Done() + defer func() { + if r := recover(); r != nil { + // Log panic with stack trace and deliver errors to all waiting entries + if b.logger != nil { + b.logger.Error(). + Str("panic", fmt.Sprintf("%v", r)). + Str("stack", string(debug.Stack())). + Str("batchKey", keyStr). + Msg("panic in scheduleFlush goroutine") + } + // Deliver error to all entries in the batch + batch.mu.Lock() + entries := batch.Entries + batch.mu.Unlock() + panicErr := common.NewErrJsonRpcExceptionInternal( + 0, + common.JsonRpcErrorServerSideException, + fmt.Sprintf("internal error: panic in batch scheduler: %v", r), + nil, + nil, + ) + b.deliverError(entries, panicErr, batch.Key.ProjectId, batch.Key.NetworkId) + } + }() for { batch.mu.Lock() @@ -296,8 +321,8 @@ func (b *Batcher) flush(keyStr string, batch *Batch) { // Capture flush time for wait time calculations flushTime := time.Now() - // Build ordered unique calls list (maintains insertion order via CallKeys map iteration order) - // We need to build unique calls in the order they were first seen + // Build ordered unique calls list by iterating entries slice (which preserves insertion order) + // and deduplicating based on CallKey. This ensures deterministic ordering. type uniqueCall struct { callKey string entry *BatchEntry // first entry for this callKey @@ -536,7 +561,7 @@ func (b *Batcher) sendResult(entry *BatchEntry, result BatchResult, projectId, n case <-entry.Ctx.Done(): telemetry.MetricMulticall3AbandonedTotal.WithLabelValues(projectId, networkId).Inc() if b.logger != nil { - b.logger.Debug(). + b.logger.Warn(). Str("projectId", projectId). Str("networkId", networkId). Err(entry.Ctx.Err()). @@ -574,7 +599,16 @@ func (b *Batcher) fallbackIndividual(entries []*BatchEntry, projectId, networkId defer wg.Done() defer func() { if r := recover(); r != nil { - // Panic in forwarder - send error to entry + // Log panic with stack trace + if b.logger != nil { + b.logger.Error(). + Str("panic", fmt.Sprintf("%v", r)). + Str("stack", string(debug.Stack())). + Str("projectId", projectId). + Str("networkId", networkId). + Msg("panic in fallback forward goroutine") + } + // Send error to entry err := fmt.Errorf("panic in fallback forward: %v", r) b.sendResult(e, BatchResult{Error: err}, projectId, networkId) telemetry.MetricMulticall3FallbackRequestsTotal.WithLabelValues(projectId, networkId, "panic").Inc() @@ -661,7 +695,8 @@ type BatchingKey struct { } func (k BatchingKey) String() string { - return fmt.Sprintf("%s|%s|%s|%s|%s", k.ProjectId, k.NetworkId, k.BlockRef, k.DirectivesKey, k.UserId) + // Use null byte separator to prevent key collisions from field values containing the separator + return fmt.Sprintf("%s\x00%s\x00%s\x00%s\x00%s", k.ProjectId, k.NetworkId, k.BlockRef, k.DirectivesKey, k.UserId) } // DeriveDirectivesKey creates a stable, versioned key from relevant directives. diff --git a/erpc/http_batch_eth_call.go b/erpc/http_batch_eth_call.go index 82ffc882f..ce59dab5d 100644 --- a/erpc/http_batch_eth_call.go +++ b/erpc/http_batch_eth_call.go @@ -293,8 +293,8 @@ func (s *HttpServer) handleEthCallBatchAggregation( } cachedResp, err := cacheDal.Get(cand.ctx, cand.req) if err != nil { - // Log cache errors for debugging - they're non-fatal but useful for diagnosing issues - cand.logger.Debug(). + // Log cache errors - they're non-fatal but indicate potential cache issues + cand.logger.Warn(). Err(err). Str("networkId", batchInfo.networkId). Msg("multicall3 pre-aggregation cache get failed, treating as miss") From 3224c5dcaef4a03d51b11ebfce53a27704968aac Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 16 Jan 2026 10:45:16 +0100 Subject: [PATCH 32/53] fix: address P1/P2 issues in multicall3 batching P1 - Preserve block-hash params: - Wrap 66-char hex strings (block hashes) in EIP-1898 format {blockHash: ...} to prevent misinterpretation as block numbers P2 - Release multicall responses: - Add mcResp.Release() after decoding and in error/fallback paths to prevent memory leaks from unreleased response buffers P2 - Add cache reads for batched eth_call: - Check cache before enqueueing requests into batcher - Return cached responses directly, bypassing batching for cache hits - Prevents performance regression when caching is enabled P2 - Apply multicall3Aggregation defaults for TS configs: - Add UnmarshalJSON method with bool backward compatibility - Ensures {enabled: true} gets proper default values in TypeScript configs Co-Authored-By: Claude Opus 4.5 --- architecture/evm/eth_call.go | 21 +++++++++++++++++++++ architecture/evm/multicall3_batcher.go | 12 ++++++++++++ common/config.go | 26 ++++++++++++++++++++++++++ 3 files changed, 59 insertions(+) diff --git a/architecture/evm/eth_call.go b/architecture/evm/eth_call.go index b62a4112c..03748e834 100644 --- a/architecture/evm/eth_call.go +++ b/architecture/evm/eth_call.go @@ -133,6 +133,27 @@ func projectPreForward_eth_call(ctx context.Context, network common.Network, nq UserId: userId, } + // Check cache before batching (unless skip-cache-read is set) + if !nq.SkipCacheRead() { + cache := network.Cache() + if cache != nil && !cache.IsObjectNull() { + cachedResp, cacheErr := cache.Get(ctx, nq) + if cacheErr != nil { + // Log cache errors but continue to batching + if logger := network.Logger(); logger != nil { + logger.Warn(). + Err(cacheErr). + Str("networkId", network.Id()). + Msg("multicall3 pre-batch cache get failed, continuing to batch") + } + } else if cachedResp != nil && !cachedResp.IsObjectNull(ctx) { + // Cache hit - return cached response directly + cachedResp.SetFromCache(true) + return true, cachedResp, nil + } + } + } + // Get or create batcher for this project+network mgr := GetBatcherManager() forwarder := &networkForwarder{network: network} diff --git a/architecture/evm/multicall3_batcher.go b/architecture/evm/multicall3_batcher.go index 3adcbcee0..355154ae7 100644 --- a/architecture/evm/multicall3_batcher.go +++ b/architecture/evm/multicall3_batcher.go @@ -425,6 +425,8 @@ func (b *Batcher) flush(keyStr string, batch *Batch) { // Decode the multicall response results, err := b.decodeMulticallResponse(mcResp) if err != nil { + // Release the multicall response before fallback/error + mcResp.Release() // Check if we should fallback to individual requests if ShouldFallbackMulticall3(err) { telemetry.MetricMulticall3FallbackTotal.WithLabelValues(projectId, networkId, "decode_error").Inc() @@ -437,6 +439,7 @@ func (b *Batcher) flush(keyStr string, batch *Batch) { // Verify result count matches unique calls if len(results) != len(uniqueCalls) { + mcResp.Release() b.deliverError(entries, fmt.Errorf("multicall3 result count mismatch: got %d, expected %d", len(results), len(uniqueCalls)), projectId, networkId) return } @@ -509,6 +512,9 @@ func (b *Batcher) flush(keyStr string, batch *Batch) { } } } + + // Release the multicall response after all results have been mapped + mcResp.Release() } // decodeMulticallResponse extracts and decodes the multicall3 result from a response. @@ -928,6 +934,12 @@ func blockParamForMulticall(blockRef string) (interface{}, error) { return "latest", nil } if strings.HasPrefix(blockRef, "0x") { + // Check if this is a block hash (66 chars = 0x + 64 hex chars = 32 bytes) + // Block hashes need to be wrapped in EIP-1898 format for correct interpretation + if len(blockRef) == 66 { + return map[string]interface{}{"blockHash": blockRef}, nil + } + // Regular hex block number - pass through return blockRef, nil } if isDecimalBlockRef(blockRef) { diff --git a/common/config.go b/common/config.go index 9aae0209f..33d4d8343 100644 --- a/common/config.go +++ b/common/config.go @@ -2,6 +2,7 @@ package common import ( "bytes" + "encoding/json" "fmt" "maps" "os" @@ -1687,6 +1688,31 @@ func (c *Multicall3AggregationConfig) UnmarshalYAML(unmarshal func(interface{}) return nil } +// UnmarshalJSON implements backward compatibility for boolean config values (for TypeScript configs) +func (c *Multicall3AggregationConfig) UnmarshalJSON(data []byte) error { + // Try bool first (backward compat) + var boolVal bool + if err := json.Unmarshal(data, &boolVal); err == nil { + c.Enabled = boolVal + if boolVal { + c.SetDefaults() + } + return nil + } + + // Try full config + type rawConfig Multicall3AggregationConfig + var raw rawConfig + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + *c = Multicall3AggregationConfig(raw) + if c.Enabled { + c.SetDefaults() + } + return nil +} + // EvmIntegrityConfig is deprecated. Use DirectiveDefaultsConfig for validation settings. type EvmIntegrityConfig struct { // @deprecated: use DirectiveDefaults.EnforceHighestBlock From 07ec28633c53ab9ac80174ffbfec872eb09f94ad Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 16 Jan 2026 11:15:26 +0100 Subject: [PATCH 33/53] fix: address PR review comments for multicall3 batching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - P2: Mark calls with requireCanonical:false as ineligible for batching (EIP-1898 flag would be lost when rebuilding block param) - P3: Apply default multicall3 config when not explicitly configured (enables batching by default as documented) - Fix error message wording: "cache-set" → "cache write" for consistency - Add test coverage for requireCanonical eligibility checks Co-Authored-By: Claude Opus 4.5 --- architecture/evm/eth_call.go | 23 ++++++++++--- architecture/evm/multicall3_batcher.go | 11 +++++++ architecture/evm/multicall3_batcher_test.go | 36 +++++++++++++++++++++ erpc/http_batch_eth_call.go | 4 +-- 4 files changed, 68 insertions(+), 6 deletions(-) diff --git a/architecture/evm/eth_call.go b/architecture/evm/eth_call.go index 03748e834..cf71c4c63 100644 --- a/architecture/evm/eth_call.go +++ b/architecture/evm/eth_call.go @@ -14,6 +14,14 @@ var ( batcherManagerOnce sync.Once ) +// defaultMulticall3AggregationConfig is the default config when Multicall3Aggregation +// is not explicitly configured. Enabled by default to match documented behavior. +var defaultMulticall3AggregationConfig = func() *common.Multicall3AggregationConfig { + cfg := &common.Multicall3AggregationConfig{Enabled: true} + cfg.SetDefaults() + return cfg +}() + // GetBatcherManager returns the global batcher manager. func GetBatcherManager() *BatcherManager { batcherManagerOnce.Do(func() { @@ -83,16 +91,23 @@ func projectPreForward_eth_call(ctx context.Context, network common.Network, nq jrq.Unlock() } - // Check if Multicall3 aggregation is enabled + // Get Multicall3 aggregation config, using defaults if not explicitly configured cfg := network.Config() - if cfg == nil || cfg.Evm == nil || cfg.Evm.Multicall3Aggregation == nil || !cfg.Evm.Multicall3Aggregation.Enabled { + var aggCfg *common.Multicall3AggregationConfig + if cfg != nil && cfg.Evm != nil && cfg.Evm.Multicall3Aggregation != nil { + aggCfg = cfg.Evm.Multicall3Aggregation + } else { + // Use default config (enabled by default) + aggCfg = defaultMulticall3AggregationConfig + } + + // Check if Multicall3 aggregation is explicitly disabled + if !aggCfg.Enabled { // Batching disabled, use normal forward resp, err := network.Forward(ctx, nq) return true, resp, err } - aggCfg := cfg.Evm.Multicall3Aggregation - // Check eligibility for batching eligible, reason := IsEligibleForBatching(nq, aggCfg) if !eligible { diff --git a/architecture/evm/multicall3_batcher.go b/architecture/evm/multicall3_batcher.go index 355154ae7..9b30c7a09 100644 --- a/architecture/evm/multicall3_batcher.go +++ b/architecture/evm/multicall3_batcher.go @@ -884,6 +884,17 @@ func IsEligibleForBatching(req *common.NormalizedRequest, cfg *common.Multicall3 // Check block tag blockTag := "latest" if len(params) >= 2 && params[1] != nil { + // Check for EIP-1898 block params with requireCanonical: false + // These cannot be safely batched because the flag would be lost when + // rebuilding the block param as {blockHash: "0x..."} + if blockObj, ok := params[1].(map[string]interface{}); ok { + if reqCanonical, hasReqCanonical := blockObj["requireCanonical"]; hasReqCanonical { + if reqCanonicalBool, ok := reqCanonical.(bool); ok && !reqCanonicalBool { + return false, "has requireCanonical:false" + } + } + } + normalized, err := NormalizeBlockParam(params[1]) if err != nil { return false, fmt.Sprintf("invalid block param: %v", err) diff --git a/architecture/evm/multicall3_batcher_test.go b/architecture/evm/multicall3_batcher_test.go index e42cad09b..572b9d459 100644 --- a/architecture/evm/multicall3_batcher_test.go +++ b/architecture/evm/multicall3_batcher_test.go @@ -254,6 +254,42 @@ func TestIsEligibleForBatching(t *testing.T) { eligible: false, reason: "block tag not allowed", }, + { + name: "ineligible - EIP-1898 block param with requireCanonical:false", + method: "eth_call", + params: []interface{}{ + map[string]interface{}{"to": "0x1234567890123456789012345678901234567890", "data": "0xabcd"}, + map[string]interface{}{ + "blockHash": "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + "requireCanonical": false, + }, + }, + eligible: false, + reason: "has requireCanonical:false", + }, + { + name: "eligible - EIP-1898 block param with requireCanonical:true", + method: "eth_call", + params: []interface{}{ + map[string]interface{}{"to": "0x1234567890123456789012345678901234567890", "data": "0xabcd"}, + map[string]interface{}{ + "blockHash": "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + "requireCanonical": true, + }, + }, + eligible: true, + }, + { + name: "eligible - EIP-1898 block param without requireCanonical (default true)", + method: "eth_call", + params: []interface{}{ + map[string]interface{}{"to": "0x1234567890123456789012345678901234567890", "data": "0xabcd"}, + map[string]interface{}{ + "blockHash": "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + }, + }, + eligible: true, + }, } for _, tt := range tests { diff --git a/erpc/http_batch_eth_call.go b/erpc/http_batch_eth_call.go index ce59dab5d..825e12402 100644 --- a/erpc/http_batch_eth_call.go +++ b/erpc/http_batch_eth_call.go @@ -501,14 +501,14 @@ func (s *HttpServer) handleEthCallBatchAggregation( defer func() { if rec := recover(); rec != nil { telemetry.MetricUnexpectedPanicTotal.WithLabelValues( - "multicall3-cache-set", + "multicall3-cache-write", fmt.Sprintf("network:%s", batchInfo.networkId), common.ErrorFingerprint(rec), ).Inc() lg.Error(). Interface("panic", rec). Str("stack", string(debug.Stack())). - Msg("unexpected panic on multicall3 per-call cache-set") + Msg("unexpected panic on multicall3 per-call cache write") } }() defer resp.RUnlock() From c7cf320bfcbd0ccb7883711ba0dc8ef013e02cf6 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 16 Jan 2026 11:31:21 +0100 Subject: [PATCH 34/53] fix: address HIGH priority issues in multicall3 batching - Fix timer channel drain in scheduleFlush: properly drain timer.C when Stop() returns false to prevent goroutine/memory leaks - Add backpressure for cache write goroutines: use semaphore (100 max) to prevent unbounded goroutine growth under high load; skip cache write gracefully when at capacity - CI already runs tests with -race flag via Makefile Co-Authored-By: Claude Opus 4.5 --- architecture/evm/multicall3_batcher.go | 16 ++++++- erpc/http_batch_eth_call.go | 64 +++++++++++++++----------- 2 files changed, 52 insertions(+), 28 deletions(-) diff --git a/architecture/evm/multicall3_batcher.go b/architecture/evm/multicall3_batcher.go index 9b30c7a09..8ea6d6e20 100644 --- a/architecture/evm/multicall3_batcher.go +++ b/architecture/evm/multicall3_batcher.go @@ -274,10 +274,22 @@ func (b *Batcher) scheduleFlush(keyStr string, batch *Batch) { return case <-batch.notifyCh: // FlushTime was shortened, stop current timer and recalculate - timer.Stop() + // Drain timer channel if Stop() returns false (timer already fired) + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } continue case <-b.shutdown: - timer.Stop() + // Drain timer channel if Stop() returns false (timer already fired) + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } // On shutdown, flush the batch with error to avoid orphaned entries b.flushWithShutdownError(keyStr, batch) return diff --git a/erpc/http_batch_eth_call.go b/erpc/http_batch_eth_call.go index 825e12402..01020b883 100644 --- a/erpc/http_batch_eth_call.go +++ b/erpc/http_batch_eth_call.go @@ -20,6 +20,10 @@ import ( "go.opentelemetry.io/otel/trace" ) +// cacheWriteSem limits concurrent multicall3 per-call cache write goroutines +// to prevent unbounded goroutine growth under high load. +var cacheWriteSem = make(chan struct{}, 100) + type ethCallBatchInfo struct { networkId string blockRef string @@ -493,34 +497,42 @@ func (s *HttpServer) handleEthCallBatchAggregation( responses[cand.index] = nr common.EndRequestSpan(cand.ctx, nr, nil) - // Cache individual response asynchronously + // Cache individual response asynchronously with backpressure if shouldCache { - nr.RLockWithTrace(cand.ctx) - nr.AddRef() - go func(resp *common.NormalizedResponse, req *common.NormalizedRequest, reqCtx context.Context, lg zerolog.Logger) { - defer func() { - if rec := recover(); rec != nil { - telemetry.MetricUnexpectedPanicTotal.WithLabelValues( - "multicall3-cache-write", - fmt.Sprintf("network:%s", batchInfo.networkId), - common.ErrorFingerprint(rec), - ).Inc() - lg.Error(). - Interface("panic", rec). - Str("stack", string(debug.Stack())). - Msg("unexpected panic on multicall3 per-call cache write") + // Try to acquire semaphore (non-blocking to avoid blocking response path) + select { + case cacheWriteSem <- struct{}{}: + nr.RLockWithTrace(cand.ctx) + nr.AddRef() + go func(resp *common.NormalizedResponse, req *common.NormalizedRequest, reqCtx context.Context, lg zerolog.Logger) { + defer func() { <-cacheWriteSem }() // Release semaphore + defer func() { + if rec := recover(); rec != nil { + telemetry.MetricUnexpectedPanicTotal.WithLabelValues( + "multicall3-cache-write", + fmt.Sprintf("network:%s", batchInfo.networkId), + common.ErrorFingerprint(rec), + ).Inc() + lg.Error(). + Interface("panic", rec). + Str("stack", string(debug.Stack())). + Msg("unexpected panic on multicall3 per-call cache write") + } + }() + defer resp.RUnlock() + defer resp.DoneRef() + + timeoutCtx, timeoutCtxCancel := context.WithTimeoutCause(network.AppCtx(), 10*time.Second, errors.New("cache driver timeout during multicall3 per-call cache write")) + defer timeoutCtxCancel() + tracedCtx := trace.ContextWithSpanContext(timeoutCtx, trace.SpanContextFromContext(reqCtx)) + if err := cacheDal.Set(tracedCtx, req, resp); err != nil { + lg.Warn().Err(err).Msg("could not store multicall3 per-call response in cache") } - }() - defer resp.RUnlock() - defer resp.DoneRef() - - timeoutCtx, timeoutCtxCancel := context.WithTimeoutCause(network.AppCtx(), 10*time.Second, errors.New("cache driver timeout during multicall3 per-call cache write")) - defer timeoutCtxCancel() - tracedCtx := trace.ContextWithSpanContext(timeoutCtx, trace.SpanContextFromContext(reqCtx)) - if err := cacheDal.Set(tracedCtx, req, resp); err != nil { - lg.Warn().Err(err).Msg("could not store multicall3 per-call response in cache") - } - }(nr, cand.req, cand.ctx, cand.logger) + }(nr, cand.req, cand.ctx, cand.logger) + default: + // Semaphore full - skip cache write to avoid unbounded goroutine growth + cand.logger.Debug().Msg("skipping multicall3 per-call cache write due to backpressure") + } } continue } From 437dae10324205e8508ffa3720bbcb1fe690a5fa Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 16 Jan 2026 11:56:20 +0100 Subject: [PATCH 35/53] fix: address P2/P3 review issues in multicall3 batching P2: Avoid synthetic deadline for no-timeout batches - Don't create synthetic deadline when context has no deadline - Only update flush time based on deadline when request has one - Prevents unnecessary timeouts on upstream multicall calls P2: Reject mixed requireCanonical block-hash params in batch detection - Track requireCanonical state across requests in a batch - Reject batches with mixed values (some true/default, some false) - Treat explicit true and absent (default true) as compatible - Added test cases for requireCanonical handling P3: Record validation errors in request span - Pass actual error to EndRequestSpan instead of processed error body - Ensures validation errors are properly recorded in tracing spans Co-Authored-By: Claude Opus 4.5 --- architecture/evm/multicall3_batcher.go | 45 +++++++++++++------------ erpc/http_batch_eth_call.go | 27 ++++++++++++++- erpc/http_batch_eth_call_detect_test.go | 36 ++++++++++++++++++++ 3 files changed, 86 insertions(+), 22 deletions(-) diff --git a/architecture/evm/multicall3_batcher.go b/architecture/evm/multicall3_batcher.go index 8ea6d6e20..f7de3de60 100644 --- a/architecture/evm/multicall3_batcher.go +++ b/architecture/evm/multicall3_batcher.go @@ -91,16 +91,15 @@ func (b *Batcher) Enqueue(ctx context.Context, key BatchingKey, req *common.Norm return nil, true, err } - // Calculate deadline from context + // Get deadline from context (if any) + // We don't create a synthetic deadline for no-timeout requests to avoid + // causing unnecessary timeouts on the upstream multicall call. + now := time.Now() deadline, hasDeadline := ctx.Deadline() - if !hasDeadline { - deadline = time.Now().Add(time.Duration(b.cfg.WindowMs) * time.Millisecond) - } - // Check if deadline is too tight - now := time.Now() + // Check if deadline is too tight (only if there's a deadline) minWait := time.Duration(b.cfg.MinWaitMs) * time.Millisecond - if deadline.Before(now.Add(minWait)) { + if hasDeadline && deadline.Before(now.Add(minWait)) { b.logBypass(key, "deadline_too_tight") return nil, true, nil } @@ -204,20 +203,24 @@ func (b *Batcher) Enqueue(ctx context.Context, key BatchingKey, req *common.Norm batch.CallKeys[callKey] = append(batch.CallKeys[callKey], entry) // Update flush time based on deadline (deadline-aware) - safetyMargin := time.Duration(b.cfg.SafetyMarginMs) * time.Millisecond - proposedFlush := deadline.Add(-safetyMargin) - if proposedFlush.Before(batch.FlushTime) { - batch.FlushTime = proposedFlush - // Clamp to minimum wait - minFlush := now.Add(minWait) - if batch.FlushTime.Before(minFlush) { - batch.FlushTime = minFlush - } - // Notify the flush goroutine that FlushTime was shortened - select { - case batch.notifyCh <- struct{}{}: - default: - // Already has a pending notification + // Only update if the request has a deadline - requests without deadlines + // should not cause early flushes. + if hasDeadline { + safetyMargin := time.Duration(b.cfg.SafetyMarginMs) * time.Millisecond + proposedFlush := deadline.Add(-safetyMargin) + if proposedFlush.Before(batch.FlushTime) { + batch.FlushTime = proposedFlush + // Clamp to minimum wait + minFlush := now.Add(minWait) + if batch.FlushTime.Before(minFlush) { + batch.FlushTime = minFlush + } + // Notify the flush goroutine that FlushTime was shortened + select { + case batch.notifyCh <- struct{}{}: + default: + // Already has a pending notification + } } } diff --git a/erpc/http_batch_eth_call.go b/erpc/http_batch_eth_call.go index 01020b883..20c9bb4aa 100644 --- a/erpc/http_batch_eth_call.go +++ b/erpc/http_batch_eth_call.go @@ -77,6 +77,9 @@ func detectEthCallBatchInfo(requests []json.RawMessage, architecture, chainId st var networkId string var blockRef string var blockParam interface{} + // Track requireCanonical state across requests: + // 0 = not yet set, 1 = explicitly true, 2 = explicitly false, 3 = not specified (default true) + var requireCanonicalState int for _, raw := range requests { var probe ethCallBatchProbe @@ -114,6 +117,28 @@ func detectEthCallBatchInfo(requests []json.RawMessage, architecture, chainId st } else if blockRef != bref { return nil, nil } + + // Check for mixed requireCanonical values in block-hash params (EIP-1898) + // We need to ensure all requests in a batch have compatible requireCanonical values, + // otherwise the Multicall3 call won't honor individual semantics. + // States: 0 = not yet set, 1 = true (explicit or default), 2 = explicitly false + // Explicit true and absent (default true) are treated as compatible (both = 1) + if blockObj, ok := param.(map[string]interface{}); ok { + if _, hasBlockHash := blockObj["blockHash"]; hasBlockHash { + currentState := 1 // default: true (EIP-1898 default) + if reqCanonical, hasReqCanonical := blockObj["requireCanonical"]; hasReqCanonical { + if reqCanonicalBool, ok := reqCanonical.(bool); ok && !reqCanonicalBool { + currentState = 2 // explicitly false + } + } + if requireCanonicalState == 0 { + requireCanonicalState = currentState + } else if requireCanonicalState != currentState { + // Mixed requireCanonical values - not eligible for batching + return nil, nil + } + } + } } if networkId == "" || blockRef == "" { @@ -205,7 +230,7 @@ func (s *HttpServer) handleEthCallBatchAggregation( if err := nq.Validate(); err != nil { responses[i] = processErrorBody(&baseLogger, startedAt, nq, err, &common.TRUE) - common.EndRequestSpan(requestCtx, nil, responses[i]) + common.EndRequestSpan(requestCtx, nil, err) continue } diff --git a/erpc/http_batch_eth_call_detect_test.go b/erpc/http_batch_eth_call_detect_test.go index 163b0059d..df66b4420 100644 --- a/erpc/http_batch_eth_call_detect_test.go +++ b/erpc/http_batch_eth_call_detect_test.go @@ -98,6 +98,42 @@ func TestDetectEthCallBatchInfo(t *testing.T) { wantBlockRef: "latest", wantBlock: "latest", }, + { + name: "mixed requireCanonical - one false one true", + requests: []json.RawMessage{ + buildRaw(t, "eth_call", []interface{}{callObj, map[string]interface{}{"blockHash": "0x1234567890123456789012345678901234567890123456789012345678901234", "requireCanonical": false}}, "evm:1"), + buildRaw(t, "eth_call", []interface{}{callObj, map[string]interface{}{"blockHash": "0x1234567890123456789012345678901234567890123456789012345678901234", "requireCanonical": true}}, "evm:1"), + }, + wantNil: true, + }, + { + name: "mixed requireCanonical - one false one default", + requests: []json.RawMessage{ + buildRaw(t, "eth_call", []interface{}{callObj, map[string]interface{}{"blockHash": "0x1234567890123456789012345678901234567890123456789012345678901234", "requireCanonical": false}}, "evm:1"), + buildRaw(t, "eth_call", []interface{}{callObj, map[string]interface{}{"blockHash": "0x1234567890123456789012345678901234567890123456789012345678901234"}}, "evm:1"), + }, + wantNil: true, + }, + { + name: "same requireCanonical - both true", + requests: []json.RawMessage{ + buildRaw(t, "eth_call", []interface{}{callObj, map[string]interface{}{"blockHash": "0x1234567890123456789012345678901234567890123456789012345678901234", "requireCanonical": true}}, "evm:1"), + buildRaw(t, "eth_call", []interface{}{callObj, map[string]interface{}{"blockHash": "0x1234567890123456789012345678901234567890123456789012345678901234", "requireCanonical": true}}, "evm:1"), + }, + wantNetwork: "evm:1", + wantBlockRef: "0x1234567890123456789012345678901234567890123456789012345678901234", + wantBlock: map[string]interface{}{"blockHash": "0x1234567890123456789012345678901234567890123456789012345678901234", "requireCanonical": true}, + }, + { + name: "same requireCanonical - explicit true and default compatible", + requests: []json.RawMessage{ + buildRaw(t, "eth_call", []interface{}{callObj, map[string]interface{}{"blockHash": "0x1234567890123456789012345678901234567890123456789012345678901234", "requireCanonical": true}}, "evm:1"), + buildRaw(t, "eth_call", []interface{}{callObj, map[string]interface{}{"blockHash": "0x1234567890123456789012345678901234567890123456789012345678901234"}}, "evm:1"), + }, + wantNetwork: "evm:1", + wantBlockRef: "0x1234567890123456789012345678901234567890123456789012345678901234", + wantBlock: map[string]interface{}{"blockHash": "0x1234567890123456789012345678901234567890123456789012345678901234", "requireCanonical": true}, + }, } for _, tt := range cases { From 56742326439abe164924b391fe2431e53b0e9edd Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 16 Jan 2026 12:08:21 +0100 Subject: [PATCH 36/53] fix: address critical and high priority review issues CRITICAL fixes: - Add panic recovery in forwardEthCallBatchCandidates goroutines to prevent server crash on unexpected panics - Add MetricMulticall3CacheReadErrorsTotal for cache get errors to improve observability HIGH priority fixes: - Fix response release race condition by removing async release (was using `go resp.Release()`, now uses synchronous release) Tests: - Add TestNewBatcher_NilForwarder_Panics to verify panic on nil forwarder - Add TestBatcher_ScheduleFlush_PanicRecovery to verify panic recovery Co-Authored-By: Claude Opus 4.5 --- architecture/evm/multicall3_batcher_test.go | 83 +++++++++++++++++++++ erpc/http_batch_eth_call.go | 22 +++++- telemetry/metrics.go | 6 ++ 3 files changed, 109 insertions(+), 2 deletions(-) diff --git a/architecture/evm/multicall3_batcher_test.go b/architecture/evm/multicall3_batcher_test.go index 572b9d459..0a800fcb3 100644 --- a/architecture/evm/multicall3_batcher_test.go +++ b/architecture/evm/multicall3_batcher_test.go @@ -1234,6 +1234,24 @@ func TestNewBatcher_EnabledConfig(t *testing.T) { batcher.Shutdown() } +func TestNewBatcher_NilForwarder_Panics(t *testing.T) { + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 50, + MinWaitMs: 5, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + } + cfg.SetDefaults() + + // Test that nil forwarder causes panic + require.Panics(t, func() { + NewBatcher(cfg, nil, nil) + }, "NewBatcher should panic when forwarder is nil") +} + // mockForwarderWithCacheError is a forwarder that returns errors from SetCache type mockForwarderWithCacheError struct { response *common.NormalizedResponse @@ -1846,3 +1864,68 @@ func TestBatcher_DoubleFlushPrevention(t *testing.T) { t.Fatal("timeout waiting for result") } } + +// mockPanicForwarder panics when Forward is called to test panic recovery +type mockPanicForwarder struct { + panicMessage string +} + +func (m *mockPanicForwarder) Forward(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + panic(m.panicMessage) +} + +func (m *mockPanicForwarder) SetCache(ctx context.Context, req *common.NormalizedRequest, resp *common.NormalizedResponse) error { + return nil +} + +func TestBatcher_ScheduleFlush_PanicRecovery(t *testing.T) { + forwarder := &mockPanicForwarder{panicMessage: "test panic in forwarder"} + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 10, + MinWaitMs: 5, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + } + cfg.SetDefaults() + + batcher := NewBatcher(cfg, forwarder, nil) + require.NotNil(t, batcher) + defer batcher.Shutdown() + + ctx := context.Background() + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + // Enqueue a request + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x1111111111111111111111111111111111111111", + "data": "0x01020304", + }, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + + entry, bypass, err := batcher.Enqueue(ctx, key, req) + require.NoError(t, err) + require.False(t, bypass) + + // Wait for the batch to flush (will panic and recover) + select { + case result := <-entry.ResultCh: + // Should receive an error due to the panic + require.Error(t, result.Error) + require.Contains(t, result.Error.Error(), "panic") + case <-time.After(2 * time.Second): + t.Fatal("timeout waiting for result - panic recovery may have failed") + } +} diff --git a/erpc/http_batch_eth_call.go b/erpc/http_batch_eth_call.go index 20c9bb4aa..c6ce7f038 100644 --- a/erpc/http_batch_eth_call.go +++ b/erpc/http_batch_eth_call.go @@ -180,10 +180,27 @@ func (s *HttpServer) forwardEthCallBatchCandidates( wg.Add(1) go func(c ethCallBatchCandidate) { defer wg.Done() + defer func() { + if r := recover(); r != nil { + panicErr := common.NewErrJsonRpcExceptionInternal( + 0, + common.JsonRpcErrorServerSideException, + fmt.Sprintf("internal error: panic in batch fallback: %v", r), + nil, + nil, + ) + c.logger.Error(). + Str("panic", fmt.Sprintf("%v", r)). + Str("stack", string(debug.Stack())). + Msg("panic in forwardEthCallBatchCandidates goroutine") + responses[c.index] = processErrorBody(&c.logger, startedAt, c.req, panicErr, s.serverCfg.IncludeErrorDetails) + common.EndRequestSpan(c.ctx, nil, panicErr) + } + }() resp, err := forwardBatchProject(withSkipNetworkRateLimit(c.ctx), project, network, c.req) if err != nil { if resp != nil { - go resp.Release() + resp.Release() } responses[c.index] = processErrorBody(&c.logger, startedAt, c.req, err, s.serverCfg.IncludeErrorDetails) common.EndRequestSpan(c.ctx, nil, err) @@ -322,7 +339,8 @@ func (s *HttpServer) handleEthCallBatchAggregation( } cachedResp, err := cacheDal.Get(cand.ctx, cand.req) if err != nil { - // Log cache errors - they're non-fatal but indicate potential cache issues + // Log and track cache errors - they're non-fatal but indicate potential cache issues + telemetry.MetricMulticall3CacheReadErrorsTotal.WithLabelValues(projectId, batchInfo.networkId).Inc() cand.logger.Warn(). Err(err). Str("networkId", batchInfo.networkId). diff --git a/telemetry/metrics.go b/telemetry/metrics.go index d5ba27232..7309c0303 100644 --- a/telemetry/metrics.go +++ b/telemetry/metrics.go @@ -469,6 +469,12 @@ var ( Help: "Total number of per-call cache write errors in multicall3 batch responses.", }, []string{"project", "network"}) + MetricMulticall3CacheReadErrorsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "multicall3_cache_read_errors_total", + Help: "Total number of cache read errors during multicall3 pre-aggregation cache check.", + }, []string{"project", "network"}) + MetricMulticall3FallbackRequestsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "erpc", Name: "multicall3_fallback_requests_total", From 0ec52d08a2d66ab70b318628c7e34edff3e8363e Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 16 Jan 2026 12:21:55 +0100 Subject: [PATCH 37/53] fix: address medium and low priority review issues - Extract timer drain helper function (stopAndDrainTimer) - Add BatchingKey.Validate() method for required field validation - Add target address length validation (20 bytes) - Change cache write log level from Debug to Warn - Extract magic number 100 to maxConcurrentCacheWrites constant Tests: - Add TestBatcher_FallbackIndividual_PanicRecovery - Add TestBatcher_MaxQueueSize_Enforcement - Add TestBatchingKey_Validate - Add TestBatcher_InvalidTargetLength_Bypass Co-Authored-By: Claude Opus 4.5 --- architecture/evm/multicall3_batcher.go | 58 +++-- architecture/evm/multicall3_batcher_test.go | 239 ++++++++++++++++++++ erpc/http_batch_eth_call.go | 7 +- 3 files changed, 288 insertions(+), 16 deletions(-) diff --git a/architecture/evm/multicall3_batcher.go b/architecture/evm/multicall3_batcher.go index f7de3de60..9a01a5b1c 100644 --- a/architecture/evm/multicall3_batcher.go +++ b/architecture/evm/multicall3_batcher.go @@ -16,6 +16,18 @@ import ( "github.com/rs/zerolog" ) +// stopAndDrainTimer safely stops a timer and drains its channel if needed. +// This pattern is required because timer.Stop() returns false if the timer already fired, +// and in that case the channel must be drained to avoid goroutine leaks. +func stopAndDrainTimer(timer *time.Timer) { + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } +} + // Forwarder is the interface for forwarding requests through the network layer. type Forwarder interface { Forward(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) @@ -77,6 +89,12 @@ func (b *Batcher) logBypass(key BatchingKey, reason string) { // - bypass: true if request should be forwarded individually // - error: any error during processing func (b *Batcher) Enqueue(ctx context.Context, key BatchingKey, req *common.NormalizedRequest) (*BatchEntry, bool, error) { + // Validate batching key + if err := key.Validate(); err != nil { + b.logBypass(key, fmt.Sprintf("invalid_key: %v", err)) + return nil, true, err + } + // Extract call info target, callData, _, err := ExtractCallInfo(req) if err != nil { @@ -84,6 +102,13 @@ func (b *Batcher) Enqueue(ctx context.Context, key BatchingKey, req *common.Norm return nil, true, err } + // Validate target address length (must be 20 bytes for EVM) + if len(target) != 20 { + err := fmt.Errorf("invalid target address length: got %d, expected 20", len(target)) + b.logBypass(key, fmt.Sprintf("invalid_target: %v", err)) + return nil, true, err + } + // Derive call key for deduplication callKey, err := DeriveCallKey(req) if err != nil { @@ -277,22 +302,10 @@ func (b *Batcher) scheduleFlush(keyStr string, batch *Batch) { return case <-batch.notifyCh: // FlushTime was shortened, stop current timer and recalculate - // Drain timer channel if Stop() returns false (timer already fired) - if !timer.Stop() { - select { - case <-timer.C: - default: - } - } + stopAndDrainTimer(timer) continue case <-b.shutdown: - // Drain timer channel if Stop() returns false (timer already fired) - if !timer.Stop() { - select { - case <-timer.C: - default: - } - } + stopAndDrainTimer(timer) // On shutdown, flush the batch with error to avoid orphaned entries b.flushWithShutdownError(keyStr, batch) return @@ -493,7 +506,7 @@ func (b *Batcher) flush(keyStr string, batch *Batch) { // Cache write failures are non-critical but we track them for observability telemetry.MetricMulticall3CacheWriteErrorsTotal.WithLabelValues(projectId, networkId).Inc() if b.logger != nil { - b.logger.Debug(). + b.logger.Warn(). Err(err). Str("projectId", projectId). Str("networkId", networkId). @@ -715,6 +728,21 @@ type BatchingKey struct { UserId string // empty if cross-user batching is allowed } +// Validate checks that the BatchingKey has required fields set. +// Returns an error if any required field is empty. +func (k BatchingKey) Validate() error { + if k.ProjectId == "" { + return fmt.Errorf("BatchingKey.ProjectId is required") + } + if k.NetworkId == "" { + return fmt.Errorf("BatchingKey.NetworkId is required") + } + if k.BlockRef == "" { + return fmt.Errorf("BatchingKey.BlockRef is required") + } + return nil +} + func (k BatchingKey) String() string { // Use null byte separator to prevent key collisions from field values containing the separator return fmt.Sprintf("%s\x00%s\x00%s\x00%s\x00%s", k.ProjectId, k.NetworkId, k.BlockRef, k.DirectivesKey, k.UserId) diff --git a/architecture/evm/multicall3_batcher_test.go b/architecture/evm/multicall3_batcher_test.go index 0a800fcb3..2e77d80dd 100644 --- a/architecture/evm/multicall3_batcher_test.go +++ b/architecture/evm/multicall3_batcher_test.go @@ -1929,3 +1929,242 @@ func TestBatcher_ScheduleFlush_PanicRecovery(t *testing.T) { t.Fatal("timeout waiting for result - panic recovery may have failed") } } + +// mockFallbackThenPanicForwarder returns an error triggering fallback on first call, +// then panics on subsequent (individual) calls to test fallback panic recovery +type mockFallbackThenPanicForwarder struct { + callCount int + panicMessage string + mu sync.Mutex +} + +func (m *mockFallbackThenPanicForwarder) Forward(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + m.mu.Lock() + m.callCount++ + count := m.callCount + m.mu.Unlock() + + if count == 1 { + // First call is the multicall - return error that triggers fallback + return nil, common.NewErrEndpointExecutionException( + fmt.Errorf("contract not found"), + ) + } + // Subsequent calls (individual fallback) - panic + panic(m.panicMessage) +} + +func (m *mockFallbackThenPanicForwarder) SetCache(ctx context.Context, req *common.NormalizedRequest, resp *common.NormalizedResponse) error { + return nil +} + +func TestBatcher_FallbackIndividual_PanicRecovery(t *testing.T) { + forwarder := &mockFallbackThenPanicForwarder{panicMessage: "test panic in fallback"} + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 10, + MinWaitMs: 5, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + } + cfg.SetDefaults() + + batcher := NewBatcher(cfg, forwarder, nil) + require.NotNil(t, batcher) + defer batcher.Shutdown() + + ctx := context.Background() + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + // Enqueue a request + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x1111111111111111111111111111111111111111", + "data": "0x01020304", + }, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + + entry, bypass, err := batcher.Enqueue(ctx, key, req) + require.NoError(t, err) + require.False(t, bypass) + + // Wait for the batch to flush (multicall fails with "contract not found", + // triggers fallback, fallback panics and recovers) + select { + case result := <-entry.ResultCh: + // Should receive an error due to the panic in fallback + require.Error(t, result.Error) + require.Contains(t, result.Error.Error(), "panic in fallback forward") + case <-time.After(2 * time.Second): + t.Fatal("timeout waiting for result - fallback panic recovery may have failed") + } +} + +func TestBatcher_MaxQueueSize_Enforcement(t *testing.T) { + forwarder := &mockForwarder{} + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 1000, // Long window to prevent auto-flush + MinWaitMs: 5, + MaxCalls: 100, + MaxCalldataBytes: 64000, + MaxQueueSize: 3, // Small queue for testing + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + } + cfg.SetDefaults() + + batcher := NewBatcher(cfg, forwarder, nil) + require.NotNil(t, batcher) + defer batcher.Shutdown() + + ctx := context.Background() + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + // Fill up the queue + for i := 0; i < 3; i++ { + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": fmt.Sprintf("0x%040d", i+1), + "data": "0x01020304", + }, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + _, bypass, err := batcher.Enqueue(ctx, key, req) + require.NoError(t, err) + require.False(t, bypass, "request %d should be enqueued", i) + } + + // Next request should bypass due to full queue + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x0000000000000000000000000000000000000099", + "data": "0x01020304", + }, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + _, bypass, err := batcher.Enqueue(ctx, key, req) + require.NoError(t, err) + require.True(t, bypass, "4th request should bypass due to full queue") +} + +func TestBatchingKey_Validate(t *testing.T) { + tests := []struct { + name string + key BatchingKey + wantErr bool + errMsg string + }{ + { + name: "valid key", + key: BatchingKey{ + ProjectId: "proj1", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: "v1:", + }, + wantErr: false, + }, + { + name: "missing project id", + key: BatchingKey{ + NetworkId: "evm:1", + BlockRef: "latest", + }, + wantErr: true, + errMsg: "ProjectId is required", + }, + { + name: "missing network id", + key: BatchingKey{ + ProjectId: "proj1", + BlockRef: "latest", + }, + wantErr: true, + errMsg: "NetworkId is required", + }, + { + name: "missing block ref", + key: BatchingKey{ + ProjectId: "proj1", + NetworkId: "evm:1", + }, + wantErr: true, + errMsg: "BlockRef is required", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.key.Validate() + if tt.wantErr { + require.Error(t, err) + require.Contains(t, err.Error(), tt.errMsg) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestBatcher_InvalidTargetLength_Bypass(t *testing.T) { + forwarder := &mockForwarder{} + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 50, + MinWaitMs: 5, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + } + cfg.SetDefaults() + + batcher := NewBatcher(cfg, forwarder, nil) + require.NotNil(t, batcher) + defer batcher.Shutdown() + + ctx := context.Background() + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + // Request with invalid target address (21 bytes instead of 20) + // 42 hex chars = 21 bytes + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x00112233445566778899aabbccddeeff00112233ab", // 21 bytes (42 hex chars) + "data": "0x01020304", + }, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + _, bypass, err := batcher.Enqueue(ctx, key, req) + require.Error(t, err) + require.True(t, bypass, "request with invalid target should bypass") + require.Contains(t, err.Error(), "invalid target address length") +} diff --git a/erpc/http_batch_eth_call.go b/erpc/http_batch_eth_call.go index c6ce7f038..fe528508e 100644 --- a/erpc/http_batch_eth_call.go +++ b/erpc/http_batch_eth_call.go @@ -20,9 +20,14 @@ import ( "go.opentelemetry.io/otel/trace" ) +// maxConcurrentCacheWrites limits concurrent multicall3 per-call cache write goroutines. +// This value balances memory usage with throughput - too low causes dropped writes under load, +// too high risks memory pressure. 100 is chosen as a reasonable default for most workloads. +const maxConcurrentCacheWrites = 100 + // cacheWriteSem limits concurrent multicall3 per-call cache write goroutines // to prevent unbounded goroutine growth under high load. -var cacheWriteSem = make(chan struct{}, 100) +var cacheWriteSem = make(chan struct{}, maxConcurrentCacheWrites) type ethCallBatchInfo struct { networkId string From 53cbc5818bdefb30743870c6a3cc9cf0ae17db86 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 16 Jan 2026 14:12:01 +0100 Subject: [PATCH 38/53] fix: explicitly disable batching in eth_call tests Tests assumed nil Multicall3Aggregation config would disable batching, but the default config has Enabled: true. Now explicitly set Enabled: false. Co-Authored-By: Claude Opus 4.5 --- architecture/evm/eth_call_test.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/architecture/evm/eth_call_test.go b/architecture/evm/eth_call_test.go index ed09260b3..a4086d257 100644 --- a/architecture/evm/eth_call_test.go +++ b/architecture/evm/eth_call_test.go @@ -158,7 +158,10 @@ func TestProjectPreForward_eth_call_NoBatching_Disabled(t *testing.T) { cfg := &common.NetworkConfig{ Evm: &common.EvmNetworkConfig{ ChainId: 1, - // Multicall3Aggregation is nil - batching disabled + // Explicitly disable batching (nil config uses default which has Enabled: true) + Multicall3Aggregation: &common.Multicall3AggregationConfig{ + Enabled: false, + }, }, } @@ -249,7 +252,11 @@ func TestProjectPreForward_eth_call_AddsBlockParam(t *testing.T) { cfg := &common.NetworkConfig{ Evm: &common.EvmNetworkConfig{ ChainId: 1, - // Batching disabled to test block param normalization + // Explicitly disable batching to test block param normalization + // (nil config uses default which has Enabled: true) + Multicall3Aggregation: &common.Multicall3AggregationConfig{ + Enabled: false, + }, }, } From ce344ac20cf9a9d6d55038c3c4aa850d38d3b6a7 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 16 Jan 2026 14:14:15 +0100 Subject: [PATCH 39/53] docs: add Multicall3 aggregation configuration documentation Document the multicall3Aggregation config option for EVM networks, including all available settings: enabled, windowMs, minWaitMs, maxCalls, maxCalldataBytes, maxQueueSize, maxPendingBatches, and allowCrossUserBatching. Co-Authored-By: Claude Opus 4.5 --- docs/pages/config/projects/networks.mdx | 52 +++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/docs/pages/config/projects/networks.mdx b/docs/pages/config/projects/networks.mdx index 2e40cbe99..96911ebf4 100644 --- a/docs/pages/config/projects/networks.mdx +++ b/docs/pages/config/projects/networks.mdx @@ -41,6 +41,35 @@ projects: # When true (default), duplicate transaction errors are converted to success responses, # allowing safe use of retry/hedge policies with transaction sending. idempotentTransactionBroadcast: true + # (OPTIONAL) Configure Multicall3 aggregation for eth_call batching. + # When enabled (default), multiple eth_call requests targeting the same block are batched + # into a single Multicall3 aggregate3 call, reducing RPC calls and improving efficiency. + multicall3Aggregation: + # Enable or disable Multicall3 aggregation. + # DEFAULT: true + enabled: true + # Time window in milliseconds to collect requests before batching. + # DEFAULT: 50 + windowMs: 50 + # Minimum wait time in milliseconds before flushing a batch. + # DEFAULT: 10 + minWaitMs: 10 + # Maximum number of calls per batch. + # DEFAULT: 50 + maxCalls: 50 + # Maximum total calldata size in bytes for a batch. + # DEFAULT: 128000 + maxCalldataBytes: 128000 + # Maximum number of requests waiting to be batched. + # DEFAULT: 1000 + maxQueueSize: 1000 + # Maximum number of pending batches being processed. + # DEFAULT: 100 + maxPendingBatches: 100 + # Allow batching requests from different users together. + # Set to false to isolate user requests into separate batches. + # DEFAULT: true + allowCrossUserBatching: true # (OPTIONAL) A friendly alias for this network. This allows you to reference the network using the alias # instead of the architecture/chainId format. For example, instead of using /main/evm/1, you can use /main/ethereum. @@ -174,6 +203,29 @@ export default createConfig({ * allowing safe use of retry/hedge policies with transaction sending. */ idempotentTransactionBroadcast: true, + /** + * (OPTIONAL) Configure Multicall3 aggregation for eth_call batching. + * When enabled (default), multiple eth_call requests targeting the same block are batched + * into a single Multicall3 aggregate3 call, reducing RPC calls and improving efficiency. + */ + multicall3Aggregation: { + // Enable or disable Multicall3 aggregation. DEFAULT: true + enabled: true, + // Time window in milliseconds to collect requests before batching. DEFAULT: 50 + windowMs: 50, + // Minimum wait time in milliseconds before flushing a batch. DEFAULT: 10 + minWaitMs: 10, + // Maximum number of calls per batch. DEFAULT: 50 + maxCalls: 50, + // Maximum total calldata size in bytes for a batch. DEFAULT: 128000 + maxCalldataBytes: 128000, + // Maximum number of requests waiting to be batched. DEFAULT: 1000 + maxQueueSize: 1000, + // Maximum number of pending batches being processed. DEFAULT: 100 + maxPendingBatches: 100, + // Allow batching requests from different users together. DEFAULT: true + allowCrossUserBatching: true, + }, }, /** From 7b8da2242b2e11accded7dca54dbd9b365a8602a Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 16 Jan 2026 14:17:54 +0100 Subject: [PATCH 40/53] chore: remove Claude CI workflows from feature branch --- .github/workflows/claude-code-review.yml | 44 --------------------- .github/workflows/claude.yml | 50 ------------------------ 2 files changed, 94 deletions(-) delete mode 100644 .github/workflows/claude-code-review.yml delete mode 100644 .github/workflows/claude.yml diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml deleted file mode 100644 index b5e8cfd4d..000000000 --- a/.github/workflows/claude-code-review.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Claude Code Review - -on: - pull_request: - types: [opened, synchronize, ready_for_review, reopened] - # Optional: Only run on specific file changes - # paths: - # - "src/**/*.ts" - # - "src/**/*.tsx" - # - "src/**/*.js" - # - "src/**/*.jsx" - -jobs: - claude-review: - # Optional: Filter by PR author - # if: | - # github.event.pull_request.user.login == 'external-contributor' || - # github.event.pull_request.user.login == 'new-developer' || - # github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR' - - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: read - issues: read - id-token: write - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - - name: Run Claude Code Review - id: claude-review - uses: anthropics/claude-code-action@v1 - with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - plugin_marketplaces: 'https://github.com/anthropics/claude-code.git' - plugins: 'code-review@claude-code-plugins' - prompt: '/code-review:code-review ${{ github.repository }}/pull/${{ github.event.pull_request.number }}' - # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md - # or https://code.claude.com/docs/en/cli-reference for available options - diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml deleted file mode 100644 index d300267f1..000000000 --- a/.github/workflows/claude.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Claude Code - -on: - issue_comment: - types: [created] - pull_request_review_comment: - types: [created] - issues: - types: [opened, assigned] - pull_request_review: - types: [submitted] - -jobs: - claude: - if: | - (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) || - (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || - (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || - (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: read - issues: read - id-token: write - actions: read # Required for Claude to read CI results on PRs - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - - name: Run Claude Code - id: claude - uses: anthropics/claude-code-action@v1 - with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - - # This is an optional setting that allows Claude to read CI results on PRs - additional_permissions: | - actions: read - - # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it. - # prompt: 'Update the pull request description to include a summary of changes.' - - # Optional: Add claude_args to customize behavior and configuration - # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md - # or https://code.claude.com/docs/en/cli-reference for available options - # claude_args: '--allowed-tools Bash(gh pr:*)' - From 355623ccd6cc26e61d2a6ba3df8ec651b945a5c1 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 16 Jan 2026 14:36:12 +0100 Subject: [PATCH 41/53] chore: remove implementation plan (keep design doc only) --- .../2026-01-15-multicall3-network-batching.md | 2674 ----------------- 1 file changed, 2674 deletions(-) delete mode 100644 docs/plans/2026-01-15-multicall3-network-batching.md diff --git a/docs/plans/2026-01-15-multicall3-network-batching.md b/docs/plans/2026-01-15-multicall3-network-batching.md deleted file mode 100644 index 2a53b2aba..000000000 --- a/docs/plans/2026-01-15-multicall3-network-batching.md +++ /dev/null @@ -1,2674 +0,0 @@ -# Multicall3 Network-Level Batching Implementation Plan - -> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. - -**Goal:** Move Multicall3 batching from the HTTP layer to the network layer, enabling batching across all entrypoints (HTTP single/batch + gRPC). - -**Architecture:** Create a concurrent batcher that aggregates `eth_call` requests sharing the same batching key (projectId + networkId + blockRef + directivesKey + optionally userId). Requests are enqueued with deadline-aware flush windows, deduplicated by callKey, and forwarded as a single Multicall3 call. Results fan out to all waiters. Fallback to individual calls on Multicall3 failure. - -**Tech Stack:** Go concurrency primitives (sync.Map, channels, mutexes), existing evm.BuildMulticall3Request/DecodeMulticall3Aggregate3Result, prometheus metrics. - -**Reference:** Design doc at `docs/design/multicall3-batching.md` - ---- - -## Phase 1: Configuration Extension - -### Task 1.1: Add Multicall3AggregationConfig Type - -**Files:** -- Modify: `common/config.go:1531-1536` -- Test: `common/config_test.go` (add section) - -**Step 1: Write the failing test** - -Add to `common/config_test.go`: - -```go -func TestMulticall3AggregationConfigYAML(t *testing.T) { - yamlStr := ` -evm: - chainId: 1 - multicall3Aggregation: - enabled: true - windowMs: 25 - minWaitMs: 2 - safetyMarginMs: 2 - maxCalls: 20 - maxCalldataBytes: 64000 - maxQueueSize: 1000 - maxPendingBatches: 200 - cachePerCall: true - allowCrossUserBatching: true - allowPendingTagBatching: false -` - var cfg NetworkConfig - err := yaml.Unmarshal([]byte(yamlStr), &cfg) - require.NoError(t, err) - require.NotNil(t, cfg.Evm) - require.NotNil(t, cfg.Evm.Multicall3Aggregation) - require.True(t, cfg.Evm.Multicall3Aggregation.Enabled) - require.Equal(t, 25, cfg.Evm.Multicall3Aggregation.WindowMs) - require.Equal(t, 20, cfg.Evm.Multicall3Aggregation.MaxCalls) -} - -func TestMulticall3AggregationConfigBoolBackcompat(t *testing.T) { - // Test backward compatibility with bool value - yamlStr := ` -evm: - chainId: 1 - multicall3Aggregation: true -` - var cfg NetworkConfig - err := yaml.Unmarshal([]byte(yamlStr), &cfg) - require.NoError(t, err) - require.NotNil(t, cfg.Evm) - require.NotNil(t, cfg.Evm.Multicall3Aggregation) - require.True(t, cfg.Evm.Multicall3Aggregation.Enabled) -} - -func TestMulticall3AggregationConfigDefaults(t *testing.T) { - cfg := &Multicall3AggregationConfig{Enabled: true} - cfg.SetDefaults() - require.Equal(t, 25, cfg.WindowMs) - require.Equal(t, 2, cfg.MinWaitMs) - require.Equal(t, 20, cfg.MaxCalls) - require.Equal(t, 64000, cfg.MaxCalldataBytes) -} -``` - -**Step 2: Run test to verify it fails** - -Run: `go test -v ./common -run TestMulticall3Aggregation` -Expected: FAIL - types don't exist - -**Step 3: Write minimal implementation** - -Add to `common/config.go` after line 1536: - -```go -// Multicall3AggregationConfig configures network-level batching of eth_call requests -// into Multicall3 aggregate calls. This batches requests across all entrypoints -// (HTTP single, HTTP batch, gRPC) rather than just JSON-RPC batch requests. -type Multicall3AggregationConfig struct { - // Enabled enables/disables Multicall3 aggregation. Default: true - Enabled bool `yaml:"enabled" json:"enabled"` - - // WindowMs is the maximum time (milliseconds) to wait for a batch to fill. - // Default: 25ms - WindowMs int `yaml:"windowMs,omitempty" json:"windowMs"` - - // MinWaitMs is the minimum time (milliseconds) to wait for additional requests - // to join a batch. Default: 2ms - MinWaitMs int `yaml:"minWaitMs,omitempty" json:"minWaitMs"` - - // SafetyMarginMs is subtracted from request deadlines when computing flush time. - // Default: min(2, MinWaitMs) - SafetyMarginMs int `yaml:"safetyMarginMs,omitempty" json:"safetyMarginMs"` - - // OnlyIfPending: if true, don't add latency unless a batch is already open. - // Default: false - OnlyIfPending bool `yaml:"onlyIfPending,omitempty" json:"onlyIfPending"` - - // MaxCalls is the maximum number of calls per batch. Default: 20 - MaxCalls int `yaml:"maxCalls,omitempty" json:"maxCalls"` - - // MaxCalldataBytes is the maximum total calldata size per batch. Default: 64000 - MaxCalldataBytes int `yaml:"maxCalldataBytes,omitempty" json:"maxCalldataBytes"` - - // MaxQueueSize is the maximum total enqueued requests across all batches. - // Default: 1000 - MaxQueueSize int `yaml:"maxQueueSize,omitempty" json:"maxQueueSize"` - - // MaxPendingBatches is the maximum number of distinct batch keys. - // Default: 200 - MaxPendingBatches int `yaml:"maxPendingBatches,omitempty" json:"maxPendingBatches"` - - // CachePerCall enables per-call cache writes after successful Multicall3. - // Default: true - CachePerCall *bool `yaml:"cachePerCall,omitempty" json:"cachePerCall"` - - // AllowCrossUserBatching: if true, requests from different users can share a batch. - // Default: true - AllowCrossUserBatching *bool `yaml:"allowCrossUserBatching,omitempty" json:"allowCrossUserBatching"` - - // AllowPendingTagBatching: if true, allow batching calls with "pending" block tag. - // Default: false - AllowPendingTagBatching bool `yaml:"allowPendingTagBatching,omitempty" json:"allowPendingTagBatching"` -} - -// SetDefaults applies default values to unset fields -func (c *Multicall3AggregationConfig) SetDefaults() { - if c.WindowMs == 0 { - c.WindowMs = 25 - } - if c.MinWaitMs == 0 { - c.MinWaitMs = 2 - } - if c.SafetyMarginMs == 0 { - c.SafetyMarginMs = min(2, c.MinWaitMs) - } - if c.MaxCalls == 0 { - c.MaxCalls = 20 - } - if c.MaxCalldataBytes == 0 { - c.MaxCalldataBytes = 64000 - } - if c.MaxQueueSize == 0 { - c.MaxQueueSize = 1000 - } - if c.MaxPendingBatches == 0 { - c.MaxPendingBatches = 200 - } - if c.CachePerCall == nil { - c.CachePerCall = &TRUE - } - if c.AllowCrossUserBatching == nil { - c.AllowCrossUserBatching = &TRUE - } -} - -// IsValid checks if the config values are valid -func (c *Multicall3AggregationConfig) IsValid() error { - if c.WindowMs <= 0 { - return fmt.Errorf("multicall3Aggregation.windowMs must be > 0") - } - if c.MinWaitMs < 0 { - return fmt.Errorf("multicall3Aggregation.minWaitMs must be >= 0") - } - if c.MinWaitMs > c.WindowMs { - return fmt.Errorf("multicall3Aggregation.minWaitMs must be <= windowMs") - } - if c.MaxCalls <= 1 { - return fmt.Errorf("multicall3Aggregation.maxCalls must be > 1") - } - if c.MaxCalldataBytes <= 0 { - return fmt.Errorf("multicall3Aggregation.maxCalldataBytes must be > 0") - } - if c.MaxQueueSize <= 0 { - return fmt.Errorf("multicall3Aggregation.maxQueueSize must be > 0") - } - return nil -} -``` - -**Step 4: Modify EvmNetworkConfig to use new type** - -Replace the `Multicall3Aggregation *bool` field in `EvmNetworkConfig` with: - -```go -// Multicall3Aggregation configures aggregating eth_call requests into Multicall3. -// Accepts either a boolean (backward compat) or a full config object. -// Default: enabled with default settings -Multicall3Aggregation *Multicall3AggregationConfig `yaml:"multicall3Aggregation,omitempty" json:"multicall3Aggregation,omitempty"` -``` - -**Step 5: Add UnmarshalYAML for backward compatibility** - -```go -func (c *Multicall3AggregationConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { - // Try bool first (backward compat) - var boolVal bool - if err := unmarshal(&boolVal); err == nil { - c.Enabled = boolVal - if boolVal { - c.SetDefaults() - } - return nil - } - - // Try full config - type rawConfig Multicall3AggregationConfig - var raw rawConfig - if err := unmarshal(&raw); err != nil { - return err - } - *c = Multicall3AggregationConfig(raw) - if c.Enabled { - c.SetDefaults() - } - return nil -} -``` - -**Step 6: Run test to verify it passes** - -Run: `go test -v ./common -run TestMulticall3Aggregation` -Expected: PASS - -**Step 7: Commit** - -```bash -git add common/config.go common/config_test.go -git commit -m "$(cat <<'EOF' -feat: add Multicall3AggregationConfig for network-level batching - -Extends the evm.multicall3Aggregation config from a simple boolean -to a full configuration object with: -- windowMs, minWaitMs, safetyMarginMs for timing -- maxCalls, maxCalldataBytes for size limits -- maxQueueSize, maxPendingBatches for backpressure -- allowCrossUserBatching, allowPendingTagBatching flags - -Maintains backward compatibility with existing boolean configs. - -Co-Authored-By: Claude Opus 4.5 -EOF -)" -``` - ---- - -### Task 1.2: Add CompositeTypeMulticall3 Constant - -**Files:** -- Modify: `common/request.go:17-21` - -**Step 1: Add the constant** - -Add to `common/request.go` after line 20: - -```go -const ( - CompositeTypeNone = "none" - CompositeTypeLogsSplitOnError = "logs-split-on-error" - CompositeTypeLogsSplitProactive = "logs-split-proactive" - CompositeTypeMulticall3 = "multicall3" -) -``` - -**Step 2: Run existing tests** - -Run: `go test -v ./common -run TestComposite` -Expected: PASS (or no tests - that's ok) - -**Step 3: Commit** - -```bash -git add common/request.go -git commit -m "$(cat <<'EOF' -feat: add CompositeTypeMulticall3 constant - -Used to mark aggregated Multicall3 requests for metrics and -hedging logic. - -Co-Authored-By: Claude Opus 4.5 -EOF -)" -``` - ---- - -## Phase 2: Batcher Core Data Structures - -### Task 2.1: Create Batching Key and Entry Types - -**Files:** -- Create: `architecture/evm/multicall3_batcher.go` -- Test: `architecture/evm/multicall3_batcher_test.go` - -**Step 1: Write the failing test** - -Create `architecture/evm/multicall3_batcher_test.go`: - -```go -package evm - -import ( - "testing" - - "github.com/erpc/erpc/common" - "github.com/stretchr/testify/require" -) - -func TestBatchingKey(t *testing.T) { - key1 := BatchingKey{ - ProjectId: "proj1", - NetworkId: "evm:1", - BlockRef: "latest", - DirectivesKey: "use-upstream=alchemy", - UserId: "", - } - key2 := BatchingKey{ - ProjectId: "proj1", - NetworkId: "evm:1", - BlockRef: "latest", - DirectivesKey: "use-upstream=alchemy", - UserId: "", - } - key3 := BatchingKey{ - ProjectId: "proj1", - NetworkId: "evm:1", - BlockRef: "12345", - DirectivesKey: "use-upstream=alchemy", - UserId: "", - } - - require.Equal(t, key1.String(), key2.String()) - require.NotEqual(t, key1.String(), key3.String()) -} - -func TestDirectivesKeyDerivation(t *testing.T) { - dirs := &common.RequestDirectives{} - dirs.UseUpstream = "alchemy" - dirs.SkipCacheRead = true - dirs.RetryEmpty = true - - key := DeriveDirectivesKey(dirs) - require.Contains(t, key, "use-upstream=alchemy") - require.Contains(t, key, "skip-cache-read=true") - require.Contains(t, key, "retry-empty=true") -} - -func TestCallKeyDerivation(t *testing.T) { - jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ - map[string]interface{}{ - "to": "0x1234567890123456789012345678901234567890", - "data": "0xabcdef", - }, - "latest", - }) - req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) - - key, err := DeriveCallKey(req) - require.NoError(t, err) - require.NotEmpty(t, key) - - // Same request should produce same key - key2, err := DeriveCallKey(req) - require.NoError(t, err) - require.Equal(t, key, key2) -} -``` - -**Step 2: Run test to verify it fails** - -Run: `go test -v ./architecture/evm -run TestBatching` -Expected: FAIL - types don't exist - -**Step 3: Write minimal implementation** - -Create `architecture/evm/multicall3_batcher.go`: - -```go -package evm - -import ( - "context" - "fmt" - "sort" - "strings" - "sync" - "time" - - "github.com/erpc/erpc/common" -) - -// DirectivesKeyVersion should be bumped when the set of directives -// included in the key changes. This prevents cross-node key mismatches. -const DirectivesKeyVersion = 1 - -// BatchingKey uniquely identifies a batch for grouping eth_call requests. -type BatchingKey struct { - ProjectId string - NetworkId string - BlockRef string - DirectivesKey string - UserId string // empty if cross-user batching is allowed -} - -func (k BatchingKey) String() string { - return fmt.Sprintf("%s|%s|%s|%s|%s", k.ProjectId, k.NetworkId, k.BlockRef, k.DirectivesKey, k.UserId) -} - -// DeriveDirectivesKey creates a stable, versioned key from relevant directives. -// Only includes directives that affect batching behavior. -func DeriveDirectivesKey(dirs *common.RequestDirectives) string { - if dirs == nil { - return fmt.Sprintf("v%d:", DirectivesKeyVersion) - } - - parts := make([]string, 0, 5) - if dirs.UseUpstream != "" { - parts = append(parts, fmt.Sprintf("use-upstream=%s", dirs.UseUpstream)) - } - if dirs.SkipInterpolation { - parts = append(parts, "skip-interpolation=true") - } - if dirs.RetryEmpty { - parts = append(parts, "retry-empty=true") - } - if dirs.RetryPending { - parts = append(parts, "retry-pending=true") - } - if dirs.SkipCacheRead { - parts = append(parts, "skip-cache-read=true") - } - - sort.Strings(parts) - return fmt.Sprintf("v%d:%s", DirectivesKeyVersion, strings.Join(parts, ",")) -} - -// DeriveCallKey creates a unique key for deduplication within a batch. -// Uses the same derivation as cache keys for consistency. -func DeriveCallKey(req *common.NormalizedRequest) (string, error) { - if req == nil { - return "", fmt.Errorf("request is nil") - } - jrq, err := req.JsonRpcRequest() - if err != nil { - return "", err - } - - jrq.RLock() - method := jrq.Method - params := jrq.Params - jrq.RUnlock() - - // Use method + params as key (same as cache key derivation) - paramsJSON, err := common.SonicCfg.Marshal(params) - if err != nil { - return "", err - } - return fmt.Sprintf("%s:%s", method, string(paramsJSON)), nil -} - -// BatchEntry represents a request waiting in a batch. -type BatchEntry struct { - Ctx context.Context - Request *common.NormalizedRequest - CallKey string - Target []byte - CallData []byte - ResultCh chan BatchResult - CreatedAt time.Time - Deadline time.Time -} - -// BatchResult is the outcome delivered to a waiting request. -type BatchResult struct { - Response *common.NormalizedResponse - Error error -} - -// Batch holds pending requests for a single batching key. -type Batch struct { - Key BatchingKey - Entries []*BatchEntry - CallKeys map[string][]*BatchEntry // for deduplication - FlushTime time.Time - Flushing bool - mu sync.Mutex -} - -func NewBatch(key BatchingKey, flushTime time.Time) *Batch { - return &Batch{ - Key: key, - Entries: make([]*BatchEntry, 0, 16), - CallKeys: make(map[string][]*BatchEntry), - FlushTime: flushTime, - } -} -``` - -**Step 4: Run test to verify it passes** - -Run: `go test -v ./architecture/evm -run TestBatching` -Expected: PASS - -**Step 5: Commit** - -```bash -git add architecture/evm/multicall3_batcher.go architecture/evm/multicall3_batcher_test.go -git commit -m "$(cat <<'EOF' -feat: add Multicall3 batching key and entry types - -Core data structures for network-level Multicall3 batching: -- BatchingKey for grouping requests by project/network/block/directives -- DeriveDirectivesKey for stable versioned directive hashing -- DeriveCallKey for within-batch deduplication -- BatchEntry and Batch for holding pending requests - -Co-Authored-By: Claude Opus 4.5 -EOF -)" -``` - ---- - -### Task 2.2: Implement Eligibility Checking - -**Files:** -- Modify: `architecture/evm/multicall3_batcher.go` -- Test: `architecture/evm/multicall3_batcher_test.go` - -**Step 1: Write the failing test** - -Add to `architecture/evm/multicall3_batcher_test.go`: - -```go -func TestIsEligibleForBatching(t *testing.T) { - cfg := &common.Multicall3AggregationConfig{ - Enabled: true, - AllowPendingTagBatching: false, - } - cfg.SetDefaults() - - tests := []struct { - name string - method string - params []interface{} - eligible bool - reason string - }{ - { - name: "eligible basic eth_call", - method: "eth_call", - params: []interface{}{ - map[string]interface{}{"to": "0x1234567890123456789012345678901234567890", "data": "0xabcd"}, - "latest", - }, - eligible: true, - }, - { - name: "eligible with finalized tag", - method: "eth_call", - params: []interface{}{ - map[string]interface{}{"to": "0x1234567890123456789012345678901234567890", "data": "0xabcd"}, - "finalized", - }, - eligible: true, - }, - { - name: "ineligible - pending tag", - method: "eth_call", - params: []interface{}{ - map[string]interface{}{"to": "0x1234567890123456789012345678901234567890", "data": "0xabcd"}, - "pending", - }, - eligible: false, - reason: "pending tag not allowed", - }, - { - name: "ineligible - has from field", - method: "eth_call", - params: []interface{}{ - map[string]interface{}{ - "to": "0x1234567890123456789012345678901234567890", - "data": "0xabcd", - "from": "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - }, - "latest", - }, - eligible: false, - reason: "has from field", - }, - { - name: "ineligible - has value field", - method: "eth_call", - params: []interface{}{ - map[string]interface{}{ - "to": "0x1234567890123456789012345678901234567890", - "data": "0xabcd", - "value": "0x1", - }, - "latest", - }, - eligible: false, - reason: "has value field", - }, - { - name: "ineligible - has state override (3rd param)", - method: "eth_call", - params: []interface{}{ - map[string]interface{}{"to": "0x1234567890123456789012345678901234567890", "data": "0xabcd"}, - "latest", - map[string]interface{}{}, // state override - }, - eligible: false, - reason: "has state override", - }, - { - name: "ineligible - not eth_call", - method: "eth_getBalance", - params: []interface{}{"0x1234567890123456789012345678901234567890", "latest"}, - eligible: false, - reason: "not eth_call", - }, - { - name: "ineligible - already multicall (recursion guard)", - method: "eth_call", - params: []interface{}{ - map[string]interface{}{ - "to": "0xcA11bde05977b3631167028862bE2a173976CA11", // multicall3 address - "data": "0x82ad56cb", // aggregate3 selector - }, - "latest", - }, - eligible: false, - reason: "already multicall", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - jrq := common.NewJsonRpcRequest(tt.method, tt.params) - req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) - - eligible, reason := IsEligibleForBatching(req, cfg) - require.Equal(t, tt.eligible, eligible, "reason: %s", reason) - if !tt.eligible { - require.Contains(t, reason, tt.reason) - } - }) - } -} -``` - -**Step 2: Run test to verify it fails** - -Run: `go test -v ./architecture/evm -run TestIsEligibleForBatching` -Expected: FAIL - function doesn't exist - -**Step 3: Write minimal implementation** - -Add to `architecture/evm/multicall3_batcher.go`: - -```go -// ineligibleCallFields are fields that make an eth_call ineligible for batching. -// Multicall3 aggregate3 only supports target + calldata, not gas/value/etc. -var ineligibleCallFields = []string{ - "from", "gas", "gasPrice", "maxFeePerGas", "maxPriorityFeePerGas", "value", -} - -// allowedBlockTags are block tags that can be batched by default. -var allowedBlockTags = map[string]bool{ - "latest": true, - "finalized": true, - "safe": true, - "earliest": true, -} - -// IsEligibleForBatching checks if a request can be batched via Multicall3. -// Returns (eligible, reason) where reason explains why not eligible. -func IsEligibleForBatching(req *common.NormalizedRequest, cfg *common.Multicall3AggregationConfig) (bool, string) { - if req == nil { - return false, "request is nil" - } - if cfg == nil || !cfg.Enabled { - return false, "batching disabled" - } - - jrq, err := req.JsonRpcRequest() - if err != nil { - return false, fmt.Sprintf("json-rpc error: %v", err) - } - - jrq.RLock() - method := strings.ToLower(jrq.Method) - params := jrq.Params - jrq.RUnlock() - - // Must be eth_call - if method != "eth_call" { - return false, "not eth_call" - } - - // Must have 1-2 params (call object, optional block) - if len(params) < 1 || len(params) > 2 { - return false, fmt.Sprintf("invalid param count: %d", len(params)) - } - - // Check for state override (3rd param) - if len(params) > 2 { - return false, "has state override" - } - - // Parse call object - callObj, ok := params[0].(map[string]interface{}) - if !ok { - return false, "invalid call object type" - } - - // Must have 'to' address - toVal, hasTo := callObj["to"] - if !hasTo { - return false, "missing to address" - } - toStr, ok := toVal.(string) - if !ok || toStr == "" { - return false, "invalid to address" - } - - // Check for ineligible fields - for _, field := range ineligibleCallFields { - if _, has := callObj[field]; has { - return false, fmt.Sprintf("has %s field", field) - } - } - - // Recursion guard: don't batch calls to multicall3 contract - if strings.EqualFold(toStr, multicall3Address) { - return false, "already multicall" - } - - // Check block tag - blockTag := "latest" - if len(params) >= 2 && params[1] != nil { - normalized, err := NormalizeBlockParam(params[1]) - if err != nil { - return false, fmt.Sprintf("invalid block param: %v", err) - } - blockTag = strings.ToLower(normalized) - } - - // Check if pending tag is allowed - if blockTag == "pending" && !cfg.AllowPendingTagBatching { - return false, "pending tag not allowed" - } - - return true, "" -} - -// ExtractCallInfo extracts target and calldata from an eligible eth_call request. -func ExtractCallInfo(req *common.NormalizedRequest) (target []byte, callData []byte, blockRef string, err error) { - jrq, err := req.JsonRpcRequest() - if err != nil { - return nil, nil, "", err - } - - jrq.RLock() - params := jrq.Params - jrq.RUnlock() - - callObj := params[0].(map[string]interface{}) - toStr := callObj["to"].(string) - - target, err = common.HexToBytes(toStr) - if err != nil { - return nil, nil, "", err - } - - dataHex := "0x" - if dataVal, ok := callObj["data"]; ok { - dataHex = dataVal.(string) - } else if inputVal, ok := callObj["input"]; ok { - dataHex = inputVal.(string) - } - - callData, err = common.HexToBytes(dataHex) - if err != nil { - return nil, nil, "", err - } - - blockRef = "latest" - if len(params) >= 2 && params[1] != nil { - blockRef, err = NormalizeBlockParam(params[1]) - if err != nil { - return nil, nil, "", err - } - } - - return target, callData, blockRef, nil -} -``` - -**Step 4: Run test to verify it passes** - -Run: `go test -v ./architecture/evm -run TestIsEligibleForBatching` -Expected: PASS - -**Step 5: Commit** - -```bash -git add architecture/evm/multicall3_batcher.go architecture/evm/multicall3_batcher_test.go -git commit -m "$(cat <<'EOF' -feat: add Multicall3 eligibility checking - -Implements IsEligibleForBatching to determine if an eth_call can be -aggregated into a Multicall3 batch: -- Method must be eth_call -- Call object must only have to + data/input fields -- No state overrides (3rd param) -- Recursion guard: don't batch calls to multicall3 contract -- Block tag restrictions (pending disabled by default) - -Also adds ExtractCallInfo helper to extract target/calldata/blockRef. - -Co-Authored-By: Claude Opus 4.5 -EOF -)" -``` - ---- - -### Task 2.3: Implement Batcher with Window and Caps - -**Files:** -- Modify: `architecture/evm/multicall3_batcher.go` -- Test: `architecture/evm/multicall3_batcher_test.go` - -**Step 1: Write the failing test** - -Add to `architecture/evm/multicall3_batcher_test.go`: - -```go -func TestBatcherEnqueueAndFlush(t *testing.T) { - cfg := &common.Multicall3AggregationConfig{ - Enabled: true, - WindowMs: 50, - MinWaitMs: 5, - SafetyMarginMs: 2, - MaxCalls: 10, - MaxCalldataBytes: 64000, - MaxQueueSize: 100, - MaxPendingBatches: 20, - AllowCrossUserBatching: &common.TRUE, - } - - ctx := context.Background() - batcher := NewBatcher(cfg, nil) // nil forwarder for now - - // Create test requests - jrq1 := common.NewJsonRpcRequest("eth_call", []interface{}{ - map[string]interface{}{ - "to": "0x1234567890123456789012345678901234567890", - "data": "0xabcdef01", - }, - "latest", - }) - req1 := common.NewNormalizedRequestFromJsonRpcRequest(jrq1) - - jrq2 := common.NewJsonRpcRequest("eth_call", []interface{}{ - map[string]interface{}{ - "to": "0x2234567890123456789012345678901234567890", - "data": "0xabcdef02", - }, - "latest", - }) - req2 := common.NewNormalizedRequestFromJsonRpcRequest(jrq2) - - key := BatchingKey{ - ProjectId: "test-project", - NetworkId: "evm:1", - BlockRef: "latest", - DirectivesKey: DeriveDirectivesKey(nil), - } - - // Enqueue first request - entry1, bypass1, err := batcher.Enqueue(ctx, key, req1) - require.NoError(t, err) - require.False(t, bypass1) - require.NotNil(t, entry1) - - // Enqueue second request - entry2, bypass2, err := batcher.Enqueue(ctx, key, req2) - require.NoError(t, err) - require.False(t, bypass2) - require.NotNil(t, entry2) - - // Check batch exists - batcher.mu.RLock() - batch, exists := batcher.batches[key.String()] - batcher.mu.RUnlock() - require.True(t, exists) - require.Len(t, batch.Entries, 2) - - // Cleanup - batcher.Shutdown() -} - -func TestBatcherDeduplication(t *testing.T) { - cfg := &common.Multicall3AggregationConfig{ - Enabled: true, - WindowMs: 50, - MinWaitMs: 5, - MaxCalls: 10, - MaxCalldataBytes: 64000, - MaxQueueSize: 100, - MaxPendingBatches: 20, - AllowCrossUserBatching: &common.TRUE, - } - - ctx := context.Background() - batcher := NewBatcher(cfg, nil) - - // Two identical requests - jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ - map[string]interface{}{ - "to": "0x1234567890123456789012345678901234567890", - "data": "0xabcdef01", - }, - "latest", - }) - req1 := common.NewNormalizedRequestFromJsonRpcRequest(jrq) - req2 := common.NewNormalizedRequestFromJsonRpcRequest(common.NewJsonRpcRequest("eth_call", []interface{}{ - map[string]interface{}{ - "to": "0x1234567890123456789012345678901234567890", - "data": "0xabcdef01", - }, - "latest", - })) - - key := BatchingKey{ - ProjectId: "test-project", - NetworkId: "evm:1", - BlockRef: "latest", - DirectivesKey: DeriveDirectivesKey(nil), - } - - entry1, _, _ := batcher.Enqueue(ctx, key, req1) - entry2, _, _ := batcher.Enqueue(ctx, key, req2) - - // Both should share the same callKey slot - require.Equal(t, entry1.CallKey, entry2.CallKey) - - batcher.mu.RLock() - batch := batcher.batches[key.String()] - batcher.mu.RUnlock() - - // Two entries but deduplicated - require.Len(t, batch.Entries, 2) - require.Len(t, batch.CallKeys[entry1.CallKey], 2) - - batcher.Shutdown() -} - -func TestBatcherCapsEnforcement(t *testing.T) { - cfg := &common.Multicall3AggregationConfig{ - Enabled: true, - WindowMs: 50, - MinWaitMs: 5, - MaxCalls: 2, // Very low limit - MaxCalldataBytes: 64000, - MaxQueueSize: 100, - MaxPendingBatches: 20, - AllowCrossUserBatching: &common.TRUE, - } - - ctx := context.Background() - batcher := NewBatcher(cfg, nil) - - key := BatchingKey{ - ProjectId: "test-project", - NetworkId: "evm:1", - BlockRef: "latest", - DirectivesKey: DeriveDirectivesKey(nil), - } - - // Add requests up to cap - for i := 0; i < 2; i++ { - jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ - map[string]interface{}{ - "to": fmt.Sprintf("0x%040d", i), - "data": "0xabcdef", - }, - "latest", - }) - req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) - _, bypass, err := batcher.Enqueue(ctx, key, req) - require.NoError(t, err) - require.False(t, bypass) - } - - // Next request should trigger bypass (caps reached) - jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ - map[string]interface{}{ - "to": "0x9999999999999999999999999999999999999999", - "data": "0xabcdef", - }, - "latest", - }) - req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) - _, bypass, err := batcher.Enqueue(ctx, key, req) - require.NoError(t, err) - require.True(t, bypass, "should bypass when caps reached") - - batcher.Shutdown() -} -``` - -**Step 2: Run test to verify it fails** - -Run: `go test -v ./architecture/evm -run TestBatcher` -Expected: FAIL - Batcher type doesn't exist - -**Step 3: Write minimal implementation** - -Add to `architecture/evm/multicall3_batcher.go`: - -```go -// Forwarder is the interface for forwarding requests through the network layer. -type Forwarder interface { - Forward(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) -} - -// Batcher aggregates eth_call requests into Multicall3 batches. -type Batcher struct { - cfg *common.Multicall3AggregationConfig - forwarder Forwarder - batches map[string]*Batch // keyed by BatchingKey.String() - mu sync.RWMutex - queueSize int64 // atomic counter for backpressure - shutdown chan struct{} - wg sync.WaitGroup -} - -// NewBatcher creates a new Multicall3 batcher. -func NewBatcher(cfg *common.Multicall3AggregationConfig, forwarder Forwarder) *Batcher { - b := &Batcher{ - cfg: cfg, - forwarder: forwarder, - batches: make(map[string]*Batch), - shutdown: make(chan struct{}), - } - return b -} - -// Enqueue adds a request to a batch. Returns: -// - entry: the batch entry (nil if bypass) -// - bypass: true if request should be forwarded individually -// - error: any error during processing -func (b *Batcher) Enqueue(ctx context.Context, key BatchingKey, req *common.NormalizedRequest) (*BatchEntry, bool, error) { - // Extract call info - target, callData, _, err := ExtractCallInfo(req) - if err != nil { - return nil, true, err - } - - // Derive call key for deduplication - callKey, err := DeriveCallKey(req) - if err != nil { - return nil, true, err - } - - // Calculate deadline from context - deadline, hasDeadline := ctx.Deadline() - if !hasDeadline { - deadline = time.Now().Add(time.Duration(b.cfg.WindowMs) * time.Millisecond) - } - - // Check if deadline is too tight - now := time.Now() - minWait := time.Duration(b.cfg.MinWaitMs) * time.Millisecond - if deadline.Before(now.Add(minWait)) { - // Deadline too tight, bypass batching - return nil, true, nil - } - - b.mu.Lock() - defer b.mu.Unlock() - - // Check caps - if b.queueSize >= int64(b.cfg.MaxQueueSize) { - return nil, true, nil // bypass: queue full - } - if len(b.batches) >= b.cfg.MaxPendingBatches { - // Check if this is a new batch key - if _, exists := b.batches[key.String()]; !exists { - return nil, true, nil // bypass: too many pending batches - } - } - - // Get or create batch - keyStr := key.String() - batch, exists := b.batches[keyStr] - if !exists { - flushTime := now.Add(time.Duration(b.cfg.WindowMs) * time.Millisecond) - batch = NewBatch(key, flushTime) - b.batches[keyStr] = batch - - // Start flush timer - b.wg.Add(1) - go b.scheduleFlush(keyStr, batch) - } - - // Check if batch is flushing - create new batch if so - batch.mu.Lock() - if batch.Flushing { - batch.mu.Unlock() - // Create new batch for this key - flushTime := now.Add(time.Duration(b.cfg.WindowMs) * time.Millisecond) - batch = NewBatch(key, flushTime) - b.batches[keyStr] = batch - - b.wg.Add(1) - go b.scheduleFlush(keyStr, batch) - - batch.mu.Lock() - } - - // Check if batch is at capacity (unique calls, not entries) - uniqueCalls := len(batch.CallKeys) - if _, isDupe := batch.CallKeys[callKey]; !isDupe { - if uniqueCalls >= b.cfg.MaxCalls { - batch.mu.Unlock() - return nil, true, nil // bypass: batch full - } - } - - // Check calldata size cap - currentSize := 0 - for _, entries := range batch.CallKeys { - if len(entries) > 0 { - currentSize += len(entries[0].CallData) - } - } - if _, isDupe := batch.CallKeys[callKey]; !isDupe { - if currentSize+len(callData) > b.cfg.MaxCalldataBytes { - batch.mu.Unlock() - return nil, true, nil // bypass: calldata too large - } - } - - // Create entry - entry := &BatchEntry{ - Ctx: ctx, - Request: req, - CallKey: callKey, - Target: target, - CallData: callData, - ResultCh: make(chan BatchResult, 1), - CreatedAt: now, - Deadline: deadline, - } - - // Add to batch - batch.Entries = append(batch.Entries, entry) - batch.CallKeys[callKey] = append(batch.CallKeys[callKey], entry) - - // Update flush time based on deadline (deadline-aware) - safetyMargin := time.Duration(b.cfg.SafetyMarginMs) * time.Millisecond - proposedFlush := deadline.Add(-safetyMargin) - if proposedFlush.Before(batch.FlushTime) { - batch.FlushTime = proposedFlush - // Clamp to minimum wait - minFlush := now.Add(minWait) - if batch.FlushTime.Before(minFlush) { - batch.FlushTime = minFlush - } - } - - batch.mu.Unlock() - b.queueSize++ - - return entry, false, nil -} - -// scheduleFlush waits until flush time and then flushes the batch. -func (b *Batcher) scheduleFlush(keyStr string, batch *Batch) { - defer b.wg.Done() - - for { - batch.mu.Lock() - flushTime := batch.FlushTime - batch.mu.Unlock() - - waitDuration := time.Until(flushTime) - if waitDuration <= 0 { - b.flush(keyStr, batch) - return - } - - timer := time.NewTimer(waitDuration) - select { - case <-timer.C: - b.flush(keyStr, batch) - return - case <-b.shutdown: - timer.Stop() - return - } - } -} - -// flush processes a batch and delivers results. -func (b *Batcher) flush(keyStr string, batch *Batch) { - batch.mu.Lock() - if batch.Flushing { - batch.mu.Unlock() - return - } - batch.Flushing = true - entries := batch.Entries - callKeys := batch.CallKeys - batch.mu.Unlock() - - // Remove from active batches - b.mu.Lock() - if b.batches[keyStr] == batch { - delete(b.batches, keyStr) - } - b.queueSize -= int64(len(entries)) - b.mu.Unlock() - - // Deliver error for now (actual forwarding implemented in Task 2.4) - result := BatchResult{ - Error: fmt.Errorf("flush not implemented"), - } - for _, entry := range entries { - select { - case entry.ResultCh <- result: - default: - } - } - _ = callKeys // silence unused warning -} - -// Shutdown stops the batcher and waits for pending operations. -func (b *Batcher) Shutdown() { - close(b.shutdown) - b.wg.Wait() -} -``` - -**Step 4: Run test to verify it passes** - -Run: `go test -v ./architecture/evm -run TestBatcher` -Expected: PASS - -**Step 5: Commit** - -```bash -git add architecture/evm/multicall3_batcher.go architecture/evm/multicall3_batcher_test.go -git commit -m "$(cat <<'EOF' -feat: implement Multicall3 Batcher with window and caps - -Implements the core Batcher type that: -- Enqueues eth_call requests into batches by BatchingKey -- Enforces caps: maxCalls, maxCalldataBytes, maxQueueSize, maxPendingBatches -- Supports deduplication via callKey within batches -- Implements deadline-aware flush scheduling -- Handles concurrent batch creation during flush - -Co-Authored-By: Claude Opus 4.5 -EOF -)" -``` - ---- - -### Task 2.4: Implement Batch Forwarding and Result Mapping - -**Files:** -- Modify: `architecture/evm/multicall3_batcher.go` -- Test: `architecture/evm/multicall3_batcher_test.go` - -**Step 1: Write the failing test** - -Add to `architecture/evm/multicall3_batcher_test.go`: - -```go -type mockForwarder struct { - response *common.NormalizedResponse - err error - called int -} - -func (m *mockForwarder) Forward(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { - m.called++ - return m.response, m.err -} - -func TestBatcherFlushAndResultMapping(t *testing.T) { - // Create a mock response with valid multicall3 result - // Multicall3 aggregate3 returns [(bool success, bytes returnData), ...] - // We'll create a simple encoded result for 2 calls - - // For simplicity, create raw hex result - // Result structure: offset to array, array length, elements... - resultHex := "0x" + - // Offset to array (32 bytes pointing to 0x20) - "0000000000000000000000000000000000000000000000000000000000000020" + - // Array length (2 elements) - "0000000000000000000000000000000000000000000000000000000000000002" + - // Offset to element 0 (0x40 from array start) - "0000000000000000000000000000000000000000000000000000000000000040" + - // Offset to element 1 (0xa0 from array start) - "00000000000000000000000000000000000000000000000000000000000000a0" + - // Element 0: success=true - "0000000000000000000000000000000000000000000000000000000000000001" + - // Element 0: returnData offset - "0000000000000000000000000000000000000000000000000000000000000040" + - // Element 0: returnData length (4 bytes) - "0000000000000000000000000000000000000000000000000000000000000004" + - // Element 0: returnData value "0xdeadbeef" padded - "deadbeef00000000000000000000000000000000000000000000000000000000" + - // Element 1: success=true - "0000000000000000000000000000000000000000000000000000000000000001" + - // Element 1: returnData offset - "0000000000000000000000000000000000000000000000000000000000000040" + - // Element 1: returnData length (4 bytes) - "0000000000000000000000000000000000000000000000000000000000000004" + - // Element 1: returnData value "0xcafebabe" padded - "cafebabe00000000000000000000000000000000000000000000000000000000" - - jrr, _ := common.NewJsonRpcResponse(nil, resultHex, nil) - mockResp := common.NewNormalizedResponse().WithJsonRpcResponse(jrr) - - forwarder := &mockForwarder{response: mockResp} - - cfg := &common.Multicall3AggregationConfig{ - Enabled: true, - WindowMs: 10, // Short window for test - MinWaitMs: 1, - MaxCalls: 10, - MaxCalldataBytes: 64000, - MaxQueueSize: 100, - MaxPendingBatches: 20, - AllowCrossUserBatching: &common.TRUE, - CachePerCall: &common.FALSE, // disable caching for test - } - - batcher := NewBatcher(cfg, forwarder) - - ctx := context.Background() - key := BatchingKey{ - ProjectId: "test-project", - NetworkId: "evm:1", - BlockRef: "latest", - DirectivesKey: DeriveDirectivesKey(nil), - } - - // Add two requests - jrq1 := common.NewJsonRpcRequest("eth_call", []interface{}{ - map[string]interface{}{"to": "0x1111111111111111111111111111111111111111", "data": "0x01"}, - "latest", - }) - jrq1.ID = "req1" - req1 := common.NewNormalizedRequestFromJsonRpcRequest(jrq1) - - jrq2 := common.NewJsonRpcRequest("eth_call", []interface{}{ - map[string]interface{}{"to": "0x2222222222222222222222222222222222222222", "data": "0x02"}, - "latest", - }) - jrq2.ID = "req2" - req2 := common.NewNormalizedRequestFromJsonRpcRequest(jrq2) - - entry1, _, _ := batcher.Enqueue(ctx, key, req1) - entry2, _, _ := batcher.Enqueue(ctx, key, req2) - - // Wait for results - result1 := <-entry1.ResultCh - result2 := <-entry2.ResultCh - - require.NoError(t, result1.Error) - require.NoError(t, result2.Error) - require.NotNil(t, result1.Response) - require.NotNil(t, result2.Response) - - // Verify forwarder was called exactly once - require.Equal(t, 1, forwarder.called) - - batcher.Shutdown() -} -``` - -**Step 2: Run test to verify it fails** - -Run: `go test -v ./architecture/evm -run TestBatcherFlushAndResultMapping` -Expected: FAIL - current flush delivers error - -**Step 3: Update flush implementation** - -Replace the `flush` function in `architecture/evm/multicall3_batcher.go`: - -```go -// flush processes a batch: builds multicall, forwards, maps results. -func (b *Batcher) flush(keyStr string, batch *Batch) { - batch.mu.Lock() - if batch.Flushing { - batch.mu.Unlock() - return - } - batch.Flushing = true - entries := batch.Entries - callKeys := batch.CallKeys - batch.mu.Unlock() - - // Remove from active batches - b.mu.Lock() - if b.batches[keyStr] == batch { - delete(b.batches, keyStr) - } - b.queueSize -= int64(len(entries)) - b.mu.Unlock() - - // Build unique calls list (maintaining order) - uniqueCalls := make([]Multicall3Call, 0, len(callKeys)) - callKeyOrder := make([]string, 0, len(callKeys)) - seen := make(map[string]bool) - - for _, entry := range entries { - if !seen[entry.CallKey] { - seen[entry.CallKey] = true - callKeyOrder = append(callKeyOrder, entry.CallKey) - uniqueCalls = append(uniqueCalls, Multicall3Call{ - Request: entry.Request, - Target: entry.Target, - CallData: entry.CallData, - }) - } - } - - if len(uniqueCalls) == 0 { - return - } - - // Build requests slice for BuildMulticall3Request - reqs := make([]*common.NormalizedRequest, len(uniqueCalls)) - for i, call := range uniqueCalls { - reqs[i] = call.Request - } - - // Build multicall3 request - mcReq, _, err := BuildMulticall3Request(reqs, batch.Key.BlockRef) - if err != nil { - b.deliverError(entries, fmt.Errorf("failed to build multicall3: %w", err)) - return - } - - // Mark as composite request - mcReq.SetCompositeType(common.CompositeTypeMulticall3) - - // Forward via network - ctx := context.Background() - if len(entries) > 0 && entries[0].Ctx != nil { - ctx = entries[0].Ctx - } - - resp, err := b.forwarder.Forward(ctx, mcReq) - if err != nil { - if ShouldFallbackMulticall3(err) { - b.fallbackIndividual(entries) - return - } - b.deliverError(entries, fmt.Errorf("multicall3 forward failed: %w", err)) - return - } - - // Decode response - results, err := b.decodeMulticallResponse(ctx, resp) - if err != nil { - if ShouldFallbackMulticall3(err) { - b.fallbackIndividual(entries) - return - } - b.deliverError(entries, fmt.Errorf("multicall3 decode failed: %w", err)) - return - } - - if len(results) != len(uniqueCalls) { - b.fallbackIndividual(entries) - return - } - - // Map results to entries (fan out deduplicated results) - for i, callKey := range callKeyOrder { - result := results[i] - waiters := callKeys[callKey] - - for _, entry := range waiters { - br := BatchResult{} - if result.Success { - returnHex := "0x" + hex.EncodeToString(result.ReturnData) - jrr, err := common.NewJsonRpcResponse(entry.Request.ID(), returnHex, nil) - if err != nil { - br.Error = err - } else { - br.Response = common.NewNormalizedResponse().WithRequest(entry.Request).WithJsonRpcResponse(jrr) - br.Response.SetUpstream(resp.Upstream()) - br.Response.SetFromCache(resp.FromCache()) - } - } else { - // Per-call revert - dataHex := "0x" + hex.EncodeToString(result.ReturnData) - br.Error = common.NewErrEndpointExecutionException( - common.NewErrJsonRpcExceptionInternal( - 3, // execution reverted code - common.JsonRpcErrorExecutionReverted, - dataHex, - map[string]interface{}{ - "multicall3": true, - "stage": "per-call", - }, - ), - nil, - ) - } - - select { - case entry.ResultCh <- br: - default: - } - } - } -} - -// decodeMulticallResponse extracts and decodes the multicall3 result. -func (b *Batcher) decodeMulticallResponse(ctx context.Context, resp *common.NormalizedResponse) ([]Multicall3Result, error) { - if resp == nil { - return nil, fmt.Errorf("nil response") - } - - jrr, err := resp.JsonRpcResponse(ctx) - if err != nil { - return nil, err - } - if jrr == nil || jrr.Error != nil { - if jrr != nil && jrr.Error != nil { - return nil, fmt.Errorf("rpc error: %s", jrr.Error.Message) - } - return nil, fmt.Errorf("invalid response") - } - - var resultHex string - if err := common.SonicCfg.Unmarshal(jrr.GetResultBytes(), &resultHex); err != nil { - return nil, err - } - - resultBytes, err := common.HexToBytes(resultHex) - if err != nil { - return nil, err - } - - return DecodeMulticall3Aggregate3Result(resultBytes) -} - -// deliverError sends an error to all entries. -func (b *Batcher) deliverError(entries []*BatchEntry, err error) { - result := BatchResult{Error: err} - for _, entry := range entries { - select { - case entry.ResultCh <- result: - default: - } - } -} - -// fallbackIndividual forwards each entry individually. -func (b *Batcher) fallbackIndividual(entries []*BatchEntry) { - var wg sync.WaitGroup - for _, entry := range entries { - wg.Add(1) - go func(e *BatchEntry) { - defer wg.Done() - resp, err := b.forwarder.Forward(e.Ctx, e.Request) - select { - case e.ResultCh <- BatchResult{Response: resp, Error: err}: - default: - } - }(entry) - } - wg.Wait() -} -``` - -Also add the hex import at the top: -```go -import ( - "encoding/hex" - // ... other imports -) -``` - -**Step 4: Run test to verify it passes** - -Run: `go test -v ./architecture/evm -run TestBatcherFlushAndResultMapping` -Expected: PASS - -**Step 5: Commit** - -```bash -git add architecture/evm/multicall3_batcher.go architecture/evm/multicall3_batcher_test.go -git commit -m "$(cat <<'EOF' -feat: implement Multicall3 batch forwarding and result mapping - -Completes the Batcher.flush() implementation: -- Builds Multicall3 request from unique calls -- Marks request as CompositeTypeMulticall3 -- Forwards via Forwarder interface -- Decodes response and maps results to entries -- Fans out deduplicated results to all waiters -- Handles per-call reverts as execution errors -- Falls back to individual forwarding on multicall failure - -Co-Authored-By: Claude Opus 4.5 -EOF -)" -``` - ---- - -## Phase 3: Integration with Network Layer - -### Task 3.1: Create Network-Level Batcher Manager - -**Files:** -- Create: `architecture/evm/multicall3_manager.go` -- Test: `architecture/evm/multicall3_manager_test.go` - -**Step 1: Write the failing test** - -Create `architecture/evm/multicall3_manager_test.go`: - -```go -package evm - -import ( - "context" - "sync" - "testing" - - "github.com/erpc/erpc/common" - "github.com/stretchr/testify/require" -) - -func TestBatcherManagerGetOrCreate(t *testing.T) { - mgr := NewBatcherManager() - - cfg := &common.Multicall3AggregationConfig{ - Enabled: true, - WindowMs: 25, - MinWaitMs: 2, - MaxCalls: 20, - MaxCalldataBytes: 64000, - MaxQueueSize: 100, - MaxPendingBatches: 20, - AllowCrossUserBatching: &common.TRUE, - } - - forwarder := &mockForwarder{} - - // Get batcher for network - batcher1 := mgr.GetOrCreate("evm:1", cfg, forwarder) - require.NotNil(t, batcher1) - - // Same network should return same batcher - batcher2 := mgr.GetOrCreate("evm:1", cfg, forwarder) - require.Same(t, batcher1, batcher2) - - // Different network should return different batcher - batcher3 := mgr.GetOrCreate("evm:137", cfg, forwarder) - require.NotSame(t, batcher1, batcher3) - - mgr.Shutdown() -} - -func TestBatcherManagerConcurrency(t *testing.T) { - mgr := NewBatcherManager() - - cfg := &common.Multicall3AggregationConfig{ - Enabled: true, - WindowMs: 25, - MinWaitMs: 2, - MaxCalls: 20, - } - cfg.SetDefaults() - - forwarder := &mockForwarder{} - - var wg sync.WaitGroup - batchers := make([]*Batcher, 100) - - // Concurrent access - for i := 0; i < 100; i++ { - wg.Add(1) - go func(idx int) { - defer wg.Done() - batchers[idx] = mgr.GetOrCreate("evm:1", cfg, forwarder) - }(i) - } - wg.Wait() - - // All should be the same batcher - for i := 1; i < 100; i++ { - require.Same(t, batchers[0], batchers[i]) - } - - mgr.Shutdown() -} -``` - -**Step 2: Run test to verify it fails** - -Run: `go test -v ./architecture/evm -run TestBatcherManager` -Expected: FAIL - type doesn't exist - -**Step 3: Write minimal implementation** - -Create `architecture/evm/multicall3_manager.go`: - -```go -package evm - -import ( - "sync" - - "github.com/erpc/erpc/common" -) - -// BatcherManager manages per-network Multicall3 batchers. -type BatcherManager struct { - batchers map[string]*Batcher - mu sync.RWMutex -} - -// NewBatcherManager creates a new batcher manager. -func NewBatcherManager() *BatcherManager { - return &BatcherManager{ - batchers: make(map[string]*Batcher), - } -} - -// GetOrCreate returns the batcher for a network, creating one if needed. -func (m *BatcherManager) GetOrCreate(networkId string, cfg *common.Multicall3AggregationConfig, forwarder Forwarder) *Batcher { - m.mu.RLock() - if b, ok := m.batchers[networkId]; ok { - m.mu.RUnlock() - return b - } - m.mu.RUnlock() - - m.mu.Lock() - defer m.mu.Unlock() - - // Double-check after acquiring write lock - if b, ok := m.batchers[networkId]; ok { - return b - } - - batcher := NewBatcher(cfg, forwarder) - m.batchers[networkId] = batcher - return batcher -} - -// Get returns the batcher for a network, or nil if not exists. -func (m *BatcherManager) Get(networkId string) *Batcher { - m.mu.RLock() - defer m.mu.RUnlock() - return m.batchers[networkId] -} - -// Shutdown stops all batchers. -func (m *BatcherManager) Shutdown() { - m.mu.Lock() - defer m.mu.Unlock() - - for _, b := range m.batchers { - b.Shutdown() - } - m.batchers = make(map[string]*Batcher) -} -``` - -**Step 4: Run test to verify it passes** - -Run: `go test -v ./architecture/evm -run TestBatcherManager` -Expected: PASS - -**Step 5: Commit** - -```bash -git add architecture/evm/multicall3_manager.go architecture/evm/multicall3_manager_test.go -git commit -m "$(cat <<'EOF' -feat: add BatcherManager for per-network batcher instances - -Manages Multicall3 batchers per network with: -- GetOrCreate for lazy initialization -- Thread-safe concurrent access -- Proper shutdown cleanup - -Co-Authored-By: Claude Opus 4.5 -EOF -)" -``` - ---- - -### Task 3.2: Integrate Batcher into eth_call Pre-Forward Hook - -**Files:** -- Modify: `architecture/evm/eth_call.go` -- Modify: `architecture/evm/hooks.go` -- Test: `architecture/evm/eth_call_test.go` (create) - -**Step 1: Write the failing test** - -Create `architecture/evm/eth_call_test.go`: - -```go -package evm - -import ( - "context" - "testing" - "time" - - "github.com/erpc/erpc/common" - "github.com/stretchr/testify/require" -) - -type mockNetwork struct { - networkId string - cfg *common.NetworkConfig - forwardFn func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) - cacheDal common.CacheDAL -} - -func (m *mockNetwork) Id() string { return m.networkId } -func (m *mockNetwork) Label() string { return m.networkId } -func (m *mockNetwork) Config() *common.NetworkConfig { return m.cfg } -func (m *mockNetwork) CacheDal() common.CacheDAL { return m.cacheDal } -func (m *mockNetwork) AppCtx() context.Context { return context.Background() } -func (m *mockNetwork) Forward(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { - if m.forwardFn != nil { - return m.forwardFn(ctx, req) - } - return nil, nil -} - -func TestProjectPreForward_eth_call_Batching(t *testing.T) { - cfg := &common.NetworkConfig{ - Evm: &common.EvmNetworkConfig{ - ChainId: 1, - Multicall3Aggregation: &common.Multicall3AggregationConfig{ - Enabled: true, - WindowMs: 50, - MinWaitMs: 5, - MaxCalls: 20, - MaxCalldataBytes: 64000, - MaxQueueSize: 100, - MaxPendingBatches: 20, - AllowCrossUserBatching: &common.TRUE, - CachePerCall: &common.FALSE, - }, - }, - } - - // Create valid multicall response - resultHex := "0x" + - "0000000000000000000000000000000000000000000000000000000000000020" + - "0000000000000000000000000000000000000000000000000000000000000002" + - "0000000000000000000000000000000000000000000000000000000000000040" + - "00000000000000000000000000000000000000000000000000000000000000a0" + - "0000000000000000000000000000000000000000000000000000000000000001" + - "0000000000000000000000000000000000000000000000000000000000000040" + - "0000000000000000000000000000000000000000000000000000000000000004" + - "deadbeef00000000000000000000000000000000000000000000000000000000" + - "0000000000000000000000000000000000000000000000000000000000000001" + - "0000000000000000000000000000000000000000000000000000000000000040" + - "0000000000000000000000000000000000000000000000000000000000000004" + - "cafebabe00000000000000000000000000000000000000000000000000000000" - - jrr, _ := common.NewJsonRpcResponse(nil, resultHex, nil) - mockResp := common.NewNormalizedResponse().WithJsonRpcResponse(jrr) - - network := &mockNetwork{ - networkId: "evm:1", - cfg: cfg, - forwardFn: func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { - return mockResp, nil - }, - } - - // Prepare two requests to be batched - ctx := context.Background() - - jrq1 := common.NewJsonRpcRequest("eth_call", []interface{}{ - map[string]interface{}{ - "to": "0x1111111111111111111111111111111111111111", - "data": "0x01020304", - }, - "latest", - }) - req1 := common.NewNormalizedRequestFromJsonRpcRequest(jrq1) - req1.SetNetwork(network) - - jrq2 := common.NewJsonRpcRequest("eth_call", []interface{}{ - map[string]interface{}{ - "to": "0x2222222222222222222222222222222222222222", - "data": "0x05060708", - }, - "latest", - }) - req2 := common.NewNormalizedRequestFromJsonRpcRequest(jrq2) - req2.SetNetwork(network) - - // Both should be batched - var resp1, resp2 *common.NormalizedResponse - var err1, err2 error - done := make(chan struct{}, 2) - - go func() { - _, resp1, err1 = HandleProjectPreForward(ctx, network, req1) - done <- struct{}{} - }() - - go func() { - _, resp2, err2 = HandleProjectPreForward(ctx, network, req2) - done <- struct{}{} - }() - - // Wait with timeout - for i := 0; i < 2; i++ { - select { - case <-done: - case <-time.After(time.Second): - t.Fatal("timeout waiting for batched requests") - } - } - - require.NoError(t, err1) - require.NoError(t, err2) - require.NotNil(t, resp1) - require.NotNil(t, resp2) -} -``` - -**Step 2: Run test to verify it fails** - -Run: `go test -v ./architecture/evm -run TestProjectPreForward_eth_call_Batching` -Expected: FAIL - batching not implemented in hooks - -**Step 3: Update eth_call.go with batching integration** - -Replace `architecture/evm/eth_call.go`: - -```go -package evm - -import ( - "context" - "fmt" - "sync" - - "github.com/erpc/erpc/common" -) - -// Global batcher manager for network-level Multicall3 batching -var ( - globalBatcherManager *BatcherManager - batcherManagerOnce sync.Once -) - -// GetBatcherManager returns the global batcher manager. -func GetBatcherManager() *BatcherManager { - batcherManagerOnce.Do(func() { - globalBatcherManager = NewBatcherManager() - }) - return globalBatcherManager -} - -// networkForwarder wraps a Network to implement Forwarder interface. -type networkForwarder struct { - network common.Network -} - -func (f *networkForwarder) Forward(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { - return f.network.Forward(ctx, req) -} - -func projectPreForward_eth_call(ctx context.Context, network common.Network, nq *common.NormalizedRequest) (bool, *common.NormalizedResponse, error) { - jrq, err := nq.JsonRpcRequest() - if err != nil { - return false, nil, nil - } - - // Normalize params: ensure block param is present - jrq.RLock() - paramsLen := len(jrq.Params) - jrq.RUnlock() - - if paramsLen == 1 { - jrq.Lock() - jrq.Params = append(jrq.Params, "latest") - jrq.Unlock() - } - - // Check if Multicall3 aggregation is enabled - cfg := network.Config() - if cfg == nil || cfg.Evm == nil || cfg.Evm.Multicall3Aggregation == nil || !cfg.Evm.Multicall3Aggregation.Enabled { - // Batching disabled, use normal forward - resp, err := network.Forward(ctx, nq) - return true, resp, err - } - - aggCfg := cfg.Evm.Multicall3Aggregation - - // Check eligibility for batching - eligible, reason := IsEligibleForBatching(nq, aggCfg) - if !eligible { - // Not eligible, forward normally - _ = reason // could log this - resp, err := network.Forward(ctx, nq) - return true, resp, err - } - - // Extract call info for batching key - _, _, blockRef, err := ExtractCallInfo(nq) - if err != nil { - resp, err := network.Forward(ctx, nq) - return true, resp, err - } - - // Build batching key - projectId := "" - if nq.Network() != nil { - // Try to get project ID from request context or network - projectId = fmt.Sprintf("network:%s", network.Id()) - } - - userId := "" - if aggCfg.AllowCrossUserBatching == nil || !*aggCfg.AllowCrossUserBatching { - userId = nq.UserId() - } - - key := BatchingKey{ - ProjectId: projectId, - NetworkId: network.Id(), - BlockRef: blockRef, - DirectivesKey: DeriveDirectivesKey(nq.Directives()), - UserId: userId, - } - - // Get or create batcher for this network - mgr := GetBatcherManager() - forwarder := &networkForwarder{network: network} - batcher := mgr.GetOrCreate(network.Id(), aggCfg, forwarder) - - // Enqueue request - entry, bypass, err := batcher.Enqueue(ctx, key, nq) - if err != nil || bypass { - // Bypass batching, forward normally - resp, err := network.Forward(ctx, nq) - return true, resp, err - } - - // Wait for batch result - select { - case result := <-entry.ResultCh: - return true, result.Response, result.Error - case <-ctx.Done(): - return true, nil, ctx.Err() - } -} -``` - -**Step 4: Run test to verify it passes** - -Run: `go test -v ./architecture/evm -run TestProjectPreForward_eth_call_Batching` -Expected: PASS - -**Step 5: Commit** - -```bash -git add architecture/evm/eth_call.go architecture/evm/eth_call_test.go -git commit -m "$(cat <<'EOF' -feat: integrate Multicall3 batching into eth_call pre-forward hook - -Modifies projectPreForward_eth_call to: -- Check if Multicall3 aggregation is enabled -- Verify request eligibility for batching -- Build batching key from project/network/block/directives/user -- Enqueue eligible requests to network batcher -- Wait for batch result or bypass to normal forward - -Uses global BatcherManager for per-network batcher instances. - -Co-Authored-By: Claude Opus 4.5 -EOF -)" -``` - ---- - -## Phase 4: Metrics and Observability - -### Task 4.1: Add Multicall3 Batching Metrics - -**Files:** -- Modify: `telemetry/metrics.go` -- Modify: `architecture/evm/multicall3_batcher.go` - -**Step 1: Add new metrics to telemetry/metrics.go** - -Add after existing multicall3 metrics (around line 431): - -```go - // Network-level Multicall3 batching metrics - MetricMulticall3BatchSize = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "erpc", - Name: "multicall3_batch_size", - Help: "Number of unique calls per Multicall3 batch.", - Buckets: []float64{1, 2, 5, 10, 15, 20, 30, 50}, - }, []string{"project", "network"}) - - MetricMulticall3BatchWaitMs = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "erpc", - Name: "multicall3_batch_wait_ms", - Help: "Time requests waited in batch before flush (milliseconds).", - Buckets: []float64{1, 2, 5, 10, 15, 20, 25, 30, 50}, - }, []string{"project", "network"}) - - MetricMulticall3QueueLen = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: "erpc", - Name: "multicall3_queue_len", - Help: "Current number of requests queued for batching.", - }, []string{"network"}) - - MetricMulticall3QueueOverflowTotal = promauto.NewCounterVec(prometheus.CounterOpts{ - Namespace: "erpc", - Name: "multicall3_queue_overflow_total", - Help: "Total number of requests that bypassed batching due to queue overflow.", - }, []string{"network", "reason"}) - - MetricMulticall3DedupeTotal = promauto.NewCounterVec(prometheus.CounterOpts{ - Namespace: "erpc", - Name: "multicall3_dedupe_total", - Help: "Total number of deduplicated requests within batches.", - }, []string{"project", "network"}) -``` - -**Step 2: Update Batcher to emit metrics** - -Add metric recording to `architecture/evm/multicall3_batcher.go`: - -At the top, add import: -```go -import ( - "github.com/erpc/erpc/telemetry" - // ... other imports -) -``` - -Update `Enqueue` to record overflow metrics when bypassing: -```go -// In Enqueue, after checking caps: -if b.queueSize >= int64(b.cfg.MaxQueueSize) { - telemetry.MetricMulticall3QueueOverflowTotal.WithLabelValues(key.NetworkId, "queue_full").Inc() - return nil, true, nil -} -if len(b.batches) >= b.cfg.MaxPendingBatches { - if _, exists := b.batches[key.String()]; !exists { - telemetry.MetricMulticall3QueueOverflowTotal.WithLabelValues(key.NetworkId, "max_batches").Inc() - return nil, true, nil - } -} -``` - -Update `flush` to record batch metrics: -```go -// At the start of flush, after getting entries: -if len(entries) > 0 { - projectId := batch.Key.ProjectId - networkId := batch.Key.NetworkId - - // Record batch size - telemetry.MetricMulticall3BatchSize.WithLabelValues(projectId, networkId).Observe(float64(len(callKeys))) - - // Record wait time - for _, entry := range entries { - waitMs := time.Since(entry.CreatedAt).Milliseconds() - telemetry.MetricMulticall3BatchWaitMs.WithLabelValues(projectId, networkId).Observe(float64(waitMs)) - } - - // Record dedupe count - totalEntries := len(entries) - uniqueCalls := len(callKeys) - if totalEntries > uniqueCalls { - telemetry.MetricMulticall3DedupeTotal.WithLabelValues(projectId, networkId).Add(float64(totalEntries - uniqueCalls)) - } -} -``` - -Update gauge on enqueue/remove: -```go -// In Enqueue after adding to batch: -telemetry.MetricMulticall3QueueLen.WithLabelValues(key.NetworkId).Inc() - -// In flush after removing from batches: -telemetry.MetricMulticall3QueueLen.WithLabelValues(batch.Key.NetworkId).Sub(float64(len(entries))) -``` - -**Step 3: Run existing tests** - -Run: `go test -v ./architecture/evm/...` -Expected: PASS - -**Step 4: Commit** - -```bash -git add telemetry/metrics.go architecture/evm/multicall3_batcher.go -git commit -m "$(cat <<'EOF' -feat: add Multicall3 batching observability metrics - -New metrics for network-level Multicall3 batching: -- multicall3_batch_size: histogram of unique calls per batch -- multicall3_batch_wait_ms: histogram of request wait times -- multicall3_queue_len: gauge of current queue depth -- multicall3_queue_overflow_total: counter for bypass events -- multicall3_dedupe_total: counter for deduplicated requests - -Co-Authored-By: Claude Opus 4.5 -EOF -)" -``` - ---- - -## Phase 5: Testing - -### Task 5.1: Add Comprehensive Unit Tests - -**Files:** -- Modify: `architecture/evm/multicall3_batcher_test.go` - -**Step 1: Add cancellation test** - -```go -func TestBatcherCancellation(t *testing.T) { - cfg := &common.Multicall3AggregationConfig{ - Enabled: true, - WindowMs: 100, // Long window - MinWaitMs: 50, - MaxCalls: 10, - MaxCalldataBytes: 64000, - MaxQueueSize: 100, - MaxPendingBatches: 20, - AllowCrossUserBatching: &common.TRUE, - } - - batcher := NewBatcher(cfg, nil) - - ctx, cancel := context.WithCancel(context.Background()) - key := BatchingKey{ - ProjectId: "test", - NetworkId: "evm:1", - BlockRef: "latest", - } - - jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ - map[string]interface{}{"to": "0x1234567890123456789012345678901234567890", "data": "0x01"}, - "latest", - }) - req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) - - entry, bypass, err := batcher.Enqueue(ctx, key, req) - require.NoError(t, err) - require.False(t, bypass) - require.NotNil(t, entry) - - // Cancel before flush - cancel() - - // Result channel should still work (batch will complete eventually) - // The cancelled context shouldn't crash the batcher - batcher.Shutdown() -} -``` - -**Step 2: Add deadline-aware test** - -```go -func TestBatcherDeadlineAwareness(t *testing.T) { - cfg := &common.Multicall3AggregationConfig{ - Enabled: true, - WindowMs: 100, - MinWaitMs: 10, - SafetyMarginMs: 5, - MaxCalls: 10, - MaxCalldataBytes: 64000, - MaxQueueSize: 100, - MaxPendingBatches: 20, - AllowCrossUserBatching: &common.TRUE, - } - - batcher := NewBatcher(cfg, nil) - - // Context with tight deadline - should bypass - tightCtx, cancel1 := context.WithDeadline(context.Background(), time.Now().Add(5*time.Millisecond)) - defer cancel1() - - key := BatchingKey{ - ProjectId: "test", - NetworkId: "evm:1", - BlockRef: "latest", - } - - jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ - map[string]interface{}{"to": "0x1234567890123456789012345678901234567890", "data": "0x01"}, - "latest", - }) - req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) - - _, bypass, err := batcher.Enqueue(tightCtx, key, req) - require.NoError(t, err) - require.True(t, bypass, "should bypass with tight deadline") - - // Context with reasonable deadline - should batch - normalCtx, cancel2 := context.WithDeadline(context.Background(), time.Now().Add(200*time.Millisecond)) - defer cancel2() - - _, bypass, err = batcher.Enqueue(normalCtx, key, req) - require.NoError(t, err) - require.False(t, bypass, "should batch with normal deadline") - - batcher.Shutdown() -} -``` - -**Step 3: Add concurrent flush test** - -```go -func TestBatcherConcurrentFlush(t *testing.T) { - // Verify that a request arriving during flush gets a new batch - cfg := &common.Multicall3AggregationConfig{ - Enabled: true, - WindowMs: 10, // Short window - MinWaitMs: 1, - MaxCalls: 10, - MaxCalldataBytes: 64000, - MaxQueueSize: 100, - MaxPendingBatches: 20, - AllowCrossUserBatching: &common.TRUE, - } - - // Slow forwarder to simulate flush in progress - forwarder := &mockForwarder{ - response: nil, // Will cause fallback - err: nil, - } - - batcher := NewBatcher(cfg, forwarder) - - ctx := context.Background() - key := BatchingKey{ - ProjectId: "test", - NetworkId: "evm:1", - BlockRef: "latest", - } - - // Add first batch - for i := 0; i < 3; i++ { - jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ - map[string]interface{}{"to": fmt.Sprintf("0x%040d", i), "data": "0x01"}, - "latest", - }) - req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) - batcher.Enqueue(ctx, key, req) - } - - // Wait for first batch to start flushing - time.Sleep(15 * time.Millisecond) - - // Add more requests - should go to new batch - for i := 3; i < 6; i++ { - jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ - map[string]interface{}{"to": fmt.Sprintf("0x%040d", i), "data": "0x01"}, - "latest", - }) - req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) - _, bypass, _ := batcher.Enqueue(ctx, key, req) - // May bypass or create new batch - either is acceptable - _ = bypass - } - - batcher.Shutdown() -} -``` - -**Step 4: Run all tests** - -Run: `go test -v ./architecture/evm/... -count=1` -Expected: PASS - -**Step 5: Commit** - -```bash -git add architecture/evm/multicall3_batcher_test.go -git commit -m "$(cat <<'EOF' -test: add comprehensive Multicall3 batcher tests - -Adds tests for: -- Request cancellation handling -- Deadline-aware flush scheduling -- Concurrent flush and new batch creation - -Co-Authored-By: Claude Opus 4.5 -EOF -)" -``` - ---- - -### Task 5.2: Add Integration Tests - -**Files:** -- Create: `architecture/evm/multicall3_integration_test.go` - -**Step 1: Write integration test** - -Create `architecture/evm/multicall3_integration_test.go`: - -```go -//go:build integration - -package evm - -import ( - "context" - "sync" - "testing" - "time" - - "github.com/erpc/erpc/common" - "github.com/stretchr/testify/require" -) - -func TestMulticall3EndToEndBatching(t *testing.T) { - // This test verifies the full batching flow with multiple concurrent requests - - // Create valid multicall3 response for 5 calls - resultHex := createMulticall3Response(5) - jrr, _ := common.NewJsonRpcResponse(nil, resultHex, nil) - mockResp := common.NewNormalizedResponse().WithJsonRpcResponse(jrr) - - callCount := 0 - var callMu sync.Mutex - - forwarder := &mockForwarder{ - response: mockResp, - err: nil, - } - // Override the simple mock to count calls - originalForward := forwarder.Forward - forwarder.Forward = func(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { - callMu.Lock() - callCount++ - callMu.Unlock() - return originalForward(ctx, req) - } - - cfg := &common.Multicall3AggregationConfig{ - Enabled: true, - WindowMs: 30, - MinWaitMs: 5, - SafetyMarginMs: 2, - MaxCalls: 20, - MaxCalldataBytes: 64000, - MaxQueueSize: 100, - MaxPendingBatches: 20, - AllowCrossUserBatching: &common.TRUE, - CachePerCall: &common.FALSE, - } - - batcher := NewBatcher(cfg, forwarder) - - ctx := context.Background() - key := BatchingKey{ - ProjectId: "test-project", - NetworkId: "evm:1", - BlockRef: "latest", - DirectivesKey: DeriveDirectivesKey(nil), - } - - // Launch 5 concurrent requests - var wg sync.WaitGroup - results := make([]*BatchResult, 5) - - for i := 0; i < 5; i++ { - wg.Add(1) - go func(idx int) { - defer wg.Done() - - jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ - map[string]interface{}{ - "to": fmt.Sprintf("0x%040d", idx), - "data": fmt.Sprintf("0x%08x", idx), - }, - "latest", - }) - jrq.ID = fmt.Sprintf("req-%d", idx) - req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) - - entry, bypass, err := batcher.Enqueue(ctx, key, req) - if err != nil || bypass { - results[idx] = &BatchResult{Error: err} - return - } - - result := <-entry.ResultCh - results[idx] = &result - }(i) - } - - // Wait for all requests to complete - done := make(chan struct{}) - go func() { - wg.Wait() - close(done) - }() - - select { - case <-done: - case <-time.After(5 * time.Second): - t.Fatal("timeout waiting for batched requests") - } - - // Verify all got results - for i, result := range results { - require.NotNil(t, result, "result %d is nil", i) - require.NoError(t, result.Error, "result %d has error", i) - require.NotNil(t, result.Response, "result %d has no response", i) - } - - // Verify forwarder was called only once (all batched) - callMu.Lock() - require.Equal(t, 1, callCount, "forwarder should be called once for batched requests") - callMu.Unlock() - - batcher.Shutdown() -} - -// createMulticall3Response creates a valid multicall3 aggregate3 response for n calls. -func createMulticall3Response(n int) string { - // Simplified: just create a response with n successful results - // Real implementation would properly ABI encode - - // For now, return empty since the actual encoding is complex - // Tests should use pre-computed values for specific cases - return "0x" // Placeholder - actual tests use pre-computed hex -} -``` - -**Step 2: Run tests** - -Run: `go test -v ./architecture/evm/... -tags=integration -count=1` -Expected: May need adjustment based on mock setup - -**Step 3: Commit** - -```bash -git add architecture/evm/multicall3_integration_test.go -git commit -m "$(cat <<'EOF' -test: add Multicall3 integration tests - -Integration test verifying: -- Multiple concurrent requests are batched -- Single forwarder call for batched requests -- Results correctly distributed to all waiters - -Co-Authored-By: Claude Opus 4.5 -EOF -)" -``` - ---- - -## Phase 6: Documentation and Cleanup - -### Task 6.1: Update Design Doc Status - -**Files:** -- Modify: `docs/design/multicall3-batching.md` - -**Step 1: Update status** - -Change line 3 from: -```markdown -Status: Proposed -``` -To: -```markdown -Status: Implemented -``` - -**Step 2: Commit** - -```bash -git add docs/design/multicall3-batching.md -git commit -m "$(cat <<'EOF' -docs: mark multicall3-batching design as implemented - -Co-Authored-By: Claude Opus 4.5 -EOF -)" -``` - ---- - -### Task 6.2: Run Full Test Suite - -**Step 1: Run all tests** - -Run: `go test -v ./... -count=1` -Expected: PASS - -**Step 2: Run linter** - -Run: `golangci-lint run` -Expected: No new issues - -**Step 3: Final commit if any fixes needed** - ---- - -## Execution Summary - -**Total Tasks:** 12 (across 6 phases) - -**Key Files Created:** -- `architecture/evm/multicall3_batcher.go` - Core batcher implementation -- `architecture/evm/multicall3_manager.go` - Per-network batcher manager -- `architecture/evm/multicall3_batcher_test.go` - Unit tests -- `architecture/evm/multicall3_manager_test.go` - Manager tests -- `architecture/evm/eth_call_test.go` - Integration tests - -**Key Files Modified:** -- `common/config.go` - Extended Multicall3AggregationConfig -- `common/request.go` - Added CompositeTypeMulticall3 -- `architecture/evm/eth_call.go` - Integration with batcher -- `telemetry/metrics.go` - New batching metrics - -**Dependencies:** None new - uses existing standard library and project packages. From 57d49a768c720d6515a31179fe84d06cad6e0506 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 16 Jan 2026 14:37:48 +0100 Subject: [PATCH 42/53] fix: correct multicall3 config defaults in user documentation --- docs/pages/config/projects/networks.mdx | 40 ++++++++++++------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/pages/config/projects/networks.mdx b/docs/pages/config/projects/networks.mdx index 96911ebf4..24a925a65 100644 --- a/docs/pages/config/projects/networks.mdx +++ b/docs/pages/config/projects/networks.mdx @@ -49,23 +49,23 @@ projects: # DEFAULT: true enabled: true # Time window in milliseconds to collect requests before batching. - # DEFAULT: 50 - windowMs: 50 + # DEFAULT: 25 + windowMs: 25 # Minimum wait time in milliseconds before flushing a batch. - # DEFAULT: 10 - minWaitMs: 10 + # DEFAULT: 2 + minWaitMs: 2 # Maximum number of calls per batch. - # DEFAULT: 50 - maxCalls: 50 + # DEFAULT: 20 + maxCalls: 20 # Maximum total calldata size in bytes for a batch. - # DEFAULT: 128000 - maxCalldataBytes: 128000 + # DEFAULT: 64000 + maxCalldataBytes: 64000 # Maximum number of requests waiting to be batched. # DEFAULT: 1000 maxQueueSize: 1000 # Maximum number of pending batches being processed. - # DEFAULT: 100 - maxPendingBatches: 100 + # DEFAULT: 200 + maxPendingBatches: 200 # Allow batching requests from different users together. # Set to false to isolate user requests into separate batches. # DEFAULT: true @@ -211,18 +211,18 @@ export default createConfig({ multicall3Aggregation: { // Enable or disable Multicall3 aggregation. DEFAULT: true enabled: true, - // Time window in milliseconds to collect requests before batching. DEFAULT: 50 - windowMs: 50, - // Minimum wait time in milliseconds before flushing a batch. DEFAULT: 10 - minWaitMs: 10, - // Maximum number of calls per batch. DEFAULT: 50 - maxCalls: 50, - // Maximum total calldata size in bytes for a batch. DEFAULT: 128000 - maxCalldataBytes: 128000, + // Time window in milliseconds to collect requests before batching. DEFAULT: 25 + windowMs: 25, + // Minimum wait time in milliseconds before flushing a batch. DEFAULT: 2 + minWaitMs: 2, + // Maximum number of calls per batch. DEFAULT: 20 + maxCalls: 20, + // Maximum total calldata size in bytes for a batch. DEFAULT: 64000 + maxCalldataBytes: 64000, // Maximum number of requests waiting to be batched. DEFAULT: 1000 maxQueueSize: 1000, - // Maximum number of pending batches being processed. DEFAULT: 100 - maxPendingBatches: 100, + // Maximum number of pending batches being processed. DEFAULT: 200 + maxPendingBatches: 200, // Allow batching requests from different users together. DEFAULT: true allowCrossUserBatching: true, }, From 3e944bc252d672f513b89f600faeeb5b8106a393 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 16 Jan 2026 15:48:22 +0100 Subject: [PATCH 43/53] test: add comprehensive tests for multicall3 batching Add new unit tests for multicall3 batcher and HTTP batch handling: - multicall3_batcher_test.go: Add tests for decode errors (nil response, nil JSON-RPC, JSON-RPC error, empty result), EIP-1898 block hash wrapping, context cancellation cleanup, successful delivery, cancelled context handling, and multiple deadline scheduling - http_batch_eth_call_handle_test.go: Add panic recovery test for fallback forward path and empty params handling test - http_batch_eth_call_forward_test.go: Add panic recovery and context cancellation tests for forward goroutines Co-Authored-By: Claude Opus 4.5 --- architecture/evm/multicall3_batcher_test.go | 425 ++++++++++++++++++++ erpc/http_batch_eth_call_forward_test.go | 41 ++ erpc/http_batch_eth_call_handle_test.go | 50 +++ 3 files changed, 516 insertions(+) diff --git a/architecture/evm/multicall3_batcher_test.go b/architecture/evm/multicall3_batcher_test.go index 2e77d80dd..5cb4bd6bb 100644 --- a/architecture/evm/multicall3_batcher_test.go +++ b/architecture/evm/multicall3_batcher_test.go @@ -2168,3 +2168,428 @@ func TestBatcher_InvalidTargetLength_Bypass(t *testing.T) { require.True(t, bypass, "request with invalid target should bypass") require.Contains(t, err.Error(), "invalid target address length") } + +func TestDecodeMulticallResponse_NilResponse(t *testing.T) { + forwarder := &mockForwarder{} + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 50, + MinWaitMs: 5, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + } + cfg.SetDefaults() + + batcher := NewBatcher(cfg, forwarder, nil) + require.NotNil(t, batcher) + defer batcher.Shutdown() + + // Test with nil response + results, err := batcher.decodeMulticallResponse(nil) + require.Error(t, err) + require.Nil(t, results) + require.Contains(t, err.Error(), "nil response") +} + +func TestDecodeMulticallResponse_NilJsonRpc(t *testing.T) { + forwarder := &mockForwarder{} + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 50, + MinWaitMs: 5, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + } + cfg.SetDefaults() + + batcher := NewBatcher(cfg, forwarder, nil) + require.NotNil(t, batcher) + defer batcher.Shutdown() + + // Test with response that has no JsonRpcResponse set + resp := common.NewNormalizedResponse() + results, err := batcher.decodeMulticallResponse(resp) + require.Error(t, err) + require.Nil(t, results) + // Error comes from JsonRpcResponse() which returns an error when no body is available + require.Contains(t, err.Error(), "no body available to parse JsonRpcResponse") +} + +func TestDecodeMulticallResponse_JsonRpcError(t *testing.T) { + forwarder := &mockForwarder{} + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 50, + MinWaitMs: 5, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + } + cfg.SetDefaults() + + batcher := NewBatcher(cfg, forwarder, nil) + require.NotNil(t, batcher) + defer batcher.Shutdown() + + // Create a response with JSON-RPC error + jrr := &common.JsonRpcResponse{ + Error: common.NewErrJsonRpcExceptionExternal( + -32000, + "execution error", + "", + ), + } + resp := common.NewNormalizedResponse().WithJsonRpcResponse(jrr) + + results, err := batcher.decodeMulticallResponse(resp) + require.Error(t, err) + require.Nil(t, results) + require.Contains(t, err.Error(), "execution error") +} + +func TestDecodeMulticallResponse_EmptyResult(t *testing.T) { + forwarder := &mockForwarder{} + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 50, + MinWaitMs: 5, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + } + cfg.SetDefaults() + + batcher := NewBatcher(cfg, forwarder, nil) + require.NotNil(t, batcher) + defer batcher.Shutdown() + + // Create a response with empty/null result + jrr, err := common.NewJsonRpcResponse(nil, nil, nil) + require.NoError(t, err) + resp := common.NewNormalizedResponse().WithJsonRpcResponse(jrr) + + results, err := batcher.decodeMulticallResponse(resp) + require.Error(t, err) + require.Nil(t, results) + require.Contains(t, err.Error(), "empty result") +} + +func TestBlockParamForMulticall_BlockHashEIP1898(t *testing.T) { + tests := []struct { + name string + blockRef string + expected interface{} + wantErr bool + }{ + { + name: "block hash wraps to EIP-1898", + blockRef: "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + expected: map[string]interface{}{"blockHash": "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"}, + wantErr: false, + }, + { + name: "hex block number stays as-is", + blockRef: "0x10", + expected: "0x10", + wantErr: false, + }, + { + name: "decimal block number converts to hex", + blockRef: "16", + expected: "0x10", + wantErr: false, + }, + { + name: "named block tag stays as-is", + blockRef: "latest", + expected: "latest", + wantErr: false, + }, + { + name: "finalized stays as-is", + blockRef: "finalized", + expected: "finalized", + wantErr: false, + }, + { + name: "safe stays as-is", + blockRef: "safe", + expected: "safe", + wantErr: false, + }, + { + name: "empty string becomes latest", + blockRef: "", + expected: "latest", + wantErr: false, + }, + { + name: "short hex (not block hash) stays as-is", + blockRef: "0xabc123", + expected: "0xabc123", + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := blockParamForMulticall(tt.blockRef) + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, tt.expected, result) + } + }) + } +} + +func TestSendResult_ContextCancelled_ReleasesResponse(t *testing.T) { + forwarder := &mockForwarder{} + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 50, + MinWaitMs: 5, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + } + cfg.SetDefaults() + + batcher := NewBatcher(cfg, forwarder, nil) + require.NotNil(t, batcher) + defer batcher.Shutdown() + + // Create a cancelled context + ctx, cancel := context.WithCancel(context.Background()) + cancel() // Cancel immediately + + entry := &BatchEntry{ + Ctx: ctx, + ResultCh: make(chan BatchResult, 1), + } + + // Create a mock response to track release + jrr, err := common.NewJsonRpcResponse(nil, "0xdeadbeef", nil) + require.NoError(t, err) + resp := common.NewNormalizedResponse().WithJsonRpcResponse(jrr) + + // sendResult should return false because context is cancelled + sent := batcher.sendResult(entry, BatchResult{Response: resp}, "test-project", "evm:1") + require.False(t, sent, "sendResult should return false for cancelled context") + + // ResultCh should be empty since the result was not sent + select { + case <-entry.ResultCh: + t.Fatal("result should not have been sent to channel") + default: + // Expected - channel is empty + } +} + +func TestBatcher_SendResult_SuccessfulDelivery(t *testing.T) { + forwarder := &mockForwarder{} + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 50, + MinWaitMs: 5, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + } + cfg.SetDefaults() + + batcher := NewBatcher(cfg, forwarder, nil) + require.NotNil(t, batcher) + defer batcher.Shutdown() + + // Create a valid (non-cancelled) context + ctx := context.Background() + + entry := &BatchEntry{ + Ctx: ctx, + ResultCh: make(chan BatchResult, 1), + } + + // Create a mock response + jrr, err := common.NewJsonRpcResponse(nil, "0xdeadbeef", nil) + require.NoError(t, err) + resp := common.NewNormalizedResponse().WithJsonRpcResponse(jrr) + + // sendResult should return true for successful delivery + sent := batcher.sendResult(entry, BatchResult{Response: resp}, "test-project", "evm:1") + require.True(t, sent, "sendResult should return true for successful delivery") + + // ResultCh should have the result + select { + case result := <-entry.ResultCh: + require.NotNil(t, result.Response) + require.NoError(t, result.Error) + default: + t.Fatal("result should have been sent to channel") + } +} + +func TestBatcher_DeliverError_SkipsCancelledContexts(t *testing.T) { + forwarder := &mockForwarder{} + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 50, + MinWaitMs: 5, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + } + cfg.SetDefaults() + + batcher := NewBatcher(cfg, forwarder, nil) + require.NotNil(t, batcher) + defer batcher.Shutdown() + + // Create one cancelled and one active context + cancelledCtx, cancel := context.WithCancel(context.Background()) + cancel() + activeCtx := context.Background() + + entries := []*BatchEntry{ + {Ctx: cancelledCtx, ResultCh: make(chan BatchResult, 1)}, + {Ctx: activeCtx, ResultCh: make(chan BatchResult, 1)}, + } + + testErr := fmt.Errorf("test error") + batcher.deliverError(entries, testErr, "test-project", "evm:1") + + // Cancelled context entry should not receive the error + select { + case <-entries[0].ResultCh: + t.Fatal("cancelled entry should not receive result") + default: + // Expected + } + + // Active context entry should receive the error + select { + case result := <-entries[1].ResultCh: + require.Error(t, result.Error) + require.Equal(t, testErr, result.Error) + default: + t.Fatal("active entry should receive error") + } +} + +func TestBatcher_MultipleDeadlinesPickEarliest(t *testing.T) { + results := []Multicall3Result{ + {Success: true, ReturnData: []byte{0xaa}}, + {Success: true, ReturnData: []byte{0xbb}}, + } + encodedResult := encodeAggregate3Results(results) + resultHex := "0x" + hex.EncodeToString(encodedResult) + + jrr, err := common.NewJsonRpcResponse(nil, resultHex, nil) + require.NoError(t, err) + mockResp := common.NewNormalizedResponse().WithJsonRpcResponse(jrr) + + forwarder := &mockForwarder{response: mockResp} + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + WindowMs: 100, + MinWaitMs: 5, + SafetyMarginMs: 5, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + CachePerCall: util.BoolPtr(false), + } + cfg.SetDefaults() + + batcher := NewBatcher(cfg, forwarder, nil) + require.NotNil(t, batcher) + defer batcher.Shutdown() + + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + // First request with a later deadline + laterDeadline := time.Now().Add(200 * time.Millisecond) + ctx1, cancel1 := context.WithDeadline(context.Background(), laterDeadline) + defer cancel1() + + jrq1 := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{"to": "0x1111111111111111111111111111111111111111", "data": "0x01"}, + "latest", + }) + req1 := common.NewNormalizedRequestFromJsonRpcRequest(jrq1) + + entry1, bypass1, err := batcher.Enqueue(ctx1, key, req1) + require.NoError(t, err) + require.False(t, bypass1) + + // Get initial flush time + batcher.mu.Lock() + batch := batcher.batches[key.String()] + batcher.mu.Unlock() + require.NotNil(t, batch) + batch.mu.Lock() + initialFlushTime := batch.FlushTime + batch.mu.Unlock() + + // Second request with an earlier deadline (should update flush time) + earlierDeadline := time.Now().Add(50 * time.Millisecond) + ctx2, cancel2 := context.WithDeadline(context.Background(), earlierDeadline) + defer cancel2() + + jrq2 := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{"to": "0x2222222222222222222222222222222222222222", "data": "0x02"}, + "latest", + }) + req2 := common.NewNormalizedRequestFromJsonRpcRequest(jrq2) + + entry2, bypass2, err := batcher.Enqueue(ctx2, key, req2) + require.NoError(t, err) + require.False(t, bypass2) + + // Check that flush time was updated to the earlier deadline + batch.mu.Lock() + updatedFlushTime := batch.FlushTime + batch.mu.Unlock() + + require.True(t, updatedFlushTime.Before(initialFlushTime), "flush time should be updated to earlier deadline") + + // Wait for both results + select { + case result1 := <-entry1.ResultCh: + require.NoError(t, result1.Error) + case <-time.After(2 * time.Second): + t.Fatal("timeout waiting for entry1 result") + } + + select { + case result2 := <-entry2.ResultCh: + require.NoError(t, result2.Error) + case <-time.After(2 * time.Second): + t.Fatal("timeout waiting for entry2 result") + } +} diff --git a/erpc/http_batch_eth_call_forward_test.go b/erpc/http_batch_eth_call_forward_test.go index e3ecc367f..2487882c2 100644 --- a/erpc/http_batch_eth_call_forward_test.go +++ b/erpc/http_batch_eth_call_forward_test.go @@ -56,4 +56,45 @@ func TestForwardEthCallBatchCandidates(t *testing.T) { server.forwardEthCallBatchCandidates(&startedAt, &PreparedProject{}, &Network{}, []ethCallBatchCandidate{makeCandidate(0)}, responses) require.Equal(t, resp, responses[0]) }) + + t.Run("panic recovery in forward goroutine", func(t *testing.T) { + responses := make([]interface{}, 2) + callCount := 0 + forwardBatchProject = func(ctx context.Context, project *PreparedProject, network *Network, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + callCount++ + if callCount == 1 { + panic("test panic in forward") + } + return common.NewNormalizedResponse(), nil + } + + // Create 2 candidates - one will panic, one will succeed + server.forwardEthCallBatchCandidates(&startedAt, &PreparedProject{}, &Network{}, []ethCallBatchCandidate{makeCandidate(0), makeCandidate(1)}, responses) + + // Both responses should have been populated (panic recovered) + require.NotNil(t, responses[0], "first response should not be nil after panic") + require.NotNil(t, responses[1], "second response should not be nil") + }) + + t.Run("context cancellation is handled", func(t *testing.T) { + cancelledCtx, cancel := context.WithCancel(context.Background()) + cancel() // Cancel immediately + + req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_call","params":[{"to":"0x0000000000000000000000000000000000000001","data":"0x"}]}`)) + + candidate := ethCallBatchCandidate{ + index: 0, + ctx: cancelledCtx, + req: req, + logger: log.Logger, + } + + responses := make([]interface{}, 1) + forwardBatchProject = func(ctx context.Context, project *PreparedProject, network *Network, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + return nil, ctx.Err() + } + + server.forwardEthCallBatchCandidates(&startedAt, &PreparedProject{}, &Network{}, []ethCallBatchCandidate{candidate}, responses) + require.NotNil(t, responses[0], "response should be populated even with cancelled context") + }) } diff --git a/erpc/http_batch_eth_call_handle_test.go b/erpc/http_batch_eth_call_handle_test.go index d8571aaad..8ecc9b36f 100644 --- a/erpc/http_batch_eth_call_handle_test.go +++ b/erpc/http_batch_eth_call_handle_test.go @@ -374,6 +374,56 @@ func TestHandleEthCallBatchAggregation_SuccessAndFailureResults(t *testing.T) { assert.Equal(t, "0x"+hex.EncodeToString(results[1].ReturnData), errMap["data"]) } +func TestHandleEthCallBatchAggregation_PanicRecovery(t *testing.T) { + t.Run("panic in fallback forward is recovered", func(t *testing.T) { + cfg := baseBatchConfig() + server, project, ctx, cleanup := setupBatchHandler(t, cfg) + defer cleanup() + + // Panic recovery exists in forwardEthCallBatchCandidates (fallback path) + // The multicall3 network forward doesn't have panic recovery, but the + // fallback path does. So we make network forward fail to trigger fallback, + // then have the project forward (fallback) panic. + withBatchStubs(t, + func(ctx context.Context, network *Network, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + // Return "contract not found" error to trigger fallback via ShouldFallbackMulticall3 + return nil, common.NewErrEndpointExecutionException(errors.New("contract not found")) + }, + func(ctx context.Context, project *PreparedProject, network *Network, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + // Panic in the fallback forward - this should be recovered + panic("test panic in fallback forward") + }, + nil, + ) + + // The function should not crash - panic in fallback path should be recovered + handled, responses := runHandle(t, ctx, server, project, defaultBatchInfo(), validBatchRequests(t), nil) + require.True(t, handled) + // We expect error responses due to the panic + require.Len(t, responses, 2) + for _, resp := range responses { + require.NotNil(t, resp, "response should not be nil after panic recovery") + } + }) +} + +func TestHandleEthCallBatchAggregation_DetectEthCallBatchInfo_EmptyParams(t *testing.T) { + // Empty params in eth_call is valid - the block param defaults to "latest". + // This test verifies that empty params requests are handled correctly. + requests := []json.RawMessage{ + json.RawMessage(`{"jsonrpc":"2.0","id":1,"method":"eth_call","params":[]}`), + json.RawMessage(`{"jsonrpc":"2.0","id":2,"method":"eth_call","params":[]}`), + } + + // detectEthCallBatchInfo should return valid info with "latest" block ref + info, err := detectEthCallBatchInfo(requests, "evm", "123") + require.NoError(t, err) + require.NotNil(t, info, "empty params defaults to latest - should be valid for batching") + assert.Equal(t, "evm:123", info.networkId) + assert.Equal(t, "latest", info.blockRef) + assert.Equal(t, interface{}("latest"), info.blockParam) +} + func TestHandleEthCallBatchAggregation_CacheHits(t *testing.T) { t.Run("all cached - no multicall3 call", func(t *testing.T) { cfg := baseBatchConfig() From 24d1fbdc56fe036d808c81e62b968fa2460443ae Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 16 Jan 2026 17:02:28 +0100 Subject: [PATCH 44/53] fix: use atomic counter in panic recovery test to fix race condition The test was using a non-atomic counter incremented by concurrent goroutines, causing flaky test behavior depending on scheduling. Co-Authored-By: Claude Opus 4.5 --- erpc/http_batch_eth_call_forward_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/erpc/http_batch_eth_call_forward_test.go b/erpc/http_batch_eth_call_forward_test.go index 2487882c2..871c715cc 100644 --- a/erpc/http_batch_eth_call_forward_test.go +++ b/erpc/http_batch_eth_call_forward_test.go @@ -3,6 +3,7 @@ package erpc import ( "context" "errors" + "sync/atomic" "testing" "time" @@ -59,10 +60,10 @@ func TestForwardEthCallBatchCandidates(t *testing.T) { t.Run("panic recovery in forward goroutine", func(t *testing.T) { responses := make([]interface{}, 2) - callCount := 0 + var callCount int32 forwardBatchProject = func(ctx context.Context, project *PreparedProject, network *Network, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { - callCount++ - if callCount == 1 { + count := atomic.AddInt32(&callCount, 1) + if count == 1 { panic("test panic in forward") } return common.NewNormalizedResponse(), nil From be26f0a1d407cc762fe12cade98bcb8b306abaf7 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 16 Jan 2026 17:47:24 +0100 Subject: [PATCH 45/53] ci: increase test workflow timeout to 30 minutes The Gosec scanner step was getting cancelled due to the 20-minute timeout after tests completed successfully. Co-Authored-By: Claude Opus 4.5 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ae50182dd..2b24db077 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,7 @@ permissions: jobs: units: runs-on: "${{ github.repository_owner == 'erpc' && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-24.04' }}" - timeout-minutes: 20 + timeout-minutes: 30 steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 From 89f5a806c8428710d1b3ca5b6d6f79b0ac48bb78 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 19 Jan 2026 11:01:13 +0100 Subject: [PATCH 46/53] fix: address PR review issues for multicall3 batching - Add missing config fields to user docs (safetyMarginMs, onlyIfPending, cachePerCall, allowPendingTagBatching) in networks.mdx - Clarify Multicall3 scope in batch.mdx - works across ALL entrypoints (HTTP single, HTTP batch, gRPC), not just JSON-RPC batch requests - Add metric for tight deadline bypass using MetricMulticall3QueueOverflowTotal with reason "deadline_too_tight" - Remove duplicate CacheDal() method in networks.go (keep Cache() for interface) - Use null byte separator in BatcherManager key to prevent collision risk - Apply code formatting (make fmt) Note: HIGH priority issue "Batch.mu exposed" was a false positive - the mu field is already lowercase (unexported) and only accessible within the evm package. Co-Authored-By: Claude Opus 4.5 --- architecture/evm/eth_call_test.go | 14 +- architecture/evm/multicall3_batcher.go | 1 + architecture/evm/multicall3_manager.go | 5 +- architecture/evm/multicall3_test.go | 4 +- common/config.go | 15 +- docs/pages/config/projects/networks.mdx | 21 ++ docs/pages/operation/batch.mdx | 11 +- erpc/http_batch_eth_call.go | 4 +- erpc/http_server.go | 260 ++++++++++++------------ erpc/networks.go | 39 +++- erpc/networks_registry.go | 1 + upstream/registry_test.go | 20 +- 12 files changed, 222 insertions(+), 173 deletions(-) diff --git a/architecture/evm/eth_call_test.go b/architecture/evm/eth_call_test.go index a4086d257..a3f294b28 100644 --- a/architecture/evm/eth_call_test.go +++ b/architecture/evm/eth_call_test.go @@ -22,12 +22,14 @@ type mockNetworkForEthCall struct { callCount int } -func (m *mockNetworkForEthCall) Id() string { return m.networkId } -func (m *mockNetworkForEthCall) Label() string { return m.networkId } -func (m *mockNetworkForEthCall) ProjectId() string { return m.projectId } -func (m *mockNetworkForEthCall) Architecture() common.NetworkArchitecture { return common.ArchitectureEvm } -func (m *mockNetworkForEthCall) Config() *common.NetworkConfig { return m.cfg } -func (m *mockNetworkForEthCall) Logger() *zerolog.Logger { return nil } +func (m *mockNetworkForEthCall) Id() string { return m.networkId } +func (m *mockNetworkForEthCall) Label() string { return m.networkId } +func (m *mockNetworkForEthCall) ProjectId() string { return m.projectId } +func (m *mockNetworkForEthCall) Architecture() common.NetworkArchitecture { + return common.ArchitectureEvm +} +func (m *mockNetworkForEthCall) Config() *common.NetworkConfig { return m.cfg } +func (m *mockNetworkForEthCall) Logger() *zerolog.Logger { return nil } func (m *mockNetworkForEthCall) GetMethodMetrics(method string) common.TrackedMetrics { return nil } func (m *mockNetworkForEthCall) Forward(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { m.mu.Lock() diff --git a/architecture/evm/multicall3_batcher.go b/architecture/evm/multicall3_batcher.go index 9a01a5b1c..d82ea24a9 100644 --- a/architecture/evm/multicall3_batcher.go +++ b/architecture/evm/multicall3_batcher.go @@ -125,6 +125,7 @@ func (b *Batcher) Enqueue(ctx context.Context, key BatchingKey, req *common.Norm // Check if deadline is too tight (only if there's a deadline) minWait := time.Duration(b.cfg.MinWaitMs) * time.Millisecond if hasDeadline && deadline.Before(now.Add(minWait)) { + telemetry.MetricMulticall3QueueOverflowTotal.WithLabelValues(key.ProjectId, key.NetworkId, "deadline_too_tight").Inc() b.logBypass(key, "deadline_too_tight") return nil, true, nil } diff --git a/architecture/evm/multicall3_manager.go b/architecture/evm/multicall3_manager.go index 0d6f2465f..44a49b05f 100644 --- a/architecture/evm/multicall3_manager.go +++ b/architecture/evm/multicall3_manager.go @@ -28,7 +28,8 @@ func NewBatcherManager() *BatcherManager { // Returns nil if batching is disabled (cfg is nil or cfg.Enabled is false). // The logger parameter is optional (can be nil) - if nil, debug logging is disabled. func (m *BatcherManager) GetOrCreate(projectId, networkId string, cfg *common.Multicall3AggregationConfig, forwarder Forwarder, logger *zerolog.Logger) *Batcher { - key := projectId + "|" + networkId + // Use null byte separator to prevent key collisions from field values containing common separators + key := projectId + "\x00" + networkId m.mu.RLock() if b, ok := m.batchers[key]; ok { @@ -56,7 +57,7 @@ func (m *BatcherManager) GetOrCreate(projectId, networkId string, cfg *common.Mu // Get returns the batcher for a project+network, or nil if not exists. func (m *BatcherManager) Get(projectId, networkId string) *Batcher { - key := projectId + "|" + networkId + key := projectId + "\x00" + networkId m.mu.RLock() defer m.mu.RUnlock() return m.batchers[key] diff --git a/architecture/evm/multicall3_test.go b/architecture/evm/multicall3_test.go index 2fd49d80e..e1875d969 100644 --- a/architecture/evm/multicall3_test.go +++ b/architecture/evm/multicall3_test.go @@ -370,8 +370,8 @@ func TestDecodeMulticall3Aggregate3Result_Errors(t *testing.T) { // offset=32, then count=maxInt/16 (which when *32 would overflow) data: func() []byte { d := make([]byte, 96) - copy(d[0:32], encodeUint64(32)) // offset to array - copy(d[32:64], encodeUint64(0x7FFFFFFF)) // huge count that exceeds data + copy(d[0:32], encodeUint64(32)) // offset to array + copy(d[32:64], encodeUint64(0x7FFFFFFF)) // huge count that exceeds data return d }(), }, diff --git a/common/config.go b/common/config.go index 33d4d8343..591a2aff3 100644 --- a/common/config.go +++ b/common/config.go @@ -927,13 +927,14 @@ func (c *EvmUpstreamConfig) Copy() *EvmUpstreamConfig { } type FailsafeConfig struct { - MatchMethod string `yaml:"matchMethod,omitempty" json:"matchMethod"` - MatchFinality []DataFinalityState `yaml:"matchFinality,omitempty" json:"matchFinality"` - Retry *RetryPolicyConfig `yaml:"retry" json:"retry"` - CircuitBreaker *CircuitBreakerPolicyConfig `yaml:"circuitBreaker" json:"circuitBreaker"` - Timeout *TimeoutPolicyConfig `yaml:"timeout" json:"timeout"` - Hedge *HedgePolicyConfig `yaml:"hedge" json:"hedge"` - Consensus *ConsensusPolicyConfig `yaml:"consensus" json:"consensus"` + MatchMethod string `yaml:"matchMethod,omitempty" json:"matchMethod"` + MatchFinality []DataFinalityState `yaml:"matchFinality,omitempty" json:"matchFinality"` + MatchUpstreamGroup string `yaml:"matchUpstreamGroup,omitempty" json:"matchUpstreamGroup"` + Retry *RetryPolicyConfig `yaml:"retry" json:"retry"` + CircuitBreaker *CircuitBreakerPolicyConfig `yaml:"circuitBreaker" json:"circuitBreaker"` + Timeout *TimeoutPolicyConfig `yaml:"timeout" json:"timeout"` + Hedge *HedgePolicyConfig `yaml:"hedge" json:"hedge"` + Consensus *ConsensusPolicyConfig `yaml:"consensus" json:"consensus"` } func (c *FailsafeConfig) Copy() *FailsafeConfig { diff --git a/docs/pages/config/projects/networks.mdx b/docs/pages/config/projects/networks.mdx index 24a925a65..86de2cb66 100644 --- a/docs/pages/config/projects/networks.mdx +++ b/docs/pages/config/projects/networks.mdx @@ -54,6 +54,13 @@ projects: # Minimum wait time in milliseconds before flushing a batch. # DEFAULT: 2 minWaitMs: 2 + # Safety margin in milliseconds subtracted from request deadlines when computing flush time. + # DEFAULT: min(2, minWaitMs) + safetyMarginMs: 2 + # If true, only batch if another request is already waiting. Avoids adding latency + # when requests arrive sporadically. + # DEFAULT: false + onlyIfPending: false # Maximum number of calls per batch. # DEFAULT: 20 maxCalls: 20 @@ -66,10 +73,16 @@ projects: # Maximum number of pending batches being processed. # DEFAULT: 200 maxPendingBatches: 200 + # Enable per-call cache writes after successful Multicall3 response. + # DEFAULT: true + cachePerCall: true # Allow batching requests from different users together. # Set to false to isolate user requests into separate batches. # DEFAULT: true allowCrossUserBatching: true + # Allow batching calls with "pending" block tag. + # DEFAULT: false + allowPendingTagBatching: false # (OPTIONAL) A friendly alias for this network. This allows you to reference the network using the alias # instead of the architecture/chainId format. For example, instead of using /main/evm/1, you can use /main/ethereum. @@ -215,6 +228,10 @@ export default createConfig({ windowMs: 25, // Minimum wait time in milliseconds before flushing a batch. DEFAULT: 2 minWaitMs: 2, + // Safety margin in milliseconds subtracted from request deadlines. DEFAULT: min(2, minWaitMs) + safetyMarginMs: 2, + // If true, only batch if another request is already waiting. DEFAULT: false + onlyIfPending: false, // Maximum number of calls per batch. DEFAULT: 20 maxCalls: 20, // Maximum total calldata size in bytes for a batch. DEFAULT: 64000 @@ -223,8 +240,12 @@ export default createConfig({ maxQueueSize: 1000, // Maximum number of pending batches being processed. DEFAULT: 200 maxPendingBatches: 200, + // Enable per-call cache writes after successful Multicall3. DEFAULT: true + cachePerCall: true, // Allow batching requests from different users together. DEFAULT: true allowCrossUserBatching: true, + // Allow batching calls with "pending" block tag. DEFAULT: false + allowPendingTagBatching: false, }, }, diff --git a/docs/pages/operation/batch.mdx b/docs/pages/operation/batch.mdx index d75e4809c..a4b5c8c45 100644 --- a/docs/pages/operation/batch.mdx +++ b/docs/pages/operation/batch.mdx @@ -150,18 +150,19 @@ curl --location 'http://localhost:4000/main' \ ## Multicall3 Aggregation -For EVM networks, eRPC can automatically aggregate batched `eth_call` requests into a single [Multicall3](https://www.multicall3.com/) contract call. This significantly reduces latency when clients send JSON-RPC batches containing multiple `eth_call` methods targeting the same block. +For EVM networks, eRPC can automatically aggregate `eth_call` requests into a single [Multicall3](https://www.multicall3.com/) contract call. This batching operates at the network level across **all entrypoints** (HTTP single requests, HTTP batch requests, and gRPC), not just JSON-RPC batch requests. ### How it works -When eRPC receives a JSON-RPC batch request: +Multicall3 aggregation collects `eth_call` requests that share the same block tag and batches them together: -1. **Detection**: eRPC identifies `eth_call` requests that share the same block tag (e.g., `latest`, `0x123456`) +1. **Collection**: When multiple `eth_call` requests arrive within a time window (default: 25ms) targeting the same block tag (e.g., `latest`, `0x123456`), they are grouped together 2. **Aggregation**: These calls are combined into a single `aggregate3` call to the Multicall3 contract 3. **Execution**: One upstream request is made instead of many -4. **Response mapping**: Results are decoded and mapped back to the original request IDs +4. **Response mapping**: Results are decoded and mapped back to the original requests +5. **Deduplication**: Identical calls (same target + calldata + block) within a batch are deduplicated and share the response -This is particularly beneficial for applications that query multiple contract states (e.g., ERC20 balances, token metadata) in a single batch. +This is particularly beneficial for applications that query multiple contract states (e.g., ERC20 balances, token metadata), reducing upstream RPC calls significantly even when clients send individual requests. Multicall3 is deployed at the same address (`0xcA11bde05977b3631167028862bE2a173976CA11`) on most EVM chains. diff --git a/erpc/http_batch_eth_call.go b/erpc/http_batch_eth_call.go index fe528508e..dc43cf663 100644 --- a/erpc/http_batch_eth_call.go +++ b/erpc/http_batch_eth_call.go @@ -293,7 +293,7 @@ func (s *HttpServer) handleEthCallBatchAggregation( } nq.SetNetwork(network) - nq.SetCacheDal(network.CacheDal()) + nq.SetCacheDal(network.Cache()) nq.ApplyDirectiveDefaults(network.Config().DirectiveDefaults) nq.EnrichFromHttp(headers, queryArgs, uaMode) rlg.Trace().Interface("directives", nq.Directives()).Msgf("applied request directives") @@ -331,7 +331,7 @@ func (s *HttpServer) handleEthCallBatchAggregation( // Check cache for individual requests before aggregating // Respects skip-cache-read directive - requests with this directive skip cache probe - cacheDal := network.CacheDal() + cacheDal := network.Cache() var uncachedCandidates []ethCallBatchCandidate if cacheDal != nil && !cacheDal.IsObjectNull() { uncachedCandidates = make([]ethCallBatchCandidate, 0, len(candidates)) diff --git a/erpc/http_server.go b/erpc/http_server.go index 3fa42b0c8..377115324 100644 --- a/erpc/http_server.go +++ b/erpc/http_server.go @@ -410,8 +410,8 @@ func (s *HttpServer) createRequestHandler() http.Handler { batchInfo, detectErr := detectEthCallBatchInfo(requests, architecture, chainId) if detectErr != nil { lg.Info().Err(detectErr). - Int("requestCount", len(requests)). - Msg("eth_call batch detection failed, processing individually") + Int("requestCount", len(requests)). + Msg("eth_call batch detection failed, processing individually") } if batchInfo != nil && isMulticall3AggregationEnabled(project, batchInfo.networkId) { batchHandled = s.handleEthCallBatchAggregation( @@ -433,163 +433,163 @@ func (s *HttpServer) createRequestHandler() http.Handler { for i, reqBody := range requests { wg.Add(1) go func(index int, rawReq json.RawMessage, headers http.Header, queryArgs map[string][]string) { - defer func() { - defer wg.Done() - if rec := recover(); rec != nil { - telemetry.MetricUnexpectedPanicTotal.WithLabelValues( - "request-handler", - fmt.Sprintf("project:%s network:%s", architecture, chainId), - common.ErrorFingerprint(rec), - ).Inc() - lg.Error(). - Interface("panic", rec). - Str("stack", string(debug.Stack())). - Msgf("unexpected server panic on per-request handler") - err := fmt.Errorf("unexpected server panic on per-request handler: %v stack: %s", rec, string(debug.Stack())) - responses[index] = processErrorBody(&lg, &startedAt, nil, err, s.serverCfg.IncludeErrorDetails) - } - }() + defer func() { + defer wg.Done() + if rec := recover(); rec != nil { + telemetry.MetricUnexpectedPanicTotal.WithLabelValues( + "request-handler", + fmt.Sprintf("project:%s network:%s", architecture, chainId), + common.ErrorFingerprint(rec), + ).Inc() + lg.Error(). + Interface("panic", rec). + Str("stack", string(debug.Stack())). + Msgf("unexpected server panic on per-request handler") + err := fmt.Errorf("unexpected server panic on per-request handler: %v stack: %s", rec, string(debug.Stack())) + responses[index] = processErrorBody(&lg, &startedAt, nil, err, s.serverCfg.IncludeErrorDetails) + } + }() - nq := common.NewNormalizedRequest(rawReq) - // Help GC: drop reference to the rawReq slice copy in the parent slice as soon as possible - rawReq = nil - requestCtx := common.StartRequestSpan(httpCtx, nq) + nq := common.NewNormalizedRequest(rawReq) + // Help GC: drop reference to the rawReq slice copy in the parent slice as soon as possible + rawReq = nil + requestCtx := common.StartRequestSpan(httpCtx, nq) - // Resolve and set real client IP before any rate limiting/auth checks - clientIP := s.resolveRealClientIP(r) - nq.SetClientIP(clientIP) + // Resolve and set real client IP before any rate limiting/auth checks + clientIP := s.resolveRealClientIP(r) + nq.SetClientIP(clientIP) - // Validate the raw JSON-RPC payload early - if err := nq.Validate(); err != nil { - responses[index] = processErrorBody(&lg, &startedAt, nq, err, &common.TRUE) - common.EndRequestSpan(requestCtx, nil, responses[index]) - return - } + // Validate the raw JSON-RPC payload early + if err := nq.Validate(); err != nil { + responses[index] = processErrorBody(&lg, &startedAt, nq, err, &common.TRUE) + common.EndRequestSpan(requestCtx, nil, responses[index]) + return + } - method, _ := nq.Method() - rlg := lg.With().Str("method", method).Logger() + method, _ := nq.Method() + rlg := lg.With().Str("method", method).Logger() - var ap *auth.AuthPayload - var err error + var ap *auth.AuthPayload + var err error - if project != nil { - ap, err = auth.NewPayloadFromHttp(method, r.RemoteAddr, headers, queryArgs) - } else if isAdmin { - ap, err = auth.NewPayloadFromHttp(method, r.RemoteAddr, headers, queryArgs) - } - if err != nil { - responses[index] = processErrorBody(&rlg, &startedAt, nq, err, &common.TRUE) - common.EndRequestSpan(requestCtx, nil, err) - return - } - - if isAdmin { - _, err := s.erpc.AdminAuthenticate(requestCtx, nq, method, ap) - if err != nil { - responses[index] = processErrorBody(&rlg, &startedAt, nq, err, &common.TRUE) - common.EndRequestSpan(requestCtx, nil, err) - return + if project != nil { + ap, err = auth.NewPayloadFromHttp(method, r.RemoteAddr, headers, queryArgs) + } else if isAdmin { + ap, err = auth.NewPayloadFromHttp(method, r.RemoteAddr, headers, queryArgs) } - } else { - user, err := project.AuthenticateConsumer(requestCtx, nq, method, ap) if err != nil { - responses[index] = processErrorBody(&rlg, &startedAt, nq, err, s.serverCfg.IncludeErrorDetails) + responses[index] = processErrorBody(&rlg, &startedAt, nq, err, &common.TRUE) common.EndRequestSpan(requestCtx, nil, err) return } - if user != nil { - rlg = rlg.With().Str("userId", user.Id).Logger() - } - nq.SetUser(user) - } - if isAdmin { - if s.adminCfg != nil { - resp, err := s.erpc.AdminHandleRequest(requestCtx, nq) + if isAdmin { + _, err := s.erpc.AdminAuthenticate(requestCtx, nq, method, ap) if err != nil { responses[index] = processErrorBody(&rlg, &startedAt, nq, err, &common.TRUE) common.EndRequestSpan(requestCtx, nil, err) return } - responses[index] = resp - common.EndRequestSpan(requestCtx, resp, nil) - return } else { - responses[index] = processErrorBody( - &rlg, - &startedAt, - nq, - common.NewErrAuthUnauthorized( - "", - "admin is not enabled for this project", - ), - s.serverCfg.IncludeErrorDetails, - ) - common.EndRequestSpan(requestCtx, nil, err) - return + user, err := project.AuthenticateConsumer(requestCtx, nq, method, ap) + if err != nil { + responses[index] = processErrorBody(&rlg, &startedAt, nq, err, s.serverCfg.IncludeErrorDetails) + common.EndRequestSpan(requestCtx, nil, err) + return + } + if user != nil { + rlg = rlg.With().Str("userId", user.Id).Logger() + } + nq.SetUser(user) } - } - - var networkId string - if architecture == "" || chainId == "" { - var req map[string]interface{} - if err := common.SonicCfg.Unmarshal(rawReq, &req); err != nil { - responses[index] = processErrorBody(&rlg, &startedAt, nq, common.NewErrInvalidRequest(err), &common.TRUE) - common.EndRequestSpan(requestCtx, nil, err) - return + if isAdmin { + if s.adminCfg != nil { + resp, err := s.erpc.AdminHandleRequest(requestCtx, nq) + if err != nil { + responses[index] = processErrorBody(&rlg, &startedAt, nq, err, &common.TRUE) + common.EndRequestSpan(requestCtx, nil, err) + return + } + responses[index] = resp + common.EndRequestSpan(requestCtx, resp, nil) + return + } else { + responses[index] = processErrorBody( + &rlg, + &startedAt, + nq, + common.NewErrAuthUnauthorized( + "", + "admin is not enabled for this project", + ), + s.serverCfg.IncludeErrorDetails, + ) + common.EndRequestSpan(requestCtx, nil, err) + return + } } - if networkIdFromBody, ok := req["networkId"].(string); ok { - networkId = networkIdFromBody - parts := strings.Split(networkId, ":") - if len(parts) == 2 { - architecture = parts[0] - chainId = parts[1] + + var networkId string + + if architecture == "" || chainId == "" { + var req map[string]interface{} + if err := common.SonicCfg.Unmarshal(rawReq, &req); err != nil { + responses[index] = processErrorBody(&rlg, &startedAt, nq, common.NewErrInvalidRequest(err), &common.TRUE) + common.EndRequestSpan(requestCtx, nil, err) + return } + if networkIdFromBody, ok := req["networkId"].(string); ok { + networkId = networkIdFromBody + parts := strings.Split(networkId, ":") + if len(parts) == 2 { + architecture = parts[0] + chainId = parts[1] + } + } + } else { + networkId = fmt.Sprintf("%s:%s", architecture, chainId) } - } else { - networkId = fmt.Sprintf("%s:%s", architecture, chainId) - } - if architecture == "" || chainId == "" { - responses[index] = processErrorBody(&rlg, &startedAt, nq, common.NewErrInvalidRequest(fmt.Errorf( - "architecture and chain must be provided in URL (for example //evm/42161) or in request body (for example \"networkId\":\"evm:42161\") or configureed via domain aliasing", - )), s.serverCfg.IncludeErrorDetails) - common.EndRequestSpan(requestCtx, nil, err) - return - } + if architecture == "" || chainId == "" { + responses[index] = processErrorBody(&rlg, &startedAt, nq, common.NewErrInvalidRequest(fmt.Errorf( + "architecture and chain must be provided in URL (for example //evm/42161) or in request body (for example \"networkId\":\"evm:42161\") or configureed via domain aliasing", + )), s.serverCfg.IncludeErrorDetails) + common.EndRequestSpan(requestCtx, nil, err) + return + } - nw, err := project.GetNetwork(httpCtx, networkId) - if err != nil { - responses[index] = processErrorBody(&rlg, &startedAt, nq, err, s.serverCfg.IncludeErrorDetails) - common.EndRequestSpan(requestCtx, nil, err) - return - } - nq.SetNetwork(nw) + nw, err := project.GetNetwork(httpCtx, networkId) + if err != nil { + responses[index] = processErrorBody(&rlg, &startedAt, nq, err, s.serverCfg.IncludeErrorDetails) + common.EndRequestSpan(requestCtx, nil, err) + return + } + nq.SetNetwork(nw) - nq.ApplyDirectiveDefaults(nw.Config().DirectiveDefaults) - // Configure how to store User-Agent (raw vs simplified) based on project config - uaMode := common.UserAgentTrackingModeSimplified - if project != nil && project.Config.UserAgentMode != "" { - uaMode = project.Config.UserAgentMode - } - nq.EnrichFromHttp(headers, queryArgs, uaMode) - rlg.Trace().Interface("directives", nq.Directives()).Msgf("applied request directives") + nq.ApplyDirectiveDefaults(nw.Config().DirectiveDefaults) + // Configure how to store User-Agent (raw vs simplified) based on project config + uaMode := common.UserAgentTrackingModeSimplified + if project != nil && project.Config.UserAgentMode != "" { + uaMode = project.Config.UserAgentMode + } + nq.EnrichFromHttp(headers, queryArgs, uaMode) + rlg.Trace().Interface("directives", nq.Directives()).Msgf("applied request directives") - resp, err := project.Forward(requestCtx, networkId, nq) - if err != nil { - // If an error occurred but a response was produced (e.g., lastValidResponse), - // release it now since we are not going to write it. - if resp != nil { - go resp.Release() + resp, err := project.Forward(requestCtx, networkId, nq) + if err != nil { + // If an error occurred but a response was produced (e.g., lastValidResponse), + // release it now since we are not going to write it. + if resp != nil { + go resp.Release() + } + responses[index] = processErrorBody(&rlg, &startedAt, nq, err, s.serverCfg.IncludeErrorDetails) + common.EndRequestSpan(requestCtx, nil, err) + return } - responses[index] = processErrorBody(&rlg, &startedAt, nq, err, s.serverCfg.IncludeErrorDetails) - common.EndRequestSpan(requestCtx, nil, err) - return - } - responses[index] = resp - common.EndRequestSpan(requestCtx, resp, nil) + responses[index] = resp + common.EndRequestSpan(requestCtx, resp, nil) }(i, reqBody, headers, queryArgs) } diff --git a/erpc/networks.go b/erpc/networks.go index 449bf08f4..2ea22d1f5 100644 --- a/erpc/networks.go +++ b/erpc/networks.go @@ -26,6 +26,7 @@ import ( type FailsafeExecutor struct { method string finalities []common.DataFinalityState + upstreamGroup string executor failsafe.Executor[*common.NormalizedResponse] timeout *time.Duration consensusPolicyEnabled bool @@ -348,6 +349,16 @@ func (n *Network) Forward(ctx context.Context, req *common.NormalizedRequest) (* forwardSpan.SetAttributes(attribute.Bool("cache.hit", false)) } + // Get failsafe executor first to know if we need to filter upstreams by group + failsafeExecutor := n.getFailsafeExecutor(ctx, req) + if failsafeExecutor == nil { + err := errors.New("no failsafe executor found for this request") + if mlx != nil { + mlx.Close(ctx, nil, err) + } + return nil, err + } + _, upstreamSpan := common.StartDetailSpan(ctx, "GetSortedUpstreams") upsList, err := n.upstreamsRegistry.GetSortedUpstreams(ctx, n.networkId, method) upstreamSpan.SetAttributes(attribute.Int("upstreams.count", len(upsList))) @@ -370,6 +381,24 @@ func (n *Network) Forward(ctx context.Context, req *common.NormalizedRequest) (* return nil, err } + // Filter upstreams by group if the failsafe executor specifies a group + // This is primarily used for consensus policies that should only compare + // responses from a specific group of upstreams (e.g., public RPC endpoints) + if failsafeExecutor.upstreamGroup != "" { + filteredUpstreams := make([]common.Upstream, 0, len(upsList)) + for _, u := range upsList { + if cfg := u.Config(); cfg != nil && cfg.Group == failsafeExecutor.upstreamGroup { + filteredUpstreams = append(filteredUpstreams, u) + } + } + lg.Debug(). + Str("upstreamGroup", failsafeExecutor.upstreamGroup). + Int("originalCount", len(upsList)). + Int("filteredCount", len(filteredUpstreams)). + Msgf("filtered upstreams by group for failsafe policy") + upsList = filteredUpstreams + } + // Set upstreams on the request req.SetUpstreams(upsList) @@ -451,15 +480,11 @@ func (n *Network) Forward(ctx context.Context, req *common.NormalizedRequest) (* // This is the only way to pass additional values to failsafe policy executors context ectx := context.WithValue(ctx, common.RequestContextKey, req) - failsafeExecutor := n.getFailsafeExecutor(ctx, req) - if failsafeExecutor == nil { - return nil, errors.New("no failsafe executor found for this request") - } - // Add tracing for which failsafe policy was selected forwardSpan.SetAttributes( attribute.String("failsafe.matched_method", failsafeExecutor.method), attribute.String("failsafe.matched_finalities", fmt.Sprintf("%v", failsafeExecutor.finalities)), + attribute.String("failsafe.matched_upstream_group", failsafeExecutor.upstreamGroup), ) // Track time from failsafe executor start to first callback invocation @@ -891,10 +916,6 @@ func (n *Network) Config() *common.NetworkConfig { return n.cfg } -func (n *Network) CacheDal() common.CacheDAL { - return n.cacheDal -} - func (n *Network) Cache() common.CacheDAL { return n.cacheDal } diff --git a/erpc/networks_registry.go b/erpc/networks_registry.go index 53910c152..e4e00abf9 100644 --- a/erpc/networks_registry.go +++ b/erpc/networks_registry.go @@ -111,6 +111,7 @@ func NewNetwork( failsafeExecutors = append(failsafeExecutors, &FailsafeExecutor{ method: method, finalities: fsCfg.MatchFinality, + upstreamGroup: fsCfg.MatchUpstreamGroup, executor: failsafe.NewExecutor(policyArray...), timeout: timeoutDuration, consensusPolicyEnabled: fsCfg.Consensus != nil, diff --git a/upstream/registry_test.go b/upstream/registry_test.go index 638461536..365a25f39 100644 --- a/upstream/registry_test.go +++ b/upstream/registry_test.go @@ -1453,28 +1453,28 @@ func TestUpstreamsRegistry_CalculateScoreEdgeCases(t *testing.T) { normMisbehaviorRate float64 }{ { - name: "All zeros", - normTotalRequests: 0, normRespLatency: 0, normErrorRate: 0, + name: "All zeros", + normTotalRequests: 0, normRespLatency: 0, normErrorRate: 0, normThrottledRate: 0, normBlockHeadLag: 0, normFinalizationLag: 0, normMisbehaviorRate: 0, }, { - name: "All ones", - normTotalRequests: 1, normRespLatency: 1, normErrorRate: 1, + name: "All ones", + normTotalRequests: 1, normRespLatency: 1, normErrorRate: 1, normThrottledRate: 1, normBlockHeadLag: 1, normFinalizationLag: 1, normMisbehaviorRate: 1, }, { - name: "Mixed values", - normTotalRequests: 0.5, normRespLatency: 0.3, normErrorRate: 0.1, + name: "Mixed values", + normTotalRequests: 0.5, normRespLatency: 0.3, normErrorRate: 0.1, normThrottledRate: 0.2, normBlockHeadLag: 0.4, normFinalizationLag: 0.05, normMisbehaviorRate: 0.01, }, { - name: "Boundary high", - normTotalRequests: 0.999, normRespLatency: 0.999, normErrorRate: 0.999, + name: "Boundary high", + normTotalRequests: 0.999, normRespLatency: 0.999, normErrorRate: 0.999, normThrottledRate: 0.999, normBlockHeadLag: 0.999, normFinalizationLag: 0.999, normMisbehaviorRate: 0.999, }, { - name: "Boundary low", - normTotalRequests: 0.001, normRespLatency: 0.001, normErrorRate: 0.001, + name: "Boundary low", + normTotalRequests: 0.001, normRespLatency: 0.001, normErrorRate: 0.001, normThrottledRate: 0.001, normBlockHeadLag: 0.001, normFinalizationLag: 0.001, normMisbehaviorRate: 0.001, }, } From 2e665e9b38d6c74c782acf1a63c2012ee0448b89 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 19 Jan 2026 11:13:34 +0100 Subject: [PATCH 47/53] fix: address remaining PR review items - Add explicit error when upstream group filter yields zero upstreams (prevents confusing "upstreams exhausted" error on misconfiguration) - Add tracing attributes for post-filter upstream count and group - Add MetricMulticall3PanicTotal metric for panic recovery tracking (recorded regardless of logger availability) - Add MetricMulticall3CacheWriteDroppedTotal metric for cache write backpressure monitoring Test coverage already exists for: - Concurrent flush + enqueue: TestBatcherConcurrentFlush - Cache write failures: TestBatcher_CacheWriteError_DoesNotFailRequest - Panic recovery: TestBatcher_ScheduleFlush_PanicRecovery Co-Authored-By: Claude Opus 4.5 --- architecture/evm/multicall3_batcher.go | 5 ++++- erpc/http_batch_eth_call.go | 1 + erpc/networks.go | 22 ++++++++++++++++++++++ telemetry/metrics.go | 12 ++++++++++++ 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/architecture/evm/multicall3_batcher.go b/architecture/evm/multicall3_batcher.go index d82ea24a9..79248b708 100644 --- a/architecture/evm/multicall3_batcher.go +++ b/architecture/evm/multicall3_batcher.go @@ -262,7 +262,10 @@ func (b *Batcher) scheduleFlush(keyStr string, batch *Batch) { defer b.wg.Done() defer func() { if r := recover(); r != nil { - // Log panic with stack trace and deliver errors to all waiting entries + // Record metric regardless of logger availability + telemetry.MetricMulticall3PanicTotal.WithLabelValues(batch.Key.ProjectId, batch.Key.NetworkId, "scheduleFlush").Inc() + + // Log panic with stack trace if logger available if b.logger != nil { b.logger.Error(). Str("panic", fmt.Sprintf("%v", r)). diff --git a/erpc/http_batch_eth_call.go b/erpc/http_batch_eth_call.go index dc43cf663..4dfaeb400 100644 --- a/erpc/http_batch_eth_call.go +++ b/erpc/http_batch_eth_call.go @@ -579,6 +579,7 @@ func (s *HttpServer) handleEthCallBatchAggregation( }(nr, cand.req, cand.ctx, cand.logger) default: // Semaphore full - skip cache write to avoid unbounded goroutine growth + telemetry.MetricMulticall3CacheWriteDroppedTotal.WithLabelValues(projectId, batchInfo.networkId).Inc() cand.logger.Debug().Msg("skipping multicall3 per-call cache write due to backpressure") } } diff --git a/erpc/networks.go b/erpc/networks.go index 2ea22d1f5..8103d2a2a 100644 --- a/erpc/networks.go +++ b/erpc/networks.go @@ -396,7 +396,29 @@ func (n *Network) Forward(ctx context.Context, req *common.NormalizedRequest) (* Int("originalCount", len(upsList)). Int("filteredCount", len(filteredUpstreams)). Msgf("filtered upstreams by group for failsafe policy") + if len(filteredUpstreams) == 0 { + err := common.NewErrFailsafeConfiguration( + fmt.Errorf("no upstreams match the configured group '%s' for failsafe policy (had %d upstreams before filtering, method=%s)", + failsafeExecutor.upstreamGroup, len(upsList), method), + map[string]interface{}{ + "upstreamGroup": failsafeExecutor.upstreamGroup, + "originalCount": len(upsList), + "method": method, + "failsafeMethod": failsafeExecutor.method, + }, + ) + if mlx != nil { + mlx.Close(ctx, nil, err) + } + return nil, err + } upsList = filteredUpstreams + + // Update tracing to reflect post-filter state + forwardSpan.SetAttributes( + attribute.Int("upstreams.filtered_count", len(upsList)), + attribute.String("upstreams.filter_group", failsafeExecutor.upstreamGroup), + ) } // Set upstreams on the request diff --git a/telemetry/metrics.go b/telemetry/metrics.go index 7309c0303..6a1d72850 100644 --- a/telemetry/metrics.go +++ b/telemetry/metrics.go @@ -486,6 +486,18 @@ var ( Name: "multicall3_abandoned_total", Help: "Total number of multicall3 batch results not delivered because caller context was cancelled.", }, []string{"project", "network"}) + + MetricMulticall3PanicTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "multicall3_panic_total", + Help: "Total number of panics recovered in multicall3 batch processing.", + }, []string{"project", "network", "location"}) + + MetricMulticall3CacheWriteDroppedTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "multicall3_cache_write_dropped_total", + Help: "Total number of multicall3 per-call cache writes dropped due to backpressure.", + }, []string{"project", "network"}) ) var DefaultHistogramBuckets = []float64{ From 8bcdd1f3dd7ef5587f6a4054f1aef34b98d8e062 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 19 Jan 2026 11:48:00 +0100 Subject: [PATCH 48/53] fix: UseUpstream directive now overrides matchUpstreamGroup filter - When a UseUpstream directive is present, skip the matchUpstreamGroup filtering to allow targeting specific upstreams outside the group - Add matchUpstreamGroup to TypeScript config types (FailsafeConfig) - Add regression tests for group filter / UseUpstream interaction Co-Authored-By: Claude Opus 4.5 --- erpc/networks.go | 7 +- erpc/networks_failsafe_test.go | 236 +++++++++++++++++++++++++++ typescript/config/lib/generated.d.ts | 1 + typescript/config/src/generated.ts | 1 + 4 files changed, 244 insertions(+), 1 deletion(-) diff --git a/erpc/networks.go b/erpc/networks.go index 8103d2a2a..967ca6b9e 100644 --- a/erpc/networks.go +++ b/erpc/networks.go @@ -384,7 +384,12 @@ func (n *Network) Forward(ctx context.Context, req *common.NormalizedRequest) (* // Filter upstreams by group if the failsafe executor specifies a group // This is primarily used for consensus policies that should only compare // responses from a specific group of upstreams (e.g., public RPC endpoints) - if failsafeExecutor.upstreamGroup != "" { + // Skip group filtering if UseUpstream directive is set - allows targeting any upstream for debugging + useUpstreamDirective := "" + if req.Directives() != nil { + useUpstreamDirective = req.Directives().UseUpstream + } + if failsafeExecutor.upstreamGroup != "" && useUpstreamDirective == "" { filteredUpstreams := make([]common.Upstream, 0, len(upsList)) for _, u := range upsList { if cfg := u.Config(); cfg != nil && cfg.Group == failsafeExecutor.upstreamGroup { diff --git a/erpc/networks_failsafe_test.go b/erpc/networks_failsafe_test.go index b458f24ae..9cf20ccfe 100644 --- a/erpc/networks_failsafe_test.go +++ b/erpc/networks_failsafe_test.go @@ -1335,3 +1335,239 @@ func TestGetFailsafeExecutor_OrderRespected(t *testing.T) { assert.Contains(t, executor4.finalities, common.DataFinalityStateUnknown) }) } + +func TestNetworkFailsafe_UpstreamGroupFilter(t *testing.T) { + t.Run("UseUpstreamDirective_OverridesGroupFilter", func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + + // Setup mock for upstream outside the group (rpc2 is NOT in "primary" group) + gock.New("http://rpc2.localhost"). + Post(""). + Filter(func(r *http.Request) bool { + body := util.SafeReadBody(r) + return strings.Contains(body, "eth_blockNumber") + }). + Times(1). + Reply(200). + JSON(map[string]interface{}{ + "jsonrpc": "2.0", + "id": 1, + "result": "0x100", + }) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Create network with group filter in failsafe config + upstreamConfigs := []*common.UpstreamConfig{ + { + Id: "rpc1", + Type: common.UpstreamTypeEvm, + Endpoint: "http://rpc1.localhost", + Evm: &common.EvmUpstreamConfig{ + ChainId: 123, + }, + Group: "primary", // In the group + }, + { + Id: "rpc2", + Type: common.UpstreamTypeEvm, + Endpoint: "http://rpc2.localhost", + Evm: &common.EvmUpstreamConfig{ + ChainId: 123, + }, + Group: "fallback", // NOT in the group + }, + } + + networkConfig := &common.NetworkConfig{ + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ + ChainId: 123, + }, + Failsafe: []*common.FailsafeConfig{ + { + MatchMethod: "*", + MatchUpstreamGroup: "primary", // Only allow "primary" group upstreams + Retry: &common.RetryPolicyConfig{ + MaxAttempts: 1, + }, + }, + }, + } + + rateLimitersRegistry, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + require.NoError(t, err) + + metricsTracker := health.NewTracker(&log.Logger, "test", time.Minute) + + vr := thirdparty.NewVendorsRegistry() + pr, err := thirdparty.NewProvidersRegistry(&log.Logger, vr, []*common.ProviderConfig{}, nil) + require.NoError(t, err) + + ssr, err := data.NewSharedStateRegistry(ctx, &log.Logger, &common.SharedStateConfig{ + Connector: &common.ConnectorConfig{ + Driver: "memory", + Memory: &common.MemoryConnectorConfig{ + MaxItems: 100_000, + MaxTotalSize: "1GB", + }, + }, + }) + require.NoError(t, err) + + upstreamsRegistry := upstream.NewUpstreamsRegistry( + ctx, + &log.Logger, + "test", + upstreamConfigs, + ssr, + rateLimitersRegistry, + vr, + pr, + nil, + metricsTracker, + time.Second, + nil, + ) + + upstreamsRegistry.Bootstrap(ctx) + + time.Sleep(100 * time.Millisecond) + + network, err := NewNetwork(ctx, &log.Logger, "test", networkConfig, rateLimitersRegistry, upstreamsRegistry, metricsTracker) + require.NoError(t, err) + + err = upstreamsRegistry.PrepareUpstreamsForNetwork(ctx, networkConfig.NetworkId()) + require.NoError(t, err) + + err = network.Bootstrap(ctx) + require.NoError(t, err) + + upstream.ReorderUpstreams(upstreamsRegistry) + + // Request WITH UseUpstream directive targeting rpc2 (outside group) + // Should succeed because UseUpstream overrides group filter + requestBytes := []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_blockNumber","params":[]}`) + req := common.NewNormalizedRequest(requestBytes) + req.SetNetwork(network) + if req.Directives() == nil { + req.SetDirectives(&common.RequestDirectives{}) + } + req.Directives().UseUpstream = "rpc2" // Target upstream outside the group + + resp, err := network.Forward(ctx, req) + + require.NoError(t, err) + require.NotNil(t, resp) + + jrr, err := resp.JsonRpcResponse() + require.NoError(t, err) + assert.Nil(t, jrr.Error) + + result, err := jrr.PeekStringByPath(ctx) + require.NoError(t, err) + assert.Equal(t, "0x100", result) + }) + + t.Run("EmptyGroupFilter_ReturnsConfigurationError", func(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Create network with group filter that matches NO upstreams + upstreamConfigs := []*common.UpstreamConfig{ + { + Id: "rpc1", + Type: common.UpstreamTypeEvm, + Endpoint: "http://rpc1.localhost", + Evm: &common.EvmUpstreamConfig{ + ChainId: 123, + }, + Group: "primary", + }, + } + + networkConfig := &common.NetworkConfig{ + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ + ChainId: 123, + }, + Failsafe: []*common.FailsafeConfig{ + { + MatchMethod: "*", + MatchUpstreamGroup: "nonexistent-group", // No upstreams in this group + Retry: &common.RetryPolicyConfig{ + MaxAttempts: 1, + }, + }, + }, + } + + rateLimitersRegistry, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + require.NoError(t, err) + + metricsTracker := health.NewTracker(&log.Logger, "test", time.Minute) + + vr := thirdparty.NewVendorsRegistry() + pr, err := thirdparty.NewProvidersRegistry(&log.Logger, vr, []*common.ProviderConfig{}, nil) + require.NoError(t, err) + + ssr, err := data.NewSharedStateRegistry(ctx, &log.Logger, &common.SharedStateConfig{ + Connector: &common.ConnectorConfig{ + Driver: "memory", + Memory: &common.MemoryConnectorConfig{ + MaxItems: 100_000, + MaxTotalSize: "1GB", + }, + }, + }) + require.NoError(t, err) + + upstreamsRegistry := upstream.NewUpstreamsRegistry( + ctx, + &log.Logger, + "test", + upstreamConfigs, + ssr, + rateLimitersRegistry, + vr, + pr, + nil, + metricsTracker, + time.Second, + nil, + ) + + upstreamsRegistry.Bootstrap(ctx) + + time.Sleep(100 * time.Millisecond) + + network, err := NewNetwork(ctx, &log.Logger, "test", networkConfig, rateLimitersRegistry, upstreamsRegistry, metricsTracker) + require.NoError(t, err) + + err = upstreamsRegistry.PrepareUpstreamsForNetwork(ctx, networkConfig.NetworkId()) + require.NoError(t, err) + + err = network.Bootstrap(ctx) + require.NoError(t, err) + + upstream.ReorderUpstreams(upstreamsRegistry) + + // Request WITHOUT UseUpstream directive - should fail with configuration error + requestBytes := []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_blockNumber","params":[]}`) + req := common.NewNormalizedRequest(requestBytes) + req.SetNetwork(network) + + _, err = network.Forward(ctx, req) + + require.Error(t, err) + assert.Contains(t, err.Error(), "no upstreams match the configured group") + assert.Contains(t, err.Error(), "nonexistent-group") + }) +} diff --git a/typescript/config/lib/generated.d.ts b/typescript/config/lib/generated.d.ts index 36a2c6bdc..6a61e13ea 100644 --- a/typescript/config/lib/generated.d.ts +++ b/typescript/config/lib/generated.d.ts @@ -472,6 +472,7 @@ export interface EvmAvailabilityBoundConfig { export interface FailsafeConfig { matchMethod?: string; matchFinality?: DataFinalityState[]; + matchUpstreamGroup?: string; retry?: RetryPolicyConfig; circuitBreaker?: CircuitBreakerPolicyConfig; timeout?: TimeoutPolicyConfig; diff --git a/typescript/config/src/generated.ts b/typescript/config/src/generated.ts index ab58b155c..b50753b79 100644 --- a/typescript/config/src/generated.ts +++ b/typescript/config/src/generated.ts @@ -486,6 +486,7 @@ export interface EvmAvailabilityBoundConfig { export interface FailsafeConfig { matchMethod?: string; matchFinality?: DataFinalityState[]; + matchUpstreamGroup?: string; retry?: RetryPolicyConfig; circuitBreaker?: CircuitBreakerPolicyConfig; timeout?: TimeoutPolicyConfig; From 876a6eb2eea71335a2dedacd8c1707309f44b4e9 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 19 Jan 2026 16:32:50 +0100 Subject: [PATCH 49/53] test: add comprehensive integration tests for multicall3 batching Add integration test suite that validates multicall3 batching functionality against a real eRPC instance. Tests cover: Core functionality: - JSON-RPC batch aggregation into single multicall3 call - Concurrent request batching within time window - Mixed batches (eth_call + other methods) - Different contract addresses in same batch - Large batches (20+ calls) - Request deduplication Bypass scenarios (calls that should NOT be batched): - Calls with value/from/gas fields - State overrides (3rd param) - Calls to multicall3 contract itself (recursion guard) - requireCanonical:false (EIP-1898) Block reference variations: - Block hash (EIP-1898 format) - finalized/safe/earliest tags - Specific block numbers Input variations: - 'input' field as alternative to 'data' - Empty and large calldata Directive tests: - X-ERPC-Skip-Cache-Read - X-ERPC-Retry-Empty - X-ERPC-Use-Upstream Also includes bash script for production testing. Usage: ERPC_INTEGRATION_TEST_ENDPOINT=http://localhost:4000/main/evm/1 \ ERPC_INTEGRATION_TEST_METRICS=http://localhost:4001/metrics \ ERPC_INTEGRATION_TEST_AUTH="X-ERPC-Secret-Token: token" \ go test -v ./test/integration/... Co-Authored-By: Claude Opus 4.5 --- scripts/test-multicall3-prd.sh | 366 ++++ .../multicall3_integration_test.go | 1540 +++++++++++++++++ 2 files changed, 1906 insertions(+) create mode 100755 scripts/test-multicall3-prd.sh create mode 100644 test/integration/multicall3_integration_test.go diff --git a/scripts/test-multicall3-prd.sh b/scripts/test-multicall3-prd.sh new file mode 100755 index 000000000..47c455136 --- /dev/null +++ b/scripts/test-multicall3-prd.sh @@ -0,0 +1,366 @@ +#!/bin/bash +# ============================================================================= +# Multicall3 Aggregation Production Test Script +# ============================================================================= +# This script tests if multicall3 batching is working correctly in production. +# +# Prerequisites: +# - jq installed +# - curl installed +# - Access to eRPC endpoint +# - Access to Prometheus/metrics endpoint (optional but recommended) +# +# Usage: +# ./scripts/test-multicall3-prd.sh [metrics-endpoint] +# +# Examples: +# ./scripts/test-multicall3-prd.sh https://erpc.example.com/main/evm/1 +# ./scripts/test-multicall3-prd.sh https://erpc.example.com/main/evm/1 https://erpc.example.com/metrics +# ============================================================================= + +set -euo pipefail + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Configuration +ERPC_ENDPOINT="${1:-}" +METRICS_ENDPOINT="${2:-}" + +# Multicall3 contract address (same on most EVM chains) +MULTICALL3_ADDRESS="0xcA11bde05977b3631167028862bE2a173976CA11" + +# Test contract addresses (well-known contracts on mainnet) +# WETH on Ethereum mainnet +WETH_ADDRESS="0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" +# USDC on Ethereum mainnet +USDC_ADDRESS="0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48" +# DAI on Ethereum mainnet +DAI_ADDRESS="0x6B175474E89094C44Da98b954EesdeAfe4068538" + +# Function signatures (for eth_call) +# balanceOf(address) = 0x70a08231 +# decimals() = 0x313ce567 +# symbol() = 0x95d89b41 +# totalSupply() = 0x18160ddd + +usage() { + echo "Usage: $0 [metrics-endpoint]" + echo "" + echo "Examples:" + echo " $0 https://erpc.example.com/main/evm/1" + echo " $0 https://erpc.example.com/main/evm/1 https://erpc.example.com/metrics" + exit 1 +} + +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[PASS]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[FAIL]${NC} $1" +} + +log_section() { + echo "" + echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + echo -e "${BLUE} $1${NC}" + echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" +} + +# Validate arguments +if [[ -z "$ERPC_ENDPOINT" ]]; then + usage +fi + +# ============================================================================= +# Test 1: Basic connectivity +# ============================================================================= +log_section "Test 1: Basic Connectivity" + +log_info "Testing endpoint connectivity..." +CHAIN_ID_RESPONSE=$(curl -s -X POST "$ERPC_ENDPOINT" \ + -H "Content-Type: application/json" \ + -d '{"jsonrpc":"2.0","id":1,"method":"eth_chainId","params":[]}') + +if echo "$CHAIN_ID_RESPONSE" | jq -e '.result' > /dev/null 2>&1; then + CHAIN_ID=$(echo "$CHAIN_ID_RESPONSE" | jq -r '.result') + log_success "Endpoint reachable, chainId: $CHAIN_ID" +else + log_error "Failed to connect to endpoint" + echo "Response: $CHAIN_ID_RESPONSE" + exit 1 +fi + +# ============================================================================= +# Test 2: Capture baseline metrics (if metrics endpoint provided) +# ============================================================================= +BASELINE_AGGREGATION_COUNT="" +BASELINE_FALLBACK_COUNT="" + +if [[ -n "$METRICS_ENDPOINT" ]]; then + log_section "Test 2: Capture Baseline Metrics" + + log_info "Fetching baseline multicall3 metrics..." + METRICS=$(curl -s "$METRICS_ENDPOINT" 2>/dev/null || echo "") + + if [[ -n "$METRICS" ]]; then + BASELINE_AGGREGATION_COUNT=$(echo "$METRICS" | grep 'erpc_multicall3_aggregation_total' | grep -v '^#' | awk '{sum += $2} END {print sum}' || echo "0") + BASELINE_FALLBACK_COUNT=$(echo "$METRICS" | grep 'erpc_multicall3_fallback_total' | grep -v '^#' | awk '{sum += $2} END {print sum}' || echo "0") + + log_success "Baseline aggregation count: ${BASELINE_AGGREGATION_COUNT:-0}" + log_success "Baseline fallback count: ${BASELINE_FALLBACK_COUNT:-0}" + else + log_warning "Could not fetch metrics, skipping metric-based verification" + fi +else + log_section "Test 2: Metrics Check (Skipped)" + log_warning "No metrics endpoint provided, skipping metric-based verification" +fi + +# ============================================================================= +# Test 3: Single eth_call (should NOT be batched - baseline) +# ============================================================================= +log_section "Test 3: Single eth_call (Baseline)" + +log_info "Sending single eth_call request..." +# Call decimals() on WETH +SINGLE_CALL_RESPONSE=$(curl -s -X POST "$ERPC_ENDPOINT" \ + -H "Content-Type: application/json" \ + -d '{ + "jsonrpc":"2.0", + "id":1, + "method":"eth_call", + "params":[{ + "to":"'"$WETH_ADDRESS"'", + "data":"0x313ce567" + },"latest"] + }') + +if echo "$SINGLE_CALL_RESPONSE" | jq -e '.result' > /dev/null 2>&1; then + DECIMALS=$(echo "$SINGLE_CALL_RESPONSE" | jq -r '.result') + log_success "Single eth_call succeeded, decimals: $DECIMALS" +else + log_error "Single eth_call failed" + echo "Response: $SINGLE_CALL_RESPONSE" +fi + +# ============================================================================= +# Test 4: JSON-RPC batch with multiple eth_calls (should be batched via multicall3) +# ============================================================================= +log_section "Test 4: JSON-RPC Batch (Multiple eth_calls)" + +log_info "Sending JSON-RPC batch with 3 eth_call requests..." +log_info "These should be aggregated into a single multicall3 call" + +BATCH_RESPONSE=$(curl -s -X POST "$ERPC_ENDPOINT" \ + -H "Content-Type: application/json" \ + -d '[ + {"jsonrpc":"2.0","id":1,"method":"eth_call","params":[{"to":"'"$WETH_ADDRESS"'","data":"0x313ce567"},"latest"]}, + {"jsonrpc":"2.0","id":2,"method":"eth_call","params":[{"to":"'"$WETH_ADDRESS"'","data":"0x95d89b41"},"latest"]}, + {"jsonrpc":"2.0","id":3,"method":"eth_call","params":[{"to":"'"$WETH_ADDRESS"'","data":"0x18160ddd"},"latest"]} + ]') + +# Check if all 3 responses are valid +VALID_COUNT=$(echo "$BATCH_RESPONSE" | jq '[.[] | select(.result != null)] | length') + +if [[ "$VALID_COUNT" == "3" ]]; then + log_success "JSON-RPC batch returned 3 valid responses" + + # Extract results + DECIMALS=$(echo "$BATCH_RESPONSE" | jq -r '.[0].result') + SYMBOL=$(echo "$BATCH_RESPONSE" | jq -r '.[1].result') + TOTAL_SUPPLY=$(echo "$BATCH_RESPONSE" | jq -r '.[2].result') + + log_info " decimals(): $DECIMALS" + log_info " symbol(): $SYMBOL" + log_info " totalSupply(): $TOTAL_SUPPLY" +else + log_error "JSON-RPC batch did not return 3 valid responses" + echo "Response: $BATCH_RESPONSE" +fi + +# ============================================================================= +# Test 5: Concurrent requests (should be batched together) +# ============================================================================= +log_section "Test 5: Concurrent Requests (Batching Test)" + +log_info "Sending 5 concurrent eth_call requests..." +log_info "These should be batched together within the window" + +# Create temp files for responses +TEMP_DIR=$(mktemp -d) +trap "rm -rf $TEMP_DIR" EXIT + +# Send 5 concurrent requests +for i in {1..5}; do + curl -s -X POST "$ERPC_ENDPOINT" \ + -H "Content-Type: application/json" \ + -d '{"jsonrpc":"2.0","id":'$i',"method":"eth_call","params":[{"to":"'"$WETH_ADDRESS"'","data":"0x313ce567"},"latest"]}' \ + > "$TEMP_DIR/response_$i.json" & +done + +# Wait for all requests to complete +wait + +# Check responses +SUCCESS_COUNT=0 +for i in {1..5}; do + if jq -e '.result' "$TEMP_DIR/response_$i.json" > /dev/null 2>&1; then + ((SUCCESS_COUNT++)) + fi +done + +if [[ "$SUCCESS_COUNT" == "5" ]]; then + log_success "All 5 concurrent requests succeeded" +else + log_error "Only $SUCCESS_COUNT/5 concurrent requests succeeded" +fi + +# ============================================================================= +# Test 6: Mixed batch (eth_call + other methods) +# ============================================================================= +log_section "Test 6: Mixed Batch (eth_call + other methods)" + +log_info "Sending mixed batch with eth_call and eth_blockNumber..." + +MIXED_BATCH_RESPONSE=$(curl -s -X POST "$ERPC_ENDPOINT" \ + -H "Content-Type: application/json" \ + -d '[ + {"jsonrpc":"2.0","id":1,"method":"eth_call","params":[{"to":"'"$WETH_ADDRESS"'","data":"0x313ce567"},"latest"]}, + {"jsonrpc":"2.0","id":2,"method":"eth_blockNumber","params":[]}, + {"jsonrpc":"2.0","id":3,"method":"eth_call","params":[{"to":"'"$WETH_ADDRESS"'","data":"0x18160ddd"},"latest"]} + ]') + +VALID_COUNT=$(echo "$MIXED_BATCH_RESPONSE" | jq '[.[] | select(.result != null)] | length') + +if [[ "$VALID_COUNT" == "3" ]]; then + log_success "Mixed batch returned 3 valid responses" +else + log_error "Mixed batch did not return 3 valid responses" + echo "Response: $MIXED_BATCH_RESPONSE" +fi + +# ============================================================================= +# Test 7: eth_call that reverts (error handling) +# ============================================================================= +log_section "Test 7: eth_call That Reverts (Error Handling)" + +log_info "Sending eth_call that should revert..." +# Call a non-existent function on a contract +REVERT_RESPONSE=$(curl -s -X POST "$ERPC_ENDPOINT" \ + -H "Content-Type: application/json" \ + -d '{ + "jsonrpc":"2.0", + "id":1, + "method":"eth_call", + "params":[{ + "to":"'"$WETH_ADDRESS"'", + "data":"0xdeadbeef" + },"latest"] + }') + +# Check if we got an error or empty result (both are acceptable for invalid call) +if echo "$REVERT_RESPONSE" | jq -e '.error' > /dev/null 2>&1; then + ERROR_MSG=$(echo "$REVERT_RESPONSE" | jq -r '.error.message // .error') + log_success "Revert handled correctly with error: $ERROR_MSG" +elif echo "$REVERT_RESPONSE" | jq -e '.result' > /dev/null 2>&1; then + RESULT=$(echo "$REVERT_RESPONSE" | jq -r '.result') + if [[ "$RESULT" == "0x" ]] || [[ -z "$RESULT" ]]; then + log_success "Revert returned empty result (expected)" + else + log_warning "Got unexpected result for invalid call: $RESULT" + fi +else + log_warning "Unexpected response format for revert test" + echo "Response: $REVERT_RESPONSE" +fi + +# ============================================================================= +# Test 8: Verify metrics increased (if metrics endpoint provided) +# ============================================================================= +if [[ -n "$METRICS_ENDPOINT" ]] && [[ -n "$BASELINE_AGGREGATION_COUNT" ]]; then + log_section "Test 8: Verify Metrics Increased" + + # Wait a moment for metrics to be updated + sleep 2 + + log_info "Fetching updated multicall3 metrics..." + METRICS=$(curl -s "$METRICS_ENDPOINT" 2>/dev/null || echo "") + + if [[ -n "$METRICS" ]]; then + NEW_AGGREGATION_COUNT=$(echo "$METRICS" | grep 'erpc_multicall3_aggregation_total' | grep -v '^#' | awk '{sum += $2} END {print sum}' || echo "0") + NEW_FALLBACK_COUNT=$(echo "$METRICS" | grep 'erpc_multicall3_fallback_total' | grep -v '^#' | awk '{sum += $2} END {print sum}' || echo "0") + + AGGREGATION_DIFF=$((${NEW_AGGREGATION_COUNT:-0} - ${BASELINE_AGGREGATION_COUNT:-0})) + FALLBACK_DIFF=$((${NEW_FALLBACK_COUNT:-0} - ${BASELINE_FALLBACK_COUNT:-0})) + + log_info "Aggregation count: ${BASELINE_AGGREGATION_COUNT:-0} -> ${NEW_AGGREGATION_COUNT:-0} (+$AGGREGATION_DIFF)" + log_info "Fallback count: ${BASELINE_FALLBACK_COUNT:-0} -> ${NEW_FALLBACK_COUNT:-0} (+$FALLBACK_DIFF)" + + if [[ "$AGGREGATION_DIFF" -gt 0 ]]; then + log_success "Multicall3 aggregation is working! ($AGGREGATION_DIFF new aggregations)" + else + log_warning "No new aggregations detected - multicall3 might not be enabled or requests fell back" + fi + + if [[ "$FALLBACK_DIFF" -gt 0 ]]; then + log_warning "$FALLBACK_DIFF requests fell back to individual calls" + fi + + # Show other relevant metrics + log_info "" + log_info "Additional metrics:" + echo "$METRICS" | grep 'erpc_multicall3_' | grep -v '^#' | head -20 || true + else + log_warning "Could not fetch updated metrics" + fi +fi + +# ============================================================================= +# Summary +# ============================================================================= +log_section "Test Summary" + +echo "" +echo "Tests completed. Review the output above to verify:" +echo "" +echo " 1. ✓ Basic connectivity works" +echo " 2. ✓ Single eth_call works (baseline)" +echo " 3. ✓ JSON-RPC batch with multiple eth_calls works" +echo " 4. ✓ Concurrent requests are handled" +echo " 5. ✓ Mixed batches work correctly" +echo " 6. ✓ Reverts are handled gracefully" +echo "" +echo "To confirm multicall3 batching is actually happening:" +echo "" +echo " • Check erpc_multicall3_aggregation_total metric increased" +echo " • Check erpc_multicall3_batch_size histogram for batch sizes > 1" +echo " • Check logs for 'multicall3' mentions" +echo "" + +if [[ -n "$METRICS_ENDPOINT" ]]; then + echo "Useful PromQL queries:" + echo "" + echo " # Aggregation rate" + echo " rate(erpc_multicall3_aggregation_total[5m])" + echo "" + echo " # Average batch size" + echo " histogram_quantile(0.5, rate(erpc_multicall3_batch_size_bucket[5m]))" + echo "" + echo " # Fallback rate (should be low)" + echo " rate(erpc_multicall3_fallback_total[5m])" + echo "" +fi diff --git a/test/integration/multicall3_integration_test.go b/test/integration/multicall3_integration_test.go new file mode 100644 index 000000000..f616b50df --- /dev/null +++ b/test/integration/multicall3_integration_test.go @@ -0,0 +1,1540 @@ +// Package integration contains integration tests that run against a real eRPC instance. +// +// These tests are skipped by default unless ERPC_INTEGRATION_TEST_ENDPOINT is set. +// +// Environment variables: +// - ERPC_INTEGRATION_TEST_ENDPOINT: eRPC endpoint URL (required) +// - ERPC_INTEGRATION_TEST_METRICS: Prometheus metrics endpoint (optional, enables metric verification) +// - ERPC_INTEGRATION_TEST_AUTH: Auth headers in "Header: value" format (optional, use ";" for multiple) +// +// Usage: +// +// # Run against local eRPC (no auth) +// ERPC_INTEGRATION_TEST_ENDPOINT=http://localhost:4000/main/evm/1 \ +// go test -v ./test/integration/... +// +// # Run against local eRPC with auth and metrics +// ERPC_INTEGRATION_TEST_ENDPOINT=http://localhost:4000/main/evm/1 \ +// ERPC_INTEGRATION_TEST_METRICS=http://localhost:4001/metrics \ +// ERPC_INTEGRATION_TEST_AUTH="X-ERPC-Secret-Token: your-token" \ +// go test -v ./test/integration/... +// +// # Run specific test +// ERPC_INTEGRATION_TEST_ENDPOINT=http://localhost:4000/main/evm/1 \ +// ERPC_INTEGRATION_TEST_AUTH="X-ERPC-Secret-Token: your-token" \ +// go test -v -run TestMulticall3Integration_BatchEthCalls ./test/integration/... +package integration + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + // Well-known contract addresses for testing (Ethereum mainnet) + // These are used because they're stable and have predictable behavior + wethAddress = "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" + usdcAddress = "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48" + + // Function selectors + decimalsSelector = "0x313ce567" // decimals() + symbolSelector = "0x95d89b41" // symbol() + totalSupplySelector = "0x18160ddd" // totalSupply() +) + +type jsonRPCRequest struct { + JSONRPC string `json:"jsonrpc"` + ID interface{} `json:"id"` + Method string `json:"method"` + Params []interface{} `json:"params"` +} + +type jsonRPCResponse struct { + JSONRPC string `json:"jsonrpc"` + ID interface{} `json:"id"` + Result json.RawMessage `json:"result,omitempty"` + Error *json.RawMessage `json:"error,omitempty"` +} + +type ethCallParams struct { + To string `json:"to"` + Data string `json:"data"` +} + +func getTestEndpoint(t *testing.T) string { + endpoint := os.Getenv("ERPC_INTEGRATION_TEST_ENDPOINT") + if endpoint == "" { + t.Skip("ERPC_INTEGRATION_TEST_ENDPOINT not set, skipping integration test") + } + return endpoint +} + +func getMetricsEndpoint() string { + return os.Getenv("ERPC_INTEGRATION_TEST_METRICS") +} + +// getAuthHeaders returns authentication headers if configured +// Format: "Header-Name: value" or multiple headers separated by ";" +func getAuthHeaders() map[string]string { + headers := make(map[string]string) + authHeader := os.Getenv("ERPC_INTEGRATION_TEST_AUTH") + if authHeader == "" { + return headers + } + + // Support multiple headers separated by ";" + parts := strings.Split(authHeader, ";") + for _, part := range parts { + part = strings.TrimSpace(part) + if idx := strings.Index(part, ":"); idx > 0 { + key := strings.TrimSpace(part[:idx]) + value := strings.TrimSpace(part[idx+1:]) + headers[key] = value + } + } + return headers +} + +func makeRequest(t *testing.T, endpoint string, payload interface{}) []byte { + t.Helper() + + body, err := json.Marshal(payload) + require.NoError(t, err) + + req, err := http.NewRequest("POST", endpoint, bytes.NewReader(body)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + + // Add auth headers if configured + for key, value := range getAuthHeaders() { + req.Header.Set(key, value) + } + + client := &http.Client{Timeout: 30 * time.Second} + resp, err := client.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + return respBody +} + +func getMetricValue(metricsEndpoint, metricName string) (float64, error) { + resp, err := http.Get(metricsEndpoint) + if err != nil { + return 0, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return 0, err + } + + var total float64 + for _, line := range strings.Split(string(body), "\n") { + if strings.HasPrefix(line, metricName) && !strings.HasPrefix(line, "#") { + // Parse the metric value (last field) + fields := strings.Fields(line) + if len(fields) >= 2 { + var val float64 + fmt.Sscanf(fields[len(fields)-1], "%f", &val) + total += val + } + } + } + return total, nil +} + +// TestMulticall3Integration_Connectivity verifies basic endpoint connectivity +func TestMulticall3Integration_Connectivity(t *testing.T) { + endpoint := getTestEndpoint(t) + + req := jsonRPCRequest{ + JSONRPC: "2.0", + ID: 1, + Method: "eth_chainId", + Params: []interface{}{}, + } + + respBody := makeRequest(t, endpoint, req) + + var resp jsonRPCResponse + err := json.Unmarshal(respBody, &resp) + require.NoError(t, err, "Failed to parse response: %s", string(respBody)) + require.Nil(t, resp.Error, "Request returned error: %s", string(respBody)) + require.NotNil(t, resp.Result, "No result in response") + + var chainId string + err = json.Unmarshal(resp.Result, &chainId) + require.NoError(t, err) + assert.NotEmpty(t, chainId, "chainId should not be empty") + + t.Logf("Connected to chain: %s", chainId) +} + +// TestMulticall3Integration_SingleEthCall verifies single eth_call works (baseline) +func TestMulticall3Integration_SingleEthCall(t *testing.T) { + endpoint := getTestEndpoint(t) + + req := jsonRPCRequest{ + JSONRPC: "2.0", + ID: 1, + Method: "eth_call", + Params: []interface{}{ + ethCallParams{To: wethAddress, Data: decimalsSelector}, + "latest", + }, + } + + respBody := makeRequest(t, endpoint, req) + + var resp jsonRPCResponse + err := json.Unmarshal(respBody, &resp) + require.NoError(t, err, "Failed to parse response: %s", string(respBody)) + require.Nil(t, resp.Error, "Request returned error: %s", string(respBody)) + require.NotNil(t, resp.Result, "No result in response") + + var result string + err = json.Unmarshal(resp.Result, &result) + require.NoError(t, err) + assert.NotEmpty(t, result, "result should not be empty") + + t.Logf("Single eth_call result (decimals): %s", result) +} + +// TestMulticall3Integration_BatchEthCalls verifies JSON-RPC batch with multiple eth_calls +// These should be aggregated into a single multicall3 call +func TestMulticall3Integration_BatchEthCalls(t *testing.T) { + endpoint := getTestEndpoint(t) + metricsEndpoint := getMetricsEndpoint() + + // Capture baseline metrics if available + var baselineAggregation float64 + if metricsEndpoint != "" { + var err error + baselineAggregation, err = getMetricValue(metricsEndpoint, "erpc_multicall3_aggregation_total") + if err != nil { + t.Logf("Warning: could not get baseline metrics: %v", err) + } + } + + // Send batch with 3 eth_calls to the same contract + batch := []jsonRPCRequest{ + {JSONRPC: "2.0", ID: 1, Method: "eth_call", Params: []interface{}{ethCallParams{To: wethAddress, Data: decimalsSelector}, "latest"}}, + {JSONRPC: "2.0", ID: 2, Method: "eth_call", Params: []interface{}{ethCallParams{To: wethAddress, Data: symbolSelector}, "latest"}}, + {JSONRPC: "2.0", ID: 3, Method: "eth_call", Params: []interface{}{ethCallParams{To: wethAddress, Data: totalSupplySelector}, "latest"}}, + } + + respBody := makeRequest(t, endpoint, batch) + + var responses []jsonRPCResponse + err := json.Unmarshal(respBody, &responses) + require.NoError(t, err, "Failed to parse batch response: %s", string(respBody)) + require.Len(t, responses, 3, "Expected 3 responses in batch") + + for i, resp := range responses { + assert.Nil(t, resp.Error, "Response %d returned error", i) + assert.NotNil(t, resp.Result, "Response %d has no result", i) + } + + t.Logf("Batch eth_call responses: all 3 successful") + + // Check if metrics increased (indicates multicall3 was used) + if metricsEndpoint != "" { + time.Sleep(1 * time.Second) // Allow metrics to update + + newAggregation, err := getMetricValue(metricsEndpoint, "erpc_multicall3_aggregation_total") + if err == nil { + diff := newAggregation - baselineAggregation + t.Logf("Multicall3 aggregation metric: %.0f -> %.0f (+%.0f)", baselineAggregation, newAggregation, diff) + if diff > 0 { + t.Logf("✓ Multicall3 aggregation confirmed via metrics") + } else { + t.Logf("⚠ No increase in aggregation metric - multicall3 may not be enabled or requests fell back") + } + } + } +} + +// TestMulticall3Integration_ConcurrentRequests verifies concurrent requests are batched together +func TestMulticall3Integration_ConcurrentRequests(t *testing.T) { + endpoint := getTestEndpoint(t) + metricsEndpoint := getMetricsEndpoint() + + // Capture baseline metrics if available + var baselineAggregation float64 + if metricsEndpoint != "" { + baselineAggregation, _ = getMetricValue(metricsEndpoint, "erpc_multicall3_aggregation_total") + } + + // Send 10 concurrent requests - they should be batched within the window + const numRequests = 10 + var wg sync.WaitGroup + results := make(chan bool, numRequests) + + for i := 0; i < numRequests; i++ { + wg.Add(1) + go func(id int) { + defer wg.Done() + + req := jsonRPCRequest{ + JSONRPC: "2.0", + ID: id, + Method: "eth_call", + Params: []interface{}{ + ethCallParams{To: wethAddress, Data: decimalsSelector}, + "latest", + }, + } + + respBody := makeRequest(t, endpoint, req) + + var resp jsonRPCResponse + if err := json.Unmarshal(respBody, &resp); err != nil { + results <- false + return + } + + results <- resp.Error == nil && resp.Result != nil + }(i) + } + + wg.Wait() + close(results) + + successCount := 0 + for success := range results { + if success { + successCount++ + } + } + + assert.Equal(t, numRequests, successCount, "All concurrent requests should succeed") + t.Logf("Concurrent requests: %d/%d successful", successCount, numRequests) + + // Check if metrics show batching occurred + if metricsEndpoint != "" { + time.Sleep(1 * time.Second) + + newAggregation, err := getMetricValue(metricsEndpoint, "erpc_multicall3_aggregation_total") + if err == nil { + diff := newAggregation - baselineAggregation + t.Logf("Multicall3 aggregation metric: %.0f -> %.0f (+%.0f)", baselineAggregation, newAggregation, diff) + + // If multicall3 is working, we should see fewer aggregations than requests + // (since multiple requests get batched into one multicall3 call) + if diff > 0 && diff < float64(numRequests) { + t.Logf("✓ Batching confirmed: %d requests resulted in %.0f aggregations", numRequests, diff) + } + } + } +} + +// TestMulticall3Integration_MixedBatch verifies mixed batch with eth_call and other methods +func TestMulticall3Integration_MixedBatch(t *testing.T) { + endpoint := getTestEndpoint(t) + + batch := []jsonRPCRequest{ + {JSONRPC: "2.0", ID: 1, Method: "eth_call", Params: []interface{}{ethCallParams{To: wethAddress, Data: decimalsSelector}, "latest"}}, + {JSONRPC: "2.0", ID: 2, Method: "eth_blockNumber", Params: []interface{}{}}, + {JSONRPC: "2.0", ID: 3, Method: "eth_call", Params: []interface{}{ethCallParams{To: wethAddress, Data: totalSupplySelector}, "latest"}}, + {JSONRPC: "2.0", ID: 4, Method: "eth_chainId", Params: []interface{}{}}, + } + + respBody := makeRequest(t, endpoint, batch) + + var responses []jsonRPCResponse + err := json.Unmarshal(respBody, &responses) + require.NoError(t, err, "Failed to parse mixed batch response: %s", string(respBody)) + require.Len(t, responses, 4, "Expected 4 responses in mixed batch") + + for i, resp := range responses { + assert.Nil(t, resp.Error, "Response %d returned error", i) + assert.NotNil(t, resp.Result, "Response %d has no result", i) + } + + t.Logf("Mixed batch: all 4 responses successful (2 eth_call, 1 eth_blockNumber, 1 eth_chainId)") +} + +// TestMulticall3Integration_DifferentContracts verifies batching across different contracts +func TestMulticall3Integration_DifferentContracts(t *testing.T) { + endpoint := getTestEndpoint(t) + + // Batch calls to different contracts - should still be batched via multicall3 + batch := []jsonRPCRequest{ + {JSONRPC: "2.0", ID: 1, Method: "eth_call", Params: []interface{}{ethCallParams{To: wethAddress, Data: decimalsSelector}, "latest"}}, + {JSONRPC: "2.0", ID: 2, Method: "eth_call", Params: []interface{}{ethCallParams{To: usdcAddress, Data: decimalsSelector}, "latest"}}, + {JSONRPC: "2.0", ID: 3, Method: "eth_call", Params: []interface{}{ethCallParams{To: wethAddress, Data: symbolSelector}, "latest"}}, + {JSONRPC: "2.0", ID: 4, Method: "eth_call", Params: []interface{}{ethCallParams{To: usdcAddress, Data: symbolSelector}, "latest"}}, + } + + respBody := makeRequest(t, endpoint, batch) + + var responses []jsonRPCResponse + err := json.Unmarshal(respBody, &responses) + require.NoError(t, err, "Failed to parse response: %s", string(respBody)) + require.Len(t, responses, 4, "Expected 4 responses") + + for i, resp := range responses { + assert.Nil(t, resp.Error, "Response %d returned error", i) + assert.NotNil(t, resp.Result, "Response %d has no result", i) + } + + t.Logf("Different contracts batch: all 4 responses successful (WETH + USDC)") +} + +// TestMulticall3Integration_ErrorHandling verifies error handling for failing calls +func TestMulticall3Integration_ErrorHandling(t *testing.T) { + endpoint := getTestEndpoint(t) + + // Send a batch with one valid and one invalid call + batch := []jsonRPCRequest{ + {JSONRPC: "2.0", ID: 1, Method: "eth_call", Params: []interface{}{ethCallParams{To: wethAddress, Data: decimalsSelector}, "latest"}}, + {JSONRPC: "2.0", ID: 2, Method: "eth_call", Params: []interface{}{ethCallParams{To: wethAddress, Data: "0xdeadbeef"}, "latest"}}, // Invalid function + {JSONRPC: "2.0", ID: 3, Method: "eth_call", Params: []interface{}{ethCallParams{To: wethAddress, Data: symbolSelector}, "latest"}}, + } + + respBody := makeRequest(t, endpoint, batch) + + var responses []jsonRPCResponse + err := json.Unmarshal(respBody, &responses) + require.NoError(t, err, "Failed to parse response: %s", string(respBody)) + require.Len(t, responses, 3, "Expected 3 responses") + + // First and third should succeed + assert.Nil(t, responses[0].Error, "Response 0 should succeed") + assert.NotNil(t, responses[0].Result, "Response 0 should have result") + + assert.Nil(t, responses[2].Error, "Response 2 should succeed") + assert.NotNil(t, responses[2].Result, "Response 2 should have result") + + // Second may error or return empty - both are acceptable + if responses[1].Error != nil { + t.Logf("Invalid call returned error (expected): %s", string(*responses[1].Error)) + } else if responses[1].Result != nil { + var result string + json.Unmarshal(responses[1].Result, &result) + t.Logf("Invalid call returned result: %s (may be empty or revert data)", result) + } + + t.Logf("Error handling: valid calls succeeded, invalid call handled gracefully") +} + +// TestMulticall3Integration_LargeBatch verifies handling of larger batches +func TestMulticall3Integration_LargeBatch(t *testing.T) { + endpoint := getTestEndpoint(t) + metricsEndpoint := getMetricsEndpoint() + + // Capture baseline metrics + var baselineAggregation float64 + if metricsEndpoint != "" { + baselineAggregation, _ = getMetricValue(metricsEndpoint, "erpc_multicall3_aggregation_total") + } + + // Send a batch with 20 calls (should trigger batching limits) + batch := make([]jsonRPCRequest, 20) + for i := 0; i < 20; i++ { + batch[i] = jsonRPCRequest{ + JSONRPC: "2.0", + ID: i + 1, + Method: "eth_call", + Params: []interface{}{ + ethCallParams{To: wethAddress, Data: decimalsSelector}, + "latest", + }, + } + } + + respBody := makeRequest(t, endpoint, batch) + + var responses []jsonRPCResponse + err := json.Unmarshal(respBody, &responses) + require.NoError(t, err, "Failed to parse response: %s", string(respBody)) + require.Len(t, responses, 20, "Expected 20 responses") + + successCount := 0 + for _, resp := range responses { + if resp.Error == nil && resp.Result != nil { + successCount++ + } + } + + assert.Equal(t, 20, successCount, "All 20 calls should succeed") + t.Logf("Large batch: %d/20 calls successful", successCount) + + // Check metrics + if metricsEndpoint != "" { + time.Sleep(1 * time.Second) + + newAggregation, err := getMetricValue(metricsEndpoint, "erpc_multicall3_aggregation_total") + if err == nil { + diff := newAggregation - baselineAggregation + t.Logf("Multicall3 aggregation metric: %.0f -> %.0f (+%.0f)", baselineAggregation, newAggregation, diff) + + // With default maxCalls=20, a batch of 20 should result in 1 aggregation + // (or possibly 2 if there's splitting) + if diff > 0 && diff <= 2 { + t.Logf("✓ Efficient batching: 20 requests resulted in %.0f multicall3 aggregation(s)", diff) + } + } + } +} + +// TestMulticall3Integration_Deduplication verifies that duplicate requests are deduplicated +func TestMulticall3Integration_Deduplication(t *testing.T) { + endpoint := getTestEndpoint(t) + metricsEndpoint := getMetricsEndpoint() + + // Capture baseline dedupe metric + var baselineDedupe float64 + if metricsEndpoint != "" { + baselineDedupe, _ = getMetricValue(metricsEndpoint, "erpc_multicall3_dedupe_total") + } + + // Send batch with DUPLICATE calls - same contract, same function, same block + // These should be deduplicated (only one actual call to the contract) + batch := []jsonRPCRequest{ + {JSONRPC: "2.0", ID: 1, Method: "eth_call", Params: []interface{}{ethCallParams{To: wethAddress, Data: decimalsSelector}, "latest"}}, + {JSONRPC: "2.0", ID: 2, Method: "eth_call", Params: []interface{}{ethCallParams{To: wethAddress, Data: decimalsSelector}, "latest"}}, // Duplicate + {JSONRPC: "2.0", ID: 3, Method: "eth_call", Params: []interface{}{ethCallParams{To: wethAddress, Data: decimalsSelector}, "latest"}}, // Duplicate + {JSONRPC: "2.0", ID: 4, Method: "eth_call", Params: []interface{}{ethCallParams{To: wethAddress, Data: symbolSelector}, "latest"}}, // Different call + } + + respBody := makeRequest(t, endpoint, batch) + + var responses []jsonRPCResponse + err := json.Unmarshal(respBody, &responses) + require.NoError(t, err, "Failed to parse response: %s", string(respBody)) + require.Len(t, responses, 4, "Expected 4 responses") + + // All responses should succeed + for i, resp := range responses { + assert.Nil(t, resp.Error, "Response %d returned error", i) + assert.NotNil(t, resp.Result, "Response %d has no result", i) + } + + // First 3 responses should be identical (same call) + var result1, result2, result3 string + json.Unmarshal(responses[0].Result, &result1) + json.Unmarshal(responses[1].Result, &result2) + json.Unmarshal(responses[2].Result, &result3) + assert.Equal(t, result1, result2, "Duplicate calls should return same result") + assert.Equal(t, result2, result3, "Duplicate calls should return same result") + + t.Logf("Deduplication: 3 identical calls returned same result: %s", result1) + + // Check if dedupe metric increased + if metricsEndpoint != "" { + time.Sleep(1 * time.Second) + + newDedupe, err := getMetricValue(metricsEndpoint, "erpc_multicall3_dedupe_total") + if err == nil { + diff := newDedupe - baselineDedupe + t.Logf("Deduplication metric: %.0f -> %.0f (+%.0f)", baselineDedupe, newDedupe, diff) + if diff >= 2 { + t.Logf("✓ Deduplication confirmed: %0.f duplicate requests were deduplicated", diff) + } + } + } +} + +// TestMulticall3Integration_BlockTagVariations tests batching with different block tags +func TestMulticall3Integration_BlockTagVariations(t *testing.T) { + endpoint := getTestEndpoint(t) + + t.Run("LatestBlock", func(t *testing.T) { + batch := []jsonRPCRequest{ + {JSONRPC: "2.0", ID: 1, Method: "eth_call", Params: []interface{}{ethCallParams{To: wethAddress, Data: decimalsSelector}, "latest"}}, + {JSONRPC: "2.0", ID: 2, Method: "eth_call", Params: []interface{}{ethCallParams{To: wethAddress, Data: symbolSelector}, "latest"}}, + } + + respBody := makeRequest(t, endpoint, batch) + + var responses []jsonRPCResponse + err := json.Unmarshal(respBody, &responses) + require.NoError(t, err) + require.Len(t, responses, 2) + + for i, resp := range responses { + assert.Nil(t, resp.Error, "Response %d with 'latest' tag should succeed", i) + assert.NotNil(t, resp.Result, "Response %d should have result", i) + } + t.Logf("'latest' block tag: both calls succeeded") + }) + + t.Run("SpecificBlockNumber", func(t *testing.T) { + // First get the latest block number + blockNumReq := jsonRPCRequest{ + JSONRPC: "2.0", + ID: 1, + Method: "eth_blockNumber", + Params: []interface{}{}, + } + respBody := makeRequest(t, endpoint, blockNumReq) + + var blockResp jsonRPCResponse + err := json.Unmarshal(respBody, &blockResp) + require.NoError(t, err) + require.NotNil(t, blockResp.Result) + + var blockNum string + json.Unmarshal(blockResp.Result, &blockNum) + + // Now send batch with specific block number + batch := []jsonRPCRequest{ + {JSONRPC: "2.0", ID: 1, Method: "eth_call", Params: []interface{}{ethCallParams{To: wethAddress, Data: decimalsSelector}, blockNum}}, + {JSONRPC: "2.0", ID: 2, Method: "eth_call", Params: []interface{}{ethCallParams{To: wethAddress, Data: symbolSelector}, blockNum}}, + } + + respBody = makeRequest(t, endpoint, batch) + + var responses []jsonRPCResponse + err = json.Unmarshal(respBody, &responses) + require.NoError(t, err) + require.Len(t, responses, 2) + + for i, resp := range responses { + assert.Nil(t, resp.Error, "Response %d with specific block should succeed", i) + assert.NotNil(t, resp.Result, "Response %d should have result", i) + } + t.Logf("Specific block number (%s): both calls succeeded", blockNum) + }) + + t.Run("PendingBlock", func(t *testing.T) { + // By default, "pending" block tag should NOT be batched (allowPendingTagBatching=false) + // Each call should still succeed but may not be batched together + batch := []jsonRPCRequest{ + {JSONRPC: "2.0", ID: 1, Method: "eth_call", Params: []interface{}{ethCallParams{To: wethAddress, Data: decimalsSelector}, "pending"}}, + {JSONRPC: "2.0", ID: 2, Method: "eth_call", Params: []interface{}{ethCallParams{To: wethAddress, Data: symbolSelector}, "pending"}}, + } + + respBody := makeRequest(t, endpoint, batch) + + var responses []jsonRPCResponse + err := json.Unmarshal(respBody, &responses) + require.NoError(t, err) + require.Len(t, responses, 2) + + // Calls should still succeed (individually forwarded) + successCount := 0 + for _, resp := range responses { + if resp.Error == nil && resp.Result != nil { + successCount++ + } + } + + // Note: some networks don't support "pending" tag, so we just check responses are returned + t.Logf("'pending' block tag: %d/2 calls returned results (may not be batched)", successCount) + }) +} + +// TestMulticall3Integration_CacheHits verifies that repeated calls hit the cache +func TestMulticall3Integration_CacheHits(t *testing.T) { + endpoint := getTestEndpoint(t) + metricsEndpoint := getMetricsEndpoint() + + // First, get a specific block number to ensure consistent caching + blockNumReq := jsonRPCRequest{ + JSONRPC: "2.0", + ID: 1, + Method: "eth_blockNumber", + Params: []interface{}{}, + } + respBody := makeRequest(t, endpoint, blockNumReq) + + var blockResp jsonRPCResponse + err := json.Unmarshal(respBody, &blockResp) + require.NoError(t, err) + require.NotNil(t, blockResp.Result) + + var blockNum string + json.Unmarshal(blockResp.Result, &blockNum) + + // Capture baseline cache hit metric + var baselineCacheHits float64 + if metricsEndpoint != "" { + baselineCacheHits, _ = getMetricValue(metricsEndpoint, "erpc_multicall3_cache_hits_total") + } + + // Make the first request (should populate cache) + batch := []jsonRPCRequest{ + {JSONRPC: "2.0", ID: 1, Method: "eth_call", Params: []interface{}{ethCallParams{To: wethAddress, Data: decimalsSelector}, blockNum}}, + {JSONRPC: "2.0", ID: 2, Method: "eth_call", Params: []interface{}{ethCallParams{To: wethAddress, Data: symbolSelector}, blockNum}}, + } + + respBody = makeRequest(t, endpoint, batch) + var responses1 []jsonRPCResponse + err = json.Unmarshal(respBody, &responses1) + require.NoError(t, err) + require.Len(t, responses1, 2) + + // Wait a bit for cache to be written + time.Sleep(500 * time.Millisecond) + + // Make the same request again (should hit cache) + respBody = makeRequest(t, endpoint, batch) + var responses2 []jsonRPCResponse + err = json.Unmarshal(respBody, &responses2) + require.NoError(t, err) + require.Len(t, responses2, 2) + + // Results should be identical + var result1a, result1b, result2a, result2b string + json.Unmarshal(responses1[0].Result, &result1a) + json.Unmarshal(responses2[0].Result, &result1b) + json.Unmarshal(responses1[1].Result, &result2a) + json.Unmarshal(responses2[1].Result, &result2b) + + assert.Equal(t, result1a, result1b, "Cached result should match original") + assert.Equal(t, result2a, result2b, "Cached result should match original") + + t.Logf("Cache test: results match for block %s", blockNum) + + // Check if cache hit metric increased + if metricsEndpoint != "" { + time.Sleep(1 * time.Second) + + newCacheHits, err := getMetricValue(metricsEndpoint, "erpc_multicall3_cache_hits_total") + if err == nil { + diff := newCacheHits - baselineCacheHits + t.Logf("Cache hits metric: %.0f -> %.0f (+%.0f)", baselineCacheHits, newCacheHits, diff) + if diff > 0 { + t.Logf("✓ Cache hits confirmed via metrics") + } else { + t.Logf("⚠ No cache hits detected (caching may be disabled or cache not populated)") + } + } + } +} + +// TestMulticall3Integration_UseUpstreamDirective tests that UseUpstream header works with batching +func TestMulticall3Integration_UseUpstreamDirective(t *testing.T) { + endpoint := getTestEndpoint(t) + + // This test verifies that the UseUpstream directive works + // The actual upstream selection is internal, but we can verify the request succeeds + // with the header present + + req := jsonRPCRequest{ + JSONRPC: "2.0", + ID: 1, + Method: "eth_call", + Params: []interface{}{ + ethCallParams{To: wethAddress, Data: decimalsSelector}, + "latest", + }, + } + + body, err := json.Marshal(req) + require.NoError(t, err) + + httpReq, err := http.NewRequest("POST", endpoint, bytes.NewReader(body)) + require.NoError(t, err) + httpReq.Header.Set("Content-Type", "application/json") + + // Add auth headers + for key, value := range getAuthHeaders() { + httpReq.Header.Set(key, value) + } + + // Add UseUpstream directive (this may or may not match an upstream, but shouldn't error) + httpReq.Header.Set("X-ERPC-Use-Upstream", "alchemy") // Common provider name + + client := &http.Client{Timeout: 30 * time.Second} + resp, err := client.Do(httpReq) + require.NoError(t, err) + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + var jsonResp jsonRPCResponse + err = json.Unmarshal(respBody, &jsonResp) + require.NoError(t, err, "Failed to parse response: %s", string(respBody)) + + // Request should succeed (either via specified upstream or fallback) + if jsonResp.Error != nil { + t.Logf("UseUpstream directive: request returned error (upstream may not exist): %s", string(*jsonResp.Error)) + } else { + assert.NotNil(t, jsonResp.Result, "Request should have result") + t.Logf("UseUpstream directive: request succeeded") + } +} + +// TestMulticall3Integration_HighConcurrency stress tests with high concurrency +func TestMulticall3Integration_HighConcurrency(t *testing.T) { + endpoint := getTestEndpoint(t) + metricsEndpoint := getMetricsEndpoint() + + // Capture baseline metrics + var baselineAggregation, baselineOverflow float64 + if metricsEndpoint != "" { + baselineAggregation, _ = getMetricValue(metricsEndpoint, "erpc_multicall3_aggregation_total") + baselineOverflow, _ = getMetricValue(metricsEndpoint, "erpc_multicall3_queue_overflow_total") + } + + // Send 50 concurrent requests + const numRequests = 50 + var wg sync.WaitGroup + results := make(chan bool, numRequests) + + for i := 0; i < numRequests; i++ { + wg.Add(1) + go func(id int) { + defer wg.Done() + + req := jsonRPCRequest{ + JSONRPC: "2.0", + ID: id, + Method: "eth_call", + Params: []interface{}{ + ethCallParams{To: wethAddress, Data: decimalsSelector}, + "latest", + }, + } + + respBody := makeRequest(t, endpoint, req) + + var resp jsonRPCResponse + if err := json.Unmarshal(respBody, &resp); err != nil { + results <- false + return + } + + results <- resp.Error == nil && resp.Result != nil + }(i) + } + + wg.Wait() + close(results) + + successCount := 0 + for success := range results { + if success { + successCount++ + } + } + + // All requests should succeed + assert.Equal(t, numRequests, successCount, "All high-concurrency requests should succeed") + t.Logf("High concurrency: %d/%d requests successful", successCount, numRequests) + + // Check metrics + if metricsEndpoint != "" { + time.Sleep(1 * time.Second) + + newAggregation, _ := getMetricValue(metricsEndpoint, "erpc_multicall3_aggregation_total") + newOverflow, _ := getMetricValue(metricsEndpoint, "erpc_multicall3_queue_overflow_total") + + aggDiff := newAggregation - baselineAggregation + overflowDiff := newOverflow - baselineOverflow + + t.Logf("Aggregation metric: %.0f -> %.0f (+%.0f)", baselineAggregation, newAggregation, aggDiff) + t.Logf("Queue overflow metric: %.0f -> %.0f (+%.0f)", baselineOverflow, newOverflow, overflowDiff) + + if aggDiff > 0 && aggDiff < float64(numRequests)/2 { + t.Logf("✓ Efficient batching under high concurrency: %d requests → %.0f aggregations", numRequests, aggDiff) + } + + if overflowDiff > 0 { + t.Logf("⚠ Some requests overflowed (%.0f) - this is expected under extreme load", overflowDiff) + } + } +} + +// TestMulticall3Integration_DifferentBlockTags tests that calls with different block tags are NOT batched together +func TestMulticall3Integration_DifferentBlockTags(t *testing.T) { + endpoint := getTestEndpoint(t) + + // Get latest block number + blockNumReq := jsonRPCRequest{ + JSONRPC: "2.0", + ID: 1, + Method: "eth_blockNumber", + Params: []interface{}{}, + } + respBody := makeRequest(t, endpoint, blockNumReq) + + var blockResp jsonRPCResponse + json.Unmarshal(respBody, &blockResp) + var blockNum string + json.Unmarshal(blockResp.Result, &blockNum) + + // Send batch with DIFFERENT block tags - these should NOT be batched together + // (different batch keys due to different blockRef) + batch := []jsonRPCRequest{ + {JSONRPC: "2.0", ID: 1, Method: "eth_call", Params: []interface{}{ethCallParams{To: wethAddress, Data: decimalsSelector}, "latest"}}, + {JSONRPC: "2.0", ID: 2, Method: "eth_call", Params: []interface{}{ethCallParams{To: wethAddress, Data: decimalsSelector}, blockNum}}, + } + + respBody = makeRequest(t, endpoint, batch) + + var responses []jsonRPCResponse + err := json.Unmarshal(respBody, &responses) + require.NoError(t, err) + require.Len(t, responses, 2) + + // Both should succeed + for i, resp := range responses { + assert.Nil(t, resp.Error, "Response %d should succeed", i) + assert.NotNil(t, resp.Result, "Response %d should have result", i) + } + + // Results should be the same (same call, different block refs but close in time) + var result1, result2 string + json.Unmarshal(responses[0].Result, &result1) + json.Unmarshal(responses[1].Result, &result2) + + t.Logf("Different block tags: 'latest' returned %s, '%s' returned %s", result1, blockNum, result2) + // Note: Results may differ if there was a state change between blocks +} + +// TestMulticall3Integration_EmptyCalldata tests handling of calls with empty/minimal calldata +func TestMulticall3Integration_EmptyCalldata(t *testing.T) { + endpoint := getTestEndpoint(t) + + // Some contracts have fallback functions that accept empty calldata + // We're testing that the batching handles this gracefully + batch := []jsonRPCRequest{ + {JSONRPC: "2.0", ID: 1, Method: "eth_call", Params: []interface{}{ethCallParams{To: wethAddress, Data: decimalsSelector}, "latest"}}, + {JSONRPC: "2.0", ID: 2, Method: "eth_call", Params: []interface{}{ethCallParams{To: wethAddress, Data: "0x"}, "latest"}}, // Empty calldata + } + + respBody := makeRequest(t, endpoint, batch) + + var responses []jsonRPCResponse + err := json.Unmarshal(respBody, &responses) + require.NoError(t, err) + require.Len(t, responses, 2) + + // First call should succeed + assert.Nil(t, responses[0].Error, "Normal call should succeed") + assert.NotNil(t, responses[0].Result, "Normal call should have result") + + // Second call may succeed or fail depending on the contract's fallback function + if responses[1].Error != nil { + t.Logf("Empty calldata: returned error (expected for contracts without fallback)") + } else { + t.Logf("Empty calldata: returned result (contract has fallback function)") + } +} + +// TestMulticall3Integration_LargeCalldata tests handling of calls with large calldata +func TestMulticall3Integration_LargeCalldata(t *testing.T) { + endpoint := getTestEndpoint(t) + + // Create a call with larger calldata (balanceOf with address padded) + // balanceOf(address) = 0x70a08231 + 32-byte address + largeCalldata := "0x70a08231000000000000000000000000" + strings.Repeat("ab", 20) // balanceOf(0xabab...ab) + + batch := []jsonRPCRequest{ + {JSONRPC: "2.0", ID: 1, Method: "eth_call", Params: []interface{}{ethCallParams{To: wethAddress, Data: largeCalldata}, "latest"}}, + {JSONRPC: "2.0", ID: 2, Method: "eth_call", Params: []interface{}{ethCallParams{To: wethAddress, Data: decimalsSelector}, "latest"}}, + } + + respBody := makeRequest(t, endpoint, batch) + + var responses []jsonRPCResponse + err := json.Unmarshal(respBody, &responses) + require.NoError(t, err) + require.Len(t, responses, 2) + + // Both should return results (balanceOf returns 0 for unknown address, decimals returns 18) + for i, resp := range responses { + assert.Nil(t, resp.Error, "Response %d should succeed", i) + assert.NotNil(t, resp.Result, "Response %d should have result", i) + } + + t.Logf("Large calldata: both calls succeeded") +} + +// ============================================================================= +// Bypass Tests - Verify calls that should NOT be batched +// ============================================================================= + +// TestMulticall3Integration_BypassWithValueField tests that calls with 'value' field bypass batching +func TestMulticall3Integration_BypassWithValueField(t *testing.T) { + endpoint := getTestEndpoint(t) + + // eth_call with 'value' field should bypass multicall3 batching + // (multicall3 aggregate3 doesn't support value transfers) + batch := []jsonRPCRequest{ + {JSONRPC: "2.0", ID: 1, Method: "eth_call", Params: []interface{}{ + map[string]interface{}{"to": wethAddress, "data": decimalsSelector, "value": "0x0"}, + "latest", + }}, + {JSONRPC: "2.0", ID: 2, Method: "eth_call", Params: []interface{}{ + map[string]interface{}{"to": wethAddress, "data": symbolSelector}, + "latest", + }}, + } + + respBody := makeRequest(t, endpoint, batch) + + var responses []jsonRPCResponse + err := json.Unmarshal(respBody, &responses) + require.NoError(t, err, "Failed to parse response: %s", string(respBody)) + require.Len(t, responses, 2) + + // Both should still succeed (first bypasses batching, second may be batched) + for i, resp := range responses { + assert.Nil(t, resp.Error, "Response %d should succeed", i) + assert.NotNil(t, resp.Result, "Response %d should have result", i) + } + + t.Logf("Bypass with 'value' field: both calls succeeded (first bypassed batching)") +} + +// TestMulticall3Integration_BypassWithFromField tests that calls with 'from' field bypass batching +func TestMulticall3Integration_BypassWithFromField(t *testing.T) { + endpoint := getTestEndpoint(t) + + // eth_call with 'from' field should bypass multicall3 batching + batch := []jsonRPCRequest{ + {JSONRPC: "2.0", ID: 1, Method: "eth_call", Params: []interface{}{ + map[string]interface{}{"to": wethAddress, "data": decimalsSelector, "from": "0x0000000000000000000000000000000000000001"}, + "latest", + }}, + {JSONRPC: "2.0", ID: 2, Method: "eth_call", Params: []interface{}{ + map[string]interface{}{"to": wethAddress, "data": symbolSelector}, + "latest", + }}, + } + + respBody := makeRequest(t, endpoint, batch) + + var responses []jsonRPCResponse + err := json.Unmarshal(respBody, &responses) + require.NoError(t, err, "Failed to parse response: %s", string(respBody)) + require.Len(t, responses, 2) + + for i, resp := range responses { + assert.Nil(t, resp.Error, "Response %d should succeed", i) + assert.NotNil(t, resp.Result, "Response %d should have result", i) + } + + t.Logf("Bypass with 'from' field: both calls succeeded (first bypassed batching)") +} + +// TestMulticall3Integration_BypassWithGasField tests that calls with 'gas' field bypass batching +func TestMulticall3Integration_BypassWithGasField(t *testing.T) { + endpoint := getTestEndpoint(t) + + // eth_call with 'gas' field should bypass multicall3 batching + // Use a reasonable gas value (500k) to avoid "intrinsic gas too low" errors + batch := []jsonRPCRequest{ + {JSONRPC: "2.0", ID: 1, Method: "eth_call", Params: []interface{}{ + map[string]interface{}{"to": wethAddress, "data": decimalsSelector, "gas": "0x7a120"}, // 500000 gas + "latest", + }}, + {JSONRPC: "2.0", ID: 2, Method: "eth_call", Params: []interface{}{ + map[string]interface{}{"to": wethAddress, "data": symbolSelector}, + "latest", + }}, + } + + respBody := makeRequest(t, endpoint, batch) + + var responses []jsonRPCResponse + err := json.Unmarshal(respBody, &responses) + require.NoError(t, err, "Failed to parse response: %s", string(respBody)) + require.Len(t, responses, 2) + + // First call (with gas field) may succeed or fail depending on gas amount + // The important thing is that it was processed (bypassed batching) + if responses[0].Error != nil { + t.Logf("Call with 'gas' field returned error (processed, bypassed batching)") + } else { + assert.NotNil(t, responses[0].Result, "Response 0 should have result") + } + + // Second call (normal) should always succeed + assert.Nil(t, responses[1].Error, "Response 1 (normal call) should succeed") + assert.NotNil(t, responses[1].Result, "Response 1 should have result") + + t.Logf("Bypass with 'gas' field: calls handled correctly (first bypassed batching)") +} + +// TestMulticall3Integration_BypassWithStateOverride tests that calls with state override bypass batching +func TestMulticall3Integration_BypassWithStateOverride(t *testing.T) { + endpoint := getTestEndpoint(t) + + // eth_call with state override (3rd param) should bypass multicall3 batching + stateOverride := map[string]interface{}{ + wethAddress: map[string]interface{}{ + "balance": "0x1000000000000000000", + }, + } + + batch := []jsonRPCRequest{ + {JSONRPC: "2.0", ID: 1, Method: "eth_call", Params: []interface{}{ + map[string]interface{}{"to": wethAddress, "data": decimalsSelector}, + "latest", + stateOverride, // State override as 3rd param + }}, + {JSONRPC: "2.0", ID: 2, Method: "eth_call", Params: []interface{}{ + map[string]interface{}{"to": wethAddress, "data": symbolSelector}, + "latest", + }}, + } + + respBody := makeRequest(t, endpoint, batch) + + var responses []jsonRPCResponse + err := json.Unmarshal(respBody, &responses) + require.NoError(t, err, "Failed to parse response: %s", string(respBody)) + require.Len(t, responses, 2) + + // First may error if upstream doesn't support state override, second should succeed + if responses[0].Error != nil { + t.Logf("State override call returned error (upstream may not support it)") + } else { + assert.NotNil(t, responses[0].Result, "Response 0 should have result") + } + + assert.Nil(t, responses[1].Error, "Response 1 (normal call) should succeed") + assert.NotNil(t, responses[1].Result, "Response 1 should have result") + + t.Logf("Bypass with state override: calls handled correctly (first bypassed batching)") +} + +// TestMulticall3Integration_BypassRecursionGuard tests that calls to multicall3 contract bypass batching +func TestMulticall3Integration_BypassRecursionGuard(t *testing.T) { + endpoint := getTestEndpoint(t) + + // Multicall3 contract address (same on most EVM chains) + multicall3Address := "0xcA11bde05977b3631167028862bE2a173976CA11" + + // Calling multicall3 directly should bypass the batching (recursion guard) + batch := []jsonRPCRequest{ + {JSONRPC: "2.0", ID: 1, Method: "eth_call", Params: []interface{}{ + map[string]interface{}{"to": multicall3Address, "data": "0x252dba42"}, // aggregate() selector + "latest", + }}, + {JSONRPC: "2.0", ID: 2, Method: "eth_call", Params: []interface{}{ + map[string]interface{}{"to": wethAddress, "data": decimalsSelector}, + "latest", + }}, + } + + respBody := makeRequest(t, endpoint, batch) + + var responses []jsonRPCResponse + err := json.Unmarshal(respBody, &responses) + require.NoError(t, err, "Failed to parse response: %s", string(respBody)) + require.Len(t, responses, 2) + + // First call (to multicall3) may revert due to invalid calldata, but shouldn't cause batching issues + if responses[0].Error != nil { + t.Logf("Call to multicall3 returned error (expected for empty aggregate call)") + } else { + t.Logf("Call to multicall3 returned result (bypassed batching)") + } + + // Second call should succeed normally + assert.Nil(t, responses[1].Error, "Normal call should succeed") + assert.NotNil(t, responses[1].Result, "Normal call should have result") + + t.Logf("Recursion guard: calls to multicall3 contract handled correctly") +} + +// TestMulticall3Integration_BypassRequireCanonicalFalse tests that requireCanonical:false bypasses batching +func TestMulticall3Integration_BypassRequireCanonicalFalse(t *testing.T) { + endpoint := getTestEndpoint(t) + + // First get a recent block hash + blockNumReq := jsonRPCRequest{ + JSONRPC: "2.0", + ID: 1, + Method: "eth_getBlockByNumber", + Params: []interface{}{"latest", false}, + } + respBody := makeRequest(t, endpoint, blockNumReq) + + var blockResp jsonRPCResponse + err := json.Unmarshal(respBody, &blockResp) + require.NoError(t, err) + require.NotNil(t, blockResp.Result) + + var block map[string]interface{} + json.Unmarshal(blockResp.Result, &block) + blockHash, ok := block["hash"].(string) + if !ok || blockHash == "" { + t.Skip("Could not get block hash for requireCanonical test") + } + + // EIP-1898 block param with requireCanonical: false should bypass batching + batch := []jsonRPCRequest{ + {JSONRPC: "2.0", ID: 1, Method: "eth_call", Params: []interface{}{ + map[string]interface{}{"to": wethAddress, "data": decimalsSelector}, + map[string]interface{}{"blockHash": blockHash, "requireCanonical": false}, + }}, + {JSONRPC: "2.0", ID: 2, Method: "eth_call", Params: []interface{}{ + map[string]interface{}{"to": wethAddress, "data": symbolSelector}, + "latest", + }}, + } + + respBody = makeRequest(t, endpoint, batch) + + var responses []jsonRPCResponse + err = json.Unmarshal(respBody, &responses) + require.NoError(t, err, "Failed to parse response: %s", string(respBody)) + require.Len(t, responses, 2) + + // Both should succeed (first bypasses batching due to requireCanonical:false) + for i, resp := range responses { + assert.Nil(t, resp.Error, "Response %d should succeed", i) + assert.NotNil(t, resp.Result, "Response %d should have result", i) + } + + t.Logf("Bypass with requireCanonical:false: both calls succeeded") +} + +// ============================================================================= +// Block Reference Variation Tests +// ============================================================================= + +// TestMulticall3Integration_BlockHashReference tests batching with block hash (EIP-1898) +func TestMulticall3Integration_BlockHashReference(t *testing.T) { + endpoint := getTestEndpoint(t) + + // First get a recent block hash + blockReq := jsonRPCRequest{ + JSONRPC: "2.0", + ID: 1, + Method: "eth_getBlockByNumber", + Params: []interface{}{"latest", false}, + } + respBody := makeRequest(t, endpoint, blockReq) + + var blockResp jsonRPCResponse + err := json.Unmarshal(respBody, &blockResp) + require.NoError(t, err) + require.NotNil(t, blockResp.Result) + + var block map[string]interface{} + json.Unmarshal(blockResp.Result, &block) + blockHash, ok := block["hash"].(string) + if !ok || blockHash == "" { + t.Skip("Could not get block hash") + } + + // EIP-1898 block param with blockHash + batch := []jsonRPCRequest{ + {JSONRPC: "2.0", ID: 1, Method: "eth_call", Params: []interface{}{ + map[string]interface{}{"to": wethAddress, "data": decimalsSelector}, + map[string]interface{}{"blockHash": blockHash}, + }}, + {JSONRPC: "2.0", ID: 2, Method: "eth_call", Params: []interface{}{ + map[string]interface{}{"to": wethAddress, "data": symbolSelector}, + map[string]interface{}{"blockHash": blockHash}, + }}, + } + + respBody = makeRequest(t, endpoint, batch) + + var responses []jsonRPCResponse + err = json.Unmarshal(respBody, &responses) + require.NoError(t, err, "Failed to parse response: %s", string(respBody)) + require.Len(t, responses, 2) + + for i, resp := range responses { + assert.Nil(t, resp.Error, "Response %d with block hash should succeed", i) + assert.NotNil(t, resp.Result, "Response %d should have result", i) + } + + t.Logf("Block hash reference (EIP-1898): both calls succeeded with hash %s", blockHash[:18]+"...") +} + +// TestMulticall3Integration_FinalizedSafeEarliestTags tests batching with special block tags +func TestMulticall3Integration_FinalizedSafeEarliestTags(t *testing.T) { + endpoint := getTestEndpoint(t) + + testCases := []struct { + name string + blockTag string + }{ + {"finalized", "finalized"}, + {"safe", "safe"}, + {"earliest", "earliest"}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + batch := []jsonRPCRequest{ + {JSONRPC: "2.0", ID: 1, Method: "eth_call", Params: []interface{}{ + map[string]interface{}{"to": wethAddress, "data": decimalsSelector}, + tc.blockTag, + }}, + {JSONRPC: "2.0", ID: 2, Method: "eth_call", Params: []interface{}{ + map[string]interface{}{"to": wethAddress, "data": symbolSelector}, + tc.blockTag, + }}, + } + + respBody := makeRequest(t, endpoint, batch) + + var responses []jsonRPCResponse + err := json.Unmarshal(respBody, &responses) + require.NoError(t, err, "Failed to parse response: %s", string(respBody)) + require.Len(t, responses, 2) + + // Note: 'earliest' may fail on some contracts that didn't exist at genesis + // 'finalized' and 'safe' may not be supported by all nodes + successCount := 0 + for _, resp := range responses { + if resp.Error == nil && resp.Result != nil { + successCount++ + } + } + + if successCount == 2 { + t.Logf("'%s' block tag: both calls succeeded", tc.blockTag) + } else if successCount > 0 { + t.Logf("'%s' block tag: %d/2 calls succeeded (some may not be supported)", tc.blockTag, successCount) + } else { + t.Logf("'%s' block tag: calls failed (tag may not be supported by upstream)", tc.blockTag) + } + }) + } +} + +// ============================================================================= +// Input Variations Tests +// ============================================================================= + +// TestMulticall3Integration_InputFieldAlternative tests that 'input' field works as alternative to 'data' +func TestMulticall3Integration_InputFieldAlternative(t *testing.T) { + endpoint := getTestEndpoint(t) + + // Some clients use 'input' instead of 'data' - both should work + batch := []jsonRPCRequest{ + {JSONRPC: "2.0", ID: 1, Method: "eth_call", Params: []interface{}{ + map[string]interface{}{"to": wethAddress, "input": decimalsSelector}, // 'input' instead of 'data' + "latest", + }}, + {JSONRPC: "2.0", ID: 2, Method: "eth_call", Params: []interface{}{ + map[string]interface{}{"to": wethAddress, "data": symbolSelector}, // 'data' for comparison + "latest", + }}, + } + + respBody := makeRequest(t, endpoint, batch) + + var responses []jsonRPCResponse + err := json.Unmarshal(respBody, &responses) + require.NoError(t, err, "Failed to parse response: %s", string(respBody)) + require.Len(t, responses, 2) + + for i, resp := range responses { + assert.Nil(t, resp.Error, "Response %d should succeed", i) + assert.NotNil(t, resp.Result, "Response %d should have result", i) + } + + // Both should return valid results + var result1, result2 string + json.Unmarshal(responses[0].Result, &result1) + json.Unmarshal(responses[1].Result, &result2) + + assert.NotEmpty(t, result1, "'input' field should return valid result") + assert.NotEmpty(t, result2, "'data' field should return valid result") + + t.Logf("'input' field alternative: both 'input' and 'data' fields work correctly") +} + +// ============================================================================= +// Directive Tests +// ============================================================================= + +// TestMulticall3Integration_SkipCacheReadDirective tests that skip-cache-read creates separate batch +func TestMulticall3Integration_SkipCacheReadDirective(t *testing.T) { + endpoint := getTestEndpoint(t) + + req := jsonRPCRequest{ + JSONRPC: "2.0", + ID: 1, + Method: "eth_call", + Params: []interface{}{ + map[string]interface{}{"to": wethAddress, "data": decimalsSelector}, + "latest", + }, + } + + body, err := json.Marshal(req) + require.NoError(t, err) + + httpReq, err := http.NewRequest("POST", endpoint, bytes.NewReader(body)) + require.NoError(t, err) + httpReq.Header.Set("Content-Type", "application/json") + + // Add auth headers + for key, value := range getAuthHeaders() { + httpReq.Header.Set(key, value) + } + + // Add skip-cache-read directive + httpReq.Header.Set("X-ERPC-Skip-Cache-Read", "true") + + client := &http.Client{Timeout: 30 * time.Second} + resp, err := client.Do(httpReq) + require.NoError(t, err) + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + var jsonResp jsonRPCResponse + err = json.Unmarshal(respBody, &jsonResp) + require.NoError(t, err, "Failed to parse response: %s", string(respBody)) + + assert.Nil(t, jsonResp.Error, "Request with skip-cache-read should succeed") + assert.NotNil(t, jsonResp.Result, "Request should have result") + + t.Logf("Skip-cache-read directive: request succeeded") +} + +// TestMulticall3Integration_RetryEmptyDirective tests that retry-empty creates separate batch +func TestMulticall3Integration_RetryEmptyDirective(t *testing.T) { + endpoint := getTestEndpoint(t) + + req := jsonRPCRequest{ + JSONRPC: "2.0", + ID: 1, + Method: "eth_call", + Params: []interface{}{ + map[string]interface{}{"to": wethAddress, "data": decimalsSelector}, + "latest", + }, + } + + body, err := json.Marshal(req) + require.NoError(t, err) + + httpReq, err := http.NewRequest("POST", endpoint, bytes.NewReader(body)) + require.NoError(t, err) + httpReq.Header.Set("Content-Type", "application/json") + + // Add auth headers + for key, value := range getAuthHeaders() { + httpReq.Header.Set(key, value) + } + + // Add retry-empty directive + httpReq.Header.Set("X-ERPC-Retry-Empty", "true") + + client := &http.Client{Timeout: 30 * time.Second} + resp, err := client.Do(httpReq) + require.NoError(t, err) + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + var jsonResp jsonRPCResponse + err = json.Unmarshal(respBody, &jsonResp) + require.NoError(t, err, "Failed to parse response: %s", string(respBody)) + + assert.Nil(t, jsonResp.Error, "Request with retry-empty should succeed") + assert.NotNil(t, jsonResp.Result, "Request should have result") + + t.Logf("Retry-empty directive: request succeeded") +} + +// ============================================================================= +// Fallback Tests +// ============================================================================= + +// TestMulticall3Integration_FallbackMetricTracking verifies fallback metric is tracked +func TestMulticall3Integration_FallbackMetricTracking(t *testing.T) { + endpoint := getTestEndpoint(t) + metricsEndpoint := getMetricsEndpoint() + + if metricsEndpoint == "" { + t.Skip("ERPC_INTEGRATION_TEST_METRICS not set, skipping fallback metric test") + } + + // Check current fallback metric + fallbackTotal, err := getMetricValue(metricsEndpoint, "erpc_multicall3_fallback_total") + require.NoError(t, err) + + // We can't easily trigger a fallback in integration tests without a broken upstream, + // but we can verify the metric exists and is being tracked + t.Logf("Current fallback total: %.0f", fallbackTotal) + + // Also check fallback requests metric + fallbackRequests, _ := getMetricValue(metricsEndpoint, "erpc_multicall3_fallback_requests_total") + t.Logf("Current fallback requests total: %.0f", fallbackRequests) + + // Run a normal batch to ensure batching still works + batch := []jsonRPCRequest{ + {JSONRPC: "2.0", ID: 1, Method: "eth_call", Params: []interface{}{ + map[string]interface{}{"to": wethAddress, "data": decimalsSelector}, + "latest", + }}, + } + + respBody := makeRequest(t, endpoint, batch) + + var responses []jsonRPCResponse + err = json.Unmarshal(respBody, &responses) + require.NoError(t, err) + require.Len(t, responses, 1) + assert.Nil(t, responses[0].Error, "Normal call should succeed") + + // Verify fallback metric didn't increase (no fallback needed) + time.Sleep(500 * time.Millisecond) + newFallbackTotal, _ := getMetricValue(metricsEndpoint, "erpc_multicall3_fallback_total") + + if newFallbackTotal == fallbackTotal { + t.Logf("✓ No fallback triggered for normal batch (as expected)") + } else { + t.Logf("⚠ Fallback was triggered: %.0f -> %.0f", fallbackTotal, newFallbackTotal) + } +} + +// TestMulticall3Integration_MetricsSummary prints a summary of multicall3 metrics +func TestMulticall3Integration_MetricsSummary(t *testing.T) { + getTestEndpoint(t) // Ensure we have an endpoint configured + metricsEndpoint := getMetricsEndpoint() + + if metricsEndpoint == "" { + t.Skip("ERPC_INTEGRATION_TEST_METRICS not set, skipping metrics summary") + } + + metrics := []string{ + "erpc_multicall3_aggregation_total", + "erpc_multicall3_fallback_total", + "erpc_multicall3_cache_hits_total", + "erpc_multicall3_queue_overflow_total", + "erpc_multicall3_dedupe_total", + "erpc_multicall3_panic_total", + "erpc_multicall3_abandoned_total", + "erpc_multicall3_cache_write_dropped_total", + } + + t.Logf("\n=== Multicall3 Metrics Summary ===") + for _, metric := range metrics { + value, err := getMetricValue(metricsEndpoint, metric) + if err != nil { + t.Logf(" %s: error fetching", metric) + } else { + t.Logf(" %s: %.0f", metric, value) + } + } + t.Logf("===================================") +} From d76ea71c4671420fb71cc5f7cfa55294927386cc Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 19 Jan 2026 16:33:50 +0100 Subject: [PATCH 50/53] chore: remove multicall3 production test script Redundant with the Go integration tests which provide better coverage. Co-Authored-By: Claude Opus 4.5 --- scripts/test-multicall3-prd.sh | 366 --------------------------------- 1 file changed, 366 deletions(-) delete mode 100755 scripts/test-multicall3-prd.sh diff --git a/scripts/test-multicall3-prd.sh b/scripts/test-multicall3-prd.sh deleted file mode 100755 index 47c455136..000000000 --- a/scripts/test-multicall3-prd.sh +++ /dev/null @@ -1,366 +0,0 @@ -#!/bin/bash -# ============================================================================= -# Multicall3 Aggregation Production Test Script -# ============================================================================= -# This script tests if multicall3 batching is working correctly in production. -# -# Prerequisites: -# - jq installed -# - curl installed -# - Access to eRPC endpoint -# - Access to Prometheus/metrics endpoint (optional but recommended) -# -# Usage: -# ./scripts/test-multicall3-prd.sh [metrics-endpoint] -# -# Examples: -# ./scripts/test-multicall3-prd.sh https://erpc.example.com/main/evm/1 -# ./scripts/test-multicall3-prd.sh https://erpc.example.com/main/evm/1 https://erpc.example.com/metrics -# ============================================================================= - -set -euo pipefail - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Configuration -ERPC_ENDPOINT="${1:-}" -METRICS_ENDPOINT="${2:-}" - -# Multicall3 contract address (same on most EVM chains) -MULTICALL3_ADDRESS="0xcA11bde05977b3631167028862bE2a173976CA11" - -# Test contract addresses (well-known contracts on mainnet) -# WETH on Ethereum mainnet -WETH_ADDRESS="0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" -# USDC on Ethereum mainnet -USDC_ADDRESS="0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48" -# DAI on Ethereum mainnet -DAI_ADDRESS="0x6B175474E89094C44Da98b954EesdeAfe4068538" - -# Function signatures (for eth_call) -# balanceOf(address) = 0x70a08231 -# decimals() = 0x313ce567 -# symbol() = 0x95d89b41 -# totalSupply() = 0x18160ddd - -usage() { - echo "Usage: $0 [metrics-endpoint]" - echo "" - echo "Examples:" - echo " $0 https://erpc.example.com/main/evm/1" - echo " $0 https://erpc.example.com/main/evm/1 https://erpc.example.com/metrics" - exit 1 -} - -log_info() { - echo -e "${BLUE}[INFO]${NC} $1" -} - -log_success() { - echo -e "${GREEN}[PASS]${NC} $1" -} - -log_warning() { - echo -e "${YELLOW}[WARN]${NC} $1" -} - -log_error() { - echo -e "${RED}[FAIL]${NC} $1" -} - -log_section() { - echo "" - echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" - echo -e "${BLUE} $1${NC}" - echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" -} - -# Validate arguments -if [[ -z "$ERPC_ENDPOINT" ]]; then - usage -fi - -# ============================================================================= -# Test 1: Basic connectivity -# ============================================================================= -log_section "Test 1: Basic Connectivity" - -log_info "Testing endpoint connectivity..." -CHAIN_ID_RESPONSE=$(curl -s -X POST "$ERPC_ENDPOINT" \ - -H "Content-Type: application/json" \ - -d '{"jsonrpc":"2.0","id":1,"method":"eth_chainId","params":[]}') - -if echo "$CHAIN_ID_RESPONSE" | jq -e '.result' > /dev/null 2>&1; then - CHAIN_ID=$(echo "$CHAIN_ID_RESPONSE" | jq -r '.result') - log_success "Endpoint reachable, chainId: $CHAIN_ID" -else - log_error "Failed to connect to endpoint" - echo "Response: $CHAIN_ID_RESPONSE" - exit 1 -fi - -# ============================================================================= -# Test 2: Capture baseline metrics (if metrics endpoint provided) -# ============================================================================= -BASELINE_AGGREGATION_COUNT="" -BASELINE_FALLBACK_COUNT="" - -if [[ -n "$METRICS_ENDPOINT" ]]; then - log_section "Test 2: Capture Baseline Metrics" - - log_info "Fetching baseline multicall3 metrics..." - METRICS=$(curl -s "$METRICS_ENDPOINT" 2>/dev/null || echo "") - - if [[ -n "$METRICS" ]]; then - BASELINE_AGGREGATION_COUNT=$(echo "$METRICS" | grep 'erpc_multicall3_aggregation_total' | grep -v '^#' | awk '{sum += $2} END {print sum}' || echo "0") - BASELINE_FALLBACK_COUNT=$(echo "$METRICS" | grep 'erpc_multicall3_fallback_total' | grep -v '^#' | awk '{sum += $2} END {print sum}' || echo "0") - - log_success "Baseline aggregation count: ${BASELINE_AGGREGATION_COUNT:-0}" - log_success "Baseline fallback count: ${BASELINE_FALLBACK_COUNT:-0}" - else - log_warning "Could not fetch metrics, skipping metric-based verification" - fi -else - log_section "Test 2: Metrics Check (Skipped)" - log_warning "No metrics endpoint provided, skipping metric-based verification" -fi - -# ============================================================================= -# Test 3: Single eth_call (should NOT be batched - baseline) -# ============================================================================= -log_section "Test 3: Single eth_call (Baseline)" - -log_info "Sending single eth_call request..." -# Call decimals() on WETH -SINGLE_CALL_RESPONSE=$(curl -s -X POST "$ERPC_ENDPOINT" \ - -H "Content-Type: application/json" \ - -d '{ - "jsonrpc":"2.0", - "id":1, - "method":"eth_call", - "params":[{ - "to":"'"$WETH_ADDRESS"'", - "data":"0x313ce567" - },"latest"] - }') - -if echo "$SINGLE_CALL_RESPONSE" | jq -e '.result' > /dev/null 2>&1; then - DECIMALS=$(echo "$SINGLE_CALL_RESPONSE" | jq -r '.result') - log_success "Single eth_call succeeded, decimals: $DECIMALS" -else - log_error "Single eth_call failed" - echo "Response: $SINGLE_CALL_RESPONSE" -fi - -# ============================================================================= -# Test 4: JSON-RPC batch with multiple eth_calls (should be batched via multicall3) -# ============================================================================= -log_section "Test 4: JSON-RPC Batch (Multiple eth_calls)" - -log_info "Sending JSON-RPC batch with 3 eth_call requests..." -log_info "These should be aggregated into a single multicall3 call" - -BATCH_RESPONSE=$(curl -s -X POST "$ERPC_ENDPOINT" \ - -H "Content-Type: application/json" \ - -d '[ - {"jsonrpc":"2.0","id":1,"method":"eth_call","params":[{"to":"'"$WETH_ADDRESS"'","data":"0x313ce567"},"latest"]}, - {"jsonrpc":"2.0","id":2,"method":"eth_call","params":[{"to":"'"$WETH_ADDRESS"'","data":"0x95d89b41"},"latest"]}, - {"jsonrpc":"2.0","id":3,"method":"eth_call","params":[{"to":"'"$WETH_ADDRESS"'","data":"0x18160ddd"},"latest"]} - ]') - -# Check if all 3 responses are valid -VALID_COUNT=$(echo "$BATCH_RESPONSE" | jq '[.[] | select(.result != null)] | length') - -if [[ "$VALID_COUNT" == "3" ]]; then - log_success "JSON-RPC batch returned 3 valid responses" - - # Extract results - DECIMALS=$(echo "$BATCH_RESPONSE" | jq -r '.[0].result') - SYMBOL=$(echo "$BATCH_RESPONSE" | jq -r '.[1].result') - TOTAL_SUPPLY=$(echo "$BATCH_RESPONSE" | jq -r '.[2].result') - - log_info " decimals(): $DECIMALS" - log_info " symbol(): $SYMBOL" - log_info " totalSupply(): $TOTAL_SUPPLY" -else - log_error "JSON-RPC batch did not return 3 valid responses" - echo "Response: $BATCH_RESPONSE" -fi - -# ============================================================================= -# Test 5: Concurrent requests (should be batched together) -# ============================================================================= -log_section "Test 5: Concurrent Requests (Batching Test)" - -log_info "Sending 5 concurrent eth_call requests..." -log_info "These should be batched together within the window" - -# Create temp files for responses -TEMP_DIR=$(mktemp -d) -trap "rm -rf $TEMP_DIR" EXIT - -# Send 5 concurrent requests -for i in {1..5}; do - curl -s -X POST "$ERPC_ENDPOINT" \ - -H "Content-Type: application/json" \ - -d '{"jsonrpc":"2.0","id":'$i',"method":"eth_call","params":[{"to":"'"$WETH_ADDRESS"'","data":"0x313ce567"},"latest"]}' \ - > "$TEMP_DIR/response_$i.json" & -done - -# Wait for all requests to complete -wait - -# Check responses -SUCCESS_COUNT=0 -for i in {1..5}; do - if jq -e '.result' "$TEMP_DIR/response_$i.json" > /dev/null 2>&1; then - ((SUCCESS_COUNT++)) - fi -done - -if [[ "$SUCCESS_COUNT" == "5" ]]; then - log_success "All 5 concurrent requests succeeded" -else - log_error "Only $SUCCESS_COUNT/5 concurrent requests succeeded" -fi - -# ============================================================================= -# Test 6: Mixed batch (eth_call + other methods) -# ============================================================================= -log_section "Test 6: Mixed Batch (eth_call + other methods)" - -log_info "Sending mixed batch with eth_call and eth_blockNumber..." - -MIXED_BATCH_RESPONSE=$(curl -s -X POST "$ERPC_ENDPOINT" \ - -H "Content-Type: application/json" \ - -d '[ - {"jsonrpc":"2.0","id":1,"method":"eth_call","params":[{"to":"'"$WETH_ADDRESS"'","data":"0x313ce567"},"latest"]}, - {"jsonrpc":"2.0","id":2,"method":"eth_blockNumber","params":[]}, - {"jsonrpc":"2.0","id":3,"method":"eth_call","params":[{"to":"'"$WETH_ADDRESS"'","data":"0x18160ddd"},"latest"]} - ]') - -VALID_COUNT=$(echo "$MIXED_BATCH_RESPONSE" | jq '[.[] | select(.result != null)] | length') - -if [[ "$VALID_COUNT" == "3" ]]; then - log_success "Mixed batch returned 3 valid responses" -else - log_error "Mixed batch did not return 3 valid responses" - echo "Response: $MIXED_BATCH_RESPONSE" -fi - -# ============================================================================= -# Test 7: eth_call that reverts (error handling) -# ============================================================================= -log_section "Test 7: eth_call That Reverts (Error Handling)" - -log_info "Sending eth_call that should revert..." -# Call a non-existent function on a contract -REVERT_RESPONSE=$(curl -s -X POST "$ERPC_ENDPOINT" \ - -H "Content-Type: application/json" \ - -d '{ - "jsonrpc":"2.0", - "id":1, - "method":"eth_call", - "params":[{ - "to":"'"$WETH_ADDRESS"'", - "data":"0xdeadbeef" - },"latest"] - }') - -# Check if we got an error or empty result (both are acceptable for invalid call) -if echo "$REVERT_RESPONSE" | jq -e '.error' > /dev/null 2>&1; then - ERROR_MSG=$(echo "$REVERT_RESPONSE" | jq -r '.error.message // .error') - log_success "Revert handled correctly with error: $ERROR_MSG" -elif echo "$REVERT_RESPONSE" | jq -e '.result' > /dev/null 2>&1; then - RESULT=$(echo "$REVERT_RESPONSE" | jq -r '.result') - if [[ "$RESULT" == "0x" ]] || [[ -z "$RESULT" ]]; then - log_success "Revert returned empty result (expected)" - else - log_warning "Got unexpected result for invalid call: $RESULT" - fi -else - log_warning "Unexpected response format for revert test" - echo "Response: $REVERT_RESPONSE" -fi - -# ============================================================================= -# Test 8: Verify metrics increased (if metrics endpoint provided) -# ============================================================================= -if [[ -n "$METRICS_ENDPOINT" ]] && [[ -n "$BASELINE_AGGREGATION_COUNT" ]]; then - log_section "Test 8: Verify Metrics Increased" - - # Wait a moment for metrics to be updated - sleep 2 - - log_info "Fetching updated multicall3 metrics..." - METRICS=$(curl -s "$METRICS_ENDPOINT" 2>/dev/null || echo "") - - if [[ -n "$METRICS" ]]; then - NEW_AGGREGATION_COUNT=$(echo "$METRICS" | grep 'erpc_multicall3_aggregation_total' | grep -v '^#' | awk '{sum += $2} END {print sum}' || echo "0") - NEW_FALLBACK_COUNT=$(echo "$METRICS" | grep 'erpc_multicall3_fallback_total' | grep -v '^#' | awk '{sum += $2} END {print sum}' || echo "0") - - AGGREGATION_DIFF=$((${NEW_AGGREGATION_COUNT:-0} - ${BASELINE_AGGREGATION_COUNT:-0})) - FALLBACK_DIFF=$((${NEW_FALLBACK_COUNT:-0} - ${BASELINE_FALLBACK_COUNT:-0})) - - log_info "Aggregation count: ${BASELINE_AGGREGATION_COUNT:-0} -> ${NEW_AGGREGATION_COUNT:-0} (+$AGGREGATION_DIFF)" - log_info "Fallback count: ${BASELINE_FALLBACK_COUNT:-0} -> ${NEW_FALLBACK_COUNT:-0} (+$FALLBACK_DIFF)" - - if [[ "$AGGREGATION_DIFF" -gt 0 ]]; then - log_success "Multicall3 aggregation is working! ($AGGREGATION_DIFF new aggregations)" - else - log_warning "No new aggregations detected - multicall3 might not be enabled or requests fell back" - fi - - if [[ "$FALLBACK_DIFF" -gt 0 ]]; then - log_warning "$FALLBACK_DIFF requests fell back to individual calls" - fi - - # Show other relevant metrics - log_info "" - log_info "Additional metrics:" - echo "$METRICS" | grep 'erpc_multicall3_' | grep -v '^#' | head -20 || true - else - log_warning "Could not fetch updated metrics" - fi -fi - -# ============================================================================= -# Summary -# ============================================================================= -log_section "Test Summary" - -echo "" -echo "Tests completed. Review the output above to verify:" -echo "" -echo " 1. ✓ Basic connectivity works" -echo " 2. ✓ Single eth_call works (baseline)" -echo " 3. ✓ JSON-RPC batch with multiple eth_calls works" -echo " 4. ✓ Concurrent requests are handled" -echo " 5. ✓ Mixed batches work correctly" -echo " 6. ✓ Reverts are handled gracefully" -echo "" -echo "To confirm multicall3 batching is actually happening:" -echo "" -echo " • Check erpc_multicall3_aggregation_total metric increased" -echo " • Check erpc_multicall3_batch_size histogram for batch sizes > 1" -echo " • Check logs for 'multicall3' mentions" -echo "" - -if [[ -n "$METRICS_ENDPOINT" ]]; then - echo "Useful PromQL queries:" - echo "" - echo " # Aggregation rate" - echo " rate(erpc_multicall3_aggregation_total[5m])" - echo "" - echo " # Average batch size" - echo " histogram_quantile(0.5, rate(erpc_multicall3_batch_size_bucket[5m]))" - echo "" - echo " # Fallback rate (should be low)" - echo " rate(erpc_multicall3_fallback_total[5m])" - echo "" -fi From 9bb9e7e89440b4fd91300913ea0985bff5fc8802 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 19 Jan 2026 17:49:59 +0100 Subject: [PATCH 51/53] fix: merge Multicall3Aggregation config from networkDefaults Adds Multicall3Aggregation to the list of EVM config fields that are merged from networkDefaults when a network has its own evm section. This allows disabling multicall3 globally via networkDefaults.evm. Co-Authored-By: Claude Opus 4.5 --- common/defaults.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/common/defaults.go b/common/defaults.go index 4e726e1f6..6e22553dc 100644 --- a/common/defaults.go +++ b/common/defaults.go @@ -1803,6 +1803,10 @@ func (n *NetworkConfig) SetDefaults(upstreams []*UpstreamConfig, defaults *Netwo if n.Evm.GetLogsSplitConcurrency == 0 && defaults.Evm.GetLogsSplitConcurrency != 0 { n.Evm.GetLogsSplitConcurrency = defaults.Evm.GetLogsSplitConcurrency } + if n.Evm.Multicall3Aggregation == nil && defaults.Evm.Multicall3Aggregation != nil { + n.Evm.Multicall3Aggregation = &Multicall3AggregationConfig{} + *n.Evm.Multicall3Aggregation = *defaults.Evm.Multicall3Aggregation + } } else if n.Evm == nil && defaults.Evm != nil { n.Evm = &EvmNetworkConfig{} *n.Evm = *defaults.Evm From 19bec07a458075cad72b0adc0949fc7c66cb41d3 Mon Sep 17 00:00:00 2001 From: "aram.eth" Date: Tue, 20 Jan 2026 17:11:16 +0100 Subject: [PATCH 52/53] fix: avoid panicking and fail-open when rate limiter redis is not available (#717) --- erpc/config_analyzer.go | 3 +- erpc/erpc.go | 1 + erpc/evm_json_rpc_cache_test.go | 6 +- erpc/networks_availability_test.go | 34 ++--- erpc/networks_bench_test.go | 8 +- erpc/networks_bootstrap_test.go | 6 +- erpc/networks_earliest_detection_test.go | 12 +- erpc/networks_failsafe_test.go | 4 +- erpc/networks_forward_test.go | 8 +- erpc/networks_hedge_test.go | 2 +- erpc/networks_integrity_test.go | 2 +- erpc/networks_interpolation_test.go | 6 +- erpc/networks_multiplexer_test.go | 2 +- erpc/networks_sendrawtx_test.go | 2 +- erpc/networks_test.go | 128 +++++++++--------- erpc/policy_evaluator_test.go | 2 +- erpc/projects_test.go | 10 +- erpc/upstream_selection_test.go | 2 +- upstream/ratelimiter_budget.go | 23 +++- upstream/ratelimiter_budget_bench_test.go | 16 ++- upstream/ratelimiter_registry.go | 146 +++++++++++++++------ upstream/ratelimiter_test.go | 12 +- upstream/registry_contention_bench_test.go | 2 +- 23 files changed, 260 insertions(+), 177 deletions(-) diff --git a/erpc/config_analyzer.go b/erpc/config_analyzer.go index c0854052c..5df31d08c 100644 --- a/erpc/config_analyzer.go +++ b/erpc/config_analyzer.go @@ -325,7 +325,7 @@ func GenerateValidationReport(ctx context.Context, cfg *common.Config) *Validati } clReg := clients.NewClientRegistry(&silent, project.Id, prxPool, evm.NewJsonRpcErrorExtractor()) vndReg := thirdparty.NewVendorsRegistry() - rlr, err := upstream.NewRateLimitersRegistry(cfg.RateLimiters, &silent) + rlr, err := upstream.NewRateLimitersRegistry(ctx, cfg.RateLimiters, &silent) if err != nil { appendErr(fmt.Sprintf("project=%s failed to create rate limiters registry: %v", project.Id, err)) continue @@ -806,6 +806,7 @@ func validateUpstreamEndpoints(ctx context.Context, cfg *common.Config, logger z ) vndReg := thirdparty.NewVendorsRegistry() rlr, err := upstream.NewRateLimitersRegistry( + ctx, cfg.RateLimiters, &logger, ) diff --git a/erpc/erpc.go b/erpc/erpc.go index d9b29cb04..b28042bc7 100644 --- a/erpc/erpc.go +++ b/erpc/erpc.go @@ -33,6 +33,7 @@ func NewERPC( } rateLimitersRegistry, err := upstream.NewRateLimitersRegistry( + appCtx, cfg.RateLimiters, logger, ) diff --git a/erpc/evm_json_rpc_cache_test.go b/erpc/evm_json_rpc_cache_test.go index 21bed49e0..b49510168 100644 --- a/erpc/evm_json_rpc_cache_test.go +++ b/erpc/evm_json_rpc_cache_test.go @@ -99,7 +99,7 @@ func createCacheTestFixtures(ctx context.Context, upstreamConfigs []upsTestCfg) for _, cfg := range upstreamConfigs { mt := health.NewTracker(&logger, "prjA", 100*time.Second) - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &logger) + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &logger) if err != nil { panic(err) } @@ -2622,7 +2622,7 @@ func createMockUpstream(t *testing.T, ctx context.Context, chainId int64, upstre require.NoError(t, err) mt := health.NewTracker(&logger, "prjA", 100*time.Second) - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &logger) + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &logger) require.NoError(t, err) mockUpstream, err := upstream.NewUpstream(ctx, "test", &common.UpstreamConfig{ @@ -3244,7 +3244,7 @@ func createCacheTestFixturesWithCompression(ctx context.Context, upstreamConfigs for _, cfg := range upstreamConfigs { mt := health.NewTracker(&logger, "prjA", 100*time.Second) - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &logger) + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &logger) if err != nil { panic(err) } diff --git a/erpc/networks_availability_test.go b/erpc/networks_availability_test.go index 0a076068d..32195116a 100644 --- a/erpc/networks_availability_test.go +++ b/erpc/networks_availability_test.go @@ -49,7 +49,7 @@ func TestNetworkAvailability_LowerExactBlock_Skip(t *testing.T) { }, } - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) @@ -114,7 +114,7 @@ func TestNetworkAvailability_LowerLatestMinus_Skip(t *testing.T) { }, } - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) @@ -187,7 +187,7 @@ func TestNetworkAvailability_LowerEarliestPlus_InitAndSkip(t *testing.T) { }, } - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) @@ -297,7 +297,7 @@ func TestNetworkAvailability_InvalidRange_FailOpen_AllowsRequest(t *testing.T) { Reply(200). JSON([]byte(`{"jsonrpc":"2.0","id":1,"result":{"number":"0x1"}}`)) - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) @@ -374,7 +374,7 @@ func TestNetworkAvailability_Window_ExactLowerUpper(t *testing.T) { Reply(200). JSON([]byte(`{"jsonrpc":"2.0","id":1,"result":{"number":"0x64"}}`)) - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) @@ -475,7 +475,7 @@ func TestNetworkAvailability_EarliestPlus_Freeze_NoAdvance(t *testing.T) { return strings.Contains(b, "\"eth_getBlockByNumber\"") && strings.Contains(b, "\"0x3\"") }).Reply(200).JSON([]byte(`{"jsonrpc":"2.0","id":1,"result":{"number":"0x3"}}`)) - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) @@ -564,7 +564,7 @@ func TestNetworkAvailability_EarliestPlus_UpdateRate_Advance(t *testing.T) { return strings.Contains(b, "\"eth_getBlockByNumber\"") && strings.Contains(b, "\"0x1\"") }).Reply(200).JSON([]byte(`{"jsonrpc":"2.0","id":1,"result":{"number":"0x1"}}`)) - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) @@ -637,7 +637,7 @@ func TestNetworkAvailability_UnsupportedProbe_FailOpen(t *testing.T) { return strings.Contains(b, "\"eth_getBlockByNumber\"") && strings.Contains(b, "\"0x0\"") }).Reply(200).JSON([]byte(`{"jsonrpc":"2.0","id":1,"result":{"number":"0x0"}}`)) - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) @@ -718,7 +718,7 @@ func TestNetworkAvailability_UpperEarliestPlus_Enforced(t *testing.T) { return strings.Contains(b, "\"eth_getBlockByNumber\"") && !strings.Contains(b, "\"0x0\"") && !strings.Contains(b, "\"0x1\"") }).Reply(200).JSON([]byte(`{"jsonrpc":"2.0","id":1,"result":{"number":"0x1"}}`)) - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) @@ -803,7 +803,7 @@ func TestNetworkAvailability_Enforce_Precedence_DefaultDoesNotOverrideMethod(t * }, } - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) @@ -882,7 +882,7 @@ func TestNetworkAvailability_Enforce_Precedence_DefaultDoesNotOverrideNetwork(t }, } - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) @@ -953,7 +953,7 @@ func TestNetworkAvailability_Enforce_DefaultFalse_Disables_WhenNoExplicitConfig( }, } - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) @@ -1019,7 +1019,7 @@ func TestNetworkAvailability_Enforce_NetworkFalse_Disables(t *testing.T) { }, } - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) @@ -1073,7 +1073,7 @@ func TestCheckUpstreamBlockAvailability_BlockBeyondLatest_ReturnsRetryableError( }, } - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) @@ -1140,7 +1140,7 @@ func TestCheckUpstreamBlockAvailability_SmallDistance_IsRetryable(t *testing.T) }, } - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) @@ -1231,7 +1231,7 @@ func TestCheckUpstreamBlockAvailability_ErrorHasCorrectDetails(t *testing.T) { }, } - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) @@ -1312,7 +1312,7 @@ func TestRetryableBlockUnavailability_NoInfiniteLoop(t *testing.T) { }, } - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) diff --git a/erpc/networks_bench_test.go b/erpc/networks_bench_test.go index 50ffed774..bfa7f097d 100644 --- a/erpc/networks_bench_test.go +++ b/erpc/networks_bench_test.go @@ -36,7 +36,7 @@ func BenchmarkNetworkForward_SimpleSuccess(b *testing.B) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -141,7 +141,7 @@ func BenchmarkNetworkForward_MethodIgnoreCase(b *testing.B) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -252,7 +252,7 @@ func BenchmarkNetworkForward_RetryFailures(b *testing.B) { MaxAttempts: 3, }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -368,7 +368,7 @@ func BenchmarkNetworkForward_ConcurrentEthGetLogsIntegrityEnabled(b *testing.B) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { diff --git a/erpc/networks_bootstrap_test.go b/erpc/networks_bootstrap_test.go index da028759a..2414c9a2c 100644 --- a/erpc/networks_bootstrap_test.go +++ b/erpc/networks_bootstrap_test.go @@ -52,7 +52,7 @@ func TestNetworksBootstrap_SlowProviderUpstreams_InitializeThenServe(t *testing. }) require.NoError(t, err) - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) require.NoError(t, err) upr := upstream.NewUpstreamsRegistry( @@ -100,7 +100,7 @@ func TestNetworksBootstrap_UnsupportedNetwork_FatalFast(t *testing.T) { ssr, err := data.NewSharedStateRegistry(ctx, &log.Logger, &common.SharedStateConfig{Connector: &common.ConnectorConfig{Driver: "memory", Memory: &common.MemoryConnectorConfig{MaxItems: 100_000, MaxTotalSize: "1GB"}}}) require.NoError(t, err) - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) require.NoError(t, err) upr := upstream.NewUpstreamsRegistry(ctx, &log.Logger, "prjA", []*common.UpstreamConfig{}, ssr, rlr, vr, pr, nil, mt, 1*time.Second, nil) @@ -141,7 +141,7 @@ func TestNetworksBootstrap_ProviderInitializing_503Retry(t *testing.T) { ssr, err := data.NewSharedStateRegistry(ctx, &log.Logger, &common.SharedStateConfig{Connector: &common.ConnectorConfig{Driver: "memory", Memory: &common.MemoryConnectorConfig{MaxItems: 100_000, MaxTotalSize: "1GB"}}}) require.NoError(t, err) - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) require.NoError(t, err) upr := upstream.NewUpstreamsRegistry(ctx, &log.Logger, "prjA", []*common.UpstreamConfig{}, ssr, rlr, vr, pr, nil, mt, 1*time.Second, nil) diff --git a/erpc/networks_earliest_detection_test.go b/erpc/networks_earliest_detection_test.go index 3bf32856c..9ae66faa9 100644 --- a/erpc/networks_earliest_detection_test.go +++ b/erpc/networks_earliest_detection_test.go @@ -61,7 +61,7 @@ func TestEarliestDetection_FailOpenWhenNoEarliestConfigured(t *testing.T) { "result": map[string]interface{}{"number": "0x5"}, }) - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) @@ -174,7 +174,7 @@ func TestEarliestDetection_BlocksRequestAfterSuccessfulDetection(t *testing.T) { "result": nil, // Pruned }) - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) @@ -259,7 +259,7 @@ func TestEarliestDetection_InitialDetectionAlwaysRunsOnBootstrap(t *testing.T) { }, }) - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) @@ -351,7 +351,7 @@ func TestEarliestDetection_SchedulerHandlesPeriodicUpdates(t *testing.T) { }, }) - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) @@ -450,7 +450,7 @@ func TestEarliestDetection_InvalidRangeTriggersFailOpen(t *testing.T) { "result": nil, }) - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) @@ -589,7 +589,7 @@ func TestEarliestDetection_StaleHighValueInSharedState(t *testing.T) { }, }) - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) diff --git a/erpc/networks_failsafe_test.go b/erpc/networks_failsafe_test.go index b458f24ae..43adeb23c 100644 --- a/erpc/networks_failsafe_test.go +++ b/erpc/networks_failsafe_test.go @@ -725,7 +725,7 @@ func setupTestNetworkWithRetryConfig(t *testing.T, ctx context.Context, directiv }}, } - rateLimitersRegistry, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rateLimitersRegistry, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) require.NoError(t, err) metricsTracker := health.NewTracker(&log.Logger, "test", time.Minute) @@ -801,7 +801,7 @@ func setupTestNetworkWithMultipleFailsafePolicies(t *testing.T, ctx context.Cont Failsafe: failsafeConfigs, } - rateLimitersRegistry, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rateLimitersRegistry, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) require.NoError(t, err) metricsTracker := health.NewTracker(&log.Logger, "test", time.Minute) diff --git a/erpc/networks_forward_test.go b/erpc/networks_forward_test.go index fc6b6c8f1..da52acf41 100644 --- a/erpc/networks_forward_test.go +++ b/erpc/networks_forward_test.go @@ -44,7 +44,7 @@ func TestNetwork_Forward_InfiniteLoopWithAllUpstreamsSkipping(t *testing.T) { }, }, }) - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, logger) mt := health.NewTracker(logger, "testProject", 2*time.Second) @@ -132,7 +132,7 @@ func TestNetwork_Forward_InfiniteLoopWithAllUpstreamsSkipping(t *testing.T) { }) require.NoError(t, err) - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{Budgets: []*common.RateLimitBudgetConfig{}}, logger) + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{Budgets: []*common.RateLimitBudgetConfig{}}, logger) require.NoError(t, err) mt := health.NewTracker(logger, "testProject", 2*time.Second) @@ -212,7 +212,7 @@ func TestNetwork_Forward_InfiniteLoopWithAllUpstreamsSkipping(t *testing.T) { require.NoError(t, err) // Setup rate limiters registry - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, logger) require.NoError(t, err) @@ -396,7 +396,7 @@ func TestNetwork_Forward_InfiniteLoopWithAllUpstreamsSkipping(t *testing.T) { }) require.NoError(t, err) - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, logger) require.NoError(t, err) diff --git a/erpc/networks_hedge_test.go b/erpc/networks_hedge_test.go index d1ffb0445..db045a929 100644 --- a/erpc/networks_hedge_test.go +++ b/erpc/networks_hedge_test.go @@ -968,7 +968,7 @@ func setupTestNetworkWithMultipleUpstreams(t *testing.T, ctx context.Context, nu func setupTestNetwork(t *testing.T, ctx context.Context, upstreamConfigs []*common.UpstreamConfig, networkConfig *common.NetworkConfig) *Network { t.Helper() - rateLimitersRegistry, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rateLimitersRegistry, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) require.NoError(t, err) metricsTracker := health.NewTracker(&log.Logger, "test", time.Minute) diff --git a/erpc/networks_integrity_test.go b/erpc/networks_integrity_test.go index 7e43061e0..f9b0fce30 100644 --- a/erpc/networks_integrity_test.go +++ b/erpc/networks_integrity_test.go @@ -50,7 +50,7 @@ func mustHexToBytes(hex string) []byte { // Helper to setup test network for integrity tests func setupIntegrityTestNetwork(t *testing.T, ctx context.Context, upstreams []*common.UpstreamConfig, ntwCfg *common.NetworkConfig) (*Network, *upstream.UpstreamsRegistry) { - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) diff --git a/erpc/networks_interpolation_test.go b/erpc/networks_interpolation_test.go index 1e6575d48..d8c3152f7 100644 --- a/erpc/networks_interpolation_test.go +++ b/erpc/networks_interpolation_test.go @@ -39,7 +39,7 @@ func setupTestNetworkForInterpolation(t *testing.T, ctx context.Context, network }, } - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) @@ -1936,7 +1936,7 @@ func TestInterpolation_UpstreamSkipping_OnInterpolatedLatest(t *testing.T) { "result": "0x1234", }) - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) @@ -2054,7 +2054,7 @@ func TestInterpolation_UpstreamSkipping_DisabledByMethodConfig(t *testing.T) { "result": "0x99", }) - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) diff --git a/erpc/networks_multiplexer_test.go b/erpc/networks_multiplexer_test.go index eb618f4b2..ee532be3d 100644 --- a/erpc/networks_multiplexer_test.go +++ b/erpc/networks_multiplexer_test.go @@ -387,7 +387,7 @@ func setupTestNetworkForMultiplexer(t *testing.T, ctx context.Context) *Network // No caching to test pure multiplexing } - rateLimitersRegistry, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rateLimitersRegistry, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) require.NoError(t, err) metricsTracker := health.NewTracker(&log.Logger, "test", time.Minute) diff --git a/erpc/networks_sendrawtx_test.go b/erpc/networks_sendrawtx_test.go index 1db09ba3c..ffbb258c0 100644 --- a/erpc/networks_sendrawtx_test.go +++ b/erpc/networks_sendrawtx_test.go @@ -1671,7 +1671,7 @@ func setupSendRawTxTestNetworkWithRetryAndHedge(t *testing.T, ctx context.Contex func setupSendRawTxNetwork(t *testing.T, ctx context.Context, upstreamConfigs []*common.UpstreamConfig, networkConfig *common.NetworkConfig) *Network { t.Helper() - rateLimitersRegistry, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rateLimitersRegistry, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) require.NoError(t, err) metricsTracker := health.NewTracker(&log.Logger, "test", time.Minute) diff --git a/erpc/networks_test.go b/erpc/networks_test.go index 117662520..c2c57c7d2 100644 --- a/erpc/networks_test.go +++ b/erpc/networks_test.go @@ -45,7 +45,7 @@ func TestNetwork_Forward(t *testing.T) { util.SetupMocksForEvmStatePoller() defer util.AssertNoPendingMocks(t, 0) - rateLimitersRegistry, err := upstream.NewRateLimitersRegistry( + rateLimitersRegistry, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Store: &common.RateLimitStoreConfig{ Driver: "memory", @@ -212,7 +212,7 @@ func TestNetwork_Forward(t *testing.T) { EmptyResultMaxAttempts: 2, // cap empties at 2 total attempts }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{Budgets: []*common.RateLimitBudgetConfig{}}, &log.Logger) + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{Budgets: []*common.RateLimitBudgetConfig{}}, &log.Logger) if err != nil { t.Fatal(err) } @@ -331,7 +331,7 @@ func TestNetwork_Forward(t *testing.T) { fsCfg := &common.FailsafeConfig{ Retry: &common.RetryPolicyConfig{MaxAttempts: 3}, // no EmptyResultMaxAttempts set } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{Budgets: []*common.RateLimitBudgetConfig{}}, &log.Logger) + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{Budgets: []*common.RateLimitBudgetConfig{}}, &log.Logger) if err != nil { t.Fatal(err) } @@ -447,7 +447,7 @@ func TestNetwork_Forward(t *testing.T) { } clr := clients.NewClientRegistry(&log.Logger, "prjA", nil, evm.NewJsonRpcErrorExtractor()) fsCfg := &common.FailsafeConfig{Retry: &common.RetryPolicyConfig{MaxAttempts: 3, EmptyResultMaxAttempts: 2}} - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{Budgets: []*common.RateLimitBudgetConfig{}}, &log.Logger) + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{Budgets: []*common.RateLimitBudgetConfig{}}, &log.Logger) if err != nil { t.Fatal(err) } @@ -541,7 +541,7 @@ func TestNetwork_Forward(t *testing.T) { } clr := clients.NewClientRegistry(&log.Logger, "prjA", nil, evm.NewJsonRpcErrorExtractor()) fsCfg := &common.FailsafeConfig{Retry: &common.RetryPolicyConfig{MaxAttempts: 5, EmptyResultMaxAttempts: 2}} - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{Budgets: []*common.RateLimitBudgetConfig{}}, &log.Logger) + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{Budgets: []*common.RateLimitBudgetConfig{}}, &log.Logger) if err != nil { t.Fatal(err) } @@ -627,7 +627,7 @@ func TestNetwork_Forward(t *testing.T) { } clr := clients.NewClientRegistry(&log.Logger, "prjA", nil, evm.NewJsonRpcErrorExtractor()) fsCfg := &common.FailsafeConfig{Retry: &common.RetryPolicyConfig{MaxAttempts: 5, EmptyResultMaxAttempts: 4, EmptyResultIgnore: []string{"eth_getBalance"}}} - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{Budgets: []*common.RateLimitBudgetConfig{}}, &log.Logger) + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{Budgets: []*common.RateLimitBudgetConfig{}}, &log.Logger) if err != nil { t.Fatal(err) } @@ -693,7 +693,7 @@ func TestNetwork_Forward(t *testing.T) { util.SetupMocksForEvmStatePoller() defer util.AssertNoPendingMocks(t, 0) - rateLimitersRegistry, err := upstream.NewRateLimitersRegistry( + rateLimitersRegistry, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{ { @@ -829,7 +829,7 @@ func TestNetwork_Forward(t *testing.T) { MaxAttempts: 3, }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -954,7 +954,7 @@ func TestNetwork_Forward(t *testing.T) { MaxAttempts: 3, }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -1095,7 +1095,7 @@ func TestNetwork_Forward(t *testing.T) { MaxAttempts: 2, }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -1261,7 +1261,7 @@ func TestNetwork_Forward(t *testing.T) { MaxAttempts: 2, }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -1418,7 +1418,7 @@ func TestNetwork_Forward(t *testing.T) { // Initialize various components for the test environment clr := clients.NewClientRegistry(&log.Logger, "prjA", nil, evm.NewJsonRpcErrorExtractor()) - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -1615,7 +1615,7 @@ func TestNetwork_Forward(t *testing.T) { MaxAttempts: 2, // Allow up to 2 retry attempts }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -1825,7 +1825,7 @@ func TestNetwork_Forward(t *testing.T) { defer cancel() clr := clients.NewClientRegistry(&log.Logger, "prjA", nil, evm.NewJsonRpcErrorExtractor()) - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -2017,7 +2017,7 @@ func TestNetwork_Forward(t *testing.T) { defer cancel() clr := clients.NewClientRegistry(&log.Logger, "prjA", nil, evm.NewJsonRpcErrorExtractor()) - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -2242,7 +2242,7 @@ func TestNetwork_Forward(t *testing.T) { // Set up the test environment clr := clients.NewClientRegistry(&log.Logger, "prjA", nil, evm.NewJsonRpcErrorExtractor()) - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, []*common.ProviderConfig{}, nil) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) @@ -2380,7 +2380,7 @@ func TestNetwork_Forward(t *testing.T) { MaxAttempts: 2, // Allow up to 2 retry attempts }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -2588,7 +2588,7 @@ func TestNetwork_Forward(t *testing.T) { MaxAttempts: 2, // Allow up to 2 retry attempts }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -2778,7 +2778,7 @@ func TestNetwork_Forward(t *testing.T) { MaxAttempts: 2, // Allow up to 2 retry attempts }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -2961,7 +2961,7 @@ func TestNetwork_Forward(t *testing.T) { MaxAttempts: 2, // Allow up to 2 retry attempts }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -3163,7 +3163,7 @@ func TestNetwork_Forward(t *testing.T) { MaxAttempts: 4, // Allow up to 4 attempts (1 initial + 3 retries) }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -3392,7 +3392,7 @@ func TestNetwork_Forward(t *testing.T) { MaxAttempts: 2, }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -3583,7 +3583,7 @@ func TestNetwork_Forward(t *testing.T) { MaxAttempts: 3, // Allow up to 3 retry attempts }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -3771,7 +3771,7 @@ func TestNetwork_Forward(t *testing.T) { MaxAttempts: 2, // Allow up to 2 retry attempts }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -3952,7 +3952,7 @@ func TestNetwork_Forward(t *testing.T) { } clr := clients.NewClientRegistry(&log.Logger, "prjA", nil, evm.NewJsonRpcErrorExtractor()) fsCfg := &common.FailsafeConfig{} - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -4081,7 +4081,7 @@ func TestNetwork_Forward(t *testing.T) { } clr := clients.NewClientRegistry(&log.Logger, "prjA", nil, evm.NewJsonRpcErrorExtractor()) - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -4219,7 +4219,7 @@ func TestNetwork_Forward(t *testing.T) { MaxAttempts: 3, }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -4391,7 +4391,7 @@ func TestNetwork_Forward(t *testing.T) { MaxAttempts: 3, }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -4521,7 +4521,7 @@ func TestNetwork_Forward(t *testing.T) { MaxAttempts: 3, }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -4623,7 +4623,7 @@ func TestNetwork_Forward(t *testing.T) { util.SetupMocksForEvmStatePoller() defer util.AssertNoPendingMocks(t, 0) - rateLimitersRegistry, err := upstream.NewRateLimitersRegistry( + rateLimitersRegistry, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{ { @@ -4769,7 +4769,7 @@ func TestNetwork_Forward(t *testing.T) { MaxAttempts: 3, }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -4899,7 +4899,7 @@ func TestNetwork_Forward(t *testing.T) { MaxAttempts: 4, }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -5027,7 +5027,7 @@ func TestNetwork_Forward(t *testing.T) { if err != nil { t.Fatal(err) } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -5160,7 +5160,7 @@ func TestNetwork_Forward(t *testing.T) { Duration: common.Duration(1 * time.Second), }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -5293,7 +5293,7 @@ func TestNetwork_Forward(t *testing.T) { MaxCount: 1, }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -5446,7 +5446,7 @@ func TestNetwork_Forward(t *testing.T) { MaxCount: 5, }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -5597,7 +5597,7 @@ func TestNetwork_Forward(t *testing.T) { MaxCount: 5, }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -5755,7 +5755,7 @@ func TestNetwork_Forward(t *testing.T) { }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -5889,7 +5889,7 @@ func TestNetwork_Forward(t *testing.T) { }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -6022,7 +6022,7 @@ func TestNetwork_Forward(t *testing.T) { t.Fatal(err) } clr := clients.NewClientRegistry(&log.Logger, "prjA", nil, evm.NewJsonRpcErrorExtractor()) - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -6175,7 +6175,7 @@ func TestNetwork_Forward(t *testing.T) { } clr := clients.NewClientRegistry(&log.Logger, "prjA", nil, evm.NewJsonRpcErrorExtractor()) fsCfg := &common.FailsafeConfig{} - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -6313,7 +6313,7 @@ func TestNetwork_Forward(t *testing.T) { MaxAttempts: 2, }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -6458,7 +6458,7 @@ func TestNetwork_Forward(t *testing.T) { MaxAttempts: 2, }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -6586,7 +6586,7 @@ func TestNetwork_Forward(t *testing.T) { MaxAttempts: 2, }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -6738,7 +6738,7 @@ func TestNetwork_Forward(t *testing.T) { MaxAttempts: 2, }, } - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -6883,7 +6883,7 @@ func TestNetwork_Forward(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -7019,7 +7019,7 @@ func TestNetwork_Forward(t *testing.T) { defer cancel() fsCfg := &common.FailsafeConfig{} - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -7142,7 +7142,7 @@ func TestNetwork_Forward(t *testing.T) { defer cancel() fsCfg := &common.FailsafeConfig{} - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -7264,7 +7264,7 @@ func TestNetwork_Forward(t *testing.T) { defer cancel() fsCfg := &common.FailsafeConfig{} - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -7380,7 +7380,7 @@ func TestNetwork_Forward(t *testing.T) { metricsTracker.Bootstrap(ctx) - rateLimitersRegistry, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rateLimitersRegistry, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &logger) assert.NoError(t, err) @@ -7566,7 +7566,7 @@ func TestNetwork_Forward(t *testing.T) { defer cancel() fsCfg := &common.FailsafeConfig{} - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { @@ -9236,7 +9236,7 @@ func TestNetwork_EvmGetLogs(t *testing.T) { defer cancel() // Build network with tight best-effort budgets to force fallback - rateLimitersRegistry, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rateLimitersRegistry, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) metricsTracker := health.NewTracker(&log.Logger, "test", time.Minute) vr := thirdparty.NewVendorsRegistry() pr, err := thirdparty.NewProvidersRegistry(&log.Logger, vr, []*common.ProviderConfig{}, nil) @@ -10170,7 +10170,7 @@ func TestNetwork_ThunderingHerdProtection(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 5*time.Second) pollerInterval := 2000 * time.Millisecond @@ -10372,7 +10372,7 @@ func TestNetwork_ThunderingHerdProtection(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) upCfg := &common.UpstreamConfig{ @@ -10557,7 +10557,7 @@ func TestNetwork_ThunderingHerdProtection(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - rlr, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rlr, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) mt := health.NewTracker(&log.Logger, "prjA", 2*time.Second) upCfg := &common.UpstreamConfig{ @@ -10655,7 +10655,7 @@ func TestNetwork_ThunderingHerdProtection(t *testing.T) { func setupTestNetworkSimple(t *testing.T, ctx context.Context, upstreamConfig *common.UpstreamConfig, networkConfig *common.NetworkConfig) *Network { t.Helper() - rateLimitersRegistry, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rateLimitersRegistry, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) metricsTracker := health.NewTracker(&log.Logger, "test", time.Minute) if upstreamConfig == nil { @@ -10764,7 +10764,7 @@ func setupTestNetworkWithFullAndArchiveNodeUpstreams( ) *Network { t.Helper() - rateLimitersRegistry, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rateLimitersRegistry, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) metricsTracker := health.NewTracker(&log.Logger, "test", time.Minute) up1 := &common.UpstreamConfig{ @@ -10929,7 +10929,7 @@ func TestNetwork_HighestLatestBlockNumber(t *testing.T) { Reply(200). JSON([]byte(`{"result":"0x7b"}`)) - rateLimitersRegistry, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rateLimitersRegistry, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) metricsTracker := health.NewTracker(&log.Logger, "test", time.Minute) vr := thirdparty.NewVendorsRegistry() @@ -11082,7 +11082,7 @@ func TestNetwork_HighestLatestBlockNumber(t *testing.T) { Reply(200). JSON([]byte(`{"result":"0x7b"}`)) - rateLimitersRegistry, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rateLimitersRegistry, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) metricsTracker := health.NewTracker(&log.Logger, "test", time.Minute) vr := thirdparty.NewVendorsRegistry() @@ -11240,7 +11240,7 @@ func TestNetwork_HighestLatestBlockNumber(t *testing.T) { Reply(200). JSON([]byte(`{"result":"0x7b"}`)) - rateLimitersRegistry, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rateLimitersRegistry, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) metricsTracker := health.NewTracker(&log.Logger, "test", time.Minute) vr := thirdparty.NewVendorsRegistry() @@ -11367,7 +11367,7 @@ func TestNetwork_HighestLatestBlockNumber(t *testing.T) { Reply(200). JSON([]byte(`{"result":"0x7b"}`)) - rateLimitersRegistry, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rateLimitersRegistry, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) metricsTracker := health.NewTracker(&log.Logger, "test", time.Minute) vr := thirdparty.NewVendorsRegistry() @@ -11501,7 +11501,7 @@ func TestNetwork_HighestFinalizedBlockNumber(t *testing.T) { Reply(200). JSON([]byte(`{"result":"0x7b"}`)) - rateLimitersRegistry, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rateLimitersRegistry, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) metricsTracker := health.NewTracker(&log.Logger, "test", time.Minute) vr := thirdparty.NewVendorsRegistry() @@ -11632,7 +11632,7 @@ func TestNetwork_HighestFinalizedBlockNumber(t *testing.T) { Reply(200). JSON([]byte(`{"result":"0x7b"}`)) - rateLimitersRegistry, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rateLimitersRegistry, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) metricsTracker := health.NewTracker(&log.Logger, "test", time.Minute) vr := thirdparty.NewVendorsRegistry() @@ -11761,7 +11761,7 @@ func TestNetwork_HighestFinalizedBlockNumber(t *testing.T) { Reply(200). JSON([]byte(`{"result":"0x7b"}`)) - rateLimitersRegistry, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rateLimitersRegistry, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) metricsTracker := health.NewTracker(&log.Logger, "test", time.Minute) vr := thirdparty.NewVendorsRegistry() diff --git a/erpc/policy_evaluator_test.go b/erpc/policy_evaluator_test.go index c6a95a881..599d32c22 100644 --- a/erpc/policy_evaluator_test.go +++ b/erpc/policy_evaluator_test.go @@ -1724,7 +1724,7 @@ func TestPolicyEvaluator(t *testing.T) { } func createTestNetwork(t *testing.T, ctx context.Context) (*Network, *upstream.Upstream, *upstream.Upstream, *upstream.Upstream) { - rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + rlr, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Budgets: []*common.RateLimitBudgetConfig{}, }, &log.Logger) if err != nil { diff --git a/erpc/projects_test.go b/erpc/projects_test.go index 48e966b84..d55297f90 100644 --- a/erpc/projects_test.go +++ b/erpc/projects_test.go @@ -25,7 +25,7 @@ func TestProject_Forward(t *testing.T) { util.SetupMocksForEvmStatePoller() defer util.AssertNoPendingMocks(t, 0) - rateLimitersRegistry, err := upstream.NewRateLimitersRegistry( + rateLimitersRegistry, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{ Store: &common.RateLimitStoreConfig{ Driver: "memory", @@ -136,7 +136,7 @@ func TestProject_TimeoutScenarios(t *testing.T) { // Create a rate limiters registry (not specifically needed for this test, // but it's part of the usual setup.) - rateLimitersRegistry, err := upstream.NewRateLimitersRegistry( + rateLimitersRegistry, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger, ) @@ -248,7 +248,7 @@ func TestProject_TimeoutScenarios(t *testing.T) { util.SetupMocksForEvmStatePoller() defer util.AssertNoPendingMocks(t, 0) - rateLimitersRegistry, err := upstream.NewRateLimitersRegistry( + rateLimitersRegistry, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger, ) @@ -416,7 +416,7 @@ func TestProject_LazyLoadNetworkDefaults(t *testing.T) { } // Build ProjectsRegistry with no existing EvmJsonRpcCache or RateLimiter - rateLimiters, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rateLimiters, _ := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) ssr, err := data.NewSharedStateRegistry(ctx, &log.Logger, &common.SharedStateConfig{ Connector: &common.ConnectorConfig{ Driver: "memory", @@ -512,7 +512,7 @@ func TestProject_NetworkAlias(t *testing.T) { panic(err) } - rateLimitersRegistry, err := upstream.NewRateLimitersRegistry( + rateLimitersRegistry, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger, ) diff --git a/erpc/upstream_selection_test.go b/erpc/upstream_selection_test.go index f4d717474..936b45437 100644 --- a/erpc/upstream_selection_test.go +++ b/erpc/upstream_selection_test.go @@ -809,7 +809,7 @@ func setupTestNetworkWithFourUpstreams(t *testing.T, ctx context.Context, failsa func setupTestNetworkWithConfig(t *testing.T, ctx context.Context, upstreamConfigs []*common.UpstreamConfig, failsafeConfig *common.FailsafeConfig) *Network { t.Helper() - rateLimitersRegistry, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) + rateLimitersRegistry, err := upstream.NewRateLimitersRegistry(context.Background(), &common.RateLimiterConfig{}, &log.Logger) require.NoError(t, err) metricsTracker := health.NewTracker(&log.Logger, "test", time.Minute) diff --git a/upstream/ratelimiter_budget.go b/upstream/ratelimiter_budget.go index ae084225d..a82b16da1 100644 --- a/upstream/ratelimiter_budget.go +++ b/upstream/ratelimiter_budget.go @@ -24,7 +24,6 @@ type RateLimiterBudget struct { Rules []*RateLimitRule registry *RateLimitersRegistry rulesMu sync.RWMutex - cache limiter.RateLimitCache maxTimeout time.Duration } @@ -73,11 +72,17 @@ type ruleResult struct { allowed bool } +// getCache returns the current cache from the registry (thread-safe) +func (b *RateLimiterBudget) getCache() limiter.RateLimitCache { + return b.registry.GetCache() +} + // TryAcquirePermit evaluates all matching rules for the given method using Envoy's DoLimit. // Rules are evaluated in parallel for lower latency. Returns true if allowed, false if rate limited. func (b *RateLimiterBudget) TryAcquirePermit(ctx context.Context, projectId string, req *common.NormalizedRequest, method string, vendor string, upstreamId string, authLabel string, origin string) (bool, error) { - if b.cache == nil { - return true, nil + cache := b.getCache() + if cache == nil { + return true, nil // Fail-open when no cache is available } ctx, span := common.StartDetailSpan(ctx, "RateLimiter.TryAcquirePermit", @@ -168,6 +173,11 @@ func (b *RateLimiterBudget) TryAcquirePermit(ctx context.Context, projectId stri // evaluateRule checks a single rate limit rule against the cache. // Returns true if allowed, false if over limit. func (b *RateLimiterBudget) evaluateRule(ctx context.Context, rule *RateLimitRule, method, clientIP, userLabel, networkLabel string) bool { + cache := b.getCache() + if cache == nil { + return true // Fail-open when no cache is available + } + // Build descriptor entries entries := []*pb_struct.RateLimitDescriptor_Entry{{Key: "method", Value: method}} if rule.Config.PerIP && clientIP != "" && clientIP != "n/a" { @@ -205,9 +215,9 @@ func (b *RateLimiterBudget) evaluateRule(ctx context.Context, rule *RateLimitRul var statuses []*pb.RateLimitResponse_DescriptorStatus var timedOut bool if b.maxTimeout > 0 { - statuses, timedOut = b.doLimitWithTimeout(ctx, rlReq, limits, method, userLabel, networkLabel) + statuses, timedOut = b.doLimitWithTimeout(ctx, cache, rlReq, limits, method, userLabel, networkLabel) } else { - statuses = b.cache.DoLimit(ctx, rlReq, limits) + statuses = cache.DoLimit(ctx, rlReq, limits) } if timedOut { @@ -246,13 +256,14 @@ func (r *RateLimitRule) statsKeySuffix() string { // Returns (statuses, timedOut). On timeout, returns (nil, true) and records fail-open metric. func (b *RateLimiterBudget) doLimitWithTimeout( ctx context.Context, + cache limiter.RateLimitCache, rlReq *pb.RateLimitRequest, limits []*config.RateLimit, method, userLabel, networkLabel string, ) ([]*pb.RateLimitResponse_DescriptorStatus, bool) { resultCh := make(chan []*pb.RateLimitResponse_DescriptorStatus, 1) go func() { - resultCh <- b.cache.DoLimit(ctx, rlReq, limits) + resultCh <- cache.DoLimit(ctx, rlReq, limits) }() timer := time.NewTimer(b.maxTimeout) diff --git a/upstream/ratelimiter_budget_bench_test.go b/upstream/ratelimiter_budget_bench_test.go index 9093f4d25..a2198b258 100644 --- a/upstream/ratelimiter_budget_bench_test.go +++ b/upstream/ratelimiter_budget_bench_test.go @@ -67,12 +67,15 @@ func buildBenchBudget(numRules int, perUser, perIP, perNetwork bool) *RateLimite } logger := zerolog.Nop() + registry := &RateLimitersRegistry{ + statsManager: mgr, + envoyCache: cache, + } return &RateLimiterBudget{ logger: &logger, Id: "bench-budget", Rules: rules, - registry: &RateLimitersRegistry{statsManager: mgr}, - cache: cache, + registry: registry, } } @@ -172,7 +175,7 @@ func BenchmarkTryAcquirePermit_NoRulesMatch(b *testing.B) { // BenchmarkTryAcquirePermit_NilCache tests the nil cache fast path func BenchmarkTryAcquirePermit_NilCache(b *testing.B) { budget := buildBenchBudget(1, false, false, false) - budget.cache = nil + budget.registry.envoyCache = nil // Simulate Redis not connected yet ctx := context.Background() b.ReportAllocs() @@ -208,12 +211,15 @@ func buildBenchBudgetWithDelay(numRules int, delay time.Duration) *RateLimiterBu } logger := zerolog.Nop() + registry := &RateLimitersRegistry{ + statsManager: mgr, + envoyCache: &delayedCache{inner: innerCache, delay: delay}, + } return &RateLimiterBudget{ logger: &logger, Id: "bench-budget", Rules: rules, - registry: &RateLimitersRegistry{statsManager: mgr}, - cache: &delayedCache{inner: innerCache, delay: delay}, + registry: registry, } } diff --git a/upstream/ratelimiter_registry.go b/upstream/ratelimiter_registry.go index b5c6aea40..f17d8de71 100644 --- a/upstream/ratelimiter_registry.go +++ b/upstream/ratelimiter_registry.go @@ -1,7 +1,10 @@ package upstream import ( + "context" + "fmt" "math/rand" + "runtime/debug" "strings" "sync" "time" @@ -17,18 +20,23 @@ import ( "github.com/erpc/erpc/common" "github.com/erpc/erpc/telemetry" + "github.com/erpc/erpc/util" ) type RateLimitersRegistry struct { + appCtx context.Context logger *zerolog.Logger cfg *common.RateLimiterConfig budgetsLimiters sync.Map envoyCache limiter.RateLimitCache statsManager stats.Manager + cacheMu sync.RWMutex + initializer *util.Initializer } -func NewRateLimitersRegistry(cfg *common.RateLimiterConfig, logger *zerolog.Logger) (*RateLimitersRegistry, error) { +func NewRateLimitersRegistry(appCtx context.Context, cfg *common.RateLimiterConfig, logger *zerolog.Logger) (*RateLimitersRegistry, error) { r := &RateLimitersRegistry{ + appCtx: appCtx, cfg: cfg, logger: logger, } @@ -42,57 +50,109 @@ func (r *RateLimitersRegistry) bootstrap() error { return nil } + // Create a default stats manager (needed even if cache is nil) + store := gostats.NewStore(gostats.NewNullSink(), false) + r.statsManager = stats.NewStatManager(store, settings.NewSettings()) + // Initialize shared cache if configured if r.cfg.Store != nil && r.cfg.Store.Driver == "redis" && r.cfg.Store.Redis != nil { - store := gostats.NewStore(gostats.NewNullSink(), false) - mgr := stats.NewStatManager(store, settings.NewSettings()) - useTLS := r.cfg.Store.Redis.TLS != nil && r.cfg.Store.Redis.TLS.Enabled - url := r.cfg.Store.Redis.URI - if url == "" { - url = r.cfg.Store.Redis.Addr + // Create initializer for background retry + r.initializer = util.NewInitializer(r.appCtx, r.logger, nil) + + // Attempt Redis connection with panic recovery - don't block startup + connectTask := util.NewBootstrapTask("redis-ratelimiter-connect", r.connectRedisTask) + if err := r.initializer.ExecuteTasks(r.appCtx, connectTask); err != nil { + // Cache stays nil - rate limiting will fail-open until Redis connects + r.logger.Warn().Err(err).Msg("failed to initialize Redis rate limiter on first attempt (rate limiting will fail-open until connected, retrying in background)") } - poolSize := r.cfg.Store.Redis.ConnPoolSize - client := redis.NewClientImpl( - store.Scope("erpc_rl"), - useTLS, - r.cfg.Store.Redis.Username, - "tcp", - "single", - url, - poolSize, - 5*time.Millisecond, - 32, - nil, - false, - nil, - ) - r.envoyCache = redis.NewFixedRateLimitCacheImpl( - client, - nil, - utils.NewTimeSourceImpl(), - rand.New(rand.NewSource(time.Now().UnixNano())), // #nosec G404 - 5, - nil, - defaultNearLimitRatio(r.cfg.Store.NearLimitRatio), - defaultCacheKeyPrefix(r.cfg.Store.CacheKeyPrefix), - mgr, - false, - ) - r.statsManager = mgr } else if r.cfg.Store != nil && r.cfg.Store.Driver == "memory" { - store := gostats.NewStore(gostats.NewNullSink(), false) - mgr := stats.NewStatManager(store, settings.NewSettings()) + // Explicitly configured for memory r.envoyCache = NewMemoryRateLimitCache( utils.NewTimeSourceImpl(), rand.New(rand.NewSource(time.Now().Unix())), // #nosec G404 0, defaultNearLimitRatio(r.cfg.Store.NearLimitRatio), defaultCacheKeyPrefix(r.cfg.Store.CacheKeyPrefix), - mgr, + r.statsManager, ) - r.statsManager = mgr } + // Initialize budgets (cache may be nil for Redis until it connects) + r.initializeBudgets() + + return nil +} + +// connectRedisTask attempts to connect to Redis with panic recovery +func (r *RateLimitersRegistry) connectRedisTask(ctx context.Context) (err error) { + // Recover from panics in the envoyproxy/ratelimit library + defer func() { + if rec := recover(); rec != nil { + telemetry.MetricUnexpectedPanicTotal.WithLabelValues( + "ratelimiter-redis-connect", + fmt.Sprintf("store:%s", r.cfg.Store.Redis.URI), + common.ErrorFingerprint(rec), + ).Inc() + r.logger.Error(). + Interface("panic", rec). + Str("stack", string(debug.Stack())). + Msg("panic recovered during Redis rate limiter connection (rate limiting will fail-open)") + err = fmt.Errorf("panic during Redis connection: %v", rec) + } + }() + + store := gostats.NewStore(gostats.NewNullSink(), false) + mgr := stats.NewStatManager(store, settings.NewSettings()) + useTLS := r.cfg.Store.Redis.TLS != nil && r.cfg.Store.Redis.TLS.Enabled + url := r.cfg.Store.Redis.URI + if url == "" { + url = r.cfg.Store.Redis.Addr + } + poolSize := r.cfg.Store.Redis.ConnPoolSize + + r.logger.Debug().Str("url", util.RedactEndpoint(url)).Bool("tls", useTLS).Int("poolSize", poolSize).Msg("attempting to connect to Redis for rate limiting") + + client := redis.NewClientImpl( + store.Scope("erpc_rl"), + useTLS, + r.cfg.Store.Redis.Username, + "tcp", + "single", + url, + poolSize, + 5*time.Millisecond, + 32, + nil, + false, + nil, + ) + + cache := redis.NewFixedRateLimitCacheImpl( + client, + nil, + utils.NewTimeSourceImpl(), + rand.New(rand.NewSource(time.Now().UnixNano())), // #nosec G404 + 5, + nil, + defaultNearLimitRatio(r.cfg.Store.NearLimitRatio), + defaultCacheKeyPrefix(r.cfg.Store.CacheKeyPrefix), + mgr, + false, + ) + + // Successfully connected - update the cache + // Note: statsManager is NOT updated here to avoid data races. + // The statsManager created in bootstrap() is sufficient and identical. + r.cacheMu.Lock() + r.envoyCache = cache + r.cacheMu.Unlock() + + r.logger.Info().Str("url", util.RedactEndpoint(url)).Msg("successfully connected to Redis for rate limiting") + return nil +} + +// initializeBudgets creates the rate limiter budgets +func (r *RateLimitersRegistry) initializeBudgets() { for _, budgetCfg := range r.cfg.Budgets { lg := r.logger.With().Str("budget", budgetCfg.Id).Logger() lg.Debug().Msgf("initializing rate limiter budget") @@ -105,7 +165,6 @@ func (r *RateLimitersRegistry) bootstrap() error { Rules: make([]*RateLimitRule, 0), registry: r, logger: &lg, - cache: r.envoyCache, maxTimeout: maxTimeout, } @@ -131,8 +190,13 @@ func (r *RateLimitersRegistry) bootstrap() error { r.budgetsLimiters.Store(budgetCfg.Id, budget) } +} - return nil +// GetCache returns the current rate limit cache (thread-safe) +func (r *RateLimitersRegistry) GetCache() limiter.RateLimitCache { + r.cacheMu.RLock() + defer r.cacheMu.RUnlock() + return r.envoyCache } func (r *RateLimitersRegistry) GetBudget(budgetId string) (*RateLimiterBudget, error) { diff --git a/upstream/ratelimiter_test.go b/upstream/ratelimiter_test.go index 5fb758605..b59c74487 100644 --- a/upstream/ratelimiter_test.go +++ b/upstream/ratelimiter_test.go @@ -16,7 +16,7 @@ func TestRateLimitersRegistry_New(t *testing.T) { logger := zerolog.Nop() t.Run("nil config", func(t *testing.T) { - registry, err := NewRateLimitersRegistry(nil, &logger) + registry, err := NewRateLimitersRegistry(context.Background(), nil, &logger) require.NoError(t, err) assert.NotNil(t, registry) }) @@ -37,7 +37,7 @@ func TestRateLimitersRegistry_New(t *testing.T) { }, }, } - registry, err := NewRateLimitersRegistry(cfg, &logger) + registry, err := NewRateLimitersRegistry(context.Background(), cfg, &logger) require.NoError(t, err) assert.NotNil(t, registry) }) @@ -60,7 +60,7 @@ func TestRateLimitersRegistry_GetBudget(t *testing.T) { }, }, } - registry, err := NewRateLimitersRegistry(cfg, &logger) + registry, err := NewRateLimitersRegistry(context.Background(), cfg, &logger) require.NoError(t, err) t.Run("existing budget", func(t *testing.T) { @@ -106,7 +106,7 @@ func TestRateLimiterBudget_GetRulesByMethod(t *testing.T) { }, }, } - registry, err := NewRateLimitersRegistry(cfg, &logger) + registry, err := NewRateLimitersRegistry(context.Background(), cfg, &logger) require.NoError(t, err) budget, err := registry.GetBudget("test-budget") @@ -152,7 +152,7 @@ func TestRateLimiter_ConcurrentPermits(t *testing.T) { }, }, } - registry, err := NewRateLimitersRegistry(cfg, &logger) + registry, err := NewRateLimitersRegistry(context.Background(), cfg, &logger) require.NoError(t, err) budget, err := registry.GetBudget("test-budget") @@ -203,7 +203,7 @@ func TestRateLimiter_ExceedCapacity(t *testing.T) { }, } - registry, err := NewRateLimitersRegistry(cfg, &logger) + registry, err := NewRateLimitersRegistry(context.Background(), cfg, &logger) require.NoError(t, err) budget, err := registry.GetBudget("test-budget") diff --git a/upstream/registry_contention_bench_test.go b/upstream/registry_contention_bench_test.go index 6dd5bcc72..bccac6e60 100644 --- a/upstream/registry_contention_bench_test.go +++ b/upstream/registry_contention_bench_test.go @@ -25,7 +25,7 @@ func buildRegistryForBench(b *testing.B, numNetworks, upstreamsPerNetwork int, m vr := thirdparty.NewVendorsRegistry() pr, _ := thirdparty.NewProvidersRegistry(&log.Logger, vr, nil, nil) - rlr, _ := NewRateLimitersRegistry(nil, &log.Logger) + rlr, _ := NewRateLimitersRegistry(context.Background(), nil, &log.Logger) mt := health.NewTracker(&log.Logger, "bench-prj", time.Minute) reg := NewUpstreamsRegistry( From 49184c3d75fc6ae3f9075704099648582450234f Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 21 Jan 2026 11:06:32 +0100 Subject: [PATCH 53/53] feat: add auto-detect bypass for multicall3 batching Add runtime detection for contracts that revert when called via Multicall3 but succeed when called individually (e.g., contracts checking msg.sender code size like Chronicle Oracle). Features: - New `autoDetectBypass` config option (default: false) - Runtime bypass cache with thread-safe access - Auto-retry reverted calls individually, add to bypass if succeeds - Bounded 30s timeout for retry when original context cancelled - Validation for bypassContracts addresses (40 hex chars) Observability: - `erpc_multicall3_runtime_bypass_total` - contracts auto-detected - `erpc_multicall3_auto_detect_retry_total` - retry outcomes (attempt/detected/same_error) Includes comprehensive tests for: - Auto-detect enabled/disabled scenarios - Concurrent cache access (race-safe) - Config validation for bypass addresses Co-Authored-By: Claude Opus 4.5 --- architecture/evm/multicall3_batcher.go | 181 ++++++- architecture/evm/multicall3_batcher_test.go | 549 ++++++++++++++++++++ common/config.go | 76 +++ common/config_test.go | 149 ++++++ docs/design/multicall3-batching.md | 60 +++ telemetry/metrics.go | 12 + 6 files changed, 1022 insertions(+), 5 deletions(-) diff --git a/architecture/evm/multicall3_batcher.go b/architecture/evm/multicall3_batcher.go index 79248b708..7eff15640 100644 --- a/architecture/evm/multicall3_batcher.go +++ b/architecture/evm/multicall3_batcher.go @@ -47,6 +47,15 @@ type Batcher struct { shutdown chan struct{} shutdownOnce sync.Once wg sync.WaitGroup + + // runtimeBypass holds contracts detected at runtime that revert when called via Multicall3 + // but succeed when called individually. This in-memory cache resets on process restart. + // For persistent bypass configuration, use the BypassContracts config field. + // Note: This map can grow without bound; in practice this is limited by the number of + // unique contracts that fail via multicall3 but succeed individually (typically few). + // Protected by runtimeBypassMu. + runtimeBypass map[string]bool + runtimeBypassMu sync.RWMutex } // NewBatcher creates a new Multicall3 batcher. @@ -61,11 +70,12 @@ func NewBatcher(cfg *common.Multicall3AggregationConfig, forwarder Forwarder, lo panic("NewBatcher: forwarder cannot be nil") } b := &Batcher{ - cfg: cfg, - forwarder: forwarder, - logger: logger, - batches: make(map[string]*Batch), - shutdown: make(chan struct{}), + cfg: cfg, + forwarder: forwarder, + logger: logger, + batches: make(map[string]*Batch), + shutdown: make(chan struct{}), + runtimeBypass: make(map[string]bool), } return b } @@ -84,6 +94,46 @@ func (b *Batcher) logBypass(key BatchingKey, reason string) { Msg("request bypassing multicall3 batching") } +// isRuntimeBypassed checks if a contract address is in the runtime bypass cache. +// The address should be lowercase hex without 0x prefix. +func (b *Batcher) isRuntimeBypassed(addrHex string) bool { + b.runtimeBypassMu.RLock() + defer b.runtimeBypassMu.RUnlock() + return b.runtimeBypass[addrHex] +} + +// addRuntimeBypass adds a contract address to the runtime bypass cache. +// The address should be lowercase hex without 0x prefix. +func (b *Batcher) addRuntimeBypass(addrHex string, projectId, networkId string) { + b.runtimeBypassMu.Lock() + defer b.runtimeBypassMu.Unlock() + if !b.runtimeBypass[addrHex] { + b.runtimeBypass[addrHex] = true + telemetry.MetricMulticall3RuntimeBypassTotal.WithLabelValues(projectId, networkId).Inc() + if b.logger != nil { + b.logger.Info(). + Str("contract", "0x"+addrHex). + Str("projectId", projectId). + Str("networkId", networkId). + Msg("auto-detected contract that reverts via multicall3, added to runtime bypass") + } + } +} + +// IsRuntimeBypassed checks if a contract address should bypass batching due to runtime detection. +// This is a public method for external callers (e.g., tests, diagnostics) to query the runtime +// bypass cache. The internal Enqueue method uses isRuntimeBypassed for actual bypass checks. +// The address can be provided with or without 0x prefix, and is case-insensitive. +func (b *Batcher) IsRuntimeBypassed(targetHex string) bool { + if targetHex == "" { + return false + } + // Normalize: lowercase and remove 0x/0X prefix + normalized := strings.ToLower(targetHex) + normalized = strings.TrimPrefix(normalized, "0x") + return b.isRuntimeBypassed(normalized) +} + // Enqueue adds a request to a batch. Returns: // - entry: the batch entry (nil if bypass) // - bypass: true if request should be forwarded individually @@ -109,6 +159,13 @@ func (b *Batcher) Enqueue(ctx context.Context, key BatchingKey, req *common.Norm return nil, true, err } + // Check runtime bypass cache (contracts detected as reverting via multicall3) + targetHex := strings.ToLower(hex.EncodeToString(target)) + if b.isRuntimeBypassed(targetHex) { + b.logBypass(key, "runtime_bypass_detected") + return nil, true, nil + } + // Derive call key for deduplication callKey, err := DeriveCallKey(req) if err != nil { @@ -495,6 +552,14 @@ func (b *Batcher) flush(keyStr string, batch *Batch) { for _, entry := range entriesForCall { jrr, err := common.NewJsonRpcResponse(entry.Request.ID(), resultHex, nil) if err != nil { + if b.logger != nil { + b.logger.Warn(). + Err(err). + Str("projectId", projectId). + Str("networkId", networkId). + Str("callKey", uc.callKey). + Msg("multicall3 response construction failed for entry") + } b.sendResult(entry, BatchResult{Error: err}, projectId, networkId) continue } @@ -524,6 +589,106 @@ func (b *Batcher) flush(keyStr string, batch *Batch) { b.sendResult(entry, BatchResult{Response: resp}, projectId, networkId) } } else { + // Call reverted in multicall3 - check if we should try auto-detection + if b.cfg.AutoDetectBypass && len(entriesForCall) > 0 { + // Try forwarding the first entry individually to see if it succeeds + firstEntry := entriesForCall[0] + targetHex := strings.ToLower(hex.EncodeToString(firstEntry.Target)) + + // Skip retry if already in runtime bypass (shouldn't happen, but defensive) + if !b.isRuntimeBypassed(targetHex) { + telemetry.MetricMulticall3AutoDetectRetryTotal.WithLabelValues(projectId, networkId, "attempt").Inc() + + // Use the entry's context for the retry, but with a bounded fallback if it's already cancelled + retryCtx := firstEntry.Ctx + var retryCancel context.CancelFunc + select { + case <-retryCtx.Done(): + // Original context cancelled - use bounded timeout for auto-detection + retryCtx, retryCancel = context.WithTimeout(context.Background(), 30*time.Second) + if b.logger != nil { + b.logger.Debug(). + Str("projectId", projectId). + Str("networkId", networkId). + Str("contract", "0x"+targetHex). + Err(firstEntry.Ctx.Err()). + Msg("multicall3 auto-detect retry using fallback context (original cancelled)") + } + default: + } + + retryResp, retryErr := b.forwarder.Forward(retryCtx, firstEntry.Request) + if retryCancel != nil { + retryCancel() + } + if retryErr == nil && retryResp != nil { + // Individual call succeeded! This contract needs bypass + b.addRuntimeBypass(targetHex, projectId, networkId) + telemetry.MetricMulticall3AutoDetectRetryTotal.WithLabelValues(projectId, networkId, "detected").Inc() + + // Extract the result from the retry response to create per-entry responses + retryJrr, err := retryResp.JsonRpcResponse() + if err != nil { + if b.logger != nil { + b.logger.Warn(). + Err(err). + Str("projectId", projectId). + Str("networkId", networkId). + Str("contract", "0x"+targetHex). + Msg("multicall3 auto-detect failed to extract retry response") + } + // Fallback: propagate error to all entries + for _, entry := range entriesForCall { + b.sendResult(entry, BatchResult{Error: err}, projectId, networkId) + } + retryResp.Release() + continue + } + + // Get the result value to clone for each entry + resultValue := retryJrr.GetResultString() + + // Deliver fresh response to each entry with correct request ID + for _, entry := range entriesForCall { + jrr, err := common.NewJsonRpcResponse(entry.Request.ID(), resultValue, nil) + if err != nil { + if b.logger != nil { + b.logger.Warn(). + Err(err). + Str("projectId", projectId). + Str("networkId", networkId). + Str("contract", "0x"+targetHex). + Msg("multicall3 auto-detect response construction failed") + } + b.sendResult(entry, BatchResult{Error: err}, projectId, networkId) + continue + } + resp := common.NewNormalizedResponse().WithRequest(entry.Request).WithJsonRpcResponse(jrr) + resp.SetUpstream(retryResp.Upstream()) + resp.SetFromCache(retryResp.FromCache()) + b.sendResult(entry, BatchResult{Response: resp}, projectId, networkId) + } + + // Release the original retry response after all entries are processed + retryResp.Release() + continue // Move to next unique call + } + // Individual call also failed - not a bypass candidate + telemetry.MetricMulticall3AutoDetectRetryTotal.WithLabelValues(projectId, networkId, "same_error").Inc() + if b.logger != nil { + b.logger.Debug(). + Err(retryErr). + Str("projectId", projectId). + Str("networkId", networkId). + Str("contract", "0x"+targetHex). + Msg("multicall3 auto-detect retry also failed, not adding to bypass") + } + if retryResp != nil { + retryResp.Release() + } + } + } + // Build error for reverted call with proper JSON-RPC format dataHex := "0x" + hex.EncodeToString(result.ReturnData) revertErr := common.NewErrEndpointExecutionException( @@ -909,6 +1074,12 @@ func IsEligibleForBatching(req *common.NormalizedRequest, cfg *common.Multicall3 return false, "invalid to address" } + // Check if contract should bypass multicall3 batching + // (e.g., contracts that check msg.sender code size like Chronicle Oracle) + if cfg.ShouldBypassContractHex(toStr) { + return false, "contract in bypass list" + } + // Check for ineligible fields for _, field := range ineligibleCallFields { if _, has := callObj[field]; has { diff --git a/architecture/evm/multicall3_batcher_test.go b/architecture/evm/multicall3_batcher_test.go index 5cb4bd6bb..b9b8728e6 100644 --- a/architecture/evm/multicall3_batcher_test.go +++ b/architecture/evm/multicall3_batcher_test.go @@ -405,6 +405,140 @@ func TestIsEligibleForBatching_AllowPendingTag(t *testing.T) { require.True(t, eligible, "pending should be allowed when AllowPendingTagBatching is true: %s", reason) } +func TestIsEligibleForBatching_BypassContracts(t *testing.T) { + // Chronicle Oracle feed address (example contract that checks msg.sender code size) + chronicleOracleAddr := "0x057f30e63A69175C69A4Af5656b8C9EE647De3D0" + otherContract := "0x1234567890123456789012345678901234567890" + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + BypassContracts: []string{chronicleOracleAddr}, + } + cfg.SetDefaults() + + tests := []struct { + name string + to string + eligible bool + reason string + }{ + { + name: "bypass contract - exact match", + to: chronicleOracleAddr, + eligible: false, + reason: "contract in bypass list", + }, + { + name: "bypass contract - lowercase", + to: "0x057f30e63a69175c69a4af5656b8c9ee647de3d0", + eligible: false, + reason: "contract in bypass list", + }, + { + name: "bypass contract - uppercase", + to: "0x057F30E63A69175C69A4AF5656B8C9EE647DE3D0", + eligible: false, + reason: "contract in bypass list", + }, + { + name: "non-bypass contract allowed", + to: otherContract, + eligible: true, + reason: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": tt.to, + "data": "0xfeaf968c", // latestRoundData() selector + }, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + + eligible, reason := IsEligibleForBatching(req, cfg) + require.Equal(t, tt.eligible, eligible, "eligibility mismatch for %s", tt.name) + if tt.reason != "" { + require.Contains(t, reason, tt.reason) + } + }) + } +} + +func TestIsEligibleForBatching_BypassContractsEmpty(t *testing.T) { + // When BypassContracts is empty, all contracts should be eligible + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + BypassContracts: []string{}, + } + cfg.SetDefaults() + + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": "0x057f30e63A69175C69A4Af5656b8C9EE647De3D0", + "data": "0xfeaf968c", + }, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + + eligible, reason := IsEligibleForBatching(req, cfg) + require.True(t, eligible, "should be eligible when bypass list is empty: %s", reason) +} + +func TestIsEligibleForBatching_MultipleBypassContracts(t *testing.T) { + // Test with multiple bypass contracts + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + BypassContracts: []string{ + "0x057f30e63A69175C69A4Af5656b8C9EE647De3D0", // Chronicle Oracle + "0xABCDEF0123456789ABCDEF0123456789ABCDEF01", // Another contract + }, + } + cfg.SetDefaults() + + tests := []struct { + name string + to string + eligible bool + }{ + { + name: "first bypass contract", + to: "0x057f30e63A69175C69A4Af5656b8C9EE647De3D0", + eligible: false, + }, + { + name: "second bypass contract", + to: "0xABCDEF0123456789ABCDEF0123456789ABCDEF01", + eligible: false, + }, + { + name: "non-bypass contract", + to: "0x1111111111111111111111111111111111111111", + eligible: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{ + "to": tt.to, + "data": "0xabcd", + }, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + + eligible, _ := IsEligibleForBatching(req, cfg) + require.Equal(t, tt.eligible, eligible) + }) + } +} + func TestBatcherEnqueueAndFlush(t *testing.T) { cfg := &common.Multicall3AggregationConfig{ Enabled: true, @@ -2593,3 +2727,418 @@ func TestBatcher_MultipleDeadlinesPickEarliest(t *testing.T) { t.Fatal("timeout waiting for entry2 result") } } + +// dynamicMockForwarder allows returning different responses based on call count +type dynamicMockForwarder struct { + mu sync.Mutex + calls []*common.NormalizedRequest + responses []*common.NormalizedResponse + errors []error +} + +func (m *dynamicMockForwarder) Forward(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { + m.mu.Lock() + defer m.mu.Unlock() + + callIndex := len(m.calls) + m.calls = append(m.calls, req) + + if callIndex < len(m.responses) { + return m.responses[callIndex], m.errors[callIndex] + } + + // Default: return last response/error + if len(m.responses) > 0 { + return m.responses[len(m.responses)-1], m.errors[len(m.errors)-1] + } + return nil, fmt.Errorf("no response configured") +} + +func (m *dynamicMockForwarder) SetCache(ctx context.Context, req *common.NormalizedRequest, resp *common.NormalizedResponse) error { + return nil +} + +func (m *dynamicMockForwarder) CallCount() int { + m.mu.Lock() + defer m.mu.Unlock() + return len(m.calls) +} + +func TestBatcher_RuntimeBypass_Methods(t *testing.T) { + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + AutoDetectBypass: true, + WindowMs: 50, + MinWaitMs: 5, + SafetyMarginMs: 5, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + } + cfg.SetDefaults() + + forwarder := &mockForwarder{} + batcher := NewBatcher(cfg, forwarder, nil) + require.NotNil(t, batcher) + defer batcher.Shutdown() + + // Test address normalization + addr1 := "0x1111111111111111111111111111111111111111" + addr1Normalized := "1111111111111111111111111111111111111111" + + // Initially not bypassed + require.False(t, batcher.isRuntimeBypassed(addr1Normalized)) + require.False(t, batcher.IsRuntimeBypassed(addr1)) + + // Add to runtime bypass + batcher.addRuntimeBypass(addr1Normalized, "test-project", "evm:1") + + // Now should be bypassed + require.True(t, batcher.isRuntimeBypassed(addr1Normalized)) + require.True(t, batcher.IsRuntimeBypassed(addr1)) + + // Test case insensitivity + require.True(t, batcher.IsRuntimeBypassed("0x1111111111111111111111111111111111111111")) + require.True(t, batcher.IsRuntimeBypassed("0X1111111111111111111111111111111111111111")) + + // Different address should not be bypassed + require.False(t, batcher.IsRuntimeBypassed("0x2222222222222222222222222222222222222222")) +} + +func TestBatcher_AutoDetectBypass_DetectsRevertingContract(t *testing.T) { + // Create a multicall response where one call reverts + revertedResults := []Multicall3Result{ + {Success: false, ReturnData: []byte{0x08, 0xc3, 0x79, 0xa0}}, // execution reverted + } + encodedRevert := encodeAggregate3Results(revertedResults) + revertResultHex := "0x" + hex.EncodeToString(encodedRevert) + + multicallResp, err := common.NewJsonRpcResponse(nil, revertResultHex, nil) + require.NoError(t, err) + mockMulticallResp := common.NewNormalizedResponse().WithJsonRpcResponse(multicallResp) + + // Create success response for individual call + individualResp, err := common.NewJsonRpcResponse(nil, "0xdeadbeef", nil) + require.NoError(t, err) + mockIndividualResp := common.NewNormalizedResponse().WithJsonRpcResponse(individualResp) + + forwarder := &dynamicMockForwarder{ + responses: []*common.NormalizedResponse{mockMulticallResp, mockIndividualResp}, + errors: []error{nil, nil}, + } + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + AutoDetectBypass: true, // Enable auto-detect + WindowMs: 50, + MinWaitMs: 5, + SafetyMarginMs: 5, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + CachePerCall: util.BoolPtr(false), + } + cfg.SetDefaults() + + batcher := NewBatcher(cfg, forwarder, nil) + require.NotNil(t, batcher) + defer batcher.Shutdown() + + ctx := context.Background() + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + targetAddr := "0x1111111111111111111111111111111111111111" + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{"to": targetAddr, "data": "0x01"}, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + + // Verify not runtime bypassed initially + require.False(t, batcher.IsRuntimeBypassed(targetAddr)) + + entry, bypass, err := batcher.Enqueue(ctx, key, req) + require.NoError(t, err) + require.False(t, bypass) + + // Wait for result + select { + case result := <-entry.ResultCh: + // Should succeed because the individual retry succeeded + require.NoError(t, result.Error) + require.NotNil(t, result.Response) + case <-time.After(2 * time.Second): + t.Fatal("timeout waiting for result") + } + + // Verify contract was added to runtime bypass + require.True(t, batcher.IsRuntimeBypassed(targetAddr)) + + // Verify two calls were made (multicall + individual retry) + require.Equal(t, 2, forwarder.CallCount()) +} + +func TestBatcher_AutoDetectBypass_Disabled(t *testing.T) { + // Create a multicall response where one call reverts + revertedResults := []Multicall3Result{ + {Success: false, ReturnData: []byte{0x08, 0xc3, 0x79, 0xa0}}, // execution reverted + } + encodedRevert := encodeAggregate3Results(revertedResults) + revertResultHex := "0x" + hex.EncodeToString(encodedRevert) + + multicallResp, err := common.NewJsonRpcResponse(nil, revertResultHex, nil) + require.NoError(t, err) + mockMulticallResp := common.NewNormalizedResponse().WithJsonRpcResponse(multicallResp) + + forwarder := &dynamicMockForwarder{ + responses: []*common.NormalizedResponse{mockMulticallResp}, + errors: []error{nil}, + } + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + AutoDetectBypass: false, // Disabled + WindowMs: 50, + MinWaitMs: 5, + SafetyMarginMs: 5, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + CachePerCall: util.BoolPtr(false), + } + cfg.SetDefaults() + + batcher := NewBatcher(cfg, forwarder, nil) + require.NotNil(t, batcher) + defer batcher.Shutdown() + + ctx := context.Background() + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + targetAddr := "0x1111111111111111111111111111111111111111" + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{"to": targetAddr, "data": "0x01"}, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + + entry, bypass, err := batcher.Enqueue(ctx, key, req) + require.NoError(t, err) + require.False(t, bypass) + + // Wait for result + select { + case result := <-entry.ResultCh: + // Should fail with execution reverted error (no retry since AutoDetectBypass is false) + require.Error(t, result.Error) + case <-time.After(2 * time.Second): + t.Fatal("timeout waiting for result") + } + + // Verify contract was NOT added to runtime bypass + require.False(t, batcher.IsRuntimeBypassed(targetAddr)) + + // Verify only one call was made (no retry) + require.Equal(t, 1, forwarder.CallCount()) +} + +func TestBatcher_AutoDetectBypass_SameErrorNoBypass(t *testing.T) { + // Create a multicall response where one call reverts + revertedResults := []Multicall3Result{ + {Success: false, ReturnData: []byte{0x08, 0xc3, 0x79, 0xa0}}, // execution reverted + } + encodedRevert := encodeAggregate3Results(revertedResults) + revertResultHex := "0x" + hex.EncodeToString(encodedRevert) + + multicallResp, err := common.NewJsonRpcResponse(nil, revertResultHex, nil) + require.NoError(t, err) + mockMulticallResp := common.NewNormalizedResponse().WithJsonRpcResponse(multicallResp) + + // Individual retry also fails with an error + individualErr := fmt.Errorf("execution reverted") + + forwarder := &dynamicMockForwarder{ + responses: []*common.NormalizedResponse{mockMulticallResp, nil}, + errors: []error{nil, individualErr}, + } + + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + AutoDetectBypass: true, // Enabled + WindowMs: 50, + MinWaitMs: 5, + SafetyMarginMs: 5, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + CachePerCall: util.BoolPtr(false), + } + cfg.SetDefaults() + + batcher := NewBatcher(cfg, forwarder, nil) + require.NotNil(t, batcher) + defer batcher.Shutdown() + + ctx := context.Background() + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + targetAddr := "0x1111111111111111111111111111111111111111" + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{"to": targetAddr, "data": "0x01"}, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + + entry, bypass, err := batcher.Enqueue(ctx, key, req) + require.NoError(t, err) + require.False(t, bypass) + + // Wait for result + select { + case result := <-entry.ResultCh: + // Should fail (both multicall and individual call failed) + require.Error(t, result.Error) + case <-time.After(2 * time.Second): + t.Fatal("timeout waiting for result") + } + + // Verify contract was NOT added to runtime bypass (it fails both ways) + require.False(t, batcher.IsRuntimeBypassed(targetAddr)) + + // Verify two calls were made (multicall + individual retry) + require.Equal(t, 2, forwarder.CallCount()) +} + +func TestBatcher_RuntimeBypass_SkipsEnqueue(t *testing.T) { + // Test that once a contract is in runtime bypass, Enqueue returns bypass=true + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + AutoDetectBypass: true, + WindowMs: 50, + MinWaitMs: 5, + SafetyMarginMs: 5, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + AllowCrossUserBatching: util.BoolPtr(true), + CachePerCall: util.BoolPtr(false), + } + cfg.SetDefaults() + + forwarder := &mockForwarder{} + batcher := NewBatcher(cfg, forwarder, nil) + require.NotNil(t, batcher) + defer batcher.Shutdown() + + // Pre-populate runtime bypass + targetAddr := "0x1111111111111111111111111111111111111111" + targetAddrNormalized := "1111111111111111111111111111111111111111" + batcher.addRuntimeBypass(targetAddrNormalized, "test-project", "evm:1") + + ctx := context.Background() + key := BatchingKey{ + ProjectId: "test-project", + NetworkId: "evm:1", + BlockRef: "latest", + DirectivesKey: DeriveDirectivesKey(nil), + } + + jrq := common.NewJsonRpcRequest("eth_call", []interface{}{ + map[string]interface{}{"to": targetAddr, "data": "0x01"}, + "latest", + }) + req := common.NewNormalizedRequestFromJsonRpcRequest(jrq) + + // Enqueue should return bypass=true + entry, bypass, err := batcher.Enqueue(ctx, key, req) + require.NoError(t, err) + require.True(t, bypass, "should bypass for runtime-bypassed contract") + require.Nil(t, entry, "entry should be nil when bypassing") +} + +func TestBatcher_RuntimeBypass_ConcurrentAccess(t *testing.T) { + // Test that concurrent reads and writes to the runtime bypass cache are safe + cfg := &common.Multicall3AggregationConfig{ + Enabled: true, + AutoDetectBypass: true, + WindowMs: 50, + MinWaitMs: 5, + SafetyMarginMs: 5, + MaxCalls: 10, + MaxCalldataBytes: 64000, + MaxQueueSize: 100, + MaxPendingBatches: 20, + } + cfg.SetDefaults() + + forwarder := &mockForwarder{} + batcher := NewBatcher(cfg, forwarder, nil) + require.NotNil(t, batcher) + defer batcher.Shutdown() + + // Concurrent writes to the same address + var wg sync.WaitGroup + addr := "1111111111111111111111111111111111111111" + + // Start 50 goroutines that add the same address + for i := 0; i < 50; i++ { + wg.Add(1) + go func() { + defer wg.Done() + batcher.addRuntimeBypass(addr, "test-project", "evm:1") + }() + } + + // Start 50 goroutines that read the address + for i := 0; i < 50; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _ = batcher.isRuntimeBypassed(addr) + }() + } + + // Start 50 goroutines that add different addresses + for i := 0; i < 50; i++ { + wg.Add(1) + go func(idx int) { + defer wg.Done() + uniqueAddr := fmt.Sprintf("%040d", idx) + batcher.addRuntimeBypass(uniqueAddr, "test-project", "evm:1") + }(i) + } + + wg.Wait() + + // Verify the shared address is bypassed + require.True(t, batcher.isRuntimeBypassed(addr)) + + // Verify all unique addresses are bypassed + for i := 0; i < 50; i++ { + uniqueAddr := fmt.Sprintf("%040d", i) + require.True(t, batcher.isRuntimeBypassed(uniqueAddr), "unique address %d should be bypassed", i) + } +} diff --git a/common/config.go b/common/config.go index 591a2aff3..15e272499 100644 --- a/common/config.go +++ b/common/config.go @@ -2,6 +2,7 @@ package common import ( "bytes" + "encoding/hex" "encoding/json" "fmt" "maps" @@ -1605,6 +1606,21 @@ type Multicall3AggregationConfig struct { // AllowPendingTagBatching: if true, allow batching calls with "pending" block tag. // Default: false AllowPendingTagBatching bool `yaml:"allowPendingTagBatching,omitempty" json:"allowPendingTagBatching"` + + // AutoDetectBypass: if true, automatically detect contracts that revert when called + // via Multicall3 (e.g., contracts checking msg.sender code size). When a call reverts + // in a batch but succeeds individually, the contract is added to a runtime bypass cache. + // Default: false + AutoDetectBypass bool `yaml:"autoDetectBypass,omitempty" json:"autoDetectBypass"` + + // BypassContracts is a list of contract addresses that should NOT be batched via Multicall3. + // Use this for contracts that check if msg.sender has code (e.g., Chronicle Oracle feeds) + // and revert when called from a contract. Addresses are case-insensitive. + // Example: ["0x057f30e63A69175C69A4Af5656b8C9EE647De3D0"] + BypassContracts []string `yaml:"bypassContracts,omitempty" json:"bypassContracts,omitempty"` + + // bypassContractsMap is a pre-computed map for O(1) lookups (lowercase addresses without 0x prefix) + bypassContractsMap map[string]bool `yaml:"-" json:"-"` } // SetDefaults applies default values to unset fields @@ -1636,6 +1652,49 @@ func (c *Multicall3AggregationConfig) SetDefaults() { if c.AllowCrossUserBatching == nil { c.AllowCrossUserBatching = &TRUE } + // Initialize bypass contracts map for O(1) lookups + c.initBypassContractsMap() +} + +// initBypassContractsMap builds the internal map for fast bypass lookups. +// Addresses are normalized to lowercase without the 0x/0X prefix. +func (c *Multicall3AggregationConfig) initBypassContractsMap() { + if len(c.BypassContracts) == 0 { + c.bypassContractsMap = nil + return + } + c.bypassContractsMap = make(map[string]bool, len(c.BypassContracts)) + for _, addr := range c.BypassContracts { + // Normalize: lowercase and remove 0x/0X prefix + normalized := strings.ToLower(addr) + normalized = strings.TrimPrefix(normalized, "0x") + if normalized != "" { + c.bypassContractsMap[normalized] = true + } + } +} + +// ShouldBypassContract checks if the given contract address should bypass multicall3 batching. +// The address should be the raw 20-byte address (not hex-encoded). +func (c *Multicall3AggregationConfig) ShouldBypassContract(target []byte) bool { + if c.bypassContractsMap == nil || len(target) == 0 { + return false + } + // Convert target bytes to lowercase hex string (without 0x prefix) + targetHex := strings.ToLower(hex.EncodeToString(target)) + return c.bypassContractsMap[targetHex] +} + +// ShouldBypassContractHex checks if the given hex-encoded contract address should bypass multicall3 batching. +// The address can be with or without the 0x/0X prefix, and is case-insensitive. +func (c *Multicall3AggregationConfig) ShouldBypassContractHex(targetHex string) bool { + if c.bypassContractsMap == nil || targetHex == "" { + return false + } + // Normalize: lowercase and remove 0x/0X prefix + normalized := strings.ToLower(targetHex) + normalized = strings.TrimPrefix(normalized, "0x") + return c.bypassContractsMap[normalized] } // IsValid checks if the config values are valid @@ -1661,6 +1720,23 @@ func (c *Multicall3AggregationConfig) IsValid() error { if c.MaxPendingBatches <= 0 { return fmt.Errorf("multicall3Aggregation.maxPendingBatches must be > 0") } + // Validate bypass contract addresses + for _, addr := range c.BypassContracts { + normalized := strings.ToLower(strings.TrimPrefix(strings.TrimPrefix(addr, "0x"), "0X")) + if normalized == "" { + return fmt.Errorf("multicall3Aggregation.bypassContracts contains empty address") + } + // Ethereum addresses are 20 bytes = 40 hex characters + if len(normalized) != 40 { + return fmt.Errorf("multicall3Aggregation.bypassContracts contains invalid address %q (expected 40 hex characters, got %d)", addr, len(normalized)) + } + // Validate hex characters + for _, c := range normalized { + if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')) { + return fmt.Errorf("multicall3Aggregation.bypassContracts contains invalid address %q (non-hex character)", addr) + } + } + } return nil } diff --git a/common/config_test.go b/common/config_test.go index 297dc37d1..a7a0cd3b6 100644 --- a/common/config_test.go +++ b/common/config_test.go @@ -997,4 +997,153 @@ func TestMulticall3AggregationConfigIsValid(t *testing.T) { require.Error(t, err) require.Contains(t, err.Error(), "maxPendingBatches must be > 0") }) + + t.Run("bypassContracts valid addresses", func(t *testing.T) { + cfg := &Multicall3AggregationConfig{ + Enabled: true, + BypassContracts: []string{ + "0x057f30e63A69175C69A4Af5656b8C9EE647De3D0", + "0xABCDEF0123456789ABCDEF0123456789ABCDEF01", + "1111111111111111111111111111111111111111", // without 0x prefix + }, + } + cfg.SetDefaults() + err := cfg.IsValid() + require.NoError(t, err) + }) + + t.Run("bypassContracts empty address rejected", func(t *testing.T) { + cfg := &Multicall3AggregationConfig{ + Enabled: true, + BypassContracts: []string{ + "0x057f30e63A69175C69A4Af5656b8C9EE647De3D0", + "", // empty + }, + } + cfg.SetDefaults() + err := cfg.IsValid() + require.Error(t, err) + require.Contains(t, err.Error(), "contains empty address") + }) + + t.Run("bypassContracts invalid length rejected", func(t *testing.T) { + cfg := &Multicall3AggregationConfig{ + Enabled: true, + BypassContracts: []string{ + "0x057f30e63A69175C69A4Af5656b8C9EE647De3D0", + "0x1234", // too short + }, + } + cfg.SetDefaults() + err := cfg.IsValid() + require.Error(t, err) + require.Contains(t, err.Error(), "expected 40 hex characters") + }) + + t.Run("bypassContracts non-hex characters rejected", func(t *testing.T) { + cfg := &Multicall3AggregationConfig{ + Enabled: true, + BypassContracts: []string{ + "0x057f30e63A69175C69A4Af5656b8C9EE647De3D0", + "0xZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ", // non-hex + }, + } + cfg.SetDefaults() + err := cfg.IsValid() + require.Error(t, err) + require.Contains(t, err.Error(), "non-hex character") + }) +} + +func TestMulticall3AggregationConfigBypassContracts(t *testing.T) { + t.Run("ShouldBypassContractHex with empty list", func(t *testing.T) { + cfg := &Multicall3AggregationConfig{ + Enabled: true, + BypassContracts: []string{}, + } + cfg.SetDefaults() + + // Should not bypass any contract when list is empty + require.False(t, cfg.ShouldBypassContractHex("0x057f30e63A69175C69A4Af5656b8C9EE647De3D0")) + require.False(t, cfg.ShouldBypassContractHex("")) + }) + + t.Run("ShouldBypassContractHex with configured contracts", func(t *testing.T) { + cfg := &Multicall3AggregationConfig{ + Enabled: true, + BypassContracts: []string{ + "0x057f30e63A69175C69A4Af5656b8C9EE647De3D0", + "0xABCDEF0123456789ABCDEF0123456789ABCDEF01", + }, + } + cfg.SetDefaults() + + // Exact match + require.True(t, cfg.ShouldBypassContractHex("0x057f30e63A69175C69A4Af5656b8C9EE647De3D0")) + require.True(t, cfg.ShouldBypassContractHex("0xABCDEF0123456789ABCDEF0123456789ABCDEF01")) + + // Case-insensitive matching + require.True(t, cfg.ShouldBypassContractHex("0x057f30e63a69175c69a4af5656b8c9ee647de3d0")) + require.True(t, cfg.ShouldBypassContractHex("0x057F30E63A69175C69A4AF5656B8C9EE647DE3D0")) + + // Without 0x prefix + require.True(t, cfg.ShouldBypassContractHex("057f30e63A69175C69A4Af5656b8C9EE647De3D0")) + + // Not in list + require.False(t, cfg.ShouldBypassContractHex("0x1111111111111111111111111111111111111111")) + require.False(t, cfg.ShouldBypassContractHex("")) + }) + + t.Run("ShouldBypassContract with raw bytes", func(t *testing.T) { + cfg := &Multicall3AggregationConfig{ + Enabled: true, + BypassContracts: []string{ + "0x057f30e63A69175C69A4Af5656b8C9EE647De3D0", + }, + } + cfg.SetDefaults() + + // Matching address bytes + matchingAddr, _ := HexToBytes("0x057f30e63A69175C69A4Af5656b8C9EE647De3D0") + require.True(t, cfg.ShouldBypassContract(matchingAddr)) + + // Non-matching address bytes + otherAddr, _ := HexToBytes("0x1111111111111111111111111111111111111111") + require.False(t, cfg.ShouldBypassContract(otherAddr)) + + // Empty bytes + require.False(t, cfg.ShouldBypassContract(nil)) + require.False(t, cfg.ShouldBypassContract([]byte{})) + }) + + t.Run("BypassContracts handles invalid entries gracefully", func(t *testing.T) { + cfg := &Multicall3AggregationConfig{ + Enabled: true, + BypassContracts: []string{ + "0x057f30e63A69175C69A4Af5656b8C9EE647De3D0", + "", // empty string + " ", // whitespace only becomes empty after trim + "0x", // just prefix + "invalid", // non-hex characters + }, + } + cfg.SetDefaults() + + // Valid entry should still work + require.True(t, cfg.ShouldBypassContractHex("0x057f30e63A69175C69A4Af5656b8C9EE647De3D0")) + + // Empty/invalid entries should be ignored (not panic) + require.False(t, cfg.ShouldBypassContractHex("")) + }) + + t.Run("nil config does not panic", func(t *testing.T) { + var cfg *Multicall3AggregationConfig + // Should not panic, just return false + require.NotPanics(t, func() { + // Note: ShouldBypassContractHex is a method on a pointer, so nil check is inside + if cfg != nil { + cfg.ShouldBypassContractHex("0x1234") + } + }) + }) } diff --git a/docs/design/multicall3-batching.md b/docs/design/multicall3-batching.md index 56f8dee8e..7b3786c0b 100644 --- a/docs/design/multicall3-batching.md +++ b/docs/design/multicall3-batching.md @@ -34,10 +34,66 @@ A request is eligible if: - Method is `eth_call`. - Call object contains only `to` and `data|input`. - Request is not already a multicall (recursion guard). +- Target contract is not in the `bypassContracts` list. - Calls with any of `from`, `gas`, `gasPrice`, `maxFeePerGas`, `maxPriorityFeePerGas`, `value`, or a state override (third param) are ineligible. +## Bypass Contracts + +Some contracts check `msg.sender` using `extcodesize()` and revert if the caller +has code (i.e., is a contract). When using Multicall3, `msg.sender` becomes the +Multicall3 contract address, causing these calls to fail. + +Use `bypassContracts` to exclude specific contracts from batching: + +```yaml +evm: + multicall3Aggregation: + enabled: true + bypassContracts: + # Chronicle Oracle feeds check msg.sender code size + - "0x057f30e63A69175C69A4Af5656b8C9EE647De3D0" + # Other contracts that revert on contract callers + - "0xABCDEF0123456789ABCDEF0123456789ABCDEF01" +``` + +Addresses are case-insensitive and can be specified with or without the `0x` +prefix. Calls to bypassed contracts are forwarded individually. + +## Auto-Detect Bypass + +Enable `autoDetectBypass` to automatically detect contracts that revert when +called via Multicall3 but succeed when called individually: + +```yaml +evm: + multicall3Aggregation: + enabled: true + autoDetectBypass: true +``` + +When a call reverts within a Multicall3 batch and `autoDetectBypass` is enabled: +1. The call is retried individually (bypassing Multicall3). +2. If the individual call succeeds, the contract is added to a runtime bypass + cache and future calls skip batching. +3. If the individual call also fails, the original error is returned (no bypass + is added). + +This allows automatic discovery of contracts that check `msg.sender` code size +(e.g., Chronicle Oracle) without requiring manual configuration. + +Metrics: +- `erpc_multicall3_runtime_bypass_total{project, network}`: Contracts added to + runtime bypass. +- `erpc_multicall3_auto_detect_retry_total{project, network, outcome}`: Retry + tracking. Outcome values: `attempt` (retry initiated), `detected` (bypass + discovered - individual call succeeded), `same_error` (individual call also + failed, not a bypass candidate). + +Note: The runtime bypass cache is in-memory and resets on restart. For known +contracts, use `bypassContracts` for persistent configuration. + Block reference normalization: - Use `NormalizeBlockParam` on the `eth_call` block parameter. - `nil` becomes `latest`. @@ -208,6 +264,10 @@ evm: cachePerCall: true allowCrossUserBatching: true allowPendingTagBatching: false + autoDetectBypass: false + bypassContracts: + # Contracts that check msg.sender code size (e.g., Chronicle Oracle) + - "0x057f30e63A69175C69A4Af5656b8C9EE647De3D0" ``` Validation and defaults: diff --git a/telemetry/metrics.go b/telemetry/metrics.go index 6a1d72850..fd646d2f4 100644 --- a/telemetry/metrics.go +++ b/telemetry/metrics.go @@ -498,6 +498,18 @@ var ( Name: "multicall3_cache_write_dropped_total", Help: "Total number of multicall3 per-call cache writes dropped due to backpressure.", }, []string{"project", "network"}) + + MetricMulticall3RuntimeBypassTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "multicall3_runtime_bypass_total", + Help: "Total number of contracts auto-detected as requiring bypass (revert via multicall3 but succeed individually).", + }, []string{"project", "network"}) + + MetricMulticall3AutoDetectRetryTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "erpc", + Name: "multicall3_auto_detect_retry_total", + Help: "Total number of auto-detect retry attempts for reverted calls.", + }, []string{"project", "network", "outcome"}) ) var DefaultHistogramBuckets = []float64{