From f005fd1acfdbafae62615d95294ad6c1df6df249 Mon Sep 17 00:00:00 2001 From: vietddude Date: Fri, 6 Feb 2026 15:21:57 +0700 Subject: [PATCH 01/11] feat: add TON blockchain indexing, RPC client, and worker support --- configs/config.example.yaml | 15 + go.mod | 2 + go.sum | 4 + internal/indexer/ton/cursor.go | 93 +++++ internal/indexer/ton/indexer.go | 181 ++++++++++ internal/indexer/ton/jetton_parser.go | 222 ++++++++++++ internal/indexer/ton/jetton_parser_test.go | 102 ++++++ internal/indexer/ton/jetton_registry.go | 81 +++++ internal/indexer/ton/parser.go | 145 ++++++++ internal/indexer/ton/parser_test.go | 98 ++++++ internal/rpc/ton/api.go | 21 ++ internal/rpc/ton/client.go | 144 ++++++++ internal/worker/factory.go | 78 +++++ internal/worker/ton/worker.go | 388 +++++++++++++++++++++ pkg/common/config/types.go | 9 +- pkg/common/enum/enum.go | 1 + sql/wallet_address.sql | 5 +- test/parse_ton_txn/main.go | 151 ++++++++ 18 files changed, 1738 insertions(+), 2 deletions(-) create mode 100644 internal/indexer/ton/cursor.go create mode 100644 internal/indexer/ton/indexer.go create mode 100644 internal/indexer/ton/jetton_parser.go create mode 100644 internal/indexer/ton/jetton_parser_test.go create mode 100644 internal/indexer/ton/jetton_registry.go create mode 100644 internal/indexer/ton/parser.go create mode 100644 internal/indexer/ton/parser_test.go create mode 100644 internal/rpc/ton/api.go create mode 100644 internal/rpc/ton/client.go create mode 100644 internal/worker/ton/worker.go create mode 100644 test/parse_ton_txn/main.go diff --git a/configs/config.example.yaml b/configs/config.example.yaml index 583075f..72c7492 100644 --- a/configs/config.example.yaml +++ b/configs/config.example.yaml @@ -161,6 +161,21 @@ chains: - url: "fullnode.mainnet.sui.io:443" # e.g. 127.0.0.1:9000 - url: "sui-mainnet.nodeinfra.com:443" + ton_mainnet: + network_id: "ton" + internal_code: "TON_MAINNET" + type: "ton" + poll_interval: "5s" + nodes: + - url: "https://ton.org/global.config.json" + throttle: + concurrency: 10 + batch_size: 50 + jettons: + - master_address: "EQCxE6mUtQJKFnGfaROTgt1lZbDiiX1Gw83iAV82Cn_0opUb" # USDT + symbol: "USDT" + decimals: 6 + # Infrastructure services services: port: 8080 # Health check and monitoring server port diff --git a/go.mod b/go.mod index fef59e3..5c59bea 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/shopspring/decimal v1.4.0 github.com/stretchr/testify v1.11.1 github.com/tyler-smith/go-bip39 v1.1.0 + github.com/xssnick/tonutils-go v1.15.5 golang.org/x/crypto v0.44.0 golang.org/x/sync v0.18.0 google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda @@ -34,6 +35,7 @@ require ( replace github.com/imdario/mergo => github.com/imdario/mergo v0.3.16 require ( + filippo.io/edwards25519 v1.1.0 // indirect github.com/armon/go-metrics v0.4.1 // indirect github.com/bits-and-blooms/bitset v1.10.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/go.sum b/go.sum index 73dba2b..ab89f67 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= @@ -281,6 +283,8 @@ github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= +github.com/xssnick/tonutils-go v1.15.5 h1:yAcHnDaY5QW0aIQE47lT0PuDhhHYE+N+NyZssdPKR0s= +github.com/xssnick/tonutils-go v1.15.5/go.mod h1:3/B8mS5IWLTd1xbGbFbzRem55oz/Q86HG884bVsTqZ8= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= diff --git a/internal/indexer/ton/cursor.go b/internal/indexer/ton/cursor.go new file mode 100644 index 0000000..b89e70c --- /dev/null +++ b/internal/indexer/ton/cursor.go @@ -0,0 +1,93 @@ +package ton + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/fystack/multichain-indexer/pkg/infra" +) + +const cursorKeyPrefix = "ton/cursor/" + +// AccountCursor tracks the polling position for a single TON account. +type AccountCursor struct { + Address string `json:"address"` + LastLT uint64 `json:"last_lt"` // Logical time of last processed tx + LastHash string `json:"last_hash"` // Hex-encoded hash of last processed tx + UpdatedAt time.Time `json:"updated_at"` +} + +// CursorStore manages account cursors for TON polling. +type CursorStore interface { + // Get returns the cursor for an account, or nil if not found. + Get(ctx context.Context, address string) (*AccountCursor, error) + + // Save persists the cursor atomically. + Save(ctx context.Context, cursor *AccountCursor) error + + // Delete removes the cursor for an account. + Delete(ctx context.Context, address string) error + + // List returns all tracked account addresses. + List(ctx context.Context) ([]string, error) +} + +// kvCursorStore implements CursorStore using infra.KVStore. +type kvCursorStore struct { + kv infra.KVStore +} + +func NewCursorStore(kv infra.KVStore) CursorStore { + return &kvCursorStore{kv: kv} +} + +func (s *kvCursorStore) cursorKey(address string) string { + return cursorKeyPrefix + address +} + +func (s *kvCursorStore) Get(ctx context.Context, address string) (*AccountCursor, error) { + var cursor AccountCursor + found, err := s.kv.GetAny(s.cursorKey(address), &cursor) + if err != nil { + return nil, fmt.Errorf("failed to get cursor for %s: %w", address, err) + } + if !found { + return nil, nil + } + return &cursor, nil +} + +func (s *kvCursorStore) Save(ctx context.Context, cursor *AccountCursor) error { + cursor.UpdatedAt = time.Now() + if err := s.kv.SetAny(s.cursorKey(cursor.Address), cursor); err != nil { + return fmt.Errorf("failed to save cursor for %s: %w", cursor.Address, err) + } + return nil +} + +func (s *kvCursorStore) Delete(ctx context.Context, address string) error { + if err := s.kv.Delete(s.cursorKey(address)); err != nil { + return fmt.Errorf("failed to delete cursor for %s: %w", address, err) + } + return nil +} + +func (s *kvCursorStore) List(ctx context.Context) ([]string, error) { + pairs, err := s.kv.List(cursorKeyPrefix) + if err != nil { + return nil, fmt.Errorf("failed to list cursors: %w", err) + } + + addresses := make([]string, 0, len(pairs)) + for _, pair := range pairs { + var cursor AccountCursor + if err := json.Unmarshal(pair.Value, &cursor); err != nil { + continue // Skip malformed entries + } + addresses = append(addresses, cursor.Address) + } + + return addresses, nil +} diff --git a/internal/indexer/ton/indexer.go b/internal/indexer/ton/indexer.go new file mode 100644 index 0000000..66b40bb --- /dev/null +++ b/internal/indexer/ton/indexer.go @@ -0,0 +1,181 @@ +package ton + +import ( + "context" + "encoding/hex" + "fmt" + + tonRpc "github.com/fystack/multichain-indexer/internal/rpc/ton" + "github.com/fystack/multichain-indexer/pkg/common/config" + "github.com/fystack/multichain-indexer/pkg/common/enum" + "github.com/fystack/multichain-indexer/pkg/common/types" + "github.com/xssnick/tonutils-go/address" + "github.com/xssnick/tonutils-go/tlb" +) + +// AccountIndexer is the interface for account-based indexing (TON). +// This is separate from the block-based Indexer interface used by EVM/Solana/etc. +type AccountIndexer interface { + GetName() string + GetNetworkType() enum.NetworkType + GetNetworkInternalCode() string + + // PollAccount fetches new transactions for a single account. + // Returns parsed transactions and the new cursor position. + // If no new transactions, returns empty slice and same cursor. + PollAccount(ctx context.Context, address string, cursor *AccountCursor) ([]types.Transaction, *AccountCursor, error) + + // IsHealthy checks if the RPC connection is healthy. + IsHealthy() bool +} + +// TonAccountIndexer implements AccountIndexer for TON blockchain. +type TonAccountIndexer struct { + chainName string + config config.ChainConfig + client tonRpc.TonAPI + jettonRegistry JettonRegistry + + // Transaction limit per poll + txLimit uint32 +} + +// NewTonAccountIndexer creates a new TON account indexer. +func NewTonAccountIndexer( + chainName string, + cfg config.ChainConfig, + client tonRpc.TonAPI, + jettonRegistry JettonRegistry, +) *TonAccountIndexer { + txLimit := uint32(50) // Default transaction limit + if cfg.Throttle.BatchSize > 0 { + txLimit = uint32(cfg.Throttle.BatchSize) + } + + return &TonAccountIndexer{ + chainName: chainName, + config: cfg, + client: client, + jettonRegistry: jettonRegistry, + txLimit: txLimit, + } +} + +func (i *TonAccountIndexer) GetName() string { return i.chainName } +func (i *TonAccountIndexer) GetNetworkType() enum.NetworkType { return enum.NetworkTypeTon } +func (i *TonAccountIndexer) GetNetworkInternalCode() string { return i.config.InternalCode } + +// IsHealthy checks if the RPC connection is healthy. +func (i *TonAccountIndexer) IsHealthy() bool { + // The client manages its own connection pool and recovery. + // We consider it healthy if it's initialized. + return i.client != nil +} + +// PollAccount fetches new transactions for a single account. +func (i *TonAccountIndexer) PollAccount(ctx context.Context, addrStr string, cursor *AccountCursor) ([]types.Transaction, *AccountCursor, error) { + // Parse the address + addr, err := address.ParseAddr(addrStr) + if err != nil { + return nil, cursor, fmt.Errorf("invalid TON address %s: %w", addrStr, err) + } + + // Prepare cursor values for API call + var lastLT uint64 + var lastHash []byte + if cursor != nil && cursor.LastLT > 0 { + lastLT = cursor.LastLT + if cursor.LastHash != "" { + lastHash, err = hex.DecodeString(cursor.LastHash) + if err != nil { + return nil, cursor, fmt.Errorf("invalid cursor hash: %w", err) + } + } + } + + // Fetch transactions from the account + // The client handles failover and retries internally + txs, err := i.client.ListTransactions(ctx, addr, i.txLimit, lastLT, lastHash) + if err != nil { + return nil, cursor, fmt.Errorf("failed to list transactions: %w", err) + } + + // No new transactions + if len(txs) == 0 { + return nil, cursor, nil + } + + // Process transactions in reverse order (oldest first for proper cursor updates) + // TON API returns newest first, so we iterate backwards + var parsedTxs []types.Transaction + newCursor := cursor + if newCursor == nil { + newCursor = &AccountCursor{Address: addrStr} + } + + // Normalize address string for consistent comparison + addrStr = addr.String() + + for j := len(txs) - 1; j >= 0; j-- { + tx := txs[j] + + // Skip transactions we've already processed (at or before cursor) + if cursor != nil && tx.LT <= cursor.LastLT { + continue + } + + // Update cursor to this transaction (even if it's skipped for failure) + newCursor.LastLT = tx.LT + newCursor.LastHash = hex.EncodeToString(tx.Hash) + + // Check if transaction was successful + if !isTransactionSuccess(tx) { + continue + } + + var collectedTxs []types.Transaction + collectedTxs = append(collectedTxs, ParseTonTransfer(tx, addrStr, i.config.InternalCode)...) + + if i.jettonRegistry != nil { + collectedTxs = append(collectedTxs, ParseJettonTransfer(tx, addrStr, i.config.InternalCode, i.jettonRegistry)...) + } + + parsedTxs = append(parsedTxs, collectedTxs...) + } + return parsedTxs, newCursor, nil +} + +// isTransactionSuccess checks if the transaction was successful by examining its phases. +func isTransactionSuccess(tx *tlb.Transaction) bool { + if tx.Description == nil { + return true // Should not happen with valid transactions + } + + desc, ok := tx.Description.(*tlb.TransactionDescriptionOrdinary) + if !ok { + return true + } + + // Check if the whole transaction was aborted + if desc.Aborted { + return false + } + + // Check Compute Phase + if desc.ComputePhase.Phase != nil { + if cp, ok := desc.ComputePhase.Phase.(*tlb.ComputePhaseVM); ok { + if !cp.Success { + return false + } + } + } + + // Check Action Phase + if desc.ActionPhase != nil { + if !desc.ActionPhase.Success { + return false + } + } + + return true +} diff --git a/internal/indexer/ton/jetton_parser.go b/internal/indexer/ton/jetton_parser.go new file mode 100644 index 0000000..b962162 --- /dev/null +++ b/internal/indexer/ton/jetton_parser.go @@ -0,0 +1,222 @@ +package ton + +import ( + "encoding/base64" + + "github.com/fystack/multichain-indexer/pkg/common/constant" + "github.com/fystack/multichain-indexer/pkg/common/types" + "github.com/shopspring/decimal" + "github.com/xssnick/tonutils-go/tlb" +) + +// Jetton standard opcodes (TEP-74) +const ( + OpcodeTransfer uint64 = 0x0f8a7ea5 + OpcodeTransferNotification uint64 = 0x7362d09c +) + +// JettonInfo describes a supported Jetton token. +type JettonInfo struct { + MasterAddress string `json:"master_address" yaml:"master_address"` // Jetton master contract + Symbol string `json:"symbol" yaml:"symbol"` + Decimals int `json:"decimals" yaml:"decimals"` +} + +// JettonRegistry manages supported Jetton tokens. +type JettonRegistry interface { + // IsSupported checks if a Jetton wallet belongs to a supported Jetton. + // This may require looking up the wallet's master address. + IsSupported(walletAddress string) bool + + // GetInfo returns info for a Jetton by its master address. + GetInfo(masterAddress string) (*JettonInfo, bool) + + // GetInfoByWallet returns info for a Jetton by a wallet address. + // Returns nil if the wallet is not from a known Jetton. + GetInfoByWallet(walletAddress string) (*JettonInfo, bool) + + // RegisterWallet associates a Jetton wallet with its master address. + RegisterWallet(walletAddress, masterAddress string) + + // List returns all supported Jettons. + List() []JettonInfo +} + +// ParseJettonTransfer extracts a Jetton transfer from a transaction. +// Detects both incoming (receive) and outgoing (send) Jetton transfers. +// Returns a slice of parsed transactions involving our address. +func ParseJettonTransfer(tx *tlb.Transaction, ourAddress string, networkID string, registry JettonRegistry) []types.Transaction { + var results []types.Transaction + + // Try incoming transfer notification first (receive) + if parsed, ok := parseIncomingJetton(tx, ourAddress, networkID, registry); ok { + results = append(results, *parsed) + } + + // Try outgoing transfers (send) - can be multiple + results = append(results, parseOutgoingJettons(tx, ourAddress, networkID, registry)...) + + return results +} + +// parseIncomingJetton handles incoming Jetton transfer notifications (receive). +func parseIncomingJetton(tx *tlb.Transaction, ourAddress string, networkID string, registry JettonRegistry) (*types.Transaction, bool) { + // Check for incoming internal message + if tx.IO.In == nil { + return nil, false + } + + // Only process internal messages + inMsg := tx.IO.In.Msg + intMsg, ok := inMsg.(*tlb.InternalMessage) + if !ok || intMsg.Bounced { + return nil, false + } + + // Verify destination is our account + dstAddr := intMsg.DstAddr.String() + if dstAddr != ourAddress { + return nil, false + } + + // Parse message body for transfer_notification + if intMsg.Body == nil { + return nil, false + } + + bodySlice := intMsg.Body.BeginParse() + opcode, err := bodySlice.LoadUInt(32) + if err != nil || opcode != OpcodeTransferNotification { + return nil, false + } + + // Structure: query_id:uint64 amount:VarUInteger16 sender:Address forward_payload:Either Cell ^Cell + _, err = bodySlice.LoadUInt(64) // query_id + if err != nil { + return nil, false + } + + jettonAmount, err := bodySlice.LoadVarUInt(16) + if err != nil { + return nil, false + } + + sender, err := bodySlice.LoadAddr() + if err != nil { + return nil, false + } + + // The sender of the notification is the Jetton wallet + jettonWallet := intMsg.SrcAddr.String() + + // Try to get Jetton info from registry + var assetAddress string + if registry != nil { + if info, ok := registry.GetInfoByWallet(jettonWallet); ok { + assetAddress = info.MasterAddress + } else { + // If not found in cache, it might be an unknown Jetton + // or we need to wait for a registration + assetAddress = jettonWallet + } + } else { + assetAddress = jettonWallet + } + + return &types.Transaction{ + TxHash: base64.StdEncoding.EncodeToString(tx.Hash), + NetworkId: networkID, + BlockNumber: tx.LT, + FromAddress: sender.String(), + ToAddress: ourAddress, + AssetAddress: assetAddress, + Amount: jettonAmount.String(), + Type: constant.TxTypeTokenTransfer, + TxFee: decimal.NewFromBigInt(tx.TotalFees.Coins.Nano(), 0).Div(decimal.NewFromInt(1e9)), + Timestamp: uint64(tx.Now), + Status: types.StatusConfirmed, + }, true +} + +// parseOutgoingJettons handles outgoing Jetton transfers (send). +func parseOutgoingJettons(tx *tlb.Transaction, ourAddress string, networkID string, registry JettonRegistry) []types.Transaction { + var results []types.Transaction + + // Check for outgoing messages + if tx.IO.Out == nil { + return nil + } + + outList, err := tx.IO.Out.ToSlice() + if err != nil { + return nil + } + + for _, outMsg := range outList { + intMsg, ok := outMsg.Msg.(*tlb.InternalMessage) + if !ok { + continue + } + + // Parse message body for transfer opcode + if intMsg.Body == nil { + continue + } + + bodySlice := intMsg.Body.BeginParse() + opcode, err := bodySlice.LoadUInt(32) + if err != nil || opcode != OpcodeTransfer { + continue + } + + // Parse transfer fields + // Skip query_id + _, err = bodySlice.LoadUInt(64) + if err != nil { + continue + } + + // Load Jetton amount + jettonAmount, err := bodySlice.LoadVarUInt(16) + if err != nil { + continue + } + + // Load destination address + destination, err := bodySlice.LoadAddr() + if err != nil { + continue + } + + // Get Jetton wallet (the destination of the outgoing message is our Jetton wallet) + jettonWallet := intMsg.DstAddr.String() + + // Try to get Jetton info from registry + var assetAddress string + if registry != nil { + if info, ok := registry.GetInfoByWallet(jettonWallet); ok { + assetAddress = info.MasterAddress + } else { + assetAddress = jettonWallet + } + } else { + assetAddress = jettonWallet + } + + results = append(results, types.Transaction{ + TxHash: base64.StdEncoding.EncodeToString(tx.Hash), + NetworkId: networkID, + BlockNumber: tx.LT, + FromAddress: ourAddress, + ToAddress: destination.String(), + AssetAddress: assetAddress, + Amount: jettonAmount.String(), + Type: constant.TxTypeTokenTransfer, + TxFee: decimal.NewFromBigInt(tx.TotalFees.Coins.Nano(), 0).Div(decimal.NewFromInt(1e9)), + Timestamp: uint64(tx.Now), + Status: types.StatusConfirmed, + }) + } + + return results +} diff --git a/internal/indexer/ton/jetton_parser_test.go b/internal/indexer/ton/jetton_parser_test.go new file mode 100644 index 0000000..fe78c0a --- /dev/null +++ b/internal/indexer/ton/jetton_parser_test.go @@ -0,0 +1,102 @@ +package ton + +import ( + "encoding/base64" + "testing" + "time" + + "github.com/fystack/multichain-indexer/pkg/common/constant" + "github.com/stretchr/testify/assert" + "github.com/xssnick/tonutils-go/address" + "github.com/xssnick/tonutils-go/tlb" + "github.com/xssnick/tonutils-go/tvm/cell" +) + +// MockRegistry implements JettonRegistry for testing +type MockRegistry struct { + info map[string]JettonInfo +} + +func (m *MockRegistry) IsSupported(walletAddress string) bool { + return false +} + +func (m *MockRegistry) GetInfo(masterAddress string) (*JettonInfo, bool) { + return nil, false +} + +func (m *MockRegistry) GetInfoByWallet(walletAddress string) (*JettonInfo, bool) { + if info, ok := m.info[walletAddress]; ok { + return &info, true + } + return nil, false +} + +func (m *MockRegistry) RegisterWallet(walletAddress, masterAddress string) {} +func (m *MockRegistry) List() []JettonInfo { return nil } + +func TestParseJettonTransfer(t *testing.T) { + ourAddr := address.MustParseAddr("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF") // Burn address + senderAddr := address.MustParseAddr("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c") // Zero address + // The jetton wallet that sends us the notification + jettonWalletAddr := address.MustParseAddr("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c") // Zero address + + t.Run("valid transfer notification", func(t *testing.T) { + // Mock registry + registry := &MockRegistry{ + info: map[string]JettonInfo{ + jettonWalletAddr.String(): { + MasterAddress: "EQMaster...", + Symbol: "USDT", + Decimals: 6, + }, + }, + } + + // Construct transfer notification body + // Opcode: 0x7362d09c (transfer_notification) + // query_id: uint64 + // amount: Coins + // sender: MsgAddress + // forward_payload: Either Cell ^Cell + body := cell.BeginCell(). + MustStoreUInt(0x7362d09c, 32). // Opcode + MustStoreUInt(0, 64). // QueryID + MustStoreCoins(1000000). // Amount (1 USDT) + MustStoreAddr(senderAddr). // Original sender + MustStoreRef( // Forward payload (comment) + cell.BeginCell(). + MustStoreUInt(0, 32). // Text comment opcode + MustStoreStringSnake("hello jetton"). + EndCell(), + ). + EndCell() + + tx := &tlb.Transaction{ + LT: 2000, + Now: uint32(time.Now().Unix()), + } + + tx.IO.In = &tlb.Message{ + MsgType: tlb.MsgTypeInternal, + Msg: &tlb.InternalMessage{ + SrcAddr: jettonWalletAddr, // Notification comes from jetton wallet + DstAddr: ourAddr, + Amount: tlb.MustFromTON("0.1"), // Small amount of TON attached + Body: body, + }, + } + + tx.Hash = []byte("txhash") + + results := ParseJettonTransfer(tx, ourAddr.String(), "TON_MAINNET", registry) + assert.Len(t, results, 1) + parsed := results[0] + assert.Equal(t, "1000000", parsed.Amount) + assert.Equal(t, senderAddr.String(), parsed.FromAddress) + assert.Equal(t, ourAddr.String(), parsed.ToAddress) + assert.Equal(t, "EQMaster...", parsed.AssetAddress) + assert.Equal(t, base64.StdEncoding.EncodeToString(tx.Hash), parsed.TxHash) + assert.Equal(t, constant.TxTypeTokenTransfer, parsed.Type) + }) +} diff --git a/internal/indexer/ton/jetton_registry.go b/internal/indexer/ton/jetton_registry.go new file mode 100644 index 0000000..6c19751 --- /dev/null +++ b/internal/indexer/ton/jetton_registry.go @@ -0,0 +1,81 @@ +package ton + +import ( + "sync" +) + +// ConfigBasedRegistry implements JettonRegistry with a static list of Jettons. +// Wallet-to-master mappings are cached as they're discovered. +type ConfigBasedRegistry struct { + jettons map[string]JettonInfo // key: master address + walletToMaster map[string]string // key: wallet address -> master address + mu sync.RWMutex +} + +// NewConfigBasedRegistry creates a registry from a list of supported Jettons. +func NewConfigBasedRegistry(jettons []JettonInfo) *ConfigBasedRegistry { + m := make(map[string]JettonInfo) + for _, j := range jettons { + m[j.MasterAddress] = j + } + return &ConfigBasedRegistry{ + jettons: m, + walletToMaster: make(map[string]string), + } +} + +// IsSupported checks if a Jetton wallet belongs to a supported Jetton. +func (r *ConfigBasedRegistry) IsSupported(walletAddress string) bool { + r.mu.RLock() + defer r.mu.RUnlock() + _, ok := r.walletToMaster[walletAddress] + return ok +} + +// GetInfo returns info for a Jetton by its master address. +func (r *ConfigBasedRegistry) GetInfo(masterAddress string) (*JettonInfo, bool) { + r.mu.RLock() + defer r.mu.RUnlock() + info, ok := r.jettons[masterAddress] + if !ok { + return nil, false + } + return &info, true +} + +// GetInfoByWallet returns info for a Jetton by a wallet address. +func (r *ConfigBasedRegistry) GetInfoByWallet(walletAddress string) (*JettonInfo, bool) { + r.mu.RLock() + masterAddr, ok := r.walletToMaster[walletAddress] + r.mu.RUnlock() + + if !ok { + return nil, false + } + + return r.GetInfo(masterAddr) +} + +// RegisterWallet associates a Jetton wallet with its master address. +// This is typically called when processing a Jetton transfer for the first time. +func (r *ConfigBasedRegistry) RegisterWallet(walletAddress, masterAddress string) { + r.mu.Lock() + defer r.mu.Unlock() + + // Only register if the master is in our supported list + if _, ok := r.jettons[masterAddress]; ok { + r.walletToMaster[walletAddress] = masterAddress + } +} + +// List returns all supported Jettons. +func (r *ConfigBasedRegistry) List() []JettonInfo { + r.mu.RLock() + defer r.mu.RUnlock() + + result := make([]JettonInfo, 0, len(r.jettons)) + for _, j := range r.jettons { + result = append(result, j) + } + return result +} diff --git a/internal/indexer/ton/parser.go b/internal/indexer/ton/parser.go new file mode 100644 index 0000000..dff3670 --- /dev/null +++ b/internal/indexer/ton/parser.go @@ -0,0 +1,145 @@ +package ton + +import ( + "encoding/base64" + + "github.com/fystack/multichain-indexer/pkg/common/constant" + "github.com/fystack/multichain-indexer/pkg/common/types" + "github.com/shopspring/decimal" + "github.com/xssnick/tonutils-go/tlb" +) + +// ParseTonTransfer extracts native TON transfers from a transaction. +// Detects both incoming (receive) and outgoing (send) transfers. +// Returns a slice of parsed transactions involving our address. +func ParseTonTransfer(tx *tlb.Transaction, ourAddress string, networkID string) []types.Transaction { + var results []types.Transaction + + // Try to parse as incoming transfer + if parsed, ok := parseIncomingTransfer(tx, ourAddress, networkID); ok { + results = append(results, *parsed) + } + + // Try to parse as outgoing transfers (can be multiple) + results = append(results, parseOutgoingTransfers(tx, ourAddress, networkID)...) + + return results +} + +// parseIncomingTransfer handles incoming TON transfers (receive). +func parseIncomingTransfer(tx *tlb.Transaction, ourAddress string, networkID string) (*types.Transaction, bool) { + // Check for incoming internal message + if tx.IO.In == nil { + return nil, false + } + + // Only process internal messages (wallet-to-wallet transfers) + inMsg := tx.IO.In.Msg + intMsg, ok := inMsg.(*tlb.InternalMessage) + if !ok || intMsg.Bounced { + return nil, false + } + + // Verify destination is our account + dstAddr := intMsg.DstAddr.String() + if dstAddr != ourAddress { + return nil, false + } + + // Extract amount (in nanoTON) + amount := intMsg.Amount.Nano().String() + + // Skip zero-value messages (typically these are just notifications) + if amount == "0" { + return nil, false + } + + // IF the message has a body, check if it's a simple comment (opcode 0) + // If it has a non-zero opcode, it's likely a contract interaction (like Jetton) + if intMsg.Body != nil { + bodySlice := intMsg.Body.BeginParse() + if bodySlice.BitsLeft() >= 32 { + opcode, err := bodySlice.LoadUInt(32) + if err == nil && opcode != 0 { + // Non-zero opcode means this is NOT a simple transfer + return nil, false + } + } + } + + return &types.Transaction{ + TxHash: base64.StdEncoding.EncodeToString(tx.Hash), + NetworkId: networkID, + BlockNumber: tx.LT, + FromAddress: intMsg.SrcAddr.String(), + ToAddress: dstAddr, + AssetAddress: "", // Empty for native TON + Amount: amount, + Type: constant.TxTypeNativeTransfer, // Incoming transfer + TxFee: decimal.NewFromBigInt(tx.TotalFees.Coins.Nano(), 0).Div(decimal.NewFromInt(1e9)), + Timestamp: uint64(tx.Now), + Status: types.StatusConfirmed, + }, true +} + +// parseOutgoingTransfers handles outgoing TON transfers (send). +func parseOutgoingTransfers(tx *tlb.Transaction, ourAddress string, networkID string) []types.Transaction { + var results []types.Transaction + + // Check for outgoing messages + if tx.IO.Out == nil { + return nil + } + + // Iterate through outgoing messages to find transfers + outList, err := tx.IO.Out.ToSlice() + if err != nil { + return nil + } + + for _, outMsg := range outList { + intMsg, ok := outMsg.Msg.(*tlb.InternalMessage) + if !ok { + continue + } + + // Verify source is our account + srcAddr := intMsg.SrcAddr.String() + if srcAddr != ourAddress { + continue + } + + // Extract amount + amount := intMsg.Amount.Nano().String() + if amount == "0" { + continue + } + + // Skip if it has a non-zero opcode (likely contract interaction) + if intMsg.Body != nil { + bodySlice := intMsg.Body.BeginParse() + if bodySlice.BitsLeft() >= 32 { + opcode, err := bodySlice.LoadUInt(32) + if err == nil && opcode != 0 { + continue + } + } + } + + results = append(results, types.Transaction{ + TxHash: base64.StdEncoding.EncodeToString(tx.Hash), + NetworkId: networkID, + BlockNumber: tx.LT, + FromAddress: srcAddr, + ToAddress: intMsg.DstAddr.String(), + AssetAddress: "", + Amount: amount, + Type: constant.TxTypeNativeTransfer, + TxFee: decimal.NewFromBigInt(tx.TotalFees.Coins.Nano(), 0).Div(decimal.NewFromInt(1e9)), + Timestamp: uint64(tx.Now), + Status: types.StatusConfirmed, + }) + } + + return results +} diff --git a/internal/indexer/ton/parser_test.go b/internal/indexer/ton/parser_test.go new file mode 100644 index 0000000..e409f04 --- /dev/null +++ b/internal/indexer/ton/parser_test.go @@ -0,0 +1,98 @@ +package ton + +import ( + "encoding/base64" + "testing" + "time" + + "github.com/fystack/multichain-indexer/pkg/common/constant" + "github.com/stretchr/testify/assert" + "github.com/xssnick/tonutils-go/address" + "github.com/xssnick/tonutils-go/tlb" + "github.com/xssnick/tonutils-go/tvm/cell" +) + +func TestParseTonTransfer(t *testing.T) { + // Burn address (valid) + ourAddr := address.MustParseAddr("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF") + // Zero address (valid) + senderAddr := address.MustParseAddr("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c") + + t.Run("valid receive", func(t *testing.T) { + // Construct a transaction + tx := &tlb.Transaction{ + LT: 1000, + Now: uint32(time.Now().Unix()), + } + + // Setup incoming internal message safely + tx.IO.In = &tlb.Message{ + MsgType: tlb.MsgTypeInternal, + Msg: &tlb.InternalMessage{ + SrcAddr: senderAddr, + DstAddr: ourAddr, + Amount: tlb.MustFromTON("1"), // 1 TON + Body: cell.BeginCell().EndCell(), + }, + } + + tx.Hash = []byte("txhash") + + results := ParseTonTransfer(tx, ourAddr.String(), "TON_MAINNET") + assert.Len(t, results, 1) + parsed := results[0] + assert.Equal(t, "1000000000", parsed.Amount) + assert.Equal(t, senderAddr.String(), parsed.FromAddress) + assert.Equal(t, ourAddr.String(), parsed.ToAddress) + assert.Equal(t, base64.StdEncoding.EncodeToString(tx.Hash), parsed.TxHash) + assert.Equal(t, constant.TxTypeNativeTransfer, parsed.Type) + }) + + t.Run("empty transaction", func(t *testing.T) { + tx := &tlb.Transaction{} + results := ParseTonTransfer(tx, ourAddr.String(), "TON_MAINNET") + assert.Len(t, results, 0) + }) + + t.Run("ignore other destination", func(t *testing.T) { + otherAddr := address.MustParseAddr("EQD__________________________________________0vo") + tx := &tlb.Transaction{ + LT: 1000, + } + tx.IO.In = &tlb.Message{ + MsgType: tlb.MsgTypeInternal, + Msg: &tlb.InternalMessage{ + SrcAddr: senderAddr, + DstAddr: otherAddr, + Amount: tlb.MustFromTON("1"), + }, + } + + results := ParseTonTransfer(tx, ourAddr.String(), "TON_MAINNET") + assert.Len(t, results, 0) + }) + + t.Run("parse comment", func(t *testing.T) { + comment := "hello world" + body := cell.BeginCell(). + MustStoreUInt(0, 32). // Opcode 0 for text comment + MustStoreStringSnake(comment). + EndCell() + + tx := &tlb.Transaction{ + LT: 1000, + } + tx.IO.In = &tlb.Message{ + MsgType: tlb.MsgTypeInternal, + Msg: &tlb.InternalMessage{ + SrcAddr: senderAddr, + DstAddr: ourAddr, + Amount: tlb.MustFromTON("0.00000001"), + Body: body, + }, + } + + results := ParseTonTransfer(tx, ourAddr.String(), "TON_MAINNET") + assert.Len(t, results, 1) + }) +} diff --git a/internal/rpc/ton/api.go b/internal/rpc/ton/api.go new file mode 100644 index 0000000..f43b217 --- /dev/null +++ b/internal/rpc/ton/api.go @@ -0,0 +1,21 @@ +package ton + +import ( + "context" + + "github.com/xssnick/tonutils-go/address" + "github.com/xssnick/tonutils-go/tlb" +) + +type TonAPI interface { + // GetAccountState returns the current state of an account. + // Returns nil Account.State if account doesn't exist (not deployed). + GetAccountState(ctx context.Context, addr *address.Address) (*tlb.Account, error) + + // ListTransactions returns transactions for an account. + // - limit: max transactions to return (typically 10-50) + // - lastLT: logical time cursor (0 for initial fetch from beginning) + // - lastHash: transaction hash cursor (nil for initial fetch) + // Returns transactions in reverse chronological order (newest first). + ListTransactions(ctx context.Context, addr *address.Address, limit uint32, lastLT uint64, lastHash []byte) ([]*tlb.Transaction, error) +} diff --git a/internal/rpc/ton/client.go b/internal/rpc/ton/client.go new file mode 100644 index 0000000..6e06841 --- /dev/null +++ b/internal/rpc/ton/client.go @@ -0,0 +1,144 @@ +package ton + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/xssnick/tonutils-go/address" + "github.com/xssnick/tonutils-go/liteclient" + "github.com/xssnick/tonutils-go/tlb" + "github.com/xssnick/tonutils-go/ton" +) + +type Client struct { + api ton.APIClientWrapped + pool *liteclient.ConnectionPool + baseURL string + mu sync.RWMutex + + // Cache for masterchain info to avoid redundant calls during mass polling + masterCache *ton.BlockIDExt + masterCacheTime time.Time +} + +type ClientConfig struct { + ConfigURL string +} + +func NewClient(ctx context.Context, cfg ClientConfig) (*Client, error) { + pool := liteclient.NewConnectionPool() + + if err := pool.AddConnectionsFromConfigUrl(ctx, cfg.ConfigURL); err != nil { + return nil, fmt.Errorf("failed to fetch/parse global config: %w", err) + } + + // tonutils-go handles failover between lite servers in the pool + api := ton.NewAPIClient(pool, ton.ProofCheckPolicyFast).WithRetry() + + return &Client{ + api: api, + pool: pool, + baseURL: cfg.ConfigURL, + }, nil +} + +func NewClientFromPool(pool *liteclient.ConnectionPool) *Client { + api := ton.NewAPIClient(pool, ton.ProofCheckPolicyFast).WithRetry() + return &Client{ + api: api, + pool: pool, + } +} + +// getMasterchainInfo returns the current masterchain block info, using cache if valid. +func (c *Client) getMasterchainInfo(ctx context.Context) (*ton.BlockIDExt, error) { + c.mu.Lock() + if c.masterCache != nil && time.Since(c.masterCacheTime) < time.Second { + defer c.mu.Unlock() + return c.masterCache, nil + } + c.mu.Unlock() + + // Fetch new info + master, err := c.api.CurrentMasterchainInfo(ctx) + if err != nil { + return nil, err + } + + c.mu.Lock() + c.masterCache = master + c.masterCacheTime = time.Now() + c.mu.Unlock() + + return master, nil +} + +// GetAccountState returns the current state of an account. +func (c *Client) GetAccountState(ctx context.Context, addr *address.Address) (*tlb.Account, error) { + // Get the current masterchain block for consistency + master, err := c.getMasterchainInfo(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get masterchain info: %w", err) + } + + // Get the account state at the current block + account, err := c.api.GetAccount(ctx, master, addr) + if err != nil { + return nil, fmt.Errorf("failed to get account state: %w", err) + } + + return account, nil +} + +// ListTransactions returns transactions for an account. +// Transactions are returned in reverse chronological order (newest first). +// Use lastLT=0 and lastHash=nil for initial fetch from the account's latest transaction. +func (c *Client) ListTransactions(ctx context.Context, addr *address.Address, limit uint32, lastLT uint64, lastHash []byte) ([]*tlb.Transaction, error) { + // Get the current masterchain block for consistency + master, err := c.getMasterchainInfo(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get masterchain info: %w", err) + } + + // Always get current account state to check for new transactions + account, err := c.api.GetAccount(ctx, master, addr) + if err != nil { + return nil, fmt.Errorf("failed to get account: %w", err) + } + + // Account not active or doesn't exist + if !account.IsActive { + return nil, nil + } + + // If cursor is at or ahead of latest, no new transactions + if lastLT >= account.LastTxLT { + return nil, nil + } + + // Fetch transactions from the LATEST, going backwards + txs, err := c.api.ListTransactions(ctx, addr, limit, account.LastTxLT, account.LastTxHash) + if err != nil { + return nil, fmt.Errorf("failed to list transactions: %w", err) + } + + // Filter to only return transactions NEWER than our cursor + var newTxs []*tlb.Transaction + for _, tx := range txs { + if tx.LT > lastLT { + newTxs = append(newTxs, tx) + } + } + + return newTxs, nil +} + +func (c *Client) Close() error { + c.mu.Lock() + defer c.mu.Unlock() + c.api = nil + c.pool = nil + return nil +} diff --git a/internal/worker/factory.go b/internal/worker/factory.go index dcf3608..bd44b42 100644 --- a/internal/worker/factory.go +++ b/internal/worker/factory.go @@ -5,12 +5,15 @@ import ( "strconv" "github.com/fystack/multichain-indexer/internal/indexer" + tonIndexer "github.com/fystack/multichain-indexer/internal/indexer/ton" "github.com/fystack/multichain-indexer/internal/rpc" "github.com/fystack/multichain-indexer/internal/rpc/bitcoin" "github.com/fystack/multichain-indexer/internal/rpc/evm" "github.com/fystack/multichain-indexer/internal/rpc/solana" "github.com/fystack/multichain-indexer/internal/rpc/sui" + tonRpc "github.com/fystack/multichain-indexer/internal/rpc/ton" "github.com/fystack/multichain-indexer/internal/rpc/tron" + tonWorker "github.com/fystack/multichain-indexer/internal/worker/ton" "github.com/fystack/multichain-indexer/pkg/addressbloomfilter" "github.com/fystack/multichain-indexer/pkg/common/config" "github.com/fystack/multichain-indexer/pkg/common/enum" @@ -288,6 +291,74 @@ func buildSuiIndexer( return indexer.NewSuiIndexer(chainName, chainCfg, failover, pubkeyStore) } +// buildTonPollingWorker constructs a TON polling worker with failover. +// TON uses account-based polling instead of block-based indexing. +func buildTonPollingWorker( + ctx context.Context, + chainName string, + chainCfg config.ChainConfig, + kvstore infra.KVStore, + pubkeyStore pubkeystore.Store, + db *gorm.DB, + emitter events.Emitter, +) Worker { + var client tonRpc.TonAPI + + if len(chainCfg.Nodes) > 0 { + configURL := chainCfg.Nodes[0].URL + + var err error // shadow assignment + client, err = tonRpc.NewClient(ctx, tonRpc.ClientConfig{ + ConfigURL: configURL, + }) + + if err != nil { + logger.Error("Failed to create TON client with global config", "url", configURL, "err", err) + // Proceed with nil client, IsHealthy() will return false + } + } else { + logger.Error("No nodes configured for TON chain", "chain", chainName) + } + + // Create cursor store backed by KVStore + cursorStore := tonIndexer.NewCursorStore(kvstore) + + // Create Jetton registry + var jettons []tonIndexer.JettonInfo + for _, j := range chainCfg.Jettons { + jettons = append(jettons, tonIndexer.JettonInfo{ + MasterAddress: j.MasterAddress, + Symbol: j.Symbol, + Decimals: j.Decimals, + }) + } + jettonRegistry := tonIndexer.NewConfigBasedRegistry(jettons) + + // Create the account indexer + accountIndexer := tonIndexer.NewTonAccountIndexer( + chainName, + chainCfg, + client, + jettonRegistry, + ) + + // Create the polling worker + worker := tonWorker.NewTonPollingWorker( + ctx, + chainCfg, + accountIndexer, + cursorStore, + pubkeyStore, + db, + emitter, + tonWorker.WorkerConfig{ + Concurrency: chainCfg.Throttle.Concurrency, + PollInterval: chainCfg.PollInterval, + }, + ) + return worker +} + // CreateManagerWithWorkers initializes manager and all workers for configured chains. func CreateManagerWithWorkers( ctx context.Context, @@ -327,6 +398,13 @@ func CreateManagerWithWorkers( idxr = buildSolanaIndexer(chainName, chainCfg, ModeRegular, pubkeyStore) case enum.NetworkTypeSui: idxr = buildSuiIndexer(chainName, chainCfg, ModeRegular, pubkeyStore) + case enum.NetworkTypeTon: + tonW := buildTonPollingWorker(ctx, chainName, chainCfg, kvstore, pubkeyStore, db, emitter) + if tonW != nil { + manager.AddWorkers(tonW) + logger.Info("TON polling worker enabled", "chain", chainName) + } + continue default: logger.Fatal("Unsupported network type", "chain", chainName, "type", chainCfg.Type) } diff --git a/internal/worker/ton/worker.go b/internal/worker/ton/worker.go new file mode 100644 index 0000000..2f009e1 --- /dev/null +++ b/internal/worker/ton/worker.go @@ -0,0 +1,388 @@ +package ton + +import ( + "context" + "log/slog" + "sync" + "time" + + tonIndexer "github.com/fystack/multichain-indexer/internal/indexer/ton" + "github.com/fystack/multichain-indexer/pkg/common/config" + "github.com/fystack/multichain-indexer/pkg/common/enum" + "github.com/fystack/multichain-indexer/pkg/common/logger" + "github.com/fystack/multichain-indexer/pkg/common/types" + "github.com/fystack/multichain-indexer/pkg/events" + "github.com/fystack/multichain-indexer/pkg/model" + "github.com/fystack/multichain-indexer/pkg/retry" + "github.com/fystack/multichain-indexer/pkg/store/pubkeystore" + "gorm.io/gorm" +) + +// TonPollingWorker polls multiple TON accounts for transactions. +type TonPollingWorker struct { + ctx context.Context + cancel context.CancelFunc + logger *slog.Logger + wg sync.WaitGroup + + config config.ChainConfig + indexer tonIndexer.AccountIndexer + cursorStore tonIndexer.CursorStore + pubkeyStore pubkeystore.Store + db *gorm.DB + emitter events.Emitter + + // Cache + wallets []string + walletsMutex sync.RWMutex + lastSyncTime time.Time + syncInterval time.Duration + + // Worker configuration + concurrency int + pollInterval time.Duration +} + +// WorkerConfig holds configuration for the TON polling worker. +type WorkerConfig struct { + Concurrency int // Max parallel account polls (default: 10) + PollInterval time.Duration // Interval between poll cycles (default: from chain config) +} + +// NewTonPollingWorker creates a new TON polling worker. +func NewTonPollingWorker( + ctx context.Context, + cfg config.ChainConfig, + indexer tonIndexer.AccountIndexer, + cursorStore tonIndexer.CursorStore, + pubkeyStore pubkeystore.Store, + db *gorm.DB, + emitter events.Emitter, + workerCfg WorkerConfig, +) *TonPollingWorker { + ctx, cancel := context.WithCancel(ctx) + + log := logger.With( + slog.String("worker", "ton-polling"), + slog.String("chain", cfg.Name), + ) + + concurrency := workerCfg.Concurrency + if concurrency <= 0 { + concurrency = 10 + } + + pollInterval := workerCfg.PollInterval + if pollInterval <= 0 { + pollInterval = cfg.PollInterval + } + + return &TonPollingWorker{ + ctx: ctx, + cancel: cancel, + logger: log, + config: cfg, + indexer: indexer, + cursorStore: cursorStore, + pubkeyStore: pubkeyStore, + db: db, + emitter: emitter, + concurrency: concurrency, + pollInterval: pollInterval, + syncInterval: time.Minute, // Sync from DB every minute + } +} + +// Start begins the polling loop. +func (w *TonPollingWorker) Start() { + w.wg.Add(1) + go w.run() +} + +// Stop gracefully stops the worker. +func (w *TonPollingWorker) Stop() { + w.cancel() + w.wg.Wait() + w.logger.Info("TON polling worker stopped") +} + +// run is the main polling loop. +func (w *TonPollingWorker) run() { + defer w.wg.Done() + + ticker := time.NewTicker(w.pollInterval) + defer ticker.Stop() + + w.logger.Info("TON polling worker started", + "poll_interval", w.pollInterval, + "concurrency", w.concurrency, + ) + + // Run immediately on start + w.pollAllAccounts() + + for { + select { + case <-w.ctx.Done(): + w.logger.Info("Context cancelled, stopping polling loop") + return + + case <-ticker.C: + w.pollAllAccounts() + } + } +} + +// pollAllAccounts polls all tracked TON addresses from the database. +func (w *TonPollingWorker) pollAllAccounts() { + // Sync addresses from DB if needed + w.syncWalletsIfNeeded() + + w.walletsMutex.RLock() + addresses := make([]string, len(w.wallets)) + copy(addresses, w.wallets) + w.walletsMutex.RUnlock() + + if len(addresses) == 0 { + // Only log periodically if needed, or debug + w.logger.Debug("No TON addresses to poll") + return + } + + w.logger.Info("Starting poll cycle", "address_count", len(addresses)) + + // Create work channel + workChan := make(chan string, len(addresses)) + for _, addr := range addresses { + workChan <- addr + } + close(workChan) + + // Start worker goroutines + var wg sync.WaitGroup + for i := 0; i < w.concurrency; i++ { + wg.Add(1) + go func(workerID int) { + defer wg.Done() + w.pollWorker(workerID, workChan) + }(i) + } + + wg.Wait() + w.logger.Debug("Poll cycle complete", "address_count", len(addresses)) +} + +// pollWorker processes addresses from the work channel. +func (w *TonPollingWorker) pollWorker(workerID int, addresses <-chan string) { + for addr := range addresses { + select { + case <-w.ctx.Done(): + return + default: + w.pollAccount(addr) + } + } +} + +// pollAccount polls a single account with retry logic. +func (w *TonPollingWorker) pollAccount(address string) { + log := w.logger.With("address", address) + + // Get existing cursor + cursor, err := w.cursorStore.Get(w.ctx, address) + if err != nil { + log.Error("Failed to get cursor", "err", err) + return + } + + // If no cursor exists, initialize one + if cursor == nil { + cursor = &tonIndexer.AccountCursor{Address: address} + } + + // Check context before starting + if w.ctx.Err() != nil { + return + } + + // Poll with retry + var txs []types.Transaction + var newCursor *tonIndexer.AccountCursor + + err = retry.Exponential(func() error { + // Check context cancellation - don't retry on shutdown + if w.ctx.Err() != nil { + return nil // Return nil to exit retry loop cleanly + } + + result, c, pollErr := w.indexer.PollAccount(w.ctx, address, cursor) + if pollErr != nil { + // Check if error is due to context cancellation + if w.ctx.Err() != nil { + return nil // Exit retry cleanly on shutdown + } + return pollErr + } + txs = result + newCursor = c + return nil + }, retry.ExponentialConfig{ + InitialInterval: 2 * time.Second, + MaxElapsedTime: 30 * time.Second, + OnRetry: func(err error, next time.Duration) { + // Only log if context is still active + if w.ctx.Err() == nil { + log.Debug("Retrying poll", "err", err, "next_retry", next) + } + }, + }) + + // Exit if context was cancelled + if w.ctx.Err() != nil { + return + } + + if err != nil { + log.Error("Failed to poll account after retries", "err", err) + return + } + + // Process transactions + if len(txs) == 0 { + return + } + + log.Info("Found transactions", "count", len(txs)) + + // Emit each transaction to NATS + for i := range txs { + tx := &txs[i] + log.Info("Emitting matched transaction", + "type", tx.Type, + "from", tx.FromAddress, + "to", tx.ToAddress, + "amount", tx.Amount, + "fee", tx.TxFee.String(), + "txhash", tx.TxHash, + ) + if err := w.emitter.EmitTransaction(w.config.InternalCode, tx); err != nil { + log.Error("Failed to emit transaction", "tx_hash", tx.TxHash, "err", err) + } else { + log.Debug("Emitted transaction", "tx_hash", tx.TxHash) + } + } + + // Update cursor after successful processing + if newCursor != nil { + if err := w.cursorStore.Save(w.ctx, newCursor); err != nil { + log.Error("Failed to save cursor", "err", err) + } + } +} + +// AddAddress registers a new address for tracking. +// This initializes the cursor and starts polling the address. +func (w *TonPollingWorker) AddAddress(ctx context.Context, address string) error { + // Check if cursor already exists + existing, err := w.cursorStore.Get(ctx, address) + if err != nil { + return err + } + + if existing != nil { + // Already tracking + return nil + } + + // Create initial cursor + cursor := &tonIndexer.AccountCursor{Address: address} + return w.cursorStore.Save(ctx, cursor) +} + +// RemoveAddress stops tracking an address. +func (w *TonPollingWorker) RemoveAddress(ctx context.Context, address string) error { + return w.cursorStore.Delete(ctx, address) +} + +// GetNetworkType implements the worker interface. +func (w *TonPollingWorker) GetNetworkType() enum.NetworkType { + return enum.NetworkTypeTon +} + +// syncWalletsIfNeeded refreshes the wallet list from DB if sync interval passed. +// Uses pagination to efficiently handle large numbers of wallets (>10K). +// Before fetching everything, it checks the count to avoid redundant work. +func (w *TonPollingWorker) syncWalletsIfNeeded() { + if time.Since(w.lastSyncTime) < w.syncInterval { + return + } + + // 1. Quickly check the count in DB + var dbCount int64 + err := w.db.Model(&model.WalletAddress{}). + Where("type = ?", enum.NetworkTypeTon). + Count(&dbCount).Error + if err != nil { + w.logger.Error("Failed to count wallets in DB", "err", err) + return + } + + w.walletsMutex.RLock() + cachedCount := int64(len(w.wallets)) + w.walletsMutex.RUnlock() + + // If count matches and we're not empty, skip the expensive full fetch + if dbCount == cachedCount && cachedCount > 0 { + w.lastSyncTime = time.Now() + return + } + + // 2. Count changed or list is empty, perform full sync + const batchSize = 1000 + var allAddresses []string + offset := 0 + + for { + var wallets []model.WalletAddress + // Paginated query for TON wallets only selecting address field + err := w.db.Select("address"). + Where("type = ?", enum.NetworkTypeTon). + Order("id"). + Limit(batchSize). + Offset(offset). + Find(&wallets).Error + + if err != nil { + w.logger.Error("Failed to fetch wallets from DB", "err", err, "offset", offset) + return + } + + // No more results + if len(wallets) == 0 { + break + } + + // Collect addresses + for _, wallet := range wallets { + allAddresses = append(allAddresses, wallet.Address) + } + + // If we got fewer than batchSize, we're done + if len(wallets) < batchSize { + break + } + + offset += batchSize + } + + w.walletsMutex.Lock() + oldSize := len(w.wallets) + w.wallets = allAddresses + w.lastSyncTime = time.Now() + w.walletsMutex.Unlock() + + w.logger.Info("Synced wallets from DB", + "old_size", oldSize, + "new_size", len(allAddresses), + ) +} diff --git a/pkg/common/config/types.go b/pkg/common/config/types.go index c5bf40a..6db4d03 100644 --- a/pkg/common/config/types.go +++ b/pkg/common/config/types.go @@ -37,7 +37,7 @@ type ChainConfig struct { Name string `yaml:"-"` NetworkId string `yaml:"network_id"` InternalCode string `yaml:"internal_code"` - Type enum.NetworkType `yaml:"type" validate:"required,oneof=tron evm btc sol sui"` + Type enum.NetworkType `yaml:"type" validate:"required,oneof=tron evm btc sol sui ton"` FromLatest bool `yaml:"from_latest"` StartBlock int `yaml:"start_block" validate:"min=0"` PollInterval time.Duration `yaml:"poll_interval"` @@ -46,6 +46,13 @@ type ChainConfig struct { Client ClientConfig `yaml:"client"` Throttle Throttle `yaml:"throttle"` Nodes []NodeConfig `yaml:"nodes" validate:"required,min=1"` + Jettons []JettonConfig `yaml:"jettons"` +} + +type JettonConfig struct { + MasterAddress string `yaml:"master_address"` + Symbol string `yaml:"symbol"` + Decimals int `yaml:"decimals"` } type ClientConfig struct { diff --git a/pkg/common/enum/enum.go b/pkg/common/enum/enum.go index 068e508..c123c9f 100644 --- a/pkg/common/enum/enum.go +++ b/pkg/common/enum/enum.go @@ -25,6 +25,7 @@ const ( NetworkTypeSol NetworkType = "sol" NetworkTypeApt NetworkType = "apt" NetworkTypeSui NetworkType = "sui" + NetworkTypeTon NetworkType = "ton" ) const ( diff --git a/sql/wallet_address.sql b/sql/wallet_address.sql index 93bb13e..5ed2f8f 100644 --- a/sql/wallet_address.sql +++ b/sql/wallet_address.sql @@ -56,4 +56,7 @@ COMMENT ON COLUMN wallet_addresses.standard IS 'The token standard (erc20, erc72 -- Insert sample data INSERT INTO wallet_addresses (address, type, standard) VALUES ('TAWdqnuYCNU3dKsi7pR8d7sDkx1Evb2giV', 'tron', 'trc20'), -('TT1j2adMBb6bF2K8C2LX1QkkmSXHjiaAfw', 'tron', 'trc20'); \ No newline at end of file +('TT1j2adMBb6bF2K8C2LX1QkkmSXHjiaAfw', 'tron', 'trc20'), +('Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF', 'ton', 'native'), +('EQDKHZ7e70CzqdvZCC83Z4WVR8POC_ZB0J1Y4zo88G-zCXmC', 'ton', 'native'), +('EQBeab7D38RIwypegbN7YZgQzwDbb8QfMMwY8ouJc3qPl91M', 'ton', 'native'); diff --git a/test/parse_ton_txn/main.go b/test/parse_ton_txn/main.go new file mode 100644 index 0000000..5a5761e --- /dev/null +++ b/test/parse_ton_txn/main.go @@ -0,0 +1,151 @@ +package main + +import ( + "context" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "log" + "net/http" + + indexer "github.com/fystack/multichain-indexer/internal/indexer/ton" + "github.com/xssnick/tonutils-go/address" + "github.com/xssnick/tonutils-go/liteclient" + "github.com/xssnick/tonutils-go/ton" +) + +const ( + txHashStr = "uoIXlCHFgKwOWjLEQx1E3Wu4gSbhcvFmHkNHlABx21E=" + configURL = "https://ton.org/global.config.json" +) + +type TonApiTx struct { + Hash string `json:"hash"` + Lt uint64 `json:"lt"` + Account struct { + Address string `json:"address"` + } `json:"account"` +} + +func main() { + // 1. Fetch transaction details from tonapi.io to get Account Address and LT + // This is necessary because liteservers typically require (Account, LT, Hash) to look up a Tx. + hexHashBytes, err := base64.StdEncoding.DecodeString(txHashStr) + if err != nil { + log.Fatalf("Invalid base64 hash: %v", err) + } + hexHash := hex.EncodeToString(hexHashBytes) + + fmt.Printf("Fetching details for hash: %s (hex: %s)\n", txHashStr, hexHash) + + resp, err := http.Get("https://tonapi.io/v2/blockchain/transactions/" + hexHash) + if err != nil { + log.Fatalf("Failed to query tonapi.io: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + log.Fatalf("tonapi.io returned status %d. Ensure the hash is correct and tonapi is accessible.", resp.StatusCode) + } + + var apiTx TonApiTx + if err := json.NewDecoder(resp.Body).Decode(&apiTx); err != nil { + log.Fatalf("Failed to decode tonapi response: %v", err) + } + + fmt.Printf("Found Transaction via API:\n Account: %s\n LT: %d\n", apiTx.Account.Address, apiTx.Lt) + + // 2. Initialize TON Lite Client + ctx := context.Background() + pool := liteclient.NewConnectionPool() + if err := pool.AddConnectionsFromConfigUrl(ctx, configURL); err != nil { + log.Fatalf("Failed to convert config url: %v", err) + } + + // Use the generic API client + api := ton.NewAPIClient(pool, ton.ProofCheckPolicyFast).WithRetry() + + // 3. Parse target address + fmt.Printf("Parsing address: '%s'\n", apiTx.Account.Address) + + var addr *address.Address + addr, err = address.ParseAddr(apiTx.Account.Address) + if err != nil { + fmt.Printf("address.ParseAddr failed: %v. Attempting manual raw parse...\n", err) + // Fallback: Parse raw address manually (assuming 0:hash format) + var wc int8 = 0 // Default to workchain 0 + var hashHex string + + if len(apiTx.Account.Address) > 2 && apiTx.Account.Address[1] == ':' { + // Basic check for workchain + n, _ := fmt.Sscanf(apiTx.Account.Address, "%d:%s", &wc, &hashHex) + if n != 2 { + // Try just using the string as hash if scan failed + hashHex = apiTx.Account.Address + } + } else { + hashHex = apiTx.Account.Address + } + + hashBytes, hErr := hex.DecodeString(hashHex) + if hErr == nil && len(hashBytes) == 32 { + addr = address.NewAddress(0, byte(wc), hashBytes) + fmt.Println("Manually parsed raw address.") + } else { + log.Fatalf("Could not parse address: %v. Raw parse also failed: %v", err, hErr) + } + } + + // 4. Fetch the specific transaction + // ListTransactions retrieves transactions starting from the specified LT/Hash. + // To get ONLY this transaction, we ask for limit 1 starting at its LT/Hash. + txs, err := api.ListTransactions(ctx, addr, 1, apiTx.Lt, hexHashBytes) + if err != nil { + log.Fatalf("Failed to fetch transaction from liteserver: %v", err) + } + + if len(txs) == 0 { + log.Fatalf("Transaction not found on liteserver (archival node might be needed if old).") + } + + tx := txs[0] + // Verify it's the right one + if hex.EncodeToString(tx.Hash) != hexHash { + fmt.Printf("Warning: Fetched transaction hash mismatch. Expected %s, got %s\n", hexHash, hex.EncodeToString(tx.Hash)) + } + + fmt.Println("\nSuccessfully fetched transaction from Lite Server.") + + // 5. Parse the transaction + // We use "TON_MAINNET" as network ID for display + networkID := "TON_MAINNET" + + fmt.Println("\n--- Parsing Results ---") + + // Parse Native TON Transfers + tonTransfers := indexer.ParseTonTransfer(tx, addr.String(), networkID) + if len(tonTransfers) > 0 { + fmt.Println("Native TON Transfers detected:") + for i, t := range tonTransfers { + parsedJSON, _ := json.MarshalIndent(t, "", " ") + fmt.Printf("[%d] %s\n", i, string(parsedJSON)) + } + } else { + fmt.Println("No Native TON Transfers detected.") + } + + // Parse Jetton Transfers + // Passing nil for registry means it will treat the Jetton Master address as the wallet address temporarily, + // or returns "unknown" for symbol, but parsing logic should hold. + jettonTransfers := indexer.ParseJettonTransfer(tx, addr.String(), networkID, nil) + if len(jettonTransfers) > 0 { + fmt.Println("\nJetton Transfers detected:") + for i, t := range jettonTransfers { + parsedJSON, _ := json.MarshalIndent(t, "", " ") + fmt.Printf("[%d] %s\n", i, string(parsedJSON)) + } + } else { + fmt.Println("No Jetton Transfers detected.") + } +} From bb6ed8b64950ac3c66ec70242153b53e0db9d85e Mon Sep 17 00:00:00 2001 From: vietddude Date: Wed, 11 Feb 2026 14:44:20 +0700 Subject: [PATCH 02/11] feat(ton): add runtime jetton reload endpoint --- README.md | 27 ++++ cmd/indexer/main.go | 92 ++++++++++++- internal/indexer/ton/indexer.go | 21 +++ internal/indexer/ton/jetton_registry.go | 175 ++++++++++++++++++++++++ internal/worker/factory.go | 12 +- internal/worker/manager.go | 7 + internal/worker/ton/worker.go | 19 +++ internal/worker/ton_jetton_reload.go | 90 ++++++++++++ 8 files changed, 435 insertions(+), 8 deletions(-) create mode 100644 internal/worker/ton_jetton_reload.go diff --git a/README.md b/README.md index e1be80f..9c15d81 100644 --- a/README.md +++ b/README.md @@ -236,6 +236,33 @@ nats consumer sub transfer transaction-consumer --- +## 🌐 HTTP API Examples + +```bash +# set this to your configured services.port +export INDEXER_PORT=8080 +``` + +### Health Check + +```bash +curl -s "http://localhost:${INDEXER_PORT}/health" | jq +``` + +### Reload TON Jetton Registry (all TON chains) + +```bash +curl -s -X POST "http://localhost:${INDEXER_PORT}/ton/jettons/reload" | jq +``` + +### Reload TON Jetton Registry For One Chain + +```bash +curl -s -X POST "http://localhost:${INDEXER_PORT}/ton/jettons/reload?chain=ton_mainnet" | jq +``` + +--- + ## 📝 Example `configs/config.yaml` (chains section) ```yaml diff --git a/cmd/indexer/main.go b/cmd/indexer/main.go index 08e889e..930d06f 100644 --- a/cmd/indexer/main.go +++ b/cmd/indexer/main.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "errors" "fmt" "log/slog" "net/http" @@ -199,8 +200,9 @@ func runIndexer(chains []string, configPath string, debug, manual, catchup, from redisClient, managerCfg, ) + tonJettonReloadService := worker.NewTonJettonReloadServiceFromManager(manager) - healthServer := startHealthServer(cfg.Services.Port, cfg) + healthServer := startHealthServer(cfg.Services.Port, cfg, tonJettonReloadService) // Start all workers logger.Info("Starting all workers") @@ -232,7 +234,24 @@ type HealthResponse struct { Version string `json:"version"` } -func startHealthServer(port int, cfg *config.Config) *http.Server { +type APIErrorResponse struct { + Status string `json:"status"` + Error string `json:"error"` + Timestamp time.Time `json:"timestamp"` +} + +type TonJettonReloadResponse struct { + Status string `json:"status"` + Chain string `json:"chain,omitempty"` + Results []worker.TonJettonReloadResult `json:"results"` + TriggeredAtUTC time.Time `json:"triggered_at_utc"` +} + +func startHealthServer( + port int, + cfg *config.Config, + tonJettonReloadService *worker.TonJettonReloadService, +) *http.Server { mux := http.NewServeMux() version := cfg.Version @@ -246,10 +265,41 @@ func startHealthServer(port int, cfg *config.Config) *http.Server { Timestamp: time.Now().UTC(), Version: version, } + writeJSON(w, http.StatusOK, response) + }) + + mux.HandleFunc("/ton/jettons/reload", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + writeErrorJSON(w, http.StatusMethodNotAllowed, "method not allowed") + return + } + + if tonJettonReloadService == nil { + writeErrorJSON(w, http.StatusNotFound, worker.ErrNoTonWorkerConfigured.Error()) + return + } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(response) + req := worker.TonJettonReloadRequest{ + ChainFilter: r.URL.Query().Get("chain"), + } + + results, err := tonJettonReloadService.ReloadTonJettons(r.Context(), req) + if err != nil { + statusCode := http.StatusInternalServerError + if errors.Is(err, worker.ErrTonWorkerNotFound) || errors.Is(err, worker.ErrNoTonWorkerConfigured) { + statusCode = http.StatusNotFound + } + writeErrorJSON(w, statusCode, err.Error()) + return + } + + response := TonJettonReloadResponse{ + Status: reloadJettonStatus(results), + Chain: req.ChainFilter, + Results: results, + TriggeredAtUTC: time.Now().UTC(), + } + writeJSON(w, http.StatusOK, response) }) server := &http.Server{ @@ -258,7 +308,12 @@ func startHealthServer(port int, cfg *config.Config) *http.Server { } go func() { - logger.Info("Health check server started", "port", port, "endpoint", "/health") + logger.Info( + "Health check server started", + "port", port, + "health_endpoint", "/health", + "jetton_reload_endpoint", "/ton/jettons/reload", + ) if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { logger.Error("Health server failed to start", "error", err) } @@ -267,6 +322,31 @@ func startHealthServer(port int, cfg *config.Config) *http.Server { return server } +func reloadJettonStatus(results []worker.TonJettonReloadResult) string { + for _, result := range results { + if result.Error != "" { + return "partial_error" + } + } + return "ok" +} + +func writeJSON(w http.ResponseWriter, statusCode int, payload any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) + if err := json.NewEncoder(w).Encode(payload); err != nil { + logger.Error("Failed to encode response", "status", statusCode, "err", err) + } +} + +func writeErrorJSON(w http.ResponseWriter, statusCode int, message string) { + writeJSON(w, statusCode, APIErrorResponse{ + Status: "error", + Error: message, + Timestamp: time.Now().UTC(), + }) +} + func waitForShutdown() { stop := make(chan os.Signal, 1) signal.Notify(stop, os.Interrupt, syscall.SIGTERM) diff --git a/internal/indexer/ton/indexer.go b/internal/indexer/ton/indexer.go index 66b40bb..8e3c78c 100644 --- a/internal/indexer/ton/indexer.go +++ b/internal/indexer/ton/indexer.go @@ -27,6 +27,9 @@ type AccountIndexer interface { // IsHealthy checks if the RPC connection is healthy. IsHealthy() bool + + // ReloadJettons refreshes supported jetton metadata at runtime. + ReloadJettons(ctx context.Context) (int, error) } // TonAccountIndexer implements AccountIndexer for TON blockchain. @@ -72,6 +75,24 @@ func (i *TonAccountIndexer) IsHealthy() bool { return i.client != nil } +func (i *TonAccountIndexer) ReloadJettons(ctx context.Context) (int, error) { + if i.jettonRegistry == nil { + return 0, nil + } + + type registryReloader interface { + Reload(context.Context) error + } + + if reloader, ok := i.jettonRegistry.(registryReloader); ok { + if err := reloader.Reload(ctx); err != nil { + return 0, fmt.Errorf("reload jetton registry: %w", err) + } + } + + return len(i.jettonRegistry.List()), nil +} + // PollAccount fetches new transactions for a single account. func (i *TonAccountIndexer) PollAccount(ctx context.Context, addrStr string, cursor *AccountCursor) ([]types.Transaction, *AccountCursor, error) { // Parse the address diff --git a/internal/indexer/ton/jetton_registry.go b/internal/indexer/ton/jetton_registry.go index 6c19751..d3e4c8e 100644 --- a/internal/indexer/ton/jetton_registry.go +++ b/internal/indexer/ton/jetton_registry.go @@ -1,7 +1,20 @@ package ton import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" "sync" + + "github.com/fystack/multichain-indexer/pkg/infra" + "github.com/redis/go-redis/v9" +) + +const ( + jettonMasterListKeyFormat = "ton/jettons/%s/masters" + jettonWalletMappingKeyFormat = "ton/jettons/%s/wallet_to_master" ) // ConfigBasedRegistry implements JettonRegistry with a static list of Jettons. @@ -79,3 +92,165 @@ func (r *ConfigBasedRegistry) List() []JettonInfo { } return result } + +// Reload is a no-op for config-backed registry. +func (r *ConfigBasedRegistry) Reload(_ context.Context) error { + return nil +} + +// RedisJettonRegistry loads supported jettons from Redis and keeps an in-memory snapshot. +// Key format: +// - ton/jettons//masters: JSON []JettonInfo +// - ton/jettons//wallet_to_master: JSON map[string]string +type RedisJettonRegistry struct { + redis infra.RedisClient + chainName string + + fallbackJettons []JettonInfo + + mu sync.RWMutex + jettons map[string]JettonInfo + walletToMaster map[string]string +} + +func NewRedisJettonRegistry(chainName string, redisClient infra.RedisClient, fallback []JettonInfo) *RedisJettonRegistry { + reg := &RedisJettonRegistry{ + redis: redisClient, + chainName: chainName, + fallbackJettons: append([]JettonInfo(nil), fallback...), + jettons: make(map[string]JettonInfo), + walletToMaster: make(map[string]string), + } + reg.applyFallback() + return reg +} + +func (r *RedisJettonRegistry) applyFallback() { + r.mu.Lock() + defer r.mu.Unlock() + + r.jettons = make(map[string]JettonInfo, len(r.fallbackJettons)) + r.walletToMaster = make(map[string]string) + for _, j := range r.fallbackJettons { + if j.MasterAddress == "" { + continue + } + r.jettons[j.MasterAddress] = j + } +} + +func (r *RedisJettonRegistry) mastersKey() string { + return fmt.Sprintf(jettonMasterListKeyFormat, r.chainName) +} + +func (r *RedisJettonRegistry) walletMappingKey() string { + return fmt.Sprintf(jettonWalletMappingKeyFormat, r.chainName) +} + +// Reload refreshes the registry snapshot from Redis. +func (r *RedisJettonRegistry) Reload(_ context.Context) error { + if r.redis == nil { + r.applyFallback() + return nil + } + + nextJettons := make(map[string]JettonInfo, len(r.fallbackJettons)) + for _, j := range r.fallbackJettons { + if j.MasterAddress == "" { + continue + } + nextJettons[j.MasterAddress] = j + } + nextWallets := make(map[string]string) + + masterRaw, err := r.redis.Get(r.mastersKey()) + if err != nil && !errors.Is(err, redis.Nil) { + return fmt.Errorf("get jetton masters from redis: %w", err) + } + if err == nil && strings.TrimSpace(masterRaw) != "" { + var masters []JettonInfo + if unmarshalErr := json.Unmarshal([]byte(masterRaw), &masters); unmarshalErr != nil { + return fmt.Errorf("unmarshal jetton masters: %w", unmarshalErr) + } + for _, j := range masters { + if j.MasterAddress == "" { + continue + } + nextJettons[j.MasterAddress] = j + } + } + + walletRaw, err := r.redis.Get(r.walletMappingKey()) + if err != nil && !errors.Is(err, redis.Nil) { + return fmt.Errorf("get jetton wallet mapping from redis: %w", err) + } + if err == nil && strings.TrimSpace(walletRaw) != "" { + if unmarshalErr := json.Unmarshal([]byte(walletRaw), &nextWallets); unmarshalErr != nil { + return fmt.Errorf("unmarshal jetton wallet mapping: %w", unmarshalErr) + } + } + + r.mu.Lock() + r.jettons = nextJettons + r.walletToMaster = nextWallets + r.mu.Unlock() + return nil +} + +func (r *RedisJettonRegistry) IsSupported(walletAddress string) bool { + r.mu.RLock() + defer r.mu.RUnlock() + + masterAddress, ok := r.walletToMaster[walletAddress] + if !ok { + return false + } + _, ok = r.jettons[masterAddress] + return ok +} + +func (r *RedisJettonRegistry) GetInfo(masterAddress string) (*JettonInfo, bool) { + r.mu.RLock() + defer r.mu.RUnlock() + + info, ok := r.jettons[masterAddress] + if !ok { + return nil, false + } + return &info, true +} + +func (r *RedisJettonRegistry) GetInfoByWallet(walletAddress string) (*JettonInfo, bool) { + r.mu.RLock() + masterAddress, ok := r.walletToMaster[walletAddress] + if !ok { + r.mu.RUnlock() + return nil, false + } + info, ok := r.jettons[masterAddress] + r.mu.RUnlock() + if !ok { + return nil, false + } + return &info, true +} + +func (r *RedisJettonRegistry) RegisterWallet(walletAddress, masterAddress string) { + r.mu.Lock() + defer r.mu.Unlock() + + if _, ok := r.jettons[masterAddress]; ok { + r.walletToMaster[walletAddress] = masterAddress + } +} + +func (r *RedisJettonRegistry) List() []JettonInfo { + r.mu.RLock() + defer r.mu.RUnlock() + + result := make([]JettonInfo, 0, len(r.jettons)) + for _, j := range r.jettons { + result = append(result, j) + } + return result +} diff --git a/internal/worker/factory.go b/internal/worker/factory.go index bd44b42..f2bbb71 100644 --- a/internal/worker/factory.go +++ b/internal/worker/factory.go @@ -298,6 +298,7 @@ func buildTonPollingWorker( chainName string, chainCfg config.ChainConfig, kvstore infra.KVStore, + redisClient infra.RedisClient, pubkeyStore pubkeystore.Store, db *gorm.DB, emitter events.Emitter, @@ -332,7 +333,13 @@ func buildTonPollingWorker( Decimals: j.Decimals, }) } - jettonRegistry := tonIndexer.NewConfigBasedRegistry(jettons) + jettonRegistry := tonIndexer.NewRedisJettonRegistry(chainName, redisClient, jettons) + if err := jettonRegistry.Reload(ctx); err != nil { + logger.Error("Failed to load jetton registry from redis, using fallback config", + "chain", chainName, + "err", err, + ) + } // Create the account indexer accountIndexer := tonIndexer.NewTonAccountIndexer( @@ -345,6 +352,7 @@ func buildTonPollingWorker( // Create the polling worker worker := tonWorker.NewTonPollingWorker( ctx, + chainName, chainCfg, accountIndexer, cursorStore, @@ -399,7 +407,7 @@ func CreateManagerWithWorkers( case enum.NetworkTypeSui: idxr = buildSuiIndexer(chainName, chainCfg, ModeRegular, pubkeyStore) case enum.NetworkTypeTon: - tonW := buildTonPollingWorker(ctx, chainName, chainCfg, kvstore, pubkeyStore, db, emitter) + tonW := buildTonPollingWorker(ctx, chainName, chainCfg, kvstore, redisClient, pubkeyStore, db, emitter) if tonW != nil { manager.AddWorkers(tonW) logger.Info("TON polling worker enabled", "chain", chainName) diff --git a/internal/worker/manager.go b/internal/worker/manager.go index 5068485..2c7375a 100644 --- a/internal/worker/manager.go +++ b/internal/worker/manager.go @@ -80,3 +80,10 @@ func (m *Manager) closeResource(name string, resource interface{}, closer func() func (m *Manager) AddWorkers(workers ...Worker) { m.workers = append(m.workers, workers...) } + +// Workers returns a copy of managed workers. +func (m *Manager) Workers() []Worker { + out := make([]Worker, 0, len(m.workers)) + out = append(out, m.workers...) + return out +} diff --git a/internal/worker/ton/worker.go b/internal/worker/ton/worker.go index 2f009e1..f4ea27c 100644 --- a/internal/worker/ton/worker.go +++ b/internal/worker/ton/worker.go @@ -25,6 +25,7 @@ type TonPollingWorker struct { logger *slog.Logger wg sync.WaitGroup + chainName string config config.ChainConfig indexer tonIndexer.AccountIndexer cursorStore tonIndexer.CursorStore @@ -52,6 +53,7 @@ type WorkerConfig struct { // NewTonPollingWorker creates a new TON polling worker. func NewTonPollingWorker( ctx context.Context, + chainName string, cfg config.ChainConfig, indexer tonIndexer.AccountIndexer, cursorStore tonIndexer.CursorStore, @@ -81,6 +83,7 @@ func NewTonPollingWorker( ctx: ctx, cancel: cancel, logger: log, + chainName: chainName, config: cfg, indexer: indexer, cursorStore: cursorStore, @@ -309,6 +312,22 @@ func (w *TonPollingWorker) GetNetworkType() enum.NetworkType { return enum.NetworkTypeTon } +func (w *TonPollingWorker) GetName() string { + if w.chainName != "" { + return w.chainName + } + return w.config.Name +} + +func (w *TonPollingWorker) ReloadJettons(ctx context.Context) (int, error) { + count, err := w.indexer.ReloadJettons(ctx) + if err != nil { + return 0, err + } + w.logger.Info("Jetton registry reloaded", "chain", w.GetName(), "jetton_count", count) + return count, nil +} + // syncWalletsIfNeeded refreshes the wallet list from DB if sync interval passed. // Uses pagination to efficiently handle large numbers of wallets (>10K). // Before fetching everything, it checks the count to avoid redundant work. diff --git a/internal/worker/ton_jetton_reload.go b/internal/worker/ton_jetton_reload.go new file mode 100644 index 0000000..6efbd6f --- /dev/null +++ b/internal/worker/ton_jetton_reload.go @@ -0,0 +1,90 @@ +package worker + +import ( + "context" + "errors" + "fmt" + "sort" + + "github.com/fystack/multichain-indexer/pkg/common/enum" +) + +var ( + ErrNoTonWorkerConfigured = errors.New("no ton worker configured") + ErrTonWorkerNotFound = errors.New("ton worker not found") +) + +type TonJettonReloadRequest struct { + ChainFilter string +} + +type TonJettonReloadResult struct { + Chain string `json:"chain"` + ReloadedJettons int `json:"reloaded_jettons"` + Error string `json:"error,omitempty"` +} + +// TonJettonReloader is implemented by TON workers that support runtime jetton reload. +type TonJettonReloader interface { + Worker + GetName() string + GetNetworkType() enum.NetworkType + ReloadJettons(ctx context.Context) (int, error) +} + +// TonJettonReloadService handles runtime jetton registry reload for TON workers. +type TonJettonReloadService struct { + reloaders []TonJettonReloader +} + +func NewTonJettonReloadService(workers []Worker) *TonJettonReloadService { + reloaders := make([]TonJettonReloader, 0) + for _, w := range workers { + reloader, ok := w.(TonJettonReloader) + if !ok || reloader.GetNetworkType() != enum.NetworkTypeTon { + continue + } + reloaders = append(reloaders, reloader) + } + return &TonJettonReloadService{reloaders: reloaders} +} + +func NewTonJettonReloadServiceFromManager(m *Manager) *TonJettonReloadService { + if m == nil { + return &TonJettonReloadService{} + } + return NewTonJettonReloadService(m.Workers()) +} + +func (s *TonJettonReloadService) ReloadTonJettons( + ctx context.Context, + req TonJettonReloadRequest, +) ([]TonJettonReloadResult, error) { + results := make([]TonJettonReloadResult, 0) + + for _, reloader := range s.reloaders { + chainName := reloader.GetName() + if req.ChainFilter != "" && req.ChainFilter != chainName { + continue + } + + item := TonJettonReloadResult{Chain: chainName} + count, err := reloader.ReloadJettons(ctx) + if err != nil { + item.Error = err.Error() + } else { + item.ReloadedJettons = count + } + results = append(results, item) + } + + if len(results) == 0 { + if req.ChainFilter != "" { + return nil, fmt.Errorf("%w: %s", ErrTonWorkerNotFound, req.ChainFilter) + } + return nil, ErrNoTonWorkerConfigured + } + + sort.Slice(results, func(i, j int) bool { return results[i].Chain < results[j].Chain }) + return results, nil +} From deb47a27ab7e7dcdf61c69033851b8504ac54681 Mon Sep 17 00:00:00 2001 From: vietddude Date: Wed, 11 Feb 2026 14:53:38 +0700 Subject: [PATCH 03/11] refactor(ton): migrate to consolidated TON core and polling flow --- internal/indexer/ton/core.go | 703 ++++++++++++++++++ internal/indexer/ton/cursor.go | 93 --- internal/indexer/ton/indexer.go | 202 ----- internal/indexer/ton/jetton_parser.go | 222 ------ internal/indexer/ton/jetton_parser_test.go | 102 --- internal/indexer/ton/parser.go | 145 ---- internal/indexer/ton/parser_test.go | 98 --- .../ton/{jetton_registry.go => registry.go} | 5 - internal/rpc/ton/api.go | 4 - internal/rpc/ton/client.go | 17 - internal/worker/factory.go | 5 +- internal/worker/ton/worker.go | 300 +++++--- pkg/common/config/chains.go | 3 + pkg/common/config/types.go | 2 +- pkg/common/types/types.go | 30 +- 15 files changed, 902 insertions(+), 1029 deletions(-) create mode 100644 internal/indexer/ton/core.go delete mode 100644 internal/indexer/ton/cursor.go delete mode 100644 internal/indexer/ton/indexer.go delete mode 100644 internal/indexer/ton/jetton_parser.go delete mode 100644 internal/indexer/ton/jetton_parser_test.go delete mode 100644 internal/indexer/ton/parser.go delete mode 100644 internal/indexer/ton/parser_test.go rename internal/indexer/ton/{jetton_registry.go => registry.go} (98%) diff --git a/internal/indexer/ton/core.go b/internal/indexer/ton/core.go new file mode 100644 index 0000000..f9d43cd --- /dev/null +++ b/internal/indexer/ton/core.go @@ -0,0 +1,703 @@ +package ton + +import ( + "context" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "strings" + "time" + + tonRpc "github.com/fystack/multichain-indexer/internal/rpc/ton" + "github.com/fystack/multichain-indexer/pkg/common/config" + "github.com/fystack/multichain-indexer/pkg/common/constant" + "github.com/fystack/multichain-indexer/pkg/common/enum" + "github.com/fystack/multichain-indexer/pkg/common/types" + "github.com/fystack/multichain-indexer/pkg/infra" + "github.com/shopspring/decimal" + "github.com/xssnick/tonutils-go/address" + "github.com/xssnick/tonutils-go/tlb" +) + +const ( + cursorKeyPrefix = "ton/cursor/" + + // Jetton standard opcodes (TEP-74) + OpcodeTransfer uint64 = 0x0f8a7ea5 + OpcodeTransferNotification uint64 = 0x7362d09c +) + +// AccountCursor tracks the polling position for a single TON account. +type AccountCursor struct { + Address string `json:"address"` + LastLT uint64 `json:"last_lt"` // Logical time of last processed tx + LastHash string `json:"last_hash"` // Hex-encoded hash of last processed tx + UpdatedAt time.Time `json:"updated_at"` +} + +// CursorStore manages account cursors for TON polling. +type CursorStore interface { + // Get returns the cursor for an account, or nil if not found. + Get(ctx context.Context, address string) (*AccountCursor, error) + + // Save persists the cursor atomically. + Save(ctx context.Context, cursor *AccountCursor) error + + // Delete removes the cursor for an account. + Delete(ctx context.Context, address string) error + + // List returns all tracked account addresses. + List(ctx context.Context) ([]string, error) +} + +// kvCursorStore implements CursorStore using infra.KVStore. +type kvCursorStore struct { + kv infra.KVStore +} + +func NewCursorStore(kv infra.KVStore) CursorStore { + return &kvCursorStore{kv: kv} +} + +func (s *kvCursorStore) cursorKey(address string) string { + return cursorKeyPrefix + address +} + +func (s *kvCursorStore) Get(ctx context.Context, address string) (*AccountCursor, error) { + var cursor AccountCursor + found, err := s.kv.GetAny(s.cursorKey(address), &cursor) + if err != nil { + return nil, fmt.Errorf("failed to get cursor for %s: %w", address, err) + } + if !found { + return nil, nil + } + return &cursor, nil +} + +func (s *kvCursorStore) Save(ctx context.Context, cursor *AccountCursor) error { + cursor.UpdatedAt = time.Now() + if err := s.kv.SetAny(s.cursorKey(cursor.Address), cursor); err != nil { + return fmt.Errorf("failed to save cursor for %s: %w", cursor.Address, err) + } + return nil +} + +func (s *kvCursorStore) Delete(ctx context.Context, address string) error { + if err := s.kv.Delete(s.cursorKey(address)); err != nil { + return fmt.Errorf("failed to delete cursor for %s: %w", address, err) + } + return nil +} + +func (s *kvCursorStore) List(ctx context.Context) ([]string, error) { + pairs, err := s.kv.List(cursorKeyPrefix) + if err != nil { + return nil, fmt.Errorf("failed to list cursors: %w", err) + } + + addresses := make([]string, 0, len(pairs)) + for _, pair := range pairs { + var cursor AccountCursor + if err := json.Unmarshal(pair.Value, &cursor); err != nil { + continue // Skip malformed entries + } + addresses = append(addresses, cursor.Address) + } + + return addresses, nil +} + +// AccountIndexer is the interface for account-based indexing (TON). +// This is separate from the block-based Indexer interface used by EVM/Solana/etc. +type AccountIndexer interface { + GetName() string + GetNetworkType() enum.NetworkType + GetNetworkInternalCode() string + + // PollAccount fetches new transactions for a single account. + // Returns parsed transactions and the new cursor position. + // If no new transactions, returns empty slice and same cursor. + PollAccount(ctx context.Context, address string, cursor *AccountCursor) ([]types.Transaction, *AccountCursor, error) + + // IsHealthy checks if the RPC connection is healthy. + IsHealthy() bool + + // ReloadJettons refreshes supported jetton metadata at runtime. + ReloadJettons(ctx context.Context) (int, error) +} + +// TonAccountIndexer implements AccountIndexer for TON blockchain. +type TonAccountIndexer struct { + chainName string + config config.ChainConfig + client tonRpc.TonAPI + jettonRegistry JettonRegistry + + // Transaction limit per poll + txLimit uint32 +} + +// NewTonAccountIndexer creates a new TON account indexer. +func NewTonAccountIndexer( + chainName string, + cfg config.ChainConfig, + client tonRpc.TonAPI, + jettonRegistry JettonRegistry, +) *TonAccountIndexer { + txLimit := uint32(50) // Default transaction limit + if cfg.Throttle.BatchSize > 0 { + txLimit = uint32(cfg.Throttle.BatchSize) + } + + return &TonAccountIndexer{ + chainName: chainName, + config: cfg, + client: client, + jettonRegistry: jettonRegistry, + txLimit: txLimit, + } +} + +func (i *TonAccountIndexer) GetName() string { return i.chainName } +func (i *TonAccountIndexer) GetNetworkType() enum.NetworkType { return enum.NetworkTypeTon } +func (i *TonAccountIndexer) GetNetworkInternalCode() string { return i.config.InternalCode } + +// IsHealthy checks if the RPC connection is healthy. +func (i *TonAccountIndexer) IsHealthy() bool { + // The client manages its own connection pool and recovery. + // We consider it healthy if it's initialized. + return i.client != nil +} + +func (i *TonAccountIndexer) ReloadJettons(ctx context.Context) (int, error) { + if i.jettonRegistry == nil { + return 0, nil + } + + type registryReloader interface { + Reload(context.Context) error + } + + if reloader, ok := i.jettonRegistry.(registryReloader); ok { + if err := reloader.Reload(ctx); err != nil { + return 0, fmt.Errorf("reload jetton registry: %w", err) + } + } + + return len(i.jettonRegistry.List()), nil +} + +// PollAccount fetches new transactions for a single account. +func (i *TonAccountIndexer) PollAccount(ctx context.Context, addrStr string, cursor *AccountCursor) ([]types.Transaction, *AccountCursor, error) { + if i.client == nil { + return nil, cursor, fmt.Errorf("ton rpc client is not initialized") + } + + addr, normalizedAddress, err := resolvePollAddress(addrStr) + if err != nil { + return nil, cursor, err + } + + lastLT, lastHash, err := decodeCursorForRPC(cursor) + if err != nil { + return nil, cursor, err + } + + txs, err := i.client.ListTransactions(ctx, addr, i.txLimit, lastLT, lastHash) + if err != nil { + return nil, cursor, fmt.Errorf("failed to list transactions: %w", err) + } + + // No new transactions + if len(txs) == 0 { + return nil, cursor, nil + } + + parsedTxs := make([]types.Transaction, 0, len(txs)) + newCursor := ensureCursor(cursor, addrStr) + + // TON API returns newest first, so process backwards (oldest to newest). + for j := len(txs) - 1; j >= 0; j-- { + tx := txs[j] + + if isAlreadyProcessed(cursor, tx) { + continue + } + + advanceCursor(newCursor, tx) + + if !isTransactionSuccess(tx) { + continue + } + + parsedTxs = append(parsedTxs, i.parseMatchedTransactions(tx, normalizedAddress)...) + } + + return parsedTxs, newCursor, nil +} + +func resolvePollAddress(addrStr string) (*address.Address, string, error) { + addr, err := parseTONAddress(addrStr) + if err != nil { + return nil, "", fmt.Errorf("invalid TON address %s: %w", addrStr, err) + } + return addr, addr.StringRaw(), nil +} + +func decodeCursorForRPC(cursor *AccountCursor) (uint64, []byte, error) { + if cursor == nil || cursor.LastLT == 0 { + return 0, nil, nil + } + + lastLT := cursor.LastLT + if cursor.LastHash == "" { + return lastLT, nil, nil + } + + lastHash, err := hex.DecodeString(cursor.LastHash) + if err != nil { + return 0, nil, fmt.Errorf("invalid cursor hash: %w", err) + } + + return lastLT, lastHash, nil +} + +func ensureCursor(cursor *AccountCursor, address string) *AccountCursor { + if cursor != nil { + return cursor + } + return &AccountCursor{Address: address} +} + +func isAlreadyProcessed(cursor *AccountCursor, tx *tlb.Transaction) bool { + return cursor != nil && tx.LT <= cursor.LastLT +} + +func advanceCursor(cursor *AccountCursor, tx *tlb.Transaction) { + cursor.LastLT = tx.LT + cursor.LastHash = hex.EncodeToString(tx.Hash) +} + +func (i *TonAccountIndexer) parseMatchedTransactions( + tx *tlb.Transaction, + normalizedAddress string, +) []types.Transaction { + collected := parseTonTransferNormalized(tx, normalizedAddress, i.config.InternalCode) + if i.jettonRegistry != nil { + collected = append(collected, parseJettonTransferNormalized(tx, normalizedAddress, i.config.InternalCode, i.jettonRegistry)...) + } + return collected +} + +// isTransactionSuccess checks if the transaction was successful by examining its phases. +func isTransactionSuccess(tx *tlb.Transaction) bool { + if tx.Description == nil { + return true // Should not happen with valid transactions + } + + desc, ok := tx.Description.(*tlb.TransactionDescriptionOrdinary) + if !ok { + return true + } + + // Check if the whole transaction was aborted + if desc.Aborted { + return false + } + + // Check Compute Phase + if desc.ComputePhase.Phase != nil { + if cp, ok := desc.ComputePhase.Phase.(*tlb.ComputePhaseVM); ok { + if !cp.Success { + return false + } + } + } + + // Check Action Phase + if desc.ActionPhase != nil { + if !desc.ActionPhase.Success { + return false + } + } + + return true +} + +// JettonInfo describes a supported Jetton token. +type JettonInfo struct { + MasterAddress string `json:"master_address" yaml:"master_address"` // Jetton master contract + Symbol string `json:"symbol" yaml:"symbol"` + Decimals int `json:"decimals" yaml:"decimals"` +} + +// JettonRegistry manages supported Jetton tokens. +type JettonRegistry interface { + // IsSupported checks if a Jetton wallet belongs to a supported Jetton. + // This may require looking up the wallet's master address. + IsSupported(walletAddress string) bool + + // GetInfo returns info for a Jetton by its master address. + GetInfo(masterAddress string) (*JettonInfo, bool) + + // GetInfoByWallet returns info for a Jetton by a wallet address. + // Returns nil if the wallet is not from a known Jetton. + GetInfoByWallet(walletAddress string) (*JettonInfo, bool) + + // RegisterWallet associates a Jetton wallet with its master address. + RegisterWallet(walletAddress, masterAddress string) + + // List returns all supported Jettons. + List() []JettonInfo +} + +// NormalizeTONAddressRaw returns canonical raw format (workchain:hex) or empty if invalid. +func NormalizeTONAddressRaw(addr string) string { + parsed, err := parseTONAddress(addr) + if err != nil { + return "" + } + return parsed.StringRaw() +} + +// NormalizeTONAddressList trims and de-duplicates addresses while preserving order. +func NormalizeTONAddressList(addresses []string) []string { + if len(addresses) == 0 { + return nil + } + + dedup := make(map[string]struct{}, len(addresses)) + result := make([]string, 0, len(addresses)) + for _, addr := range addresses { + addr = strings.TrimSpace(addr) + if addr == "" { + continue + } + if _, exists := dedup[addr]; exists { + continue + } + dedup[addr] = struct{}{} + result = append(result, addr) + } + + return result +} + +func parseTONAddress(addrStr string) (*address.Address, error) { + addrStr = strings.TrimSpace(addrStr) + + // User-friendly format (base64url with checksum), e.g. EQ... + if addr, err := address.ParseAddr(addrStr); err == nil { + return addr, nil + } + // Raw format, e.g. 0:abcdef... + if addr, err := address.ParseRawAddr(addrStr); err == nil { + return addr, nil + } + + // Defensive normalization for malformed historical values. + parts := strings.SplitN(addrStr, ":", 2) + if len(parts) != 2 { + return nil, fmt.Errorf("invalid address format") + } + + rawHex := strings.TrimPrefix(strings.ToLower(strings.TrimSpace(parts[1])), "0x") + if rawHex == "" { + return nil, fmt.Errorf("empty address payload") + } + + if len(rawHex)%2 == 1 { + rawHex = "0" + rawHex + } + if len(rawHex) > 64 { + rawHex = rawHex[len(rawHex)-64:] + } else if len(rawHex) < 64 { + rawHex = strings.Repeat("0", 64-len(rawHex)) + rawHex + } + + return address.ParseRawAddr(parts[0] + ":" + rawHex) +} + +// ParseTonTransfer extracts native TON transfers from a transaction. +// Detects both incoming (receive) and outgoing (send) transfers. +func ParseTonTransfer(tx *tlb.Transaction, ourAddress string, networkID string) []types.Transaction { + normalizedOurAddress := NormalizeTONAddressRaw(ourAddress) + if normalizedOurAddress == "" { + return nil + } + return parseTonTransferNormalized(tx, normalizedOurAddress, networkID) +} + +func parseTonTransferNormalized(tx *tlb.Transaction, normalizedOurAddress string, networkID string) []types.Transaction { + var results []types.Transaction + + if parsed, ok := parseIncomingTransfer(tx, normalizedOurAddress, networkID); ok { + results = append(results, *parsed) + } + + for _, intMsg := range outgoingInternalMessages(tx) { + parsed, ok := parseOutgoingTransferMessage(tx, intMsg, normalizedOurAddress, networkID) + if !ok { + continue + } + results = append(results, parsed) + } + + return results +} + +func parseIncomingTransfer(tx *tlb.Transaction, ourAddress string, networkID string) (*types.Transaction, bool) { + intMsg, ok := incomingInternalMessage(tx) + if !ok { + return nil, false + } + + dstAddrRaw := intMsg.DstAddr.StringRaw() + if dstAddrRaw != ourAddress || !isSimpleTransferMessage(intMsg) { + return nil, false + } + + amount := intMsg.Amount.Nano().String() + if amount == "0" { + return nil, false + } + + txData := newBaseParsedTransaction(tx, networkID) + txData.FromAddress = intMsg.SrcAddr.StringRaw() + txData.ToAddress = dstAddrRaw + txData.AssetAddress = "" + txData.Amount = amount + txData.Type = constant.TxTypeNativeTransfer + return &txData, true +} + +func parseOutgoingTransferMessage( + tx *tlb.Transaction, + intMsg *tlb.InternalMessage, + ourAddress string, + networkID string, +) (types.Transaction, bool) { + srcAddr := intMsg.SrcAddr.StringRaw() + if srcAddr != ourAddress || !isSimpleTransferMessage(intMsg) { + return types.Transaction{}, false + } + + amount := intMsg.Amount.Nano().String() + if amount == "0" { + return types.Transaction{}, false + } + + txData := newBaseParsedTransaction(tx, networkID) + txData.FromAddress = srcAddr + txData.ToAddress = intMsg.DstAddr.StringRaw() + txData.AssetAddress = "" + txData.Amount = amount + txData.Type = constant.TxTypeNativeTransfer + return txData, true +} + +// ParseJettonTransfer extracts Jetton transfers from a transaction. +func ParseJettonTransfer(tx *tlb.Transaction, ourAddress string, networkID string, registry JettonRegistry) []types.Transaction { + normalizedOurAddress := NormalizeTONAddressRaw(ourAddress) + if normalizedOurAddress == "" { + return nil + } + return parseJettonTransferNormalized(tx, normalizedOurAddress, networkID, registry) +} + +func parseJettonTransferNormalized( + tx *tlb.Transaction, + normalizedOurAddress string, + networkID string, + registry JettonRegistry, +) []types.Transaction { + var results []types.Transaction + + if parsed, ok := parseIncomingJetton(tx, normalizedOurAddress, networkID, registry); ok { + results = append(results, *parsed) + } + + for _, intMsg := range outgoingInternalMessages(tx) { + parsed, ok := parseOutgoingJettonMessage(tx, intMsg, normalizedOurAddress, networkID, registry) + if !ok { + continue + } + results = append(results, parsed) + } + + return results +} + +func parseIncomingJetton(tx *tlb.Transaction, ourAddress string, networkID string, registry JettonRegistry) (*types.Transaction, bool) { + intMsg, ok := incomingInternalMessage(tx) + if !ok { + return nil, false + } + + dstAddrRaw := intMsg.DstAddr.StringRaw() + if dstAddrRaw != ourAddress || !messageHasOpcode(intMsg, OpcodeTransferNotification) { + return nil, false + } + + jettonAmount, sender, ok := parseJettonTransferBody(intMsg) + if !ok || sender == nil { + return nil, false + } + + jettonWallet := intMsg.SrcAddr.StringRaw() + + txData := newBaseParsedTransaction(tx, networkID) + txData.FromAddress = sender.StringRaw() + txData.ToAddress = dstAddrRaw + txData.AssetAddress = resolveJettonAssetAddress(registry, jettonWallet) + txData.Amount = jettonAmount + txData.Type = constant.TxTypeTokenTransfer + return &txData, true +} + +func parseOutgoingJettonMessage( + tx *tlb.Transaction, + intMsg *tlb.InternalMessage, + ourAddress string, + networkID string, + registry JettonRegistry, +) (types.Transaction, bool) { + if !messageHasOpcode(intMsg, OpcodeTransfer) { + return types.Transaction{}, false + } + + jettonAmount, destination, ok := parseJettonTransferBody(intMsg) + if !ok || destination == nil { + return types.Transaction{}, false + } + + jettonWallet := intMsg.DstAddr.StringRaw() + + txData := newBaseParsedTransaction(tx, networkID) + txData.FromAddress = ourAddress + txData.ToAddress = destination.StringRaw() + txData.AssetAddress = resolveJettonAssetAddress(registry, jettonWallet) + txData.Amount = jettonAmount + txData.Type = constant.TxTypeTokenTransfer + return txData, true +} + +func parseJettonTransferBody(msg *tlb.InternalMessage) (string, *address.Address, bool) { + if msg.Body == nil { + return "", nil, false + } + + bodySlice := msg.Body.BeginParse() + if _, err := bodySlice.LoadUInt(32); err != nil { // opcode + return "", nil, false + } + if _, err := bodySlice.LoadUInt(64); err != nil { // query_id + return "", nil, false + } + + jettonAmount, err := bodySlice.LoadVarUInt(16) + if err != nil { + return "", nil, false + } + + peerAddress, err := bodySlice.LoadAddr() + if err != nil { + return "", nil, false + } + + return jettonAmount.String(), peerAddress, true +} + +func incomingInternalMessage(tx *tlb.Transaction) (*tlb.InternalMessage, bool) { + if tx.IO.In == nil { + return nil, false + } + + intMsg, ok := tx.IO.In.Msg.(*tlb.InternalMessage) + if !ok || intMsg.Bounced { + return nil, false + } + + return intMsg, true +} + +func outgoingInternalMessages(tx *tlb.Transaction) []*tlb.InternalMessage { + if tx.IO.Out == nil { + return nil + } + + outList, err := tx.IO.Out.ToSlice() + if err != nil { + return nil + } + + messages := make([]*tlb.InternalMessage, 0, len(outList)) + for _, outMsg := range outList { + intMsg, ok := outMsg.Msg.(*tlb.InternalMessage) + if !ok { + continue + } + messages = append(messages, intMsg) + } + return messages +} + +func messageOpcode(msg *tlb.InternalMessage) (uint64, bool) { + if msg.Body == nil { + return 0, false + } + + bodySlice := msg.Body.BeginParse() + if bodySlice.BitsLeft() < 32 { + return 0, false + } + + opcode, err := bodySlice.LoadUInt(32) + if err != nil { + return 0, false + } + return opcode, true +} + +func isSimpleTransferMessage(msg *tlb.InternalMessage) bool { + if msg.Body == nil { + return true + } + + opcode, ok := messageOpcode(msg) + if !ok { + return true + } + + // Opcode 0 means comment payload for regular TON transfer. + return opcode == 0 +} + +func messageHasOpcode(msg *tlb.InternalMessage, expected uint64) bool { + opcode, ok := messageOpcode(msg) + return ok && opcode == expected +} + +func resolveJettonAssetAddress(registry JettonRegistry, jettonWallet string) string { + if registry == nil { + return jettonWallet + } + if info, ok := registry.GetInfoByWallet(jettonWallet); ok { + return info.MasterAddress + } + return jettonWallet +} + +func newBaseParsedTransaction(tx *tlb.Transaction, networkID string) types.Transaction { + return types.Transaction{ + TxHash: base64.StdEncoding.EncodeToString(tx.Hash), + NetworkId: networkID, + BlockNumber: 0, + LogicalTime: tx.LT, + TxFee: decimal.NewFromBigInt(tx.TotalFees.Coins.Nano(), 0).Div(decimal.NewFromInt(1e9)), + Timestamp: uint64(tx.Now), + Status: types.StatusConfirmed, + } +} diff --git a/internal/indexer/ton/cursor.go b/internal/indexer/ton/cursor.go deleted file mode 100644 index b89e70c..0000000 --- a/internal/indexer/ton/cursor.go +++ /dev/null @@ -1,93 +0,0 @@ -package ton - -import ( - "context" - "encoding/json" - "fmt" - "time" - - "github.com/fystack/multichain-indexer/pkg/infra" -) - -const cursorKeyPrefix = "ton/cursor/" - -// AccountCursor tracks the polling position for a single TON account. -type AccountCursor struct { - Address string `json:"address"` - LastLT uint64 `json:"last_lt"` // Logical time of last processed tx - LastHash string `json:"last_hash"` // Hex-encoded hash of last processed tx - UpdatedAt time.Time `json:"updated_at"` -} - -// CursorStore manages account cursors for TON polling. -type CursorStore interface { - // Get returns the cursor for an account, or nil if not found. - Get(ctx context.Context, address string) (*AccountCursor, error) - - // Save persists the cursor atomically. - Save(ctx context.Context, cursor *AccountCursor) error - - // Delete removes the cursor for an account. - Delete(ctx context.Context, address string) error - - // List returns all tracked account addresses. - List(ctx context.Context) ([]string, error) -} - -// kvCursorStore implements CursorStore using infra.KVStore. -type kvCursorStore struct { - kv infra.KVStore -} - -func NewCursorStore(kv infra.KVStore) CursorStore { - return &kvCursorStore{kv: kv} -} - -func (s *kvCursorStore) cursorKey(address string) string { - return cursorKeyPrefix + address -} - -func (s *kvCursorStore) Get(ctx context.Context, address string) (*AccountCursor, error) { - var cursor AccountCursor - found, err := s.kv.GetAny(s.cursorKey(address), &cursor) - if err != nil { - return nil, fmt.Errorf("failed to get cursor for %s: %w", address, err) - } - if !found { - return nil, nil - } - return &cursor, nil -} - -func (s *kvCursorStore) Save(ctx context.Context, cursor *AccountCursor) error { - cursor.UpdatedAt = time.Now() - if err := s.kv.SetAny(s.cursorKey(cursor.Address), cursor); err != nil { - return fmt.Errorf("failed to save cursor for %s: %w", cursor.Address, err) - } - return nil -} - -func (s *kvCursorStore) Delete(ctx context.Context, address string) error { - if err := s.kv.Delete(s.cursorKey(address)); err != nil { - return fmt.Errorf("failed to delete cursor for %s: %w", address, err) - } - return nil -} - -func (s *kvCursorStore) List(ctx context.Context) ([]string, error) { - pairs, err := s.kv.List(cursorKeyPrefix) - if err != nil { - return nil, fmt.Errorf("failed to list cursors: %w", err) - } - - addresses := make([]string, 0, len(pairs)) - for _, pair := range pairs { - var cursor AccountCursor - if err := json.Unmarshal(pair.Value, &cursor); err != nil { - continue // Skip malformed entries - } - addresses = append(addresses, cursor.Address) - } - - return addresses, nil -} diff --git a/internal/indexer/ton/indexer.go b/internal/indexer/ton/indexer.go deleted file mode 100644 index 8e3c78c..0000000 --- a/internal/indexer/ton/indexer.go +++ /dev/null @@ -1,202 +0,0 @@ -package ton - -import ( - "context" - "encoding/hex" - "fmt" - - tonRpc "github.com/fystack/multichain-indexer/internal/rpc/ton" - "github.com/fystack/multichain-indexer/pkg/common/config" - "github.com/fystack/multichain-indexer/pkg/common/enum" - "github.com/fystack/multichain-indexer/pkg/common/types" - "github.com/xssnick/tonutils-go/address" - "github.com/xssnick/tonutils-go/tlb" -) - -// AccountIndexer is the interface for account-based indexing (TON). -// This is separate from the block-based Indexer interface used by EVM/Solana/etc. -type AccountIndexer interface { - GetName() string - GetNetworkType() enum.NetworkType - GetNetworkInternalCode() string - - // PollAccount fetches new transactions for a single account. - // Returns parsed transactions and the new cursor position. - // If no new transactions, returns empty slice and same cursor. - PollAccount(ctx context.Context, address string, cursor *AccountCursor) ([]types.Transaction, *AccountCursor, error) - - // IsHealthy checks if the RPC connection is healthy. - IsHealthy() bool - - // ReloadJettons refreshes supported jetton metadata at runtime. - ReloadJettons(ctx context.Context) (int, error) -} - -// TonAccountIndexer implements AccountIndexer for TON blockchain. -type TonAccountIndexer struct { - chainName string - config config.ChainConfig - client tonRpc.TonAPI - jettonRegistry JettonRegistry - - // Transaction limit per poll - txLimit uint32 -} - -// NewTonAccountIndexer creates a new TON account indexer. -func NewTonAccountIndexer( - chainName string, - cfg config.ChainConfig, - client tonRpc.TonAPI, - jettonRegistry JettonRegistry, -) *TonAccountIndexer { - txLimit := uint32(50) // Default transaction limit - if cfg.Throttle.BatchSize > 0 { - txLimit = uint32(cfg.Throttle.BatchSize) - } - - return &TonAccountIndexer{ - chainName: chainName, - config: cfg, - client: client, - jettonRegistry: jettonRegistry, - txLimit: txLimit, - } -} - -func (i *TonAccountIndexer) GetName() string { return i.chainName } -func (i *TonAccountIndexer) GetNetworkType() enum.NetworkType { return enum.NetworkTypeTon } -func (i *TonAccountIndexer) GetNetworkInternalCode() string { return i.config.InternalCode } - -// IsHealthy checks if the RPC connection is healthy. -func (i *TonAccountIndexer) IsHealthy() bool { - // The client manages its own connection pool and recovery. - // We consider it healthy if it's initialized. - return i.client != nil -} - -func (i *TonAccountIndexer) ReloadJettons(ctx context.Context) (int, error) { - if i.jettonRegistry == nil { - return 0, nil - } - - type registryReloader interface { - Reload(context.Context) error - } - - if reloader, ok := i.jettonRegistry.(registryReloader); ok { - if err := reloader.Reload(ctx); err != nil { - return 0, fmt.Errorf("reload jetton registry: %w", err) - } - } - - return len(i.jettonRegistry.List()), nil -} - -// PollAccount fetches new transactions for a single account. -func (i *TonAccountIndexer) PollAccount(ctx context.Context, addrStr string, cursor *AccountCursor) ([]types.Transaction, *AccountCursor, error) { - // Parse the address - addr, err := address.ParseAddr(addrStr) - if err != nil { - return nil, cursor, fmt.Errorf("invalid TON address %s: %w", addrStr, err) - } - - // Prepare cursor values for API call - var lastLT uint64 - var lastHash []byte - if cursor != nil && cursor.LastLT > 0 { - lastLT = cursor.LastLT - if cursor.LastHash != "" { - lastHash, err = hex.DecodeString(cursor.LastHash) - if err != nil { - return nil, cursor, fmt.Errorf("invalid cursor hash: %w", err) - } - } - } - - // Fetch transactions from the account - // The client handles failover and retries internally - txs, err := i.client.ListTransactions(ctx, addr, i.txLimit, lastLT, lastHash) - if err != nil { - return nil, cursor, fmt.Errorf("failed to list transactions: %w", err) - } - - // No new transactions - if len(txs) == 0 { - return nil, cursor, nil - } - - // Process transactions in reverse order (oldest first for proper cursor updates) - // TON API returns newest first, so we iterate backwards - var parsedTxs []types.Transaction - newCursor := cursor - if newCursor == nil { - newCursor = &AccountCursor{Address: addrStr} - } - - // Normalize address string for consistent comparison - addrStr = addr.String() - - for j := len(txs) - 1; j >= 0; j-- { - tx := txs[j] - - // Skip transactions we've already processed (at or before cursor) - if cursor != nil && tx.LT <= cursor.LastLT { - continue - } - - // Update cursor to this transaction (even if it's skipped for failure) - newCursor.LastLT = tx.LT - newCursor.LastHash = hex.EncodeToString(tx.Hash) - - // Check if transaction was successful - if !isTransactionSuccess(tx) { - continue - } - - var collectedTxs []types.Transaction - collectedTxs = append(collectedTxs, ParseTonTransfer(tx, addrStr, i.config.InternalCode)...) - - if i.jettonRegistry != nil { - collectedTxs = append(collectedTxs, ParseJettonTransfer(tx, addrStr, i.config.InternalCode, i.jettonRegistry)...) - } - - parsedTxs = append(parsedTxs, collectedTxs...) - } - return parsedTxs, newCursor, nil -} - -// isTransactionSuccess checks if the transaction was successful by examining its phases. -func isTransactionSuccess(tx *tlb.Transaction) bool { - if tx.Description == nil { - return true // Should not happen with valid transactions - } - - desc, ok := tx.Description.(*tlb.TransactionDescriptionOrdinary) - if !ok { - return true - } - - // Check if the whole transaction was aborted - if desc.Aborted { - return false - } - - // Check Compute Phase - if desc.ComputePhase.Phase != nil { - if cp, ok := desc.ComputePhase.Phase.(*tlb.ComputePhaseVM); ok { - if !cp.Success { - return false - } - } - } - - // Check Action Phase - if desc.ActionPhase != nil { - if !desc.ActionPhase.Success { - return false - } - } - - return true -} diff --git a/internal/indexer/ton/jetton_parser.go b/internal/indexer/ton/jetton_parser.go deleted file mode 100644 index b962162..0000000 --- a/internal/indexer/ton/jetton_parser.go +++ /dev/null @@ -1,222 +0,0 @@ -package ton - -import ( - "encoding/base64" - - "github.com/fystack/multichain-indexer/pkg/common/constant" - "github.com/fystack/multichain-indexer/pkg/common/types" - "github.com/shopspring/decimal" - "github.com/xssnick/tonutils-go/tlb" -) - -// Jetton standard opcodes (TEP-74) -const ( - OpcodeTransfer uint64 = 0x0f8a7ea5 - OpcodeTransferNotification uint64 = 0x7362d09c -) - -// JettonInfo describes a supported Jetton token. -type JettonInfo struct { - MasterAddress string `json:"master_address" yaml:"master_address"` // Jetton master contract - Symbol string `json:"symbol" yaml:"symbol"` - Decimals int `json:"decimals" yaml:"decimals"` -} - -// JettonRegistry manages supported Jetton tokens. -type JettonRegistry interface { - // IsSupported checks if a Jetton wallet belongs to a supported Jetton. - // This may require looking up the wallet's master address. - IsSupported(walletAddress string) bool - - // GetInfo returns info for a Jetton by its master address. - GetInfo(masterAddress string) (*JettonInfo, bool) - - // GetInfoByWallet returns info for a Jetton by a wallet address. - // Returns nil if the wallet is not from a known Jetton. - GetInfoByWallet(walletAddress string) (*JettonInfo, bool) - - // RegisterWallet associates a Jetton wallet with its master address. - RegisterWallet(walletAddress, masterAddress string) - - // List returns all supported Jettons. - List() []JettonInfo -} - -// ParseJettonTransfer extracts a Jetton transfer from a transaction. -// Detects both incoming (receive) and outgoing (send) Jetton transfers. -// Returns a slice of parsed transactions involving our address. -func ParseJettonTransfer(tx *tlb.Transaction, ourAddress string, networkID string, registry JettonRegistry) []types.Transaction { - var results []types.Transaction - - // Try incoming transfer notification first (receive) - if parsed, ok := parseIncomingJetton(tx, ourAddress, networkID, registry); ok { - results = append(results, *parsed) - } - - // Try outgoing transfers (send) - can be multiple - results = append(results, parseOutgoingJettons(tx, ourAddress, networkID, registry)...) - - return results -} - -// parseIncomingJetton handles incoming Jetton transfer notifications (receive). -func parseIncomingJetton(tx *tlb.Transaction, ourAddress string, networkID string, registry JettonRegistry) (*types.Transaction, bool) { - // Check for incoming internal message - if tx.IO.In == nil { - return nil, false - } - - // Only process internal messages - inMsg := tx.IO.In.Msg - intMsg, ok := inMsg.(*tlb.InternalMessage) - if !ok || intMsg.Bounced { - return nil, false - } - - // Verify destination is our account - dstAddr := intMsg.DstAddr.String() - if dstAddr != ourAddress { - return nil, false - } - - // Parse message body for transfer_notification - if intMsg.Body == nil { - return nil, false - } - - bodySlice := intMsg.Body.BeginParse() - opcode, err := bodySlice.LoadUInt(32) - if err != nil || opcode != OpcodeTransferNotification { - return nil, false - } - - // Structure: query_id:uint64 amount:VarUInteger16 sender:Address forward_payload:Either Cell ^Cell - _, err = bodySlice.LoadUInt(64) // query_id - if err != nil { - return nil, false - } - - jettonAmount, err := bodySlice.LoadVarUInt(16) - if err != nil { - return nil, false - } - - sender, err := bodySlice.LoadAddr() - if err != nil { - return nil, false - } - - // The sender of the notification is the Jetton wallet - jettonWallet := intMsg.SrcAddr.String() - - // Try to get Jetton info from registry - var assetAddress string - if registry != nil { - if info, ok := registry.GetInfoByWallet(jettonWallet); ok { - assetAddress = info.MasterAddress - } else { - // If not found in cache, it might be an unknown Jetton - // or we need to wait for a registration - assetAddress = jettonWallet - } - } else { - assetAddress = jettonWallet - } - - return &types.Transaction{ - TxHash: base64.StdEncoding.EncodeToString(tx.Hash), - NetworkId: networkID, - BlockNumber: tx.LT, - FromAddress: sender.String(), - ToAddress: ourAddress, - AssetAddress: assetAddress, - Amount: jettonAmount.String(), - Type: constant.TxTypeTokenTransfer, - TxFee: decimal.NewFromBigInt(tx.TotalFees.Coins.Nano(), 0).Div(decimal.NewFromInt(1e9)), - Timestamp: uint64(tx.Now), - Status: types.StatusConfirmed, - }, true -} - -// parseOutgoingJettons handles outgoing Jetton transfers (send). -func parseOutgoingJettons(tx *tlb.Transaction, ourAddress string, networkID string, registry JettonRegistry) []types.Transaction { - var results []types.Transaction - - // Check for outgoing messages - if tx.IO.Out == nil { - return nil - } - - outList, err := tx.IO.Out.ToSlice() - if err != nil { - return nil - } - - for _, outMsg := range outList { - intMsg, ok := outMsg.Msg.(*tlb.InternalMessage) - if !ok { - continue - } - - // Parse message body for transfer opcode - if intMsg.Body == nil { - continue - } - - bodySlice := intMsg.Body.BeginParse() - opcode, err := bodySlice.LoadUInt(32) - if err != nil || opcode != OpcodeTransfer { - continue - } - - // Parse transfer fields - // Skip query_id - _, err = bodySlice.LoadUInt(64) - if err != nil { - continue - } - - // Load Jetton amount - jettonAmount, err := bodySlice.LoadVarUInt(16) - if err != nil { - continue - } - - // Load destination address - destination, err := bodySlice.LoadAddr() - if err != nil { - continue - } - - // Get Jetton wallet (the destination of the outgoing message is our Jetton wallet) - jettonWallet := intMsg.DstAddr.String() - - // Try to get Jetton info from registry - var assetAddress string - if registry != nil { - if info, ok := registry.GetInfoByWallet(jettonWallet); ok { - assetAddress = info.MasterAddress - } else { - assetAddress = jettonWallet - } - } else { - assetAddress = jettonWallet - } - - results = append(results, types.Transaction{ - TxHash: base64.StdEncoding.EncodeToString(tx.Hash), - NetworkId: networkID, - BlockNumber: tx.LT, - FromAddress: ourAddress, - ToAddress: destination.String(), - AssetAddress: assetAddress, - Amount: jettonAmount.String(), - Type: constant.TxTypeTokenTransfer, - TxFee: decimal.NewFromBigInt(tx.TotalFees.Coins.Nano(), 0).Div(decimal.NewFromInt(1e9)), - Timestamp: uint64(tx.Now), - Status: types.StatusConfirmed, - }) - } - - return results -} diff --git a/internal/indexer/ton/jetton_parser_test.go b/internal/indexer/ton/jetton_parser_test.go deleted file mode 100644 index fe78c0a..0000000 --- a/internal/indexer/ton/jetton_parser_test.go +++ /dev/null @@ -1,102 +0,0 @@ -package ton - -import ( - "encoding/base64" - "testing" - "time" - - "github.com/fystack/multichain-indexer/pkg/common/constant" - "github.com/stretchr/testify/assert" - "github.com/xssnick/tonutils-go/address" - "github.com/xssnick/tonutils-go/tlb" - "github.com/xssnick/tonutils-go/tvm/cell" -) - -// MockRegistry implements JettonRegistry for testing -type MockRegistry struct { - info map[string]JettonInfo -} - -func (m *MockRegistry) IsSupported(walletAddress string) bool { - return false -} - -func (m *MockRegistry) GetInfo(masterAddress string) (*JettonInfo, bool) { - return nil, false -} - -func (m *MockRegistry) GetInfoByWallet(walletAddress string) (*JettonInfo, bool) { - if info, ok := m.info[walletAddress]; ok { - return &info, true - } - return nil, false -} - -func (m *MockRegistry) RegisterWallet(walletAddress, masterAddress string) {} -func (m *MockRegistry) List() []JettonInfo { return nil } - -func TestParseJettonTransfer(t *testing.T) { - ourAddr := address.MustParseAddr("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF") // Burn address - senderAddr := address.MustParseAddr("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c") // Zero address - // The jetton wallet that sends us the notification - jettonWalletAddr := address.MustParseAddr("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c") // Zero address - - t.Run("valid transfer notification", func(t *testing.T) { - // Mock registry - registry := &MockRegistry{ - info: map[string]JettonInfo{ - jettonWalletAddr.String(): { - MasterAddress: "EQMaster...", - Symbol: "USDT", - Decimals: 6, - }, - }, - } - - // Construct transfer notification body - // Opcode: 0x7362d09c (transfer_notification) - // query_id: uint64 - // amount: Coins - // sender: MsgAddress - // forward_payload: Either Cell ^Cell - body := cell.BeginCell(). - MustStoreUInt(0x7362d09c, 32). // Opcode - MustStoreUInt(0, 64). // QueryID - MustStoreCoins(1000000). // Amount (1 USDT) - MustStoreAddr(senderAddr). // Original sender - MustStoreRef( // Forward payload (comment) - cell.BeginCell(). - MustStoreUInt(0, 32). // Text comment opcode - MustStoreStringSnake("hello jetton"). - EndCell(), - ). - EndCell() - - tx := &tlb.Transaction{ - LT: 2000, - Now: uint32(time.Now().Unix()), - } - - tx.IO.In = &tlb.Message{ - MsgType: tlb.MsgTypeInternal, - Msg: &tlb.InternalMessage{ - SrcAddr: jettonWalletAddr, // Notification comes from jetton wallet - DstAddr: ourAddr, - Amount: tlb.MustFromTON("0.1"), // Small amount of TON attached - Body: body, - }, - } - - tx.Hash = []byte("txhash") - - results := ParseJettonTransfer(tx, ourAddr.String(), "TON_MAINNET", registry) - assert.Len(t, results, 1) - parsed := results[0] - assert.Equal(t, "1000000", parsed.Amount) - assert.Equal(t, senderAddr.String(), parsed.FromAddress) - assert.Equal(t, ourAddr.String(), parsed.ToAddress) - assert.Equal(t, "EQMaster...", parsed.AssetAddress) - assert.Equal(t, base64.StdEncoding.EncodeToString(tx.Hash), parsed.TxHash) - assert.Equal(t, constant.TxTypeTokenTransfer, parsed.Type) - }) -} diff --git a/internal/indexer/ton/parser.go b/internal/indexer/ton/parser.go deleted file mode 100644 index dff3670..0000000 --- a/internal/indexer/ton/parser.go +++ /dev/null @@ -1,145 +0,0 @@ -package ton - -import ( - "encoding/base64" - - "github.com/fystack/multichain-indexer/pkg/common/constant" - "github.com/fystack/multichain-indexer/pkg/common/types" - "github.com/shopspring/decimal" - "github.com/xssnick/tonutils-go/tlb" -) - -// ParseTonTransfer extracts native TON transfers from a transaction. -// Detects both incoming (receive) and outgoing (send) transfers. -// Returns a slice of parsed transactions involving our address. -func ParseTonTransfer(tx *tlb.Transaction, ourAddress string, networkID string) []types.Transaction { - var results []types.Transaction - - // Try to parse as incoming transfer - if parsed, ok := parseIncomingTransfer(tx, ourAddress, networkID); ok { - results = append(results, *parsed) - } - - // Try to parse as outgoing transfers (can be multiple) - results = append(results, parseOutgoingTransfers(tx, ourAddress, networkID)...) - - return results -} - -// parseIncomingTransfer handles incoming TON transfers (receive). -func parseIncomingTransfer(tx *tlb.Transaction, ourAddress string, networkID string) (*types.Transaction, bool) { - // Check for incoming internal message - if tx.IO.In == nil { - return nil, false - } - - // Only process internal messages (wallet-to-wallet transfers) - inMsg := tx.IO.In.Msg - intMsg, ok := inMsg.(*tlb.InternalMessage) - if !ok || intMsg.Bounced { - return nil, false - } - - // Verify destination is our account - dstAddr := intMsg.DstAddr.String() - if dstAddr != ourAddress { - return nil, false - } - - // Extract amount (in nanoTON) - amount := intMsg.Amount.Nano().String() - - // Skip zero-value messages (typically these are just notifications) - if amount == "0" { - return nil, false - } - - // IF the message has a body, check if it's a simple comment (opcode 0) - // If it has a non-zero opcode, it's likely a contract interaction (like Jetton) - if intMsg.Body != nil { - bodySlice := intMsg.Body.BeginParse() - if bodySlice.BitsLeft() >= 32 { - opcode, err := bodySlice.LoadUInt(32) - if err == nil && opcode != 0 { - // Non-zero opcode means this is NOT a simple transfer - return nil, false - } - } - } - - return &types.Transaction{ - TxHash: base64.StdEncoding.EncodeToString(tx.Hash), - NetworkId: networkID, - BlockNumber: tx.LT, - FromAddress: intMsg.SrcAddr.String(), - ToAddress: dstAddr, - AssetAddress: "", // Empty for native TON - Amount: amount, - Type: constant.TxTypeNativeTransfer, // Incoming transfer - TxFee: decimal.NewFromBigInt(tx.TotalFees.Coins.Nano(), 0).Div(decimal.NewFromInt(1e9)), - Timestamp: uint64(tx.Now), - Status: types.StatusConfirmed, - }, true -} - -// parseOutgoingTransfers handles outgoing TON transfers (send). -func parseOutgoingTransfers(tx *tlb.Transaction, ourAddress string, networkID string) []types.Transaction { - var results []types.Transaction - - // Check for outgoing messages - if tx.IO.Out == nil { - return nil - } - - // Iterate through outgoing messages to find transfers - outList, err := tx.IO.Out.ToSlice() - if err != nil { - return nil - } - - for _, outMsg := range outList { - intMsg, ok := outMsg.Msg.(*tlb.InternalMessage) - if !ok { - continue - } - - // Verify source is our account - srcAddr := intMsg.SrcAddr.String() - if srcAddr != ourAddress { - continue - } - - // Extract amount - amount := intMsg.Amount.Nano().String() - if amount == "0" { - continue - } - - // Skip if it has a non-zero opcode (likely contract interaction) - if intMsg.Body != nil { - bodySlice := intMsg.Body.BeginParse() - if bodySlice.BitsLeft() >= 32 { - opcode, err := bodySlice.LoadUInt(32) - if err == nil && opcode != 0 { - continue - } - } - } - - results = append(results, types.Transaction{ - TxHash: base64.StdEncoding.EncodeToString(tx.Hash), - NetworkId: networkID, - BlockNumber: tx.LT, - FromAddress: srcAddr, - ToAddress: intMsg.DstAddr.String(), - AssetAddress: "", - Amount: amount, - Type: constant.TxTypeNativeTransfer, - TxFee: decimal.NewFromBigInt(tx.TotalFees.Coins.Nano(), 0).Div(decimal.NewFromInt(1e9)), - Timestamp: uint64(tx.Now), - Status: types.StatusConfirmed, - }) - } - - return results -} diff --git a/internal/indexer/ton/parser_test.go b/internal/indexer/ton/parser_test.go deleted file mode 100644 index e409f04..0000000 --- a/internal/indexer/ton/parser_test.go +++ /dev/null @@ -1,98 +0,0 @@ -package ton - -import ( - "encoding/base64" - "testing" - "time" - - "github.com/fystack/multichain-indexer/pkg/common/constant" - "github.com/stretchr/testify/assert" - "github.com/xssnick/tonutils-go/address" - "github.com/xssnick/tonutils-go/tlb" - "github.com/xssnick/tonutils-go/tvm/cell" -) - -func TestParseTonTransfer(t *testing.T) { - // Burn address (valid) - ourAddr := address.MustParseAddr("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF") - // Zero address (valid) - senderAddr := address.MustParseAddr("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c") - - t.Run("valid receive", func(t *testing.T) { - // Construct a transaction - tx := &tlb.Transaction{ - LT: 1000, - Now: uint32(time.Now().Unix()), - } - - // Setup incoming internal message safely - tx.IO.In = &tlb.Message{ - MsgType: tlb.MsgTypeInternal, - Msg: &tlb.InternalMessage{ - SrcAddr: senderAddr, - DstAddr: ourAddr, - Amount: tlb.MustFromTON("1"), // 1 TON - Body: cell.BeginCell().EndCell(), - }, - } - - tx.Hash = []byte("txhash") - - results := ParseTonTransfer(tx, ourAddr.String(), "TON_MAINNET") - assert.Len(t, results, 1) - parsed := results[0] - assert.Equal(t, "1000000000", parsed.Amount) - assert.Equal(t, senderAddr.String(), parsed.FromAddress) - assert.Equal(t, ourAddr.String(), parsed.ToAddress) - assert.Equal(t, base64.StdEncoding.EncodeToString(tx.Hash), parsed.TxHash) - assert.Equal(t, constant.TxTypeNativeTransfer, parsed.Type) - }) - - t.Run("empty transaction", func(t *testing.T) { - tx := &tlb.Transaction{} - results := ParseTonTransfer(tx, ourAddr.String(), "TON_MAINNET") - assert.Len(t, results, 0) - }) - - t.Run("ignore other destination", func(t *testing.T) { - otherAddr := address.MustParseAddr("EQD__________________________________________0vo") - tx := &tlb.Transaction{ - LT: 1000, - } - tx.IO.In = &tlb.Message{ - MsgType: tlb.MsgTypeInternal, - Msg: &tlb.InternalMessage{ - SrcAddr: senderAddr, - DstAddr: otherAddr, - Amount: tlb.MustFromTON("1"), - }, - } - - results := ParseTonTransfer(tx, ourAddr.String(), "TON_MAINNET") - assert.Len(t, results, 0) - }) - - t.Run("parse comment", func(t *testing.T) { - comment := "hello world" - body := cell.BeginCell(). - MustStoreUInt(0, 32). // Opcode 0 for text comment - MustStoreStringSnake(comment). - EndCell() - - tx := &tlb.Transaction{ - LT: 1000, - } - tx.IO.In = &tlb.Message{ - MsgType: tlb.MsgTypeInternal, - Msg: &tlb.InternalMessage{ - SrcAddr: senderAddr, - DstAddr: ourAddr, - Amount: tlb.MustFromTON("0.00000001"), - Body: body, - }, - } - - results := ParseTonTransfer(tx, ourAddr.String(), "TON_MAINNET") - assert.Len(t, results, 1) - }) -} diff --git a/internal/indexer/ton/jetton_registry.go b/internal/indexer/ton/registry.go similarity index 98% rename from internal/indexer/ton/jetton_registry.go rename to internal/indexer/ton/registry.go index d3e4c8e..e504a53 100644 --- a/internal/indexer/ton/jetton_registry.go +++ b/internal/indexer/ton/registry.go @@ -93,11 +93,6 @@ func (r *ConfigBasedRegistry) List() []JettonInfo { return result } -// Reload is a no-op for config-backed registry. -func (r *ConfigBasedRegistry) Reload(_ context.Context) error { - return nil -} - // RedisJettonRegistry loads supported jettons from Redis and keeps an in-memory snapshot. // Key format: // - ton/jettons//masters: JSON []JettonInfo diff --git a/internal/rpc/ton/api.go b/internal/rpc/ton/api.go index f43b217..5987aab 100644 --- a/internal/rpc/ton/api.go +++ b/internal/rpc/ton/api.go @@ -8,10 +8,6 @@ import ( ) type TonAPI interface { - // GetAccountState returns the current state of an account. - // Returns nil Account.State if account doesn't exist (not deployed). - GetAccountState(ctx context.Context, addr *address.Address) (*tlb.Account, error) - // ListTransactions returns transactions for an account. // - limit: max transactions to return (typically 10-50) // - lastLT: logical time cursor (0 for initial fetch from beginning) diff --git a/internal/rpc/ton/client.go b/internal/rpc/ton/client.go index 6e06841..e9fda4d 100644 --- a/internal/rpc/ton/client.go +++ b/internal/rpc/ton/client.go @@ -75,23 +75,6 @@ func (c *Client) getMasterchainInfo(ctx context.Context) (*ton.BlockIDExt, error return master, nil } -// GetAccountState returns the current state of an account. -func (c *Client) GetAccountState(ctx context.Context, addr *address.Address) (*tlb.Account, error) { - // Get the current masterchain block for consistency - master, err := c.getMasterchainInfo(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get masterchain info: %w", err) - } - - // Get the account state at the current block - account, err := c.api.GetAccount(ctx, master, addr) - if err != nil { - return nil, fmt.Errorf("failed to get account state: %w", err) - } - - return account, nil -} - // ListTransactions returns transactions for an account. // Transactions are returned in reverse chronological order (newest first). // Use lastLT=0 and lastHash=nil for initial fetch from the account's latest transaction. diff --git a/internal/worker/factory.go b/internal/worker/factory.go index f2bbb71..17feecf 100644 --- a/internal/worker/factory.go +++ b/internal/worker/factory.go @@ -299,7 +299,6 @@ func buildTonPollingWorker( chainCfg config.ChainConfig, kvstore infra.KVStore, redisClient infra.RedisClient, - pubkeyStore pubkeystore.Store, db *gorm.DB, emitter events.Emitter, ) Worker { @@ -356,8 +355,8 @@ func buildTonPollingWorker( chainCfg, accountIndexer, cursorStore, - pubkeyStore, db, + kvstore, emitter, tonWorker.WorkerConfig{ Concurrency: chainCfg.Throttle.Concurrency, @@ -407,7 +406,7 @@ func CreateManagerWithWorkers( case enum.NetworkTypeSui: idxr = buildSuiIndexer(chainName, chainCfg, ModeRegular, pubkeyStore) case enum.NetworkTypeTon: - tonW := buildTonPollingWorker(ctx, chainName, chainCfg, kvstore, redisClient, pubkeyStore, db, emitter) + tonW := buildTonPollingWorker(ctx, chainName, chainCfg, kvstore, redisClient, db, emitter) if tonW != nil { manager.AddWorkers(tonW) logger.Info("TON polling worker enabled", "chain", chainName) diff --git a/internal/worker/ton/worker.go b/internal/worker/ton/worker.go index f4ea27c..8f32fea 100644 --- a/internal/worker/ton/worker.go +++ b/internal/worker/ton/worker.go @@ -2,6 +2,7 @@ package ton import ( "context" + "fmt" "log/slog" "sync" "time" @@ -12,12 +13,14 @@ import ( "github.com/fystack/multichain-indexer/pkg/common/logger" "github.com/fystack/multichain-indexer/pkg/common/types" "github.com/fystack/multichain-indexer/pkg/events" + "github.com/fystack/multichain-indexer/pkg/infra" "github.com/fystack/multichain-indexer/pkg/model" "github.com/fystack/multichain-indexer/pkg/retry" - "github.com/fystack/multichain-indexer/pkg/store/pubkeystore" "gorm.io/gorm" ) +const walletListKeyFormat = "ton/wallets/%s" + // TonPollingWorker polls multiple TON accounts for transactions. type TonPollingWorker struct { ctx context.Context @@ -29,15 +32,14 @@ type TonPollingWorker struct { config config.ChainConfig indexer tonIndexer.AccountIndexer cursorStore tonIndexer.CursorStore - pubkeyStore pubkeystore.Store db *gorm.DB + kvstore infra.KVStore emitter events.Emitter // Cache - wallets []string - walletsMutex sync.RWMutex - lastSyncTime time.Time - syncInterval time.Duration + wallets []string + walletsInitialized bool + walletsMutex sync.RWMutex // Worker configuration concurrency int @@ -57,8 +59,8 @@ func NewTonPollingWorker( cfg config.ChainConfig, indexer tonIndexer.AccountIndexer, cursorStore tonIndexer.CursorStore, - pubkeyStore pubkeystore.Store, db *gorm.DB, + kvstore infra.KVStore, emitter events.Emitter, workerCfg WorkerConfig, ) *TonPollingWorker { @@ -87,12 +89,11 @@ func NewTonPollingWorker( config: cfg, indexer: indexer, cursorStore: cursorStore, - pubkeyStore: pubkeyStore, db: db, + kvstore: kvstore, emitter: emitter, concurrency: concurrency, pollInterval: pollInterval, - syncInterval: time.Minute, // Sync from DB every minute } } @@ -136,47 +137,51 @@ func (w *TonPollingWorker) run() { } } -// pollAllAccounts polls all tracked TON addresses from the database. +// pollAllAccounts polls all tracked TON addresses from in-memory cache. func (w *TonPollingWorker) pollAllAccounts() { - // Sync addresses from DB if needed - w.syncWalletsIfNeeded() + if err := w.ensureWalletsLoaded(); err != nil { + w.logger.Error("Failed to ensure wallet list", "err", err) + return + } - w.walletsMutex.RLock() - addresses := make([]string, len(w.wallets)) - copy(addresses, w.wallets) - w.walletsMutex.RUnlock() + addresses := w.snapshotWallets() if len(addresses) == 0 { - // Only log periodically if needed, or debug w.logger.Debug("No TON addresses to poll") return } w.logger.Info("Starting poll cycle", "address_count", len(addresses)) - // Create work channel workChan := make(chan string, len(addresses)) for _, addr := range addresses { workChan <- addr } close(workChan) - // Start worker goroutines var wg sync.WaitGroup for i := 0; i < w.concurrency; i++ { wg.Add(1) - go func(workerID int) { + go func() { defer wg.Done() - w.pollWorker(workerID, workChan) - }(i) + w.pollWorker(workChan) + }() } wg.Wait() w.logger.Debug("Poll cycle complete", "address_count", len(addresses)) } +func (w *TonPollingWorker) snapshotWallets() []string { + w.walletsMutex.RLock() + defer w.walletsMutex.RUnlock() + out := make([]string, len(w.wallets)) + copy(out, w.wallets) + return out +} + // pollWorker processes addresses from the work channel. -func (w *TonPollingWorker) pollWorker(workerID int, addresses <-chan string) { +func (w *TonPollingWorker) pollWorker(addresses <-chan string) { for addr := range addresses { select { case <-w.ctx.Done(): @@ -191,73 +196,39 @@ func (w *TonPollingWorker) pollWorker(workerID int, addresses <-chan string) { func (w *TonPollingWorker) pollAccount(address string) { log := w.logger.With("address", address) - // Get existing cursor cursor, err := w.cursorStore.Get(w.ctx, address) if err != nil { log.Error("Failed to get cursor", "err", err) return } - - // If no cursor exists, initialize one if cursor == nil { cursor = &tonIndexer.AccountCursor{Address: address} } - // Check context before starting if w.ctx.Err() != nil { return } - // Poll with retry - var txs []types.Transaction - var newCursor *tonIndexer.AccountCursor - - err = retry.Exponential(func() error { - // Check context cancellation - don't retry on shutdown - if w.ctx.Err() != nil { - return nil // Return nil to exit retry loop cleanly - } - - result, c, pollErr := w.indexer.PollAccount(w.ctx, address, cursor) - if pollErr != nil { - // Check if error is due to context cancellation - if w.ctx.Err() != nil { - return nil // Exit retry cleanly on shutdown - } - return pollErr - } - txs = result - newCursor = c - return nil - }, retry.ExponentialConfig{ - InitialInterval: 2 * time.Second, - MaxElapsedTime: 30 * time.Second, - OnRetry: func(err error, next time.Duration) { - // Only log if context is still active - if w.ctx.Err() == nil { - log.Debug("Retrying poll", "err", err, "next_retry", next) - } - }, - }) - - // Exit if context was cancelled + txs, newCursor, err := w.pollAccountWithRetry(address, cursor, log) + if err != nil { + log.Error("Failed to poll account after retries", "err", err) + return + } if w.ctx.Err() != nil { return } - if err != nil { - log.Error("Failed to poll account after retries", "err", err) - return + if newCursor != nil { + if err := w.cursorStore.Save(w.ctx, newCursor); err != nil { + log.Error("Failed to save cursor", "err", err) + } } - // Process transactions if len(txs) == 0 { return } log.Info("Found transactions", "count", len(txs)) - - // Emit each transaction to NATS for i := range txs { tx := &txs[i] log.Info("Emitting matched transaction", @@ -274,20 +245,58 @@ func (w *TonPollingWorker) pollAccount(address string) { log.Debug("Emitted transaction", "tx_hash", tx.TxHash) } } +} - // Update cursor after successful processing - if newCursor != nil { - if err := w.cursorStore.Save(w.ctx, newCursor); err != nil { - log.Error("Failed to save cursor", "err", err) +func (w *TonPollingWorker) pollAccountWithRetry( + address string, + cursor *tonIndexer.AccountCursor, + log *slog.Logger, +) ([]types.Transaction, *tonIndexer.AccountCursor, error) { + var ( + txs []types.Transaction + newCursor *tonIndexer.AccountCursor + ) + + err := retry.Exponential(func() error { + if w.ctx.Err() != nil { + return nil } + + result, c, pollErr := w.indexer.PollAccount(w.ctx, address, cursor) + if pollErr != nil { + if w.ctx.Err() != nil { + return nil + } + return pollErr + } + txs = result + newCursor = c + return nil + }, retry.ExponentialConfig{ + InitialInterval: 2 * time.Second, + MaxElapsedTime: 30 * time.Second, + OnRetry: func(err error, next time.Duration) { + if w.ctx.Err() == nil { + log.Debug("Retrying poll", "err", err, "next_retry", next) + } + }, + }) + if w.ctx.Err() != nil { + return nil, nil, nil } + return txs, newCursor, err } // AddAddress registers a new address for tracking. // This initializes the cursor and starts polling the address. func (w *TonPollingWorker) AddAddress(ctx context.Context, address string) error { + normalizedAddress := tonIndexer.NormalizeTONAddressRaw(address) + if normalizedAddress == "" { + return fmt.Errorf("invalid TON address: %s", address) + } + // Check if cursor already exists - existing, err := w.cursorStore.Get(ctx, address) + existing, err := w.cursorStore.Get(ctx, normalizedAddress) if err != nil { return err } @@ -298,12 +307,16 @@ func (w *TonPollingWorker) AddAddress(ctx context.Context, address string) error } // Create initial cursor - cursor := &tonIndexer.AccountCursor{Address: address} + cursor := &tonIndexer.AccountCursor{Address: normalizedAddress} return w.cursorStore.Save(ctx, cursor) } // RemoveAddress stops tracking an address. func (w *TonPollingWorker) RemoveAddress(ctx context.Context, address string) error { + normalizedAddress := tonIndexer.NormalizeTONAddressRaw(address) + if normalizedAddress != "" { + return w.cursorStore.Delete(ctx, normalizedAddress) + } return w.cursorStore.Delete(ctx, address) } @@ -313,95 +326,134 @@ func (w *TonPollingWorker) GetNetworkType() enum.NetworkType { } func (w *TonPollingWorker) GetName() string { - if w.chainName != "" { - return w.chainName + return w.chainName +} + +func (w *TonPollingWorker) walletListKey() string { + return fmt.Sprintf(walletListKeyFormat, w.chainName) +} + +func (w *TonPollingWorker) ensureWalletsLoaded() error { + w.walletsMutex.RLock() + initialized := w.walletsInitialized + w.walletsMutex.RUnlock() + + if initialized { + return nil } - return w.config.Name + + if _, err := w.ReloadWalletsFromKV(w.ctx); err != nil { + return err + } + + w.walletsMutex.RLock() + hasWallets := len(w.wallets) > 0 + w.walletsMutex.RUnlock() + if hasWallets { + return nil + } + + // KV empty fallback: sync once from DB to bootstrap. + if w.db == nil { + return nil + } + _, err := w.ReloadWalletsFromDB(w.ctx) + return err } -func (w *TonPollingWorker) ReloadJettons(ctx context.Context) (int, error) { - count, err := w.indexer.ReloadJettons(ctx) +func (w *TonPollingWorker) replaceWalletCache(addresses []string) int { + w.walletsMutex.Lock() + oldSize := len(w.wallets) + w.wallets = addresses + w.walletsInitialized = true + w.walletsMutex.Unlock() + + w.logger.Info("Wallet cache updated", + "old_size", oldSize, + "new_size", len(addresses), + ) + return len(addresses) +} + +// ReloadWalletsFromKV refreshes in-memory wallets from KV store. +func (w *TonPollingWorker) ReloadWalletsFromKV(_ context.Context) (int, error) { + if w.kvstore == nil { + return 0, fmt.Errorf("kvstore is not configured") + } + + var addresses []string + found, err := w.kvstore.GetAny(w.walletListKey(), &addresses) if err != nil { - return 0, err + return 0, fmt.Errorf("get wallet list from kv: %w", err) } - w.logger.Info("Jetton registry reloaded", "chain", w.GetName(), "jetton_count", count) - return count, nil + if !found { + w.replaceWalletCache(nil) + return 0, nil + } + + return w.replaceWalletCache(tonIndexer.NormalizeTONAddressList(addresses)), nil } -// syncWalletsIfNeeded refreshes the wallet list from DB if sync interval passed. -// Uses pagination to efficiently handle large numbers of wallets (>10K). -// Before fetching everything, it checks the count to avoid redundant work. -func (w *TonPollingWorker) syncWalletsIfNeeded() { - if time.Since(w.lastSyncTime) < w.syncInterval { - return +// ReloadWalletsFromDB reloads wallets from database and persists them to KV store. +func (w *TonPollingWorker) ReloadWalletsFromDB(_ context.Context) (int, error) { + if w.db == nil { + return 0, fmt.Errorf("database is not configured") + } + if w.kvstore == nil { + return 0, fmt.Errorf("kvstore is not configured") } - // 1. Quickly check the count in DB - var dbCount int64 - err := w.db.Model(&model.WalletAddress{}). - Where("type = ?", enum.NetworkTypeTon). - Count(&dbCount).Error + allAddresses, err := w.loadWalletAddressesFromDB() if err != nil { - w.logger.Error("Failed to count wallets in DB", "err", err) - return + return 0, err } - w.walletsMutex.RLock() - cachedCount := int64(len(w.wallets)) - w.walletsMutex.RUnlock() + cleaned := tonIndexer.NormalizeTONAddressList(allAddresses) + if err := w.kvstore.SetAny(w.walletListKey(), cleaned); err != nil { + return 0, fmt.Errorf("persist wallet list to kv: %w", err) + } - // If count matches and we're not empty, skip the expensive full fetch - if dbCount == cachedCount && cachedCount > 0 { - w.lastSyncTime = time.Now() - return + return w.replaceWalletCache(cleaned), nil +} + +// ReloadJettons refreshes TON jetton registry from Redis (with config fallback). +func (w *TonPollingWorker) ReloadJettons(ctx context.Context) (int, error) { + count, err := w.indexer.ReloadJettons(ctx) + if err != nil { + return 0, err } - // 2. Count changed or list is empty, perform full sync + w.logger.Info("Jetton registry reloaded", "chain", w.chainName, "jetton_count", count) + return count, nil +} + +func (w *TonPollingWorker) loadWalletAddressesFromDB() ([]string, error) { const batchSize = 1000 - var allAddresses []string - offset := 0 + allAddresses := make([]string, 0, batchSize) + offset := 0 for { var wallets []model.WalletAddress - // Paginated query for TON wallets only selecting address field err := w.db.Select("address"). Where("type = ?", enum.NetworkTypeTon). Order("id"). Limit(batchSize). Offset(offset). Find(&wallets).Error - if err != nil { - w.logger.Error("Failed to fetch wallets from DB", "err", err, "offset", offset) - return + return nil, fmt.Errorf("fetch wallets from db (offset=%d): %w", offset, err) } - - // No more results if len(wallets) == 0 { break } - - // Collect addresses for _, wallet := range wallets { allAddresses = append(allAddresses, wallet.Address) } - - // If we got fewer than batchSize, we're done if len(wallets) < batchSize { break } - offset += batchSize } - w.walletsMutex.Lock() - oldSize := len(w.wallets) - w.wallets = allAddresses - w.lastSyncTime = time.Now() - w.walletsMutex.Unlock() - - w.logger.Info("Synced wallets from DB", - "old_size", oldSize, - "new_size", len(allAddresses), - ) + return allAddresses, nil } diff --git a/pkg/common/config/chains.go b/pkg/common/config/chains.go index a04b044..572ac14 100644 --- a/pkg/common/config/chains.go +++ b/pkg/common/config/chains.go @@ -47,6 +47,9 @@ func (c Chains) OverrideFromLatest(names []string) { // ApplyDefaults merges global defaults into all chain configs. func (c Chains) ApplyDefaults(def Defaults) error { for name, chain := range c { + if !chain.FromLatest { + chain.FromLatest = def.FromLatest + } if chain.PollInterval == 0 { chain.PollInterval = def.PollInterval } diff --git a/pkg/common/config/types.go b/pkg/common/config/types.go index 6db4d03..29118d5 100644 --- a/pkg/common/config/types.go +++ b/pkg/common/config/types.go @@ -37,7 +37,7 @@ type ChainConfig struct { Name string `yaml:"-"` NetworkId string `yaml:"network_id"` InternalCode string `yaml:"internal_code"` - Type enum.NetworkType `yaml:"type" validate:"required,oneof=tron evm btc sol sui ton"` + Type enum.NetworkType `yaml:"type" validate:"required"` FromLatest bool `yaml:"from_latest"` StartBlock int `yaml:"start_block" validate:"min=0"` PollInterval time.Duration `yaml:"poll_interval"` diff --git a/pkg/common/types/types.go b/pkg/common/types/types.go index 00604c7..4fc842c 100644 --- a/pkg/common/types/types.go +++ b/pkg/common/types/types.go @@ -20,18 +20,20 @@ type Block struct { } type Transaction struct { - TxHash string `json:"txHash"` - NetworkId string `json:"networkId"` - BlockNumber uint64 `json:"blockNumber"` // 0 for mempool transactions - FromAddress string `json:"fromAddress"` - ToAddress string `json:"toAddress"` - AssetAddress string `json:"assetAddress"` - Amount string `json:"amount"` - Type constant.TxType `json:"type"` - TxFee decimal.Decimal `json:"txFee"` - Timestamp uint64 `json:"timestamp"` - Confirmations uint64 `json:"confirmations"` // Number of confirmations (0 = mempool/unconfirmed) - Status string `json:"status"` // "pending" (0 conf), "confirmed" (1+ conf) + TxHash string `json:"txHash"` + NetworkId string `json:"networkId"` + BlockNumber uint64 `json:"blockNumber"` // 0 for mempool transactions + LogicalTime uint64 `json:"logicalTime,omitempty"` + MasterchainSeqno uint64 `json:"masterchainSeqno,omitempty"` + FromAddress string `json:"fromAddress"` + ToAddress string `json:"toAddress"` + AssetAddress string `json:"assetAddress"` + Amount string `json:"amount"` + Type constant.TxType `json:"type"` + TxFee decimal.Decimal `json:"txFee"` + Timestamp uint64 `json:"timestamp"` + Confirmations uint64 `json:"confirmations"` // Number of confirmations (0 = mempool/unconfirmed) + Status string `json:"status"` // "pending" (0 conf), "confirmed" (1+ conf) } func (t Transaction) MarshalBinary() ([]byte, error) { @@ -48,10 +50,12 @@ func (t *Transaction) UnmarshalBinary(data []byte) error { func (t Transaction) String() string { return fmt.Sprintf( - "{TxHash: %s, NetworkId: %s, BlockNumber: %d, FromAddress: %s, ToAddress: %s, AssetAddress: %s, Amount: %s, Type: %s, TxFee: %s, Timestamp: %d, Confirmations: %d, Status: %s}", + "{TxHash: %s, NetworkId: %s, BlockNumber: %d, LogicalTime: %d, MasterchainSeqno: %d, FromAddress: %s, ToAddress: %s, AssetAddress: %s, Amount: %s, Type: %s, TxFee: %s, Timestamp: %d, Confirmations: %d, Status: %s}", t.TxHash, t.NetworkId, t.BlockNumber, + t.LogicalTime, + t.MasterchainSeqno, t.FromAddress, t.ToAddress, t.AssetAddress, From 17ffef72701707e65dd9dee9fc9c85d18402cd74 Mon Sep 17 00:00:00 2001 From: vietddude Date: Wed, 11 Feb 2026 15:19:37 +0700 Subject: [PATCH 04/11] feat: add runtime TON wallet reload API with KV/DB source options --- cmd/indexer/main.go | 67 ++++++++- internal/worker/ton_jetton_reload.go | 90 ------------ internal/worker/ton_reload.go | 203 +++++++++++++++++++++++++++ 3 files changed, 269 insertions(+), 91 deletions(-) delete mode 100644 internal/worker/ton_jetton_reload.go create mode 100644 internal/worker/ton_reload.go diff --git a/cmd/indexer/main.go b/cmd/indexer/main.go index 930d06f..830545c 100644 --- a/cmd/indexer/main.go +++ b/cmd/indexer/main.go @@ -9,6 +9,7 @@ import ( "net/http" "os" "os/signal" + "strings" "syscall" "time" @@ -200,9 +201,10 @@ func runIndexer(chains []string, configPath string, debug, manual, catchup, from redisClient, managerCfg, ) + tonWalletReloadService := worker.NewTonWalletReloadServiceFromManager(manager) tonJettonReloadService := worker.NewTonJettonReloadServiceFromManager(manager) - healthServer := startHealthServer(cfg.Services.Port, cfg, tonJettonReloadService) + healthServer := startHealthServer(cfg.Services.Port, cfg, tonWalletReloadService, tonJettonReloadService) // Start all workers logger.Info("Starting all workers") @@ -240,6 +242,15 @@ type APIErrorResponse struct { Timestamp time.Time `json:"timestamp"` } +type TonWalletReloadResponse struct { + Status string `json:"status"` + Source worker.WalletReloadSource `json:"source"` + Chain string `json:"chain,omitempty"` + Results []worker.TonWalletReloadResult `json:"results"` + TriggeredAtUTC time.Time `json:"triggered_at_utc"` + SupportedSources []worker.WalletReloadSource `json:"supported_sources"` +} + type TonJettonReloadResponse struct { Status string `json:"status"` Chain string `json:"chain,omitempty"` @@ -250,6 +261,7 @@ type TonJettonReloadResponse struct { func startHealthServer( port int, cfg *config.Config, + tonWalletReloadService *worker.TonWalletReloadService, tonJettonReloadService *worker.TonJettonReloadService, ) *http.Server { mux := http.NewServeMux() @@ -268,6 +280,49 @@ func startHealthServer( writeJSON(w, http.StatusOK, response) }) + mux.HandleFunc("/ton/wallets/reload", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + writeErrorJSON(w, http.StatusMethodNotAllowed, "method not allowed") + return + } + + if tonWalletReloadService == nil { + writeErrorJSON(w, http.StatusNotFound, worker.ErrNoTonWorkerConfigured.Error()) + return + } + + source := worker.WalletReloadSource(r.URL.Query().Get("source")).Normalize() + if !source.IsValid() { + writeErrorJSON(w, http.StatusBadRequest, "invalid source (supported: kv, db)") + return + } + + req := worker.TonWalletReloadRequest{ + Source: source, + ChainFilter: strings.TrimSpace(r.URL.Query().Get("chain")), + } + + results, err := tonWalletReloadService.ReloadTonWallets(r.Context(), req) + if err != nil { + statusCode := http.StatusInternalServerError + if errors.Is(err, worker.ErrTonWorkerNotFound) || errors.Is(err, worker.ErrNoTonWorkerConfigured) { + statusCode = http.StatusNotFound + } + writeErrorJSON(w, statusCode, err.Error()) + return + } + + response := TonWalletReloadResponse{ + Status: reloadWalletStatus(results), + Source: source, + Chain: req.ChainFilter, + Results: results, + TriggeredAtUTC: time.Now().UTC(), + SupportedSources: []worker.WalletReloadSource{worker.WalletReloadSourceKV, worker.WalletReloadSourceDB}, + } + writeJSON(w, http.StatusOK, response) + }) + mux.HandleFunc("/ton/jettons/reload", func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { writeErrorJSON(w, http.StatusMethodNotAllowed, "method not allowed") @@ -312,6 +367,7 @@ func startHealthServer( "Health check server started", "port", port, "health_endpoint", "/health", + "wallet_reload_endpoint", "/ton/wallets/reload", "jetton_reload_endpoint", "/ton/jettons/reload", ) if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { @@ -322,6 +378,15 @@ func startHealthServer( return server } +func reloadWalletStatus(results []worker.TonWalletReloadResult) string { + for _, result := range results { + if result.Error != "" { + return "partial_error" + } + } + return "ok" +} + func reloadJettonStatus(results []worker.TonJettonReloadResult) string { for _, result := range results { if result.Error != "" { diff --git a/internal/worker/ton_jetton_reload.go b/internal/worker/ton_jetton_reload.go deleted file mode 100644 index 6efbd6f..0000000 --- a/internal/worker/ton_jetton_reload.go +++ /dev/null @@ -1,90 +0,0 @@ -package worker - -import ( - "context" - "errors" - "fmt" - "sort" - - "github.com/fystack/multichain-indexer/pkg/common/enum" -) - -var ( - ErrNoTonWorkerConfigured = errors.New("no ton worker configured") - ErrTonWorkerNotFound = errors.New("ton worker not found") -) - -type TonJettonReloadRequest struct { - ChainFilter string -} - -type TonJettonReloadResult struct { - Chain string `json:"chain"` - ReloadedJettons int `json:"reloaded_jettons"` - Error string `json:"error,omitempty"` -} - -// TonJettonReloader is implemented by TON workers that support runtime jetton reload. -type TonJettonReloader interface { - Worker - GetName() string - GetNetworkType() enum.NetworkType - ReloadJettons(ctx context.Context) (int, error) -} - -// TonJettonReloadService handles runtime jetton registry reload for TON workers. -type TonJettonReloadService struct { - reloaders []TonJettonReloader -} - -func NewTonJettonReloadService(workers []Worker) *TonJettonReloadService { - reloaders := make([]TonJettonReloader, 0) - for _, w := range workers { - reloader, ok := w.(TonJettonReloader) - if !ok || reloader.GetNetworkType() != enum.NetworkTypeTon { - continue - } - reloaders = append(reloaders, reloader) - } - return &TonJettonReloadService{reloaders: reloaders} -} - -func NewTonJettonReloadServiceFromManager(m *Manager) *TonJettonReloadService { - if m == nil { - return &TonJettonReloadService{} - } - return NewTonJettonReloadService(m.Workers()) -} - -func (s *TonJettonReloadService) ReloadTonJettons( - ctx context.Context, - req TonJettonReloadRequest, -) ([]TonJettonReloadResult, error) { - results := make([]TonJettonReloadResult, 0) - - for _, reloader := range s.reloaders { - chainName := reloader.GetName() - if req.ChainFilter != "" && req.ChainFilter != chainName { - continue - } - - item := TonJettonReloadResult{Chain: chainName} - count, err := reloader.ReloadJettons(ctx) - if err != nil { - item.Error = err.Error() - } else { - item.ReloadedJettons = count - } - results = append(results, item) - } - - if len(results) == 0 { - if req.ChainFilter != "" { - return nil, fmt.Errorf("%w: %s", ErrTonWorkerNotFound, req.ChainFilter) - } - return nil, ErrNoTonWorkerConfigured - } - - sort.Slice(results, func(i, j int) bool { return results[i].Chain < results[j].Chain }) - return results, nil -} diff --git a/internal/worker/ton_reload.go b/internal/worker/ton_reload.go new file mode 100644 index 0000000..8500ad0 --- /dev/null +++ b/internal/worker/ton_reload.go @@ -0,0 +1,203 @@ +package worker + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + + "github.com/fystack/multichain-indexer/pkg/common/enum" +) + +var ( + ErrNoTonWorkerConfigured = errors.New("no ton worker configured") + ErrTonWorkerNotFound = errors.New("ton worker not found") +) + +type TonJettonReloadRequest struct { + ChainFilter string +} + +type WalletReloadSource string + +const ( + WalletReloadSourceKV WalletReloadSource = "kv" + WalletReloadSourceDB WalletReloadSource = "db" +) + +func (s WalletReloadSource) Normalize() WalletReloadSource { + switch strings.ToLower(strings.TrimSpace(string(s))) { + case string(WalletReloadSourceDB): + return WalletReloadSourceDB + case string(WalletReloadSourceKV), "": + return WalletReloadSourceKV + default: + return WalletReloadSource(strings.ToLower(strings.TrimSpace(string(s)))) + } +} + +func (s WalletReloadSource) IsValid() bool { + normalized := s.Normalize() + return normalized == WalletReloadSourceKV || normalized == WalletReloadSourceDB +} + +type TonWalletReloadRequest struct { + Source WalletReloadSource + ChainFilter string +} + +type TonWalletReloadResult struct { + Chain string `json:"chain"` + ReloadedWallets int `json:"reloaded_wallets"` + Error string `json:"error,omitempty"` +} + +type TonJettonReloadResult struct { + Chain string `json:"chain"` + ReloadedJettons int `json:"reloaded_jettons"` + Error string `json:"error,omitempty"` +} + +// TonWalletReloader is implemented by TON workers that support runtime wallet reload. +type TonWalletReloader interface { + Worker + GetName() string + GetNetworkType() enum.NetworkType + ReloadWalletsFromKV(ctx context.Context) (int, error) + ReloadWalletsFromDB(ctx context.Context) (int, error) +} + +// TonJettonReloader is implemented by TON workers that support runtime jetton reload. +type TonJettonReloader interface { + Worker + GetName() string + GetNetworkType() enum.NetworkType + ReloadJettons(ctx context.Context) (int, error) +} + +// TonWalletReloadService handles runtime wallet cache reload for TON workers. +type TonWalletReloadService struct { + reloaders []TonWalletReloader +} + +// TonJettonReloadService handles runtime jetton registry reload for TON workers. +type TonJettonReloadService struct { + reloaders []TonJettonReloader +} + +func NewTonWalletReloadService(workers []Worker) *TonWalletReloadService { + reloaders := make([]TonWalletReloader, 0) + for _, w := range workers { + reloader, ok := w.(TonWalletReloader) + if !ok || reloader.GetNetworkType() != enum.NetworkTypeTon { + continue + } + reloaders = append(reloaders, reloader) + } + + return &TonWalletReloadService{reloaders: reloaders} +} + +func NewTonWalletReloadServiceFromManager(m *Manager) *TonWalletReloadService { + if m == nil { + return &TonWalletReloadService{} + } + return NewTonWalletReloadService(m.Workers()) +} + +func NewTonJettonReloadService(workers []Worker) *TonJettonReloadService { + reloaders := make([]TonJettonReloader, 0) + for _, w := range workers { + reloader, ok := w.(TonJettonReloader) + if !ok || reloader.GetNetworkType() != enum.NetworkTypeTon { + continue + } + reloaders = append(reloaders, reloader) + } + return &TonJettonReloadService{reloaders: reloaders} +} + +func NewTonJettonReloadServiceFromManager(m *Manager) *TonJettonReloadService { + if m == nil { + return &TonJettonReloadService{} + } + return NewTonJettonReloadService(m.Workers()) +} + +func (s *TonWalletReloadService) ReloadTonWallets( + ctx context.Context, + req TonWalletReloadRequest, +) ([]TonWalletReloadResult, error) { + source := req.Source.Normalize() + results := make([]TonWalletReloadResult, 0) + + for _, reloader := range s.reloaders { + chainName := reloader.GetName() + if req.ChainFilter != "" && req.ChainFilter != chainName { + continue + } + + item := TonWalletReloadResult{Chain: chainName} + var ( + count int + err error + ) + switch source { + case WalletReloadSourceDB: + count, err = reloader.ReloadWalletsFromDB(ctx) + default: + count, err = reloader.ReloadWalletsFromKV(ctx) + } + + if err != nil { + item.Error = err.Error() + } else { + item.ReloadedWallets = count + } + results = append(results, item) + } + + if len(results) == 0 { + if req.ChainFilter != "" { + return nil, fmt.Errorf("%w: %s", ErrTonWorkerNotFound, req.ChainFilter) + } + return nil, ErrNoTonWorkerConfigured + } + + sort.Slice(results, func(i, j int) bool { return results[i].Chain < results[j].Chain }) + return results, nil +} + +func (s *TonJettonReloadService) ReloadTonJettons( + ctx context.Context, + req TonJettonReloadRequest, +) ([]TonJettonReloadResult, error) { + results := make([]TonJettonReloadResult, 0) + + for _, reloader := range s.reloaders { + chainName := reloader.GetName() + if req.ChainFilter != "" && req.ChainFilter != chainName { + continue + } + + item := TonJettonReloadResult{Chain: chainName} + count, err := reloader.ReloadJettons(ctx) + if err != nil { + item.Error = err.Error() + } else { + item.ReloadedJettons = count + } + results = append(results, item) + } + + if len(results) == 0 { + if req.ChainFilter != "" { + return nil, fmt.Errorf("%w: %s", ErrTonWorkerNotFound, req.ChainFilter) + } + return nil, ErrNoTonWorkerConfigured + } + + sort.Slice(results, func(i, j int) bool { return results[i].Chain < results[j].Chain }) + return results, nil +} From 283c734666492053fa61791cbeb4d521d439d43c Mon Sep 17 00:00:00 2001 From: vietddude Date: Thu, 12 Feb 2026 11:28:56 +0700 Subject: [PATCH 05/11] fix: correct sui network id --- internal/indexer/sui.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/indexer/sui.go b/internal/indexer/sui.go index 87ec141..b53eb26 100644 --- a/internal/indexer/sui.go +++ b/internal/indexer/sui.go @@ -220,7 +220,7 @@ func (s *SuiIndexer) convertCheckpoint(cp *sui.Checkpoint) *types.Block { func (s *SuiIndexer) convertTransaction(execTx *v2.ExecutedTransaction, blockNumber, blockTs uint64) types.Transaction { t := types.Transaction{ TxHash: execTx.GetDigest(), - NetworkId: s.cfg.InternalCode, + NetworkId: s.cfg.NetworkId, BlockNumber: blockNumber, Timestamp: blockTs, } From 08798ac92b118f666fc9cb2867693bf5dfb3a33c Mon Sep 17 00:00:00 2001 From: vietddude Date: Thu, 12 Feb 2026 11:37:23 +0700 Subject: [PATCH 06/11] feat(ton): normalize tx fields and modularize TON indexer --- internal/indexer/ton/address.go | 83 ++++++ internal/indexer/ton/core.go | 464 +----------------------------- internal/indexer/ton/core_test.go | 62 ++++ internal/indexer/ton/parser.go | 320 +++++++++++++++++++++ pkg/store/toncursorstore/store.go | 84 ++++++ 5 files changed, 553 insertions(+), 460 deletions(-) create mode 100644 internal/indexer/ton/address.go create mode 100644 internal/indexer/ton/core_test.go create mode 100644 internal/indexer/ton/parser.go create mode 100644 pkg/store/toncursorstore/store.go diff --git a/internal/indexer/ton/address.go b/internal/indexer/ton/address.go new file mode 100644 index 0000000..a263f0a --- /dev/null +++ b/internal/indexer/ton/address.go @@ -0,0 +1,83 @@ +package ton + +import ( + "fmt" + "strings" + + "github.com/xssnick/tonutils-go/address" +) + +func resolvePollAddress(addrStr string) (*address.Address, string, error) { + addr, err := parseTONAddress(addrStr) + if err != nil { + return nil, "", fmt.Errorf("invalid TON address %s: %w", addrStr, err) + } + return addr, addr.StringRaw(), nil +} + +// NormalizeTONAddressRaw returns canonical raw format (workchain:hex) or empty if invalid. +func NormalizeTONAddressRaw(addr string) string { + parsed, err := parseTONAddress(addr) + if err != nil { + return "" + } + return parsed.StringRaw() +} + +// NormalizeTONAddressList canonicalizes to raw format, trims invalid inputs, and de-duplicates while preserving order. +func NormalizeTONAddressList(addresses []string) []string { + if len(addresses) == 0 { + return nil + } + + dedup := make(map[string]struct{}, len(addresses)) + result := make([]string, 0, len(addresses)) + for _, addr := range addresses { + normalized := NormalizeTONAddressRaw(addr) + if normalized == "" { + continue + } + if _, exists := dedup[normalized]; exists { + continue + } + dedup[normalized] = struct{}{} + result = append(result, normalized) + } + + return result +} + +func parseTONAddress(addrStr string) (*address.Address, error) { + addrStr = strings.TrimSpace(addrStr) + + // User-friendly format (base64url with checksum), e.g. EQ... + if addr, err := address.ParseAddr(addrStr); err == nil { + return addr, nil + } + // Raw format, e.g. 0:abcdef... + if addr, err := address.ParseRawAddr(addrStr); err == nil { + return addr, nil + } + + // Defensive normalization for malformed historical values. + parts := strings.SplitN(addrStr, ":", 2) + if len(parts) != 2 { + return nil, fmt.Errorf("invalid address format") + } + + rawHex := strings.TrimPrefix(strings.ToLower(strings.TrimSpace(parts[1])), "0x") + if rawHex == "" { + return nil, fmt.Errorf("empty address payload") + } + + if len(rawHex)%2 == 1 { + rawHex = "0" + rawHex + } + if len(rawHex) > 64 { + rawHex = rawHex[len(rawHex)-64:] + } else if len(rawHex) < 64 { + rawHex = strings.Repeat("0", 64-len(rawHex)) + rawHex + } + + return address.ParseRawAddr(parts[0] + ":" + rawHex) +} diff --git a/internal/indexer/ton/core.go b/internal/indexer/ton/core.go index f9d43cd..ce6bcb4 100644 --- a/internal/indexer/ton/core.go +++ b/internal/indexer/ton/core.go @@ -2,111 +2,23 @@ package ton import ( "context" - "encoding/base64" "encoding/hex" - "encoding/json" "fmt" - "strings" - "time" tonRpc "github.com/fystack/multichain-indexer/internal/rpc/ton" "github.com/fystack/multichain-indexer/pkg/common/config" - "github.com/fystack/multichain-indexer/pkg/common/constant" "github.com/fystack/multichain-indexer/pkg/common/enum" "github.com/fystack/multichain-indexer/pkg/common/types" "github.com/fystack/multichain-indexer/pkg/infra" - "github.com/shopspring/decimal" - "github.com/xssnick/tonutils-go/address" + tonCursorStore "github.com/fystack/multichain-indexer/pkg/store/toncursorstore" "github.com/xssnick/tonutils-go/tlb" ) -const ( - cursorKeyPrefix = "ton/cursor/" - - // Jetton standard opcodes (TEP-74) - OpcodeTransfer uint64 = 0x0f8a7ea5 - OpcodeTransferNotification uint64 = 0x7362d09c -) - -// AccountCursor tracks the polling position for a single TON account. -type AccountCursor struct { - Address string `json:"address"` - LastLT uint64 `json:"last_lt"` // Logical time of last processed tx - LastHash string `json:"last_hash"` // Hex-encoded hash of last processed tx - UpdatedAt time.Time `json:"updated_at"` -} - -// CursorStore manages account cursors for TON polling. -type CursorStore interface { - // Get returns the cursor for an account, or nil if not found. - Get(ctx context.Context, address string) (*AccountCursor, error) - - // Save persists the cursor atomically. - Save(ctx context.Context, cursor *AccountCursor) error - - // Delete removes the cursor for an account. - Delete(ctx context.Context, address string) error - - // List returns all tracked account addresses. - List(ctx context.Context) ([]string, error) -} - -// kvCursorStore implements CursorStore using infra.KVStore. -type kvCursorStore struct { - kv infra.KVStore -} +type AccountCursor = tonCursorStore.AccountCursor +type CursorStore = tonCursorStore.Store func NewCursorStore(kv infra.KVStore) CursorStore { - return &kvCursorStore{kv: kv} -} - -func (s *kvCursorStore) cursorKey(address string) string { - return cursorKeyPrefix + address -} - -func (s *kvCursorStore) Get(ctx context.Context, address string) (*AccountCursor, error) { - var cursor AccountCursor - found, err := s.kv.GetAny(s.cursorKey(address), &cursor) - if err != nil { - return nil, fmt.Errorf("failed to get cursor for %s: %w", address, err) - } - if !found { - return nil, nil - } - return &cursor, nil -} - -func (s *kvCursorStore) Save(ctx context.Context, cursor *AccountCursor) error { - cursor.UpdatedAt = time.Now() - if err := s.kv.SetAny(s.cursorKey(cursor.Address), cursor); err != nil { - return fmt.Errorf("failed to save cursor for %s: %w", cursor.Address, err) - } - return nil -} - -func (s *kvCursorStore) Delete(ctx context.Context, address string) error { - if err := s.kv.Delete(s.cursorKey(address)); err != nil { - return fmt.Errorf("failed to delete cursor for %s: %w", address, err) - } - return nil -} - -func (s *kvCursorStore) List(ctx context.Context) ([]string, error) { - pairs, err := s.kv.List(cursorKeyPrefix) - if err != nil { - return nil, fmt.Errorf("failed to list cursors: %w", err) - } - - addresses := make([]string, 0, len(pairs)) - for _, pair := range pairs { - var cursor AccountCursor - if err := json.Unmarshal(pair.Value, &cursor); err != nil { - continue // Skip malformed entries - } - addresses = append(addresses, cursor.Address) - } - - return addresses, nil + return tonCursorStore.New(kv) } // AccountIndexer is the interface for account-based indexing (TON). @@ -238,14 +150,6 @@ func (i *TonAccountIndexer) PollAccount(ctx context.Context, addrStr string, cur return parsedTxs, newCursor, nil } -func resolvePollAddress(addrStr string) (*address.Address, string, error) { - addr, err := parseTONAddress(addrStr) - if err != nil { - return nil, "", fmt.Errorf("invalid TON address %s: %w", addrStr, err) - } - return addr, addr.StringRaw(), nil -} - func decodeCursorForRPC(cursor *AccountCursor) (uint64, []byte, error) { if cursor == nil || cursor.LastLT == 0 { return 0, nil, nil @@ -280,17 +184,6 @@ func advanceCursor(cursor *AccountCursor, tx *tlb.Transaction) { cursor.LastHash = hex.EncodeToString(tx.Hash) } -func (i *TonAccountIndexer) parseMatchedTransactions( - tx *tlb.Transaction, - normalizedAddress string, -) []types.Transaction { - collected := parseTonTransferNormalized(tx, normalizedAddress, i.config.InternalCode) - if i.jettonRegistry != nil { - collected = append(collected, parseJettonTransferNormalized(tx, normalizedAddress, i.config.InternalCode, i.jettonRegistry)...) - } - return collected -} - // isTransactionSuccess checks if the transaction was successful by examining its phases. func isTransactionSuccess(tx *tlb.Transaction) bool { if tx.Description == nil { @@ -352,352 +245,3 @@ type JettonRegistry interface { // List returns all supported Jettons. List() []JettonInfo } - -// NormalizeTONAddressRaw returns canonical raw format (workchain:hex) or empty if invalid. -func NormalizeTONAddressRaw(addr string) string { - parsed, err := parseTONAddress(addr) - if err != nil { - return "" - } - return parsed.StringRaw() -} - -// NormalizeTONAddressList trims and de-duplicates addresses while preserving order. -func NormalizeTONAddressList(addresses []string) []string { - if len(addresses) == 0 { - return nil - } - - dedup := make(map[string]struct{}, len(addresses)) - result := make([]string, 0, len(addresses)) - for _, addr := range addresses { - addr = strings.TrimSpace(addr) - if addr == "" { - continue - } - if _, exists := dedup[addr]; exists { - continue - } - dedup[addr] = struct{}{} - result = append(result, addr) - } - - return result -} - -func parseTONAddress(addrStr string) (*address.Address, error) { - addrStr = strings.TrimSpace(addrStr) - - // User-friendly format (base64url with checksum), e.g. EQ... - if addr, err := address.ParseAddr(addrStr); err == nil { - return addr, nil - } - // Raw format, e.g. 0:abcdef... - if addr, err := address.ParseRawAddr(addrStr); err == nil { - return addr, nil - } - - // Defensive normalization for malformed historical values. - parts := strings.SplitN(addrStr, ":", 2) - if len(parts) != 2 { - return nil, fmt.Errorf("invalid address format") - } - - rawHex := strings.TrimPrefix(strings.ToLower(strings.TrimSpace(parts[1])), "0x") - if rawHex == "" { - return nil, fmt.Errorf("empty address payload") - } - - if len(rawHex)%2 == 1 { - rawHex = "0" + rawHex - } - if len(rawHex) > 64 { - rawHex = rawHex[len(rawHex)-64:] - } else if len(rawHex) < 64 { - rawHex = strings.Repeat("0", 64-len(rawHex)) + rawHex - } - - return address.ParseRawAddr(parts[0] + ":" + rawHex) -} - -// ParseTonTransfer extracts native TON transfers from a transaction. -// Detects both incoming (receive) and outgoing (send) transfers. -func ParseTonTransfer(tx *tlb.Transaction, ourAddress string, networkID string) []types.Transaction { - normalizedOurAddress := NormalizeTONAddressRaw(ourAddress) - if normalizedOurAddress == "" { - return nil - } - return parseTonTransferNormalized(tx, normalizedOurAddress, networkID) -} - -func parseTonTransferNormalized(tx *tlb.Transaction, normalizedOurAddress string, networkID string) []types.Transaction { - var results []types.Transaction - - if parsed, ok := parseIncomingTransfer(tx, normalizedOurAddress, networkID); ok { - results = append(results, *parsed) - } - - for _, intMsg := range outgoingInternalMessages(tx) { - parsed, ok := parseOutgoingTransferMessage(tx, intMsg, normalizedOurAddress, networkID) - if !ok { - continue - } - results = append(results, parsed) - } - - return results -} - -func parseIncomingTransfer(tx *tlb.Transaction, ourAddress string, networkID string) (*types.Transaction, bool) { - intMsg, ok := incomingInternalMessage(tx) - if !ok { - return nil, false - } - - dstAddrRaw := intMsg.DstAddr.StringRaw() - if dstAddrRaw != ourAddress || !isSimpleTransferMessage(intMsg) { - return nil, false - } - - amount := intMsg.Amount.Nano().String() - if amount == "0" { - return nil, false - } - - txData := newBaseParsedTransaction(tx, networkID) - txData.FromAddress = intMsg.SrcAddr.StringRaw() - txData.ToAddress = dstAddrRaw - txData.AssetAddress = "" - txData.Amount = amount - txData.Type = constant.TxTypeNativeTransfer - return &txData, true -} - -func parseOutgoingTransferMessage( - tx *tlb.Transaction, - intMsg *tlb.InternalMessage, - ourAddress string, - networkID string, -) (types.Transaction, bool) { - srcAddr := intMsg.SrcAddr.StringRaw() - if srcAddr != ourAddress || !isSimpleTransferMessage(intMsg) { - return types.Transaction{}, false - } - - amount := intMsg.Amount.Nano().String() - if amount == "0" { - return types.Transaction{}, false - } - - txData := newBaseParsedTransaction(tx, networkID) - txData.FromAddress = srcAddr - txData.ToAddress = intMsg.DstAddr.StringRaw() - txData.AssetAddress = "" - txData.Amount = amount - txData.Type = constant.TxTypeNativeTransfer - return txData, true -} - -// ParseJettonTransfer extracts Jetton transfers from a transaction. -func ParseJettonTransfer(tx *tlb.Transaction, ourAddress string, networkID string, registry JettonRegistry) []types.Transaction { - normalizedOurAddress := NormalizeTONAddressRaw(ourAddress) - if normalizedOurAddress == "" { - return nil - } - return parseJettonTransferNormalized(tx, normalizedOurAddress, networkID, registry) -} - -func parseJettonTransferNormalized( - tx *tlb.Transaction, - normalizedOurAddress string, - networkID string, - registry JettonRegistry, -) []types.Transaction { - var results []types.Transaction - - if parsed, ok := parseIncomingJetton(tx, normalizedOurAddress, networkID, registry); ok { - results = append(results, *parsed) - } - - for _, intMsg := range outgoingInternalMessages(tx) { - parsed, ok := parseOutgoingJettonMessage(tx, intMsg, normalizedOurAddress, networkID, registry) - if !ok { - continue - } - results = append(results, parsed) - } - - return results -} - -func parseIncomingJetton(tx *tlb.Transaction, ourAddress string, networkID string, registry JettonRegistry) (*types.Transaction, bool) { - intMsg, ok := incomingInternalMessage(tx) - if !ok { - return nil, false - } - - dstAddrRaw := intMsg.DstAddr.StringRaw() - if dstAddrRaw != ourAddress || !messageHasOpcode(intMsg, OpcodeTransferNotification) { - return nil, false - } - - jettonAmount, sender, ok := parseJettonTransferBody(intMsg) - if !ok || sender == nil { - return nil, false - } - - jettonWallet := intMsg.SrcAddr.StringRaw() - - txData := newBaseParsedTransaction(tx, networkID) - txData.FromAddress = sender.StringRaw() - txData.ToAddress = dstAddrRaw - txData.AssetAddress = resolveJettonAssetAddress(registry, jettonWallet) - txData.Amount = jettonAmount - txData.Type = constant.TxTypeTokenTransfer - return &txData, true -} - -func parseOutgoingJettonMessage( - tx *tlb.Transaction, - intMsg *tlb.InternalMessage, - ourAddress string, - networkID string, - registry JettonRegistry, -) (types.Transaction, bool) { - if !messageHasOpcode(intMsg, OpcodeTransfer) { - return types.Transaction{}, false - } - - jettonAmount, destination, ok := parseJettonTransferBody(intMsg) - if !ok || destination == nil { - return types.Transaction{}, false - } - - jettonWallet := intMsg.DstAddr.StringRaw() - - txData := newBaseParsedTransaction(tx, networkID) - txData.FromAddress = ourAddress - txData.ToAddress = destination.StringRaw() - txData.AssetAddress = resolveJettonAssetAddress(registry, jettonWallet) - txData.Amount = jettonAmount - txData.Type = constant.TxTypeTokenTransfer - return txData, true -} - -func parseJettonTransferBody(msg *tlb.InternalMessage) (string, *address.Address, bool) { - if msg.Body == nil { - return "", nil, false - } - - bodySlice := msg.Body.BeginParse() - if _, err := bodySlice.LoadUInt(32); err != nil { // opcode - return "", nil, false - } - if _, err := bodySlice.LoadUInt(64); err != nil { // query_id - return "", nil, false - } - - jettonAmount, err := bodySlice.LoadVarUInt(16) - if err != nil { - return "", nil, false - } - - peerAddress, err := bodySlice.LoadAddr() - if err != nil { - return "", nil, false - } - - return jettonAmount.String(), peerAddress, true -} - -func incomingInternalMessage(tx *tlb.Transaction) (*tlb.InternalMessage, bool) { - if tx.IO.In == nil { - return nil, false - } - - intMsg, ok := tx.IO.In.Msg.(*tlb.InternalMessage) - if !ok || intMsg.Bounced { - return nil, false - } - - return intMsg, true -} - -func outgoingInternalMessages(tx *tlb.Transaction) []*tlb.InternalMessage { - if tx.IO.Out == nil { - return nil - } - - outList, err := tx.IO.Out.ToSlice() - if err != nil { - return nil - } - - messages := make([]*tlb.InternalMessage, 0, len(outList)) - for _, outMsg := range outList { - intMsg, ok := outMsg.Msg.(*tlb.InternalMessage) - if !ok { - continue - } - messages = append(messages, intMsg) - } - return messages -} - -func messageOpcode(msg *tlb.InternalMessage) (uint64, bool) { - if msg.Body == nil { - return 0, false - } - - bodySlice := msg.Body.BeginParse() - if bodySlice.BitsLeft() < 32 { - return 0, false - } - - opcode, err := bodySlice.LoadUInt(32) - if err != nil { - return 0, false - } - return opcode, true -} - -func isSimpleTransferMessage(msg *tlb.InternalMessage) bool { - if msg.Body == nil { - return true - } - - opcode, ok := messageOpcode(msg) - if !ok { - return true - } - - // Opcode 0 means comment payload for regular TON transfer. - return opcode == 0 -} - -func messageHasOpcode(msg *tlb.InternalMessage, expected uint64) bool { - opcode, ok := messageOpcode(msg) - return ok && opcode == expected -} - -func resolveJettonAssetAddress(registry JettonRegistry, jettonWallet string) string { - if registry == nil { - return jettonWallet - } - if info, ok := registry.GetInfoByWallet(jettonWallet); ok { - return info.MasterAddress - } - return jettonWallet -} - -func newBaseParsedTransaction(tx *tlb.Transaction, networkID string) types.Transaction { - return types.Transaction{ - TxHash: base64.StdEncoding.EncodeToString(tx.Hash), - NetworkId: networkID, - BlockNumber: 0, - LogicalTime: tx.LT, - TxFee: decimal.NewFromBigInt(tx.TotalFees.Coins.Nano(), 0).Div(decimal.NewFromInt(1e9)), - Timestamp: uint64(tx.Now), - Status: types.StatusConfirmed, - } -} diff --git a/internal/indexer/ton/core_test.go b/internal/indexer/ton/core_test.go new file mode 100644 index 0000000..874c3bf --- /dev/null +++ b/internal/indexer/ton/core_test.go @@ -0,0 +1,62 @@ +package ton + +import ( + "encoding/hex" + "testing" + + "github.com/fystack/multichain-indexer/pkg/common/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTonAccountIndexerTransactionNetworkID(t *testing.T) { + t.Run("prefer_network_id", func(t *testing.T) { + indexer := &TonAccountIndexer{ + config: config.ChainConfig{ + NetworkId: "ton_testnet", + InternalCode: "TON_TESTNET", + }, + } + + assert.Equal(t, "ton_testnet", indexer.transactionNetworkID()) + }) + + t.Run("empty_when_network_id_missing", func(t *testing.T) { + indexer := &TonAccountIndexer{ + config: config.ChainConfig{ + NetworkId: " ", + InternalCode: "TON_TESTNET", + }, + } + + assert.Equal(t, "", indexer.transactionNetworkID()) + }) +} + +func TestNormalizeTONAddressList_CanonicalizesAndDedups(t *testing.T) { + addr1 := "0:fc58a2bb35b051810bef84fce18747ac2c2cfcbe0ce3d3167193d9b2538ef33e" + addr2 := "0:2942e40f94b5a2f111ea2ff98beb5f634f3a971f99f7fedafff5164c4bfa1bef" + + got := NormalizeTONAddressList([]string{ + " 0:FC58A2BB35B051810BEF84FCE18747AC2C2CFCBE0CE3D3167193D9B2538EF33E ", + "0:0xfc58a2bb35b051810bef84fce18747ac2c2cfcbe0ce3d3167193d9b2538ef33e", + addr2, + "not-an-address", + addr1, + "", + }) + + assert.Equal(t, []string{addr1, addr2}, got) +} + +func TestEncodeTONTxHash_UsesHex(t *testing.T) { + hash := []byte{0xfb, 0xef, 0xff, 0xfa, 0x00, 0x01, 0x02, 0x7f, 0x80, 0x90, 0xaa} + + encoded := encodeTONTxHash(hash) + + assert.Equal(t, "fbeffffa0001027f8090aa", encoded) + + decoded, err := hex.DecodeString(encoded) + require.NoError(t, err) + assert.Equal(t, hash, decoded) +} diff --git a/internal/indexer/ton/parser.go b/internal/indexer/ton/parser.go new file mode 100644 index 0000000..4649494 --- /dev/null +++ b/internal/indexer/ton/parser.go @@ -0,0 +1,320 @@ +package ton + +import ( + "encoding/hex" + "strings" + + "github.com/fystack/multichain-indexer/pkg/common/constant" + "github.com/fystack/multichain-indexer/pkg/common/types" + "github.com/shopspring/decimal" + "github.com/xssnick/tonutils-go/address" + "github.com/xssnick/tonutils-go/tlb" +) + +const ( + // Jetton standard opcodes (TEP-74) + OpcodeTransfer uint64 = 0x0f8a7ea5 + OpcodeTransferNotification uint64 = 0x7362d09c +) + +func (i *TonAccountIndexer) transactionNetworkID() string { + return strings.TrimSpace(i.config.NetworkId) +} + +func (i *TonAccountIndexer) parseMatchedTransactions( + tx *tlb.Transaction, + normalizedAddress string, +) []types.Transaction { + networkID := i.transactionNetworkID() + collected := parseTonTransferNormalized(tx, normalizedAddress, networkID) + if i.jettonRegistry != nil { + collected = append(collected, parseJettonTransferNormalized(tx, normalizedAddress, networkID, i.jettonRegistry)...) + } + return collected +} + +// ParseTonTransfer extracts native TON transfers from a transaction. +// Detects both incoming (receive) and outgoing (send) transfers. +func ParseTonTransfer(tx *tlb.Transaction, ourAddress string, networkID string) []types.Transaction { + normalizedOurAddress := NormalizeTONAddressRaw(ourAddress) + if normalizedOurAddress == "" { + return nil + } + return parseTonTransferNormalized(tx, normalizedOurAddress, networkID) +} + +func parseTonTransferNormalized(tx *tlb.Transaction, normalizedOurAddress string, networkID string) []types.Transaction { + var results []types.Transaction + + if parsed, ok := parseIncomingTransfer(tx, normalizedOurAddress, networkID); ok { + results = append(results, *parsed) + } + + for _, intMsg := range outgoingInternalMessages(tx) { + parsed, ok := parseOutgoingTransferMessage(tx, intMsg, normalizedOurAddress, networkID) + if !ok { + continue + } + results = append(results, parsed) + } + + return results +} + +func parseIncomingTransfer(tx *tlb.Transaction, ourAddress string, networkID string) (*types.Transaction, bool) { + intMsg, ok := incomingInternalMessage(tx) + if !ok { + return nil, false + } + + dstAddrRaw := intMsg.DstAddr.StringRaw() + if dstAddrRaw != ourAddress || !isSimpleTransferMessage(intMsg) { + return nil, false + } + + amount := intMsg.Amount.Nano().String() + if amount == "0" { + return nil, false + } + + txData := newBaseParsedTransaction(tx, networkID) + txData.FromAddress = intMsg.SrcAddr.StringRaw() + txData.ToAddress = dstAddrRaw + txData.AssetAddress = "" + txData.Amount = amount + txData.Type = constant.TxTypeNativeTransfer + return &txData, true +} + +func parseOutgoingTransferMessage( + tx *tlb.Transaction, + intMsg *tlb.InternalMessage, + ourAddress string, + networkID string, +) (types.Transaction, bool) { + srcAddr := intMsg.SrcAddr.StringRaw() + if srcAddr != ourAddress || !isSimpleTransferMessage(intMsg) { + return types.Transaction{}, false + } + + amount := intMsg.Amount.Nano().String() + if amount == "0" { + return types.Transaction{}, false + } + + txData := newBaseParsedTransaction(tx, networkID) + txData.FromAddress = srcAddr + txData.ToAddress = intMsg.DstAddr.StringRaw() + txData.AssetAddress = "" + txData.Amount = amount + txData.Type = constant.TxTypeNativeTransfer + return txData, true +} + +// ParseJettonTransfer extracts Jetton transfers from a transaction. +func ParseJettonTransfer(tx *tlb.Transaction, ourAddress string, networkID string, registry JettonRegistry) []types.Transaction { + normalizedOurAddress := NormalizeTONAddressRaw(ourAddress) + if normalizedOurAddress == "" { + return nil + } + return parseJettonTransferNormalized(tx, normalizedOurAddress, networkID, registry) +} + +func parseJettonTransferNormalized( + tx *tlb.Transaction, + normalizedOurAddress string, + networkID string, + registry JettonRegistry, +) []types.Transaction { + var results []types.Transaction + + if parsed, ok := parseIncomingJetton(tx, normalizedOurAddress, networkID, registry); ok { + results = append(results, *parsed) + } + + for _, intMsg := range outgoingInternalMessages(tx) { + parsed, ok := parseOutgoingJettonMessage(tx, intMsg, normalizedOurAddress, networkID, registry) + if !ok { + continue + } + results = append(results, parsed) + } + + return results +} + +func parseIncomingJetton(tx *tlb.Transaction, ourAddress string, networkID string, registry JettonRegistry) (*types.Transaction, bool) { + intMsg, ok := incomingInternalMessage(tx) + if !ok { + return nil, false + } + + dstAddrRaw := intMsg.DstAddr.StringRaw() + if dstAddrRaw != ourAddress || !messageHasOpcode(intMsg, OpcodeTransferNotification) { + return nil, false + } + + jettonAmount, sender, ok := parseJettonTransferBody(intMsg) + if !ok || sender == nil { + return nil, false + } + + jettonWallet := intMsg.SrcAddr.StringRaw() + + txData := newBaseParsedTransaction(tx, networkID) + txData.FromAddress = sender.StringRaw() + txData.ToAddress = dstAddrRaw + txData.AssetAddress = resolveJettonAssetAddress(registry, jettonWallet) + txData.Amount = jettonAmount + txData.Type = constant.TxTypeTokenTransfer + return &txData, true +} + +func parseOutgoingJettonMessage( + tx *tlb.Transaction, + intMsg *tlb.InternalMessage, + ourAddress string, + networkID string, + registry JettonRegistry, +) (types.Transaction, bool) { + if !messageHasOpcode(intMsg, OpcodeTransfer) { + return types.Transaction{}, false + } + + jettonAmount, destination, ok := parseJettonTransferBody(intMsg) + if !ok || destination == nil { + return types.Transaction{}, false + } + + jettonWallet := intMsg.DstAddr.StringRaw() + + txData := newBaseParsedTransaction(tx, networkID) + txData.FromAddress = ourAddress + txData.ToAddress = destination.StringRaw() + txData.AssetAddress = resolveJettonAssetAddress(registry, jettonWallet) + txData.Amount = jettonAmount + txData.Type = constant.TxTypeTokenTransfer + return txData, true +} + +func parseJettonTransferBody(msg *tlb.InternalMessage) (string, *address.Address, bool) { + if msg.Body == nil { + return "", nil, false + } + + bodySlice := msg.Body.BeginParse() + if _, err := bodySlice.LoadUInt(32); err != nil { // opcode + return "", nil, false + } + if _, err := bodySlice.LoadUInt(64); err != nil { // query_id + return "", nil, false + } + + jettonAmount, err := bodySlice.LoadVarUInt(16) + if err != nil { + return "", nil, false + } + + peerAddress, err := bodySlice.LoadAddr() + if err != nil { + return "", nil, false + } + + return jettonAmount.String(), peerAddress, true +} + +func incomingInternalMessage(tx *tlb.Transaction) (*tlb.InternalMessage, bool) { + if tx.IO.In == nil { + return nil, false + } + + intMsg, ok := tx.IO.In.Msg.(*tlb.InternalMessage) + if !ok || intMsg.Bounced { + return nil, false + } + + return intMsg, true +} + +func outgoingInternalMessages(tx *tlb.Transaction) []*tlb.InternalMessage { + if tx.IO.Out == nil { + return nil + } + + outList, err := tx.IO.Out.ToSlice() + if err != nil { + return nil + } + + messages := make([]*tlb.InternalMessage, 0, len(outList)) + for _, outMsg := range outList { + intMsg, ok := outMsg.Msg.(*tlb.InternalMessage) + if !ok { + continue + } + messages = append(messages, intMsg) + } + return messages +} + +func messageOpcode(msg *tlb.InternalMessage) (uint64, bool) { + if msg.Body == nil { + return 0, false + } + + bodySlice := msg.Body.BeginParse() + if bodySlice.BitsLeft() < 32 { + return 0, false + } + + opcode, err := bodySlice.LoadUInt(32) + if err != nil { + return 0, false + } + return opcode, true +} + +func isSimpleTransferMessage(msg *tlb.InternalMessage) bool { + if msg.Body == nil { + return true + } + + opcode, ok := messageOpcode(msg) + if !ok { + return true + } + + // Opcode 0 means comment payload for regular TON transfer. + return opcode == 0 +} + +func messageHasOpcode(msg *tlb.InternalMessage, expected uint64) bool { + opcode, ok := messageOpcode(msg) + return ok && opcode == expected +} + +func resolveJettonAssetAddress(registry JettonRegistry, jettonWallet string) string { + if registry == nil { + return jettonWallet + } + if info, ok := registry.GetInfoByWallet(jettonWallet); ok { + return info.MasterAddress + } + return jettonWallet +} + +func encodeTONTxHash(hash []byte) string { + return hex.EncodeToString(hash) +} + +func newBaseParsedTransaction(tx *tlb.Transaction, networkID string) types.Transaction { + return types.Transaction{ + TxHash: encodeTONTxHash(tx.Hash), + NetworkId: networkID, + BlockNumber: 0, + LogicalTime: tx.LT, + TxFee: decimal.NewFromBigInt(tx.TotalFees.Coins.Nano(), 0).Div(decimal.NewFromInt(1e9)), + Timestamp: uint64(tx.Now), + Status: types.StatusConfirmed, + } +} diff --git a/pkg/store/toncursorstore/store.go b/pkg/store/toncursorstore/store.go new file mode 100644 index 0000000..53eb84e --- /dev/null +++ b/pkg/store/toncursorstore/store.go @@ -0,0 +1,84 @@ +package toncursorstore + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/fystack/multichain-indexer/pkg/infra" +) + +const keyPrefix = "ton/cursor/" + +// AccountCursor tracks the polling position for a single TON account. +type AccountCursor struct { + Address string `json:"address"` + LastLT uint64 `json:"last_lt"` // Logical time of last processed tx + LastHash string `json:"last_hash"` // Hex-encoded hash of last processed tx + UpdatedAt time.Time `json:"updated_at"` +} + +type Store interface { + Get(ctx context.Context, address string) (*AccountCursor, error) + Save(ctx context.Context, cursor *AccountCursor) error + Delete(ctx context.Context, address string) error + List(ctx context.Context) ([]string, error) +} + +type kvStore struct { + kv infra.KVStore +} + +func New(kv infra.KVStore) Store { + return &kvStore{kv: kv} +} + +func (s *kvStore) cursorKey(address string) string { + return keyPrefix + address +} + +func (s *kvStore) Get(_ context.Context, address string) (*AccountCursor, error) { + var cursor AccountCursor + found, err := s.kv.GetAny(s.cursorKey(address), &cursor) + if err != nil { + return nil, fmt.Errorf("failed to get cursor for %s: %w", address, err) + } + if !found { + return nil, nil + } + return &cursor, nil +} + +func (s *kvStore) Save(_ context.Context, cursor *AccountCursor) error { + cursor.UpdatedAt = time.Now() + if err := s.kv.SetAny(s.cursorKey(cursor.Address), cursor); err != nil { + return fmt.Errorf("failed to save cursor for %s: %w", cursor.Address, err) + } + return nil +} + +func (s *kvStore) Delete(_ context.Context, address string) error { + if err := s.kv.Delete(s.cursorKey(address)); err != nil { + return fmt.Errorf("failed to delete cursor for %s: %w", address, err) + } + return nil +} + +func (s *kvStore) List(_ context.Context) ([]string, error) { + pairs, err := s.kv.List(keyPrefix) + if err != nil { + return nil, fmt.Errorf("failed to list cursors: %w", err) + } + + addresses := make([]string, 0, len(pairs)) + for _, pair := range pairs { + var cursor AccountCursor + if err := json.Unmarshal(pair.Value, &cursor); err != nil { + continue + } + addresses = append(addresses, cursor.Address) + } + + return addresses, nil +} From 1af90de746fb71c86d6d84fb9eee2ad8a57101fe Mon Sep 17 00:00:00 2001 From: vietddude Date: Thu, 12 Feb 2026 15:58:09 +0700 Subject: [PATCH 07/11] feat(ton-rpc): paginate account tx fetch and resolve jetton masters --- internal/rpc/ton/api.go | 6 ++ internal/rpc/ton/client.go | 106 +++++++++++++++++++++++++++++--- internal/rpc/ton/client_test.go | 26 ++++++++ 3 files changed, 128 insertions(+), 10 deletions(-) create mode 100644 internal/rpc/ton/client_test.go diff --git a/internal/rpc/ton/api.go b/internal/rpc/ton/api.go index 5987aab..fae3917 100644 --- a/internal/rpc/ton/api.go +++ b/internal/rpc/ton/api.go @@ -14,4 +14,10 @@ type TonAPI interface { // - lastHash: transaction hash cursor (nil for initial fetch) // Returns transactions in reverse chronological order (newest first). ListTransactions(ctx context.Context, addr *address.Address, limit uint32, lastLT uint64, lastHash []byte) ([]*tlb.Transaction, error) + + // GetLatestMasterchainSeqno returns the latest observed masterchain sequence number. + GetLatestMasterchainSeqno(ctx context.Context) (uint64, error) + + // ResolveJettonMasterAddress resolves a jetton wallet address to its master contract address. + ResolveJettonMasterAddress(ctx context.Context, jettonWallet string) (string, error) } diff --git a/internal/rpc/ton/client.go b/internal/rpc/ton/client.go index e9fda4d..ed643e7 100644 --- a/internal/rpc/ton/client.go +++ b/internal/rpc/ton/client.go @@ -2,7 +2,9 @@ package ton import ( "context" + "errors" "fmt" + "sort" "sync" "time" @@ -79,6 +81,10 @@ func (c *Client) getMasterchainInfo(ctx context.Context) (*ton.BlockIDExt, error // Transactions are returned in reverse chronological order (newest first). // Use lastLT=0 and lastHash=nil for initial fetch from the account's latest transaction. func (c *Client) ListTransactions(ctx context.Context, addr *address.Address, limit uint32, lastLT uint64, lastHash []byte) ([]*tlb.Transaction, error) { + if limit == 0 { + limit = 1 + } + // Get the current masterchain block for consistency master, err := c.getMasterchainInfo(ctx) if err != nil { @@ -101,21 +107,101 @@ func (c *Client) ListTransactions(ctx context.Context, addr *address.Address, li return nil, nil } - // Fetch transactions from the LATEST, going backwards - txs, err := c.api.ListTransactions(ctx, addr, limit, account.LastTxLT, account.LastTxHash) + // Page backwards from account tip until we reach the saved cursor. + var ( + allNewTxs []*tlb.Transaction + fetchLT = account.LastTxLT + fetchHash = account.LastTxHash + ) + + for { + txs, listErr := c.api.ListTransactions(ctx, addr, limit, fetchLT, fetchHash) + if listErr != nil { + if errors.Is(listErr, ton.ErrNoTransactionsWereFound) && len(allNewTxs) > 0 { + break + } + return nil, fmt.Errorf("failed to list transactions: %w", listErr) + } + if len(txs) == 0 { + break + } + + oldest := txs[0] + for _, tx := range txs { + if tx.LT > lastLT { + allNewTxs = append(allNewTxs, tx) + } + if tx.LT < oldest.LT { + oldest = tx + } + } + + // Reached already-processed range. + if oldest.LT <= lastLT { + break + } + // No more history. + if oldest.PrevTxLT == 0 || len(oldest.PrevTxHash) == 0 { + break + } + + fetchLT = oldest.PrevTxLT + fetchHash = oldest.PrevTxHash + } + + // Keep existing contract used by indexer: newest first. + sort.Slice(allNewTxs, func(i, j int) bool { + return allNewTxs[i].LT > allNewTxs[j].LT + }) + + return allNewTxs, nil +} + +func (c *Client) GetLatestMasterchainSeqno(ctx context.Context) (uint64, error) { + master, err := c.getMasterchainInfo(ctx) if err != nil { - return nil, fmt.Errorf("failed to list transactions: %w", err) + return 0, fmt.Errorf("failed to get masterchain info: %w", err) } + return uint64(master.SeqNo), nil +} - // Filter to only return transactions NEWER than our cursor - var newTxs []*tlb.Transaction - for _, tx := range txs { - if tx.LT > lastLT { - newTxs = append(newTxs, tx) - } +func (c *Client) ResolveJettonMasterAddress(ctx context.Context, jettonWallet string) (string, error) { + walletAddr, err := parseTONAddressAny(jettonWallet) + if err != nil { + return "", fmt.Errorf("invalid jetton wallet address: %w", err) + } + + master, err := c.getMasterchainInfo(ctx) + if err != nil { + return "", fmt.Errorf("failed to get masterchain info: %w", err) + } + + res, err := c.api.WaitForBlock(master.SeqNo).RunGetMethod(ctx, master, walletAddr, "get_wallet_data") + if err != nil { + return "", fmt.Errorf("run get_wallet_data: %w", err) + } + + masterSlice, err := res.Slice(2) + if err != nil { + return "", fmt.Errorf("parse get_wallet_data master address: %w", err) } - return newTxs, nil + masterAddr, err := masterSlice.LoadAddr() + if err != nil { + return "", fmt.Errorf("load master address from stack: %w", err) + } + if masterAddr == nil { + return "", fmt.Errorf("jetton master address is nil") + } + + return masterAddr.StringRaw(), nil +} + +func parseTONAddressAny(addr string) (*address.Address, error) { + if raw, err := address.ParseRawAddr(addr); err == nil { + return raw, nil + } + return address.ParseAddr(addr) } func (c *Client) Close() error { diff --git a/internal/rpc/ton/client_test.go b/internal/rpc/ton/client_test.go new file mode 100644 index 0000000..c509e11 --- /dev/null +++ b/internal/rpc/ton/client_test.go @@ -0,0 +1,26 @@ +package ton + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestParseTONAddressAny(t *testing.T) { + t.Run("raw_address", func(t *testing.T) { + addr, err := parseTONAddressAny("0:eaa27e0e4fbadad817ac4a106de2bae8b52106b5267c1656a3892538d59c69dc") + require.NoError(t, err) + require.Equal(t, "0:eaa27e0e4fbadad817ac4a106de2bae8b52106b5267c1656a3892538d59c69dc", addr.StringRaw()) + }) + + t.Run("friendly_address", func(t *testing.T) { + addr, err := parseTONAddressAny("EQAd59XMWb40TRfxqTJ0otrUTqD8ZuxLHhC8T401A5-Kr2aY") + require.NoError(t, err) + require.Equal(t, "0:1de7d5cc59be344d17f1a93274a2dad44ea0fc66ec4b1e10bc4f8d35039f8aaf", addr.StringRaw()) + }) + + t.Run("invalid_address", func(t *testing.T) { + _, err := parseTONAddressAny("not-an-address") + require.Error(t, err) + }) +} From 4e5399574ff9550de4e0992279e4706e6dcfa253 Mon Sep 17 00:00:00 2001 From: vietddude Date: Thu, 12 Feb 2026 15:58:28 +0700 Subject: [PATCH 08/11] feat(ton-indexer): enrich emitted txs and dedup tracked internal flows --- internal/indexer/ton/core.go | 79 +++++++++++++++++++++++- internal/indexer/ton/core_jetton_test.go | 75 ++++++++++++++++++++++ internal/worker/ton/worker.go | 50 +++++++++++++++ internal/worker/ton/worker_test.go | 43 +++++++++++++ 4 files changed, 246 insertions(+), 1 deletion(-) create mode 100644 internal/indexer/ton/core_jetton_test.go create mode 100644 internal/worker/ton/worker_test.go diff --git a/internal/indexer/ton/core.go b/internal/indexer/ton/core.go index ce6bcb4..de5abac 100644 --- a/internal/indexer/ton/core.go +++ b/internal/indexer/ton/core.go @@ -4,9 +4,11 @@ import ( "context" "encoding/hex" "fmt" + "sync" tonRpc "github.com/fystack/multichain-indexer/internal/rpc/ton" "github.com/fystack/multichain-indexer/pkg/common/config" + "github.com/fystack/multichain-indexer/pkg/common/constant" "github.com/fystack/multichain-indexer/pkg/common/enum" "github.com/fystack/multichain-indexer/pkg/common/types" "github.com/fystack/multichain-indexer/pkg/infra" @@ -46,6 +48,8 @@ type TonAccountIndexer struct { config config.ChainConfig client tonRpc.TonAPI jettonRegistry JettonRegistry + jettonMasterMu sync.RWMutex + jettonMasters map[string]string // jetton wallet -> jetton master // Transaction limit per poll txLimit uint32 @@ -68,6 +72,7 @@ func NewTonAccountIndexer( config: cfg, client: client, jettonRegistry: jettonRegistry, + jettonMasters: make(map[string]string), txLimit: txLimit, } } @@ -129,6 +134,8 @@ func (i *TonAccountIndexer) PollAccount(ctx context.Context, addrStr string, cur parsedTxs := make([]types.Transaction, 0, len(txs)) newCursor := ensureCursor(cursor, addrStr) + var latestSeqno uint64 + var latestSeqnoFetched bool // TON API returns newest first, so process backwards (oldest to newest). for j := len(txs) - 1; j >= 0; j-- { @@ -144,12 +151,82 @@ func (i *TonAccountIndexer) PollAccount(ctx context.Context, addrStr string, cur continue } - parsedTxs = append(parsedTxs, i.parseMatchedTransactions(tx, normalizedAddress)...) + matchedTxs := i.parseMatchedTransactions(tx, normalizedAddress) + if len(matchedTxs) == 0 { + continue + } + i.resolveJettonAssetAddresses(ctx, matchedTxs) + + if !latestSeqnoFetched { + latestSeqno, err = i.client.GetLatestMasterchainSeqno(ctx) + if err != nil { + return nil, cursor, fmt.Errorf("failed to get latest masterchain seqno: %w", err) + } + latestSeqnoFetched = true + } + + for idx := range matchedTxs { + matchedTxs[idx].BlockNumber = latestSeqno + matchedTxs[idx].MasterchainSeqno = latestSeqno + } + + parsedTxs = append(parsedTxs, matchedTxs...) } return parsedTxs, newCursor, nil } +func (i *TonAccountIndexer) resolveJettonAssetAddresses(ctx context.Context, txs []types.Transaction) { + for idx := range txs { + tx := &txs[idx] + if tx.Type != constant.TxTypeTokenTransfer || tx.AssetAddress == "" { + continue + } + + walletAddress := tx.AssetAddress + if i.jettonRegistry != nil { + if info, ok := i.jettonRegistry.GetInfo(walletAddress); ok { + tx.AssetAddress = info.MasterAddress + continue + } + if info, ok := i.jettonRegistry.GetInfoByWallet(walletAddress); ok { + tx.AssetAddress = info.MasterAddress + i.cacheJettonMaster(walletAddress, info.MasterAddress) + continue + } + } + + if cachedMaster, ok := i.getCachedJettonMaster(walletAddress); ok { + tx.AssetAddress = cachedMaster + continue + } + + resolvedMaster, err := i.client.ResolveJettonMasterAddress(ctx, walletAddress) + if err != nil || resolvedMaster == "" { + continue + } + + i.cacheJettonMaster(walletAddress, resolvedMaster) + tx.AssetAddress = resolvedMaster + if i.jettonRegistry != nil { + i.jettonRegistry.RegisterWallet(walletAddress, resolvedMaster) + } + } +} + +func (i *TonAccountIndexer) getCachedJettonMaster(walletAddress string) (string, bool) { + i.jettonMasterMu.RLock() + defer i.jettonMasterMu.RUnlock() + master, ok := i.jettonMasters[walletAddress] + return master, ok +} + +func (i *TonAccountIndexer) cacheJettonMaster(walletAddress, masterAddress string) { + i.jettonMasterMu.Lock() + defer i.jettonMasterMu.Unlock() + i.jettonMasters[walletAddress] = masterAddress +} + func decodeCursorForRPC(cursor *AccountCursor) (uint64, []byte, error) { if cursor == nil || cursor.LastLT == 0 { return 0, nil, nil diff --git a/internal/indexer/ton/core_jetton_test.go b/internal/indexer/ton/core_jetton_test.go new file mode 100644 index 0000000..92735fa --- /dev/null +++ b/internal/indexer/ton/core_jetton_test.go @@ -0,0 +1,75 @@ +package ton + +import ( + "context" + "testing" + + "github.com/fystack/multichain-indexer/pkg/common/constant" + "github.com/fystack/multichain-indexer/pkg/common/types" + "github.com/stretchr/testify/assert" + "github.com/xssnick/tonutils-go/address" + "github.com/xssnick/tonutils-go/tlb" +) + +type mockTonAPI struct { + resolveMasterFn func(ctx context.Context, jettonWallet string) (string, error) + resolveCalls int +} + +func (m *mockTonAPI) ListTransactions(_ context.Context, _ *address.Address, _ uint32, _ uint64, _ []byte) ([]*tlb.Transaction, error) { + return nil, nil +} + +func (m *mockTonAPI) GetLatestMasterchainSeqno(_ context.Context) (uint64, error) { + return 0, nil +} + +func (m *mockTonAPI) ResolveJettonMasterAddress(ctx context.Context, jettonWallet string) (string, error) { + m.resolveCalls++ + if m.resolveMasterFn == nil { + return "", nil + } + return m.resolveMasterFn(ctx, jettonWallet) +} + +func TestResolveJettonAssetAddresses_ResolveAndCache(t *testing.T) { + const ( + walletAddr = "0:eaa27e0e4fbadad817ac4a106de2bae8b52106b5267c1656a3892538d59c69dc" + masterAddr = "0:ca6e321c3ce184f66f4f74344770f31472800583947a7f9d5f68fecf052ce20f" + ) + + api := &mockTonAPI{ + resolveMasterFn: func(_ context.Context, jettonWallet string) (string, error) { + if jettonWallet == walletAddr { + return masterAddr, nil + } + return "", nil + }, + } + + idx := &TonAccountIndexer{ + client: api, + jettonRegistry: NewConfigBasedRegistry(nil), + jettonMasters: make(map[string]string), + } + + first := []types.Transaction{ + { + Type: constant.TxTypeTokenTransfer, + AssetAddress: walletAddr, + }, + } + idx.resolveJettonAssetAddresses(context.Background(), first) + assert.Equal(t, masterAddr, first[0].AssetAddress) + assert.Equal(t, 1, api.resolveCalls) + + second := []types.Transaction{ + { + Type: constant.TxTypeTokenTransfer, + AssetAddress: walletAddr, + }, + } + idx.resolveJettonAssetAddresses(context.Background(), second) + assert.Equal(t, masterAddr, second[0].AssetAddress) + assert.Equal(t, 1, api.resolveCalls) +} diff --git a/internal/worker/ton/worker.go b/internal/worker/ton/worker.go index 8f32fea..4d1562d 100644 --- a/internal/worker/ton/worker.go +++ b/internal/worker/ton/worker.go @@ -38,6 +38,7 @@ type TonPollingWorker struct { // Cache wallets []string + walletSet map[string]struct{} walletsInitialized bool walletsMutex sync.RWMutex @@ -231,10 +232,20 @@ func (w *TonPollingWorker) pollAccount(address string) { log.Info("Found transactions", "count", len(txs)) for i := range txs { tx := &txs[i] + if w.shouldSkipTrackedInternalTransfer(address, tx) { + log.Debug("Skipping duplicate tracked-wallet transfer from receiver-side poll", + "from", tx.FromAddress, + "to", tx.ToAddress, + "txhash", tx.TxHash, + ) + continue + } + log.Info("Emitting matched transaction", "type", tx.Type, "from", tx.FromAddress, "to", tx.ToAddress, + "asset_address", tx.AssetAddress, "amount", tx.Amount, "fee", tx.TxFee.String(), "txhash", tx.TxHash, @@ -362,9 +373,15 @@ func (w *TonPollingWorker) ensureWalletsLoaded() error { } func (w *TonPollingWorker) replaceWalletCache(addresses []string) int { + walletSet := make(map[string]struct{}, len(addresses)) + for _, addr := range addresses { + walletSet[addr] = struct{}{} + } + w.walletsMutex.Lock() oldSize := len(w.wallets) w.wallets = addresses + w.walletSet = walletSet w.walletsInitialized = true w.walletsMutex.Unlock() @@ -375,6 +392,39 @@ func (w *TonPollingWorker) replaceWalletCache(addresses []string) int { return len(addresses) } +func (w *TonPollingWorker) isTrackedWallet(address string) bool { + normalized := tonIndexer.NormalizeTONAddressRaw(address) + if normalized == "" { + return false + } + + w.walletsMutex.RLock() + _, ok := w.walletSet[normalized] + w.walletsMutex.RUnlock() + return ok +} + +func (w *TonPollingWorker) shouldSkipTrackedInternalTransfer(polledAddress string, tx *types.Transaction) bool { + from := tonIndexer.NormalizeTONAddressRaw(tx.FromAddress) + to := tonIndexer.NormalizeTONAddressRaw(tx.ToAddress) + if from == "" || to == "" { + return false + } + + // Keep only one side when both sender and receiver are tracked. + // Sender-side poll has polledAddress == from. + if !w.isTrackedWallet(from) || !w.isTrackedWallet(to) { + return false + } + + polled := tonIndexer.NormalizeTONAddressRaw(polledAddress) + if polled == "" { + return false + } + + return polled != from +} + // ReloadWalletsFromKV refreshes in-memory wallets from KV store. func (w *TonPollingWorker) ReloadWalletsFromKV(_ context.Context) (int, error) { if w.kvstore == nil { diff --git a/internal/worker/ton/worker_test.go b/internal/worker/ton/worker_test.go new file mode 100644 index 0000000..6bf831c --- /dev/null +++ b/internal/worker/ton/worker_test.go @@ -0,0 +1,43 @@ +package ton + +import ( + "io" + "log/slog" + "testing" + + "github.com/fystack/multichain-indexer/pkg/common/types" + "github.com/stretchr/testify/assert" +) + +func TestShouldSkipTrackedInternalTransfer(t *testing.T) { + const ( + sender = "0:2942e40f94b5a2f111ea2ff98beb5f634f3a971f99f7fedafff5164c4bfa1bef" + receiver = "0:fc58a2bb35b051810bef84fce18747ac2c2cfcbe0ce3d3167193d9b2538ef33e" + external = "0:5df8318107d8988e2ab298b0190ba9f923267bc1575f4532a49f37384db6a799" + ) + + w := &TonPollingWorker{} + w.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) + w.replaceWalletCache([]string{sender, receiver}) + + tx := &types.Transaction{ + FromAddress: sender, + ToAddress: receiver, + } + + t.Run("keep_sender_side_poll", func(t *testing.T) { + assert.False(t, w.shouldSkipTrackedInternalTransfer(sender, tx)) + }) + + t.Run("skip_receiver_side_poll", func(t *testing.T) { + assert.True(t, w.shouldSkipTrackedInternalTransfer(receiver, tx)) + }) + + t.Run("do_not_skip_external_to_tracked", func(t *testing.T) { + externalTx := &types.Transaction{ + FromAddress: external, + ToAddress: receiver, + } + assert.False(t, w.shouldSkipTrackedInternalTransfer(receiver, externalTx)) + }) +} From af9d0e0ab386082b7406dae002f86412df071538 Mon Sep 17 00:00:00 2001 From: vietddude Date: Thu, 12 Feb 2026 15:58:50 +0700 Subject: [PATCH 09/11] fix(ton-registry): persist wallet master mappings and fallback lookup --- internal/indexer/ton/registry.go | 35 +++++++++---- internal/indexer/ton/registry_test.go | 72 +++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 9 deletions(-) create mode 100644 internal/indexer/ton/registry_test.go diff --git a/internal/indexer/ton/registry.go b/internal/indexer/ton/registry.go index e504a53..cecdc77 100644 --- a/internal/indexer/ton/registry.go +++ b/internal/indexer/ton/registry.go @@ -66,7 +66,11 @@ func (r *ConfigBasedRegistry) GetInfoByWallet(walletAddress string) (*JettonInfo return nil, false } - return r.GetInfo(masterAddr) + if info, ok := r.GetInfo(masterAddr); ok { + return info, true + } + // Fallback: return master address even when token metadata is not preconfigured. + return &JettonInfo{MasterAddress: masterAddr}, true } // RegisterWallet associates a Jetton wallet with its master address. @@ -75,10 +79,7 @@ func (r *ConfigBasedRegistry) RegisterWallet(walletAddress, masterAddress string r.mu.Lock() defer r.mu.Unlock() - // Only register if the master is in our supported list - if _, ok := r.jettons[masterAddress]; ok { - r.walletToMaster[walletAddress] = masterAddress - } + r.walletToMaster[walletAddress] = masterAddress } // List returns all supported Jettons. @@ -225,18 +226,34 @@ func (r *RedisJettonRegistry) GetInfoByWallet(walletAddress string) (*JettonInfo info, ok := r.jettons[masterAddress] r.mu.RUnlock() if !ok { - return nil, false + // Fallback: return master address even when token metadata is not in masters list. + return &JettonInfo{MasterAddress: masterAddress}, true } return &info, true } func (r *RedisJettonRegistry) RegisterWallet(walletAddress, masterAddress string) { + if strings.TrimSpace(walletAddress) == "" || strings.TrimSpace(masterAddress) == "" { + return + } + r.mu.Lock() - defer r.mu.Unlock() + r.walletToMaster[walletAddress] = masterAddress + snapshot := make(map[string]string, len(r.walletToMaster)) + for k, v := range r.walletToMaster { + snapshot[k] = v + } + r.mu.Unlock() + + if r.redis == nil { + return + } - if _, ok := r.jettons[masterAddress]; ok { - r.walletToMaster[walletAddress] = masterAddress + payload, err := json.Marshal(snapshot) + if err != nil { + return } + _ = r.redis.Set(r.walletMappingKey(), string(payload), 0) } func (r *RedisJettonRegistry) List() []JettonInfo { diff --git a/internal/indexer/ton/registry_test.go b/internal/indexer/ton/registry_test.go new file mode 100644 index 0000000..b593c62 --- /dev/null +++ b/internal/indexer/ton/registry_test.go @@ -0,0 +1,72 @@ +package ton + +import ( + "testing" + "time" + + "github.com/fystack/multichain-indexer/pkg/infra" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/assert" +) + +type mockRedisClient struct { + data map[string]string +} + +func (m *mockRedisClient) GetClient() *redis.Client { return nil } +func (m *mockRedisClient) Set(key string, value any, _ time.Duration) error { + if m.data == nil { + m.data = make(map[string]string) + } + switch v := value.(type) { + case string: + m.data[key] = v + default: + m.data[key] = "" + } + return nil +} +func (m *mockRedisClient) Get(key string) (string, error) { return m.data[key], nil } +func (m *mockRedisClient) Del(_ ...string) error { return nil } +func (m *mockRedisClient) ZAdd(_ string, _ ...redis.Z) error { + return nil +} +func (m *mockRedisClient) ZRem(_ string, _ ...interface{}) error { + return nil +} +func (m *mockRedisClient) ZRange(_ string, _, _ int64) ([]string, error) { + return nil, nil +} +func (m *mockRedisClient) ZRangeWithScores(_ string, _, _ int64) ([]redis.Z, error) { + return nil, nil +} +func (m *mockRedisClient) ZRevRangeWithScores(_ string, _, _ int64) ([]redis.Z, error) { + return nil, nil +} +func (m *mockRedisClient) Close() error { return nil } + +var _ infra.RedisClient = (*mockRedisClient)(nil) + +func TestConfigBasedRegistryGetInfoByWalletFallback(t *testing.T) { + reg := NewConfigBasedRegistry(nil) + reg.RegisterWallet("wallet-1", "master-1") + + info, ok := reg.GetInfoByWallet("wallet-1") + assert.True(t, ok) + assert.Equal(t, "master-1", info.MasterAddress) +} + +func TestRedisJettonRegistryRegisterWalletPersistsAndFallback(t *testing.T) { + redisClient := &mockRedisClient{data: make(map[string]string)} + reg := NewRedisJettonRegistry("ton_testnet", redisClient, nil) + + reg.RegisterWallet("wallet-1", "master-1") + + info, ok := reg.GetInfoByWallet("wallet-1") + assert.True(t, ok) + assert.Equal(t, "master-1", info.MasterAddress) + + raw, exists := redisClient.data[reg.walletMappingKey()] + assert.True(t, exists) + assert.Contains(t, raw, "\"wallet-1\":\"master-1\"") +} From 6c2e2b4579175bd2f3f974a8384fe2d594bb6bc3 Mon Sep 17 00:00:00 2001 From: vietddude Date: Thu, 12 Feb 2026 16:00:22 +0700 Subject: [PATCH 10/11] feat: add ton_testnet configuration to example config --- configs/config.example.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/configs/config.example.yaml b/configs/config.example.yaml index 72c7492..74dc7b5 100644 --- a/configs/config.example.yaml +++ b/configs/config.example.yaml @@ -176,6 +176,26 @@ chains: symbol: "USDT" decimals: 6 + ton_testnet: + network_id: "ton_testnet" + internal_code: "TON_TESTNET" + type: "ton" + start_block: 0 + poll_interval: "4s" + nodes: + - url: "https://ton.org/testnet-global-config.json" + client: + timeout: "15s" + throttle: + rps: 10 + burst: 20 + batch_size: 1 + concurrency: 4 + jettons: + - master_address: "0:1de7d5cc59be344d17f1a93274a2dad44ea0fc66ec4b1e10bc4f8d35039f8aaf" + symbol: "USDT" + decimals: 6 + # Infrastructure services services: port: 8080 # Health check and monitoring server port From d8e02f9986606a389758f1f2ba33058434199be0 Mon Sep 17 00:00:00 2001 From: vietddude Date: Sat, 21 Feb 2026 15:53:01 +0700 Subject: [PATCH 11/11] feat(cosmos): add cosmos indexing flow and align network ids --- README.md | 17 + configs/config.example.yaml | 87 ++++ internal/indexer/cosmos.go | 624 +++++++++++++++++++++++++++++ internal/indexer/cosmos_test.go | 229 +++++++++++ internal/rpc/cosmos/api.go | 16 + internal/rpc/cosmos/client.go | 118 ++++++ internal/rpc/cosmos/types.go | 65 +++ internal/rpc/types.go | 1 + internal/worker/factory.go | 41 ++ pkg/addressbloomfilter/inmemory.go | 1 + pkg/addressbloomfilter/redis.go | 1 + pkg/common/config/chains.go | 7 + pkg/common/config/types.go | 1 + pkg/common/enum/enum.go | 15 +- sql/wallet_address.sql | 72 ++-- 15 files changed, 1252 insertions(+), 43 deletions(-) create mode 100644 internal/indexer/cosmos.go create mode 100644 internal/indexer/cosmos_test.go create mode 100644 internal/rpc/cosmos/api.go create mode 100644 internal/rpc/cosmos/client.go create mode 100644 internal/rpc/cosmos/types.go diff --git a/README.md b/README.md index 9c15d81..e30a84c 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ This indexer is designed to be used in a multi-chain environment, where each cha - Bitcoin - Solana - Sui +- Cosmos (Osmosis, Celestia, Cosmos Hub) --- @@ -304,6 +305,22 @@ chains: throttle: rps: 5 burst: 8 + + cosmoshub_mainnet: + type: "cosmos" + network_id: "cosmoshub-4" + native_denom: "uatom" + nodes: + - url: "https://rpc.cosmos.directory/cosmoshub" + - url: "https://cosmos-rpc.publicnode.com" + poll_interval: "5s" + client: + timeout: "20s" + max_retries: 3 + retry_delay: "2s" + throttle: + rps: 20 + burst: 40 ``` ## 📡 Consuming Transaction Events diff --git a/configs/config.example.yaml b/configs/config.example.yaml index 74dc7b5..0f2288b 100644 --- a/configs/config.example.yaml +++ b/configs/config.example.yaml @@ -19,6 +19,8 @@ defaults: chains: tron_mainnet: + network_id: "tron_mainnet" + internal_code: "TRON_MAINNET" type: "tron" start_block: 75144237 poll_interval: "8s" # override default poll interval @@ -37,6 +39,8 @@ chains: burst: 8 ethereum_mainnet: + network_id: "ethereum_mainnet" + internal_code: "ETH_MAINNET" type: "evm" start_block: 23080871 poll_interval: "3s" # faster polling for Ethereum @@ -52,6 +56,7 @@ chains: # - url: "https://eth-mainnet.g.alchemy.com/v2/${ALCHEMY_KEY}" # optional bnb_mainnet: + network_id: "bnb_mainnet" internal_code: "BNB_MAINNET" type: "evm" start_block: 59040717 @@ -62,6 +67,7 @@ chains: - url: "https://bnb.rpc.subquery.network/public" bitcoin_testnet: + network_id: "bitcoin_testnet" internal_code: "BTC_TESTNET" type: "btc" start_block: 4440000 # recent testnet block @@ -83,6 +89,7 @@ chains: concurrency: 1 # No parallel fetching needed bitcoin_mainnet: + network_id: "bitcoin_mainnet" internal_code: "BTC_MAINNET" type: "btc" start_block: 850000 @@ -161,6 +168,86 @@ chains: - url: "fullnode.mainnet.sui.io:443" # e.g. 127.0.0.1:9000 - url: "sui-mainnet.nodeinfra.com:443" + osmosis_mainnet: + network_id: "osmosis-1" + internal_code: "OSMO_MAINNET" + native_denom: "uosmo" + type: "cosmos" + start_block: 0 + poll_interval: "5s" + nodes: + - url: "https://rpc.osmosis.zone" + - url: "https://osmosis-rpc.publicnode.com" + client: + timeout: "20s" + max_retries: 3 + retry_delay: "2s" + throttle: + rps: 20 + burst: 40 + batch_size: 20 + concurrency: 4 + + cosmoshub_mainnet: + network_id: "cosmoshub-4" + internal_code: "ATOM_MAINNET" + native_denom: "uatom" + type: "cosmos" + start_block: 0 + poll_interval: "5s" + nodes: + - url: "https://rpc.cosmos.directory/cosmoshub" + - url: "https://cosmos-rpc.publicnode.com" + client: + timeout: "20s" + max_retries: 3 + retry_delay: "2s" + throttle: + rps: 20 + burst: 40 + batch_size: 20 + concurrency: 4 + + cosmos_hub_testnet: + network_id: "cosmos_hub_testnet" + internal_code: "ATOM_TESTNET" + native_denom: "uatom" + type: "cosmos" + start_block: 0 + poll_interval: "5s" + nodes: + - url: "https://rpc.provider-sentry-01.hub-testnet.polypore.xyz" + - url: "https://cosmos-testnet-rpc.itrocket.net" + client: + timeout: "20s" + max_retries: 3 + retry_delay: "2s" + throttle: + rps: 20 + burst: 40 + batch_size: 20 + concurrency: 4 + + celestia_mainnet: + network_id: "celestia" + internal_code: "TIA_MAINNET" + native_denom: "utia" + type: "cosmos" + start_block: 0 + poll_interval: "5s" + nodes: + - url: "https://rpc-celestia.01node.com" + - url: "https://celestia-rpc.publicnode.com" + client: + timeout: "20s" + max_retries: 3 + retry_delay: "2s" + throttle: + rps: 20 + burst: 40 + batch_size: 20 + concurrency: 4 + ton_mainnet: network_id: "ton" internal_code: "TON_MAINNET" diff --git a/internal/indexer/cosmos.go b/internal/indexer/cosmos.go new file mode 100644 index 0000000..90d9221 --- /dev/null +++ b/internal/indexer/cosmos.go @@ -0,0 +1,624 @@ +package indexer + +import ( + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + "regexp" + "strconv" + "strings" + "sync" + "time" + "unicode/utf8" + + "github.com/fystack/multichain-indexer/internal/rpc" + "github.com/fystack/multichain-indexer/internal/rpc/cosmos" + "github.com/fystack/multichain-indexer/pkg/common/config" + "github.com/fystack/multichain-indexer/pkg/common/constant" + "github.com/fystack/multichain-indexer/pkg/common/enum" + "github.com/fystack/multichain-indexer/pkg/common/types" + "github.com/shopspring/decimal" +) + +var cosmosCoinRegexp = regexp.MustCompile(`^([0-9]+)([a-zA-Z0-9/._:-]+)$`) + +const ( + cosmosNativeDenomOsmosis = "uosmo" + cosmosNativeDenomAtom = "uatom" + + cosmosNetworkHintOsmosis = "osmosis" + cosmosNetworkHintOsmo = "osmo" + cosmosNetworkHintHub = "cosmoshub" + cosmosNetworkHintAtom = "atom" + cosmosNetworkHintCosmos = "cosmos" + cosmosNetworkHintHubWord = "hub" + cosmosNetworkHintCosmos1 = "cosmos1" +) + +type CosmosIndexer struct { + chainName string + config config.ChainConfig + failover *rpc.Failover[cosmos.CosmosAPI] + pubkeyStore PubkeyStore +} + +func NewCosmosIndexer( + chainName string, + cfg config.ChainConfig, + failover *rpc.Failover[cosmos.CosmosAPI], + pubkeyStore PubkeyStore, +) *CosmosIndexer { + return &CosmosIndexer{ + chainName: chainName, + config: cfg, + failover: failover, + pubkeyStore: pubkeyStore, + } +} + +func (c *CosmosIndexer) GetName() string { return strings.ToUpper(c.chainName) } +func (c *CosmosIndexer) GetNetworkType() enum.NetworkType { return enum.NetworkTypeCosmos } +func (c *CosmosIndexer) GetNetworkInternalCode() string { return c.config.InternalCode } + +func (c *CosmosIndexer) GetLatestBlockNumber(ctx context.Context) (uint64, error) { + var latest uint64 + err := c.failover.ExecuteWithRetry(ctx, func(client cosmos.CosmosAPI) error { + height, err := client.GetLatestHeight(ctx) + latest = height + return err + }) + return latest, err +} + +func (c *CosmosIndexer) GetBlock(ctx context.Context, number uint64) (*types.Block, error) { + var ( + blockData *cosmos.BlockResponse + blockResult *cosmos.BlockResultsResponse + ) + + if err := c.failover.ExecuteWithRetry(ctx, func(client cosmos.CosmosAPI) error { + b, err := client.GetBlock(ctx, number) + blockData = b + return err + }); err != nil { + return nil, fmt.Errorf("get cosmos block %d failed: %w", number, err) + } + + if err := c.failover.ExecuteWithRetry(ctx, func(client cosmos.CosmosAPI) error { + r, err := client.GetBlockResults(ctx, number) + blockResult = r + return err + }); err != nil { + if isMissingFinalizeBlockResponsesError(err) { + fallback, fallbackErr := c.getBlockResultsByTxLookup(ctx, number, blockData.Block.Data.Txs) + if fallbackErr != nil { + return nil, fmt.Errorf( + "get cosmos block_results %d failed: %w; tx lookup fallback failed: %w", + number, + err, + fallbackErr, + ) + } + blockResult = fallback + } else { + return nil, fmt.Errorf("get cosmos block_results %d failed: %w", number, err) + } + } + + if blockData == nil { + return nil, fmt.Errorf("cosmos block %d not found", number) + } + + return c.convertBlock(blockData, blockResult) +} + +func (c *CosmosIndexer) GetBlocks( + ctx context.Context, + from, to uint64, + isParallel bool, +) ([]BlockResult, error) { + if to < from { + return nil, fmt.Errorf("invalid range: from %d > to %d", from, to) + } + + nums := make([]uint64, 0, to-from+1) + for n := from; n <= to; n++ { + nums = append(nums, n) + } + + workers := 1 + if isParallel { + workers = c.config.Throttle.Concurrency + } + return c.getBlocks(ctx, nums, workers) +} + +func (c *CosmosIndexer) GetBlocksByNumbers( + ctx context.Context, + blockNumbers []uint64, +) ([]BlockResult, error) { + return c.getBlocks(ctx, blockNumbers, c.config.Throttle.Concurrency) +} + +func (c *CosmosIndexer) getBlocks( + ctx context.Context, + blockNumbers []uint64, + workers int, +) ([]BlockResult, error) { + if len(blockNumbers) == 0 { + return nil, nil + } + if workers <= 0 { + workers = 1 + } + workers = min(workers, len(blockNumbers)) + + results := make([]BlockResult, len(blockNumbers)) + + type job struct { + index int + num uint64 + } + + jobs := make(chan job, workers*2) + var wg sync.WaitGroup + + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := range jobs { + block, err := c.GetBlock(ctx, j.num) + results[j.index] = BlockResult{ + Number: j.num, + Block: block, + } + if err != nil { + results[j.index].Error = &Error{ + ErrorType: ErrorTypeUnknown, + Message: err.Error(), + } + } + } + }() + } + + go func() { + defer close(jobs) + for i, num := range blockNumbers { + select { + case <-ctx.Done(): + return + case jobs <- job{index: i, num: num}: + } + } + }() + + wg.Wait() + if ctx.Err() != nil { + return nil, ctx.Err() + } + + var firstErr error + for _, res := range results { + if res.Error != nil { + firstErr = fmt.Errorf("block %d: %s", res.Number, res.Error.Message) + break + } + } + return results, firstErr +} + +func (c *CosmosIndexer) IsHealthy() bool { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, err := c.GetLatestBlockNumber(ctx) + return err == nil +} + +func (c *CosmosIndexer) convertBlock( + blockData *cosmos.BlockResponse, + blockResults *cosmos.BlockResultsResponse, +) (*types.Block, error) { + height, err := strconv.ParseUint(blockData.Block.Header.Height, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid block height %q: %w", blockData.Block.Header.Height, err) + } + + ts, err := parseCosmosBlockTime(blockData.Block.Header.Time) + if err != nil { + return nil, err + } + + txs := c.extractTransferTransactions( + height, + ts, + hashCosmosTxs(blockData.Block.Data.Txs), + blockResults, + ) + + return &types.Block{ + Number: height, + Hash: blockData.BlockID.Hash, + ParentHash: blockData.Block.Header.LastBlockID.Hash, + Timestamp: ts, + Transactions: txs, + }, nil +} + +func (c *CosmosIndexer) extractTransferTransactions( + blockNumber uint64, + timestamp uint64, + txHashes []string, + blockResults *cosmos.BlockResultsResponse, +) []types.Transaction { + if blockResults == nil || len(txHashes) == 0 { + return nil + } + + results := blockResults.TxsResults + limit := min(len(txHashes), len(results)) + if limit == 0 { + return nil + } + + transactions := make([]types.Transaction, 0, limit) + for i := 0; i < limit; i++ { + txResult := results[i] + if txResult.Code != 0 { + continue + } + + fee := extractCosmosFee(txResult.Events) + transfers := extractCosmosTransfers(txResult.Events) + + feeAssigned := false + for _, transfer := range transfers { + if transfer.sender == "" || transfer.recipient == "" || transfer.amount == "" { + continue + } + if c.pubkeyStore != nil && !c.pubkeyStore.Exist(enum.NetworkTypeCosmos, transfer.recipient) { + continue + } + + txType, assetAddress := c.classifyDenom(transfer.denom) + tx := types.Transaction{ + TxHash: txHashes[i], + NetworkId: c.config.NetworkId, + BlockNumber: blockNumber, + FromAddress: transfer.sender, + ToAddress: transfer.recipient, + AssetAddress: assetAddress, + Amount: transfer.amount, + Type: txType, + Timestamp: timestamp, + Confirmations: 1, + Status: types.StatusConfirmed, + } + + if !feeAssigned { + tx.TxFee = fee + feeAssigned = true + } + + transactions = append(transactions, tx) + } + } + + return transactions +} + +type cosmosTransfer struct { + sender string + recipient string + amount string + denom string +} + +func extractCosmosTransfers(events []cosmos.Event) []cosmosTransfer { + transfers := make([]cosmosTransfer, 0) + + for _, event := range events { + if event.Type != "transfer" { + continue + } + + var ( + senders []string + recipients []string + amounts []string + ) + + for _, attr := range event.Attributes { + key := strings.ToLower(strings.TrimSpace(decodeCosmosEventValue(attr.Key))) + value := strings.TrimSpace(decodeCosmosEventValue(attr.Value)) + if value == "" { + continue + } + + switch key { + case "sender": + senders = append(senders, value) + case "recipient": + recipients = append(recipients, value) + case "amount": + amounts = append(amounts, value) + } + } + + count := min(len(senders), len(recipients), len(amounts)) + for i := 0; i < count; i++ { + coins := parseCosmosCoins(amounts[i]) + for _, coin := range coins { + transfers = append(transfers, cosmosTransfer{ + sender: senders[i], + recipient: recipients[i], + amount: coin.Amount, + denom: coin.Denom, + }) + } + } + } + + return transfers +} + +type cosmosCoin struct { + Amount string + Denom string +} + +func parseCosmosCoins(raw string) []cosmosCoin { + parts := strings.Split(raw, ",") + coins := make([]cosmosCoin, 0, len(parts)) + + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + + matches := cosmosCoinRegexp.FindStringSubmatch(part) + if len(matches) != 3 { + continue + } + + coins = append(coins, cosmosCoin{ + Amount: matches[1], + Denom: matches[2], + }) + } + + return coins +} + +func extractCosmosFee(events []cosmos.Event) decimal.Decimal { + for _, event := range events { + if event.Type != "tx" { + continue + } + for _, attr := range event.Attributes { + key := strings.ToLower(strings.TrimSpace(decodeCosmosEventValue(attr.Key))) + if key != "fee" { + continue + } + + value := strings.TrimSpace(decodeCosmosEventValue(attr.Value)) + if value == "" { + continue + } + + coins := parseCosmosCoins(value) + if len(coins) == 0 { + continue + } + + fee, err := decimal.NewFromString(coins[0].Amount) + if err == nil { + return fee + } + } + } + + return decimal.Zero +} + +func decodeCosmosEventValue(raw string) string { + if raw == "" { + return "" + } + + if decoded, err := base64.StdEncoding.DecodeString(raw); err == nil { + if isReadableText(decoded) { + return string(decoded) + } + } + if decoded, err := base64.RawStdEncoding.DecodeString(raw); err == nil { + if isReadableText(decoded) { + return string(decoded) + } + } + return raw +} + +func isReadableText(b []byte) bool { + if len(b) == 0 || !utf8.Valid(b) { + return false + } + + for _, r := range string(b) { + switch { + case r == '\n' || r == '\r' || r == '\t': + continue + case r < 32 || r == 127: + return false + } + } + return true +} + +func hashCosmosTxs(encodedTxs []string) []string { + hashes := make([]string, 0, len(encodedTxs)) + for _, encoded := range encodedTxs { + if encoded == "" { + continue + } + + raw, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + raw, err = base64.RawStdEncoding.DecodeString(encoded) + if err != nil { + continue + } + } + + sum := sha256.Sum256(raw) + hashes = append(hashes, strings.ToUpper(hex.EncodeToString(sum[:]))) + } + return hashes +} + +func parseCosmosBlockTime(raw string) (uint64, error) { + t, err := time.Parse(time.RFC3339Nano, raw) + if err != nil { + return 0, fmt.Errorf("invalid block time %q: %w", raw, err) + } + return uint64(t.Unix()), nil +} + +func (c *CosmosIndexer) classifyDenom(denom string) (constant.TxType, string) { + nativeDenom := c.nativeDenom() + if nativeDenom != "" && denom == nativeDenom { + return constant.TxTypeNativeTransfer, "" + } + return constant.TxTypeTokenTransfer, denom +} + +func (c *CosmosIndexer) nativeDenom() string { + if d := strings.TrimSpace(c.config.NativeDenom); d != "" { + return d + } + + network := strings.ToLower(c.config.NetworkId + " " + c.chainName) + + switch { + case strings.Contains(network, cosmosNetworkHintOsmosis), containsCosmosNetworkToken(network, cosmosNetworkHintOsmo): + return cosmosNativeDenomOsmosis + case strings.Contains(network, cosmosNetworkHintHub), + strings.Contains(network, cosmosNetworkHintAtom), + strings.Contains(network, cosmosNetworkHintCosmos1), + (containsCosmosNetworkToken(network, cosmosNetworkHintCosmos) && + containsCosmosNetworkToken(network, cosmosNetworkHintHubWord)): + return cosmosNativeDenomAtom + default: + return "" + } +} + +func containsCosmosNetworkToken(network, token string) bool { + if network == "" || token == "" { + return false + } + + parts := strings.FieldsFunc(network, func(r rune) bool { + return (r < 'a' || r > 'z') && (r < '0' || r > '9') + }) + + for _, part := range parts { + if part == token { + return true + } + } + return false +} + +func (c *CosmosIndexer) getBlockResultsByTxLookup( + ctx context.Context, + height uint64, + encodedTxs []string, +) (*cosmos.BlockResultsResponse, error) { + hashes := hashCosmosTxs(encodedTxs) + if len(hashes) == 0 { + return &cosmos.BlockResultsResponse{ + Height: strconv.FormatUint(height, 10), + TxsResults: []cosmos.TxResult{}, + }, nil + } + + workers := c.config.Throttle.Concurrency + if workers <= 0 { + workers = 1 + } + workers = min(workers, len(hashes)) + + type job struct { + index int + hash string + } + + results := make([]cosmos.TxResult, len(hashes)) + errs := make([]error, len(hashes)) + jobs := make(chan job, workers*2) + var wg sync.WaitGroup + + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := range jobs { + var txResp *cosmos.TxResponse + err := c.failover.ExecuteWithRetry(ctx, func(client cosmos.CosmosAPI) error { + resp, err := client.GetTxByHash(ctx, j.hash) + txResp = resp + return err + }) + if err != nil { + errs[j.index] = fmt.Errorf("lookup tx %s: %w", j.hash, err) + continue + } + if txResp == nil { + errs[j.index] = fmt.Errorf("lookup tx %s: empty response", j.hash) + continue + } + + results[j.index] = txResp.TxResult + } + }() + } + + go func() { + defer close(jobs) + for i, hash := range hashes { + select { + case <-ctx.Done(): + return + case jobs <- job{index: i, hash: hash}: + } + } + }() + + wg.Wait() + if ctx.Err() != nil { + return nil, ctx.Err() + } + for _, err := range errs { + if err != nil { + return nil, err + } + } + + return &cosmos.BlockResultsResponse{ + Height: strconv.FormatUint(height, 10), + TxsResults: results, + }, nil +} + +func isMissingFinalizeBlockResponsesError(err error) bool { + if err == nil { + return false + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "not persisting finalize block responses") +} diff --git a/internal/indexer/cosmos_test.go b/internal/indexer/cosmos_test.go new file mode 100644 index 0000000..490efa0 --- /dev/null +++ b/internal/indexer/cosmos_test.go @@ -0,0 +1,229 @@ +package indexer + +import ( + "encoding/base64" + "testing" + + "github.com/fystack/multichain-indexer/internal/rpc/cosmos" + "github.com/fystack/multichain-indexer/pkg/common/config" + "github.com/fystack/multichain-indexer/pkg/common/constant" + "github.com/fystack/multichain-indexer/pkg/common/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCosmosConvertBlock_ParsesTransfersAndFee(t *testing.T) { + txPayload := []byte("tx-one") + txEncoded := base64.StdEncoding.EncodeToString(txPayload) + + idx := &CosmosIndexer{ + chainName: "osmosis_mainnet", + config: config.ChainConfig{ + NetworkId: "osmosis-1", + NativeDenom: "uosmo", + }, + } + + blockData := &cosmos.BlockResponse{ + BlockID: cosmos.BlockID{Hash: "BLOCK_HASH"}, + Block: cosmos.Block{ + Header: cosmos.BlockHeader{ + Height: "100", + Time: "2026-02-13T00:00:00Z", + LastBlockID: cosmos.LastBlockID{ + Hash: "PARENT_HASH", + }, + }, + Data: cosmos.BlockData{ + Txs: []string{txEncoded}, + }, + }, + } + + blockResults := &cosmos.BlockResultsResponse{ + Height: "100", + TxsResults: []cosmos.TxResult{ + { + Code: 0, + Events: []cosmos.Event{ + { + Type: "transfer", + Attributes: []cosmos.EventAttribute{ + {Key: b64("sender"), Value: b64("osmo1sender")}, + {Key: b64("recipient"), Value: b64("osmo1recipient")}, + {Key: b64("amount"), Value: b64("123uosmo,45ibc/ABCDEF")}, + }, + }, + { + Type: "tx", + Attributes: []cosmos.EventAttribute{ + {Key: b64("fee"), Value: b64("7uosmo")}, + }, + }, + }, + }, + }, + } + + block, err := idx.convertBlock(blockData, blockResults) + require.NoError(t, err) + require.NotNil(t, block) + require.Len(t, block.Transactions, 2) + + assert.Equal(t, uint64(100), block.Number) + assert.Equal(t, "BLOCK_HASH", block.Hash) + assert.Equal(t, "PARENT_HASH", block.ParentHash) + assert.Equal(t, uint64(1770940800), block.Timestamp) + + expectedHash := hashCosmosTxs([]string{txEncoded})[0] + + nativeTx := block.Transactions[0] + assert.Equal(t, expectedHash, nativeTx.TxHash) + assert.Equal(t, "osmosis-1", nativeTx.NetworkId) + assert.Equal(t, "osmo1sender", nativeTx.FromAddress) + assert.Equal(t, "osmo1recipient", nativeTx.ToAddress) + assert.Equal(t, "123", nativeTx.Amount) + assert.Equal(t, constant.TxTypeNativeTransfer, nativeTx.Type) + assert.Equal(t, "", nativeTx.AssetAddress) + assert.Equal(t, "7", nativeTx.TxFee.String()) + assert.Equal(t, uint64(1), nativeTx.Confirmations) + assert.Equal(t, types.StatusConfirmed, nativeTx.Status) + + tokenTx := block.Transactions[1] + assert.Equal(t, expectedHash, tokenTx.TxHash) + assert.Equal(t, "45", tokenTx.Amount) + assert.Equal(t, constant.TxTypeTokenTransfer, tokenTx.Type) + assert.Equal(t, "ibc/ABCDEF", tokenTx.AssetAddress) + assert.Equal(t, "0", tokenTx.TxFee.String()) +} + +func TestExtractCosmosTransfers_PlainEventAttributes(t *testing.T) { + events := []cosmos.Event{ + { + Type: "transfer", + Attributes: []cosmos.EventAttribute{ + {Key: "sender", Value: "osmo1a"}, + {Key: "recipient", Value: "osmo1b"}, + {Key: "amount", Value: "100uosmo"}, + }, + }, + } + + transfers := extractCosmosTransfers(events) + require.Len(t, transfers, 1) + assert.Equal(t, "osmo1a", transfers[0].sender) + assert.Equal(t, "osmo1b", transfers[0].recipient) + assert.Equal(t, "100", transfers[0].amount) + assert.Equal(t, "uosmo", transfers[0].denom) +} + +func TestCosmosClassifyDenom_UsesConfiguredNativeDenom(t *testing.T) { + idx := &CosmosIndexer{ + chainName: "celestia_mainnet", + config: config.ChainConfig{ + NetworkId: "celestia", + NativeDenom: "utia", + }, + } + + txType, asset := idx.classifyDenom("utia") + assert.Equal(t, constant.TxTypeNativeTransfer, txType) + assert.Equal(t, "", asset) + + txType, asset = idx.classifyDenom("ibc/XYZ") + assert.Equal(t, constant.TxTypeTokenTransfer, txType) + assert.Equal(t, "ibc/XYZ", asset) +} + +func TestCosmosClassifyDenom_InfersCosmosHubNativeDenom(t *testing.T) { + idx := &CosmosIndexer{ + chainName: "cosmoshub_mainnet", + config: config.ChainConfig{ + NetworkId: "cosmoshub-4", + }, + } + + txType, asset := idx.classifyDenom("uatom") + assert.Equal(t, constant.TxTypeNativeTransfer, txType) + assert.Equal(t, "", asset) + + txType, asset = idx.classifyDenom("ibc/ATOM") + assert.Equal(t, constant.TxTypeTokenTransfer, txType) + assert.Equal(t, "ibc/ATOM", asset) +} + +func TestCosmosClassifyDenom_InfersCosmosHubFromCosmos1Hint(t *testing.T) { + idx := &CosmosIndexer{ + chainName: "cosmos1_hub_mainnet", + config: config.ChainConfig{ + NetworkId: "mainnet", + }, + } + + txType, asset := idx.classifyDenom("uatom") + assert.Equal(t, constant.TxTypeNativeTransfer, txType) + assert.Equal(t, "", asset) +} + +func TestCosmosConvertBlock_SupportsCosmosHubAddresses(t *testing.T) { + txPayload := []byte("tx-cosmos-hub") + txEncoded := base64.StdEncoding.EncodeToString(txPayload) + + idx := &CosmosIndexer{ + chainName: "cosmoshub_mainnet", + config: config.ChainConfig{ + NetworkId: "cosmoshub-4", + }, + } + + blockData := &cosmos.BlockResponse{ + BlockID: cosmos.BlockID{Hash: "BLOCK_HASH"}, + Block: cosmos.Block{ + Header: cosmos.BlockHeader{ + Height: "200", + Time: "2026-02-20T00:00:00Z", + LastBlockID: cosmos.LastBlockID{ + Hash: "PARENT_HASH", + }, + }, + Data: cosmos.BlockData{ + Txs: []string{txEncoded}, + }, + }, + } + + blockResults := &cosmos.BlockResultsResponse{ + Height: "200", + TxsResults: []cosmos.TxResult{ + { + Code: 0, + Events: []cosmos.Event{ + { + Type: "transfer", + Attributes: []cosmos.EventAttribute{ + {Key: "sender", Value: "cosmos1sender"}, + {Key: "recipient", Value: "cosmos1recipient"}, + {Key: "amount", Value: "999uatom"}, + }, + }, + }, + }, + }, + } + + block, err := idx.convertBlock(blockData, blockResults) + require.NoError(t, err) + require.NotNil(t, block) + require.Len(t, block.Transactions, 1) + + tx := block.Transactions[0] + assert.Equal(t, "cosmos1sender", tx.FromAddress) + assert.Equal(t, "cosmos1recipient", tx.ToAddress) + assert.Equal(t, "999", tx.Amount) + assert.Equal(t, constant.TxTypeNativeTransfer, tx.Type) + assert.Equal(t, "", tx.AssetAddress) +} + +func b64(v string) string { + return base64.StdEncoding.EncodeToString([]byte(v)) +} diff --git a/internal/rpc/cosmos/api.go b/internal/rpc/cosmos/api.go new file mode 100644 index 0000000..18a21b9 --- /dev/null +++ b/internal/rpc/cosmos/api.go @@ -0,0 +1,16 @@ +package cosmos + +import ( + "context" + + "github.com/fystack/multichain-indexer/internal/rpc" +) + +type CosmosAPI interface { + rpc.NetworkClient + + GetLatestHeight(ctx context.Context) (uint64, error) + GetBlock(ctx context.Context, height uint64) (*BlockResponse, error) + GetBlockResults(ctx context.Context, height uint64) (*BlockResultsResponse, error) + GetTxByHash(ctx context.Context, hash string) (*TxResponse, error) +} diff --git a/internal/rpc/cosmos/client.go b/internal/rpc/cosmos/client.go new file mode 100644 index 0000000..9805bc9 --- /dev/null +++ b/internal/rpc/cosmos/client.go @@ -0,0 +1,118 @@ +package cosmos + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "github.com/fystack/multichain-indexer/internal/rpc" + "github.com/fystack/multichain-indexer/pkg/ratelimiter" +) + +type Client struct { + base *rpc.BaseClient +} + +func NewCosmosClient( + baseURL string, + auth *rpc.AuthConfig, + timeout time.Duration, + rl *ratelimiter.PooledRateLimiter, +) *Client { + return &Client{ + base: rpc.NewBaseClient(baseURL, rpc.NetworkCosmos, rpc.ClientTypeREST, auth, timeout, rl), + } +} + +func (c *Client) CallRPC(ctx context.Context, method string, params any) (*rpc.RPCResponse, error) { + return c.base.CallRPC(ctx, method, params) +} + +func (c *Client) Do( + ctx context.Context, + method, endpoint string, + body any, + params map[string]string, +) ([]byte, error) { + return c.base.Do(ctx, method, endpoint, body, params) +} + +func (c *Client) GetNetworkType() string { return c.base.GetNetworkType() } +func (c *Client) GetClientType() string { return c.base.GetClientType() } +func (c *Client) GetURL() string { return c.base.GetURL() } +func (c *Client) Close() error { return c.base.Close() } + +func (c *Client) GetLatestHeight(ctx context.Context) (uint64, error) { + result, err := getResponse[StatusResponse](ctx, c, "/status", nil) + if err != nil { + return 0, err + } + height, err := strconv.ParseUint(result.SyncInfo.LatestBlockHeight, 10, 64) + if err != nil { + return 0, fmt.Errorf("invalid latest block height: %w", err) + } + return height, nil +} + +func (c *Client) GetBlock(ctx context.Context, height uint64) (*BlockResponse, error) { + return getResponse[BlockResponse](ctx, c, "/block", map[string]string{ + "height": strconv.FormatUint(height, 10), + }) +} + +func (c *Client) GetBlockResults( + ctx context.Context, + height uint64, +) (*BlockResultsResponse, error) { + return getResponse[BlockResultsResponse](ctx, c, "/block_results", map[string]string{ + "height": strconv.FormatUint(height, 10), + }) +} + +func (c *Client) GetTxByHash(ctx context.Context, hash string) (*TxResponse, error) { + hash = strings.TrimSpace(hash) + if hash == "" { + return nil, fmt.Errorf("tx hash is empty") + } + if !strings.HasPrefix(hash, "0x") && !strings.HasPrefix(hash, "0X") { + hash = "0x" + hash + } + + return getResponse[TxResponse](ctx, c, "/tx", map[string]string{ + "hash": hash, + "prove": "false", + }) +} + +func getResponse[T any]( + ctx context.Context, + client *Client, + endpoint string, + params map[string]string, +) (*T, error) { + raw, err := client.base.Do(ctx, http.MethodGet, endpoint, nil, params) + if err != nil { + return nil, err + } + + var response rpc.RPCResponse + if err := json.Unmarshal(raw, &response); err != nil { + return nil, fmt.Errorf("decode %s response: %w", endpoint, err) + } + if response.Error != nil { + return nil, fmt.Errorf("%s failed: %w", endpoint, response.Error) + } + if len(response.Result) == 0 { + return nil, fmt.Errorf("%s returned empty result", endpoint) + } + + var result T + if err := json.Unmarshal(response.Result, &result); err != nil { + return nil, fmt.Errorf("decode %s result: %w", endpoint, err) + } + return &result, nil +} diff --git a/internal/rpc/cosmos/types.go b/internal/rpc/cosmos/types.go new file mode 100644 index 0000000..84adc5f --- /dev/null +++ b/internal/rpc/cosmos/types.go @@ -0,0 +1,65 @@ +package cosmos + +type StatusResponse struct { + SyncInfo SyncInfo `json:"sync_info"` +} + +type SyncInfo struct { + LatestBlockHeight string `json:"latest_block_height"` +} + +type BlockResponse struct { + BlockID BlockID `json:"block_id"` + Block Block `json:"block"` +} + +type BlockID struct { + Hash string `json:"hash"` +} + +type Block struct { + Header BlockHeader `json:"header"` + Data BlockData `json:"data"` +} + +type BlockHeader struct { + Height string `json:"height"` + Time string `json:"time"` + LastBlockID LastBlockID `json:"last_block_id"` +} + +type LastBlockID struct { + Hash string `json:"hash"` +} + +type BlockData struct { + Txs []string `json:"txs"` +} + +type BlockResultsResponse struct { + Height string `json:"height"` + TxsResults []TxResult `json:"txs_results"` +} + +type TxResponse struct { + Hash string `json:"hash"` + Height string `json:"height"` + TxResult TxResult `json:"tx_result"` +} + +type TxResult struct { + Code uint32 `json:"code"` + Log string `json:"log"` + Events []Event `json:"events"` +} + +type Event struct { + Type string `json:"type"` + Attributes []EventAttribute `json:"attributes"` +} + +type EventAttribute struct { + Key string `json:"key"` + Value string `json:"value"` + Index bool `json:"index,omitempty"` +} diff --git a/internal/rpc/types.go b/internal/rpc/types.go index 729f5e6..bf5291a 100644 --- a/internal/rpc/types.go +++ b/internal/rpc/types.go @@ -20,6 +20,7 @@ const ( NetworkSolana = "solana" // Solana blockchain NetworkTron = "tron" // Tron blockchain NetworkBitcoin = "bitcoin" // Bitcoin blockchain + NetworkCosmos = "cosmos" // Cosmos SDK / CometBFT chains NetworkGeneric = "generic" // Generic/unknown blockchain type ) diff --git a/internal/worker/factory.go b/internal/worker/factory.go index 17feecf..5fe414a 100644 --- a/internal/worker/factory.go +++ b/internal/worker/factory.go @@ -8,6 +8,7 @@ import ( tonIndexer "github.com/fystack/multichain-indexer/internal/indexer/ton" "github.com/fystack/multichain-indexer/internal/rpc" "github.com/fystack/multichain-indexer/internal/rpc/bitcoin" + "github.com/fystack/multichain-indexer/internal/rpc/cosmos" "github.com/fystack/multichain-indexer/internal/rpc/evm" "github.com/fystack/multichain-indexer/internal/rpc/solana" "github.com/fystack/multichain-indexer/internal/rpc/sui" @@ -291,6 +292,44 @@ func buildSuiIndexer( return indexer.NewSuiIndexer(chainName, chainCfg, failover, pubkeyStore) } +// buildCosmosIndexer constructs a Cosmos indexer with failover and providers. +func buildCosmosIndexer( + chainName string, + chainCfg config.ChainConfig, + mode WorkerMode, + pubkeyStore pubkeystore.Store, +) indexer.Indexer { + failover := rpc.NewFailover[cosmos.CosmosAPI](nil) + + rl := ratelimiter.GetOrCreateSharedPooledRateLimiter( + chainName, chainCfg.Throttle.RPS, chainCfg.Throttle.Burst, + ) + + for i, node := range chainCfg.Nodes { + client := cosmos.NewCosmosClient( + node.URL, + &rpc.AuthConfig{ + Type: rpc.AuthType(node.Auth.Type), + Key: node.Auth.Key, + Value: node.Auth.Value, + }, + chainCfg.Client.Timeout, + rl, + ) + + failover.AddProvider(&rpc.Provider{ + Name: chainName + "-" + strconv.Itoa(i+1), + URL: node.URL, + Network: chainName, + ClientType: rpc.ClientTypeREST, + Client: client, + State: rpc.StateHealthy, + }) + } + + return indexer.NewCosmosIndexer(chainName, chainCfg, failover, pubkeyStore) +} + // buildTonPollingWorker constructs a TON polling worker with failover. // TON uses account-based polling instead of block-based indexing. func buildTonPollingWorker( @@ -405,6 +444,8 @@ func CreateManagerWithWorkers( idxr = buildSolanaIndexer(chainName, chainCfg, ModeRegular, pubkeyStore) case enum.NetworkTypeSui: idxr = buildSuiIndexer(chainName, chainCfg, ModeRegular, pubkeyStore) + case enum.NetworkTypeCosmos: + idxr = buildCosmosIndexer(chainName, chainCfg, ModeRegular, pubkeyStore) case enum.NetworkTypeTon: tonW := buildTonPollingWorker(ctx, chainName, chainCfg, kvstore, redisClient, db, emitter) if tonW != nil { diff --git a/pkg/addressbloomfilter/inmemory.go b/pkg/addressbloomfilter/inmemory.go index 507556f..644e605 100644 --- a/pkg/addressbloomfilter/inmemory.go +++ b/pkg/addressbloomfilter/inmemory.go @@ -47,6 +47,7 @@ func (abf *addressBloomFilter) Initialize(ctx context.Context) error { enum.NetworkTypeTron, enum.NetworkTypeBtc, enum.NetworkTypeSui, + enum.NetworkTypeCosmos, } for _, addrType := range types { diff --git a/pkg/addressbloomfilter/redis.go b/pkg/addressbloomfilter/redis.go index 32e4f3c..91d26d9 100644 --- a/pkg/addressbloomfilter/redis.go +++ b/pkg/addressbloomfilter/redis.go @@ -73,6 +73,7 @@ func (rbf *redisBloomFilter) Initialize(ctx context.Context) error { enum.NetworkTypeBtc, enum.NetworkTypeSol, enum.NetworkTypeSui, + enum.NetworkTypeCosmos, } for _, addrType := range types { diff --git a/pkg/common/config/chains.go b/pkg/common/config/chains.go index 572ac14..08202e2 100644 --- a/pkg/common/config/chains.go +++ b/pkg/common/config/chains.go @@ -2,6 +2,7 @@ package config import ( "fmt" + "strings" "dario.cat/mergo" ) @@ -47,6 +48,12 @@ func (c Chains) OverrideFromLatest(names []string) { // ApplyDefaults merges global defaults into all chain configs. func (c Chains) ApplyDefaults(def Defaults) error { for name, chain := range c { + if strings.TrimSpace(chain.NetworkId) == "" { + chain.NetworkId = name + } + if strings.TrimSpace(chain.InternalCode) == "" { + chain.InternalCode = strings.ToUpper(name) + } if !chain.FromLatest { chain.FromLatest = def.FromLatest } diff --git a/pkg/common/config/types.go b/pkg/common/config/types.go index 29118d5..d115bb5 100644 --- a/pkg/common/config/types.go +++ b/pkg/common/config/types.go @@ -37,6 +37,7 @@ type ChainConfig struct { Name string `yaml:"-"` NetworkId string `yaml:"network_id"` InternalCode string `yaml:"internal_code"` + NativeDenom string `yaml:"native_denom"` Type enum.NetworkType `yaml:"type" validate:"required"` FromLatest bool `yaml:"from_latest"` StartBlock int `yaml:"start_block" validate:"min=0"` diff --git a/pkg/common/enum/enum.go b/pkg/common/enum/enum.go index c123c9f..330809b 100644 --- a/pkg/common/enum/enum.go +++ b/pkg/common/enum/enum.go @@ -19,13 +19,14 @@ const ( ) const ( - NetworkTypeEVM NetworkType = "evm" - NetworkTypeTron NetworkType = "tron" - NetworkTypeBtc NetworkType = "btc" - NetworkTypeSol NetworkType = "sol" - NetworkTypeApt NetworkType = "apt" - NetworkTypeSui NetworkType = "sui" - NetworkTypeTon NetworkType = "ton" + NetworkTypeEVM NetworkType = "evm" + NetworkTypeTron NetworkType = "tron" + NetworkTypeBtc NetworkType = "btc" + NetworkTypeSol NetworkType = "sol" + NetworkTypeApt NetworkType = "apt" + NetworkTypeSui NetworkType = "sui" + NetworkTypeTon NetworkType = "ton" + NetworkTypeCosmos NetworkType = "cosmos" ) const ( diff --git a/sql/wallet_address.sql b/sql/wallet_address.sql index 5ed2f8f..8aba454 100644 --- a/sql/wallet_address.sql +++ b/sql/wallet_address.sql @@ -1,12 +1,41 @@ --- Create wallet_address table +-- Remove enum constraints (if old schema existed) +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'wallet_addresses' + AND column_name = 'type' + AND udt_name = 'address_type' + ) THEN + ALTER TABLE wallet_addresses + ALTER COLUMN type TYPE VARCHAR(64) USING type::text; + END IF; + + IF EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'wallet_addresses' + AND column_name = 'standard' + AND udt_name = 'address_standard' + ) THEN + ALTER TABLE wallet_addresses + ALTER COLUMN standard TYPE VARCHAR(64) USING standard::text; + END IF; +END $$; + +DROP TYPE IF EXISTS address_type; +DROP TYPE IF EXISTS address_standard; + +-- Create wallet_address table without enum constraints CREATE TABLE IF NOT EXISTS wallet_addresses ( id BIGSERIAL PRIMARY KEY, created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, deleted_at TIMESTAMP WITH TIME ZONE, address VARCHAR(255) NOT NULL, - type address_type NOT NULL, - standard address_standard + type VARCHAR(64) NOT NULL, + standard VARCHAR(64) ); -- Create unique index on address @@ -17,41 +46,11 @@ CREATE INDEX IF NOT EXISTS idx_wallet_address_type ON wallet_addresses (type); CREATE INDEX IF NOT EXISTS idx_wallet_address_standard ON wallet_addresses (standard); CREATE INDEX IF NOT EXISTS idx_wallet_address_created_at ON wallet_addresses (created_at); --- Create enum types if they don't exist -DO $$ BEGIN - CREATE TYPE address_type AS ENUM ( - 'evm', - 'btc', - 'sol', - 'aptos', - 'tron' - ); -EXCEPTION - WHEN duplicate_object THEN null; -END $$; - -DO $$ BEGIN - CREATE TYPE address_standard AS ENUM ( - 'erc20', - 'erc721', - 'erc1155', - 'native', - 'spl', - 'trc20', - 'trc721', - 'btc_p2pkh', - 'btc_p2sh', - 'btc_bech32' - ); -EXCEPTION - WHEN duplicate_object THEN null; -END $$; - -- Add comments for documentation COMMENT ON TABLE wallet_addresses IS 'Stores wallet addresses for different blockchain networks'; COMMENT ON COLUMN wallet_addresses.address IS 'The wallet address string'; -COMMENT ON COLUMN wallet_addresses.type IS 'The blockchain network type (evm, bitcoin, solana, tron)'; -COMMENT ON COLUMN wallet_addresses.standard IS 'The token standard (erc20, erc721, etc.)'; +COMMENT ON COLUMN wallet_addresses.type IS 'The blockchain network type (free text, no enum constraint)'; +COMMENT ON COLUMN wallet_addresses.standard IS 'The token standard (free text, no enum constraint)'; -- Insert sample data INSERT INTO wallet_addresses (address, type, standard) VALUES @@ -59,4 +58,5 @@ INSERT INTO wallet_addresses (address, type, standard) VALUES ('TT1j2adMBb6bF2K8C2LX1QkkmSXHjiaAfw', 'tron', 'trc20'), ('Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF', 'ton', 'native'), ('EQDKHZ7e70CzqdvZCC83Z4WVR8POC_ZB0J1Y4zo88G-zCXmC', 'ton', 'native'), -('EQBeab7D38RIwypegbN7YZgQzwDbb8QfMMwY8ouJc3qPl91M', 'ton', 'native'); +('EQBeab7D38RIwypegbN7YZgQzwDbb8QfMMwY8ouJc3qPl91M', 'ton', 'native') +ON CONFLICT (address) DO NOTHING;