diff --git a/go.mod b/go.mod index 0cfaf6a..75ce8d4 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( cel.dev/expr v0.24.0 // indirect filippo.io/edwards25519 v1.1.0 // indirect github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/StackExchange/wmi v1.2.1 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/bits-and-blooms/bitset v1.24.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect @@ -27,6 +28,7 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/fatih/structtag v1.2.0 // indirect github.com/go-logr/logr v1.4.3 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-sql-driver/mysql v1.9.3 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/google/cel-go v0.26.1 // indirect @@ -53,11 +55,14 @@ require ( github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/riza-io/grpc-go v0.2.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect github.com/spf13/cobra v1.9.1 // indirect github.com/spf13/pflag v1.0.7 // indirect github.com/sqlc-dev/sqlc v1.30.0 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect github.com/tetratelabs/wazero v1.9.0 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect github.com/wasilibs/go-pgquery v0.0.0-20250409022910-10ac41983c07 // indirect github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect diff --git a/go.sum b/go.sum index 97f1d79..dacd7b0 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,8 @@ github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1 github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/RoaringBitmap/roaring/v2 v2.14.4 h1:4aKySrrg9G/5oRtJ3TrZLObVqxgQ9f1znCRBwEwjuVw= github.com/RoaringBitmap/roaring/v2 v2.14.4/go.mod h1:oMvV6omPWr+2ifRdeZvVJyaz+aoEUopyv5iH0u/+wbY= +github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= +github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/participle/v2 v2.1.4 h1:W/H79S8Sat/krZ3el6sQMvMaahJ+XcM9WSI2naI7w2U= @@ -44,6 +46,9 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= @@ -132,6 +137,8 @@ github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= +github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -158,6 +165,10 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w= github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ= github.com/wasilibs/go-pgquery v0.0.0-20250409022910-10ac41983c07 h1:mJdDDPblDfPe7z7go8Dvv1AJQDI3eQ/5xith3q2mFlo= @@ -211,7 +222,11 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/sqlitestore.go b/sqlitestore.go index c4110cf..3669a45 100644 --- a/sqlitestore.go +++ b/sqlitestore.go @@ -14,6 +14,7 @@ import ( "github.com/Arkiv-Network/sqlite-bitmap-store/store" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/metrics" "github.com/golang-migrate/migrate/v4" "github.com/golang-migrate/migrate/v4/database/sqlite3" "github.com/golang-migrate/migrate/v4/source/iofs" @@ -23,6 +24,21 @@ import ( "github.com/Arkiv-Network/arkiv-events/events" ) +var ( + // Metrics for tracking operations + metricOperationStarted = metrics.NewRegisteredCounter("arkiv_store/operations_started", nil) + metricOperationSuccessful = metrics.NewRegisteredCounter("arkiv_store/operations_successful", nil) + metricCreates = metrics.NewRegisteredCounter("arkiv_store/creates", nil) + metricUpdates = metrics.NewRegisteredCounter("arkiv_store/updates", nil) + metricDeletes = metrics.NewRegisteredCounter("arkiv_store/deletes", nil) + metricExtends = metrics.NewRegisteredCounter("arkiv_store/extends", nil) + metricOwnerChanges = metrics.NewRegisteredCounter("arkiv_store/owner_changes", nil) + // Tracks operation duration (ms) using an exponential decay sample so the histogram + // is more responsive to recent performance by weighting newer measurements higher + // (sample size 100, alpha 0.4). + metricOperationTime = metrics.NewRegisteredHistogram("arkiv_store/operation_time_ms", nil, metrics.NewExpDecaySample(100, 0.4)) +) + type SQLiteStore struct { writePool *sql.DB readPool *sql.DB @@ -99,6 +115,14 @@ func (s *SQLiteStore) GetLastBlock(ctx context.Context) (uint64, error) { return store.New(s.writePool).GetLastBlock(ctx) } +type blockStats struct { + creates int + updates int + deletes int + extends int + ownerChanges int +} + func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.BatchIterator) error { for batch := range iterator { @@ -106,11 +130,8 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat return fmt.Errorf("failed to follow events: %w", batch.Error) } - totalCreates := 0 - totalUpdates := 0 - totalDeletes := 0 - totalExtends := 0 - totalOwnerChanges := 0 + // We will calculate totals for the log at the end, but track per-block for metrics + stats := make(map[uint64]*blockStats) err := func() error { @@ -138,20 +159,22 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat startTime := time.Now() + metricOperationStarted.Inc(1) + mainLoop: for _, block := range batch.Batch.Blocks { - updates := 0 - deletes := 0 - extends := 0 - creates := 0 - ownerChanges := 0 - if block.Number <= uint64(lastBlockFromDB) { s.log.Info("skipping block", "block", block.Number, "lastBlockFromDB", lastBlockFromDB) continue mainLoop } + // Initialize stats for this block + if _, ok := stats[block.Number]; !ok { + stats[block.Number] = &blockStats{} + } + blockStat := stats[block.Number] + updatesMap := map[common.Hash][]*events.OPUpdate{} for _, operation := range block.Operations { @@ -162,7 +185,6 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat } } - // blockNumber := block.Number operationLoop: for _, operation := range block.Operations { @@ -170,7 +192,7 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat case operation.Create != nil: // expiresAtBlock := blockNumber + operation.Create.BTL - creates++ + blockStat.creates++ key := operation.Create.Key stringAttributes := maps.Clone(operation.Create.StringAttributes) @@ -225,7 +247,6 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat } } case operation.Update != nil: - updates++ updates := updatesMap[operation.Update.Key] lastUpdate := updates[len(updates)-1] @@ -233,6 +254,7 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat if operation.Update != lastUpdate { continue operationLoop } + blockStat.updates++ key := operation.Update.Key.Bytes() @@ -319,7 +341,7 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat case operation.Delete != nil || operation.Expire != nil: - deletes++ + blockStat.deletes++ var key []byte if operation.Delete != nil { key = common.Hash(*operation.Delete).Bytes() @@ -363,7 +385,7 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat case operation.ExtendBTL != nil: - extends++ + blockStat.extends++ key := operation.ExtendBTL.Key.Bytes() @@ -403,7 +425,7 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat } case operation.ChangeOwner != nil: - ownerChanges++ + blockStat.ownerChanges++ key := operation.ChangeOwner.Key.Bytes() latestPayload, err := st.GetPayloadForEntityKey(ctx, key) @@ -449,12 +471,8 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat } - s.log.Info("block updated", "block", block.Number, "creates", creates, "updates", updates, "deletes", deletes, "extends", extends, "ownerChanges", ownerChanges) - totalCreates += creates - totalUpdates += updates - totalDeletes += deletes - totalExtends += extends - totalOwnerChanges += ownerChanges + // Log per block if needed, but we can now rely on the map for totals later + s.log.Info("block updated", "block", block.Number, "creates", blockStat.creates, "updates", blockStat.updates, "deletes", blockStat.deletes, "extends", blockStat.extends, "ownerChanges", blockStat.ownerChanges) } err = st.UpsertLastBlock(ctx, lastBlock) @@ -472,7 +490,55 @@ func (s *SQLiteStore) FollowEvents(ctx context.Context, iterator arkivevents.Bat return fmt.Errorf("failed to commit transaction: %w", err) } - s.log.Info("batch processed", "firstBlock", firstBlock, "lastBlock", lastBlock, "processingTime", time.Since(startTime).Milliseconds(), "creates", totalCreates, "updates", totalUpdates, "deletes", totalDeletes, "extends", totalExtends, "ownerChanges", totalOwnerChanges) + // Calculate batch totals for logging and update metrics PER BLOCK + var ( + totalCreates int + totalUpdates int + totalDeletes int + totalExtends int + totalOwnerChanges int + ) + + // Iterate blocks again to preserve order and update metrics per block + for _, block := range batch.Batch.Blocks { + if stat, ok := stats[block.Number]; ok { + totalCreates += stat.creates + totalUpdates += stat.updates + totalDeletes += stat.deletes + totalExtends += stat.extends + totalOwnerChanges += stat.ownerChanges + + // Update metrics specifically per block + if stat.creates > 0 { + metricCreates.Inc(int64(stat.creates)) + } + if stat.updates > 0 { + metricUpdates.Inc(int64(stat.updates)) + } + if stat.deletes > 0 { + metricDeletes.Inc(int64(stat.deletes)) + } + if stat.extends > 0 { + metricExtends.Inc(int64(stat.extends)) + } + if stat.ownerChanges > 0 { + metricOwnerChanges.Inc(int64(stat.ownerChanges)) + } + } + } + + metricOperationSuccessful.Inc(1) + metricOperationTime.Update(time.Since(startTime).Milliseconds()) + + s.log.Info("batch processed", + "firstBlock", firstBlock, + "lastBlock", lastBlock, + "processingTime", time.Since(startTime).Milliseconds(), + "creates", totalCreates, + "updates", totalUpdates, + "deletes", totalDeletes, + "extends", totalExtends, + "ownerChanges", totalOwnerChanges) return nil }()