feat: Add state-analyzer panel in the lab

backend/pkg/api/service.go

@@ -27,6 +27,7 @@ import (
 
 	beaconslotspb "github.com/ethpandaops/lab/backend/pkg/server/proto/beacon_slots"
 	configpb "github.com/ethpandaops/lab/backend/pkg/server/proto/config"
+	state_analytics_pb "github.com/ethpandaops/lab/backend/pkg/server/proto/state_analytics"
 	xatu_cbt_pb "github.com/ethpandaops/lab/backend/pkg/server/proto/xatu_cbt"
 )
 
@@ -43,11 +44,12 @@ type Service struct {
 	metrics       *metrics.Metrics
 
 	// gRPC connection to srv service
-	srvConn           *grpc.ClientConn
-	beaconSlotsClient beaconslotspb.BeaconSlotsClient
-	xatuCBTClient     xatu_cbt_pb.XatuCBTClient
-	configClient      configpb.ConfigServiceClient
-	publicV1Router    *v1rest.PublicRouter
+	srvConn              *grpc.ClientConn
+	beaconSlotsClient    beaconslotspb.BeaconSlotsClient
+	xatuCBTClient        xatu_cbt_pb.XatuCBTClient
+	configClient         configpb.ConfigServiceClient
+	stateAnalyticsClient state_analytics_pb.StateAnalyticsClient
+	publicV1Router       *v1rest.PublicRouter
 }
 
 // New creates a new api service
@@ -309,7 +311,8 @@ func (s *Service) initializeServices(ctx context.Context) error {
 	s.beaconSlotsClient = beaconslotspb.NewBeaconSlotsClient(conn)
 	s.xatuCBTClient = xatu_cbt_pb.NewXatuCBTClient(conn)
 	s.configClient = configpb.NewConfigServiceClient(conn)
-	s.publicV1Router = v1rest.NewPublicRouter(s.log, s.configClient, s.xatuCBTClient)
+	s.stateAnalyticsClient = state_analytics_pb.NewStateAnalyticsClient(conn)
+	s.publicV1Router = v1rest.NewPublicRouter(s.log, s.configClient, s.xatuCBTClient, s.stateAnalyticsClient)
 
 	return nil
 }

backend/pkg/api/v1/rest/handler.go

@@ -12,10 +12,13 @@ import (
 	apiv1 "github.com/ethpandaops/lab/backend/pkg/api/v1/proto"
 	"github.com/ethpandaops/lab/backend/pkg/api/v1/rest/middleware"
 	configpb "github.com/ethpandaops/lab/backend/pkg/server/proto/config"
+	state_analytics_pb "github.com/ethpandaops/lab/backend/pkg/server/proto/state_analytics"
 	xatu_cbt_pb "github.com/ethpandaops/lab/backend/pkg/server/proto/xatu_cbt"
 	cbtproto "github.com/ethpandaops/xatu-cbt/pkg/proto/clickhouse"
 	"github.com/gorilla/mux"
 	"github.com/sirupsen/logrus"
+	"google.golang.org/protobuf/encoding/protojson"
+	"google.golang.org/protobuf/proto"
 )
 
 // Common query parameter names used across REST API handlers
@@ -27,21 +30,24 @@ const (
 
 // PublicRouter handles public REST API v1 requests for all Lab endpoints.
 type PublicRouter struct {
-	log           logrus.FieldLogger
-	configClient  configpb.ConfigServiceClient
-	xatuCBTClient xatu_cbt_pb.XatuCBTClient
+	log                  logrus.FieldLogger
+	configClient         configpb.ConfigServiceClient
+	xatuCBTClient        xatu_cbt_pb.XatuCBTClient
+	stateAnalyticsClient state_analytics_pb.StateAnalyticsClient
 }
 
 // NewPublicRouter creates a new public REST router for API v1.
 func NewPublicRouter(
 	log logrus.FieldLogger,
 	configClient configpb.ConfigServiceClient,
 	xatuCBTClient xatu_cbt_pb.XatuCBTClient,
+	stateAnalyticsClient state_analytics_pb.StateAnalyticsClient,
 ) *PublicRouter {
 	return &PublicRouter{
-		log:           log.WithField("component", "public_rest_router_v1"),
-		configClient:  configClient,
-		xatuCBTClient: xatuCBTClient,
+		log:                  log.WithField("component", "public_rest_router_v1"),
+		configClient:         configClient,
+		xatuCBTClient:        xatuCBTClient,
+		stateAnalyticsClient: stateAnalyticsClient,
 	}
 }
 
@@ -70,22 +76,40 @@ func (r *PublicRouter) HandleGRPCError(w http.ResponseWriter, req *http.Request,
 
 // WriteJSONResponse safely writes a JSON response, handling encoding errors properly.
 func (r *PublicRouter) WriteJSONResponse(w http.ResponseWriter, req *http.Request, statusCode int, response interface{}) {
-	// Buffer the response first to catch encoding errors.
-	var buf bytes.Buffer
-	if err := json.NewEncoder(&buf).Encode(response); err != nil {
-		// Encoding failed - we haven't written anything yet, so we can send an error
-		r.log.WithError(err).WithField("type", response).Error("Failed to encode response")
-		r.HandleGRPCError(w, req, err)
-
-		return
+	var jsonBytes []byte
+	var err error
+
+	// Check if response is a protobuf message
+	if msg, ok := response.(proto.Message); ok {
+		// Use protojson for protobuf messages to properly serialize timestamps
+		marshaler := protojson.MarshalOptions{
+			UseProtoNames:   true,  // Use proto field names (snake_case)
+			EmitUnpopulated: false, // Don't emit zero values
+		}
+		jsonBytes, err = marshaler.Marshal(msg)
+		if err != nil {
+			r.log.WithError(err).WithField("type", response).Error("Failed to marshal protobuf response")
+			r.HandleGRPCError(w, req, err)
+			return
+		}
+	} else {
+		// Use standard JSON encoding for non-protobuf responses
+		var buf bytes.Buffer
+		if err := json.NewEncoder(&buf).Encode(response); err != nil {
+			// Encoding failed - we haven't written anything yet, so we can send an error
+			r.log.WithError(err).WithField("type", response).Error("Failed to encode response")
+			r.HandleGRPCError(w, req, err)
+			return
+		}
+		jsonBytes = buf.Bytes()
 	}
 
 	// Set headers only after successful encoding.
 	w.Header().Set("Content-Type", "application/json")
 	w.WriteHeader(statusCode)
 
-	// Write the buffered response
-	if _, err := w.Write(buf.Bytes()); err != nil {
+	// Write the response
+	if _, err := w.Write(jsonBytes); err != nil {
 		// Headers sent at this point, client disconnected most likely.
 		r.log.WithError(err).Debug("Failed to write response")
 	}

backend/pkg/api/v1/rest/routes.go

@@ -150,5 +150,63 @@ func (r *PublicRouter) GetRoutes() []RouteConfig {
 			Cache:       middleware.CacheRealtime,
 			Description: "Get prepared blocks for a specific slot",
 		},
+
+		// State Analytics endpoints
+		{
+			Path:        "/{network}/state/latest",
+			Handler:     r.handleStateLatestBlockDelta,
+			Methods:     []string{http.MethodGet, http.MethodOptions},
+			Cache:       middleware.CacheNearRealtime,
+			Description: "Get state changes for the most recent block",
+		},
+		{
+			Path:        "/{network}/state/top-adders",
+			Handler:     r.handleStateTopAdders,
+			Methods:     []string{http.MethodGet, http.MethodOptions},
+			Cache:       middleware.CacheNearRealtime,
+			Description: "Get contracts that created the most new storage slots",
+		},
+		{
+			Path:        "/{network}/state/top-removers",
+			Handler:     r.handleStateTopRemovers,
+			Methods:     []string{http.MethodGet, http.MethodOptions},
+			Cache:       middleware.CacheNearRealtime,
+			Description: "Get contracts that cleared the most storage slots",
+		},
+		{
+			Path:        "/{network}/state/growth-chart",
+			Handler:     r.handleStateGrowthChart,
+			Methods:     []string{http.MethodGet, http.MethodOptions},
+			Cache:       middleware.CacheNearRealtime,
+			Description: "Get time-series data of state growth",
+		},
+		{
+			Path:        "/{network}/state/growth-by-category",
+			Handler:     r.handleStateGrowthByCategory,
+			Methods:     []string{http.MethodGet, http.MethodOptions},
+			Cache:       middleware.CacheBrowserShort,
+			Description: "Get categorized state growth over time (Paradigm Figures 2 & 3)",
+		},
+		{
+			Path:        "/{network}/state/contract/{address}",
+			Handler:     r.handleContractStateActivity,
+			Methods:     []string{http.MethodGet, http.MethodOptions},
+			Cache:       middleware.CacheNearRealtime,
+			Description: "Get detailed state activity for a specific contract",
+		},
+		{
+			Path:        "/{network}/state/composition",
+			Handler:     r.handleContractStateComposition,
+			Methods:     []string{http.MethodGet, http.MethodOptions},
+			Cache:       middleware.CacheBrowserShort,
+			Description: "Get current state size for all contracts (Paradigm diagram data)",
+		},
+		{
+			Path:        "/{network}/state/hierarchical",
+			Handler:     r.handleHierarchicalState,
+			Methods:     []string{http.MethodGet, http.MethodOptions},
+			Cache:       middleware.CacheBrowserShort,
+			Description: "Get state organized hierarchically by category -> protocol -> contract",
+		},
 	}
 }