address comments

Author: jakeheft

CMV_SUPPORT.md

@@ -10,6 +10,10 @@ behavior via the standard `CreateMaterializedView` gRPC method.
 
 ### 1. Creating a CMV
 
+There are two ways to create a CMV in the emulator:
+
+#### Option A: gRPC (recommended, matches production)
+
 Use the standard Go admin client, pointed at the emulator:
 
 ```go
@@ -32,6 +36,45 @@ ORDER BY region, account_id, ts, typ, item_id, src_key`,
 The emulator parses the SQL to extract the key transformation config. The same code works
 against both production Bigtable and the emulator.
 
+#### Option B: `--cmv-config` flag (local testing convenience)
+
+For local testing without writing client code, pass a JSON config file at startup:
+
+```bash
+./little_bigtable --port 9000 --db-file /tmp/lbt.db --cmv-config /path/to/cmv_config.json
+```
+
+The JSON file is an array of CMV config objects:
+
+```json
+[
+  {
+    "source_table": "events",
+    "view_id": "events_by_account",
+    "key_separator": "#",
+    "key_mapping": [3, 4, 1, 2, 0],
+    "append_source_key": true,
+    "include_families": ["cf1"]
+  }
+]
+```
+
+| Field | Description |
+|---|---|
+| `source_table` | Table ID of the source table (not the fully-qualified name) |
+| `view_id` | Table ID to use for the CMV shadow table |
+| `key_separator` | Delimiter used in the composite row key |
+| `key_mapping` | Ordered list of 0-based source key component indices for the CMV key |
+| `append_source_key` | If true, appends the full original source key as the final component |
+| `include_families` | Column families to copy; omit or leave empty to include all |
+
+> **Note:** The `--cmv-config` approach registers CMVs directly without going through SQL
+> parsing. It is intended for local testing only. In production (and for full emulator
+> fidelity), use `CreateMaterializedView` via the gRPC client.
+
+> **Persistence:** CMV registrations loaded via either approach are in-memory only. If the
+> emulator restarts, CMVs must be re-registered (or the `--cmv-config` flag re-supplied).
+
 ### 2. Write-time Sync
 
 When data is written to a source table (via MutateRow, MutateRows, CheckAndMutateRow,
@@ -59,16 +102,18 @@ row, err := table.ReadRow(ctx, "region-a#account-42#...")
 ## What's Changed
 
 ### New Files
-- `bttest/cmv.go` — CMV config types, SQL parser, key transformation logic
+- `bttest/cmv.go` — CMV config types, registry, key transformation logic
+- `bttest/sql_parse.go` — SQL parser for extracting CMV config from a `CreateMaterializedView` query
 - `bttest/cmv_test.go` — Tests for key transformation, write sync, delete propagation
+- `bttest/sql_parse_test.go` — Tests for the SQL parser
 
 ### Modified Files
-- `little_bigtable.go` — Version bump to 0.2.0
+- `little_bigtable.go` — Version bump to 0.2.0; added `--cmv-config` flag
 - `bttest/inmem.go` — Added `cmvs` field to server struct, CMV registration,
   shadow table creation, write-time sync hooks in MutateRow/MutateRows/
   CheckAndMutateRow/ReadModifyWriteRow/DropRowRange
 - `bttest/instance_server.go` — Implemented CreateMaterializedView, GetMaterializedView,
-  ListMaterializedViews, DeleteMaterializedView
+  ListMaterializedViews, UpdateMaterializedView (DeletionProtection only), DeleteMaterializedView
 
 ## Known Limitations
 
@@ -80,8 +125,8 @@ row, err := table.ReadRow(ctx, "region-a#account-42#...")
   creation are not reflected in the CMV table.
 - **Backfill**: Data written to the source table before the CMV is registered is not
   retroactively copied.
-- **Persistence**: CMV registrations are in-memory only. If the emulator restarts, CMVs
-  must be re-registered via `CreateMaterializedView`.
+- **Persistence**: CMV registrations are in-memory only and do not survive a restart.
+  Re-register via `CreateMaterializedView` or re-supply `--cmv-config` on each startup.
 
 ## Example: Key Transformation
 

bttest/cmv.go

@@ -1,8 +1,12 @@
 package bttest
 
 import (
+	"encoding/json"
+	"fmt"
 	"log"
+	"os"
 	"strings"
+	"sync"
 )
 
 // CMVConfig defines a Continuous Materialized View for the emulator.
@@ -28,7 +32,9 @@ type CMVConfig struct {
 
 // cmvRegistry maps plain source table IDs to CMV definitions.
 // Lookups match by table ID suffix against fully-qualified table names.
+// Its own mu protects concurrent reads/writes to configs independently of s.mu.
 type cmvRegistry struct {
+	mu      sync.RWMutex
 	configs map[string][]CMVConfig
 }
 
@@ -39,10 +45,14 @@ func newCMVRegistry() *cmvRegistry {
 }
 
 func (r *cmvRegistry) register(cfg CMVConfig) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
 	r.configs[cfg.SourceTable] = append(r.configs[cfg.SourceTable], cfg)
 }
 
 func (r *cmvRegistry) deregister(viewID string) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
 	for src, cfgs := range r.configs {
 		filtered := cfgs[:0]
 		for _, c := range cfgs {
@@ -58,16 +68,36 @@ func (r *cmvRegistry) deregister(viewID string) {
 	}
 }
 
+// deregisterBySource removes all CMV configs for a given source table and
+// returns the view IDs that were registered against it.
+func (r *cmvRegistry) deregisterBySource(sourceTable string) []string {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	cfgs := r.configs[sourceTable]
+	if len(cfgs) == 0 {
+		return nil
+	}
+	viewIDs := make([]string, len(cfgs))
+	for i, c := range cfgs {
+		viewIDs[i] = c.ViewID
+	}
+	delete(r.configs, sourceTable)
+	return viewIDs
+}
+
 func (r *cmvRegistry) cmvsForTable(fqTable string) []*cmvInstance {
 	parent, tableID := splitFQTable(fqTable)
+	r.mu.RLock()
 	cfgs, ok := r.configs[tableID]
 	if !ok {
+		r.mu.RUnlock()
 		return nil
 	}
 	result := make([]*cmvInstance, len(cfgs))
 	for i := range cfgs {
 		result[i] = &cmvInstance{config: cfgs[i], parent: parent}
 	}
+	r.mu.RUnlock()
 	return result
 }
 
@@ -145,3 +175,18 @@ func (c *cmvInstance) buildCMVRow(sourceRow *row) *row {
 func (c *cmvInstance) deriveCMVKey(sourceKey string) string {
 	return c.transformKey(sourceKey)
 }
+
+// LoadCMVConfigs reads a JSON array of CMVConfig from the given file path.
+// This is used with the --cmv-config flag for local testing convenience.
+func LoadCMVConfigs(path string) ([]CMVConfig, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("opening cmv config: %w", err)
+	}
+	defer f.Close()
+	var configs []CMVConfig
+	if err := json.NewDecoder(f).Decode(&configs); err != nil {
+		return nil, fmt.Errorf("parsing cmv config: %w", err)
+	}
+	return configs, nil
+}

bttest/inmem.go

@@ -140,6 +140,17 @@ func NewServer(laddr string, db *sql.DB, opt ...grpc.ServerOption) (*Server, err
 	return s, nil
 }
 
+// RegisterCMVs pre-loads CMV configs for local testing (used with --cmv-config).
+// This is equivalent to calling CreateMaterializedView for each config but does
+// not require a project/instance path and bypasses SQL parsing.
+func (s *Server) RegisterCMVs(configs []CMVConfig) {
+	s.s.mu.Lock()
+	defer s.s.mu.Unlock()
+	for _, cfg := range configs {
+		s.s.cmvs.register(cfg)
+	}
+}
+
 // Close shuts down the server.
 func (s *Server) Close() {
 	s.s.mu.Lock()
@@ -361,12 +372,33 @@ func (s *server) UpdateTable(ctx context.Context, req *btapb.UpdateTableRequest)
 func (s *server) DeleteTable(ctx context.Context, req *btapb.DeleteTableRequest) (*emptypb.Empty, error) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
-	if tbl, ok := s.tables[req.Name]; !ok {
+	tbl, ok := s.tables[req.Name]
+	if !ok {
 		return nil, status.Errorf(codes.NotFound, "table %q not found", req.Name)
-	} else {
-		s.tableBackend.Delete(tbl)
-		tbl.rows.DeleteAll()
-		delete(s.tables, req.Name)
+	}
+	s.tableBackend.Delete(tbl)
+	tbl.rows.DeleteAll()
+	delete(s.tables, req.Name)
+
+	// Clean up any CMVs that use this table as their source.
+	idx := strings.LastIndex(req.Name, "/tables/")
+	if idx >= 0 {
+		parent := req.Name[:idx]
+		tableID := req.Name[idx+len("/tables/"):]
+		for _, viewID := range s.cmvs.deregisterBySource(tableID) {
+			fqShadow := parent + "/tables/" + viewID
+			if shadowTbl, exists := s.tables[fqShadow]; exists {
+				s.tableBackend.Delete(shadowTbl)
+				shadowTbl.rows.DeleteAll()
+				delete(s.tables, fqShadow)
+			}
+			for mvName := range s.materializedViews {
+				if strings.HasSuffix(mvName, "/materializedViews/"+viewID) {
+					delete(s.materializedViews, mvName)
+					break
+				}
+			}
+		}
 	}
 	return &emptypb.Empty{}, nil
 }

bttest/instance_server.go

@@ -260,10 +260,16 @@ func (s *server) DeleteMaterializedView(ctx context.Context, req *btapb.DeleteMa
 		return nil, status.Errorf(codes.FailedPrecondition, "materialized view %q is protected against deletion", req.Name)
 	}
 
-	// Extract view ID from the full resource name.
+	// Extract parent and view ID from the full resource name.
 	parts := strings.Split(mv.Name, "/materializedViews/")
 	if len(parts) == 2 {
 		s.cmvs.deregister(parts[1])
+		fqShadow := parts[0] + "/tables/" + parts[1]
+		if shadowTbl, exists := s.tables[fqShadow]; exists {
+			s.tableBackend.Delete(shadowTbl)
+			shadowTbl.rows.DeleteAll()
+			delete(s.tables, fqShadow)
+		}
 	}
 	delete(s.materializedViews, req.Name)
 	return new(empty.Empty), nil

little_bigtable.go

@@ -26,6 +26,7 @@ func main() {
 	host := flag.String("host", "localhost", "the address to bind to on the local machine")
 	port := flag.Int("port", 9000, "the port number to bind to on the local machine")
 	dbFile := flag.String("db-file", "little_bigtable.db", "path to data file")
+	cmvConfig := flag.String("cmv-config", "", "optional path to a JSON file pre-loading CMV configs for local testing")
 	showVersion := flag.Bool("version", false, "show version")
 
 	ctx := context.Background()
@@ -62,6 +63,15 @@ func main() {
 		log.Fatalf("failed to start emulator: %v", err)
 	}
 
+	if *cmvConfig != "" {
+		configs, err := bttest.LoadCMVConfigs(*cmvConfig)
+		if err != nil {
+			log.Fatalf("failed to load cmv config: %v", err)
+		}
+		srv.RegisterCMVs(configs)
+		log.Printf("loaded %d CMV config(s) from %s", len(configs), *cmvConfig)
+	}
+
 	log.Printf("\"little\" Bigtable emulator running. DB:%s Connect with environment variable BIGTABLE_EMULATOR_HOST=%q", *dbFile, srv.Addr)
 	select {}
 }