Merge origin/main and make regex caps burst-proof

Author: sjmiller609

DEVELOPMENT.md

@@ -292,10 +292,10 @@ build:
 
 ### Local OpenTelemetry (optional)
 
-To collect traces and metrics locally, run the Grafana LGTM stack (Loki, Grafana, Tempo, Mimir):
+To collect traces and metrics locally, run the LGTM stack (Loki, Tempo, Mimir):
 
 ```bash
-# Start Grafana LGTM (UI at http://localhost:3000, login: admin/admin)
+# Start LGTM (UI at http://localhost:3000, login: admin/admin)
 # Note, if you are developing on a shared server, you can use the same LGTM stack as your peer(s)
 # You will be able to sort your metrics, traces, and logs using the ENV configuration (see below)
 BIND=127.0.0.1
@@ -323,15 +323,7 @@ docker run -d --name lgtm \
 make dev
 ```
 
-Open http://localhost:3000 to view traces (Tempo), metrics (Mimir), and logs (Loki) in Grafana.
-
-**Import the Hypeman dashboard:**
-
-1. Go to Dashboards → New → Import
-2. Upload `dashboards/hypeman.json` or paste its contents
-3. Select the Prometheus datasource and click Import
-
-Use the Environment/Instance dropdowns to filter by `deployment.environment` or `service.instance.id`.
+Open http://localhost:3000 to view traces (Tempo), metrics (Mimir), and logs (Loki).
 
 ## Testing
 

cmd/api/config/config.go

@@ -98,10 +98,11 @@ type APIConfig struct {
 
 // MetricsConfig holds metrics endpoint settings.
 type MetricsConfig struct {
-	ListenAddress           string `koanf:"listen_address"`
-	Port                    int    `koanf:"port"`
-	VMLabelBudget           int    `koanf:"vm_label_budget"`
-	ResourceRefreshInterval string `koanf:"resource_refresh_interval"`
+	ListenAddress               string `koanf:"listen_address"`
+	Port                        int    `koanf:"port"`
+	VMLabelBudget               int    `koanf:"vm_label_budget"`
+	ResourceRefreshInterval     string `koanf:"resource_refresh_interval"`
+	AllocationReconcileInterval string `koanf:"allocation_reconcile_interval"`
 }
 
 // OtelConfig holds OpenTelemetry settings.
@@ -144,6 +145,11 @@ type BuildConfig struct {
 	DockerSocket              string `koanf:"docker_socket"`
 }
 
+// InstancesConfig holds instance-manager internal settings.
+type InstancesConfig struct {
+	LifecycleEventBufferSize int `koanf:"lifecycle_event_buffer_size"`
+}
+
 // RegistryConfig holds OCI registry settings.
 type RegistryConfig struct {
 	URL        string `koanf:"url"`
@@ -241,6 +247,7 @@ type Config struct {
 	Logging          LoggingConfig          `koanf:"logging"`
 	Images           ImagesConfig           `koanf:"images"`
 	Build            BuildConfig            `koanf:"build"`
+	Instances        InstancesConfig        `koanf:"instances"`
 	Registry         RegistryConfig         `koanf:"registry"`
 	Limits           LimitsConfig           `koanf:"limits"`
 	Oversubscription OversubscriptionConfig `koanf:"oversubscription"`
@@ -310,10 +317,11 @@ func defaultConfig() *Config {
 		},
 
 		Metrics: MetricsConfig{
-			ListenAddress:           "127.0.0.1",
-			Port:                    9464,
-			VMLabelBudget:           200,
-			ResourceRefreshInterval: "120s",
+			ListenAddress:               "127.0.0.1",
+			Port:                        9464,
+			VMLabelBudget:               200,
+			ResourceRefreshInterval:     "120s",
+			AllocationReconcileInterval: "120s",
 		},
 
 		Otel: OtelConfig{
@@ -349,6 +357,10 @@ func defaultConfig() *Config {
 			DockerSocket:              "/var/run/docker.sock",
 		},
 
+		Instances: InstancesConfig{
+			LifecycleEventBufferSize: 256,
+		},
+
 		Registry: RegistryConfig{
 			URL:        "localhost:8080",
 			Insecure:   false,
@@ -499,6 +511,16 @@ func (c *Config) Validate() error {
 	if interval <= 0 {
 		return fmt.Errorf("metrics.resource_refresh_interval must be positive, got %q", c.Metrics.ResourceRefreshInterval)
 	}
+	if strings.TrimSpace(c.Metrics.AllocationReconcileInterval) == "" {
+		return fmt.Errorf("metrics.allocation_reconcile_interval must not be empty")
+	}
+	reconcileInterval, err := time.ParseDuration(c.Metrics.AllocationReconcileInterval)
+	if err != nil {
+		return fmt.Errorf("metrics.allocation_reconcile_interval must be a valid duration, got %q: %w", c.Metrics.AllocationReconcileInterval, err)
+	}
+	if reconcileInterval <= 0 {
+		return fmt.Errorf("metrics.allocation_reconcile_interval must be positive, got %q", c.Metrics.AllocationReconcileInterval)
+	}
 	if c.Otel.MetricExportInterval != "" {
 		if _, err := time.ParseDuration(c.Otel.MetricExportInterval); err != nil {
 			return fmt.Errorf("otel.metric_export_interval must be a valid duration, got %q: %w", c.Otel.MetricExportInterval, err)
@@ -552,6 +574,9 @@ func (c *Config) Validate() error {
 	if err := validateNamePatternLimits(c.Limits.NamePatterns); err != nil {
 		return err
 	}
+	if c.Instances.LifecycleEventBufferSize <= 0 {
+		return fmt.Errorf("instances.lifecycle_event_buffer_size must be positive, got %d", c.Instances.LifecycleEventBufferSize)
+	}
 	if err := validateDuration("images.auto_delete.unused_for", c.Images.AutoDelete.UnusedFor); err != nil {
 		return err
 	}

cmd/api/config/config_test.go

@@ -25,6 +25,9 @@ func TestDefaultConfigIncludesMetricsSettings(t *testing.T) {
 	if cfg.Metrics.ResourceRefreshInterval != "120s" {
 		t.Fatalf("expected default metrics.resource_refresh_interval to be 120s, got %q", cfg.Metrics.ResourceRefreshInterval)
 	}
+	if cfg.Metrics.AllocationReconcileInterval != "120s" {
+		t.Fatalf("expected default metrics.allocation_reconcile_interval to be 120s, got %q", cfg.Metrics.AllocationReconcileInterval)
+	}
 	if cfg.Otel.MetricExportInterval != "60s" {
 		t.Fatalf("expected default otel.metric_export_interval to be 60s, got %q", cfg.Otel.MetricExportInterval)
 	}
@@ -40,15 +43,20 @@ func TestDefaultConfigIncludesMetricsSettings(t *testing.T) {
 	if len(cfg.Images.AutoDelete.Allowed) != 0 {
 		t.Fatalf("expected default images.auto_delete.allowed to be empty, got %v", cfg.Images.AutoDelete.Allowed)
 	}
+	if cfg.Instances.LifecycleEventBufferSize != 256 {
+		t.Fatalf("expected default instances.lifecycle_event_buffer_size to be 256, got %d", cfg.Instances.LifecycleEventBufferSize)
+	}
 }
 
 func TestLoadEnvOverridesMetricsAndOtelInterval(t *testing.T) {
 	t.Setenv("METRICS__LISTEN_ADDRESS", "0.0.0.0")
 	t.Setenv("METRICS__PORT", "9999")
 	t.Setenv("METRICS__VM_LABEL_BUDGET", "350")
 	t.Setenv("METRICS__RESOURCE_REFRESH_INTERVAL", "30s")
+	t.Setenv("METRICS__ALLOCATION_RECONCILE_INTERVAL", "45s")
 	t.Setenv("OTEL__METRIC_EXPORT_INTERVAL", "15s")
 	t.Setenv("OTEL__SUCCESSFUL_GET_SAMPLE_RATIO", "0.25")
+	t.Setenv("INSTANCES__LIFECYCLE_EVENT_BUFFER_SIZE", "512")
 
 	tmp := t.TempDir()
 	cfgPath := filepath.Join(tmp, "config.yaml")
@@ -73,12 +81,18 @@ func TestLoadEnvOverridesMetricsAndOtelInterval(t *testing.T) {
 	if cfg.Metrics.ResourceRefreshInterval != "30s" {
 		t.Fatalf("expected metrics.resource_refresh_interval override, got %q", cfg.Metrics.ResourceRefreshInterval)
 	}
+	if cfg.Metrics.AllocationReconcileInterval != "45s" {
+		t.Fatalf("expected metrics.allocation_reconcile_interval override, got %q", cfg.Metrics.AllocationReconcileInterval)
+	}
 	if cfg.Otel.MetricExportInterval != "15s" {
 		t.Fatalf("expected otel.metric_export_interval override, got %q", cfg.Otel.MetricExportInterval)
 	}
 	if cfg.Otel.SuccessfulGetSampleRatio != 0.25 {
 		t.Fatalf("expected otel.successful_get_sample_ratio override, got %v", cfg.Otel.SuccessfulGetSampleRatio)
 	}
+	if cfg.Instances.LifecycleEventBufferSize != 512 {
+		t.Fatalf("expected instances.lifecycle_event_buffer_size override, got %d", cfg.Instances.LifecycleEventBufferSize)
+	}
 }
 
 func TestValidateRejectsInvalidMetricsPort(t *testing.T) {
@@ -147,6 +161,59 @@ func TestValidateRejectsInvalidResourceRefreshInterval(t *testing.T) {
 	}
 }
 
+func TestValidateRejectsInvalidAllocationReconcileInterval(t *testing.T) {
+	cfg := defaultConfig()
+	cfg.Metrics.AllocationReconcileInterval = ""
+
+	err := cfg.Validate()
+	if err == nil {
+		t.Fatalf("expected validation error for empty allocation reconcile interval")
+	}
+
+	cfg = defaultConfig()
+	cfg.Metrics.AllocationReconcileInterval = "not-a-duration"
+
+	err = cfg.Validate()
+	if err == nil {
+		t.Fatalf("expected validation error for invalid allocation reconcile interval")
+	}
+
+	cfg = defaultConfig()
+	cfg.Metrics.AllocationReconcileInterval = "0s"
+
+	err = cfg.Validate()
+	if err == nil {
+		t.Fatalf("expected validation error for non-positive allocation reconcile interval")
+	}
+}
+
+func TestLoadUsesConfiguredLifecycleEventBufferSize(t *testing.T) {
+	tmp := t.TempDir()
+	cfgPath := filepath.Join(tmp, "config.yaml")
+	if err := os.WriteFile(cfgPath, []byte("instances:\n  lifecycle_event_buffer_size: 384\n"), 0600); err != nil {
+		t.Fatalf("write temp config: %v", err)
+	}
+
+	cfg, err := Load(cfgPath)
+	if err != nil {
+		t.Fatalf("load config: %v", err)
+	}
+
+	if cfg.Instances.LifecycleEventBufferSize != 384 {
+		t.Fatalf("expected instances.lifecycle_event_buffer_size from config file, got %d", cfg.Instances.LifecycleEventBufferSize)
+	}
+}
+
+func TestValidateRejectsInvalidLifecycleEventBufferSize(t *testing.T) {
+	cfg := defaultConfig()
+	cfg.Instances.LifecycleEventBufferSize = 0
+
+	err := cfg.Validate()
+	if err == nil {
+		t.Fatalf("expected validation error for invalid lifecycle event buffer size")
+	}
+}
+
 func TestLoadUsesDefaultImageAutoDeleteRetentionWindow(t *testing.T) {
 	tmp := t.TempDir()
 	cfgPath := filepath.Join(tmp, "config.yaml")

cmd/api/main.go

@@ -183,9 +183,18 @@ func run() error {
 	if err != nil {
 		return fmt.Errorf("invalid metrics resource refresh interval %q: %w", app.Config.Metrics.ResourceRefreshInterval, err)
 	}
+	allocationReconcileInterval, err := time.ParseDuration(app.Config.Metrics.AllocationReconcileInterval)
+	if err != nil {
+		return fmt.Errorf("invalid metrics allocation reconcile interval %q: %w", app.Config.Metrics.AllocationReconcileInterval, err)
+	}
 	if err := app.ResourceManager.StartMonitoring(ctx, otelProvider.Meter, resourceRefreshInterval); err != nil {
 		return fmt.Errorf("start resource monitoring: %w", err)
 	}
+	if reconciler, ok := app.InstanceManager.(interface {
+		StartAdmissionAllocationReconciler(context.Context, time.Duration)
+	}); ok {
+		reconciler.StartAdmissionAllocationReconciler(ctx, allocationReconcileInterval)
+	}
 
 	// Log OTel status
 	if cfg.Otel.Enabled {