diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 90f3457a..61c38ff8 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -542,6 +542,16 @@ func (s *ApiService) StandbyInstance(ctx context.Context, request oapi.StandbyIn } standbyReq.Compression = compression } + if request.Body != nil && request.Body.CompressionDelay != nil { + compressionDelay, err := parseOptionalDuration(*request.Body.CompressionDelay, "compression_delay") + if err != nil { + return oapi.StandbyInstance400JSONResponse{ + Code: "invalid_compression_delay", + Message: err.Error(), + }, nil + } + standbyReq.CompressionDelay = compressionDelay + } result, err := s.InstanceManager.StandbyInstance(ctx, inst.Id, standbyReq) if err != nil { @@ -1152,6 +1162,13 @@ func toInstanceSnapshotPolicy(policy oapi.SnapshotPolicy) (*instances.SnapshotPo } out.Compression = compression } + if policy.StandbyCompressionDelay != nil { + delay, err := parseOptionalDuration(*policy.StandbyCompressionDelay, "standby_compression_delay") + if err != nil { + return nil, err + } + out.StandbyCompressionDelay = delay + } return out, nil } @@ -1176,5 +1193,20 @@ func toOAPISnapshotPolicy(policy instances.SnapshotPolicy) oapi.SnapshotPolicy { compression := toOAPISnapshotCompressionConfig(*policy.Compression) out.Compression = &compression } + if policy.StandbyCompressionDelay != nil { + delay := policy.StandbyCompressionDelay.String() + out.StandbyCompressionDelay = &delay + } return out } + +func parseOptionalDuration(value string, field string) (*time.Duration, error) { + duration, err := time.ParseDuration(value) + if err != nil { + return nil, fmt.Errorf("%s must be a valid duration: %w", field, err) + } + if duration < 0 { + return nil, fmt.Errorf("%s cannot be negative", field) + } + return &duration, nil +} diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index 3244e9c2..7a664db8 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -480,10 +480,85 @@ func TestCreateInstance_MapsNetworkEgressEnforcementMode(t *testing.T) { assert.Equal(t, instances.EgressEnforcementModeHTTPHTTPSOnly, mockMgr.lastReq.NetworkEgress.EnforcementMode) } -func TestCreateInstance_MapsAutoStandbyPolicy(t *testing.T) { +func TestCreateInstance_MapsStandbyCompressionDelayInSnapshotPolicy(t *testing.T) { + t.Parallel() + + svc := newTestService(t) + origMgr := svc.InstanceManager + mockMgr := &captureCreateManager{Manager: origMgr} + svc.InstanceManager = mockMgr + + delay := "2m30s" + resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ + Body: &oapi.CreateInstanceRequest{ + Name: "test-standby-compression-delay", + Image: "docker.io/library/alpine:latest", + SnapshotPolicy: &oapi.SnapshotPolicy{ + StandbyCompressionDelay: &delay, + }, + }, + }) + require.NoError(t, err) + _, ok := resp.(oapi.CreateInstance201JSONResponse) + require.True(t, ok, "expected 201 response") + + require.NotNil(t, mockMgr.lastReq) + require.NotNil(t, mockMgr.lastReq.SnapshotPolicy) + require.NotNil(t, mockMgr.lastReq.SnapshotPolicy.StandbyCompressionDelay) + assert.Equal(t, 150*time.Second, *mockMgr.lastReq.SnapshotPolicy.StandbyCompressionDelay) +} + +func TestCreateInstance_InvalidStandbyCompressionDelayInSnapshotPolicy(t *testing.T) { t.Parallel() + svc := newTestService(t) + delay := "not-a-duration" + resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ + Body: &oapi.CreateInstanceRequest{ + Name: "test-invalid-standby-delay", + Image: "docker.io/library/alpine:latest", + SnapshotPolicy: &oapi.SnapshotPolicy{ + StandbyCompressionDelay: &delay, + }, + }, + }) + require.NoError(t, err) + + badReq, ok := resp.(oapi.CreateInstance400JSONResponse) + require.True(t, ok, "expected 400 response") + assert.Equal(t, "invalid_snapshot_policy", badReq.Code) + assert.Contains(t, badReq.Message, "standby_compression_delay") +} + +func TestInstanceToOAPI_EmitsStandbyCompressionDelayInSnapshotPolicy(t *testing.T) { + t.Parallel() + + delay := 90 * time.Second + inst := instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "inst-standby-delay", + Name: "inst-standby-delay", + Image: "docker.io/library/alpine:latest", + CreatedAt: time.Now(), + HypervisorType: hypervisor.TypeCloudHypervisor, + SnapshotPolicy: &instances.SnapshotPolicy{ + StandbyCompressionDelay: &delay, + }, + }, + State: instances.StateStandby, + } + + oapiInst := instanceToOAPI(inst) + require.NotNil(t, oapiInst.SnapshotPolicy) + require.NotNil(t, oapiInst.SnapshotPolicy.StandbyCompressionDelay) + assert.Equal(t, "1m30s", *oapiInst.SnapshotPolicy.StandbyCompressionDelay) +} + +func TestCreateInstance_MapsAutoStandbyPolicy(t *testing.T) { + t.Parallel() + + svc := newTestService(t) origMgr := svc.InstanceManager mockMgr := &captureCreateManager{Manager: origMgr} svc.InstanceManager = mockMgr @@ -907,6 +982,81 @@ func TestStandbyInstance_InvalidRequest(t *testing.T) { assert.Contains(t, badReq.Message, "invalid snapshot compression level") } +func TestStandbyInstance_MapsCompressionDelay(t *testing.T) { + t.Parallel() + + svc := newTestService(t) + now := time.Now() + source := instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "standby-delay-src", + Name: "standby-delay-src", + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + HypervisorType: hypervisor.TypeCloudHypervisor, + }, + State: instances.StateRunning, + } + + mockMgr := &captureStandbyManager{ + Manager: svc.InstanceManager, + result: &source, + } + svc.InstanceManager = mockMgr + + delay := "45s" + resp, err := svc.StandbyInstance( + mw.WithResolvedInstance(ctx(), source.Id, source), + oapi.StandbyInstanceRequestObject{ + Id: source.Id, + Body: &oapi.StandbyInstanceRequest{ + CompressionDelay: &delay, + }, + }, + ) + require.NoError(t, err) + _, ok := resp.(oapi.StandbyInstance200JSONResponse) + require.True(t, ok, "expected 200 response") + + require.NotNil(t, mockMgr.lastReq) + require.NotNil(t, mockMgr.lastReq.CompressionDelay) + assert.Equal(t, 45*time.Second, *mockMgr.lastReq.CompressionDelay) +} + +func TestStandbyInstance_InvalidCompressionDelay(t *testing.T) { + t.Parallel() + + svc := newTestService(t) + now := time.Now() + source := instances.Instance{ + StoredMetadata: instances.StoredMetadata{ + Id: "standby-invalid-delay-src", + Name: "standby-invalid-delay-src", + Image: "docker.io/library/alpine:latest", + CreatedAt: now, + HypervisorType: hypervisor.TypeCloudHypervisor, + }, + State: instances.StateRunning, + } + + delay := "-5s" + resp, err := svc.StandbyInstance( + mw.WithResolvedInstance(ctx(), source.Id, source), + oapi.StandbyInstanceRequestObject{ + Id: source.Id, + Body: &oapi.StandbyInstanceRequest{ + CompressionDelay: &delay, + }, + }, + ) + require.NoError(t, err) + + badReq, ok := resp.(oapi.StandbyInstance400JSONResponse) + require.True(t, ok, "expected 400 response") + assert.Equal(t, "invalid_compression_delay", badReq.Code) + assert.Contains(t, badReq.Message, "compression_delay") +} + func TestForkInstance_FromRunningFlagForwarded(t *testing.T) { t.Parallel() svc := newTestService(t) diff --git a/lib/instances/compression_integration_linux_test.go b/lib/instances/compression_integration_linux_test.go index 75ceef96..0734965e 100644 --- a/lib/instances/compression_integration_linux_test.go +++ b/lib/instances/compression_integration_linux_test.go @@ -259,30 +259,64 @@ func waitForRunningAndExecReady(t *testing.T, ctx context.Context, mgr *manager, require.NoError(t, waitHypervisorUp(ctx, inst)) } require.NoError(t, waitForExecAgent(ctx, mgr, instanceID, 30*time.Second)) + waitForGuestExecReady(t, ctx, inst) return inst } -func writeGuestMarker(t *testing.T, ctx context.Context, inst *Instance, path string, value string) { +func waitForGuestExecReady(t *testing.T, ctx context.Context, inst *Instance) { t.Helper() - execCtx, cancel := context.WithTimeout(ctx, integrationTestTimeout(compressionGuestExecTimeout)) - defer cancel() - output, exitCode, err := execCommand(execCtx, inst, "sh", "-c", fmt.Sprintf("printf %q > %s && sync", value, path)) + require.Eventually(t, func() bool { + execCtx, cancel := context.WithTimeout(ctx, integrationTestTimeout(5*time.Second)) + defer cancel() + + output, exitCode, err := execCommand(execCtx, inst, "true") + return err == nil && exitCode == 0 && output == "" + }, integrationTestTimeout(15*time.Second), 500*time.Millisecond, "guest exec should succeed after restore") +} + +func writeGuestMarker(t *testing.T, ctx context.Context, inst *Instance, path string, value string) { + t.Helper() + output, exitCode, err := execCommandWithRetry(ctx, inst, compressionGuestExecTimeout, "sh", "-c", fmt.Sprintf("printf %q > %s && sync", value, path)) require.NoError(t, err) require.Equal(t, 0, exitCode, output) } func assertGuestMarker(t *testing.T, ctx context.Context, inst *Instance, path string, expected string) { t.Helper() - execCtx, cancel := context.WithTimeout(ctx, integrationTestTimeout(compressionGuestExecTimeout)) - defer cancel() - - output, exitCode, err := execCommand(execCtx, inst, "cat", path) + output, exitCode, err := execCommandWithRetry(ctx, inst, compressionGuestExecTimeout, "cat", path) require.NoError(t, err) require.Equal(t, 0, exitCode, output) assert.Equal(t, expected, output) } +func execCommandWithRetry(ctx context.Context, inst *Instance, timeout time.Duration, command ...string) (string, int, error) { + deadline := time.Now().Add(integrationTestTimeout(timeout)) + var lastOutput string + var lastExitCode int + var lastErr error + + for { + execCtx, cancel := context.WithTimeout(ctx, integrationTestTimeout(5*time.Second)) + output, exitCode, err := execCommand(execCtx, inst, command...) + cancel() + + if err == nil { + return output, exitCode, nil + } + + lastOutput = output + lastExitCode = exitCode + lastErr = err + + if time.Now().After(deadline) { + return lastOutput, lastExitCode, lastErr + } + + time.Sleep(500 * time.Millisecond) + } +} + func waitForCompressionJobStart(t *testing.T, mgr *manager, key string, timeout time.Duration) { t.Helper() deadline := time.Now().Add(timeout) diff --git a/lib/instances/create.go b/lib/instances/create.go index c6d208ac..b007fc82 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -599,6 +599,11 @@ func validateCreateRequest(req *CreateInstanceRequest) error { return err } } + if req.SnapshotPolicy != nil && req.SnapshotPolicy.StandbyCompressionDelay != nil { + if _, err := normalizeStandbyCompressionDelay(req.SnapshotPolicy.StandbyCompressionDelay); err != nil { + return err + } + } normalizedAutoStandby, err := normalizeAutoStandbyPolicy(req.AutoStandby) if err != nil { return err diff --git a/lib/instances/delete.go b/lib/instances/delete.go index 974224e8..283e0b19 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -39,8 +39,8 @@ func (m *manager) deleteInstance( if err != nil { return fmt.Errorf("wait for instance compression to stop: %w", err) } - if target != nil { - m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionDeleteInstance, *target) + if target != nil && target.State == compressionJobStateRunning { + m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionDeleteInstance, target.Target) } // 2. Get network allocation BEFORE killing VMM (while we can still query it) diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 84bea17c..177f1067 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -472,6 +472,7 @@ func (m *manager) cleanupForkInstanceOnError(ctx context.Context, forkID string) func cloneStoredMetadata(src StoredMetadata) StoredMetadata { dst := src + dst.PendingStandbyCompression = nil if src.Env != nil { dst.Env = make(map[string]string, len(src.Env)) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 78c25061..495d8a4e 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -128,6 +128,7 @@ type manager struct { snapshotDefaults SnapshotPolicy compressionMu sync.Mutex compressionJobs map[string]*compressionJob + compressionTimerFactory func(time.Duration) compressionTimer nativeCodecMu sync.Mutex nativeCodecPaths map[string]string imageUsageRecorder ImageUsageRecorder @@ -212,6 +213,9 @@ func NewManagerWithConfig(p *paths.Paths, imageManager images.Manager, systemMan m.lifecycleEvents.onDrop = func(ctx context.Context, consumer LifecycleEventConsumer) { m.recordLifecycleEventDropped(ctx, consumer, lifecycleEventDropReasonBufferFull) } + if err := m.recoverPendingStandbyCompressionJobs(context.Background()); err != nil { + logger.FromContext(context.Background()).WarnContext(context.Background(), "failed to recover pending standby compression jobs", "error", err) + } return m } diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index 1b652fca..84bf8060 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -25,10 +25,18 @@ type snapshotCompressionResult string const ( snapshotCompressionResultSuccess snapshotCompressionResult = "success" + snapshotCompressionResultSkipped snapshotCompressionResult = "skipped" snapshotCompressionResultCanceled snapshotCompressionResult = "canceled" snapshotCompressionResultFailed snapshotCompressionResult = "failed" ) +type snapshotCompressionWaitOutcome string + +const ( + snapshotCompressionWaitOutcomeStarted snapshotCompressionWaitOutcome = "started" + snapshotCompressionWaitOutcomeSkipped snapshotCompressionWaitOutcome = "skipped" +) + type snapshotMemoryPreparePath string const ( @@ -76,6 +84,7 @@ type Metrics struct { stateTransitions metric.Int64Counter snapshotCompressionJobsTotal metric.Int64Counter snapshotCompressionDuration metric.Float64Histogram + snapshotCompressionWaitDuration metric.Float64Histogram snapshotCompressionSavedBytes metric.Int64Histogram snapshotCompressionRatio metric.Float64Histogram snapshotCodecFallbacksTotal metric.Int64Counter @@ -174,6 +183,16 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M return nil, err } + snapshotCompressionWaitDuration, err := meter.Float64Histogram( + "hypeman_snapshot_compression_wait_duration_seconds", + metric.WithDescription("Time a delayed snapshot compression job waits before compression starts or is skipped"), + metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(hypotel.CommonDurationHistogramBuckets()...), + ) + if err != nil { + return nil, err + } + snapshotCompressionSavedBytes, err := meter.Int64Histogram( "hypeman_snapshot_compression_saved_bytes", metric.WithDescription("Bytes saved by compressing snapshot memory"), @@ -253,7 +272,15 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M snapshotCompressionActiveTotal, err := meter.Int64ObservableGauge( "hypeman_snapshot_compression_active_total", - metric.WithDescription("Total number of in-flight snapshot compression jobs"), + metric.WithDescription("Total number of actively running snapshot compression jobs"), + ) + if err != nil { + return nil, err + } + + snapshotCompressionPendingTotal, err := meter.Int64ObservableGauge( + "hypeman_snapshot_compression_pending_total", + metric.WithDescription("Total number of delayed snapshot compression jobs waiting to start"), ) if err != nil { return nil, err @@ -331,7 +358,8 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M source string } - counts := make(map[compressionKey]int64) + activeCounts := make(map[compressionKey]int64) + pendingCounts := make(map[compressionKey]int64) m.compressionMu.Lock() for _, job := range m.compressionJobs { key := compressionKey{ @@ -339,11 +367,16 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M algorithm: string(job.target.Policy.Algorithm), source: string(job.target.Source), } - counts[key]++ + switch job.state { + case compressionJobStatePendingDelay: + pendingCounts[key]++ + case compressionJobStateRunning: + activeCounts[key]++ + } } m.compressionMu.Unlock() - for key, count := range counts { + for key, count := range activeCounts { attrs := []attribute.KeyValue{ attribute.String("algorithm", key.algorithm), attribute.String("source", key.source), @@ -353,9 +386,20 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M } o.ObserveInt64(snapshotCompressionActiveTotal, count, metric.WithAttributes(attrs...)) } + for key, count := range pendingCounts { + attrs := []attribute.KeyValue{ + attribute.String("algorithm", key.algorithm), + attribute.String("source", key.source), + } + if key.hypervisor != "" { + attrs = append(attrs, attribute.String("hypervisor", key.hypervisor)) + } + o.ObserveInt64(snapshotCompressionPendingTotal, count, metric.WithAttributes(attrs...)) + } return nil }, snapshotCompressionActiveTotal, + snapshotCompressionPendingTotal, ) if err != nil { return nil, err @@ -392,6 +436,7 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M stateTransitions: stateTransitions, snapshotCompressionJobsTotal: snapshotCompressionJobsTotal, snapshotCompressionDuration: snapshotCompressionDuration, + snapshotCompressionWaitDuration: snapshotCompressionWaitDuration, snapshotCompressionSavedBytes: snapshotCompressionSavedBytes, snapshotCompressionRatio: snapshotCompressionRatio, snapshotCodecFallbacksTotal: snapshotCodecFallbacksTotal, @@ -524,7 +569,7 @@ func snapshotCompressionAttributes(hvType hypervisor.Type, algorithm snapshotsto return attrs } -func (m *manager) recordSnapshotCompressionJob(ctx context.Context, target compressionTarget, result snapshotCompressionResult, start time.Time, uncompressedSize, compressedSize int64) { +func (m *manager) recordSnapshotCompressionJob(ctx context.Context, target compressionTarget, result snapshotCompressionResult, compressionStart *time.Time, uncompressedSize, compressedSize int64) { if m.metrics == nil { return } @@ -534,7 +579,9 @@ func (m *manager) recordSnapshotCompressionJob(ctx context.Context, target compr attrsWithResult = append(attrsWithResult, attribute.String("result", string(result))) m.metrics.snapshotCompressionJobsTotal.Add(ctx, 1, metric.WithAttributes(attrsWithResult...)) - m.metrics.snapshotCompressionDuration.Record(ctx, time.Since(start).Seconds(), metric.WithAttributes(attrsWithResult...)) + if compressionStart != nil { + m.metrics.snapshotCompressionDuration.Record(ctx, time.Since(*compressionStart).Seconds(), metric.WithAttributes(attrsWithResult...)) + } if result != snapshotCompressionResultSuccess || uncompressedSize <= 0 || compressedSize < 0 { return @@ -548,6 +595,16 @@ func (m *manager) recordSnapshotCompressionJob(ctx context.Context, target compr m.metrics.snapshotCompressionRatio.Record(ctx, float64(compressedSize)/float64(uncompressedSize), metric.WithAttributes(attrs...)) } +func (m *manager) recordSnapshotCompressionWait(ctx context.Context, target compressionTarget, outcome snapshotCompressionWaitOutcome, start time.Time) { + if m.metrics == nil { + return + } + + attrs := snapshotCompressionAttributes(target.HypervisorType, target.Policy.Algorithm, target.Source) + attrs = append(attrs, attribute.String("outcome", string(outcome))) + m.metrics.snapshotCompressionWaitDuration.Record(ctx, time.Since(start).Seconds(), metric.WithAttributes(attrs...)) +} + func (m *manager) recordSnapshotRestoreMemoryPrepare(ctx context.Context, hvType hypervisor.Type, path snapshotMemoryPreparePath, result snapshotCompressionResult, start time.Time) { if m.metrics == nil { return diff --git a/lib/instances/metrics_test.go b/lib/instances/metrics_test.go index 37e251dd..cad949fb 100644 --- a/lib/instances/metrics_test.go +++ b/lib/instances/metrics_test.go @@ -27,7 +27,8 @@ func TestSnapshotCompressionMetrics_RecordAndObserve(t *testing.T) { paths: paths.New(t.TempDir()), compressionJobs: map[string]*compressionJob{ "job-1": { - done: make(chan struct{}), + done: make(chan struct{}), + state: compressionJobStateRunning, target: compressionTarget{ Key: "job-1", HypervisorType: hypervisor.TypeCloudHypervisor, @@ -38,6 +39,19 @@ func TestSnapshotCompressionMetrics_RecordAndObserve(t *testing.T) { }, }, }, + "job-2": { + done: make(chan struct{}), + state: compressionJobStatePendingDelay, + target: compressionTarget{ + Key: "job-2", + HypervisorType: hypervisor.TypeQEMU, + Source: snapshotCompressionSourceStandby, + Policy: snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + }, + }, + }, }, } @@ -46,7 +60,10 @@ func TestSnapshotCompressionMetrics_RecordAndObserve(t *testing.T) { m.metrics = metrics target := m.compressionJobs["job-1"].target - m.recordSnapshotCompressionJob(t.Context(), target, snapshotCompressionResultSuccess, time.Now().Add(-2*time.Second), 1024, 256) + startedAt := time.Now().Add(-2 * time.Second) + m.recordSnapshotCompressionJob(t.Context(), target, snapshotCompressionResultSuccess, &startedAt, 1024, 256) + m.recordSnapshotCompressionJob(t.Context(), m.compressionJobs["job-2"].target, snapshotCompressionResultSkipped, nil, 0, 0) + m.recordSnapshotCompressionWait(t.Context(), m.compressionJobs["job-2"].target, snapshotCompressionWaitOutcomeSkipped, time.Now().Add(-1500*time.Millisecond)) m.recordSnapshotCodecFallback(t.Context(), snapshotstore.SnapshotCompressionAlgorithmLz4, snapshotCodecOperationCompress, snapshotCodecFallbackReasonMissingBinary) m.recordSnapshotRestoreMemoryPrepare(t.Context(), hypervisor.TypeCloudHypervisor, snapshotMemoryPreparePathRaw, snapshotCompressionResultSuccess, time.Now().Add(-250*time.Millisecond)) m.recordSnapshotCompressionPreemption(t.Context(), snapshotCompressionPreemptionRestoreInstance, target) @@ -57,6 +74,7 @@ func TestSnapshotCompressionMetrics_RecordAndObserve(t *testing.T) { assertMetricNames(t, rm, []string{ "hypeman_snapshot_compression_jobs_total", "hypeman_snapshot_compression_duration_seconds", + "hypeman_snapshot_compression_wait_duration_seconds", "hypeman_snapshot_compression_saved_bytes", "hypeman_snapshot_compression_ratio", "hypeman_snapshot_codec_fallbacks_total", @@ -64,17 +82,29 @@ func TestSnapshotCompressionMetrics_RecordAndObserve(t *testing.T) { "hypeman_snapshot_restore_memory_prepare_duration_seconds", "hypeman_snapshot_compression_preemptions_total", "hypeman_snapshot_compression_active_total", + "hypeman_snapshot_compression_pending_total", }) jobsMetric := findMetric(t, rm, "hypeman_snapshot_compression_jobs_total") jobs, ok := jobsMetric.Data.(metricdata.Sum[int64]) require.True(t, ok) - require.Len(t, jobs.DataPoints, 1) - assert.Equal(t, int64(1), jobs.DataPoints[0].Value) - assert.Equal(t, "cloud-hypervisor", metricLabel(t, jobs.DataPoints[0].Attributes, "hypervisor")) - assert.Equal(t, "lz4", metricLabel(t, jobs.DataPoints[0].Attributes, "algorithm")) - assert.Equal(t, "standby", metricLabel(t, jobs.DataPoints[0].Attributes, "source")) - assert.Equal(t, "success", metricLabel(t, jobs.DataPoints[0].Attributes, "result")) + require.Len(t, jobs.DataPoints, 2) + for _, point := range jobs.DataPoints { + switch metricLabel(t, point.Attributes, "result") { + case "success": + assert.Equal(t, int64(1), point.Value) + assert.Equal(t, "cloud-hypervisor", metricLabel(t, point.Attributes, "hypervisor")) + assert.Equal(t, "lz4", metricLabel(t, point.Attributes, "algorithm")) + assert.Equal(t, "standby", metricLabel(t, point.Attributes, "source")) + case "skipped": + assert.Equal(t, int64(1), point.Value) + assert.Equal(t, "qemu", metricLabel(t, point.Attributes, "hypervisor")) + assert.Equal(t, "zstd", metricLabel(t, point.Attributes, "algorithm")) + assert.Equal(t, "standby", metricLabel(t, point.Attributes, "source")) + default: + t.Fatalf("unexpected compression job result datapoint: %s", metricLabel(t, point.Attributes, "result")) + } + } savedBytesMetric := findMetric(t, rm, "hypeman_snapshot_compression_saved_bytes") savedBytes, ok := savedBytesMetric.Data.(metricdata.Histogram[int64]) @@ -114,6 +144,20 @@ func TestSnapshotCompressionMetrics_RecordAndObserve(t *testing.T) { assert.Equal(t, int64(1), active.DataPoints[0].Value) assert.Equal(t, "lz4", metricLabel(t, active.DataPoints[0].Attributes, "algorithm")) assert.Equal(t, "standby", metricLabel(t, active.DataPoints[0].Attributes, "source")) + + pendingMetric := findMetric(t, rm, "hypeman_snapshot_compression_pending_total") + pending, ok := pendingMetric.Data.(metricdata.Gauge[int64]) + require.True(t, ok) + require.Len(t, pending.DataPoints, 1) + assert.Equal(t, int64(1), pending.DataPoints[0].Value) + assert.Equal(t, "zstd", metricLabel(t, pending.DataPoints[0].Attributes, "algorithm")) + assert.Equal(t, "standby", metricLabel(t, pending.DataPoints[0].Attributes, "source")) + + waitMetric := findMetric(t, rm, "hypeman_snapshot_compression_wait_duration_seconds") + waitDurations, ok := waitMetric.Data.(metricdata.Histogram[float64]) + require.True(t, ok) + require.Len(t, waitDurations.DataPoints, 1) + assert.Equal(t, "skipped", metricLabel(t, waitDurations.DataPoints[0].Attributes, "outcome")) } func TestLifecycleEventMetrics_ObserveSubscribersQueueDepthAndDrops(t *testing.T) { @@ -439,6 +483,71 @@ func TestLifecycleDurationMetrics_RecordCompressionLabels(t *testing.T) { assert.Equal(t, "none", metricLabel(t, standby.DataPoints[0].Attributes, "level")) } +func TestEnsureSnapshotMemoryReadySkipsPendingCompressionWithoutPreemptionMetric(t *testing.T) { + t.Parallel() + + reader := otelmetric.NewManualReader() + provider := otelmetric.NewMeterProvider(otelmetric.WithReader(reader)) + + mgr, _ := setupTestManager(t) + metrics, err := newInstanceMetrics(provider.Meter("test"), nil, mgr) + require.NoError(t, err) + mgr.metrics = metrics + + delay := 30 * time.Second + timer := newFakeCompressionTimer() + mgr.compressionTimerFactory = func(got time.Duration) compressionTimer { + require.Equal(t, delay, got) + return timer + } + + snapshotDir := t.TempDir() + rawPath := filepath.Join(snapshotDir, "memory") + require.NoError(t, os.WriteFile(rawPath, []byte("pending raw snapshot"), 0o644)) + + target := compressionTarget{ + Key: "instance:pending", + OwnerID: "pending", + SnapshotDir: snapshotDir, + HypervisorType: hypervisor.TypeCloudHypervisor, + Source: snapshotCompressionSourceStandby, + Policy: snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }, + Delay: delay, + } + + mgr.startCompressionJob(t.Context(), target) + + require.Eventually(t, func() bool { + mgr.compressionMu.Lock() + defer mgr.compressionMu.Unlock() + job, ok := mgr.compressionJobs[target.Key] + return ok && job.state == compressionJobStatePendingDelay + }, time.Second, 10*time.Millisecond) + + require.NoError(t, mgr.ensureSnapshotMemoryReady(t.Context(), snapshotDir, target.Key, hypervisor.TypeCloudHypervisor)) + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(t.Context(), &rm)) + + jobsMetric := findMetric(t, rm, "hypeman_snapshot_compression_jobs_total") + jobs, ok := jobsMetric.Data.(metricdata.Sum[int64]) + require.True(t, ok) + require.Len(t, jobs.DataPoints, 1) + assert.Equal(t, "skipped", metricLabel(t, jobs.DataPoints[0].Attributes, "result")) + + waitMetric := findMetric(t, rm, "hypeman_snapshot_compression_wait_duration_seconds") + waitDurations, ok := waitMetric.Data.(metricdata.Histogram[float64]) + require.True(t, ok) + require.Len(t, waitDurations.DataPoints, 1) + assert.Equal(t, "skipped", metricLabel(t, waitDurations.DataPoints[0].Attributes, "outcome")) + + assert.False(t, metricExists(rm, "hypeman_snapshot_compression_preemptions_total"), "pending-delay cancellation should not record a preemption") +} + func assertMetricNames(t *testing.T, rm metricdata.ResourceMetrics, expected []string) { t.Helper() @@ -454,6 +563,17 @@ func assertMetricNames(t *testing.T, rm metricdata.ResourceMetrics, expected []s } } +func metricExists(rm metricdata.ResourceMetrics, name string) bool { + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name == name { + return true + } + } + } + return false +} + func findMetric(t *testing.T, rm metricdata.ResourceMetrics, name string) metricdata.Metrics { t.Helper() diff --git a/lib/instances/network_test.go b/lib/instances/network_test.go index 60eaf4e3..9122a8f7 100644 --- a/lib/instances/network_test.go +++ b/lib/instances/network_test.go @@ -245,42 +245,42 @@ func TestDockerForwardChainRestored(t *testing.T) { require.NoError(t, manager.networkManager.Initialize(ctx, nil)) // Check if DOCKER-FORWARD chain exists (Docker must be running on host). - checkChain := exec.Command("iptables", "-L", "DOCKER-FORWARD", "-n") + checkChain := exec.Command("iptables", "-w", "5", "-L", "DOCKER-FORWARD", "-n") if checkChain.Run() != nil { t.Skip("DOCKER-FORWARD chain not present (Docker not running), skipping") } // Verify jump currently exists. - checkJump := exec.Command("iptables", "-C", "FORWARD", "-j", "DOCKER-FORWARD") + checkJump := exec.Command("iptables", "-w", "5", "-C", "FORWARD", "-j", "DOCKER-FORWARD") require.NoError(t, checkJump.Run(), "DOCKER-FORWARD jump should exist before test") // Safety net: restore the jump if the test fails or aborts after we delete it, // so we don't leave the host's Docker networking broken. t.Cleanup(func() { - check := exec.Command("iptables", "-C", "FORWARD", "-j", "DOCKER-FORWARD") + check := exec.Command("iptables", "-w", "5", "-C", "FORWARD", "-j", "DOCKER-FORWARD") if check.Run() != nil { - restore := exec.Command("iptables", "-A", "FORWARD", "-j", "DOCKER-FORWARD") + restore := exec.Command("iptables", "-w", "5", "-A", "FORWARD", "-j", "DOCKER-FORWARD") _ = restore.Run() } }) // Simulate the hypervisor flush: remove every jump. for { - delJump := exec.Command("iptables", "-D", "FORWARD", "-j", "DOCKER-FORWARD") + delJump := exec.Command("iptables", "-w", "5", "-D", "FORWARD", "-j", "DOCKER-FORWARD") if err := delJump.Run(); err != nil { break } } // Confirm it's gone. - checkGone := exec.Command("iptables", "-C", "FORWARD", "-j", "DOCKER-FORWARD") + checkGone := exec.Command("iptables", "-w", "5", "-C", "FORWARD", "-j", "DOCKER-FORWARD") require.Error(t, checkGone.Run(), "DOCKER-FORWARD jump should be gone after delete") // Re-initialize network — this should restore the jump. require.NoError(t, manager.networkManager.Initialize(ctx, nil)) // Verify jump is restored. - checkRestored := exec.Command("iptables", "-C", "FORWARD", "-j", "DOCKER-FORWARD") + checkRestored := exec.Command("iptables", "-w", "5", "-C", "FORWARD", "-j", "DOCKER-FORWARD") require.NoError(t, checkRestored.Run(), "ensureDockerForwardJump should have restored the DOCKER-FORWARD jump") } diff --git a/lib/instances/restore.go b/lib/instances/restore.go index c4658ba7..268f769b 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -89,6 +89,7 @@ func (m *manager) restoreInstance( if err != nil { return nil, fmt.Errorf("prepare standby snapshot memory: %w", err) } + stored.PendingStandbyCompression = nil starter, err := m.getVMStarter(stored.HypervisorType) if err != nil { return nil, fmt.Errorf("get vm starter: %w", err) diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 9ea0296e..ca188251 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -107,8 +107,11 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps if err != nil { return nil, fmt.Errorf("wait for source instance compression to stop: %w", err) } - if target != nil { - m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionCreateSnapshot, *target) + if target != nil && target.State == compressionJobStateRunning { + m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionCreateSnapshot, target.Target) + } + if err := m.clearPendingStandbyCompression(ctx, id); err != nil && !errors.Is(err, ErrNotFound) { + return nil, fmt.Errorf("clear source standby compression plan: %w", err) } if err := m.ensureSnapshotMemoryReady(ctx, m.paths.InstanceSnapshotLatest(id), "", stored.HypervisorType); err != nil { return nil, fmt.Errorf("prepare source snapshot memory for copy: %w", err) @@ -224,8 +227,8 @@ func (m *manager) deleteSnapshot(ctx context.Context, snapshotID string) error { if err != nil { return fmt.Errorf("wait for snapshot compression to stop: %w", err) } - if target != nil { - m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionDeleteSnapshot, *target) + if target != nil && target.State == compressionJobStateRunning { + m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionDeleteSnapshot, target.Target) } if err := m.snapshotStore().Delete(snapshotID); err != nil { if errors.Is(err, snapshotstore.ErrNotFound) { @@ -274,15 +277,15 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str if err != nil { return nil, fmt.Errorf("wait for snapshot compression to stop: %w", err) } - if target != nil { - m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionRestoreSnapshot, *target) + if target != nil && target.State == compressionJobStateRunning { + m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionRestoreSnapshot, target.Target) } target, err = m.cancelAndWaitCompressionJob(ctx, m.snapshotJobKeyForInstance(id)) if err != nil { return nil, fmt.Errorf("wait for source instance compression to stop: %w", err) } - if target != nil { - m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionRestoreSnapshot, *target) + if target != nil && target.State == compressionJobStateRunning { + m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionRestoreSnapshot, target.Target) } if err := m.replaceInstanceWithSnapshotPayload(snapshotID, id); err != nil { @@ -403,8 +406,8 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS if err != nil { return nil, fmt.Errorf("wait for snapshot compression to stop: %w", err) } - if target != nil { - m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionForkSnapshot, *target) + if target != nil && target.State == compressionJobStateRunning { + m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionForkSnapshot, target.Target) } if err := m.ensureSnapshotMemoryReady(ctx, m.paths.SnapshotGuestDir(snapshotID), "", rec.StoredMetadata.HypervisorType); err != nil { return nil, fmt.Errorf("prepare snapshot memory for fork: %w", err) diff --git a/lib/instances/snapshot_compression.go b/lib/instances/snapshot_compression.go index 5c930002..61d1ebff 100644 --- a/lib/instances/snapshot_compression.go +++ b/lib/instances/snapshot_compression.go @@ -35,9 +35,17 @@ const ( type compressionJob struct { cancel context.CancelFunc done chan struct{} + state compressionJobState target compressionTarget } +type compressionJobState string + +const ( + compressionJobStatePendingDelay compressionJobState = "pending_delay" + compressionJobStateRunning compressionJobState = "running" +) + type compressionTarget struct { Key string OwnerID string @@ -46,6 +54,29 @@ type compressionTarget struct { HypervisorType hypervisor.Type Source snapshotCompressionSource Policy snapshotstore.SnapshotCompressionConfig + Delay time.Duration +} + +type canceledCompressionJob struct { + Target compressionTarget + State compressionJobState +} + +type compressionTimer interface { + Chan() <-chan time.Time + Stop() bool +} + +type realCompressionTimer struct { + timer *time.Timer +} + +func (t *realCompressionTimer) Chan() <-chan time.Time { + return t.timer.C +} + +func (t *realCompressionTimer) Stop() bool { + return t.timer.Stop() } type nativeCodecRuntime struct { @@ -80,13 +111,42 @@ func cloneCompressionConfig(cfg *snapshotstore.SnapshotCompressionConfig) *snaps return &cloned } +func cloneDurationPtr(v *time.Duration) *time.Duration { + if v == nil { + return nil + } + cloned := *v + return &cloned +} + func cloneSnapshotPolicy(policy *SnapshotPolicy) *SnapshotPolicy { if policy == nil { return nil } return &SnapshotPolicy{ - Compression: cloneCompressionConfig(policy.Compression), + Compression: cloneCompressionConfig(policy.Compression), + StandbyCompressionDelay: cloneDurationPtr(policy.StandbyCompressionDelay), + } +} + +func clonePendingStandbyCompression(plan *PendingStandbyCompression) *PendingStandbyCompression { + if plan == nil { + return nil + } + return &PendingStandbyCompression{ + Policy: *cloneCompressionConfig(&plan.Policy), + NotBefore: plan.NotBefore, + } +} + +func normalizeStandbyCompressionDelay(delay *time.Duration) (*time.Duration, error) { + if delay == nil { + return nil, nil + } + if *delay < 0 { + return nil, fmt.Errorf("%w: standby compression delay cannot be negative", ErrInvalidRequest) } + return cloneDurationPtr(delay), nil } func normalizeCompressionConfig(cfg *snapshotstore.SnapshotCompressionConfig) (snapshotstore.SnapshotCompressionConfig, error) { @@ -147,6 +207,31 @@ func (m *manager) resolveStandbyCompressionPolicy(stored *StoredMetadata, overri return m.resolveConfiguredCompressionPolicy(stored, override) } +func (m *manager) resolveStandbyCompressionDelay(stored *StoredMetadata, override *time.Duration) (time.Duration, error) { + // Standby delay is intentionally scoped to the standby request and the + // per-instance snapshot policy. This change does not introduce a + // server-global standby delay default. + candidates := []*time.Duration{override} + if stored != nil && stored.SnapshotPolicy != nil { + candidates = append(candidates, stored.SnapshotPolicy.StandbyCompressionDelay) + } + + for _, candidate := range candidates { + if candidate == nil { + continue + } + normalized, err := normalizeStandbyCompressionDelay(candidate) + if err != nil { + return 0, err + } + if normalized != nil { + return *normalized, nil + } + } + + return 0, nil +} + func (m *manager) resolveConfiguredCompressionPolicy(stored *StoredMetadata, override *snapshotstore.SnapshotCompressionConfig) (*snapshotstore.SnapshotCompressionConfig, error) { candidates := []*snapshotstore.SnapshotCompressionConfig{override} if stored != nil && stored.SnapshotPolicy != nil { @@ -178,6 +263,115 @@ func (m *manager) snapshotJobKeyForSnapshot(snapshotID string) string { return "snapshot:" + snapshotID } +func instanceIDFromCompressionJobKey(key string) (string, bool) { + const prefix = "instance:" + if !strings.HasPrefix(key, prefix) { + return "", false + } + instanceID := strings.TrimPrefix(key, prefix) + if instanceID == "" { + return "", false + } + return instanceID, true +} + +func (m *manager) clearPendingStandbyCompression(ctx context.Context, instanceID string) error { + if instanceID == "" { + return nil + } + + meta, err := m.loadMetadata(instanceID) + if err != nil { + return err + } + if meta.StoredMetadata.PendingStandbyCompression == nil { + return nil + } + + meta.StoredMetadata.PendingStandbyCompression = nil + if err := m.saveMetadata(meta); err != nil { + return fmt.Errorf("save metadata: %w", err) + } + + logger.FromContext(ctx).DebugContext(ctx, "cleared pending standby compression plan", "instance_id", instanceID) + return nil +} + +func (m *manager) recoverPendingStandbyCompressionJobs(ctx context.Context) error { + metaFiles, err := m.listMetadataFiles() + if err != nil { + return fmt.Errorf("list metadata files: %w", err) + } + + log := logger.FromContext(ctx) + for _, metaPath := range metaFiles { + instanceID := filepath.Base(filepath.Dir(metaPath)) + meta, err := m.loadMetadata(instanceID) + if err != nil { + log.WarnContext(ctx, "failed to load instance metadata during standby compression recovery", "instance_id", instanceID, "metadata_path", metaPath, "error", err) + continue + } + + plan := clonePendingStandbyCompression(meta.StoredMetadata.PendingStandbyCompression) + if plan == nil { + continue + } + if !plan.Policy.Enabled { + log.WarnContext(ctx, "clearing invalid pending standby compression plan with disabled policy", "instance_id", instanceID) + if err := m.clearPendingStandbyCompression(ctx, instanceID); err != nil && !errors.Is(err, ErrNotFound) { + log.WarnContext(ctx, "failed to clear invalid pending standby compression plan", "instance_id", instanceID, "error", err) + } + continue + } + + state := m.deriveStateWithoutHydration(ctx, &meta.StoredMetadata) + if state.State != StateStandby { + log.InfoContext(ctx, "clearing stale pending standby compression plan for non-standby instance", "instance_id", instanceID, "state", state.State) + if err := m.clearPendingStandbyCompression(ctx, instanceID); err != nil && !errors.Is(err, ErrNotFound) { + log.WarnContext(ctx, "failed to clear stale pending standby compression plan", "instance_id", instanceID, "error", err) + } + continue + } + + snapshotDir := m.paths.InstanceSnapshotLatest(instanceID) + if _, ok := findRawSnapshotMemoryFile(snapshotDir); ok { + delay := time.Duration(0) + now := m.nowUTC() + if plan.NotBefore.After(now) { + delay = plan.NotBefore.Sub(now) + } + log.InfoContext(ctx, "recovering pending standby snapshot compression", + "instance_id", instanceID, + "operation", "recover_snapshot_compression", + "source", string(snapshotCompressionSourceStandby), + "algorithm", string(plan.Policy.Algorithm), + "compression_delay", delay.String(), + ) + m.startCompressionJob(ctx, compressionTarget{ + Key: m.snapshotJobKeyForInstance(instanceID), + OwnerID: instanceID, + SnapshotDir: snapshotDir, + HypervisorType: meta.StoredMetadata.HypervisorType, + Source: snapshotCompressionSourceStandby, + Policy: plan.Policy, + Delay: delay, + }) + continue + } + + if _, _, ok := findCompressedSnapshotMemoryFile(snapshotDir); ok { + log.InfoContext(ctx, "clearing pending standby compression plan after finding compressed snapshot artifact", "instance_id", instanceID) + } else { + log.InfoContext(ctx, "clearing pending standby compression plan after standby snapshot disappeared", "instance_id", instanceID) + } + if err := m.clearPendingStandbyCompression(ctx, instanceID); err != nil && !errors.Is(err, ErrNotFound) { + log.WarnContext(ctx, "failed to clear completed pending standby compression plan", "instance_id", instanceID, "error", err) + } + } + + return nil +} + func nativeCodecBinaryName(algorithm snapshotstore.SnapshotCompressionAlgorithm) string { switch algorithm { case snapshotstore.SnapshotCompressionAlgorithmLz4: @@ -256,6 +450,26 @@ func invalidateNativeCodecPath(runtime nativeCodecRuntime, algorithm snapshotsto } } +func (m *manager) newCompressionTimer(delay time.Duration) compressionTimer { + if m != nil && m.compressionTimerFactory != nil { + return m.compressionTimerFactory(delay) + } + return &realCompressionTimer{timer: time.NewTimer(delay)} +} + +func stopCompressionTimer(timer compressionTimer) { + if timer == nil { + return + } + if timer.Stop() { + return + } + select { + case <-timer.Chan(): + default: + } +} + func recordNativeCodecFallback(ctx context.Context, runtime nativeCodecRuntime, algorithm snapshotstore.SnapshotCompressionAlgorithm, operation snapshotCodecOperation, reason snapshotCodecFallbackReason, nativeBinary string, err error) { logger.FromContext(ctx).WarnContext( ctx, @@ -302,6 +516,11 @@ func (m *manager) startCompressionJob(ctx context.Context, target compressionTar return } + initialState := compressionJobStateRunning + if target.Delay > 0 { + initialState = compressionJobStatePendingDelay + } + m.compressionMu.Lock() if _, exists := m.compressionJobs[target.Key]; exists { m.compressionMu.Unlock() @@ -311,6 +530,7 @@ func (m *manager) startCompressionJob(ctx context.Context, target compressionTar job := &compressionJob{ cancel: cancel, done: make(chan struct{}), + state: initialState, target: target, } parentSpanContext := trace.SpanContextFromContext(ctx) @@ -318,12 +538,102 @@ func (m *manager) startCompressionJob(ctx context.Context, target compressionTar m.compressionMu.Unlock() go func() { - start := time.Now() result := snapshotCompressionResultSuccess var uncompressedSize int64 var compressedSize int64 var spanErr error + log := logger.FromContext(ctx) metricsCtx := context.Background() + var compressionStart *time.Time + + defer func() { + m.recordSnapshotCompressionJob(metricsCtx, target, result, compressionStart, uncompressedSize, compressedSize) + if target.Source == snapshotCompressionSourceStandby && target.SnapshotID == "" { + if err := m.clearPendingStandbyCompression(context.Background(), target.OwnerID); err != nil && !errors.Is(err, ErrNotFound) { + log.WarnContext(context.Background(), "failed to clear pending standby compression plan after job completion", "instance_id", target.OwnerID, "error", err) + } + } + m.compressionMu.Lock() + delete(m.compressionJobs, target.Key) + m.compressionMu.Unlock() + close(job.done) + }() + + if target.Delay > 0 { + waitStart := time.Now() + waitSpanOptions := []trace.SpanStartOption{ + trace.WithNewRoot(), + trace.WithAttributes( + attribute.String("operation", "snapshot_compression_wait"), + attribute.String("owner_id", target.OwnerID), + attribute.String("snapshot_id", target.SnapshotID), + attribute.String("hypervisor", string(target.HypervisorType)), + attribute.String("source", string(target.Source)), + attribute.String("algorithm", string(target.Policy.Algorithm)), + attribute.Float64("compression_delay_seconds", target.Delay.Seconds()), + ), + } + if parentSpanContext.IsValid() { + waitSpanOptions = append(waitSpanOptions, trace.WithLinks(trace.Link{SpanContext: parentSpanContext})) + } + waitCtx, waitSpan := m.tracerOrDefault().Start(context.Background(), "instances.snapshot_compression_wait", waitSpanOptions...) + timer := m.newCompressionTimer(target.Delay) + log.InfoContext(ctx, "snapshot compression queued with standby delay", + "owner_id", target.OwnerID, + "snapshot_id", target.SnapshotID, + "source", string(target.Source), + "algorithm", string(target.Policy.Algorithm), + "compression_delay", target.Delay.String(), + ) + + select { + case <-timer.Chan(): + m.recordSnapshotCompressionWait(metricsCtx, target, snapshotCompressionWaitOutcomeStarted, waitStart) + waitSpan.SetAttributes( + attribute.String("wait_outcome", string(snapshotCompressionWaitOutcomeStarted)), + attribute.Float64("wait_duration_seconds", time.Since(waitStart).Seconds()), + ) + waitSpan.End() + case <-jobCtx.Done(): + stopCompressionTimer(timer) + result = snapshotCompressionResultSkipped + m.recordSnapshotCompressionWait(metricsCtx, target, snapshotCompressionWaitOutcomeSkipped, waitStart) + waitSpan.SetAttributes( + attribute.String("wait_outcome", string(snapshotCompressionWaitOutcomeSkipped)), + attribute.Float64("wait_duration_seconds", time.Since(waitStart).Seconds()), + ) + waitSpan.End() + if target.SnapshotID != "" { + if err := m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateNone, "", nil, nil, nil); err != nil { + log.ErrorContext(jobCtx, "failed to update snapshot compression metadata", "snapshot_id", target.SnapshotID, "snapshot_dir", target.SnapshotDir, "state", snapshotstore.SnapshotCompressionStateNone, "error", err) + } + } + log.InfoContext(waitCtx, "snapshot compression skipped before start", + "owner_id", target.OwnerID, + "snapshot_id", target.SnapshotID, + "source", string(target.Source), + "algorithm", string(target.Policy.Algorithm), + "compression_delay", target.Delay.String(), + ) + return + } + + m.compressionMu.Lock() + if currentJob, ok := m.compressionJobs[target.Key]; ok && currentJob == job { + currentJob.state = compressionJobStateRunning + } + m.compressionMu.Unlock() + log.InfoContext(waitCtx, "standby compression delay elapsed; starting compression", + "owner_id", target.OwnerID, + "snapshot_id", target.SnapshotID, + "source", string(target.Source), + "algorithm", string(target.Policy.Algorithm), + "compression_delay", target.Delay.String(), + ) + } + + startedAt := time.Now() + compressionStart = &startedAt spanOptions := []trace.SpanStartOption{ trace.WithNewRoot(), trace.WithAttributes( @@ -338,17 +648,9 @@ func (m *manager) startCompressionJob(ctx context.Context, target compressionTar if parentSpanContext.IsValid() { spanOptions = append(spanOptions, trace.WithLinks(trace.Link{SpanContext: parentSpanContext})) } - metricsCtx, span := m.tracerOrDefault().Start(metricsCtx, "instances.snapshot_compression", spanOptions...) - log := logger.FromContext(ctx) - - defer func() { - m.recordSnapshotCompressionJob(metricsCtx, target, result, start, uncompressedSize, compressedSize) - finishInstancesSpan(span, spanErr) - m.compressionMu.Lock() - delete(m.compressionJobs, target.Key) - m.compressionMu.Unlock() - close(job.done) - }() + compressionCtx, span := m.tracerOrDefault().Start(context.Background(), "instances.snapshot_compression", spanOptions...) + metricsCtx = compressionCtx + defer func() { finishInstancesSpan(span, spanErr) }() rawPath, ok := findRawSnapshotMemoryFile(target.SnapshotDir) if !ok { @@ -414,7 +716,7 @@ func (m *manager) waitCompressionJobContext(ctx context.Context, key string) err } } -func (m *manager) cancelAndWaitCompressionJob(ctx context.Context, key string) (*compressionTarget, error) { +func (m *manager) cancelAndWaitCompressionJob(ctx context.Context, key string) (*canceledCompressionJob, error) { if key == "" { return nil, nil } @@ -432,8 +734,10 @@ func (m *manager) cancelAndWaitCompressionJob(ctx context.Context, key string) ( select { case <-job.done: - target := job.target - return &target, nil + return &canceledCompressionJob{ + Target: job.target, + State: job.state, + }, nil case <-ctx.Done(): return nil, ctx.Err() } @@ -441,14 +745,20 @@ func (m *manager) cancelAndWaitCompressionJob(ctx context.Context, key string) ( func (m *manager) ensureSnapshotMemoryReady(ctx context.Context, snapshotDir, jobKey string, hvType hypervisor.Type) error { start := time.Now() + log := logger.FromContext(ctx) if jobKey != "" { target, err := m.cancelAndWaitCompressionJob(ctx, jobKey) if err != nil { return err } - if target != nil { - m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionRestoreInstance, *target) + if target != nil && target.State == compressionJobStateRunning { + m.recordSnapshotCompressionPreemption(ctx, snapshotCompressionPreemptionRestoreInstance, target.Target) + } + if instanceID, ok := instanceIDFromCompressionJobKey(jobKey); ok { + if err := m.clearPendingStandbyCompression(ctx, instanceID); err != nil && !errors.Is(err, ErrNotFound) { + log.WarnContext(ctx, "failed to clear pending standby compression plan while preparing snapshot memory", "instance_id", instanceID, "error", err) + } } } diff --git a/lib/instances/snapshot_compression_test.go b/lib/instances/snapshot_compression_test.go index 274f9c50..86a61e48 100644 --- a/lib/instances/snapshot_compression_test.go +++ b/lib/instances/snapshot_compression_test.go @@ -4,10 +4,13 @@ import ( "context" "errors" "os" + "path/filepath" + "sync" "testing" "time" "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/paths" snapshotstore "github.com/kernel/hypeman/lib/snapshot" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -183,6 +186,60 @@ func TestResolveStandbyCompressionPolicyInvalidConfiguredDefaultIsInvalidRequest assert.True(t, errors.Is(err, ErrInvalidRequest)) } +func TestResolveStandbyCompressionDelayPrecedence(t *testing.T) { + t.Parallel() + + instanceDelay := 2 * time.Minute + overrideDelay := 15 * time.Second + m := &manager{} + + delay, err := m.resolveStandbyCompressionDelay(&StoredMetadata{ + SnapshotPolicy: &SnapshotPolicy{ + StandbyCompressionDelay: &instanceDelay, + }, + }, &overrideDelay) + require.NoError(t, err) + assert.Equal(t, overrideDelay, delay) + + delay, err = m.resolveStandbyCompressionDelay(&StoredMetadata{ + SnapshotPolicy: &SnapshotPolicy{ + StandbyCompressionDelay: &instanceDelay, + }, + }, nil) + require.NoError(t, err) + assert.Equal(t, instanceDelay, delay) + + delay, err = m.resolveStandbyCompressionDelay(&StoredMetadata{}, nil) + require.NoError(t, err) + assert.Zero(t, delay) +} + +func TestResolveStandbyCompressionDelayIgnoresServerDefaults(t *testing.T) { + t.Parallel() + + serverDelay := 3 * time.Minute + m := &manager{ + snapshotDefaults: SnapshotPolicy{ + StandbyCompressionDelay: &serverDelay, + }, + } + + delay, err := m.resolveStandbyCompressionDelay(&StoredMetadata{}, nil) + require.NoError(t, err) + assert.Zero(t, delay) +} + +func TestResolveStandbyCompressionDelayRejectsNegativeDuration(t *testing.T) { + t.Parallel() + + m := &manager{} + negative := -1 * time.Second + + _, err := m.resolveStandbyCompressionDelay(&StoredMetadata{}, &negative) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrInvalidRequest)) +} + func TestCompressionMetadataForExistingArtifactUsesActualAlgorithm(t *testing.T) { t.Parallel() @@ -225,6 +282,23 @@ func TestValidateCreateRequestSnapshotPolicy(t *testing.T) { assert.True(t, errors.Is(err, ErrInvalidRequest)) } +func TestValidateCreateRequestRejectsNegativeStandbyCompressionDelay(t *testing.T) { + t.Parallel() + + negative := -1 * time.Second + req := &CreateInstanceRequest{ + Name: "compression-delay-test", + Image: "docker.io/library/alpine:latest", + SnapshotPolicy: &SnapshotPolicy{ + StandbyCompressionDelay: &negative, + }, + } + + err := validateCreateRequest(req) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrInvalidRequest)) +} + func TestValidateCreateSnapshotRequestRejectsStoppedCompression(t *testing.T) { t.Parallel() @@ -320,6 +394,7 @@ func installCancelableCompressionJob(mgr *manager, target compressionTarget) *co mgr.compressionJobs[target.Key] = &compressionJob{ cancel: cancel, done: done, + state: compressionJobStateRunning, target: target, } mgr.compressionMu.Unlock() @@ -345,3 +420,300 @@ func assertCompressionJobCanceled(t *testing.T, mgr *manager, target *compressio return !ok }, time.Second, 10*time.Millisecond) } + +func newSnapshotCompressionTestManager(t *testing.T) *manager { + t.Helper() + + p := paths.New(t.TempDir()) + return &manager{ + paths: p, + now: time.Now, + compressionJobs: make(map[string]*compressionJob), + lifecycleEvents: newLifecycleSubscribers(), + } +} + +func TestStartCompressionJobDelayedCancellationRecordsSkipped(t *testing.T) { + t.Parallel() + + mgr := newSnapshotCompressionTestManager(t) + delay := 45 * time.Second + timer := newFakeCompressionTimer() + mgr.compressionTimerFactory = func(got time.Duration) compressionTimer { + require.Equal(t, delay, got) + return timer + } + + snapshotDir := t.TempDir() + rawPath := filepath.Join(snapshotDir, "memory") + require.NoError(t, os.WriteFile(rawPath, []byte("delayed standby snapshot"), 0o644)) + + target := compressionTarget{ + Key: "instance:delayed", + OwnerID: "delayed", + SnapshotDir: snapshotDir, + Source: snapshotCompressionSourceStandby, + Policy: snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }, + Delay: delay, + } + + mgr.startCompressionJob(context.Background(), target) + + require.Eventually(t, func() bool { + mgr.compressionMu.Lock() + defer mgr.compressionMu.Unlock() + job, ok := mgr.compressionJobs[target.Key] + return ok && job.state == compressionJobStatePendingDelay + }, time.Second, 10*time.Millisecond) + + canceled, err := mgr.cancelAndWaitCompressionJob(context.Background(), target.Key) + require.NoError(t, err) + require.NotNil(t, canceled) + assert.Equal(t, compressionJobStatePendingDelay, canceled.State) + + _, err = os.Stat(rawPath) + require.NoError(t, err, "raw snapshot should remain available when delay is skipped") + _, err = os.Stat(rawPath + ".zst") + require.Error(t, err) + assert.True(t, os.IsNotExist(err)) +} + +func TestRecoverPendingStandbyCompressionJobsRequeuesDelayedJob(t *testing.T) { + t.Parallel() + + mgr := newSnapshotCompressionTestManager(t) + now := time.Date(2026, time.April, 6, 12, 0, 0, 0, time.UTC) + mgr.now = func() time.Time { return now } + + const instanceID = "recover-delayed" + delay := 30 * time.Second + snapshotDir := mgr.paths.InstanceSnapshotLatest(instanceID) + rawPath := filepath.Join(snapshotDir, "memory") + + require.NoError(t, mgr.ensureDirectories(instanceID)) + require.NoError(t, os.MkdirAll(snapshotDir, 0o755)) + require.NoError(t, os.WriteFile(rawPath, []byte("pending standby snapshot"), 0o644)) + require.NoError(t, mgr.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: instanceID, + Name: instanceID, + DataDir: mgr.paths.InstanceDir(instanceID), + HypervisorType: hypervisor.TypeCloudHypervisor, + CreatedAt: now, + StoppedAt: &now, + PendingStandbyCompression: &PendingStandbyCompression{ + Policy: snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }, + NotBefore: now.Add(delay), + }, + }})) + + timer := newFakeCompressionTimer() + delayCh := make(chan time.Duration, 1) + mgr.compressionTimerFactory = func(got time.Duration) compressionTimer { + delayCh <- got + return timer + } + + require.NoError(t, mgr.recoverPendingStandbyCompressionJobs(context.Background())) + + require.Eventually(t, func() bool { + mgr.compressionMu.Lock() + defer mgr.compressionMu.Unlock() + job, ok := mgr.compressionJobs[mgr.snapshotJobKeyForInstance(instanceID)] + return ok && job.state == compressionJobStatePendingDelay + }, time.Second, 10*time.Millisecond) + select { + case gotDelay := <-delayCh: + assert.Equal(t, delay, gotDelay) + case <-time.After(time.Second): + t.Fatal("timed out waiting for recovered compression delay") + } + + canceled, err := mgr.cancelAndWaitCompressionJob(context.Background(), mgr.snapshotJobKeyForInstance(instanceID)) + require.NoError(t, err) + require.NotNil(t, canceled) + assert.Equal(t, compressionJobStatePendingDelay, canceled.State) + + meta, err := mgr.loadMetadata(instanceID) + require.NoError(t, err) + assert.Nil(t, meta.StoredMetadata.PendingStandbyCompression) + _, err = os.Stat(rawPath) + require.NoError(t, err) +} + +func TestRecoverPendingStandbyCompressionJobsStartsImmediateCompression(t *testing.T) { + t.Parallel() + + mgr := newSnapshotCompressionTestManager(t) + now := time.Date(2026, time.April, 6, 12, 5, 0, 0, time.UTC) + mgr.now = func() time.Time { return now } + + const instanceID = "recover-immediate" + snapshotDir := mgr.paths.InstanceSnapshotLatest(instanceID) + rawPath := filepath.Join(snapshotDir, "memory") + + require.NoError(t, mgr.ensureDirectories(instanceID)) + require.NoError(t, os.MkdirAll(snapshotDir, 0o755)) + require.NoError(t, os.WriteFile(rawPath, []byte("standby snapshot that should compress now"), 0o644)) + require.NoError(t, mgr.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: instanceID, + Name: instanceID, + DataDir: mgr.paths.InstanceDir(instanceID), + HypervisorType: hypervisor.TypeCloudHypervisor, + CreatedAt: now, + StoppedAt: &now, + PendingStandbyCompression: &PendingStandbyCompression{ + Policy: snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }, + NotBefore: now.Add(-time.Second), + }, + }})) + mgr.compressionTimerFactory = func(time.Duration) compressionTimer { + t.Fatal("unexpected delay timer for immediate recovery") + return newFakeCompressionTimer() + } + + require.NoError(t, mgr.recoverPendingStandbyCompressionJobs(context.Background())) + + require.Eventually(t, func() bool { + meta, err := mgr.loadMetadata(instanceID) + if err != nil { + return false + } + _, rawExistsErr := os.Stat(rawPath) + _, _, compressed := findCompressedSnapshotMemoryFile(snapshotDir) + return meta.StoredMetadata.PendingStandbyCompression == nil && os.IsNotExist(rawExistsErr) && compressed + }, 5*time.Second, 20*time.Millisecond) +} + +func TestRecoverPendingStandbyCompressionJobsClearsStalePlans(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + prepare func(t *testing.T, mgr *manager, instanceID string, now time.Time) + }{ + { + name: "stopped_instance_without_snapshot", + prepare: func(t *testing.T, mgr *manager, instanceID string, now time.Time) { + t.Helper() + require.NoError(t, mgr.ensureDirectories(instanceID)) + require.NoError(t, mgr.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: instanceID, + Name: instanceID, + DataDir: mgr.paths.InstanceDir(instanceID), + HypervisorType: hypervisor.TypeCloudHypervisor, + CreatedAt: now, + StoppedAt: &now, + PendingStandbyCompression: &PendingStandbyCompression{ + Policy: snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }, + NotBefore: now.Add(time.Minute), + }, + }})) + }, + }, + { + name: "already_compressed_snapshot", + prepare: func(t *testing.T, mgr *manager, instanceID string, now time.Time) { + t.Helper() + snapshotDir := mgr.paths.InstanceSnapshotLatest(instanceID) + require.NoError(t, mgr.ensureDirectories(instanceID)) + require.NoError(t, os.MkdirAll(snapshotDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(snapshotDir, "memory.zst"), []byte("compressed"), 0o644)) + require.NoError(t, mgr.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: instanceID, + Name: instanceID, + DataDir: mgr.paths.InstanceDir(instanceID), + HypervisorType: hypervisor.TypeCloudHypervisor, + CreatedAt: now, + StoppedAt: &now, + PendingStandbyCompression: &PendingStandbyCompression{ + Policy: snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }, + NotBefore: now.Add(time.Minute), + }, + }})) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + mgr := newSnapshotCompressionTestManager(t) + now := time.Date(2026, time.April, 6, 12, 10, 0, 0, time.UTC) + mgr.now = func() time.Time { return now } + instanceID := "recover-stale-" + tt.name + + tt.prepare(t, mgr, instanceID, now) + + require.NoError(t, mgr.recoverPendingStandbyCompressionJobs(context.Background())) + + meta, err := mgr.loadMetadata(instanceID) + require.NoError(t, err) + assert.Nil(t, meta.StoredMetadata.PendingStandbyCompression) + + mgr.compressionMu.Lock() + _, ok := mgr.compressionJobs[mgr.snapshotJobKeyForInstance(instanceID)] + mgr.compressionMu.Unlock() + assert.False(t, ok) + }) + } +} + +type fakeCompressionTimer struct { + ch chan time.Time + mu sync.Mutex + stopped bool + fired bool +} + +func newFakeCompressionTimer() *fakeCompressionTimer { + return &fakeCompressionTimer{ch: make(chan time.Time, 1)} +} + +func (t *fakeCompressionTimer) Chan() <-chan time.Time { + return t.ch +} + +func (t *fakeCompressionTimer) Stop() bool { + t.mu.Lock() + defer t.mu.Unlock() + if t.stopped || t.fired { + return false + } + t.stopped = true + return true +} + +func (t *fakeCompressionTimer) Fire() bool { + t.mu.Lock() + if t.stopped || t.fired { + t.mu.Unlock() + return false + } + t.fired = true + ch := t.ch + t.mu.Unlock() + + ch <- time.Now() + return true +} diff --git a/lib/instances/standby.go b/lib/instances/standby.go index 433422f2..d9de8053 100644 --- a/lib/instances/standby.go +++ b/lib/instances/standby.go @@ -63,6 +63,7 @@ func (m *manager) standbyInstance( // Resolve/validate compression policy early so invalid request/config // fails before any state transition side effects. var compressionPolicy *snapshotstore.SnapshotCompressionConfig + var compressionDelay time.Duration if !skipCompression { policy, err := m.resolveStandbyCompressionPolicy(stored, req.Compression) if err != nil { @@ -72,6 +73,16 @@ func (m *manager) standbyInstance( return nil, err } compressionPolicy = policy + if compressionPolicy != nil { + delay, err := m.resolveStandbyCompressionDelay(stored, req.CompressionDelay) + if err != nil { + if !errors.Is(err, ErrInvalidRequest) { + err = fmt.Errorf("%w: %v", ErrInvalidRequest, err) + } + return nil, err + } + compressionDelay = delay + } } // 3. Get network allocation BEFORE killing VMM (while we can still query it) @@ -203,6 +214,13 @@ func (m *manager) standbyInstance( now := time.Now() stored.StoppedAt = &now stored.HypervisorPID = nil + stored.PendingStandbyCompression = nil + if compressionPolicy != nil { + stored.PendingStandbyCompression = &PendingStandbyCompression{ + Policy: *cloneCompressionConfig(compressionPolicy), + NotBefore: m.nowUTC().Add(compressionDelay), + } + } meta = &metadata{StoredMetadata: *stored} if err := m.saveMetadata(meta); err != nil { @@ -220,10 +238,18 @@ func (m *manager) standbyInstance( finalInst := m.toInstance(ctx, meta) if compressionPolicy != nil { + log.InfoContext(ctx, "enqueueing standby snapshot compression", + "instance_id", id, + "operation", "enqueue_snapshot_compression", + "source", string(snapshotCompressionSourceStandby), + "algorithm", string(compressionPolicy.Algorithm), + "compression_delay", compressionDelay.String(), + ) compressionCtx, compressionSpanEnd := m.startLifecycleStep(ctx, "enqueue_snapshot_compression", attribute.String("instance_id", id), attribute.String("hypervisor", string(stored.HypervisorType)), attribute.String("operation", "enqueue_snapshot_compression"), + attribute.Float64("compression_delay_seconds", compressionDelay.Seconds()), ) m.startCompressionJob(compressionCtx, compressionTarget{ Key: m.snapshotJobKeyForInstance(stored.Id), @@ -232,6 +258,7 @@ func (m *manager) standbyInstance( HypervisorType: stored.HypervisorType, Source: snapshotCompressionSourceStandby, Policy: *compressionPolicy, + Delay: compressionDelay, }) compressionSpanEnd(nil) } diff --git a/lib/instances/types.go b/lib/instances/types.go index 141a6b9d..56243a0d 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -140,6 +140,10 @@ type StoredMetadata struct { // Snapshot policy defaults for this instance. SnapshotPolicy *SnapshotPolicy + // Pending standby compression plan for the latest standby snapshot. + // Persisted so server restarts can recover delayed or interrupted jobs. + PendingStandbyCompression *PendingStandbyCompression + // Automatic standby policy driven by host-observed inbound TCP activity. AutoStandby *autostandby.Policy @@ -272,7 +276,8 @@ type CreateSnapshotRequest struct { // StandbyInstanceRequest is the domain request for putting an instance into standby. type StandbyInstanceRequest struct { - Compression *snapshot.SnapshotCompressionConfig // Optional compression override + Compression *snapshot.SnapshotCompressionConfig // Optional compression override + CompressionDelay *time.Duration // Optional standby-only compression delay override } // RestoreSnapshotRequest is the domain request for restoring a snapshot in-place. @@ -290,7 +295,15 @@ type ForkSnapshotRequest struct { // SnapshotPolicy defines default snapshot behavior for an instance. type SnapshotPolicy struct { - Compression *snapshot.SnapshotCompressionConfig + Compression *snapshot.SnapshotCompressionConfig + StandbyCompressionDelay *time.Duration +} + +// PendingStandbyCompression stores the effective standby compression plan that +// should be recovered after a server restart. +type PendingStandbyCompression struct { + Policy snapshot.SnapshotCompressionConfig + NotBefore time.Time } // AttachVolumeRequest is the domain request for attaching a volume (used for API compatibility) diff --git a/lib/network/bridge_linux.go b/lib/network/bridge_linux.go index 62831b97..06b31b7a 100644 --- a/lib/network/bridge_linux.go +++ b/lib/network/bridge_linux.go @@ -20,6 +20,18 @@ import ( ) const netlinkDumpRetryCount = 3 +const iptablesWaitSeconds = "5" + +func newIPTablesCommand(args ...string) *exec.Cmd { + fullArgs := make([]string, 0, len(args)+2) + fullArgs = append(fullArgs, "-w", iptablesWaitSeconds) + fullArgs = append(fullArgs, args...) + cmd := exec.Command("iptables", fullArgs...) + cmd.SysProcAttr = &syscall.SysProcAttr{ + AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, + } + return cmd +} func listBridgeAddrsWithRetry(link netlink.Link) ([]netlink.Addr, error) { var err error @@ -288,13 +300,10 @@ func (m *manager) setupIPTablesRules(ctx context.Context, subnet, bridgeName str // ensureNATRule ensures the MASQUERADE rule exists with correct uplink func (m *manager) ensureNATRule(subnet, uplink, comment string) (string, error) { // Check if rule exists with correct subnet and uplink - checkCmd := exec.Command("iptables", "-t", "nat", "-C", "POSTROUTING", + checkCmd := newIPTablesCommand("-t", "nat", "-C", "POSTROUTING", "-s", subnet, "-o", uplink, "-m", "comment", "--comment", comment, "-j", "MASQUERADE") - checkCmd.SysProcAttr = &syscall.SysProcAttr{ - AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, - } if checkCmd.Run() == nil { return "existing", nil } @@ -303,13 +312,10 @@ func (m *manager) ensureNATRule(subnet, uplink, comment string) (string, error) m.deleteNATRuleByComment(comment) // Add rule with comment - addCmd := exec.Command("iptables", "-t", "nat", "-A", "POSTROUTING", + addCmd := newIPTablesCommand("-t", "nat", "-A", "POSTROUTING", "-s", subnet, "-o", uplink, "-m", "comment", "--comment", comment, "-j", "MASQUERADE") - addCmd.SysProcAttr = &syscall.SysProcAttr{ - AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, - } if err := addCmd.Run(); err != nil { return "", fmt.Errorf("add masquerade rule: %w", err) } @@ -327,10 +333,7 @@ func (m *manager) ruleComment(base string) string { // deleteNATRuleByComment deletes any NAT POSTROUTING rule containing our comment func (m *manager) deleteNATRuleByComment(comment string) { // List NAT POSTROUTING rules - cmd := exec.Command("iptables", "-t", "nat", "-L", "POSTROUTING", "--line-numbers", "-n") - cmd.SysProcAttr = &syscall.SysProcAttr{ - AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, - } + cmd := newIPTablesCommand("-t", "nat", "-L", "POSTROUTING", "--line-numbers", "-n") output, err := cmd.Output() if err != nil { return @@ -350,10 +353,7 @@ func (m *manager) deleteNATRuleByComment(comment string) { // Delete in reverse order for i := len(ruleNums) - 1; i >= 0; i-- { - delCmd := exec.Command("iptables", "-t", "nat", "-D", "POSTROUTING", ruleNums[i]) - delCmd.SysProcAttr = &syscall.SysProcAttr{ - AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, - } + delCmd := newIPTablesCommand("-t", "nat", "-D", "POSTROUTING", ruleNums[i]) delCmd.Run() // ignore error } } @@ -369,14 +369,11 @@ func (m *manager) ensureForwardRule(inIface, outIface, ctstate, comment string, m.deleteForwardRuleByComment(comment) // Insert at specified position with comment - addCmd := exec.Command("iptables", "-I", "FORWARD", fmt.Sprintf("%d", position), + addCmd := newIPTablesCommand("-I", "FORWARD", fmt.Sprintf("%d", position), "-i", inIface, "-o", outIface, "-m", "conntrack", "--ctstate", ctstate, "-m", "comment", "--comment", comment, "-j", "ACCEPT") - addCmd.SysProcAttr = &syscall.SysProcAttr{ - AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, - } if err := addCmd.Run(); err != nil { return "", fmt.Errorf("insert forward rule: %w", err) } @@ -386,10 +383,7 @@ func (m *manager) ensureForwardRule(inIface, outIface, ctstate, comment string, // isForwardRuleCorrect checks if our rule exists at the expected position with correct interfaces func (m *manager) isForwardRuleCorrect(inIface, outIface, comment string, position int) bool { // List FORWARD chain with line numbers - cmd := exec.Command("iptables", "-L", "FORWARD", "--line-numbers", "-n", "-v") - cmd.SysProcAttr = &syscall.SysProcAttr{ - AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, - } + cmd := newIPTablesCommand("-L", "FORWARD", "--line-numbers", "-n", "-v") output, err := cmd.Output() if err != nil { return false @@ -417,10 +411,7 @@ func (m *manager) isForwardRuleCorrect(inIface, outIface, comment string, positi // deleteForwardRuleByComment deletes any FORWARD rule containing our comment func (m *manager) deleteForwardRuleByComment(comment string) { // List FORWARD rules - cmd := exec.Command("iptables", "-L", "FORWARD", "--line-numbers", "-n") - cmd.SysProcAttr = &syscall.SysProcAttr{ - AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, - } + cmd := newIPTablesCommand("-L", "FORWARD", "--line-numbers", "-n") output, err := cmd.Output() if err != nil { return @@ -440,10 +431,7 @@ func (m *manager) deleteForwardRuleByComment(comment string) { // Delete in reverse order for i := len(ruleNums) - 1; i >= 0; i-- { - delCmd := exec.Command("iptables", "-D", "FORWARD", ruleNums[i]) - delCmd.SysProcAttr = &syscall.SysProcAttr{ - AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, - } + delCmd := newIPTablesCommand("-D", "FORWARD", ruleNums[i]) delCmd.Run() // ignore error } } @@ -460,19 +448,13 @@ func (m *manager) ensureDockerForwardJump(ctx context.Context) { log := logger.FromContext(ctx) // Check if DOCKER-FORWARD chain exists (Docker is installed and configured) - checkChain := exec.Command("iptables", "-L", "DOCKER-FORWARD", "-n") - checkChain.SysProcAttr = &syscall.SysProcAttr{ - AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, - } + checkChain := newIPTablesCommand("-L", "DOCKER-FORWARD", "-n") if checkChain.Run() != nil { return // Chain doesn't exist — Docker not installed or not configured } // Check if jump already exists in FORWARD - checkJump := exec.Command("iptables", "-C", "FORWARD", "-j", "DOCKER-FORWARD") - checkJump.SysProcAttr = &syscall.SysProcAttr{ - AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, - } + checkJump := newIPTablesCommand("-C", "FORWARD", "-j", "DOCKER-FORWARD") if checkJump.Run() == nil { return // Jump already present } @@ -481,10 +463,7 @@ func (m *manager) ensureDockerForwardJump(ctx context.Context) { // Insert right after hypeman's last rule so the jump is evaluated before any // explicit DROP/REJECT rules that an external firewall tool may have added. insertPos := m.lastHypemanForwardRulePosition() + 1 - addJump := exec.Command("iptables", "-I", "FORWARD", fmt.Sprintf("%d", insertPos), "-j", "DOCKER-FORWARD") - addJump.SysProcAttr = &syscall.SysProcAttr{ - AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, - } + addJump := newIPTablesCommand("-I", "FORWARD", fmt.Sprintf("%d", insertPos), "-j", "DOCKER-FORWARD") if err := addJump.Run(); err != nil { log.WarnContext(ctx, "failed to restore Docker FORWARD chain jump", "error", err) return @@ -496,10 +475,7 @@ func (m *manager) ensureDockerForwardJump(ctx context.Context) { // lastHypemanForwardRulePosition returns the line number of the last hypeman-managed // rule in the FORWARD chain, or 0 if none are found. func (m *manager) lastHypemanForwardRulePosition() int { - cmd := exec.Command("iptables", "-L", "FORWARD", "--line-numbers", "-n", "-v") - cmd.SysProcAttr = &syscall.SysProcAttr{ - AmbientCaps: []uintptr{unix.CAP_NET_ADMIN}, - } + cmd := newIPTablesCommand("-L", "FORWARD", "--line-numbers", "-n", "-v") output, err := cmd.Output() if err != nil { return 0 diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 8ceb515c..093dd8f5 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -1263,6 +1263,9 @@ type SnapshotKind string // SnapshotPolicy defines model for SnapshotPolicy. type SnapshotPolicy struct { Compression *SnapshotCompressionConfig `json:"compression,omitempty"` + + // StandbyCompressionDelay Delay before standby snapshot compression begins, expressed as a Go duration like "30s" or "5m". Applies only to standby compression and defaults to immediate start when omitted. + StandbyCompressionDelay *string `json:"standby_compression_delay,omitempty"` } // SnapshotSchedule defines model for SnapshotSchedule. @@ -1316,6 +1319,9 @@ type SnapshotTargetState string // StandbyInstanceRequest defines model for StandbyInstanceRequest. type StandbyInstanceRequest struct { Compression *SnapshotCompressionConfig `json:"compression,omitempty"` + + // CompressionDelay Delay before standby snapshot compression begins, expressed as a Go duration like "30s" or "5m". Overrides the instance default for this standby operation only. + CompressionDelay *string `json:"compression_delay,omitempty"` } // Tags User-defined key-value tags. @@ -15648,272 +15654,273 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+y9/XIbOZI4+CqIutkYaYakqA/LtjY6fqeWbLe2rbbOsj232/RRYBVIolUFVAMoSrTD", - "/+4DzCPOk1wgAdQniizJlmyNvTsxI7PwmchMZCby42MQ8iTljDAlg4OPgQznJMHw56FSOJy/43GWkNfk", - "z4xIpX9OBU+JUJRAo4RnTI1TrOb6XxGRoaCpopwFB8EZVnN0NSeCoAWMguScZ3GEJgRBPxIFvYBc4ySN", - "SXAQbCVMbUVY4aAXqGWqf5JKUDYLPvUCQXDEWbw000xxFqvgYIpjSXq1aU/10AhLpLv0oU8+3oTzmGAW", - "fIIR/8yoIFFw8Ht5G+/zxnzyBwmVnvwwU/xcYRZNlmc8puGyudmXlGXXMBvCmeIJVjRE0vRBKXRCEyxJ", - "hDhDOFR0QRBlE56xCL05OkMhZ4yEejA5YnwiiViQCE0FT5CaEzTnUkEbJXB4iRSexGQwYkGvdh6E6S/R", - "eij9Y07UnAjPYqlEdhQ05QKpOZWIMv01JIPygSmRkSZkewGNYjJWNCE8U01A/cKvUMzZDLblxkVJJhWa", - "4wVBH4jg6M8Mx3S6pGzWDqQJmXJB0C/LlCSYoTTGIZGIKkSZ4m43BkYFjj1KfMhFZ4wLMo6IVJRhPf44", - "5cJQRHX1r+APHKNSW1gatEdqjpXDcsYVuiQkrW4UX+HLKhh/39npPR0Oh+97AVUkMWSFr2mSJcHB/qNH", - "u496QUKZ+fd2vnrKFJkRoZdvf8FC4GVpO5JnIiTjkEZi1U7CmBKm0NHJ8etbbiDYHg7g/7eeBL1g++nO", - "YHv/Cfx7ez8ob6sB+OrKP60mvXOFVSabPMhQ09giyriEJM1d/5YlEyIQn6IwE4IwFS8RkBSJOiBdZdtD", - "31GEnE3pLBOOBH0kVwHnHEuEmWEa/Rq/KAbrRHehZmIRv2JjQRJMmYZxYxGv3SekKRRZItJLCjlTgsex", - "ZgpKkSRV0lFRT7NxhnCaxjQE1lMhqr1kKINewLI41h9rKyxOm8R0RqFBJ9BQWTok1xcpjghTROQU3gU0", - "FbbYNnEBbu9pFHyxOxeUlIX+7bI6zBPN4QUJzXbzG6ACkQkJeUKQHrp6AjvDnf3+cK8/3H+z/fhguHcw", - "fPQ/QS+YcpFgFRwEEVakrw+8yzGt5t9HBZR0Q2QbFleVB3aDGg/uhi4xliqnaiByqpZj7FnTG5oQqXCS", - "asLWaygBs42s3YD1c3CQXwng7c8CMCPXamwh5N2PDz/IdUpCfcVwR575ja3H6yE6RRjlPECjq2GMKzfy", - "9LM2IgiWesFa7tC30+9BxmSW6ruQROM0xkqPq4UUQINxQqXUXfMfIioNYfYCh+RjxtVYZIyZhoyoKy4u", - "yy3tKGOaBr1gjuV4MUuzoLfqHqgiNUxBYpxKGM+euBgTIbgIjKy5HE+5cIekL7EChCuGakBI5neWB0JB", - "L6gAIOePbi9u3fmpehcHswAuCSOmG7kaNtNceHms5nLzpa3mlIYtG6nUHTOynWWVA0QUzxiXioayE9+E", - "21gfb8IjD+s8zodDNCJM0SklwgqqBImMwbXmBkF6EEQZymSNDnJZekwWWvkZL/bGKkybQKlpCuXDK132", - "xRVTuuby488pZQ2SVvfu1UQWmAJNHpMFNVdLVRiyRzOOBF0Q4WHf+Y1qWKFphzY0rWsWwjgjmxVIsQWN", - "KO7CDiJY05h6sOfs6ASZz+jkGG3MyXV1kp3HkydB+5AMJx5c+CVLMOtrgtDLcuND2/LYL/e8Mj9Pkmw8", - "EzxLmyOfvDo9fYvgI2IgMpZHfLLjE/3SkI5xFAkipX//7mN5bcPhcHiAdw6Gw8HQt8oFYREXrSA1n/0g", - "3R5GZMWQnUBqx2+A9Ld3J8cnh+iIi5QLUILWEk4ZPOV9ldGmeio+/P85o3HUxPqJ/pmIcX6J+AB24sSo", - "k2MnJ9h+6N0p2tA8JCKTbDajbLbZBd9DrsGhrzrfJQ5LRbaNVhOVk1Jufd+GguA10+kWnSZrklpmTnKc", - "yLbRXRPNURMax1SSkLNIluegTO3vtW+mRDDmhmpM9Uz/jBIiJZ4RtAEmFVA/DDPVgs0U05hEm92E2bbN", - "/MEnpSukgt6AFn08Cbd3dr28I8EzMo7ozNrE6leU/l2jmB5HIWjt3whc5t32AVMKMm3O9xxYN0wiyJQI", - "onH8M6dLBV8Qhq328heYN/i/tgpj4Za1FG4BMM+K5p96wZ8Zycg45ZKaFTY4l/2i0QhAjaCHf83wadVZ", - "lzBKKixW0we0+AKUWMh1a2FjzRZatMGztV3e6DZ13gmsMZclSlyglUU+00KNRzrgTNkPNfMln6GYMqNx", - "aNHOnAXIVcuU/BRzYIlfCA45+JvEr9d9C+ZlfmgZTX/r5QJ4zGdlaM4JFmpCKsBsucLsQMXqWsF/ViGf", - "2l2FJRmv5iBnlDESgb3YErZpqcVYr5oBVHRJ1XhBhPTSHCzrV6qQbdE6VMzDyymNyXiO5dwa2KKIGmPh", - "WWUnHmmtYojHoI+7AUGKAP31/JfDnUf7yE7ggaG1XOoGzZ2UeuvhTVuksJjgOPbiRju63fyObmKIHwMK", - "Y2Xb3ZNjoENMw+kCe5pWT87k3PwFvFuvCu4+zQY0esX67/eeTR8BkzBaQuvrjV8GzC3Ds5hrmC5Rxuif", - "WUXAHqCTKRiI9UVBIxL1EIYPYHfQ+t+MMCI0nyosQyUhGG2QwWzQQyMtF/a1FNzHO/3hsD8cBVUxNt7r", - "G/U+xUoRoRf4//2O+x8O+/8z7D99X/w5HvTf//0vPgToKpk7qdDuc8PRfg+5xZbF9fpC14nyt+b+5eX7", - "OI456hPNJ2560kcnTcHB7DXi4SURA8q3YjoRWCy32Iyy64MYKyJVdeer235RWMA+VgCBzTSYbgiGmtID", - "aLwR8ysiQs2BY6IRT/Y0E6ZK9hDWejMwL6Rvyf9EIWaaFoxwwQUiLEJXVM0RhnZVaCXLPk5pn5qlBr0g", - "wdcvCZupeXCwv9vAc43kG/aP/vu/uZ82/48X1UUWEw+Sv+aZomyG4HP5Wc+tIX+iWXUiDrpZDGJeQtmJ", - "6bbdfIP6vBN2G1l10kaZaz1qzYRyE9mahTTfd7WylXhUh1cLIgSN3LV8dHqMNmJ6SSy9IJExNMqGw90Q", - "GsCfxP4S8iTBLDK/bQ7Qq4QqfR1mxS1vnmxrr2sknHMQVOKY3+Q5DSRFUHBwvPIeXwUaL7SP8nGbt/4v", - "XKp+ghmeEVBHbUM0EfyS6IWaNwFKJLokSy3lLNFMD9pfUAkvPIQt0AIbq8NgxN7MuSSmifskwbZPFwQl", - "PLw0T79zDpr8AscZkT10NdciB9gECY7tz8g8jI3YXC9ShjwlkVZCTDPYGrogbHGBEpwCmWNBgMZRghUR", - "FMf0g3nCh1cGElF9w40YAcJAKdY0H4ZcRPDCxhHB4bwEhb9KdGEElgsY/oIyjdYXhjBrj9Ufg1dv3/z8", - "6u1vx+NXZ89+OzwZ//rsv/XPplNw8PvHwLhq5JLKzwQLItBfPsJ+PxnxNiIiOAgOMzXngn4w1ppPvUDD", - "QGr8wikd8JQwTAchT4Je8LfyP99/eu8EMmPGXmgy8Czsk1cYMnephyUdO2ugRNbC5N42NMg0i3px9nZL", - "384pllLNBc9m8yphWNHgRiQRUXk5pnw8SX1rovISnWy9QlpwQTHVBJoLKtvD4enPW3IU6H88cv/YHKBj", - "Q7WwfM2DuLDyk5xr9Mm9Po7O3iIcxzy0NpRp2wOvm8rH4AlTYply6lPiasypaNrkUf1+8fUGrGhrQtmW", - "1MfQD28Gd8CbW6sSz9iCCs4Src4tsKD6npZVWvnt1fGz8bPf3gUH+iKIstBaJc9evX4THAS7w+Ew8CGo", - "xqA1PPDF2Vvz6mnIRqVxNhtL+sEjShzm+0MJSbgwKrTtgzbmVUnD0C2CwxkFuy9+Nsi1/QLwyh2KfSPK", - "RzED1571Xvzsw5b5MiViQaXPzvZL/s2dfNPdp4Lb5pUsR1rA4kFJfwljnkX90pS9YEoFCcG9Qv/rT5Jo", - "QX7xofos5ennN391EmDXSKY4TikjK0TTb0REvOLiMuY46m9/YQnRPqh6XGPMh+r55i9rDiUaHmcTzKIr", - "Gqn5OOJXTC/Zw1ftF5Q3zpnrtd4Jjv/1v/98d1roWdsvJqnltNs7jz6T09Z4qx7aa0PJN5Kl/m28Tf2b", - "eHf6r//9p9vJ192EEURuJdTZ839mRqg7zVhfQmMObXkZzm/v3GFFcatQQ3fkcG/tM7CPUfMFETFelhiv", - "XVOwPQTuV1uVoOAliWw/zUYvke68hg3r0dwl/6Ku5O8M/YzWsyjPmn7WvMLeC11Wki9ke+fU/rnTXFLL", - "ii5pOgapeYxnuc13lUvo+SVNrSgOPcwxxrFhBFEGwvuEczUYMeOhos8ODphckxB4nlRYocOzE4muaByD", - "hQiYSvNq0YJ9ybUJmkul/1tkrIcmmdLSOlcEWb0JJslgLdB4QlDGsHsPr8nOdoNN9wIAyyURjMRjIxvL", - "jpAxnZDt1Aoc2OoUS+uiJlSWVuF1/OvpOdo4XjKc0BD9akY95VEWE3RuvAs2q9DrjVgqwE1BT6Lpmdp5", - "+RTxTPX5tK8EIW6JCQyW29jsY+3ixdlb+9wvNwcj9ppowBIWWUdfd+NYJ9CIs79qiiVRddjy/DWgt7l0", - "SIZTOedqnObO06u407ltXqji3Y0JvWARpln1SHd6rU6gCypUhmPNayvipPeB3zixe9QG4yNfVl8s3yuc", - "ZlX1ZbarxcWMDB7tXndZj+HESEqdDSclVb5hQnF65sdui10z/glzC1lpOCpUzc+Y69wM0nDeMT/33M5u", - "AaWTHCY1c9OXAc+hLKnmnZzPjQ+WkQgl2rjQ2rzFY62/X/TQxd8qP2jad6qFli+ukIEG8BOmfyqPXzdK", - "rDUX3Mjdu3w4WN7+PA5lq6cTWmwjJTCTxkdtjlMyQL8AE0eKJKnmZGyGqES5axdi/Oo/ETdCjes6Ynpp", - "0viJWHDkRiNJZ4yy2aYW8/XFhKPIWJammcqEbregsoBmFXWc9abh1WpWRww/hggJysI4iwi6cBaei6pc", - "2LT/NFVCaxBqaDgGJKDZgLKntpJM6en1hhOswrmGE8+UcRyzW6869dWsTOseVO1a8qe2W5z/ec4u6oEw", - "C4+KozdnH3nALFiyT7aZAa2g4jdRXpIlHLkzR+KGQbJsifTbCwWRPF4Qe+2WbZkTCPXhRnAqzJjGIGlt", - "kJr860EuPuvcuqPQ8OoM/qqq4AnxkarvNltgjJX+nU+440J6c2a+nlaMJQHgg+pxgEAcu+gZXYmABQIx", - "jSwxiqggoWoMT9lsxMCH5ML+MrCjXWgi1zLKFwmcgjgEENrLR4tKJ+vEPhhGb40nVCkS9aqywSUhqVy/", - "KS1eW8O1x7ouyJWgjpE5p+KO4hlhUy5Cklgl4fMUx2elwbxq3M2GaLp0GPiW1uziMyA6hUTGf8icB5hZ", - "K2Eb9ejFqKa1GReC6pQXOI4v0IZttIkE+QM88e1ZMc4KZH9zdOZQIH/2fnfa0xipucDFXKl0rP9LjjUV", - "X9QHs30dhReRZU+GoF/t7e3aU7VGN7Pg2rBV+5rXLaL9aJz43fqypvFCr9L6mXQR5Y+KLoUl9ZKyqOsA", - "v+q2rda5XDBymsZdG+hSQfpZOhMYXGy/pHnu1u+mAM12Dr4mjtfnJllECGZS8aTsb79Rc/GgVWeQKrAW", - "PO5HWGEwZXa0t5rlNh2Pk6UZyuhibZaY8Wzi8RuiHyAUYEZneLJU1feDbW803+c+Yru1+I6lzYHfaJAk", - "Giu+2oWZTpFr28Vj0cQbKD5eTClfHd5h/V8q8XfmOrJ6rR6in4bUmhNAxgnnxsPUAAGExnen5be7wYj1", - "4fo9QMf5BPmw+ZAYZEscmZeTDS5KizCBHGiy3EQYvTsdoDf5av8qkVZYFsRFNMyxRBNCGMrA9Ay3Yd/c", - "xeUFZBIuTVXvbm0nJvhhE54ouf02yGOOwUqTR1CDq9SE1vZjIifhoOybMGZlK1gnq9Uqx+/XZEalEjW3", - "b7Tx+vnR7u7u07r9cudRf7jd3370Znt4MNT/+Z/uHuJfPr7DN9ZhlbdY57My9zl6e3K8Y42l1XnUhz38", - "9Mn1NVZP9+mVfPohmYjZH7v4XiJA/KzsuPCaQxuZJKLv2KTGKp+vXMklrcUX7tYubnfksVY44K5qayDx", - "Rre8i9AWn9O0ddm9efBJnWGudbsuba6pyS9T0DsLKilJcNa7MaReP85jKi9/FgRfQshe895O8IzIsbnP", - "/P4MmTRONuTaWjcE52oqzbtp1eq5vfd478nu/t6T4dAT0dFEeB7ScahvoE4LeHV0gmK8JAJBH7QBD14R", - "msR8UkX0R7v7Tx4Pn27vdF2HeeLpBodc8XK90IaFyN9dnhL3pbKonZ3H+7u7u8P9/Z29Tquy9uJOi3K2", - "5YpI8nj38d72k529TlDwCfTPXIRNXYD3RVYemuh+/a++TElIpzREEKODdAe0kcAVRvLXqipNTnDk4k/9", - "d4fCNJYrPSbMZLalMbQlWaxoGhPzDQ6kky0adn4MI3kzZDCWx/vebCQbl7TWQ8DtJW+CKvFlFdCdmoDm", - "kvBESRwdGApdy+fgNIuFvW/DA7uHjtjwUqtO/ZgsSFxGAnN1mchaQVCOJ+bQKruibIFjGo0pSzMvSrSC", - "8nkmQBY1gyI84Zkyz4w2QLuYBLyeQfeYanbdTc99zsXlWv9RfRPncehrrUKHYEifWlMN3OIY2d4uRKEk", - "9OXPgebR1H6X6LXpYSxExc9pVs1q04OZrCWJIUGk4sBJrcHQDtNVuvTLLWAsde4fZr6Cd96T70t/atwF", - "vqyGLWYE8i+otRKLxpQ30P4cmnd2R9cd1xpSOsCdkav7ADr46/c12vYlw+ndQHyVM1puaygawS0saEQG", - "CKgLvGJcfGCN0s4VT1MS5fafwYhZf+78J2leUHRHAwc1J1QgLuiMVieuGtju0qvtJqjosOnW6Fju2JRQ", - "4SO4b7QTPZ4qk2vh0oVMkXL8kj2EoBec55kpLCeqguZ1nt2jAZHC1bKxxBdnb2/qm5YKPqW+fEPgC2G/", - "Ws3MeW293Bue97f/H+OBqfENRDTKjP9EwqNaIgnbvtvN8+Ls7VnbmvLUDqi8usaeco+XVcmtHETso5J9", - "lbQajEN/fbHkkxSy91OfLDsVOCGTbDolYpx4jGvP9XdkGhjXJsrQ6c9VeVbLzV215rPK4YDaPMWhjczv", - "Bn2PQa62jV4Jmu/9x/WamGu4LZ5PH5WwbWxI3wD9lifTQC/O3kpUeCl5LHXV4231lz+bLyUNcWxGNOG5", - "lJUNbICcnSXks6KjNUV65GR/DhZHCGhjMUszIMPz1/2TV++2kogsepU1gWfRnMdEr3uzxC0WLqqvcO6v", - "MIlFm6XDIIbsSkAlWOUU3BlIJXr1QEdxheOxjLnPWeON/ojgI9p499xEXekV9FBaOUr9ewkKFfze91KM", - "5kht057DhHWTaYXAvbpjNRumMa+UtleZ1EcqvxAcmySgVXxuJkDil9WD5pfrk+6YQXzznjjH8JpS4wve", - "Ojo9NgJDyJnClBGBEqKwTTlacnEBcSjoBX19R0WYJOBqN/3P1d4tLSb4cjRWqxH3qJG3404MuC3x5q+N", - "C0KEEszolEhl480rM8s53nm0f2CyYkRkuvdofzAY3DRG5VkRlNLpKLaMC38pXGUg5593DncQitJlLx+D", - "s8M3vwQHwVYmxVbMQxxvyQllB6V/5/8sPsAf5p8TyrwhLJ0SqdBpI4FK9UlT31nm94NSzkuX369TXju/", - "PgOeDRA35403Vnim9RODcZ8bWHzr1CNF/itVSjlSdgjtkH6EflhtCXWCEbSxc2ZM0bjIzNK0gd4qt45c", - "mX6gkXogJSxPOBDH5q+Qs4WmCl/2gQoDd98+6/3AermMI+rB5H9Ybc84SUBU1Xp6C7Zwmq5HW7+gmPO/", - "rllXbGy05yb66lz/Nm9s1dlfzf7rz/9Xnj3+Y/vPl+/e/ffixX8d/0b/+1189uqzIqhWh8V/1dj2LxbO", - "Dg9LlZj2rqh0ilXoEajmXKoWCNsvSHHjrzlAR6D4HYxYH72kiggcH6BRUHMRHgVog1zjUJleiDOkh7KR", - "Dpu685kx/+jOH51u+ak+RmRDGoQ9kDySSWaTiCeYss0RGzE7FnIbkfCmr/+KUIhTlQmiT0/LsPESTQSk", - "9bbqeTF5D33Eafppc8RAwyXXSugdpFioPI+HmwGQwq7K+AzY5iRygeFGQx6x/F7K48KNjWaQG0HANl/3", - "uPQDxau+cFENxXky9EXQg9eXPsiYSkXAMTvHbI1GuTsaejKssIonwyfDtQJ+jkMr0A8ooZnv3yFlB1oy", - "CAxTG8YNHmodbOmaNxkaQb+8eXOmwaD/9xy5gQpY5EdslDzjAyiNjVDFsuT9txl4s43C6XbckDGSQbe4", - "Q9TQM+Me+ublOVJEJM5hfyPU4JzSUO8Pnv+plJlGRYrR4dHps81Bh4IFANt8/SvO8U2+w3pwhzWatdkC", - "c4zX8O2hk2Nwz7UUWghw4FbznAsUGwZT0PUBeitJ1dcVjsq86puTjJeF5c3cAKNg042Y1jnFAXqdy404", - "X0qlSELVmFfQJQxrH16Mz09j9F4j/bhwepFlbeDhg1XuJK5v3HZWsJr8PRAHmrd+3SWb5s1ou2wM1ZP5", - "UaM4+y+dNeXLizu7N1Vyb5rhoRqEWQrgzZM8dM/OcBdZDpoK3zVV49ZXfKQ/2zd7p9a8O0VzLNlfFXys", - "KTfbu4875evUs3Z9/y6/fPOpWVJOli6iM3+3NbGtlzSOjTuEpDOGY/QUbZyfvPj15OXLTdRHr16d1o9i", - "VQ/f+XRI9uBo48XZWwiXwXLsnpDavSZx4XlMrqlUshnw2ukldnVyiV8qCSC8EcSbXzArhHu+bmzjPvI9", - "fE2/wG8v18TK7BCfm+LBSst3lOGhlbn6siNU+az5+cvmariT5awtL1IWKpzT9q2TI/QC6nFYPZSaBZII", - "nZwVSRYLq5YbvrYnW6tnezgcbA+72PgSHK6Y+/TwqPvkwx1jyTjAk4MwOiDTz7AxWsQ20h+Or/BSopGT", - "z0eBUQhKmkCJbK0M3+n9tpmD4nYpJ+oCxbqkEjdJItEtO8TnhuSvSrV8Xk2y3FnI+4xKJJ1cKNzVbp0n", - "bK/xTcznBIU8iyMtSE006RrFjkRW/5REFfmrgdrfskvGr1h168aKqhnAnxkRS/Tu9LRicxdkatPzdtg4", - "OF20nANPb3QMO2tk7bWruWWihvtIzlBnu6Xr7ounYigb/ZwTp8HQDsa/Qvz0PrxTZo5G48mKPdXMNhFZ", - "jLPMJ1XpTy504+3bk+MKcmC8v/1k+ORp/8lke7+/Fw23+3h7d7+/8wgPp7vh492WBPndHW9u70tTpeb2", - "UCkAPJhATSRcdKDpLXeGmWQK5Y5ympCPtHiKSnKwCQwCq8QJowqSQFI208OAkcCKySbC0+SppIwqSCkA", - "CW0o01sGa4wexLo/HaAX0BY+4QQCltwitHJUNUTgaGkMsZoxuKlT+NfqJZ/PMyj3A33kPFMIykPpbWsw", - "WHVl9RCGxxyg3zj0Ec5LlfG63mOag02g2byuI21YvyTnvwqTWYZ5gJ7nTDJns5atbkhi/zS827pWg9v4", - "ZsV5z554oLGlOLmSX1ovMBANeoEDFPivNT3Z7Lq8QRplVPS9UBAcAwstPIUyRWObJQF2QqFAEmwEw+G2", - "UbLNCEaisREB2t4bjfuJFRPyTo5RvDtFGxAP+XdklUr9r838bbJMlXs7T/ee7j/eebrfKeqhWOB6Bn8E", - "zlHNxa3l9mGajV3tkZatH529NSUOQ85klhgrgd17yck0FTzU0iplqChmUkz+dPC0HOwR8cwUdrJLspFh", - "n0rly1ZWnml5YPuTxgs6nbI/P4SXO38Immxf78udiVe5K+qkeSXhk7KptaE2kknfZHH0++MDQgnZGrLy", - "mkjYATonCgH+9BEO4ZLOfZosyrnAFgtxL2Lt7e7uPnn8aKcTXtnVlQhnDPprc5WndgUlEoOWaOP1+Tna", - "KiGcGdM5ekKCCWYFOD+dIZvQeVitBDrYHu76sKRFXiqwxo69SFpB/s4KQXZTFujgmpULSA0q90J7d3f4", - "eO/Rk0fdyNjV3RPXqzmMS+phwGPzoJRPfgPM828Oz5AeXUxxWNVQtnd29x7tP35yo1WpG60KcviY3Bs3", - "WNiTx/uP9nZ3trvFXvlM8DaqsEKwVd7lIToPUnhOwwOKJuvttd0WPsHTINhrEsaYJoehc5+p3T4mx8ZY", - "mGbFIXS5GKyRoHFxdejbSUWrFQ4yogEXqFRxcbDeHHo762Y7mzb3wXo23pShY8w0uGyQgEnleAvYpYIs", - "KM/kFxiIKxJqZJrGnIsb9W3zR3pNZBYrY4KkEr07/SswEY1cSCqSVn3tLfqtCKW45eZuRMAVnPBjdRuw", - "Op1Gl6NfteFeC5n2VvnRVsi/NWIp0qwqY+vfvo9wHGaQvAzn56l3BbEHPFPwUr80XiJxzDlD4RyzGYFk", - "8CZVIpshjOY8jgaB/6kkjsZT7xNGXmGeF/XL3SJ0N1d/f+MFL0raGVSq5ed9lBiuYjM3DTrUki9q4rZE", - "OGl4YsVLaQBMl4o2H/OZBC1Qgf/LoJ59JsXCuLVgZvLULRKjPFZDt3b0be9ZYo17+65Qc3XyqdVorYyh", - "eA5JHAouZVGY+91pdZmrHBjzevbr37Ori+2AujLlTBJ/mXhbE76Twcd3IXo8wz7nSgQcBgfQVTWgbR7D", - "BLMMMn2VEJlcp1QY9Oj2OD7nUo3zcJQbLlaqMWRxygQpYtbcfTmHAIClYXHQxnsvOtZ2G3Dl5Y1v0buB", - "Vf6h2hbYzlO9EPVDq5fjoA+NmwE5K2OAiqCiegTJTULGirQ/VMKotBSthDYYVxW2VEpds9nlocqvo+p5", - "2krKvtwbnneN5lodvHWG1fyETbknN+QNDP7WJd75LqREQPlxzlBEGCWRUx5zy7+1bYGTfSwJijJiIWcE", - "UoEtwLEhb8gByZxRjLJZjdfXJ+xihjdrWJ3kCea1Dbs8OUq/a/YbkQGsjJOARLhw0u7k8UDl2G8pbg4s", - "yCyLsUD1iMUVS5bLJKbsssvocplMeExDpDvUn3OmPI751Vh/kj/BXjY77U53GBc+hrXnGbM462FqDqQ2", - "b7GFn/QuN2v+7WB62TL9t3T/Ti+4Xr+h5zQmNqjvLaPXJUSvZkHZ2xm2hT60DFoJemgGhN6Uc1uU9VG8", - "i9U8zKsmeNzxjQdQ7VWiaois7Ne3W3AxWxXo0TTFoA33KOyyzFThWsr20skS0s3Lre7+4FazJUlYnX3v", - "yaPH+x3T7XyWrXNFVeXPsGwukhUWzZaTOu1iNnvy6MnTp7t7j57u3MhA5TxlWs6nzVumfD614ig1o9mj", - "IfzfjRZlfGX8S2rxl6kuqFLo5NYL+rSCdIsw65Znj9Y833H5JN07S9UC2s3GuEJaOqyIXKVaXhtkOiWg", - "VI4N3PrFYmru+Z3WEOIUh1QtPQYTfGVSvudNauHCXaxp1cV6QGrHthkfNOeS2aRw6Nxwk6O/GdN6DRee", - "dM7aJbNJmxn/VX1WY8QvbEDlJ6IOLzRFYYGmuSDfzxWWFa8O/XcIOZuLWm11/yHTontVaofreWHqwjPS", - "F/LuL0JdPv7acZbMvhUhuQ7xVVdoOwneSIf23Mi+MpXrvXJr/MFegLfrNZ6U8+mtTFhYSb5X3Lo3n7db", - "lblmP3OD3Xy+kgvoTTrWU4sBPto1WJAXY/cqKNGCTYqL9Rml7yBBkPEpuFWKIOuOcC9ZguzPd5IZqHEc", - "50S5tudao8/iFQmhmSJigT2GKTcEck2qdlTDiXvImvjQdrJZK1W4N/fLajaEt6P7mJYCx6kgU3q9AltM", - "A3NdV93HpYVAVE0aLtFGgq/R3mMUzrGQtbUzOpureFk1su55oic+r4gzUVp07p5evThN17H5omGPszy6", - "j2TPS7EO/rTvJBqvCnQ/yps5m3GKlyBbtiqCj3f3hsPdneGtIt2/VDb60jhtHqGlftaYU3l6LI+Q+382", - "UxZeCWqKmjkwSSUITg7AmyrFIUExmUIcWJ4qdq1O35h69eLtI6l1/M/x3x2Uq1xq7SyWxTHOQPBw49gk", - "AW4bgXulrcZ6lL83l70iWCxnM2Ejaqzuv7rfH+72h/tvtncPHu0fbG/fRWh8DqQ2F57HH7avHsc7eLoX", - "P1k+/nN7/ni2k+x6wwXuoPBBrY5grQ6C3UNKRD0XZT2HqyQxZaQvc7e39Q7IK3iBeUlaS/83sz6YHawU", - "Fs6rmyzLDFgVwKmXZLuPwCa7+pUmlPryT45XL/tWfmT1hfgRrL4UwKdui4FULdufmxckYx3vnbelhp1v", - "npW+jevuHp/TN5C295RbIO7D5wpjrFDYqhu7eat5VLgZF1TNk9XXQ94szzIAj+EfpIqqgTQDdDJjkHm2", - "/HP+9lEuDq07B70g/rBXpRn7e/eQKhtVnyOgPeqyGNDhbQASG6+GAjQpVAth3BOwIACIn7b720/hhT7+", - "sPfTsP90gP5R8hToGWiVwbftWld+HXaBYc4oQe40L+fbT2/0jO7guQqDfrX3UttFbOPtLY4XaT/dXeHc", - "pisHXHxunHEtquiOCg19WrFjJzjfLG2P6+URTQZ+2eTpm+H2jdP23OyKGHyGR/FnqXrd1LsYS9UmV7/E", - "UuX6GBKZla57UOKG1Srd27TiNhQAXDsh1v4AyZqG7/Kpy0LocVaAHppxhYoggLVSDixfZMyLD9X1F2WK", - "IT1EK0LsrEGIbmvKA/noKtI9OUap4FEWFh6wMSw6C0Mi5TSDqsuDrhLt+jfGu1TmwbVca/Trlfk27X19", - "mCm5bj/v38i1Kk2pEbb9qLeH64/6TiwAvSBLo/U8zDTqxsFulIljjU+lxx5RBXtNCipt5n0Hjv66DMGm", - "ggcFm1CoxYEsdRUENU41MUl66gbi67E3R8AxiYkivkGQKelp3T6oLLjoepa6vf/EbzLD1+MQAhIbC/mV", - "kFTL6QmXtsRmgtnSu7B6Fm20MXQVJCWC4fsmk5eFVnVxj9dKIa1H1T0hec2iayK+yvnf8wwdXzYbue25", - "tlbE3Qkqb6yq1JZRpu7rWbY7Hvb/x9gZ0XhwsPXT3//v/vu//cVfm6WiSEki+hGZwgPYJVn2TdFZrbQN", - "qvlMIdtNIBW2FU0UwQlYEcJLYqwWCb4ur/fRMKek5W84aWwBXg4TyvJ/r93Q3//S/u5WAuNbYB5rz/Gz", - "sx/dRWpZxR2P3kiImLn87c5dDMp5s3ipj0qiUv46e887sv6rzLuUC8deGNloAOWOJxTSgMoR02oODkOS", - "KhINbB4vCmsRHEiyXj7Z5tFz7t2aWDEkD7XhNrU0WR+9RYoPAkau+maGqK9xb+/Rvq0YX4bkduOIfYdu", - "IrbbKixqKHvMCC+phHAE53Vbaow2SJKqpcsS6/wiN28WQX6YD+h9Cv3C6bOGT79EttC3K9ODfof1PcsB", - "/m5Ba0P7G+ffmpPP71h1XM/UY2jS1iyrZpapqUxS9dv9rhJ9x4/BfbDpI6W/GddEmw9zltUTg28lTG3Z", - "7Lu+cIgIShKvdEYtqMxFu/eh03ofy5VSZmlnpZW0n82pE6bqZaLbAXSmQXM1J4KUDgI6FClEbwgy6yjY", - "IcjG5MhMiejXi9qZuguCgudhrgc7EOTOpE3D2OoMN6f4Op8BjKpYNp4eYB9FrrftFz9DLZXXrrgZnboh", - "YBk1UdefrqaKRV3KszcPo4xVzX2b9l7Cs7xqBfdro60achZzVFDTh4//wFQ95wKE4/aQljvPegOCd0QE", - "xPTWc9p0SghDExKNeaZW079NMm/jWSI0IVMuSCn/rlMEMCCxrXm8hhe4oItiDe99coMkYSaoWmrN0Yqk", - "E4IFEYeZIXgAJEwEPxcTQzrbT5/AhDb1eLC9IIwIGqLDsxOgR6jbr4nj3SmK6ZSEy1Ar4JCNtJF/A4S8", - "V0cnVvlyGd/gQZEqQD1Xhvjw7ASqmgqjgATDwc5gCMScEoZTGhwEu4NtqPGqEQ62uAXZ7+FP65xu4tIo", - "ZyeRlYN+Nk10L4ETooiQwcHvHidvRYTJpi9B6sSzkt6QYiqs4pDG4HpuUIXqvpD+yF2lB+Y+7hmAd7Yd", - "SbW0jngkfWWP9b3GBEM1sMWd4dDoaUzZixcX1S63/rABe8W8neQ5AI8nF1BDrncypQX5p16wN9y+0XrW", - "Fqj0TfuW4UzNuaAfCCzz0Q2BcKtJT5jxDkYmz4T1fyjTGaBQmcJ+f6/PS2ZJgsXSgauAVcplmzBMJMJQ", - "Is+UcviDTwbIWsYh/amc8yzW3AQZ12cS6QsLa54ymH1AWIRzuiAjZu9pU2wUC8gvnSB9Pxu1pUoaZmpz", - "+nlQ2s88Wtagmw+3pYfrO3toAeB6mkVJxpAtatxWp6WwhFLGoN6jJDYnZV6woOlkAQV6Zci9lYkJw0wV", - "9V5NZd5LsrTGVu+AndK6aIYHx0KgEHyer3xn0x/PANkz/aFAx/k3ZMFbFScYvFCEcRYVMpdzscViguPY", - "G/c/i/kEx7aA8SXxiKgvoIUFSjnRqBNuGI+ISRqZLtWcM/N3NsmYyszfE8GvJBFaBLLZpy2sbfVOi7pQ", - "SZ4mkAHa1LbQc26ZJW59vCTLT4MRO4wSV7dE2uLzseS2srNJn0Mlcj6ZBnf96U1bnvuPMql4YlGKlQtR", - "mmXyTKWZsk+dkiibMhuaQ51SOSfRiCmOPgpTln75aetjMeMn0F0IjjSelJqYLW19pNGntlXLMda7H0NT", - "j/ZHAACjQN8uo0D/PRNY6y6ZnIMpQ4L5YlY+0o08FlvLhZt1CIeYoZSnJo4dkMoUrK6MAeUHcBwjBaTk", - "+mppE06yZT82NMVXS8/GpZhAghoZQVW9EjEN95746UmSUBCfgeO/zl/9huCq0mdgmhVmI4ARZfoWRVEG", - "kjzMPhixZzicIyM3Qa6yUUCjUZBrF9EmrDWT1nG23wcR9ye9tJ/MND0a/TQY6KGM9HyAfv9oRjnQtJQm", - "Y8UvCRsFn3qo9GFG1Tyb5N/e+wHa5t5/XmEEaMPw/k1XPAbSDBTXoLk3MIsQt7w2XiKMCg5UtqNMKMNi", - "ZeUbD+gtBLUqj2eyDIyPI7CgjoKDkbOhjoLeKCBsAb9ZQ+so+OSHgBWi2xNjmeI/TtbOkWh/ONxcH3dn", - "4esRoSsNNfl9akhfO19M8LBCV1PwMJtzWf30CZoyTkbcugfJ52ccucIAP0S8NSKetVyUhDfoX74HDPrG", - "xCi4NQlM67Oxk8BWaicGLSCtJWgcLkrWKBzUSXAF8pbVj7o631Qr9tqoLIQlxg7/9u4B/2DeohQ6zPv0", - "vubFMeSozAsDPyx0hMNyiNjza8QviPoWMG54X6zUJtT8mvj7UPDnBbFyXwG0GjfbIgv33uTPBQAxANKO", - "YhprXfUc1tQ/J0yhZ/DrwP6v03ggs+1FzGcXB8iAMOYzFFNmX+NKr0X6UrSwhE4mDCDvZ6MCXCKmDXN/", - "/ut//wmLomz2r//9p5amzV9A7lsmOwYkbr2YEyzUhGB1cYB+JSTt45guiNsMpFYkCyKWaHcIYmYq4JOn", - "2KQcsRF7TVQmWOnV0uREknZAUD0Y7IeyjEgbRqEb0qlN2GAMzB4V3tGyAeW9UnSv6XNqdlDagL4VHQ5A", - "BC412Wut/hX4rWdmzxX7Wd1W3rCYrucvilwrg719s8AbMhgAsY/u4IPdNNo4P3+2OUCgYxisgKQcIDEX", - "w1jhefCDJ63nSYajVBkKQNnwplKd8Vb777Ft080AbEf8nizAbYXT203AxuRBBIkcvH7oCl3MwX64OdOw", - "zz577ILn2g20t99veQrnTdRJEf5y5+xwrwlz86UEsq+hAqMN56jtav6dHZ242jCbXw3p7+XW0Du1FRXy", - "qwNxU2nw3tSyI86mMQ0V6ru1QFL/hOSqWhVBHgo7eG1XjbDbVz39Xfl+26pkc2m96fLELsWVd/e3R23S", - "m1wjRYq+Atd+3CTrUOeYypDrviVs6Yc4tQULjfiS02kZi9YZpIzfd37lrBSXLHs+OXYEeX+mKTt1xup3", - "wz0wxeMaQ/yKjLBWhK2U1PIhYfPb/BRdooAVlqtvCzWH9ycF3bcVy4fmD8mMFdXAprmgSe3beoG+IOoX", - "0+IOD9rO4Nn4ORGOql0OYth1vi3TFYVzEl6aDcGD9Grd98Q06ab6mvG+J80XwHMTicWC/IeI0kHZLWC1", - "SsE9sYXl7k6/hRlupN5+uXdei2AeIIOzycRZrE3NNiyXLNz8rp567+U2M8B+kJfZWRbH7sVjQYRCef3m", - "8h2w9RHcktbL9o7aVl4Hb1+/7BMWcvBDy32o/EKU/fKFJXxzYGYrP9Cki05oInapu8/aJJzPOH/jLojy", - "+uD/sfPcVgj/j53npkb4f+wemirhm3eGLMP7Ys33LXE/YOTTAjetAg1YE4Naoesk1LxVRyHVtf+u5FSz", - "6RtJqjlcfwirXYTVMrhWyqv2KO5UYjVzfKUnmRzZfNCGT84/8TuTVO/Xymcx0uXvpbL67GELtHABdl74", - "RBnKJHmADpQ0x7jytdHRXF0Q5Mrrw6HuyXEPANnToIOEQjZA5J6M124d9y7c2nnv33J9mEzoLOOZLMee", - "JFiFcyJtsFJMqgz4oYndxfXcKnh/w1g6vM+r497l6h94f0cSf/1ADfM2L1DrZH7XqqvMb9trmd8Wzzex", - "a69dUX6bJ2mzxanQBVF3ReNKrHnT2dG3Lp8ugt5qRaVQFxBoEAcj9n+0/vG7Ijh5/5MLksmGw519+J2w", - "xfufXJwMO3WoQpgS1CbvPPztGJ79ZhB9Dvk9i5C8+jpMQQBAPZfA5t9OQSpePrtrSA4Lf2hInTSkErhW", - "a0j2LO5WRaomwbp3Hcnhmw/gNonJDy3pPrQkmU2nNKSEqaJeVsNJzJbbe4CxZcy+D5WcOyoXbWctKSfK", - "NQJoka393h178snvXzlyieEfpo88N1ExkVNHisuwXR/51vBheL/M+f71kIeMYkbgb4Iu1TKlrwwjZHpM", - "MgVOiUWGEPD6RMJI7fmIA1RUP5RZmnKhpMkWCQIwlMNScy0A+zJLVpNF+rJDQmJcSmRvxCCBvP5sYvm3", - "LsnS5IKkvKjqn+/U5n/0xV5Vc3F+VTL68jKWP9FoJxnrnsnY5lP+ejLWV2Md9yJpnVTS1G/khAEK5YTk", - "lMzz4D76gbLZ5oPyQDXMKt9bKZ+RR9TaggJsNrvulswLvbZdtKUEu7Y84b/hjdvcpE9qd7loSwBEEcUz", - "xqWioQvcrSfy/nFDd76hV0PWi81TW17Tr9A/5+Ky6xXnKff0AG668g6/QVuCXh5kA/v6JgVQts1toJHm", - "3m/BRg2vrxmCQev3Yhhnkb4I3YXoRMmp4MnY/mjy1WqqsNlAwUQR2lG/NrPRs9+Dweg3rhBN0phoKZ5E", - "qG+wSZ+mFf1d0ncqSxXvbsYMNdmUA2JMMjrpquZYFgmPa+7ANuCdvXlcXq4Z89n6JBj55C7jgycLxoiZ", - "pPTEZbC/QDmTRYojSWISKnQ1p+EcMmLo30yhTUhWgdP0Ik+BtXmAXgClljOBweQbkgitCIWcSR4Tk+hi", - "kSQXB82Mre9OT6GTSYZhcrNeHCCXpTW/IKRuVc5wkZfj+c3m7djQmCR4HJsTvdBaY2l/mzb3RZGibMR8", - "eTAYubID0im6KKXEuGjJieEY6ks++2rSVq89saTZi+JIAOAMbhIWBW0PMTT2Z8PYHnpLlXTMzGGWcceJ", - "ORqLeclneVLLCirjNO2KvnaZgMWLJFmBw2ijVDNTqohn6u9SRUQI6Gyxuw250QYOzT8UvtSIamve5FVH", - "Af28z40my5wXVJqplgq8mH8tkiToBXY9pex0N5De12Q4qQ/YfBbTJ1NKY/JD7r5JgpIqsy9lKKndHLYq", - "e7vIbYvNf/f2WQuo6HuwslTfs4pVUOZEFThbXhSIelCZDuAgG7KYKU/koxG3y74s1bXs9rzVqIj5DSit", - "61698vKGee3F+37+aq7gIQfByMZuplzUw+PXvYt984j05Y6ksdUuGPIDN29unuuEmGm2oswlVOmUYOeD", - "0o+Q1zmccy5LaD8hc7ygXNgM7NbqmmMmmCyM9mi95y40ql5Y++2FFc8PrK0J4fInO8cAulufO38P96no", - "8bykbeccv+dEasgCKRFGE0HJFKU4k0RLS1lCkKkwYhN5ExzOXRHnwYi9mRNkSzeWDAh5pV8q0cV2ctFD", - "k0yhGIsZaDvmo/GkEyTkSUJYZMqxjtic4AXVqppAMVaEhcu+JFCed0GKAiZadbcvlAgeKPMCoD3k6saC", - "geGiVBX2AqWCABIZdZlVSrCOmMjYf5rMlXrYC7fQC0SkwpOYynleKyLEEWGhNy3k+bfNxr68EfecqGbh", - "1K/yZnkrXvo1HzHLtsy8dPU38b75wBy1uHD1LTuw+RVCr2xXDauej+dFsdh/Q5I2e3V7/EovMzmIV1Hx", - "t/EkU6kW/+NZRlmSjDIzHalWVP9u31pyhoIyVnlusTbZ2z645JUQcjDfiOdtfXR/ntzCRvaNcMJeq2Lf", - "lnO72PS3wHItVG/Fc7+ScdDakkpWsa/Igu2ivp74xEWJy30TbNgQXM6NyzxHCQw6FWc/mHGdGVv3gNsy", - "Y2dxbTyAl9gzZf00xm18uagd72fA1iDwb+r9WttdiRF+dcZXvAjcG7M7ydmbYXgpXsYcf+/vMiEXwgR0", - "2nLEDyehWMkWWHpg2gCLWy/nED0XTfLu9HSzjUsItZJHCPWAOUS1rGmYeKo1vloQIWjkSkcenR5b71Uq", - "kcjYAL1KKNRzvCQkhUIxlGcSQWTuQO/PhbY2i+BVYlh7AWFKLFNOmVq7iqLp3Szm061K590zn7QpFb/7", - "x2Owwj88JgW8Q4srdgOrtUiFVasznnNOo8zUu9TSFp7wTI+uOYsrtDuDu21KYyKXUpHEeOZNsxiICJLu", - "2ppMtp+JKO0hqiTS9NCDCLyUiIRKSTmTI2bLv6dE6Ll1dyj+WzgZeY33Cudc88ywvm/DgU0vxvhsYdUG", - "NUgtAHVAg4NgC6fpFpSL9jtJ2eV9xpKeg0cakstkwmMaopiyS4k2YnpplA60kCjWf2yudGkbQ78vXXHq", - "9pSlIX3CptxblMPgbI7M30cQUpWtuUfEB8fWXpAysTj+AwftZ2tyLV8TBMd9RROSB7+jTNGYfjCsTg9C", - "paKhiaspQi+hCLONvhyxU6KEboMFQSGPYxIqZ1zZSgUPt0bZcLgbphSylOwSWBwwvPbPCcx4dPYW2plC", - "0b0R0/+Agd8cnpmX2Cm2NoLSQhlRV1xcopOtV2ucfM8BTP/GXnJmgytjIL0H/uP57uaRza00JFtIlKer", - "FCCefvdunFaC+2EteJjWAkgtke9mYyZwCEKxnGcq4lfMbxlY8DhL9D/MHyfrEpQoHM7fQdNvRto1y1k7", - "jdvggyBKu6eImKJBX+WBwgDsofqXasC5LYAQU/Hc894Ch+p7xO4vb5Qvw/EbfJq0EHUFub4Z2rrvm8+u", - "weXdKsPjoZC5wTS3E8VXW5+uMG23Pv0c8/BSoowpGleSGmi9DfKA6h+LvI324Q/EBIiOdKXEEblOqYAM", - "NrX0CIjoHUuEkSIioQzHW7BnMwhkoHRWLLzgFIKUw5hCmBiNCEp5HEOWnas5YUjvBgxVboDSO620FSDK", - "bcpPjIqjCQl5QlxWzk2f6vYPTNVzLqopNr8VvvimBH+9H71Vvc81WUXbZ/ysLKOn+BrcmqPMPhO7FW28", - "4MWPxhTUQ3A2o2B3KEdBD42CnWQU6BM4wmBCxQo9QgllmSJygI6NfQvCUPeHSJKQs0i65KDOgrc7lG1B", - "qQYtWyIc96HffYo9FqsAlK/tJD72oNsh3R8CbNBGmeAsTUY9ILoI8UyBA7ejK9sqIgrMI5v3/gJbopEf", - "un0XTv4PS74VHgWnrNll6egNZ8/TR661urmgijmXRdZJFOIUh1QtewjHMQ8L60Em89eBfr6UiSD4UutQ", - "gxF7nSeutIEQ6Ojsbc8ZzVBE5aUZwdrFBujVggiZTfLFIeAGxoIHh0GiEVMchTgOs1jjLZlOSQgxDDFN", - "qJItdrV8KXdZBrGYxHPw7mOetuZhGZP8OAGnV6CFrGHcljnqLUHCGNOkbFSqAwdEX3jSBbPvRA/K9TU8", - "je3zVii4lMgO1ScxndFJbB9r5AC90SIHTsiIpTFmjAiUSeN3pJfeTwWRMjOBMXoAqDNrMKqHikQnqeDK", - "moljzoU0ll2N4e9OkVQkXYFmr83Ip7DnO0oTbAa3M30lhaG2hvZryTZB+kAMphiAazzS1/RXcPYxC/ra", - "6YQfCuG/EXQ2I0JTBTZM1jyNGrJ24DREX4n0aM2Rf5636pYjPx+15M1d8nRemahi7BqOQYC+yQusZ/JL", - "2prLxH66WfTFr7pTx7mrXv7+RdhPn7nL76X02HnJubprZv0Cwx9akvvSyiukWglQWJ+OoHNEwl1GCHTO", - "O/DV0g085CwDuBJ20JZO4NtDhOH9Rsfdd5rth41blSwBlcI6LaFS69N3fhMYeDd5O79ydOgt8nZ+U/FK", - "kHfx68WNflORShU7oCse8t1n5ryrACWTnhPSWLQFKBmuZx0JVipK72ybbmqSHfF7kuDt2/MN5HcH9h9a", - "fweVoQQsv8nOxEa7vC0kSdXSPS7yae0BUNIPEIzhS/yQ+xDcXb6FWzyvfzn0cHja+rj+o57Wvb3fF0WH", - "T44ffhGtMs1VLpYtfev0sQjndEHaje5VCrYgSgXppzyFx5XIAMzCw91lCovB7AOyw9tcVfZfiLoUxyRC", - "ERUkVPESUaY4cAQzx18lElxrAvCdi6XPmF6m3OeCJ4d2N2vuQ0tT1hhWvPkmy36EFe4vHLdZYUL7jJd2", - "97atGR6iDL34GW2QayVMxl001ZoPotMcpOQ6JCSSgJOb5QVvD1ssm/QDGc8mXVa5InfyK5ubGoWZVDxx", - "Z39yjDag2MKMMH0WWtSfgiSbCr6gkSlEWgB1wWMD1e0WgN7U7qqFirxShlMuzOK+igzT5UKafaBplS0Y", - "14XgIJhQhmFxa7MUV2nKBFTp+TCFsIaCdhzmBD+uMKv5bThlR2OiVnIcEBXnJjXe5o9r7iFfc2XHVHen", - "VW67bqUiu/mqdnQhvYuEubkf8/2ard99O+6VVD5Iz0prOl/kCmmb2fzbQsHh/d0P920uf/eA3fFfEKd8", - "l0zlMIAe0YcwL3mIYxSRBYl5ClUkTdugF2QiDg6CuVLpwdZWrNvNuVQHT4ZPhsGn95/+/wAAAP//tsVd", - "yWFyAQA=", + "H4sIAAAAAAAC/+y9+3YbubE3+ipYfbJXpISkqKttZc3aRyPZHu2xbB3Lds7O0B8FdoMkRt1AD4CmRHv5", + "3zxAHjFP8i0UgL4RTbZkXazYe2clMrsbl0KhUFWo+tXnIORJyhlhSgb7nwMZTkmC4c8DpXA4/cDjLCFv", + "yR8ZkUr/nAqeEqEogZcSnjE1TLGa6n9FRIaCpopyFuwHp1hN0eWUCIJm0AqSU57FERoRBN+RKOgE5Aon", + "aUyC/WAjYWojwgoHnUDNU/2TVIKySfClEwiCI87iuelmjLNYBftjHEvSqXV7optGWCL9SRe+ydsbcR4T", + "zIIv0OIfGRUkCvZ/K0/jY/4yH/1OQqU7P8gUP1OYRaP5KY9pOF+c7CvKsivoDeFM8QQrGiJpvkEpfIRG", + "WJIIcYZwqOiMIMpGPGMRend4ikLOGAl1Y3LA+EgSMSMRGgueIDUlaMqlgneUwOEFUngUk96ABZ3aehCm", + "n0SrqfT3KVFTIjyDpRLZVtCYC6SmVCLK9NOQ9MoLpkRGFinbCWgUk6GiCeGZWiTUL/wSxZxNYFquXZRk", + "UqEpnhH0iQiO/shwTMdzyibNRBqRMRcE/TJPSYIZSmMcEomoQpQp7mZjaFTw2G7iYy46YVyQYUSkogzr", + "9ocpF2ZHVEf/Bv7AMSq9C0OD95GaYuW4nHGFLghJqxPFl/iiSsbftrY6z/r9/sdOQBVJzLbCVzTJkmB/", + "b3d3e7cTJJSZf2/mo6dMkQkRevj2FywEnpemI3kmQjIMaSSWzSSMKWEKHR4fvb3hBILNfg/+f+Np0Ak2", + "n231Nveewr8394LytBYIXx35l+Vb70xhlclFGWR209AyyrDEJIuzfp0lIyIQH6MwE4IwFc8RbCkStWC6", + "yrT7vqUIORvTSSbcFvRtuQo5p1gizIzQ6NbkRdFYq30XaiEW8Us2FCTBlGkaLwzirXuE9A5FdhPpIYWc", + "KcHjWAsFpUiSKul2UUeLcYZwmsY0BNFT2VQ7SV8GnYBlcawf1kZYrDaJ6YTCC61IQ2Vpkdy3SHFEmCIi", + "3+FtSFMRi00dF+T2rkYhF9tLQUlZ6J8uq9M80RJekNBMNz8BKhQZkZAnBOmmqyuw1d/a6/Z3uv29d5tP", + "9vs7+/3dfwSdYMxFglWwH0RYka5e8DbLtFx+HxZU0i8i+2JxVHlo16vJ4HbsEmOp8l0Nm5yq+RB7xvSO", + "JkQqnKR6Y+sxlIjZtK1dg/V1cJRfSuDNryIwI1dqaCnknY+PP8hVSkJ9xHC3PfMTW7fXQXSMMMplgGZX", + "IxiXTuTZV01EECz1gLXeoU+n34KMySzVZyGJhmmMlW5XKynABsOESqk/zX+IqDQbsxM4Jh8yroYiY8y8", + "yIi65OKi/KZtZUjToBNMsRzOJmkWdJadA1Wmhi5IjFMJ7dkVF0MiBBeB0TXnwzEXbpH0IVaQcElTCxSS", + "+ZnloVDQCSoEyOWjm4sbd76q3sFBL8BLwqjpRq+GySwOvNzW4nDzoS2XlEYsG63ULTOyH8uqBIgonjAu", + "FQ1lK7kJp7Fe3oRHHtF5lDeHaESYomNKhFVUCRIZg2PNNYJ0I4gylMnaPsh16SGZaeNnONsZqjBdJErN", + "UigvXumwL46Y0jGXL3++U1YwaXXuXktkhinsySMyo+ZoqSpDdmmGkaAzIjziOz9RjSg076E1vde1CGGc", + "kfUKpdiMRhS3EQcRjGlIPdxzeniMzGN0fITWpuSq2snWk9HToLlJhhMPL/ySJZh19YbQw3Ltw7vltl/t", + "eHV+niTZcCJ4li62fPzm5OQ9goeIgcpYbvHplk/1S0M6xFEkiJT++buH5bH1+/3+Pt7a7/d7fd8oZ4RF", + "XDSS1Dz2k3SzH5ElTbYiqW1/gaSvPxwfHR+gQy5SLsAIWrlxyuQpz6vMNtVV8fH/zxmNo0WuH+mfiRjm", + "h4iPYMdOjTo+cnqC/Q59OEFrWoZEZJRNJpRN1tvwe8g1OfRR5zvEYajIvqPNROW0lBuft6EgeEV3+o1W", + "nS1utcys5DCRTa27V7RETWgcU0lCziJZ7oMytbfTPJnShjEn1EJXz/XPKCFS4glBa+BSAfPDCFOt2Iwx", + "jUm03k6ZbZrM73xUOkIq7A1s0cWjcHNr2ys7Ejwhw4hOrE+sfkTp3zWL6XYUgrf9E4HDvN08oEtBxov9", + "vQDRDZ0IMiaCaB7/yu5SwWeEYWu9/An6Df6fjcJZuGE9hRtAzNPi9S+d4I+MZGSYcknNCBckl32i2QhI", + "jeAL/5jh0bK1LnGUVFgs3x/wxi3sxEKvW0kb67bQqg2erPzknX6nLjtBNOa6REkKNIrI51qp8WgHnCn7", + "oOa+5BMUU2YsDq3ambUAvWqekp9iDiLxluiQk39x8+tx30B4mR8aWtPPOrkCHvNJmZpTgoUakQoxG44w", + "21Axukbyn1a2T+2swpIMl0uQU8oYicBfbDe2eVOrsV4zA3bRBVXDGRHSu+dgWL9ShewbjU3FPLwY05gM", + "p1hOrYMtiqhxFp5WZuLR1iqOeAz2uGsQtAiwX89+Odja3UO2Aw8NredSv7A4k9LXunnzLlJYjHAce3mj", + "md2uf0YvcoifAwpnZdPZk3OgY0wj6QK7mtZOzuTU/AWyW48Kzj4tBjR7xfrvj55JH4KQMFZC4+2NXwfM", + "PcOTmGuazlHG6B9ZRcHuoeMxOIj1QUEjEnUQhgfgd9D234QwIrScKjxDJSUYrZHepNdBA60XdrUW3MVb", + "3X6/2x8EVTU23uka8z7FShGhB/h/fsPdTwfdf/S7zz4Wfw573Y9//ZOPAdpq5k4rtPNcc3u/g9xgy+p6", + "faCrVPkbS//y8H0Sxyz1sZYT113pw+NFxcHMNeLhBRE9yjdiOhJYzDfYhLKr/RgrIlV15svfvVVawDyW", + "EIFNNJmuSYaa0QNsvBbzSyJCLYFjohlPdrQQpkp2ENZ2MwgvpE/Jv6EQM70XjHLBBSIsQpdUTRGG96rU", + "SuZdnNIuNUMNOkGCr14RNlHTYH9ve4HPNZOv2T+6H//iflr/by+riywmHiZ/yzNF2QTB4/K1nhtDfkWz", + "bEUcdbMY1LyEsmPz2ebiHdTXrbCbyLKVNsZc41JrIZS7yFYMZPF+Vxtbicd0eDMjQtDIHcuHJ0doLaYX", + "xO4XJDKGBlm/vx3CC/Ansb+EPEkwi8xv6z30JqFKH4dZccqbK9va7RoJpxwUlTjm17lOA00RDBwcLz3H", + "l5HGS+3DvN3FU/8XLlU3wQxPCJij9kU0EvyC6IGaOwFKJLogc63lzNFEN9qdUQk3PITN0Awbr0NvwN5N", + "uSTmFfdIgm+fzghKeHhhrn6nHCz5GY4zIjvocqpVDvAJEhzbn5G5GBuwqR6kDHlKIm2EmNdgauicsNk5", + "SnAK2xwLAnscJVgRQXFMP5krfLhlIBHVJ9yAEdgYKMV6z4chFxHcsHFEcDgtUeHPEp0bheUcmj+nTLP1", + "udmYtcvqz8Gb9+9+fvP+9dHwzenz1wfHw1+f/6/+2XwU7P/2OTChGrmm8jPBggj0p88w3y9GvY2ICPaD", + "g0xNuaCfjLfmSyfQNJCav3BKezwlDNNeyJOgE/yl/M+PXz46hcy4sWd6G3gG9sWrDJmz1COSjpw3UCLr", + "YXJ3G5pkWkS9PH2/oU/nFEuppoJnk2l1Y1jV4FpbIqLyYkj5cJT6xkTlBTreeIO04oJiqjdorqhs9vsn", + "P2/IQaD/sev+sd5DR2bXwvC1DOLC6k9yqtknj/o4PH2PcBzz0PpQxk0XvK4rn4AnTIl5yqnPiKsJp+LV", + "RRnV7RZPryGKNkaUbUi9DN3wenQHvrmxKfGczajgLNHm3AwLqs9pWd0rr98cPR8+f/0h2NcHQZSF1it5", + "+ubtu2A/2O73+4GPQTUHrZCBL0/fm1tPs21UGmeToaSfPKrEQT4/lJCEC2NC22/Q2rSqaZh9i2BxBsH2", + "y58Nc22+BL5yi2LviPJWTMO1a72XP/u4ZTpPiZhR6fOz/ZI/cyu/GO5T4W1zS5YzLXBxr2S/hDHPom6p", + "y04wpoKEEF6h//UHSbQiP/tUvZbyfOd3f7VSYFdopjhOKSNLVNNvREW85OIi5jjqbt6yhmgvVD2hMeZB", + "dX3zmzXHEgsRZyPMoksaqekw4pdMD9kjV+0TlL+cC9crPRMc//uf//pwUthZmy9HqZW0m1u7Xylpa7JV", + "N+31oeQTyVL/NN6n/kl8OPn3P//lZvKwkzCKyI2UOrv+z00L9aAZG0to3KENN8P56Z0HrChuDWr4HDne", + "W3kN7BPUfEZEjOclwWvHFGz2QfrVRiUoREki+50WoxdIf7xCDOvW3CH/sm7kb/X9gtYzKM+Yftaywp4L", + "bUaSD2Rz68T+ubU4pIYRXdB0CFrzEE9yn++ykNCzC5paVRy+MMsYx0YQRBko7yPOVW/ATISKXjtYYHJF", + "QpB5UmGFDk6PJbqkcQweIhAqi0eLVuxLoU3wulT6v0XGOmiUKa2tc0WQtZugkwzGAi+PCMoYdvfhNd3Z", + "TnAxvADIckEEI/HQ6MayJWXMR8h+1EgcmOoYSxuiJlSWVul19OvJGVo7mjOc0BD9alo94VEWE3RmogvW", + "q9TrDFgqIExBd6L3M7X98jHimerycVcJQtwQE2gs97HZy9rZy9P39rpfrvcG7C3RhCUssoG+7sSxQaAR", + "Z3/WO5ZE1WbL/deI3hTSIRlO5ZSrYZoHTy+TTmf29cIUb+9M6ASzMM2qS7rVaQwCnVGhMhxrWVtRJ70X", + "/CaI3WM2mBj5svli5V4RNKuqN7NtPS6mZYho94bLehwnRlNq7TgpmfILLhRnZ35uN9gV7R8zN5CljqPC", + "1PyKvs5MIwvBO+bnjpvZDah0nNOk5m66HfIcyJJp3ir43MRgGY1QorVzbc1bPtb2+3kHnf+l8oPe+860", + "0PrFJTLUAHnC9E/l9utOiZXugmuFe5cXB8ubr8eBbIx0QrNNpARm0sSoTXFKeugXEOJIkSTVkoxNEJUo", + "D+1CjF/+DXGj1LhPB0wPTZo4EUuO3Gkk6YRRNlnXar4+mHAUGc/SOFOZ0O/NqCyoWWUd571ZiGo1oyNG", + "HkOGBGVhnEUEnTsPz3lVL1z0/yyahNYhtGDhGJKAZQPGntpIMqW71xNOsAqnmk48UyZwzE69GtRX8zKt", + "ulC1Y8mv2m6w/me5uKgnwsw8Jo6enL3kAbdgyT/Z5Aa0iorfRXlB5rDkzh2JFxySZU+k318oiOTxjNhj", + "t+zLHEGqDzeKU+HGNA5J64PU27+e5OLzzq1aCk2v1uSvmgqeFB+pum6yBcdY7d/FhDsppCdn+utow1gS", + "ID6YHvsI1LHzjrGVCHggENPMEqOIChKqheYpmwwYxJCc2196trVzvcm1jnIriVOQhwBKe3lpUWllndoH", + "zeip8YQqRaJOVTe4ICSVqyel1WvruPZ41wW5FNQJMhdU3FI9I2zMRUgSayR8neH4vNSY14y7XhOLIR2G", + "vqUxu/wMyE4hkYkfMusBbtZK2kY9ezGqWW0mhKDa5TmO43O0Zl9aR4L8DpH4dq0YZwWzvzs8dSyQX3t/", + "OOlojtRS4HyqVDrU/yWHehef1xuz37odXmSWPe2DfbWzs21X1TrdzIBrzVb9a96wiOalcep3482a5gs9", + "Shtn0kaVPyw+KTypF5RFbRv4Vb/b6J3LFSNnady1gy4VpJulE4EhxPY23XM3vjcFajZL8BV5vL4wySJD", + "MJOKJ+V4+7VaiAetBoNUiTXjcTfCCoMrs6W/1Qx3MfA4mZumjC3W5IkZTkaeuCH6CVIBJnSCR3NVvT/Y", + "9Gbzfe0lthuLb1maAviNBUmioeLLQ5jpGLl320QsmnwDxYezMeXL0zts/Esl/84cR9au1U1005BadwLo", + "OOHURJgaIoDS+OGkfHfXG7AuHL/76CjvIG82bxKDbokjc3OyxkVpECaRA43m6wijDyc99C4f7Z8l0gbL", + "jLiMhimWaEQIQxm4nuE07JqzuDyATMKhqeqfW9+JSX5YhytKbp/18pxj8NLkGdQQKjWitfmYzElYKHsn", + "jFnZC9bKa7Us8PstmVCpRC3sG629fXG4vb39rO6/3Nrt9je7m7vvNvv7ff2ff7SPEL/9/A5fWwdV2WKD", + "z8rS5/D98dGWdZZW+1GfdvCzp1dXWD3bo5fy2adkJCa/b+N7yQDxi7KjImoOrWWSiK4Tk5qrfLFypZC0", + "hli4G4e43VHEWhGAu+xdQ4l3+s27SG3xBU3bkN3rJ5/UBebKsOvS5BYt+XkKdmexS0oanI1uDKk3jvOI", + "youfBcEXkLK3eG4neELk0Jxn/niGTJogG3JlvRuCczWW5t606vXc3Hmy83R7b+dpv+/J6FhkeB7SYahP", + "oFYDeHN4jGI8JwLBN2gNLrwiNIr5qMrou9t7T5/0n21utR2HueJpR4fc8HJfoTVLkb86nBL3pDKora0n", + "e9vb2/29va2dVqOy/uJWg3K+5YpK8mT7yc7m062dVlTwKfTPXYZNXYH3ZVYemOx+/a+uTElIxzREkKOD", + "9AdoLYEjjOS3VdU9OcKRyz/1nx0K01gujZgwndk3jaMtyWJF05iYZ7AgrXzRMPMjaMmLkMFYnu97vZZs", + "XtLKCAE3l/wVVMkvq5DuxCQ0l5QnSuJo3+zQlXIOVrMY2McmPrBzaMkNr7Tp1I3JjMRlJjBHl8msFQTl", + "fGIWrTIrymY4ptGQsjTzskQjKV9kAnRR0yjCI54pc81oE7SLTiDqGWyPsRbX7ezcF1xcrIwf1Sdxnoe+", + "0it0AI70sXXVwCmOkf3apSiUlL78OtBcmtrnEr01XxgPUfFzmlVRbTrQk/UkMSSIVBwkqXUY2mbaapd+", + "vQWcpS78w/RXyM57in3pjk24wO1a2GJCAH9BrdRYNKe8g/fP4PXW4ej6w5WOlBZ0Z+TyPogO8fpdzbZd", + "yXB6NxRfFoyW+xqKl+AUFjQiPQS7C6JiXH5gbaedKZ6mJMr9P70Bs/Hc+U/S3KDoDw0d1JRQgbigE1rt", + "uOpgu8uotuuwouOmG7Nj+cNFDRUeQvhG86bHY2WwFi5cyhQp5y/ZRQg6wVmOTGElUZU0b3N0jwWKFKGW", + "C0N8efr+urFpqeBj6sMbglgI+9RaZi5q69VO/6y7+f+ZCEzNb6CiUWbiJxIe1YAk7PvtTp6Xp+9Pm8aU", + "Qzug8ugW5pRHvCwDt3IUsZdK9lbSWjCO/fXBkndS6N7PfLrsWOCEjLLxmIhh4nGuvdDPkXnBhDZRhk5+", + "ruqzWm9uazWfVhYHzOYxDm1mfjvqexxytWl0StT86F+ut8Qcw035fHqphH3HpvT10OscTAO9PH0vURGl", + "5PHUVZe3MV7+dDqXNMSxadGk51JWdrABc7bWkE+LD60r0qMn+zFY3EZAa7NJmsE2PHvbPX7zYSOJyKxT", + "GRNEFk15TPS410vSYuay+org/oqQmDV5OgxjyLYbqESrfAe3JlJpv3qoo7jC8VDG3Bes8U4/RPAQrX14", + "YbKu9Ag6KK0spf69RIUKf+95d4yWSE3dnkGHdZdpZYN7bccqGqZxr5SmV+nUt1V+ITg2IKBVfl4EQOIX", + "1YXmF6tBd0wjvn6PXWB4zajxJW8dnhwZhSHkTGHKiEAJUdhCjpZCXEAdCjpBV59RESYJhNqN/7Y8uqXB", + "BV/Oxmp04h4u4HbciQO3Id/8rQlBiFCCGR0TqWy+eaVnOcVbu3v7BhUjIuOd3b1er3fdHJXnRVJKq6XY", + "MCH8pXSVnpx+3TrcQSpKm7l8Dk4P3v0S7AcbmRQbMQ9xvCFHlO2X/p3/s3gAf5h/jijzprC0AlKh4wUA", + "leqVpj6zzO/7JcxLh+/XCtfOb89AZAPkzXnzjRWeaPvEcNzXJhbfGHqkwL9SJciRckBoC/gR+mm5J9Qp", + "RvCO7TNjisYFMsuiD/RG2DpyKfzAAvRASlgOOBDH5q+Qs5neFT70gYoAd8++6v7ARrkMI+rh5L9ba88E", + "SUBW1er9FmzgNF3Ntn5FMZd/bVFXbG605yR6cKl/kzu2au9vJv/zx/8vT5/8vvnHqw8f/nf28n+OXtP/", + "/RCfvvmqDKrlafEPmtt+a+nscLFUyWlvy0onWIUehWrKpWqgsH2CFDfxmj10CIbf/oB10SuqiMDxPhoE", + "tRDhQYDWyBUOlfkKcYZ0UzbTYV1/fGrcP/rjz862/FJvI7IpDcIuSJ7JJLNRxBNM2fqADZhtC7mJSLjT", + "139FKMSpygTRq6d12HiORgJgva15XnTeQZ9xmn5ZHzCwcMmVEnoGKRYqx/FwPQBT2FGZmAH7OolcYrix", + "kAcsP5fyvHDjo+nlThDwzdcjLv1E8ZovXFRTcZ72fRn0EPWlFzKmUhEIzM45W7NRHo6GnvYrouJp/2l/", + "pYKf89AS9oOdsIj375iyxV4yDAxdG8ENEWotfOlaNpk9gn559+5Uk0H/7xlyDRW0yJfYGHkmBlAaH6GK", + "ZSn6bz3woo3C6rackHGSwWdxi6yh5yY89N2rM6SISFzA/lqoyTmmoZ4fXP9TKTPNihSjg8OT5+u9FgUL", + "gLb5+Jes47t8hvXkDus0a/IF5hyv6dtBx0cQnmt3aKHAQVjNCy5QbARMsa/30XtJqrGusFTmVt+sZDwv", + "PG/mBBgE667FtC4p9tHbXG/E+VAqRRKqzrxiX0Kz9uLFxPwstN5ZgB8Xzi6yog0ifLDKg8T1idssCpZv", + "fw/FYc/buO6ST/N6e7vsDNWd+VmjWPvbRk25fXVn+7pG7nURHqpJmKUE3hzkoT06w12gHCwafFdUDRtv", + "8ZF+bO/snVnz4QRNsWR/VvCwZtxsbj9phdepe217/12++eZjM6R8W7qMzvze1uS2XtA4NuEQkk4YjtEz", + "tHZ2/PLX41ev1lEXvXlzUl+KZV/41qcF2IPbGy9P30O6DJZDd4XUHDWJi8hjckWlkosJr61uYpeDS/xS", + "AYDwZhCv3yIqhLu+XpjGfeA9PGRc4LeHNbEUHeJrIR6stnxHCA+NwtWHjlCVs+bn28VquJPhrCwvUlYq", + "XND2jcEROgH1BKweSC0CSYSOTwuQxcKr5ZqvzcnW6tns93ub/TY+vgSHS/o+OThs33l/y3gy9vFoP4z2", + "yfgrfIyWsY32h+NLPJdo4PTzQWAMgpIlUNq2VodvdX+7iEFxM8iJukKxClTiOiAS7dAhvjYlfxnU8lkV", + "ZLm1kvcVlUhahVC4o90GT9ivhtdxnxMU8iyOtCI10lvXGHYksvanJKrAr4bd/p5dMH7JqlM3XlQtAP7I", + "iJijDycnFZ+7IGMLz9ti4hB00bAOPL3WMmyt0LVXjuaGQA33Ac5QF7ul4+7WoRjKTj8XxGk4tIXzr1A/", + "vRfvlJml0XyyZE41t01EZsMs82lV+pFL3Xj//viowhwY720+7T991n062tzr7kT9zS7e3N7rbu3i/ng7", + "fLLdAJDfPvDm5rE01d3cnCoFhAcXqMmEi/b1fsuDYUaZQnmgnN7Ih1o9RSU92CQGgVfimFEFIJCUTXQz", + "4CSwarLJ8DQ4lZRRBZACAGhDmZ4yeGN0Izb8aR+9hHfhEU4gYckNQhtHVUcEjubGEasFg+s6hX8tH/LZ", + "NINyP/CNnGYKQXkoPW1NBmuuLG/CyJh99JrDN8JFqTJet3vM6+ATWHy9biOt2bgkF78KnVmBuY9e5EIy", + "F7NWrK5JYv80stuGVkPY+HoleM+ueKC5pVi5UlxaJzAUDTqBIxTEry1GstlxeZM0yqzou6EgOAYRWkQK", + "ZYrGFiUBZkKhQBJMBMPiNu1kiwhGoqFRAZruG034iVUT8o+coPhwgtYgH/KvyBqV+l/r+d1keVfubD3b", + "ebb3ZOvZXqush2KAqwX8IQRHLQ5upbQP02zoao80TP3w9L0pcRhyJrPEeAns3EtBpqngodZWKUNFMZOi", + "82e9Z+Vkj4hnprCTHZLNDPtSKl+2tPJMwwXbHzSe0fGY/fEpvNj6XdBk82pPbo28xl1RJ82rCR+XXa0L", + "ZiMZdQ2Koz8eHxhKyMaUlbdEwgzQGVEI+KeLcAiHdB7TZFnOJbZYinsZa2d7e/vpk92tVnxlR1faOEOw", + "XxdHeWJHUNpi8CZae3t2hjZKDGfadIGeADDBrALn32fIAjr3q5VAe5v9bR+XNOhLBdfYtmdJI8k/WCXI", + "TsoSHUKzcgVpYZd7qb293X+ys/t0t902dnX3xNVyCeNAPQx5LA5KeeXXwD3/7uAU6dbFGIdVC2Vza3tn", + "d+/J02uNSl1rVIDhY7A3rjGwp0/2dne2tzbb5V75XPA2q7CyYauyy7PpPEzhWQ0PKRZFb6fptPApnobB", + "3pIwxjQ5CF34TO30MRgbQ2FeKxahzcFgnQQLB1eLb1uZaLXCQUY14AKVKi72VrtDb+bdbBbT5jxYLcYX", + "degYM00umyRgoBxvQLtUkBnlmbyFhrgioWamccy5uNa3TfFIb4nMYmVckFSiDyd/BiGimQtJRdJqrL1l", + "vyWpFDec3LU2cIUn/FzdRKxWq9Fm6ZdNuNOwTTvL4mgr278xYynSoipjq+++D3EcZgBehvP11LOC3AOe", + "Kbipn5sokTjmnKFwitmEABi8gUpkE4TRlMdRL/BflcTRcOy9wsgrzPOifrkbhP7M1d9fe8mLknaGlWr4", + "vLuJkSoWuanXopZ8URO3IcNJ0xMrXoIBMJ9UrPmYTyRYgQriX3p19JkUCxPWgpnBqZslxnispm5t6dPe", + "M8Sa9PYdoebo5GNr0VodQ/GckjgUXMqiMPeHk+owlwUw5vXsV99nVwfbgnVlypkk/jLxtiZ8K4eP70D0", + "RIZ9zZEIPAwBoMtqQFscwwSzDJC+SoxMrlIqDHu0uxyfcqmGeTrKNQcr1RBQnDJBipw1d15OIQFgbkQc", + "vOM9F51ouwm58vLGN/h6gav8TTUNsFmmeinqp1Yn50EfGy8m5CzNASqSiuoZJNdJGStgf6iEVmkpWwmt", + "Ma4qYqkEXbPe5qLKb6PqfppKyr7a6Z+1zeZanrx1itX0mI25BxvyGg5/GxLvYhdSIqD8OGcoIoySyBmP", + "ueff+rYgyD6WBEUZsZQzCqnAluDYbG/AgGTOKUbZpCbr6x22ccObMSwHeYJ+7YttrhylPzT7nciAViZI", + "QCJcBGm3inigcuj3FC82LMgki7FA9YzFJUOW8ySm7KJN63KejHhMQ6Q/qF/njHkc88uhfiR/grmst5qd", + "/mBYxBjWrmfM4GyEqVmQWr/FFH7Ss1yvxbeD62XDfL+hv291g+uNG3pBY2KT+t4zelVi9CoKys5Wvyn1", + "oaHRStLDYkLodSW3ZVnfjne5mgd51QRPOL6JAKrdSlQdkZX5+mYLIWbLEj0WXTFozV0KO5SZKl1LaC+t", + "PCHtotzq4Q9uNBuShNXed57uPtlrCbfzVb7OJVWVv8KzOUuWeDQbVuqkjdvs6e7TZ8+2d3afbV3LQeUi", + "ZRrWpylaprw+teIoNafZbh/+71qDMrEy/iE1xMtUB1QpdHLjAX1ZsnWLNOuGa49GnO+4vJLunqXqAW3n", + "Y1yiLR1UVK5SLa81Mh4TMCqHhm7dYjC18PxWYwhxikOq5h6HCb40kO/5K7V04TbetOpgPSS1bVvEBy25", + "ZDYqAjrXXOfoL8a1XuOFp61Ru2Q2anLjv6n3apz4hQ+ofEXU4oamKCyw6C7I53OJZSWqQ/8dAmZzUaut", + "Hj9k3mhfldrxel6YuoiM9KW8+4tQl5e/tpwlt29FSa5TfNkR2rwFr2VDe05kX5nK1VG5NflgD8CbfTUc", + "lfH0lgIWVsD3ilP3+v22qzK3+J05wa7fXykE9Dof1qHFgB/tGCzJi7Y7FZZo4CbFxWpE6TsACDIxBTeC", + "CLLhCPeCEmR/vhNkoIXlOCPKvXumLfosXgIIzRQRM+xxTLkmkHul6kc1kriDrIsPbSbrtVKFO1O/rmZT", + "eFuGj2ktcJgKMqZXS7jFvGCO62r4uLQUiKqg4RKtJfgK7TxB4RQLWRs7o5OpiudVJ+uOJ3vi64o4E6VV", + "5/bw6sVqug8XbzTscpZb923Zs1Kugx/2nUTDZYnuh/lrzmec4jnolo2G4JPtnX5/e6t/o0z320KjL7XT", + "FBFa+s46cypXj+UW8vjPRcjCS0FNUTNHJqkEwck+RFOlOCQoJmPIA8uhYlfa9AtdLx+8vSS1gf85/7uF", + "cpVLrZ/FijjGGSgerh0LEuCmEbhb2mquR/n54rCXJIvlYiZcyBqrx6/udfvb3f7eu83t/d29/c3Nu0iN", + "z4nUFMLz5NPm5ZN4C4934qfzJ39sTp9MtpJtb7rAHRQ+qNURrNVBsHNIiahjUdYxXCWJKSNdmYe9rQ5A", + "XiILzE3Syv1/Pe+DmcFSZeGsOsmyzoBVQZx6Sbb7SGyyo1/qQqkP//ho+bBvFEdWH4ifwepDAX5qNxiA", + "atn8WlyQjLU8d96XXmx98iyNbVx19viCvmFre1e5geI+fq4IxsoOW3ZiL55qHhNuwgVV02T58ZC/lqMM", + "wGX4J6miaiJNDx1PGCDPln/O7z7KxaH1x0EniD/tVPeM/b19SpXNqs8Z0C51WQ1ocTcAwMbLqQCvFKaF", + "MOEJWBAgxE+b3c1ncEMff9r5qd991kN/L0UKdAy1yuTbdG9Xfu23oWEuKEHvNDfnm8+udY3u6LmMg361", + "51LTQWzz7S2PF7Cf7qxwYdOVBS4eL6xxLavozgoN2dNsWNaSIhLjua9YQ4znaETGgG5dsw/LTIZGZEKZ", + "7CBy5YQOlgijsink6rr35SBAXKBBsJsMgh46sCgVYK0WqM6V5gHPt8QnNLE1piyib3NYylbSLn+ibjxc", + "D7rIfeVRz3p+/ezZu/7mtaGLrndM9r4iqvqrzN12Jm6MpWqyLV5hqXKbFInMWhgdKPPDatX+LbS6TYeA", + "8FbAG9hHsublcJjyslD8nCekgyZcoSIRYqWmB8MXGfPyQ3X8RalmgMhoZIitFQzRbkx5MiNdJr6Oj1Aq", + "eJSFRRRwDIPOwpBIOc6g8nSvrVa/+p71Lh0aEF4/5gKtdmg0eTBWp9qSq+b1fk2uVKlLzbDNS73ZX73U", + "d+IF6QRZGq2WYealdhLsWmgkK+JKPT6ZKtlrmmBpMh9bSPS3ZQouGrlQtAqFWiXKUldFUfPUIidJT+1E", + "fDX04iQckZjoY2qxEWTKmtrQFyoLKbpapG7uPfW7DfHVMISkzIWB/EpIqm2VhEtbZjTBbO4dWB1JHK31", + "XRVNiaD5rkEzs9SqDu7JSk2scanag7LXvNom662MgZ+jlNwuIrv9cmW9jLvwwz2kkvbGXi7IKmM4tIYc", + "38X1b8J6TRHtuHZe7+zKdjrZO2sZNwEI1UN7y27mg+4/jFsZDXv7Gz/99f/tfvzLn/yleCp2sySiG5Ex", + "3HdekHnX1BjWNnqvCl8L4EZambYFbBTBCTiNwgtinFQJviqPd7efC435a5wsTAEuihPK8n+vnNBf/9R8", + "zVoi43uQkytZ9qvBru4CSVhxdxytJURMHFy/iw6E6u3abrggc4lKcIVWpXGM+meZf1KuE3xu1MAeVLce", + "UUB9lQOmrVochiTV1oSFbaMwFsFB+tSrZVvYRBfNr+USBqxYm11VQ0X77K1JvR8wctk1PURdzXs7u3vA", + "R5SVKbm5sMS+RTcJ+k0FNTWVPV6jV1RC9okLsi69jNZIkqq5AwV2YbDr1wMMOMgb9N583zJaWv/ZbYDD", + "vl+KBvsdlnMt4zm4Aa1EclhY/0YIRn8c3VEdmMnsSVuirgokVLMOpeo2h9klWp0ZQrToYkicfmYiUS38", + "6SSr48BvJExtWLBlX/ZLBBWol8YeF7vMgRt04aPVIbVLFerSzEojaV6bE6c31quCNxPoVJPmckoEKS0E", + "fFAgxl6TZDYutEVOlYFETYno1msYmjIbgkKgaW7yOxLkscOLftDlgEYn+CrvAXzoWC7cNME8Cmi/zZc/", + "Q+mct66WHR27JmAYNa3ej05U5aI21fgXF6PMVYvzNu97N56VVUukX9PeqjFn0UeFNX38+HdM1QsuwA5o", + "zmC6c5AjsDEiIiCFuw5h1Ar/hyYkGvJMLd//tqaATV+KnCpfwC07mwcDE9sS1ytkgcuxKcbw0ac3SBJm", + "gqq5NpKtSjoiWBBxkJkND4SEjuDnomNAL/7yBbyFY0/A4kvCiKAhOjg9hv2YYAaqMvpwgmI6JuE8jIkF", + "n12AWwEl783hsbUzHcAf2A1UAeu5qtMHp8dQxFYYWyvo97Z6fdjMKWE4pcF+sN3bhJK+muFgihtQ7AD+", + "tLkIub1yHFk96Gfziv5K4IQoImSw/5snpl8RYYonSNA68aRkN6SYCms4pDFkGhhWofpbQLtyR+m+OY87", + "huCt3WRSzW3cJUnf2GX9qDnB7BqY4la/b0xSpuzBi4viphu/2/zMot9W+hyQxwP9tKDXO53SkvxLJ9jp", + "b15rPCvrkfq6fc9wpqZc0E8Ehrl7TSLcqNNjZoLBkYEVseEu5X0GLFTeYb991OslsyTBYu7IVdAq5bJJ", + "GSbadmfk0lbu+J2PesheAgDarZzyLNbSBJlId2fuKyx6k08Ii3BKZ2TA7DltastiAcZ8gvT5bMyW6tYw", + "XZvVz3MQf+bRvEbdvLkN3VzXuX4LAtdRNSUZAjjYsKksT+H0pYxBeU9JLARpXp9iMaYG6jHLkHsLUROG", + "mSrK+5pCzBdkbv3K3gZbofhogQfLQqDufw5Pv7XuT18BsFR/5tdR/gxZ8lbVCQaXMWGcRYXO5SKqsRjh", + "OPbCPExiPsKxrVd9QTwq6kt4wxKljCvrlBvGI2IwQtO5mnJm/s5GGVOZ+Xsk+KUkQqtAFmzc0toWa7Ws", + "ewmwYAkAfptSJrrPDTPEjc8XZP6lN2AHUeLK1EjzCY4lt4W8DVoSlciF4Bre9aPZNkR3HGZS8cSyFCvX", + "HTXD5JlKM2VvtiVRFiEdXoeytHJKogFTHH0WZEKlEvMvG5+LHr+A7UJwpPmk9IqZ0sZnGn1pGrUcYj37", + "Ibzqsf4IEGAQ6NNlEOi/JwJr2yWTU3BlSHBfTMpLupan3mu9cL1O4RAzlPLUwBYAU5n65JU2oNoEjmOk", + "YCu5b7W2CSvZMB+bieQrnWjTkEzeSG0bQRHF0mbq7zz17ydJQkF8Do7/OXvzGsFRpdfAvFa4jczVMtOn", + "KIoy0OSh996APcfhFBm9CaDpBgGNBkFuXUTrMNZM2jjpbhdU3J/00H4y3XRo9FOvp5sy2vM++u2zaWVf", + "76U0GSp+Qdgg+NJBpQcTqqbZKH/20U/QpmyOs4ogQGtG9q+7WkGAKlEcg+bcwCxC3MraeI4wKiRQ2Y8y", + "ogyLpYWOPKS3FNSmPJ7IMjE+D8CDOgj2B86HOgg6g4CwGfxmHa2D4IufAlaJbsZBM7WenK6dM9Fev7++", + "Os3S0tejQlde1Nvvy4L2tXVriodVuhYVDzM5B+KoV9BU7TLq1j1oPj/jyNWB+KHirVDxrOeipLzB9+Vz", + "wLBvTIyBW9PAtD0bOw1sqXVi2AJQTMHicEnRxuCgToMrmLdsftTN+UWzYqdpl4UwxNjx38498B/0W1S+", + "h36f3Ve/OAZI0rwO9ONiR1gsx4gdv0X8kqhvgeP69yVKLX7qQ/LvY+Gfl8TqfQXRatJsg8zcfZMf+gFS", + "PqRtxbysbdUzGFP3jDCFnsOvPfu/zuIBIOPzmE/O95EhYcwnKKbM3saVbov0oWhpCR+ZrI/8O5sE4nC3", + "1sz5+e9//gsGRdnk3//8l9amzV+w3TcMGArg9J5PCRZqRLA630e/EpJ2cUxnxE0GkDTJjIg52u6DmpkK", + "eOSpLSoHbMDeEpUJVrq1NBBY0jYIpgeD+VCWEWmzZvSLdGzxOYyD2WPCu71sSHmvO7qzGGJsZlCagD4V", + "HQ9AwjU1YMXW/gr83jMz54r/rO4rX/CYrpYvilwpw71dM8BrChggsW/fwQM7abR2dvZ8vYfAxjBcARgs", + "oDEXzVjlufdDJq2WSUaiVAUKUNnIplJZ+Ub/75F9p50D2Lb4PXmAm+rkN7uAjcuDCBI5ev2wFdq4g/10", + "c65hn3/2yOVKNjtobz7fchcumqiVIXx76+x4b5Hm5kmJZA9hAqM1F5PuSjyeHh67UkDrD8b093Jq6Jna", + "Ahr50YG4KSx5b2bZIWfjmIYKdd1YoIZDQnJTrcogj0UcvLWjRtjNq452WD7fNirgPY0nXY7jUxx5d396", + "1Dq9zjFSIDIWvPbjJFnFOkdUhlx/W+KWbohTW5/SqC/5Pi1z0SqHlAlxz4+cpeqSFc/HR25D3p9rynad", + "sfrZcA9C8agmEB9QENZq7pUwTB8TN7/PV9HhQizxXH1brNm/Py3ovr1YPjZ/TG6sqEY2LQUNknPjAfqS", + "qF/MG3e40LYHz8TPiHC72kFOw6zzaZlPUTgl4YWZEFxIL7d9j80r7Uxf0973ZPkCea6jsViS/1BRWhi7", + "Ba2WGbjHto7g3dm30MO1zNvbu+e1DOYhMgSbjJzH2pTow3LOwvXv6qr3Xk4zQ+xHeZidZnHsbjxmRCiU", + "l+sunwEbnyEsabVu73bb0uPg/dtXXcJCDnFoeQyVX4myT25ZwzcLZqbyg03a2IQmOZm686xJw/mK9Tfh", + "gigvB/9fWy9sQfj/2nphSsL/1/aBKQq/fmfM0r8v0XzfGvcjZj6tcNMq0UA0MSgNu0pDzd9qqaS6978r", + "PdVM+lqaak7XH8pqG2W1TK6l+qpdijvVWE0fD3QlkzObj9rwyMUnfmea6v16+SxHOrhmKqvXHrYeDxfg", + "54VHlKFMkkcYQElzjisfGy3d1cWGXHp8ONY9PuoAITuadICdZBNE7sl57cZx78qt7ff+PdcHyYhOMp7J", + "cu5JglU4JdImK8WkKoAfm9pdHM+Nivc3zKX9+zw67l2v/sH3d6Tx1xfUCG9zA7VK53dvtdX57fta5zcp", + "1DZ3zQI8dRz433pDUKFLom7LxpVc88VgR9+4fLYIeq8NlcJcQGBB7A/Yf2v74zdFcPLxJ5ckk/X7W3vw", + "O2Gzjz+5PBl24liFMCWoxWo9eH0E134TyD4HONciJa8+DlP/AVjPAdj8xxlIxc1newvJceEPC6mVhVQi", + "13ILya7F3ZpIVRCse7eRHL/5CG5BTH5YSfdhJclsPKYhJUwV5dEWgsRsdcVHmFvG7P1QKbijctC2tpLy", + "TblCAS3A+e89sOe4QCO8b+PI1QF4nDHyPLXA2tYcKQ7DZnvkW+OH/v0K5/u3Qx4zixmFf5F0qdYpfVU3", + "AekxyRQEJRYIIRD1iYTR2vMWe6godimzNOVCSYMWCQqwwZOfagXYhyxZBYv0oUMCBjAlsjNgUC9APza5", + "/BsXZG6wIClnOexjPlOL/+jLvapicT7oNrp9HcsPNNpKx7rnbWyhox9Ox3ow0XEvmtZxBZF/Ld8YYFCO", + "SL6TeZ7cRz9RNll/VBGoRljlcyvhGXlUrQ2ot2fRdTdkXte36aAtAezaapT/gSfu4iR9WrvDoi0REEUU", + "TxiXioYucbeOWf7jhG59Qi+nrJebx7aaqt+gf8HFRdsjzlPd6xGcdOUZfoO+BD08QAN7eJcCGNvmNNBM", + "c++n4ELJtodMwaD1czGMs0gfhO5AdKrkWPBkaH80eLV6V1g0UHBRhLbVhxY2uvd7cBi95grRJI2J1uJJ", + "hLqGm/RqWtXfgb5TWSpweD1hqLdNOSHGgNFJVyDIiki4XHMLtgb37IvL5ZWaMZ+sBsHIO3eIDx4UjAEz", + "oPTEIdifo1zIQgktEpNQocspDaeAiAFltaCuKoBV4DQ9zyGw1vfRS9ipZSQw6HxNEqENoZAzyWNigC5m", + "SXK+v4jY+uHkBD4yYBgGm/V8HzmU1vyAkPqtMsJFXnnotcXtWNOcJHgcmxU911ZjaX7rFvuigCgbMB8O", + "BiOXtkE6RuclSIzzBkwMJ1Bf8cmDaVudZmBJMxfFkQDCGd4kLAqaLmJo7EfD2Ox7q7K0ROYww7hjYI6F", + "wbzikxzUssLKOE3bsq8dJnDxLEmW8DBaK5VIlSrimfqrVBERAj623N3E3GgNh+YfCl9oRrXlffIis8B+", + "3utGgzLnJZUWqqVaNuZfsyQJOoEdTwmd7hra+wqEk3qDi9diemVKMCY/9O7rAJRUhX0JoaR2ctgi/M0q", + "91vzwnfvn7WEir4HL0v1PqsYBWVFGSYBddRdLaxHhXQAC7mgi5nyRL494mbZlaUSnu2utxaKf34DRuuq", + "W6+8kmNeZvK+r78WR/CYk2DkwmzGXNTT41fdi33zjHR7S7Iw1TYc8oM3r++ea8WYabakoicUJJXg54Mq", + "l4DrHE45lyW2H5EpnlEuLAK79brmnAkuC2M92ui5c82q59Z/e27V833ra0K4/Mj20YPPbcyd/wv3qPji", + "RcnaziV+x6nUgAIpEUYjQckYpTiTRGtLWUKQqTBigbwJDqeuZndvwN5NCbJVKksOhLyoMZXofDM576BR", + "plCMxQSsHfPQRNIJEvIkISwylWcHbErwjGpTTaAYK8LCeVcSqEQ8I0UBE2262xtKU/A6r3XaQa5ELjgY", + "zksFcM9RKggwkTGXWaXa7ICJjP3NIFfqZs/dQM8RkQqPYiqnea2IEEeEhV5YyLNvW4zdvhP3jKjFGrEP", + "cmd5I1n6kJeYZV9mXqX7m7jffGSBWly4+pYtxPwSpVc2m4bVyMezoi7uf+CWNnN1c3ygm5mcxMt28bdx", + "JVMpjP/jWkbZLRllpjtSLR7/3d61FNWcM1a5brE+2ZteuOSVEHIyX0vmbXx2fx7fwEf2jUjCTqNh34S5", + "XUz6WxC5lqo3krkP5By0vqSSV+wBRbAd1MOpT1yUpNw3IYbNhsulcVnmKIHBpuLshzCuC2MbHnBTYew8", + "rgsX4CXxTFk3jXGTXC5qx/sFsHUI/IdGv9ZmVxKEDy74ihuBexN2x7l4MwIvxfOY4+/9XibkQpiETluO", + "+PEAipV8gaULpjXwuHVyCdFx2SQfTk7Wm6SEUEtlhFCPWEJUy5qGiada45sZEYJGrnTk4cmRjV6lEomM", + "9dCbhEI9xwtCUigUQ3kmEWTm9vT8XGrrYhG8Sg5rJyBMiXnKKVMrR1G8ejeD+XKj0nn3LCctpOJ3f3kM", + "XvjHJ6RAdmh1xU5guRWpsGoMxnPBaZSZepda28IjnunWtWRxhXYncLaNaUzkXCqSmMi8cRbDJgLQXVuT", + "yX5nMko7iCqJ9H7oQAZeSkRCpaScyQGz5d9TInTf+nMo/lsEGXmd9wrnUvPUiL5vI4BND8bEbGHVRDWA", + "FoA6oMF+sIHTdAPKRfuDpOzwvmJILyAiDcl5MuIxDVFM2YVEazG9MEYHmkkU6z/Wl4a0DeG72644dfOd", + "pSl9zMbcW5TD8GzOzN9HElJVrLlLxEcn1l6S8mZx8gcW2i/W5Eq5JgiOu4omJE9+R5miMf1kRJ1uhEpF", + "Q5NXU6ReQhFmm305YCdECf0OFgSFPI5JqJxzZSMVPNwYZP3+dphSQCnZJjA4EHjNjxPo8fD0PbxnCkV3", + "Bkz/Axp+d3BqbmLH2PoISgNlRF1ycYGON96sCPI9AzL9B0fJmQkuzYH0LviP67vrZzY37iHZsEV5uswA", + "4ul3H8ZpNbgf3oLH6S0AaIl8NmsTgUNQiuU0UxG/ZH7PwIzHWaL/Yf44XgVQonA4/QCvfjParhnOym7c", + "BB/FprRziogpGvQgFxSGYI81vlQTzk0BlJhK5J73FDhQ3yN3375TvkzHb/Bq0lLUFeT6ZvbWfZ98dgwO", + "d6tMj8eyzQ2nuZkovtz7dIlps/fp55iHFxJlTNG4Amqg7TbAAdU/FriN9uIP1ATIjnSlxBG5SqkABJsa", + "PAIiesYSYaSISCjD8QbM2TQCCJTOi4VnnEKSchhTSBOjEUEpj2NA2bmcEob0bMBR5Roo3dNKWwGi/E75", + "ilFxNCIhT4hD5Vz3mW5/x1S94KIKsfmtyMV3Jfrr+eip6nmuQBVt7vGrUEZP8BWENUeZvSZ2I1p7yYsf", + "jSuog2BtBsF2Xw6CDhoEW8kg0CtwiMGFihXaRQllmSKyh46MfwvSUPf6SJKQs0g6cFDnwdvuy6akVMOW", + "DRmOe/Ddfao9lquAlG9tJz7xoN9D+ntIsEFr5Q1n92TUgU0XIZ4pCOB2+8q+FREF7pH1e7+BLe2RH7Z9", + "G0n+d7t9KzIKVlmLy9LSG8mew0eu9Lq5pIoplwXqJApxikOq5h2E45iHhfcgk/ntQDcfykgQfKFtqN6A", + "vc2BK20iBDo8fd9xTjMUUXlhWrB+sR56MyNCZqN8cAikgfHgwWKQaMAURyGOwyzWfEvGYxJCDkNME6pk", + "g18tH8pdlkEsOvEsvHuYw9Y8LmeSnydg9Qq2kDWO2zBLvSFIGGOalJ1KdeKA6gtXuuD2HelGuT6Gx7G9", + "3goFlxLZprokphM6iu1ljeyhd1rlwAkZsDTGjBGBMmnijvTQu6kgUmYmMUY3AHVmDUd1UAF0kgqurJs4", + "5lxI49nVHP7hBElF0iVs9ta0fAJzviOYYNO47emBDIbaGJqPJfsK0gtiOMUQXPORPqYfINjHDOih4YQf", + "y8Z/J+hkQoTeFdgIWXM1ara1I6fZ9JVMj0aM/LP8rXYY+XmrpWjuUqTzUqCKoXtxCAr0dW5gPZ1f0EYs", + "E/voetkXv+qPWvZdjfL3D8I++spZfi+lx85KwdVtkfULDn9sIPelkVe2aiVBYTUcQeuMhLvMEGiNO/Bg", + "cAOPGWUAV9IOmuAEvj1G6N9vdtx9w2w/bt6qoARUCus0pEqthu/8JjjwbnA7Hzg79Aa4nd9UvhLgLj5c", + "3ug3lalU8QO64iHfPTLnXSUoGXhOgLFoSlAyUs8GEiw1lD7Yd9qZSbbF70mDt3fP19DfHdl/WP0tTIYS", + "sfwuO5Mb7XBbSJKqubtc5OPaBaCknyAZwwf8kMcQ3B3ewg2u12+PPRyfNl6u/6indW/390XR4eOjx19E", + "q7znKgfLhj51uliEUzojzU736g62JEoF6aY8hcuVyBDM0sOdZQqL3uQTss1brCr7L0QdxDGJUEQFCVU8", + "R5QpDhLB9PFniQTXlgA852Luc6aXd+4LwZMDO5sV56HdU9YZVtz5JvNuhBXuzpy0WeJC+4qbdne3rQUe", + "ogy9/BmtkSslDOIuGmvLB9FxTlJyFRISSeDJ9fKAN/sNnk36iQwnozajXIKd/MZiU6Mwk4onbu2Pj9Aa", + "FFuYEKbXQqv6Y9BkU8FnNDKFSAuiznhsqLrZQNDr+l21UpFXynDGhRncg+gwbQ6kySeaVsWCCV0I9oMR", + "ZRgGtxKluLqnTEKV7g9TSGso9o7jnODHEWYtvzVn7GhO1EaOI6Li3EDjrf845h7zMVcOTHVnWuW0a1cq", + "sl2sassQ0rsAzM3jmO/Xbf3h2wmvpPJRRlZa1/ksN0ib3ObfFgv27+98uG93+YdHHI7/kjjju+QqhwZ0", + "iz6GecVDHKOIzEjMU6giad4NOkEm4mA/mCqV7m9sxPq9KZdq/2n/aT/48vHL/w0AAP//ZNbJHlB0AQA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/otel/README.md b/lib/otel/README.md index 04f9113c..ed4613b6 100644 --- a/lib/otel/README.md +++ b/lib/otel/README.md @@ -70,6 +70,17 @@ This keeps pull and push views aligned because both are sourced from the same OT | `hypeman_instances_restore_duration_seconds` | histogram | status, hypervisor, algorithm, level | Restore time | | `hypeman_instances_standby_duration_seconds` | histogram | status, hypervisor, algorithm, level | Standby time | | `hypeman_instances_state_transitions_total` | counter | from, to | State transitions | +| `hypeman_snapshot_compression_jobs_total` | counter | hypervisor, algorithm, source, result | Snapshot compression job outcomes (`success`, `skipped`, `canceled`, `failed`) | +| `hypeman_snapshot_compression_duration_seconds` | histogram | hypervisor, algorithm, source, result | Active compression execution time only | +| `hypeman_snapshot_compression_wait_duration_seconds` | histogram | hypervisor, algorithm, source, outcome | Standby compression delay wait time before start or skip | +| `hypeman_snapshot_compression_active_total` | gauge | hypervisor, algorithm, source | Currently running compression jobs | +| `hypeman_snapshot_compression_pending_total` | gauge | hypervisor, algorithm, source | Delayed standby compression jobs waiting to start | +| `hypeman_snapshot_compression_saved_bytes` | histogram | hypervisor, algorithm, source | Bytes saved by compression | +| `hypeman_snapshot_compression_ratio` | histogram | hypervisor, algorithm, source | Compressed-to-raw size ratio | +| `hypeman_snapshot_codec_fallbacks_total` | counter | algorithm, operation, reason | Native codec fallback count | +| `hypeman_snapshot_restore_memory_prepare_total` | counter | restore_source, result, hypervisor | Restore memory preparation outcomes | +| `hypeman_snapshot_restore_memory_prepare_duration_seconds` | histogram | restore_source, result, hypervisor | Restore memory preparation time | +| `hypeman_snapshot_compression_preemptions_total` | counter | hypervisor, algorithm, source, operation | Foreground operations that interrupted active compression | ### Network | Metric | Type | Labels | Description | diff --git a/lib/snapshot/README.md b/lib/snapshot/README.md index 82cd4300..5d7c22cb 100644 --- a/lib/snapshot/README.md +++ b/lib/snapshot/README.md @@ -42,6 +42,7 @@ Snapshot memory compression is optional and is **off by default**. ### When compression runs - A standby operation can request compression explicitly. +- A standby operation can also set a standby-only `compression_delay` so raw memory remains on disk for a grace period before background compression starts. - A snapshot create request can request compression explicitly. - If the request does not specify compression, Hypeman falls back to: - the instance's `snapshot_policy` @@ -56,6 +57,9 @@ Compression runs **asynchronously after the snapshot is already durable on disk* - This keeps the standby path fast. - Standby can return successfully while compression is still running in the background. +- For standby only, Hypeman can wait for a configured grace period before starting that background compression. +- During that delay window, the raw memory snapshot remains in place so resume can skip decompression entirely. +- If Hypeman restarts during that delay window, it rebuilds the pending standby compression job from instance metadata and either resumes waiting for the remaining delay or starts compression immediately once the `not_before` time has passed. - That means a later restore can arrive before compression has finished. - While compression is running, the snapshot remains valid and reports `compression_state=compressing`. - Once finished, the snapshot reports `compression_state=compressed` and exposes compressed/uncompressed size metadata. diff --git a/openapi.yaml b/openapi.yaml index b1884bac..e6446965 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -420,7 +420,7 @@ components: description: Hypervisor to use for this instance. Defaults to server configuration. example: cloud-hypervisor snapshot_policy: - description: Snapshot compression policy for this instance. Controls compression settings applied when creating snapshots or entering standby. + description: Snapshot policy for this instance. Controls compression settings applied when creating snapshots or entering standby, plus any default standby-only compression delay. $ref: "#/components/schemas/SnapshotPolicy" auto_standby: $ref: "#/components/schemas/AutoStandbyPolicy" @@ -587,6 +587,10 @@ components: properties: compression: $ref: "#/components/schemas/SnapshotCompressionConfig" + standby_compression_delay: + type: string + description: Delay before standby snapshot compression begins, expressed as a Go duration like "30s" or "5m". Applies only to standby compression and defaults to immediate start when omitted. + example: "2m" CreateSnapshotRequest: type: object @@ -610,7 +614,12 @@ components: type: object properties: compression: + description: Compression settings for standby snapshot memory. Overrides instance defaults. $ref: "#/components/schemas/SnapshotCompressionConfig" + compression_delay: + type: string + description: Delay before standby snapshot compression begins, expressed as a Go duration like "30s" or "5m". Overrides the instance default for this standby operation only. + example: "45s" RestoreSnapshotRequest: type: object diff --git a/skills/test-agent/agents/test-agent/NOTES.md b/skills/test-agent/agents/test-agent/NOTES.md index 200c6905..92429048 100644 --- a/skills/test-agent/agents/test-agent/NOTES.md +++ b/skills/test-agent/agents/test-agent/NOTES.md @@ -218,6 +218,82 @@ - `TestVolumeMultiAttachReadOnly` - `exec-agent not ready for instance ... within 15s (last state: Initializing)` - `TestVolumeFromArchive` + +## 2026-04-06 - PR #184 standby compression delay branch (`codex/standby-compression-delay`) + +### CI red signature +- Linux `test` job on PR [#184](https://github.com/kernel/hypeman/pull/184) failed while the other checks passed. +- Observed failures from the GitHub Actions log: + - `TestQEMUStandbyRestoreCompressionScenarios` + - `TestQEMUStandbyAndRestore` + - `TestBasicEndToEnd` + - `TestForkCloudHypervisorFromRunningNetwork` +- Failure shapes were integration stalls, not deterministic assertion failures: + - `instance ... did not reach Running within 20s (last state: Initializing)` + - `rpc error: code = DeadlineExceeded desc = stream terminated by RST_STREAM with error code: CANCEL` + +### Investigation +- Initial stopgap of removing `t.Parallel()` from the new restart-recovery tests was rejected; that was the wrong direction and was not kept. +- Reproduced the branch on `deft-kernel-dev` using the CI-like Linux/root flow with correct prewarm env: + - `go mod download` + - `make oapi-generate` + - `make build` + - `go run ./cmd/test-prewarm` + - `sudo env ... go test -count=1 -tags containers_image_openpgp -timeout=20m ./...` +- Tight loop on the exact CI-failing tests did not reproduce a flake once the command shape matched CI and prewarm settings were correct. + +### Root cause and fix +- The new standby compression recovery tests were unit-style tests but used `setupTestManager`, which pulls in much heavier integration-style manager setup than needed. +- That extra setup was unnecessary for these tests and added avoidable load to an already heavy `lib/instances` package. +- Fix: + - Added a lightweight `newSnapshotCompressionTestManager` helper in `lib/instances/snapshot_compression_test.go` + - Moved the new delayed-job and restart-recovery tests to that lightweight fixture + - Restored `t.Parallel()` on the new recovery tests and subtests +- This keeps coverage and parallelism intact while removing needless setup cost. + +### Validation +- Targeted stress loop after the fixture change: + - `go test -count=20 -run '^(TestRecoverPendingStandbyCompressionJobs|TestStartCompressionJobDelayedCancellationRecordsSkipped)$' ./lib/instances` + - Result: pass +- Deft full fresh-cache CI-like runs after the fix: + - Run 1: pass (`lib/instances` 193.279s) + - Run 2: pass (`lib/instances` 261.633s) + - Run 3: pass (`lib/instances` 173.573s) + +## 2026-04-07 - PR #184 follow-up CI round on `codex/standby-compression-delay` + +### Initial CI red signature +- Linux `test` job failed on `TestDockerForwardChainRestored`. +- Failure: + - `ensureDockerForwardJump should have restored the DOCKER-FORWARD jump` + - raw `iptables -C FORWARD -j DOCKER-FORWARD` exited non-zero in the test after re-initialization. + +### Root cause and fix +- The Docker-forward recovery path and the test both used plain `iptables` invocations with no wait for the xtables lock. +- Under parallel CI activity, a transient lock holder can cause checks/deletes/inserts to fail immediately and make the test observe a missing rule even though the recovery logic is otherwise correct. +- Fix: + - Added a small `newIPTablesCommand` helper in `lib/network/bridge_linux.go` that uses `iptables -w 5 ...` with the existing `CAP_NET_ADMIN` setup. + - Switched the bridge NAT/FORWARD rule management and `ensureDockerForwardJump` commands to that helper. + - Updated `TestDockerForwardChainRestored` in `lib/instances/network_test.go` to use `iptables -w 5` for its direct host-global mutations/checks. + +### Secondary flake surfaced during Deft reruns +- A subsequent Deft full-suite rerun exposed a post-restore guest exec race in `TestCloudHypervisorStandbyRestoreCompressionScenarios`: + - `receive response (stdout=0, stderr=0): rpc error: code = DeadlineExceeded desc = stream terminated by RST_STREAM with error code: CANCEL` +- The compression integration harness was only waiting for the exec agent socket and then issuing marker reads/writes immediately after restore. +- Fix: + - Added a no-op post-restore guest exec readiness probe in `waitForRunningAndExecReady`. + - Added a small retry wrapper for the compression integration test’s guest marker read/write commands so transient post-restore transport resets do not fail the scenario immediately. + +### Validation +- Deft targeted loop: + - `go test -count=20 -run '^TestDockerForwardChainRestored$' -v ./lib/instances` + - Result: pass +- Deft targeted loop: + - `go test -count=10 -run '^TestCloudHypervisorStandbyRestoreCompressionScenarios$' -tags containers_image_openpgp -timeout=30m ./lib/instances` + - Result: pass +- Local sanity: + - `go test ./lib/instances -count=1` + - Result: pass (`117.538s`) - `exec-agent not ready for instance ... within 15s (last state: Initializing)` ### Additional flakes reproduced during Deft full-suite verification diff --git a/stainless.yaml b/stainless.yaml index 8f9645cd..de1fd3e3 100644 --- a/stainless.yaml +++ b/stainless.yaml @@ -75,6 +75,7 @@ resources: auto_standby_policy: "#/components/schemas/AutoStandbyPolicy" auto_standby_status: "#/components/schemas/AutoStandbyStatus" snapshot_policy: "#/components/schemas/SnapshotPolicy" + standby_instance_request: "#/components/schemas/StandbyInstanceRequest" snapshot_schedule: "#/components/schemas/SnapshotSchedule" snapshot_schedule_retention: "#/components/schemas/SnapshotScheduleRetention" set_snapshot_schedule_request: "#/components/schemas/SetSnapshotScheduleRequest"