From 18e42bff79e91185d5bc58355a46d1ad37bc1d5c Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Thu, 9 Apr 2026 19:35:42 +0000 Subject: [PATCH 1/2] Add transparent ReadWriteMany (RWX) volume support via NFS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Volumes can now be attached read-write to multiple instances simultaneously. When a second rw attachment is requested, the volume is automatically loop-mounted on the host and exported via NFS. Subsequent instances mount the volume over NFS instead of as a block device, enabling shared read-write access without filesystem corruption. Key changes: - New nfsManager handles loop mount, /etc/exports, and exportfs lifecycle - AttachVolume transparently upgrades to NFS on concurrent rw attachments - DetachVolume tears down NFS when the last NFS consumer detaches - Guest init gains an "nfs" mount mode (NFSv4, hard mount) - Config disk skips device slot allocation for NFS-served volumes - Instance creation excludes NFS volumes from the hypervisor disk list - NFS host IP derived from network bridge gateway at startup NFS is purely an internal implementation detail — no public API changes. The access mode is determined per-attachment by the existing readonly field, not by a per-volume property. Co-Authored-By: Claude Opus 4.6 --- cmd/api/main.go | 15 +++ lib/builds/manager_test.go | 4 + lib/instances/configdisk.go | 15 +++ lib/instances/create.go | 9 +- lib/paths/paths.go | 5 + lib/system/init/volumes.go | 20 +++- lib/vmconfig/config.go | 4 +- lib/volumes/manager.go | 137 ++++++++++++++++++++++++--- lib/volumes/manager_test.go | 183 +++++++++++++++++++++++++++++++++++- lib/volumes/nfs.go | 157 +++++++++++++++++++++++++++++++ lib/volumes/storage.go | 8 ++ lib/volumes/types.go | 8 ++ 12 files changed, 546 insertions(+), 19 deletions(-) create mode 100644 lib/volumes/nfs.go diff --git a/cmd/api/main.go b/cmd/api/main.go index 61ee9b0b..98f34db2 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -29,6 +29,7 @@ import ( "github.com/kernel/hypeman/lib/imageretention" "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/instances" + "github.com/kernel/hypeman/lib/network" loglib "github.com/kernel/hypeman/lib/logger" mw "github.com/kernel/hypeman/lib/middleware" "github.com/kernel/hypeman/lib/oapi" @@ -37,6 +38,7 @@ import ( "github.com/kernel/hypeman/lib/registry" "github.com/kernel/hypeman/lib/scopes" "github.com/kernel/hypeman/lib/vmm" + "github.com/kernel/hypeman/lib/volumes" nethttpmiddleware "github.com/oapi-codegen/nethttp-middleware" "github.com/riandyrn/otelchi" "go.opentelemetry.io/otel/metric" @@ -265,6 +267,19 @@ func run() error { return fmt.Errorf("initialize network manager: %w", err) } + // Configure NFS host IP for ReadWriteMany volume support. + // The gateway IP is the host's address on the VM bridge — VMs can reach NFS at this address. + if nfsSetter, ok := app.VolumeManager.(volumes.NFSHostSetter); ok { + gateway := app.Config.Network.SubnetGateway + if gateway == "" { + gateway, _ = network.DeriveGateway(app.Config.Network.SubnetCIDR) + } + if gateway != "" { + nfsSetter.SetNFSHost(gateway) + logger.Info("NFS host configured for ReadWriteMany volumes", "host", gateway) + } + } + // Set up HTB qdisc on bridge for network fair sharing networkCapacity := app.ResourceManager.NetworkCapacity() if err := app.NetworkManager.SetupHTB(app.Ctx, networkCapacity); err != nil { diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go index 3b6a3e34..19b6b37f 100644 --- a/lib/builds/manager_test.go +++ b/lib/builds/manager_test.go @@ -267,6 +267,10 @@ func (m *mockVolumeManager) TotalVolumeBytes(ctx context.Context) (int64, error) return 0, nil } +func (m *mockVolumeManager) GetVolumeNFSInfo(ctx context.Context, id string) (*volumes.NFSInfo, error) { + return nil, nil +} + // mockSecretProvider implements SecretProvider for testing type mockSecretProvider struct{} diff --git a/lib/instances/configdisk.go b/lib/instances/configdisk.go index f85fa7af..a3f6eccc 100644 --- a/lib/instances/configdisk.go +++ b/lib/instances/configdisk.go @@ -99,8 +99,23 @@ func (m *manager) buildGuestConfig(ctx context.Context, inst *Instance, imageInf // Volume mounts // Volumes are attached as /dev/vdd, /dev/vde, etc. (after vda=rootfs, vdb=overlay, vdc=config) + // NFS-served volumes do not consume a device slot — they mount via network. deviceIdx := 0 for _, vol := range inst.Volumes { + // Check if this volume is NFS-served (ReadWriteMany) + nfsInfo, _ := m.volumeManager.GetVolumeNFSInfo(ctx, vol.VolumeID) + if nfsInfo != nil && !vol.Readonly { + // NFS mount — no block device needed + mount := vmconfig.VolumeMount{ + Path: vol.MountPath, + Mode: "nfs", + NFSHost: nfsInfo.Host, + NFSExport: nfsInfo.ExportPath, + } + cfg.VolumeMounts = append(cfg.VolumeMounts, mount) + continue + } + device := fmt.Sprintf("/dev/vd%c", 'd'+deviceIdx) mount := vmconfig.VolumeMount{ Device: device, diff --git a/lib/instances/create.go b/lib/instances/create.go index c6d208ac..69ae870c 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -765,8 +765,15 @@ func (m *manager) buildHypervisorConfig(ctx context.Context, inst *Instance, ima {Path: m.paths.InstanceConfigDisk(inst.Id), Readonly: true, IOBps: ioBps, IOBurstBps: burstBps}, } - // Add attached volumes as additional disks + // Add attached volumes as additional disks. + // NFS-served volumes are mounted via the network and do NOT get a block device. for _, volAttach := range inst.Volumes { + // Skip NFS-served volumes — they're mounted via NFS in the guest, not as block devices + nfsInfo, _ := m.volumeManager.GetVolumeNFSInfo(ctx, volAttach.VolumeID) + if nfsInfo != nil && !volAttach.Readonly { + continue + } + volumePath := m.volumeManager.GetVolumePath(volAttach.VolumeID) if volAttach.Overlay { // Base volume is always read-only when overlay is enabled diff --git a/lib/paths/paths.go b/lib/paths/paths.go index adc070c4..5aeb0d0a 100644 --- a/lib/paths/paths.go +++ b/lib/paths/paths.go @@ -299,6 +299,11 @@ func (p *Paths) VolumeMetadata(id string) string { return filepath.Join(p.VolumeDir(id), "metadata.json") } +// VolumeNFSMount returns the directory where a volume's data.raw is loop-mounted for NFS export. +func (p *Paths) VolumeNFSMount(id string) string { + return filepath.Join(p.VolumeDir(id), "nfs_mount") +} + // Caddy path methods // CaddyDir returns the caddy data directory. diff --git a/lib/system/init/volumes.go b/lib/system/init/volumes.go index d39b9dbe..46751d9c 100644 --- a/lib/system/init/volumes.go +++ b/lib/system/init/volumes.go @@ -10,7 +10,7 @@ import ( ) // mountVolumes mounts attached volumes according to the configuration. -// Supports three modes: ro (read-only), rw (read-write), and overlay. +// Supports four modes: ro (read-only), rw (read-write), overlay, and nfs. func mountVolumes(log *Logger, cfg *vmconfig.Config) error { log.Info("hypeman-init:volumes", "mounting volumes") @@ -32,6 +32,10 @@ func mountVolumes(log *Logger, cfg *vmconfig.Config) error { if err := mountVolumeReadOnly(log, vol, mountPath); err != nil { log.Error("hypeman-init:volumes", fmt.Sprintf("mount ro %s failed", vol.Path), err) } + case "nfs": + if err := mountVolumeNFS(log, vol, mountPath); err != nil { + log.Error("hypeman-init:volumes", fmt.Sprintf("mount nfs %s failed", vol.Path), err) + } default: // "rw" if err := mountVolumeReadWrite(log, vol, mountPath); err != nil { log.Error("hypeman-init:volumes", fmt.Sprintf("mount rw %s failed", vol.Path), err) @@ -110,3 +114,17 @@ func mountVolumeReadWrite(log *Logger, vol vmconfig.VolumeMount, mountPath strin log.Info("hypeman-init:volumes", fmt.Sprintf("mounted %s at %s (rw)", vol.Device, vol.Path)) return nil } + +// mountVolumeNFS mounts a volume via NFS from the host. +// Used for ReadWriteMany volumes where multiple VMs need concurrent rw access. +func mountVolumeNFS(log *Logger, vol vmconfig.VolumeMount, mountPath string) error { + nfsSource := fmt.Sprintf("%s:%s", vol.NFSHost, vol.NFSExport) + // Use NFSv4 with tcp, hard mount for data safety, relatively short timeo for responsiveness + cmd := exec.Command("/bin/mount", "-t", "nfs", "-o", "vers=4,tcp,hard,timeo=100,retrans=3", nfsSource, mountPath) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("mount nfs %s: %s: %s", nfsSource, err, output) + } + + log.Info("hypeman-init:volumes", fmt.Sprintf("mounted %s at %s (nfs)", nfsSource, vol.Path)) + return nil +} diff --git a/lib/vmconfig/config.go b/lib/vmconfig/config.go index 9d073077..32feba9d 100644 --- a/lib/vmconfig/config.go +++ b/lib/vmconfig/config.go @@ -38,8 +38,10 @@ type Config struct { type VolumeMount struct { Device string `json:"device"` Path string `json:"path"` - Mode string `json:"mode"` // "ro", "rw", or "overlay" + Mode string `json:"mode"` // "ro", "rw", "overlay", or "nfs" OverlayDevice string `json:"overlay_device,omitempty"` + NFSHost string `json:"nfs_host,omitempty"` // Host IP for NFS mount (mode=nfs) + NFSExport string `json:"nfs_export,omitempty"` // Export path on host (mode=nfs) } // EgressProxyConfig configures guest-side trust and proxy endpoint wiring. diff --git a/lib/volumes/manager.go b/lib/volumes/manager.go index f33d0b86..27ddafbc 100644 --- a/lib/volumes/manager.go +++ b/lib/volumes/manager.go @@ -25,16 +25,22 @@ type Manager interface { DeleteVolume(ctx context.Context, id string) error // Attachment operations (called by instance manager) - // Multi-attach rules: + // Multi-attach rules (dynamic based on current state): // - If no attachments: allow any mode (rw or ro) - // - If existing attachment is rw: reject all new attachments // - If existing attachments are ro: only allow new ro attachments + // - Multiple rw attachments (ReadWriteMany): internally backed by NFS, + // transparent to the caller. NFS is set up automatically when a second + // rw attachment is requested. AttachVolume(ctx context.Context, id string, req AttachVolumeRequest) error DetachVolume(ctx context.Context, volumeID string, instanceID string) error // GetVolumePath returns the path to the volume data file GetVolumePath(id string) string + // GetVolumeNFSInfo returns NFS serving details if the volume is NFS-served, nil otherwise. + // Used by the instance manager to decide whether to pass a block device or NFS mount info. + GetVolumeNFSInfo(ctx context.Context, id string) (*NFSInfo, error) + // TotalVolumeBytes returns the total size of all volumes. // Used by the resource manager for disk capacity tracking. TotalVolumeBytes(ctx context.Context) (int64, error) @@ -48,16 +54,20 @@ type manager struct { totalVolumeBytes int64 totalVolumeBytesReady bool metrics *Metrics + nfs *nfsManager + nfsHost string // Host IP for NFS mounts (VM bridge gateway) } // NewManager creates a new volumes manager. // maxTotalVolumeStorage is the maximum total volume storage in bytes (0 = unlimited). +// nfsHost is the host IP that VMs use to reach the NFS server (typically the bridge gateway). // If meter is nil, metrics are disabled. func NewManager(p *paths.Paths, maxTotalVolumeStorage int64, meter metric.Meter) Manager { m := &manager{ paths: p, maxTotalVolumeStorage: maxTotalVolumeStorage, volumeLocks: sync.Map{}, + nfs: newNFSManager(p), } // Initialize metrics if meter is provided @@ -71,6 +81,17 @@ func NewManager(p *paths.Paths, maxTotalVolumeStorage int64, meter metric.Meter) return m } +// NFSHostSetter allows setting the NFS host IP after initialization. +type NFSHostSetter interface { + SetNFSHost(host string) +} + +// SetNFSHost sets the host IP used for NFS mounts. Called after network initialization +// when the bridge gateway IP is known. +func (m *manager) SetNFSHost(host string) { + m.nfsHost = host +} + // getVolumeLock returns or creates a lock for a specific volume func (m *manager) getVolumeLock(id string) *sync.RWMutex { lock, _ := m.volumeLocks.LoadOrStore(id, &sync.RWMutex{}) @@ -377,11 +398,14 @@ func (m *manager) DeleteVolume(ctx context.Context, id string) error { return nil } -// AttachVolume marks a volume as attached to an instance +// AttachVolume marks a volume as attached to an instance. // Multi-attach rules (dynamic based on current state): -// - If no attachments: allow any mode (rw or ro) -// - If existing attachment is rw: reject all new attachments -// - If existing attachments are ro: only allow new ro attachments +// - If no attachments: allow any mode (rw or ro) via block device +// - If existing attachments are all ro: only allow new ro attachments +// - If existing attachment is rw (block device) and new is rw: enable NFS +// for ReadWriteMany. The volume is loop-mounted on the host and exported +// via NFS. The new attachment (and any subsequent ones) use NFS. +// - If volume is already NFS-served: additional rw attachments use NFS func (m *manager) AttachVolume(ctx context.Context, id string, req AttachVolumeRequest) error { lock := m.getVolumeLock(id) lock.Lock() @@ -399,18 +423,53 @@ func (m *manager) AttachVolume(ctx context.Context, id string, req AttachVolumeR } } - // Apply multi-attach rules + useNFS := false + if len(meta.Attachments) > 0 { - // Check if any existing attachment is read-write + hasRW := false + allRO := true for _, att := range meta.Attachments { if !att.Readonly { - return fmt.Errorf("volume has exclusive read-write attachment to instance %s", att.InstanceID) + hasRW = true + allRO = false } } - // Existing attachments are all read-only, new attachment must also be read-only - if !req.Readonly { + + if allRO && !req.Readonly { + // Existing attachments are all ro, new is rw → conflict return fmt.Errorf("cannot attach read-write: volume has existing read-only attachments") } + + if hasRW && req.Readonly { + // Existing has rw, new is ro → conflict (rw is exclusive or NFS-only) + return fmt.Errorf("cannot attach read-only: volume has existing read-write attachment") + } + + if hasRW && !req.Readonly { + // ReadWriteMany scenario: both existing and new want rw. + // Transparently enable NFS serving. + if m.nfsHost == "" { + return fmt.Errorf("cannot attach read-write to multiple instances: NFS host not configured (networking required)") + } + + // Start NFS serving if not already active + if meta.NFS == nil { + exportPath, err := m.nfs.startServing(id) + if err != nil { + return fmt.Errorf("start nfs serving for ReadWriteMany: %w", err) + } + meta.NFS = &storedNFSInfo{ + Host: m.nfsHost, + ExportPath: exportPath, + } + } + useNFS = true + } + } + + // If volume is already NFS-served, new rw attachments use NFS + if meta.NFS != nil && !req.Readonly { + useNFS = true } // Add new attachment @@ -418,12 +477,14 @@ func (m *manager) AttachVolume(ctx context.Context, id string, req AttachVolumeR InstanceID: req.InstanceID, MountPath: req.MountPath, Readonly: req.Readonly, + NFS: useNFS, }) return saveMetadata(m.paths, meta) } -// DetachVolume removes the attachment for a specific instance +// DetachVolume removes the attachment for a specific instance. +// When the last NFS-using attachment is removed, NFS serving is stopped. func (m *manager) DetachVolume(ctx context.Context, volumeID string, instanceID string) error { lock := m.getVolumeLock(volumeID) lock.Lock() @@ -450,6 +511,27 @@ func (m *manager) DetachVolume(ctx context.Context, volumeID string, instanceID } meta.Attachments = newAttachments + + // Check if NFS serving should be stopped. + // Stop when there are no remaining NFS-based rw attachments. + if meta.NFS != nil { + hasNFSAttachments := false + for _, att := range meta.Attachments { + if att.NFS { + hasNFSAttachments = true + break + } + } + if !hasNFSAttachments { + // No more NFS consumers — tear down NFS serving + if err := m.nfs.stopServing(volumeID); err != nil { + // Log but don't fail the detach + fmt.Fprintf(os.Stderr, "warning: failed to stop NFS serving for volume %s: %v\n", volumeID, err) + } + meta.NFS = nil + } + } + return saveMetadata(m.paths, meta) } @@ -458,6 +540,25 @@ func (m *manager) GetVolumePath(id string) string { return m.paths.VolumeData(id) } +// GetVolumeNFSInfo returns NFS serving details if the volume is NFS-served, nil otherwise. +func (m *manager) GetVolumeNFSInfo(ctx context.Context, id string) (*NFSInfo, error) { + lock := m.getVolumeLock(id) + lock.RLock() + defer lock.RUnlock() + + meta, err := loadMetadata(m.paths, id) + if err != nil { + return nil, err + } + if meta.NFS == nil { + return nil, nil + } + return &NFSInfo{ + Host: meta.NFS.Host, + ExportPath: meta.NFS.ExportPath, + }, nil +} + // TotalVolumeBytes returns the total size of all volumes. func (m *manager) TotalVolumeBytes(ctx context.Context) (int64, error) { return m.getTotalVolumeBytes(ctx) @@ -474,10 +575,11 @@ func (m *manager) metadataToVolume(meta *storedMetadata) *Volume { InstanceID: att.InstanceID, MountPath: att.MountPath, Readonly: att.Readonly, + NFS: att.NFS, } } - return &Volume{ + vol := &Volume{ Id: meta.Id, Name: meta.Name, SizeGb: meta.SizeGb, @@ -485,4 +587,13 @@ func (m *manager) metadataToVolume(meta *storedMetadata) *Volume { CreatedAt: createdAt, Attachments: attachments, } + + if meta.NFS != nil { + vol.NFS = &NFSInfo{ + Host: meta.NFS.Host, + ExportPath: meta.NFS.ExportPath, + } + } + + return vol } diff --git a/lib/volumes/manager_test.go b/lib/volumes/manager_test.go index f7b05aaf..0220e20b 100644 --- a/lib/volumes/manager_test.go +++ b/lib/volumes/manager_test.go @@ -108,14 +108,14 @@ func TestMultiAttach_RejectSecondAttachWhenRW(t *testing.T) { }) require.NoError(t, err) - // Second attachment (either RO or RW) should fail + // Second attachment as RO should fail when existing is RW err = manager.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ InstanceID: "instance-2", MountPath: "/data", - Readonly: true, // Even RO should fail when existing is RW + Readonly: true, // RO should fail when existing is RW }) assert.Error(t, err) - assert.Contains(t, err.Error(), "exclusive read-write attachment") + assert.Contains(t, err.Error(), "existing read-write attachment") } func TestMultiAttach_AllowMultipleRO(t *testing.T) { @@ -390,6 +390,183 @@ func TestMultiAttach_ConcurrentRWConflict(t *testing.T) { assert.False(t, vol.Attachments[0].Readonly, "Attachment should be read-write") } +func TestRWX_RejectWithoutNFSHost(t *testing.T) { + mgr, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + vol, err := mgr.CreateVolume(ctx, CreateVolumeRequest{ + Name: "rwx-vol", + SizeGb: 1, + }) + require.NoError(t, err) + + // First rw attachment succeeds (block device, no NFS needed) + err = mgr.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-1", + MountPath: "/data", + Readonly: false, + }) + require.NoError(t, err) + + // Second rw attachment should fail because NFS host is not configured + err = mgr.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-2", + MountPath: "/data", + Readonly: false, + }) + assert.Error(t, err) + assert.Contains(t, err.Error(), "NFS host not configured") +} + +func TestRWX_NFSInfoNilWhenNotServed(t *testing.T) { + mgr, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + vol, err := mgr.CreateVolume(ctx, CreateVolumeRequest{ + Name: "no-nfs-vol", + SizeGb: 1, + }) + require.NoError(t, err) + + // Single rw attachment — no NFS + err = mgr.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "instance-1", + MountPath: "/data", + Readonly: false, + }) + require.NoError(t, err) + + nfsInfo, err := mgr.GetVolumeNFSInfo(ctx, vol.Id) + require.NoError(t, err) + assert.Nil(t, nfsInfo, "NFS info should be nil for single rw attachment") +} + +func TestRWX_NFSMetadataPersistence(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "volume-nfs-persist-*") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + p := paths.New(tmpDir) + require.NoError(t, os.MkdirAll(p.VolumeDir("vol-nfs-1"), 0755)) + + // Save metadata with NFS info + meta := &storedMetadata{ + Id: "vol-nfs-1", + Name: "nfs-vol", + SizeGb: 5, + NFS: &storedNFSInfo{ + Host: "10.100.0.1", + ExportPath: "/data/volumes/vol-nfs-1/nfs_mount", + }, + Attachments: []storedAttachment{ + {InstanceID: "inst-1", MountPath: "/data", Readonly: false, NFS: false}, + {InstanceID: "inst-2", MountPath: "/data", Readonly: false, NFS: true}, + }, + } + require.NoError(t, saveMetadata(p, meta)) + + // Reload and verify + loaded, err := loadMetadata(p, "vol-nfs-1") + require.NoError(t, err) + require.NotNil(t, loaded.NFS) + assert.Equal(t, "10.100.0.1", loaded.NFS.Host) + assert.Equal(t, "/data/volumes/vol-nfs-1/nfs_mount", loaded.NFS.ExportPath) + require.Len(t, loaded.Attachments, 2) + assert.False(t, loaded.Attachments[0].NFS) + assert.True(t, loaded.Attachments[1].NFS) + + // Verify domain conversion + vol := (&manager{}).metadataToVolume(loaded) + require.NotNil(t, vol.NFS) + assert.Equal(t, "10.100.0.1", vol.NFS.Host) + assert.True(t, vol.Attachments[1].NFS) +} + +func TestRWX_DetachClearsNFSWhenNoConsumers(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "volume-nfs-detach-*") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + p := paths.New(tmpDir) + require.NoError(t, os.MkdirAll(p.VolumeDir("vol-detach-1"), 0755)) + + // Set up metadata with NFS and two attachments (one NFS, one not) + meta := &storedMetadata{ + Id: "vol-detach-1", + Name: "detach-vol", + SizeGb: 5, + NFS: &storedNFSInfo{ + Host: "10.100.0.1", + ExportPath: "/data/volumes/vol-detach-1/nfs_mount", + }, + Attachments: []storedAttachment{ + {InstanceID: "inst-1", MountPath: "/data", Readonly: false, NFS: false}, + {InstanceID: "inst-2", MountPath: "/data", Readonly: false, NFS: true}, + }, + } + require.NoError(t, saveMetadata(p, meta)) + + mgr := &manager{ + paths: p, + nfs: newNFSManager(p), + } + ctx := context.Background() + + // Detach the NFS consumer + err = mgr.DetachVolume(ctx, "vol-detach-1", "inst-2") + require.NoError(t, err) + + // Verify NFS info is cleared (no NFS consumers remain) + loaded, err := loadMetadata(p, "vol-detach-1") + require.NoError(t, err) + assert.Nil(t, loaded.NFS, "NFS info should be cleared when no NFS consumers remain") + require.Len(t, loaded.Attachments, 1) +} + +func TestRWX_DetachKeepsNFSWithRemainingConsumers(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "volume-nfs-keep-*") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + p := paths.New(tmpDir) + require.NoError(t, os.MkdirAll(p.VolumeDir("vol-keep-1"), 0755)) + + // Set up metadata with NFS and three attachments (two NFS) + meta := &storedMetadata{ + Id: "vol-keep-1", + Name: "keep-vol", + SizeGb: 5, + NFS: &storedNFSInfo{ + Host: "10.100.0.1", + ExportPath: "/data/volumes/vol-keep-1/nfs_mount", + }, + Attachments: []storedAttachment{ + {InstanceID: "inst-1", MountPath: "/data", Readonly: false, NFS: false}, + {InstanceID: "inst-2", MountPath: "/data", Readonly: false, NFS: true}, + {InstanceID: "inst-3", MountPath: "/data", Readonly: false, NFS: true}, + }, + } + require.NoError(t, saveMetadata(p, meta)) + + mgr := &manager{ + paths: p, + nfs: newNFSManager(p), + } + ctx := context.Background() + + // Detach one NFS consumer + err = mgr.DetachVolume(ctx, "vol-keep-1", "inst-2") + require.NoError(t, err) + + // Verify NFS info is still present (inst-3 still uses NFS) + loaded, err := loadMetadata(p, "vol-keep-1") + require.NoError(t, err) + assert.NotNil(t, loaded.NFS, "NFS info should be kept when NFS consumers remain") + require.Len(t, loaded.Attachments, 2) +} + func TestCreateVolume_MetadataRoundTrip(t *testing.T) { tmpDir, err := os.MkdirTemp("", "volume-metadata-*") require.NoError(t, err) diff --git a/lib/volumes/nfs.go b/lib/volumes/nfs.go new file mode 100644 index 00000000..e8656e35 --- /dev/null +++ b/lib/volumes/nfs.go @@ -0,0 +1,157 @@ +package volumes + +import ( + "fmt" + "os" + "os/exec" + "strings" + "sync" + + "github.com/kernel/hypeman/lib/paths" +) + +// nfsManager handles NFS export lifecycle for volumes that need ReadWriteMany access. +// NFS is an internal implementation detail — callers never see NFS configuration. +type nfsManager struct { + paths *paths.Paths + mu sync.Mutex + exports map[string]bool // volumeID -> actively exported +} + +func newNFSManager(p *paths.Paths) *nfsManager { + return &nfsManager{ + paths: p, + exports: make(map[string]bool), + } +} + +// startServing sets up NFS export for a volume: +// 1. Loop-mounts data.raw to a host directory +// 2. Adds an NFS export entry +// 3. Refreshes the NFS export table +// +// Returns the export path on the host. The caller must combine this with the +// host gateway IP to form the full NFS mount spec for the guest. +func (n *nfsManager) startServing(volumeID string) (exportPath string, err error) { + n.mu.Lock() + defer n.mu.Unlock() + + if n.exports[volumeID] { + // Already exported — return existing mount point + return n.paths.VolumeNFSMount(volumeID), nil + } + + mountDir := n.paths.VolumeNFSMount(volumeID) + dataPath := n.paths.VolumeData(volumeID) + + // Create the mount point directory + if err := os.MkdirAll(mountDir, 0755); err != nil { + return "", fmt.Errorf("create nfs mount dir: %w", err) + } + + // Loop-mount data.raw as ext4 + cmd := exec.Command("mount", "-o", "loop", "-t", "ext4", dataPath, mountDir) + if output, err := cmd.CombinedOutput(); err != nil { + return "", fmt.Errorf("loop mount %s: %s: %w", dataPath, strings.TrimSpace(string(output)), err) + } + + // Add NFS export (rw, no_root_squash for VM access, sync for data safety) + exportLine := fmt.Sprintf("%s *(rw,no_root_squash,no_subtree_check,sync,fsid=%s)\n", mountDir, volumeID) + if err := appendExport(exportLine); err != nil { + // Cleanup: unmount on failure + exec.Command("umount", mountDir).Run() + return "", fmt.Errorf("add nfs export: %w", err) + } + + // Refresh NFS exports + if err := refreshExports(); err != nil { + // Cleanup: remove export entry and unmount + removeExport(mountDir) + exec.Command("umount", mountDir).Run() + return "", fmt.Errorf("refresh nfs exports: %w", err) + } + + n.exports[volumeID] = true + return mountDir, nil +} + +// stopServing tears down NFS export and unmounts the volume. +func (n *nfsManager) stopServing(volumeID string) error { + n.mu.Lock() + defer n.mu.Unlock() + + if !n.exports[volumeID] { + return nil // Not exported, nothing to do + } + + mountDir := n.paths.VolumeNFSMount(volumeID) + + // Remove NFS export + removeExport(mountDir) + refreshExports() + + // Unmount + cmd := exec.Command("umount", mountDir) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("unmount %s: %s: %w", mountDir, strings.TrimSpace(string(output)), err) + } + + // Clean up mount directory + os.Remove(mountDir) + + delete(n.exports, volumeID) + return nil +} + +// isServing returns whether a volume is currently NFS-exported. +func (n *nfsManager) isServing(volumeID string) bool { + n.mu.Lock() + defer n.mu.Unlock() + return n.exports[volumeID] +} + +const exportsFile = "/etc/exports" + +// appendExport adds a line to /etc/exports. +func appendExport(line string) error { + f, err := os.OpenFile(exportsFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer f.Close() + _, err = f.WriteString(line) + return err +} + +// removeExport removes all lines containing mountDir from /etc/exports. +func removeExport(mountDir string) error { + data, err := os.ReadFile(exportsFile) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + + var kept []string + for _, line := range strings.Split(string(data), "\n") { + if strings.TrimSpace(line) == "" { + continue + } + if strings.Contains(line, mountDir) { + continue // Drop this export + } + kept = append(kept, line) + } + + return os.WriteFile(exportsFile, []byte(strings.Join(kept, "\n")+"\n"), 0644) +} + +// refreshExports runs exportfs -ra to apply /etc/exports changes. +func refreshExports() error { + cmd := exec.Command("exportfs", "-ra") + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("exportfs -ra: %s: %w", strings.TrimSpace(string(output)), err) + } + return nil +} diff --git a/lib/volumes/storage.go b/lib/volumes/storage.go index cd600ddb..bcec7b45 100644 --- a/lib/volumes/storage.go +++ b/lib/volumes/storage.go @@ -21,6 +21,13 @@ type storedAttachment struct { InstanceID string `json:"instance_id"` MountPath string `json:"mount_path"` Readonly bool `json:"readonly"` + NFS bool `json:"nfs,omitempty"` // True if this attachment uses NFS +} + +// storedNFSInfo represents persisted NFS serving state +type storedNFSInfo struct { + Host string `json:"host"` // Host IP for NFS mount + ExportPath string `json:"export_path"` // Export path on the host } // storedMetadata represents volume metadata that is persisted to disk @@ -31,6 +38,7 @@ type storedMetadata struct { Tags tags.Tags `json:"tags,omitempty"` CreatedAt string `json:"created_at"` // RFC3339 format Attachments []storedAttachment `json:"attachments,omitempty"` + NFS *storedNFSInfo `json:"nfs,omitempty"` // Non-nil when volume is NFS-served } // ensureVolumeDir creates the volume directory diff --git a/lib/volumes/types.go b/lib/volumes/types.go index c6648bfb..f86d77ab 100644 --- a/lib/volumes/types.go +++ b/lib/volumes/types.go @@ -11,6 +11,13 @@ type Attachment struct { InstanceID string MountPath string Readonly bool + NFS bool // True if this attachment uses NFS (internal, not exposed in API) +} + +// NFSInfo contains NFS serving details for a volume (host-internal). +type NFSInfo struct { + Host string // Host IP/address for NFS mount (gateway IP on VM bridge) + ExportPath string // Exported filesystem path on the host } // Volume represents a persistent block storage volume @@ -21,6 +28,7 @@ type Volume struct { Tags tags.Tags CreatedAt time.Time Attachments []Attachment // List of current attachments (empty if not attached) + NFS *NFSInfo // Non-nil when the volume is being served via NFS (internal) } // CreateVolumeRequest is the domain request for creating a volume From 06de65e2ae2c779a9205b77aab3434e403bde57f Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Thu, 9 Apr 2026 19:44:31 +0000 Subject: [PATCH 2/2] Add access_mode field to attach volume request Add an explicit AccessMode enum (ReadWriteOnce, ReadOnlyMany, ReadWriteMany) to the attach volume request. This replaces the implicit rw+rw=NFS behavior with an explicit opt-in model: - ReadWriteOnce: exclusive rw via block device (default, no change) - ReadOnlyMany: read-only, multiple instances (maps from readonly=true) - ReadWriteMany: shared rw via NFS (only mode that triggers NFS) The existing readonly field is deprecated but still works with identical semantics. Neither legacy path triggers NFS. When both fields are set, access_mode takes precedence. Validation rules enforce that different access modes cannot be mixed on the same volume (e.g., RWO + RWX = conflict). Changes: OpenAPI spec (AccessMode enum, deprecated readonly), regenerated oapi.go, domain types, volume manager attach logic, storage persistence, and 9 new tests (36 total pass). Co-Authored-By: Claude Opus 4.6 --- lib/oapi/oapi.go | 555 +++++++++++++++++++----------------- lib/volumes/manager.go | 126 ++++---- lib/volumes/manager_test.go | 283 +++++++++++++++++- lib/volumes/storage.go | 3 +- lib/volumes/types.go | 27 ++ openapi.yaml | 13 +- 6 files changed, 677 insertions(+), 330 deletions(-) diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 8ceb515c..7aa0155b 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -29,6 +29,13 @@ const ( BearerAuthScopes = "bearerAuth.Scopes" ) +// Defines values for AccessMode. +const ( + ReadOnlyMany AccessMode = "ReadOnlyMany" + ReadWriteMany AccessMode = "ReadWriteMany" + ReadWriteOnce AccessMode = "ReadWriteOnce" +) + // Defines values for AutoStandbyStatusReason. const ( AutoStandbyStatusReasonActiveInboundConnections AutoStandbyStatusReason = "active_inbound_connections" @@ -212,12 +219,24 @@ const ( Vmm GetInstanceLogsParamsSource = "vmm" ) +// AccessMode Volume access mode for attachment. +// - ReadWriteOnce: exclusive read-write (only one instance at a time) +// - ReadOnlyMany: read-only, multiple instances can share +// - ReadWriteMany: shared read-write via NFS, multiple instances simultaneously +type AccessMode string + // AttachVolumeRequest defines model for AttachVolumeRequest. type AttachVolumeRequest struct { + // AccessMode Volume access mode for attachment. + // - ReadWriteOnce: exclusive read-write (only one instance at a time) + // - ReadOnlyMany: read-only, multiple instances can share + // - ReadWriteMany: shared read-write via NFS, multiple instances simultaneously + AccessMode *AccessMode `json:"access_mode,omitempty"` + // MountPath Path where volume should be mounted MountPath string `json:"mount_path"` - // Readonly Mount as read-only + // Readonly Deprecated: use access_mode instead. Mount as read-only. Readonly *bool `json:"readonly,omitempty"` } @@ -15648,272 +15667,274 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+y9/XIbOZI4+CqIutkYaYakqA/LtjY6fqeWbLe2rbbOsj232/RRYBVIolUFVAMoSrTD", - "/+4DzCPOk1wgAdQniizJlmyNvTsxI7PwmchMZCby42MQ8iTljDAlg4OPgQznJMHw56FSOJy/43GWkNfk", - "z4xIpX9OBU+JUJRAo4RnTI1TrOb6XxGRoaCpopwFB8EZVnN0NSeCoAWMguScZ3GEJgRBPxIFvYBc4ySN", - "SXAQbCVMbUVY4aAXqGWqf5JKUDYLPvUCQXDEWbw000xxFqvgYIpjSXq1aU/10AhLpLv0oU8+3oTzmGAW", - "fIIR/8yoIFFw8Ht5G+/zxnzyBwmVnvwwU/xcYRZNlmc8puGyudmXlGXXMBvCmeIJVjRE0vRBKXRCEyxJ", - "hDhDOFR0QRBlE56xCL05OkMhZ4yEejA5YnwiiViQCE0FT5CaEzTnUkEbJXB4iRSexGQwYkGvdh6E6S/R", - "eij9Y07UnAjPYqlEdhQ05QKpOZWIMv01JIPygSmRkSZkewGNYjJWNCE8U01A/cKvUMzZDLblxkVJJhWa", - "4wVBH4jg6M8Mx3S6pGzWDqQJmXJB0C/LlCSYoTTGIZGIKkSZ4m43BkYFjj1KfMhFZ4wLMo6IVJRhPf44", - "5cJQRHX1r+APHKNSW1gatEdqjpXDcsYVuiQkrW4UX+HLKhh/39npPR0Oh+97AVUkMWSFr2mSJcHB/qNH", - "u496QUKZ+fd2vnrKFJkRoZdvf8FC4GVpO5JnIiTjkEZi1U7CmBKm0NHJ8etbbiDYHg7g/7eeBL1g++nO", - "YHv/Cfx7ez8ob6sB+OrKP60mvXOFVSabPMhQ09giyriEJM1d/5YlEyIQn6IwE4IwFS8RkBSJOiBdZdtD", - "31GEnE3pLBOOBH0kVwHnHEuEmWEa/Rq/KAbrRHehZmIRv2JjQRJMmYZxYxGv3SekKRRZItJLCjlTgsex", - "ZgpKkSRV0lFRT7NxhnCaxjQE1lMhqr1kKINewLI41h9rKyxOm8R0RqFBJ9BQWTok1xcpjghTROQU3gU0", - "FbbYNnEBbu9pFHyxOxeUlIX+7bI6zBPN4QUJzXbzG6ACkQkJeUKQHrp6AjvDnf3+cK8/3H+z/fhguHcw", - "fPQ/QS+YcpFgFRwEEVakrw+8yzGt5t9HBZR0Q2QbFleVB3aDGg/uhi4xliqnaiByqpZj7FnTG5oQqXCS", - "asLWaygBs42s3YD1c3CQXwng7c8CMCPXamwh5N2PDz/IdUpCfcVwR575ja3H6yE6RRjlPECjq2GMKzfy", - "9LM2IgiWesFa7tC30+9BxmSW6ruQROM0xkqPq4UUQINxQqXUXfMfIioNYfYCh+RjxtVYZIyZhoyoKy4u", - "yy3tKGOaBr1gjuV4MUuzoLfqHqgiNUxBYpxKGM+euBgTIbgIjKy5HE+5cIekL7EChCuGakBI5neWB0JB", - "L6gAIOePbi9u3fmpehcHswAuCSOmG7kaNtNceHms5nLzpa3mlIYtG6nUHTOynWWVA0QUzxiXioayE9+E", - "21gfb8IjD+s8zodDNCJM0SklwgqqBImMwbXmBkF6EEQZymSNDnJZekwWWvkZL/bGKkybQKlpCuXDK132", - "xRVTuuby488pZQ2SVvfu1UQWmAJNHpMFNVdLVRiyRzOOBF0Q4WHf+Y1qWKFphzY0rWsWwjgjmxVIsQWN", - "KO7CDiJY05h6sOfs6ASZz+jkGG3MyXV1kp3HkydB+5AMJx5c+CVLMOtrgtDLcuND2/LYL/e8Mj9Pkmw8", - "EzxLmyOfvDo9fYvgI2IgMpZHfLLjE/3SkI5xFAkipX//7mN5bcPhcHiAdw6Gw8HQt8oFYREXrSA1n/0g", - "3R5GZMWQnUBqx2+A9Ld3J8cnh+iIi5QLUILWEk4ZPOV9ldGmeio+/P85o3HUxPqJ/pmIcX6J+AB24sSo", - "k2MnJ9h+6N0p2tA8JCKTbDajbLbZBd9DrsGhrzrfJQ5LRbaNVhOVk1Jufd+GguA10+kWnSZrklpmTnKc", - "yLbRXRPNURMax1SSkLNIluegTO3vtW+mRDDmhmpM9Uz/jBIiJZ4RtAEmFVA/DDPVgs0U05hEm92E2bbN", - "/MEnpSukgt6AFn08Cbd3dr28I8EzMo7ozNrE6leU/l2jmB5HIWjt3whc5t32AVMKMm3O9xxYN0wiyJQI", - "onH8M6dLBV8Qhq328heYN/i/tgpj4Za1FG4BMM+K5p96wZ8Zycg45ZKaFTY4l/2i0QhAjaCHf83wadVZ", - "lzBKKixW0we0+AKUWMh1a2FjzRZatMGztV3e6DZ13gmsMZclSlyglUU+00KNRzrgTNkPNfMln6GYMqNx", - "aNHOnAXIVcuU/BRzYIlfCA45+JvEr9d9C+ZlfmgZTX/r5QJ4zGdlaM4JFmpCKsBsucLsQMXqWsF/ViGf", - "2l2FJRmv5iBnlDESgb3YErZpqcVYr5oBVHRJ1XhBhPTSHCzrV6qQbdE6VMzDyymNyXiO5dwa2KKIGmPh", - "WWUnHmmtYojHoI+7AUGKAP31/JfDnUf7yE7ggaG1XOoGzZ2UeuvhTVuksJjgOPbiRju63fyObmKIHwMK", - "Y2Xb3ZNjoENMw+kCe5pWT87k3PwFvFuvCu4+zQY0esX67/eeTR8BkzBaQuvrjV8GzC3Ds5hrmC5Rxuif", - "WUXAHqCTKRiI9UVBIxL1EIYPYHfQ+t+MMCI0nyosQyUhGG2QwWzQQyMtF/a1FNzHO/3hsD8cBVUxNt7r", - "G/U+xUoRoRf4//2O+x8O+/8z7D99X/w5HvTf//0vPgToKpk7qdDuc8PRfg+5xZbF9fpC14nyt+b+5eX7", - "OI456hPNJ2560kcnTcHB7DXi4SURA8q3YjoRWCy32Iyy64MYKyJVdeer235RWMA+VgCBzTSYbgiGmtID", - "aLwR8ysiQs2BY6IRT/Y0E6ZK9hDWejMwL6Rvyf9EIWaaFoxwwQUiLEJXVM0RhnZVaCXLPk5pn5qlBr0g", - "wdcvCZupeXCwv9vAc43kG/aP/vu/uZ82/48X1UUWEw+Sv+aZomyG4HP5Wc+tIX+iWXUiDrpZDGJeQtmJ", - "6bbdfIP6vBN2G1l10kaZaz1qzYRyE9mahTTfd7WylXhUh1cLIgSN3LV8dHqMNmJ6SSy9IJExNMqGw90Q", - "GsCfxP4S8iTBLDK/bQ7Qq4QqfR1mxS1vnmxrr2sknHMQVOKY3+Q5DSRFUHBwvPIeXwUaL7SP8nGbt/4v", - "XKp+ghmeEVBHbUM0EfyS6IWaNwFKJLokSy3lLNFMD9pfUAkvPIQt0AIbq8NgxN7MuSSmifskwbZPFwQl", - "PLw0T79zDpr8AscZkT10NdciB9gECY7tz8g8jI3YXC9ShjwlkVZCTDPYGrogbHGBEpwCmWNBgMZRghUR", - "FMf0g3nCh1cGElF9w40YAcJAKdY0H4ZcRPDCxhHB4bwEhb9KdGEElgsY/oIyjdYXhjBrj9Ufg1dv3/z8", - "6u1vx+NXZ89+OzwZ//rsv/XPplNw8PvHwLhq5JLKzwQLItBfPsJ+PxnxNiIiOAgOMzXngn4w1ppPvUDD", - "QGr8wikd8JQwTAchT4Je8LfyP99/eu8EMmPGXmgy8Czsk1cYMnephyUdO2ugRNbC5N42NMg0i3px9nZL", - "384pllLNBc9m8yphWNHgRiQRUXk5pnw8SX1rovISnWy9QlpwQTHVBJoLKtvD4enPW3IU6H88cv/YHKBj", - "Q7WwfM2DuLDyk5xr9Mm9Po7O3iIcxzy0NpRp2wOvm8rH4AlTYply6lPiasypaNrkUf1+8fUGrGhrQtmW", - "1MfQD28Gd8CbW6sSz9iCCs4Src4tsKD6npZVWvnt1fGz8bPf3gUH+iKIstBaJc9evX4THAS7w+Ew8CGo", - "xqA1PPDF2Vvz6mnIRqVxNhtL+sEjShzm+0MJSbgwKrTtgzbmVUnD0C2CwxkFuy9+Nsi1/QLwyh2KfSPK", - "RzED1571Xvzsw5b5MiViQaXPzvZL/s2dfNPdp4Lb5pUsR1rA4kFJfwljnkX90pS9YEoFCcG9Qv/rT5Jo", - "QX7xofos5ennN391EmDXSKY4TikjK0TTb0REvOLiMuY46m9/YQnRPqh6XGPMh+r55i9rDiUaHmcTzKIr", - "Gqn5OOJXTC/Zw1ftF5Q3zpnrtd4Jjv/1v/98d1roWdsvJqnltNs7jz6T09Z4qx7aa0PJN5Kl/m28Tf2b", - "eHf6r//9p9vJ192EEURuJdTZ839mRqg7zVhfQmMObXkZzm/v3GFFcatQQ3fkcG/tM7CPUfMFETFelhiv", - "XVOwPQTuV1uVoOAliWw/zUYvke68hg3r0dwl/6Ku5O8M/YzWsyjPmn7WvMLeC11Wki9ke+fU/rnTXFLL", - "ii5pOgapeYxnuc13lUvo+SVNrSgOPcwxxrFhBFEGwvuEczUYMeOhos8ODphckxB4nlRYocOzE4muaByD", - "hQiYSvNq0YJ9ybUJmkul/1tkrIcmmdLSOlcEWb0JJslgLdB4QlDGsHsPr8nOdoNN9wIAyyURjMRjIxvL", - "jpAxnZDt1Aoc2OoUS+uiJlSWVuF1/OvpOdo4XjKc0BD9akY95VEWE3RuvAs2q9DrjVgqwE1BT6Lpmdp5", - "+RTxTPX5tK8EIW6JCQyW29jsY+3ixdlb+9wvNwcj9ppowBIWWUdfd+NYJ9CIs79qiiVRddjy/DWgt7l0", - "SIZTOedqnObO06u407ltXqji3Y0JvWARpln1SHd6rU6gCypUhmPNayvipPeB3zixe9QG4yNfVl8s3yuc", - "ZlX1ZbarxcWMDB7tXndZj+HESEqdDSclVb5hQnF65sdui10z/glzC1lpOCpUzc+Y69wM0nDeMT/33M5u", - "AaWTHCY1c9OXAc+hLKnmnZzPjQ+WkQgl2rjQ2rzFY62/X/TQxd8qP2jad6qFli+ukIEG8BOmfyqPXzdK", - "rDUX3Mjdu3w4WN7+PA5lq6cTWmwjJTCTxkdtjlMyQL8AE0eKJKnmZGyGqES5axdi/Oo/ETdCjes6Ynpp", - "0viJWHDkRiNJZ4yy2aYW8/XFhKPIWJammcqEbregsoBmFXWc9abh1WpWRww/hggJysI4iwi6cBaei6pc", - "2LT/NFVCaxBqaDgGJKDZgLKntpJM6en1hhOswrmGE8+UcRyzW6869dWsTOseVO1a8qe2W5z/ec4u6oEw", - "C4+KozdnH3nALFiyT7aZAa2g4jdRXpIlHLkzR+KGQbJsifTbCwWRPF4Qe+2WbZkTCPXhRnAqzJjGIGlt", - "kJr860EuPuvcuqPQ8OoM/qqq4AnxkarvNltgjJX+nU+440J6c2a+nlaMJQHgg+pxgEAcu+gZXYmABQIx", - "jSwxiqggoWoMT9lsxMCH5ML+MrCjXWgi1zLKFwmcgjgEENrLR4tKJ+vEPhhGb40nVCkS9aqywSUhqVy/", - "KS1eW8O1x7ouyJWgjpE5p+KO4hlhUy5Cklgl4fMUx2elwbxq3M2GaLp0GPiW1uziMyA6hUTGf8icB5hZ", - "K2Eb9ejFqKa1GReC6pQXOI4v0IZttIkE+QM88e1ZMc4KZH9zdOZQIH/2fnfa0xipucDFXKl0rP9LjjUV", - "X9QHs30dhReRZU+GoF/t7e3aU7VGN7Pg2rBV+5rXLaL9aJz43fqypvFCr9L6mXQR5Y+KLoUl9ZKyqOsA", - "v+q2rda5XDBymsZdG+hSQfpZOhMYXGy/pHnu1u+mAM12Dr4mjtfnJllECGZS8aTsb79Rc/GgVWeQKrAW", - "PO5HWGEwZXa0t5rlNh2Pk6UZyuhibZaY8Wzi8RuiHyAUYEZneLJU1feDbW803+c+Yru1+I6lzYHfaJAk", - "Giu+2oWZTpFr28Vj0cQbKD5eTClfHd5h/V8q8XfmOrJ6rR6in4bUmhNAxgnnxsPUAAGExnen5be7wYj1", - "4fo9QMf5BPmw+ZAYZEscmZeTDS5KizCBHGiy3EQYvTsdoDf5av8qkVZYFsRFNMyxRBNCGMrA9Ay3Yd/c", - "xeUFZBIuTVXvbm0nJvhhE54ouf02yGOOwUqTR1CDq9SE1vZjIifhoOybMGZlK1gnq9Uqx+/XZEalEjW3", - "b7Tx+vnR7u7u07r9cudRf7jd3370Znt4MNT/+Z/uHuJfPr7DN9ZhlbdY57My9zl6e3K8Y42l1XnUhz38", - "9Mn1NVZP9+mVfPohmYjZH7v4XiJA/KzsuPCaQxuZJKLv2KTGKp+vXMklrcUX7tYubnfksVY44K5qayDx", - "Rre8i9AWn9O0ddm9efBJnWGudbsuba6pyS9T0DsLKilJcNa7MaReP85jKi9/FgRfQshe895O8IzIsbnP", - "/P4MmTRONuTaWjcE52oqzbtp1eq5vfd478nu/t6T4dAT0dFEeB7ScahvoE4LeHV0gmK8JAJBH7QBD14R", - "msR8UkX0R7v7Tx4Pn27vdF2HeeLpBodc8XK90IaFyN9dnhL3pbKonZ3H+7u7u8P9/Z29Tquy9uJOi3K2", - "5YpI8nj38d72k529TlDwCfTPXIRNXYD3RVYemuh+/a++TElIpzREEKODdAe0kcAVRvLXqipNTnDk4k/9", - "d4fCNJYrPSbMZLalMbQlWaxoGhPzDQ6kky0adn4MI3kzZDCWx/vebCQbl7TWQ8DtJW+CKvFlFdCdmoDm", - "kvBESRwdGApdy+fgNIuFvW/DA7uHjtjwUqtO/ZgsSFxGAnN1mchaQVCOJ+bQKruibIFjGo0pSzMvSrSC", - "8nkmQBY1gyI84Zkyz4w2QLuYBLyeQfeYanbdTc99zsXlWv9RfRPncehrrUKHYEifWlMN3OIY2d4uRKEk", - "9OXPgebR1H6X6LXpYSxExc9pVs1q04OZrCWJIUGk4sBJrcHQDtNVuvTLLWAsde4fZr6Cd96T70t/atwF", - "vqyGLWYE8i+otRKLxpQ30P4cmnd2R9cd1xpSOsCdkav7ADr46/c12vYlw+ndQHyVM1puaygawS0saEQG", - "CKgLvGJcfGCN0s4VT1MS5fafwYhZf+78J2leUHRHAwc1J1QgLuiMVieuGtju0qvtJqjosOnW6Fju2JRQ", - "4SO4b7QTPZ4qk2vh0oVMkXL8kj2EoBec55kpLCeqguZ1nt2jAZHC1bKxxBdnb2/qm5YKPqW+fEPgC2G/", - "Ws3MeW293Bue97f/H+OBqfENRDTKjP9EwqNaIgnbvtvN8+Ls7VnbmvLUDqi8usaeco+XVcmtHETso5J9", - "lbQajEN/fbHkkxSy91OfLDsVOCGTbDolYpx4jGvP9XdkGhjXJsrQ6c9VeVbLzV215rPK4YDaPMWhjczv", - "Bn2PQa62jV4Jmu/9x/WamGu4LZ5PH5WwbWxI3wD9lifTQC/O3kpUeCl5LHXV4231lz+bLyUNcWxGNOG5", - "lJUNbICcnSXks6KjNUV65GR/DhZHCGhjMUszIMPz1/2TV++2kogsepU1gWfRnMdEr3uzxC0WLqqvcO6v", - "MIlFm6XDIIbsSkAlWOUU3BlIJXr1QEdxheOxjLnPWeON/ojgI9p499xEXekV9FBaOUr9ewkKFfze91KM", - "5kht057DhHWTaYXAvbpjNRumMa+UtleZ1EcqvxAcmySgVXxuJkDil9WD5pfrk+6YQXzznjjH8JpS4wve", - "Ojo9NgJDyJnClBGBEqKwTTlacnEBcSjoBX19R0WYJOBqN/3P1d4tLSb4cjRWqxH3qJG3404MuC3x5q+N", - "C0KEEszolEhl480rM8s53nm0f2CyYkRkuvdofzAY3DRG5VkRlNLpKLaMC38pXGUg5593DncQitJlLx+D", - "s8M3vwQHwVYmxVbMQxxvyQllB6V/5/8sPsAf5p8TyrwhLJ0SqdBpI4FK9UlT31nm94NSzkuX369TXju/", - "PgOeDRA35403Vnim9RODcZ8bWHzr1CNF/itVSjlSdgjtkH6EflhtCXWCEbSxc2ZM0bjIzNK0gd4qt45c", - "mX6gkXogJSxPOBDH5q+Qs4WmCl/2gQoDd98+6/3AermMI+rB5H9Ybc84SUBU1Xp6C7Zwmq5HW7+gmPO/", - "rllXbGy05yb66lz/Nm9s1dlfzf7rz/9Xnj3+Y/vPl+/e/ffixX8d/0b/+1189uqzIqhWh8V/1dj2LxbO", - "Dg9LlZj2rqh0ilXoEajmXKoWCNsvSHHjrzlAR6D4HYxYH72kiggcH6BRUHMRHgVog1zjUJleiDOkh7KR", - "Dpu685kx/+jOH51u+ak+RmRDGoQ9kDySSWaTiCeYss0RGzE7FnIbkfCmr/+KUIhTlQmiT0/LsPESTQSk", - "9bbqeTF5D33Eafppc8RAwyXXSugdpFioPI+HmwGQwq7K+AzY5iRygeFGQx6x/F7K48KNjWaQG0HANl/3", - "uPQDxau+cFENxXky9EXQg9eXPsiYSkXAMTvHbI1GuTsaejKssIonwyfDtQJ+jkMr0A8ooZnv3yFlB1oy", - "CAxTG8YNHmodbOmaNxkaQb+8eXOmwaD/9xy5gQpY5EdslDzjAyiNjVDFsuT9txl4s43C6XbckDGSQbe4", - "Q9TQM+Me+ublOVJEJM5hfyPU4JzSUO8Pnv+plJlGRYrR4dHps81Bh4IFANt8/SvO8U2+w3pwhzWatdkC", - "c4zX8O2hk2Nwz7UUWghw4FbznAsUGwZT0PUBeitJ1dcVjsq86puTjJeF5c3cAKNg042Y1jnFAXqdy404", - "X0qlSELVmFfQJQxrH16Mz09j9F4j/bhwepFlbeDhg1XuJK5v3HZWsJr8PRAHmrd+3SWb5s1ou2wM1ZP5", - "UaM4+y+dNeXLizu7N1Vyb5rhoRqEWQrgzZM8dM/OcBdZDpoK3zVV49ZXfKQ/2zd7p9a8O0VzLNlfFXys", - "KTfbu4875evUs3Z9/y6/fPOpWVJOli6iM3+3NbGtlzSOjTuEpDOGY/QUbZyfvPj15OXLTdRHr16d1o9i", - "VQ/f+XRI9uBo48XZWwiXwXLsnpDavSZx4XlMrqlUshnw2ukldnVyiV8qCSC8EcSbXzArhHu+bmzjPvI9", - "fE2/wG8v18TK7BCfm+LBSst3lOGhlbn6siNU+az5+cvmariT5awtL1IWKpzT9q2TI/QC6nFYPZSaBZII", - "nZwVSRYLq5YbvrYnW6tnezgcbA+72PgSHK6Y+/TwqPvkwx1jyTjAk4MwOiDTz7AxWsQ20h+Or/BSopGT", - "z0eBUQhKmkCJbK0M3+n9tpmD4nYpJ+oCxbqkEjdJItEtO8TnhuSvSrV8Xk2y3FnI+4xKJJ1cKNzVbp0n", - "bK/xTcznBIU8iyMtSE006RrFjkRW/5REFfmrgdrfskvGr1h168aKqhnAnxkRS/Tu9LRicxdkatPzdtg4", - "OF20nANPb3QMO2tk7bWruWWihvtIzlBnu6Xr7ounYigb/ZwTp8HQDsa/Qvz0PrxTZo5G48mKPdXMNhFZ", - "jLPMJ1XpTy504+3bk+MKcmC8v/1k+ORp/8lke7+/Fw23+3h7d7+/8wgPp7vh492WBPndHW9u70tTpeb2", - "UCkAPJhATSRcdKDpLXeGmWQK5Y5ympCPtHiKSnKwCQwCq8QJowqSQFI208OAkcCKySbC0+SppIwqSCkA", - "CW0o01sGa4wexLo/HaAX0BY+4QQCltwitHJUNUTgaGkMsZoxuKlT+NfqJZ/PMyj3A33kPFMIykPpbWsw", - "WHVl9RCGxxyg3zj0Ec5LlfG63mOag02g2byuI21YvyTnvwqTWYZ5gJ7nTDJns5atbkhi/zS827pWg9v4", - "ZsV5z554oLGlOLmSX1ovMBANeoEDFPivNT3Z7Lq8QRplVPS9UBAcAwstPIUyRWObJQF2QqFAEmwEw+G2", - "UbLNCEaisREB2t4bjfuJFRPyTo5RvDtFGxAP+XdklUr9r838bbJMlXs7T/ee7j/eebrfKeqhWOB6Bn8E", - "zlHNxa3l9mGajV3tkZatH529NSUOQ85klhgrgd17yck0FTzU0iplqChmUkz+dPC0HOwR8cwUdrJLspFh", - "n0rly1ZWnml5YPuTxgs6nbI/P4SXO38Immxf78udiVe5K+qkeSXhk7KptaE2kknfZHH0++MDQgnZGrLy", - "mkjYATonCgH+9BEO4ZLOfZosyrnAFgtxL2Lt7e7uPnn8aKcTXtnVlQhnDPprc5WndgUlEoOWaOP1+Tna", - "KiGcGdM5ekKCCWYFOD+dIZvQeVitBDrYHu76sKRFXiqwxo69SFpB/s4KQXZTFujgmpULSA0q90J7d3f4", - "eO/Rk0fdyNjV3RPXqzmMS+phwGPzoJRPfgPM828Oz5AeXUxxWNVQtnd29x7tP35yo1WpG60KcviY3Bs3", - "WNiTx/uP9nZ3trvFXvlM8DaqsEKwVd7lIToPUnhOwwOKJuvttd0WPsHTINhrEsaYJoehc5+p3T4mx8ZY", - "mGbFIXS5GKyRoHFxdejbSUWrFQ4yogEXqFRxcbDeHHo762Y7mzb3wXo23pShY8w0uGyQgEnleAvYpYIs", - "KM/kFxiIKxJqZJrGnIsb9W3zR3pNZBYrY4KkEr07/SswEY1cSCqSVn3tLfqtCKW45eZuRMAVnPBjdRuw", - "Op1Gl6NfteFeC5n2VvnRVsi/NWIp0qwqY+vfvo9wHGaQvAzn56l3BbEHPFPwUr80XiJxzDlD4RyzGYFk", - "8CZVIpshjOY8jgaB/6kkjsZT7xNGXmGeF/XL3SJ0N1d/f+MFL0raGVSq5ed9lBiuYjM3DTrUki9q4rZE", - "OGl4YsVLaQBMl4o2H/OZBC1Qgf/LoJ59JsXCuLVgZvLULRKjPFZDt3b0be9ZYo17+65Qc3XyqdVorYyh", - "eA5JHAouZVGY+91pdZmrHBjzevbr37Ori+2AujLlTBJ/mXhbE76Twcd3IXo8wz7nSgQcBgfQVTWgbR7D", - "BLMMMn2VEJlcp1QY9Oj2OD7nUo3zcJQbLlaqMWRxygQpYtbcfTmHAIClYXHQxnsvOtZ2G3Dl5Y1v0buB", - "Vf6h2hbYzlO9EPVDq5fjoA+NmwE5K2OAiqCiegTJTULGirQ/VMKotBSthDYYVxW2VEpds9nlocqvo+p5", - "2krKvtwbnneN5lodvHWG1fyETbknN+QNDP7WJd75LqREQPlxzlBEGCWRUx5zy7+1bYGTfSwJijJiIWcE", - "UoEtwLEhb8gByZxRjLJZjdfXJ+xihjdrWJ3kCea1Dbs8OUq/a/YbkQGsjJOARLhw0u7k8UDl2G8pbg4s", - "yCyLsUD1iMUVS5bLJKbsssvocplMeExDpDvUn3OmPI751Vh/kj/BXjY77U53GBc+hrXnGbM462FqDqQ2", - "b7GFn/QuN2v+7WB62TL9t3T/Ti+4Xr+h5zQmNqjvLaPXJUSvZkHZ2xm2hT60DFoJemgGhN6Uc1uU9VG8", - "i9U8zKsmeNzxjQdQ7VWiaois7Ne3W3AxWxXo0TTFoA33KOyyzFThWsr20skS0s3Lre7+4FazJUlYnX3v", - "yaPH+x3T7XyWrXNFVeXPsGwukhUWzZaTOu1iNnvy6MnTp7t7j57u3MhA5TxlWs6nzVumfD614ig1o9mj", - "IfzfjRZlfGX8S2rxl6kuqFLo5NYL+rSCdIsw65Znj9Y833H5JN07S9UC2s3GuEJaOqyIXKVaXhtkOiWg", - "VI4N3PrFYmru+Z3WEOIUh1QtPQYTfGVSvudNauHCXaxp1cV6QGrHthkfNOeS2aRw6Nxwk6O/GdN6DRee", - "dM7aJbNJmxn/VX1WY8QvbEDlJ6IOLzRFYYGmuSDfzxWWFa8O/XcIOZuLWm11/yHTontVaofreWHqwjPS", - "F/LuL0JdPv7acZbMvhUhuQ7xVVdoOwneSIf23Mi+MpXrvXJr/MFegLfrNZ6U8+mtTFhYSb5X3Lo3n7db", - "lblmP3OD3Xy+kgvoTTrWU4sBPto1WJAXY/cqKNGCTYqL9Rml7yBBkPEpuFWKIOuOcC9ZguzPd5IZqHEc", - "50S5tudao8/iFQmhmSJigT2GKTcEck2qdlTDiXvImvjQdrJZK1W4N/fLajaEt6P7mJYCx6kgU3q9AltM", - "A3NdV93HpYVAVE0aLtFGgq/R3mMUzrGQtbUzOpureFk1su55oic+r4gzUVp07p5evThN17H5omGPszy6", - "j2TPS7EO/rTvJBqvCnQ/yps5m3GKlyBbtiqCj3f3hsPdneGtIt2/VDb60jhtHqGlftaYU3l6LI+Q+382", - "UxZeCWqKmjkwSSUITg7AmyrFIUExmUIcWJ4qdq1O35h69eLtI6l1/M/x3x2Uq1xq7SyWxTHOQPBw49gk", - "AW4bgXulrcZ6lL83l70iWCxnM2Ejaqzuv7rfH+72h/tvtncPHu0fbG/fRWh8DqQ2F57HH7avHsc7eLoX", - "P1k+/nN7/ni2k+x6wwXuoPBBrY5grQ6C3UNKRD0XZT2HqyQxZaQvc7e39Q7IK3iBeUlaS/83sz6YHawU", - "Fs6rmyzLDFgVwKmXZLuPwCa7+pUmlPryT45XL/tWfmT1hfgRrL4UwKdui4FULdufmxckYx3vnbelhp1v", - "npW+jevuHp/TN5C295RbIO7D5wpjrFDYqhu7eat5VLgZF1TNk9XXQ94szzIAj+EfpIqqgTQDdDJjkHm2", - "/HP+9lEuDq07B70g/rBXpRn7e/eQKhtVnyOgPeqyGNDhbQASG6+GAjQpVAth3BOwIACIn7b720/hhT7+", - "sPfTsP90gP5R8hToGWiVwbftWld+HXaBYc4oQe40L+fbT2/0jO7guQqDfrX3UttFbOPtLY4XaT/dXeHc", - "pisHXHxunHEtquiOCg19WrFjJzjfLG2P6+URTQZ+2eTpm+H2jdP23OyKGHyGR/FnqXrd1LsYS9UmV7/E", - "UuX6GBKZla57UOKG1Srd27TiNhQAXDsh1v4AyZqG7/Kpy0LocVaAHppxhYoggLVSDixfZMyLD9X1F2WK", - "IT1EK0LsrEGIbmvKA/noKtI9OUap4FEWFh6wMSw6C0Mi5TSDqsuDrhLt+jfGu1TmwbVca/Trlfk27X19", - "mCm5bj/v38i1Kk2pEbb9qLeH64/6TiwAvSBLo/U8zDTqxsFulIljjU+lxx5RBXtNCipt5n0Hjv66DMGm", - "ggcFm1CoxYEsdRUENU41MUl66gbi67E3R8AxiYkivkGQKelp3T6oLLjoepa6vf/EbzLD1+MQAhIbC/mV", - "kFTL6QmXtsRmgtnSu7B6Fm20MXQVJCWC4fsmk5eFVnVxj9dKIa1H1T0hec2iayK+yvnf8wwdXzYbue25", - "tlbE3Qkqb6yq1JZRpu7rWbY7Hvb/x9gZ0XhwsPXT3//v/vu//cVfm6WiSEki+hGZwgPYJVn2TdFZrbQN", - "qvlMIdtNIBW2FU0UwQlYEcJLYqwWCb4ur/fRMKek5W84aWwBXg4TyvJ/r93Q3//S/u5WAuNbYB5rz/Gz", - "sx/dRWpZxR2P3kiImLn87c5dDMp5s3ipj0qiUv46e887sv6rzLuUC8deGNloAOWOJxTSgMoR02oODkOS", - "KhINbB4vCmsRHEiyXj7Z5tFz7t2aWDEkD7XhNrU0WR+9RYoPAkau+maGqK9xb+/Rvq0YX4bkduOIfYdu", - "IrbbKixqKHvMCC+phHAE53Vbaow2SJKqpcsS6/wiN28WQX6YD+h9Cv3C6bOGT79EttC3K9ODfof1PcsB", - "/m5Ba0P7G+ffmpPP71h1XM/UY2jS1iyrZpapqUxS9dv9rhJ9x4/BfbDpI6W/GddEmw9zltUTg28lTG3Z", - "7Lu+cIgIShKvdEYtqMxFu/eh03ofy5VSZmlnpZW0n82pE6bqZaLbAXSmQXM1J4KUDgI6FClEbwgy6yjY", - "IcjG5MhMiejXi9qZuguCgudhrgc7EOTOpE3D2OoMN6f4Op8BjKpYNp4eYB9FrrftFz9DLZXXrrgZnboh", - "YBk1UdefrqaKRV3KszcPo4xVzX2b9l7Cs7xqBfdro60achZzVFDTh4//wFQ95wKE4/aQljvPegOCd0QE", - "xPTWc9p0SghDExKNeaZW079NMm/jWSI0IVMuSCn/rlMEMCCxrXm8hhe4oItiDe99coMkYSaoWmrN0Yqk", - "E4IFEYeZIXgAJEwEPxcTQzrbT5/AhDb1eLC9IIwIGqLDsxOgR6jbr4nj3SmK6ZSEy1Ar4JCNtJF/A4S8", - "V0cnVvlyGd/gQZEqQD1Xhvjw7ASqmgqjgATDwc5gCMScEoZTGhwEu4NtqPGqEQ62uAXZ7+FP65xu4tIo", - "ZyeRlYN+Nk10L4ETooiQwcHvHidvRYTJpi9B6sSzkt6QYiqs4pDG4HpuUIXqvpD+yF2lB+Y+7hmAd7Yd", - "SbW0jngkfWWP9b3GBEM1sMWd4dDoaUzZixcX1S63/rABe8W8neQ5AI8nF1BDrncypQX5p16wN9y+0XrW", - "Fqj0TfuW4UzNuaAfCCzz0Q2BcKtJT5jxDkYmz4T1fyjTGaBQmcJ+f6/PS2ZJgsXSgauAVcplmzBMJMJQ", - "Is+UcviDTwbIWsYh/amc8yzW3AQZ12cS6QsLa54ymH1AWIRzuiAjZu9pU2wUC8gvnSB9Pxu1pUoaZmpz", - "+nlQ2s88Wtagmw+3pYfrO3toAeB6mkVJxpAtatxWp6WwhFLGoN6jJDYnZV6woOlkAQV6Zci9lYkJw0wV", - "9V5NZd5LsrTGVu+AndK6aIYHx0KgEHyer3xn0x/PANkz/aFAx/k3ZMFbFScYvFCEcRYVMpdzscViguPY", - "G/c/i/kEx7aA8SXxiKgvoIUFSjnRqBNuGI+ISRqZLtWcM/N3NsmYyszfE8GvJBFaBLLZpy2sbfVOi7pQ", - "SZ4mkAHa1LbQc26ZJW59vCTLT4MRO4wSV7dE2uLzseS2srNJn0Mlcj6ZBnf96U1bnvuPMql4YlGKlQtR", - "mmXyTKWZsk+dkiibMhuaQ51SOSfRiCmOPgpTln75aetjMeMn0F0IjjSelJqYLW19pNGntlXLMda7H0NT", - "j/ZHAACjQN8uo0D/PRNY6y6ZnIMpQ4L5YlY+0o08FlvLhZt1CIeYoZSnJo4dkMoUrK6MAeUHcBwjBaTk", - "+mppE06yZT82NMVXS8/GpZhAghoZQVW9EjEN95746UmSUBCfgeO/zl/9huCq0mdgmhVmI4ARZfoWRVEG", - "kjzMPhixZzicIyM3Qa6yUUCjUZBrF9EmrDWT1nG23wcR9ye9tJ/MND0a/TQY6KGM9HyAfv9oRjnQtJQm", - "Y8UvCRsFn3qo9GFG1Tyb5N/e+wHa5t5/XmEEaMPw/k1XPAbSDBTXoLk3MIsQt7w2XiKMCg5UtqNMKMNi", - "ZeUbD+gtBLUqj2eyDIyPI7CgjoKDkbOhjoLeKCBsAb9ZQ+so+OSHgBWi2xNjmeI/TtbOkWh/ONxcH3dn", - "4esRoSsNNfl9akhfO19M8LBCV1PwMJtzWf30CZoyTkbcugfJ52ccucIAP0S8NSKetVyUhDfoX74HDPrG", - "xCi4NQlM67Oxk8BWaicGLSCtJWgcLkrWKBzUSXAF8pbVj7o631Qr9tqoLIQlxg7/9u4B/2DeohQ6zPv0", - "vubFMeSozAsDPyx0hMNyiNjza8QviPoWMG54X6zUJtT8mvj7UPDnBbFyXwG0GjfbIgv33uTPBQAxANKO", - "YhprXfUc1tQ/J0yhZ/DrwP6v03ggs+1FzGcXB8iAMOYzFFNmX+NKr0X6UrSwhE4mDCDvZ6MCXCKmDXN/", - "/ut//wmLomz2r//9p5amzV9A7lsmOwYkbr2YEyzUhGB1cYB+JSTt45guiNsMpFYkCyKWaHcIYmYq4JOn", - "2KQcsRF7TVQmWOnV0uREknZAUD0Y7IeyjEgbRqEb0qlN2GAMzB4V3tGyAeW9UnSv6XNqdlDagL4VHQ5A", - "BC412Wut/hX4rWdmzxX7Wd1W3rCYrucvilwrg719s8AbMhgAsY/u4IPdNNo4P3+2OUCgYxisgKQcIDEX", - "w1jhefCDJ63nSYajVBkKQNnwplKd8Vb777Ft080AbEf8nizAbYXT203AxuRBBIkcvH7oCl3MwX64OdOw", - "zz577ILn2g20t99veQrnTdRJEf5y5+xwrwlz86UEsq+hAqMN56jtav6dHZ242jCbXw3p7+XW0Du1FRXy", - "qwNxU2nw3tSyI86mMQ0V6ru1QFL/hOSqWhVBHgo7eG1XjbDbVz39Xfl+26pkc2m96fLELsWVd/e3R23S", - "m1wjRYq+Atd+3CTrUOeYypDrviVs6Yc4tQULjfiS02kZi9YZpIzfd37lrBSXLHs+OXYEeX+mKTt1xup3", - "wz0wxeMaQ/yKjLBWhK2U1PIhYfPb/BRdooAVlqtvCzWH9ycF3bcVy4fmD8mMFdXAprmgSe3beoG+IOoX", - "0+IOD9rO4Nn4ORGOql0OYth1vi3TFYVzEl6aDcGD9Grd98Q06ab6mvG+J80XwHMTicWC/IeI0kHZLWC1", - "SsE9sYXl7k6/hRlupN5+uXdei2AeIIOzycRZrE3NNiyXLNz8rp567+U2M8B+kJfZWRbH7sVjQYRCef3m", - "8h2w9RHcktbL9o7aVl4Hb1+/7BMWcvBDy32o/EKU/fKFJXxzYGYrP9Cki05oInapu8/aJJzPOH/jLojy", - "+uD/sfPcVgj/j53npkb4f+wemirhm3eGLMP7Ys33LXE/YOTTAjetAg1YE4Naoesk1LxVRyHVtf+u5FSz", - "6RtJqjlcfwirXYTVMrhWyqv2KO5UYjVzfKUnmRzZfNCGT84/8TuTVO/Xymcx0uXvpbL67GELtHABdl74", - "RBnKJHmADpQ0x7jytdHRXF0Q5Mrrw6HuyXEPANnToIOEQjZA5J6M124d9y7c2nnv33J9mEzoLOOZLMee", - "JFiFcyJtsFJMqgz4oYndxfXcKnh/w1g6vM+r497l6h94f0cSf/1ADfM2L1DrZH7XqqvMb9trmd8Wzzex", - "a69dUX6bJ2mzxanQBVF3ReNKrHnT2dG3Lp8ugt5qRaVQFxBoEAcj9n+0/vG7Ijh5/5MLksmGw519+J2w", - "xfufXJwMO3WoQpgS1CbvPPztGJ79ZhB9Dvk9i5C8+jpMQQBAPZfA5t9OQSpePrtrSA4Lf2hInTSkErhW", - "a0j2LO5WRaomwbp3Hcnhmw/gNonJDy3pPrQkmU2nNKSEqaJeVsNJzJbbe4CxZcy+D5WcOyoXbWctKSfK", - "NQJoka393h178snvXzlyieEfpo88N1ExkVNHisuwXR/51vBheL/M+f71kIeMYkbgb4Iu1TKlrwwjZHpM", - "MgVOiUWGEPD6RMJI7fmIA1RUP5RZmnKhpMkWCQIwlMNScy0A+zJLVpNF+rJDQmJcSmRvxCCBvP5sYvm3", - "LsnS5IKkvKjqn+/U5n/0xV5Vc3F+VTL68jKWP9FoJxnrnsnY5lP+ejLWV2Md9yJpnVTS1G/khAEK5YTk", - "lMzz4D76gbLZ5oPyQDXMKt9bKZ+RR9TaggJsNrvulswLvbZdtKUEu7Y84b/hjdvcpE9qd7loSwBEEcUz", - "xqWioQvcrSfy/nFDd76hV0PWi81TW17Tr9A/5+Ky6xXnKff0AG668g6/QVuCXh5kA/v6JgVQts1toJHm", - "3m/BRg2vrxmCQev3Yhhnkb4I3YXoRMmp4MnY/mjy1WqqsNlAwUQR2lG/NrPRs9+Dweg3rhBN0phoKZ5E", - "qG+wSZ+mFf1d0ncqSxXvbsYMNdmUA2JMMjrpquZYFgmPa+7ANuCdvXlcXq4Z89n6JBj55C7jgycLxoiZ", - "pPTEZbC/QDmTRYojSWISKnQ1p+EcMmLo30yhTUhWgdP0Ik+BtXmAXgClljOBweQbkgitCIWcSR4Tk+hi", - "kSQXB82Mre9OT6GTSYZhcrNeHCCXpTW/IKRuVc5wkZfj+c3m7djQmCR4HJsTvdBaY2l/mzb3RZGibMR8", - "eTAYubID0im6KKXEuGjJieEY6ks++2rSVq89saTZi+JIAOAMbhIWBW0PMTT2Z8PYHnpLlXTMzGGWcceJ", - "ORqLeclneVLLCirjNO2KvnaZgMWLJFmBw2ijVDNTqohn6u9SRUQI6Gyxuw250QYOzT8UvtSIamve5FVH", - "Af28z40my5wXVJqplgq8mH8tkiToBXY9pex0N5De12Q4qQ/YfBbTJ1NKY/JD7r5JgpIqsy9lKKndHLYq", - "e7vIbYvNf/f2WQuo6HuwslTfs4pVUOZEFThbXhSIelCZDuAgG7KYKU/koxG3y74s1bXs9rzVqIj5DSit", - "61698vKGee3F+37+aq7gIQfByMZuplzUw+PXvYt984j05Y6ksdUuGPIDN29unuuEmGm2oswlVOmUYOeD", - "0o+Q1zmccy5LaD8hc7ygXNgM7NbqmmMmmCyM9mi95y40ql5Y++2FFc8PrK0J4fInO8cAulufO38P96no", - "8bykbeccv+dEasgCKRFGE0HJFKU4k0RLS1lCkKkwYhN5ExzOXRHnwYi9mRNkSzeWDAh5pV8q0cV2ctFD", - "k0yhGIsZaDvmo/GkEyTkSUJYZMqxjtic4AXVqppAMVaEhcu+JFCed0GKAiZadbcvlAgeKPMCoD3k6saC", - "geGiVBX2AqWCABIZdZlVSrCOmMjYf5rMlXrYC7fQC0SkwpOYynleKyLEEWGhNy3k+bfNxr68EfecqGbh", - "1K/yZnkrXvo1HzHLtsy8dPU38b75wBy1uHD1LTuw+RVCr2xXDauej+dFsdh/Q5I2e3V7/EovMzmIV1Hx", - "t/EkU6kW/+NZRlmSjDIzHalWVP9u31pyhoIyVnlusTbZ2z645JUQcjDfiOdtfXR/ntzCRvaNcMJeq2Lf", - "lnO72PS3wHItVG/Fc7+ScdDakkpWsa/Igu2ivp74xEWJy30TbNgQXM6NyzxHCQw6FWc/mHGdGVv3gNsy", - "Y2dxbTyAl9gzZf00xm18uagd72fA1iDwb+r9WttdiRF+dcZXvAjcG7M7ydmbYXgpXsYcf+/vMiEXwgR0", - "2nLEDyehWMkWWHpg2gCLWy/nED0XTfLu9HSzjUsItZJHCPWAOUS1rGmYeKo1vloQIWjkSkcenR5b71Uq", - "kcjYAL1KKNRzvCQkhUIxlGcSQWTuQO/PhbY2i+BVYlh7AWFKLFNOmVq7iqLp3Szm061K590zn7QpFb/7", - "x2Owwj88JgW8Q4srdgOrtUiFVasznnNOo8zUu9TSFp7wTI+uOYsrtDuDu21KYyKXUpHEeOZNsxiICJLu", - "2ppMtp+JKO0hqiTS9NCDCLyUiIRKSTmTI2bLv6dE6Ll1dyj+WzgZeY33Cudc88ywvm/DgU0vxvhsYdUG", - "NUgtAHVAg4NgC6fpFpSL9jtJ2eV9xpKeg0cakstkwmMaopiyS4k2YnpplA60kCjWf2yudGkbQ78vXXHq", - "9pSlIX3CptxblMPgbI7M30cQUpWtuUfEB8fWXpAysTj+AwftZ2tyLV8TBMd9RROSB7+jTNGYfjCsTg9C", - "paKhiaspQi+hCLONvhyxU6KEboMFQSGPYxIqZ1zZSgUPt0bZcLgbphSylOwSWBwwvPbPCcx4dPYW2plC", - "0b0R0/+Agd8cnpmX2Cm2NoLSQhlRV1xcopOtV2ucfM8BTP/GXnJmgytjIL0H/uP57uaRza00JFtIlKer", - "FCCefvdunFaC+2EteJjWAkgtke9mYyZwCEKxnGcq4lfMbxlY8DhL9D/MHyfrEpQoHM7fQdNvRto1y1k7", - "jdvggyBKu6eImKJBX+WBwgDsofqXasC5LYAQU/Hc894Ch+p7xO4vb5Qvw/EbfJq0EHUFub4Z2rrvm8+u", - "weXdKsPjoZC5wTS3E8VXW5+uMG23Pv0c8/BSoowpGleSGmi9DfKA6h+LvI324Q/EBIiOdKXEEblOqYAM", - "NrX0CIjoHUuEkSIioQzHW7BnMwhkoHRWLLzgFIKUw5hCmBiNCEp5HEOWnas5YUjvBgxVboDSO620FSDK", - "bcpPjIqjCQl5QlxWzk2f6vYPTNVzLqopNr8VvvimBH+9H71Vvc81WUXbZ/ysLKOn+BrcmqPMPhO7FW28", - "4MWPxhTUQ3A2o2B3KEdBD42CnWQU6BM4wmBCxQo9QgllmSJygI6NfQvCUPeHSJKQs0i65KDOgrc7lG1B", - "qQYtWyIc96HffYo9FqsAlK/tJD72oNsh3R8CbNBGmeAsTUY9ILoI8UyBA7ejK9sqIgrMI5v3/gJbopEf", - "un0XTv4PS74VHgWnrNll6egNZ8/TR661urmgijmXRdZJFOIUh1QtewjHMQ8L60Em89eBfr6UiSD4UutQ", - "gxF7nSeutIEQ6Ojsbc8ZzVBE5aUZwdrFBujVggiZTfLFIeAGxoIHh0GiEVMchTgOs1jjLZlOSQgxDDFN", - "qJItdrV8KXdZBrGYxHPw7mOetuZhGZP8OAGnV6CFrGHcljnqLUHCGNOkbFSqAwdEX3jSBbPvRA/K9TU8", - "je3zVii4lMgO1ScxndFJbB9r5AC90SIHTsiIpTFmjAiUSeN3pJfeTwWRMjOBMXoAqDNrMKqHikQnqeDK", - "moljzoU0ll2N4e9OkVQkXYFmr83Ip7DnO0oTbAa3M30lhaG2hvZryTZB+kAMphiAazzS1/RXcPYxC/ra", - "6YQfCuG/EXQ2I0JTBTZM1jyNGrJ24DREX4n0aM2Rf5636pYjPx+15M1d8nRemahi7BqOQYC+yQusZ/JL", - "2prLxH66WfTFr7pTx7mrXv7+RdhPn7nL76X02HnJubprZv0Cwx9akvvSyiukWglQWJ+OoHNEwl1GCHTO", - "O/DV0g085CwDuBJ20JZO4NtDhOH9Rsfdd5rth41blSwBlcI6LaFS69N3fhMYeDd5O79ydOgt8nZ+U/FK", - "kHfx68WNflORShU7oCse8t1n5ryrACWTnhPSWLQFKBmuZx0JVipK72ybbmqSHfF7kuDt2/MN5HcH9h9a", - "fweVoQQsv8nOxEa7vC0kSdXSPS7yae0BUNIPEIzhS/yQ+xDcXb6FWzyvfzn0cHja+rj+o57Wvb3fF0WH", - "T44ffhGtMs1VLpYtfev0sQjndEHaje5VCrYgSgXppzyFx5XIAMzCw91lCovB7AOyw9tcVfZfiLoUxyRC", - "ERUkVPESUaY4cAQzx18lElxrAvCdi6XPmF6m3OeCJ4d2N2vuQ0tT1hhWvPkmy36EFe4vHLdZYUL7jJd2", - "97atGR6iDL34GW2QayVMxl001ZoPotMcpOQ6JCSSgJOb5QVvD1ssm/QDGc8mXVa5InfyK5ubGoWZVDxx", - "Z39yjDag2MKMMH0WWtSfgiSbCr6gkSlEWgB1wWMD1e0WgN7U7qqFirxShlMuzOK+igzT5UKafaBplS0Y", - "14XgIJhQhmFxa7MUV2nKBFTp+TCFsIaCdhzmBD+uMKv5bThlR2OiVnIcEBXnJjXe5o9r7iFfc2XHVHen", - "VW67bqUiu/mqdnQhvYuEubkf8/2ard99O+6VVD5Iz0prOl/kCmmb2fzbQsHh/d0P920uf/eA3fFfEKd8", - "l0zlMIAe0YcwL3mIYxSRBYl5ClUkTdugF2QiDg6CuVLpwdZWrNvNuVQHT4ZPhsGn95/+/wAAAP//tsVd", - "yWFyAQA=", + "H4sIAAAAAAAC/+y9+3IbubE4/Cqo+XIqUkJS1MWyzdTW+bSS7dVZy9Zn2c53svSPAmdAEqsZYBbAUKJd", + "/jcPkEfMk/wKDWBuxJAj2ZKs2OekEpmDa6O70d3oy6cg5EnKGWFKBoNPgQxnJMHw50EYEilPeET0vyIi", + "Q0FTRTkLBsF7HmcJQRiaoIRHBE24QFgpHM4SwlRvyLroDcHR3wVV5DULyQCRqzDOJJ0TJAiOupf6C9rg", + "LF4gzgiiTCrMQoKwQhgpmpBNN8hrFi9OMFsMTE/dpYOSLFY0jYuOEoWYITnDglRmNz3h96g89Zxi9Or5", + "mXckSfWPmBGeyXgxZEEnICxLgsFvQWVXQScoL9D+M582+NAJ1CIlwSCQSlA2DT53ggOAkgHhG/JHRqTS", + "EE4FT4lQlAD0DWhHiQX/nwSZBIPg/9kqjmvLntVW6aA+d4KEZ0yNUqxmy8d2itUMXc6IIGhuTlDOeBZH", + "aEwQ9COR3ugVTtJYr3krYWorwgoHnm1oSOqTMNNMcBarYDDBsSSd2rRHJBUkxIpEA5RJhzawNwA5wVEP", + "nej5EZbFEfeKWcecxwSz4DPM+0dGBYn0WZQ2W0Caj38noQJIZ4qfKcyi8eKUxzRcLIPkJWXZFUyHcKZ4", + "ghUNkTR9UAqd0BhLEiHOEA6VRl/KxjxjEXp7eIpCzhgJ9WByyPhYEjEnEZoIniA1I2jGpYI2SuDwAik8", + "jkkP0Kl63ITpL9F6WP59RtSMCM9iqUR2FKBFNaMyR+he+ViVyMgyZDsBjWIy0nTHM7UMqF/4JYo5m8K2", + "clpNMqnQDM8J+kgER39kOKaTBWXTZiCNyYQLgn5ZpCTBDKUx1gRHFaJMcbcbA6MCEx8lPhSkU8YFGUVE", + "KsqwHn+UcmE4WXX1r+EPHKNSW1gatEdqhpWjBcYVuiAkrW4UX+KLKhh/29npPO33+x86AVUkgUkTfEUT", + "zSb2Hz3afdQJEsrMv7fz1VOmyJQIvXz7CxYCL0rbkTwTIRmFNBKrdhLGlDCFDo+P3txwA8F2vwf/v/Uk", + "6ATbT3d62/tP4N/b+0F5W0uAr67882rSO1NYZdLH4jQ1jSyijEpIsrzrV1kyJgLxCQozIQhT8QIBSZGo", + "BdJVtt33HUXI2YROM+FI0EdyFXDOsESYGabRrfGLYrBWdBdqJhbxSzYSJMGUaRgvLeKN+wQ3I7JEpJcU", + "cqYEj2MC9y9JUiUdFXU0s2cIp2lMQ2A9FaLaS/oy6AQsi2P9sbbC4rRJTKcUGrQCDZWlQ3J9keKIMEVE", + "TuFtQFNhi00TF+D2nkbBF9tzQUn13e6bldVhnmgOL0hotpvfABWIjEnIE4L00NUT2Onv7Hf7e93+/tvt", + "x4P+3qD/6B9BJ5hwkWAVDIIIK9LVB97mmFbz78MCSrohsg2Lq8oDu16NB7dDlxhLlVM1EDlVixH2rOkt", + "TYhUOEk1Yes1lIDZRNZuwPo5OMivBPD2FwGYkSs1shDy7seHH+QqJaG+Yrgjz/zG1uN1EJ0gjHIeoNHV", + "MMaVG3n6RRsRBEu94E+5UJsxmaX6LiTRKI2x0uNqIQXQYJRQKXXX/IeISkOYncAh+YhxNRIZY6YhI+qS", + "i4tySzvKiKZBJ5hhOZpP0yzorLoHqkgNU5AYpxLGsycuRkQILgIjkS5GEy7cIelLrADhiqGWICTzO8sD", + "oaATVACQ80e3F7fu/FS9i4NZAJeE0QKM9A2bWV54eazl5eZLW80pDVs2Uqk7ZmQ7yyoHiCieMi4VDWUr", + "vgm3sT7exKsxHuXDIRoRpuiEEmEFVYJExuBac4MgqxtodaFKB7ksPSJzrQWN5nsjFabLQKlpCuXDK132", + "xRVTuuby488pZQ2SVvfu1UTmmAJNHpE5NVdLVRiyRzOKBJ0T4WHf+Y1qWKFphzY0rWsWwjgjmxVIsTmN", + "KG7DDiJY04h6sOf08BiZz+j4CG3MyFV1kp3H4ydB85AMJx5c+CVLMOtqgtDLcuND2/LYL/e8Mj9Pkmw0", + "FTxLl0c+fn1y8g7BR8RAZCyP+GTHJ/qlIR3hKBJESv/+3cfy2vr9fn+Adwb9fq/vW+WcsIiLRpCaz36Q", + "bvcjsmLIViC14y+B9NX746PjA3TIRcoFKEFrCacMnvK+ymhTPRUf/v+c0Thaxvqx/pmIUX6J+AB27MSo", + "4yMnJ9h+6P0J2tA8JCLjbDqlbLrZBt9DrsGhrzrfJQ5LRbaNVhOVk1JufN+GguA10+kWrSZbJrXMnOQo", + "kU2juyaaoyY0jqkkIWeRLM9Bmdrfa95MiWDMDbU01TP9M0qIlHjq7HqgfhhmqgWbCaYxiTbbCbNNm/md", + "j0tXSAW9AS26eBxu7+x6eUeCp2QU0ak1udWvKP27RjE9jkLQ2r8RuMzb7QOmBOtdfb7nwLphEkEmRBCN", + "4184XSr4nDBstZdVVkMA5mnR/HMn+CMjGRmlXFKzwiXOZb9oNAJQI+jhXzN8WnXWJYySCovV9AEtvgIl", + "FnLdWthYs4UWbfB0bZe3uk2ddwJrzGWJEhdoZJHPtFDjkQ44U/ZDzXzJpyimzGgcWrQzZwFy1SIlP8Uc", + "WOJXgkMO/mXi1+u+AfMyPzSMpr8VdveYT8vQnBEs1JhUgNlwhdmBitU1gv+0Qj61uwpLMlrNQU4pYyQC", + "e7ElbNNSi7FeNQOo6IKq0ZwI6aU5WNavVCHbonGomIcXExqT0QzLmTWwRRE1xsLTyk480lp5yhMM+rgb", + "EKQI0F/PfjnYebSP7AQeGFrLpW6wvJNSbz28aYsUFmMcx17caEa369/Ryxjix4DCWNl09+QY6BDTcLrA", + "nqbVkzM5M38B79argrtPswGNXrH+2/c4dAhMwmgJjY9DfhkwtwxPY65hukAZo39kFQG7h44nYCDWFwWN", + "SNRBGD6A3UHrf1PCiNB8qrAMlYRgtEF6014HDbVc2NVScBfvdPv9bn8YVMXYeK9r1PsUK0WEXuD/+Q13", + "Px50/9HvPv1Q/DnqdT/89U8+BGgrmTup0O5zw9F+B7nFlsX1+kLXifI35v7l5fs4jjnqY80nrnvSh8fL", + "goPZa8TDCyJ6lG/FdCywWGyxKWVXgxgrIlV156vbflVYwD5WAIFNNZiuCYaa0gNovBHzSyJCzYFjohFP", + "djQTpkp2ENZ6MzAvpG/Jv6EQM00LRrjgAhEWoUuqZghDuyq0kkUXp7RLzVKDTpDgq5eETdUsGOzvLuG5", + "RvIN+0f3w1/cT5v/7UV1kcXEg+RveKYomyL4XH7Wc2vIn2hWnYiDbhabF2LKjk237eU3qC87YbeRVSdt", + "lLnmh+9M8dxEtu7le+l9VytbiUd1eD0nQtDIXcuHJ0doI6YXxNILEhlDw6zf3w2hAfxJ7C8hTxLMIvPb", + "Zg+9TqjS12FW3PLmybb2ukbCGQdBJY75dZ7TQFIEBQfHK+/xVaDxQvswH3f51v+FS9VNMMNTAuqobYjG", + "gl8QvVDzJkCJRBdkoaWcBZrqQbtzKuGFh7A5mmNjdegN2dsZl8Q0cZ8k2PbpnKCEhxfm6XfGQZOf4zgj", + "soMuZ1rkAJsgwbH9GZmHsSGb6UXKkKck0kqIaQZbQ+eEzc9RglMgcywI0DhKsCKC4ph+NE/48MpAIqpv", + "uCEjQBgoxZrmw5CLCF7YOCI4nJWg8GeJzo3Acg7Dn1Om0frcEGbtsfpT8Prd259fv3t1NHp9+uzVwfHo", + "12f/q382nYLBb58C42KTSyo/EyyIQH/6BPv9bMTbiIhgEBxkasYF/WisNZ87gYaB1PiFU9rjKWGY9kKe", + "BJ3gL+V/fvj8wQlkxow912TgWdhnrzBk7lIPSzpy1kCJrIXJvW1okGkW9eL03Za+nVMspZoJnk1nVcKw", + "osG1SCKi8mJE+Wic+tZE5QU63nqNtOCCYqoJNBdUtvv9k5+35DDQ/3jk/rHZQ0eGamH5mgdxYeUncBUq", + "vD4OT98hHMc8tDaUSdMDr5vKx+AJU2KRcupT4mrMqWi6zKO63eLrNVjR1piyLamPoRteD+6ANzdWJZ6x", + "ORWcJVqdm2NB9T0tq7Ty6vXRs9GzV++Dgb4Ioiy0VsnT12/eBoNgt9/vBz4E1Ri0hge+OH1nXj0N2ag0", + "zqYjST96RImDfH8oIQkXRoW2fdDGrCppGLpFcDjDYPfFzwa5tl8AXrlDsW9E+Shm4Nqz3ouffdgyW6RE", + "zKn02dl+yb+5k19296ngtnkly5EWsLhX0l/CmGdRtzRlJ5hQQUJwr9D/+oMkWpCff6w+S3n6+c1frQTY", + "NZIpjlPKyArR9BsRES+5uIg5jrrbX1lCtA+qHtcY86F6vvnLmkOJJY+zMWbRJY3UbBTxS6aX7OGr9gvK", + "G+fM9UrvBMf//ue/3p8Uetb2i3FqOe32zqMv5LQ13qqH9tpQ8o1kqX8b71L/Jt6f/Puf/3I7ud9NGEHk", + "RkKdPf9nZoS604z1JTTm0IaX4fz2zh1WFLcKNXRHDvfWPgP7GDWfExHjRYnx2jUF233gfrVVCQpeksj2", + "02z0AunOa9iwHs1d8i/qSv5O389oPYvyrOlnzSvsvdBmJflCtndO7J87y0tqWNEFTUcgNY/wNLf5rnIJ", + "PbugqRXFoYc5xjg2jCDKQHgfc656Q2Y8VPTZwQGTKxICz5MKK3RweizRJY1jsBABU1m+WrRgX3JtguZS", + "6f8WGeugcaa0tM4VQVZvgkkyWAs0HhOUMezew2uys93gsnsBgOWCCEbikZGNZUvImE7IdmoEDmx1gqV1", + "URMqS6vwOvr15AxtHC0YTmiIfjWjnvAoiwk6M94Fm1XodYYsFeCmoCfR9EztvHyCeKa6fNJVghC3xAQG", + "y21s9rF2/uL0nX3ul5u9IXtDNGAJi6yjr7txrBNoxNmfNcWSqDpsef4a0JtcOiTDqZxxNUpz5+lV3OnM", + "Ni9U8fbGhE4wD9OseqQ7nUYn0DkVKsOx5rUVcdL7wG9c3WVTFENZfbF8rxyKUHmZbWtxMSODS7vXXdZj", + "ODGSUmvDSUmVXzKhOD3zU7vFrhn/mLmFrDQcFarmF8x1ZgZZct4xP3fczm4ApeMcJjVz09cBz4Esqeat", + "nM+ND5aRCCXaONfavMVjrb+fd9D5Xyo/aNp3qoWWLy6RgQbwE6Z/Ko9fN0qsNRdcy927fDhY3vw8DmSj", + "pxOabyMlMJPGR22GU9JDvwATR4okqeZkbIqoRLlrF2L88m+IG6HGdR0yvTRp/EQsOHKjkaRTRtl0EyKH", + "xgThKDKWpUmmMqHbzaksoFlFHWe9WfJqNasjhh9DhARlYZxFBJ07C895VS5ctv8sq4TWILSk4RiQgGYD", + "yp7aSjKlp9cbTrAKZxpOPFPGccxuverUV7MyrXtQtWvJn9pucP5nObuoB8LMPSqO3px95AGzYMk+2WQG", + "tIKK30R5QRZw5M4ciZcMkmVLpN9eKIjk8ZzYa7dsyxxDqA83glNhxkQ21C0PDKoHufisc+uOQsOrNfir", + "qoInxEeqrttsgTFW+nc+4Y4L6c2Z+TpaMZYEgA+qxwCBOHbeMboSAQsEYhpZYhRRQUK1NDxl0yEDH5Jz", + "+0vPjnauiVzLKF8lcAriEEBoLx8tKp2sE/tgGL01nlClSNSpygYXhKRy/aa0eG0N1x7ruiCXgjpG5pyK", + "W4pnhE24CElilYQvUxyflQbzqnHXG2LZpcPAt7RmF58B0SkkMv5D5jzAzFoJ26geeuFi7LQ240JQnfIc", + "x/E52rCNNpEgv4Mnvj0rxlmB7G8PTx0K5M/e7086GiM1FzifKZWO9H/Jkabi8/pgtq+j8CKy7Ekf9Ku9", + "vd1eJYrULLg2bNW+5nWLaD4aJ343vqxpvNCrtH4mbUT5w6JLYUm9oCxqO8Cvum2jdS4XjJymcdsGulSQ", + "bpZOBQYX269pnrvxuylAs5mDrwkT9rlJFhGCmVQ8Kfvbb9RcPGjVGaQKrDmPuxFWGEyZLe2tNjB8yfE4", + "WZihjC7WZIkZTccevyH6EUIBpnSKxwtVfT/Y9kbzfekjtluL71iaHPiNBkmikeKrXZjpBLm2bTwWTbyB", + "4qP5hPLV4R3W/6USf2euI6vX6iG6aUitOQFknHBmPEwNEEBofH9SfruDUH69uAE6yifIh82HxMwE15uX", + "kw0uSoswgRxovNhEGL0/6aG3+Wr/LJFWWObERTTMsERjQhjKwPQMt2HX3MXlBWQSLk1V725tJyb4YROe", + "KLn91stjjsFKk0dQg6vUmNb2YyIn4aDsmzBmZStYK6vVKsfvN2RKpRI1t2+08eb54e7u7tO6/XLnUbe/", + "3d1+9Ha7P+jr//yjvYf414/v8I11UOUt1vmszH0O3x0f7VhjaXUe9XEPP31ydYXV0316KZ9+TMZi+vsu", + "vpMIED8rOyq85tBGJonoOjapscrnK1dySWvwhbuxi9steawVDrir2hpIvNUtbyO0xec0bV12rx98UmeY", + "a92uS5tb1uQXKeidBZWUJDjr3RhSrx/nEZUXPwuCLyBkb/neTvCUyJG5z/z+DJk0Tjbkylo3BOdqIs27", + "adXqub33eO/J7v7ek37fE9GxjPA8pKNQ30CtFvD68BjFeEEEgj5ow6ZSGcd8XEX0R7v7Tx73n27vtF2H", + "eeJpB4dc8XK90IaFyF9dNhP3pbKonZ3H+7u7u/39/Z29Vquy9uJWi3K25YpI8nj38d72k529VlDwCfTP", + "XIRNXYD3RVYemOh+/a+uTElIJzREEKODdAe0kcAVRvLXqipNjnHk4k/9d4fCNJYrPSbMZLalMbTlKXXg", + "GxxIK1s07PwIRvJmyGAsj/e93kg2Lmmth4DbS94EVeLLKqA7MQHNJeGJkjgaGApdy+fgNIuFfWjCA7uH", + "ltjwUqtO3ZjMSVxGAnN1mchaQVCOJ+bQKruibI5jGo0oSzMvSjSC8nkmQBY1gyI85pkyz4w2QLuYBLye", + "QfeYaHbdTs99zsXFWv9RfRPncehrrUIHYEifWFMN3OIY2d4uRKEk9OXPgebR1H6X6I3pYSxExc9pVs1q", + "04GZrCWJIUGk4sBJrcHQDtNWuvTLLWAsde4fZr6Cd96R70t3YtwFvq6GLaYE8i+otRKLxpS30P4Mmrd2", + "R9cd1xpSWsCdkcu7ADr463c12nYlw+ntQHyVM1puaygawS0saER6CKgLvGJcfGCN0s4UT1MS5faf3pBZ", + "f+78J2leUHRHAwc1I1QgLuiUVieuGthu06vtOqjosOnG6FjuuCyhwkdw32gmejxRJtfChQuZIuX4JXsI", + "QSc4yzNTWE5UBc2bPLvHEkQKV8ulJb44fXdd37RU8An15RsCXwj71Wpmzmvr5V7/rLv9/xkPTI1vIKJR", + "ZvwnEh7VEknY9u1unhen706b1pSndkDl1S3tKfd4WZXcqkhACI9K9lXSajAO/fXFkk9SyN5PfbLsROCE", + "jLPJhIhR4jGuPdffkWlgXJsoQyc/V+VZLTe31ZpPK4cDavMEhzYyvx30PQa52jY6JWh+8B/XG2Ku4aZ4", + "Pn1UwraxIX099CpPpoFenL6TqPBS8ljqqsfb6C9/OltIGuLYjGjCcykrG9gAOVtLyKdFR2uK9MjJ/hws", + "jhDQxnyaZkCGZ2+6x6/fbyURmXcqawLPohmPiV73ZolbzF1UX+HcX2ES8yZLh0EM2ZaASrDKKbg1kEr0", + "6oGO4grHIxlzn7PGW/0RwUe08f65ibrSK+igtHKU+vcSFCr4ve+lGM2RmqY9gwnrJtMKgXt1x2o2TGNe", + "KW2vMqmPVH4hODapQqv4vJwAiV9UD5pfrE+6YwbxzXvsHMNrSo0veOvw5MgIDCFnClNGBEqIwjYxacnF", + "BcShoBN09R0VYZKAq93kb6u9WxpM8OVorEYj7uFS3o5bMeA2xJu/MS4IEUowoxMilY03r8wsZ3jn0f7A", + "ZMWIyGTv0X6v17tujMqzIiil1VFsGRf+UrhKT86+7BxuIRSlzV4+BacHb38JBsFWJsVWzEMcb8kxZYPS", + "v/N/Fh/gD/PPMWXeEJZWiVToZCmBSvVJU99Z5vdBKeely+/XKq+dX58BzwaIm/PGGys81fqJwbgvDSy+", + "ceqRIv+VKqUcKTuEtkg/Qj+utoQ6wQja2DkzpmhcZGZZtoHeKLeOXJl+YCn1QEpYnnAgjs1fIWdzTRW+", + "7AMVBu6+fdH7gfVyGUXUg8l/t9qecZKAqKr19BZs4TRdj7Z+QTHnf22zrtjYaM9NdO9c/yZvbNXZX0//", + "54//X54+/n37j5fv3//v/MX/HL2i//s+Pn39RRFUq8Pi7zW2/auFs8PDUiWmvS0qnWAVegSqGZeqAcL2", + "C1Lc+Gv20CEofoMh66KXVBGB4wEaBjUX4WGANsgVDpXphThDeigb6QDJ80+N+Ud3/uR0y8/1MSIb0iDs", + "geSRTDIbRzzBlG0O2ZDZsZDbiIQ3ff1XhEKcqkwQfXpaho0XaCwgrbdVz4vJO+gTTtPPm0MGGi65UkLv", + "IMVC5Xk83AyAFHZVxmfANieRCww3GvKQ5fdSHhdubDS93AgCtvm6x6UfKF71hYtqKM6Tvi+CHry+9EHG", + "VCoCjtk5Zms0yt3R0JN+hVU86T/prxXwcxxagX5ACUvYlzikbEFLBoFhasO4wUOthS1d8yZDI+iXt29P", + "NRj0/54hN1ABi/yIjZJnfAClsRGqWJa8/zYDb7ZRON2WGzJGMugWt4gaembcQ9++PEOKiMQ57G+EGpwT", + "Gur9wfM/lTLTqEgxOjg8ebbZpmABwDZf/4pzfJvvsB7cYY1mTbbAHOM1fDvo+Ajccy2FFgIcuNU85wLF", + "hsEUdD1A7ySp+rrCUZlXfXOS8aKwvJkbYBhsuhHTOqcYoDe53IjzpVSKJFSNeQVdwrD24cX4/CyN3llK", + "Py6cXmRZG3j4YJU7iesbt5kVrCZ/D8SB5q1fd8mmeT3aLhtD9WR+1CjO/mtnTfn64s7udZXc62Z4qAZh", + "lgJ48yQP7bMz3EaWg2WF74qqUeMrPtKf7Zu9U2ven6AZluzPCj7WlJvt3cet8nXqWdu+f5dfvvnELCkn", + "SxfRmb/bmtjWCxrHxh1C0inDMXqKNs6OX/x6/PLlJuqi169P6kexqofvfFoke3C08eL0HYTLYDlyT0jN", + "XpO48DwmV1QquRzw2uoldnVyiV8qCSC8EcSbXzErhHu+XtrGXeR7uE+/wG8v18TK7BBfmuLBSsu3lOGh", + "kbn6siNU+az5+evmariV5awtL1IWKpzT9o2TI3QC6nFYPZCaBZIIHZ8WSRYLq5YbvrYnW6tnu9/vbffb", + "2PgSHK6Y++TgsP3k/R1jyRjg8SCMBmTyBTZGi9hG+sPxJV5INHTy+TAwCkFJEyiRrZXhW73fLueguFnK", + "ibpAsS6pxHWSSLTLDvGlIfmrUi2fVZMstxbyvqASSSsXCne1W+cJ22t0HfM5QSHP4kgLUmNNukaxI5HV", + "PyVRRf5qoPZ37ILxS1bdurGiagbwR0bEAr0/OanY3AWZ2PS8LTYOThcN58DTax3DzhpZe+1qbpio4S6S", + "M9TZbum6++qpGMpGP+fEaTC0hfGvED+9D++UmaPReLJiTzWzTUTmoyzzSVX6kwvdePfu+KiCHBjvbz/p", + "P3nafTLe3u/uRf3tLt7e3e/uPML9yW74eLchQX57x5ub+9JUqbk5VAoADyZQEwkXDTS95c4w40yh3FFO", + "E/KhFk9RSQ42gUFglThmVEESSMqmehgwElgx2UR4mjyVlFEFKQUgoQ1lestgjYGCoqbDAL2AtvAJJxCw", + "5BahlaOqIQJHC2OI1YzBTZ3Cv1Yv+WyWQbkf6CNnmUJQHkpvW4PBqiurhzA8ZoBecegjnJcq43W9xzQH", + "m8By87qOtGH9kpz/KkxmGeYAPc+ZZM5mLVvdkMT+aXi3da0Gt/HNivOePfFAY0txciW/tE5gIBp0Agco", + "8F9b9mSz6/IGaZRR0fdCQXAMLLTwFMoUjW2WBNgJhQJJpvgtHG4TJduMYCQaGRGg6b3RuJ9YMSHv5BjF", + "+xO0AfGQf0VWqdT/2szfJstUubfzdO/p/uOdp/utoh6KBa5n8IfgHLW8uLXcPkyzkas90rD1w9N3psRh", + "yJnMEmMlsHsvOZmmgkPtYcpQUcykmPxp72k52CPimSnsZJdkI8M+l8qXraw80/DA9geN53QyYX98DC92", + "fhc02b7alztjr3JX1EnzSsLHZVPrktpIxl2TxdHvjw8IJWRjyMobImEH6IwoBPjTRTiESzr3abIo5wJb", + "LMS9iLW3u7v75PGjnVZ4ZVdXIpwR6K/LqzyxKyiRGLREG2/OztBWCeHMmM7RExJMMCvA+ekM2YTO/Wol", + "0N52f9eHJQ3yUoE1dux50gjy91YIspuyQAfXrFxAWqJyL7R3d/uP9x49edSOjF3dPXG1msO4pB4GPDYP", + "SvnkN8A8//bgFOnRxQSHVQ1le2d379H+4yfXWpW61qogh4/JvXGNhT15vP9ob3dnu13slc8Eb6MKKwRb", + "5V0eovMghec0PKBYZr2dptvCJ3gaBHtDwhjT5CB07jO128fk2BgJ06w4hDYXgzUSLF1cLfq2UtFqhYOM", + "aMAFKlVc7K03h97MutnMps19sJ6NL8vQMWYaXDZIwKRyvAHsUkHmlGfyKwzEFQk1Mk1izsW1+jb5I70h", + "MouVMUFSid6f/BmYiEYuJBVJq772Fv1WhFLccHPXIuAKTvixuglYrU6jzdGv2nCngUw7q/xoK+TfGLEU", + "aVaVsfVv34c4DjNIXobz89S7gtgDnil4qV8YL5E45pyhcIbZlEAyeJMqkU0RRjMeR73A/1QSR6OJ9wkj", + "rzDPi/rlbhG6m6u/v/GCFyXtDCrV8vM+SgxXsZmbei1qyRc1cRsinDQ8seKlNACmS0Wbj/lUghaowP+l", + "V88+k2Jh3FowM3nq5olRHquhWzv6tvcssca9fVeouTr5xGq0VsZQPIckDgWXsijM/f6kusxVDox5Pfv1", + "79nVxbZAXZlyJom/TLytCd/K4OO7ED2eYV9yJQIOgwPoqhrQNo9hglkGmb5KiEyuUioMerR7HJ9xqUZ5", + "OMo1FyvVCLI4ZYIUMWvuvpxBAMDCsDho470XHWu7Cbjy8sY36L2EVf6hmhbYzFO9EPVDq5PjoA+NlwNy", + "VsYAFUFF9QiS64SMFWl/qIRRaSlaCW0wripsqZS6ZrPNQ5VfR9XzNJWUfbnXP2sbzbU6eOsUq9kxm3BP", + "bshrGPytS7zzXUiJgPLjnKGIMEoipzzmln9r2wIn+1gSFGXEQs4IpAJbgGND3pADkjmjGGXTGq+vT9jG", + "DG/WsDrJE8xrG7Z5cpR+1+y3IgNYGScBiXDhpN3K44HKkd9SvDywINMsxgLVIxZXLFkukpiyizajy0Uy", + "5jENke5Qf86Z8DjmlyP9Sf4Ee9lstTvdYVT4GNaeZ8zirIepOZDavMUWftK73Kz5t4PpZcv039L9W73g", + "ev2GntOY2KC+d4xelRC9mgVlb6ffFPrQMGgl6GE5IPS6nNuirI/iXazmQV41weOObzyAaq8SVUNkZb++", + "3YKL2apAj2VTDNpwj8Iuy0wVrqVsL60sIe283OruD241W5KE1dn3njx6vN8y3c4X2TpXVFX+AsvmPFlh", + "0Ww4qZM2ZrMnj548fbq79+jpzrUMVM5TpuF8mrxlyudTK45SM5o96sP/XWtRxlfGv6QGf5nqgiqFTm68", + "oM8rSLcIs2549mjM8x2XT9K9s1QtoO1sjCukpYOKyFWq5bVBJhMCSuXIwK1bLKbmnt9qDSFOcUjVwmMw", + "wZcm5XvepBYu3MaaVl2sB6R2bJvxQXMumY0Lh84NNzn6izGt13DhSeusXTIbN5nxX9dnNUb8wgZUfiJq", + "8UJTFBZYNhfk+7nEsuLVof8OIWdzUaut7j9kWrSvSu1wPS9MXXhG+kLe/UWoy8dfO86S2bciJNchvuoK", + "bSbBa+nQnhvZV6ZyvVdujT/YC/BmvUbjcj69lQkLK8n3ilv3+vO2qzK33M/cYNefr+QCep2O9dRigI92", + "DRbkxdidCko0YJPiYn1G6VtIEGR8Cm6UIsi6I9xJliD7861kBlo6jjOiXNszrdFn8YqE0EwRMccew5Qb", + "ArkmVTuq4cQdZE18aDvZrJUq3Jv5ZTUbwtvSfUxLgaNUkAm9WoEtpoG5rqvu49JCIKomDZdoI8FXaO8x", + "CmdYyNraGZ3OVLyoGln3PNETX1bEmSgtOrdPr16cpuu4/KJhj7M8uo9kz0qxDv607yQarQp0P8ybOZtx", + "ihcgWzYqgo939/r93Z3+jSLdv1Y2+tI4TR6hpX7WmFN5eiyPkPt/LqcsvBTUFDVzYJJKEJwMwJsqxSFB", + "MZlAHFieKnatTr809erF20dS6/if4787KFe51NpZLItjnIHg4caxSQLcNgL3SluN9Sh/X172imCxnM2E", + "S1Fjdf/V/W5/t9vff7u9O3i0P9jevo3Q+BxITS48jz9uXz6Od/BkL36yePzH9uzxdCfZ9YYL3ELhg1od", + "wVodBLuHlIh6Lsp6DldJYspIV+Zub+sdkFfwAvOStJb+r2d9MDtYKSycVTdZlhmwKoBTL8l2F4FNdvUr", + "TSj15R8frV72jfzI6gvxI1h9KYBP7RYDqVq2vzQvSMZa3jvvSg1b3zwrfRvX3T0+p28gbe8pN0Dch88V", + "xlihsFU39vKt5lHhplxQNUtWXw95szzLADyGf5QqqgbS9NDxlEHm2fLP+dtHuTi07hx0gvjjXpVm7O/t", + "Q6psVH2OgPaoy2JAi7cBSGy8GgrQpFAthHFPwIIAIH7a7m4/hRf6+OPeT/3u0x76e8lToGOgVQbftmtd", + "+bXfBoY5owS507ycbz+91jO6g+cqDPrV3ktNF7GNt7c4XqT9dHeFc5uuHHDxeemMa1FFt1Ro6POKHTvB", + "+Xppe1wvj2jS88smT9/2t6+dtud6V0TvCzyKv0jVa6fexViqJrn6JZYq18eQyKx03YESN6xW6d6mFbeh", + "AODaCbH2AyRrGr7Lpy4LocdZATpoyhUqggDWSjmwfJExLz5U11+UKYb0EI0IsbMGIdqtKQ/ko6tI9/gI", + "pYJHWVh4wMaw6CwMiZSTDKou99pKtOvfGG9TmQfXcq3Rr1fmm7T39WGm5Kr5vF+RK1WaUiNs81Fv99cf", + "9a1YADpBlkbreZhp1I6DXSsTxxqfSo89ogr2mhRU2syHFhz9TRmCywoeFGxCoRYHstRVENQ4tYxJ0lM3", + "EF+NvDkCjkhMFPENgkxJT+v2QWXBRdez1O39J36TGb4ahRCQuLSQXwlJtZyecGlLbCaYLbwLq2fRRht9", + "V0FSIhi+azJ5WWhVF/d4rRTSeFTtE5LXLLom4quc/z3P0PF1s5HbnmtrRdyeoPLWqkpNGWXqvp5lu+NB", + "9x/GzohGvcHWT3/9f7sf/vInf22WiiIliehGZAIPYBdk0TVFZ7XS1qvmM4VsN4FU2FY0UQQnYEUIL4ix", + "WiT4qrzeR/2ckhavcLK0BXg5TCjL/712Q3/9U/O7WwmM74B5rD3HL85+dBupZRV3PHojIWLq8rc7dzEo", + "583ihT4qiUr56+w978j6zzLvUi4ce25kox6UOx5TSAMqh0yrOTgMSapI1LN5vCisRXAgyXr5ZJtHz7l3", + "a2LFkDzUhtvU0mR98hYpHgSMXHbNDFFX497eo31bMb4Mye2lI/YduonYbqqwqKHsMSO8pBLCEZzXbakx", + "2iBJqhYuS6zzi9y8XgT5QT6g9yn0K6fP6j/9GtlC361MD/od1vcsB/i7Ba0N7V86/8acfH7HqqN6ph5D", + "k7ZmWTWzTE1lkqrb7HeV6Dt+BO6Dyz5S+ptxTbT5MKdZPTH4VsLUls2+6wuHiKAk8Upn1ILKXLR7Fzqt", + "97FcKWWWdlZaSfPZnDhhql4muhlApxo0lzMiSOkgoEORQvSaILOOgi2CbEyOzJSIbr2onam7ICh4HuZ6", + "sANB7ky6bBhbneHmBF/lM4BRFculpwfYR5HrbfvFz1BL5Y0rbkYnbghYRk3U9aerqWJRm/Lsy4dRxqrl", + "fZv2XsKzvGoF92uirRpyFnNUUNOHj3/HVD3nAoTj5pCWW896A4J3RATE9NZz2rRKCEMTEo14plbTv00y", + "b+NZIjQmEy5IKf+uUwQwILGtebyGF7igi2INH3xygyRhJqhaaM3RiqRjggURB5kheAAkTAQ/FxNDOtvP", + "n8GENvF4sL0gjAgaooPTY6BHqNuvieP9CYrphISLUCvgkI10Kf8GCHmvD4+t8uUyvsGDIlWAeq4M8cHp", + "MVQ1FUYBCfq9nV4fiDklDKc0GAS7vW2o8aoRDra4Bdnv4U/rnG7i0ihnx5GVg342TXQvgROiiJDB4DeP", + "k7ciwmTTlyB14mlJb0gxFVZxSGNwPTeoQnVfSH/krtKBuY87BuCtbUdSLawjHklf22P9oDHBUA1scaff", + "N3oaU/bixUW1y63fbcBeMW8reQ7A48kFtCTXO5nSgvxzJ9jrb19rPWsLVPqmfcdwpmZc0I8ElvnomkC4", + "0aTHzHgHI5Nnwvo/lOkMUKhMYb990OclsyTBYuHAVcAq5bJJGCYSYSiRZ0o5/M7HPWQt45D+VM54Fmtu", + "gozrM4n0hYU1T+lNPyIswhmdkyGz97QpNooF5JdOkL6fjdpSJQ0ztTn9PCjtZx4tatDNh9vSw3WdPbQA", + "cD3NoiQjyBY1aqrTUlhCKWNQ71ESm5MyL1iw7GQBBXplyL2ViQnDTBX1Xk1l3guysMZW74Ct0rpohgfH", + "QqAQfJ6vfGfTH88A2TP9oUBH+TdkwVsVJxi8UIRxFhUyl3OxxWKM49gb9z+N+RjHtoDxBfGIqC+ghQVK", + "OdGoE24Yj4hJGpku1Iwz83c2zpjKzN9jwS8lEVoEstmnLaxt9U6LulBJniaQAdrUttBzbpklbn26IIvP", + "vSE7iBJXt0Ta4vOx5Lays0mfQyVyPpkGd/3pTRue+w8zqXhiUYqVC1GaZfJMpZmyT52SKJsyG5pDnVI5", + "I9GQKY4+CVOWfvF561Mx42fQXQiONJ6UmpgtbX2i0eemVcsR1rsfQVOP9kcAAMNA3y7DQP89FVjrLpmc", + "gSlDgvliWj7SjTwWW8uFm3UIh5ihlKcmjh2QyhSsrowB5QdwHCMFpOT6amkTTrJhPzY0xVdLz8almECC", + "GhlBVb0SMfX3nvjpSZJQEJ+B43/OXr9CcFXpMzDNCrMRwIgyfYuiKANJHmbvDdkzHM6QkZsgV9kwoNEw", + "yLWLaBPWmknrONvtgoj7k17aT2aaDo1+6vX0UEZ6HqDfPplRBpqW0mSk+AVhw+BzB5U+TKmaZeP82wc/", + "QJvc+88qjABtGN6/6YrHQJqB4ho09wZmEeKW18YLhFHBgcp2lDFlWKysfOMBvYWgVuXxVJaB8WkIFtRh", + "MBg6G+ow6AwDwubwmzW0DoPPfghYIbo5MZYp/uNk7RyJ9vv9zfVxdxa+HhG60lCT3+cl6WvnqwkeVuha", + "FjzM5lxWP32CpoyTEbfuQPL5GUeuMMAPEW+NiGctFyXhDfqX7wGDvjExCm5NAtP6bOwksJXaiUELSGsJ", + "GoeLkjUKB3USXIG8ZfWjrs4vqxV7TVQWwhJjh397d4B/MG9RCh3mfXpX8+IYclTmhYEfFjrCYTlE7Pg1", + "4hdEfQsY178rVmoTat4n/j4U/HlBrNxXAK3GzbbI3L03+XMBQAyAtKOYxlpXPYM1dc8IU+gZ/Nqz/+s0", + "Hshsex7z6fkAGRDGfIpiyuxrXOm1SF+KFpbQyYQB5P1sVIBLxLRh7s9///NfsCjKpv/+57+0NG3+AnLf", + "MtkxIHHr+YxgocYEq/MB+pWQtItjOiduM5BakcyJWKDdPoiZqYBPnmKTcsiG7A1RmWClV0uTE0naAUH1", + "YLAfyjIibRiFbkgnNmGDMTB7VHhHywaUd0rRnWWfU7OD0gb0rehwACJwqclea/WvwG89M3uu2M/qtvIl", + "i+l6/qLIlTLY2zULvCaDARD76A4+2E2jjbOzZ5s9BDqGwQpIygESczGMFZ57P3jSep5kOEqVoQCUDW8q", + "1RlvtP8e2TbtDMB2xO/JAtxUOL3ZBGxMHkSQyMHrh67Qxhzsh5szDfvss0cueK7ZQHvz/ZancN5ErRTh", + "r3fODveWYW6+lEB2Hyow2nCO2q7m3+nhsasNs3lvSH8nt4beqa2okF8diJtKg3emlh1yNolpqFDXrQWS", + "+ickV9WqCPJQ2MEbu2qE3b7q6e/K99tWJZtL402XJ3Yprrzbvz1qk17nGilS9BW49uMmWYc6R1SGXPct", + "YUs3xKktWGjEl5xOy1i0ziBl/L7zK2eluGTZ8/GRI8i7M03ZqTNWvxvugCke1RjiPTLCWhG2UlLLh4TN", + "7/JTdIkCVliuvi3U7N+dFHTXViwfmj8kM1ZUA5vmgia1b+MF+oKoX0yLWzxoO4Nn42dEOKp2OYhh1/m2", + "TFcUzkh4YTYED9Krdd9j06Sd6mvG+540XwDPdSQWC/IfIkoLZbeA1SoF99gWlrs9/RZmuJZ6+/XeeS2C", + "eYAMziZjZ7E2NduwXLBw87t66r2T28wA+0FeZqdZHLsXjzkRCuX1m8t3wNYncEtaL9s7alt5Hbx787JL", + "WMjBDy33ofILUfbLV5bwzYGZrfxAkzY6oYnYpe4+a5JwvuD8jbsgyuuD/9fOc1sh/L92npsa4f+1e2Cq", + "hG/eGrL074o137XE/YCRTwvctAo0YE0MaoWuk1DzVi2FVNf+u5JTzaavJanmcP0hrLYRVsvgWimv2qO4", + "VYnVzHFPTzI5svmgDZ+cf+J3JqnerZXPYqTL30tl9dnDFmjhAuy88IkylEnyAB0oaY5x5Wujpbm6IMiV", + "14dD3eOjDgCyo0EHCYVsgMgdGa/dOu5cuLXz3r3l+iAZ02nGM1mOPUmwCmdE2mClmFQZ8EMTu4vruVHw", + "/oaxtH+XV8edy9U/8P6WJP76gRrmbV6g1sn8rlVbmd+21zK/LZ5vYtfeuKL8Nk/SZoNToQuibovGlVjz", + "ZWdH37p8ugh6pxWVQl1AoEEMhuy/tf7xmyI4+fCTC5LJ+v2dffidsPmHn1ycDDtxqEKYEtQm7zx4dQTP", + "flOIPof8nkVIXn0dpiAAoJ5LYPMfpyAVL5/tNSSHhT80pFYaUglcqzUkexa3qyJVk2DduY7k8M0HcJvE", + "5IeWdBdakswmExpSwlRRL2vJScyW23uAsWXMvg+VnDsqF21rLSknyjUCaJGt/c4de/LJ7145conhH6aP", + "PDdRMZFTR4rLsFkf+dbwoX+3zPnu9ZCHjGJG4F8GXaplSl8ZRsj0mGQKnBKLDCHg9YmEkdrzEXuoqH4o", + "szTlQkmTLRIEYCiHpWZaAPZllqwmi/Rlh4TEuJTIzpBBAnn92cTyb12QhckFSXlR1T/fqc3/6Iu9qubi", + "vFcy+voylj/RaCsZ647J2OZTvj8Z695Yx51IWseVNPUbOWGAQjkmOSXzPLiPfqRsuvmgPFANs8r3Vspn", + "5BG1tqAAm82uuyXzQq9NF20pwa4tT/gfeOMub9IntbtctCUAoojiKeNS0dAF7tYTef+4oVvf0Ksh68Xm", + "iS2v6Vfon3Nx0faK85R7egA3XXmH36AtQS8PsoHdv0kBlG1zG2ikufNbcKmG132GYND6vRjGWaQvQnch", + "OlFyIngysj+afLWaKmw2UDBRhHbU+2Y2evY7MBi94grRJI2JluJJhLoGm/RpWtHfJX2nslTx7nrMUJNN", + "OSDGJKOTrmqOZZHwuOYObAPe2ZePy8s1Yz5dnwQjn9xlfPBkwRgyk5SeuAz25yhnskhxJElMQoUuZzSc", + "QUYM/ZsptAnJKnCanucpsDYH6AVQajkTGEy+IYnQilDImeQxMYku5klyPljO2Pr+5AQ6mWQYJjfr+QC5", + "LK35BSF1q3KGi7wczyubt2NDY5LgcWxO9FxrjaX9bdrcF0WKsiHz5cFg5NIOSCfovJQS47whJ4ZjqC/5", + "9N6krU5zYkmzF8WRAMAZ3CQsCpoeYmjsz4ax3feWKmmZmcMs45YTcywt5iWf5kktK6iM07Qt+tplAhbP", + "k2QFDqONUs1MqSKeqb9KFREhoLPF7ibkRhs4NP9Q+EIjqq15k1cdBfTzPjeaLHNeUGmmWirwYv41T5Kg", + "E9j1lLLTXUN6X5PhpD7g8rOYPplSGpMfcvd1EpRUmX0pQ0nt5rBV2ZtFblts/ru3z1pARd+DlaX6nlWs", + "gjInqsDZ8qJA1IPKdAAHuSSLmfJEPhpxu+zKUl3Lds9bSxUxvwGldd2rV17eMK+9eNfPX8sreMhBMHJp", + "NxMu6uHx697FvnlE+npHsrTVNhjyAzevb55rhZhptqLMJVTplGDng9KPkNc5nHEuS2g/JjM8p1zYDOzW", + "6ppjJpgsjPZovefONaqeW/vtuRXPB9bWhHD5k52jB92tz52/h/tU9Hhe0rZzjt9xIjVkgZQIo7GgZIJS", + "nEmipaUsIchUGLGJvAkOZ66Ic2/I3s4IsqUbSwaEvNIvleh8OznvoHGmUIzFFLQd89F40gkS8iQhLDLl", + "WIdsRvCcalVNoBgrwsJFVxIozzsnRQETrbrbF0oED5R5AdAOcnVjwcBwXqoKe45SQQCJjLrMKiVYh0xk", + "7G8mc6Ue9twt9BwRqfA4pnKW14oIcURY6E0LefZts7Gvb8Q9I2q5cOq9vFneiJfe5yNm2ZaZl67+Jt43", + "H5ijFheuvmULNr9C6JXNqmHV8/GsKBb7H0jSZq9uj/f0MpODeBUVfxtPMpVq8T+eZZQlySgz05FqRfXv", + "9q0lZygoY5XnFmuTvemDS14JIQfztXje1if35/ENbGTfCCfsNCr2TTm3i01/CyzXQvVGPPeejIPWllSy", + "it0jC7aLuj/xiYsSl/sm2LAhuJwbl3mOEhh0Ks5+MOM6M7buATdlxs7iuvQAXmLPlHXTGDfx5aJ2vJ8B", + "W4PAf6j3a213JUZ474yveBG4M2Z3nLM3w/BSvIg5/t7fZUIuhAnotOWIH05CsZItsPTAtAEWt07OITou", + "muT9yclmE5cQaiWPEOoBc4hqWdMw8VRrfD0nQtDIlY48PDmy3qtUIpGxHnqdUKjneEFICoViKM8kgsjc", + "nt6fC21dLoJXiWHtBIQpsUg5ZWrtKoqmt7OYzzcqnXfHfNKmVPzuH4/BCv/wmBTwDi2u2A2s1iIVVo3O", + "eM45jTJT71JLW3jMMz265iyu0O4U7rYJjYlcSEUS45k3yWIgIki6a2sy2X4morSDqJJI00MHIvBSIhIq", + "JeVMDpkt/54SoefW3aH4b+Fk5DXeK5xzzVPD+r4NBza9GOOzhVUT1CC1ANQBDQbBFk7TLSgX7XeSssv7", + "giU9B480JBfJmMc0RDFlFxJtxPTCKB1oLlGs/9hc6dI2gn5fu+LUzSlLQ/qYTbi3KIfB2RyZv48gpCpb", + "c4+ID46tvSBlYnH8Bw7az9bkWr4mCI67iiYkD35HmaIx/WhYnR6ESkVDE1dThF5CEWYbfTlkJ0QJ3QYL", + "gkIexyRUzriylQoebg2zfn83TClkKdklsDhgeM2fE5jx8PQdtDOFojtDpv8BA789ODUvsRNsbQSlhTKi", + "Lrm4QMdbr9c4+Z4BmP6DveTMBlfGQHoP/Mfz3fUjmxtpSDaQKE9XKUA8/e7dOK0E98Na8DCtBZBaIt/N", + "xlTgEIRiOctUxC+Z3zIw53GW6H+YP47XJShROJy9h6bfjLRrlrN2GrfBB0GUdk8RMUWD7uWBwgDsofqX", + "asC5LYAQU/Hc894CB+p7xO6vb5Qvw/EbfJq0EHUFub4Z2rrrm8+uweXdKsPjoZC5wTS3E8VXW58uMW22", + "Pv0c8/BCoowpGleSGmi9DfKA6h+LvI324Q/EBIiOdKXEEblKqYAMNrX0CIjoHUuEkSIioQzHW7BnMwhk", + "oHRWLDznFIKUw5hCmBiNCEp5HEOWncsZYUjvBgxVboDSO620FSDKbcpPjIqjMQl5QlxWzk2f6vZ3TNVz", + "LqopNr8Vvvi2BH+9H71Vvc81WUWbZ/yiLKMn+ArcmqPMPhO7FW284MWPxhTUQXA2w2C3L4dBBw2DnWQY", + "6BM4xGBCxQo9QgllmSKyh46MfQvCUPf7SJKQs0i65KDOgrfbl01BqQYtGyIc96HfXYo9FqsAlG/sJD72", + "oNsh3R8CbNBGmeAsTUYdILoI8UyBA7ejK9sqIgrMI5t3/gJbopEfun0bTv53S74VHgWnrNll6egNZ8/T", + "R661urmgihmXRdZJFOIUh1QtOgjHMQ8L60Em89eBbr6UsSD4QutQvSF7kyeutIEQ6PD0XccZzVBE5YUZ", + "wdrFeuj1nAiZjfPFIeAGxoIHh0GiIVMchTgOs1jjLZlMSAgxDDFNqJINdrV8KbdZBrGYxHPw7mOetuZh", + "GZP8OAGnV6CFrGHcljnqLUHCGNOkbFSqAwdEX3jSBbPvWA/K9TU8ie3zVii4lMgO1SUxndJxbB9rZA+9", + "1SIHTsiQpTFmjAiUSeN3pJfeTQWRMjOBMXoAqDNrMKqDikQnqeDKmoljzoU0ll2N4e9PkFQkXYFmb8zI", + "J7DnW0oTbAa3M92TwlBbQ/O1ZJsgfSAGUwzANR7pa/oenH3Mgu47nfBDIfy3gk6nRGiqwIbJmqdRQ9YO", + "nIboK5EejTnyz/JW7XLk56OWvLlLns4rE1WMXMMRCNDXeYH1TH5BG3OZ2E/Xi774VXdqOXfVy9+/CPvp", + "C3f5vZQeOys5V7fNrF9g+ENLcl9aeYVUKwEK69MRtI5IuM0IgdZ5B+4t3cBDzjKAK2EHTekEvj1E6N9t", + "dNxdp9l+2LhVyRJQKazTECq1Pn3nN4GBt5O3856jQ2+Qt/ObileCvIv3Fzf6TUUqVeyArnjId5+Z87YC", + "lEx6Tkhj0RSgZLiedSRYqSi9t23aqUl2xO9Jgrdvz9eQ3x3Yf2j9LVSGErD8JjsTG+3ytpAkVQv3uMgn", + "tQdAST9CMIYv8UPuQ3B7+RZu8Lz+9dDD4Wnj4/qPelp39n5fFB0+Pnr4RbTKNFe5WLb0rdPFIpzROWk2", + "ulcp2IIoFaSb8hQeVyIDMAsPd5cpLHrTj8gOb3NV2X8h6lIckwhFVJBQxQtEmeLAEcwcf5ZIcK0JwHcu", + "Fj5jeplynwueHNjdrLkPLU1ZY1jx5pssuhFWuDt33GaFCe0LXtrd27ZmeIgy9OJntEGulDAZd9FEaz6I", + "TnKQkquQkEgCTm6WF7zdb7Bs0o9kNB23WeWK3MmvbW5qFGZS8cSd/fER2oBiC1PC9FloUX8Ckmwq+JxG", + "phBpAdQ5jw1UtxsAel27qxYq8koZTrkwi7sXGabNhTT9SNMqWzCuC8EgGFOGYXFrsxRXacoEVOn5MIWw", + "hoJ2HOYEP64wq/ltOGVHY6JWchwQFecmNd7mj2vuIV9zZcdUd6dVbrt2pSLb+aq2dCG9jYS5uR/z3Zqt", + "33877pVUPkjPSms6n+cKaZPZ/NtCwf7d3Q93bS5//4Dd8V8Qp3yXTOUwgB7RhzAveYhjFJE5iXkKVSRN", + "26ATZCIOBsFMqXSwtRXrdjMu1eBJ/0k/+Pzh8/8NAAD//6lKy2cZdAEA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/volumes/manager.go b/lib/volumes/manager.go index 27ddafbc..0f89a040 100644 --- a/lib/volumes/manager.go +++ b/lib/volumes/manager.go @@ -25,12 +25,11 @@ type Manager interface { DeleteVolume(ctx context.Context, id string) error // Attachment operations (called by instance manager) - // Multi-attach rules (dynamic based on current state): - // - If no attachments: allow any mode (rw or ro) - // - If existing attachments are ro: only allow new ro attachments - // - Multiple rw attachments (ReadWriteMany): internally backed by NFS, - // transparent to the caller. NFS is set up automatically when a second - // rw attachment is requested. + // Access mode rules: + // - ReadWriteOnce: exclusive rw via block device (reject if already attached rw) + // - ReadOnlyMany: read-only via block device (multiple ro attaches allowed) + // - ReadWriteMany: shared rw via NFS (requires network, NFS set up automatically) + // Legacy: if access_mode is unset, readonly field maps to ReadOnlyMany/ReadWriteOnce. AttachVolume(ctx context.Context, id string, req AttachVolumeRequest) error DetachVolume(ctx context.Context, volumeID string, instanceID string) error @@ -399,13 +398,13 @@ func (m *manager) DeleteVolume(ctx context.Context, id string) error { } // AttachVolume marks a volume as attached to an instance. -// Multi-attach rules (dynamic based on current state): -// - If no attachments: allow any mode (rw or ro) via block device -// - If existing attachments are all ro: only allow new ro attachments -// - If existing attachment is rw (block device) and new is rw: enable NFS -// for ReadWriteMany. The volume is loop-mounted on the host and exported -// via NFS. The new attachment (and any subsequent ones) use NFS. -// - If volume is already NFS-served: additional rw attachments use NFS +// Access mode rules: +// - ReadWriteOnce: exclusive rw via block device. Rejects if already attached rw. +// - ReadOnlyMany: read-only via block device. Multiple ro attaches allowed. +// - ReadWriteMany: shared rw via NFS. Requires NFS host (network enabled). +// +// Legacy readonly field: readonly=true → ReadOnlyMany, readonly=false → ReadWriteOnce. +// Neither legacy path triggers NFS. Only explicit ReadWriteMany uses NFS. func (m *manager) AttachVolume(ctx context.Context, id string, req AttachVolumeRequest) error { lock := m.getVolumeLock(id) lock.Lock() @@ -423,61 +422,88 @@ func (m *manager) AttachVolume(ctx context.Context, id string, req AttachVolumeR } } + mode := req.ResolveAccessMode() + + // Log warning if both fields are set (access_mode wins) + if req.AccessMode != "" && req.Readonly { + fmt.Fprintf(os.Stderr, "warning: both access_mode and readonly set on attach for volume %s; access_mode takes precedence\n", id) + } + + // Classify existing attachments + hasRW := false // any non-readonly block device attachment + hasRO := false // any readonly attachment + hasRWX := false // any ReadWriteMany (NFS) attachment + for _, att := range meta.Attachments { + if att.NFS { + hasRWX = true + } else if att.Readonly { + hasRO = true + } else { + hasRW = true + } + } + useNFS := false + readonly := false - if len(meta.Attachments) > 0 { - hasRW := false - allRO := true - for _, att := range meta.Attachments { - if !att.Readonly { - hasRW = true - allRO = false - } + switch mode { + case AccessReadWriteOnce: + // Exclusive rw via block device. Reject conflicts. + if hasRW { + return fmt.Errorf("cannot attach ReadWriteOnce: volume has existing read-write attachment") + } + if hasRO { + return fmt.Errorf("cannot attach ReadWriteOnce: volume has existing read-only attachments") + } + if hasRWX { + return fmt.Errorf("cannot attach ReadWriteOnce: volume has existing ReadWriteMany attachments") } - if allRO && !req.Readonly { - // Existing attachments are all ro, new is rw → conflict - return fmt.Errorf("cannot attach read-write: volume has existing read-only attachments") + case AccessReadOnlyMany: + // Read-only via block device. Reject if rw or rwx exists. + if hasRW { + return fmt.Errorf("cannot attach ReadOnlyMany: volume has existing read-write attachment") } + if hasRWX { + return fmt.Errorf("cannot attach ReadOnlyMany: volume has existing ReadWriteMany attachments") + } + readonly = true - if hasRW && req.Readonly { - // Existing has rw, new is ro → conflict (rw is exclusive or NFS-only) - return fmt.Errorf("cannot attach read-only: volume has existing read-write attachment") + case AccessReadWriteMany: + // Shared rw via NFS. Reject if non-NFS rw or ro exists. + if hasRW { + return fmt.Errorf("cannot attach ReadWriteMany: volume has existing ReadWriteOnce attachment") + } + if hasRO { + return fmt.Errorf("cannot attach ReadWriteMany: volume has existing ReadOnlyMany attachments") + } + if m.nfsHost == "" { + return fmt.Errorf("cannot attach ReadWriteMany: NFS host not configured (networking required)") } - if hasRW && !req.Readonly { - // ReadWriteMany scenario: both existing and new want rw. - // Transparently enable NFS serving. - if m.nfsHost == "" { - return fmt.Errorf("cannot attach read-write to multiple instances: NFS host not configured (networking required)") + // Start NFS serving if not already active + if meta.NFS == nil { + exportPath, err := m.nfs.startServing(id) + if err != nil { + return fmt.Errorf("start nfs serving for ReadWriteMany: %w", err) } - - // Start NFS serving if not already active - if meta.NFS == nil { - exportPath, err := m.nfs.startServing(id) - if err != nil { - return fmt.Errorf("start nfs serving for ReadWriteMany: %w", err) - } - meta.NFS = &storedNFSInfo{ - Host: m.nfsHost, - ExportPath: exportPath, - } + meta.NFS = &storedNFSInfo{ + Host: m.nfsHost, + ExportPath: exportPath, } - useNFS = true } - } - - // If volume is already NFS-served, new rw attachments use NFS - if meta.NFS != nil && !req.Readonly { useNFS = true + + default: + return fmt.Errorf("unsupported access mode: %s", mode) } - // Add new attachment meta.Attachments = append(meta.Attachments, storedAttachment{ InstanceID: req.InstanceID, MountPath: req.MountPath, - Readonly: req.Readonly, + Readonly: readonly, NFS: useNFS, + AccessMode: string(mode), }) return saveMetadata(m.paths, meta) diff --git a/lib/volumes/manager_test.go b/lib/volumes/manager_test.go index 0220e20b..cef6209d 100644 --- a/lib/volumes/manager_test.go +++ b/lib/volumes/manager_test.go @@ -179,7 +179,7 @@ func TestMultiAttach_RejectRWWhenExistingRO(t *testing.T) { Readonly: false, }) assert.Error(t, err) - assert.Contains(t, err.Error(), "cannot attach read-write") + assert.Contains(t, err.Error(), "cannot attach ReadWriteOnce") } func TestMultiAttach_RejectDuplicateInstance(t *testing.T) { @@ -401,19 +401,11 @@ func TestRWX_RejectWithoutNFSHost(t *testing.T) { }) require.NoError(t, err) - // First rw attachment succeeds (block device, no NFS needed) + // ReadWriteMany should fail because NFS host is not configured err = mgr.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ InstanceID: "instance-1", MountPath: "/data", - Readonly: false, - }) - require.NoError(t, err) - - // Second rw attachment should fail because NFS host is not configured - err = mgr.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ - InstanceID: "instance-2", - MountPath: "/data", - Readonly: false, + AccessMode: AccessReadWriteMany, }) assert.Error(t, err) assert.Contains(t, err.Error(), "NFS host not configured") @@ -567,6 +559,275 @@ func TestRWX_DetachKeepsNFSWithRemainingConsumers(t *testing.T) { require.Len(t, loaded.Attachments, 2) } +// --- AccessMode tests --- + +func TestAccessMode_ReadWriteOnceExclusive(t *testing.T) { + mgr, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + vol, err := mgr.CreateVolume(ctx, CreateVolumeRequest{Name: "am-vol", SizeGb: 1}) + require.NoError(t, err) + + // First RWO succeeds + err = mgr.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "inst-1", + MountPath: "/data", + AccessMode: AccessReadWriteOnce, + }) + require.NoError(t, err) + + // Second RWO is rejected + err = mgr.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "inst-2", + MountPath: "/data", + AccessMode: AccessReadWriteOnce, + }) + assert.Error(t, err) + assert.Contains(t, err.Error(), "cannot attach ReadWriteOnce") +} + +func TestAccessMode_ReadOnlyManyAllowsMultiple(t *testing.T) { + mgr, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + vol, err := mgr.CreateVolume(ctx, CreateVolumeRequest{Name: "rom-vol", SizeGb: 1}) + require.NoError(t, err) + + for i := 0; i < 3; i++ { + err = mgr.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: fmt.Sprintf("inst-%d", i), + MountPath: "/data", + AccessMode: AccessReadOnlyMany, + }) + require.NoError(t, err) + } + + vol, err = mgr.GetVolume(ctx, vol.Id) + require.NoError(t, err) + assert.Len(t, vol.Attachments, 3) + for _, att := range vol.Attachments { + assert.True(t, att.Readonly) + assert.False(t, att.NFS) + } +} + +func TestAccessMode_ReadWriteManyUsesNFS(t *testing.T) { + // Test via metadata (NFS loop mount requires real disk, so we test the stored state) + tmpDir, err := os.MkdirTemp("", "volume-rwx-am-*") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + p := paths.New(tmpDir) + require.NoError(t, os.MkdirAll(p.VolumeDir("vol-rwx-1"), 0755)) + + // Simulate two RWX attachments with NFS already set up + meta := &storedMetadata{ + Id: "vol-rwx-1", + Name: "rwx-vol", + SizeGb: 5, + NFS: &storedNFSInfo{ + Host: "10.100.0.1", + ExportPath: "/data/volumes/vol-rwx-1/nfs_mount", + }, + Attachments: []storedAttachment{ + {InstanceID: "inst-1", MountPath: "/data", Readonly: false, NFS: true, AccessMode: "ReadWriteMany"}, + {InstanceID: "inst-2", MountPath: "/data", Readonly: false, NFS: true, AccessMode: "ReadWriteMany"}, + }, + } + require.NoError(t, saveMetadata(p, meta)) + + // Verify round-trip + loaded, err := loadMetadata(p, "vol-rwx-1") + require.NoError(t, err) + assert.Len(t, loaded.Attachments, 2) + for _, att := range loaded.Attachments { + assert.True(t, att.NFS) + assert.False(t, att.Readonly) + assert.Equal(t, "ReadWriteMany", att.AccessMode) + } + assert.NotNil(t, loaded.NFS) + + vol := (&manager{}).metadataToVolume(loaded) + assert.Len(t, vol.Attachments, 2) + for _, att := range vol.Attachments { + assert.True(t, att.NFS) + } + assert.NotNil(t, vol.NFS) +} + +func TestAccessMode_RWXRejectsWithRWOExisting(t *testing.T) { + mgr, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + vol, err := mgr.CreateVolume(ctx, CreateVolumeRequest{Name: "conflict-vol", SizeGb: 1}) + require.NoError(t, err) + + // Attach as RWO (legacy readonly=false) + err = mgr.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "inst-1", + MountPath: "/data", + Readonly: false, + }) + require.NoError(t, err) + + // RWX should be rejected — there's an existing RWO attachment + err = mgr.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "inst-2", + MountPath: "/data", + AccessMode: AccessReadWriteMany, + }) + assert.Error(t, err) + assert.Contains(t, err.Error(), "existing ReadWriteOnce attachment") +} + +func TestAccessMode_RWORejectsWithRWXExisting(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "volume-rwo-rwx-*") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + p := paths.New(tmpDir) + require.NoError(t, os.MkdirAll(p.VolumeDir("vol-rwo-rwx-1"), 0755)) + + // Pre-set metadata with an existing RWX attachment + meta := &storedMetadata{ + Id: "vol-rwo-rwx-1", + Name: "rwo-rwx-vol", + SizeGb: 5, + NFS: &storedNFSInfo{ + Host: "10.100.0.1", + ExportPath: "/data/volumes/vol-rwo-rwx-1/nfs_mount", + }, + Attachments: []storedAttachment{ + {InstanceID: "inst-1", MountPath: "/data", Readonly: false, NFS: true, AccessMode: "ReadWriteMany"}, + }, + } + require.NoError(t, saveMetadata(p, meta)) + + mgr := &manager{ + paths: p, + nfs: newNFSManager(p), + nfsHost: "10.100.0.1", + } + ctx := context.Background() + + // RWO should be rejected — there's an existing RWX attachment + err = mgr.AttachVolume(ctx, "vol-rwo-rwx-1", AttachVolumeRequest{ + InstanceID: "inst-2", + MountPath: "/data", + AccessMode: AccessReadWriteOnce, + }) + assert.Error(t, err) + assert.Contains(t, err.Error(), "existing ReadWriteMany attachments") +} + +func TestAccessMode_LegacyReadonlyDoesNotTriggerNFS(t *testing.T) { + mgr, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + vol, err := mgr.CreateVolume(ctx, CreateVolumeRequest{Name: "legacy-vol", SizeGb: 1}) + require.NoError(t, err) + + // Legacy readonly=false → ReadWriteOnce (no NFS) + err = mgr.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "inst-1", + MountPath: "/data", + Readonly: false, + }) + require.NoError(t, err) + + vol, err = mgr.GetVolume(ctx, vol.Id) + require.NoError(t, err) + assert.False(t, vol.Attachments[0].NFS) + assert.Nil(t, vol.NFS) +} + +func TestAccessMode_AccessModeWinsOverReadonly(t *testing.T) { + mgr, _, cleanup := setupTestManager(t) + defer cleanup() + ctx := context.Background() + + vol, err := mgr.CreateVolume(ctx, CreateVolumeRequest{Name: "precedence-vol", SizeGb: 1}) + require.NoError(t, err) + + // readonly=true but access_mode=ReadWriteOnce → access_mode wins (rw) + err = mgr.AttachVolume(ctx, vol.Id, AttachVolumeRequest{ + InstanceID: "inst-1", + MountPath: "/data", + Readonly: true, + AccessMode: AccessReadWriteOnce, + }) + require.NoError(t, err) + + vol, err = mgr.GetVolume(ctx, vol.Id) + require.NoError(t, err) + assert.False(t, vol.Attachments[0].Readonly, "access_mode=ReadWriteOnce should override readonly=true") +} + +func TestAccessMode_ROManyRejectsWithRWXExisting(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "volume-rom-rwx-*") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + p := paths.New(tmpDir) + require.NoError(t, os.MkdirAll(p.VolumeDir("vol-rom-rwx-1"), 0755)) + + // Pre-set metadata with an existing RWX attachment + meta := &storedMetadata{ + Id: "vol-rom-rwx-1", + Name: "rom-rwx-vol", + SizeGb: 5, + NFS: &storedNFSInfo{ + Host: "10.100.0.1", + ExportPath: "/data/volumes/vol-rom-rwx-1/nfs_mount", + }, + Attachments: []storedAttachment{ + {InstanceID: "inst-1", MountPath: "/data", Readonly: false, NFS: true, AccessMode: "ReadWriteMany"}, + }, + } + require.NoError(t, saveMetadata(p, meta)) + + mgr := &manager{ + paths: p, + nfs: newNFSManager(p), + nfsHost: "10.100.0.1", + } + ctx := context.Background() + + // ReadOnlyMany should be rejected when RWX exists + err = mgr.AttachVolume(ctx, "vol-rom-rwx-1", AttachVolumeRequest{ + InstanceID: "inst-2", + MountPath: "/data", + AccessMode: AccessReadOnlyMany, + }) + assert.Error(t, err) + assert.Contains(t, err.Error(), "existing ReadWriteMany attachments") +} + +func TestAccessMode_ResolveAccessMode(t *testing.T) { + tests := []struct { + name string + req AttachVolumeRequest + expected AccessMode + }{ + {"default (neither set)", AttachVolumeRequest{}, AccessReadWriteOnce}, + {"readonly=true", AttachVolumeRequest{Readonly: true}, AccessReadOnlyMany}, + {"readonly=false", AttachVolumeRequest{Readonly: false}, AccessReadWriteOnce}, + {"explicit RWO", AttachVolumeRequest{AccessMode: AccessReadWriteOnce}, AccessReadWriteOnce}, + {"explicit ROM", AttachVolumeRequest{AccessMode: AccessReadOnlyMany}, AccessReadOnlyMany}, + {"explicit RWX", AttachVolumeRequest{AccessMode: AccessReadWriteMany}, AccessReadWriteMany}, + {"access_mode wins over readonly", AttachVolumeRequest{Readonly: true, AccessMode: AccessReadWriteOnce}, AccessReadWriteOnce}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, tt.req.ResolveAccessMode()) + }) + } +} + func TestCreateVolume_MetadataRoundTrip(t *testing.T) { tmpDir, err := os.MkdirTemp("", "volume-metadata-*") require.NoError(t, err) diff --git a/lib/volumes/storage.go b/lib/volumes/storage.go index bcec7b45..36e59e0d 100644 --- a/lib/volumes/storage.go +++ b/lib/volumes/storage.go @@ -21,7 +21,8 @@ type storedAttachment struct { InstanceID string `json:"instance_id"` MountPath string `json:"mount_path"` Readonly bool `json:"readonly"` - NFS bool `json:"nfs,omitempty"` // True if this attachment uses NFS + NFS bool `json:"nfs,omitempty"` // True if this attachment uses NFS + AccessMode string `json:"access_mode,omitempty"` // "ReadWriteOnce", "ReadOnlyMany", "ReadWriteMany" } // storedNFSInfo represents persisted NFS serving state diff --git a/lib/volumes/types.go b/lib/volumes/types.go index f86d77ab..bd6d27a8 100644 --- a/lib/volumes/types.go +++ b/lib/volumes/types.go @@ -6,6 +6,18 @@ import ( "github.com/kernel/hypeman/lib/tags" ) +// AccessMode defines how a volume attachment can be accessed. +type AccessMode string + +const ( + // AccessReadWriteOnce is exclusive read-write: only one instance at a time. + AccessReadWriteOnce AccessMode = "ReadWriteOnce" + // AccessReadOnlyMany allows read-only access from multiple instances. + AccessReadOnlyMany AccessMode = "ReadOnlyMany" + // AccessReadWriteMany allows shared read-write access via NFS. + AccessReadWriteMany AccessMode = "ReadWriteMany" +) + // Attachment represents a volume attached to an instance type Attachment struct { InstanceID string @@ -44,6 +56,21 @@ type AttachVolumeRequest struct { InstanceID string MountPath string Readonly bool + AccessMode AccessMode // If set, takes precedence over Readonly +} + +// ResolveAccessMode returns the effective access mode, applying field precedence rules. +// If AccessMode is set, it wins. Otherwise, Readonly maps to legacy behavior: +// - readonly=true → ReadOnlyMany +// - readonly=false → ReadWriteOnce +func (r *AttachVolumeRequest) ResolveAccessMode() AccessMode { + if r.AccessMode != "" { + return r.AccessMode + } + if r.Readonly { + return AccessReadOnlyMany + } + return AccessReadWriteOnce } // CreateVolumeFromArchiveRequest is the domain request for creating a volume diff --git a/openapi.yaml b/openapi.yaml index b1884bac..451c9150 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1122,6 +1122,15 @@ components: description: Creation timestamp (RFC3339) example: "2025-01-15T09:00:00Z" + AccessMode: + type: string + enum: [ReadWriteOnce, ReadOnlyMany, ReadWriteMany] + description: | + Volume access mode for attachment. + - ReadWriteOnce: exclusive read-write (only one instance at a time) + - ReadOnlyMany: read-only, multiple instances can share + - ReadWriteMany: shared read-write via NFS, multiple instances simultaneously + AttachVolumeRequest: type: object required: [mount_path] @@ -1132,8 +1141,10 @@ components: example: /mnt/data readonly: type: boolean - description: Mount as read-only + description: "Deprecated: use access_mode instead. Mount as read-only." default: false + access_mode: + $ref: "#/components/schemas/AccessMode" Health: type: object