Add ReadWriteMany (RWX) volume support via NFS

Introduce a volume access mode system with three modes:
- ReadWriteOnce (default): single rw or multiple ro, block-backed
- ReadOnlyMany: multiple ro only, block-backed
- ReadWriteMany: multiple rw/ro via NFS backing

NFS volumes don't create local disk images — they store server
connection details (server, export path, version, options) in
metadata and pass them through to the guest init config. The
guest mounts volumes via NFS instead of block devices.

Key changes:
- Volume domain types: AccessMode enum, NFSConfig struct
- Volume manager: RWX-aware attachment rules, NFS creation flow
- Instance layer: NFS config propagation, device count skip for NFS
- Guest config: "nfs"/"nfs_ro" mount modes with server details
- OpenAPI spec: access_mode, nfs fields on create/response
- Validation: NFS requires networking, no overlay for NFS volumes

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: sjmiller609

cmd/api/api/volumes.go

@@ -46,11 +46,31 @@ func (s *ApiService) CreateVolume(ctx context.Context, request oapi.CreateVolume
 		}, nil
 	}
 
+	var accessMode volumes.AccessMode
+	if request.Body.AccessMode != nil {
+		accessMode = volumes.AccessMode(*request.Body.AccessMode)
+	}
+	var nfsConfig *volumes.NFSConfig
+	if request.Body.Nfs != nil {
+		nfsConfig = &volumes.NFSConfig{
+			Server:     request.Body.Nfs.Server,
+			ExportPath: request.Body.Nfs.ExportPath,
+		}
+		if request.Body.Nfs.Version != nil {
+			nfsConfig.Version = *request.Body.Nfs.Version
+		}
+		if request.Body.Nfs.Options != nil {
+			nfsConfig.Options = *request.Body.Nfs.Options
+		}
+	}
+
 	domainReq := volumes.CreateVolumeRequest{
-		Name:   request.Body.Name,
-		SizeGb: request.Body.SizeGb,
-		Id:     request.Body.Id,
-		Tags:   toMapTags(request.Body.Tags),
+		Name:       request.Body.Name,
+		SizeGb:     request.Body.SizeGb,
+		AccessMode: accessMode,
+		NFS:        nfsConfig,
+		Id:         request.Body.Id,
+		Tags:       toMapTags(request.Body.Tags),
 	}
 
 	vol, err := s.VolumeManager.CreateVolume(ctx, domainReq)
@@ -61,7 +81,7 @@ func (s *ApiService) CreateVolume(ctx context.Context, request oapi.CreateVolume
 				Message: "volume with this ID already exists",
 			}, nil
 		}
-		if errors.Is(err, tags.ErrInvalidTags) {
+		if errors.Is(err, tags.ErrInvalidTags) || errors.Is(err, volumes.ErrInvalidRequest) {
 			return oapi.CreateVolume400JSONResponse{
 				Code:    "invalid_request",
 				Message: err.Error(),
@@ -182,12 +202,28 @@ func (s *ApiService) DeleteVolume(ctx context.Context, request oapi.DeleteVolume
 }
 
 func volumeToOAPI(vol volumes.Volume) oapi.Volume {
+	accessMode := oapi.VolumeAccessMode(vol.AccessMode)
 	oapiVol := oapi.Volume{
-		Id:        vol.Id,
-		Name:      vol.Name,
-		SizeGb:    vol.SizeGb,
-		Tags:      toOAPITags(vol.Tags),
-		CreatedAt: vol.CreatedAt,
+		Id:         vol.Id,
+		Name:       vol.Name,
+		SizeGb:     vol.SizeGb,
+		AccessMode: &accessMode,
+		Tags:       toOAPITags(vol.Tags),
+		CreatedAt:  vol.CreatedAt,
+	}
+
+	if vol.NFS != nil {
+		nfs := &oapi.NFSConfig{
+			Server:     vol.NFS.Server,
+			ExportPath: vol.NFS.ExportPath,
+		}
+		if vol.NFS.Version != "" {
+			nfs.Version = &vol.NFS.Version
+		}
+		if vol.NFS.Options != "" {
+			nfs.Options = &vol.NFS.Options
+		}
+		oapiVol.Nfs = nfs
 	}
 
 	// Convert attachments

lib/instances/admission.go

@@ -3,7 +3,7 @@ package instances
 func volumeOverlayReservationBytes(volumes []VolumeAttachment) int64 {
 	var total int64
 	for _, vol := range volumes {
-		if vol.Overlay {
+		if vol.Overlay && vol.NFS == nil {
 			total += vol.OverlaySize
 		}
 	}

lib/instances/configdisk.go

@@ -98,9 +98,28 @@ func (m *manager) buildGuestConfig(ctx context.Context, inst *Instance, imageInf
 	}
 
 	// Volume mounts
-	// Volumes are attached as /dev/vdd, /dev/vde, etc. (after vda=rootfs, vdb=overlay, vdc=config)
+	// Block volumes are attached as /dev/vdd, /dev/vde, etc. (after vda=rootfs, vdb=overlay, vdc=config)
+	// NFS volumes don't consume a block device — they're mounted via the network.
 	deviceIdx := 0
 	for _, vol := range inst.Volumes {
+		if vol.NFS != nil {
+			// NFS-backed volume (ReadWriteMany) — no block device, guest mounts via NFS
+			mount := vmconfig.VolumeMount{
+				Path:          vol.MountPath,
+				Mode:          "nfs",
+				NFSServer:     vol.NFS.Server,
+				NFSExportPath: vol.NFS.ExportPath,
+				NFSVersion:    vol.NFS.Version,
+				NFSOptions:    vol.NFS.Options,
+			}
+			if vol.Readonly {
+				mount.Mode = "nfs_ro"
+			}
+			cfg.VolumeMounts = append(cfg.VolumeMounts, mount)
+			continue
+		}
+
+		// Block-backed volume
 		device := fmt.Sprintf("/dev/vd%c", 'd'+deviceIdx)
 		mount := vmconfig.VolumeMount{
 			Device: device,

lib/instances/create.go

@@ -420,14 +420,31 @@ func (m *manager) createInstance(
 	// 15. Validate and attach volumes
 	if len(req.Volumes) > 0 {
 		log.DebugContext(ctx, "validating volumes", "instance_id", id, "count", len(req.Volumes))
-		for _, volAttach := range req.Volumes {
-			// Check volume exists
-			_, err := m.volumeManager.GetVolume(ctx, volAttach.VolumeID)
+		resolvedVolumes := make([]VolumeAttachment, len(req.Volumes))
+		for i, volAttach := range req.Volumes {
+			// Check volume exists and get its details
+			vol, err := m.volumeManager.GetVolume(ctx, volAttach.VolumeID)
 			if err != nil {
 				log.ErrorContext(ctx, "volume not found", "instance_id", id, "volume_id", volAttach.VolumeID, "error", err)
 				return nil, fmt.Errorf("volume %s: %w", volAttach.VolumeID, err)
 			}
 
+			resolvedVolumes[i] = volAttach
+
+			// Populate NFS config from the volume for RWX volumes
+			if vol.NFS != nil {
+				resolvedVolumes[i].NFS = &VolumeNFSConfig{
+					Server:     vol.NFS.Server,
+					ExportPath: vol.NFS.ExportPath,
+					Version:    vol.NFS.Version,
+					Options:    vol.NFS.Options,
+				}
+				// NFS volumes require networking for NFS client access
+				if !req.NetworkEnabled {
+					return nil, fmt.Errorf("volume %s: NFS volumes require network.enabled=true", volAttach.VolumeID)
+				}
+			}
+
 			// Mark volume as attached (AttachVolume handles multi-attach validation)
 			if err := m.volumeManager.AttachVolume(ctx, volAttach.VolumeID, volumes.AttachVolumeRequest{
 				InstanceID: id,
@@ -444,7 +461,7 @@ func (m *manager) createInstance(
 				m.volumeManager.DetachVolume(ctx, volumeID, id)
 			})
 
-			// Create overlay disk for volumes with overlay enabled
+			// Create overlay disk for volumes with overlay enabled (not for NFS)
 			if volAttach.Overlay {
 				log.DebugContext(ctx, "creating volume overlay disk", "instance_id", id, "volume_id", volAttach.VolumeID, "size", volAttach.OverlaySize)
 				if err := m.createVolumeOverlayDisk(id, volAttach.VolumeID, volAttach.OverlaySize); err != nil {
@@ -453,8 +470,12 @@ func (m *manager) createInstance(
 				}
 			}
 		}
-		// Store volume attachments in metadata
-		stored.Volumes = req.Volumes
+		// Re-validate with NFS info populated
+		if err := validateVolumeAttachments(resolvedVolumes); err != nil {
+			return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
+		}
+		// Store resolved volume attachments in metadata
+		stored.Volumes = resolvedVolumes
 	}
 
 	// 16. Create config disk (needs Instance for buildVMConfig)
@@ -614,10 +635,13 @@ func validateCreateRequest(req *CreateInstanceRequest) error {
 }
 
 // validateVolumeAttachments validates volume attachment requests
-func validateVolumeAttachments(volumes []VolumeAttachment) error {
-	// Count total devices needed (each overlay volume needs 2 devices: base + overlay)
+func validateVolumeAttachments(vols []VolumeAttachment) error {
+	// Count total block devices needed (NFS volumes don't consume a device)
 	totalDevices := 0
-	for _, vol := range volumes {
+	for _, vol := range vols {
+		if vol.NFS != nil {
+			continue // NFS volumes don't use block devices
+		}
 		totalDevices++
 		if vol.Overlay {
 			totalDevices++ // Overlay needs an additional device
@@ -628,7 +652,7 @@ func validateVolumeAttachments(volumes []VolumeAttachment) error {
 	}
 
 	seenPaths := make(map[string]bool)
-	for _, vol := range volumes {
+	for _, vol := range vols {
 		// Validate mount path is absolute
 		if !filepath.IsAbs(vol.MountPath) {
 			return fmt.Errorf("volume %s: mount path %q must be absolute", vol.VolumeID, vol.MountPath)
@@ -648,6 +672,11 @@ func validateVolumeAttachments(volumes []VolumeAttachment) error {
 		}
 		seenPaths[cleanPath] = true
 
+		// NFS volumes cannot use overlay mode
+		if vol.NFS != nil && vol.Overlay {
+			return fmt.Errorf("volume %s: overlay mode is not supported for NFS volumes", vol.VolumeID)
+		}
+
 		// Validate overlay mode requirements
 		if vol.Overlay {
 			if !vol.Readonly {
@@ -765,8 +794,11 @@ func (m *manager) buildHypervisorConfig(ctx context.Context, inst *Instance, ima
 		{Path: m.paths.InstanceConfigDisk(inst.Id), Readonly: true, IOBps: ioBps, IOBurstBps: burstBps},
 	}
 
-	// Add attached volumes as additional disks
+	// Add attached volumes as additional disks (skip NFS volumes — they're network-mounted)
 	for _, volAttach := range inst.Volumes {
+		if volAttach.NFS != nil {
+			continue // NFS volumes don't have a local block device
+		}
 		volumePath := m.volumeManager.GetVolumePath(volAttach.VolumeID)
 		if volAttach.Overlay {
 			// Base volume is always read-only when overlay is enabled

lib/instances/types.go

@@ -30,13 +30,22 @@ const (
 	EgressEnforcementModeHTTPHTTPSOnly EgressEnforcementMode = "http_https_only"
 )
 
+// VolumeNFSConfig holds NFS connection details for an NFS-backed volume attachment.
+type VolumeNFSConfig struct {
+	Server     string // NFS server hostname or IP
+	ExportPath string // NFS export path
+	Version    string // NFS protocol version (e.g., "4.1")
+	Options    string // Additional mount options
+}
+
 // VolumeAttachment represents a volume attached to an instance
 type VolumeAttachment struct {
-	VolumeID    string // Volume ID
-	MountPath   string // Mount path in guest
-	Readonly    bool   // Whether mounted read-only
-	Overlay     bool   // If true, create per-instance overlay for writes (requires Readonly=true)
-	OverlaySize int64  // Size of overlay disk in bytes (max diff from base)
+	VolumeID    string           // Volume ID
+	MountPath   string           // Mount path in guest
+	Readonly    bool             // Whether mounted read-only
+	Overlay     bool             // If true, create per-instance overlay for writes (requires Readonly=true)
+	OverlaySize int64            // Size of overlay disk in bytes (max diff from base)
+	NFS         *VolumeNFSConfig // NFS config (set for ReadWriteMany volumes, nil for block volumes)
 }
 
 // NetworkEgressPolicy configures host-mediated outbound networking behavior.