Maildir re-implementation with two bachend: fs and s3

backend.go

@@ -1,42 +1,51 @@
 package imapmaildir
 
 import (
-	"errors"
-	"io/ioutil"
+	"fmt"
 	"log"
 	"os"
+	"sort"
 	"strings"
 	"sync"
+	"time"
 
-	"github.com/asdine/storm/v3"
 	"github.com/emersion/go-imap"
 	"github.com/emersion/go-imap/backend"
+	mess "github.com/foxcpp/go-imap-mess"
+
+	"github.com/foxcpp/go-imap-maildir/maildir"
 )
 
 type Backend struct {
 	Log   *log.Logger
 	Debug *log.Logger
 
-	PathTemplate  string
-	Authenticator func(*imap.ConnInfo, string, string) (bool, error)
-
-	// BoltDB does not allow to open the same database file multiple times,
-	// therefore we need to serialize access to one handle and close it only if
-	// the mailbox is no longer used.
-	//
-	// dbsLock protects the concurrent map access. At the moment there is no
-	// clever locking and dbsLock is held for the whole duration of storm.DB
-	// initialization.  That is, once dbsLock is acquired, all elements in the
-	// map have vaild db.
-	//
-	// Lookup key is username + \0 + mailboxName.
-	dbs     map[string]mailboxHandle
-	dbsLock sync.Mutex
+	PathTemplate     string
+	Authenticator    func(*imap.ConnInfo, string, string) (bool, error)
+	DefaultMailboxes []DefaultMailboxSpec
+	StorageProvider  maildir.Provider
+
+	Manager *mess.Manager
+
+	limitLock   sync.Mutex
+	appendLimit *uint32
+
+	statesLock sync.Mutex
+	states     map[string]*mailboxState
+}
+
+type mailboxState struct {
+	uidValidity uint32
+	uidNext     uint32
+	messages    map[string]*messageMeta
+	meta        *maildir.Metadata
 }
 
-type mailboxHandle struct {
-	db   *storm.DB
-	uses int64
+type messageMeta struct {
+	uid          uint32
+	flags        []string
+	internalDate time.Time
+	recent       bool
 }
 
 func (b *Backend) Login(connInfo *imap.ConnInfo, username, password string) (backend.User, error) {
@@ -55,55 +64,213 @@ func (b *Backend) Login(connInfo *imap.ConnInfo, username, password string) (bac
 
 func (b *Backend) GetUser(username string) (backend.User, error) {
 	basePath := strings.ReplaceAll(b.PathTemplate, "{username}", username)
-
-	if _, err := os.Stat(basePath); err != nil {
-		if os.IsNotExist(err) {
-			return nil, backend.ErrInvalidCredentials
-		}
+	storage := b.StorageProvider.Storage(basePath)
+	mailbox, err := storage.Dir(InboxName)
+	if err != nil {
 		b.Log.Printf("%v", err)
-		return nil, errors.New("I/O error")
+		return nil, fmt.Errorf("I/O error: %w", err)
+	}
+	exists, err := mailbox.Exists()
+	if err != nil {
+		b.Log.Printf("%v", err)
+		return nil, fmt.Errorf("I/O error: %w", err)
+	}
+	if !exists {
+		return nil, backend.ErrInvalidCredentials
 	}
 
 	b.Debug.Printf("user logged in (%v, %v)", username, basePath)
 
-	return &User{
-		b:        b,
-		name:     username,
-		basePath: basePath,
-	}, nil
+	user := &User{
+		b:         b,
+		name:      username,
+		storage:   storage,
+		mailboxes: map[string]*Mailbox{},
+	}
+	if err := mailbox.Init(); err != nil {
+		b.Log.Printf("failed to init inbox: %v", err)
+		return nil, fmt.Errorf("I/O error: %w", err)
+	}
+
+	return user, nil
 }
 
 func (b *Backend) CreateUser(username string) error {
 	basePath := strings.ReplaceAll(b.PathTemplate, "{username}", username)
-
-	err := os.Mkdir(basePath, 0700)
+	storage := b.StorageProvider.Storage(basePath)
+	mailbox, err := storage.Dir(InboxName)
 	if err != nil {
-		if os.IsExist(err) {
-			return errors.New("imapmaildir: user already exits")
-		}
 		return err
 	}
+	if err := mailbox.Init(); err != nil {
+		return err
+	}
+
+	user := &User{
+		b:         b,
+		name:      username,
+		storage:   storage,
+		mailboxes: map[string]*Mailbox{},
+	}
+	user.ensureMailbox(InboxName)
+	user.ensureDefaultMailboxes()
 	return nil
 }
 
-func (b *Backend) Close() error {
-	b.dbsLock.Lock()
-	defer b.dbsLock.Unlock()
+func (b *Backend) DeleteUser(username string) error {
+	basePath := strings.ReplaceAll(b.PathTemplate, "{username}", username)
+	storage := b.StorageProvider.Storage(basePath)
+
+	mailboxes := []string{}
+	seen := map[string]struct{}{}
+	addMailbox := func(name string) {
+		if name == "" {
+			return
+		}
+		if _, ok := seen[name]; ok {
+			return
+		}
+		seen[name] = struct{}{}
+		mailboxes = append(mailboxes, name)
+	}
+
+	if inbox, err := storage.Dir(InboxName); err == nil {
+		exists, err := inbox.Exists()
+		if err != nil {
+			return fmt.Errorf("I/O error: %w", err)
+		}
+		if exists {
+			addMailbox(InboxName)
+		}
+	}
+
+	names, err := storage.ListDirs()
+	if err != nil {
+		return fmt.Errorf("I/O error: %w", err)
+	}
+	for _, name := range names {
+		addMailbox(name)
+	}
 
-	for k, db := range b.dbs {
-		if err := db.db.Close(); err != nil {
-			b.Log.Printf("close failed for %s DB: %v", k, err)
+	sort.Slice(mailboxes, func(i, j int) bool {
+		depthI := strings.Count(mailboxes[i], HierarchySep)
+		depthJ := strings.Count(mailboxes[j], HierarchySep)
+		if depthI == depthJ {
+			return mailboxes[i] > mailboxes[j]
 		}
+		return depthI > depthJ
+	})
+
+	for _, name := range mailboxes {
+		dir, err := storage.Dir(name)
+		if err != nil {
+			return fmt.Errorf("I/O error: %w", err)
+		}
+		exists, err := dir.Exists()
+		if err != nil {
+			return fmt.Errorf("I/O error: %w", err)
+		}
+		if !exists {
+			continue
+		}
+		if err := dir.Remove(false); err != nil {
+			return fmt.Errorf("I/O error: %w", err)
+		}
+		b.deleteMailboxState(username, name)
 	}
 
 	return nil
 }
 
-func New(pathTemplate string) (*Backend, error) {
+func (b *Backend) CreateMessageLimit() *uint32 {
+	b.limitLock.Lock()
+	defer b.limitLock.Unlock()
+
+	if b.appendLimit == nil {
+		return nil
+	}
+	val := *b.appendLimit
+	return &val
+}
+
+func (b *Backend) SetMessageLimit(val *uint32) error {
+	b.limitLock.Lock()
+	defer b.limitLock.Unlock()
+
+	if val == nil {
+		b.appendLimit = nil
+		return nil
+	}
+
+	copyVal := *val
+	b.appendLimit = &copyVal
+	return nil
+}
+
+func (b *Backend) Close() error {
+	return nil
+}
+
+func New(pathTemplate string, provider maildir.Provider, defaultMailboxes []DefaultMailboxSpec) (*Backend, error) {
+	if len(defaultMailboxes) == 0 {
+		defaultMailboxes = defaultMailboxSpecs
+	}
 	return &Backend{
-		Log:          log.New(os.Stderr, "imapmaildir: ", 0),
-		Debug:        log.New(ioutil.Discard, "imapmaildir[debug]: ", 0),
-		PathTemplate: pathTemplate,
-		dbs:          map[string]mailboxHandle{},
+		Log:              log.New(os.Stderr, "imapmaildir: ", 0),
+		Debug:            log.New(os.Stderr, "imapmaildir[debug]: ", 0),
+		PathTemplate:     pathTemplate,
+		DefaultMailboxes: defaultMailboxes,
+		StorageProvider:  provider,
+		Manager:          mess.NewManager(),
+		states:           map[string]*mailboxState{},
 	}, nil
 }
+
+func (b *Backend) mailboxKey(username, mailbox string) string {
+	return username + "\x00" + mailbox
+}
+
+func (b *Backend) getMailboxState(username, mailbox string) *mailboxState {
+	key := b.mailboxKey(username, mailbox)
+
+	b.statesLock.Lock()
+	defer b.statesLock.Unlock()
+
+	state, ok := b.states[key]
+	if ok {
+		return state
+	}
+
+	uidValidity := uint32(time.Now().UnixNano())
+	if uidValidity == 0 {
+		uidValidity = 1
+	}
+
+	state = &mailboxState{
+		uidValidity: uidValidity,
+		uidNext:     1,
+		messages:    map[string]*messageMeta{},
+	}
+	b.states[key] = state
+	return state
+}
+
+func (b *Backend) deleteMailboxState(username, mailbox string) {
+	key := b.mailboxKey(username, mailbox)
+	b.statesLock.Lock()
+	defer b.statesLock.Unlock()
+	delete(b.states, key)
+}
+
+func (b *Backend) renameMailboxState(username, oldName, newName string) {
+	oldKey := b.mailboxKey(username, oldName)
+	newKey := b.mailboxKey(username, newName)
+
+	b.statesLock.Lock()
+	defer b.statesLock.Unlock()
+
+	if state, ok := b.states[oldKey]; ok {
+		delete(b.states, oldKey)
+		b.states[newKey] = state
+	}
+}

backendtests_test.go → backendtests_fs_test.go

@@ -1,22 +1,23 @@
 package imapmaildir
 
 import (
-	"io/ioutil"
 	"log"
 	"os"
 	"strings"
 	"testing"
 
 	backendtests "github.com/foxcpp/go-imap-backend-tests"
+
+	"github.com/foxcpp/go-imap-maildir/maildir/fs"
 )
 
 func initTestBackend() backendtests.Backend {
-	tempDir, err := ioutil.TempDir("", "go-imap-maildir-")
+	tempDir, err := os.MkdirTemp("", "go-imap-maildir-")
 	if err != nil {
 		panic(err)
 	}
 
-	be, err := New(tempDir + "/{username}")
+	be, err := New(tempDir+"/{username}", fs.Provider{}, nil)
 	if err != nil {
 		panic(err)
 	}

backendtests_s3_test.go

@@ -0,0 +1,51 @@
+package imapmaildir
+
+import (
+	"strings"
+	"testing"
+
+	backendtests "github.com/foxcpp/go-imap-backend-tests"
+
+	"github.com/foxcpp/go-imap-maildir/maildir/s3"
+	"github.com/foxcpp/go-imap-maildir/maildir/s3/testing"
+)
+
+func TestBackendS3(t *testing.T) {
+	client, bucket, rootPrefix, cleanup := s3testing.StartMinio(t)
+	defer cleanup()
+
+	provider := s3.NewProvider(client, bucket, rootPrefix)
+
+	init := func() backendtests.Backend {
+		be, err := New("users/{username}", provider, defaultMailboxSpecs)
+		if err != nil {
+			panic(err)
+		}
+		return be
+	}
+
+	clean := func(b backendtests.Backend) {
+		be := b.(*Backend)
+		prefix := ""
+		if be.PathTemplate != "" {
+			prefix = joinKey(rootPrefix, "users")
+		}
+		s3testing.RemoveAllObjects(t, client, bucket, prefix)
+	}
+
+	backendtests.RunTests(t, init, clean)
+}
+
+func joinKey(parts ...string) string {
+	var clean []string
+	for _, part := range parts {
+		if part == "" {
+			continue
+		}
+		clean = append(clean, strings.Trim(part, "/"))
+	}
+	if len(clean) == 0 {
+		return ""
+	}
+	return strings.Join(clean, "/")
+}